turbopack_core/chunk/
chunk_group.rs

1use std::{collections::HashSet, sync::atomic::AtomicBool};
2
3use anyhow::{Context, Result};
4use rustc_hash::FxHashMap;
5use smallvec::{SmallVec, smallvec};
6use turbo_rcstr::rcstr;
7use turbo_tasks::{FxIndexSet, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, Vc};
8
9use super::{
10    Chunk, ChunkGroupContent, ChunkItem, ChunkItemWithAsyncModuleInfo, ChunkingContext,
11    availability_info::AvailabilityInfo, chunking::make_chunks,
12};
13use crate::{
14    chunk::{
15        ChunkableModule, ChunkingType,
16        chunk_item_batch::{ChunkItemBatchGroup, ChunkItemOrBatchWithAsyncModuleInfo},
17    },
18    environment::ChunkLoading,
19    module::Module,
20    module_graph::{
21        GraphTraversalAction, ModuleGraph,
22        module_batch::{ChunkableModuleBatchGroup, ChunkableModuleOrBatch, ModuleOrBatch},
23        module_batches::{BatchingConfig, ModuleBatchesGraphEdge},
24    },
25    output::OutputAssets,
26    reference::ModuleReference,
27    traced_asset::TracedAsset,
28};
29
30pub struct MakeChunkGroupResult {
31    pub chunks: Vec<ResolvedVc<Box<dyn Chunk>>>,
32    pub availability_info: AvailabilityInfo,
33}
34
35/// Creates a chunk group from a set of entries.
36pub async fn make_chunk_group(
37    chunk_group_entries: impl IntoIterator<
38        IntoIter = impl Iterator<Item = ResolvedVc<Box<dyn Module>>> + Send,
39    > + Send
40    + Clone,
41    module_graph: Vc<ModuleGraph>,
42    chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
43    availability_info: AvailabilityInfo,
44) -> Result<MakeChunkGroupResult> {
45    let can_split_async = !matches!(
46        *chunking_context.environment().chunk_loading().await?,
47        ChunkLoading::Edge
48    );
49    let should_trace = *chunking_context.is_tracing_enabled().await?;
50    let should_merge_modules = *chunking_context.is_module_merging_enabled().await?;
51    let batching_config = chunking_context.batching_config();
52
53    let ChunkGroupContent {
54        chunkable_items,
55        batch_groups,
56        async_modules,
57        traced_modules,
58        availability_info,
59    } = chunk_group_content(
60        module_graph,
61        chunk_group_entries.clone(),
62        availability_info,
63        can_split_async,
64        should_trace,
65        should_merge_modules,
66        batching_config,
67    )
68    .await?;
69
70    let async_modules_info = module_graph.async_module_info().await?;
71
72    // Attach async info to chunkable modules
73    let mut chunk_items = chunkable_items
74        .iter()
75        .copied()
76        .map(|m| {
77            ChunkItemOrBatchWithAsyncModuleInfo::from_chunkable_module_or_batch(
78                m,
79                &async_modules_info,
80                module_graph,
81                *chunking_context,
82            )
83        })
84        .try_join()
85        .await?
86        .into_iter()
87        .flatten()
88        .collect::<Vec<_>>();
89
90    let chunk_item_batch_groups = batch_groups
91        .iter()
92        .map(|&batch_group| {
93            ChunkItemBatchGroup::from_module_batch_group(
94                ChunkableModuleBatchGroup::from_module_batch_group(*batch_group),
95                module_graph,
96                *chunking_context,
97            )
98            .to_resolved()
99        })
100        .try_join()
101        .await?;
102
103    // Insert async chunk loaders for every referenced async module
104    let async_loaders = async_modules
105        .into_iter()
106        .map(async |module| {
107            chunking_context
108                .async_loader_chunk_item(*module, module_graph, availability_info)
109                .to_resolved()
110                .await
111        })
112        .try_join()
113        .await?;
114    let async_loader_chunk_items = async_loaders.iter().map(|&chunk_item| {
115        ChunkItemOrBatchWithAsyncModuleInfo::ChunkItem(ChunkItemWithAsyncModuleInfo {
116            chunk_item,
117            module: None,
118            async_info: None,
119        })
120    });
121
122    // And also add output assets referenced by async chunk loaders
123    let async_loader_references = async_loaders
124        .iter()
125        .map(|&loader| loader.references())
126        .try_join()
127        .await?;
128
129    let mut referenced_output_assets = traced_modules
130        .into_iter()
131        .map(|module| async move {
132            Ok(ResolvedVc::upcast(
133                TracedAsset::new(*module).to_resolved().await?,
134            ))
135        })
136        .try_join()
137        .await?;
138
139    chunk_items.extend(async_loader_chunk_items);
140    referenced_output_assets.reserve(
141        async_loader_references
142            .iter()
143            .map(|r| r.len())
144            .sum::<usize>(),
145    );
146    referenced_output_assets.extend(async_loader_references.into_iter().flatten());
147
148    // Pass chunk items to chunking algorithm
149    let chunks = make_chunks(
150        module_graph,
151        chunking_context,
152        chunk_items,
153        chunk_item_batch_groups,
154        rcstr!(""),
155        ResolvedVc::cell(referenced_output_assets),
156    )
157    .await?;
158
159    Ok(MakeChunkGroupResult {
160        chunks,
161        availability_info,
162    })
163}
164
165pub async fn references_to_output_assets(
166    references: impl IntoIterator<Item = &ResolvedVc<Box<dyn ModuleReference>>>,
167) -> Result<Vc<OutputAssets>> {
168    let output_assets = references
169        .into_iter()
170        .map(|reference| reference.resolve_reference().primary_output_assets())
171        .try_join()
172        .await?;
173    let mut set = HashSet::new();
174    let output_assets = output_assets
175        .iter()
176        .flatten()
177        .copied()
178        .filter(|&asset| set.insert(asset))
179        .map(|asset| *asset)
180        .collect::<Vec<_>>();
181    Ok(OutputAssets::new(output_assets))
182}
183
184pub async fn chunk_group_content(
185    module_graph: Vc<ModuleGraph>,
186    chunk_group_entries: impl IntoIterator<
187        IntoIter = impl Iterator<Item = ResolvedVc<Box<dyn Module>>> + Send,
188    > + Send,
189    availability_info: AvailabilityInfo,
190    can_split_async: bool,
191    should_trace: bool,
192    should_merge_modules: bool,
193    batching_config: Vc<BatchingConfig>,
194) -> Result<ChunkGroupContent> {
195    let module_batches_graph = module_graph.module_batches(batching_config).await?;
196
197    type ModuleToChunkableMap = FxHashMap<ModuleOrBatch, ChunkableModuleOrBatch>;
198
199    struct TraverseState {
200        unsorted_items: ModuleToChunkableMap,
201        chunkable_items: FxIndexSet<ChunkableModuleOrBatch>,
202        async_modules: FxIndexSet<ResolvedVc<Box<dyn ChunkableModule>>>,
203        traced_modules: FxIndexSet<ResolvedVc<Box<dyn Module>>>,
204    }
205
206    let mut state = TraverseState {
207        unsorted_items: FxHashMap::default(),
208        chunkable_items: FxIndexSet::default(),
209        async_modules: FxIndexSet::default(),
210        traced_modules: FxIndexSet::default(),
211    };
212
213    let available_modules = match availability_info.available_modules() {
214        Some(available_modules) => Some(available_modules.snapshot().await?),
215        None => None,
216    };
217
218    let chunk_group_entries = chunk_group_entries.into_iter();
219    let mut entries = Vec::with_capacity(chunk_group_entries.size_hint().0);
220    for entry in chunk_group_entries {
221        entries.push(module_batches_graph.get_entry_index(entry).await?);
222    }
223
224    module_batches_graph.traverse_edges_from_entries_dfs(
225        entries,
226        &mut state,
227        |parent_info, &node, state| {
228            // Traced modules need to have a special handling
229            if let Some((
230                _,
231                ModuleBatchesGraphEdge {
232                    ty: ChunkingType::Traced,
233                    ..
234                },
235            )) = parent_info
236            {
237                if should_trace {
238                    let ModuleOrBatch::Module(module) = node else {
239                        unreachable!();
240                    };
241                    state.traced_modules.insert(module);
242                }
243                return Ok(GraphTraversalAction::Exclude);
244            }
245
246            let Some(chunkable_node) = ChunkableModuleOrBatch::from_module_or_batch(node) else {
247                return Ok(GraphTraversalAction::Exclude);
248            };
249
250            let is_available = available_modules
251                .as_ref()
252                .is_some_and(|available_modules| available_modules.get(chunkable_node));
253
254            let Some((_, edge)) = parent_info else {
255                // An entry from the entries list
256                return Ok(if is_available {
257                    GraphTraversalAction::Exclude
258                } else if state
259                    .unsorted_items
260                    .try_insert(node, chunkable_node)
261                    .is_ok()
262                {
263                    GraphTraversalAction::Continue
264                } else {
265                    GraphTraversalAction::Exclude
266                });
267            };
268
269            Ok(match edge.ty {
270                ChunkingType::Parallel { .. } | ChunkingType::Shared { .. } => {
271                    if is_available {
272                        GraphTraversalAction::Exclude
273                    } else if state
274                        .unsorted_items
275                        .try_insert(node, chunkable_node)
276                        .is_ok()
277                    {
278                        GraphTraversalAction::Continue
279                    } else {
280                        GraphTraversalAction::Exclude
281                    }
282                }
283                ChunkingType::Async => {
284                    if can_split_async {
285                        let chunkable_module = ResolvedVc::try_downcast(edge.module.unwrap())
286                            .context("Module in async chunking edge is not chunkable")?;
287                        state.async_modules.insert(chunkable_module);
288                        GraphTraversalAction::Exclude
289                    } else if is_available {
290                        GraphTraversalAction::Exclude
291                    } else if state
292                        .unsorted_items
293                        .try_insert(node, chunkable_node)
294                        .is_ok()
295                    {
296                        GraphTraversalAction::Continue
297                    } else {
298                        GraphTraversalAction::Exclude
299                    }
300                }
301                ChunkingType::Traced => {
302                    // handled above before the sidecast
303                    unreachable!();
304                }
305                ChunkingType::Isolated { .. } => {
306                    // TODO currently not implemented
307                    GraphTraversalAction::Exclude
308                }
309            })
310        },
311        |_, node, state| {
312            // Insert modules in topological order
313            if let Some(chunkable_module) = state.unsorted_items.get(node).copied() {
314                state.chunkable_items.insert(chunkable_module);
315            }
316        },
317    )?;
318
319    // This needs to use the unmerged items
320    let availability_info = availability_info
321        .with_modules(Vc::cell(state.chunkable_items.clone()))
322        .await?;
323
324    if should_merge_modules {
325        let merged_modules = module_graph.merged_modules().await?;
326        state.chunkable_items = state
327            .chunkable_items
328            .into_iter()
329            .map(async |chunkable_module| match chunkable_module {
330                ChunkableModuleOrBatch::Module(module) => {
331                    if !merged_modules.should_create_chunk_item_for(ResolvedVc::upcast(module)) {
332                        return Ok(smallvec![]);
333                    }
334
335                    let module = if let Some(replacement) =
336                        merged_modules.should_replace_module(ResolvedVc::upcast(module))
337                    {
338                        replacement
339                    } else {
340                        module
341                    };
342
343                    Ok(smallvec![ChunkableModuleOrBatch::Module(module)])
344                }
345                ChunkableModuleOrBatch::Batch(batch) => {
346                    let batch_ref = batch.await?;
347                    let modules = &batch_ref.modules;
348
349                    let modified = AtomicBool::new(false);
350                    let modules = modules
351                        .iter()
352                        .filter(|module| {
353                            if merged_modules
354                                .should_create_chunk_item_for(ResolvedVc::upcast(**module))
355                            {
356                                true
357                            } else {
358                                modified.store(true, std::sync::atomic::Ordering::Relaxed);
359                                false
360                            }
361                        })
362                        .map(|&module| {
363                            if let Some(replacement) =
364                                merged_modules.should_replace_module(ResolvedVc::upcast(module))
365                            {
366                                modified.store(true, std::sync::atomic::Ordering::Relaxed);
367                                replacement
368                            } else {
369                                module
370                            }
371                        })
372                        .map(ChunkableModuleOrBatch::Module)
373                        .collect::<SmallVec<[_; 1]>>();
374
375                    if modified.load(std::sync::atomic::Ordering::Relaxed) {
376                        Ok(modules)
377                    } else {
378                        Ok(smallvec![ChunkableModuleOrBatch::Batch(batch)])
379                    }
380                }
381                ChunkableModuleOrBatch::None(i) => Ok(smallvec![ChunkableModuleOrBatch::None(i)]),
382            })
383            .try_flat_join()
384            .await?
385            .into_iter()
386            .collect();
387    };
388
389    let mut batch_groups = FxIndexSet::default();
390    for &module in &state.chunkable_items {
391        if let Some(batch_group) = module_batches_graph.get_batch_group(&module.into()) {
392            batch_groups.insert(batch_group);
393        }
394    }
395
396    Ok(ChunkGroupContent {
397        chunkable_items: state.chunkable_items,
398        batch_groups,
399        async_modules: state.async_modules,
400        traced_modules: state.traced_modules,
401        availability_info,
402    })
403}