1use std::{collections::HashSet, sync::atomic::AtomicBool};
2
3use anyhow::{Context, Result};
4use rustc_hash::FxHashMap;
5use smallvec::{SmallVec, smallvec};
6use turbo_rcstr::rcstr;
7use turbo_tasks::{FxIndexSet, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, Vc};
8
9use super::{
10 Chunk, ChunkGroupContent, ChunkItem, ChunkItemWithAsyncModuleInfo, ChunkingContext,
11 availability_info::AvailabilityInfo, chunking::make_chunks,
12};
13use crate::{
14 chunk::{
15 ChunkableModule, ChunkingType,
16 chunk_item_batch::{ChunkItemBatchGroup, ChunkItemOrBatchWithAsyncModuleInfo},
17 },
18 environment::ChunkLoading,
19 module::Module,
20 module_graph::{
21 GraphTraversalAction, ModuleGraph,
22 module_batch::{ChunkableModuleBatchGroup, ChunkableModuleOrBatch, ModuleOrBatch},
23 module_batches::{BatchingConfig, ModuleBatchesGraphEdge},
24 },
25 output::OutputAssets,
26 reference::ModuleReference,
27 traced_asset::TracedAsset,
28};
29
30pub struct MakeChunkGroupResult {
31 pub chunks: Vec<ResolvedVc<Box<dyn Chunk>>>,
32 pub availability_info: AvailabilityInfo,
33}
34
35pub async fn make_chunk_group(
37 chunk_group_entries: impl IntoIterator<
38 IntoIter = impl Iterator<Item = ResolvedVc<Box<dyn Module>>> + Send,
39 > + Send
40 + Clone,
41 module_graph: Vc<ModuleGraph>,
42 chunking_context: ResolvedVc<Box<dyn ChunkingContext>>,
43 availability_info: AvailabilityInfo,
44) -> Result<MakeChunkGroupResult> {
45 let can_split_async = !matches!(
46 *chunking_context.environment().chunk_loading().await?,
47 ChunkLoading::Edge
48 );
49 let should_trace = *chunking_context.is_tracing_enabled().await?;
50 let should_merge_modules = *chunking_context.is_module_merging_enabled().await?;
51 let batching_config = chunking_context.batching_config();
52
53 let ChunkGroupContent {
54 chunkable_items,
55 batch_groups,
56 async_modules,
57 traced_modules,
58 availability_info,
59 } = chunk_group_content(
60 module_graph,
61 chunk_group_entries.clone(),
62 availability_info,
63 can_split_async,
64 should_trace,
65 should_merge_modules,
66 batching_config,
67 )
68 .await?;
69
70 let async_modules_info = module_graph.async_module_info().await?;
71
72 let mut chunk_items = chunkable_items
74 .iter()
75 .copied()
76 .map(|m| {
77 ChunkItemOrBatchWithAsyncModuleInfo::from_chunkable_module_or_batch(
78 m,
79 &async_modules_info,
80 module_graph,
81 *chunking_context,
82 )
83 })
84 .try_join()
85 .await?
86 .into_iter()
87 .flatten()
88 .collect::<Vec<_>>();
89
90 let chunk_item_batch_groups = batch_groups
91 .iter()
92 .map(|&batch_group| {
93 ChunkItemBatchGroup::from_module_batch_group(
94 ChunkableModuleBatchGroup::from_module_batch_group(*batch_group),
95 module_graph,
96 *chunking_context,
97 )
98 .to_resolved()
99 })
100 .try_join()
101 .await?;
102
103 let async_loaders = async_modules
105 .into_iter()
106 .map(async |module| {
107 chunking_context
108 .async_loader_chunk_item(*module, module_graph, availability_info)
109 .to_resolved()
110 .await
111 })
112 .try_join()
113 .await?;
114 let async_loader_chunk_items = async_loaders.iter().map(|&chunk_item| {
115 ChunkItemOrBatchWithAsyncModuleInfo::ChunkItem(ChunkItemWithAsyncModuleInfo {
116 chunk_item,
117 module: None,
118 async_info: None,
119 })
120 });
121
122 let async_loader_references = async_loaders
124 .iter()
125 .map(|&loader| loader.references())
126 .try_join()
127 .await?;
128
129 let mut referenced_output_assets = traced_modules
130 .into_iter()
131 .map(|module| async move {
132 Ok(ResolvedVc::upcast(
133 TracedAsset::new(*module).to_resolved().await?,
134 ))
135 })
136 .try_join()
137 .await?;
138
139 chunk_items.extend(async_loader_chunk_items);
140 referenced_output_assets.reserve(
141 async_loader_references
142 .iter()
143 .map(|r| r.len())
144 .sum::<usize>(),
145 );
146 referenced_output_assets.extend(async_loader_references.into_iter().flatten());
147
148 let chunks = make_chunks(
150 module_graph,
151 chunking_context,
152 chunk_items,
153 chunk_item_batch_groups,
154 rcstr!(""),
155 ResolvedVc::cell(referenced_output_assets),
156 )
157 .await?;
158
159 Ok(MakeChunkGroupResult {
160 chunks,
161 availability_info,
162 })
163}
164
165pub async fn references_to_output_assets(
166 references: impl IntoIterator<Item = &ResolvedVc<Box<dyn ModuleReference>>>,
167) -> Result<Vc<OutputAssets>> {
168 let output_assets = references
169 .into_iter()
170 .map(|reference| reference.resolve_reference().primary_output_assets())
171 .try_join()
172 .await?;
173 let mut set = HashSet::new();
174 let output_assets = output_assets
175 .iter()
176 .flatten()
177 .copied()
178 .filter(|&asset| set.insert(asset))
179 .map(|asset| *asset)
180 .collect::<Vec<_>>();
181 Ok(OutputAssets::new(output_assets))
182}
183
184pub async fn chunk_group_content(
185 module_graph: Vc<ModuleGraph>,
186 chunk_group_entries: impl IntoIterator<
187 IntoIter = impl Iterator<Item = ResolvedVc<Box<dyn Module>>> + Send,
188 > + Send,
189 availability_info: AvailabilityInfo,
190 can_split_async: bool,
191 should_trace: bool,
192 should_merge_modules: bool,
193 batching_config: Vc<BatchingConfig>,
194) -> Result<ChunkGroupContent> {
195 let module_batches_graph = module_graph.module_batches(batching_config).await?;
196
197 type ModuleToChunkableMap = FxHashMap<ModuleOrBatch, ChunkableModuleOrBatch>;
198
199 struct TraverseState {
200 unsorted_items: ModuleToChunkableMap,
201 chunkable_items: FxIndexSet<ChunkableModuleOrBatch>,
202 async_modules: FxIndexSet<ResolvedVc<Box<dyn ChunkableModule>>>,
203 traced_modules: FxIndexSet<ResolvedVc<Box<dyn Module>>>,
204 }
205
206 let mut state = TraverseState {
207 unsorted_items: FxHashMap::default(),
208 chunkable_items: FxIndexSet::default(),
209 async_modules: FxIndexSet::default(),
210 traced_modules: FxIndexSet::default(),
211 };
212
213 let available_modules = match availability_info.available_modules() {
214 Some(available_modules) => Some(available_modules.snapshot().await?),
215 None => None,
216 };
217
218 let chunk_group_entries = chunk_group_entries.into_iter();
219 let mut entries = Vec::with_capacity(chunk_group_entries.size_hint().0);
220 for entry in chunk_group_entries {
221 entries.push(module_batches_graph.get_entry_index(entry).await?);
222 }
223
224 module_batches_graph.traverse_edges_from_entries_dfs(
225 entries,
226 &mut state,
227 |parent_info, &node, state| {
228 if let Some((
230 _,
231 ModuleBatchesGraphEdge {
232 ty: ChunkingType::Traced,
233 ..
234 },
235 )) = parent_info
236 {
237 if should_trace {
238 let ModuleOrBatch::Module(module) = node else {
239 unreachable!();
240 };
241 state.traced_modules.insert(module);
242 }
243 return Ok(GraphTraversalAction::Exclude);
244 }
245
246 let Some(chunkable_node) = ChunkableModuleOrBatch::from_module_or_batch(node) else {
247 return Ok(GraphTraversalAction::Exclude);
248 };
249
250 let is_available = available_modules
251 .as_ref()
252 .is_some_and(|available_modules| available_modules.get(chunkable_node));
253
254 let Some((_, edge)) = parent_info else {
255 return Ok(if is_available {
257 GraphTraversalAction::Exclude
258 } else if state
259 .unsorted_items
260 .try_insert(node, chunkable_node)
261 .is_ok()
262 {
263 GraphTraversalAction::Continue
264 } else {
265 GraphTraversalAction::Exclude
266 });
267 };
268
269 Ok(match edge.ty {
270 ChunkingType::Parallel { .. } | ChunkingType::Shared { .. } => {
271 if is_available {
272 GraphTraversalAction::Exclude
273 } else if state
274 .unsorted_items
275 .try_insert(node, chunkable_node)
276 .is_ok()
277 {
278 GraphTraversalAction::Continue
279 } else {
280 GraphTraversalAction::Exclude
281 }
282 }
283 ChunkingType::Async => {
284 if can_split_async {
285 let chunkable_module = ResolvedVc::try_downcast(edge.module.unwrap())
286 .context("Module in async chunking edge is not chunkable")?;
287 state.async_modules.insert(chunkable_module);
288 GraphTraversalAction::Exclude
289 } else if is_available {
290 GraphTraversalAction::Exclude
291 } else if state
292 .unsorted_items
293 .try_insert(node, chunkable_node)
294 .is_ok()
295 {
296 GraphTraversalAction::Continue
297 } else {
298 GraphTraversalAction::Exclude
299 }
300 }
301 ChunkingType::Traced => {
302 unreachable!();
304 }
305 ChunkingType::Isolated { .. } => {
306 GraphTraversalAction::Exclude
308 }
309 })
310 },
311 |_, node, state| {
312 if let Some(chunkable_module) = state.unsorted_items.get(node).copied() {
314 state.chunkable_items.insert(chunkable_module);
315 }
316 },
317 )?;
318
319 let availability_info = availability_info
321 .with_modules(Vc::cell(state.chunkable_items.clone()))
322 .await?;
323
324 if should_merge_modules {
325 let merged_modules = module_graph.merged_modules().await?;
326 state.chunkable_items = state
327 .chunkable_items
328 .into_iter()
329 .map(async |chunkable_module| match chunkable_module {
330 ChunkableModuleOrBatch::Module(module) => {
331 if !merged_modules.should_create_chunk_item_for(ResolvedVc::upcast(module)) {
332 return Ok(smallvec![]);
333 }
334
335 let module = if let Some(replacement) =
336 merged_modules.should_replace_module(ResolvedVc::upcast(module))
337 {
338 replacement
339 } else {
340 module
341 };
342
343 Ok(smallvec![ChunkableModuleOrBatch::Module(module)])
344 }
345 ChunkableModuleOrBatch::Batch(batch) => {
346 let batch_ref = batch.await?;
347 let modules = &batch_ref.modules;
348
349 let modified = AtomicBool::new(false);
350 let modules = modules
351 .iter()
352 .filter(|module| {
353 if merged_modules
354 .should_create_chunk_item_for(ResolvedVc::upcast(**module))
355 {
356 true
357 } else {
358 modified.store(true, std::sync::atomic::Ordering::Relaxed);
359 false
360 }
361 })
362 .map(|&module| {
363 if let Some(replacement) =
364 merged_modules.should_replace_module(ResolvedVc::upcast(module))
365 {
366 modified.store(true, std::sync::atomic::Ordering::Relaxed);
367 replacement
368 } else {
369 module
370 }
371 })
372 .map(ChunkableModuleOrBatch::Module)
373 .collect::<SmallVec<[_; 1]>>();
374
375 if modified.load(std::sync::atomic::Ordering::Relaxed) {
376 Ok(modules)
377 } else {
378 Ok(smallvec![ChunkableModuleOrBatch::Batch(batch)])
379 }
380 }
381 ChunkableModuleOrBatch::None(i) => Ok(smallvec![ChunkableModuleOrBatch::None(i)]),
382 })
383 .try_flat_join()
384 .await?
385 .into_iter()
386 .collect();
387 };
388
389 let mut batch_groups = FxIndexSet::default();
390 for &module in &state.chunkable_items {
391 if let Some(batch_group) = module_batches_graph.get_batch_group(&module.into()) {
392 batch_groups.insert(batch_group);
393 }
394 }
395
396 Ok(ChunkGroupContent {
397 chunkable_items: state.chunkable_items,
398 batch_groups,
399 async_modules: state.async_modules,
400 traced_modules: state.traced_modules,
401 availability_info,
402 })
403}