1use std::{borrow::Cow, io::Write};
2
3use anyhow::Result;
4use byteorder::{BE, WriteBytesExt};
5use either::Either;
6use next_core::app_structure::FileSystemPathVec;
7use rustc_hash::{FxHashMap, FxHashSet};
8use serde::Serialize;
9use turbo_rcstr::RcStr;
10use turbo_tasks::{
11 FxIndexSet, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, ValueToString, ValueToStringRef, Vc,
12};
13use turbo_tasks_fs::{
14 File, FileContent, FileSystemPath,
15 rope::{Rope, RopeBuilder},
16};
17use turbopack_analyze::split_chunk::{split_output_asset_into_parts, split_traced_file_into_parts};
18use turbopack_core::{
19 SOURCE_URL_PROTOCOL,
20 asset::{Asset, AssetContent},
21 chunk::{ChunkingType, TracedMode},
22 module::Module,
23 module_graph::{GraphTraversalAction, ModuleGraph},
24 output::{OutputAsset, OutputAssets, OutputAssetsReference},
25 reference::all_assets_from_entries,
26};
27
28pub struct EdgesData {
29 pub offsets: Vec<u32>,
30 pub data: Vec<u32>,
31}
32
33impl EdgesData {
34 fn from_iterator<'a>(iterable: impl IntoIterator<Item = &'a Vec<u32>> + Clone) -> Self {
35 let mut current_offset = 0;
36 let sum: usize = iterable.clone().into_iter().map(|v| v.len()).sum();
37 let mut data = Vec::with_capacity(sum);
38 let offsets = iterable
39 .into_iter()
40 .map(|edges| {
41 current_offset += edges.len() as u32;
42 data.extend(edges);
43 current_offset
44 })
45 .collect();
46 Self { offsets, data }
47 }
48
49 fn write(&self, writer: &mut impl Write) -> Result<()> {
50 writer.write_u32::<BE>(self.offsets.len() as u32)?;
51 for &offset in &self.offsets {
52 writer.write_u32::<BE>(offset)?;
53 }
54 for &data in &self.data {
55 writer.write_u32::<BE>(data)?;
56 }
57 Ok(())
58 }
59}
60
61#[derive(Serialize)]
62pub struct AnalyzeSource {
63 pub parent_source_index: Option<u32>,
64 pub path: RcStr,
68}
69
70#[derive(Serialize)]
71pub struct AnalyzeModule {
72 pub ident: RcStr,
73 pub path: RcStr,
74}
75
76#[derive(Serialize)]
77pub struct AnalyzeChunkPart {
78 pub source_index: u32,
79 pub output_file_index: u32,
80 pub size: u32,
81 pub compressed_size: u32,
82}
83
84#[derive(Serialize)]
85pub struct AnalyzeOutputFile {
86 pub filename: RcStr,
87}
88
89#[derive(Serialize)]
90struct EdgesDataReference {
91 pub offset: u32,
92 pub length: u32,
93}
94
95#[derive(Serialize)]
96struct AnalyzeDataHeader {
97 pub sources: Vec<AnalyzeSource>,
98 pub chunk_parts: Vec<AnalyzeChunkPart>,
99 pub output_files: Vec<AnalyzeOutputFile>,
100 pub output_file_chunk_parts: EdgesDataReference,
102 pub source_chunk_parts: EdgesDataReference,
104 pub source_children: EdgesDataReference,
106 pub source_roots: Vec<u32>,
108}
109
110#[derive(Serialize)]
111struct ModulesDataHeader {
112 pub modules: Vec<AnalyzeModule>,
113 pub module_dependents: EdgesDataReference,
115 pub async_module_dependents: EdgesDataReference,
117 pub traced_module_dependents: EdgesDataReference,
119 pub module_dependencies: EdgesDataReference,
121 pub async_module_dependencies: EdgesDataReference,
123 pub traced_module_dependencies: EdgesDataReference,
125}
126
127struct AnalyzeOutputFileBuilder {
128 output_file: AnalyzeOutputFile,
129 chunk_part_indices: Vec<u32>,
130}
131
132struct AnalyzeSourceBuilder {
133 source: AnalyzeSource,
134 child_source_indices: Vec<u32>,
135 chunk_part_indices: Vec<u32>,
136}
137
138struct AnalyzeModuleBuilder {
139 module: AnalyzeModule,
140 dependencies: FxIndexSet<u32>,
141 async_dependencies: FxIndexSet<u32>,
142 traced_dependencies: FxIndexSet<u32>,
143 dependents: FxIndexSet<u32>,
144 async_dependents: FxIndexSet<u32>,
145 traced_dependents: FxIndexSet<u32>,
146}
147
148struct AnalyzeDataBuilder {
149 sources: Vec<AnalyzeSourceBuilder>,
150 source_index_map: FxHashMap<RcStr, u32>,
151 chunk_parts: Vec<AnalyzeChunkPart>,
152 output_files: Vec<AnalyzeOutputFileBuilder>,
153}
154
155struct ModulesDataBuilder {
156 modules: Vec<AnalyzeModuleBuilder>,
157 module_index_map: FxHashMap<RcStr, u32>,
158}
159
160struct EdgesDataSectionBuilder {
161 data: Vec<u8>,
162}
163
164impl EdgesDataSectionBuilder {
165 fn new() -> Self {
166 Self { data: vec![] }
167 }
168
169 fn add_edges(&mut self, edges: &EdgesData) -> EdgesDataReference {
170 let offset = self.data.len().try_into().unwrap();
171 edges.write(&mut self.data).unwrap();
172 let length = (self.data.len() - offset as usize).try_into().unwrap();
173 EdgesDataReference { offset, length }
174 }
175}
176
177impl AnalyzeDataBuilder {
178 fn new() -> Self {
179 Self {
180 sources: vec![],
181 source_index_map: FxHashMap::default(),
182 chunk_parts: vec![],
183 output_files: vec![],
184 }
185 }
186
187 fn ensure_source(&mut self, path: &str) -> (&mut AnalyzeSourceBuilder, u32) {
188 if let Some(&index) = self.source_index_map.get(path) {
189 return (&mut self.sources[index as usize], index);
190 }
191 let index = self.sources.len() as u32;
192 let path = RcStr::from(path);
193 self.source_index_map.insert(path.clone(), index);
194 self.sources.push(AnalyzeSourceBuilder {
195 source: AnalyzeSource {
196 parent_source_index: None,
197 path,
198 },
199 child_source_indices: vec![],
200 chunk_part_indices: vec![],
201 });
202 (&mut self.sources[index as usize], index)
203 }
204
205 fn add_chunk_part(&mut self, chunk_part: AnalyzeChunkPart) -> u32 {
206 let i = self.chunk_parts.len() as u32;
207 self.chunk_parts.push(chunk_part);
208 i
209 }
210
211 fn add_output_file(&mut self, output_file: AnalyzeOutputFile) -> u32 {
212 let i = self.output_files.len() as u32;
213 self.output_files.push(AnalyzeOutputFileBuilder {
214 output_file,
215 chunk_part_indices: vec![],
216 });
217 i
218 }
219
220 fn add_chunk_part_to_output_file(&mut self, output_file_index: u32, chunk_part_index: u32) {
221 self.output_files[output_file_index as usize]
222 .chunk_part_indices
223 .push(chunk_part_index);
224 }
225
226 fn add_chunk_part_to_source(&mut self, source_index: u32, chunk_part_index: u32) {
227 self.sources[source_index as usize]
228 .chunk_part_indices
229 .push(chunk_part_index);
230 }
231
232 fn build(self) -> Rope {
233 let source_roots = self
234 .sources
235 .iter()
236 .enumerate()
237 .filter_map(|(i, s)| {
238 if s.source.parent_source_index.is_none() {
239 Some(i as u32)
240 } else {
241 None
242 }
243 })
244 .collect();
245
246 let source_children =
247 EdgesData::from_iterator(self.sources.iter().map(|s| &s.child_source_indices));
248
249 let source_chunk_parts =
250 EdgesData::from_iterator(self.sources.iter().map(|s| &s.chunk_part_indices));
251
252 let output_file_chunk_parts =
253 EdgesData::from_iterator(self.output_files.iter().map(|of| &of.chunk_part_indices));
254
255 let mut binary_section = EdgesDataSectionBuilder::new();
256
257 let header = AnalyzeDataHeader {
258 sources: self.sources.into_iter().map(|s| s.source).collect(),
259 chunk_parts: self.chunk_parts,
260 output_files: self
261 .output_files
262 .into_iter()
263 .map(|of| of.output_file)
264 .collect(),
265 output_file_chunk_parts: binary_section.add_edges(&output_file_chunk_parts),
266 source_chunk_parts: binary_section.add_edges(&source_chunk_parts),
267 source_children: binary_section.add_edges(&source_children),
268 source_roots,
269 };
270
271 let header_json = serde_json::to_vec(&header).unwrap();
272
273 let mut rope = RopeBuilder::default();
274 rope.push_bytes(&(header_json.len() as u32).to_be_bytes());
275 rope.reserve_bytes(header_json.len() + binary_section.data.len());
276 rope.push_bytes(&header_json);
277 rope.push_bytes(&binary_section.data);
278 rope.build()
279 }
280}
281
282impl ModulesDataBuilder {
283 fn new() -> Self {
284 Self {
285 modules: vec![],
286 module_index_map: FxHashMap::default(),
287 }
288 }
289
290 fn get_module(&mut self, ident: &str) -> (&mut AnalyzeModuleBuilder, u32) {
291 if let Some(&index) = self.module_index_map.get(ident) {
292 return (&mut self.modules[index as usize], index);
293 }
294 panic!("Module with ident `{}` not found", ident);
295 }
296
297 fn ensure_module(&mut self, ident: &str, path: &str) -> (&mut AnalyzeModuleBuilder, u32) {
298 if let Some(&index) = self.module_index_map.get(ident) {
299 return (&mut self.modules[index as usize], index);
300 }
301 let index = self.modules.len() as u32;
302 let ident = RcStr::from(ident);
303 let path = RcStr::from(path);
304 self.module_index_map.insert(ident.clone(), index);
305 self.modules.push(AnalyzeModuleBuilder {
306 module: AnalyzeModule { ident, path },
307 dependencies: FxIndexSet::default(),
308 async_dependencies: FxIndexSet::default(),
309 traced_dependencies: FxIndexSet::default(),
310 dependents: FxIndexSet::default(),
311 async_dependents: FxIndexSet::default(),
312 traced_dependents: FxIndexSet::default(),
313 });
314 (&mut self.modules[index as usize], index)
315 }
316
317 fn build(self) -> Rope {
318 let module_dependencies_vecs: Vec<Vec<u32>> = self
319 .modules
320 .iter()
321 .map(|s| s.dependencies.iter().copied().collect())
322 .collect();
323 let async_module_dependencies_vecs: Vec<Vec<u32>> = self
324 .modules
325 .iter()
326 .map(|s| s.async_dependencies.iter().copied().collect())
327 .collect();
328 let traced_module_dependencies_vecs: Vec<Vec<u32>> = self
329 .modules
330 .iter()
331 .map(|s| s.traced_dependencies.iter().copied().collect())
332 .collect();
333 let module_dependents_vecs: Vec<Vec<u32>> = self
334 .modules
335 .iter()
336 .map(|s| s.dependents.iter().copied().collect())
337 .collect();
338 let async_module_dependents_vecs: Vec<Vec<u32>> = self
339 .modules
340 .iter()
341 .map(|s| s.async_dependents.iter().copied().collect())
342 .collect();
343 let traced_module_dependents_vecs: Vec<Vec<u32>> = self
344 .modules
345 .iter()
346 .map(|s| s.traced_dependents.iter().copied().collect())
347 .collect();
348
349 let module_dependencies = EdgesData::from_iterator(&module_dependencies_vecs);
350 let async_module_dependencies = EdgesData::from_iterator(&async_module_dependencies_vecs);
351 let traced_module_dependencies = EdgesData::from_iterator(&traced_module_dependencies_vecs);
352 let module_dependents = EdgesData::from_iterator(&module_dependents_vecs);
353 let async_module_dependents = EdgesData::from_iterator(&async_module_dependents_vecs);
354 let traced_module_dependents = EdgesData::from_iterator(&traced_module_dependents_vecs);
355
356 let mut binary_section = EdgesDataSectionBuilder::new();
357
358 let header = ModulesDataHeader {
359 modules: self.modules.into_iter().map(|s| s.module).collect(),
360 module_dependents: binary_section.add_edges(&module_dependents),
361 async_module_dependents: binary_section.add_edges(&async_module_dependents),
362 traced_module_dependents: binary_section.add_edges(&traced_module_dependents),
363 module_dependencies: binary_section.add_edges(&module_dependencies),
364 async_module_dependencies: binary_section.add_edges(&async_module_dependencies),
365 traced_module_dependencies: binary_section.add_edges(&traced_module_dependencies),
366 };
367
368 let header_json = serde_json::to_vec(&header).unwrap();
369
370 let mut rope = RopeBuilder::default();
371 rope.push_bytes(&(header_json.len() as u32).to_be_bytes());
372 rope.reserve_bytes(header_json.len() + binary_section.data.len());
373 rope.push_bytes(&header_json);
374 rope.push_bytes(&binary_section.data);
375 rope.build()
376 }
377}
378
379#[turbo_tasks::function]
382pub async fn combine_output_assets(
383 primary: Vc<OutputAssets>,
384 extra: Vc<OutputAssets>,
385) -> Result<Vc<OutputAssets>> {
386 let mut combined: Vec<ResolvedVc<Box<dyn OutputAsset>>> =
387 primary.await?.iter().copied().collect();
388 combined.extend(extra.await?.iter().copied());
389 Ok(Vc::cell(combined))
390}
391
392#[turbo_tasks::function]
395pub async fn combine_traced_files(
396 primary: Vc<FileSystemPathVec>,
397 extra: Vc<FileSystemPathVec>,
398) -> Result<Vc<FileSystemPathVec>> {
399 let mut combined: Vec<FileSystemPath> = primary.await?.iter().cloned().collect();
400 combined.extend(extra.await?.iter().cloned());
401 Ok(Vc::cell(combined))
402}
403
404#[turbo_tasks::function]
405pub async fn analyze_output_assets(
406 output_assets: Vc<OutputAssets>,
407 traced_files: Vc<FileSystemPathVec>,
408) -> Result<Vc<FileContent>> {
409 let output_assets = all_assets_from_entries(output_assets);
410
411 let mut builder = AnalyzeDataBuilder::new();
412
413 let prefix = format!("{SOURCE_URL_PROTOCOL}///");
414
415 for asset in output_assets
418 .await?
419 .iter()
420 .copied()
421 .map(Either::Left)
422 .chain(traced_files.await?.iter().cloned().map(Either::Right))
423 {
424 let file_system_path = match &asset {
425 Either::Left(asset) => Either::Left(asset.path().await?),
426 Either::Right(path) => Either::Right(path),
427 };
428 let path = match &file_system_path {
429 Either::Left(path) => &path.path,
430 Either::Right(path) => &path.path,
431 };
432 if path.ends_with(".map") || path.ends_with(".nft.json") {
433 continue;
435 }
436
437 let filename = match &file_system_path {
438 Either::Left(path) => path.to_string_ref().await?,
439 Either::Right(path) => path.to_string_ref().await?,
440 };
441
442 let output_file_index = builder.add_output_file(AnalyzeOutputFile {
443 filename: filename.clone(),
444 });
445 let chunk_parts = match asset {
446 Either::Left(asset) => split_output_asset_into_parts(*asset).await?,
447 Either::Right(path) => split_traced_file_into_parts(path).await?,
448 };
449 for chunk_part in &chunk_parts {
450 let decoded_source = urlencoding::decode(&chunk_part.source)?;
451 let source = if let Some(stripped) = decoded_source.strip_prefix(&prefix) {
452 Cow::Borrowed(stripped)
453 } else if decoded_source.starts_with("[project]/") {
454 decoded_source
455 } else {
456 Cow::Owned(format!(
457 "[project]/{}",
458 decoded_source.trim_start_matches("../")
459 ))
460 };
461 let source_index = builder.ensure_source(&source).1;
462 let size = chunk_part.real_size + chunk_part.unaccounted_size;
463 let chunk_part_index = builder.add_chunk_part(AnalyzeChunkPart {
464 source_index,
465 output_file_index,
466 size,
467 compressed_size: chunk_part.get_compressed_size().await?.unwrap_or(size),
468 });
469 builder.add_chunk_part_to_output_file(output_file_index, chunk_part_index);
470 builder.add_chunk_part_to_source(source_index, chunk_part_index);
471 }
472 }
473
474 let mut i: u32 = 0;
476 while i < builder.sources.len().try_into().unwrap() {
477 let source = &builder.sources[i as usize];
478 let path = source.source.path.as_str();
479 if !path.is_empty() {
480 let (parent_path, path) = if let Some(pos) = path.trim_end_matches('/').rfind('/') {
481 (&path[..pos + 1], &path[pos + 1..])
482 } else {
483 ("", path)
484 };
485 let parent_path = parent_path.to_string();
486 let path = path.into();
487 let (parent_source, parent_index) = builder.ensure_source(&parent_path);
488 parent_source.child_source_indices.push(i);
489 builder.sources[i as usize].source.parent_source_index = Some(parent_index);
490 builder.sources[i as usize].source.path = path;
491 }
492 i += 1;
493 }
494
495 let rope = builder.build();
496 Ok(FileContent::Content(File::from(rope)).cell())
497}
498
499#[turbo_tasks::function]
500pub async fn analyze_module_graphs(module_graph: Vc<ModuleGraph>) -> Result<Vc<FileContent>> {
501 let mut builder = ModulesDataBuilder::new();
502
503 let mut all_modules = FxIndexSet::default();
504 let mut all_edges = FxIndexSet::default();
505 let mut all_async_edges = FxIndexSet::default();
506 let mut all_traced_edges = FxIndexSet::default();
507 let mut traced_modules = FxHashSet::default();
508
509 let module_graph = module_graph.await?;
510 module_graph.traverse_edges_dfs(
511 module_graph.all_entry_modules(),
512 &mut (),
513 |parent, node, _| {
514 all_modules.insert(node);
515 let Some((parent_node, reference)) = parent else {
516 return Ok(GraphTraversalAction::Continue);
517 };
518
519 if matches!(
523 reference.chunking_type,
524 ChunkingType::Traced {
525 mode: TracedMode::Entry
526 }
527 ) || traced_modules.contains(&parent_node)
528 {
529 traced_modules.insert(node);
530 all_traced_edges.insert((parent_node, node));
531 return Ok(GraphTraversalAction::Continue);
532 };
533
534 match reference.chunking_type {
535 ChunkingType::Async => {
536 all_async_edges.insert((parent_node, node));
537 }
538 _ => {
539 all_edges.insert((parent_node, node));
540 }
541 }
542 Ok(GraphTraversalAction::Continue)
543 },
544 |_, _, _| Ok(()),
545 true,
546 )?;
547
548 type ModulePair = (ResolvedVc<Box<dyn Module>>, ResolvedVc<Box<dyn Module>>);
549 async fn mapper((from, to): ModulePair) -> Result<Option<(RcStr, RcStr)>> {
550 if from == to {
551 return Ok(None);
552 }
553 let from_ident = from.ident().to_string().owned().await?;
554 let to_ident = to.ident().to_string().owned().await?;
555 Ok(Some((from_ident, to_ident)))
556 }
557
558 let all_modules = all_modules
559 .iter()
560 .copied()
561 .map(async |module| {
562 let ident = module.ident().to_string().owned().await?;
563 let path = module.ident().await?.path.to_string_ref().await?;
564 Ok((ident, path))
565 })
566 .try_join()
567 .await?;
568
569 for (ident, path) in &all_modules {
570 builder.ensure_module(ident, path);
571 }
572
573 let all_edges = all_edges
574 .iter()
575 .copied()
576 .map(mapper)
577 .try_flat_join()
578 .await?;
579 let all_async_edges = all_async_edges
580 .iter()
581 .copied()
582 .map(mapper)
583 .try_flat_join()
584 .await?;
585 let all_traced_edges = all_traced_edges
586 .iter()
587 .copied()
588 .map(mapper)
589 .try_flat_join()
590 .await?;
591 for (from_ident, to_ident) in all_edges {
592 let from_index = builder.get_module(&from_ident).1;
593 let to_index = builder.get_module(&to_ident).1;
594 if from_index == to_index {
595 continue;
596 }
597 builder.modules[from_index as usize]
598 .dependencies
599 .insert(to_index);
600 builder.modules[to_index as usize]
601 .dependents
602 .insert(from_index);
603 }
604 for (from_ident, to_ident) in all_async_edges {
605 let from_index = builder.get_module(&from_ident).1;
606 let to_index = builder.get_module(&to_ident).1;
607 if from_index == to_index {
608 continue;
609 }
610 builder.modules[from_index as usize]
611 .async_dependencies
612 .insert(to_index);
613 builder.modules[to_index as usize]
614 .async_dependents
615 .insert(from_index);
616 }
617 for (from_ident, to_ident) in all_traced_edges {
618 let from_index = builder.get_module(&from_ident).1;
619 let to_index = builder.get_module(&to_ident).1;
620 if from_index == to_index {
621 continue;
622 }
623 builder.modules[from_index as usize]
624 .traced_dependencies
625 .insert(to_index);
626 builder.modules[to_index as usize]
627 .traced_dependents
628 .insert(from_index);
629 }
630
631 let rope = builder.build();
632 Ok(FileContent::Content(File::from(rope)).cell())
633}
634
635#[turbo_tasks::value]
636pub struct AnalyzeDataOutputAsset {
637 pub path: FileSystemPath,
638 pub output_assets: ResolvedVc<OutputAssets>,
639 pub traced_files: ResolvedVc<FileSystemPathVec>,
640}
641
642#[turbo_tasks::value_impl]
643impl AnalyzeDataOutputAsset {
644 #[turbo_tasks::function]
645 pub async fn new(
646 path: FileSystemPath,
647 output_assets: ResolvedVc<OutputAssets>,
648 traced_files: ResolvedVc<FileSystemPathVec>,
649 ) -> Result<Vc<Self>> {
650 Ok(Self {
651 path,
652 output_assets,
653 traced_files,
654 }
655 .cell())
656 }
657}
658
659#[turbo_tasks::value_impl]
660impl Asset for AnalyzeDataOutputAsset {
661 #[turbo_tasks::function]
662 fn content(&self) -> Vc<AssetContent> {
663 let file_content = analyze_output_assets(*self.output_assets, *self.traced_files);
664 AssetContent::file(file_content)
665 }
666}
667
668#[turbo_tasks::value_impl]
669impl OutputAssetsReference for AnalyzeDataOutputAsset {}
670
671#[turbo_tasks::value_impl]
672impl OutputAsset for AnalyzeDataOutputAsset {
673 #[turbo_tasks::function]
674 fn path(&self) -> Vc<FileSystemPath> {
675 self.path.clone().cell()
676 }
677}
678
679#[turbo_tasks::value]
680pub struct ModulesDataOutputAsset {
681 pub path: FileSystemPath,
682 pub module_graph: ResolvedVc<ModuleGraph>,
683}
684
685#[turbo_tasks::value_impl]
686impl ModulesDataOutputAsset {
687 #[turbo_tasks::function]
688 pub async fn new(
689 path: FileSystemPath,
690 module_graph: ResolvedVc<ModuleGraph>,
691 ) -> Result<Vc<Self>> {
692 Ok(Self { path, module_graph }.cell())
693 }
694}
695
696#[turbo_tasks::value_impl]
697impl Asset for ModulesDataOutputAsset {
698 #[turbo_tasks::function]
699 fn content(&self) -> Vc<AssetContent> {
700 let file_content = analyze_module_graphs(*self.module_graph);
701 AssetContent::file(file_content)
702 }
703}
704
705#[turbo_tasks::value_impl]
706impl OutputAssetsReference for ModulesDataOutputAsset {}
707
708#[turbo_tasks::value_impl]
709impl OutputAsset for ModulesDataOutputAsset {
710 #[turbo_tasks::function]
711 fn path(&self) -> Vc<FileSystemPath> {
712 self.path.clone().cell()
713 }
714}