next_api/
analyze.rs

1use std::{borrow::Cow, io::Write};
2
3use anyhow::Result;
4use byteorder::{BE, WriteBytesExt};
5use rustc_hash::FxHashMap;
6use serde::Serialize;
7use turbo_rcstr::RcStr;
8use turbo_tasks::{FxIndexSet, ResolvedVc, TryFlatJoinIterExt, TryJoinIterExt, ValueToString, Vc};
9use turbo_tasks_fs::{
10    File, FileContent, FileSystemPath,
11    rope::{Rope, RopeBuilder},
12};
13use turbopack_analyze::split_chunk::split_output_asset_into_parts;
14use turbopack_core::{
15    SOURCE_URL_PROTOCOL,
16    asset::{Asset, AssetContent},
17    chunk::ChunkingType,
18    module::Module,
19    output::{OutputAsset, OutputAssets, OutputAssetsReference},
20    reference::all_assets_from_entries,
21};
22
23use crate::route::ModuleGraphs;
24
25pub struct EdgesData {
26    pub offsets: Vec<u32>,
27    pub data: Vec<u32>,
28}
29
30impl EdgesData {
31    fn from_iterator<'a>(iterable: impl IntoIterator<Item = &'a Vec<u32>> + Clone) -> Self {
32        let mut current_offset = 0;
33        let sum: usize = iterable.clone().into_iter().map(|v| v.len()).sum();
34        let mut data = Vec::with_capacity(sum);
35        let offsets = iterable
36            .into_iter()
37            .map(|edges| {
38                current_offset += edges.len() as u32;
39                data.extend(edges);
40                current_offset
41            })
42            .collect();
43        Self { offsets, data }
44    }
45
46    fn write(&self, writer: &mut impl Write) -> Result<()> {
47        writer.write_u32::<BE>(self.offsets.len() as u32)?;
48        for &offset in &self.offsets {
49            writer.write_u32::<BE>(offset)?;
50        }
51        for &data in &self.data {
52            writer.write_u32::<BE>(data)?;
53        }
54        Ok(())
55    }
56}
57
58#[derive(Serialize)]
59pub struct AnalyzeSource {
60    pub parent_source_index: Option<u32>,
61    /// Path. When there is a parent, this is concatenated to the parent's path.
62    /// Folders end with a slash. Might have multiple path segments when folders contain only a
63    /// single child.
64    pub path: RcStr,
65}
66
67#[derive(Serialize)]
68pub struct AnalyzeModule {
69    pub ident: RcStr,
70    pub path: RcStr,
71}
72
73#[derive(Serialize)]
74pub struct AnalyzeChunkPart {
75    pub source_index: u32,
76    pub output_file_index: u32,
77    pub size: u32,
78    pub compressed_size: u32,
79}
80
81#[derive(Serialize)]
82pub struct AnalyzeOutputFile {
83    pub filename: RcStr,
84}
85
86#[derive(Serialize)]
87struct EdgesDataReference {
88    pub offset: u32,
89    pub length: u32,
90}
91
92#[derive(Serialize)]
93struct AnalyzeDataHeader {
94    pub sources: Vec<AnalyzeSource>,
95    pub chunk_parts: Vec<AnalyzeChunkPart>,
96    pub output_files: Vec<AnalyzeOutputFile>,
97    /// Edges from chunks to chunk parts
98    pub output_file_chunk_parts: EdgesDataReference,
99    /// Edges from sources to chunk parts
100    pub source_chunk_parts: EdgesDataReference,
101    /// Edges from sources to their children sources
102    pub source_children: EdgesDataReference,
103    /// Root level sources, walking their children will reach all sources
104    pub source_roots: Vec<u32>,
105}
106
107#[derive(Serialize)]
108struct ModulesDataHeader {
109    pub modules: Vec<AnalyzeModule>,
110    /// Edges from modules to modules
111    pub module_dependents: EdgesDataReference,
112    /// Edges from modules to modules
113    pub async_module_dependents: EdgesDataReference,
114    /// Edges from modules to modules
115    pub module_dependencies: EdgesDataReference,
116    /// Edges from modules to modules
117    pub async_module_dependencies: EdgesDataReference,
118}
119
120struct AnalyzeOutputFileBuilder {
121    output_file: AnalyzeOutputFile,
122    chunk_part_indices: Vec<u32>,
123}
124
125struct AnalyzeSourceBuilder {
126    source: AnalyzeSource,
127    child_source_indices: Vec<u32>,
128    chunk_part_indices: Vec<u32>,
129}
130
131struct AnalyzeModuleBuilder {
132    module: AnalyzeModule,
133    dependencies: FxIndexSet<u32>,
134    async_dependencies: FxIndexSet<u32>,
135    dependents: FxIndexSet<u32>,
136    async_dependents: FxIndexSet<u32>,
137}
138
139struct AnalyzeDataBuilder {
140    sources: Vec<AnalyzeSourceBuilder>,
141    source_index_map: FxHashMap<RcStr, u32>,
142    chunk_parts: Vec<AnalyzeChunkPart>,
143    output_files: Vec<AnalyzeOutputFileBuilder>,
144}
145
146struct ModulesDataBuilder {
147    modules: Vec<AnalyzeModuleBuilder>,
148    module_index_map: FxHashMap<RcStr, u32>,
149}
150
151struct EdgesDataSectionBuilder {
152    data: Vec<u8>,
153}
154
155impl EdgesDataSectionBuilder {
156    fn new() -> Self {
157        Self { data: vec![] }
158    }
159
160    fn add_edges(&mut self, edges: &EdgesData) -> EdgesDataReference {
161        let offset = self.data.len().try_into().unwrap();
162        edges.write(&mut self.data).unwrap();
163        let length = (self.data.len() - offset as usize).try_into().unwrap();
164        EdgesDataReference { offset, length }
165    }
166}
167
168impl AnalyzeDataBuilder {
169    fn new() -> Self {
170        Self {
171            sources: vec![],
172            source_index_map: FxHashMap::default(),
173            chunk_parts: vec![],
174            output_files: vec![],
175        }
176    }
177
178    fn ensure_source(&mut self, path: &str) -> (&mut AnalyzeSourceBuilder, u32) {
179        if let Some(&index) = self.source_index_map.get(path) {
180            return (&mut self.sources[index as usize], index);
181        }
182        let index = self.sources.len() as u32;
183        let path = RcStr::from(path);
184        self.source_index_map.insert(path.clone(), index);
185        self.sources.push(AnalyzeSourceBuilder {
186            source: AnalyzeSource {
187                parent_source_index: None,
188                path,
189            },
190            child_source_indices: vec![],
191            chunk_part_indices: vec![],
192        });
193        (&mut self.sources[index as usize], index)
194    }
195
196    fn add_chunk_part(&mut self, chunk_part: AnalyzeChunkPart) -> u32 {
197        let i = self.chunk_parts.len() as u32;
198        self.chunk_parts.push(chunk_part);
199        i
200    }
201
202    fn add_output_file(&mut self, output_file: AnalyzeOutputFile) -> u32 {
203        let i = self.output_files.len() as u32;
204        self.output_files.push(AnalyzeOutputFileBuilder {
205            output_file,
206            chunk_part_indices: vec![],
207        });
208        i
209    }
210
211    fn add_chunk_part_to_output_file(&mut self, output_file_index: u32, chunk_part_index: u32) {
212        self.output_files[output_file_index as usize]
213            .chunk_part_indices
214            .push(chunk_part_index);
215    }
216
217    fn add_chunk_part_to_source(&mut self, source_index: u32, chunk_part_index: u32) {
218        self.sources[source_index as usize]
219            .chunk_part_indices
220            .push(chunk_part_index);
221    }
222
223    fn build(self) -> Rope {
224        let source_roots = self
225            .sources
226            .iter()
227            .enumerate()
228            .filter_map(|(i, s)| {
229                if s.source.parent_source_index.is_none() {
230                    Some(i as u32)
231                } else {
232                    None
233                }
234            })
235            .collect();
236
237        let source_children =
238            EdgesData::from_iterator(self.sources.iter().map(|s| &s.child_source_indices));
239
240        let source_chunk_parts =
241            EdgesData::from_iterator(self.sources.iter().map(|s| &s.chunk_part_indices));
242
243        let output_file_chunk_parts =
244            EdgesData::from_iterator(self.output_files.iter().map(|of| &of.chunk_part_indices));
245
246        let mut binary_section = EdgesDataSectionBuilder::new();
247
248        let header = AnalyzeDataHeader {
249            sources: self.sources.into_iter().map(|s| s.source).collect(),
250            chunk_parts: self.chunk_parts,
251            output_files: self
252                .output_files
253                .into_iter()
254                .map(|of| of.output_file)
255                .collect(),
256            output_file_chunk_parts: binary_section.add_edges(&output_file_chunk_parts),
257            source_chunk_parts: binary_section.add_edges(&source_chunk_parts),
258            source_children: binary_section.add_edges(&source_children),
259            source_roots,
260        };
261
262        let header_json = serde_json::to_vec(&header).unwrap();
263
264        let mut rope = RopeBuilder::default();
265        rope.push_bytes(&(header_json.len() as u32).to_be_bytes());
266        rope.reserve_bytes(header_json.len() + binary_section.data.len());
267        rope.push_bytes(&header_json);
268        rope.push_bytes(&binary_section.data);
269        rope.build()
270    }
271}
272
273impl ModulesDataBuilder {
274    fn new() -> Self {
275        Self {
276            modules: vec![],
277            module_index_map: FxHashMap::default(),
278        }
279    }
280
281    fn get_module(&mut self, ident: &str) -> (&mut AnalyzeModuleBuilder, u32) {
282        if let Some(&index) = self.module_index_map.get(ident) {
283            return (&mut self.modules[index as usize], index);
284        }
285        panic!("Module with ident `{}` not found", ident);
286    }
287
288    fn ensure_module(&mut self, ident: &str, path: &str) -> (&mut AnalyzeModuleBuilder, u32) {
289        if let Some(&index) = self.module_index_map.get(ident) {
290            return (&mut self.modules[index as usize], index);
291        }
292        let index = self.modules.len() as u32;
293        let ident = RcStr::from(ident);
294        let path = RcStr::from(path);
295        self.module_index_map.insert(ident.clone(), index);
296        self.modules.push(AnalyzeModuleBuilder {
297            module: AnalyzeModule { ident, path },
298            dependencies: FxIndexSet::default(),
299            async_dependencies: FxIndexSet::default(),
300            dependents: FxIndexSet::default(),
301            async_dependents: FxIndexSet::default(),
302        });
303        (&mut self.modules[index as usize], index)
304    }
305
306    fn build(self) -> Rope {
307        let module_dependencies_vecs: Vec<Vec<u32>> = self
308            .modules
309            .iter()
310            .map(|s| s.dependencies.iter().copied().collect())
311            .collect();
312        let async_module_dependencies_vecs: Vec<Vec<u32>> = self
313            .modules
314            .iter()
315            .map(|s| s.async_dependencies.iter().copied().collect())
316            .collect();
317        let module_dependents_vecs: Vec<Vec<u32>> = self
318            .modules
319            .iter()
320            .map(|s| s.dependents.iter().copied().collect())
321            .collect();
322        let async_module_dependents_vecs: Vec<Vec<u32>> = self
323            .modules
324            .iter()
325            .map(|s| s.async_dependents.iter().copied().collect())
326            .collect();
327
328        let module_dependencies = EdgesData::from_iterator(&module_dependencies_vecs);
329        let async_module_dependencies = EdgesData::from_iterator(&async_module_dependencies_vecs);
330        let module_dependents = EdgesData::from_iterator(&module_dependents_vecs);
331        let async_module_dependents = EdgesData::from_iterator(&async_module_dependents_vecs);
332
333        let mut binary_section = EdgesDataSectionBuilder::new();
334
335        let header = ModulesDataHeader {
336            modules: self.modules.into_iter().map(|s| s.module).collect(),
337            module_dependents: binary_section.add_edges(&module_dependents),
338            async_module_dependents: binary_section.add_edges(&async_module_dependents),
339            module_dependencies: binary_section.add_edges(&module_dependencies),
340            async_module_dependencies: binary_section.add_edges(&async_module_dependencies),
341        };
342
343        let header_json = serde_json::to_vec(&header).unwrap();
344
345        let mut rope = RopeBuilder::default();
346        rope.push_bytes(&(header_json.len() as u32).to_be_bytes());
347        rope.reserve_bytes(header_json.len() + binary_section.data.len());
348        rope.push_bytes(&header_json);
349        rope.push_bytes(&binary_section.data);
350        rope.build()
351    }
352}
353
354#[turbo_tasks::function]
355pub async fn analyze_output_assets(output_assets: Vc<OutputAssets>) -> Result<Vc<FileContent>> {
356    let output_assets = all_assets_from_entries(output_assets);
357
358    let mut builder = AnalyzeDataBuilder::new();
359
360    let prefix = format!("{SOURCE_URL_PROTOCOL}///");
361
362    // Process the output assets and extract chunk parts.
363    // Also creates sources for the chunk parts.
364    for &asset in output_assets.await? {
365        let filename = asset.path().to_string().owned().await?;
366        if filename.ends_with(".map") || filename.ends_with(".nft.json") {
367            // Skip source maps.
368            continue;
369        }
370
371        let output_file_index = builder.add_output_file(AnalyzeOutputFile { filename });
372        let chunk_parts = split_output_asset_into_parts(*asset).await?;
373        for chunk_part in chunk_parts {
374            let decoded_source = urlencoding::decode(&chunk_part.source)?;
375            let source = if let Some(stripped) = decoded_source.strip_prefix(&prefix) {
376                Cow::Borrowed(stripped)
377            } else {
378                Cow::Owned(format!(
379                    "[project]/{}",
380                    decoded_source.trim_start_matches("../")
381                ))
382            };
383            let source_index = builder.ensure_source(&source).1;
384            let chunk_part_index = builder.add_chunk_part(AnalyzeChunkPart {
385                source_index,
386                output_file_index,
387                size: chunk_part.real_size + chunk_part.unaccounted_size,
388                compressed_size: chunk_part.get_compressed_size().await?,
389            });
390            builder.add_chunk_part_to_output_file(output_file_index, chunk_part_index);
391            builder.add_chunk_part_to_source(source_index, chunk_part_index);
392        }
393    }
394
395    // Build a directory structure for the sources.
396    let mut i: u32 = 0;
397    while i < builder.sources.len().try_into().unwrap() {
398        let source = &builder.sources[i as usize];
399        let path = source.source.path.as_str();
400        if !path.is_empty() {
401            let (parent_path, path) = if let Some(pos) = path.trim_end_matches('/').rfind('/') {
402                (&path[..pos + 1], &path[pos + 1..])
403            } else {
404                ("", path)
405            };
406            let parent_path = parent_path.to_string();
407            let path = path.into();
408            let (parent_source, parent_index) = builder.ensure_source(&parent_path);
409            parent_source.child_source_indices.push(i);
410            builder.sources[i as usize].source.parent_source_index = Some(parent_index);
411            builder.sources[i as usize].source.path = path;
412        }
413        i += 1;
414    }
415
416    let rope = builder.build();
417    Ok(FileContent::Content(File::from(rope)).cell())
418}
419
420#[turbo_tasks::function]
421pub async fn analyze_module_graphs(module_graphs: Vc<ModuleGraphs>) -> Result<Vc<FileContent>> {
422    let mut builder = ModulesDataBuilder::new();
423
424    let mut all_modules = FxIndexSet::default();
425    let mut all_edges = FxIndexSet::default();
426    let mut all_async_edges = FxIndexSet::default();
427    for &module_graph in module_graphs.await? {
428        let module_graph = module_graph.await?;
429        module_graph.traverse_edges_unordered(|parent, node| {
430            if let Some((parent_node, reference)) = parent {
431                all_modules.insert(parent_node);
432                all_modules.insert(node);
433                match reference.chunking_type {
434                    ChunkingType::Async => {
435                        all_async_edges.insert((parent_node, node));
436                    }
437                    _ => {
438                        all_edges.insert((parent_node, node));
439                    }
440                }
441            }
442            Ok(())
443        })?;
444    }
445
446    type ModulePair = (ResolvedVc<Box<dyn Module>>, ResolvedVc<Box<dyn Module>>);
447    async fn mapper((from, to): ModulePair) -> Result<Option<(RcStr, RcStr)>> {
448        if from == to {
449            return Ok(None);
450        }
451        let from_ident = from.ident().to_string().owned().await?;
452        let to_ident = to.ident().to_string().owned().await?;
453        Ok(Some((from_ident, to_ident)))
454    }
455
456    let all_modules = all_modules
457        .iter()
458        .copied()
459        .map(async |module| {
460            let ident = module.ident().to_string().owned().await?;
461            let path = module.ident().path().to_string().owned().await?;
462            Ok((ident, path))
463        })
464        .try_join()
465        .await?;
466
467    for (ident, path) in all_modules {
468        builder.ensure_module(&ident, &path);
469    }
470
471    let all_edges = all_edges
472        .iter()
473        .copied()
474        .map(mapper)
475        .try_flat_join()
476        .await?;
477    let all_async_edges = all_async_edges
478        .iter()
479        .copied()
480        .map(mapper)
481        .try_flat_join()
482        .await?;
483    for (from_ident, to_ident) in all_edges {
484        let from_index = builder.get_module(&from_ident).1;
485        let to_index = builder.get_module(&to_ident).1;
486        if from_index == to_index {
487            continue;
488        }
489        builder.modules[from_index as usize]
490            .dependencies
491            .insert(to_index);
492        builder.modules[to_index as usize]
493            .dependents
494            .insert(from_index);
495    }
496    for (from_ident, to_ident) in all_async_edges {
497        let from_index = builder.get_module(&from_ident).1;
498        let to_index = builder.get_module(&to_ident).1;
499        if from_index == to_index {
500            continue;
501        }
502        builder.modules[from_index as usize]
503            .async_dependencies
504            .insert(to_index);
505        builder.modules[to_index as usize]
506            .async_dependents
507            .insert(from_index);
508    }
509
510    let rope = builder.build();
511    Ok(FileContent::Content(File::from(rope)).cell())
512}
513
514#[turbo_tasks::value]
515pub struct AnalyzeDataOutputAsset {
516    pub path: FileSystemPath,
517    pub output_assets: ResolvedVc<OutputAssets>,
518}
519
520#[turbo_tasks::value_impl]
521impl AnalyzeDataOutputAsset {
522    #[turbo_tasks::function]
523    pub async fn new(
524        path: FileSystemPath,
525        output_assets: ResolvedVc<OutputAssets>,
526    ) -> Result<Vc<Self>> {
527        Ok(Self {
528            path,
529            output_assets,
530        }
531        .cell())
532    }
533}
534
535#[turbo_tasks::value_impl]
536impl Asset for AnalyzeDataOutputAsset {
537    #[turbo_tasks::function]
538    fn content(&self) -> Vc<AssetContent> {
539        let file_content = analyze_output_assets(*self.output_assets);
540        AssetContent::file(file_content)
541    }
542}
543
544#[turbo_tasks::value_impl]
545impl OutputAssetsReference for AnalyzeDataOutputAsset {}
546
547#[turbo_tasks::value_impl]
548impl OutputAsset for AnalyzeDataOutputAsset {
549    #[turbo_tasks::function]
550    fn path(&self) -> Vc<FileSystemPath> {
551        self.path.clone().cell()
552    }
553}
554
555#[turbo_tasks::value]
556pub struct ModulesDataOutputAsset {
557    pub path: FileSystemPath,
558    pub module_graphs: ResolvedVc<ModuleGraphs>,
559}
560
561#[turbo_tasks::value_impl]
562impl ModulesDataOutputAsset {
563    #[turbo_tasks::function]
564    pub async fn new(path: FileSystemPath, module_graphs: Vc<ModuleGraphs>) -> Result<Vc<Self>> {
565        Ok(Self {
566            path,
567            module_graphs: module_graphs.to_resolved().await?,
568        }
569        .cell())
570    }
571}
572
573#[turbo_tasks::value_impl]
574impl Asset for ModulesDataOutputAsset {
575    #[turbo_tasks::function]
576    fn content(&self) -> Vc<AssetContent> {
577        let file_content = analyze_module_graphs(*self.module_graphs);
578        AssetContent::file(file_content)
579    }
580}
581
582#[turbo_tasks::value_impl]
583impl OutputAssetsReference for ModulesDataOutputAsset {}
584
585#[turbo_tasks::value_impl]
586impl OutputAsset for ModulesDataOutputAsset {
587    #[turbo_tasks::function]
588    fn path(&self) -> Vc<FileSystemPath> {
589        self.path.clone().cell()
590    }
591}
next_api/analyze.rs

next_api/
analyze.rs
