turbopack_analyze/
split_chunk.rs

1use std::mem::replace;
2
3use anyhow::Result;
4use bincode::{Decode, Encode};
5use turbo_rcstr::RcStr;
6use turbo_tasks::{FxIndexMap, NonLocalValue, ResolvedVc, ValueToString, ValueToStringRef, Vc};
7use turbo_tasks_fs::{FileContent, FileLine, FileLinesContent, FileSystemPath, rope::Rope};
8use turbopack_core::{
9    asset::{Asset, AssetContent},
10    file_source::FileSource,
11    output::OutputAsset,
12    source_map::{GenerateSourceMap, OriginalToken, SourceMap, Token},
13};
14
15use crate::compressed_size::compressed_size_bytes;
16
17#[derive(Clone, Debug, Eq, NonLocalValue, PartialEq, Encode, Decode)]
18pub struct ChunkPartRange {
19    pub line: u32,
20    pub start_column: u32,
21    pub end_column: u32,
22}
23
24#[derive(Clone, Debug, Eq, NonLocalValue, PartialEq, Encode, Decode)]
25pub struct ChunkPart {
26    pub source: RcStr,
27    pub real_size: u32,
28    pub unaccounted_size: u32,
29    pub lines: ResolvedVc<FileLinesContent>,
30    pub ranges: Vec<ChunkPartRange>,
31}
32
33impl ChunkPart {
34    pub async fn get_compressed_size(&self) -> Result<Option<u32>> {
35        let lines = &*self.lines.await?;
36        let FileLinesContent::Lines(lines) = lines else {
37            return Ok(None);
38        };
39
40        if self.ranges.is_empty() {
41            let mut all_content = String::new();
42            for line in lines {
43                all_content.push_str(&line.content);
44            }
45            Ok(Some(compressed_size_bytes(all_content)?))
46        } else {
47            let mut all_range_content = String::new();
48            for range in &self.ranges {
49                append_content_between(
50                    range.line,
51                    range.start_column,
52                    range.line,
53                    range.end_column,
54                    lines,
55                    &mut all_range_content,
56                );
57            }
58            Ok(Some(compressed_size_bytes(all_range_content)?))
59        }
60    }
61}
62
63pub async fn split_traced_file_into_parts(path: FileSystemPath) -> Result<Vec<ChunkPart>> {
64    let source = FileSource::new(path.clone());
65    let content = source.content().await?;
66    let AssetContent::File(file_content) = &*content else {
67        return Ok(vec![]);
68    };
69    let FileContent::Content(content) = &*file_content.await? else {
70        return Ok(vec![]);
71    };
72    let content = content.content();
73    let lines_vc = file_content.lines().to_resolved().await?;
74
75    self_mapped(path.to_string_ref().await?, content, lines_vc).await
76}
77
78pub async fn split_output_asset_into_parts(
79    asset: Vc<Box<dyn OutputAsset>>,
80) -> Result<Vec<ChunkPart>> {
81    let content = asset.content().await?;
82    let AssetContent::File(file_content) = &*content else {
83        return Ok(vec![]);
84    };
85    let FileContent::Content(content) = &*file_content.await? else {
86        return Ok(vec![]);
87    };
88    let content = content.content();
89    let lines_vc = file_content.lines().to_resolved().await?;
90
91    let Some(generate_source_map) =
92        ResolvedVc::try_sidecast::<Box<dyn GenerateSourceMap>>(asset.to_resolved().await?)
93    else {
94        return self_mapped(asset.path().to_string().owned().await?, content, lines_vc).await;
95    };
96    let source_map = generate_source_map.generate_source_map().await?;
97    let Some(source_map) = source_map.as_content() else {
98        return self_mapped(asset.path().to_string().owned().await?, content, lines_vc).await;
99    };
100    let Some(source_map) = SourceMap::new_from_rope(source_map.content())? else {
101        return unaccounted(asset, content, lines_vc).await;
102    };
103
104    let lines = lines_vc.await?;
105    let FileLinesContent::Lines(lines) = &*lines else {
106        return unaccounted(asset, content, lines_vc).await;
107    };
108
109    fn end_of_mapping_column(
110        start_line: u32,
111        end_line: u32,
112        end_column: u32,
113        lines: &[FileLine],
114    ) -> u32 {
115        let start_line = start_line.min(lines.len() as u32 - 1);
116        let line_end = lines[start_line as usize].len() as u32;
117        if start_line == end_line {
118            end_column.min(line_end)
119        } else {
120            line_end
121        }
122    }
123
124    fn len_between(
125        start_line: u32,
126        start_column: u32,
127        end_line: u32,
128        end_column: u32,
129        lines: &[FileLine],
130    ) -> u32 {
131        let start_line = start_line.min(lines.len() as u32 - 1);
132        let end_line = end_line.min(lines.len() as u32 - 1);
133        if start_line == end_line {
134            // TODO: Figure out why start is larger than end sometimes
135            return end_column.saturating_sub(start_column);
136        }
137        let mut len = lines[start_line as usize].len() as u32 - start_column + 1;
138        for line in &lines[start_line as usize + 1..end_line as usize] {
139            len += line.len() as u32 + 1;
140        }
141        len += end_column;
142        len
143    }
144
145    let mut chunk_parts = FxIndexMap::default();
146    fn add_chunk_part_range(
147        source: RcStr,
148        chunk_part_range: ChunkPartRange,
149        size: u32,
150        chunk_parts: &mut FxIndexMap<RcStr, ChunkPart>,
151        lines: ResolvedVc<FileLinesContent>,
152    ) {
153        let entry = chunk_parts
154            .entry(source)
155            .or_insert_with_key(|source| ChunkPart {
156                source: source.clone(),
157                real_size: 0,
158                unaccounted_size: 0,
159                ranges: vec![],
160                lines,
161            });
162        entry.real_size += size;
163        entry.ranges.push(chunk_part_range);
164    }
165
166    fn add_unaccounted_chunk_part(
167        source: RcStr,
168        unaccounted: u32,
169        chunk_parts: &mut FxIndexMap<RcStr, ChunkPart>,
170        lines: ResolvedVc<FileLinesContent>,
171    ) {
172        let entry = chunk_parts
173            .entry(source)
174            .or_insert_with_key(|source| ChunkPart {
175                source: source.clone(),
176                real_size: 0,
177                unaccounted_size: 0,
178                ranges: vec![],
179                lines,
180            });
181        entry.unaccounted_size += unaccounted;
182    }
183
184    fn end_current_mapping(
185        source: RcStr,
186        current_line: u32,
187        start_column: u32,
188        next_line: u32,
189        next_column: u32,
190        lines: &[FileLine],
191        chunk_parts: &mut FxIndexMap<RcStr, ChunkPart>,
192        lines_vc: ResolvedVc<FileLinesContent>,
193    ) -> State {
194        let mapping_end_column = end_of_mapping_column(current_line, next_line, next_column, lines);
195        let len = mapping_end_column.saturating_sub(start_column);
196        add_chunk_part_range(
197            source.clone(),
198            ChunkPartRange {
199                line: current_line,
200                start_column,
201                end_column: mapping_end_column,
202            },
203            len,
204            chunk_parts,
205            lines_vc,
206        );
207        State::AfterMapping {
208            source,
209            generated_line: current_line,
210            current_generated_column: mapping_end_column,
211        }
212    }
213
214    fn should_extend_mapping(
215        state: &State,
216        new_source: &RcStr,
217        new_line: u32,
218        new_column: u32,
219    ) -> bool {
220        if let State::InMapping {
221            source,
222            generated_line,
223            end_column,
224            ..
225        } = state
226        {
227            // Extend if same source and line, and columns are adjacent or overlapping
228            // end_column <= new_column handles both adjacent (equal) and overlapping cases
229            source == new_source && *generated_line == new_line && *end_column <= new_column
230        } else {
231            false
232        }
233    }
234
235    enum State {
236        StartOfFile,
237        InMapping {
238            source: RcStr,
239            generated_line: u32,
240            start_column: u32,
241            end_column: u32,
242        },
243        AfterMapping {
244            source: RcStr,
245            generated_line: u32,
246            current_generated_column: u32,
247        },
248    }
249
250    let mut state: State = State::StartOfFile;
251
252    for token in source_map.tokens() {
253        if let Token::Original(OriginalToken {
254            original_file,
255            generated_line,
256            generated_column,
257            ..
258        }) = token
259        {
260            // Check if we can extend the current mapping
261            if should_extend_mapping(&state, &original_file, generated_line, generated_column) {
262                // Same source and line with adjacent columns - update end to next token position
263                if let State::InMapping {
264                    source,
265                    generated_line: current_line,
266                    start_column,
267                    ..
268                } = state
269                {
270                    state = State::InMapping {
271                        source,
272                        generated_line: current_line,
273                        start_column,
274                        end_column: generated_column,
275                    };
276                    continue;
277                }
278            }
279
280            // End the current mapping if we're in one
281            if let State::InMapping {
282                source,
283                generated_line: current_line,
284                start_column,
285                ..
286            } = state
287            {
288                state = end_current_mapping(
289                    source,
290                    current_line,
291                    start_column,
292                    generated_line,
293                    generated_column,
294                    lines,
295                    &mut chunk_parts,
296                    lines_vc,
297                );
298            }
299
300            // Start a new mapping and put the unaccounted part in between somewhere
301            // Set end_column to start_column initially; it will be updated when we see the next
302            // token
303            match replace(
304                &mut state,
305                State::InMapping {
306                    source: original_file.clone(),
307                    generated_line,
308                    start_column: generated_column,
309                    end_column: generated_column,
310                },
311            ) {
312                State::InMapping { .. } => {
313                    unreachable!();
314                }
315                State::AfterMapping {
316                    source,
317                    generated_line,
318                    current_generated_column,
319                } => {
320                    let len = len_between(
321                        generated_line,
322                        current_generated_column,
323                        generated_line,
324                        generated_column,
325                        lines,
326                    );
327                    let half = len / 2;
328                    add_unaccounted_chunk_part(source, half, &mut chunk_parts, lines_vc);
329                    add_unaccounted_chunk_part(
330                        original_file.clone(),
331                        len - half,
332                        &mut chunk_parts,
333                        lines_vc,
334                    );
335                }
336                State::StartOfFile => {
337                    let len = len_between(0, 0, generated_line, generated_column, lines);
338                    add_unaccounted_chunk_part(
339                        original_file.clone(),
340                        len,
341                        &mut chunk_parts,
342                        lines_vc,
343                    );
344                }
345            }
346        }
347    }
348    let last_line = lines.len() as u32 - 1;
349    let last_column = lines[last_line as usize].len() as u32;
350
351    // End the current token at end of file
352    if let State::InMapping {
353        ref source,
354        generated_line,
355        start_column,
356        ..
357    } = state
358    {
359        state = end_current_mapping(
360            source.clone(),
361            generated_line,
362            start_column,
363            last_line,
364            last_column,
365            lines,
366            &mut chunk_parts,
367            lines_vc,
368        );
369    }
370
371    match state {
372        State::InMapping { .. } => {
373            unreachable!();
374        }
375        State::AfterMapping {
376            source,
377            generated_line,
378            current_generated_column,
379        } => {
380            let len = len_between(
381                generated_line,
382                current_generated_column,
383                last_line,
384                last_column,
385                lines,
386            );
387            add_unaccounted_chunk_part(source, len, &mut chunk_parts, lines_vc);
388        }
389        State::StartOfFile => {
390            return unaccounted(asset, content, lines_vc).await;
391        }
392    }
393
394    Ok(chunk_parts.into_values().collect())
395}
396
397pub async fn self_mapped(
398    path: RcStr,
399    content: &Rope,
400    lines: ResolvedVc<FileLinesContent>,
401) -> Result<Vec<ChunkPart>> {
402    let len = content.len().try_into().unwrap_or(u32::MAX);
403    Ok(vec![ChunkPart {
404        source: path,
405        real_size: len,
406        unaccounted_size: 0,
407        ranges: vec![],
408        lines,
409    }])
410}
411
412async fn unaccounted(
413    asset: Vc<Box<dyn OutputAsset>>,
414    content: &Rope,
415    lines: ResolvedVc<FileLinesContent>,
416) -> Result<Vec<ChunkPart>> {
417    let len = content.len().try_into().unwrap_or(u32::MAX);
418    Ok(vec![ChunkPart {
419        source: asset.path().to_string().owned().await?,
420        real_size: 0,
421        unaccounted_size: len,
422        ranges: vec![],
423        lines,
424    }])
425}
426
427fn append_content_between(
428    start_line: u32,
429    start_column: u32,
430    end_line: u32,
431    end_column: u32,
432    lines: &[FileLine],
433    out: &mut String,
434) {
435    let start_line = start_line.min(lines.len() as u32 - 1);
436    let end_line = end_line.min(lines.len() as u32 - 1);
437
438    let start_column = start_column.min(lines[start_line as usize].len() as u32);
439    let end_column = if start_line == end_line {
440        end_column.min(lines[start_line as usize].len() as u32)
441    } else {
442        lines[start_line as usize].len() as u32
443    };
444
445    if end_column <= start_column {
446        return;
447    }
448
449    out.extend(
450        lines[start_line as usize]
451            .content
452            .chars()
453            .skip(start_column as usize)
454            .take((end_column - start_column) as usize),
455    );
456
457    if start_line == end_line {
458        return;
459    }
460
461    for line in &lines[start_line as usize + 1..end_line as usize] {
462        out.push_str(&line.content);
463    }
464
465    out.extend(
466        lines[end_line as usize]
467            .content
468            .chars()
469            .take(end_column as usize),
470    );
471}
turbopack_analyze/split_chunk.rs

turbopack_analyze/
split_chunk.rs
