turbopack_core/chunk/
mod.rs

1pub mod availability_info;
2pub mod available_modules;
3pub mod chunk_group;
4pub(crate) mod chunk_item_batch;
5pub mod chunking;
6pub(crate) mod chunking_context;
7pub(crate) mod containment_tree;
8pub(crate) mod data;
9pub(crate) mod evaluate;
10pub mod module_id_strategies;
11pub mod optimize;
12
13use std::fmt::Display;
14
15use anyhow::Result;
16use auto_hash_map::AutoSet;
17use serde::{Deserialize, Serialize};
18use turbo_rcstr::RcStr;
19use turbo_tasks::{
20    FxIndexSet, NonLocalValue, ResolvedVc, TaskInput, Upcast, ValueToString, Vc,
21    debug::ValueDebugFormat, trace::TraceRawVcs,
22};
23use turbo_tasks_hash::DeterministicHash;
24
25pub use self::{
26    chunk_item_batch::{
27        ChunkItemBatchGroup, ChunkItemBatchWithAsyncModuleInfo,
28        ChunkItemOrBatchWithAsyncModuleInfo, batch_info,
29    },
30    chunking_context::{
31        ChunkGroupResult, ChunkGroupType, ChunkingConfig, ChunkingConfigs, ChunkingContext,
32        ChunkingContextExt, EntryChunkGroupResult, MangleType, MinifyType, SourceMapsType,
33    },
34    data::{ChunkData, ChunkDataOption, ChunksData},
35    evaluate::{EvaluatableAsset, EvaluatableAssetExt, EvaluatableAssets},
36};
37use crate::{
38    asset::Asset,
39    chunk::availability_info::AvailabilityInfo,
40    ident::AssetIdent,
41    module::Module,
42    module_graph::{
43        ModuleGraph,
44        module_batch::{ChunkableModuleOrBatch, ModuleBatchGroup},
45    },
46    output::OutputAssets,
47    reference::ModuleReference,
48    resolve::ExportUsage,
49};
50
51/// A module id, which can be a number or string
52#[turbo_tasks::value(shared, operation)]
53#[derive(Debug, Clone, Hash, Ord, PartialOrd, DeterministicHash)]
54#[serde(untagged)]
55pub enum ModuleId {
56    Number(u64),
57    String(RcStr),
58}
59
60impl Display for ModuleId {
61    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
62        match self {
63            ModuleId::Number(i) => write!(f, "{i}"),
64            ModuleId::String(s) => write!(f, "{s}"),
65        }
66    }
67}
68
69#[turbo_tasks::value_impl]
70impl ValueToString for ModuleId {
71    #[turbo_tasks::function]
72    fn to_string(&self) -> Vc<RcStr> {
73        Vc::cell(self.to_string().into())
74    }
75}
76
77impl ModuleId {
78    pub fn parse(id: &str) -> Result<ModuleId> {
79        Ok(match id.parse::<u64>() {
80            Ok(i) => ModuleId::Number(i),
81            Err(_) => ModuleId::String(id.into()),
82        })
83    }
84}
85
86/// A list of module ids.
87#[turbo_tasks::value(transparent, shared)]
88pub struct ModuleIds(Vec<ResolvedVc<ModuleId>>);
89
90/// A [Module] that can be converted into a [Chunk].
91#[turbo_tasks::value_trait]
92pub trait ChunkableModule: Module + Asset {
93    #[turbo_tasks::function]
94    fn as_chunk_item(
95        self: Vc<Self>,
96        module_graph: Vc<ModuleGraph>,
97        chunking_context: Vc<Box<dyn ChunkingContext>>,
98    ) -> Vc<Box<dyn ChunkItem>>;
99}
100
101#[turbo_tasks::value(transparent)]
102pub struct ChunkableModules(Vec<ResolvedVc<Box<dyn ChunkableModule>>>);
103
104#[turbo_tasks::value_impl]
105impl ChunkableModules {
106    #[turbo_tasks::function]
107    pub fn interned(modules: Vec<ResolvedVc<Box<dyn ChunkableModule>>>) -> Vc<Self> {
108        Vc::cell(modules)
109    }
110}
111
112/// A [Module] that can be merged with other [Module]s (to perform scope hoisting)
113// TODO currently this is only used for ecmascript modules, and with the current API cannot be used
114// with other module types (as a MergeableModule cannot prevent itself from being merged with other
115// module types)
116#[turbo_tasks::value_trait]
117pub trait MergeableModule: Module + Asset {
118    /// Even though MergeableModule is implemented, this allows a dynamic condition to determine
119    /// mergeability
120    #[turbo_tasks::function]
121    fn is_mergeable(self: Vc<Self>) -> Vc<bool> {
122        Vc::cell(true)
123    }
124
125    /// Create a new module representing the merged content of the given `modules`.
126    ///
127    /// Group entry points are not referenced by any other module in the group. This list is needed
128    /// because the merged module is created by recursively inlining modules when they are imported,
129    /// but this process has to start somewhere (= with these entry points).
130    #[turbo_tasks::function]
131    fn merge(
132        self: Vc<Self>,
133        modules: Vc<MergeableModulesExposed>,
134        entry_points: Vc<MergeableModules>,
135    ) -> Vc<Box<dyn ChunkableModule>>;
136}
137#[turbo_tasks::value(transparent)]
138pub struct MergeableModules(Vec<ResolvedVc<Box<dyn MergeableModule>>>);
139
140#[turbo_tasks::value_impl]
141impl MergeableModules {
142    #[turbo_tasks::function]
143    pub fn interned(modules: Vec<ResolvedVc<Box<dyn MergeableModule>>>) -> Vc<Self> {
144        Vc::cell(modules)
145    }
146}
147
148/// Whether a given module needs to be exposed (depending on how it is imported by other modules)
149#[derive(
150    Copy,
151    Clone,
152    Debug,
153    PartialEq,
154    Eq,
155    Serialize,
156    Deserialize,
157    TraceRawVcs,
158    NonLocalValue,
159    TaskInput,
160    Hash,
161)]
162pub enum MergeableModuleExposure {
163    // This module is only used from within the current group, and only individual exports are
164    // used (and no namespace object is required).
165    None,
166    // This module is only used from within the current group, and but the namespace object is
167    // needed.
168    Internal,
169    // The exports of this module are read from outside this group (necessitating a namespace
170    // object anyway).
171    External,
172}
173
174#[turbo_tasks::value(transparent)]
175pub struct MergeableModulesExposed(
176    Vec<(
177        ResolvedVc<Box<dyn MergeableModule>>,
178        MergeableModuleExposure,
179    )>,
180);
181
182#[turbo_tasks::value_impl]
183impl MergeableModulesExposed {
184    #[turbo_tasks::function]
185    pub fn interned(
186        modules: Vec<(
187            ResolvedVc<Box<dyn MergeableModule>>,
188            MergeableModuleExposure,
189        )>,
190    ) -> Vc<Self> {
191        Vc::cell(modules)
192    }
193}
194
195#[turbo_tasks::value(transparent)]
196pub struct Chunks(Vec<ResolvedVc<Box<dyn Chunk>>>);
197
198#[turbo_tasks::value_impl]
199impl Chunks {
200    /// Creates a new empty [Vc<Chunks>].
201    #[turbo_tasks::function]
202    pub fn empty() -> Vc<Self> {
203        Vc::cell(vec![])
204    }
205}
206
207/// A [Chunk] group chunk items together into something that will become an [OutputAsset].
208/// It usually contains multiple chunk items.
209// TODO This could be simplified to and merged with [OutputChunk]
210#[turbo_tasks::value_trait]
211pub trait Chunk {
212    #[turbo_tasks::function]
213    fn ident(self: Vc<Self>) -> Vc<AssetIdent>;
214    #[turbo_tasks::function]
215    fn chunking_context(self: Vc<Self>) -> Vc<Box<dyn ChunkingContext>>;
216    // fn path(self: Vc<Self>) -> Vc<FileSystemPath> {
217    //     self.ident().path()
218    // }
219
220    /// Other [OutputAsset]s referenced from this [Chunk].
221    #[turbo_tasks::function]
222    fn references(self: Vc<Self>) -> Vc<OutputAssets> {
223        OutputAssets::empty()
224    }
225
226    #[turbo_tasks::function]
227    fn chunk_items(self: Vc<Self>) -> Vc<ChunkItems> {
228        ChunkItems(vec![]).cell()
229    }
230}
231
232/// Aggregated information about a chunk content that can be used by the runtime
233/// code to optimize chunk loading.
234#[turbo_tasks::value(shared)]
235#[derive(Default)]
236pub struct OutputChunkRuntimeInfo {
237    pub included_ids: Option<ResolvedVc<ModuleIds>>,
238    pub excluded_ids: Option<ResolvedVc<ModuleIds>>,
239    /// List of paths of chunks containing individual modules that are part of
240    /// this chunk. This is useful for selectively loading modules from a chunk
241    /// without loading the whole chunk.
242    pub module_chunks: Option<ResolvedVc<OutputAssets>>,
243    pub placeholder_for_future_extensions: (),
244}
245
246#[turbo_tasks::value_impl]
247impl OutputChunkRuntimeInfo {
248    #[turbo_tasks::function]
249    pub fn empty() -> Vc<Self> {
250        Self::default().cell()
251    }
252}
253
254#[turbo_tasks::value_trait]
255pub trait OutputChunk: Asset {
256    #[turbo_tasks::function]
257    fn runtime_info(self: Vc<Self>) -> Vc<OutputChunkRuntimeInfo>;
258}
259
260/// Specifies how a chunk interacts with other chunks when building a chunk
261/// group
262#[derive(
263    Debug,
264    Clone,
265    Hash,
266    TraceRawVcs,
267    Serialize,
268    Deserialize,
269    Eq,
270    PartialEq,
271    ValueDebugFormat,
272    NonLocalValue,
273)]
274pub enum ChunkingType {
275    /// The referenced module is placed in the same chunk group and is loaded in parallel.
276    Parallel {
277        /// Whether the parent module becomes an async module when the referenced module is async.
278        /// This should happen for e.g. ESM imports, but not for CommonJS requires.
279        inherit_async: bool,
280        /// Whether the referenced module is executed always immediately before the parent module
281        /// (corresponding to ESM import semantics).
282        hoisted: bool,
283    },
284    /// An async loader is placed into the referencing chunk and loads the
285    /// separate chunk group in which the module is placed.
286    Async,
287    /// Create a new chunk group in a separate context, merging references with the same tag into a
288    /// single chunk group. It does not inherit the available modules from the parent.
289    // TODO this is currently skipped in chunking
290    Isolated {
291        _ty: ChunkGroupType,
292        merge_tag: Option<RcStr>,
293    },
294    /// Create a new chunk group in a separate context, merging references with the same tag into a
295    /// single chunk group. It provides available modules to the current chunk group. It's assumed
296    /// to be loaded before the current chunk group.
297    Shared {
298        inherit_async: bool,
299        merge_tag: Option<RcStr>,
300    },
301    // Module not placed in chunk group, but its references are still followed.
302    Traced,
303}
304
305impl Display for ChunkingType {
306    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
307        match self {
308            ChunkingType::Parallel {
309                inherit_async,
310                hoisted,
311            } => {
312                write!(
313                    f,
314                    "Parallel(inherit_async: {inherit_async}, hoisted: {hoisted})",
315                )
316            }
317            ChunkingType::Async => write!(f, "Async"),
318            ChunkingType::Isolated {
319                _ty,
320                merge_tag: Some(merge_tag),
321            } => {
322                write!(f, "Isolated(merge_tag: {merge_tag})")
323            }
324            ChunkingType::Isolated {
325                _ty,
326                merge_tag: None,
327            } => {
328                write!(f, "Isolated")
329            }
330            ChunkingType::Shared {
331                inherit_async,
332                merge_tag: Some(merge_tag),
333            } => {
334                write!(
335                    f,
336                    "Shared(inherit_async: {inherit_async}, merge_tag: {merge_tag})"
337                )
338            }
339            ChunkingType::Shared {
340                inherit_async,
341                merge_tag: None,
342            } => {
343                write!(f, "Shared(inherit_async: {inherit_async})")
344            }
345            ChunkingType::Traced => write!(f, "Traced"),
346        }
347    }
348}
349
350impl ChunkingType {
351    pub fn is_inherit_async(&self) -> bool {
352        matches!(
353            self,
354            ChunkingType::Parallel {
355                inherit_async: true,
356                ..
357            } | ChunkingType::Shared {
358                inherit_async: true,
359                ..
360            }
361        )
362    }
363
364    pub fn is_parallel(&self) -> bool {
365        matches!(self, ChunkingType::Parallel { .. })
366    }
367
368    pub fn is_merged(&self) -> bool {
369        matches!(
370            self,
371            ChunkingType::Isolated {
372                merge_tag: Some(_),
373                ..
374            } | ChunkingType::Shared {
375                merge_tag: Some(_),
376                ..
377            }
378        )
379    }
380
381    pub fn without_inherit_async(&self) -> Self {
382        match self {
383            ChunkingType::Parallel { hoisted, .. } => ChunkingType::Parallel {
384                hoisted: *hoisted,
385                inherit_async: false,
386            },
387            ChunkingType::Async => ChunkingType::Async,
388            ChunkingType::Isolated { _ty, merge_tag } => ChunkingType::Isolated {
389                _ty: *_ty,
390                merge_tag: merge_tag.clone(),
391            },
392            ChunkingType::Shared {
393                inherit_async: _,
394                merge_tag,
395            } => ChunkingType::Shared {
396                inherit_async: false,
397                merge_tag: merge_tag.clone(),
398            },
399            ChunkingType::Traced => ChunkingType::Traced,
400        }
401    }
402}
403
404#[turbo_tasks::value(transparent)]
405pub struct ChunkingTypeOption(Option<ChunkingType>);
406
407/// A [ModuleReference] implementing this trait and returning Some(_) for
408/// [ChunkableModuleReference::chunking_type] are considered as potentially
409/// chunkable references. When all [Module]s of such a reference implement
410/// [ChunkableModule] they are placed in [Chunk]s during chunking.
411/// They are even potentially placed in the same [Chunk] when a chunk type
412/// specific interface is implemented.
413#[turbo_tasks::value_trait]
414pub trait ChunkableModuleReference: ModuleReference + ValueToString {
415    #[turbo_tasks::function]
416    fn chunking_type(self: Vc<Self>) -> Vc<ChunkingTypeOption> {
417        Vc::cell(Some(ChunkingType::Parallel {
418            inherit_async: false,
419            hoisted: false,
420        }))
421    }
422
423    #[turbo_tasks::function]
424    fn export_usage(self: Vc<Self>) -> Vc<ExportUsage> {
425        ExportUsage::all()
426    }
427}
428
429pub struct ChunkGroupContent {
430    pub chunkable_items: FxIndexSet<ChunkableModuleOrBatch>,
431    pub batch_groups: FxIndexSet<ResolvedVc<ModuleBatchGroup>>,
432    pub async_modules: FxIndexSet<ResolvedVc<Box<dyn ChunkableModule>>>,
433    pub traced_modules: FxIndexSet<ResolvedVc<Box<dyn Module>>>,
434    pub availability_info: AvailabilityInfo,
435}
436
437#[turbo_tasks::value_trait]
438pub trait ChunkItem {
439    /// The [AssetIdent] of the [Module] that this [ChunkItem] was created from.
440    /// For most chunk types this must uniquely identify the chunk item at
441    /// runtime as it's the source of the module id used at runtime.
442    #[turbo_tasks::function]
443    fn asset_ident(self: Vc<Self>) -> Vc<AssetIdent>;
444    /// A [AssetIdent] that uniquely identifies the content of this [ChunkItem].
445    /// It is usually identical to [ChunkItem::asset_ident] but can be
446    /// different when the chunk item content depends on available modules e. g.
447    /// for chunk loaders.
448    #[turbo_tasks::function]
449    fn content_ident(self: Vc<Self>) -> Vc<AssetIdent> {
450        self.asset_ident()
451    }
452    /// A [ChunkItem] can reference OutputAssets, unlike [Module]s referencing other [Module]s.
453    #[turbo_tasks::function]
454    fn references(self: Vc<Self>) -> Vc<OutputAssets> {
455        OutputAssets::empty()
456    }
457
458    /// The type of chunk this item should be assembled into.
459    #[turbo_tasks::function]
460    fn ty(self: Vc<Self>) -> Vc<Box<dyn ChunkType>>;
461
462    /// A temporary method to retrieve the module associated with this
463    /// ChunkItem. TODO: Remove this as part of the chunk refactoring.
464    #[turbo_tasks::function]
465    fn module(self: Vc<Self>) -> Vc<Box<dyn Module>>;
466
467    #[turbo_tasks::function]
468    fn chunking_context(self: Vc<Self>) -> Vc<Box<dyn ChunkingContext>>;
469}
470
471#[turbo_tasks::value_trait]
472pub trait ChunkType: ValueToString {
473    /// Whether the source (reference) order of items needs to be retained during chunking.
474    #[turbo_tasks::function]
475    fn is_style(self: Vc<Self>) -> Vc<bool>;
476
477    /// Create a new chunk for the given chunk items
478    #[turbo_tasks::function]
479    fn chunk(
480        &self,
481        chunking_context: Vc<Box<dyn ChunkingContext>>,
482        chunk_items: Vec<ChunkItemOrBatchWithAsyncModuleInfo>,
483        batch_groups: Vec<ResolvedVc<ChunkItemBatchGroup>>,
484        referenced_output_assets: Vc<OutputAssets>,
485    ) -> Vc<Box<dyn Chunk>>;
486
487    #[turbo_tasks::function]
488    fn chunk_item_size(
489        &self,
490        chunking_context: Vc<Box<dyn ChunkingContext>>,
491        chunk_item: Vc<Box<dyn ChunkItem>>,
492        async_module_info: Option<Vc<AsyncModuleInfo>>,
493    ) -> Vc<usize>;
494}
495
496pub fn round_chunk_item_size(size: usize) -> usize {
497    let a = size.next_power_of_two();
498    size & (a | (a >> 1) | (a >> 2))
499}
500
501#[turbo_tasks::value(transparent)]
502pub struct ChunkItems(pub Vec<ResolvedVc<Box<dyn ChunkItem>>>);
503
504#[turbo_tasks::value]
505pub struct AsyncModuleInfo {
506    pub referenced_async_modules: AutoSet<ResolvedVc<Box<dyn Module>>>,
507}
508
509#[turbo_tasks::value_impl]
510impl AsyncModuleInfo {
511    #[turbo_tasks::function]
512    pub fn new(referenced_async_modules: Vec<ResolvedVc<Box<dyn Module>>>) -> Result<Vc<Self>> {
513        Ok(Self {
514            referenced_async_modules: referenced_async_modules.into_iter().collect(),
515        }
516        .cell())
517    }
518}
519
520#[derive(
521    Serialize, Deserialize, Debug, Clone, PartialEq, Eq, Hash, TraceRawVcs, TaskInput, NonLocalValue,
522)]
523pub struct ChunkItemWithAsyncModuleInfo {
524    pub chunk_item: ResolvedVc<Box<dyn ChunkItem>>,
525    pub module: Option<ResolvedVc<Box<dyn ChunkableModule>>>,
526    pub async_info: Option<ResolvedVc<AsyncModuleInfo>>,
527}
528
529#[turbo_tasks::value(transparent)]
530pub struct ChunkItemsWithAsyncModuleInfo(Vec<ChunkItemWithAsyncModuleInfo>);
531
532pub trait ChunkItemExt {
533    /// Returns the module id of this chunk item.
534    fn id(self: Vc<Self>) -> Vc<ModuleId>;
535}
536
537impl<T> ChunkItemExt for T
538where
539    T: Upcast<Box<dyn ChunkItem>>,
540{
541    /// Returns the module id of this chunk item.
542    fn id(self: Vc<Self>) -> Vc<ModuleId> {
543        let chunk_item = Vc::upcast(self);
544        chunk_item.chunking_context().chunk_item_id(chunk_item)
545    }
546}
547
548pub trait ModuleChunkItemIdExt {
549    /// Returns the chunk item id of this module.
550    fn chunk_item_id(
551        self: Vc<Self>,
552        chunking_context: Vc<Box<dyn ChunkingContext>>,
553    ) -> Vc<ModuleId>;
554}
555impl<T> ModuleChunkItemIdExt for T
556where
557    T: Upcast<Box<dyn Module>>,
558{
559    fn chunk_item_id(
560        self: Vc<Self>,
561        chunking_context: Vc<Box<dyn ChunkingContext>>,
562    ) -> Vc<ModuleId> {
563        chunking_context.chunk_item_id_from_module(Vc::upcast(self))
564    }
565}
566
567#[cfg(test)]
568mod tests {
569    use super::*;
570
571    #[test]
572    fn test_round_chunk_item_size() {
573        assert_eq!(round_chunk_item_size(0), 0);
574        assert_eq!(round_chunk_item_size(1), 1);
575        assert_eq!(round_chunk_item_size(2), 2);
576        assert_eq!(round_chunk_item_size(3), 3);
577        assert_eq!(round_chunk_item_size(4), 4);
578        assert_eq!(round_chunk_item_size(5), 4);
579        assert_eq!(round_chunk_item_size(6), 6);
580        assert_eq!(round_chunk_item_size(7), 6);
581        assert_eq!(round_chunk_item_size(8), 8);
582        assert_eq!(round_chunk_item_size(49000), 32_768);
583        assert_eq!(round_chunk_item_size(50000), 49_152);
584
585        assert_eq!(changes_in_range(0..1000), 19);
586        assert_eq!(changes_in_range(1000..2000), 2);
587        assert_eq!(changes_in_range(2000..3000), 1);
588
589        assert_eq!(changes_in_range(3000..10000), 4);
590
591        fn changes_in_range(range: std::ops::Range<usize>) -> usize {
592            let len = range.len();
593            let mut count = 0;
594            for i in range {
595                let a = round_chunk_item_size(i);
596                assert!(a >= i * 2 / 3);
597                assert!(a <= i);
598                let b = round_chunk_item_size(i + 1);
599
600                if a == b {
601                    count += 1;
602                }
603            }
604            len - count
605        }
606    }
607}