turbopack_core/
ident.rs

1use std::{fmt::Write, sync::LazyLock};
2
3use anyhow::Result;
4use bincode::{Decode, Encode};
5use regex::Regex;
6use turbo_rcstr::RcStr;
7use turbo_tasks::{ReadRef, ResolvedVc, ValueToString, ValueToStringRef, Vc, turbofmt};
8use turbo_tasks_fs::FileSystemPath;
9use turbo_tasks_hash::{DeterministicHash, Xxh3Hash64Hasher, encode_base38, hash_xxh3_hash64};
10
11use crate::resolve::ModulePart;
12
13/// A layer identifies a distinct part of the module graph.
14#[turbo_tasks::task_input]
15#[derive(Clone, Hash, Debug, DeterministicHash, Eq, PartialEq, Encode, Decode)]
16pub struct Layer {
17    name: RcStr,
18    user_friendly_name: Option<RcStr>,
19}
20
21impl Layer {
22    pub fn new(name: RcStr) -> Self {
23        debug_assert!(!name.is_empty());
24        Self {
25            name,
26            user_friendly_name: None,
27        }
28    }
29    pub fn new_with_user_friendly_name(name: RcStr, user_friendly_name: RcStr) -> Self {
30        debug_assert!(!name.is_empty());
31        debug_assert!(!user_friendly_name.is_empty());
32        Self {
33            name,
34            user_friendly_name: Some(user_friendly_name),
35        }
36    }
37
38    /// Returns a user friendly name for this layer
39    pub fn user_friendly_name(&self) -> &RcStr {
40        self.user_friendly_name.as_ref().unwrap_or(&self.name)
41    }
42
43    pub fn name(&self) -> &RcStr {
44        &self.name
45    }
46}
47
48#[turbo_tasks::value(task_input)]
49#[derive(Clone, Debug, Hash)]
50pub struct AssetIdent {
51    /// The primary path of the asset
52    pub path: FileSystemPath,
53    /// The query string of the asset this is either the empty string or a query string that starts
54    /// with a `?` (e.g. `?foo=bar`)
55    pub query: RcStr,
56    /// The fragment of the asset, this is either the empty string or a fragment string that starts
57    /// with a `#` (e.g. `#foo`)
58    pub fragment: RcStr,
59    /// The assets that are nested in this asset
60    pub assets: Vec<(RcStr, ResolvedVc<AssetIdent>)>,
61    /// The modifiers of this asset (e.g. `client chunks`)
62    pub modifiers: Vec<RcStr>,
63    /// The parts of the asset that are (ECMAScript) modules
64    pub parts: Vec<ModulePart>,
65    /// The asset layer the asset was created from.
66    pub layer: Option<Layer>,
67    /// The MIME content type, if this asset was created from a data URL.
68    pub content_type: Option<RcStr>,
69}
70
71impl AssetIdent {
72    /// Creates an [AssetIdent] from a [FileSystemPath].
73    ///
74    /// Returns an owned value; call [`AssetIdent::into_vc`] at the end of the builder chain to
75    /// turn it into a [`Vc<AssetIdent>`].
76    pub fn from_path(path: FileSystemPath) -> Self {
77        AssetIdent {
78            path,
79            query: RcStr::default(),
80            fragment: RcStr::default(),
81            assets: Vec::new(),
82            modifiers: Vec::new(),
83            parts: Vec::new(),
84            layer: None,
85            content_type: None,
86        }
87    }
88
89    /// Finalizes the builder by turning the owned [`AssetIdent`] into a cached [`Vc<AssetIdent>`].
90    pub fn into_vc(self) -> Vc<Self> {
91        // This optimizes cache misses in cold builds by only storing one copy of the AssetIdent.
92        AssetIdent::new_inner(ReadRef::new_owned(self))
93    }
94
95    pub fn with_query(mut self, query: RcStr) -> Self {
96        self.query = query;
97        self
98    }
99
100    pub fn with_fragment(mut self, fragment: RcStr) -> Self {
101        self.fragment = fragment;
102        self
103    }
104
105    pub fn with_modifier(mut self, modifier: RcStr) -> Self {
106        debug_assert!(!modifier.is_empty(), "modifiers cannot be empty.");
107        self.modifiers.push(modifier);
108        self
109    }
110
111    pub fn with_part(mut self, part: ModulePart) -> Self {
112        self.parts.push(part);
113        self
114    }
115
116    pub fn with_path(mut self, path: FileSystemPath) -> Self {
117        self.path = path;
118        self
119    }
120
121    pub fn with_layer(mut self, layer: Layer) -> Self {
122        self.layer = Some(layer);
123        self
124    }
125
126    pub fn with_content_type(mut self, content_type: RcStr) -> Self {
127        self.content_type = Some(content_type);
128        self
129    }
130
131    pub fn with_asset(mut self, key: RcStr, asset: ResolvedVc<AssetIdent>) -> Self {
132        self.assets.push((key, asset));
133        self
134    }
135
136    pub fn rename_as(mut self, pattern: &str) -> Self {
137        self.path = FileSystemPath::new_normalized_unchecked(
138            self.path.fs,
139            pattern.replace('*', &self.path.path).into(),
140        );
141        self.content_type = None;
142        self
143    }
144}
145
146#[turbo_tasks::value_impl]
147impl AssetIdent {
148    #[turbo_tasks::function]
149    fn new_inner(ident: ReadRef<AssetIdent>) -> Vc<Self> {
150        debug_assert!(
151            ident.query.is_empty() || ident.query.starts_with("?"),
152            "query should be empty or start with a `?`"
153        );
154        debug_assert!(
155            ident.fragment.is_empty() || ident.fragment.starts_with("#"),
156            "query should be empty or start with a `?`"
157        );
158        ReadRef::cell(ident)
159    }
160
161    /// Computes a unique output asset name for the given asset identifier.
162    /// TODO(alexkirsz) This is `turbopack-browser` specific, as
163    /// `turbopack-nodejs` would use a content hash instead. But for now
164    /// both are using the same name generation logic.
165    #[turbo_tasks::function]
166    pub async fn output_name(
167        &self,
168        context_path: FileSystemPath,
169        prefix: Option<RcStr>,
170        expected_extension: RcStr,
171    ) -> Result<Vc<RcStr>> {
172        debug_assert!(
173            expected_extension.starts_with("."),
174            "the extension should include the leading '.', got '{expected_extension}'"
175        );
176        // TODO(PACK-2140): restrict character set to A–Za–z0–9-_.~'()
177        // to be compatible with all operating systems + URLs.
178
179        // For clippy -- This explicit deref is necessary
180        let path = &self.path;
181        let mut name = if let Some(inner) = context_path.get_path_to(path) {
182            escape_file_path(inner)
183        } else {
184            escape_file_path(&self.path.to_string_ref().await?)
185        };
186        let removed_extension = name.ends_with(&*expected_extension);
187        if removed_extension {
188            name.truncate(name.len() - expected_extension.len());
189        }
190        // This step ensures that leading dots are not preserved in file names. This is
191        // important as some file servers do not serve files with leading dots (e.g.
192        // Next.js).
193        let mut name = clean_additional_extensions(&name);
194        if let Some(prefix) = prefix {
195            name = format!("{prefix}-{name}");
196        }
197
198        let default_modifier = match expected_extension.as_str() {
199            ".js" => Some("ecmascript"),
200            ".css" => Some("css"),
201            _ => None,
202        };
203
204        let mut hasher = Xxh3Hash64Hasher::new();
205        let mut has_hash = false;
206        let AssetIdent {
207            path: _,
208            query,
209            fragment,
210            assets,
211            modifiers,
212            parts,
213            layer,
214            content_type,
215        } = self;
216        if !query.is_empty() {
217            0_u8.deterministic_hash(&mut hasher);
218            query.deterministic_hash(&mut hasher);
219            has_hash = true;
220        }
221        if !fragment.is_empty() {
222            1_u8.deterministic_hash(&mut hasher);
223            fragment.deterministic_hash(&mut hasher);
224            has_hash = true;
225        }
226        for (key, ident) in assets.iter() {
227            2_u8.deterministic_hash(&mut hasher);
228            key.deterministic_hash(&mut hasher);
229            ident.to_string().await?.deterministic_hash(&mut hasher);
230            has_hash = true;
231        }
232        for modifier in modifiers.iter() {
233            if let Some(default_modifier) = default_modifier
234                && *modifier == default_modifier
235            {
236                continue;
237            }
238            3_u8.deterministic_hash(&mut hasher);
239            modifier.deterministic_hash(&mut hasher);
240            has_hash = true;
241        }
242        for part in parts.iter() {
243            4_u8.deterministic_hash(&mut hasher);
244            match part {
245                ModulePart::Evaluation => {
246                    1_u8.deterministic_hash(&mut hasher);
247                }
248                ModulePart::Export(export) => {
249                    2_u8.deterministic_hash(&mut hasher);
250                    export.deterministic_hash(&mut hasher);
251                }
252                ModulePart::PartialExport { export, member } => {
253                    3_u8.deterministic_hash(&mut hasher);
254                    export.deterministic_hash(&mut hasher);
255                    member.deterministic_hash(&mut hasher);
256                }
257                ModulePart::RenamedExport {
258                    original_export,
259                    export,
260                } => {
261                    4_u8.deterministic_hash(&mut hasher);
262                    original_export.deterministic_hash(&mut hasher);
263                    export.deterministic_hash(&mut hasher);
264                }
265                ModulePart::RenamedNamespace { export } => {
266                    5_u8.deterministic_hash(&mut hasher);
267                    export.deterministic_hash(&mut hasher);
268                }
269                ModulePart::RenamedPartialNamespace { export, member } => {
270                    6_u8.deterministic_hash(&mut hasher);
271                    export.deterministic_hash(&mut hasher);
272                    member.deterministic_hash(&mut hasher);
273                }
274                ModulePart::Internal(id) => {
275                    7_u8.deterministic_hash(&mut hasher);
276                    id.deterministic_hash(&mut hasher);
277                }
278                ModulePart::Locals => {
279                    8_u8.deterministic_hash(&mut hasher);
280                }
281                ModulePart::Exports => {
282                    9_u8.deterministic_hash(&mut hasher);
283                }
284                ModulePart::Facade => {
285                    10_u8.deterministic_hash(&mut hasher);
286                }
287            }
288
289            has_hash = true;
290        }
291        if let Some(layer) = layer {
292            5_u8.deterministic_hash(&mut hasher);
293            layer.deterministic_hash(&mut hasher);
294            has_hash = true;
295        }
296        if let Some(content_type) = content_type {
297            6_u8.deterministic_hash(&mut hasher);
298            content_type.deterministic_hash(&mut hasher);
299            has_hash = true;
300        }
301
302        if has_hash {
303            let hash = encode_base38(hasher.finish());
304            // Use the complete fixed-width base38 encoding of the 64-bit hash.
305            //
306            // This suffix is what distinguishes idents that share a readable
307            // name prefix, so truncating it makes distinct output paths collide
308            // for large module graphs, failing the build with "Two or more
309            // assets with different content were emitted to the same output
310            // path". Keeping all 64 bits removes that failure mode, at the cost
311            // of a slightly longer file name.
312            write!(name, "_{hash}")?;
313        }
314
315        // Location in "path" where hashed and named parts are split.
316        // Everything before i is hashed and after i named.
317        let mut i = 0;
318        static NODE_MODULES: &str = "_node_modules_";
319        if let Some(j) = name.rfind(NODE_MODULES) {
320            i = j + NODE_MODULES.len();
321        }
322        const MAX_FILENAME: usize = 80;
323        if name.len() - i > MAX_FILENAME {
324            i = name.len() - MAX_FILENAME;
325            if let Some(j) = name[i..].find('_')
326                && j < 20
327            {
328                i += j + 1;
329            }
330        }
331        if i > 0 {
332            let hash = encode_base38(hash_xxh3_hash64(&name.as_bytes()[..i]));
333            // 4 base38 chars ≈ 21 bits — just a short disambiguator prefix
334            let truncated_hash = &hash[..4];
335            name = format!("{}_{}", truncated_hash, &name[i..]);
336        }
337        // We need to make sure that `.json` and `.json.js` doesn't end up with the same
338        // name. So when we add an extra extension when want to mark that with a "._"
339        // suffix.
340        if !removed_extension {
341            name += "._";
342        }
343        name += &expected_extension;
344        Ok(Vc::cell(name.into()))
345    }
346}
347
348#[turbo_tasks::value_impl]
349impl ValueToString for AssetIdent {
350    #[turbo_tasks::function]
351    async fn to_string(&self) -> Result<Vc<RcStr>> {
352        // The query string/fragment is either empty or non-empty starting with
353        // `?` so we can just concat
354        let mut s = turbofmt!("{}{}{}", self.path, self.query, self.fragment)
355            .await?
356            .into_owned();
357
358        if !self.assets.is_empty() {
359            s.push_str(" {");
360
361            for (i, (key, asset)) in self.assets.iter().enumerate() {
362                if i > 0 {
363                    s.push(',');
364                }
365
366                let asset_str = asset.to_string().await?;
367                write!(s, " {key} => {asset_str:?}")?;
368            }
369
370            s.push_str(" }");
371        }
372
373        if let Some(layer) = &self.layer {
374            write!(s, " [{}]", layer.name)?;
375        }
376
377        if !self.modifiers.is_empty() {
378            s.push_str(" (");
379
380            for (i, modifier) in self.modifiers.iter().enumerate() {
381                if i > 0 {
382                    s.push_str(", ");
383                }
384
385                s.push_str(modifier);
386            }
387
388            s.push(')');
389        }
390
391        if let Some(content_type) = &self.content_type {
392            write!(s, " <{content_type}>")?;
393        }
394
395        if !self.parts.is_empty() {
396            for part in self.parts.iter() {
397                if !matches!(part, ModulePart::Facade) {
398                    // facade is not included in ident as switching between facade and non-facade
399                    // shouldn't change the ident
400                    write!(s, " <{part}>")?;
401                }
402            }
403        }
404
405        Ok(Vc::cell(s.into()))
406    }
407}
408
409fn escape_file_path(s: &str) -> String {
410    static SEPARATOR_REGEX: LazyLock<Regex> = LazyLock::new(|| Regex::new(r"[/#?:]").unwrap());
411    SEPARATOR_REGEX.replace_all(s, "_").to_string()
412}
413
414fn clean_additional_extensions(s: &str) -> String {
415    s.replace('.', "_")
416}
417
418#[cfg(test)]
419pub mod tests {
420    use turbo_rcstr::{RcStr, rcstr};
421    use turbo_tasks::Vc;
422    use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage};
423    use turbo_tasks_fs::{FileSystem, VirtualFileSystem};
424
425    use crate::ident::AssetIdent;
426
427    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
428    async fn test_output_name_escaping() {
429        let tt = turbo_tasks::TurboTasks::new(TurboTasksBackend::new(
430            BackendOptions::default(),
431            noop_backing_storage(),
432        ));
433        tt.run_once(async move {
434            #[turbo_tasks::function(operation, root)]
435            async fn output_name_operation() -> anyhow::Result<Vc<RcStr>> {
436                let fs = VirtualFileSystem::new_with_name(rcstr!("test"));
437                let root = fs.root().owned().await?;
438
439                let asset_ident = AssetIdent::from_path(root.join("a:b?c#d.js")?).into_vc();
440                let output_name = asset_ident
441                    .output_name(root, Some(rcstr!("prefix")), rcstr!(".js"))
442                    .await?;
443                Ok(Vc::cell((*output_name).clone()))
444            }
445
446            let output_name = output_name_operation().read_strongly_consistent().await?;
447            assert_eq!(&*output_name, "prefix-a_b_c_d.js");
448
449            Ok(())
450        })
451        .await
452        .unwrap();
453    }
454}
turbopack_core/ident.rs

turbopack_core/
ident.rs
