turbopack_core/source_map/
utils.rs

1use std::{borrow::Cow, collections::HashSet, iter, sync::LazyLock};
2
3use anyhow::{Context, Result};
4use const_format::concatcp;
5use regex::Regex;
6use serde::{Deserialize, Serialize};
7use serde_json::value::RawValue;
8use turbo_rcstr::RcStr;
9use turbo_tasks::{ResolvedVc, turbofmt};
10use turbo_tasks_fs::{DiskFileSystem, FileContent, FileSystemPath, rope::Rope};
11use url::Url;
12
13use crate::{SOURCE_URL_PROTOCOL_STR, source_map::structured::StructuredSourceMap};
14
15pub fn add_default_ignore_list(map: &mut swc_sourcemap::SourceMap) {
16    let mut ignored_ids = HashSet::new();
17
18    for (source_id, source) in map.sources().enumerate() {
19        if source.starts_with(concatcp!(SOURCE_URL_PROTOCOL_STR, "///[next]"))
20            || source.starts_with(concatcp!(SOURCE_URL_PROTOCOL_STR, "///[turbopack]"))
21            || source.contains("/node_modules/")
22            || source.ends_with("__nextjs-internal-proxy.cjs")
23            || source.ends_with("__nextjs-internal-proxy.mjs")
24        {
25            ignored_ids.insert(source_id);
26        }
27    }
28
29    for ignored_id in ignored_ids {
30        map.add_to_ignore_list(ignored_id as _);
31    }
32}
33
34#[derive(Serialize, Deserialize)]
35struct SourceMapSectionOffsetJson {
36    line: u32,
37    offset: u32,
38}
39
40#[derive(Serialize, Deserialize)]
41struct SourceMapSectionItemJson {
42    offset: SourceMapSectionOffsetJson,
43    map: SourceMapJson,
44}
45
46// Some of these values use `Box<RawValue>`: If we don't read these fields (or rarely read these
47// fields) there's no point in decoding/encoding the data. Ideally they would be a `&RawValue`
48// reference, but we deserialize using `from_reader`, which does not support that.
49#[derive(Serialize, Deserialize)]
50#[serde(rename_all = "camelCase")]
51struct SourceMapJson {
52    version: u32,
53    #[serde(skip_serializing_if = "Option::is_none")]
54    file: Option<String>,
55    #[serde(skip_serializing_if = "Option::is_none")]
56    source_root: Option<String>,
57    // Technically a required field, but we don't want to error here.
58    #[serde(skip_serializing_if = "Option::is_none")]
59    sources: Option<Vec<Option<String>>>,
60    #[serde(skip_serializing_if = "Option::is_none")]
61    sources_content: Option<Vec<Option<Box<RawValue>>>>,
62    #[serde(skip_serializing_if = "Option::is_none")]
63    names: Option<Box<RawValue>>,
64    // We just need to hold onto `mappings` for serialization/deserialization, so there's no point
65    // in decoding/encoding the string. Store it as a `RawValue`. Ideally this would be a reference
66    // to the RawValue, but we deserialize using `from_reader`, which does not support that.
67    mappings: Box<RawValue>,
68    #[serde(skip_serializing_if = "Option::is_none")]
69    ignore_list: Option<Box<RawValue>>,
70
71    // A somewhat widespread non-standard extension
72    #[serde(skip_serializing_if = "Option::is_none")]
73    debug_id: Option<Box<RawValue>>,
74
75    #[serde(skip_serializing_if = "Option::is_none")]
76    sections: Option<Vec<SourceMapSectionItemJson>>,
77}
78
79/// Replace the origin prefix in the `file` and `sources` with `turbopack:///` and read the
80/// `sourceContent`s from disk.
81pub async fn resolve_source_map_sources(
82    map: Option<&Rope>,
83    origin: &FileSystemPath,
84) -> Result<Option<Rope>> {
85    let fs_vc = origin.fs().to_resolved().await?;
86    let fs_str = &*turbofmt!("[{fs_vc}]").await?;
87
88    let disk_fs = if let Some(fs_vc) = ResolvedVc::try_downcast_type::<DiskFileSystem>(fs_vc) {
89        Some((fs_vc, fs_vc.await?))
90    } else {
91        None
92    };
93    let disk_fs = &disk_fs;
94
95    let resolve_source =
96        async |source_url: &mut String, source_content: Option<&mut Option<Box<RawValue>>>| {
97            // original_source should always be a URL (possibly a `file://` url). If it's a relative
98            // URL, it should be relative to `origin` (the generated file that's being mapped).
99            // https://developer.mozilla.org/en-US/docs/Learn_web_development/Howto/Web_mechanics/What_is_a_URL#absolute_urls_vs._relative_urls
100            let maybe_file_url = if source_url.starts_with("//") {
101                // looks like a "scheme-relative" URL
102                // Rewrite '//scheme/relative' -> 'file:///scheme/relative' (three slashes)
103                Cow::Owned(format!("file:/{source_url}"))
104            } else if source_url.starts_with('/') {
105                // looks like a "domain-relative" (aka "server-relative") URL
106                // Rewrite '/domain/relative' -> 'file:///domain/relative' (three slashes)
107                Cow::Owned(format!("file://{source_url}"))
108            } else {
109                Cow::Borrowed(source_url)
110            };
111
112            let fs_path = if let Ok(original_source_url_obj) = Url::parse(&maybe_file_url) {
113                // We have an absolute URL, try to parse it as a `file://` URL
114                if let Ok(sys_path) = original_source_url_obj.to_file_path() {
115                    if let Some((disk_fs_vc, disk_fs)) = disk_fs {
116                        disk_fs.try_from_sys_path(*disk_fs_vc, &sys_path, Some(origin))
117                    } else {
118                        None
119                    }
120                } else {
121                    // this is an absolute URL with a non-`file://` scheme, just assume it's valid
122                    // and don't modify anything
123                    return Ok(());
124                }
125            } else {
126                // assume it's a relative URL, and just remove any percent encoding from path
127                // segments. Our internal path format is POSIX-like, without percent encoding.
128                origin
129                    .parent()
130                    .try_join(&urlencoding::decode(source_url).unwrap_or(Cow::Borrowed(source_url)))
131            };
132
133            if let Some(fs_path) = fs_path {
134                // TODO: Encode `fs_str` and `fs_path_str` using `urlencoding`, so that these are
135                // valid URLs. However, `project_trace_source_operation` (and `uri_from_file`) need
136                // to handle percent encoding correctly first.
137                let fs_path_str = &fs_path.path;
138                *source_url = format!("{SOURCE_URL_PROTOCOL_STR}///{fs_str}/{fs_path_str}");
139
140                if let Some(source_content) = source_content
141                    && source_content.is_none()
142                {
143                    if let FileContent::Content(file) = &*fs_path.read().await? {
144                        let text = file.content().to_str()?;
145                        *source_content = Some(unencoded_str_to_raw_value(&text));
146                    } else {
147                        *source_content = Some(unencoded_str_to_raw_value(&format!(
148                            "unable to read source {fs_str}/{fs_path_str}"
149                        )));
150                    }
151                }
152            } else {
153                // The URL was broken somehow, create a dummy `turbopack://` URL and content
154                let origin_str = &origin.path;
155                if let Some(source_content) = source_content
156                    && source_content.is_none()
157                {
158                    *source_content = Some(unencoded_str_to_raw_value(&format!(
159                        "unable to access {source_url} in {fs_str}/{origin_str} (it's leaving the \
160                         filesystem root)"
161                    )));
162                }
163                static INVALID_REGEX: LazyLock<Regex> =
164                    LazyLock::new(|| Regex::new(r#"(?:^|/)(?:\.\.?(?:/|$))+"#).unwrap());
165                let source = INVALID_REGEX
166                    .replace_all(source_url, |s: &regex::Captures<'_>| s[0].replace('.', "_"));
167                *source_url = format!("{SOURCE_URL_PROTOCOL_STR}///{fs_str}/{origin_str}/{source}");
168            }
169            anyhow::Ok(())
170        };
171
172    let resolve_map = async |map: &mut SourceMapJson| {
173        if let Some(sources) = &mut map.sources {
174            let mut contents = if let Some(mut contents) = map.sources_content.take() {
175                contents.resize(sources.len(), None);
176                contents
177            } else {
178                iter::repeat_n(None, sources.len()).collect()
179            };
180
181            for (source, content) in sources.iter_mut().zip(contents.iter_mut()) {
182                if let Some(source) = source {
183                    if let Some(source_root) = &map.source_root {
184                        *source = format!("{source_root}{source}");
185                    }
186                    resolve_source(source, Some(content)).await?;
187                }
188            }
189
190            map.source_root = None;
191            map.sources_content = Some(contents);
192        }
193        anyhow::Ok(())
194    };
195
196    let Some(map) = map else {
197        return Ok(None);
198    };
199
200    let Ok(mut map): serde_json::Result<SourceMapJson> = serde_json::from_reader(map.read()) else {
201        // Silently ignore invalid sourcemaps
202        return Ok(None);
203    };
204
205    if let Some(file) = &mut map.file {
206        resolve_source(file, None).await?;
207    }
208
209    resolve_map(&mut map).await?;
210    for section in map.sections.iter_mut().flatten() {
211        resolve_map(&mut section.map).await?;
212    }
213
214    let map = Rope::from(serde_json::to_vec(&map)?);
215    Ok(Some(map))
216}
217
218fn unencoded_str_to_raw_value(unencoded: &str) -> Box<RawValue> {
219    RawValue::from_string(
220        serde_json::to_string(unencoded)
221            .expect("serialization of a utf-8 string should always succeed"),
222    )
223    .expect("serde_json::to_string should produce valid JSON")
224}
225
226fn uri_encode_path(path: &str) -> String {
227    path.split('/')
228        .map(|s| urlencoding::encode(s))
229        .collect::<Vec<_>>()
230        .join("/")
231}
232
233/// Applies the standard `turbopack:///[fs]/` source-URL transform to a structured map,
234/// sharing every other field (notably `sourcesContent`) with the input. `transform` is given
235/// the source string as found in the map (i.e. a URI) with the `turbopack:///[fs]/` prefix
236/// stripped.
237async fn transform_relative_files<F>(
238    map: &StructuredSourceMap,
239    context_path: &FileSystemPath,
240    mut transform: F,
241) -> Result<StructuredSourceMap>
242where
243    F: FnMut(&DiskFileSystem, &str) -> Result<String>,
244{
245    let context_fs = context_path.fs;
246    let context_fs = &*ResolvedVc::try_downcast_type::<DiskFileSystem>(context_fs)
247        .context("Expected the chunking context to have a DiskFileSystem")?
248        .await?;
249
250    let prefix = format!("{}///[{}]/", SOURCE_URL_PROTOCOL_STR, context_fs.name());
251
252    map.rewrite_sources(|src| {
253        if let Some(src_rest) = src.strip_prefix(&prefix) {
254            Ok(Some(transform(context_fs, src_rest)?))
255        } else {
256            Ok(None)
257        }
258    })
259}
260
261/// Turns `turbopack:///[project]` references in the map's sources into absolute `file://` uris.
262/// This is useful for debugging environments.
263pub async fn absolute_fileify_source_map(
264    map: &StructuredSourceMap,
265    context_path: FileSystemPath,
266) -> Result<StructuredSourceMap> {
267    transform_relative_files(map, &context_path.clone(), |context_fs, src_rest| {
268        let path = context_path.join(src_rest)?;
269
270        // `to_sys_path` returns a win32 path on Windows. `Url::from_file_path` can also handle
271        // verbatim (`\\?\`-prefixed) disk and UNC paths, in case that conversion failed.
272        let sys_path = context_fs.to_sys_path(&path);
273        Ok(Url::from_file_path(&sys_path)
274            .map_err(|()| {
275                anyhow::anyhow!("path {sys_path:?} cannot be converted to a file:// URI")
276            })?
277            .into())
278    })
279    .await
280}
281
282/// Turns `turbopack:///[project]` references in the map's sources into `./`-relative uris.
283/// This is useful in server environments and especially build environments.
284pub async fn relative_fileify_source_map(
285    map: &StructuredSourceMap,
286    context_path: FileSystemPath,
287    relative_path_to_output_root: RcStr,
288) -> Result<StructuredSourceMap> {
289    let relative_path_to_output_root = relative_path_to_output_root
290        .split('/')
291        .map(|s| urlencoding::encode(s))
292        .collect::<Vec<_>>()
293        .join("/");
294    transform_relative_files(map, &context_path, |_context_fs, src_rest| {
295        // NOTE: we just include the relative path prefix here instead of using `sourceRoot`
296        // since the spec on sourceRoot is broken.
297
298        // TODO(bgw): this shouldn't be necessary to uri encode since the strings we get out of the
299        // source map should already be uri encoded, however in the case of the turbopack scheme in
300        // particular we are inconsistent so be defensive here.
301        let src_rest = uri_encode_path(src_rest);
302        if relative_path_to_output_root.is_empty() {
303            Ok(src_rest.to_string())
304        } else {
305            Ok(format!("{relative_path_to_output_root}/{src_rest}"))
306        }
307    })
308    .await
309}
310
311#[cfg(test)]
312mod tests {
313    use std::path::Path;
314
315    use turbo_rcstr::{RcStr, rcstr};
316    use turbo_tasks::Vc;
317    use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage};
318    use turbo_tasks_fs::FileSystem;
319
320    use super::*;
321
322    fn source_map_rope<'a>(
323        source_root: Option<&str>,
324        sources: impl IntoIterator<Item = &'a str>,
325    ) -> Rope {
326        Rope::from(
327            serde_json::to_string_pretty(
328                &serde_json::from_value::<SourceMapJson>(serde_json::json!({
329                    "version": 3,
330                    "mappings": "",
331                    "sourceRoot": source_root,
332                    "sources": sources.into_iter().map(Some).collect::<Vec<_>>(),
333                }))
334                .unwrap(),
335            )
336            .unwrap(),
337        )
338    }
339
340    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
341    async fn test_resolve_source_map_sources() {
342        let tt = turbo_tasks::TurboTasks::new(TurboTasksBackend::new(
343            BackendOptions::default(),
344            noop_backing_storage(),
345        ));
346        tt.run_once(async move {
347            #[turbo_tasks::value]
348            struct SourceMapSourcesOutput {
349                resolved_sources: Vec<Option<String>>,
350                rooted_sources: Vec<Option<String>>,
351            }
352
353            #[turbo_tasks::function(operation, root)]
354            async fn resolve_source_map_sources_operation()
355            -> anyhow::Result<Vc<SourceMapSourcesOutput>> {
356                let sys_root = if cfg!(windows) {
357                    Path::new(r"C:\fake\root")
358                } else {
359                    Path::new(r"/fake/root")
360                };
361                let url_root = Url::from_directory_path(sys_root).unwrap();
362
363                let fs_root_path = DiskFileSystem::new(
364                    rcstr!("mock"),
365                    Vc::cell(RcStr::from(sys_root.to_str().unwrap())),
366                )
367                .root()
368                .await?;
369
370                let resolved_source_map: SourceMapJson = serde_json::from_str(
371                    &resolve_source_map_sources(
372                        Some(&source_map_rope(
373                            /* source_root */ None,
374                            [
375                                "page.js",
376                                "./current-dir-page.js",
377                                "../other%20route/page.js",
378                                // contains the file:// protocol/scheme
379                                url_root.join("absolute%20file%20url.js")?.as_str(),
380                                // A server-relative path starting with `/`, potentially includes a
381                                // windows disk
382                                &format!("{}/server%20relative%20path.js", url_root.path()),
383                                // A scheme-relative path
384                                url_root
385                                    .join("scheme%20relative%20path.js")?
386                                    .as_str()
387                                    .strip_prefix("file:")
388                                    .unwrap(),
389                                // non-file URLs are preserved
390                                "https://example.com/page%20path.js",
391                            ],
392                        )),
393                        // NOTE: the percent encoding here should NOT be decoded, as this is not
394                        // part of a `file://` URL
395                        &fs_root_path.join("app/source%20mapped/page.js").unwrap(),
396                    )
397                    .await?
398                    .unwrap()
399                    .to_str()?,
400                )?;
401
402                let rooted_source_map: SourceMapJson = serde_json::from_str(
403                    &resolve_source_map_sources(
404                        Some(&source_map_rope(
405                            // NOTE: these should get literally concated, a slash should NOT get
406                            // added.
407                            Some("../source%20root%20"),
408                            ["page.js"],
409                        )),
410                        &fs_root_path.join("app/page.js").unwrap(),
411                    )
412                    .await?
413                    .unwrap()
414                    .to_str()?,
415                )?;
416
417                Ok(SourceMapSourcesOutput {
418                    resolved_sources: resolved_source_map.sources.unwrap_or_default(),
419                    rooted_sources: rooted_source_map.sources.unwrap_or_default(),
420                }
421                .cell())
422            }
423
424            let resolved_source_maps = resolve_source_map_sources_operation()
425                .read_strongly_consistent()
426                .await?;
427
428            let prefix = format!("{SOURCE_URL_PROTOCOL_STR}///[mock]");
429            assert_eq!(
430                resolved_source_maps.resolved_sources,
431                vec![
432                    Some(format!("{prefix}/app/source%20mapped/page.js")),
433                    Some(format!("{prefix}/app/source%20mapped/current-dir-page.js")),
434                    Some(format!("{prefix}/app/other route/page.js")),
435                    Some(format!("{prefix}/absolute file url.js")),
436                    Some(format!("{prefix}/server relative path.js")),
437                    Some(format!("{prefix}/scheme relative path.js")),
438                    Some("https://example.com/page%20path.js".to_owned()),
439                ]
440            );
441
442            assert_eq!(
443                resolved_source_maps.rooted_sources,
444                vec![Some(format!("{prefix}/source root page.js"))]
445            );
446
447            anyhow::Ok(())
448        })
449        .await
450        .unwrap();
451    }
452}
turbopack_core/source_map/utils.rs

turbopack_core/source_map/
utils.rs
