turbopack_core/source_map/
utils.rs

1use std::{borrow::Cow, collections::HashSet, iter, sync::LazyLock};
2
3use anyhow::{Context, Result};
4use const_format::concatcp;
5use regex::Regex;
6use serde::{Deserialize, Serialize};
7use serde_json::value::RawValue;
8use turbo_rcstr::RcStr;
9use turbo_tasks::{ResolvedVc, turbofmt};
10use turbo_tasks_fs::{DiskFileSystem, FileContent, FileSystemPath, rope::Rope};
11use url::Url;
12
13use crate::{SOURCE_URL_PROTOCOL_STR, source_map::structured::StructuredSourceMap};
14
15pub fn add_default_ignore_list(map: &mut swc_sourcemap::SourceMap) {
16    let mut ignored_ids = HashSet::new();
17
18    for (source_id, source) in map.sources().enumerate() {
19        if source.starts_with(concatcp!(SOURCE_URL_PROTOCOL_STR, "///[next]"))
20            || source.starts_with(concatcp!(SOURCE_URL_PROTOCOL_STR, "///[turbopack]"))
21            || source.contains("/node_modules/")
22            // The proxy module's ident carries a nested-asset suffix (e.g.
23            // ` { server => "..." }`), so it does not end with the extension.
24            // Match the base name instead.
25            || source.contains("__nextjs-internal-proxy.")
26        {
27            ignored_ids.insert(source_id);
28        }
29    }
30
31    for ignored_id in ignored_ids {
32        map.add_to_ignore_list(ignored_id as _);
33    }
34}
35
36#[derive(Serialize, Deserialize)]
37struct SourceMapSectionOffsetJson {
38    line: u32,
39    offset: u32,
40}
41
42#[derive(Serialize, Deserialize)]
43struct SourceMapSectionItemJson {
44    offset: SourceMapSectionOffsetJson,
45    map: SourceMapJson,
46}
47
48// Some of these values use `Box<RawValue>`: If we don't read these fields (or rarely read these
49// fields) there's no point in decoding/encoding the data. Ideally they would be a `&RawValue`
50// reference, but we deserialize using `from_reader`, which does not support that.
51#[derive(Serialize, Deserialize)]
52#[serde(rename_all = "camelCase")]
53struct SourceMapJson {
54    version: u32,
55    #[serde(skip_serializing_if = "Option::is_none")]
56    file: Option<String>,
57    #[serde(skip_serializing_if = "Option::is_none")]
58    source_root: Option<String>,
59    // Technically a required field, but we don't want to error here.
60    #[serde(skip_serializing_if = "Option::is_none")]
61    sources: Option<Vec<Option<String>>>,
62    #[serde(skip_serializing_if = "Option::is_none")]
63    sources_content: Option<Vec<Option<Box<RawValue>>>>,
64    #[serde(skip_serializing_if = "Option::is_none")]
65    names: Option<Box<RawValue>>,
66    // We just need to hold onto `mappings` for serialization/deserialization, so there's no point
67    // in decoding/encoding the string. Store it as a `RawValue`. Ideally this would be a reference
68    // to the RawValue, but we deserialize using `from_reader`, which does not support that.
69    mappings: Box<RawValue>,
70    #[serde(skip_serializing_if = "Option::is_none")]
71    ignore_list: Option<Box<RawValue>>,
72
73    // A somewhat widespread non-standard extension
74    #[serde(skip_serializing_if = "Option::is_none")]
75    debug_id: Option<Box<RawValue>>,
76
77    #[serde(skip_serializing_if = "Option::is_none")]
78    sections: Option<Vec<SourceMapSectionItemJson>>,
79}
80
81/// Replace the origin prefix in the `file` and `sources` with `turbopack:///` and read the
82/// `sourceContent`s from disk.
83pub async fn resolve_source_map_sources(
84    map: Option<&Rope>,
85    origin: &FileSystemPath,
86) -> Result<Option<Rope>> {
87    let fs_vc = origin.fs().to_resolved().await?;
88    let fs_str = &*turbofmt!("[{fs_vc}]").await?;
89
90    let disk_fs = if let Some(fs_vc) = ResolvedVc::try_downcast_type::<DiskFileSystem>(fs_vc) {
91        Some((fs_vc, fs_vc.await?))
92    } else {
93        None
94    };
95    let disk_fs = &disk_fs;
96
97    let resolve_source =
98        async |source_url: &mut String, source_content: Option<&mut Option<Box<RawValue>>>| {
99            // original_source should always be a URL (possibly a `file://` url). If it's a relative
100            // URL, it should be relative to `origin` (the generated file that's being mapped).
101            // https://developer.mozilla.org/en-US/docs/Learn_web_development/Howto/Web_mechanics/What_is_a_URL#absolute_urls_vs._relative_urls
102            let maybe_file_url = if source_url.starts_with("//") {
103                // looks like a "scheme-relative" URL
104                // Rewrite '//scheme/relative' -> 'file:///scheme/relative' (three slashes)
105                Cow::Owned(format!("file:/{source_url}"))
106            } else if source_url.starts_with('/') {
107                // looks like a "domain-relative" (aka "server-relative") URL
108                // Rewrite '/domain/relative' -> 'file:///domain/relative' (three slashes)
109                Cow::Owned(format!("file://{source_url}"))
110            } else {
111                Cow::Borrowed(source_url)
112            };
113
114            let fs_path = if let Ok(original_source_url_obj) = Url::parse(&maybe_file_url) {
115                // We have an absolute URL, try to parse it as a `file://` URL
116                if let Ok(sys_path) = original_source_url_obj.to_file_path() {
117                    if let Some((disk_fs_vc, disk_fs)) = disk_fs {
118                        disk_fs.try_from_sys_path(*disk_fs_vc, &sys_path, Some(origin))
119                    } else {
120                        None
121                    }
122                } else {
123                    // this is an absolute URL with a non-`file://` scheme, just assume it's valid
124                    // and don't modify anything
125                    return Ok(());
126                }
127            } else {
128                // assume it's a relative URL, and just remove any percent encoding from path
129                // segments. Our internal path format is POSIX-like, without percent encoding.
130                origin
131                    .parent()
132                    .try_join(&urlencoding::decode(source_url).unwrap_or(Cow::Borrowed(source_url)))
133            };
134
135            if let Some(fs_path) = fs_path {
136                // TODO: Encode `fs_str` and `fs_path_str` using `urlencoding`, so that these are
137                // valid URLs. However, `project_trace_source_operation` (and `uri_from_file`) need
138                // to handle percent encoding correctly first.
139                let fs_path_str = &fs_path.path;
140                *source_url = format!("{SOURCE_URL_PROTOCOL_STR}///{fs_str}/{fs_path_str}");
141
142                if let Some(source_content) = source_content
143                    && source_content.is_none()
144                {
145                    if let FileContent::Content(file) = &*fs_path.read().await? {
146                        let text = file.content().to_str()?;
147                        *source_content = Some(unencoded_str_to_raw_value(&text));
148                    } else {
149                        *source_content = Some(unencoded_str_to_raw_value(&format!(
150                            "unable to read source {fs_str}/{fs_path_str}"
151                        )));
152                    }
153                }
154            } else {
155                // The URL was broken somehow, create a dummy `turbopack://` URL and content
156                let origin_str = &origin.path;
157                if let Some(source_content) = source_content
158                    && source_content.is_none()
159                {
160                    *source_content = Some(unencoded_str_to_raw_value(&format!(
161                        "unable to access {source_url} in {fs_str}/{origin_str} (it's leaving the \
162                         filesystem root)"
163                    )));
164                }
165                static INVALID_REGEX: LazyLock<Regex> =
166                    LazyLock::new(|| Regex::new(r#"(?:^|/)(?:\.\.?(?:/|$))+"#).unwrap());
167                let source = INVALID_REGEX
168                    .replace_all(source_url, |s: &regex::Captures<'_>| s[0].replace('.', "_"));
169                *source_url = format!("{SOURCE_URL_PROTOCOL_STR}///{fs_str}/{origin_str}/{source}");
170            }
171            anyhow::Ok(())
172        };
173
174    let resolve_map = async |map: &mut SourceMapJson| {
175        if let Some(sources) = &mut map.sources {
176            let mut contents = if let Some(mut contents) = map.sources_content.take() {
177                contents.resize(sources.len(), None);
178                contents
179            } else {
180                iter::repeat_n(None, sources.len()).collect()
181            };
182
183            for (source, content) in sources.iter_mut().zip(contents.iter_mut()) {
184                if let Some(source) = source {
185                    if let Some(source_root) = &map.source_root {
186                        *source = format!("{source_root}{source}");
187                    }
188                    resolve_source(source, Some(content)).await?;
189                }
190            }
191
192            map.source_root = None;
193            map.sources_content = Some(contents);
194        }
195        anyhow::Ok(())
196    };
197
198    let Some(map) = map else {
199        return Ok(None);
200    };
201
202    let Ok(mut map): serde_json::Result<SourceMapJson> = serde_json::from_reader(map.read()) else {
203        // Silently ignore invalid sourcemaps
204        return Ok(None);
205    };
206
207    if let Some(file) = &mut map.file {
208        resolve_source(file, None).await?;
209    }
210
211    resolve_map(&mut map).await?;
212    for section in map.sections.iter_mut().flatten() {
213        resolve_map(&mut section.map).await?;
214    }
215
216    let map = Rope::from(serde_json::to_vec(&map)?);
217    Ok(Some(map))
218}
219
220fn unencoded_str_to_raw_value(unencoded: &str) -> Box<RawValue> {
221    serde_json::value::to_raw_value(unencoded)
222        .expect("serialization of a utf-8 string should always succeed")
223}
224
225fn uri_encode_path(path: &str) -> String {
226    path.split('/')
227        .map(|s| urlencoding::encode(s))
228        .collect::<Vec<_>>()
229        .join("/")
230}
231
232/// Applies the standard `turbopack:///[fs]/` source-URL transform to a structured map,
233/// sharing every other field (notably `sourcesContent`) with the input. `transform` is given
234/// the source string as found in the map (i.e. a URI) with the `turbopack:///[fs]/` prefix
235/// stripped.
236async fn transform_relative_files<F>(
237    map: &StructuredSourceMap,
238    context_path: &FileSystemPath,
239    mut transform: F,
240) -> Result<StructuredSourceMap>
241where
242    F: FnMut(&DiskFileSystem, &str) -> Result<String>,
243{
244    let context_fs = context_path.fs;
245    let context_fs = &*ResolvedVc::try_downcast_type::<DiskFileSystem>(context_fs)
246        .context("Expected the chunking context to have a DiskFileSystem")?
247        .await?;
248
249    let prefix = format!("{}///[{}]/", SOURCE_URL_PROTOCOL_STR, context_fs.name());
250
251    map.rewrite_sources(|src| {
252        if let Some(src_rest) = src.strip_prefix(&prefix) {
253            Ok(Some(transform(context_fs, src_rest)?))
254        } else {
255            Ok(None)
256        }
257    })
258}
259
260/// Turns `turbopack:///[project]` references in the map's sources into absolute `file://` uris.
261/// This is useful for debugging environments.
262pub async fn absolute_fileify_source_map(
263    map: &StructuredSourceMap,
264    context_path: FileSystemPath,
265) -> Result<StructuredSourceMap> {
266    transform_relative_files(map, &context_path.clone(), |context_fs, src_rest| {
267        let path = context_path.join(src_rest)?;
268
269        // `to_sys_path` returns a win32 path on Windows. `Url::from_file_path` can also handle
270        // verbatim (`\\?\`-prefixed) disk and UNC paths, in case that conversion failed.
271        let sys_path = context_fs.to_sys_path(&path);
272        Ok(Url::from_file_path(&sys_path)
273            .map_err(|()| {
274                anyhow::anyhow!("path {sys_path:?} cannot be converted to a file:// URI")
275            })?
276            .into())
277    })
278    .await
279}
280
281/// Turns `turbopack:///[project]` references in the map's sources into `./`-relative uris.
282/// This is useful in server environments and especially build environments.
283pub async fn relative_fileify_source_map(
284    map: &StructuredSourceMap,
285    context_path: FileSystemPath,
286    relative_path_to_output_root: RcStr,
287) -> Result<StructuredSourceMap> {
288    let relative_path_to_output_root = relative_path_to_output_root
289        .split('/')
290        .map(|s| urlencoding::encode(s))
291        .collect::<Vec<_>>()
292        .join("/");
293    transform_relative_files(map, &context_path, |_context_fs, src_rest| {
294        // NOTE: we just include the relative path prefix here instead of using `sourceRoot`
295        // since the spec on sourceRoot is broken.
296
297        // TODO(bgw): this shouldn't be necessary to uri encode since the strings we get out of the
298        // source map should already be uri encoded, however in the case of the turbopack scheme in
299        // particular we are inconsistent so be defensive here.
300        let src_rest = uri_encode_path(src_rest);
301        if relative_path_to_output_root.is_empty() {
302            Ok(src_rest.to_string())
303        } else {
304            Ok(format!("{relative_path_to_output_root}/{src_rest}"))
305        }
306    })
307    .await
308}
309
310#[cfg(test)]
311mod tests {
312    use std::path::Path;
313
314    use turbo_rcstr::{RcStr, rcstr};
315    use turbo_tasks::Vc;
316    use turbo_tasks_backend::{BackendOptions, TurboTasksBackend, noop_backing_storage};
317    use turbo_tasks_fs::FileSystem;
318
319    use super::*;
320
321    fn source_map_rope<'a>(
322        source_root: Option<&str>,
323        sources: impl IntoIterator<Item = &'a str>,
324    ) -> Rope {
325        Rope::from(
326            serde_json::to_string_pretty(
327                &serde_json::from_value::<SourceMapJson>(serde_json::json!({
328                    "version": 3,
329                    "mappings": "",
330                    "sourceRoot": source_root,
331                    "sources": sources.into_iter().map(Some).collect::<Vec<_>>(),
332                }))
333                .unwrap(),
334            )
335            .unwrap(),
336        )
337    }
338
339    #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
340    async fn test_resolve_source_map_sources() {
341        let tt = turbo_tasks::TurboTasks::new(TurboTasksBackend::new(
342            BackendOptions::default(),
343            noop_backing_storage(),
344        ));
345        tt.run_once(async move {
346            #[turbo_tasks::value]
347            struct SourceMapSourcesOutput {
348                resolved_sources: Vec<Option<String>>,
349                rooted_sources: Vec<Option<String>>,
350            }
351
352            #[turbo_tasks::function(operation, root)]
353            async fn resolve_source_map_sources_operation()
354            -> anyhow::Result<Vc<SourceMapSourcesOutput>> {
355                let sys_root = if cfg!(windows) {
356                    Path::new(r"C:\fake\root")
357                } else {
358                    Path::new(r"/fake/root")
359                };
360                let url_root = Url::from_directory_path(sys_root).unwrap();
361
362                let fs_root_path = DiskFileSystem::new(
363                    rcstr!("mock"),
364                    Vc::cell(RcStr::from(sys_root.to_str().unwrap())),
365                )
366                .root()
367                .await?;
368
369                let resolved_source_map: SourceMapJson = serde_json::from_str(
370                    &resolve_source_map_sources(
371                        Some(&source_map_rope(
372                            /* source_root */ None,
373                            [
374                                "page.js",
375                                "./current-dir-page.js",
376                                "../other%20route/page.js",
377                                // contains the file:// protocol/scheme
378                                url_root.join("absolute%20file%20url.js")?.as_str(),
379                                // A server-relative path starting with `/`, potentially includes a
380                                // windows disk
381                                &format!("{}/server%20relative%20path.js", url_root.path()),
382                                // A scheme-relative path
383                                url_root
384                                    .join("scheme%20relative%20path.js")?
385                                    .as_str()
386                                    .strip_prefix("file:")
387                                    .unwrap(),
388                                // non-file URLs are preserved
389                                "https://example.com/page%20path.js",
390                            ],
391                        )),
392                        // NOTE: the percent encoding here should NOT be decoded, as this is not
393                        // part of a `file://` URL
394                        &fs_root_path.join("app/source%20mapped/page.js").unwrap(),
395                    )
396                    .await?
397                    .unwrap()
398                    .to_str()?,
399                )?;
400
401                let rooted_source_map: SourceMapJson = serde_json::from_str(
402                    &resolve_source_map_sources(
403                        Some(&source_map_rope(
404                            // NOTE: these should get literally concated, a slash should NOT get
405                            // added.
406                            Some("../source%20root%20"),
407                            ["page.js"],
408                        )),
409                        &fs_root_path.join("app/page.js").unwrap(),
410                    )
411                    .await?
412                    .unwrap()
413                    .to_str()?,
414                )?;
415
416                Ok(SourceMapSourcesOutput {
417                    resolved_sources: resolved_source_map.sources.unwrap_or_default(),
418                    rooted_sources: rooted_source_map.sources.unwrap_or_default(),
419                }
420                .cell())
421            }
422
423            let resolved_source_maps = resolve_source_map_sources_operation()
424                .read_strongly_consistent()
425                .await?;
426
427            let prefix = format!("{SOURCE_URL_PROTOCOL_STR}///[mock]");
428            assert_eq!(
429                resolved_source_maps.resolved_sources,
430                vec![
431                    Some(format!("{prefix}/app/source%20mapped/page.js")),
432                    Some(format!("{prefix}/app/source%20mapped/current-dir-page.js")),
433                    Some(format!("{prefix}/app/other route/page.js")),
434                    Some(format!("{prefix}/absolute file url.js")),
435                    Some(format!("{prefix}/server relative path.js")),
436                    Some(format!("{prefix}/scheme relative path.js")),
437                    Some("https://example.com/page%20path.js".to_owned()),
438                ]
439            );
440
441            assert_eq!(
442                resolved_source_maps.rooted_sources,
443                vec![Some(format!("{prefix}/source root page.js"))]
444            );
445
446            anyhow::Ok(())
447        })
448        .await
449        .unwrap();
450    }
451}
turbopack_core/source_map/utils.rs

turbopack_core/source_map/
utils.rs
