turbopack_core/source_map/
mod.rs

1use std::{borrow::Cow, io::Write, ops::Deref, sync::Arc};
2
3use anyhow::Result;
4use bincode::{
5    Decode, Encode,
6    de::Decoder,
7    enc::Encoder,
8    error::{DecodeError, EncodeError},
9};
10use bytes_str::BytesStr;
11use either::Either;
12use once_cell::sync::Lazy;
13use ref_cast::RefCast;
14use regex::Regex;
15use swc_sourcemap::{DecodedMap, SourceMap as RegularMap, SourceMapBuilder, SourceMapIndex};
16use turbo_rcstr::{RcStr, rcstr};
17use turbo_tasks::{ResolvedVc, TryJoinIterExt, Vc};
18use turbo_tasks_fs::{
19    File, FileContent, FileSystem, FileSystemPath, VirtualFileSystem,
20    rope::{Rope, RopeBuilder},
21};
22
23use crate::{
24    SOURCE_URL_PROTOCOL, asset::AssetContent, source::Source,
25    source_map::utils::add_default_ignore_list, source_pos::SourcePos,
26    virtual_source::VirtualSource,
27};
28
29pub(crate) mod source_map_asset;
30pub mod utils;
31
32pub use source_map_asset::SourceMapAsset;
33
34/// Represents an empty value in a u32 variable in the sourcemap crate.
35static SOURCEMAP_CRATE_NONE_U32: u32 = !0;
36
37/// Allows callers to generate source maps.
38#[turbo_tasks::value_trait]
39pub trait GenerateSourceMap {
40    /// Generates a usable source map, capable of both tracing and stringifying.
41    #[turbo_tasks::function]
42    fn generate_source_map(self: Vc<Self>) -> Vc<FileContent>;
43
44    /// Returns an individual section of the larger source map, if found.
45    #[turbo_tasks::function]
46    fn by_section(self: Vc<Self>, _section: RcStr) -> Vc<FileContent> {
47        FileContent::NotFound.cell()
48    }
49}
50
51/// Implements the source map specification as either a decoded or sectioned sourcemap.
52///
53/// - A "decoded" map represents a source map as if it was came out of a JSON
54/// decode.
55/// - A "sectioned" source map is a tree of many [SourceMap]
56/// covering regions of an output file.
57///
58/// The distinction between the source map spec's [sourcemap::Index] and our
59/// [SourceMap::Sectioned] is whether the sections are represented with Vcs
60/// pointers.
61#[turbo_tasks::value(shared, cell = "new", eq = "manual")]
62#[derive(Debug)]
63pub struct SourceMap {
64    /// A decoded source map contains no Vcs.
65    #[turbo_tasks(trace_ignore)]
66    map: Arc<CrateMapWrapper>,
67}
68impl Eq for SourceMap {}
69impl PartialEq for SourceMap {
70    fn eq(&self, other: &Self) -> bool {
71        Arc::ptr_eq(&self.map, &other.map)
72    }
73}
74
75#[turbo_tasks::value(transparent)]
76pub struct OptionSourceMap(Option<SourceMap>);
77
78#[turbo_tasks::value_impl]
79impl OptionSourceMap {
80    #[turbo_tasks::function]
81    pub fn none() -> Vc<Self> {
82        Vc::cell(None)
83    }
84}
85
86impl OptionSourceMap {
87    pub fn none_resolved() -> ResolvedVc<Self> {
88        ResolvedVc::cell(None)
89    }
90}
91
92/// A token represents a mapping in a source map.
93///
94/// It may either be Synthetic, meaning it was generated by some build tool and doesn't
95/// represent a location in a user-authored source file, or it is Original, meaning it represents a
96/// real location in source file.
97#[turbo_tasks::value]
98#[derive(Clone, Debug)]
99pub enum Token {
100    Synthetic(SyntheticToken),
101    Original(OriginalToken),
102}
103
104#[turbo_tasks::value]
105#[derive(Clone, Debug)]
106pub struct TokenWithSource {
107    pub token: Token,
108    pub source_content: Option<ResolvedVc<Box<dyn Source>>>,
109}
110
111/// A SyntheticToken represents a region of the generated file that was created
112/// by some build tool.
113#[turbo_tasks::value]
114#[derive(Clone, Debug)]
115pub struct SyntheticToken {
116    pub generated_line: u32,
117    pub generated_column: u32,
118    pub guessed_original_file: Option<RcStr>,
119}
120
121/// An OriginalToken represents a region of the generated file that exists in
122/// user-authored source file.
123#[turbo_tasks::value]
124#[derive(Clone, Debug)]
125pub struct OriginalToken {
126    pub generated_line: u32,
127    pub generated_column: u32,
128    pub original_file: RcStr,
129    pub original_line: u32,
130    pub original_column: u32,
131    pub name: Option<RcStr>,
132}
133
134impl Token {
135    pub fn generated_line(&self) -> u32 {
136        match self {
137            Self::Original(t) => t.generated_line,
138            Self::Synthetic(t) => t.generated_line,
139        }
140    }
141
142    pub fn generated_column(&self) -> u32 {
143        match self {
144            Self::Original(t) => t.generated_column,
145            Self::Synthetic(t) => t.generated_column,
146        }
147    }
148
149    pub fn with_offset(&self, line_offset: u32, column_offset: u32) -> Self {
150        match self {
151            Self::Original(t) => Self::Original(OriginalToken {
152                generated_line: t.generated_line + line_offset,
153                generated_column: if t.generated_line == 0 {
154                    t.generated_column + column_offset
155                } else {
156                    t.generated_column
157                },
158                original_file: t.original_file.clone(),
159                original_line: t.original_line,
160                original_column: t.original_column,
161                name: t.name.clone(),
162            }),
163            Self::Synthetic(t) => Self::Synthetic(SyntheticToken {
164                generated_line: t.generated_line + line_offset,
165                generated_column: if t.generated_line == 0 {
166                    t.generated_column + column_offset
167                } else {
168                    t.generated_column
169                },
170                guessed_original_file: t.guessed_original_file.clone(),
171            }),
172        }
173    }
174}
175
176impl From<swc_sourcemap::Token<'_>> for Token {
177    fn from(t: swc_sourcemap::Token) -> Self {
178        if t.has_source() {
179            Token::Original(OriginalToken {
180                generated_line: t.get_dst_line(),
181                generated_column: t.get_dst_col(),
182                original_file: RcStr::from(
183                    t.get_source()
184                        .expect("already checked token has source")
185                        .clone(),
186                ),
187                original_line: t.get_src_line(),
188                original_column: t.get_src_col(),
189                name: t.get_name().cloned().map(RcStr::from),
190            })
191        } else {
192            Token::Synthetic(SyntheticToken {
193                generated_line: t.get_dst_line(),
194                generated_column: t.get_dst_col(),
195                guessed_original_file: None,
196            })
197        }
198    }
199}
200
201impl TryInto<swc_sourcemap::RawToken> for Token {
202    type Error = std::num::ParseIntError;
203
204    fn try_into(self) -> Result<swc_sourcemap::RawToken, Self::Error> {
205        Ok(match self {
206            Self::Original(t) => swc_sourcemap::RawToken {
207                dst_col: t.generated_column,
208                dst_line: t.generated_line,
209                name_id: match t.name {
210                    None => SOURCEMAP_CRATE_NONE_U32,
211                    Some(name) => name.parse()?,
212                },
213                src_col: t.original_column,
214                src_line: t.original_line,
215                src_id: t.original_file.parse()?,
216                is_range: false,
217            },
218            Self::Synthetic(t) => swc_sourcemap::RawToken {
219                dst_col: t.generated_column,
220                dst_line: t.generated_line,
221                name_id: SOURCEMAP_CRATE_NONE_U32,
222                src_col: SOURCEMAP_CRATE_NONE_U32,
223                src_line: SOURCEMAP_CRATE_NONE_U32,
224                src_id: SOURCEMAP_CRATE_NONE_U32,
225                is_range: false,
226            },
227        })
228    }
229}
230
231impl SourceMap {
232    /// Creates a new SourceMap::Decoded Vc out of a [RegularMap] instance.
233    fn new_regular(map: RegularMap) -> Self {
234        Self::new_decoded(DecodedMap::Regular(map))
235    }
236
237    /// Creates a new SourceMap::Decoded Vc out of a [DecodedMap] instance.
238    fn new_decoded(map: DecodedMap) -> Self {
239        SourceMap {
240            map: Arc::new(CrateMapWrapper(map)),
241        }
242    }
243
244    pub fn new_from_rope(content: &Rope) -> Result<Option<Self>> {
245        let Ok(map) = DecodedMap::from_reader(content.read()) else {
246            return Ok(None);
247        };
248        Ok(Some(SourceMap::new_decoded(map)))
249    }
250}
251
252#[turbo_tasks::value_impl]
253impl SourceMap {
254    /// This function should be used sparingly to reduce memory usage, only in cold code paths
255    /// (issue resolving, etc).
256    #[turbo_tasks::function]
257    pub async fn new_from_rope_cached(content: Vc<FileContent>) -> Result<Vc<OptionSourceMap>> {
258        let content = content.await?;
259        let Some(content) = content.as_content() else {
260            return Ok(OptionSourceMap::none());
261        };
262        Ok(Vc::cell(SourceMap::new_from_rope(content.content())?))
263    }
264}
265
266impl SourceMap {
267    pub fn to_source_map(&self) -> Arc<CrateMapWrapper> {
268        self.map.clone()
269    }
270}
271
272static EMPTY_SOURCE_MAP_ROPE: Lazy<Rope> =
273    Lazy::new(|| Rope::from(r#"{"version":3,"sources":[],"names":[],"mappings":"A"}"#));
274
275impl SourceMap {
276    /// A source map that contains no actual source location information (no
277    /// `sources`, no mappings that point into a source). This is used to tell
278    /// Chrome that the generated code starting at a particular offset is no
279    /// longer part of the previous section's mappings.
280    pub fn empty() -> Self {
281        let mut builder = SourceMapBuilder::new(None);
282        builder.add(0, 0, 0, 0, None, None, false);
283        SourceMap::new_regular(builder.into_sourcemap())
284    }
285
286    /// A source map that contains no actual source location information (no
287    /// `sources`, no mappings that point into a source). This is used to tell
288    /// Chrome that the generated code starting at a particular offset is no
289    /// longer part of the previous section's mappings.
290    pub fn empty_rope() -> Rope {
291        EMPTY_SOURCE_MAP_ROPE.clone()
292    }
293
294    pub fn sections_to_rope(
295        sections: impl IntoIterator<Item = (SourcePos, Rope)>,
296        debug_id: Option<RcStr>,
297    ) -> Rope {
298        let mut sections = sections.into_iter().peekable();
299
300        let mut first = sections.next();
301        if let Some((offset, map)) = &mut first
302            && sections.peek().is_none()
303            && *offset == (0, 0)
304            && debug_id.is_none()
305        {
306            // There is just a single sourcemap that starts at the beginning of the file.
307            return std::mem::take(map);
308        }
309
310        // My kingdom for a decent dedent macro with interpolation!
311        // NOTE: The empty `sources` array is technically incorrect, but there is a bug
312        // in Node.js that requires sectioned source maps to have a `sources` array.
313        let mut rope = RopeBuilder::from(
314            r#"{
315  "version": 3,
316  "sources": [],
317"#,
318        );
319        if let Some(debug_id) = debug_id {
320            writeln!(rope, r#"  "debugId": "{debug_id}","#).unwrap();
321        }
322        rope += "  \"sections\": [";
323
324        let mut first_section = true;
325        for (offset, section_map) in first.into_iter().chain(sections) {
326            if !first_section {
327                rope += ",";
328            }
329            first_section = false;
330
331            write!(
332                rope,
333                r#"
334    {{"offset": {{"line": {}, "column": {}}}, "map": "#,
335                offset.line, offset.column,
336            )
337            .unwrap();
338
339            rope += &section_map;
340
341            rope += "}";
342        }
343
344        rope += "]";
345
346        rope += "\n}";
347
348        rope.build()
349    }
350
351    /// Stringifies the source map into JSON bytes.
352    pub fn to_rope(&self) -> Result<Rope> {
353        let mut bytes = vec![];
354        self.map.0.to_writer(&mut bytes)?;
355        Ok(Rope::from(bytes))
356    }
357
358    /// Traces a generated line/column into an mapping token representing either
359    /// synthetic code or user-authored original code.
360    pub fn lookup_token(&self, line: u32, column: u32) -> Token {
361        let (token, _) = self.lookup_token_and_source_internal(line, column, true);
362        token
363    }
364
365    /// Traces a generated line/column into an mapping token representing either
366    /// synthetic code or user-authored original code.
367    pub async fn lookup_token_and_source(&self, line: u32, column: u32) -> Result<TokenWithSource> {
368        let (token, content) = self.lookup_token_and_source_internal(line, column, true);
369        Ok(TokenWithSource {
370            token,
371            source_content: match content {
372                Some(v) => Some(v.to_resolved().await?),
373                None => None,
374            },
375        })
376    }
377
378    pub async fn with_resolved_sources(&self, origin: FileSystemPath) -> Result<Self> {
379        async fn resolve_source(
380            source_request: BytesStr,
381            source_content: Option<BytesStr>,
382            origin: FileSystemPath,
383        ) -> Result<(BytesStr, BytesStr)> {
384            Ok(
385                if let Some(path) = origin.parent().try_join(&source_request) {
386                    let path_str = path.value_to_string().await?;
387                    let source = format!("{SOURCE_URL_PROTOCOL}///{path_str}");
388                    let source_content = if let Some(source_content) = source_content {
389                        source_content
390                    } else if let FileContent::Content(file) = &*path.read().await? {
391                        let text = file.content().to_str()?;
392                        text.to_string().into()
393                    } else {
394                        format!("unable to read source {path_str}").into()
395                    };
396                    (source.into(), source_content)
397                } else {
398                    let origin_str = origin.value_to_string().await?;
399                    static INVALID_REGEX: Lazy<Regex> =
400                        Lazy::new(|| Regex::new(r#"(?:^|/)(?:\.\.?(?:/|$))+"#).unwrap());
401                    let source = INVALID_REGEX
402                        .replace_all(&source_request, |s: &regex::Captures<'_>| {
403                            s[0].replace('.', "_")
404                        });
405                    let source = format!("{SOURCE_URL_PROTOCOL}///{origin_str}/{source}");
406                    let source_content = source_content.unwrap_or_else(|| {
407                        format!(
408                            "unable to access {source_request} in {origin_str} (it's leaving the \
409                             filesystem root)"
410                        )
411                        .into()
412                    });
413                    (source.into(), source_content)
414                },
415            )
416        }
417        async fn regular_map_with_resolved_sources(
418            map: &RegularMapWrapper,
419            origin: FileSystemPath,
420        ) -> Result<RegularMap> {
421            let map = &map.0;
422            let file = map.get_file().cloned();
423            let tokens = map.tokens().map(|t| t.get_raw_token()).collect();
424            let names = map.names().cloned().collect();
425            let count = map.get_source_count() as usize;
426            let sources = map.sources().cloned().collect::<Vec<_>>();
427            let source_contents = map
428                .source_contents()
429                .map(|s| s.cloned())
430                .collect::<Vec<_>>();
431            let mut new_sources = Vec::with_capacity(count);
432            let mut new_source_contents = Vec::with_capacity(count);
433            for (source, source_content) in sources.into_iter().zip(source_contents.into_iter()) {
434                let (source, source_content) =
435                    resolve_source(source, source_content, origin.clone()).await?;
436                new_sources.push(source);
437                new_source_contents.push(Some(source_content));
438            }
439            let mut map =
440                RegularMap::new(file, tokens, names, new_sources, Some(new_source_contents));
441
442            add_default_ignore_list(&mut map);
443
444            Ok(map)
445        }
446        async fn decoded_map_with_resolved_sources(
447            map: &CrateMapWrapper,
448            origin: FileSystemPath,
449        ) -> Result<CrateMapWrapper> {
450            Ok(CrateMapWrapper(match &map.0 {
451                DecodedMap::Regular(map) => {
452                    let map = RegularMapWrapper::ref_cast(map);
453                    DecodedMap::Regular(regular_map_with_resolved_sources(map, origin).await?)
454                }
455                DecodedMap::Index(map) => {
456                    let count = map.get_section_count() as usize;
457                    let file = map.get_file().cloned();
458                    let sections = map
459                        .sections()
460                        .filter_map(|section| {
461                            section
462                                .get_sourcemap()
463                                .map(|s| (section.get_offset(), CrateMapWrapper::ref_cast(s)))
464                        })
465                        .collect::<Vec<_>>();
466                    let sections = sections
467                        .into_iter()
468                        .map(|(offset, map)| {
469                            let origin = origin.clone();
470                            async move {
471                                Ok((
472                                    offset,
473                                    Box::pin(decoded_map_with_resolved_sources(
474                                        map,
475                                        origin.clone(),
476                                    ))
477                                    .await?,
478                                ))
479                            }
480                        })
481                        .try_join()
482                        .await?;
483                    let mut new_sections = Vec::with_capacity(count);
484                    for (offset, map) in sections {
485                        new_sections.push(swc_sourcemap::SourceMapSection::new(
486                            offset,
487                            // Urls are deprecated and we don't accept them
488                            None,
489                            Some(map.0),
490                        ));
491                    }
492                    DecodedMap::Index(SourceMapIndex::new(file, new_sections))
493                }
494                DecodedMap::Hermes(_) => {
495                    todo!("hermes source maps are not implemented");
496                }
497            }))
498        }
499
500        let map = Box::pin(decoded_map_with_resolved_sources(&self.map, origin)).await?;
501        Ok(Self::new_decoded(map.0))
502    }
503}
504
505#[turbo_tasks::function]
506fn sourcemap_content_fs_root() -> Vc<FileSystemPath> {
507    VirtualFileSystem::new_with_name(rcstr!("sourcemap-content")).root()
508}
509
510#[turbo_tasks::function]
511async fn sourcemap_content_source(path: RcStr, content: RcStr) -> Result<Vc<Box<dyn Source>>> {
512    let path = sourcemap_content_fs_root().await?.join(&path)?;
513    let content = AssetContent::file(FileContent::new(File::from(content)).cell());
514    Ok(Vc::upcast(VirtualSource::new(path, content)))
515}
516
517impl SourceMap {
518    fn lookup_token_and_source_internal(
519        &self,
520        line: u32,
521        column: u32,
522        need_source_content: bool,
523    ) -> (Token, Option<Vc<Box<dyn Source>>>) {
524        let mut content: Option<Vc<Box<dyn Source>>> = None;
525
526        let token: Token = {
527            let map = &self.map;
528
529            let tok = map.lookup_token(line, column);
530            let mut token = tok.map(Token::from).unwrap_or_else(|| {
531                Token::Synthetic(SyntheticToken {
532                    generated_line: line,
533                    generated_column: column,
534                    guessed_original_file: None,
535                })
536            });
537
538            if let Token::Synthetic(SyntheticToken {
539                guessed_original_file,
540                ..
541            }) = &mut token
542                && let DecodedMap::Regular(map) = &map.0
543                && map.get_source_count() == 1
544            {
545                let source = map.sources().next().unwrap().clone();
546                *guessed_original_file = Some(RcStr::from(source));
547            }
548
549            if need_source_content
550                && content.is_none()
551                && let Some(map) = map.as_regular_source_map()
552            {
553                content = tok.and_then(|tok| {
554                    let src_id = tok.get_src_id();
555
556                    let name = map.get_source(src_id);
557                    let content = map.get_source_contents(src_id);
558
559                    let (name, content) = name.zip(content)?;
560                    Some(sourcemap_content_source(
561                        name.clone().into(),
562                        content.clone().into(),
563                    ))
564                });
565            }
566
567            token
568        };
569
570        (token, content)
571    }
572}
573
574impl SourceMap {
575    pub fn tokens(&self) -> impl Iterator<Item = Token> + '_ {
576        let map = &self.map;
577
578        fn regular_map_to_tokens(
579            map: &RegularMap,
580            offset_line: u32,
581            offset_column: u32,
582        ) -> impl Iterator<Item = Token> + '_ {
583            map.tokens()
584                .map(move |t| Token::from(t).with_offset(offset_line, offset_column))
585        }
586
587        fn index_map_to_tokens(
588            map: &SourceMapIndex,
589            offset_line: u32,
590            offset_column: u32,
591        ) -> impl Iterator<Item = Token> + '_ {
592            map.sections().flat_map(move |section| {
593                let (line, col) = section.get_offset();
594                let offset_line = offset_line + line;
595                let offset_column = if line == 0 { offset_column + col } else { col };
596                if let Some(source_map) = section.get_sourcemap() {
597                    Either::Left(Box::new(decoded_map_to_tokens(
598                        source_map,
599                        offset_line,
600                        offset_column,
601                    )) as Box<dyn Iterator<Item = Token>>)
602                } else {
603                    Either::Right(std::iter::empty())
604                }
605            })
606        }
607
608        fn decoded_map_to_tokens(
609            map: &DecodedMap,
610            offset_line: u32,
611            offset_column: u32,
612        ) -> impl Iterator<Item = Token> + '_ {
613            match map {
614                DecodedMap::Regular(map) => {
615                    Either::Left(regular_map_to_tokens(map, offset_line, offset_column))
616                }
617                DecodedMap::Index(map) => {
618                    Either::Right(index_map_to_tokens(map, offset_line, offset_column))
619                }
620                DecodedMap::Hermes(_) => {
621                    todo!("hermes source maps are not implemented");
622                }
623            }
624        }
625
626        decoded_map_to_tokens(&map.0, 0, 0)
627    }
628}
629
630#[turbo_tasks::value_impl]
631impl GenerateSourceMap for SourceMap {
632    #[turbo_tasks::function]
633    fn generate_source_map(&self) -> Result<Vc<FileContent>> {
634        Ok(FileContent::Content(File::from(self.to_rope()?)).cell())
635    }
636}
637
638/// Wraps the DecodedMap struct so that it is Sync and Send.
639///
640/// # Safety
641///
642/// Must not use per line access to the SourceMap, as it is not thread safe.
643#[derive(Debug, RefCast)]
644#[repr(transparent)]
645pub struct CrateMapWrapper(DecodedMap);
646
647// Safety: DecodedMap contains a raw pointer, which isn't Send, which is
648// required to cache in a Vc. So, we have wrap it in 4 layers of cruft to do it.
649unsafe impl Send for CrateMapWrapper {}
650unsafe impl Sync for CrateMapWrapper {}
651
652/// Wraps the RegularMap struct so that it is Sync and Send.
653///
654/// # Safety
655///
656/// Must not use per line access to the SourceMap, as it is not thread safe.
657#[derive(Debug, RefCast)]
658#[repr(transparent)]
659pub struct RegularMapWrapper(RegularMap);
660
661// Safety: RegularMap contains a raw pointer, which isn't Send, which is
662// required to cache in a Vc. So, we have wrap it in 4 layers of cruft to do it.
663unsafe impl Send for RegularMapWrapper {}
664unsafe impl Sync for RegularMapWrapper {}
665
666impl CrateMapWrapper {
667    pub fn as_regular_source_map(&self) -> Option<Cow<'_, RegularMap>> {
668        match &self.0 {
669            DecodedMap::Regular(m) => Some(Cow::Borrowed(m)),
670            DecodedMap::Index(m) => m.flatten().map(Cow::Owned).ok(),
671            _ => None,
672        }
673    }
674}
675
676impl Deref for CrateMapWrapper {
677    type Target = DecodedMap;
678
679    fn deref(&self) -> &Self::Target {
680        &self.0
681    }
682}
683
684impl Encode for CrateMapWrapper {
685    fn encode<E: Encoder>(&self, encoder: &mut E) -> Result<(), EncodeError> {
686        let mut bytes = Vec::new();
687        self.0
688            .to_writer(&mut bytes)
689            .map_err(|e| EncodeError::OtherString(e.to_string()))?;
690        bytes.encode(encoder)
691    }
692}
693
694impl<Context> Decode<Context> for CrateMapWrapper {
695    fn decode<D: Decoder<Context = Context>>(decoder: &mut D) -> Result<Self, DecodeError> {
696        let bytes = Vec::<u8>::decode(decoder)?;
697        let map = DecodedMap::from_reader(&*bytes)
698            .map_err(|e| DecodeError::OtherString(e.to_string()))?;
699        Ok(CrateMapWrapper(map))
700    }
701}
702
703bincode::impl_borrow_decode!(CrateMapWrapper);
turbopack_core/source_map/mod.rs

turbopack_core/source_map/
mod.rs
