xtask/
visualize_bundler_bench.rs

1use std::{
2    collections::BTreeMap,
3    fs::{self, File},
4    io::BufReader,
5    path::PathBuf,
6    str::FromStr,
7};
8
9use anyhow::{Context, Result};
10use num_format::{Locale, ToFormattedString};
11use plotters::{
12    backend::SVGBackend,
13    data::fitting_range,
14    prelude::{BindKeyPoints, ChartBuilder, IntoDrawingArea, PathElement, SeriesLabelPosition},
15    series::LineSeries,
16    style::{Color, RGBAColor, RGBColor},
17};
18use rustc_hash::FxHashSet;
19
20use crate::summarize_bench::data::{BaseBenchmarks, CStats};
21
22type ByModuleCount = BTreeMap<u32, CStats>;
23type ByBundler = BTreeMap<Bundler, ByModuleCount>;
24type ByBench = BTreeMap<String, ByBundler>;
25
26#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash)]
27enum Bundler {
28    NextJs11Ssr,
29    NextJs12Ssr,
30    ViteCsr,
31    ViteSsr,
32    ViteSwcCsr,
33    NextJs13Ssr,
34    NextJs13Rsc,
35    NextJs13Rcc,
36    TurbopackCsr,
37    TurbopackSsr,
38    TurbopackRsc,
39    TurbopackRcc,
40    Webpack,
41    Parcel,
42}
43
44impl std::fmt::Display for Bundler {
45    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
46        write!(f, "{}", self.as_str())
47    }
48}
49
50impl FromStr for Bundler {
51    type Err = ();
52
53    fn from_str(s: &str) -> Result<Self, Self::Err> {
54        match s {
55            "Next.js 11 SSR" => Ok(Self::NextJs11Ssr),
56            "Next.js 12 SSR" => Ok(Self::NextJs12Ssr),
57            "Next.js 13 SSR" => Ok(Self::NextJs13Ssr),
58            "Next.js 13 RSC" => Ok(Self::NextJs13Rsc),
59            "Next.js 13 RCC" => Ok(Self::NextJs13Rcc),
60            "Turbopack CSR" => Ok(Self::TurbopackCsr),
61            "Turbopack SSR" => Ok(Self::TurbopackSsr),
62            "Turbopack RSC" => Ok(Self::TurbopackRsc),
63            "Turbopack RCC" => Ok(Self::TurbopackRcc),
64            "Vite CSR" => Ok(Self::ViteCsr),
65            "Vite SSR" => Ok(Self::ViteSsr),
66            "Vite SWC CSR" => Ok(Self::ViteSwcCsr),
67            "Webpack" => Ok(Self::Webpack),
68            "Parcel" => Ok(Self::Parcel),
69            _ => Err(()),
70        }
71    }
72}
73
74impl Bundler {
75    fn as_str(&self) -> &'static str {
76        match self {
77            Self::NextJs11Ssr => "Next.js 11 SSR",
78            Self::NextJs12Ssr => "Next.js 12 SSR",
79            Self::NextJs13Ssr => "Next.js 13 SSR",
80            Self::NextJs13Rsc => "Next.js 13 RSC",
81            Self::NextJs13Rcc => "Next.js 13 RCC",
82            Self::TurbopackCsr => "Turbopack CSR",
83            Self::TurbopackSsr => "Turbopack SSR",
84            Self::TurbopackRsc => "Turbopack RSC",
85            Self::TurbopackRcc => "Turbopack RCC",
86            Self::ViteCsr => "Vite CSR",
87            Self::ViteSsr => "Vite SSR",
88            Self::ViteSwcCsr => "Vite SWC CSR",
89            Self::Webpack => "Webpack",
90            Self::Parcel => "Parcel",
91        }
92    }
93
94    fn color(&self) -> RGBColor {
95        match self {
96            // These are the currently used ones.
97            Self::NextJs12Ssr => plotters::style::full_palette::CYAN,
98            Self::NextJs11Ssr => plotters::style::full_palette::BLUE,
99
100            Self::TurbopackSsr => plotters::style::full_palette::RED,
101            Self::ViteSwcCsr => plotters::style::full_palette::GREEN,
102
103            // TODO(alexkirsz) These should probably change to be consistent with the above.
104            Self::NextJs13Ssr => plotters::style::full_palette::PURPLE,
105            Self::NextJs13Rsc => plotters::style::full_palette::PURPLE_300,
106            Self::NextJs13Rcc => plotters::style::full_palette::PURPLE_700,
107
108            Self::TurbopackCsr => plotters::style::full_palette::RED_200,
109            Self::TurbopackRsc => plotters::style::full_palette::RED_300,
110            Self::TurbopackRcc => plotters::style::full_palette::RED_700,
111
112            Self::ViteCsr => plotters::style::full_palette::GREEN_200,
113            Self::ViteSsr => plotters::style::full_palette::GREEN_300,
114
115            Self::Webpack => plotters::style::full_palette::YELLOW,
116            Self::Parcel => plotters::style::full_palette::BROWN,
117        }
118    }
119}
120
121pub fn generate(summary_path: PathBuf, filter_bundlers: Option<FxHashSet<&str>>) -> Result<()> {
122    let summary_file = File::open(&summary_path)?;
123    let reader = BufReader::new(summary_file);
124    let summary: BaseBenchmarks = serde_json::from_reader(reader)?;
125
126    let mut by_bench: ByBench = BTreeMap::new();
127    for (_, bench) in summary.benchmarks {
128        // TODO: Improve heuristic for detecting bundler benchmarks
129        if !bench.info.group_id.starts_with("bench_") {
130            continue;
131        }
132
133        let Some(function_id) = bench.info.function_id else {
134            continue;
135        };
136
137        let Ok(bundler) = Bundler::from_str(&function_id) else {
138            eprintln!("Skipping benchmark with unknown bundler: {function_id}");
139            continue;
140        };
141
142        if filter_bundlers
143            .as_ref()
144            .map(|bundlers| !bundlers.contains(bundler.as_str()))
145            .unwrap_or(false)
146        {
147            continue;
148        }
149
150        let by_bundler = by_bench.entry(bench.info.group_id).or_default();
151
152        let by_module_count = by_bundler.entry(bundler).or_default();
153
154        by_module_count.insert(
155            bench
156                .info
157                .value_str
158                .context("Missing value_str")?
159                .split_ascii_whitespace()
160                .collect::<Vec<&str>>()[0]
161                .parse()?,
162            // we want to use slope instead of mean when available since this is a better
163            // estimation of the real performance values when iterations go to infinity
164            bench.estimates.slope.unwrap_or(bench.estimates.mean),
165        );
166    }
167
168    let output_path = summary_path.parent().context("summary_path needs parent")?;
169    generate_scaling(output_path.join("scaling"), &by_bench)?;
170
171    Ok(())
172}
173
174#[derive(Debug, Clone, Copy)]
175enum FormatTimeStyle {
176    Milliseconds,
177    Seconds,
178}
179
180impl FormatTimeStyle {
181    fn format(self, ns: f64) -> String {
182        let value = (match self {
183            FormatTimeStyle::Milliseconds => ns / 1e6,
184            FormatTimeStyle::Seconds => ns / 1e9,
185        }
186        .round() as u64)
187            .to_formatted_string(&Locale::en);
188
189        format!("{}{}", value, self.unit())
190    }
191
192    fn unit(self) -> &'static str {
193        match self {
194            FormatTimeStyle::Milliseconds => "ms",
195            FormatTimeStyle::Seconds => "s",
196        }
197    }
198}
199
200#[derive(Debug, Clone, Copy)]
201enum Theme {
202    Light,
203    Dark,
204}
205
206impl Theme {
207    fn name(self) -> &'static str {
208        match self {
209            Theme::Light => "light",
210            Theme::Dark => "dark",
211        }
212    }
213
214    fn legend_background_color(self) -> RGBAColor {
215        match self {
216            Theme::Light => plotters::style::colors::WHITE.into(),
217            Theme::Dark => RGBColor(34, 34, 34).into(),
218        }
219    }
220
221    fn light_line_color(self) -> RGBAColor {
222        match self {
223            Theme::Light => plotters::style::colors::BLACK.mix(0.1),
224            Theme::Dark => plotters::style::colors::WHITE.mix(0.1),
225        }
226    }
227
228    fn bold_line_color(self) -> RGBAColor {
229        match self {
230            Theme::Light => plotters::style::colors::BLACK.mix(0.2),
231            Theme::Dark => plotters::style::colors::WHITE.mix(0.2),
232        }
233    }
234
235    fn axis_line_color(self) -> RGBAColor {
236        match self {
237            Theme::Light => plotters::style::colors::BLACK.into(),
238            Theme::Dark => plotters::style::colors::WHITE.into(),
239        }
240    }
241
242    fn label_color(self) -> RGBAColor {
243        match self {
244            Theme::Light => plotters::style::colors::BLACK.mix(0.75),
245            Theme::Dark => plotters::style::colors::WHITE.mix(0.75),
246        }
247    }
248
249    fn axis_desc_color(self) -> RGBAColor {
250        match self {
251            Theme::Light => plotters::style::colors::BLACK.into(),
252            Theme::Dark => plotters::style::colors::WHITE.into(),
253        }
254    }
255}
256
257const THEMES: [Theme; 2] = [Theme::Light, Theme::Dark];
258
259fn generate_scaling(output_path: PathBuf, by_bench: &ByBench) -> Result<()> {
260    fs::create_dir_all(&output_path)?;
261
262    for theme in THEMES {
263        for (bench_name, by_bundler) in by_bench {
264            let module_counts: FxHashSet<_> = by_bundler
265                .values()
266                .flat_map(|by_module_count| by_module_count.keys())
267                .copied()
268                .collect();
269            let module_count_range = fitting_range(module_counts.iter());
270
271            // Ensure we have labels for every sampled module count.
272            let module_count_range =
273                module_count_range.with_key_points(module_counts.into_iter().collect());
274
275            let time_range_iter = by_bundler.values().flat_map(|by_module_count| {
276                by_module_count.values().map(|stats| stats.point_estimate)
277            });
278
279            // Make the time range end 5% higher than the maximum time value so the highest
280            // point is not cut off.
281            let time_range_end = time_range_iter
282                // f64 does not implement Ord.
283                .fold(0.0, |max, time| if time > max { time } else { max })
284                * 1.05;
285            // Ensure the time range starts at 0 instead of the minimum time value.
286            let time_range = 0.0..time_range_end;
287
288            let format_time_style = if time_range.end > 10e8 {
289                FormatTimeStyle::Seconds
290            } else {
291                FormatTimeStyle::Milliseconds
292            };
293
294            let file_name = output_path.join(format!("{}_{}.svg", bench_name, theme.name()));
295            let root = SVGBackend::new(&file_name, (960, 720)).into_drawing_area();
296            let mut chart = ChartBuilder::on(&root)
297                .x_label_area_size(60)
298                // The y labels are horizontal and have units, so they take some room.
299                .y_label_area_size(80)
300                .margin(30)
301                .build_cartesian_2d(module_count_range, time_range)?;
302
303            for (bundler, by_module_count) in by_bundler.iter() {
304                let color = bundler.color();
305                let points = by_module_count
306                    .iter()
307                    .map(|(count, stats)| (count.to_owned(), stats.point_estimate));
308
309                chart
310                    .draw_series(LineSeries::new(points.clone(), color.stroke_width(4)))?
311                    .label(bundler.as_str())
312                    .legend(move |(x, y)| {
313                        PathElement::new(vec![(x, y), (x + 20, y)], color.stroke_width(4))
314                    });
315            }
316
317            // This is the font used by the turbo.build website.
318            let font = r#"ui-sans-serif, system-ui, -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, "Helvetica Neue", Arial, "Noto Sans", sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol", "Noto Color Emoji""#;
319
320            chart
321                .configure_mesh()
322                .x_labels(10)
323                .y_labels(10)
324                .x_desc("Number of modules")
325                .y_desc("Mean time — lower is better")
326                .x_label_style((font, 20, &theme.label_color()))
327                .y_label_style((font, 20, &theme.label_color()))
328                .axis_desc_style((font, 24, &theme.axis_desc_color()))
329                .x_label_formatter(&|v| v.to_formatted_string(&Locale::en))
330                .y_label_formatter(&|v| format_time_style.format(*v))
331                .bold_line_style(theme.bold_line_color())
332                .light_line_style(theme.light_line_color())
333                .axis_style(theme.axis_line_color())
334                .draw()?;
335
336            chart
337                .configure_series_labels()
338                .background_style(theme.legend_background_color())
339                .border_style(theme.bold_line_color())
340                .label_font((font, 20, &theme.axis_desc_color()))
341                .position(SeriesLabelPosition::UpperLeft)
342                .margin(16)
343                .draw()?;
344
345            root.present()?;
346        }
347    }
348
349    Ok(())
350}
xtask/visualize_bundler_bench.rs

xtask/
visualize_bundler_bench.rs
