next_build_test/
lib.rs

1#![feature(min_specialization)]
2#![feature(arbitrary_self_types)]
3#![feature(arbitrary_self_types_pointers)]
4
5use std::{str::FromStr, time::Instant};
6
7use anyhow::{Context, Result, bail};
8use futures_util::{StreamExt, TryStreamExt};
9use next_api::{
10    entrypoints::Entrypoints,
11    project::{HmrTarget, ProjectContainer, ProjectOptions},
12    route::{Endpoint, EndpointOutputPaths, Route, endpoint_write_to_disk},
13};
14use turbo_rcstr::{RcStr, rcstr};
15use turbo_tasks::{
16    Effects, ReadConsistency, ReadRef, ResolvedVc, TransientInstance, TurboTasks, Vc,
17    read_strongly_consistent_and_apply_effects, take_effects,
18};
19use turbo_tasks_backend::TurboTasksBackend;
20use turbo_tasks_malloc::TurboMalloc;
21
22pub async fn main_inner(
23    tt: &TurboTasks<TurboTasksBackend>,
24    strategy: Strategy,
25    factor: usize,
26    limit: usize,
27    files: Option<Vec<String>>,
28) -> Result<()> {
29    let path = std::env::current_dir()?.join("project_options.json");
30    let mut file = std::fs::File::open(&path)
31        .with_context(|| format!("loading file at {}", path.display()))?;
32
33    let mut options: ProjectOptions = serde_json::from_reader(&mut file)?;
34
35    if matches!(strategy, Strategy::Development { .. }) {
36        options.dev = true;
37        options.watch.enable = true;
38    } else {
39        options.dev = false;
40        options.watch.enable = false;
41    }
42
43    let project = tt
44        .run(async {
45            let container_op = ProjectContainer::new_operation(rcstr!("next.js"), options.dev);
46            ProjectContainer::initialize(container_op, options).await?;
47            container_op.resolve().strongly_consistent().await
48        })
49        .await?;
50
51    tracing::info!("collecting endpoints");
52
53    #[turbo_tasks::function(operation, root)]
54    fn project_entrypoints_operation(project: ResolvedVc<ProjectContainer>) -> Vc<Entrypoints> {
55        project.entrypoints()
56    }
57    let entrypoints = tt
58        .run(async move {
59            project_entrypoints_operation(project)
60                .read_strongly_consistent()
61                .await
62        })
63        .await?;
64
65    let mut routes = if let Some(files) = files {
66        tracing::info!("building only the files:");
67        for file in &files {
68            tracing::info!("  {}", file);
69        }
70
71        // filter out the files that are not in the list
72        // we expect this to be small so linear search OK
73        Box::new(files.into_iter().filter_map(|f| {
74            entrypoints
75                .routes
76                .iter()
77                .find(|(name, _)| f.as_str() == name.as_str())
78                .map(|(name, route)| (name.clone(), route.clone()))
79        })) as Box<dyn Iterator<Item = _> + Send + Sync>
80    } else {
81        Box::new(entrypoints.routes.clone().into_iter())
82    };
83
84    if strategy.randomized() {
85        routes = Box::new(shuffle(routes))
86    }
87
88    let start = Instant::now();
89    let count = render_routes(tt, routes, strategy, factor, limit).await?;
90    tracing::info!("rendered {} pages in {:?}", count, start.elapsed());
91
92    if count == 0 {
93        tracing::info!("No pages found, these pages exist:");
94        for (route, _) in entrypoints.routes.iter() {
95            tracing::info!("  {}", route);
96        }
97    }
98
99    if matches!(strategy, Strategy::Development { .. }) {
100        hmr(tt, project).await?;
101    }
102
103    Ok(())
104}
105
106#[derive(PartialEq, Copy, Clone)]
107pub enum Strategy {
108    Sequential { randomized: bool },
109    Concurrent,
110    Parallel { randomized: bool },
111    Development { randomized: bool },
112}
113
114impl std::fmt::Display for Strategy {
115    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
116        match self {
117            Strategy::Sequential { randomized: false } => write!(f, "sequential"),
118            Strategy::Sequential { randomized: true } => write!(f, "sequential-randomized"),
119            Strategy::Concurrent => write!(f, "concurrent"),
120            Strategy::Parallel { randomized: false } => write!(f, "parallel"),
121            Strategy::Parallel { randomized: true } => write!(f, "parallel-randomized"),
122            Strategy::Development { randomized: false } => write!(f, "development"),
123            Strategy::Development { randomized: true } => write!(f, "development-randomized"),
124        }
125    }
126}
127
128impl FromStr for Strategy {
129    type Err = anyhow::Error;
130
131    fn from_str(s: &str) -> Result<Self> {
132        match s {
133            "sequential" => Ok(Strategy::Sequential { randomized: false }),
134            "sequential-randomized" => Ok(Strategy::Sequential { randomized: true }),
135            "concurrent" => Ok(Strategy::Concurrent),
136            "parallel" => Ok(Strategy::Parallel { randomized: false }),
137            "parallel-randomized" => Ok(Strategy::Parallel { randomized: true }),
138            "development" => Ok(Strategy::Development { randomized: false }),
139            "development-randomized" => Ok(Strategy::Development { randomized: true }),
140            _ => bail!("invalid strategy"),
141        }
142    }
143}
144
145impl Strategy {
146    pub fn randomized(&self) -> bool {
147        match self {
148            Strategy::Sequential { randomized } => *randomized,
149            Strategy::Concurrent => false,
150            Strategy::Parallel { randomized } => *randomized,
151            Strategy::Development { randomized } => *randomized,
152        }
153    }
154}
155
156pub fn shuffle<'a, T: 'a>(items: impl Iterator<Item = T>) -> impl Iterator<Item = T> {
157    use rand::{SeedableRng, seq::SliceRandom};
158    let mut rng = rand::rngs::SmallRng::from_seed([0; 32]);
159    let mut input = items.collect::<Vec<_>>();
160    input.shuffle(&mut rng);
161    input.into_iter()
162}
163
164pub async fn render_routes(
165    tt: &TurboTasks<TurboTasksBackend>,
166    routes: impl Iterator<Item = (RcStr, Route)>,
167    strategy: Strategy,
168    factor: usize,
169    limit: usize,
170) -> Result<usize> {
171    tracing::info!(
172        "rendering routes with {} parallel and strategy {}",
173        factor,
174        strategy
175    );
176
177    let stream = tokio_stream::iter(routes)
178        .map(move |(name, route)| async move {
179            tracing::info!("{name}...");
180            let start = Instant::now();
181
182            let memory = TurboMalloc::memory_usage();
183
184            tt.run({
185                let name = name.clone();
186                async move {
187                    match route {
188                        Route::Page {
189                            html_endpoint,
190                            data_endpoint: _,
191                        } => {
192                            endpoint_write_to_disk_with_apply(html_endpoint).await?;
193                        }
194                        Route::PageApi { endpoint } => {
195                            endpoint_write_to_disk_with_apply(endpoint).await?;
196                        }
197                        Route::AppPage(routes) => {
198                            for route in routes {
199                                endpoint_write_to_disk_with_apply(route.html_endpoint).await?;
200                            }
201                        }
202                        Route::AppRoute {
203                            original_name: _,
204                            endpoint,
205                        } => {
206                            endpoint_write_to_disk_with_apply(endpoint).await?;
207                        }
208                        Route::Conflict => {
209                            tracing::info!("WARN: conflict {}", name);
210                        }
211                    }
212                    Ok(())
213                }
214            })
215            .await?;
216
217            let duration = start.elapsed();
218            let memory_after = TurboMalloc::memory_usage();
219            if matches!(strategy, Strategy::Sequential { .. }) {
220                if memory_after > memory {
221                    tracing::info!(
222                        "{name} {:?} {} MiB (memory usage increased by {} MiB)",
223                        duration,
224                        memory_after / 1024 / 1024,
225                        (memory_after - memory) / 1024 / 1024
226                    );
227                } else {
228                    tracing::info!(
229                        "{name} {:?} {} MiB (memory usage decreased by {} MiB)",
230                        duration,
231                        memory_after / 1024 / 1024,
232                        (memory - memory_after) / 1024 / 1024
233                    );
234                }
235            } else {
236                tracing::info!("{name} {:?} {} MiB", duration, memory_after / 1024 / 1024);
237            }
238
239            Ok::<_, anyhow::Error>(())
240        })
241        .take(limit)
242        .buffer_unordered(factor)
243        .try_collect::<Vec<_>>()
244        .await?;
245
246    Ok(stream.len())
247}
248
249async fn endpoint_write_to_disk_with_apply(
250    endpoint: ResolvedVc<Box<dyn Endpoint>>,
251) -> Result<ReadRef<EndpointOutputPaths>> {
252    #[turbo_tasks::function(operation, root)]
253    fn inner_operation(endpoint: ResolvedVc<Box<dyn Endpoint>>) -> Vc<EndpointOutputPaths> {
254        // we must wrap this in an operation so we can get the Effects collectibles
255        endpoint_write_to_disk(*endpoint)
256    }
257
258    #[turbo_tasks::value(serialization = "skip")]
259    struct WithEffects {
260        output_paths: ReadRef<EndpointOutputPaths>,
261        effects: Effects,
262    }
263
264    #[turbo_tasks::function(operation, root)]
265    pub async fn inner_operation_with_effects(
266        endpoint: ResolvedVc<Box<dyn Endpoint>>,
267    ) -> Result<Vc<WithEffects>> {
268        let op = inner_operation(endpoint);
269        let output_paths = op.read_strongly_consistent().await?;
270        let effects = take_effects(op).await?;
271        Ok(WithEffects {
272            output_paths,
273            effects,
274        }
275        .cell())
276    }
277
278    let op = inner_operation_with_effects(endpoint);
279    let read = read_strongly_consistent_and_apply_effects(op, |v| &v.effects).await?;
280
281    Ok(read.output_paths.clone())
282}
283
284async fn hmr(
285    tt: &TurboTasks<TurboTasksBackend>,
286    project: ResolvedVc<ProjectContainer>,
287) -> Result<()> {
288    tracing::info!("HMR...");
289    let session = TransientInstance::new(());
290
291    #[turbo_tasks::function(operation, root)]
292    fn project_hmr_chunk_names_operation(project: ResolvedVc<ProjectContainer>) -> Vc<Vec<RcStr>> {
293        project.hmr_chunk_names(HmrTarget::Client)
294    }
295
296    let idents = tt
297        .run(async move {
298            project_hmr_chunk_names_operation(project)
299                .read_strongly_consistent()
300                .await
301        })
302        .await?;
303
304    let start = Instant::now();
305    for ident in &idents {
306        if !ident.ends_with(".js") {
307            continue;
308        }
309        let session = session.clone();
310        let start = Instant::now();
311        let ident_for_task = ident.clone();
312        let task = tt.spawn_root_task(move || {
313            let session = session.clone();
314            let ident = ident_for_task.clone();
315            async move {
316                let project = project.project();
317                let state = project.hmr_version_state(ident.clone(), HmrTarget::Client, session);
318                project
319                    .hmr_update(ident.clone(), HmrTarget::Client, state)
320                    .await?;
321                Ok(Vc::<()>::cell(()))
322            }
323        });
324        tt.wait_task_completion(task, ReadConsistency::Strong)
325            .await?;
326        let e = start.elapsed();
327        if e.as_millis() > 10 {
328            tracing::info!("HMR: {:?} {:?}", ident, e);
329        }
330    }
331    tracing::info!("HMR {:?}", start.elapsed());
332
333    Ok(())
334}
next_build_test/lib.rs

next_build_test/
lib.rs
