next_build_test/
lib.rs

1#![feature(min_specialization)]
2#![feature(arbitrary_self_types)]
3#![feature(arbitrary_self_types_pointers)]
4
5use std::{path::Path, str::FromStr, time::Instant};
6
7use anyhow::{Context, Result, bail};
8use futures_util::{StreamExt, TryStreamExt};
9use next_api::{
10    entrypoints::Entrypoints,
11    project::{ProjectContainer, ProjectOptions},
12    route::{Endpoint, EndpointOutputPaths, Route, endpoint_write_to_disk},
13};
14use turbo_rcstr::{RcStr, rcstr};
15use turbo_tasks::{
16    Effects, ReadConsistency, ReadRef, ResolvedVc, TransientInstance, TurboTasks, Vc,
17    read_strongly_consistent_and_apply_effects, take_effects,
18};
19use turbo_tasks_backend::TurboTasksBackend;
20use turbo_tasks_fs::canonicalize_to_rcstr;
21use turbo_tasks_malloc::TurboMalloc;
22
23pub async fn main_inner(
24    tt: &TurboTasks<TurboTasksBackend>,
25    strategy: Strategy,
26    factor: usize,
27    limit: usize,
28    files: Option<Vec<String>>,
29) -> Result<()> {
30    let path = std::env::current_dir()?.join("project_options.json");
31    let mut file = std::fs::File::open(&path)
32        .with_context(|| format!("loading file at {}", path.display()))?;
33
34    let mut options: ProjectOptions = serde_json::from_reader(&mut file)?;
35    options.root_path = canonicalize_to_rcstr(Path::new(&*options.root_path))?;
36
37    if matches!(strategy, Strategy::Development { .. }) {
38        options.dev = true;
39        options.watch.enable = true;
40    } else {
41        options.dev = false;
42        options.watch.enable = false;
43    }
44
45    let project = tt
46        .run(async {
47            let container_op = ProjectContainer::new_operation(rcstr!("next.js"), options.dev);
48            ProjectContainer::initialize(container_op, options).await?;
49            container_op.resolve().strongly_consistent().await
50        })
51        .await?;
52
53    tracing::info!("collecting endpoints");
54
55    #[turbo_tasks::function(operation, root)]
56    fn project_entrypoints_operation(project: ResolvedVc<ProjectContainer>) -> Vc<Entrypoints> {
57        project.entrypoints()
58    }
59    let entrypoints = tt
60        .run(async move {
61            project_entrypoints_operation(project)
62                .read_strongly_consistent()
63                .await
64        })
65        .await?;
66
67    let mut routes = if let Some(files) = files {
68        tracing::info!("building only the files:");
69        for file in &files {
70            tracing::info!("  {}", file);
71        }
72
73        // filter out the files that are not in the list
74        // we expect this to be small so linear search OK
75        Box::new(files.into_iter().filter_map(|f| {
76            entrypoints
77                .routes
78                .iter()
79                .find(|(name, _)| f.as_str() == name.as_str())
80                .map(|(name, route)| (name.clone(), route.clone()))
81        })) as Box<dyn Iterator<Item = _> + Send + Sync>
82    } else {
83        Box::new(entrypoints.routes.clone().into_iter())
84    };
85
86    if strategy.randomized() {
87        routes = Box::new(shuffle(routes))
88    }
89
90    let start = Instant::now();
91    let count = render_routes(tt, routes, strategy, factor, limit).await?;
92    tracing::info!("rendered {} pages in {:?}", count, start.elapsed());
93
94    if count == 0 {
95        tracing::info!("No pages found, these pages exist:");
96        for (route, _) in entrypoints.routes.iter() {
97            tracing::info!("  {}", route);
98        }
99    }
100
101    if matches!(strategy, Strategy::Development { .. }) {
102        hmr(tt, project).await?;
103    }
104
105    Ok(())
106}
107
108#[derive(PartialEq, Copy, Clone)]
109pub enum Strategy {
110    Sequential { randomized: bool },
111    Concurrent,
112    Parallel { randomized: bool },
113    Development { randomized: bool },
114}
115
116impl std::fmt::Display for Strategy {
117    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
118        match self {
119            Strategy::Sequential { randomized: false } => write!(f, "sequential"),
120            Strategy::Sequential { randomized: true } => write!(f, "sequential-randomized"),
121            Strategy::Concurrent => write!(f, "concurrent"),
122            Strategy::Parallel { randomized: false } => write!(f, "parallel"),
123            Strategy::Parallel { randomized: true } => write!(f, "parallel-randomized"),
124            Strategy::Development { randomized: false } => write!(f, "development"),
125            Strategy::Development { randomized: true } => write!(f, "development-randomized"),
126        }
127    }
128}
129
130impl FromStr for Strategy {
131    type Err = anyhow::Error;
132
133    fn from_str(s: &str) -> Result<Self> {
134        match s {
135            "sequential" => Ok(Strategy::Sequential { randomized: false }),
136            "sequential-randomized" => Ok(Strategy::Sequential { randomized: true }),
137            "concurrent" => Ok(Strategy::Concurrent),
138            "parallel" => Ok(Strategy::Parallel { randomized: false }),
139            "parallel-randomized" => Ok(Strategy::Parallel { randomized: true }),
140            "development" => Ok(Strategy::Development { randomized: false }),
141            "development-randomized" => Ok(Strategy::Development { randomized: true }),
142            _ => bail!("invalid strategy"),
143        }
144    }
145}
146
147impl Strategy {
148    pub fn randomized(&self) -> bool {
149        match self {
150            Strategy::Sequential { randomized } => *randomized,
151            Strategy::Concurrent => false,
152            Strategy::Parallel { randomized } => *randomized,
153            Strategy::Development { randomized } => *randomized,
154        }
155    }
156}
157
158pub fn shuffle<'a, T: 'a>(items: impl Iterator<Item = T>) -> impl Iterator<Item = T> {
159    use rand::{SeedableRng, seq::SliceRandom};
160    let mut rng = rand::rngs::SmallRng::from_seed([0; 32]);
161    let mut input = items.collect::<Vec<_>>();
162    input.shuffle(&mut rng);
163    input.into_iter()
164}
165
166pub async fn render_routes(
167    tt: &TurboTasks<TurboTasksBackend>,
168    routes: impl Iterator<Item = (RcStr, Route)>,
169    strategy: Strategy,
170    factor: usize,
171    limit: usize,
172) -> Result<usize> {
173    tracing::info!(
174        "rendering routes with {} parallel and strategy {}",
175        factor,
176        strategy
177    );
178
179    let stream = tokio_stream::iter(routes)
180        .map(move |(name, route)| async move {
181            tracing::info!("{name}...");
182            let start = Instant::now();
183
184            let memory = TurboMalloc::memory_usage();
185
186            tt.run({
187                let name = name.clone();
188                async move {
189                    match route {
190                        Route::Page {
191                            html_endpoint,
192                            data_endpoint: _,
193                        } => {
194                            endpoint_write_to_disk_with_apply(html_endpoint).await?;
195                        }
196                        Route::PageApi { endpoint } => {
197                            endpoint_write_to_disk_with_apply(endpoint).await?;
198                        }
199                        Route::AppPage(routes) => {
200                            for route in routes {
201                                endpoint_write_to_disk_with_apply(route.html_endpoint).await?;
202                            }
203                        }
204                        Route::AppRoute {
205                            original_name: _,
206                            endpoint,
207                            ..
208                        } => {
209                            endpoint_write_to_disk_with_apply(endpoint).await?;
210                        }
211                        Route::Conflict => {
212                            tracing::info!("WARN: conflict {}", name);
213                        }
214                    }
215                    Ok(())
216                }
217            })
218            .await?;
219
220            let duration = start.elapsed();
221            let memory_after = TurboMalloc::memory_usage();
222            if matches!(strategy, Strategy::Sequential { .. }) {
223                if memory_after > memory {
224                    tracing::info!(
225                        "{name} {:?} {} MiB (memory usage increased by {} MiB)",
226                        duration,
227                        memory_after / 1024 / 1024,
228                        (memory_after - memory) / 1024 / 1024
229                    );
230                } else {
231                    tracing::info!(
232                        "{name} {:?} {} MiB (memory usage decreased by {} MiB)",
233                        duration,
234                        memory_after / 1024 / 1024,
235                        (memory - memory_after) / 1024 / 1024
236                    );
237                }
238            } else {
239                tracing::info!("{name} {:?} {} MiB", duration, memory_after / 1024 / 1024);
240            }
241
242            Ok::<_, anyhow::Error>(())
243        })
244        .take(limit)
245        .buffer_unordered(factor)
246        .try_collect::<Vec<_>>()
247        .await?;
248
249    Ok(stream.len())
250}
251
252async fn endpoint_write_to_disk_with_apply(
253    endpoint: ResolvedVc<Box<dyn Endpoint>>,
254) -> Result<ReadRef<EndpointOutputPaths>> {
255    #[turbo_tasks::function(operation, root)]
256    fn inner_operation(endpoint: ResolvedVc<Box<dyn Endpoint>>) -> Vc<EndpointOutputPaths> {
257        // we must wrap this in an operation so we can get the Effects collectibles
258        endpoint_write_to_disk(*endpoint)
259    }
260
261    #[turbo_tasks::value(serialization = "skip")]
262    struct WithEffects {
263        output_paths: ReadRef<EndpointOutputPaths>,
264        effects: Effects,
265    }
266
267    #[turbo_tasks::function(operation, root)]
268    pub async fn inner_operation_with_effects(
269        endpoint: ResolvedVc<Box<dyn Endpoint>>,
270    ) -> Result<Vc<WithEffects>> {
271        let op = inner_operation(endpoint);
272        let output_paths = op.read_strongly_consistent().await?;
273        let effects = take_effects(op).await?;
274        Ok(WithEffects {
275            output_paths,
276            effects,
277        }
278        .cell())
279    }
280
281    let op = inner_operation_with_effects(endpoint);
282    let read = read_strongly_consistent_and_apply_effects(op, |v| &v.effects).await?;
283
284    Ok(read.output_paths.clone())
285}
286
287async fn hmr(
288    tt: &TurboTasks<TurboTasksBackend>,
289    project: ResolvedVc<ProjectContainer>,
290) -> Result<()> {
291    tracing::info!("HMR...");
292    let session = TransientInstance::new(());
293
294    #[turbo_tasks::function(operation, root)]
295    fn project_hmr_chunk_names_operation(project: ResolvedVc<ProjectContainer>) -> Vc<Vec<RcStr>> {
296        project.hmr_chunk_names()
297    }
298
299    let idents = tt
300        .run(async move {
301            project_hmr_chunk_names_operation(project)
302                .read_strongly_consistent()
303                .await
304        })
305        .await?;
306
307    let start = Instant::now();
308    for ident in &idents {
309        if !ident.ends_with(".js") {
310            continue;
311        }
312        let session = session.clone();
313        let start = Instant::now();
314        let ident_for_task = ident.clone();
315        let task = tt.spawn_root_task(move || {
316            let session = session.clone();
317            let ident = ident_for_task.clone();
318            async move {
319                let project = project.project();
320                let state = project.hmr_version_state(ident.clone(), session);
321                project.hmr_update(ident.clone(), state).await?;
322                Ok(Vc::<()>::cell(()))
323            }
324        });
325        tt.wait_task_completion(task, ReadConsistency::Strong)
326            .await?;
327        let e = start.elapsed();
328        if e.as_millis() > 10 {
329            tracing::info!("HMR: {:?} {:?}", ident, e);
330        }
331    }
332    tracing::info!("HMR {:?}", start.elapsed());
333
334    Ok(())
335}
next_build_test/lib.rs

next_build_test/
lib.rs
