next_build_test/
lib.rs

1#![feature(future_join)]
2#![feature(min_specialization)]
3#![feature(arbitrary_self_types)]
4#![feature(arbitrary_self_types_pointers)]
5
6use std::{str::FromStr, time::Instant};
7
8use anyhow::{Context, Result, bail};
9use futures_util::{StreamExt, TryStreamExt};
10use next_api::{
11    entrypoints::Entrypoints,
12    project::{HmrTarget, ProjectContainer, ProjectOptions},
13    route::{Endpoint, EndpointOutputPaths, Route, endpoint_write_to_disk},
14};
15use turbo_rcstr::{RcStr, rcstr};
16use turbo_tasks::{ReadConsistency, ResolvedVc, TransientInstance, TurboTasks, Vc, get_effects};
17use turbo_tasks_backend::{NoopBackingStorage, TurboTasksBackend};
18use turbo_tasks_malloc::TurboMalloc;
19
20pub async fn main_inner(
21    tt: &TurboTasks<TurboTasksBackend<NoopBackingStorage>>,
22    strategy: Strategy,
23    factor: usize,
24    limit: usize,
25    files: Option<Vec<String>>,
26) -> Result<()> {
27    let path = std::env::current_dir()?.join("project_options.json");
28    let mut file = std::fs::File::open(&path)
29        .with_context(|| format!("loading file at {}", path.display()))?;
30
31    let mut options: ProjectOptions = serde_json::from_reader(&mut file)?;
32
33    if matches!(strategy, Strategy::Development { .. }) {
34        options.dev = true;
35        options.watch.enable = true;
36    } else {
37        options.dev = false;
38        options.watch.enable = false;
39    }
40
41    let project = tt
42        .run(async {
43            let container_op = ProjectContainer::new_operation(rcstr!("next.js"), options.dev);
44            ProjectContainer::initialize(container_op, options).await?;
45            container_op.resolve_strongly_consistent().await
46        })
47        .await?;
48
49    tracing::info!("collecting endpoints");
50
51    #[turbo_tasks::function(operation)]
52    fn project_entrypoints_operation(project: ResolvedVc<ProjectContainer>) -> Vc<Entrypoints> {
53        project.entrypoints()
54    }
55    let entrypoints = tt
56        .run(async move {
57            project_entrypoints_operation(project)
58                .read_strongly_consistent()
59                .await
60        })
61        .await?;
62
63    let mut routes = if let Some(files) = files {
64        tracing::info!("building only the files:");
65        for file in &files {
66            tracing::info!("  {}", file);
67        }
68
69        // filter out the files that are not in the list
70        // we expect this to be small so linear search OK
71        Box::new(files.into_iter().filter_map(|f| {
72            entrypoints
73                .routes
74                .iter()
75                .find(|(name, _)| f.as_str() == name.as_str())
76                .map(|(name, route)| (name.clone(), route.clone()))
77        })) as Box<dyn Iterator<Item = _> + Send + Sync>
78    } else {
79        Box::new(entrypoints.routes.clone().into_iter())
80    };
81
82    if strategy.randomized() {
83        routes = Box::new(shuffle(routes))
84    }
85
86    let start = Instant::now();
87    let count = render_routes(tt, routes, strategy, factor, limit).await?;
88    tracing::info!("rendered {} pages in {:?}", count, start.elapsed());
89
90    if count == 0 {
91        tracing::info!("No pages found, these pages exist:");
92        for (route, _) in entrypoints.routes.iter() {
93            tracing::info!("  {}", route);
94        }
95    }
96
97    if matches!(strategy, Strategy::Development { .. }) {
98        hmr(tt, project).await?;
99    }
100
101    Ok(())
102}
103
104#[derive(PartialEq, Copy, Clone)]
105pub enum Strategy {
106    Sequential { randomized: bool },
107    Concurrent,
108    Parallel { randomized: bool },
109    Development { randomized: bool },
110}
111
112impl std::fmt::Display for Strategy {
113    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
114        match self {
115            Strategy::Sequential { randomized: false } => write!(f, "sequential"),
116            Strategy::Sequential { randomized: true } => write!(f, "sequential-randomized"),
117            Strategy::Concurrent => write!(f, "concurrent"),
118            Strategy::Parallel { randomized: false } => write!(f, "parallel"),
119            Strategy::Parallel { randomized: true } => write!(f, "parallel-randomized"),
120            Strategy::Development { randomized: false } => write!(f, "development"),
121            Strategy::Development { randomized: true } => write!(f, "development-randomized"),
122        }
123    }
124}
125
126impl FromStr for Strategy {
127    type Err = anyhow::Error;
128
129    fn from_str(s: &str) -> Result<Self> {
130        match s {
131            "sequential" => Ok(Strategy::Sequential { randomized: false }),
132            "sequential-randomized" => Ok(Strategy::Sequential { randomized: true }),
133            "concurrent" => Ok(Strategy::Concurrent),
134            "parallel" => Ok(Strategy::Parallel { randomized: false }),
135            "parallel-randomized" => Ok(Strategy::Parallel { randomized: true }),
136            "development" => Ok(Strategy::Development { randomized: false }),
137            "development-randomized" => Ok(Strategy::Development { randomized: true }),
138            _ => bail!("invalid strategy"),
139        }
140    }
141}
142
143impl Strategy {
144    pub fn randomized(&self) -> bool {
145        match self {
146            Strategy::Sequential { randomized } => *randomized,
147            Strategy::Concurrent => false,
148            Strategy::Parallel { randomized } => *randomized,
149            Strategy::Development { randomized } => *randomized,
150        }
151    }
152}
153
154pub fn shuffle<'a, T: 'a>(items: impl Iterator<Item = T>) -> impl Iterator<Item = T> {
155    use rand::{SeedableRng, seq::SliceRandom};
156    let mut rng = rand::rngs::SmallRng::from_seed([0; 32]);
157    let mut input = items.collect::<Vec<_>>();
158    input.shuffle(&mut rng);
159    input.into_iter()
160}
161
162pub async fn render_routes(
163    tt: &TurboTasks<TurboTasksBackend<NoopBackingStorage>>,
164    routes: impl Iterator<Item = (RcStr, Route)>,
165    strategy: Strategy,
166    factor: usize,
167    limit: usize,
168) -> Result<usize> {
169    tracing::info!(
170        "rendering routes with {} parallel and strategy {}",
171        factor,
172        strategy
173    );
174
175    let stream = tokio_stream::iter(routes)
176        .map(move |(name, route)| async move {
177            tracing::info!("{name}...");
178            let start = Instant::now();
179
180            let memory = TurboMalloc::memory_usage();
181
182            tt.run({
183                let name = name.clone();
184                async move {
185                    match route {
186                        Route::Page {
187                            html_endpoint,
188                            data_endpoint: _,
189                        } => {
190                            endpoint_write_to_disk_with_effects(*html_endpoint).await?;
191                        }
192                        Route::PageApi { endpoint } => {
193                            endpoint_write_to_disk_with_effects(*endpoint).await?;
194                        }
195                        Route::AppPage(routes) => {
196                            for route in routes {
197                                endpoint_write_to_disk_with_effects(*route.html_endpoint).await?;
198                            }
199                        }
200                        Route::AppRoute {
201                            original_name: _,
202                            endpoint,
203                        } => {
204                            endpoint_write_to_disk_with_effects(*endpoint).await?;
205                        }
206                        Route::Conflict => {
207                            tracing::info!("WARN: conflict {}", name);
208                        }
209                    }
210                    Ok(())
211                }
212            })
213            .await?;
214
215            let duration = start.elapsed();
216            let memory_after = TurboMalloc::memory_usage();
217            if matches!(strategy, Strategy::Sequential { .. }) {
218                if memory_after > memory {
219                    tracing::info!(
220                        "{name} {:?} {} MiB (memory usage increased by {} MiB)",
221                        duration,
222                        memory_after / 1024 / 1024,
223                        (memory_after - memory) / 1024 / 1024
224                    );
225                } else {
226                    tracing::info!(
227                        "{name} {:?} {} MiB (memory usage decreased by {} MiB)",
228                        duration,
229                        memory_after / 1024 / 1024,
230                        (memory - memory_after) / 1024 / 1024
231                    );
232                }
233            } else {
234                tracing::info!("{name} {:?} {} MiB", duration, memory_after / 1024 / 1024);
235            }
236
237            Ok::<_, anyhow::Error>(())
238        })
239        .take(limit)
240        .buffer_unordered(factor)
241        .try_collect::<Vec<_>>()
242        .await?;
243
244    Ok(stream.len())
245}
246
247#[turbo_tasks::function]
248async fn endpoint_write_to_disk_with_effects(
249    endpoint: ResolvedVc<Box<dyn Endpoint>>,
250) -> Result<Vc<EndpointOutputPaths>> {
251    let op = endpoint_write_to_disk_operation(endpoint);
252    let result = op.resolve_strongly_consistent().await?;
253    get_effects(op).await?.apply().await?;
254    Ok(*result)
255}
256
257#[turbo_tasks::function(operation)]
258pub fn endpoint_write_to_disk_operation(
259    endpoint: ResolvedVc<Box<dyn Endpoint>>,
260) -> Vc<EndpointOutputPaths> {
261    endpoint_write_to_disk(*endpoint)
262}
263
264async fn hmr(
265    tt: &TurboTasks<TurboTasksBackend<NoopBackingStorage>>,
266    project: ResolvedVc<ProjectContainer>,
267) -> Result<()> {
268    tracing::info!("HMR...");
269    let session = TransientInstance::new(());
270
271    #[turbo_tasks::function(operation)]
272    fn project_hmr_chunk_names_operation(project: ResolvedVc<ProjectContainer>) -> Vc<Vec<RcStr>> {
273        project.hmr_chunk_names(HmrTarget::Client)
274    }
275
276    let idents = tt
277        .run(async move {
278            project_hmr_chunk_names_operation(project)
279                .read_strongly_consistent()
280                .await
281        })
282        .await?;
283
284    let start = Instant::now();
285    for ident in idents {
286        if !ident.ends_with(".js") {
287            continue;
288        }
289        let session = session.clone();
290        let start = Instant::now();
291        let task = tt.spawn_root_task(move || {
292            let session = session.clone();
293            async move {
294                let project = project.project();
295                let state = project.hmr_version_state(ident.clone(), HmrTarget::Client, session);
296                project
297                    .hmr_update(ident.clone(), HmrTarget::Client, state)
298                    .await?;
299                Ok(Vc::<()>::cell(()))
300            }
301        });
302        tt.wait_task_completion(task, ReadConsistency::Strong)
303            .await?;
304        let e = start.elapsed();
305        if e.as_millis() > 10 {
306            tracing::info!("HMR: {:?} {:?}", ident, e);
307        }
308    }
309    tracing::info!("HMR {:?}", start.elapsed());
310
311    Ok(())
312}
next_build_test/lib.rs

next_build_test/
lib.rs
