next_build_test/
lib.rs

1#![feature(future_join)]
2#![feature(min_specialization)]
3#![feature(arbitrary_self_types)]
4#![feature(arbitrary_self_types_pointers)]
5
6use std::{str::FromStr, time::Instant};
7
8use anyhow::{Context, Result};
9use futures_util::{StreamExt, TryStreamExt};
10use next_api::{
11    project::{ProjectContainer, ProjectOptions},
12    route::{Endpoint, EndpointOutputPaths, Route, endpoint_write_to_disk},
13};
14use turbo_rcstr::{RcStr, rcstr};
15use turbo_tasks::{ReadConsistency, ResolvedVc, TransientInstance, TurboTasks, Vc, get_effects};
16use turbo_tasks_backend::{NoopBackingStorage, TurboTasksBackend};
17use turbo_tasks_malloc::TurboMalloc;
18
19pub async fn main_inner(
20    tt: &TurboTasks<TurboTasksBackend<NoopBackingStorage>>,
21    strategy: Strategy,
22    factor: usize,
23    limit: usize,
24    files: Option<Vec<String>>,
25) -> Result<()> {
26    let path = std::env::current_dir()?.join("project_options.json");
27    let mut file = std::fs::File::open(&path)
28        .with_context(|| format!("loading file at {}", path.display()))?;
29
30    let mut options: ProjectOptions = serde_json::from_reader(&mut file)?;
31
32    if matches!(strategy, Strategy::Development { .. }) {
33        options.dev = true;
34        options.watch.enable = true;
35    } else {
36        options.dev = false;
37        options.watch.enable = false;
38    }
39
40    let project = tt
41        .run(async {
42            let project = ProjectContainer::new(rcstr!("next-build-test"), options.dev);
43            let project = project.to_resolved().await?;
44            project.initialize(options).await?;
45            Ok(project)
46        })
47        .await?;
48
49    tracing::info!("collecting endpoints");
50    let entrypoints = tt.run(async move { project.entrypoints().await }).await?;
51
52    let mut routes = if let Some(files) = files {
53        tracing::info!("building only the files:");
54        for file in &files {
55            tracing::info!("  {}", file);
56        }
57
58        // filter out the files that are not in the list
59        // we expect this to be small so linear search OK
60        Box::new(files.into_iter().filter_map(|f| {
61            entrypoints
62                .routes
63                .iter()
64                .find(|(name, _)| f.as_str() == name.as_str())
65                .map(|(name, route)| (name.clone(), route.clone()))
66        })) as Box<dyn Iterator<Item = _> + Send + Sync>
67    } else {
68        Box::new(entrypoints.routes.clone().into_iter())
69    };
70
71    if strategy.randomized() {
72        routes = Box::new(shuffle(routes))
73    }
74
75    let start = Instant::now();
76    let count = render_routes(tt, routes, strategy, factor, limit).await?;
77    tracing::info!("rendered {} pages in {:?}", count, start.elapsed());
78
79    if count == 0 {
80        tracing::info!("No pages found, these pages exist:");
81        for (route, _) in entrypoints.routes.iter() {
82            tracing::info!("  {}", route);
83        }
84    }
85
86    if matches!(strategy, Strategy::Development { .. }) {
87        hmr(tt, *project).await?;
88    }
89
90    Ok(())
91}
92
93#[derive(PartialEq, Copy, Clone)]
94pub enum Strategy {
95    Sequential { randomized: bool },
96    Concurrent,
97    Parallel { randomized: bool },
98    Development { randomized: bool },
99}
100
101impl std::fmt::Display for Strategy {
102    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
103        match self {
104            Strategy::Sequential { randomized: false } => write!(f, "sequential"),
105            Strategy::Sequential { randomized: true } => write!(f, "sequential-randomized"),
106            Strategy::Concurrent => write!(f, "concurrent"),
107            Strategy::Parallel { randomized: false } => write!(f, "parallel"),
108            Strategy::Parallel { randomized: true } => write!(f, "parallel-randomized"),
109            Strategy::Development { randomized: false } => write!(f, "development"),
110            Strategy::Development { randomized: true } => write!(f, "development-randomized"),
111        }
112    }
113}
114
115impl FromStr for Strategy {
116    type Err = anyhow::Error;
117
118    fn from_str(s: &str) -> Result<Self> {
119        match s {
120            "sequential" => Ok(Strategy::Sequential { randomized: false }),
121            "sequential-randomized" => Ok(Strategy::Sequential { randomized: true }),
122            "concurrent" => Ok(Strategy::Concurrent),
123            "parallel" => Ok(Strategy::Parallel { randomized: false }),
124            "parallel-randomized" => Ok(Strategy::Parallel { randomized: true }),
125            "development" => Ok(Strategy::Development { randomized: false }),
126            "development-randomized" => Ok(Strategy::Development { randomized: true }),
127            _ => Err(anyhow::anyhow!("invalid strategy")),
128        }
129    }
130}
131
132impl Strategy {
133    pub fn randomized(&self) -> bool {
134        match self {
135            Strategy::Sequential { randomized } => *randomized,
136            Strategy::Concurrent => false,
137            Strategy::Parallel { randomized } => *randomized,
138            Strategy::Development { randomized } => *randomized,
139        }
140    }
141}
142
143pub fn shuffle<'a, T: 'a>(items: impl Iterator<Item = T>) -> impl Iterator<Item = T> {
144    use rand::{SeedableRng, seq::SliceRandom};
145    let mut rng = rand::rngs::SmallRng::from_seed([0; 32]);
146    let mut input = items.collect::<Vec<_>>();
147    input.shuffle(&mut rng);
148    input.into_iter()
149}
150
151pub async fn render_routes(
152    tt: &TurboTasks<TurboTasksBackend<NoopBackingStorage>>,
153    routes: impl Iterator<Item = (RcStr, Route)>,
154    strategy: Strategy,
155    factor: usize,
156    limit: usize,
157) -> Result<usize> {
158    tracing::info!(
159        "rendering routes with {} parallel and strategy {}",
160        factor,
161        strategy
162    );
163
164    let stream = tokio_stream::iter(routes)
165        .map(move |(name, route)| async move {
166            tracing::info!("{name}...");
167            let start = Instant::now();
168
169            let memory = TurboMalloc::memory_usage();
170
171            tt.run({
172                let name = name.clone();
173                async move {
174                    match route {
175                        Route::Page {
176                            html_endpoint,
177                            data_endpoint: _,
178                        } => {
179                            endpoint_write_to_disk_with_effects(*html_endpoint).await?;
180                        }
181                        Route::PageApi { endpoint } => {
182                            endpoint_write_to_disk_with_effects(*endpoint).await?;
183                        }
184                        Route::AppPage(routes) => {
185                            for route in routes {
186                                endpoint_write_to_disk_with_effects(*route.html_endpoint).await?;
187                            }
188                        }
189                        Route::AppRoute {
190                            original_name: _,
191                            endpoint,
192                        } => {
193                            endpoint_write_to_disk_with_effects(*endpoint).await?;
194                        }
195                        Route::Conflict => {
196                            tracing::info!("WARN: conflict {}", name);
197                        }
198                    }
199                    Ok(())
200                }
201            })
202            .await?;
203
204            let duration = start.elapsed();
205            let memory_after = TurboMalloc::memory_usage();
206            if matches!(strategy, Strategy::Sequential { .. }) {
207                if memory_after > memory {
208                    tracing::info!(
209                        "{name} {:?} {} MiB (memory usage increased by {} MiB)",
210                        duration,
211                        memory_after / 1024 / 1024,
212                        (memory_after - memory) / 1024 / 1024
213                    );
214                } else {
215                    tracing::info!(
216                        "{name} {:?} {} MiB (memory usage decreased by {} MiB)",
217                        duration,
218                        memory_after / 1024 / 1024,
219                        (memory - memory_after) / 1024 / 1024
220                    );
221                }
222            } else {
223                tracing::info!("{name} {:?} {} MiB", duration, memory_after / 1024 / 1024);
224            }
225
226            Ok::<_, anyhow::Error>(())
227        })
228        .take(limit)
229        .buffer_unordered(factor)
230        .try_collect::<Vec<_>>()
231        .await?;
232
233    Ok(stream.len())
234}
235
236#[turbo_tasks::function]
237async fn endpoint_write_to_disk_with_effects(
238    endpoint: ResolvedVc<Box<dyn Endpoint>>,
239) -> Result<Vc<EndpointOutputPaths>> {
240    let op = endpoint_write_to_disk_operation(endpoint);
241    let result = op.resolve_strongly_consistent().await?;
242    get_effects(op).await?.apply().await?;
243    Ok(*result)
244}
245
246#[turbo_tasks::function(operation)]
247pub fn endpoint_write_to_disk_operation(
248    endpoint: ResolvedVc<Box<dyn Endpoint>>,
249) -> Vc<EndpointOutputPaths> {
250    endpoint_write_to_disk(*endpoint)
251}
252
253async fn hmr(
254    tt: &TurboTasks<TurboTasksBackend<NoopBackingStorage>>,
255    project: Vc<ProjectContainer>,
256) -> Result<()> {
257    tracing::info!("HMR...");
258    let session = TransientInstance::new(());
259    let idents = tt
260        .run(async move { project.hmr_identifiers().await })
261        .await?;
262    let start = Instant::now();
263    for ident in idents {
264        if !ident.ends_with(".js") {
265            continue;
266        }
267        let session = session.clone();
268        let start = Instant::now();
269        let task = tt.spawn_root_task(move || {
270            let session = session.clone();
271            async move {
272                let project = project.project();
273                let state = project.hmr_version_state(ident.clone(), session);
274                project.hmr_update(ident.clone(), state).await?;
275                Ok(Vc::<()>::cell(()))
276            }
277        });
278        tt.wait_task_completion(task, ReadConsistency::Strong)
279            .await?;
280        let e = start.elapsed();
281        if e.as_millis() > 10 {
282            tracing::info!("HMR: {:?} {:?}", ident, e);
283        }
284    }
285    tracing::info!("HMR {:?}", start.elapsed());
286
287    Ok(())
288}
next_build_test/lib.rs

next_build_test/
lib.rs
