next_build_test/
lib.rs

1#![feature(min_specialization)]
2#![feature(arbitrary_self_types)]
3#![feature(arbitrary_self_types_pointers)]
4
5use std::{path::Path, str::FromStr, time::Instant};
6
7use anyhow::{Context, Result, bail};
8use futures_util::{StreamExt, TryStreamExt};
9use next_api::{
10    entrypoints::Entrypoints,
11    project::{HmrTarget, ProjectContainer, ProjectOptions},
12    route::{Endpoint, EndpointOutputPaths, Route, endpoint_write_to_disk},
13};
14use turbo_rcstr::{RcStr, rcstr};
15use turbo_tasks::{
16    Effects, ReadConsistency, ReadRef, ResolvedVc, TransientInstance, TurboTasks, Vc,
17    read_strongly_consistent_and_apply_effects, take_effects,
18};
19use turbo_tasks_backend::TurboTasksBackend;
20use turbo_tasks_fs::canonicalize_to_rcstr;
21use turbo_tasks_malloc::TurboMalloc;
22
23pub async fn main_inner(
24    tt: &TurboTasks<TurboTasksBackend>,
25    strategy: Strategy,
26    factor: usize,
27    limit: usize,
28    files: Option<Vec<String>>,
29) -> Result<()> {
30    let path = std::env::current_dir()?.join("project_options.json");
31    let mut file = std::fs::File::open(&path)
32        .with_context(|| format!("loading file at {}", path.display()))?;
33
34    let mut options: ProjectOptions = serde_json::from_reader(&mut file)?;
35    options.root_path = canonicalize_to_rcstr(Path::new(&*options.root_path))?;
36
37    if matches!(strategy, Strategy::Development { .. }) {
38        options.dev = true;
39        options.watch.enable = true;
40    } else {
41        options.dev = false;
42        options.watch.enable = false;
43    }
44
45    let project = tt
46        .run(async {
47            let container_op = ProjectContainer::new_operation(rcstr!("next.js"), options.dev);
48            ProjectContainer::initialize(container_op, options).await?;
49            container_op.resolve().strongly_consistent().await
50        })
51        .await?;
52
53    tracing::info!("collecting endpoints");
54
55    #[turbo_tasks::function(operation, root)]
56    fn project_entrypoints_operation(project: ResolvedVc<ProjectContainer>) -> Vc<Entrypoints> {
57        project.entrypoints()
58    }
59    let entrypoints = tt
60        .run(async move {
61            project_entrypoints_operation(project)
62                .read_strongly_consistent()
63                .await
64        })
65        .await?;
66
67    let mut routes = if let Some(files) = files {
68        tracing::info!("building only the files:");
69        for file in &files {
70            tracing::info!("  {}", file);
71        }
72
73        // filter out the files that are not in the list
74        // we expect this to be small so linear search OK
75        Box::new(files.into_iter().filter_map(|f| {
76            entrypoints
77                .routes
78                .iter()
79                .find(|(name, _)| f.as_str() == name.as_str())
80                .map(|(name, route)| (name.clone(), route.clone()))
81        })) as Box<dyn Iterator<Item = _> + Send + Sync>
82    } else {
83        Box::new(entrypoints.routes.clone().into_iter())
84    };
85
86    if strategy.randomized() {
87        routes = Box::new(shuffle(routes))
88    }
89
90    let start = Instant::now();
91    let count = render_routes(tt, routes, strategy, factor, limit).await?;
92    tracing::info!("rendered {} pages in {:?}", count, start.elapsed());
93
94    if count == 0 {
95        tracing::info!("No pages found, these pages exist:");
96        for (route, _) in entrypoints.routes.iter() {
97            tracing::info!("  {}", route);
98        }
99    }
100
101    if matches!(strategy, Strategy::Development { .. }) {
102        hmr(tt, project).await?;
103    }
104
105    Ok(())
106}
107
108#[derive(PartialEq, Copy, Clone)]
109pub enum Strategy {
110    Sequential { randomized: bool },
111    Concurrent,
112    Parallel { randomized: bool },
113    Development { randomized: bool },
114}
115
116impl std::fmt::Display for Strategy {
117    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
118        match self {
119            Strategy::Sequential { randomized: false } => write!(f, "sequential"),
120            Strategy::Sequential { randomized: true } => write!(f, "sequential-randomized"),
121            Strategy::Concurrent => write!(f, "concurrent"),
122            Strategy::Parallel { randomized: false } => write!(f, "parallel"),
123            Strategy::Parallel { randomized: true } => write!(f, "parallel-randomized"),
124            Strategy::Development { randomized: false } => write!(f, "development"),
125            Strategy::Development { randomized: true } => write!(f, "development-randomized"),
126        }
127    }
128}
129
130impl FromStr for Strategy {
131    type Err = anyhow::Error;
132
133    fn from_str(s: &str) -> Result<Self> {
134        match s {
135            "sequential" => Ok(Strategy::Sequential { randomized: false }),
136            "sequential-randomized" => Ok(Strategy::Sequential { randomized: true }),
137            "concurrent" => Ok(Strategy::Concurrent),
138            "parallel" => Ok(Strategy::Parallel { randomized: false }),
139            "parallel-randomized" => Ok(Strategy::Parallel { randomized: true }),
140            "development" => Ok(Strategy::Development { randomized: false }),
141            "development-randomized" => Ok(Strategy::Development { randomized: true }),
142            _ => bail!("invalid strategy"),
143        }
144    }
145}
146
147impl Strategy {
148    pub fn randomized(&self) -> bool {
149        match self {
150            Strategy::Sequential { randomized } => *randomized,
151            Strategy::Concurrent => false,
152            Strategy::Parallel { randomized } => *randomized,
153            Strategy::Development { randomized } => *randomized,
154        }
155    }
156}
157
158pub fn shuffle<'a, T: 'a>(items: impl Iterator<Item = T>) -> impl Iterator<Item = T> {
159    use rand::{SeedableRng, seq::SliceRandom};
160    let mut rng = rand::rngs::SmallRng::from_seed([0; 32]);
161    let mut input = items.collect::<Vec<_>>();
162    input.shuffle(&mut rng);
163    input.into_iter()
164}
165
166pub async fn render_routes(
167    tt: &TurboTasks<TurboTasksBackend>,
168    routes: impl Iterator<Item = (RcStr, Route)>,
169    strategy: Strategy,
170    factor: usize,
171    limit: usize,
172) -> Result<usize> {
173    tracing::info!(
174        "rendering routes with {} parallel and strategy {}",
175        factor,
176        strategy
177    );
178
179    let stream = tokio_stream::iter(routes)
180        .map(move |(name, route)| async move {
181            tracing::info!("{name}...");
182            let start = Instant::now();
183
184            let memory = TurboMalloc::memory_usage();
185
186            tt.run({
187                let name = name.clone();
188                async move {
189                    match route {
190                        Route::Page {
191                            html_endpoint,
192                            data_endpoint: _,
193                        } => {
194                            endpoint_write_to_disk_with_apply(html_endpoint).await?;
195                        }
196                        Route::PageApi { endpoint } => {
197                            endpoint_write_to_disk_with_apply(endpoint).await?;
198                        }
199                        Route::AppPage(routes) => {
200                            for route in routes {
201                                endpoint_write_to_disk_with_apply(route.html_endpoint).await?;
202                            }
203                        }
204                        Route::AppRoute {
205                            original_name: _,
206                            endpoint,
207                        } => {
208                            endpoint_write_to_disk_with_apply(endpoint).await?;
209                        }
210                        Route::Conflict => {
211                            tracing::info!("WARN: conflict {}", name);
212                        }
213                    }
214                    Ok(())
215                }
216            })
217            .await?;
218
219            let duration = start.elapsed();
220            let memory_after = TurboMalloc::memory_usage();
221            if matches!(strategy, Strategy::Sequential { .. }) {
222                if memory_after > memory {
223                    tracing::info!(
224                        "{name} {:?} {} MiB (memory usage increased by {} MiB)",
225                        duration,
226                        memory_after / 1024 / 1024,
227                        (memory_after - memory) / 1024 / 1024
228                    );
229                } else {
230                    tracing::info!(
231                        "{name} {:?} {} MiB (memory usage decreased by {} MiB)",
232                        duration,
233                        memory_after / 1024 / 1024,
234                        (memory - memory_after) / 1024 / 1024
235                    );
236                }
237            } else {
238                tracing::info!("{name} {:?} {} MiB", duration, memory_after / 1024 / 1024);
239            }
240
241            Ok::<_, anyhow::Error>(())
242        })
243        .take(limit)
244        .buffer_unordered(factor)
245        .try_collect::<Vec<_>>()
246        .await?;
247
248    Ok(stream.len())
249}
250
251async fn endpoint_write_to_disk_with_apply(
252    endpoint: ResolvedVc<Box<dyn Endpoint>>,
253) -> Result<ReadRef<EndpointOutputPaths>> {
254    #[turbo_tasks::function(operation, root)]
255    fn inner_operation(endpoint: ResolvedVc<Box<dyn Endpoint>>) -> Vc<EndpointOutputPaths> {
256        // we must wrap this in an operation so we can get the Effects collectibles
257        endpoint_write_to_disk(*endpoint)
258    }
259
260    #[turbo_tasks::value(serialization = "skip")]
261    struct WithEffects {
262        output_paths: ReadRef<EndpointOutputPaths>,
263        effects: Effects,
264    }
265
266    #[turbo_tasks::function(operation, root)]
267    pub async fn inner_operation_with_effects(
268        endpoint: ResolvedVc<Box<dyn Endpoint>>,
269    ) -> Result<Vc<WithEffects>> {
270        let op = inner_operation(endpoint);
271        let output_paths = op.read_strongly_consistent().await?;
272        let effects = take_effects(op).await?;
273        Ok(WithEffects {
274            output_paths,
275            effects,
276        }
277        .cell())
278    }
279
280    let op = inner_operation_with_effects(endpoint);
281    let read = read_strongly_consistent_and_apply_effects(op, |v| &v.effects).await?;
282
283    Ok(read.output_paths.clone())
284}
285
286async fn hmr(
287    tt: &TurboTasks<TurboTasksBackend>,
288    project: ResolvedVc<ProjectContainer>,
289) -> Result<()> {
290    tracing::info!("HMR...");
291    let session = TransientInstance::new(());
292
293    #[turbo_tasks::function(operation, root)]
294    fn project_hmr_chunk_names_operation(project: ResolvedVc<ProjectContainer>) -> Vc<Vec<RcStr>> {
295        project.hmr_chunk_names(HmrTarget::Client)
296    }
297
298    let idents = tt
299        .run(async move {
300            project_hmr_chunk_names_operation(project)
301                .read_strongly_consistent()
302                .await
303        })
304        .await?;
305
306    let start = Instant::now();
307    for ident in &idents {
308        if !ident.ends_with(".js") {
309            continue;
310        }
311        let session = session.clone();
312        let start = Instant::now();
313        let ident_for_task = ident.clone();
314        let task = tt.spawn_root_task(move || {
315            let session = session.clone();
316            let ident = ident_for_task.clone();
317            async move {
318                let project = project.project();
319                let state = project.hmr_version_state(ident.clone(), HmrTarget::Client, session);
320                project
321                    .hmr_update(ident.clone(), HmrTarget::Client, state)
322                    .await?;
323                Ok(Vc::<()>::cell(()))
324            }
325        });
326        tt.wait_task_completion(task, ReadConsistency::Strong)
327            .await?;
328        let e = start.elapsed();
329        if e.as_millis() > 10 {
330            tracing::info!("HMR: {:?} {:?}", ident, e);
331        }
332    }
333    tracing::info!("HMR {:?}", start.elapsed());
334
335    Ok(())
336}
next_build_test/lib.rs

next_build_test/
lib.rs
