turbo_tasks_backend/
lib.rs

1#![feature(anonymous_lifetime_in_impl_trait)]
2#![feature(deref_patterns)]
3
4mod backend;
5mod backing_storage;
6mod data;
7mod database;
8mod error;
9mod kv_backing_storage;
10mod utils;
11
12use std::path::Path;
13
14use anyhow::Result;
15use turbo_persistence::{CompactConfig, TurboPersistence};
16
17use crate::database::turbo::{self, TurboKeyValueDatabase};
18pub use crate::{
19    backend::{
20        BackendOptions, EvictionMode, GcPassResult, GcStats, StorageMode, TestSnapshotOutcome,
21        TtlCounter, TurboTasksBackend,
22    },
23    database::{
24        db_invalidation,
25        db_invalidation::StartupCacheState,
26        db_versioning::{GitVersionInfo, handle_db_versioning},
27    },
28    kv_backing_storage::TurboBackingStorage,
29};
30
31/// Options controlling how the on-disk persistent cache database is opened and compacted.
32#[derive(Clone, Copy, Debug, Default)]
33pub struct BackingStorageOptions {
34    /// Whether the process is running in a CI environment. Enables more aggressive (full)
35    /// compaction on shutdown to reduce the size of the cache that gets uploaded.
36    pub is_ci: bool,
37    /// Whether this is a short-lived session (e.g. a single build). Disables background
38    /// persistence during the session
39    pub is_short_session: bool,
40    /// Whether to skip database compaction on shutdown entirely
41    pub skip_compaction: bool,
42}
43
44/// Creates a `BackingStorage` to be passed to [`TurboTasksBackend::new`].
45///
46/// Information about the state of the on-disk cache is returned using [`StartupCacheState`].
47pub fn turbo_backing_storage(
48    base_path: &Path,
49    version_info: &GitVersionInfo,
50    options: BackingStorageOptions,
51) -> Result<(TurboBackingStorage, StartupCacheState)> {
52    TurboBackingStorage::open_versioned_on_disk(
53        base_path.to_owned(),
54        version_info,
55        options.is_ci,
56        |path| TurboKeyValueDatabase::new(path, options),
57    )
58}
59
60/// Creates an in-memory `BackingStorage` to be passed to [`TurboTasksBackend::new`]. Backed by
61/// an empty, read-only [`TurboPersistence`] — reads return `None`, writes are not expected
62/// (callers should set [`BackendOptions::storage_mode`] to `None`).
63pub fn noop_backing_storage() -> TurboBackingStorage {
64    TurboBackingStorage::new_in_memory(TurboKeyValueDatabase::empty_in_memory())
65}
66
67/// Opens a Turbopack persistent cache database at the given base path and performs a full
68/// compaction. This is intended for use by the `next internal post-build` CLI command to optimize
69/// the database after a build, without requiring the full turbo-tasks runtime.
70///
71/// The parallel scheduler requires a Tokio runtime. If one is already active (e.g. when called
72/// from a NAPI async function), it is reused. Otherwise a new multi-threaded runtime is created.
73pub fn compact_database(
74    base_path: &Path,
75    version_info: &GitVersionInfo,
76    is_ci: bool,
77) -> Result<()> {
78    let versioned_path = handle_db_versioning(base_path, version_info, is_ci)?;
79    // The parallel scheduler uses `tokio::task::block_in_place` internally, which
80    // requires a multi-threaded Tokio runtime. Create one only if there is no
81    // active runtime (e.g. when called from a standalone CLI context).
82    let _owned_runtime = if tokio::runtime::Handle::try_current().is_ok() {
83        None
84    } else {
85        Some(
86            tokio::runtime::Builder::new_multi_thread()
87                .enable_all()
88                .build()?,
89        )
90    };
91    // If we created a runtime, enter it so the scheduler can find it.
92    let _guard = _owned_runtime.as_ref().map(|rt| rt.enter());
93    let db =
94        TurboPersistence::<turbo::TurboTasksParallelScheduler, { turbo::FAMILIES }>::open_with_config(
95            versioned_path,
96            turbo::db_config(),
97        )?;
98    // Fully compact with no segment count limit (unlike the runtime shutdown path
99    // which caps segments based on available parallelism).
100    db.compact(&CompactConfig {
101        max_merge_segment_count: usize::MAX,
102        ..turbo::COMPACT_CONFIG
103    })?;
104    db.shutdown()
105}
turbo_tasks_backend/lib.rs

turbo_tasks_backend/
lib.rs
