turbo_tasks_backend/
lib.rs

1#![feature(current_thread_id)]
2#![feature(deref_patterns)]
3#![feature(thread_id_value)]
4
5mod backend;
6mod backing_storage;
7mod data;
8mod database;
9mod error;
10mod kv_backing_storage;
11mod utils;
12
13use std::path::Path;
14
15use anyhow::Result;
16use turbo_persistence::{CompactConfig, TurboPersistence};
17
18use crate::database::turbo::{self, TurboKeyValueDatabase};
19pub use crate::{
20    backend::{
21        BackendOptions, EvictionMode, GcPassResult, GcStats, StorageMode, TestSnapshotOutcome,
22        TtlCounter, TurboTasksBackend,
23    },
24    database::{
25        db_invalidation,
26        db_invalidation::StartupCacheState,
27        db_versioning::{GitVersionInfo, handle_db_versioning},
28    },
29    kv_backing_storage::TurboBackingStorage,
30};
31
32/// Options controlling how the on-disk persistent cache database is opened and compacted.
33#[derive(Clone, Copy, Debug, Default)]
34pub struct BackingStorageOptions {
35    /// Whether the process is running in a CI environment. Enables more aggressive (full)
36    /// compaction on shutdown to reduce the size of the cache that gets uploaded.
37    pub is_ci: bool,
38    /// Whether this is a short-lived session (e.g. a single build). Disables background
39    /// persistence during the session
40    pub is_short_session: bool,
41    /// Whether to skip database compaction on shutdown entirely
42    pub skip_compaction: bool,
43}
44
45/// Creates a `BackingStorage` to be passed to [`TurboTasksBackend::new`].
46///
47/// Information about the state of the on-disk cache is returned using [`StartupCacheState`].
48pub fn turbo_backing_storage(
49    base_path: &Path,
50    version_info: &GitVersionInfo,
51    options: BackingStorageOptions,
52) -> Result<(TurboBackingStorage, StartupCacheState)> {
53    TurboBackingStorage::open_versioned_on_disk(
54        base_path.to_owned(),
55        version_info,
56        options.is_ci,
57        |path| TurboKeyValueDatabase::new(path, options),
58    )
59}
60
61/// Creates an in-memory `BackingStorage` to be passed to [`TurboTasksBackend::new`]. Backed by
62/// an empty, read-only [`TurboPersistence`] — reads return `None`, writes are not expected
63/// (callers should set [`BackendOptions::storage_mode`] to `None`).
64pub fn noop_backing_storage() -> TurboBackingStorage {
65    TurboBackingStorage::new_in_memory(TurboKeyValueDatabase::empty_in_memory())
66}
67
68/// Opens a Turbopack persistent cache database at the given base path and performs a full
69/// compaction. This is intended for use by the `next internal post-build` CLI command to optimize
70/// the database after a build, without requiring the full turbo-tasks runtime.
71///
72/// The parallel scheduler requires a Tokio runtime. If one is already active (e.g. when called
73/// from a NAPI async function), it is reused. Otherwise a new multi-threaded runtime is created.
74pub fn compact_database(
75    base_path: &Path,
76    version_info: &GitVersionInfo,
77    is_ci: bool,
78) -> Result<()> {
79    let versioned_path = handle_db_versioning(base_path, version_info, is_ci)?;
80    // The parallel scheduler uses `tokio::task::block_in_place` internally, which
81    // requires a multi-threaded Tokio runtime. Create one only if there is no
82    // active runtime (e.g. when called from a standalone CLI context).
83    let _owned_runtime = if tokio::runtime::Handle::try_current().is_ok() {
84        None
85    } else {
86        Some(
87            tokio::runtime::Builder::new_multi_thread()
88                .enable_all()
89                .build()?,
90        )
91    };
92    // If we created a runtime, enter it so the scheduler can find it.
93    let _guard = _owned_runtime.as_ref().map(|rt| rt.enter());
94    let db =
95        TurboPersistence::<turbo::TurboTasksParallelScheduler, { turbo::FAMILIES }>::open_with_config(
96            versioned_path,
97            turbo::db_config(),
98        )?;
99    // Fully compact with no segment count limit (unlike the runtime shutdown path
100    // which caps segments based on available parallelism).
101    db.compact(&CompactConfig {
102        max_merge_jobs: usize::MAX,
103        ..turbo::COMPACT_CONFIG
104    })?;
105    db.shutdown();
106    Ok(())
107}
108
109/// Internals exposed for benchmarks only. Not a stable API.
110#[doc(hidden)]
111pub mod private {
112    use anyhow::Result;
113    pub use turbo_persistence::ArcBytes;
114    use turbo_tasks::TaskId;
115
116    use crate::kv_backing_storage::TurboBackingStorage;
117    pub use crate::{
118        backend::{SpecificTaskDataCategory, storage_schema::TaskStorage},
119        kv_backing_storage::{PARALLEL_DECODE_THRESHOLD, decode_task_storage_batch},
120    };
121
122    /// See `TurboBackingStorage::batch_lookup_raw`.
123    pub fn batch_lookup_raw(
124        storage: &TurboBackingStorage,
125        task_ids: &[TaskId],
126        category: SpecificTaskDataCategory,
127    ) -> Result<Vec<Option<ArcBytes>>> {
128        storage.batch_lookup_raw(task_ids, category)
129    }
130}
turbo_tasks_backend/lib.rs

turbo_tasks_backend/
lib.rs
