turbo_tasks/vc/
raw.rs

1use std::{
2    fmt::{Debug, Display},
3    future::Future,
4    num::{NonZeroU32, NonZeroU64},
5    pin::Pin,
6    sync::Arc,
7    task::{Poll, ready},
8};
9
10use anyhow::Result;
11use bincode::{Decode, Encode, enc::Encoder, error::EncodeError};
12use serde::{Deserialize, Serialize};
13
14use crate::{
15    ReadCellOptions, ReadConsistency, ReadOutputOptions, TaskId, TaskPersistence, TraitTypeId,
16    ValueTypeId,
17    backend::TypedCellContent,
18    event::EventListener,
19    id::{ExecutionId, LocalTaskId, TASK_ID_MAX},
20    manager::{
21        ReadCellTracking, ReadTracking, SUPPRESS_EVENTUAL_CONSISTENCY_TOP_LEVEL_TASK_CHECK,
22        ScheduleKey, TurboTasksApi, execute_read_target_inline, read_local_output,
23        with_turbo_tasks,
24    },
25    read_options::ReadOutcome,
26    registry::get_value_type,
27    turbo_tasks,
28};
29
30/// Identifies a specific cell within a task: a [`ValueTypeId`] paired with a
31/// sequentially-allocated index within that type.
32///
33/// Packed into a single [`NonZeroU32`]:
34/// ```text
35/// bits 31..=22 (10 bits): ValueTypeId logical value
36/// bits 21..=0  (22 bits): cell index
37/// ```
38/// Because the `ValueTypeId` is always `>= 1`, the type is trivially non-zero
39#[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Encode, Decode)]
40pub struct CellId(NonZeroU32);
41
42/// Number of low bits used for the cell index.
43const CELL_INDEX_BITS: u32 = 22;
44/// Mask selecting the cell index bits.
45const CELL_INDEX_MASK: u32 = (1 << CELL_INDEX_BITS) - 1;
46
47impl CellId {
48    /// Maximum `ValueTypeId` logical value that fits in the 10-bit type field.
49    pub const MAX_VALUE_TYPE_ID: u16 = (1 << (u32::BITS - CELL_INDEX_BITS)) as u16 - 1;
50    /// Maximum cell index that fits in the 22-bit index field.
51    pub const MAX_CELL_INDEX: u32 = CELL_INDEX_MASK;
52
53    /// Packs a `type_id` and `index` into a single word.
54    pub fn new(type_id: ValueTypeId, index: u32) -> Self {
55        let type_id = *type_id;
56        debug_assert!(
57            type_id <= Self::MAX_VALUE_TYPE_ID,
58            "ValueTypeId {} exceeds the {} cap packed into CellId",
59            type_id,
60            Self::MAX_VALUE_TYPE_ID,
61        );
62        debug_assert!(
63            index <= Self::MAX_CELL_INDEX,
64            "cell index {} exceeds the {} cap packed into CellId",
65            index,
66            Self::MAX_CELL_INDEX,
67        );
68        let packed = ((type_id as u32) << CELL_INDEX_BITS) | (index & CELL_INDEX_MASK);
69        // SAFETY: `type_id >= 1`, so `packed >= (1 << CELL_INDEX_BITS) > 0`.
70        CellId(unsafe { NonZeroU32::new_unchecked(packed) })
71    }
72
73    pub fn type_id(self) -> ValueTypeId {
74        let type_id = (self.0.get() >> CELL_INDEX_BITS) as u16;
75        // SAFETY: the high bits always hold a `ValueTypeId` of `1..=1023` by construction.
76        unsafe { ValueTypeId::new_unchecked(type_id) }
77    }
78
79    pub fn index(self) -> u32 {
80        self.0.get() & CELL_INDEX_MASK
81    }
82
83    /// The raw packed word, used by [`RawVc`] to pack a `TaskCell` into its u64.
84    pub(crate) fn raw(self) -> u32 {
85        self.0.get()
86    }
87
88    /// Reconstructs a `CellId` from a raw packed word produced by [`Self::raw`].
89    ///
90    /// # Safety
91    ///
92    /// `raw` must be a value previously returned by [`Self::raw`] (in
93    /// particular, non-zero with a valid 10-bit type id in the high bits).
94    pub(crate) unsafe fn from_raw(raw: u32) -> Self {
95        debug_assert!(raw != 0);
96        // SAFETY: the caller guarantees `raw` came from a valid `CellId`.
97        CellId(unsafe { NonZeroU32::new_unchecked(raw) })
98    }
99}
100
101impl Debug for CellId {
102    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
103        f.debug_struct("CellId")
104            .field("type_id", &self.type_id())
105            .field("index", &self.index())
106            .finish()
107    }
108}
109
110impl Display for CellId {
111    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
112        write!(
113            f,
114            "{}#{}",
115            get_value_type(self.type_id()).ty.name,
116            self.index()
117        )
118    }
119}
120
121/// A type-erased representation of [`Vc`].
122///
123/// Type erasure reduces the [monomorphization] (and therefore binary size and compilation time)
124/// required to support [`Vc`].
125///
126/// This type is heavily used within the [`Backend`][crate::backend::Backend] trait, but should
127/// otherwise be treated as an internal implementation detail of `turbo-tasks`.
128///
129/// # Representation
130///
131/// `RawVc` is one of three logical variants (see [`RawVcUnpacked`]) bit-packed
132/// into a single [`NonZeroU64`].
133///
134/// Bit 31 is the discriminator between a local output and a task variant; the
135/// two task variants are then told apart by whether the [`CellId`] field (the
136/// high 32 bits) is zero — which is unambiguous because every `CellId` is
137/// non-zero.
138///
139/// ```text
140/// bit31 = 1                  LocalOutput: 1<<31 | transient(1) | ExecutionId(16) << 1 | LocalTaskId(32) << 32
141/// bit31 = 0, bits32..63 == 0 TaskOutput:  TaskId(31)
142/// bit31 = 0, bits32..63 != 0 TaskCell:    TaskId(31) | CellId(32) << 32
143/// ```
144/// [`Vc`]: crate::Vc
145/// [monomorphization]: https://doc.rust-lang.org/book/ch10-01-syntax.html#performance-of-code-using-generics
146#[derive(Clone, Copy, PartialEq, Eq, Hash, Deserialize, Decode)]
147pub struct RawVc(NonZeroU64);
148
149// RawVc serializes its packed integer directly, so TaskId's guards cannot check embedded task
150// ids. Keep the original serde newtype shape and bincode format while rejecting transient values.
151impl Serialize for RawVc {
152    fn serialize<S: serde::Serializer>(&self, serializer: S) -> Result<S::Ok, S::Error> {
153        debug_assert!(
154            !self.is_transient(),
155            "transient RawVc must not be serialized"
156        );
157        serializer.serialize_newtype_struct("RawVc", &self.0)
158    }
159}
160
161impl Encode for RawVc {
162    fn encode<E: Encoder>(&self, encoder: &mut E) -> Result<(), EncodeError> {
163        debug_assert!(!self.is_transient(), "transient RawVc must not be encoded");
164        self.0.encode(encoder)
165    }
166}
167
168/// The unpacked form of [`RawVc`], produced by [`RawVc::unpack`].
169#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
170pub enum RawVcUnpacked {
171    /// The synchronous return value of a task (after argument resolution). This is the
172    /// representation used by [`OperationVc`][crate::OperationVc].
173    TaskOutput(TaskId),
174    /// A pointer to a specific [`Vc::cell`][crate::Vc::cell] or `.cell()` call within a task. This
175    /// is the representation used by [`ResolvedVc`][crate::ResolvedVc].
176    ///
177    /// [`CellId`] contains the [`ValueTypeId`], which can be useful for efficient downcasting.
178    TaskCell(TaskId, CellId),
179    /// The synchronous return value of a local task. This is created when a function is called
180    /// with unresolved arguments or more explicitly with
181    /// [`#[turbo_tasks::function(local)]`][crate::function].
182    ///
183    /// Local outputs are only valid within the context of their parent "non-local" task. Turbo
184    /// Task's APIs are designed to prevent escapes of local [`Vc`]s, but [`ExecutionId`] is used
185    /// for a fallback runtime assertion.
186    ///
187    /// [`Vc`]: crate::Vc
188    LocalOutput(ExecutionId, LocalTaskId, TaskPersistence),
189}
190
191/// Bit 31 discriminates `LocalOutput` (set) from the task variants (clear).
192/// It is free for the task variants because a `TaskId` is only 31 bits
193/// (`bits 0..=30`) and the `CellId` lives in the high 32 bits (`32..=63`).
194const RAW_VC_LOCAL_FLAG: u64 = 1 << 31;
195
196/// Mask of the `TaskId` value inside `TaskOutput` / `TaskCell` (bits `0..=30`).
197///
198/// This equals `TASK_ID_MAX` because a `TaskId` is `2^31 - 1`, so its max value
199/// is also the mask of its bits. TaskId is a u32 so this cast is safe.
200const RAW_VC_TASK_MASK: u64 = TASK_ID_MAX as u64;
201/// Shift of the packed `CellId` word inside `TaskCell`. A zero high word means
202/// `TaskOutput`; a non-zero one means `TaskCell`.
203const RAW_VC_CELL_SHIFT: u64 = 32;
204
205/// `LocalOutput` field layout (the `RAW_VC_LOCAL_FLAG` bit is always set).
206const RAW_VC_LOCAL_TRANSIENT_SHIFT: u64 = 0;
207const RAW_VC_LOCAL_EXECUTION_SHIFT: u64 = 1;
208const RAW_VC_LOCAL_TASK_SHIFT: u64 = 32;
209
210impl RawVc {
211    /// Packs the synchronous return value of a task. The word is simply the
212    /// `TaskId` value: bit 31 clear (a task variant) and the cell field zero
213    /// (no cell).
214    pub fn task_output(task: TaskId) -> Self {
215        let task = *task as u64;
216        debug_assert!(task <= RAW_VC_TASK_MASK, "TaskId exceeds 31 bits");
217        Self::from_bits(task)
218    }
219
220    /// Packs a pointer to a specific cell within a task: the `TaskId` in the low
221    /// bits and the non-zero `CellId` in the high 32 bits.
222    pub fn task_cell(task: TaskId, cell: CellId) -> Self {
223        let task = *task as u64;
224        debug_assert!(task <= RAW_VC_TASK_MASK, "TaskId exceeds 31 bits");
225        let cell = cell.raw() as u64;
226        Self::from_bits(task | (cell << RAW_VC_CELL_SHIFT))
227    }
228
229    /// Packs the synchronous return value of a local task, marked by bit 31.
230    pub fn local_output(
231        execution_id: ExecutionId,
232        local_task_id: LocalTaskId,
233        persistence: TaskPersistence,
234    ) -> Self {
235        let transient = (persistence == TaskPersistence::Transient) as u64;
236        let execution_id = *execution_id as u64;
237        let local_task_id = *local_task_id as u64;
238        Self::from_bits(
239            RAW_VC_LOCAL_FLAG
240                | (transient << RAW_VC_LOCAL_TRANSIENT_SHIFT)
241                | (execution_id << RAW_VC_LOCAL_EXECUTION_SHIFT)
242                | (local_task_id << RAW_VC_LOCAL_TASK_SHIFT),
243        )
244    }
245
246    #[inline]
247    fn from_bits(bits: u64) -> Self {
248        // SAFETY: every constructor produces a non-zero word — the task variants
249        // carry a `TaskId >= 1` in the low bits, and `LocalOutput` always sets
250        // `RAW_VC_LOCAL_FLAG`.
251        RawVc(unsafe { NonZeroU64::new_unchecked(bits) })
252    }
253
254    #[inline]
255    pub(crate) fn bits(self) -> u64 {
256        self.0.get()
257    }
258
259    /// The high 32 bits — the `CellId` slot. Zero for `TaskOutput`, the non-zero
260    /// packed `CellId` for `TaskCell`, and the `LocalTaskId` for `LocalOutput`.
261    #[inline]
262    fn cell_word(self) -> u32 {
263        (self.bits() >> RAW_VC_CELL_SHIFT) as u32
264    }
265
266    /// True for `TaskCell`: a task variant (bit 31 clear) whose cell field is
267    /// non-zero.
268    #[inline]
269    fn is_task_cell(self) -> bool {
270        !self.is_local_output() && self.cell_word() != 0
271    }
272
273    /// True for `TaskOutput`: a task variant (bit 31 clear) whose cell field is
274    /// zero.
275    #[inline]
276    fn is_task_output(self) -> bool {
277        !self.is_local_output() && self.cell_word() == 0
278    }
279
280    /// Reads the `TaskId` from a `TaskOutput` / `TaskCell` word.
281    ///
282    /// Produces a garbage value if this is a `LocalOutput` word
283    #[inline]
284    fn read_task_id(self) -> TaskId {
285        let id = (self.bits() & RAW_VC_TASK_MASK) as u32;
286        // SAFETY: a non-zero `TaskId` was packed in by construction.
287        unsafe { TaskId::new_unchecked(id) }
288    }
289
290    /// Reads the [`CellId`] from a `TaskCell` word.
291    ///
292    /// Produces a garbage value if this is a `LocalOutput` or `TaskOutput` word
293    #[inline]
294    fn read_cell(self) -> CellId {
295        // SAFETY: a valid packed `CellId` was stored in the high 32 bits.
296        unsafe { CellId::from_raw(self.cell_word()) }
297    }
298
299    /// Unpacks into the logical [`RawVcUnpacked`] enum for matching.
300    pub fn unpack(self) -> RawVcUnpacked {
301        if self.is_local_output() {
302            let (execution_id, local_task_id, persistence) = self.decode_local_output();
303            RawVcUnpacked::LocalOutput(execution_id, local_task_id, persistence)
304        } else {
305            let task_id = self.read_task_id();
306            let cell_word = self.cell_word();
307            if cell_word != 0 {
308                RawVcUnpacked::TaskCell(task_id, unsafe { CellId::from_raw(self.cell_word()) })
309            } else {
310                RawVcUnpacked::TaskOutput(task_id)
311            }
312        }
313    }
314
315    /// Returns the [`TaskId`] if this is a `TaskOutput`, otherwise `None`.
316    ///
317    /// Prefer this over [`unpack`][Self::unpack] when a caller only cares about
318    /// the `TaskOutput` case: it reads just the discriminator and the task bits.
319    pub fn as_task_output(self) -> Option<TaskId> {
320        self.is_task_output().then(|| self.read_task_id())
321    }
322
323    /// Returns the `(TaskId, CellId)` pair if this is a `TaskCell`, otherwise
324    /// `None`.
325    ///
326    /// Prefer this over [`unpack`][Self::unpack] when a caller only cares about
327    /// the `TaskCell` case: it reads just the discriminator, the task bits, and
328    /// the cell bits.
329    pub fn as_task_cell(self) -> Option<(TaskId, CellId)> {
330        self.is_task_cell()
331            .then(|| (self.read_task_id(), self.read_cell()))
332    }
333
334    /// Returns the `(ExecutionId, LocalTaskId, TaskPersistence)` triple if this
335    /// is a `LocalOutput`, otherwise `None`.
336    ///
337    /// Prefer this over [`unpack`][Self::unpack] when a caller only cares about
338    /// the `LocalOutput` case.
339    pub fn as_local_output(self) -> Option<(ExecutionId, LocalTaskId, TaskPersistence)> {
340        self.is_local_output().then(|| self.decode_local_output())
341    }
342
343    /// Decodes the fields of a `LocalOutput` word. Only valid when
344    /// [`RAW_VC_LOCAL_FLAG`] is set.
345    fn decode_local_output(self) -> (ExecutionId, LocalTaskId, TaskPersistence) {
346        let bits = self.bits();
347        let persistence = if (bits >> RAW_VC_LOCAL_TRANSIENT_SHIFT) & 1 == 1 {
348            TaskPersistence::Transient
349        } else {
350            TaskPersistence::Persistent
351        };
352        let execution_id = ((bits >> RAW_VC_LOCAL_EXECUTION_SHIFT) & 0xFFFF) as u16;
353        let local_task_id = ((bits >> RAW_VC_LOCAL_TASK_SHIFT) & 0xFFFF_FFFF) as u32;
354        // SAFETY: non-zero `ExecutionId`/`LocalTaskId` were packed in.
355        (
356            unsafe { ExecutionId::new_unchecked(execution_id) },
357            unsafe { LocalTaskId::new_unchecked(local_task_id) },
358            persistence,
359        )
360    }
361}
362
363impl Debug for RawVc {
364    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
365        match self.unpack() {
366            RawVcUnpacked::TaskOutput(task_id) => {
367                f.debug_tuple("RawVc::TaskOutput").field(&*task_id).finish()
368            }
369            RawVcUnpacked::TaskCell(task_id, cell_id) => f
370                .debug_tuple("RawVc::TaskCell")
371                .field(&*task_id)
372                .field(&cell_id.to_string())
373                .finish(),
374            RawVcUnpacked::LocalOutput(execution_id, local_task_id, task_persistence) => f
375                .debug_tuple("RawVc::LocalOutput")
376                .field(&*execution_id)
377                .field(&*local_task_id)
378                .field(&task_persistence)
379                .finish(),
380        }
381    }
382}
383
384impl Display for RawVc {
385    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
386        match self.unpack() {
387            RawVcUnpacked::TaskOutput(task_id) => write!(f, "output of task {}", *task_id),
388            RawVcUnpacked::TaskCell(task_id, cell_id) => {
389                write!(f, "{} of task {}", cell_id, *task_id)
390            }
391            RawVcUnpacked::LocalOutput(execution_id, local_task_id, task_persistence) => write!(
392                f,
393                "output of local task {} ({}, {})",
394                *local_task_id, *execution_id, task_persistence
395            ),
396        }
397    }
398}
399
400impl RawVc {
401    pub fn is_resolved(&self) -> bool {
402        self.is_task_cell()
403    }
404
405    pub fn is_local_output(&self) -> bool {
406        self.bits() & RAW_VC_LOCAL_FLAG != 0
407    }
408
409    /// Returns `true` if the task this `RawVc` reads from cannot be serialized and will not be
410    /// stored in the filesystem cache.
411    ///
412    /// See [`TaskPersistence`] for more details.
413    pub fn is_transient(&self) -> bool {
414        if self.is_local_output() {
415            // LocalOutput: the transient flag is stored as a bit.
416            (self.bits() >> RAW_VC_LOCAL_TRANSIENT_SHIFT) & 1 == 1
417        } else {
418            // TaskOutput / TaskCell: transience is a property of the TaskId value.
419            self.read_task_id().is_transient()
420        }
421    }
422
423    pub(crate) fn into_read(self) -> ReadRawVcFuture {
424        // returns a custom future to have something concrete and sized
425        // this avoids boxing in IntoFuture
426        ReadRawVcFuture::new(self)
427    }
428
429    /// See [`crate::Vc::to_resolved`].
430    pub(crate) fn resolve(self) -> ResolveRawVcFuture {
431        ResolveRawVcFuture::new(self)
432    }
433
434    /// Convert a potentially local `RawVc` into a non-local `RawVc`. This is a subset of resolution
435    /// resolution, because the returned `RawVc` can be a `TaskOutput`.
436    pub async fn to_non_local(self) -> Result<RawVc> {
437        let Some((execution_id, local_task_id, ..)) = self.as_local_output() else {
438            return Ok(self);
439        };
440        let tt = turbo_tasks();
441        let local_output = read_local_output(&*tt, execution_id, local_task_id).await?;
442        debug_assert!(
443            !local_output.is_local_output(),
444            "a LocalOutput cannot point at other LocalOutputs"
445        );
446        Ok(local_output)
447    }
448
449    /// Convert a potentially local `RawVc` into a non-local `RawVc`. This is a subset of resolution
450    /// resolution, because the returned `RawVc` can be a `TaskOutput`.
451    ///
452    /// 'unchecked' because the caller must have already confirmed that the local tasks were already
453    /// completed
454    pub(crate) fn to_non_local_unchecked_sync(self, tt: &dyn TurboTasksApi) -> Result<RawVc> {
455        let Some((execution_id, local_task_id, ..)) = self.as_local_output() else {
456            return Ok(self);
457        };
458        let local_output = match tt.try_read_local_output(execution_id, local_task_id)? {
459            Ok(raw_vc) => raw_vc,
460            Err(_event_listener) => unreachable!("local output is not ready yet"),
461        };
462        debug_assert!(
463            !local_output.is_local_output(),
464            "a LocalOutput cannot point at other LocalOutputs"
465        );
466        Ok(local_output)
467    }
468
469    pub fn try_get_task_id(&self) -> Option<TaskId> {
470        (!self.is_local_output()).then(|| self.read_task_id())
471    }
472
473    pub fn try_get_type_id(&self) -> Option<ValueTypeId> {
474        self.is_task_cell().then(|| self.read_cell().type_id())
475    }
476
477    /// For a cell that's already resolved, synchronously check if it implements a trait using the
478    /// type information in `RawVc::TaskCell` (we don't actually need to read the cell!).
479    pub(crate) fn resolved_has_trait(&self, trait_id: TraitTypeId) -> bool {
480        debug_assert!(
481            self.is_task_cell(),
482            "resolved_has_trait must be called with a RawVc::TaskCell"
483        );
484        get_value_type(self.read_cell().type_id()).has_trait(&trait_id)
485    }
486
487    /// For a cell that's already resolved, synchronously check if it is a given type using the type
488    /// information in `RawVc::TaskCell` (we don't actually need to read the cell!).
489    pub(crate) fn resolved_is_type(&self, type_id: ValueTypeId) -> bool {
490        debug_assert!(
491            self.is_task_cell(),
492            "resolved_is_type must be called with a RawVc::TaskCell"
493        );
494        self.read_cell().type_id() == type_id
495    }
496}
497
498/// Polls a pending [`EventListener`] slot. Returns [`Poll::Pending`] if the event has not yet
499/// fired. On [`Poll::Ready`], clears the slot so it is not polled again.
500fn poll_listener(
501    listener: &mut Option<EventListener>,
502    cx: &mut std::task::Context<'_>,
503) -> Poll<()> {
504    if let Some(l) = listener {
505        ready!(Pin::new(l).poll(cx));
506        *listener = None;
507    }
508    Poll::Ready(())
509}
510
511/// Wraps `f` in a scope that suppresses the eventual-consistency top-level task assertion,
512/// but only when `strongly_consistent` is `true` and debug assertions are enabled.
513///
514/// This is needed because a strongly-consistent read of a `TaskOutput` is not a single atomic
515/// operation — inner reads switch to eventual consistency after the first output is resolved —
516/// which would otherwise trigger the assertion in top-level tasks.
517fn suppress_top_level_task_check<R>(strongly_consistent: bool, f: impl FnOnce() -> R) -> R {
518    if cfg!(debug_assertions) && strongly_consistent {
519        // Temporarily suppress the top-level task check
520        SUPPRESS_EVENTUAL_CONSISTENCY_TOP_LEVEL_TASK_CHECK.sync_scope(true, f)
521    } else {
522        f()
523    }
524}
525
526/// Executes the task a read is waiting for when it is only scheduled, so the read can continue
527/// without waiting for a worker.
528///
529/// Must be called *outside* [`with_turbo_tasks`]: executing a task enters a task-local scope of its
530/// own, which panics while the read borrows `TURBO_TASKS`.
531fn execute_inline(key: ScheduleKey) {
532    execute_read_target_inline(&*turbo_tasks(), key);
533}
534
535#[must_use]
536pub struct ResolveRawVcFuture {
537    current: RawVc,
538    read_output_options: ReadOutputOptions,
539    /// This flag is redundant with `read_output_options`, but `read_output_options` is mutated
540    /// during the resolve. This flag indicates that the initial read was strongly consistent.
541    strongly_consistent: bool,
542    listener: Option<EventListener>,
543}
544
545impl ResolveRawVcFuture {
546    fn new(vc: RawVc) -> Self {
547        ResolveRawVcFuture {
548            current: vc,
549            read_output_options: ReadOutputOptions::default(),
550            strongly_consistent: false,
551            listener: None,
552        }
553    }
554
555    pub fn strongly_consistent(mut self) -> Self {
556        self.strongly_consistent = true;
557        self.read_output_options.consistency = ReadConsistency::Strong;
558        self
559    }
560
561    /// Track task output reads with a specific key (forwarded from
562    /// [`ReadRawVcFuture::track_with_key`]).
563    pub(crate) fn track_with_key(mut self) -> Self {
564        self.read_output_options.tracking = ReadTracking::Tracked;
565        self
566    }
567
568    /// Do not track task output reads as dependencies (forwarded from
569    /// [`ReadRawVcFuture::untracked`]).
570    pub(crate) fn untracked(mut self) -> Self {
571        self.read_output_options.tracking = ReadTracking::TrackOnlyError;
572        self
573    }
574}
575
576impl Future for ResolveRawVcFuture {
577    type Output = Result<RawVc>;
578
579    #[inline(never)]
580    fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll<Self::Output> {
581        // SAFETY: we are not moving self
582        let this = unsafe { self.get_unchecked_mut() };
583
584        let strongly_consistent = this.strongly_consistent;
585        // `execute_inline` is the task to execute inline, reported out-of-band so that the value
586        // path stays exactly as cheap as it was. It is only set for tasks that are merely
587        // scheduled; one that a worker is already executing cannot be taken over.
588        let mut poll_fn = |tt: &Arc<dyn TurboTasksApi>,
589                           execute_inline: &mut Option<ScheduleKey>|
590         -> Poll<Self::Output> {
591            'outer: loop {
592                ready!(poll_listener(&mut this.listener, cx));
593                let (listener, key) = match this.current.unpack() {
594                    RawVcUnpacked::TaskOutput(task) => {
595                        let read_result = tt.try_read_task_output(task, this.read_output_options);
596                        match read_result {
597                            Ok(ReadOutcome::Value(vc)) => {
598                                // turbo-tasks-backend doesn't currently have any sort of
599                                // "transaction" or global lock mechanism to group together chains
600                                // of `TaskOutput`/`TaskCell` reads.
601                                //
602                                // If we ignore the theoretical TOCTOU issues, we no longer need to
603                                // read strongly consistent, as any Vc returned from the first task
604                                // will be inside of the scope of the first task. So it's already
605                                // strongly consistent.
606                                this.read_output_options.consistency = ReadConsistency::Eventual;
607                                this.current = vc;
608                                continue 'outer;
609                            }
610                            // Nobody has started the task yet, so the caller may take it over.
611                            Ok(ReadOutcome::Scheduled(listener)) => {
612                                (listener, Some(ScheduleKey::Task(task)))
613                            }
614                            Ok(ReadOutcome::InProgress(listener)) => {
615                                // A worker is on it; nothing to take over. Loop back so
616                                // `poll_listener` registers the waker on this listener — returning
617                                // `Pending` here would sleep through the event.
618                                #[cfg(feature = "inline_execution_stats")]
619                                tt.note_waited_for_in_progress_task();
620                                this.listener = Some(listener);
621                                continue 'outer;
622                            }
623                            Err(err) => return Poll::Ready(Err(err)),
624                        }
625                    }
626                    RawVcUnpacked::TaskCell(_, _) => return Poll::Ready(Ok(this.current)),
627                    RawVcUnpacked::LocalOutput(execution_id, local_task_id, ..) => {
628                        debug_assert_eq!(
629                            this.read_output_options.consistency,
630                            ReadConsistency::Eventual
631                        );
632                        let read_result = tt.try_read_local_output(execution_id, local_task_id);
633                        match read_result {
634                            Ok(Ok(vc)) => {
635                                this.current = vc;
636                                continue 'outer;
637                            }
638                            Ok(Err(listener)) => (
639                                listener,
640                                Some(ScheduleKey::current_local_task(execution_id, local_task_id)),
641                            ),
642                            Err(err) => return Poll::Ready(Err(err)),
643                        }
644                    }
645                };
646                // The task is not done yet, so we have to wait for it — unless it is merely
647                // scheduled, in which case our caller executes it and we read again.
648                this.listener = Some(listener);
649                *execute_inline = key;
650                return Poll::Pending;
651            }
652        };
653
654        loop {
655            let mut execute_inline_key = None;
656            // HACK: Temporarily suppress top-level task check if doing strongly consistent read.
657            //
658            // This masks a bug: There's an unlikely TOCTOU race condition in `poll_fn`. Because the
659            // strongly consistent read isn't a single atomic operation, any inner `TaskOutput` or
660            // `TaskCell` could get mutated after the strongly consistent read of the outer
661            // `TaskOutput`.
662            let result = suppress_top_level_task_check(strongly_consistent, || {
663                with_turbo_tasks(|tt| poll_fn(tt, &mut execute_inline_key))
664            });
665            if let Some(key) = execute_inline_key {
666                // Not inside `with_turbo_tasks`, see `execute_inline`.
667                execute_inline(key);
668                continue;
669            }
670            return result;
671        }
672    }
673}
674
675impl Unpin for ResolveRawVcFuture {}
676
677#[must_use]
678pub struct ReadRawVcFuture {
679    read_cell_options: ReadCellOptions,
680    state: ReadRawVcState,
681}
682
683/// Phase 1 and phase 2 of [`ReadRawVcFuture`] use disjoint sets of fields. Storing them in an
684/// enum keeps the future smaller than holding both sets simultaneously.
685enum ReadRawVcState {
686    /// Phase 1: resolves the [`RawVc`] pointer chain to a [`RawVc::TaskCell`].
687    Resolving(ResolveRawVcFuture),
688    /// Phase 2: the resolved task/cell identity plus a listener for the cell read wait.
689    Reading {
690        task: TaskId,
691        index: CellId,
692        /// Whether phase 1 was a strongly-consistent read. Needed here to re-apply
693        /// [`suppress_top_level_task_check`] in phase 2. Lives in this variant (rather than the
694        /// outer struct) so it can share padding with the other `Reading` fields — keeping
695        /// `Reading` no larger than `Resolving`, and the whole future 8 bytes smaller.
696        strongly_consistent: bool,
697        listener: Option<EventListener>,
698    },
699}
700
701impl ReadRawVcFuture {
702    pub(crate) fn new(vc: RawVc) -> Self {
703        ReadRawVcFuture {
704            read_cell_options: ReadCellOptions::default(),
705            state: ReadRawVcState::Resolving(ResolveRawVcFuture::new(vc)),
706        }
707    }
708
709    fn map_resolve(mut self, f: impl FnOnce(ResolveRawVcFuture) -> ResolveRawVcFuture) -> Self {
710        match self.state {
711            ReadRawVcState::Resolving(resolve) => {
712                self.state = ReadRawVcState::Resolving(f(resolve));
713            }
714            ReadRawVcState::Reading { .. } => {
715                unreachable!("builder methods are only called before polling");
716            }
717        }
718        self
719    }
720
721    /// Make reads strongly consistent.
722    pub fn strongly_consistent(self) -> Self {
723        self.map_resolve(|r| r.strongly_consistent())
724    }
725
726    /// Track the value as a dependency with an key.
727    pub fn track_with_key(mut self, key: u64) -> Self {
728        self.read_cell_options.tracking = ReadCellTracking::Tracked { key: Some(key) };
729        self.map_resolve(|r| r.track_with_key())
730    }
731
732    /// This will not track the value as dependency, but will still track the error as dependency,
733    /// if there is an error.
734    ///
735    /// INVALIDATION: Be careful with this, it will not track dependencies, so
736    /// using it could break cache invalidation.
737    pub fn untracked(mut self) -> Self {
738        self.read_cell_options.tracking = ReadCellTracking::TrackOnlyError;
739        self.map_resolve(|r| r.untracked())
740    }
741
742    /// Hint that this is the final read of the cell content.
743    pub fn final_read_hint(mut self) -> Self {
744        self.read_cell_options.final_read_hint = true;
745        self
746    }
747}
748
749impl Future for ReadRawVcFuture {
750    type Output = Result<TypedCellContent>;
751
752    #[inline(never)]
753    fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll<Self::Output> {
754        // SAFETY: we are not moving self
755        let this = unsafe { self.get_unchecked_mut() };
756
757        // --- Phase 1: resolve the RawVc pointer chain to a TaskCell ---
758        //
759        // `ResolveRawVcFuture` is `Unpin`, so `Pin::new` is safe.
760        // It handles `with_turbo_tasks` and `suppress_top_level_task_check` internally.
761        if let ReadRawVcState::Resolving(resolve) = &mut this.state {
762            let strongly_consistent = resolve.strongly_consistent;
763            match ready!(Pin::new(resolve).poll(cx)) {
764                Err(err) => return Poll::Ready(Err(err)),
765                Ok(resolved) => {
766                    let Some((task, index)) = resolved.as_task_cell() else {
767                        unreachable!("ResolveRawVcFuture always resolves to a TaskCell")
768                    };
769                    this.state = ReadRawVcState::Reading {
770                        task,
771                        index,
772                        strongly_consistent,
773                        listener: None,
774                    };
775                }
776            }
777        }
778
779        // --- Phase 2: read the cell content ---
780        let ReadRawVcState::Reading {
781            task,
782            index,
783            strongly_consistent,
784            listener,
785        } = &mut this.state
786        else {
787            unreachable!("phase 1 transitioned to Reading above");
788        };
789        let task = *task;
790        let index = *index;
791        let read_cell_options = this.read_cell_options;
792
793        let strongly_consistent = *strongly_consistent;
794
795        let mut poll_fn = |tt: &Arc<dyn TurboTasksApi>,
796                           execute_inline: &mut Option<ScheduleKey>|
797         -> Poll<Self::Output> {
798            loop {
799                ready!(poll_listener(listener, cx));
800                let (new_listener, key) =
801                    match tt.try_read_task_cell(task, index, read_cell_options) {
802                        Ok(ReadOutcome::Value(content)) => return Poll::Ready(Ok(content)),
803                        Ok(ReadOutcome::Scheduled(l)) => (l, Some(ScheduleKey::Task(task))),
804                        Ok(ReadOutcome::InProgress(l)) => {
805                            // A worker is already filling the cell; nothing to take over. Loop back
806                            // so `poll_listener` registers the waker on this listener — returning
807                            // `Pending` here would sleep through the event.
808                            #[cfg(feature = "inline_execution_stats")]
809                            tt.note_waited_for_in_progress_task();
810                            *listener = Some(l);
811                            continue;
812                        }
813                        Err(err) => return Poll::Ready(Err(err)),
814                    };
815                // The cell isn't available yet, so we have to wait for the task that fills it —
816                // unless that task is merely scheduled, in which case our caller executes it and we
817                // read again.
818                *listener = Some(new_listener);
819                *execute_inline = key;
820                return Poll::Pending;
821            }
822        };
823
824        loop {
825            let mut execute_inline_key = None;
826            // Phase 2 must also suppress the top-level task check when phase 1 was
827            // strongly-consistent. The suppression from `ResolveRawVcFuture::poll` only lasts for
828            // the duration of that individual `poll` call and does not carry over to subsequent
829            // calls or to this phase.
830            let result = suppress_top_level_task_check(strongly_consistent, || {
831                with_turbo_tasks(|tt| poll_fn(tt, &mut execute_inline_key))
832            });
833            if let Some(key) = execute_inline_key {
834                // Not inside `with_turbo_tasks`, see `execute_inline`.
835                execute_inline(key);
836                continue;
837            }
838            return result;
839        }
840    }
841}
842
843impl Unpin for ReadRawVcFuture {}
844
845#[cfg(test)]
846mod tests {
847    use super::*;
848
849    /// `CellId` must pack into 4 bytes and keep its niche so `Option<CellId>`
850    /// stays 4 bytes — this is the whole point of [`RawVc`] shrinking.
851    #[test]
852    fn cell_id_is_four_bytes() {
853        assert_eq!(size_of::<CellId>(), 4);
854        assert_eq!(size_of::<Option<CellId>>(), 4);
855    }
856
857    /// Packing and unpacking a `(type_id, index)` pair must round-trip across
858    /// the full range of both fields, including the boundary values.
859    #[test]
860    fn cell_id_pack_unpack_round_trip() {
861        let type_ids = [1u16, 2, 100, CellId::MAX_VALUE_TYPE_ID];
862        let indices = [0u32, 1, 12345, CellId::MAX_CELL_INDEX];
863        for &raw_ty in &type_ids {
864            // SAFETY: all test values are >= 1.
865            let type_id = unsafe { ValueTypeId::new_unchecked(raw_ty) };
866            for &index in &indices {
867                let cell = CellId::new(type_id, index);
868                assert_eq!(cell.type_id(), type_id, "type_id round-trip for {raw_ty}");
869                assert_eq!(cell.index(), index, "index round-trip for {index}");
870            }
871        }
872    }
873
874    /// Distinct `(type_id, index)` pairs must pack to distinct words — the
875    /// packing is a bijection, which is what lets us derive `Eq`/`Hash`.
876    #[test]
877    fn cell_id_packing_is_bijective() {
878        // SAFETY: ids are >= 1.
879        let a = CellId::new(unsafe { ValueTypeId::new_unchecked(1) }, 0);
880        let b = CellId::new(unsafe { ValueTypeId::new_unchecked(1) }, 1);
881        let c = CellId::new(unsafe { ValueTypeId::new_unchecked(2) }, 0);
882        assert_ne!(a, b);
883        assert_ne!(a, c);
884        assert_ne!(b, c);
885    }
886
887    /// `RawVc` must pack into 8 bytes and keep its niche.
888    #[test]
889    fn raw_vc_is_eight_bytes() {
890        assert_eq!(size_of::<RawVc>(), 8);
891        assert_eq!(size_of::<Option<RawVc>>(), 8);
892    }
893
894    /// Every variant must round-trip through pack → `unpack()` across the full
895    /// range of each packed field, including boundary values and both
896    /// persistence states. This is the core correctness property of the
897    /// bit-packing.
898    #[test]
899    fn raw_vc_pack_unpack_round_trip() {
900        // SAFETY: all ids below are >= 1 and within their bit budgets.
901        let tasks = [
902            1u32,
903            2,
904            crate::TRANSIENT_TASK_BIT - 1,
905            crate::TRANSIENT_TASK_BIT,
906            TASK_ID_MAX,
907        ];
908        for &t in &tasks {
909            let task = unsafe { TaskId::new_unchecked(t) };
910
911            // TaskOutput
912            let vc = RawVc::task_output(task);
913            assert_eq!(vc.unpack(), RawVcUnpacked::TaskOutput(task));
914            assert!(!vc.is_resolved() && !vc.is_local_output());
915            assert_eq!(vc.is_transient(), task.is_transient());
916            assert_eq!(vc.try_get_task_id(), Some(task));
917            // single-arm accessors
918            assert_eq!(vc.as_task_output(), Some(task));
919            assert_eq!(vc.as_task_cell(), None);
920            assert_eq!(vc.as_local_output(), None);
921
922            // TaskCell, across CellId boundaries
923            for cell in [
924                CellId::new(unsafe { ValueTypeId::new_unchecked(1) }, 0),
925                CellId::new(
926                    unsafe { ValueTypeId::new_unchecked(CellId::MAX_VALUE_TYPE_ID) },
927                    CellId::MAX_CELL_INDEX,
928                ),
929            ] {
930                let vc = RawVc::task_cell(task, cell);
931                assert_eq!(vc.unpack(), RawVcUnpacked::TaskCell(task, cell));
932                assert!(vc.is_resolved());
933                assert_eq!(vc.try_get_task_id(), Some(task));
934                assert_eq!(vc.try_get_type_id(), Some(cell.type_id()));
935                // single-arm accessors
936                assert_eq!(vc.as_task_cell(), Some((task, cell)));
937                assert_eq!(vc.as_task_output(), None);
938                assert_eq!(vc.as_local_output(), None);
939            }
940        }
941
942        // LocalOutput, both persistence states and boundary ids
943        for persistence in [TaskPersistence::Persistent, TaskPersistence::Transient] {
944            for (e, l) in [(1u16, 1u32), (u16::MAX, u32::MAX)] {
945                let exec = unsafe { ExecutionId::new_unchecked(e) };
946                let local = unsafe { LocalTaskId::new_unchecked(l) };
947                let vc = RawVc::local_output(exec, local, persistence);
948                assert_eq!(
949                    vc.unpack(),
950                    RawVcUnpacked::LocalOutput(exec, local, persistence)
951                );
952                assert!(vc.is_local_output());
953                assert_eq!(vc.is_transient(), persistence == TaskPersistence::Transient);
954                assert_eq!(vc.try_get_task_id(), None);
955                // single-arm accessors
956                assert_eq!(vc.as_local_output(), Some((exec, local, persistence)));
957                assert_eq!(vc.as_task_output(), None);
958                assert_eq!(vc.as_task_cell(), None);
959            }
960        }
961    }
962
963    /// The discriminator relies on the cell field being zero for `TaskOutput`
964    /// and non-zero for `TaskCell`. A `TaskOutput` and a `TaskCell` that share
965    /// the same `TaskId` must still be told apart, and a `LocalOutput` whose
966    /// `LocalTaskId` populates the high bits (the cell-field region) must remain
967    /// a `LocalOutput` because bit 31 wins.
968    #[test]
969    fn raw_vc_discriminator_is_unambiguous() {
970        // SAFETY: all ids are >= 1.
971        let task = unsafe { TaskId::new_unchecked(123) };
972        let cell = CellId::new(unsafe { ValueTypeId::new_unchecked(1) }, 0);
973
974        let output = RawVc::task_output(task);
975        let task_cell = RawVc::task_cell(task, cell);
976        assert!(output.is_task_output() && !output.is_task_cell() && !output.is_local_output());
977        assert!(
978            task_cell.is_task_cell() && !task_cell.is_task_output() && !task_cell.is_local_output()
979        );
980        // Same TaskId, different variants, distinct words.
981        assert_ne!(output, task_cell);
982        assert_eq!(output.read_task_id(), task_cell.read_task_id());
983
984        // A LocalOutput with a max LocalTaskId fills the high 32 bits; it must
985        // not be misread as a TaskCell.
986        let local = RawVc::local_output(
987            unsafe { ExecutionId::new_unchecked(u16::MAX) },
988            unsafe { LocalTaskId::new_unchecked(u32::MAX) },
989            TaskPersistence::Persistent,
990        );
991        assert!(local.is_local_output() && !local.is_task_cell() && !local.is_task_output());
992    }
993
994    #[test]
995    #[cfg(debug_assertions)]
996    #[should_panic(expected = "TaskId exceeds 31 bits")]
997    #[cfg_attr(
998        target_family = "wasm",
999        ignore = "no unwinding on wasm: std is built panic=abort, so catch_unwind cannot catch"
1000    )]
1001    fn task_output_panics_on_out_of_range_task_id() {
1002        // `TASK_ID_MAX + 1` is the first value that sets bit 31.
1003        // SAFETY: non-zero.
1004        let task = unsafe { TaskId::new_unchecked(TASK_ID_MAX + 1) };
1005        let _ = RawVc::task_output(task);
1006    }
1007
1008    #[test]
1009    #[cfg(debug_assertions)]
1010    #[should_panic(expected = "TaskId exceeds 31 bits")]
1011    #[cfg_attr(
1012        target_family = "wasm",
1013        ignore = "no unwinding on wasm: std is built panic=abort, so catch_unwind cannot catch"
1014    )]
1015    fn task_cell_panics_on_out_of_range_task_id() {
1016        // SAFETY: non-zero.
1017        let task = unsafe { TaskId::new_unchecked(TASK_ID_MAX + 1) };
1018        let cell = CellId::new(unsafe { ValueTypeId::new_unchecked(1) }, 0);
1019        let _ = RawVc::task_cell(task, cell);
1020    }
1021
1022    #[test]
1023    #[cfg(debug_assertions)]
1024    #[should_panic(expected = "exceeds")]
1025    #[cfg_attr(
1026        target_family = "wasm",
1027        ignore = "no unwinding on wasm: std is built panic=abort, so catch_unwind cannot catch"
1028    )]
1029    fn cell_id_panics_on_out_of_range_type_id() {
1030        // SAFETY: `MAX_VALUE_TYPE_ID + 1` is non-zero.
1031        let type_id = unsafe { ValueTypeId::new_unchecked(CellId::MAX_VALUE_TYPE_ID + 1) };
1032        let _ = CellId::new(type_id, 0);
1033    }
1034    #[test]
1035    #[cfg(debug_assertions)]
1036    #[should_panic(expected = "exceeds")]
1037    #[cfg_attr(
1038        target_family = "wasm",
1039        ignore = "no unwinding on wasm: std is built panic=abort, so catch_unwind cannot catch"
1040    )]
1041    fn cell_id_panics_on_out_of_range_index() {
1042        let type_id = unsafe { ValueTypeId::new_unchecked(1) };
1043        let _ = CellId::new(type_id, CellId::MAX_CELL_INDEX + 1);
1044    }
1045}
turbo_tasks/vc/raw.rs

turbo_tasks/vc/
raw.rs
