turbo_tasks/vc/
raw.rs

1use std::{
2    fmt::{Debug, Display},
3    future::Future,
4    num::{NonZeroU32, NonZeroU64},
5    pin::Pin,
6    sync::Arc,
7    task::{Poll, ready},
8};
9
10use anyhow::Result;
11use bincode::{Decode, Encode};
12use serde::{Deserialize, Serialize};
13
14use crate::{
15    ReadCellOptions, ReadConsistency, ReadOutputOptions, TaskId, TaskPersistence, TraitTypeId,
16    ValueTypeId,
17    backend::TypedCellContent,
18    event::EventListener,
19    id::{ExecutionId, LocalTaskId, TASK_ID_MAX},
20    manager::{
21        ReadCellTracking, ReadTracking, SUPPRESS_EVENTUAL_CONSISTENCY_TOP_LEVEL_TASK_CHECK,
22        TurboTasksApi, read_local_output, with_turbo_tasks,
23    },
24    registry::get_value_type,
25    turbo_tasks,
26};
27
28/// Identifies a specific cell within a task: a [`ValueTypeId`] paired with a
29/// sequentially-allocated index within that type.
30///
31/// Packed into a single [`NonZeroU32`]:
32/// ```text
33/// bits 31..=22 (10 bits): ValueTypeId logical value
34/// bits 21..=0  (22 bits): cell index
35/// ```
36/// Because the `ValueTypeId` is always `>= 1`, the type is trivially non-zero
37#[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Encode, Decode)]
38pub struct CellId(NonZeroU32);
39
40/// Number of low bits used for the cell index.
41const CELL_INDEX_BITS: u32 = 22;
42/// Mask selecting the cell index bits.
43const CELL_INDEX_MASK: u32 = (1 << CELL_INDEX_BITS) - 1;
44
45impl CellId {
46    /// Maximum `ValueTypeId` logical value that fits in the 10-bit type field.
47    pub const MAX_VALUE_TYPE_ID: u16 = (1 << (u32::BITS - CELL_INDEX_BITS)) as u16 - 1;
48    /// Maximum cell index that fits in the 22-bit index field.
49    pub const MAX_CELL_INDEX: u32 = CELL_INDEX_MASK;
50
51    /// Packs a `type_id` and `index` into a single word.
52    pub fn new(type_id: ValueTypeId, index: u32) -> Self {
53        let type_id = *type_id;
54        debug_assert!(
55            type_id <= Self::MAX_VALUE_TYPE_ID,
56            "ValueTypeId {} exceeds the {} cap packed into CellId",
57            type_id,
58            Self::MAX_VALUE_TYPE_ID,
59        );
60        debug_assert!(
61            index <= Self::MAX_CELL_INDEX,
62            "cell index {} exceeds the {} cap packed into CellId",
63            index,
64            Self::MAX_CELL_INDEX,
65        );
66        let packed = ((type_id as u32) << CELL_INDEX_BITS) | (index & CELL_INDEX_MASK);
67        // SAFETY: `type_id >= 1`, so `packed >= (1 << CELL_INDEX_BITS) > 0`.
68        CellId(unsafe { NonZeroU32::new_unchecked(packed) })
69    }
70
71    pub fn type_id(self) -> ValueTypeId {
72        let type_id = (self.0.get() >> CELL_INDEX_BITS) as u16;
73        // SAFETY: the high bits always hold a `ValueTypeId` of `1..=1023` by construction.
74        unsafe { ValueTypeId::new_unchecked(type_id) }
75    }
76
77    pub fn index(self) -> u32 {
78        self.0.get() & CELL_INDEX_MASK
79    }
80
81    /// The raw packed word, used by [`RawVc`] to pack a `TaskCell` into its u64.
82    pub(crate) fn raw(self) -> u32 {
83        self.0.get()
84    }
85
86    /// Reconstructs a `CellId` from a raw packed word produced by [`Self::raw`].
87    ///
88    /// # Safety
89    ///
90    /// `raw` must be a value previously returned by [`Self::raw`] (in
91    /// particular, non-zero with a valid 10-bit type id in the high bits).
92    pub(crate) unsafe fn from_raw(raw: u32) -> Self {
93        debug_assert!(raw != 0);
94        // SAFETY: the caller guarantees `raw` came from a valid `CellId`.
95        CellId(unsafe { NonZeroU32::new_unchecked(raw) })
96    }
97}
98
99impl Debug for CellId {
100    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
101        f.debug_struct("CellId")
102            .field("type_id", &self.type_id())
103            .field("index", &self.index())
104            .finish()
105    }
106}
107
108impl Display for CellId {
109    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
110        write!(
111            f,
112            "{}#{}",
113            get_value_type(self.type_id()).ty.name,
114            self.index()
115        )
116    }
117}
118
119/// A type-erased representation of [`Vc`].
120///
121/// Type erasure reduces the [monomorphization] (and therefore binary size and compilation time)
122/// required to support [`Vc`].
123///
124/// This type is heavily used within the [`Backend`][crate::backend::Backend] trait, but should
125/// otherwise be treated as an internal implementation detail of `turbo-tasks`.
126///
127/// # Representation
128///
129/// `RawVc` is one of three logical variants (see [`RawVcUnpacked`]) bit-packed
130/// into a single [`NonZeroU64`].
131///
132/// Bit 31 is the discriminator between a local output and a task variant; the
133/// two task variants are then told apart by whether the [`CellId`] field (the
134/// high 32 bits) is zero — which is unambiguous because every `CellId` is
135/// non-zero.
136///
137/// ```text
138/// bit31 = 1                  LocalOutput: 1<<31 | transient(1) | ExecutionId(16) << 1 | LocalTaskId(32) << 32
139/// bit31 = 0, bits32..63 == 0 TaskOutput:  TaskId(31)
140/// bit31 = 0, bits32..63 != 0 TaskCell:    TaskId(31) | CellId(32) << 32
141/// ```
142/// [`Vc`]: crate::Vc
143/// [monomorphization]: https://doc.rust-lang.org/book/ch10-01-syntax.html#performance-of-code-using-generics
144#[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Encode, Decode)]
145pub struct RawVc(NonZeroU64);
146
147/// The unpacked form of [`RawVc`], produced by [`RawVc::unpack`].
148#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
149pub enum RawVcUnpacked {
150    /// The synchronous return value of a task (after argument resolution). This is the
151    /// representation used by [`OperationVc`][crate::OperationVc].
152    TaskOutput(TaskId),
153    /// A pointer to a specific [`Vc::cell`][crate::Vc::cell] or `.cell()` call within a task. This
154    /// is the representation used by [`ResolvedVc`].
155    ///
156    /// [`CellId`] contains the [`ValueTypeId`], which can be useful for efficient downcasting.
157    TaskCell(TaskId, CellId),
158    /// The synchronous return value of a local task. This is created when a function is called
159    /// with unresolved arguments or more explicitly with
160    /// [`#[turbo_tasks::function(local)]`][crate::function].
161    ///
162    /// Local outputs are only valid within the context of their parent "non-local" task. Turbo
163    /// Task's APIs are designed to prevent escapes of local [`Vc`]s, but [`ExecutionId`] is used
164    /// for a fallback runtime assertion.
165    ///
166    /// [`Vc`]: crate::Vc
167    LocalOutput(ExecutionId, LocalTaskId, TaskPersistence),
168}
169
170/// Bit 31 discriminates `LocalOutput` (set) from the task variants (clear).
171/// It is free for the task variants because a `TaskId` is only 31 bits
172/// (`bits 0..=30`) and the `CellId` lives in the high 32 bits (`32..=63`).
173const RAW_VC_LOCAL_FLAG: u64 = 1 << 31;
174
175/// Mask of the `TaskId` value inside `TaskOutput` / `TaskCell` (bits `0..=30`).
176///
177/// This equals `TASK_ID_MAX` because a `TaskId` is `2^31 - 1`, so its max value
178/// is also the mask of its bits. TaskId is a u32 so this cast is safe.
179const RAW_VC_TASK_MASK: u64 = TASK_ID_MAX as u64;
180/// Shift of the packed `CellId` word inside `TaskCell`. A zero high word means
181/// `TaskOutput`; a non-zero one means `TaskCell`.
182const RAW_VC_CELL_SHIFT: u64 = 32;
183
184/// `LocalOutput` field layout (the `RAW_VC_LOCAL_FLAG` bit is always set).
185const RAW_VC_LOCAL_TRANSIENT_SHIFT: u64 = 0;
186const RAW_VC_LOCAL_EXECUTION_SHIFT: u64 = 1;
187const RAW_VC_LOCAL_TASK_SHIFT: u64 = 32;
188
189impl RawVc {
190    /// Packs the synchronous return value of a task. The word is simply the
191    /// `TaskId` value: bit 31 clear (a task variant) and the cell field zero
192    /// (no cell).
193    pub fn task_output(task: TaskId) -> Self {
194        let task = *task as u64;
195        debug_assert!(task <= RAW_VC_TASK_MASK, "TaskId exceeds 31 bits");
196        Self::from_bits(task)
197    }
198
199    /// Packs a pointer to a specific cell within a task: the `TaskId` in the low
200    /// bits and the non-zero `CellId` in the high 32 bits.
201    pub fn task_cell(task: TaskId, cell: CellId) -> Self {
202        let task = *task as u64;
203        debug_assert!(task <= RAW_VC_TASK_MASK, "TaskId exceeds 31 bits");
204        let cell = cell.raw() as u64;
205        Self::from_bits(task | (cell << RAW_VC_CELL_SHIFT))
206    }
207
208    /// Packs the synchronous return value of a local task, marked by bit 31.
209    pub fn local_output(
210        execution_id: ExecutionId,
211        local_task_id: LocalTaskId,
212        persistence: TaskPersistence,
213    ) -> Self {
214        let transient = (persistence == TaskPersistence::Transient) as u64;
215        let execution_id = *execution_id as u64;
216        let local_task_id = *local_task_id as u64;
217        Self::from_bits(
218            RAW_VC_LOCAL_FLAG
219                | (transient << RAW_VC_LOCAL_TRANSIENT_SHIFT)
220                | (execution_id << RAW_VC_LOCAL_EXECUTION_SHIFT)
221                | (local_task_id << RAW_VC_LOCAL_TASK_SHIFT),
222        )
223    }
224
225    #[inline]
226    fn from_bits(bits: u64) -> Self {
227        // SAFETY: every constructor produces a non-zero word — the task variants
228        // carry a `TaskId >= 1` in the low bits, and `LocalOutput` always sets
229        // `RAW_VC_LOCAL_FLAG`.
230        RawVc(unsafe { NonZeroU64::new_unchecked(bits) })
231    }
232
233    #[inline]
234    pub(crate) fn bits(self) -> u64 {
235        self.0.get()
236    }
237
238    /// The high 32 bits — the `CellId` slot. Zero for `TaskOutput`, the non-zero
239    /// packed `CellId` for `TaskCell`, and the `LocalTaskId` for `LocalOutput`.
240    #[inline]
241    fn cell_word(self) -> u32 {
242        (self.bits() >> RAW_VC_CELL_SHIFT) as u32
243    }
244
245    /// True for `TaskCell`: a task variant (bit 31 clear) whose cell field is
246    /// non-zero.
247    #[inline]
248    fn is_task_cell(self) -> bool {
249        !self.is_local_output() && self.cell_word() != 0
250    }
251
252    /// True for `TaskOutput`: a task variant (bit 31 clear) whose cell field is
253    /// zero.
254    #[inline]
255    fn is_task_output(self) -> bool {
256        !self.is_local_output() && self.cell_word() == 0
257    }
258
259    /// Reads the `TaskId` from a `TaskOutput` / `TaskCell` word.
260    ///
261    /// Produces a garbage value if this is a `LocalOutput` word
262    #[inline]
263    fn read_task_id(self) -> TaskId {
264        let id = (self.bits() & RAW_VC_TASK_MASK) as u32;
265        // SAFETY: a non-zero `TaskId` was packed in by construction.
266        unsafe { TaskId::new_unchecked(id) }
267    }
268
269    /// Reads the [`CellId`] from a `TaskCell` word.
270    ///
271    /// Produces a garbage value if this is a `LocalOutput` or `TaskOutput` word
272    #[inline]
273    fn read_cell(self) -> CellId {
274        // SAFETY: a valid packed `CellId` was stored in the high 32 bits.
275        unsafe { CellId::from_raw(self.cell_word()) }
276    }
277
278    /// Unpacks into the logical [`RawVcUnpacked`] enum for matching.
279    pub fn unpack(self) -> RawVcUnpacked {
280        if self.is_local_output() {
281            let (execution_id, local_task_id, persistence) = self.decode_local_output();
282            RawVcUnpacked::LocalOutput(execution_id, local_task_id, persistence)
283        } else {
284            let task_id = self.read_task_id();
285            let cell_word = self.cell_word();
286            if cell_word != 0 {
287                RawVcUnpacked::TaskCell(task_id, unsafe { CellId::from_raw(self.cell_word()) })
288            } else {
289                RawVcUnpacked::TaskOutput(task_id)
290            }
291        }
292    }
293
294    /// Returns the [`TaskId`] if this is a `TaskOutput`, otherwise `None`.
295    ///
296    /// Prefer this over [`unpack`][Self::unpack] when a caller only cares about
297    /// the `TaskOutput` case: it reads just the discriminator and the task bits.
298    pub fn as_task_output(self) -> Option<TaskId> {
299        self.is_task_output().then(|| self.read_task_id())
300    }
301
302    /// Returns the `(TaskId, CellId)` pair if this is a `TaskCell`, otherwise
303    /// `None`.
304    ///
305    /// Prefer this over [`unpack`][Self::unpack] when a caller only cares about
306    /// the `TaskCell` case: it reads just the discriminator, the task bits, and
307    /// the cell bits.
308    pub fn as_task_cell(self) -> Option<(TaskId, CellId)> {
309        self.is_task_cell()
310            .then(|| (self.read_task_id(), self.read_cell()))
311    }
312
313    /// Returns the `(ExecutionId, LocalTaskId, TaskPersistence)` triple if this
314    /// is a `LocalOutput`, otherwise `None`.
315    ///
316    /// Prefer this over [`unpack`][Self::unpack] when a caller only cares about
317    /// the `LocalOutput` case.
318    pub fn as_local_output(self) -> Option<(ExecutionId, LocalTaskId, TaskPersistence)> {
319        self.is_local_output().then(|| self.decode_local_output())
320    }
321
322    /// Decodes the fields of a `LocalOutput` word. Only valid when
323    /// [`RAW_VC_LOCAL_FLAG`] is set.
324    fn decode_local_output(self) -> (ExecutionId, LocalTaskId, TaskPersistence) {
325        let bits = self.bits();
326        let persistence = if (bits >> RAW_VC_LOCAL_TRANSIENT_SHIFT) & 1 == 1 {
327            TaskPersistence::Transient
328        } else {
329            TaskPersistence::Persistent
330        };
331        let execution_id = ((bits >> RAW_VC_LOCAL_EXECUTION_SHIFT) & 0xFFFF) as u16;
332        let local_task_id = ((bits >> RAW_VC_LOCAL_TASK_SHIFT) & 0xFFFF_FFFF) as u32;
333        // SAFETY: non-zero `ExecutionId`/`LocalTaskId` were packed in.
334        (
335            unsafe { ExecutionId::new_unchecked(execution_id) },
336            unsafe { LocalTaskId::new_unchecked(local_task_id) },
337            persistence,
338        )
339    }
340}
341
342impl Debug for RawVc {
343    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
344        match self.unpack() {
345            RawVcUnpacked::TaskOutput(task_id) => {
346                f.debug_tuple("RawVc::TaskOutput").field(&*task_id).finish()
347            }
348            RawVcUnpacked::TaskCell(task_id, cell_id) => f
349                .debug_tuple("RawVc::TaskCell")
350                .field(&*task_id)
351                .field(&cell_id.to_string())
352                .finish(),
353            RawVcUnpacked::LocalOutput(execution_id, local_task_id, task_persistence) => f
354                .debug_tuple("RawVc::LocalOutput")
355                .field(&*execution_id)
356                .field(&*local_task_id)
357                .field(&task_persistence)
358                .finish(),
359        }
360    }
361}
362
363impl Display for RawVc {
364    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
365        match self.unpack() {
366            RawVcUnpacked::TaskOutput(task_id) => write!(f, "output of task {}", *task_id),
367            RawVcUnpacked::TaskCell(task_id, cell_id) => {
368                write!(f, "{} of task {}", cell_id, *task_id)
369            }
370            RawVcUnpacked::LocalOutput(execution_id, local_task_id, task_persistence) => write!(
371                f,
372                "output of local task {} ({}, {})",
373                *local_task_id, *execution_id, task_persistence
374            ),
375        }
376    }
377}
378
379impl RawVc {
380    pub fn is_resolved(&self) -> bool {
381        self.is_task_cell()
382    }
383
384    pub fn is_local_output(&self) -> bool {
385        self.bits() & RAW_VC_LOCAL_FLAG != 0
386    }
387
388    /// Returns `true` if the task this `RawVc` reads from cannot be serialized and will not be
389    /// stored in the filesystem cache.
390    ///
391    /// See [`TaskPersistence`] for more details.
392    pub fn is_transient(&self) -> bool {
393        if self.is_local_output() {
394            // LocalOutput: the transient flag is stored as a bit.
395            (self.bits() >> RAW_VC_LOCAL_TRANSIENT_SHIFT) & 1 == 1
396        } else {
397            // TaskOutput / TaskCell: transience is a property of the TaskId value.
398            self.read_task_id().is_transient()
399        }
400    }
401
402    pub(crate) fn into_read(self) -> ReadRawVcFuture {
403        // returns a custom future to have something concrete and sized
404        // this avoids boxing in IntoFuture
405        ReadRawVcFuture::new(self)
406    }
407
408    /// See [`crate::Vc::to_resolved`].
409    pub(crate) fn resolve(self) -> ResolveRawVcFuture {
410        ResolveRawVcFuture::new(self)
411    }
412
413    /// Convert a potentially local `RawVc` into a non-local `RawVc`. This is a subset of resolution
414    /// resolution, because the returned `RawVc` can be a `TaskOutput`.
415    pub async fn to_non_local(self) -> Result<RawVc> {
416        let Some((execution_id, local_task_id, ..)) = self.as_local_output() else {
417            return Ok(self);
418        };
419        let tt = turbo_tasks();
420        let local_output = read_local_output(&*tt, execution_id, local_task_id).await?;
421        debug_assert!(
422            !local_output.is_local_output(),
423            "a LocalOutput cannot point at other LocalOutputs"
424        );
425        Ok(local_output)
426    }
427
428    /// Convert a potentially local `RawVc` into a non-local `RawVc`. This is a subset of resolution
429    /// resolution, because the returned `RawVc` can be a `TaskOutput`.
430    ///
431    /// 'unchecked' because the caller must have already confirmed that the local tasks were already
432    /// completed
433    pub(crate) fn to_non_local_unchecked_sync(self, tt: &dyn TurboTasksApi) -> Result<RawVc> {
434        let Some((execution_id, local_task_id, ..)) = self.as_local_output() else {
435            return Ok(self);
436        };
437        let local_output = match tt.try_read_local_output(execution_id, local_task_id)? {
438            Ok(raw_vc) => raw_vc,
439            Err(_event_listener) => unreachable!("local output is not ready yet"),
440        };
441        debug_assert!(
442            !local_output.is_local_output(),
443            "a LocalOutput cannot point at other LocalOutputs"
444        );
445        Ok(local_output)
446    }
447
448    pub fn try_get_task_id(&self) -> Option<TaskId> {
449        (!self.is_local_output()).then(|| self.read_task_id())
450    }
451
452    pub fn try_get_type_id(&self) -> Option<ValueTypeId> {
453        self.is_task_cell().then(|| self.read_cell().type_id())
454    }
455
456    /// For a cell that's already resolved, synchronously check if it implements a trait using the
457    /// type information in `RawVc::TaskCell` (we don't actually need to read the cell!).
458    pub(crate) fn resolved_has_trait(&self, trait_id: TraitTypeId) -> bool {
459        debug_assert!(
460            self.is_task_cell(),
461            "resolved_has_trait must be called with a RawVc::TaskCell"
462        );
463        get_value_type(self.read_cell().type_id()).has_trait(&trait_id)
464    }
465
466    /// For a cell that's already resolved, synchronously check if it is a given type using the type
467    /// information in `RawVc::TaskCell` (we don't actually need to read the cell!).
468    pub(crate) fn resolved_is_type(&self, type_id: ValueTypeId) -> bool {
469        debug_assert!(
470            self.is_task_cell(),
471            "resolved_is_type must be called with a RawVc::TaskCell"
472        );
473        self.read_cell().type_id() == type_id
474    }
475}
476
477/// Polls a pending [`EventListener`] slot. Returns [`Poll::Pending`] if the event has not yet
478/// fired. On [`Poll::Ready`], clears the slot so it is not polled again.
479fn poll_listener(
480    listener: &mut Option<EventListener>,
481    cx: &mut std::task::Context<'_>,
482) -> Poll<()> {
483    if let Some(l) = listener {
484        ready!(Pin::new(l).poll(cx));
485        *listener = None;
486    }
487    Poll::Ready(())
488}
489
490/// Wraps `f` in a scope that suppresses the eventual-consistency top-level task assertion,
491/// but only when `strongly_consistent` is `true` and debug assertions are enabled.
492///
493/// This is needed because a strongly-consistent read of a `TaskOutput` is not a single atomic
494/// operation — inner reads switch to eventual consistency after the first output is resolved —
495/// which would otherwise trigger the assertion in top-level tasks.
496fn suppress_top_level_task_check<R>(strongly_consistent: bool, f: impl FnOnce() -> R) -> R {
497    if cfg!(debug_assertions) && strongly_consistent {
498        // Temporarily suppress the top-level task check
499        SUPPRESS_EVENTUAL_CONSISTENCY_TOP_LEVEL_TASK_CHECK.sync_scope(true, f)
500    } else {
501        f()
502    }
503}
504
505#[must_use]
506pub struct ResolveRawVcFuture {
507    current: RawVc,
508    read_output_options: ReadOutputOptions,
509    /// This flag is redundant with `read_output_options`, but `read_output_options` is mutated
510    /// during the resolve. This flag indicates that the initial read was strongly consistent.
511    strongly_consistent: bool,
512    listener: Option<EventListener>,
513}
514
515impl ResolveRawVcFuture {
516    fn new(vc: RawVc) -> Self {
517        ResolveRawVcFuture {
518            current: vc,
519            read_output_options: ReadOutputOptions::default(),
520            strongly_consistent: false,
521            listener: None,
522        }
523    }
524
525    pub fn strongly_consistent(mut self) -> Self {
526        self.strongly_consistent = true;
527        self.read_output_options.consistency = ReadConsistency::Strong;
528        self
529    }
530
531    /// Track task output reads with a specific key (forwarded from
532    /// [`ReadRawVcFuture::track_with_key`]).
533    pub(crate) fn track_with_key(mut self) -> Self {
534        self.read_output_options.tracking = ReadTracking::Tracked;
535        self
536    }
537
538    /// Do not track task output reads as dependencies (forwarded from
539    /// [`ReadRawVcFuture::untracked`]).
540    pub(crate) fn untracked(mut self) -> Self {
541        self.read_output_options.tracking = ReadTracking::TrackOnlyError;
542        self
543    }
544}
545
546impl Future for ResolveRawVcFuture {
547    type Output = Result<RawVc>;
548
549    #[inline(never)]
550    fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll<Self::Output> {
551        // SAFETY: we are not moving self
552        let this = unsafe { self.get_unchecked_mut() };
553
554        let poll_fn = |tt: &Arc<dyn TurboTasksApi>| -> Poll<Self::Output> {
555            'outer: loop {
556                ready!(poll_listener(&mut this.listener, cx));
557                let listener = match this.current.unpack() {
558                    RawVcUnpacked::TaskOutput(task) => {
559                        let read_result = tt.try_read_task_output(task, this.read_output_options);
560                        match read_result {
561                            Ok(Ok(vc)) => {
562                                // turbo-tasks-backend doesn't currently have any sort of
563                                // "transaction" or global lock mechanism to group together chains
564                                // of `TaskOutput`/`TaskCell` reads.
565                                //
566                                // If we ignore the theoretical TOCTOU issues, we no longer need to
567                                // read strongly consistent, as any Vc returned from the first task
568                                // will be inside of the scope of the first task. So it's already
569                                // strongly consistent.
570                                this.read_output_options.consistency = ReadConsistency::Eventual;
571                                this.current = vc;
572                                continue 'outer;
573                            }
574                            Ok(Err(listener)) => listener,
575                            Err(err) => return Poll::Ready(Err(err)),
576                        }
577                    }
578                    RawVcUnpacked::TaskCell(_, _) => return Poll::Ready(Ok(this.current)),
579                    RawVcUnpacked::LocalOutput(execution_id, local_task_id, ..) => {
580                        debug_assert_eq!(
581                            this.read_output_options.consistency,
582                            ReadConsistency::Eventual
583                        );
584                        let read_result = tt.try_read_local_output(execution_id, local_task_id);
585                        match read_result {
586                            Ok(Ok(vc)) => {
587                                this.current = vc;
588                                continue 'outer;
589                            }
590                            Ok(Err(listener)) => listener,
591                            Err(err) => return Poll::Ready(Err(err)),
592                        }
593                    }
594                };
595                this.listener = Some(listener);
596            }
597        };
598
599        // HACK: Temporarily suppress top-level task check if doing strongly consistent read.
600        //
601        // This masks a bug: There's an unlikely TOCTOU race condition in `poll_fn`. Because the
602        // strongly consistent read isn't a single atomic operation, any inner `TaskOutput` or
603        // `TaskCell` could get mutated after the strongly consistent read of the outer
604        // `TaskOutput`.
605        suppress_top_level_task_check(this.strongly_consistent, || with_turbo_tasks(poll_fn))
606    }
607}
608
609impl Unpin for ResolveRawVcFuture {}
610
611#[must_use]
612pub struct ReadRawVcFuture {
613    read_cell_options: ReadCellOptions,
614    state: ReadRawVcState,
615}
616
617/// Phase 1 and phase 2 of [`ReadRawVcFuture`] use disjoint sets of fields. Storing them in an
618/// enum keeps the future smaller than holding both sets simultaneously.
619enum ReadRawVcState {
620    /// Phase 1: resolves the [`RawVc`] pointer chain to a [`RawVc::TaskCell`].
621    Resolving(ResolveRawVcFuture),
622    /// Phase 2: the resolved task/cell identity plus a listener for the cell read wait.
623    Reading {
624        task: TaskId,
625        index: CellId,
626        /// Whether phase 1 was a strongly-consistent read. Needed here to re-apply
627        /// [`suppress_top_level_task_check`] in phase 2. Lives in this variant (rather than the
628        /// outer struct) so it can share padding with the other `Reading` fields — keeping
629        /// `Reading` no larger than `Resolving`, and the whole future 8 bytes smaller.
630        strongly_consistent: bool,
631        listener: Option<EventListener>,
632    },
633}
634
635impl ReadRawVcFuture {
636    pub(crate) fn new(vc: RawVc) -> Self {
637        ReadRawVcFuture {
638            read_cell_options: ReadCellOptions::default(),
639            state: ReadRawVcState::Resolving(ResolveRawVcFuture::new(vc)),
640        }
641    }
642
643    fn map_resolve(mut self, f: impl FnOnce(ResolveRawVcFuture) -> ResolveRawVcFuture) -> Self {
644        match self.state {
645            ReadRawVcState::Resolving(resolve) => {
646                self.state = ReadRawVcState::Resolving(f(resolve));
647            }
648            ReadRawVcState::Reading { .. } => {
649                unreachable!("builder methods are only called before polling");
650            }
651        }
652        self
653    }
654
655    /// Make reads strongly consistent.
656    pub fn strongly_consistent(self) -> Self {
657        self.map_resolve(|r| r.strongly_consistent())
658    }
659
660    /// Track the value as a dependency with an key.
661    pub fn track_with_key(mut self, key: u64) -> Self {
662        self.read_cell_options.tracking = ReadCellTracking::Tracked { key: Some(key) };
663        self.map_resolve(|r| r.track_with_key())
664    }
665
666    /// This will not track the value as dependency, but will still track the error as dependency,
667    /// if there is an error.
668    ///
669    /// INVALIDATION: Be careful with this, it will not track dependencies, so
670    /// using it could break cache invalidation.
671    pub fn untracked(mut self) -> Self {
672        self.read_cell_options.tracking = ReadCellTracking::TrackOnlyError;
673        self.map_resolve(|r| r.untracked())
674    }
675
676    /// Hint that this is the final read of the cell content.
677    pub fn final_read_hint(mut self) -> Self {
678        self.read_cell_options.final_read_hint = true;
679        self
680    }
681}
682
683impl Future for ReadRawVcFuture {
684    type Output = Result<TypedCellContent>;
685
686    #[inline(never)]
687    fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll<Self::Output> {
688        // SAFETY: we are not moving self
689        let this = unsafe { self.get_unchecked_mut() };
690
691        // --- Phase 1: resolve the RawVc pointer chain to a TaskCell ---
692        //
693        // `ResolveRawVcFuture` is `Unpin`, so `Pin::new` is safe.
694        // It handles `with_turbo_tasks` and `suppress_top_level_task_check` internally.
695        if let ReadRawVcState::Resolving(resolve) = &mut this.state {
696            let strongly_consistent = resolve.strongly_consistent;
697            match ready!(Pin::new(resolve).poll(cx)) {
698                Err(err) => return Poll::Ready(Err(err)),
699                Ok(resolved) => {
700                    let Some((task, index)) = resolved.as_task_cell() else {
701                        unreachable!("ResolveRawVcFuture always resolves to a TaskCell")
702                    };
703                    this.state = ReadRawVcState::Reading {
704                        task,
705                        index,
706                        strongly_consistent,
707                        listener: None,
708                    };
709                }
710            }
711        }
712
713        // --- Phase 2: read the cell content ---
714        let ReadRawVcState::Reading {
715            task,
716            index,
717            strongly_consistent,
718            listener,
719        } = &mut this.state
720        else {
721            unreachable!("phase 1 transitioned to Reading above");
722        };
723        let task = *task;
724        let index = *index;
725        let read_cell_options = this.read_cell_options;
726
727        let poll_fn = |tt: &Arc<dyn TurboTasksApi>| -> Poll<Self::Output> {
728            loop {
729                ready!(poll_listener(listener, cx));
730                let new_listener = match tt.try_read_task_cell(task, index, read_cell_options) {
731                    Ok(Ok(content)) => return Poll::Ready(Ok(content)),
732                    Ok(Err(l)) => l,
733                    Err(err) => return Poll::Ready(Err(err)),
734                };
735                *listener = Some(new_listener);
736            }
737        };
738
739        // Phase 2 must also suppress the top-level task check when phase 1 was
740        // strongly-consistent. The suppression from `ResolveRawVcFuture::poll` only lasts for
741        // the duration of that individual `poll` call and does not carry over to subsequent calls
742        // or to this phase.
743        suppress_top_level_task_check(*strongly_consistent, || with_turbo_tasks(poll_fn))
744    }
745}
746
747impl Unpin for ReadRawVcFuture {}
748
749#[cfg(test)]
750mod tests {
751    use super::*;
752
753    /// `CellId` must pack into 4 bytes and keep its niche so `Option<CellId>`
754    /// stays 4 bytes — this is the whole point of [`RawVc`] shrinking.
755    #[test]
756    fn cell_id_is_four_bytes() {
757        assert_eq!(size_of::<CellId>(), 4);
758        assert_eq!(size_of::<Option<CellId>>(), 4);
759    }
760
761    /// Packing and unpacking a `(type_id, index)` pair must round-trip across
762    /// the full range of both fields, including the boundary values.
763    #[test]
764    fn cell_id_pack_unpack_round_trip() {
765        let type_ids = [1u16, 2, 100, CellId::MAX_VALUE_TYPE_ID];
766        let indices = [0u32, 1, 12345, CellId::MAX_CELL_INDEX];
767        for &raw_ty in &type_ids {
768            // SAFETY: all test values are >= 1.
769            let type_id = unsafe { ValueTypeId::new_unchecked(raw_ty) };
770            for &index in &indices {
771                let cell = CellId::new(type_id, index);
772                assert_eq!(cell.type_id(), type_id, "type_id round-trip for {raw_ty}");
773                assert_eq!(cell.index(), index, "index round-trip for {index}");
774            }
775        }
776    }
777
778    /// Distinct `(type_id, index)` pairs must pack to distinct words — the
779    /// packing is a bijection, which is what lets us derive `Eq`/`Hash`.
780    #[test]
781    fn cell_id_packing_is_bijective() {
782        // SAFETY: ids are >= 1.
783        let a = CellId::new(unsafe { ValueTypeId::new_unchecked(1) }, 0);
784        let b = CellId::new(unsafe { ValueTypeId::new_unchecked(1) }, 1);
785        let c = CellId::new(unsafe { ValueTypeId::new_unchecked(2) }, 0);
786        assert_ne!(a, b);
787        assert_ne!(a, c);
788        assert_ne!(b, c);
789    }
790
791    /// `RawVc` must pack into 8 bytes and keep its niche.
792    #[test]
793    fn raw_vc_is_eight_bytes() {
794        assert_eq!(size_of::<RawVc>(), 8);
795        assert_eq!(size_of::<Option<RawVc>>(), 8);
796    }
797
798    /// Every variant must round-trip through pack → `unpack()` across the full
799    /// range of each packed field, including boundary values and both
800    /// persistence states. This is the core correctness property of the
801    /// bit-packing.
802    #[test]
803    fn raw_vc_pack_unpack_round_trip() {
804        // SAFETY: all ids below are >= 1 and within their bit budgets.
805        let tasks = [
806            1u32,
807            2,
808            crate::TRANSIENT_TASK_BIT - 1,
809            crate::TRANSIENT_TASK_BIT,
810            TASK_ID_MAX,
811        ];
812        for &t in &tasks {
813            let task = unsafe { TaskId::new_unchecked(t) };
814
815            // TaskOutput
816            let vc = RawVc::task_output(task);
817            assert_eq!(vc.unpack(), RawVcUnpacked::TaskOutput(task));
818            assert!(!vc.is_resolved() && !vc.is_local_output());
819            assert_eq!(vc.is_transient(), task.is_transient());
820            assert_eq!(vc.try_get_task_id(), Some(task));
821            // single-arm accessors
822            assert_eq!(vc.as_task_output(), Some(task));
823            assert_eq!(vc.as_task_cell(), None);
824            assert_eq!(vc.as_local_output(), None);
825
826            // TaskCell, across CellId boundaries
827            for cell in [
828                CellId::new(unsafe { ValueTypeId::new_unchecked(1) }, 0),
829                CellId::new(
830                    unsafe { ValueTypeId::new_unchecked(CellId::MAX_VALUE_TYPE_ID) },
831                    CellId::MAX_CELL_INDEX,
832                ),
833            ] {
834                let vc = RawVc::task_cell(task, cell);
835                assert_eq!(vc.unpack(), RawVcUnpacked::TaskCell(task, cell));
836                assert!(vc.is_resolved());
837                assert_eq!(vc.try_get_task_id(), Some(task));
838                assert_eq!(vc.try_get_type_id(), Some(cell.type_id()));
839                // single-arm accessors
840                assert_eq!(vc.as_task_cell(), Some((task, cell)));
841                assert_eq!(vc.as_task_output(), None);
842                assert_eq!(vc.as_local_output(), None);
843            }
844        }
845
846        // LocalOutput, both persistence states and boundary ids
847        for persistence in [TaskPersistence::Persistent, TaskPersistence::Transient] {
848            for (e, l) in [(1u16, 1u32), (u16::MAX, u32::MAX)] {
849                let exec = unsafe { ExecutionId::new_unchecked(e) };
850                let local = unsafe { LocalTaskId::new_unchecked(l) };
851                let vc = RawVc::local_output(exec, local, persistence);
852                assert_eq!(
853                    vc.unpack(),
854                    RawVcUnpacked::LocalOutput(exec, local, persistence)
855                );
856                assert!(vc.is_local_output());
857                assert_eq!(vc.is_transient(), persistence == TaskPersistence::Transient);
858                assert_eq!(vc.try_get_task_id(), None);
859                // single-arm accessors
860                assert_eq!(vc.as_local_output(), Some((exec, local, persistence)));
861                assert_eq!(vc.as_task_output(), None);
862                assert_eq!(vc.as_task_cell(), None);
863            }
864        }
865    }
866
867    /// The discriminator relies on the cell field being zero for `TaskOutput`
868    /// and non-zero for `TaskCell`. A `TaskOutput` and a `TaskCell` that share
869    /// the same `TaskId` must still be told apart, and a `LocalOutput` whose
870    /// `LocalTaskId` populates the high bits (the cell-field region) must remain
871    /// a `LocalOutput` because bit 31 wins.
872    #[test]
873    fn raw_vc_discriminator_is_unambiguous() {
874        // SAFETY: all ids are >= 1.
875        let task = unsafe { TaskId::new_unchecked(123) };
876        let cell = CellId::new(unsafe { ValueTypeId::new_unchecked(1) }, 0);
877
878        let output = RawVc::task_output(task);
879        let task_cell = RawVc::task_cell(task, cell);
880        assert!(output.is_task_output() && !output.is_task_cell() && !output.is_local_output());
881        assert!(
882            task_cell.is_task_cell() && !task_cell.is_task_output() && !task_cell.is_local_output()
883        );
884        // Same TaskId, different variants, distinct words.
885        assert_ne!(output, task_cell);
886        assert_eq!(output.read_task_id(), task_cell.read_task_id());
887
888        // A LocalOutput with a max LocalTaskId fills the high 32 bits; it must
889        // not be misread as a TaskCell.
890        let local = RawVc::local_output(
891            unsafe { ExecutionId::new_unchecked(u16::MAX) },
892            unsafe { LocalTaskId::new_unchecked(u32::MAX) },
893            TaskPersistence::Persistent,
894        );
895        assert!(local.is_local_output() && !local.is_task_cell() && !local.is_task_output());
896    }
897
898    #[test]
899    #[cfg(debug_assertions)]
900    #[should_panic(expected = "TaskId exceeds 31 bits")]
901    fn task_output_panics_on_out_of_range_task_id() {
902        // `TASK_ID_MAX + 1` is the first value that sets bit 31.
903        // SAFETY: non-zero.
904        let task = unsafe { TaskId::new_unchecked(TASK_ID_MAX + 1) };
905        let _ = RawVc::task_output(task);
906    }
907
908    #[test]
909    #[cfg(debug_assertions)]
910    #[should_panic(expected = "TaskId exceeds 31 bits")]
911    fn task_cell_panics_on_out_of_range_task_id() {
912        // SAFETY: non-zero.
913        let task = unsafe { TaskId::new_unchecked(TASK_ID_MAX + 1) };
914        let cell = CellId::new(unsafe { ValueTypeId::new_unchecked(1) }, 0);
915        let _ = RawVc::task_cell(task, cell);
916    }
917
918    #[test]
919    #[cfg(debug_assertions)]
920    #[should_panic(expected = "exceeds")]
921    fn cell_id_panics_on_out_of_range_type_id() {
922        // SAFETY: `MAX_VALUE_TYPE_ID + 1` is non-zero.
923        let type_id = unsafe { ValueTypeId::new_unchecked(CellId::MAX_VALUE_TYPE_ID + 1) };
924        let _ = CellId::new(type_id, 0);
925    }
926    #[test]
927    #[cfg(debug_assertions)]
928    #[should_panic(expected = "exceeds")]
929    fn cell_id_panics_on_out_of_range_index() {
930        let type_id = unsafe { ValueTypeId::new_unchecked(1) };
931        let _ = CellId::new(type_id, CellId::MAX_CELL_INDEX + 1);
932    }
933}
turbo_tasks/vc/raw.rs

turbo_tasks/vc/
raw.rs
