turbo_tasks/vc/
raw.rs

1use std::{
2    fmt::{Debug, Display},
3    future::Future,
4    num::{NonZeroU32, NonZeroU64},
5    pin::Pin,
6    sync::Arc,
7    task::{Poll, ready},
8};
9
10use anyhow::Result;
11use bincode::{Decode, Encode};
12use serde::{Deserialize, Serialize};
13
14use crate::{
15    ReadCellOptions, ReadConsistency, ReadOutputOptions, TaskId, TaskPersistence, TraitTypeId,
16    ValueTypeId,
17    backend::TypedCellContent,
18    event::EventListener,
19    id::{ExecutionId, LocalTaskId, TASK_ID_MAX},
20    manager::{
21        ReadCellTracking, ReadTracking, SUPPRESS_EVENTUAL_CONSISTENCY_TOP_LEVEL_TASK_CHECK,
22        ScheduleKey, TurboTasksApi, execute_read_target_inline, read_local_output,
23        with_turbo_tasks,
24    },
25    read_options::ReadOutcome,
26    registry::get_value_type,
27    turbo_tasks,
28};
29
30/// Identifies a specific cell within a task: a [`ValueTypeId`] paired with a
31/// sequentially-allocated index within that type.
32///
33/// Packed into a single [`NonZeroU32`]:
34/// ```text
35/// bits 31..=22 (10 bits): ValueTypeId logical value
36/// bits 21..=0  (22 bits): cell index
37/// ```
38/// Because the `ValueTypeId` is always `>= 1`, the type is trivially non-zero
39#[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Encode, Decode)]
40pub struct CellId(NonZeroU32);
41
42/// Number of low bits used for the cell index.
43const CELL_INDEX_BITS: u32 = 22;
44/// Mask selecting the cell index bits.
45const CELL_INDEX_MASK: u32 = (1 << CELL_INDEX_BITS) - 1;
46
47impl CellId {
48    /// Maximum `ValueTypeId` logical value that fits in the 10-bit type field.
49    pub const MAX_VALUE_TYPE_ID: u16 = (1 << (u32::BITS - CELL_INDEX_BITS)) as u16 - 1;
50    /// Maximum cell index that fits in the 22-bit index field.
51    pub const MAX_CELL_INDEX: u32 = CELL_INDEX_MASK;
52
53    /// Packs a `type_id` and `index` into a single word.
54    pub fn new(type_id: ValueTypeId, index: u32) -> Self {
55        let type_id = *type_id;
56        debug_assert!(
57            type_id <= Self::MAX_VALUE_TYPE_ID,
58            "ValueTypeId {} exceeds the {} cap packed into CellId",
59            type_id,
60            Self::MAX_VALUE_TYPE_ID,
61        );
62        debug_assert!(
63            index <= Self::MAX_CELL_INDEX,
64            "cell index {} exceeds the {} cap packed into CellId",
65            index,
66            Self::MAX_CELL_INDEX,
67        );
68        let packed = ((type_id as u32) << CELL_INDEX_BITS) | (index & CELL_INDEX_MASK);
69        // SAFETY: `type_id >= 1`, so `packed >= (1 << CELL_INDEX_BITS) > 0`.
70        CellId(unsafe { NonZeroU32::new_unchecked(packed) })
71    }
72
73    pub fn type_id(self) -> ValueTypeId {
74        let type_id = (self.0.get() >> CELL_INDEX_BITS) as u16;
75        // SAFETY: the high bits always hold a `ValueTypeId` of `1..=1023` by construction.
76        unsafe { ValueTypeId::new_unchecked(type_id) }
77    }
78
79    pub fn index(self) -> u32 {
80        self.0.get() & CELL_INDEX_MASK
81    }
82
83    /// The raw packed word, used by [`RawVc`] to pack a `TaskCell` into its u64.
84    pub(crate) fn raw(self) -> u32 {
85        self.0.get()
86    }
87
88    /// Reconstructs a `CellId` from a raw packed word produced by [`Self::raw`].
89    ///
90    /// # Safety
91    ///
92    /// `raw` must be a value previously returned by [`Self::raw`] (in
93    /// particular, non-zero with a valid 10-bit type id in the high bits).
94    pub(crate) unsafe fn from_raw(raw: u32) -> Self {
95        debug_assert!(raw != 0);
96        // SAFETY: the caller guarantees `raw` came from a valid `CellId`.
97        CellId(unsafe { NonZeroU32::new_unchecked(raw) })
98    }
99}
100
101impl Debug for CellId {
102    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
103        f.debug_struct("CellId")
104            .field("type_id", &self.type_id())
105            .field("index", &self.index())
106            .finish()
107    }
108}
109
110impl Display for CellId {
111    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
112        write!(
113            f,
114            "{}#{}",
115            get_value_type(self.type_id()).ty.name,
116            self.index()
117        )
118    }
119}
120
121/// A type-erased representation of [`Vc`].
122///
123/// Type erasure reduces the [monomorphization] (and therefore binary size and compilation time)
124/// required to support [`Vc`].
125///
126/// This type is heavily used within the [`Backend`][crate::backend::Backend] trait, but should
127/// otherwise be treated as an internal implementation detail of `turbo-tasks`.
128///
129/// # Representation
130///
131/// `RawVc` is one of three logical variants (see [`RawVcUnpacked`]) bit-packed
132/// into a single [`NonZeroU64`].
133///
134/// Bit 31 is the discriminator between a local output and a task variant; the
135/// two task variants are then told apart by whether the [`CellId`] field (the
136/// high 32 bits) is zero — which is unambiguous because every `CellId` is
137/// non-zero.
138///
139/// ```text
140/// bit31 = 1                  LocalOutput: 1<<31 | transient(1) | ExecutionId(16) << 1 | LocalTaskId(32) << 32
141/// bit31 = 0, bits32..63 == 0 TaskOutput:  TaskId(31)
142/// bit31 = 0, bits32..63 != 0 TaskCell:    TaskId(31) | CellId(32) << 32
143/// ```
144/// [`Vc`]: crate::Vc
145/// [monomorphization]: https://doc.rust-lang.org/book/ch10-01-syntax.html#performance-of-code-using-generics
146#[derive(Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize, Encode, Decode)]
147pub struct RawVc(NonZeroU64);
148
149/// The unpacked form of [`RawVc`], produced by [`RawVc::unpack`].
150#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
151pub enum RawVcUnpacked {
152    /// The synchronous return value of a task (after argument resolution). This is the
153    /// representation used by [`OperationVc`][crate::OperationVc].
154    TaskOutput(TaskId),
155    /// A pointer to a specific [`Vc::cell`][crate::Vc::cell] or `.cell()` call within a task. This
156    /// is the representation used by [`ResolvedVc`].
157    ///
158    /// [`CellId`] contains the [`ValueTypeId`], which can be useful for efficient downcasting.
159    TaskCell(TaskId, CellId),
160    /// The synchronous return value of a local task. This is created when a function is called
161    /// with unresolved arguments or more explicitly with
162    /// [`#[turbo_tasks::function(local)]`][crate::function].
163    ///
164    /// Local outputs are only valid within the context of their parent "non-local" task. Turbo
165    /// Task's APIs are designed to prevent escapes of local [`Vc`]s, but [`ExecutionId`] is used
166    /// for a fallback runtime assertion.
167    ///
168    /// [`Vc`]: crate::Vc
169    LocalOutput(ExecutionId, LocalTaskId, TaskPersistence),
170}
171
172/// Bit 31 discriminates `LocalOutput` (set) from the task variants (clear).
173/// It is free for the task variants because a `TaskId` is only 31 bits
174/// (`bits 0..=30`) and the `CellId` lives in the high 32 bits (`32..=63`).
175const RAW_VC_LOCAL_FLAG: u64 = 1 << 31;
176
177/// Mask of the `TaskId` value inside `TaskOutput` / `TaskCell` (bits `0..=30`).
178///
179/// This equals `TASK_ID_MAX` because a `TaskId` is `2^31 - 1`, so its max value
180/// is also the mask of its bits. TaskId is a u32 so this cast is safe.
181const RAW_VC_TASK_MASK: u64 = TASK_ID_MAX as u64;
182/// Shift of the packed `CellId` word inside `TaskCell`. A zero high word means
183/// `TaskOutput`; a non-zero one means `TaskCell`.
184const RAW_VC_CELL_SHIFT: u64 = 32;
185
186/// `LocalOutput` field layout (the `RAW_VC_LOCAL_FLAG` bit is always set).
187const RAW_VC_LOCAL_TRANSIENT_SHIFT: u64 = 0;
188const RAW_VC_LOCAL_EXECUTION_SHIFT: u64 = 1;
189const RAW_VC_LOCAL_TASK_SHIFT: u64 = 32;
190
191impl RawVc {
192    /// Packs the synchronous return value of a task. The word is simply the
193    /// `TaskId` value: bit 31 clear (a task variant) and the cell field zero
194    /// (no cell).
195    pub fn task_output(task: TaskId) -> Self {
196        let task = *task as u64;
197        debug_assert!(task <= RAW_VC_TASK_MASK, "TaskId exceeds 31 bits");
198        Self::from_bits(task)
199    }
200
201    /// Packs a pointer to a specific cell within a task: the `TaskId` in the low
202    /// bits and the non-zero `CellId` in the high 32 bits.
203    pub fn task_cell(task: TaskId, cell: CellId) -> Self {
204        let task = *task as u64;
205        debug_assert!(task <= RAW_VC_TASK_MASK, "TaskId exceeds 31 bits");
206        let cell = cell.raw() as u64;
207        Self::from_bits(task | (cell << RAW_VC_CELL_SHIFT))
208    }
209
210    /// Packs the synchronous return value of a local task, marked by bit 31.
211    pub fn local_output(
212        execution_id: ExecutionId,
213        local_task_id: LocalTaskId,
214        persistence: TaskPersistence,
215    ) -> Self {
216        let transient = (persistence == TaskPersistence::Transient) as u64;
217        let execution_id = *execution_id as u64;
218        let local_task_id = *local_task_id as u64;
219        Self::from_bits(
220            RAW_VC_LOCAL_FLAG
221                | (transient << RAW_VC_LOCAL_TRANSIENT_SHIFT)
222                | (execution_id << RAW_VC_LOCAL_EXECUTION_SHIFT)
223                | (local_task_id << RAW_VC_LOCAL_TASK_SHIFT),
224        )
225    }
226
227    #[inline]
228    fn from_bits(bits: u64) -> Self {
229        // SAFETY: every constructor produces a non-zero word — the task variants
230        // carry a `TaskId >= 1` in the low bits, and `LocalOutput` always sets
231        // `RAW_VC_LOCAL_FLAG`.
232        RawVc(unsafe { NonZeroU64::new_unchecked(bits) })
233    }
234
235    #[inline]
236    pub(crate) fn bits(self) -> u64 {
237        self.0.get()
238    }
239
240    /// The high 32 bits — the `CellId` slot. Zero for `TaskOutput`, the non-zero
241    /// packed `CellId` for `TaskCell`, and the `LocalTaskId` for `LocalOutput`.
242    #[inline]
243    fn cell_word(self) -> u32 {
244        (self.bits() >> RAW_VC_CELL_SHIFT) as u32
245    }
246
247    /// True for `TaskCell`: a task variant (bit 31 clear) whose cell field is
248    /// non-zero.
249    #[inline]
250    fn is_task_cell(self) -> bool {
251        !self.is_local_output() && self.cell_word() != 0
252    }
253
254    /// True for `TaskOutput`: a task variant (bit 31 clear) whose cell field is
255    /// zero.
256    #[inline]
257    fn is_task_output(self) -> bool {
258        !self.is_local_output() && self.cell_word() == 0
259    }
260
261    /// Reads the `TaskId` from a `TaskOutput` / `TaskCell` word.
262    ///
263    /// Produces a garbage value if this is a `LocalOutput` word
264    #[inline]
265    fn read_task_id(self) -> TaskId {
266        let id = (self.bits() & RAW_VC_TASK_MASK) as u32;
267        // SAFETY: a non-zero `TaskId` was packed in by construction.
268        unsafe { TaskId::new_unchecked(id) }
269    }
270
271    /// Reads the [`CellId`] from a `TaskCell` word.
272    ///
273    /// Produces a garbage value if this is a `LocalOutput` or `TaskOutput` word
274    #[inline]
275    fn read_cell(self) -> CellId {
276        // SAFETY: a valid packed `CellId` was stored in the high 32 bits.
277        unsafe { CellId::from_raw(self.cell_word()) }
278    }
279
280    /// Unpacks into the logical [`RawVcUnpacked`] enum for matching.
281    pub fn unpack(self) -> RawVcUnpacked {
282        if self.is_local_output() {
283            let (execution_id, local_task_id, persistence) = self.decode_local_output();
284            RawVcUnpacked::LocalOutput(execution_id, local_task_id, persistence)
285        } else {
286            let task_id = self.read_task_id();
287            let cell_word = self.cell_word();
288            if cell_word != 0 {
289                RawVcUnpacked::TaskCell(task_id, unsafe { CellId::from_raw(self.cell_word()) })
290            } else {
291                RawVcUnpacked::TaskOutput(task_id)
292            }
293        }
294    }
295
296    /// Returns the [`TaskId`] if this is a `TaskOutput`, otherwise `None`.
297    ///
298    /// Prefer this over [`unpack`][Self::unpack] when a caller only cares about
299    /// the `TaskOutput` case: it reads just the discriminator and the task bits.
300    pub fn as_task_output(self) -> Option<TaskId> {
301        self.is_task_output().then(|| self.read_task_id())
302    }
303
304    /// Returns the `(TaskId, CellId)` pair if this is a `TaskCell`, otherwise
305    /// `None`.
306    ///
307    /// Prefer this over [`unpack`][Self::unpack] when a caller only cares about
308    /// the `TaskCell` case: it reads just the discriminator, the task bits, and
309    /// the cell bits.
310    pub fn as_task_cell(self) -> Option<(TaskId, CellId)> {
311        self.is_task_cell()
312            .then(|| (self.read_task_id(), self.read_cell()))
313    }
314
315    /// Returns the `(ExecutionId, LocalTaskId, TaskPersistence)` triple if this
316    /// is a `LocalOutput`, otherwise `None`.
317    ///
318    /// Prefer this over [`unpack`][Self::unpack] when a caller only cares about
319    /// the `LocalOutput` case.
320    pub fn as_local_output(self) -> Option<(ExecutionId, LocalTaskId, TaskPersistence)> {
321        self.is_local_output().then(|| self.decode_local_output())
322    }
323
324    /// Decodes the fields of a `LocalOutput` word. Only valid when
325    /// [`RAW_VC_LOCAL_FLAG`] is set.
326    fn decode_local_output(self) -> (ExecutionId, LocalTaskId, TaskPersistence) {
327        let bits = self.bits();
328        let persistence = if (bits >> RAW_VC_LOCAL_TRANSIENT_SHIFT) & 1 == 1 {
329            TaskPersistence::Transient
330        } else {
331            TaskPersistence::Persistent
332        };
333        let execution_id = ((bits >> RAW_VC_LOCAL_EXECUTION_SHIFT) & 0xFFFF) as u16;
334        let local_task_id = ((bits >> RAW_VC_LOCAL_TASK_SHIFT) & 0xFFFF_FFFF) as u32;
335        // SAFETY: non-zero `ExecutionId`/`LocalTaskId` were packed in.
336        (
337            unsafe { ExecutionId::new_unchecked(execution_id) },
338            unsafe { LocalTaskId::new_unchecked(local_task_id) },
339            persistence,
340        )
341    }
342}
343
344impl Debug for RawVc {
345    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
346        match self.unpack() {
347            RawVcUnpacked::TaskOutput(task_id) => {
348                f.debug_tuple("RawVc::TaskOutput").field(&*task_id).finish()
349            }
350            RawVcUnpacked::TaskCell(task_id, cell_id) => f
351                .debug_tuple("RawVc::TaskCell")
352                .field(&*task_id)
353                .field(&cell_id.to_string())
354                .finish(),
355            RawVcUnpacked::LocalOutput(execution_id, local_task_id, task_persistence) => f
356                .debug_tuple("RawVc::LocalOutput")
357                .field(&*execution_id)
358                .field(&*local_task_id)
359                .field(&task_persistence)
360                .finish(),
361        }
362    }
363}
364
365impl Display for RawVc {
366    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
367        match self.unpack() {
368            RawVcUnpacked::TaskOutput(task_id) => write!(f, "output of task {}", *task_id),
369            RawVcUnpacked::TaskCell(task_id, cell_id) => {
370                write!(f, "{} of task {}", cell_id, *task_id)
371            }
372            RawVcUnpacked::LocalOutput(execution_id, local_task_id, task_persistence) => write!(
373                f,
374                "output of local task {} ({}, {})",
375                *local_task_id, *execution_id, task_persistence
376            ),
377        }
378    }
379}
380
381impl RawVc {
382    pub fn is_resolved(&self) -> bool {
383        self.is_task_cell()
384    }
385
386    pub fn is_local_output(&self) -> bool {
387        self.bits() & RAW_VC_LOCAL_FLAG != 0
388    }
389
390    /// Returns `true` if the task this `RawVc` reads from cannot be serialized and will not be
391    /// stored in the filesystem cache.
392    ///
393    /// See [`TaskPersistence`] for more details.
394    pub fn is_transient(&self) -> bool {
395        if self.is_local_output() {
396            // LocalOutput: the transient flag is stored as a bit.
397            (self.bits() >> RAW_VC_LOCAL_TRANSIENT_SHIFT) & 1 == 1
398        } else {
399            // TaskOutput / TaskCell: transience is a property of the TaskId value.
400            self.read_task_id().is_transient()
401        }
402    }
403
404    pub(crate) fn into_read(self) -> ReadRawVcFuture {
405        // returns a custom future to have something concrete and sized
406        // this avoids boxing in IntoFuture
407        ReadRawVcFuture::new(self)
408    }
409
410    /// See [`crate::Vc::to_resolved`].
411    pub(crate) fn resolve(self) -> ResolveRawVcFuture {
412        ResolveRawVcFuture::new(self)
413    }
414
415    /// Convert a potentially local `RawVc` into a non-local `RawVc`. This is a subset of resolution
416    /// resolution, because the returned `RawVc` can be a `TaskOutput`.
417    pub async fn to_non_local(self) -> Result<RawVc> {
418        let Some((execution_id, local_task_id, ..)) = self.as_local_output() else {
419            return Ok(self);
420        };
421        let tt = turbo_tasks();
422        let local_output = read_local_output(&*tt, execution_id, local_task_id).await?;
423        debug_assert!(
424            !local_output.is_local_output(),
425            "a LocalOutput cannot point at other LocalOutputs"
426        );
427        Ok(local_output)
428    }
429
430    /// Convert a potentially local `RawVc` into a non-local `RawVc`. This is a subset of resolution
431    /// resolution, because the returned `RawVc` can be a `TaskOutput`.
432    ///
433    /// 'unchecked' because the caller must have already confirmed that the local tasks were already
434    /// completed
435    pub(crate) fn to_non_local_unchecked_sync(self, tt: &dyn TurboTasksApi) -> Result<RawVc> {
436        let Some((execution_id, local_task_id, ..)) = self.as_local_output() else {
437            return Ok(self);
438        };
439        let local_output = match tt.try_read_local_output(execution_id, local_task_id)? {
440            Ok(raw_vc) => raw_vc,
441            Err(_event_listener) => unreachable!("local output is not ready yet"),
442        };
443        debug_assert!(
444            !local_output.is_local_output(),
445            "a LocalOutput cannot point at other LocalOutputs"
446        );
447        Ok(local_output)
448    }
449
450    pub fn try_get_task_id(&self) -> Option<TaskId> {
451        (!self.is_local_output()).then(|| self.read_task_id())
452    }
453
454    pub fn try_get_type_id(&self) -> Option<ValueTypeId> {
455        self.is_task_cell().then(|| self.read_cell().type_id())
456    }
457
458    /// For a cell that's already resolved, synchronously check if it implements a trait using the
459    /// type information in `RawVc::TaskCell` (we don't actually need to read the cell!).
460    pub(crate) fn resolved_has_trait(&self, trait_id: TraitTypeId) -> bool {
461        debug_assert!(
462            self.is_task_cell(),
463            "resolved_has_trait must be called with a RawVc::TaskCell"
464        );
465        get_value_type(self.read_cell().type_id()).has_trait(&trait_id)
466    }
467
468    /// For a cell that's already resolved, synchronously check if it is a given type using the type
469    /// information in `RawVc::TaskCell` (we don't actually need to read the cell!).
470    pub(crate) fn resolved_is_type(&self, type_id: ValueTypeId) -> bool {
471        debug_assert!(
472            self.is_task_cell(),
473            "resolved_is_type must be called with a RawVc::TaskCell"
474        );
475        self.read_cell().type_id() == type_id
476    }
477}
478
479/// Polls a pending [`EventListener`] slot. Returns [`Poll::Pending`] if the event has not yet
480/// fired. On [`Poll::Ready`], clears the slot so it is not polled again.
481fn poll_listener(
482    listener: &mut Option<EventListener>,
483    cx: &mut std::task::Context<'_>,
484) -> Poll<()> {
485    if let Some(l) = listener {
486        ready!(Pin::new(l).poll(cx));
487        *listener = None;
488    }
489    Poll::Ready(())
490}
491
492/// Wraps `f` in a scope that suppresses the eventual-consistency top-level task assertion,
493/// but only when `strongly_consistent` is `true` and debug assertions are enabled.
494///
495/// This is needed because a strongly-consistent read of a `TaskOutput` is not a single atomic
496/// operation — inner reads switch to eventual consistency after the first output is resolved —
497/// which would otherwise trigger the assertion in top-level tasks.
498fn suppress_top_level_task_check<R>(strongly_consistent: bool, f: impl FnOnce() -> R) -> R {
499    if cfg!(debug_assertions) && strongly_consistent {
500        // Temporarily suppress the top-level task check
501        SUPPRESS_EVENTUAL_CONSISTENCY_TOP_LEVEL_TASK_CHECK.sync_scope(true, f)
502    } else {
503        f()
504    }
505}
506
507/// Executes the task a read is waiting for when it is only scheduled, so the read can continue
508/// without waiting for a worker.
509///
510/// Must be called *outside* [`with_turbo_tasks`]: executing a task enters a task-local scope of its
511/// own, which panics while the read borrows `TURBO_TASKS`.
512fn execute_inline(key: ScheduleKey) {
513    execute_read_target_inline(&*turbo_tasks(), key);
514}
515
516#[must_use]
517pub struct ResolveRawVcFuture {
518    current: RawVc,
519    read_output_options: ReadOutputOptions,
520    /// This flag is redundant with `read_output_options`, but `read_output_options` is mutated
521    /// during the resolve. This flag indicates that the initial read was strongly consistent.
522    strongly_consistent: bool,
523    listener: Option<EventListener>,
524}
525
526impl ResolveRawVcFuture {
527    fn new(vc: RawVc) -> Self {
528        ResolveRawVcFuture {
529            current: vc,
530            read_output_options: ReadOutputOptions::default(),
531            strongly_consistent: false,
532            listener: None,
533        }
534    }
535
536    pub fn strongly_consistent(mut self) -> Self {
537        self.strongly_consistent = true;
538        self.read_output_options.consistency = ReadConsistency::Strong;
539        self
540    }
541
542    /// Track task output reads with a specific key (forwarded from
543    /// [`ReadRawVcFuture::track_with_key`]).
544    pub(crate) fn track_with_key(mut self) -> Self {
545        self.read_output_options.tracking = ReadTracking::Tracked;
546        self
547    }
548
549    /// Do not track task output reads as dependencies (forwarded from
550    /// [`ReadRawVcFuture::untracked`]).
551    pub(crate) fn untracked(mut self) -> Self {
552        self.read_output_options.tracking = ReadTracking::TrackOnlyError;
553        self
554    }
555}
556
557impl Future for ResolveRawVcFuture {
558    type Output = Result<RawVc>;
559
560    #[inline(never)]
561    fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll<Self::Output> {
562        // SAFETY: we are not moving self
563        let this = unsafe { self.get_unchecked_mut() };
564
565        let strongly_consistent = this.strongly_consistent;
566        // `execute_inline` is the task to execute inline, reported out-of-band so that the value
567        // path stays exactly as cheap as it was. It is only set for tasks that are merely
568        // scheduled; one that a worker is already executing cannot be taken over.
569        let mut poll_fn = |tt: &Arc<dyn TurboTasksApi>,
570                           execute_inline: &mut Option<ScheduleKey>|
571         -> Poll<Self::Output> {
572            'outer: loop {
573                ready!(poll_listener(&mut this.listener, cx));
574                let (listener, key) = match this.current.unpack() {
575                    RawVcUnpacked::TaskOutput(task) => {
576                        let read_result = tt.try_read_task_output(task, this.read_output_options);
577                        match read_result {
578                            Ok(ReadOutcome::Value(vc)) => {
579                                // turbo-tasks-backend doesn't currently have any sort of
580                                // "transaction" or global lock mechanism to group together chains
581                                // of `TaskOutput`/`TaskCell` reads.
582                                //
583                                // If we ignore the theoretical TOCTOU issues, we no longer need to
584                                // read strongly consistent, as any Vc returned from the first task
585                                // will be inside of the scope of the first task. So it's already
586                                // strongly consistent.
587                                this.read_output_options.consistency = ReadConsistency::Eventual;
588                                this.current = vc;
589                                continue 'outer;
590                            }
591                            // Nobody has started the task yet, so the caller may take it over.
592                            Ok(ReadOutcome::Scheduled(listener)) => {
593                                (listener, Some(ScheduleKey::Task(task)))
594                            }
595                            Ok(ReadOutcome::InProgress(listener)) => {
596                                // A worker is on it; nothing to take over. Loop back so
597                                // `poll_listener` registers the waker on this listener — returning
598                                // `Pending` here would sleep through the event.
599                                #[cfg(feature = "inline_execution_stats")]
600                                tt.note_waited_for_in_progress_task();
601                                this.listener = Some(listener);
602                                continue 'outer;
603                            }
604                            Err(err) => return Poll::Ready(Err(err)),
605                        }
606                    }
607                    RawVcUnpacked::TaskCell(_, _) => return Poll::Ready(Ok(this.current)),
608                    RawVcUnpacked::LocalOutput(execution_id, local_task_id, ..) => {
609                        debug_assert_eq!(
610                            this.read_output_options.consistency,
611                            ReadConsistency::Eventual
612                        );
613                        let read_result = tt.try_read_local_output(execution_id, local_task_id);
614                        match read_result {
615                            Ok(Ok(vc)) => {
616                                this.current = vc;
617                                continue 'outer;
618                            }
619                            Ok(Err(listener)) => (
620                                listener,
621                                Some(ScheduleKey::LocalTask(execution_id, local_task_id)),
622                            ),
623                            Err(err) => return Poll::Ready(Err(err)),
624                        }
625                    }
626                };
627                // The task is not done yet, so we have to wait for it — unless it is merely
628                // scheduled, in which case our caller executes it and we read again.
629                this.listener = Some(listener);
630                *execute_inline = key;
631                return Poll::Pending;
632            }
633        };
634
635        loop {
636            let mut execute_inline_key = None;
637            // HACK: Temporarily suppress top-level task check if doing strongly consistent read.
638            //
639            // This masks a bug: There's an unlikely TOCTOU race condition in `poll_fn`. Because the
640            // strongly consistent read isn't a single atomic operation, any inner `TaskOutput` or
641            // `TaskCell` could get mutated after the strongly consistent read of the outer
642            // `TaskOutput`.
643            let result = suppress_top_level_task_check(strongly_consistent, || {
644                with_turbo_tasks(|tt| poll_fn(tt, &mut execute_inline_key))
645            });
646            if let Some(key) = execute_inline_key {
647                // Not inside `with_turbo_tasks`, see `execute_inline`.
648                execute_inline(key);
649                continue;
650            }
651            return result;
652        }
653    }
654}
655
656impl Unpin for ResolveRawVcFuture {}
657
658#[must_use]
659pub struct ReadRawVcFuture {
660    read_cell_options: ReadCellOptions,
661    state: ReadRawVcState,
662}
663
664/// Phase 1 and phase 2 of [`ReadRawVcFuture`] use disjoint sets of fields. Storing them in an
665/// enum keeps the future smaller than holding both sets simultaneously.
666enum ReadRawVcState {
667    /// Phase 1: resolves the [`RawVc`] pointer chain to a [`RawVc::TaskCell`].
668    Resolving(ResolveRawVcFuture),
669    /// Phase 2: the resolved task/cell identity plus a listener for the cell read wait.
670    Reading {
671        task: TaskId,
672        index: CellId,
673        /// Whether phase 1 was a strongly-consistent read. Needed here to re-apply
674        /// [`suppress_top_level_task_check`] in phase 2. Lives in this variant (rather than the
675        /// outer struct) so it can share padding with the other `Reading` fields — keeping
676        /// `Reading` no larger than `Resolving`, and the whole future 8 bytes smaller.
677        strongly_consistent: bool,
678        listener: Option<EventListener>,
679    },
680}
681
682impl ReadRawVcFuture {
683    pub(crate) fn new(vc: RawVc) -> Self {
684        ReadRawVcFuture {
685            read_cell_options: ReadCellOptions::default(),
686            state: ReadRawVcState::Resolving(ResolveRawVcFuture::new(vc)),
687        }
688    }
689
690    fn map_resolve(mut self, f: impl FnOnce(ResolveRawVcFuture) -> ResolveRawVcFuture) -> Self {
691        match self.state {
692            ReadRawVcState::Resolving(resolve) => {
693                self.state = ReadRawVcState::Resolving(f(resolve));
694            }
695            ReadRawVcState::Reading { .. } => {
696                unreachable!("builder methods are only called before polling");
697            }
698        }
699        self
700    }
701
702    /// Make reads strongly consistent.
703    pub fn strongly_consistent(self) -> Self {
704        self.map_resolve(|r| r.strongly_consistent())
705    }
706
707    /// Track the value as a dependency with an key.
708    pub fn track_with_key(mut self, key: u64) -> Self {
709        self.read_cell_options.tracking = ReadCellTracking::Tracked { key: Some(key) };
710        self.map_resolve(|r| r.track_with_key())
711    }
712
713    /// This will not track the value as dependency, but will still track the error as dependency,
714    /// if there is an error.
715    ///
716    /// INVALIDATION: Be careful with this, it will not track dependencies, so
717    /// using it could break cache invalidation.
718    pub fn untracked(mut self) -> Self {
719        self.read_cell_options.tracking = ReadCellTracking::TrackOnlyError;
720        self.map_resolve(|r| r.untracked())
721    }
722
723    /// Hint that this is the final read of the cell content.
724    pub fn final_read_hint(mut self) -> Self {
725        self.read_cell_options.final_read_hint = true;
726        self
727    }
728}
729
730impl Future for ReadRawVcFuture {
731    type Output = Result<TypedCellContent>;
732
733    #[inline(never)]
734    fn poll(self: Pin<&mut Self>, cx: &mut std::task::Context<'_>) -> Poll<Self::Output> {
735        // SAFETY: we are not moving self
736        let this = unsafe { self.get_unchecked_mut() };
737
738        // --- Phase 1: resolve the RawVc pointer chain to a TaskCell ---
739        //
740        // `ResolveRawVcFuture` is `Unpin`, so `Pin::new` is safe.
741        // It handles `with_turbo_tasks` and `suppress_top_level_task_check` internally.
742        if let ReadRawVcState::Resolving(resolve) = &mut this.state {
743            let strongly_consistent = resolve.strongly_consistent;
744            match ready!(Pin::new(resolve).poll(cx)) {
745                Err(err) => return Poll::Ready(Err(err)),
746                Ok(resolved) => {
747                    let Some((task, index)) = resolved.as_task_cell() else {
748                        unreachable!("ResolveRawVcFuture always resolves to a TaskCell")
749                    };
750                    this.state = ReadRawVcState::Reading {
751                        task,
752                        index,
753                        strongly_consistent,
754                        listener: None,
755                    };
756                }
757            }
758        }
759
760        // --- Phase 2: read the cell content ---
761        let ReadRawVcState::Reading {
762            task,
763            index,
764            strongly_consistent,
765            listener,
766        } = &mut this.state
767        else {
768            unreachable!("phase 1 transitioned to Reading above");
769        };
770        let task = *task;
771        let index = *index;
772        let read_cell_options = this.read_cell_options;
773
774        let strongly_consistent = *strongly_consistent;
775
776        let mut poll_fn = |tt: &Arc<dyn TurboTasksApi>,
777                           execute_inline: &mut Option<ScheduleKey>|
778         -> Poll<Self::Output> {
779            loop {
780                ready!(poll_listener(listener, cx));
781                let (new_listener, key) =
782                    match tt.try_read_task_cell(task, index, read_cell_options) {
783                        Ok(ReadOutcome::Value(content)) => return Poll::Ready(Ok(content)),
784                        Ok(ReadOutcome::Scheduled(l)) => (l, Some(ScheduleKey::Task(task))),
785                        Ok(ReadOutcome::InProgress(l)) => {
786                            // A worker is already filling the cell; nothing to take over. Loop back
787                            // so `poll_listener` registers the waker on this listener — returning
788                            // `Pending` here would sleep through the event.
789                            #[cfg(feature = "inline_execution_stats")]
790                            tt.note_waited_for_in_progress_task();
791                            *listener = Some(l);
792                            continue;
793                        }
794                        Err(err) => return Poll::Ready(Err(err)),
795                    };
796                // The cell isn't available yet, so we have to wait for the task that fills it —
797                // unless that task is merely scheduled, in which case our caller executes it and we
798                // read again.
799                *listener = Some(new_listener);
800                *execute_inline = key;
801                return Poll::Pending;
802            }
803        };
804
805        loop {
806            let mut execute_inline_key = None;
807            // Phase 2 must also suppress the top-level task check when phase 1 was
808            // strongly-consistent. The suppression from `ResolveRawVcFuture::poll` only lasts for
809            // the duration of that individual `poll` call and does not carry over to subsequent
810            // calls or to this phase.
811            let result = suppress_top_level_task_check(strongly_consistent, || {
812                with_turbo_tasks(|tt| poll_fn(tt, &mut execute_inline_key))
813            });
814            if let Some(key) = execute_inline_key {
815                // Not inside `with_turbo_tasks`, see `execute_inline`.
816                execute_inline(key);
817                continue;
818            }
819            return result;
820        }
821    }
822}
823
824impl Unpin for ReadRawVcFuture {}
825
826#[cfg(test)]
827mod tests {
828    use super::*;
829
830    /// `CellId` must pack into 4 bytes and keep its niche so `Option<CellId>`
831    /// stays 4 bytes — this is the whole point of [`RawVc`] shrinking.
832    #[test]
833    fn cell_id_is_four_bytes() {
834        assert_eq!(size_of::<CellId>(), 4);
835        assert_eq!(size_of::<Option<CellId>>(), 4);
836    }
837
838    /// Packing and unpacking a `(type_id, index)` pair must round-trip across
839    /// the full range of both fields, including the boundary values.
840    #[test]
841    fn cell_id_pack_unpack_round_trip() {
842        let type_ids = [1u16, 2, 100, CellId::MAX_VALUE_TYPE_ID];
843        let indices = [0u32, 1, 12345, CellId::MAX_CELL_INDEX];
844        for &raw_ty in &type_ids {
845            // SAFETY: all test values are >= 1.
846            let type_id = unsafe { ValueTypeId::new_unchecked(raw_ty) };
847            for &index in &indices {
848                let cell = CellId::new(type_id, index);
849                assert_eq!(cell.type_id(), type_id, "type_id round-trip for {raw_ty}");
850                assert_eq!(cell.index(), index, "index round-trip for {index}");
851            }
852        }
853    }
854
855    /// Distinct `(type_id, index)` pairs must pack to distinct words — the
856    /// packing is a bijection, which is what lets us derive `Eq`/`Hash`.
857    #[test]
858    fn cell_id_packing_is_bijective() {
859        // SAFETY: ids are >= 1.
860        let a = CellId::new(unsafe { ValueTypeId::new_unchecked(1) }, 0);
861        let b = CellId::new(unsafe { ValueTypeId::new_unchecked(1) }, 1);
862        let c = CellId::new(unsafe { ValueTypeId::new_unchecked(2) }, 0);
863        assert_ne!(a, b);
864        assert_ne!(a, c);
865        assert_ne!(b, c);
866    }
867
868    /// `RawVc` must pack into 8 bytes and keep its niche.
869    #[test]
870    fn raw_vc_is_eight_bytes() {
871        assert_eq!(size_of::<RawVc>(), 8);
872        assert_eq!(size_of::<Option<RawVc>>(), 8);
873    }
874
875    /// Every variant must round-trip through pack → `unpack()` across the full
876    /// range of each packed field, including boundary values and both
877    /// persistence states. This is the core correctness property of the
878    /// bit-packing.
879    #[test]
880    fn raw_vc_pack_unpack_round_trip() {
881        // SAFETY: all ids below are >= 1 and within their bit budgets.
882        let tasks = [
883            1u32,
884            2,
885            crate::TRANSIENT_TASK_BIT - 1,
886            crate::TRANSIENT_TASK_BIT,
887            TASK_ID_MAX,
888        ];
889        for &t in &tasks {
890            let task = unsafe { TaskId::new_unchecked(t) };
891
892            // TaskOutput
893            let vc = RawVc::task_output(task);
894            assert_eq!(vc.unpack(), RawVcUnpacked::TaskOutput(task));
895            assert!(!vc.is_resolved() && !vc.is_local_output());
896            assert_eq!(vc.is_transient(), task.is_transient());
897            assert_eq!(vc.try_get_task_id(), Some(task));
898            // single-arm accessors
899            assert_eq!(vc.as_task_output(), Some(task));
900            assert_eq!(vc.as_task_cell(), None);
901            assert_eq!(vc.as_local_output(), None);
902
903            // TaskCell, across CellId boundaries
904            for cell in [
905                CellId::new(unsafe { ValueTypeId::new_unchecked(1) }, 0),
906                CellId::new(
907                    unsafe { ValueTypeId::new_unchecked(CellId::MAX_VALUE_TYPE_ID) },
908                    CellId::MAX_CELL_INDEX,
909                ),
910            ] {
911                let vc = RawVc::task_cell(task, cell);
912                assert_eq!(vc.unpack(), RawVcUnpacked::TaskCell(task, cell));
913                assert!(vc.is_resolved());
914                assert_eq!(vc.try_get_task_id(), Some(task));
915                assert_eq!(vc.try_get_type_id(), Some(cell.type_id()));
916                // single-arm accessors
917                assert_eq!(vc.as_task_cell(), Some((task, cell)));
918                assert_eq!(vc.as_task_output(), None);
919                assert_eq!(vc.as_local_output(), None);
920            }
921        }
922
923        // LocalOutput, both persistence states and boundary ids
924        for persistence in [TaskPersistence::Persistent, TaskPersistence::Transient] {
925            for (e, l) in [(1u16, 1u32), (u16::MAX, u32::MAX)] {
926                let exec = unsafe { ExecutionId::new_unchecked(e) };
927                let local = unsafe { LocalTaskId::new_unchecked(l) };
928                let vc = RawVc::local_output(exec, local, persistence);
929                assert_eq!(
930                    vc.unpack(),
931                    RawVcUnpacked::LocalOutput(exec, local, persistence)
932                );
933                assert!(vc.is_local_output());
934                assert_eq!(vc.is_transient(), persistence == TaskPersistence::Transient);
935                assert_eq!(vc.try_get_task_id(), None);
936                // single-arm accessors
937                assert_eq!(vc.as_local_output(), Some((exec, local, persistence)));
938                assert_eq!(vc.as_task_output(), None);
939                assert_eq!(vc.as_task_cell(), None);
940            }
941        }
942    }
943
944    /// The discriminator relies on the cell field being zero for `TaskOutput`
945    /// and non-zero for `TaskCell`. A `TaskOutput` and a `TaskCell` that share
946    /// the same `TaskId` must still be told apart, and a `LocalOutput` whose
947    /// `LocalTaskId` populates the high bits (the cell-field region) must remain
948    /// a `LocalOutput` because bit 31 wins.
949    #[test]
950    fn raw_vc_discriminator_is_unambiguous() {
951        // SAFETY: all ids are >= 1.
952        let task = unsafe { TaskId::new_unchecked(123) };
953        let cell = CellId::new(unsafe { ValueTypeId::new_unchecked(1) }, 0);
954
955        let output = RawVc::task_output(task);
956        let task_cell = RawVc::task_cell(task, cell);
957        assert!(output.is_task_output() && !output.is_task_cell() && !output.is_local_output());
958        assert!(
959            task_cell.is_task_cell() && !task_cell.is_task_output() && !task_cell.is_local_output()
960        );
961        // Same TaskId, different variants, distinct words.
962        assert_ne!(output, task_cell);
963        assert_eq!(output.read_task_id(), task_cell.read_task_id());
964
965        // A LocalOutput with a max LocalTaskId fills the high 32 bits; it must
966        // not be misread as a TaskCell.
967        let local = RawVc::local_output(
968            unsafe { ExecutionId::new_unchecked(u16::MAX) },
969            unsafe { LocalTaskId::new_unchecked(u32::MAX) },
970            TaskPersistence::Persistent,
971        );
972        assert!(local.is_local_output() && !local.is_task_cell() && !local.is_task_output());
973    }
974
975    #[test]
976    #[cfg(debug_assertions)]
977    #[should_panic(expected = "TaskId exceeds 31 bits")]
978    fn task_output_panics_on_out_of_range_task_id() {
979        // `TASK_ID_MAX + 1` is the first value that sets bit 31.
980        // SAFETY: non-zero.
981        let task = unsafe { TaskId::new_unchecked(TASK_ID_MAX + 1) };
982        let _ = RawVc::task_output(task);
983    }
984
985    #[test]
986    #[cfg(debug_assertions)]
987    #[should_panic(expected = "TaskId exceeds 31 bits")]
988    fn task_cell_panics_on_out_of_range_task_id() {
989        // SAFETY: non-zero.
990        let task = unsafe { TaskId::new_unchecked(TASK_ID_MAX + 1) };
991        let cell = CellId::new(unsafe { ValueTypeId::new_unchecked(1) }, 0);
992        let _ = RawVc::task_cell(task, cell);
993    }
994
995    #[test]
996    #[cfg(debug_assertions)]
997    #[should_panic(expected = "exceeds")]
998    fn cell_id_panics_on_out_of_range_type_id() {
999        // SAFETY: `MAX_VALUE_TYPE_ID + 1` is non-zero.
1000        let type_id = unsafe { ValueTypeId::new_unchecked(CellId::MAX_VALUE_TYPE_ID + 1) };
1001        let _ = CellId::new(type_id, 0);
1002    }
1003    #[test]
1004    #[cfg(debug_assertions)]
1005    #[should_panic(expected = "exceeds")]
1006    fn cell_id_panics_on_out_of_range_index() {
1007        let type_id = unsafe { ValueTypeId::new_unchecked(1) };
1008        let _ = CellId::new(type_id, CellId::MAX_CELL_INDEX + 1);
1009    }
1010}
turbo_tasks/vc/raw.rs

turbo_tasks/vc/
raw.rs
