turbo_tasks_fetch/
client.rs

1use std::{hash::Hash, sync::LazyLock};
2
3use anyhow::Result;
4use quick_cache::sync::Cache;
5use serde::{Deserialize, Serialize};
6use turbo_rcstr::RcStr;
7use turbo_tasks::{
8    NonLocalValue, ReadRef, Vc, duration_span, mark_session_dependent, trace::TraceRawVcs,
9};
10
11use crate::{FetchError, FetchResult, HttpResponse, HttpResponseBody};
12
13const MAX_CLIENTS: usize = 16;
14static CLIENT_CACHE: LazyLock<Cache<ReadRef<FetchClientConfig>, reqwest::Client>> =
15    LazyLock::new(|| Cache::new(MAX_CLIENTS));
16
17#[derive(Hash, PartialEq, Eq, Serialize, Deserialize, NonLocalValue, Debug, TraceRawVcs)]
18pub enum ProxyConfig {
19    Http(RcStr),
20    Https(RcStr),
21}
22
23/// Represents the configuration needed to construct a [`reqwest::Client`].
24///
25/// This is used to cache clients keyed by their configuration, so the configuration should contain
26/// as few fields as possible and change infrequently.
27///
28/// This is needed because [`reqwest::ClientBuilder`] does not implement the required traits. This
29/// factory cannot be a closure because closures do not implement `Eq` or `Hash`.
30#[turbo_tasks::value(shared)]
31#[derive(Hash)]
32pub struct FetchClientConfig {
33    /// Whether to load embedded webpki root certs with rustls. Default is true.
34    ///
35    /// Ignored for:
36    /// - Windows on ARM, which uses `native-tls` instead of `rustls-tls`.
37    /// - Ignored for WASM targets, which use the runtime's TLS implementation.
38    pub tls_built_in_webpki_certs: bool,
39    /// Whether to load native root certs using the `rustls-native-certs` crate. This may make
40    /// reqwest client initialization slower, so it's not used by default.
41    ///
42    /// Ignored for:
43    /// - Windows on ARM, which uses `native-tls` instead of `rustls-tls`.
44    /// - Ignored for WASM targets, which use the runtime's TLS implementation.
45    pub tls_built_in_native_certs: bool,
46}
47
48impl Default for FetchClientConfig {
49    fn default() -> Self {
50        Self {
51            tls_built_in_webpki_certs: true,
52            tls_built_in_native_certs: false,
53        }
54    }
55}
56
57impl FetchClientConfig {
58    /// Returns a cached instance of `reqwest::Client` it exists, otherwise constructs a new one.
59    ///
60    /// The cache is bound in size to prevent accidental blowups or leaks. However, in practice,
61    /// very few clients should be created, likely only when the bundler configuration changes.
62    ///
63    /// Client construction is largely deterministic, aside from changes to system TLS
64    /// configuration.
65    ///
66    /// The reqwest client fails to construct if the TLS backend cannot be initialized, or the
67    /// resolver cannot load the system configuration. These failures should be treated as
68    /// cached for some amount of time, but ultimately transient (e.g. using
69    /// [`turbo_tasks::mark_session_dependent`]).
70    pub fn try_get_cached_reqwest_client(
71        self: ReadRef<FetchClientConfig>,
72    ) -> reqwest::Result<reqwest::Client> {
73        CLIENT_CACHE.get_or_insert_with(&self, {
74            let this = ReadRef::clone(&self);
75            move || this.try_build_uncached_reqwest_client()
76        })
77    }
78
79    fn try_build_uncached_reqwest_client(&self) -> reqwest::Result<reqwest::Client> {
80        let client_builder = reqwest::Client::builder();
81
82        // make sure this cfg matches the one in `Cargo.toml`!
83        #[cfg(not(any(
84            all(target_os = "windows", target_arch = "aarch64"),
85            target_arch = "wasm32"
86        )))]
87        let client_builder = client_builder
88            .use_rustls_tls()
89            .tls_built_in_root_certs(false)
90            .tls_built_in_webpki_certs(self.tls_built_in_webpki_certs)
91            .tls_built_in_native_certs(self.tls_built_in_native_certs);
92
93        client_builder.build()
94    }
95}
96
97#[turbo_tasks::value_impl]
98impl FetchClientConfig {
99    #[turbo_tasks::function(network)]
100    pub async fn fetch(
101        self: Vc<FetchClientConfig>,
102        url: RcStr,
103        user_agent: Option<RcStr>,
104    ) -> Result<Vc<FetchResult>> {
105        let url_ref = &*url;
106        let this = self.await?;
107        let tls_built_in_native_certs = this.tls_built_in_native_certs;
108        let response_result: reqwest::Result<HttpResponse> = async move {
109            let reqwest_client = this.try_get_cached_reqwest_client()?;
110
111            let mut builder = reqwest_client.get(url_ref);
112            if let Some(user_agent) = user_agent {
113                builder = builder.header("User-Agent", user_agent.as_str());
114            }
115
116            let response = {
117                let _span = duration_span!("fetch request", url = url_ref);
118                builder.send().await
119            }
120            .and_then(|r| r.error_for_status())?;
121
122            let status = response.status().as_u16();
123
124            let body = {
125                let _span = duration_span!("fetch response", url = url_ref);
126                response.bytes().await?
127            }
128            .to_vec();
129
130            Ok(HttpResponse {
131                status,
132                body: HttpResponseBody(body).resolved_cell(),
133            })
134        }
135        .await;
136
137        match response_result {
138            Ok(resp) => Ok(Vc::cell(Ok(resp.resolved_cell()))),
139            Err(err) => {
140                // the client failed to construct or the HTTP request failed
141                mark_session_dependent();
142                Ok(Vc::cell(Err(FetchError::from_reqwest_error(
143                    &err,
144                    &url,
145                    tls_built_in_native_certs,
146                )
147                .resolved_cell())))
148            }
149        }
150    }
151}
152
153#[doc(hidden)]
154pub fn __test_only_reqwest_client_cache_clear() {
155    CLIENT_CACHE.clear()
156}
157
158#[doc(hidden)]
159pub fn __test_only_reqwest_client_cache_len() -> usize {
160    CLIENT_CACHE.len()
161}
turbo_tasks_fetch/client.rs

turbo_tasks_fetch/
client.rs
