turbo_tasks_fetch/
client.rs

1use std::{hash::Hash, sync::LazyLock};
2
3use anyhow::Result;
4use quick_cache::sync::Cache;
5use turbo_rcstr::RcStr;
6use turbo_tasks::{ReadRef, Vc, duration_span, mark_session_dependent};
7
8use crate::{FetchError, FetchResult, HttpResponse, HttpResponseBody};
9
10const MAX_CLIENTS: usize = 16;
11static CLIENT_CACHE: LazyLock<Cache<ReadRef<FetchClientConfig>, reqwest::Client>> =
12    LazyLock::new(|| Cache::new(MAX_CLIENTS));
13
14/// Represents the configuration needed to construct a [`reqwest::Client`].
15///
16/// This is used to cache clients keyed by their configuration, so the configuration should contain
17/// as few fields as possible and change infrequently.
18///
19/// This is needed because [`reqwest::ClientBuilder`] does not implement the required traits. This
20/// factory cannot be a closure because closures do not implement `Eq` or `Hash`.
21#[turbo_tasks::value(shared)]
22#[derive(Hash)]
23pub struct FetchClientConfig {
24    /// Whether to load embedded webpki root certs with rustls. Default is true.
25    ///
26    /// Ignored for:
27    /// - Windows on ARM, which uses `native-tls` instead of `rustls-tls`.
28    /// - Ignored for WASM targets, which use the runtime's TLS implementation.
29    pub tls_built_in_webpki_certs: bool,
30    /// Whether to load native root certs using the `rustls-native-certs` crate. This may make
31    /// reqwest client initialization slower, so it's not used by default.
32    ///
33    /// Ignored for:
34    /// - Windows on ARM, which uses `native-tls` instead of `rustls-tls`.
35    /// - Ignored for WASM targets, which use the runtime's TLS implementation.
36    pub tls_built_in_native_certs: bool,
37}
38
39impl Default for FetchClientConfig {
40    fn default() -> Self {
41        Self {
42            tls_built_in_webpki_certs: true,
43            tls_built_in_native_certs: false,
44        }
45    }
46}
47
48impl FetchClientConfig {
49    /// Returns a cached instance of `reqwest::Client` it exists, otherwise constructs a new one.
50    ///
51    /// The cache is bound in size to prevent accidental blowups or leaks. However, in practice,
52    /// very few clients should be created, likely only when the bundler configuration changes.
53    ///
54    /// Client construction is largely deterministic, aside from changes to system TLS
55    /// configuration.
56    ///
57    /// The reqwest client fails to construct if the TLS backend cannot be initialized, or the
58    /// resolver cannot load the system configuration. These failures should be treated as
59    /// cached for some amount of time, but ultimately transient (e.g. using
60    /// [`turbo_tasks::mark_session_dependent`]).
61    pub fn try_get_cached_reqwest_client(
62        self: ReadRef<FetchClientConfig>,
63    ) -> reqwest::Result<reqwest::Client> {
64        CLIENT_CACHE.get_or_insert_with(&self, {
65            let this = ReadRef::clone(&self);
66            move || this.try_build_uncached_reqwest_client()
67        })
68    }
69
70    fn try_build_uncached_reqwest_client(&self) -> reqwest::Result<reqwest::Client> {
71        let client_builder = reqwest::Client::builder();
72
73        // make sure this cfg matches the one in `Cargo.toml`!
74        #[cfg(not(any(
75            all(target_os = "windows", target_arch = "aarch64"),
76            target_arch = "wasm32"
77        )))]
78        let client_builder = client_builder
79            .use_rustls_tls()
80            .tls_built_in_root_certs(false)
81            .tls_built_in_webpki_certs(self.tls_built_in_webpki_certs)
82            .tls_built_in_native_certs(self.tls_built_in_native_certs);
83
84        client_builder.build()
85    }
86}
87
88#[turbo_tasks::value_impl]
89impl FetchClientConfig {
90    #[turbo_tasks::function(network)]
91    pub async fn fetch(
92        self: Vc<FetchClientConfig>,
93        url: RcStr,
94        user_agent: Option<RcStr>,
95    ) -> Result<Vc<FetchResult>> {
96        let url_ref = &*url;
97        let this = self.await?;
98        let tls_built_in_native_certs = this.tls_built_in_native_certs;
99        let response_result: reqwest::Result<HttpResponse> = async move {
100            let reqwest_client = this.try_get_cached_reqwest_client()?;
101
102            let mut builder = reqwest_client.get(url_ref);
103            if let Some(user_agent) = user_agent {
104                builder = builder.header("User-Agent", user_agent.as_str());
105            }
106
107            let response = {
108                let _span = duration_span!("fetch request", url = url_ref);
109                builder.send().await
110            }
111            .and_then(|r| r.error_for_status())?;
112
113            let status = response.status().as_u16();
114
115            let body = {
116                let _span = duration_span!("fetch response", url = url_ref);
117                response.bytes().await?
118            }
119            .to_vec();
120
121            Ok(HttpResponse {
122                status,
123                body: HttpResponseBody(body).resolved_cell(),
124            })
125        }
126        .await;
127
128        match response_result {
129            Ok(resp) => Ok(Vc::cell(Ok(resp.resolved_cell()))),
130            Err(err) => {
131                // the client failed to construct or the HTTP request failed
132                mark_session_dependent();
133                Ok(Vc::cell(Err(FetchError::from_reqwest_error(
134                    &err,
135                    &url,
136                    tls_built_in_native_certs,
137                )
138                .resolved_cell())))
139            }
140        }
141    }
142}
143
144#[doc(hidden)]
145pub fn __test_only_reqwest_client_cache_clear() {
146    CLIENT_CACHE.clear()
147}
148
149#[doc(hidden)]
150pub fn __test_only_reqwest_client_cache_len() -> usize {
151    CLIENT_CACHE.len()
152}
turbo_tasks_fetch/client.rs

turbo_tasks_fetch/
client.rs
