fix(npm): use an http client with connection pool (#16705)

Should make downloading npm packages faster and more reliable.

8 files changed, 450 insertions, 418 deletions

diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs
index e16b808d9..b9a91d41a 100644
--- a/cli/file_fetcher.rs
+++ b/cli/file_fetcher.rs
@@ -3,13 +3,12 @@
 use crate::auth_tokens::AuthTokens;
 use crate::colors;
 use crate::http_cache::HttpCache;
-use crate::http_util::fetch_once;
 use crate::http_util::CacheSemantics;
 use crate::http_util::FetchOnceArgs;
 use crate::http_util::FetchOnceResult;
+use crate::http_util::HttpClient;
 use crate::progress_bar::ProgressBar;
 use crate::text_encoding;
-use crate::version::get_user_agent;
 
 use data_url::DataUrl;
 use deno_ast::MediaType;
@@ -22,8 +21,6 @@ use deno_core::futures;
 use deno_core::futures::future::FutureExt;
 use deno_core::parking_lot::Mutex;
 use deno_core::ModuleSpecifier;
-use deno_runtime::deno_fetch::create_http_client;
-use deno_runtime::deno_fetch::reqwest;
 use deno_runtime::deno_tls::rustls;
 use deno_runtime::deno_tls::rustls::RootCertStore;
 use deno_runtime::deno_tls::rustls_native_certs::load_native_certs;
@@ -333,7 +330,7 @@ pub struct FileFetcher {
   cache: FileCache,
   cache_setting: CacheSetting,
   pub http_cache: HttpCache,
-  http_client: reqwest::Client,
+  http_client: HttpClient,
   blob_store: BlobStore,
   download_log_level: log::Level,
   progress_bar: Option<ProgressBar>,
@@ -344,9 +341,8 @@ impl FileFetcher {
     http_cache: HttpCache,
     cache_setting: CacheSetting,
     allow_remote: bool,
-    root_cert_store: Option<RootCertStore>,
+    http_client: HttpClient,
     blob_store: BlobStore,
-    unsafely_ignore_certificate_errors: Option<Vec<String>>,
     progress_bar: Option<ProgressBar>,
   ) -> Result<Self, AnyError> {
     Ok(Self {
@@ -355,14 +351,7 @@ impl FileFetcher {
       cache: Default::default(),
       cache_setting,
       http_cache,
-      http_client: create_http_client(
-        get_user_agent(),
-        root_cert_store,
-        vec![],
-        None,
-        unsafely_ignore_certificate_errors,
-        None,
-      )?,
+      http_client,
       blob_store,
       download_log_level: log::Level::Info,
       progress_bar,
@@ -628,14 +617,14 @@ impl FileFetcher {
     let file_fetcher = self.clone();
     // A single pass of fetch either yields code or yields a redirect.
     async move {
-      match fetch_once(FetchOnceArgs {
-        client,
-        url: specifier.clone(),
-        maybe_accept: maybe_accept.clone(),
-        maybe_etag,
-        maybe_auth_token,
-      })
-      .await?
+      match client
+        .fetch_once(FetchOnceArgs {
+          url: specifier.clone(),
+          maybe_accept: maybe_accept.clone(),
+          maybe_etag,
+          maybe_auth_token,
+        })
+        .await?
       {
         FetchOnceResult::NotModified => {
           let file = file_fetcher.fetch_cached(&specifier, 10)?.unwrap();
@@ -765,6 +754,8 @@ impl FileFetcher {
 
 #[cfg(test)]
 mod tests {
+  use crate::http_util::HttpClient;
+
   use super::*;
   use deno_core::error::get_custom_error_class;
   use deno_core::resolve_url;
@@ -793,10 +784,9 @@ mod tests {
       HttpCache::new(&location),
       cache_setting,
       true,
-      None,
+      HttpClient::new(None, None).unwrap(),
       blob_store.clone(),
       None,
-      None,
     )
     .unwrap();
     (file_fetcher, temp_dir, blob_store)
@@ -1232,10 +1222,9 @@ mod tests {
       HttpCache::new(&location),
       CacheSetting::ReloadAll,
       true,
-      None,
+      HttpClient::new(None, None).unwrap(),
       BlobStore::default(),
       None,
-      None,
     )
     .unwrap();
     let result = file_fetcher
@@ -1259,10 +1248,9 @@ mod tests {
       HttpCache::new(&location),
       CacheSetting::Use,
       true,
-      None,
+      HttpClient::new(None, None).unwrap(),
       BlobStore::default(),
       None,
-      None,
     )
     .unwrap();
     let specifier =
@@ -1287,10 +1275,9 @@ mod tests {
       HttpCache::new(&location),
       CacheSetting::Use,
       true,
-      None,
+      HttpClient::new(None, None).unwrap(),
       BlobStore::default(),
       None,
-      None,
     )
     .unwrap();
     let result = file_fetcher_02
@@ -1431,10 +1418,9 @@ mod tests {
       HttpCache::new(&location),
       CacheSetting::Use,
       true,
-      None,
+      HttpClient::new(None, None).unwrap(),
       BlobStore::default(),
       None,
-      None,
     )
     .unwrap();
     let specifier =
@@ -1461,10 +1447,9 @@ mod tests {
       HttpCache::new(&location),
       CacheSetting::Use,
       true,
-      None,
+      HttpClient::new(None, None).unwrap(),
       BlobStore::default(),
       None,
-      None,
     )
     .unwrap();
     let result = file_fetcher_02
@@ -1562,10 +1547,9 @@ mod tests {
       HttpCache::new(&location),
       CacheSetting::Use,
       false,
-      None,
+      HttpClient::new(None, None).unwrap(),
       BlobStore::default(),
       None,
-      None,
     )
     .unwrap();
     let specifier =
@@ -1589,20 +1573,18 @@ mod tests {
       HttpCache::new(&location),
       CacheSetting::Only,
       true,
-      None,
+      HttpClient::new(None, None).unwrap(),
       BlobStore::default(),
       None,
-      None,
     )
     .unwrap();
     let file_fetcher_02 = FileFetcher::new(
       HttpCache::new(&location),
       CacheSetting::Use,
       true,
-      None,
+      HttpClient::new(None, None).unwrap(),
       BlobStore::default(),
       None,
-      None,
     )
     .unwrap();
     let specifier =
diff --git a/cli/http_cache.rs b/cli/http_cache.rs
index f70ccc715..6cda9f279 100644
--- a/cli/http_cache.rs
+++ b/cli/http_cache.rs
@@ -73,11 +73,6 @@ pub fn url_to_filename(url: &Url) -> Option<PathBuf> {
   Some(cache_filename)
 }
 
-#[derive(Debug, Clone, Default)]
-pub struct HttpCache {
-  pub location: PathBuf,
-}
-
 #[derive(Serialize, Deserialize)]
 pub struct Metadata {
   pub headers: HeadersMap,
@@ -107,6 +102,11 @@ impl Metadata {
   }
 }
 
+#[derive(Debug, Clone, Default)]
+pub struct HttpCache {
+  pub location: PathBuf,
+}
+
 impl HttpCache {
   /// Returns a new instance.
   ///
diff --git a/cli/http_util.rs b/cli/http_util.rs
index 54d641bef..5cb843a42 100644
--- a/cli/http_util.rs
+++ b/cli/http_util.rs
@@ -1,5 +1,6 @@
 // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
 use crate::auth_tokens::AuthToken;
+use crate::version::get_user_agent;
 
 use cache_control::Cachability;
 use cache_control::CacheControl;
@@ -8,13 +9,15 @@ use deno_core::error::custom_error;
 use deno_core::error::generic_error;
 use deno_core::error::AnyError;
 use deno_core::url::Url;
+use deno_runtime::deno_fetch::create_http_client;
+use deno_runtime::deno_fetch::reqwest;
 use deno_runtime::deno_fetch::reqwest::header::HeaderValue;
 use deno_runtime::deno_fetch::reqwest::header::ACCEPT;
 use deno_runtime::deno_fetch::reqwest::header::AUTHORIZATION;
 use deno_runtime::deno_fetch::reqwest::header::IF_NONE_MATCH;
 use deno_runtime::deno_fetch::reqwest::header::LOCATION;
-use deno_runtime::deno_fetch::reqwest::Client;
 use deno_runtime::deno_fetch::reqwest::StatusCode;
+use deno_runtime::deno_tls::rustls::RootCertStore;
 use log::debug;
 use std::collections::HashMap;
 use std::time::Duration;
@@ -208,97 +211,125 @@ pub enum FetchOnceResult {
 
 #[derive(Debug)]
 pub struct FetchOnceArgs {
-  pub client: Client,
   pub url: Url,
   pub maybe_accept: Option<String>,
   pub maybe_etag: Option<String>,
   pub maybe_auth_token: Option<AuthToken>,
 }
 
-/// Asynchronously fetches the given HTTP URL one pass only.
-/// If no redirect is present and no error occurs,
-/// yields Code(ResultPayload).
-/// If redirect occurs, does not follow and
-/// yields Redirect(url).
-pub async fn fetch_once(
-  args: FetchOnceArgs,
-) -> Result<FetchOnceResult, AnyError> {
-  let mut request = args.client.get(args.url.clone());
+#[derive(Debug, Clone)]
+pub struct HttpClient(reqwest::Client);
 
-  if let Some(etag) = args.maybe_etag {
-    let if_none_match_val = HeaderValue::from_str(&etag)?;
-    request = request.header(IF_NONE_MATCH, if_none_match_val);
-  }
-  if let Some(auth_token) = args.maybe_auth_token {
-    let authorization_val = HeaderValue::from_str(&auth_token.to_string())?;
-    request = request.header(AUTHORIZATION, authorization_val);
+impl HttpClient {
+  pub fn new(
+    root_cert_store: Option<RootCertStore>,
+    unsafely_ignore_certificate_errors: Option<Vec<String>>,
+  ) -> Result<Self, AnyError> {
+    Ok(HttpClient::from_client(create_http_client(
+      get_user_agent(),
+      root_cert_store,
+      vec![],
+      None,
+      unsafely_ignore_certificate_errors,
+      None,
+    )?))
   }
-  if let Some(accept) = args.maybe_accept {
-    let accepts_val = HeaderValue::from_str(&accept)?;
-    request = request.header(ACCEPT, accepts_val);
+
+  pub fn from_client(client: reqwest::Client) -> Self {
+    Self(client)
   }
-  let response = request.send().await?;
 
-  if response.status() == StatusCode::NOT_MODIFIED {
-    return Ok(FetchOnceResult::NotModified);
+  pub fn get<U: reqwest::IntoUrl>(&self, url: U) -> reqwest::RequestBuilder {
+    self.0.get(url)
   }
 
-  let mut headers_: HashMap<String, String> = HashMap::new();
-  let headers = response.headers();
+  /// Asynchronously fetches the given HTTP URL one pass only.
+  /// If no redirect is present and no error occurs,
+  /// yields Code(ResultPayload).
+  /// If redirect occurs, does not follow and
+  /// yields Redirect(url).
+  pub async fn fetch_once(
+    &self,
+    args: FetchOnceArgs,
+  ) -> Result<FetchOnceResult, AnyError> {
+    let mut request = self.get(args.url.clone());
 
-  if let Some(warning) = headers.get("X-Deno-Warning") {
-    eprintln!(
-      "{} {}",
-      crate::colors::yellow("Warning"),
-      warning.to_str().unwrap()
-    );
-  }
+    if let Some(etag) = args.maybe_etag {
+      let if_none_match_val = HeaderValue::from_str(&etag)?;
+      request = request.header(IF_NONE_MATCH, if_none_match_val);
+    }
+    if let Some(auth_token) = args.maybe_auth_token {
+      let authorization_val = HeaderValue::from_str(&auth_token.to_string())?;
+      request = request.header(AUTHORIZATION, authorization_val);
+    }
+    if let Some(accept) = args.maybe_accept {
+      let accepts_val = HeaderValue::from_str(&accept)?;
+      request = request.header(ACCEPT, accepts_val);
+    }
+    let response = request.send().await?;
 
-  for key in headers.keys() {
-    let key_str = key.to_string();
-    let values = headers.get_all(key);
-    let values_str = values
-      .iter()
-      .map(|e| e.to_str().unwrap().to_string())
-      .collect::<Vec<String>>()
-      .join(",");
-    headers_.insert(key_str, values_str);
-  }
+    if response.status() == StatusCode::NOT_MODIFIED {
+      return Ok(FetchOnceResult::NotModified);
+    }
 
-  if response.status().is_redirection() {
-    if let Some(location) = response.headers().get(LOCATION) {
-      let location_string = location.to_str().unwrap();
-      debug!("Redirecting to {:?}...", &location_string);
-      let new_url = resolve_url_from_location(&args.url, location_string);
-      return Ok(FetchOnceResult::Redirect(new_url, headers_));
-    } else {
-      return Err(generic_error(format!(
-        "Redirection from '{}' did not provide location header",
-        args.url
-      )));
+    let mut result_headers = HashMap::new();
+    let response_headers = response.headers();
+
+    if let Some(warning) = response_headers.get("X-Deno-Warning") {
+      eprintln!(
+        "{} {}",
+        crate::colors::yellow("Warning"),
+        warning.to_str().unwrap()
+      );
     }
-  }
 
-  if response.status().is_client_error() || response.status().is_server_error()
-  {
-    let err = if response.status() == StatusCode::NOT_FOUND {
-      custom_error(
-        "NotFound",
-        format!("Import '{}' failed, not found.", args.url),
-      )
-    } else {
-      generic_error(format!(
-        "Import '{}' failed: {}",
-        args.url,
-        response.status()
-      ))
-    };
-    return Err(err);
-  }
+    for key in response_headers.keys() {
+      let key_str = key.to_string();
+      let values = response_headers.get_all(key);
+      let values_str = values
+        .iter()
+        .map(|e| e.to_str().unwrap().to_string())
+        .collect::<Vec<String>>()
+        .join(",");
+      result_headers.insert(key_str, values_str);
+    }
+
+    if response.status().is_redirection() {
+      if let Some(location) = response.headers().get(LOCATION) {
+        let location_string = location.to_str().unwrap();
+        debug!("Redirecting to {:?}...", &location_string);
+        let new_url = resolve_url_from_location(&args.url, location_string);
+        return Ok(FetchOnceResult::Redirect(new_url, result_headers));
+      } else {
+        return Err(generic_error(format!(
+          "Redirection from '{}' did not provide location header",
+          args.url
+        )));
+      }
+    }
+
+    if response.status().is_client_error()
+      || response.status().is_server_error()
+    {
+      let err = if response.status() == StatusCode::NOT_FOUND {
+        custom_error(
+          "NotFound",
+          format!("Import '{}' failed, not found.", args.url),
+        )
+      } else {
+        generic_error(format!(
+          "Import '{}' failed: {}",
+          args.url,
+          response.status()
+        ))
+      };
+      return Err(err);
+    }
 
-  let body = response.bytes().await?.to_vec();
+    let body = response.bytes().await?.to_vec();
 
-  Ok(FetchOnceResult::Code(body, headers_))
+    Ok(FetchOnceResult::Code(body, result_headers))
+  }
 }
 
 #[cfg(test)]
@@ -308,16 +339,18 @@ mod tests {
   use deno_runtime::deno_fetch::create_http_client;
   use std::fs::read;
 
-  fn create_test_client() -> Client {
-    create_http_client(
-      "test_client".to_string(),
-      None,
-      vec![],
-      None,
-      None,
-      None,
+  fn create_test_client() -> HttpClient {
+    HttpClient::from_client(
+      create_http_client(
+        "test_client".to_string(),
+        None,
+        vec![],
+        None,
+        None,
+        None,
+      )
+      .unwrap(),
     )
-    .unwrap()
   }
 
   #[tokio::test]
@@ -326,14 +359,14 @@ mod tests {
     // Relies on external http server. See target/debug/test_server
     let url = Url::parse("http://127.0.0.1:4545/assets/fixture.json").unwrap();
     let client = create_test_client();
-    let result = fetch_once(FetchOnceArgs {
-      client,
-      url,
-      maybe_accept: None,
-      maybe_etag: None,
-      maybe_auth_token: None,
-    })
-    .await;
+    let result = client
+      .fetch_once(FetchOnceArgs {
+        url,
+        maybe_accept: None,
+        maybe_etag: None,
+        maybe_auth_token: None,
+      })
+      .await;
     if let Ok(FetchOnceResult::Code(body, headers)) = result {
       assert!(!body.is_empty());
       assert_eq!(headers.get("content-type").unwrap(), "application/json");
@@ -351,14 +384,14 @@ mod tests {
     let url = Url::parse("http://127.0.0.1:4545/run/import_compression/gziped")
       .unwrap();
     let client = create_test_client();
-    let result = fetch_once(FetchOnceArgs {
-      client,
-      url,
-      maybe_accept: None,
-      maybe_etag: None,
-      maybe_auth_token: None,
-    })
-    .await;
+    let result = client
+      .fetch_once(FetchOnceArgs {
+        url,
+        maybe_accept: None,
+        maybe_etag: None,
+        maybe_auth_token: None,
+      })
+      .await;
     if let Ok(FetchOnceResult::Code(body, headers)) = result {
       assert_eq!(String::from_utf8(body).unwrap(), "console.log('gzip')");
       assert_eq!(
@@ -377,14 +410,14 @@ mod tests {
     let _http_server_guard = test_util::http_server();
     let url = Url::parse("http://127.0.0.1:4545/etag_script.ts").unwrap();
     let client = create_test_client();
-    let result = fetch_once(FetchOnceArgs {
-      client: client.clone(),
-      url: url.clone(),
-      maybe_accept: None,
-      maybe_etag: None,
-      maybe_auth_token: None,
-    })
-    .await;
+    let result = client
+      .fetch_once(FetchOnceArgs {
+        url: url.clone(),
+        maybe_accept: None,
+        maybe_etag: None,
+        maybe_auth_token: None,
+      })
+      .await;
     if let Ok(FetchOnceResult::Code(body, headers)) = result {
       assert!(!body.is_empty());
       assert_eq!(String::from_utf8(body).unwrap(), "console.log('etag')");
@@ -397,14 +430,14 @@ mod tests {
       panic!();
     }
 
-    let res = fetch_once(FetchOnceArgs {
-      client,
-      url,
-      maybe_accept: None,
-      maybe_etag: Some("33a64df551425fcc55e".to_string()),
-      maybe_auth_token: None,
-    })
-    .await;
+    let res = client
+      .fetch_once(FetchOnceArgs {
+        url,
+        maybe_accept: None,
+        maybe_etag: Some("33a64df551425fcc55e".to_string()),
+        maybe_auth_token: None,
+      })
+      .await;
     assert_eq!(res.unwrap(), FetchOnceResult::NotModified);
   }
 
@@ -415,14 +448,14 @@ mod tests {
     let url = Url::parse("http://127.0.0.1:4545/run/import_compression/brotli")
       .unwrap();
     let client = create_test_client();
-    let result = fetch_once(FetchOnceArgs {
-      client,
-      url,
-      maybe_accept: None,
-      maybe_etag: None,
-      maybe_auth_token: None,
-    })
-    .await;
+    let result = client
+      .fetch_once(FetchOnceArgs {
+        url,
+        maybe_accept: None,
+        maybe_etag: None,
+        maybe_auth_token: None,
+      })
+      .await;
     if let Ok(FetchOnceResult::Code(body, headers)) = result {
       assert!(!body.is_empty());
       assert_eq!(String::from_utf8(body).unwrap(), "console.log('brotli');");
@@ -443,14 +476,14 @@ mod tests {
     // Relies on external http server. See target/debug/test_server
     let url = Url::parse("http://127.0.0.1:4545/echo_accept").unwrap();
     let client = create_test_client();
-    let result = fetch_once(FetchOnceArgs {
-      client,
-      url,
-      maybe_accept: Some("application/json".to_string()),
-      maybe_etag: None,
-      maybe_auth_token: None,
-    })
-    .await;
+    let result = client
+      .fetch_once(FetchOnceArgs {
+        url,
+        maybe_accept: Some("application/json".to_string()),
+        maybe_etag: None,
+        maybe_auth_token: None,
+      })
+      .await;
     if let Ok(FetchOnceResult::Code(body, _)) = result {
       assert_eq!(body, r#"{"accept":"application/json"}"#.as_bytes());
     } else {
@@ -467,14 +500,14 @@ mod tests {
     let target_url =
       Url::parse("http://localhost:4545/assets/fixture.json").unwrap();
     let client = create_test_client();
-    let result = fetch_once(FetchOnceArgs {
-      client,
-      url,
-      maybe_accept: None,
-      maybe_etag: None,
-      maybe_auth_token: None,
-    })
-    .await;
+    let result = client
+      .fetch_once(FetchOnceArgs {
+        url,
+        maybe_accept: None,
+        maybe_etag: None,
+        maybe_auth_token: None,
+      })
+      .await;
     if let Ok(FetchOnceResult::Redirect(url, _)) = result {
       assert_eq!(url, target_url);
     } else {
@@ -526,29 +559,31 @@ mod tests {
     // Relies on external http server. See target/debug/test_server
     let url = Url::parse("https://localhost:5545/assets/fixture.json").unwrap();
 
-    let client = create_http_client(
-      version::get_user_agent(),
-      None,
-      vec![read(
-        test_util::testdata_path()
-          .join("tls/RootCA.pem")
-          .to_str()
-          .unwrap(),
+    let client = HttpClient::from_client(
+      create_http_client(
+        version::get_user_agent(),
+        None,
+        vec![read(
+          test_util::testdata_path()
+            .join("tls/RootCA.pem")
+            .to_str()
+            .unwrap(),
+        )
+        .unwrap()],
+        None,
+        None,
+        None,
       )
-      .unwrap()],
-      None,
-      None,
-      None,
-    )
-    .unwrap();
-    let result = fetch_once(FetchOnceArgs {
-      client,
-      url,
-      maybe_accept: None,
-      maybe_etag: None,
-      maybe_auth_token: None,
-    })
-    .await;
+      .unwrap(),
+    );
+    let result = client
+      .fetch_once(FetchOnceArgs {
+        url,
+        maybe_accept: None,
+        maybe_etag: None,
+        maybe_auth_token: None,
+      })
+      .await;
     if let Ok(FetchOnceResult::Code(body, headers)) = result {
       assert!(!body.is_empty());
       assert_eq!(headers.get("content-type").unwrap(), "application/json");
@@ -564,24 +599,26 @@ mod tests {
     let _http_server_guard = test_util::http_server();
     // Relies on external http server with a valid mozilla root CA cert.
     let url = Url::parse("https://deno.land").unwrap();
-    let client = create_http_client(
-      version::get_user_agent(),
-      None, // This will load mozilla certs by default
-      vec![],
-      None,
-      None,
-      None,
-    )
-    .unwrap();
+    let client = HttpClient::from_client(
+      create_http_client(
+        version::get_user_agent(),
+        None, // This will load mozilla certs by default
+        vec![],
+        None,
+        None,
+        None,
+      )
+      .unwrap(),
+    );
 
-    let result = fetch_once(FetchOnceArgs {
-      client,
-      url,
-      maybe_accept: None,
-      maybe_etag: None,
-      maybe_auth_token: None,
-    })
-    .await;
+    let result = client
+      .fetch_once(FetchOnceArgs {
+        url,
+        maybe_accept: None,
+        maybe_etag: None,
+        maybe_auth_token: None,
+      })
+      .await;
 
     println!("{:?}", result);
     if let Ok(FetchOnceResult::Code(body, _headers)) = result {
@@ -601,24 +638,20 @@ mod tests {
     let _http_server_guard = test_util::http_server();
     // Relies on external http server with a valid mozilla root CA cert.
     let url = Url::parse("https://deno.land").unwrap();
-    let client = create_http_client(
-      version::get_user_agent(),
+    let client = HttpClient::new(
       Some(RootCertStore::empty()), // no certs loaded at all
-      vec![],
-      None,
-      None,
       None,
     )
     .unwrap();
 
-    let result = fetch_once(FetchOnceArgs {
-      client,
-      url,
-      maybe_accept: None,
-      maybe_etag: None,
-      maybe_auth_token: None,
-    })
-    .await;
+    let result = client
+      .fetch_once(FetchOnceArgs {
+        url,
+        maybe_accept: None,
+        maybe_etag: None,
+        maybe_auth_token: None,
+      })
+      .await;
 
     if let Ok(FetchOnceResult::Code(_body, _headers)) = result {
       // This test is expected to fail since to CA certs have been loaded
@@ -633,29 +666,31 @@ mod tests {
     let url =
       Url::parse("https://localhost:5545/run/import_compression/gziped")
         .unwrap();
-    let client = create_http_client(
-      version::get_user_agent(),
-      None,
-      vec![read(
-        test_util::testdata_path()
-          .join("tls/RootCA.pem")
-          .to_str()
-          .unwrap(),
+    let client = HttpClient::from_client(
+      create_http_client(
+        version::get_user_agent(),
+        None,
+        vec![read(
+          test_util::testdata_path()
+            .join("tls/RootCA.pem")
+            .to_str()
+            .unwrap(),
+        )
+        .unwrap()],
+        None,
+        None,
+        None,
       )
-      .unwrap()],
-      None,
-      None,
-      None,
-    )
-    .unwrap();
-    let result = fetch_once(FetchOnceArgs {
-      client,
-      url,
-      maybe_accept: None,
-      maybe_etag: None,
-      maybe_auth_token: None,
-    })
-    .await;
+      .unwrap(),
+    );
+    let result = client
+      .fetch_once(FetchOnceArgs {
+        url,
+        maybe_accept: None,
+        maybe_etag: None,
+        maybe_auth_token: None,
+      })
+      .await;
     if let Ok(FetchOnceResult::Code(body, headers)) = result {
       assert_eq!(String::from_utf8(body).unwrap(), "console.log('gzip')");
       assert_eq!(
@@ -673,29 +708,31 @@ mod tests {
   async fn test_fetch_with_cafile_with_etag() {
     let _http_server_guard = test_util::http_server();
     let url = Url::parse("https://localhost:5545/etag_script.ts").unwrap();
-    let client = create_http_client(
-      version::get_user_agent(),
-      None,
-      vec![read(
-        test_util::testdata_path()
-          .join("tls/RootCA.pem")
-          .to_str()
-          .unwrap(),
+    let client = HttpClient::from_client(
+      create_http_client(
+        version::get_user_agent(),
+        None,
+        vec![read(
+          test_util::testdata_path()
+            .join("tls/RootCA.pem")
+            .to_str()
+            .unwrap(),
+        )
+        .unwrap()],
+        None,
+        None,
+        None,
       )
-      .unwrap()],
-      None,
-      None,
-      None,
-    )
-    .unwrap();
-    let result = fetch_once(FetchOnceArgs {
-      client: client.clone(),
-      url: url.clone(),
-      maybe_accept: None,
-      maybe_etag: None,
-      maybe_auth_token: None,
-    })
-    .await;
+      .unwrap(),
+    );
+    let result = client
+      .fetch_once(FetchOnceArgs {
+        url: url.clone(),
+        maybe_accept: None,
+        maybe_etag: None,
+        maybe_auth_token: None,
+      })
+      .await;
     if let Ok(FetchOnceResult::Code(body, headers)) = result {
       assert!(!body.is_empty());
       assert_eq!(String::from_utf8(body).unwrap(), "console.log('etag')");
@@ -709,14 +746,14 @@ mod tests {
       panic!();
     }
 
-    let res = fetch_once(FetchOnceArgs {
-      client,
-      url,
-      maybe_accept: None,
-      maybe_etag: Some("33a64df551425fcc55e".to_string()),
-      maybe_auth_token: None,
-    })
-    .await;
+    let res = client
+      .fetch_once(FetchOnceArgs {
+        url,
+        maybe_accept: None,
+        maybe_etag: Some("33a64df551425fcc55e".to_string()),
+        maybe_auth_token: None,
+      })
+      .await;
     assert_eq!(res.unwrap(), FetchOnceResult::NotModified);
   }
 
@@ -727,29 +764,31 @@ mod tests {
     let url =
       Url::parse("https://localhost:5545/run/import_compression/brotli")
         .unwrap();
-    let client = create_http_client(
-      version::get_user_agent(),
-      None,
-      vec![read(
-        test_util::testdata_path()
-          .join("tls/RootCA.pem")
-          .to_str()
-          .unwrap(),
+    let client = HttpClient::from_client(
+      create_http_client(
+        version::get_user_agent(),
+        None,
+        vec![read(
+          test_util::testdata_path()
+            .join("tls/RootCA.pem")
+            .to_str()
+            .unwrap(),
+        )
+        .unwrap()],
+        None,
+        None,
+        None,
       )
-      .unwrap()],
-      None,
-      None,
-      None,
-    )
-    .unwrap();
-    let result = fetch_once(FetchOnceArgs {
-      client,
-      url,
-      maybe_accept: None,
-      maybe_etag: None,
-      maybe_auth_token: None,
-    })
-    .await;
+      .unwrap(),
+    );
+    let result = client
+      .fetch_once(FetchOnceArgs {
+        url,
+        maybe_accept: None,
+        maybe_etag: None,
+        maybe_auth_token: None,
+      })
+      .await;
     if let Ok(FetchOnceResult::Code(body, headers)) = result {
       assert!(!body.is_empty());
       assert_eq!(String::from_utf8(body).unwrap(), "console.log('brotli');");
@@ -770,14 +809,14 @@ mod tests {
     let url_str = "http://127.0.0.1:4545/bad_redirect";
     let url = Url::parse(url_str).unwrap();
     let client = create_test_client();
-    let result = fetch_once(FetchOnceArgs {
-      client,
-      url,
-      maybe_accept: None,
-      maybe_etag: None,
-      maybe_auth_token: None,
-    })
-    .await;
+    let result = client
+      .fetch_once(FetchOnceArgs {
+        url,
+        maybe_accept: None,
+        maybe_etag: None,
+        maybe_auth_token: None,
+      })
+      .await;
     assert!(result.is_err());
     let err = result.unwrap_err();
     // Check that the error message contains the original URL
diff --git a/cli/lsp/language_server.rs b/cli/lsp/language_server.rs
index cf1fbba97..4595d2146 100644
--- a/cli/lsp/language_server.rs
+++ b/cli/lsp/language_server.rs
@@ -50,7 +50,6 @@ use super::parent_process_checker;
 use super::performance::Performance;
 use super::refactor;
 use super::registries::ModuleRegistry;
-use super::registries::ModuleRegistryOptions;
 use super::testing;
 use super::text;
 use super::tsc;
@@ -65,10 +64,13 @@ use crate::args::FmtConfig;
 use crate::args::LintConfig;
 use crate::args::TsConfig;
 use crate::deno_dir;
+use crate::deno_dir::DenoDir;
+use crate::file_fetcher::get_root_cert_store;
 use crate::file_fetcher::get_source_from_data_url;
 use crate::file_fetcher::CacheSetting;
 use crate::fs_util;
 use crate::graph_util::graph_valid;
+use crate::http_util::HttpClient;
 use crate::npm::NpmCache;
 use crate::npm::NpmPackageResolver;
 use crate::npm::RealNpmRegistryApi;
@@ -235,17 +237,43 @@ impl LanguageServer {
   }
 }
 
+fn create_lsp_npm_resolver(
+  dir: &DenoDir,
+  http_client: HttpClient,
+) -> NpmPackageResolver {
+  let registry_url = RealNpmRegistryApi::default_url();
+  // Use an "only" cache setting in order to make the
+  // user do an explicit "cache" command and prevent
+  // the cache from being filled with lots of packages while
+  // the user is typing.
+  let cache_setting = CacheSetting::Only;
+  let progress_bar = ProgressBar::default();
+  let npm_cache = NpmCache::from_deno_dir(
+    dir,
+    cache_setting.clone(),
+    http_client.clone(),
+    progress_bar.clone(),
+  );
+  let api = RealNpmRegistryApi::new(
+    registry_url,
+    npm_cache.clone(),
+    cache_setting,
+    http_client,
+    progress_bar,
+  );
+  NpmPackageResolver::new(npm_cache, api, false, None)
+}
+
 impl Inner {
   fn new(client: Client) -> Self {
     let maybe_custom_root = env::var("DENO_DIR").map(String::into).ok();
     let dir = deno_dir::DenoDir::new(maybe_custom_root)
       .expect("could not access DENO_DIR");
     let module_registries_location = dir.root.join(REGISTRIES_PATH);
-    let module_registries = ModuleRegistry::new(
-      &module_registries_location,
-      ModuleRegistryOptions::default(),
-    )
-    .expect("could not create module registries");
+    let http_client = HttpClient::new(None, None).unwrap();
+    let module_registries =
+      ModuleRegistry::new(&module_registries_location, http_client.clone())
+        .unwrap();
     let location = dir.root.join(CACHE_PATH);
     let documents = Documents::new(&location);
     let cache_metadata = cache::CacheMetadata::new(&location);
@@ -258,25 +286,7 @@ impl Inner {
       ts_server.clone(),
     );
     let assets = Assets::new(ts_server.clone());
-    let registry_url = RealNpmRegistryApi::default_url();
-    // Use an "only" cache setting in order to make the
-    // user do an explicit "cache" command and prevent
-    // the cache from being filled with lots of packages while
-    // the user is typing.
-    let cache_setting = CacheSetting::Only;
-    let progress_bar = ProgressBar::default();
-    let npm_cache = NpmCache::from_deno_dir(
-      &dir,
-      cache_setting.clone(),
-      progress_bar.clone(),
-    );
-    let api = RealNpmRegistryApi::new(
-      registry_url,
-      npm_cache.clone(),
-      cache_setting,
-      progress_bar,
-    );
-    let npm_resolver = NpmPackageResolver::new(npm_cache, api, false, None);
+    let npm_resolver = create_lsp_npm_resolver(&dir, http_client);
 
     Self {
       assets,
@@ -498,36 +508,48 @@ impl Inner {
       None
     };
     if self.maybe_cache_path != maybe_cache_path {
-      let maybe_custom_root = maybe_cache_path
-        .clone()
-        .or_else(|| env::var("DENO_DIR").map(String::into).ok());
-      let dir = deno_dir::DenoDir::new(maybe_custom_root)?;
-      let module_registries_location = dir.root.join(REGISTRIES_PATH);
-      let workspace_settings = self.config.get_workspace_settings();
-      let maybe_root_path = self
-        .config
-        .root_uri
-        .as_ref()
-        .and_then(|uri| fs_util::specifier_to_file_path(uri).ok());
-      self.module_registries = ModuleRegistry::new(
-        &module_registries_location,
-        ModuleRegistryOptions {
-          maybe_root_path,
-          maybe_ca_stores: workspace_settings.certificate_stores.clone(),
-          maybe_ca_file: workspace_settings.tls_certificate.clone(),
-          unsafely_ignore_certificate_errors: workspace_settings
-            .unsafely_ignore_certificate_errors,
-        },
-      )?;
-      self.module_registries_location = module_registries_location;
-      let location = dir.root.join(CACHE_PATH);
-      self.documents.set_location(&location);
-      self.cache_metadata.set_location(&location);
-      self.maybe_cache_path = maybe_cache_path;
+      self.recreate_http_client_and_dependents(maybe_cache_path)?;
     }
     Ok(())
   }
 
+  /// Recreates the http client and all dependent structs.
+  fn recreate_http_client_and_dependents(
+    &mut self,
+    new_cache_path: Option<PathBuf>,
+  ) -> Result<(), AnyError> {
+    let maybe_custom_root = new_cache_path
+      .clone()
+      .or_else(|| env::var("DENO_DIR").map(String::into).ok());
+    let dir = deno_dir::DenoDir::new(maybe_custom_root)?;
+    let workspace_settings = self.config.get_workspace_settings();
+    let maybe_root_path = self
+      .config
+      .root_uri
+      .as_ref()
+      .and_then(|uri| fs_util::specifier_to_file_path(uri).ok());
+    let root_cert_store = Some(get_root_cert_store(
+      maybe_root_path,
+      workspace_settings.certificate_stores.clone(),
+      workspace_settings.tls_certificate.clone(),
+    )?);
+    let client = HttpClient::new(
+      root_cert_store,
+      workspace_settings.unsafely_ignore_certificate_errors,
+    )?;
+    let module_registries_location = dir.root.join(REGISTRIES_PATH);
+    self.module_registries =
+      ModuleRegistry::new(&module_registries_location, client.clone())?;
+    self.module_registries_location = module_registries_location;
+    self.npm_resolver = create_lsp_npm_resolver(&dir, client);
+    // update the cache path
+    let location = dir.root.join(CACHE_PATH);
+    self.documents.set_location(&location);
+    self.cache_metadata.set_location(&location);
+    self.maybe_cache_path = new_cache_path;
+    Ok(())
+  }
+
   pub async fn update_import_map(&mut self) -> Result<(), AnyError> {
     let mark = self.performance.mark("update_import_map", None::<()>);
 
@@ -627,23 +649,8 @@ impl Inner {
 
   async fn update_registries(&mut self) -> Result<(), AnyError> {
     let mark = self.performance.mark("update_registries", None::<()>);
+    self.recreate_http_client_and_dependents(self.maybe_cache_path.clone())?;
     let workspace_settings = self.config.get_workspace_settings();
-    let maybe_root_path = self
-      .config
-      .root_uri
-      .as_ref()
-      .and_then(|uri| fs_util::specifier_to_file_path(uri).ok());
-    self.module_registries = ModuleRegistry::new(
-      &self.module_registries_location,
-      ModuleRegistryOptions {
-        maybe_root_path,
-        maybe_ca_stores: workspace_settings.certificate_stores.clone(),
-        maybe_ca_file: workspace_settings.tls_certificate.clone(),
-        unsafely_ignore_certificate_errors: workspace_settings
-          .unsafely_ignore_certificate_errors
-          .clone(),
-      },
-    )?;
     for (registry, enabled) in workspace_settings.suggest.imports.hosts.iter() {
       if *enabled {
         lsp_log!("Enabling import suggestions for: {}", registry);
diff --git a/cli/lsp/registries.rs b/cli/lsp/registries.rs
index 3da435666..6fd48d8b9 100644
--- a/cli/lsp/registries.rs
+++ b/cli/lsp/registries.rs
@@ -13,10 +13,10 @@ use super::path_to_regex::StringOrVec;
 use super::path_to_regex::Token;
 
 use crate::deno_dir;
-use crate::file_fetcher::get_root_cert_store;
 use crate::file_fetcher::CacheSetting;
 use crate::file_fetcher::FileFetcher;
 use crate::http_cache::HttpCache;
+use crate::http_util::HttpClient;
 
 use deno_core::anyhow::anyhow;
 use deno_core::error::AnyError;
@@ -36,7 +36,6 @@ use once_cell::sync::Lazy;
 use regex::Regex;
 use std::collections::HashMap;
 use std::path::Path;
-use std::path::PathBuf;
 use tower_lsp::lsp_types as lsp;
 
 const CONFIG_PATH: &str = "/.well-known/deno-import-intellisense.json";
@@ -409,14 +408,6 @@ enum VariableItems {
   List(VariableItemsList),
 }
 
-#[derive(Debug, Default)]
-pub struct ModuleRegistryOptions {
-  pub maybe_root_path: Option<PathBuf>,
-  pub maybe_ca_stores: Option<Vec<String>>,
-  pub maybe_ca_file: Option<String>,
-  pub unsafely_ignore_certificate_errors: Option<Vec<String>>,
-}
-
 /// A structure which holds the information about currently configured module
 /// registries and can provide completion information for URLs that match
 /// one of the enabled registries.
@@ -433,28 +424,23 @@ impl Default for ModuleRegistry {
     // custom root.
     let dir = deno_dir::DenoDir::new(None).unwrap();
     let location = dir.root.join("registries");
-    Self::new(&location, ModuleRegistryOptions::default()).unwrap()
+    let http_client = HttpClient::new(None, None).unwrap();
+    Self::new(&location, http_client).unwrap()
   }
 }
 
 impl ModuleRegistry {
   pub fn new(
     location: &Path,
-    options: ModuleRegistryOptions,
+    http_client: HttpClient,
   ) -> Result<Self, AnyError> {
     let http_cache = HttpCache::new(location);
-    let root_cert_store = Some(get_root_cert_store(
-      options.maybe_root_path,
-      options.maybe_ca_stores,
-      options.maybe_ca_file,
-    )?);
     let mut file_fetcher = FileFetcher::new(
       http_cache,
       CacheSetting::RespectHeaders,
       true,
-      root_cert_store,
+      http_client,
       BlobStore::default(),
-      options.unsafely_ignore_certificate_errors,
       None,
     )?;
     file_fetcher.set_download_log_level(super::logging::lsp_log_level());
@@ -1262,7 +1248,8 @@ mod tests {
     let temp_dir = TempDir::new();
     let location = temp_dir.path().join("registries");
     let mut module_registry =
-      ModuleRegistry::new(&location, ModuleRegistryOptions::default()).unwrap();
+      ModuleRegistry::new(&location, HttpClient::new(None, None).unwrap())
+        .unwrap();
     module_registry
       .enable("http://localhost:4545/")
       .await
@@ -1323,7 +1310,8 @@ mod tests {
     let temp_dir = TempDir::new();
     let location = temp_dir.path().join("registries");
     let mut module_registry =
-      ModuleRegistry::new(&location, ModuleRegistryOptions::default()).unwrap();
+      ModuleRegistry::new(&location, HttpClient::new(None, None).unwrap())
+        .unwrap();
     module_registry
       .enable("http://localhost:4545/")
       .await
@@ -1546,7 +1534,8 @@ mod tests {
     let temp_dir = TempDir::new();
     let location = temp_dir.path().join("registries");
     let mut module_registry =
-      ModuleRegistry::new(&location, ModuleRegistryOptions::default()).unwrap();
+      ModuleRegistry::new(&location, HttpClient::new(None, None).unwrap())
+        .unwrap();
     module_registry
       .enable_custom("http://localhost:4545/lsp/registries/deno-import-intellisense-key-first.json")
       .await
@@ -1616,7 +1605,8 @@ mod tests {
     let temp_dir = TempDir::new();
     let location = temp_dir.path().join("registries");
     let mut module_registry =
-      ModuleRegistry::new(&location, ModuleRegistryOptions::default()).unwrap();
+      ModuleRegistry::new(&location, HttpClient::new(None, None).unwrap())
+        .unwrap();
     module_registry
       .enable_custom("http://localhost:4545/lsp/registries/deno-import-intellisense-complex.json")
       .await
@@ -1667,7 +1657,8 @@ mod tests {
     let temp_dir = TempDir::new();
     let location = temp_dir.path().join("registries");
     let module_registry =
-      ModuleRegistry::new(&location, ModuleRegistryOptions::default()).unwrap();
+      ModuleRegistry::new(&location, HttpClient::new(None, None).unwrap())
+        .unwrap();
     let result = module_registry.check_origin("http://localhost:4545").await;
     assert!(result.is_ok());
   }
@@ -1678,7 +1669,8 @@ mod tests {
     let temp_dir = TempDir::new();
     let location = temp_dir.path().join("registries");
     let module_registry =
-      ModuleRegistry::new(&location, ModuleRegistryOptions::default()).unwrap();
+      ModuleRegistry::new(&location, HttpClient::new(None, None).unwrap())
+        .unwrap();
     let result = module_registry.check_origin("https://deno.com").await;
     assert!(result.is_err());
     let err = result.unwrap_err().to_string();
diff --git a/cli/npm/cache.rs b/cli/npm/cache.rs
index 2ca597bd7..ce2208db7 100644
--- a/cli/npm/cache.rs
+++ b/cli/npm/cache.rs
@@ -10,11 +10,11 @@ use deno_core::anyhow::Context;
 use deno_core::error::custom_error;
 use deno_core::error::AnyError;
 use deno_core::url::Url;
-use deno_runtime::deno_fetch::reqwest;
 
 use crate::deno_dir::DenoDir;
 use crate::file_fetcher::CacheSetting;
 use crate::fs_util;
+use crate::http_util::HttpClient;
 use crate::progress_bar::ProgressBar;
 
 use super::registry::NpmPackageVersionDistInfo;
@@ -315,6 +315,7 @@ impl ReadonlyNpmCache {
 pub struct NpmCache {
   readonly: ReadonlyNpmCache,
   cache_setting: CacheSetting,
+  http_client: HttpClient,
   progress_bar: ProgressBar,
 }
 
@@ -322,11 +323,13 @@ impl NpmCache {
   pub fn from_deno_dir(
     dir: &DenoDir,
     cache_setting: CacheSetting,
+    http_client: HttpClient,
     progress_bar: ProgressBar,
   ) -> Self {
     Self {
       readonly: ReadonlyNpmCache::from_deno_dir(dir),
       cache_setting,
+      http_client,
       progress_bar,
     }
   }
@@ -383,7 +386,7 @@ impl NpmCache {
     }
 
     let _guard = self.progress_bar.update(&dist.tarball);
-    let response = reqwest::get(&dist.tarball).await?;
+    let response = self.http_client.get(&dist.tarball).send().await?;
 
     if response.status() == 404 {
       bail!("Could not find npm package tarball at: {}", dist.tarball);
diff --git a/cli/npm/registry.rs b/cli/npm/registry.rs
index 066c136ee..3f0a84165 100644
--- a/cli/npm/registry.rs
+++ b/cli/npm/registry.rs
@@ -18,12 +18,12 @@ use deno_core::serde::Deserialize;
 use deno_core::serde_json;
 use deno_core::url::Url;
 use deno_runtime::colors;
-use deno_runtime::deno_fetch::reqwest;
 use serde::Serialize;
 
 use crate::file_fetcher::CacheSetting;
 use crate::fs_util;
 use crate::http_cache::CACHE_PERM;
+use crate::http_util::HttpClient;
 use crate::progress_bar::ProgressBar;
 
 use super::cache::NpmCache;
@@ -249,6 +249,7 @@ impl RealNpmRegistryApi {
     base_url: Url,
     cache: NpmCache,
     cache_setting: CacheSetting,
+    http_client: HttpClient,
     progress_bar: ProgressBar,
   ) -> Self {
     Self(Arc::new(RealNpmRegistryApiInner {
@@ -256,6 +257,7 @@ impl RealNpmRegistryApi {
       cache,
       mem_cache: Default::default(),
       cache_setting,
+      http_client,
       progress_bar,
     }))
   }
@@ -285,6 +287,7 @@ struct RealNpmRegistryApiInner {
   cache: NpmCache,
   mem_cache: Mutex<HashMap<String, Option<Arc<NpmPackageInfo>>>>,
   cache_setting: CacheSetting,
+  http_client: HttpClient,
   progress_bar: ProgressBar,
 }
 
@@ -420,7 +423,7 @@ impl RealNpmRegistryApiInner {
     let package_url = self.get_package_url(name);
     let _guard = self.progress_bar.update(package_url.as_str());
 
-    let response = match reqwest::get(package_url).await {
+    let response = match self.http_client.get(package_url).send().await {
       Ok(response) => response,
       Err(err) => {
         // attempt to use the local cache
diff --git a/cli/proc_state.rs b/cli/proc_state.rs
index 9e66bdb66..a193adc53 100644
--- a/cli/proc_state.rs
+++ b/cli/proc_state.rs
@@ -19,6 +19,7 @@ use crate::graph_util::graph_lock_or_exit;
 use crate::graph_util::GraphData;
 use crate::graph_util::ModuleEntry;
 use crate::http_cache;
+use crate::http_util::HttpClient;
 use crate::lockfile::as_maybe_locker;
 use crate::lockfile::Lockfile;
 use crate::node;
@@ -157,15 +158,18 @@ impl ProcState {
     let root_cert_store = cli_options.resolve_root_cert_store()?;
     let cache_usage = cli_options.cache_setting();
     let progress_bar = ProgressBar::default();
+    let http_client = HttpClient::new(
+      Some(root_cert_store.clone()),
+      cli_options
+        .unsafely_ignore_certificate_errors()
+        .map(ToOwned::to_owned),
+    )?;
     let file_fetcher = FileFetcher::new(
       http_cache,
       cache_usage,
       !cli_options.no_remote(),
-      Some(root_cert_store.clone()),
+      http_client.clone(),
       blob_store.clone(),
-      cli_options
-        .unsafely_ignore_certificate_errors()
-        .map(ToOwned::to_owned),
       Some(progress_bar.clone()),
     )?;
 
@@ -216,12 +220,14 @@ impl ProcState {
     let npm_cache = NpmCache::from_deno_dir(
       &dir,
       cli_options.cache_setting(),
+      http_client.clone(),
       progress_bar.clone(),
     );
     let api = RealNpmRegistryApi::new(
       registry_url,
       npm_cache.clone(),
       cli_options.cache_setting(),
+      http_client,
       progress_bar.clone(),
     );
     let maybe_lockfile = lockfile.as_ref().cloned();