refactor: don't share `reqwest::HttpClient` across tokio runtimes (#24092)

This also fixes several issues where we weren't properly creating http
clients with the user's settings.

9 files changed, 260 insertions, 144 deletions

diff --git a/cli/tools/compile.rs b/cli/tools/compile.rs
index 7f31b9035..a29511af4 100644
--- a/cli/tools/compile.rs
+++ b/cli/tools/compile.rs
@@ -3,6 +3,7 @@
 use crate::args::CompileFlags;
 use crate::args::Flags;
 use crate::factory::CliFactory;
+use crate::http_util::HttpClientProvider;
 use crate::standalone::is_standalone_binary;
 use deno_core::anyhow::bail;
 use deno_core::anyhow::Context;
@@ -26,6 +27,7 @@ pub async fn compile(
   let module_graph_creator = factory.module_graph_creator().await?;
   let parsed_source_cache = factory.parsed_source_cache();
   let binary_writer = factory.create_compile_binary_writer().await?;
+  let http_client = factory.http_client_provider();
   let module_specifier = cli_options.resolve_main_module()?;
   let module_roots = {
     let mut vec = Vec::with_capacity(compile_flags.include.len() + 1);
@@ -49,6 +51,7 @@ pub async fn compile(
   }
 
   let output_path = resolve_compile_executable_output_path(
+    http_client,
     &compile_flags,
     cli_options.initial_cwd(),
   )
@@ -174,6 +177,7 @@ fn validate_output_path(output_path: &Path) -> Result<(), AnyError> {
 }
 
 async fn resolve_compile_executable_output_path(
+  http_client_provider: &HttpClientProvider,
   compile_flags: &CompileFlags,
   current_dir: &Path,
 ) -> Result<PathBuf, AnyError> {
@@ -184,9 +188,10 @@ async fn resolve_compile_executable_output_path(
   let mut output_path = if let Some(out) = output_flag.as_ref() {
     let mut out_path = PathBuf::from(out);
     if out.ends_with('/') || out.ends_with('\\') {
-      if let Some(infer_file_name) = infer_name_from_url(&module_specifier)
-        .await
-        .map(PathBuf::from)
+      if let Some(infer_file_name) =
+        infer_name_from_url(http_client_provider, &module_specifier)
+          .await
+          .map(PathBuf::from)
       {
         out_path = out_path.join(infer_file_name);
       }
@@ -199,7 +204,7 @@ async fn resolve_compile_executable_output_path(
   };
 
   if output_flag.is_none() {
-    output_path = infer_name_from_url(&module_specifier)
+    output_path = infer_name_from_url(http_client_provider, &module_specifier)
       .await
       .map(PathBuf::from)
   }
@@ -237,7 +242,9 @@ mod test {
 
   #[tokio::test]
   async fn resolve_compile_executable_output_path_target_linux() {
+    let http_client = HttpClientProvider::new(None, None);
     let path = resolve_compile_executable_output_path(
+      &http_client,
       &CompileFlags {
         source_file: "mod.ts".to_string(),
         output: Some(String::from("./file")),
@@ -259,7 +266,9 @@ mod test {
 
   #[tokio::test]
   async fn resolve_compile_executable_output_path_target_windows() {
+    let http_client = HttpClientProvider::new(None, None);
     let path = resolve_compile_executable_output_path(
+      &http_client,
       &CompileFlags {
         source_file: "mod.ts".to_string(),
         output: Some(String::from("./file")),
diff --git a/cli/tools/installer.rs b/cli/tools/installer.rs
index f810e9ca0..34ecc66be 100644
--- a/cli/tools/installer.rs
+++ b/cli/tools/installer.rs
@@ -12,7 +12,7 @@ use crate::args::TypeCheckMode;
 use crate::args::UninstallFlags;
 use crate::args::UninstallKind;
 use crate::factory::CliFactory;
-use crate::http_util::HttpClient;
+use crate::http_util::HttpClientProvider;
 use crate::util::fs::canonicalize_path_maybe_not_exists;
 
 use deno_config::ConfigFlag;
@@ -133,15 +133,21 @@ fn get_installer_root() -> Result<PathBuf, io::Error> {
   Ok(home_path)
 }
 
-pub async fn infer_name_from_url(url: &Url) -> Option<String> {
+pub async fn infer_name_from_url(
+  http_client_provider: &HttpClientProvider,
+  url: &Url,
+) -> Option<String> {
   // If there's an absolute url with no path, eg. https://my-cli.com
   // perform a request, and see if it redirects another file instead.
   let mut url = url.clone();
 
   if url.path() == "/" {
-    let client = HttpClient::new(None, None);
-    if let Ok(res) = client.get_redirected_response(url.clone(), None).await {
-      url = res.url().clone();
+    if let Ok(client) = http_client_provider.get_or_create() {
+      if let Ok(redirected_url) =
+        client.get_redirected_url(url.clone(), None).await
+      {
+        url = redirected_url;
+      }
     }
   }
 
@@ -295,16 +301,20 @@ pub async fn install_command(
     .await?
     .load_and_type_check_files(&[install_flags_global.module_url.clone()])
     .await?;
+  let http_client = factory.http_client_provider();
 
   // create the install shim
-  create_install_shim(flags, install_flags_global).await
+  create_install_shim(http_client, flags, install_flags_global).await
 }
 
 async fn create_install_shim(
+  http_client_provider: &HttpClientProvider,
   flags: Flags,
   install_flags_global: InstallFlagsGlobal,
 ) -> Result<(), AnyError> {
-  let shim_data = resolve_shim_data(&flags, &install_flags_global).await?;
+  let shim_data =
+    resolve_shim_data(http_client_provider, &flags, &install_flags_global)
+      .await?;
 
   // ensure directory exists
   if let Ok(metadata) = fs::metadata(&shim_data.installation_dir) {
@@ -355,6 +365,7 @@ struct ShimData {
 }
 
 async fn resolve_shim_data(
+  http_client_provider: &HttpClientProvider,
   flags: &Flags,
   install_flags_global: &InstallFlagsGlobal,
 ) -> Result<ShimData, AnyError> {
@@ -372,7 +383,7 @@ async fn resolve_shim_data(
   let name = if install_flags_global.name.is_some() {
     install_flags_global.name.clone()
   } else {
-    infer_name_from_url(&module_url).await
+    infer_name_from_url(http_client_provider, &module_url).await
   };
 
   let name = match name {
@@ -561,8 +572,10 @@ mod tests {
 
   #[tokio::test]
   async fn install_infer_name_from_url() {
+    let http_client = HttpClientProvider::new(None, None);
     assert_eq!(
       infer_name_from_url(
+        &http_client,
         &Url::parse("https://example.com/abc/server.ts").unwrap()
       )
       .await,
@@ -570,6 +583,7 @@ mod tests {
     );
     assert_eq!(
       infer_name_from_url(
+        &http_client,
         &Url::parse("https://example.com/abc/main.ts").unwrap()
       )
       .await,
@@ -577,6 +591,7 @@ mod tests {
     );
     assert_eq!(
       infer_name_from_url(
+        &http_client,
         &Url::parse("https://example.com/abc/mod.ts").unwrap()
       )
       .await,
@@ -584,6 +599,7 @@ mod tests {
     );
     assert_eq!(
       infer_name_from_url(
+        &http_client,
         &Url::parse("https://example.com/ab%20c/mod.ts").unwrap()
       )
       .await,
@@ -591,6 +607,7 @@ mod tests {
     );
     assert_eq!(
       infer_name_from_url(
+        &http_client,
         &Url::parse("https://example.com/abc/index.ts").unwrap()
       )
       .await,
@@ -598,42 +615,67 @@ mod tests {
     );
     assert_eq!(
       infer_name_from_url(
+        &http_client,
         &Url::parse("https://example.com/abc/cli.ts").unwrap()
       )
       .await,
       Some("abc".to_string())
     );
     assert_eq!(
-      infer_name_from_url(&Url::parse("https://example.com/main.ts").unwrap())
-        .await,
+      infer_name_from_url(
+        &http_client,
+        &Url::parse("https://example.com/main.ts").unwrap()
+      )
+      .await,
       Some("main".to_string())
     );
     assert_eq!(
-      infer_name_from_url(&Url::parse("https://example.com").unwrap()).await,
+      infer_name_from_url(
+        &http_client,
+        &Url::parse("https://example.com").unwrap()
+      )
+      .await,
       None
     );
     assert_eq!(
-      infer_name_from_url(&Url::parse("file:///abc/server.ts").unwrap()).await,
+      infer_name_from_url(
+        &http_client,
+        &Url::parse("file:///abc/server.ts").unwrap()
+      )
+      .await,
       Some("server".to_string())
     );
     assert_eq!(
-      infer_name_from_url(&Url::parse("file:///abc/main.ts").unwrap()).await,
+      infer_name_from_url(
+        &http_client,
+        &Url::parse("file:///abc/main.ts").unwrap()
+      )
+      .await,
       Some("abc".to_string())
     );
     assert_eq!(
-      infer_name_from_url(&Url::parse("file:///ab%20c/main.ts").unwrap()).await,
+      infer_name_from_url(
+        &http_client,
+        &Url::parse("file:///ab%20c/main.ts").unwrap()
+      )
+      .await,
       Some("ab c".to_string())
     );
     assert_eq!(
-      infer_name_from_url(&Url::parse("file:///main.ts").unwrap()).await,
+      infer_name_from_url(
+        &http_client,
+        &Url::parse("file:///main.ts").unwrap()
+      )
+      .await,
       Some("main".to_string())
     );
     assert_eq!(
-      infer_name_from_url(&Url::parse("file:///").unwrap()).await,
+      infer_name_from_url(&http_client, &Url::parse("file:///").unwrap()).await,
       None
     );
     assert_eq!(
       infer_name_from_url(
+        &http_client,
         &Url::parse("https://example.com/abc@0.1.0").unwrap()
       )
       .await,
@@ -641,6 +683,7 @@ mod tests {
     );
     assert_eq!(
       infer_name_from_url(
+        &http_client,
         &Url::parse("https://example.com/abc@0.1.0/main.ts").unwrap()
       )
       .await,
@@ -648,47 +691,71 @@ mod tests {
     );
     assert_eq!(
       infer_name_from_url(
+        &http_client,
         &Url::parse("https://example.com/abc@def@ghi").unwrap()
       )
       .await,
       Some("abc".to_string())
     );
     assert_eq!(
-      infer_name_from_url(&Url::parse("https://example.com/@abc.ts").unwrap())
-        .await,
+      infer_name_from_url(
+        &http_client,
+        &Url::parse("https://example.com/@abc.ts").unwrap()
+      )
+      .await,
       Some("@abc".to_string())
     );
     assert_eq!(
       infer_name_from_url(
+        &http_client,
         &Url::parse("https://example.com/@abc/mod.ts").unwrap()
       )
       .await,
       Some("@abc".to_string())
     );
     assert_eq!(
-      infer_name_from_url(&Url::parse("file:///@abc.ts").unwrap()).await,
+      infer_name_from_url(
+        &http_client,
+        &Url::parse("file:///@abc.ts").unwrap()
+      )
+      .await,
       Some("@abc".to_string())
     );
     assert_eq!(
-      infer_name_from_url(&Url::parse("file:///@abc/cli.ts").unwrap()).await,
+      infer_name_from_url(
+        &http_client,
+        &Url::parse("file:///@abc/cli.ts").unwrap()
+      )
+      .await,
       Some("@abc".to_string())
     );
     assert_eq!(
-      infer_name_from_url(&Url::parse("npm:cowsay@1.2/cowthink").unwrap())
-        .await,
+      infer_name_from_url(
+        &http_client,
+        &Url::parse("npm:cowsay@1.2/cowthink").unwrap()
+      )
+      .await,
       Some("cowthink".to_string())
     );
     assert_eq!(
-      infer_name_from_url(&Url::parse("npm:cowsay@1.2/cowthink/test").unwrap())
-        .await,
+      infer_name_from_url(
+        &http_client,
+        &Url::parse("npm:cowsay@1.2/cowthink/test").unwrap()
+      )
+      .await,
       Some("cowsay".to_string())
     );
     assert_eq!(
-      infer_name_from_url(&Url::parse("npm:cowsay@1.2").unwrap()).await,
+      infer_name_from_url(&http_client, &Url::parse("npm:cowsay@1.2").unwrap())
+        .await,
       Some("cowsay".to_string())
     );
     assert_eq!(
-      infer_name_from_url(&Url::parse("npm:@types/node@1.2").unwrap()).await,
+      infer_name_from_url(
+        &http_client,
+        &Url::parse("npm:@types/node@1.2").unwrap()
+      )
+      .await,
       None
     );
   }
@@ -700,6 +767,7 @@ mod tests {
     std::fs::create_dir(&bin_dir).unwrap();
 
     create_install_shim(
+      &HttpClientProvider::new(None, None),
       Flags {
         unstable_config: UnstableConfig {
           legacy_flag_enabled: true,
@@ -740,6 +808,7 @@ mod tests {
   #[tokio::test]
   async fn install_inferred_name() {
     let shim_data = resolve_shim_data(
+      &HttpClientProvider::new(None, None),
       &Flags::default(),
       &InstallFlagsGlobal {
         module_url: "http://localhost:4545/echo_server.ts".to_string(),
@@ -762,6 +831,7 @@ mod tests {
   #[tokio::test]
   async fn install_unstable_legacy() {
     let shim_data = resolve_shim_data(
+      &HttpClientProvider::new(None, None),
       &Flags {
         unstable_config: UnstableConfig {
           legacy_flag_enabled: true,
@@ -795,6 +865,7 @@ mod tests {
   #[tokio::test]
   async fn install_unstable_features() {
     let shim_data = resolve_shim_data(
+      &HttpClientProvider::new(None, None),
       &Flags {
         unstable_config: UnstableConfig {
           features: vec!["kv".to_string(), "cron".to_string()],
@@ -829,6 +900,7 @@ mod tests {
   #[tokio::test]
   async fn install_inferred_name_from_parent() {
     let shim_data = resolve_shim_data(
+      &HttpClientProvider::new(None, None),
       &Flags::default(),
       &InstallFlagsGlobal {
         module_url: "http://localhost:4545/subdir/main.ts".to_string(),
@@ -852,6 +924,7 @@ mod tests {
   async fn install_inferred_name_after_redirect_for_no_path_url() {
     let _http_server_guard = test_util::http_server();
     let shim_data = resolve_shim_data(
+      &HttpClientProvider::new(None, None),
       &Flags::default(),
       &InstallFlagsGlobal {
         module_url: "http://localhost:4550/?redirect_to=/subdir/redirects/a.ts"
@@ -879,6 +952,7 @@ mod tests {
   #[tokio::test]
   async fn install_custom_dir_option() {
     let shim_data = resolve_shim_data(
+      &HttpClientProvider::new(None, None),
       &Flags::default(),
       &InstallFlagsGlobal {
         module_url: "http://localhost:4545/echo_server.ts".to_string(),
@@ -901,6 +975,7 @@ mod tests {
   #[tokio::test]
   async fn install_with_flags() {
     let shim_data = resolve_shim_data(
+      &HttpClientProvider::new(None, None),
       &Flags {
         permissions: PermissionFlags {
           allow_net: Some(vec![]),
@@ -940,6 +1015,7 @@ mod tests {
   #[tokio::test]
   async fn install_prompt() {
     let shim_data = resolve_shim_data(
+      &HttpClientProvider::new(None, None),
       &Flags {
         permissions: PermissionFlags {
           no_prompt: true,
@@ -972,6 +1048,7 @@ mod tests {
   #[tokio::test]
   async fn install_allow_all() {
     let shim_data = resolve_shim_data(
+      &HttpClientProvider::new(None, None),
       &Flags {
         permissions: PermissionFlags {
           allow_all: true,
@@ -1005,6 +1082,7 @@ mod tests {
   async fn install_npm_lockfile_default() {
     let temp_dir = canonicalize_path(&env::temp_dir()).unwrap();
     let shim_data = resolve_shim_data(
+      &HttpClientProvider::new(None, None),
       &Flags {
         permissions: PermissionFlags {
           allow_all: true,
@@ -1041,6 +1119,7 @@ mod tests {
   #[tokio::test]
   async fn install_npm_no_lock() {
     let shim_data = resolve_shim_data(
+      &HttpClientProvider::new(None, None),
       &Flags {
         permissions: PermissionFlags {
           allow_all: true,
@@ -1083,6 +1162,7 @@ mod tests {
     let local_module_str = local_module.to_string_lossy();
 
     create_install_shim(
+      &HttpClientProvider::new(None, None),
       Flags::default(),
       InstallFlagsGlobal {
         module_url: local_module_str.to_string(),
@@ -1112,6 +1192,7 @@ mod tests {
     std::fs::create_dir(&bin_dir).unwrap();
 
     create_install_shim(
+      &HttpClientProvider::new(None, None),
       Flags::default(),
       InstallFlagsGlobal {
         module_url: "http://localhost:4545/echo_server.ts".to_string(),
@@ -1132,6 +1213,7 @@ mod tests {
 
     // No force. Install failed.
     let no_force_result = create_install_shim(
+      &HttpClientProvider::new(None, None),
       Flags::default(),
       InstallFlagsGlobal {
         module_url: "http://localhost:4545/cat.ts".to_string(), // using a different URL
@@ -1153,6 +1235,7 @@ mod tests {
 
     // Force. Install success.
     let force_result = create_install_shim(
+      &HttpClientProvider::new(None, None),
       Flags::default(),
       InstallFlagsGlobal {
         module_url: "http://localhost:4545/cat.ts".to_string(), // using a different URL
@@ -1180,6 +1263,7 @@ mod tests {
     assert!(result.is_ok());
 
     let result = create_install_shim(
+      &HttpClientProvider::new(None, None),
       Flags {
         config_flag: ConfigFlag::Path(config_file_path.to_string()),
         ..Flags::default()
@@ -1212,6 +1296,7 @@ mod tests {
     std::fs::create_dir(&bin_dir).unwrap();
 
     create_install_shim(
+      &HttpClientProvider::new(None, None),
       Flags::default(),
       InstallFlagsGlobal {
         module_url: "http://localhost:4545/echo_server.ts".to_string(),
@@ -1252,6 +1337,7 @@ mod tests {
     std::fs::write(&local_module, "// Some JavaScript I guess").unwrap();
 
     create_install_shim(
+      &HttpClientProvider::new(None, None),
       Flags::default(),
       InstallFlagsGlobal {
         module_url: local_module_str.to_string(),
@@ -1293,6 +1379,7 @@ mod tests {
     assert!(result.is_ok());
 
     let result = create_install_shim(
+      &HttpClientProvider::new(None, None),
       Flags {
         import_map_path: Some(import_map_path.to_string()),
         ..Flags::default()
@@ -1338,6 +1425,7 @@ mod tests {
     assert!(file_module_string.starts_with("file:///"));
 
     let result = create_install_shim(
+      &HttpClientProvider::new(None, None),
       Flags::default(),
       InstallFlagsGlobal {
         module_url: file_module_string.to_string(),
diff --git a/cli/tools/registry/api.rs b/cli/tools/registry/api.rs
index de9b4a333..c7097267d 100644
--- a/cli/tools/registry/api.rs
+++ b/cli/tools/registry/api.rs
@@ -6,6 +6,8 @@ use deno_runtime::deno_fetch::reqwest;
 use lsp_types::Url;
 use serde::de::DeserializeOwned;
 
+use crate::http_util::HttpClient;
+
 #[derive(serde::Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub struct CreateAuthorizationResponse {
@@ -116,8 +118,8 @@ pub async fn parse_response<T: DeserializeOwned>(
 }
 
 pub async fn get_scope(
-  client: &reqwest::Client,
-  registry_api_url: &str,
+  client: &HttpClient,
+  registry_api_url: &Url,
   scope: &str,
 ) -> Result<reqwest::Response, AnyError> {
   let scope_url = format!("{}scopes/{}", registry_api_url, scope);
@@ -126,7 +128,7 @@ pub async fn get_scope(
 }
 
 pub fn get_package_api_url(
-  registry_api_url: &str,
+  registry_api_url: &Url,
   scope: &str,
   package: &str,
 ) -> String {
@@ -134,8 +136,8 @@ pub fn get_package_api_url(
 }
 
 pub async fn get_package(
-  client: &reqwest::Client,
-  registry_api_url: &str,
+  client: &HttpClient,
+  registry_api_url: &Url,
   scope: &str,
   package: &str,
 ) -> Result<reqwest::Response, AnyError> {
diff --git a/cli/tools/registry/mod.rs b/cli/tools/registry/mod.rs
index 23e8f4313..d300e5eaf 100644
--- a/cli/tools/registry/mod.rs
+++ b/cli/tools/registry/mod.rs
@@ -17,11 +17,13 @@ use deno_config::WorkspaceMemberConfig;
 use deno_core::anyhow::bail;
 use deno_core::anyhow::Context;
 use deno_core::error::AnyError;
+use deno_core::futures::future::LocalBoxFuture;
+use deno_core::futures::stream::FuturesUnordered;
 use deno_core::futures::FutureExt;
+use deno_core::futures::StreamExt;
 use deno_core::serde_json;
 use deno_core::serde_json::json;
 use deno_core::serde_json::Value;
-use deno_core::unsync::JoinSet;
 use deno_runtime::deno_fetch::reqwest;
 use deno_runtime::deno_fs::FileSystem;
 use deno_terminal::colors;
@@ -154,7 +156,7 @@ pub async fn publish(
   }
 
   perform_publish(
-    cli_factory.http_client(),
+    &cli_factory.http_client_provider().get_or_create()?,
     prepared_data.publish_order_graph,
     prepared_data.package_by_name,
     auth_method,
@@ -523,9 +525,9 @@ pub enum Permission<'s> {
 }
 
 async fn get_auth_headers(
-  client: &reqwest::Client,
-  registry_url: String,
-  packages: Vec<Rc<PreparedPublishPackage>>,
+  client: &HttpClient,
+  registry_url: &Url,
+  packages: &[Rc<PreparedPublishPackage>],
   auth_method: AuthMethod,
 ) -> Result<HashMap<(String, String, String), Rc<str>>, AnyError> {
   let permissions = packages
@@ -600,7 +602,7 @@ async fn get_auth_headers(
               colors::cyan(res.user.name)
             );
             let authorization: Rc<str> = format!("Bearer {}", res.token).into();
-            for pkg in &packages {
+            for pkg in packages {
               authorizations.insert(
                 (pkg.scope.clone(), pkg.package.clone(), pkg.version.clone()),
                 authorization.clone(),
@@ -620,7 +622,7 @@ async fn get_auth_headers(
     }
     AuthMethod::Token(token) => {
       let authorization: Rc<str> = format!("Bearer {}", token).into();
-      for pkg in &packages {
+      for pkg in packages {
         authorizations.insert(
           (pkg.scope.clone(), pkg.package.clone(), pkg.version.clone()),
           authorization.clone(),
@@ -682,9 +684,9 @@ async fn get_auth_headers(
 /// Check if both `scope` and `package` already exist, if not return
 /// a URL to the management panel to create them.
 async fn check_if_scope_and_package_exist(
-  client: &reqwest::Client,
-  registry_api_url: &str,
-  registry_manage_url: &str,
+  client: &HttpClient,
+  registry_api_url: &Url,
+  registry_manage_url: &Url,
   scope: &str,
   package: &str,
 ) -> Result<Option<String>, AnyError> {
@@ -714,18 +716,18 @@ async fn check_if_scope_and_package_exist(
 }
 
 async fn ensure_scopes_and_packages_exist(
-  client: &reqwest::Client,
-  registry_api_url: String,
-  registry_manage_url: String,
-  packages: Vec<Rc<PreparedPublishPackage>>,
+  client: &HttpClient,
+  registry_api_url: &Url,
+  registry_manage_url: &Url,
+  packages: &[Rc<PreparedPublishPackage>],
 ) -> Result<(), AnyError> {
   if !std::io::stdin().is_terminal() {
     let mut missing_packages_lines = vec![];
     for package in packages {
       let maybe_create_package_url = check_if_scope_and_package_exist(
         client,
-        &registry_api_url,
-        &registry_manage_url,
+        registry_api_url,
+        registry_manage_url,
         &package.scope,
         &package.package,
       )
@@ -748,8 +750,8 @@ async fn ensure_scopes_and_packages_exist(
   for package in packages {
     let maybe_create_package_url = check_if_scope_and_package_exist(
       client,
-      &registry_api_url,
-      &registry_manage_url,
+      registry_api_url,
+      registry_manage_url,
       &package.scope,
       &package.package,
     )
@@ -770,7 +772,7 @@ async fn ensure_scopes_and_packages_exist(
     let _ = open::that_detached(&create_package_url);
 
     let package_api_url = api::get_package_api_url(
-      &registry_api_url,
+      registry_api_url,
       &package.scope,
       &package.package,
     );
@@ -790,15 +792,14 @@ async fn ensure_scopes_and_packages_exist(
 }
 
 async fn perform_publish(
-  http_client: &Arc<HttpClient>,
+  http_client: &HttpClient,
   mut publish_order_graph: PublishOrderGraph,
   mut prepared_package_by_name: HashMap<String, Rc<PreparedPublishPackage>>,
   auth_method: AuthMethod,
   provenance: bool,
 ) -> Result<(), AnyError> {
-  let client = http_client.client()?;
-  let registry_api_url = jsr_api_url().to_string();
-  let registry_url = jsr_url().to_string();
+  let registry_api_url = jsr_api_url();
+  let registry_url = jsr_url();
 
   let packages = prepared_package_by_name
     .values()
@@ -806,19 +807,20 @@ async fn perform_publish(
     .collect::<Vec<_>>();
 
   ensure_scopes_and_packages_exist(
-    client,
-    registry_api_url.clone(),
-    registry_url.clone(),
-    packages.clone(),
+    http_client,
+    registry_api_url,
+    registry_url,
+    &packages,
   )
   .await?;
 
   let mut authorizations =
-    get_auth_headers(client, registry_api_url.clone(), packages, auth_method)
+    get_auth_headers(http_client, registry_api_url, &packages, auth_method)
       .await?;
 
   assert_eq!(prepared_package_by_name.len(), authorizations.len());
-  let mut futures: JoinSet<Result<String, AnyError>> = JoinSet::default();
+  let mut futures: FuturesUnordered<LocalBoxFuture<Result<String, AnyError>>> =
+    Default::default();
   loop {
     let next_batch = publish_order_graph.next();
 
@@ -844,32 +846,32 @@ async fn perform_publish(
           package.version.clone(),
         ))
         .unwrap();
-      let registry_api_url = registry_api_url.clone();
-      let registry_url = registry_url.clone();
-      let http_client = http_client.clone();
-      futures.spawn(async move {
-        let display_name = package.display_name();
-        publish_package(
-          &http_client,
-          package,
-          &registry_api_url,
-          &registry_url,
-          &authorization,
-          provenance,
-        )
-        .await
-        .with_context(|| format!("Failed to publish {}", display_name))?;
-        Ok(package_name)
-      });
+      futures.push(
+        async move {
+          let display_name = package.display_name();
+          publish_package(
+            http_client,
+            package,
+            registry_api_url,
+            registry_url,
+            &authorization,
+            provenance,
+          )
+          .await
+          .with_context(|| format!("Failed to publish {}", display_name))?;
+          Ok(package_name)
+        }
+        .boxed_local(),
+      );
     }
 
-    let Some(result) = futures.join_next().await else {
+    let Some(result) = futures.next().await else {
       // done, ensure no circular dependency
       publish_order_graph.ensure_no_pending()?;
       break;
     };
 
-    let package_name = result??;
+    let package_name = result?;
     publish_order_graph.finish_package(&package_name);
   }
 
@@ -879,12 +881,11 @@ async fn perform_publish(
 async fn publish_package(
   http_client: &HttpClient,
   package: Rc<PreparedPublishPackage>,
-  registry_api_url: &str,
-  registry_url: &str,
+  registry_api_url: &Url,
+  registry_url: &Url,
   authorization: &str,
   provenance: bool,
 ) -> Result<(), AnyError> {
-  let client = http_client.client()?;
   log::info!(
     "{} @{}/{}@{} ...",
     colors::intense_blue("Publishing"),
@@ -902,7 +903,7 @@ async fn publish_package(
     package.config
   );
 
-  let response = client
+  let response = http_client
     .post(url)
     .header(reqwest::header::AUTHORIZATION, authorization)
     .header(reqwest::header::CONTENT_ENCODING, "gzip")
@@ -950,7 +951,7 @@ async fn publish_package(
   let interval = std::time::Duration::from_secs(2);
   while task.status != "success" && task.status != "failure" {
     tokio::time::sleep(interval).await;
-    let resp = client
+    let resp = http_client
       .get(format!("{}publish_status/{}", registry_api_url, task.id))
       .send()
       .await
@@ -1000,7 +1001,7 @@ async fn publish_package(
       package.scope, package.package, package.version
     ))?;
 
-    let meta_bytes = client.get(meta_url).send().await?.bytes().await?;
+    let meta_bytes = http_client.get(meta_url).send().await?.bytes().await?;
 
     if std::env::var("DISABLE_JSR_MANIFEST_VERIFICATION_FOR_TESTING").is_err() {
       verify_version_manifest(&meta_bytes, &package)?;
@@ -1015,7 +1016,7 @@ async fn publish_package(
         sha256: faster_hex::hex_string(&sha2::Sha256::digest(&meta_bytes)),
       },
     };
-    let bundle = provenance::generate_provenance(subject).await?;
+    let bundle = provenance::generate_provenance(http_client, subject).await?;
 
     let tlog_entry = &bundle.verification_material.tlog_entries[0];
     log::info!("{}",
@@ -1030,7 +1031,7 @@ async fn publish_package(
       "{}scopes/{}/packages/{}/versions/{}/provenance",
       registry_api_url, package.scope, package.package, package.version
     );
-    client
+    http_client
       .post(provenance_url)
       .header(reqwest::header::AUTHORIZATION, authorization)
       .json(&json!({ "bundle": bundle }))
diff --git a/cli/tools/registry/pm.rs b/cli/tools/registry/pm.rs
index e37ee3d82..62d0f604a 100644
--- a/cli/tools/registry/pm.rs
+++ b/cli/tools/registry/pm.rs
@@ -188,7 +188,7 @@ pub async fn add(flags: Flags, add_flags: AddFlags) -> Result<(), AnyError> {
   }
   let config_file_path = config_specifier.to_file_path().unwrap();
 
-  let http_client = cli_factory.http_client();
+  let http_client = cli_factory.http_client_provider();
 
   let mut selected_packages = Vec::with_capacity(add_flags.packages.len());
   let mut package_reqs = Vec::with_capacity(add_flags.packages.len());
@@ -227,6 +227,7 @@ pub async fn add(flags: Flags, add_flags: AddFlags) -> Result<(), AnyError> {
     None,
   );
   deps_file_fetcher.set_download_log_level(log::Level::Trace);
+  let deps_file_fetcher = Arc::new(deps_file_fetcher);
   let jsr_resolver = Arc::new(JsrFetchResolver::new(deps_file_fetcher.clone()));
   let npm_resolver = Arc::new(NpmFetchResolver::new(deps_file_fetcher));
 
diff --git a/cli/tools/registry/provenance.rs b/cli/tools/registry/provenance.rs
index 69926372e..7fa2be381 100644
--- a/cli/tools/registry/provenance.rs
+++ b/cli/tools/registry/provenance.rs
@@ -1,5 +1,7 @@
 // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
 
+use crate::http_util::HttpClient;
+
 use super::api::OidcTokenResponse;
 use super::auth::gha_oidc_token;
 use super::auth::is_gha;
@@ -13,7 +15,6 @@ use deno_core::serde_json;
 use once_cell::sync::Lazy;
 use p256::elliptic_curve;
 use p256::pkcs8::AssociatedOid;
-use reqwest::Client;
 use ring::rand::SystemRandom;
 use ring::signature::EcdsaKeyPair;
 use ring::signature::KeyPair;
@@ -291,6 +292,7 @@ pub struct ProvenanceBundle {
 }
 
 pub async fn generate_provenance(
+  http_client: &HttpClient,
   subject: Subject,
 ) -> Result<ProvenanceBundle, AnyError> {
   if !is_gha() {
@@ -306,19 +308,20 @@ pub async fn generate_provenance(
   let slsa = ProvenanceAttestation::new_github_actions(subject);
 
   let attestation = serde_json::to_string(&slsa)?;
-  let bundle = attest(&attestation, INTOTO_PAYLOAD_TYPE).await?;
+  let bundle = attest(http_client, &attestation, INTOTO_PAYLOAD_TYPE).await?;
 
   Ok(bundle)
 }
 
 pub async fn attest(
+  http_client: &HttpClient,
   data: &str,
   type_: &str,
 ) -> Result<ProvenanceBundle, AnyError> {
   // DSSE Pre-Auth Encoding (PAE) payload
   let pae = pre_auth_encoding(type_, data);
 
-  let signer = FulcioSigner::new()?;
+  let signer = FulcioSigner::new(http_client)?;
   let (signature, key_material) = signer.sign(&pae).await?;
 
   let content = SignatureBundle {
@@ -332,7 +335,8 @@ pub async fn attest(
       }],
     },
   };
-  let transparency_logs = testify(&content, &key_material.certificate).await?;
+  let transparency_logs =
+    testify(http_client, &content, &key_material.certificate).await?;
 
   // First log entry is the one we're interested in
   let (_, log_entry) = transparency_logs.iter().next().unwrap();
@@ -363,13 +367,6 @@ static DEFAULT_FULCIO_URL: Lazy<String> = Lazy::new(|| {
     .unwrap_or_else(|_| "https://fulcio.sigstore.dev".to_string())
 });
 
-struct FulcioSigner {
-  // The ephemeral key pair used to sign.
-  ephemeral_signer: EcdsaKeyPair,
-  rng: SystemRandom,
-  client: Client,
-}
-
 static ALGORITHM: &ring::signature::EcdsaSigningAlgorithm =
   &ring::signature::ECDSA_P256_SHA256_ASN1_SIGNING;
 
@@ -424,8 +421,15 @@ struct SigningCertificateResponse {
   signed_certificate_detached_sct: Option<SignedCertificate>,
 }
 
-impl FulcioSigner {
-  pub fn new() -> Result<Self, AnyError> {
+struct FulcioSigner<'a> {
+  // The ephemeral key pair used to sign.
+  ephemeral_signer: EcdsaKeyPair,
+  rng: SystemRandom,
+  http_client: &'a HttpClient,
+}
+
+impl<'a> FulcioSigner<'a> {
+  pub fn new(http_client: &'a HttpClient) -> Result<Self, AnyError> {
     let rng = SystemRandom::new();
     let document = EcdsaKeyPair::generate_pkcs8(ALGORITHM, &rng)?;
     let ephemeral_signer =
@@ -434,7 +438,7 @@ impl FulcioSigner {
     Ok(Self {
       ephemeral_signer,
       rng,
-      client: Client::new(),
+      http_client,
     })
   }
 
@@ -443,7 +447,7 @@ impl FulcioSigner {
     data: &[u8],
   ) -> Result<(ring::signature::Signature, KeyMaterial), AnyError> {
     // Request token from GitHub Actions for audience "sigstore"
-    let token = gha_request_token("sigstore").await?;
+    let token = self.gha_request_token("sigstore").await?;
     // Extract the subject from the token
     let subject = extract_jwt_subject(&token)?;
 
@@ -498,7 +502,12 @@ impl FulcioSigner {
       },
     };
 
-    let response = self.client.post(url).json(&request_body).send().await?;
+    let response = self
+      .http_client
+      .post(url)
+      .json(&request_body)
+      .send()
+      .await?;
 
     let body: SigningCertificateResponse = response.json().await?;
 
@@ -508,6 +517,27 @@ impl FulcioSigner {
       .ok_or_else(|| anyhow::anyhow!("No certificate chain returned"))?;
     Ok(key.chain.certificates)
   }
+
+  async fn gha_request_token(&self, aud: &str) -> Result<String, AnyError> {
+    let Ok(req_url) = env::var("ACTIONS_ID_TOKEN_REQUEST_URL") else {
+      bail!("Not running in GitHub Actions");
+    };
+
+    let Some(token) = gha_oidc_token() else {
+      bail!("No OIDC token available");
+    };
+
+    let res = self
+      .http_client
+      .get(&req_url)
+      .bearer_auth(token)
+      .query(&[("audience", aud)])
+      .send()
+      .await?
+      .json::<OidcTokenResponse>()
+      .await?;
+    Ok(res.value)
+  }
 }
 
 #[derive(Deserialize)]
@@ -532,27 +562,6 @@ fn extract_jwt_subject(token: &str) -> Result<String, AnyError> {
   }
 }
 
-async fn gha_request_token(aud: &str) -> Result<String, AnyError> {
-  let Ok(req_url) = env::var("ACTIONS_ID_TOKEN_REQUEST_URL") else {
-    bail!("Not running in GitHub Actions");
-  };
-
-  let Some(token) = gha_oidc_token() else {
-    bail!("No OIDC token available");
-  };
-
-  let client = Client::new();
-  let res = client
-    .get(&req_url)
-    .bearer_auth(token)
-    .query(&[("audience", aud)])
-    .send()
-    .await?
-    .json::<OidcTokenResponse>()
-    .await?;
-  Ok(res.value)
-}
-
 static DEFAULT_REKOR_URL: Lazy<String> = Lazy::new(|| {
   env::var("REKOR_URL")
     .unwrap_or_else(|_| "https://rekor.sigstore.dev".to_string())
@@ -616,6 +625,7 @@ struct ProposedIntotoEntryHash {
 
 // Rekor witness
 async fn testify(
+  http_client: &HttpClient,
   content: &SignatureBundle,
   public_key: &str,
 ) -> Result<RekorEntry, AnyError> {
@@ -672,9 +682,8 @@ async fn testify(
     },
   };
 
-  let client = Client::new();
   let url = format!("{}/api/v1/log/entries", *DEFAULT_REKOR_URL);
-  let res = client
+  let res = http_client
     .post(&url)
     .json(&proposed_intoto_entry)
     .send()
diff --git a/cli/tools/run/mod.rs b/cli/tools/run/mod.rs
index 90551a85d..82dcae711 100644
--- a/cli/tools/run/mod.rs
+++ b/cli/tools/run/mod.rs
@@ -42,7 +42,7 @@ To grant permissions, set them before the script argument. For example:
   // map specified and bare specifier is used on the command line
   let factory = CliFactory::from_flags(flags)?;
   let deno_dir = factory.deno_dir()?;
-  let http_client = factory.http_client();
+  let http_client = factory.http_client_provider();
   let cli_options = factory.cli_options();
 
   if cli_options.unstable_sloppy_imports() {
diff --git a/cli/tools/test/mod.rs b/cli/tools/test/mod.rs
index 56b09f1c7..fa69ad950 100644
--- a/cli/tools/test/mod.rs
+++ b/cli/tools/test/mod.rs
@@ -881,6 +881,7 @@ async fn run_tests_for_worker_inner(
     // failing. If we don't do this, a connection to a test server we just tore down might be re-used in
     // the next test.
     // TODO(mmastrac): this should be some sort of callback that we can implement for any subsystem
+    #[allow(clippy::disallowed_types)] // allow using reqwest::Client here
     worker
       .js_runtime
       .op_state()
diff --git a/cli/tools/upgrade.rs b/cli/tools/upgrade.rs
index 00e7a2d57..2afeffc92 100644
--- a/cli/tools/upgrade.rs
+++ b/cli/tools/upgrade.rs
@@ -7,6 +7,7 @@ use crate::args::UpgradeFlags;
 use crate::colors;
 use crate::factory::CliFactory;
 use crate::http_util::HttpClient;
+use crate::http_util::HttpClientProvider;
 use crate::standalone::binary::unpack_into_dir;
 use crate::util::progress_bar::ProgressBar;
 use crate::util::progress_bar::ProgressBarStyle;
@@ -101,17 +102,17 @@ trait VersionProvider: Clone {
 
 #[derive(Clone)]
 struct RealVersionProvider {
-  http_client: Arc<HttpClient>,
+  http_client_provider: Arc<HttpClientProvider>,
   check_kind: UpgradeCheckKind,
 }
 
 impl RealVersionProvider {
   pub fn new(
-    http_client: Arc<HttpClient>,
+    http_client_provider: Arc<HttpClientProvider>,
     check_kind: UpgradeCheckKind,
   ) -> Self {
     Self {
-      http_client,
+      http_client_provider,
       check_kind,
     }
   }
@@ -124,8 +125,12 @@ impl VersionProvider for RealVersionProvider {
   }
 
   async fn latest_version(&self) -> Result<String, AnyError> {
-    get_latest_version(&self.http_client, self.release_kind(), self.check_kind)
-      .await
+    get_latest_version(
+      &self.http_client_provider.get_or_create()?,
+      self.release_kind(),
+      self.check_kind,
+    )
+    .await
   }
 
   fn current_version(&self) -> Cow<str> {
@@ -241,7 +246,7 @@ pub fn upgrade_check_enabled() -> bool {
 }
 
 pub fn check_for_upgrades(
-  http_client: Arc<HttpClient>,
+  http_client_provider: Arc<HttpClientProvider>,
   cache_file_path: PathBuf,
 ) {
   if !upgrade_check_enabled() {
@@ -250,7 +255,7 @@ pub fn check_for_upgrades(
 
   let env = RealUpdateCheckerEnvironment::new(cache_file_path);
   let version_provider =
-    RealVersionProvider::new(http_client, UpgradeCheckKind::Execution);
+    RealVersionProvider::new(http_client_provider, UpgradeCheckKind::Execution);
   let update_checker = UpdateChecker::new(env, version_provider);
 
   if update_checker.should_check_for_new_version() {
@@ -300,14 +305,14 @@ pub struct LspVersionUpgradeInfo {
 }
 
 pub async fn check_for_upgrades_for_lsp(
-  http_client: Arc<HttpClient>,
+  http_client_provider: Arc<HttpClientProvider>,
 ) -> Result<Option<LspVersionUpgradeInfo>, AnyError> {
   if !upgrade_check_enabled() {
     return Ok(None);
   }
 
   let version_provider =
-    RealVersionProvider::new(http_client, UpgradeCheckKind::Lsp);
+    RealVersionProvider::new(http_client_provider, UpgradeCheckKind::Lsp);
   check_for_upgrades_for_lsp_with_provider(&version_provider).await
 }
 
@@ -370,7 +375,7 @@ pub async fn upgrade(
   upgrade_flags: UpgradeFlags,
 ) -> Result<(), AnyError> {
   let factory = CliFactory::from_flags(flags)?;
-  let client = factory.http_client();
+  let client = factory.http_client_provider().get_or_create()?;
   let current_exe_path = std::env::current_exe()?;
   let full_path_output_flag = upgrade_flags
     .output
@@ -445,7 +450,7 @@ pub async fn upgrade(
       };
 
       let latest_version =
-        get_latest_version(client, release_kind, UpgradeCheckKind::Execution)
+        get_latest_version(&client, release_kind, UpgradeCheckKind::Execution)
           .await?;
 
       let current_is_most_recent = if upgrade_flags.canary {
@@ -491,7 +496,7 @@ pub async fn upgrade(
     )
   };
 
-  let archive_data = download_package(client, &download_url)
+  let archive_data = download_package(&client, &download_url)
     .await
     .with_context(|| format!("Failed downloading {download_url}. The version you requested may not have been built for the current architecture."))?;