feat(unstable): add more options to Deno.createHttpClient (#17385)

6 files changed, 242 insertions, 94 deletions

diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs
index fd8c0f793..71d284ef6 100644
--- a/cli/file_fetcher.rs
+++ b/cli/file_fetcher.rs
@@ -744,6 +744,7 @@ mod tests {
   use deno_core::resolve_url;
   use deno_core::url::Url;
   use deno_runtime::deno_fetch::create_http_client;
+  use deno_runtime::deno_fetch::CreateHttpClientOptions;
   use deno_runtime::deno_web::Blob;
   use deno_runtime::deno_web::InMemoryBlobPart;
   use std::fs::read;
@@ -1746,7 +1747,7 @@ mod tests {
 
   fn create_test_client() -> HttpClient {
     HttpClient::from_client(
-      create_http_client("test_client", None, vec![], None, None, None)
+      create_http_client("test_client", CreateHttpClientOptions::default())
         .unwrap(),
     )
   }
@@ -1943,17 +1944,16 @@ mod tests {
     let client = HttpClient::from_client(
       create_http_client(
         version::get_user_agent(),
-        None,
-        vec![read(
-          test_util::testdata_path()
-            .join("tls/RootCA.pem")
-            .to_str()
-            .unwrap(),
-        )
-        .unwrap()],
-        None,
-        None,
-        None,
+        CreateHttpClientOptions {
+          ca_certs: vec![read(
+            test_util::testdata_path()
+              .join("tls/RootCA.pem")
+              .to_str()
+              .unwrap(),
+          )
+          .unwrap()],
+          ..Default::default()
+        },
       )
       .unwrap(),
     );
@@ -1986,11 +1986,7 @@ mod tests {
     let client = HttpClient::from_client(
       create_http_client(
         version::get_user_agent(),
-        None, // This will load mozilla certs by default
-        vec![],
-        None,
-        None,
-        None,
+        CreateHttpClientOptions::default(),
       )
       .unwrap(),
     );
@@ -2068,17 +2064,16 @@ mod tests {
     let client = HttpClient::from_client(
       create_http_client(
         version::get_user_agent(),
-        None,
-        vec![read(
-          test_util::testdata_path()
-            .join("tls/RootCA.pem")
-            .to_str()
-            .unwrap(),
-        )
-        .unwrap()],
-        None,
-        None,
-        None,
+        CreateHttpClientOptions {
+          ca_certs: vec![read(
+            test_util::testdata_path()
+              .join("tls/RootCA.pem")
+              .to_str()
+              .unwrap(),
+          )
+          .unwrap()],
+          ..Default::default()
+        },
       )
       .unwrap(),
     );
@@ -2113,17 +2108,16 @@ mod tests {
     let client = HttpClient::from_client(
       create_http_client(
         version::get_user_agent(),
-        None,
-        vec![read(
-          test_util::testdata_path()
-            .join("tls/RootCA.pem")
-            .to_str()
-            .unwrap(),
-        )
-        .unwrap()],
-        None,
-        None,
-        None,
+        CreateHttpClientOptions {
+          ca_certs: vec![read(
+            test_util::testdata_path()
+              .join("tls/RootCA.pem")
+              .to_str()
+              .unwrap(),
+          )
+          .unwrap()],
+          ..Default::default()
+        },
       )
       .unwrap(),
     );
@@ -2175,17 +2169,16 @@ mod tests {
     let client = HttpClient::from_client(
       create_http_client(
         version::get_user_agent(),
-        None,
-        vec![read(
-          test_util::testdata_path()
-            .join("tls/RootCA.pem")
-            .to_str()
-            .unwrap(),
-        )
-        .unwrap()],
-        None,
-        None,
-        None,
+        CreateHttpClientOptions {
+          ca_certs: vec![read(
+            test_util::testdata_path()
+              .join("tls/RootCA.pem")
+              .to_str()
+              .unwrap(),
+          )
+          .unwrap()],
+          ..Default::default()
+        },
       )
       .unwrap(),
     );
diff --git a/cli/http_util.rs b/cli/http_util.rs
index 7c17e8e1e..e90e0ee96 100644
--- a/cli/http_util.rs
+++ b/cli/http_util.rs
@@ -15,6 +15,7 @@ use deno_runtime::deno_fetch::create_http_client;
 use deno_runtime::deno_fetch::reqwest;
 use deno_runtime::deno_fetch::reqwest::header::LOCATION;
 use deno_runtime::deno_fetch::reqwest::Response;
+use deno_runtime::deno_fetch::CreateHttpClientOptions;
 use deno_runtime::deno_tls::RootCertStoreProvider;
 use std::collections::HashMap;
 use std::sync::Arc;
@@ -219,18 +220,15 @@ impl CacheSemantics {
 }
 
 pub struct HttpClient {
+  options: CreateHttpClientOptions,
   root_cert_store_provider: Option<Arc<dyn RootCertStoreProvider>>,
-  unsafely_ignore_certificate_errors: Option<Vec<String>>,
   cell: once_cell::sync::OnceCell<reqwest::Client>,
 }
 
 impl std::fmt::Debug for HttpClient {
   fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
     f.debug_struct("HttpClient")
-      .field(
-        "unsafely_ignore_certificate_errors",
-        &self.unsafely_ignore_certificate_errors,
-      )
+      .field("options", &self.options)
       .finish()
   }
 }
@@ -241,8 +239,11 @@ impl HttpClient {
     unsafely_ignore_certificate_errors: Option<Vec<String>>,
   ) -> Self {
     Self {
+      options: CreateHttpClientOptions {
+        unsafely_ignore_certificate_errors,
+        ..Default::default()
+      },
       root_cert_store_provider,
-      unsafely_ignore_certificate_errors,
       cell: Default::default(),
     }
   }
@@ -250,8 +251,8 @@ impl HttpClient {
   #[cfg(test)]
   pub fn from_client(client: reqwest::Client) -> Self {
     let result = Self {
+      options: Default::default(),
       root_cert_store_provider: Default::default(),
-      unsafely_ignore_certificate_errors: Default::default(),
       cell: Default::default(),
     };
     result.cell.set(client).unwrap();
@@ -262,14 +263,13 @@ impl HttpClient {
     self.cell.get_or_try_init(|| {
       create_http_client(
         get_user_agent(),
-        match &self.root_cert_store_provider {
-          Some(provider) => Some(provider.get_or_try_init()?.clone()),
-          None => None,
+        CreateHttpClientOptions {
+          root_cert_store: match &self.root_cert_store_provider {
+            Some(provider) => Some(provider.get_or_try_init()?.clone()),
+            None => None,
+          },
+          ..self.options.clone()
         },
-        vec![],
-        None,
-        self.unsafely_ignore_certificate_errors.clone(),
-        None,
       )
     })
   }
diff --git a/cli/tests/unit/fetch_test.ts b/cli/tests/unit/fetch_test.ts
index 7de04013e..d86795578 100644
--- a/cli/tests/unit/fetch_test.ts
+++ b/cli/tests/unit/fetch_test.ts
@@ -1532,6 +1532,40 @@ Deno.test(
 );
 
 Deno.test(
+  {
+    permissions: { net: true, read: true },
+    // Doesn't pass on linux CI for unknown reasons (works fine locally on linux)
+    ignore: Deno.build.os !== "darwin",
+  },
+  async function fetchForceHttp1OnHttp2Server() {
+    const client = Deno.createHttpClient({ http2: false, http1: true });
+    await assertRejects(
+      () => fetch("http://localhost:5549/http_version", { client }),
+      TypeError,
+      "invalid HTTP version parsed",
+    );
+    client.close();
+  },
+);
+
+Deno.test(
+  {
+    permissions: { net: true, read: true },
+    // Doesn't pass on linux CI for unknown reasons (works fine locally on linux)
+    ignore: Deno.build.os !== "darwin",
+  },
+  async function fetchForceHttp2OnHttp1Server() {
+    const client = Deno.createHttpClient({ http2: true, http1: false });
+    await assertRejects(
+      () => fetch("http://localhost:5548/http_version", { client }),
+      TypeError,
+      "stream closed because of a broken pipe",
+    );
+    client.close();
+  },
+);
+
+Deno.test(
   { permissions: { net: true, read: true } },
   async function fetchPrefersHttp2() {
     const caCert = await Deno.readTextFile("cli/tests/testdata/tls/RootCA.pem");
diff --git a/cli/tsc/dts/lib.deno.unstable.d.ts b/cli/tsc/dts/lib.deno.unstable.d.ts
index 70d7ef7c4..8681cbd9b 100644
--- a/cli/tsc/dts/lib.deno.unstable.d.ts
+++ b/cli/tsc/dts/lib.deno.unstable.d.ts
@@ -821,6 +821,15 @@ declare namespace Deno {
     certChain?: string;
     /** PEM formatted (RSA or PKCS8) private key of client certificate. */
     privateKey?: string;
+    /** Sets the maximum numer of idle connections per host allowed in the pool. */
+    poolMaxIdlePerHost?: number;
+    /** Set an optional timeout for idle sockets being kept-alive.
+     * Set to false to disable the timeout. */
+    poolIdleTimeout?: number | false;
+    /** Whether HTTP/1.1 is allowed or not. */
+    http1?: boolean;
+    /** Whether HTTP/2 is allowed or not. */
+    http2?: boolean;
   }
 
   /** **UNSTABLE**: New API, yet to be vetted.
diff --git a/ext/fetch/lib.rs b/ext/fetch/lib.rs
index e41d85ea4..9fdf80eec 100644
--- a/ext/fetch/lib.rs
+++ b/ext/fetch/lib.rs
@@ -200,11 +200,19 @@ pub fn get_or_create_client_from_state(
     let options = state.borrow::<Options>();
     let client = create_http_client(
       &options.user_agent,
-      options.root_cert_store()?,
-      vec![],
-      options.proxy.clone(),
-      options.unsafely_ignore_certificate_errors.clone(),
-      options.client_cert_chain_and_key.clone(),
+      CreateHttpClientOptions {
+        root_cert_store: options.root_cert_store()?,
+        ca_certs: vec![],
+        proxy: options.proxy.clone(),
+        unsafely_ignore_certificate_errors: options
+          .unsafely_ignore_certificate_errors
+          .clone(),
+        client_cert_chain_and_key: options.client_cert_chain_and_key.clone(),
+        pool_max_idle_per_host: None,
+        pool_idle_timeout: None,
+        http1: true,
+        http2: true,
+      },
     )?;
     state.put::<reqwest::Client>(client.clone());
     Ok(client)
@@ -606,19 +614,36 @@ impl HttpClientResource {
   }
 }
 
+#[derive(Deserialize, Debug, Clone)]
+#[serde(rename_all = "camelCase")]
+pub enum PoolIdleTimeout {
+  State(bool),
+  Specify(u64),
+}
+
 #[derive(Deserialize, Debug)]
 #[serde(rename_all = "camelCase")]
-pub struct CreateHttpClientOptions {
+pub struct CreateHttpClientArgs {
   ca_certs: Vec<String>,
   proxy: Option<Proxy>,
   cert_chain: Option<String>,
   private_key: Option<String>,
+  pool_max_idle_per_host: Option<usize>,
+  pool_idle_timeout: Option<PoolIdleTimeout>,
+  #[serde(default = "default_true")]
+  http1: bool,
+  #[serde(default = "default_true")]
+  http2: bool,
+}
+
+fn default_true() -> bool {
+  true
 }
 
 #[op]
 pub fn op_fetch_custom_client<FP>(
   state: &mut OpState,
-  args: CreateHttpClientOptions,
+  args: CreateHttpClientArgs,
 ) -> Result<ResourceId, AnyError>
 where
   FP: FetchPermissions + 'static,
@@ -653,32 +678,71 @@ where
 
   let client = create_http_client(
     &options.user_agent,
-    options.root_cert_store()?,
-    ca_certs,
-    args.proxy,
-    options.unsafely_ignore_certificate_errors.clone(),
-    client_cert_chain_and_key,
+    CreateHttpClientOptions {
+      root_cert_store: options.root_cert_store()?,
+      ca_certs,
+      proxy: args.proxy,
+      unsafely_ignore_certificate_errors: options
+        .unsafely_ignore_certificate_errors
+        .clone(),
+      client_cert_chain_and_key,
+      pool_max_idle_per_host: args.pool_max_idle_per_host,
+      pool_idle_timeout: args.pool_idle_timeout.and_then(
+        |timeout| match timeout {
+          PoolIdleTimeout::State(true) => None,
+          PoolIdleTimeout::State(false) => Some(None),
+          PoolIdleTimeout::Specify(specify) => Some(Some(specify)),
+        },
+      ),
+      http1: args.http1,
+      http2: args.http2,
+    },
   )?;
 
   let rid = state.resource_table.add(HttpClientResource::new(client));
   Ok(rid)
 }
 
+#[derive(Debug, Clone)]
+pub struct CreateHttpClientOptions {
+  pub root_cert_store: Option<RootCertStore>,
+  pub ca_certs: Vec<Vec<u8>>,
+  pub proxy: Option<Proxy>,
+  pub unsafely_ignore_certificate_errors: Option<Vec<String>>,
+  pub client_cert_chain_and_key: Option<(String, String)>,
+  pub pool_max_idle_per_host: Option<usize>,
+  pub pool_idle_timeout: Option<Option<u64>>,
+  pub http1: bool,
+  pub http2: bool,
+}
+
+impl Default for CreateHttpClientOptions {
+  fn default() -> Self {
+    CreateHttpClientOptions {
+      root_cert_store: None,
+      ca_certs: vec![],
+      proxy: None,
+      unsafely_ignore_certificate_errors: None,
+      client_cert_chain_and_key: None,
+      pool_max_idle_per_host: None,
+      pool_idle_timeout: None,
+      http1: true,
+      http2: true,
+    }
+  }
+}
+
 /// Create new instance of async reqwest::Client. This client supports
 /// proxies and doesn't follow redirects.
 pub fn create_http_client(
   user_agent: &str,
-  root_cert_store: Option<RootCertStore>,
-  ca_certs: Vec<Vec<u8>>,
-  proxy: Option<Proxy>,
-  unsafely_ignore_certificate_errors: Option<Vec<String>>,
-  client_cert_chain_and_key: Option<(String, String)>,
+  options: CreateHttpClientOptions,
 ) -> Result<Client, AnyError> {
   let mut tls_config = deno_tls::create_client_config(
-    root_cert_store,
-    ca_certs,
-    unsafely_ignore_certificate_errors,
-    client_cert_chain_and_key,
+    options.root_cert_store,
+    options.ca_certs,
+    options.unsafely_ignore_certificate_errors,
+    options.client_cert_chain_and_key,
   )?;
 
   tls_config.alpn_protocols = vec!["h2".into(), "http/1.1".into()];
@@ -690,7 +754,7 @@ pub fn create_http_client(
     .default_headers(headers)
     .use_preconfigured_tls(tls_config);
 
-  if let Some(proxy) = proxy {
+  if let Some(proxy) = options.proxy {
     let mut reqwest_proxy = reqwest::Proxy::all(&proxy.url)?;
     if let Some(basic_auth) = &proxy.basic_auth {
       reqwest_proxy =
@@ -699,6 +763,24 @@ pub fn create_http_client(
     builder = builder.proxy(reqwest_proxy);
   }
 
-  // unwrap here because it can only fail when native TLS is used.
-  Ok(builder.build().unwrap())
+  if let Some(pool_max_idle_per_host) = options.pool_max_idle_per_host {
+    builder = builder.pool_max_idle_per_host(pool_max_idle_per_host);
+  }
+
+  if let Some(pool_idle_timeout) = options.pool_idle_timeout {
+    builder = builder.pool_idle_timeout(
+      pool_idle_timeout.map(std::time::Duration::from_millis),
+    );
+  }
+
+  match (options.http1, options.http2) {
+    (true, false) => builder = builder.http1_only(),
+    (false, true) => builder = builder.http2_prior_knowledge(),
+    (true, true) => {}
+    (false, false) => {
+      return Err(type_error("Either `http1` or `http2` needs to be true"))
+    }
+  }
+
+  builder.build().map_err(|e| e.into())
 }
diff --git a/test_util/src/lib.rs b/test_util/src/lib.rs
index 054298b54..96dc1c325 100644
--- a/test_util/src/lib.rs
+++ b/test_util/src/lib.rs
@@ -80,8 +80,10 @@ const TLS_CLIENT_AUTH_PORT: u16 = 4552;
 const BASIC_AUTH_REDIRECT_PORT: u16 = 4554;
 const TLS_PORT: u16 = 4557;
 const HTTPS_PORT: u16 = 5545;
-const H1_ONLY_PORT: u16 = 5546;
-const H2_ONLY_PORT: u16 = 5547;
+const H1_ONLY_TLS_PORT: u16 = 5546;
+const H2_ONLY_TLS_PORT: u16 = 5547;
+const H1_ONLY_PORT: u16 = 5548;
+const H2_ONLY_PORT: u16 = 5549;
 const HTTPS_CLIENT_AUTH_PORT: u16 = 5552;
 const WS_PORT: u16 = 4242;
 const WSS_PORT: u16 = 4243;
@@ -1395,8 +1397,9 @@ async fn wrap_main_https_server() {
   }
 }
 
-async fn wrap_https_h1_only_server() {
-  let main_server_https_addr = SocketAddr::from(([127, 0, 0, 1], H1_ONLY_PORT));
+async fn wrap_https_h1_only_tls_server() {
+  let main_server_https_addr =
+    SocketAddr::from(([127, 0, 0, 1], H1_ONLY_TLS_PORT));
   let cert_file = "tls/localhost.crt";
   let key_file = "tls/localhost.key";
   let ca_cert_file = "tls/RootCA.pem";
@@ -1440,8 +1443,9 @@ async fn wrap_https_h1_only_server() {
   }
 }
 
-async fn wrap_https_h2_only_server() {
-  let main_server_https_addr = SocketAddr::from(([127, 0, 0, 1], H2_ONLY_PORT));
+async fn wrap_https_h2_only_tls_server() {
+  let main_server_https_addr =
+    SocketAddr::from(([127, 0, 0, 1], H2_ONLY_TLS_PORT));
   let cert_file = "tls/localhost.crt";
   let key_file = "tls/localhost.key";
   let ca_cert_file = "tls/RootCA.pem";
@@ -1485,6 +1489,28 @@ async fn wrap_https_h2_only_server() {
   }
 }
 
+async fn wrap_https_h1_only_server() {
+  let main_server_http_addr = SocketAddr::from(([127, 0, 0, 1], H1_ONLY_PORT));
+
+  let main_server_http_svc =
+    make_service_fn(|_| async { Ok::<_, Infallible>(service_fn(main_server)) });
+  let main_server_http = Server::bind(&main_server_http_addr)
+    .http1_only(true)
+    .serve(main_server_http_svc);
+  let _ = main_server_http.await;
+}
+
+async fn wrap_https_h2_only_server() {
+  let main_server_http_addr = SocketAddr::from(([127, 0, 0, 1], H2_ONLY_PORT));
+
+  let main_server_http_svc =
+    make_service_fn(|_| async { Ok::<_, Infallible>(service_fn(main_server)) });
+  let main_server_http = Server::bind(&main_server_http_addr)
+    .http2_only(true)
+    .serve(main_server_http_svc);
+  let _ = main_server_http.await;
+}
+
 async fn wrap_client_auth_https_server() {
   let main_server_https_addr =
     SocketAddr::from(([127, 0, 0, 1], HTTPS_CLIENT_AUTH_PORT));
@@ -1573,6 +1599,8 @@ pub async fn run_all_servers() {
   let client_auth_server_https_fut = wrap_client_auth_https_server();
   let main_server_fut = wrap_main_server();
   let main_server_https_fut = wrap_main_https_server();
+  let h1_only_server_tls_fut = wrap_https_h1_only_tls_server();
+  let h2_only_server_tls_fut = wrap_https_h2_only_tls_server();
   let h1_only_server_fut = wrap_https_h1_only_server();
   let h2_only_server_fut = wrap_https_h2_only_server();
 
@@ -1594,6 +1622,8 @@ pub async fn run_all_servers() {
       main_server_fut,
       main_server_https_fut,
       client_auth_server_https_fut,
+      h1_only_server_tls_fut,
+      h2_only_server_tls_fut,
       h1_only_server_fut,
       h2_only_server_fut
     )