feat(tls): Optionally support loading native certs (#11491)

This commit adds "DENO_TLS_CA_STORE" env variable to support 
optionally loading certificates from the users local certificate store. 
This will allow them to successfully connect via tls with corporate 
and self signed certs provided they have them installed in their keystore. 
It also allows them to deal with revoked certs by simply updating 
their keystore without having to upgrade Deno.

Currently supported values are "mozilla", "system" or empty value.

9 files changed, 160 insertions, 45 deletions

diff --git a/cli/Cargo.toml b/cli/Cargo.toml
index 22b734f13..525a0d352 100644
--- a/cli/Cargo.toml
+++ b/cli/Cargo.toml
@@ -47,6 +47,7 @@ deno_core = { version = "0.95.0", path = "../core" }
 deno_doc = "0.9.0"
 deno_lint = "0.11.0"
 deno_runtime = { version = "0.21.0", path = "../runtime" }
+deno_tls = { version = "0.1.0", path = "../extensions/tls" }
 
 atty = "0.2.14"
 base64 = "0.13.0"
diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs
index a7bd503ae..207f08c64 100644
--- a/cli/file_fetcher.rs
+++ b/cli/file_fetcher.rs
@@ -3,7 +3,6 @@
 use crate::auth_tokens::AuthTokens;
 use crate::colors;
 use crate::http_cache::HttpCache;
-use crate::http_util::create_http_client;
 use crate::http_util::fetch_once;
 use crate::http_util::FetchOnceArgs;
 use crate::http_util::FetchOnceResult;
@@ -22,6 +21,8 @@ use deno_core::ModuleSpecifier;
 use deno_runtime::deno_fetch::reqwest;
 use deno_runtime::deno_web::BlobStore;
 use deno_runtime::permissions::Permissions;
+use deno_tls::create_http_client;
+use deno_tls::rustls::RootCertStore;
 use log::debug;
 use log::info;
 use std::borrow::Borrow;
@@ -220,7 +221,7 @@ impl FileFetcher {
     http_cache: HttpCache,
     cache_setting: CacheSetting,
     allow_remote: bool,
-    ca_data: Option<Vec<u8>>,
+    root_cert_store: Option<RootCertStore>,
     blob_store: BlobStore,
   ) -> Result<Self, AnyError> {
     Ok(Self {
@@ -229,7 +230,12 @@ impl FileFetcher {
       cache: Default::default(),
       cache_setting,
       http_cache,
-      http_client: create_http_client(get_user_agent(), ca_data)?,
+      http_client: create_http_client(
+        get_user_agent(),
+        root_cert_store,
+        None,
+        None,
+      )?,
       blob_store,
     })
   }
diff --git a/cli/flags.rs b/cli/flags.rs
index 1dafa205f..1c7eaf9a0 100644
--- a/cli/flags.rs
+++ b/cli/flags.rs
@@ -139,6 +139,7 @@ pub struct Flags {
   pub allow_read: Option<Vec<PathBuf>>,
   pub allow_run: Option<Vec<String>>,
   pub allow_write: Option<Vec<PathBuf>>,
+  pub ca_stores: Option<Vec<String>>,
   pub ca_file: Option<String>,
   pub cache_blocklist: Vec<String>,
   /// This is not exposed as an option in the CLI, it is used internally when
@@ -276,6 +277,9 @@ static ENV_VARIABLES_HELP: &str = r#"ENVIRONMENT VARIABLES:
                          hostnames to use when fetching remote modules from
                          private repositories
                          (e.g. "abcde12345@deno.land;54321edcba@github.com")
+    DENO_TLS_CA_STORE    Comma-seperated list of order dependent certificate stores
+                         (system, mozilla)
+                         (defaults to mozilla)
     DENO_CERT            Load certificate authority from PEM encoded file
     DENO_DIR             Set the cache directory
     DENO_INSTALL_ROOT    Set deno install's output directory
diff --git a/cli/http_util.rs b/cli/http_util.rs
index a199f20c8..671093923 100644
--- a/cli/http_util.rs
+++ b/cli/http_util.rs
@@ -1,46 +1,18 @@
 // Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
-
 use crate::auth_tokens::AuthToken;
 
 use deno_core::error::generic_error;
 use deno_core::error::AnyError;
 use deno_core::url::Url;
-use deno_runtime::deno_fetch::reqwest;
-use deno_runtime::deno_fetch::reqwest::header::HeaderMap;
 use deno_runtime::deno_fetch::reqwest::header::HeaderValue;
 use deno_runtime::deno_fetch::reqwest::header::AUTHORIZATION;
 use deno_runtime::deno_fetch::reqwest::header::IF_NONE_MATCH;
 use deno_runtime::deno_fetch::reqwest::header::LOCATION;
-use deno_runtime::deno_fetch::reqwest::header::USER_AGENT;
-use deno_runtime::deno_fetch::reqwest::redirect::Policy;
 use deno_runtime::deno_fetch::reqwest::Client;
 use deno_runtime::deno_fetch::reqwest::StatusCode;
 use log::debug;
 use std::collections::HashMap;
 
-/// Create new instance of async reqwest::Client. This client supports
-/// proxies and doesn't follow redirects.
-pub fn create_http_client(
-  user_agent: String,
-  ca_data: Option<Vec<u8>>,
-) -> Result<Client, AnyError> {
-  let mut headers = HeaderMap::new();
-  headers.insert(USER_AGENT, user_agent.parse().unwrap());
-  let mut builder = Client::builder()
-    .redirect(Policy::none())
-    .default_headers(headers)
-    .use_rustls_tls();
-
-  if let Some(ca_data) = ca_data {
-    let cert = reqwest::Certificate::from_pem(&ca_data)?;
-    builder = builder.add_root_certificate(cert);
-  }
-
-  builder
-    .build()
-    .map_err(|e| generic_error(format!("Unable to build http client: {}", e)))
-}
-
 /// Construct the next uri based on base uri and location header fragment
 /// See <https://tools.ietf.org/html/rfc3986#section-4.2>
 fn resolve_url_from_location(base_url: &Url, location: &str) -> Url {
@@ -168,10 +140,12 @@ pub async fn fetch_once(
 mod tests {
   use super::*;
   use crate::version;
+  use deno_tls::create_http_client;
+  use deno_tls::rustls::RootCertStore;
   use std::fs::read;
 
   fn create_test_client(ca_data: Option<Vec<u8>>) -> Client {
-    create_http_client("test_client".to_string(), ca_data).unwrap()
+    create_http_client("test_client".to_string(), None, ca_data, None).unwrap()
   }
 
   #[tokio::test]
@@ -362,6 +336,7 @@ mod tests {
 
     let client = create_http_client(
       version::get_user_agent(),
+      None,
       Some(
         read(
           test_util::root_path()
@@ -371,6 +346,7 @@ mod tests {
         )
         .unwrap(),
       ),
+      None,
     )
     .unwrap();
     let result = fetch_once(FetchOnceArgs {
@@ -391,6 +367,64 @@ mod tests {
   }
 
   #[tokio::test]
+  async fn test_fetch_with_default_certificate_store() {
+    let _http_server_guard = test_util::http_server();
+    // Relies on external http server with a valid mozilla root CA cert.
+    let url = Url::parse("https://deno.land").unwrap();
+    let client = create_http_client(
+      version::get_user_agent(),
+      None, // This will load mozilla certs by default
+      None,
+      None,
+    )
+    .unwrap();
+
+    let result = fetch_once(FetchOnceArgs {
+      client,
+      url,
+      maybe_etag: None,
+      maybe_auth_token: None,
+    })
+    .await;
+
+    println!("{:?}", result);
+    if let Ok(FetchOnceResult::Code(body, _headers)) = result {
+      assert!(!body.is_empty());
+    } else {
+      panic!();
+    }
+  }
+
+  // TODO(@justinmchase): Windows should verify certs too and fail to make this request without ca certs
+  #[cfg(not(windows))]
+  #[tokio::test]
+  async fn test_fetch_with_empty_certificate_store() {
+    let _http_server_guard = test_util::http_server();
+    // Relies on external http server with a valid mozilla root CA cert.
+    let url = Url::parse("https://deno.land").unwrap();
+    let client = create_http_client(
+      version::get_user_agent(),
+      Some(RootCertStore::empty()), // no certs loaded at all
+      None,
+      None,
+    )
+    .unwrap();
+
+    let result = fetch_once(FetchOnceArgs {
+      client,
+      url,
+      maybe_etag: None,
+      maybe_auth_token: None,
+    })
+    .await;
+
+    if let Ok(FetchOnceResult::Code(_body, _headers)) = result {
+      // This test is expected to fail since to CA certs have been loaded
+      panic!();
+    }
+  }
+
+  #[tokio::test]
   async fn test_fetch_with_cafile_gzip() {
     let _http_server_guard = test_util::http_server();
     // Relies on external http server. See target/debug/test_server
@@ -400,6 +434,7 @@ mod tests {
     .unwrap();
     let client = create_http_client(
       version::get_user_agent(),
+      None,
       Some(
         read(
           test_util::root_path()
@@ -409,6 +444,7 @@ mod tests {
         )
         .unwrap(),
       ),
+      None,
     )
     .unwrap();
     let result = fetch_once(FetchOnceArgs {
@@ -437,6 +473,7 @@ mod tests {
     let url = Url::parse("https://localhost:5545/etag_script.ts").unwrap();
     let client = create_http_client(
       version::get_user_agent(),
+      None,
       Some(
         read(
           test_util::root_path()
@@ -446,6 +483,7 @@ mod tests {
         )
         .unwrap(),
       ),
+      None,
     )
     .unwrap();
     let result = fetch_once(FetchOnceArgs {
@@ -488,6 +526,7 @@ mod tests {
     .unwrap();
     let client = create_http_client(
       version::get_user_agent(),
+      None,
       Some(
         read(
           test_util::root_path()
@@ -497,6 +536,7 @@ mod tests {
         )
         .unwrap(),
       ),
+      None,
     )
     .unwrap();
     let result = fetch_once(FetchOnceArgs {
diff --git a/cli/main.rs b/cli/main.rs
index 7d375c0c4..77cce1d05 100644
--- a/cli/main.rs
+++ b/cli/main.rs
@@ -109,7 +109,7 @@ fn create_web_worker_callback(
         .log_level
         .map_or(false, |l| l == log::Level::Debug),
       unstable: program_state.flags.unstable,
-      ca_data: program_state.ca_data.clone(),
+      root_cert_store: program_state.root_cert_store.clone(),
       user_agent: version::get_user_agent(),
       seed: program_state.flags.seed,
       module_loader,
@@ -189,7 +189,7 @@ pub fn create_main_worker(
       .log_level
       .map_or(false, |l| l == log::Level::Debug),
     unstable: program_state.flags.unstable,
-    ca_data: program_state.ca_data.clone(),
+    root_cert_store: program_state.root_cert_store.clone(),
     user_agent: version::get_user_agent(),
     seed: program_state.flags.seed,
     js_error_create_fn: Some(js_error_create_fn),
diff --git a/cli/program_state.rs b/cli/program_state.rs
index b8fb5e33b..244351a03 100644
--- a/cli/program_state.rs
+++ b/cli/program_state.rs
@@ -30,12 +30,16 @@ use deno_core::resolve_url;
 use deno_core::url::Url;
 use deno_core::ModuleSource;
 use deno_core::ModuleSpecifier;
+use deno_tls::rustls::RootCertStore;
+use deno_tls::rustls_native_certs::load_native_certs;
+use deno_tls::webpki_roots::TLS_SERVER_ROOTS;
 use log::debug;
 use log::warn;
 use std::collections::HashMap;
 use std::collections::HashSet;
 use std::env;
-use std::fs::read;
+use std::fs::File;
+use std::io::BufReader;
 use std::sync::Arc;
 
 /// This structure represents state of single "deno" program.
@@ -53,7 +57,7 @@ pub struct ProgramState {
   pub maybe_config_file: Option<ConfigFile>,
   pub maybe_import_map: Option<ImportMap>,
   pub maybe_inspector_server: Option<Arc<InspectorServer>>,
-  pub ca_data: Option<Vec<u8>>,
+  pub root_cert_store: Option<RootCertStore>,
   pub blob_store: BlobStore,
   pub broadcast_channel: InMemoryBroadcastChannel,
   pub shared_array_buffer_store: SharedArrayBufferStore,
@@ -68,11 +72,50 @@ impl ProgramState {
     let dir = deno_dir::DenoDir::new(maybe_custom_root)?;
     let deps_cache_location = dir.root.join("deps");
     let http_cache = http_cache::HttpCache::new(&deps_cache_location);
+
+    let mut root_cert_store = RootCertStore::empty();
+    let ca_stores: Vec<String> = flags
+      .ca_stores
+      .clone()
+      .or_else(|| {
+        let env_ca_store = env::var("DENO_TLS_CA_STORE").ok()?;
+        Some(
+          env_ca_store
+            .split(',')
+            .map(|s| s.trim().to_string())
+            .filter(|s| !s.is_empty())
+            .collect(),
+        )
+      })
+      .unwrap_or_else(|| vec!["mozilla".to_string()]);
+
+    for store in ca_stores.iter() {
+      match store.as_str() {
+        "mozilla" => {
+          root_cert_store.add_server_trust_anchors(&TLS_SERVER_ROOTS);
+        }
+        "system" => {
+          let roots = load_native_certs()
+            .expect("could not load platform certs")
+            .roots;
+          root_cert_store.roots.extend(roots);
+        }
+        _ => {
+          return Err(anyhow!("Unknown certificate store \"{}\" specified (allowed: \"system,mozilla\")", store));
+        }
+      }
+    }
+
     let ca_file = flags.ca_file.clone().or_else(|| env::var("DENO_CERT").ok());
-    let ca_data = match &ca_file {
-      Some(ca_file) => Some(read(ca_file).context("Failed to open ca file")?),
-      None => None,
-    };
+    if let Some(ca_file) = ca_file {
+      let certfile = File::open(&ca_file)?;
+      let mut reader = BufReader::new(certfile);
+
+      // This function does not return specific errors, if it fails give a generic message.
+      if let Err(_err) = root_cert_store.add_pem_file(&mut reader) {
+        return Err(anyhow!("Unable to add pem file to certificate store"));
+      }
+    }
 
     let cache_usage = if flags.cached_only {
       CacheSetting::Only
@@ -92,7 +135,7 @@ impl ProgramState {
       http_cache,
       cache_usage,
       !flags.no_remote,
-      ca_data.clone(),
+      Some(root_cert_store.clone()),
       blob_store.clone(),
     )?;
 
@@ -152,7 +195,7 @@ impl ProgramState {
       maybe_config_file,
       maybe_import_map,
       maybe_inspector_server,
-      ca_data,
+      root_cert_store: Some(root_cert_store.clone()),
       blob_store,
       broadcast_channel,
       shared_array_buffer_store,
diff --git a/cli/standalone.rs b/cli/standalone.rs
index 3c8dabd3a..460ee23d0 100644
--- a/cli/standalone.rs
+++ b/cli/standalone.rs
@@ -8,6 +8,7 @@ use crate::ops;
 use crate::program_state::ProgramState;
 use crate::version;
 use data_url::DataUrl;
+use deno_core::error::anyhow;
 use deno_core::error::type_error;
 use deno_core::error::uri_error;
 use deno_core::error::AnyError;
@@ -29,11 +30,14 @@ use deno_runtime::permissions::Permissions;
 use deno_runtime::permissions::PermissionsOptions;
 use deno_runtime::worker::MainWorker;
 use deno_runtime::worker::WorkerOptions;
+use deno_tls::create_default_root_cert_store;
 use log::Level;
 use std::cell::RefCell;
 use std::convert::TryInto;
 use std::env::current_exe;
 use std::fs::File;
+use std::io::BufReader;
+use std::io::Cursor;
 use std::io::Read;
 use std::io::Seek;
 use std::io::SeekFrom;
@@ -51,6 +55,7 @@ pub struct Metadata {
   pub location: Option<Url>,
   pub v8_flags: Vec<String>,
   pub log_level: Option<Level>,
+  pub ca_stores: Option<Vec<String>>,
   pub ca_data: Option<Vec<u8>>,
 }
 
@@ -201,6 +206,7 @@ fn metadata_to_flags(metadata: &Metadata) -> Flags {
     allow_write: permissions.allow_write,
     v8_flags: metadata.v8_flags.clone(),
     log_level: metadata.log_level,
+    ca_stores: metadata.ca_stores.clone(),
     ..Default::default()
   }
 }
@@ -227,13 +233,26 @@ pub async fn run(
       .collect::<Vec<_>>(),
   );
 
+  let mut root_cert_store = program_state
+    .root_cert_store
+    .clone()
+    .unwrap_or_else(create_default_root_cert_store);
+
+  if let Some(cert) = metadata.ca_data {
+    let reader = &mut BufReader::new(Cursor::new(cert));
+    // This function does not return specific errors, if it fails give a generic message.
+    if let Err(_err) = root_cert_store.add_pem_file(reader) {
+      return Err(anyhow!("Unable to add pem file to certificate store"));
+    }
+  }
+
   let options = WorkerOptions {
     apply_source_maps: false,
     args: metadata.argv,
     debug_flag: metadata.log_level.map_or(false, |l| l == log::Level::Debug),
     user_agent: version::get_user_agent(),
     unstable: metadata.unstable,
-    ca_data: metadata.ca_data,
+    root_cert_store: Some(root_cert_store),
     seed: metadata.seed,
     js_error_create_fn: None,
     create_web_worker_cb,
diff --git a/cli/tests/integration/mod.rs b/cli/tests/integration/mod.rs
index c11d26dc9..cc016382f 100644
--- a/cli/tests/integration/mod.rs
+++ b/cli/tests/integration/mod.rs
@@ -2,9 +2,9 @@
 
 use crate::itest;
 use deno_core::url;
-use deno_runtime::deno_net::ops_tls::rustls;
-use deno_runtime::deno_net::ops_tls::webpki;
 use deno_runtime::deno_net::ops_tls::TlsStream;
+use deno_runtime::deno_tls::rustls;
+use deno_runtime::deno_tls::webpki;
 use std::fs;
 use std::io::BufReader;
 use std::io::Cursor;
diff --git a/cli/tools/standalone.rs b/cli/tools/standalone.rs
index 5f89b592d..46ac27b83 100644
--- a/cli/tools/standalone.rs
+++ b/cli/tools/standalone.rs
@@ -100,6 +100,7 @@ pub fn create_standalone_binary(
     permissions: flags.clone().into(),
     v8_flags: flags.v8_flags.clone(),
     log_level: flags.log_level,
+    ca_stores: flags.ca_stores,
     ca_data,
   };
   let mut metadata = serde_json::to_string(&metadata)?.as_bytes().to_vec();
@@ -205,6 +206,7 @@ pub fn compile_to_runtime_flags(
     allow_read: flags.allow_read,
     allow_run: flags.allow_run,
     allow_write: flags.allow_write,
+    ca_stores: flags.ca_stores,
     ca_file: flags.ca_file,
     cache_blocklist: vec![],
     cache_path: None,