feat(cli): support auth tokens for accessing private modules (#9508)

Closes #5239

6 files changed, 318 insertions, 34 deletions

diff --git a/cli/auth_tokens.rs b/cli/auth_tokens.rs
new file mode 100644
index 000000000..f52f564e1
--- /dev/null
+++ b/cli/auth_tokens.rs
@@ -0,0 +1,144 @@
+// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
+
+use deno_core::ModuleSpecifier;
+use std::fmt;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct AuthToken {
+  host: String,
+  token: String,
+}
+
+impl fmt::Display for AuthToken {
+  fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+    write!(f, "Bearer {}", self.token)
+  }
+}
+
+/// A structure which contains bearer tokens that can be used when sending
+/// requests to websites, intended to authorize access to private resources
+/// such as remote modules.
+#[derive(Debug, Clone)]
+pub struct AuthTokens(Vec<AuthToken>);
+
+impl AuthTokens {
+  /// Create a new set of tokens based on the provided string. It is intended
+  /// that the string be the value of an environment variable and the string is
+  /// parsed for token values.  The string is expected to be a semi-colon
+  /// separated string, where each value is `{token}@{hostname}`.
+  pub fn new(maybe_tokens_str: Option<String>) -> Self {
+    let mut tokens = Vec::new();
+    if let Some(tokens_str) = maybe_tokens_str {
+      for token_str in tokens_str.split(';') {
+        if token_str.contains('@') {
+          let pair: Vec<&str> = token_str.rsplitn(2, '@').collect();
+          let token = pair[1].to_string();
+          let host = pair[0].to_lowercase();
+          tokens.push(AuthToken { host, token });
+        } else {
+          error!("Badly formed auth token discarded.");
+        }
+      }
+      debug!("Parsed {} auth token(s).", tokens.len());
+    }
+
+    Self(tokens)
+  }
+
+  /// Attempt to match the provided specifier to the tokens in the set.  The
+  /// matching occurs from the right of the hostname plus port, irrespective of
+  /// scheme.  For example `https://www.deno.land:8080/` would match a token
+  /// with a host value of `deno.land:8080` but not match `www.deno.land`.  The
+  /// matching is case insensitive.
+  pub fn get(&self, specifier: &ModuleSpecifier) -> Option<AuthToken> {
+    self.0.iter().find_map(|t| {
+      let url = specifier.as_url();
+      let hostname = if let Some(port) = url.port() {
+        format!("{}:{}", url.host_str()?, port)
+      } else {
+        url.host_str()?.to_string()
+      };
+      if hostname.to_lowercase().ends_with(&t.host) {
+        Some(t.clone())
+      } else {
+        None
+      }
+    })
+  }
+}
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+
+  #[test]
+  fn test_auth_token() {
+    let auth_tokens = AuthTokens::new(Some("abc123@deno.land".to_string()));
+    let fixture =
+      ModuleSpecifier::resolve_url("https://deno.land/x/mod.ts").unwrap();
+    assert_eq!(
+      auth_tokens.get(&fixture).unwrap().to_string(),
+      "Bearer abc123"
+    );
+    let fixture =
+      ModuleSpecifier::resolve_url("https://www.deno.land/x/mod.ts").unwrap();
+    assert_eq!(
+      auth_tokens.get(&fixture).unwrap().to_string(),
+      "Bearer abc123".to_string()
+    );
+    let fixture =
+      ModuleSpecifier::resolve_url("http://127.0.0.1:8080/x/mod.ts").unwrap();
+    assert_eq!(auth_tokens.get(&fixture), None);
+    let fixture =
+      ModuleSpecifier::resolve_url("https://deno.land.example.com/x/mod.ts")
+        .unwrap();
+    assert_eq!(auth_tokens.get(&fixture), None);
+    let fixture =
+      ModuleSpecifier::resolve_url("https://deno.land:8080/x/mod.ts").unwrap();
+    assert_eq!(auth_tokens.get(&fixture), None);
+  }
+
+  #[test]
+  fn test_auth_tokens_multiple() {
+    let auth_tokens =
+      AuthTokens::new(Some("abc123@deno.land;def456@example.com".to_string()));
+    let fixture =
+      ModuleSpecifier::resolve_url("https://deno.land/x/mod.ts").unwrap();
+    assert_eq!(
+      auth_tokens.get(&fixture).unwrap().to_string(),
+      "Bearer abc123".to_string()
+    );
+    let fixture =
+      ModuleSpecifier::resolve_url("http://example.com/a/file.ts").unwrap();
+    assert_eq!(
+      auth_tokens.get(&fixture).unwrap().to_string(),
+      "Bearer def456".to_string()
+    );
+  }
+
+  #[test]
+  fn test_auth_tokens_port() {
+    let auth_tokens =
+      AuthTokens::new(Some("abc123@deno.land:8080".to_string()));
+    let fixture =
+      ModuleSpecifier::resolve_url("https://deno.land/x/mod.ts").unwrap();
+    assert_eq!(auth_tokens.get(&fixture), None);
+    let fixture =
+      ModuleSpecifier::resolve_url("http://deno.land:8080/x/mod.ts").unwrap();
+    assert_eq!(
+      auth_tokens.get(&fixture).unwrap().to_string(),
+      "Bearer abc123".to_string()
+    );
+  }
+
+  #[test]
+  fn test_auth_tokens_contain_at() {
+    let auth_tokens = AuthTokens::new(Some("abc@123@deno.land".to_string()));
+    let fixture =
+      ModuleSpecifier::resolve_url("https://deno.land/x/mod.ts").unwrap();
+    assert_eq!(
+      auth_tokens.get(&fixture).unwrap().to_string(),
+      "Bearer abc@123".to_string()
+    );
+  }
+}
diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs
index 23ace672c..722813457 100644
--- a/cli/file_fetcher.rs
+++ b/cli/file_fetcher.rs
@@ -1,9 +1,11 @@
 // Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
 
+use crate::auth_tokens::AuthTokens;
 use crate::colors;
 use crate::http_cache::HttpCache;
 use crate::http_util::create_http_client;
 use crate::http_util::fetch_once;
+use crate::http_util::FetchOnceArgs;
 use crate::http_util::FetchOnceResult;
 use crate::media_type::MediaType;
 use crate::text_encoding;
@@ -19,6 +21,7 @@ use deno_core::futures::future::FutureExt;
 use deno_core::ModuleSpecifier;
 use deno_runtime::deno_fetch::reqwest;
 use std::collections::HashMap;
+use std::env;
 use std::fs;
 use std::future::Future;
 use std::io::Read;
@@ -27,6 +30,7 @@ use std::pin::Pin;
 use std::sync::Arc;
 use std::sync::Mutex;
 
+static DENO_AUTH_TOKENS: &str = "DENO_AUTH_TOKENS";
 pub const SUPPORTED_SCHEMES: [&str; 4] = ["data", "file", "http", "https"];
 
 /// A structure representing a source file.
@@ -308,6 +312,7 @@ fn strip_shebang(mut value: String) -> String {
 /// A structure for resolving, fetching and caching source files.
 #[derive(Clone)]
 pub struct FileFetcher {
+  auth_tokens: AuthTokens,
   allow_remote: bool,
   cache: FileCache,
   cache_setting: CacheSetting,
@@ -323,8 +328,9 @@ impl FileFetcher {
     ca_data: Option<Vec<u8>>,
   ) -> Result<Self, AnyError> {
     Ok(Self {
+      auth_tokens: AuthTokens::new(env::var(DENO_AUTH_TOKENS).ok()),
       allow_remote,
-      cache: FileCache::default(),
+      cache: Default::default(),
       cache_setting,
       http_cache,
       http_client: create_http_client(get_user_agent(), ca_data)?,
@@ -488,17 +494,25 @@ impl FileFetcher {
 
     info!("{} {}", colors::green("Download"), specifier);
 
-    let file_fetcher = self.clone();
-    let cached_etag = match self.http_cache.get(specifier.as_url()) {
+    let maybe_etag = match self.http_cache.get(specifier.as_url()) {
       Ok((_, headers)) => headers.get("etag").cloned(),
       _ => None,
     };
+    let maybe_auth_token = self.auth_tokens.get(&specifier);
     let specifier = specifier.clone();
     let permissions = permissions.clone();
-    let http_client = self.http_client.clone();
+    let client = self.http_client.clone();
+    let file_fetcher = self.clone();
     // A single pass of fetch either yields code or yields a redirect.
     async move {
-      match fetch_once(http_client, specifier.as_url(), cached_etag).await? {
+      match fetch_once(FetchOnceArgs {
+        client,
+        url: specifier.as_url().clone(),
+        maybe_etag,
+        maybe_auth_token,
+      })
+      .await?
+      {
         FetchOnceResult::NotModified => {
           let file = file_fetcher.fetch_cached(&specifier, 10)?.unwrap();
           Ok(file)
diff --git a/cli/flags.rs b/cli/flags.rs
index f7c83294c..80ad6e240 100644
--- a/cli/flags.rs
+++ b/cli/flags.rs
@@ -219,18 +219,22 @@ impl From<Flags> for PermissionsOptions {
   }
 }
 
-static ENV_VARIABLES_HELP: &str = "ENVIRONMENT VARIABLES:
+static ENV_VARIABLES_HELP: &str = r#"ENVIRONMENT VARIABLES:
+    DENO_AUTH_TOKENS     A semi-colon separated list of bearer tokens and
+                         hostnames to use when fetching remote modules from
+                         private repositories
+                         (e.g. "abcde12345@deno.land;54321edcba@github.com")
+    DENO_CERT            Load certificate authority from PEM encoded file
     DENO_DIR             Set the cache directory
     DENO_INSTALL_ROOT    Set deno install's output directory
                          (defaults to $HOME/.deno/bin)
-    DENO_CERT            Load certificate authority from PEM encoded file
-    NO_COLOR             Set to disable color
     HTTP_PROXY           Proxy address for HTTP requests
                          (module downloads, fetch)
     HTTPS_PROXY          Proxy address for HTTPS requests
                          (module downloads, fetch)
+    NO_COLOR             Set to disable color
     NO_PROXY             Comma-separated list of hosts which do not use a proxy
-                         (module downloads, fetch)";
+                         (module downloads, fetch)"#;
 
 static DENO_HELP: &str = "A secure JavaScript and TypeScript runtime
 
diff --git a/cli/http_util.rs b/cli/http_util.rs
index 437b9355c..69778a101 100644
--- a/cli/http_util.rs
+++ b/cli/http_util.rs
@@ -1,11 +1,14 @@
 // Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
 
+use crate::auth_tokens::AuthToken;
+
 use deno_core::error::generic_error;
 use deno_core::error::AnyError;
 use deno_core::url::Url;
 use deno_runtime::deno_fetch::reqwest;
 use deno_runtime::deno_fetch::reqwest::header::HeaderMap;
 use deno_runtime::deno_fetch::reqwest::header::HeaderValue;
+use deno_runtime::deno_fetch::reqwest::header::AUTHORIZATION;
 use deno_runtime::deno_fetch::reqwest::header::IF_NONE_MATCH;
 use deno_runtime::deno_fetch::reqwest::header::LOCATION;
 use deno_runtime::deno_fetch::reqwest::header::USER_AGENT;
@@ -76,24 +79,33 @@ pub enum FetchOnceResult {
   Redirect(Url, HeadersMap),
 }
 
+#[derive(Debug)]
+pub struct FetchOnceArgs {
+  pub client: Client,
+  pub url: Url,
+  pub maybe_etag: Option<String>,
+  pub maybe_auth_token: Option<AuthToken>,
+}
+
 /// Asynchronously fetches the given HTTP URL one pass only.
 /// If no redirect is present and no error occurs,
 /// yields Code(ResultPayload).
 /// If redirect occurs, does not follow and
 /// yields Redirect(url).
 pub async fn fetch_once(
-  client: Client,
-  url: &Url,
-  cached_etag: Option<String>,
+  args: FetchOnceArgs,
 ) -> Result<FetchOnceResult, AnyError> {
-  let url = url.clone();
+  let mut request = args.client.get(args.url.clone());
 
-  let mut request = client.get(url.clone());
-
-  if let Some(etag) = cached_etag {
+  if let Some(etag) = args.maybe_etag {
     let if_none_match_val = HeaderValue::from_str(&etag).unwrap();
     request = request.header(IF_NONE_MATCH, if_none_match_val);
   }
+  if let Some(auth_token) = args.maybe_auth_token {
+    let authorization_val =
+      HeaderValue::from_str(&auth_token.to_string()).unwrap();
+    request = request.header(AUTHORIZATION, authorization_val);
+  }
   let response = request.send().await?;
 
   if response.status() == StatusCode::NOT_MODIFIED {
@@ -126,20 +138,23 @@ pub async fn fetch_once(
     if let Some(location) = response.headers().get(LOCATION) {
       let location_string = location.to_str().unwrap();
       debug!("Redirecting to {:?}...", &location_string);
-      let new_url = resolve_url_from_location(&url, location_string);
+      let new_url = resolve_url_from_location(&args.url, location_string);
       return Ok(FetchOnceResult::Redirect(new_url, headers_));
     } else {
       return Err(generic_error(format!(
         "Redirection from '{}' did not provide location header",
-        url
+        args.url
       )));
     }
   }
 
   if response.status().is_client_error() || response.status().is_server_error()
   {
-    let err =
-      generic_error(format!("Import '{}' failed: {}", &url, response.status()));
+    let err = generic_error(format!(
+      "Import '{}' failed: {}",
+      args.url,
+      response.status()
+    ));
     return Err(err);
   }
 
@@ -165,7 +180,13 @@ mod tests {
     let url =
       Url::parse("http://127.0.0.1:4545/cli/tests/fixture.json").unwrap();
     let client = create_test_client(None);
-    let result = fetch_once(client, &url, None).await;
+    let result = fetch_once(FetchOnceArgs {
+      client,
+      url,
+      maybe_etag: None,
+      maybe_auth_token: None,
+    })
+    .await;
     if let Ok(FetchOnceResult::Code(body, headers)) = result {
       assert!(!body.is_empty());
       assert_eq!(headers.get("content-type").unwrap(), "application/json");
@@ -185,7 +206,13 @@ mod tests {
     )
     .unwrap();
     let client = create_test_client(None);
-    let result = fetch_once(client, &url, None).await;
+    let result = fetch_once(FetchOnceArgs {
+      client,
+      url,
+      maybe_etag: None,
+      maybe_auth_token: None,
+    })
+    .await;
     if let Ok(FetchOnceResult::Code(body, headers)) = result {
       assert_eq!(String::from_utf8(body).unwrap(), "console.log('gzip')");
       assert_eq!(
@@ -204,7 +231,13 @@ mod tests {
     let _http_server_guard = test_util::http_server();
     let url = Url::parse("http://127.0.0.1:4545/etag_script.ts").unwrap();
     let client = create_test_client(None);
-    let result = fetch_once(client.clone(), &url, None).await;
+    let result = fetch_once(FetchOnceArgs {
+      client: client.clone(),
+      url: url.clone(),
+      maybe_etag: None,
+      maybe_auth_token: None,
+    })
+    .await;
     if let Ok(FetchOnceResult::Code(body, headers)) = result {
       assert!(!body.is_empty());
       assert_eq!(String::from_utf8(body).unwrap(), "console.log('etag')");
@@ -217,8 +250,13 @@ mod tests {
       panic!();
     }
 
-    let res =
-      fetch_once(client, &url, Some("33a64df551425fcc55e".to_string())).await;
+    let res = fetch_once(FetchOnceArgs {
+      client,
+      url,
+      maybe_etag: Some("33a64df551425fcc55e".to_string()),
+      maybe_auth_token: None,
+    })
+    .await;
     assert_eq!(res.unwrap(), FetchOnceResult::NotModified);
   }
 
@@ -231,7 +269,13 @@ mod tests {
     )
     .unwrap();
     let client = create_test_client(None);
-    let result = fetch_once(client, &url, None).await;
+    let result = fetch_once(FetchOnceArgs {
+      client,
+      url,
+      maybe_etag: None,
+      maybe_auth_token: None,
+    })
+    .await;
     if let Ok(FetchOnceResult::Code(body, headers)) = result {
       assert!(!body.is_empty());
       assert_eq!(String::from_utf8(body).unwrap(), "console.log('brotli');");
@@ -256,7 +300,13 @@ mod tests {
     let target_url =
       Url::parse("http://localhost:4545/cli/tests/fixture.json").unwrap();
     let client = create_test_client(None);
-    let result = fetch_once(client, &url, None).await;
+    let result = fetch_once(FetchOnceArgs {
+      client,
+      url,
+      maybe_etag: None,
+      maybe_auth_token: None,
+    })
+    .await;
     if let Ok(FetchOnceResult::Redirect(url, _)) = result {
       assert_eq!(url, target_url);
     } else {
@@ -322,7 +372,13 @@ mod tests {
       ),
     )
     .unwrap();
-    let result = fetch_once(client, &url, None).await;
+    let result = fetch_once(FetchOnceArgs {
+      client,
+      url,
+      maybe_etag: None,
+      maybe_auth_token: None,
+    })
+    .await;
     if let Ok(FetchOnceResult::Code(body, headers)) = result {
       assert!(!body.is_empty());
       assert_eq!(headers.get("content-type").unwrap(), "application/json");
@@ -354,7 +410,13 @@ mod tests {
       ),
     )
     .unwrap();
-    let result = fetch_once(client, &url, None).await;
+    let result = fetch_once(FetchOnceArgs {
+      client,
+      url,
+      maybe_etag: None,
+      maybe_auth_token: None,
+    })
+    .await;
     if let Ok(FetchOnceResult::Code(body, headers)) = result {
       assert_eq!(String::from_utf8(body).unwrap(), "console.log('gzip')");
       assert_eq!(
@@ -385,7 +447,13 @@ mod tests {
       ),
     )
     .unwrap();
-    let result = fetch_once(client.clone(), &url, None).await;
+    let result = fetch_once(FetchOnceArgs {
+      client: client.clone(),
+      url: url.clone(),
+      maybe_etag: None,
+      maybe_auth_token: None,
+    })
+    .await;
     if let Ok(FetchOnceResult::Code(body, headers)) = result {
       assert!(!body.is_empty());
       assert_eq!(String::from_utf8(body).unwrap(), "console.log('etag')");
@@ -399,8 +467,13 @@ mod tests {
       panic!();
     }
 
-    let res =
-      fetch_once(client, &url, Some("33a64df551425fcc55e".to_string())).await;
+    let res = fetch_once(FetchOnceArgs {
+      client,
+      url,
+      maybe_etag: Some("33a64df551425fcc55e".to_string()),
+      maybe_auth_token: None,
+    })
+    .await;
     assert_eq!(res.unwrap(), FetchOnceResult::NotModified);
   }
 
@@ -425,7 +498,13 @@ mod tests {
       ),
     )
     .unwrap();
-    let result = fetch_once(client, &url, None).await;
+    let result = fetch_once(FetchOnceArgs {
+      client,
+      url,
+      maybe_etag: None,
+      maybe_auth_token: None,
+    })
+    .await;
     if let Ok(FetchOnceResult::Code(body, headers)) = result {
       assert!(!body.is_empty());
       assert_eq!(String::from_utf8(body).unwrap(), "console.log('brotli');");
@@ -446,7 +525,13 @@ mod tests {
     let url_str = "http://127.0.0.1:4545/bad_redirect";
     let url = Url::parse(url_str).unwrap();
     let client = create_test_client(None);
-    let result = fetch_once(client, &url, None).await;
+    let result = fetch_once(FetchOnceArgs {
+      client,
+      url,
+      maybe_etag: None,
+      maybe_auth_token: None,
+    })
+    .await;
     assert!(result.is_err());
     let err = result.unwrap_err();
     // Check that the error message contains the original URL
diff --git a/cli/main.rs b/cli/main.rs
index d98313f54..2addfa0d5 100644
--- a/cli/main.rs
+++ b/cli/main.rs
@@ -8,6 +8,7 @@ extern crate lazy_static;
 extern crate log;
 
 mod ast;
+mod auth_tokens;
 mod checksum;
 mod colors;
 mod deno_dir;
diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs
index 15da2359d..a25a51b20 100644
--- a/cli/tests/integration_tests.rs
+++ b/cli/tests/integration_tests.rs
@@ -208,6 +208,42 @@ mod integration {
     assert_eq!("noColor false", util::strip_ansi_codes(stdout_str));
   }
 
+  #[test]
+  fn auth_tokens() {
+    let _g = util::http_server();
+    let output = util::deno_cmd()
+      .current_dir(util::root_path())
+      .arg("run")
+      .arg("http://127.0.0.1:4551/cli/tests/001_hello.js")
+      .stdout(std::process::Stdio::piped())
+      .stderr(std::process::Stdio::piped())
+      .spawn()
+      .unwrap()
+      .wait_with_output()
+      .unwrap();
+    assert!(!output.status.success());
+    let stdout_str = std::str::from_utf8(&output.stdout).unwrap().trim();
+    assert!(stdout_str.is_empty());
+    let stderr_str = std::str::from_utf8(&output.stderr).unwrap().trim();
+    eprintln!("{}", stderr_str);
+    assert!(stderr_str.contains("Import 'http://127.0.0.1:4551/cli/tests/001_hello.js' failed: 404 Not Found"));
+
+    let output = util::deno_cmd()
+      .current_dir(util::root_path())
+      .arg("run")
+      .arg("http://127.0.0.1:4551/cli/tests/001_hello.js")
+      .env("DENO_AUTH_TOKENS", "abcdef123456789@127.0.0.1:4551")
+      .stdout(std::process::Stdio::piped())
+      .stderr(std::process::Stdio::piped())
+      .spawn()
+      .unwrap()
+      .wait_with_output()
+      .unwrap();
+    assert!(output.status.success());
+    let stdout_str = std::str::from_utf8(&output.stdout).unwrap().trim();
+    assert_eq!(util::strip_ansi_codes(stdout_str), "Hello World");
+  }
+
   #[cfg(unix)]
   #[test]
   pub fn test_raw_tty() {