summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--Cargo.lock4
-rw-r--r--cli/Cargo.toml2
-rw-r--r--cli/flags.rs138
-rw-r--r--cli/fs_util.rs68
-rw-r--r--cli/main.rs13
-rw-r--r--cli/tests/integration/mod.rs2
-rw-r--r--cli/tests/integration/vendor_tests.rs372
-rw-r--r--cli/tests/testdata/vendor/dynamic.ts3
-rw-r--r--cli/tests/testdata/vendor/dynamic_non_analyzable.ts4
-rw-r--r--cli/tests/testdata/vendor/logger.ts5
-rw-r--r--cli/tests/testdata/vendor/query_reexport.ts1
-rw-r--r--cli/tools/mod.rs1
-rw-r--r--cli/tools/vendor/analyze.rs113
-rw-r--r--cli/tools/vendor/build.rs577
-rw-r--r--cli/tools/vendor/import_map.rs285
-rw-r--r--cli/tools/vendor/mappings.rs286
-rw-r--r--cli/tools/vendor/mod.rs172
-rw-r--r--cli/tools/vendor/specifiers.rs251
-rw-r--r--cli/tools/vendor/test.rs240
-rw-r--r--ext/http/lib.rs3
-rw-r--r--runtime/ops/http.rs3
21 files changed, 2531 insertions, 12 deletions
diff --git a/Cargo.lock b/Cargo.lock
index c17564493..d34f28955 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -1965,9 +1965,9 @@ checksum = "cb56e1aa765b4b4f3aadfab769793b7087bb03a4ea4920644a6d238e2df5b9ed"
[[package]]
name = "import_map"
-version = "0.8.0"
+version = "0.9.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09ae88504e9128c4c181a0a4726d868d52aa76de270c7fb00c3c40a8f4fbace4"
+checksum = "f99e0f89d56c163538ea6bf1f250049669298a26daeee15a9a18f4118cc503f1"
dependencies = [
"indexmap",
"log",
diff --git a/cli/Cargo.toml b/cli/Cargo.toml
index 92687c283..9aae5db8f 100644
--- a/cli/Cargo.toml
+++ b/cli/Cargo.toml
@@ -69,7 +69,7 @@ env_logger = "=0.8.4"
eszip = "=0.16.0"
fancy-regex = "=0.7.1"
http = "=0.2.4"
-import_map = "=0.8.0"
+import_map = "=0.9.0"
jsonc-parser = { version = "=0.19.0", features = ["serde"] }
libc = "=0.2.106"
log = { version = "=0.4.14", features = ["serde"] }
diff --git a/cli/flags.rs b/cli/flags.rs
index 6bb03b993..ab8153c4e 100644
--- a/cli/flags.rs
+++ b/cli/flags.rs
@@ -163,6 +163,13 @@ pub struct UpgradeFlags {
}
#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
+pub struct VendorFlags {
+ pub specifiers: Vec<String>,
+ pub output_path: Option<PathBuf>,
+ pub force: bool,
+}
+
+#[derive(Clone, Debug, PartialEq, Deserialize, Serialize)]
pub enum DenoSubcommand {
Bundle(BundleFlags),
Cache(CacheFlags),
@@ -182,6 +189,7 @@ pub enum DenoSubcommand {
Test(TestFlags),
Types,
Upgrade(UpgradeFlags),
+ Vendor(VendorFlags),
}
impl Default for DenoSubcommand {
@@ -481,6 +489,7 @@ pub fn flags_from_vec(args: Vec<String>) -> clap::Result<Flags> {
Some(("lint", m)) => lint_parse(&mut flags, m),
Some(("compile", m)) => compile_parse(&mut flags, m),
Some(("lsp", m)) => lsp_parse(&mut flags, m),
+ Some(("vendor", m)) => vendor_parse(&mut flags, m),
_ => handle_repl_flags(&mut flags, ReplFlags { eval: None }),
}
@@ -552,6 +561,7 @@ If the flag is set, restrict these messages to errors.",
.subcommand(test_subcommand())
.subcommand(types_subcommand())
.subcommand(upgrade_subcommand())
+ .subcommand(vendor_subcommand())
.long_about(DENO_HELP)
.after_help(ENV_VARIABLES_HELP)
}
@@ -1413,6 +1423,52 @@ update to a different location, use the --output flag
.arg(ca_file_arg())
}
+fn vendor_subcommand<'a>() -> App<'a> {
+ App::new("vendor")
+ .about("Vendor remote modules into a local directory")
+ .long_about(
+ "Vendor remote modules into a local directory.
+
+Analyzes the provided modules along with their dependencies, downloads
+remote modules to the output directory, and produces an import map that
+maps remote specifiers to the downloaded files.
+
+ deno vendor main.ts
+ deno run --import-map vendor/import_map.json main.ts
+
+Remote modules and multiple modules may also be specified:
+
+ deno vendor main.ts test.deps.ts https://deno.land/std/path/mod.ts",
+ )
+ .arg(
+ Arg::new("specifiers")
+ .takes_value(true)
+ .multiple_values(true)
+ .multiple_occurrences(true)
+ .required(true),
+ )
+ .arg(
+ Arg::new("output")
+ .long("output")
+ .help("The directory to output the vendored modules to")
+ .takes_value(true),
+ )
+ .arg(
+ Arg::new("force")
+ .long("force")
+ .short('f')
+ .help(
+ "Forcefully overwrite conflicting files in existing output directory",
+ )
+ .takes_value(false),
+ )
+ .arg(config_arg())
+ .arg(import_map_arg())
+ .arg(lock_arg())
+ .arg(reload_arg())
+ .arg(ca_file_arg())
+}
+
fn compile_args(app: App) -> App {
app
.arg(import_map_arg())
@@ -2237,6 +2293,23 @@ fn upgrade_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
});
}
+fn vendor_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
+ ca_file_arg_parse(flags, matches);
+ config_arg_parse(flags, matches);
+ import_map_arg_parse(flags, matches);
+ lock_arg_parse(flags, matches);
+ reload_arg_parse(flags, matches);
+
+ flags.subcommand = DenoSubcommand::Vendor(VendorFlags {
+ specifiers: matches
+ .values_of("specifiers")
+ .map(|p| p.map(ToString::to_string).collect())
+ .unwrap_or_default(),
+ output_path: matches.value_of("output").map(PathBuf::from),
+ force: matches.is_present("force"),
+ });
+}
+
fn compile_args_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
import_map_arg_parse(flags, matches);
no_remote_arg_parse(flags, matches);
@@ -2443,13 +2516,17 @@ fn no_check_arg_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
}
fn lock_args_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
+ lock_arg_parse(flags, matches);
+ if matches.is_present("lock-write") {
+ flags.lock_write = true;
+ }
+}
+
+fn lock_arg_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
if matches.is_present("lock") {
let lockfile = matches.value_of("lock").unwrap();
flags.lock = Some(PathBuf::from(lockfile));
}
- if matches.is_present("lock-write") {
- flags.lock_write = true;
- }
}
fn config_arg_parse(flags: &mut Flags, matches: &ArgMatches) {
@@ -2512,8 +2589,8 @@ mod tests {
/// Creates vector of strings, Vec<String>
macro_rules! svec {
- ($($x:expr),*) => (vec![$($x.to_string()),*]);
-}
+ ($($x:expr),* $(,)?) => (vec![$($x.to_string()),*]);
+ }
#[test]
fn global_flags() {
@@ -4895,4 +4972,55 @@ mod tests {
.contains("error: The following required arguments were not provided:"));
assert!(&error_message.contains("--watch=<FILES>..."));
}
+
+ #[test]
+ fn vendor_minimal() {
+ let r = flags_from_vec(svec!["deno", "vendor", "mod.ts",]);
+ assert_eq!(
+ r.unwrap(),
+ Flags {
+ subcommand: DenoSubcommand::Vendor(VendorFlags {
+ specifiers: svec!["mod.ts"],
+ force: false,
+ output_path: None,
+ }),
+ ..Flags::default()
+ }
+ );
+ }
+
+ #[test]
+ fn vendor_all() {
+ let r = flags_from_vec(svec![
+ "deno",
+ "vendor",
+ "--config",
+ "deno.json",
+ "--import-map",
+ "import_map.json",
+ "--lock",
+ "lock.json",
+ "--force",
+ "--output",
+ "out_dir",
+ "--reload",
+ "mod.ts",
+ "deps.test.ts",
+ ]);
+ assert_eq!(
+ r.unwrap(),
+ Flags {
+ subcommand: DenoSubcommand::Vendor(VendorFlags {
+ specifiers: svec!["mod.ts", "deps.test.ts"],
+ force: true,
+ output_path: Some(PathBuf::from("out_dir")),
+ }),
+ config_path: Some("deno.json".to_string()),
+ import_map_path: Some("import_map.json".to_string()),
+ lock: Some(PathBuf::from("lock.json")),
+ reload: true,
+ ..Flags::default()
+ }
+ );
+ }
}
diff --git a/cli/fs_util.rs b/cli/fs_util.rs
index fbdcdc81a..2f10a523f 100644
--- a/cli/fs_util.rs
+++ b/cli/fs_util.rs
@@ -362,6 +362,34 @@ pub fn path_has_trailing_slash(path: &Path) -> bool {
}
}
+/// Gets a path with the specified file stem suffix.
+///
+/// Ex. `file.ts` with suffix `_2` returns `file_2.ts`
+pub fn path_with_stem_suffix(path: &Path, suffix: &str) -> PathBuf {
+ if let Some(file_name) = path.file_name().map(|f| f.to_string_lossy()) {
+ if let Some(file_stem) = path.file_stem().map(|f| f.to_string_lossy()) {
+ if let Some(ext) = path.extension().map(|f| f.to_string_lossy()) {
+ return if file_stem.to_lowercase().ends_with(".d") {
+ path.with_file_name(format!(
+ "{}{}.{}.{}",
+ &file_stem[..file_stem.len() - ".d".len()],
+ suffix,
+ // maintain casing
+ &file_stem[file_stem.len() - "d".len()..],
+ ext
+ ))
+ } else {
+ path.with_file_name(format!("{}{}.{}", file_stem, suffix, ext))
+ };
+ }
+ }
+
+ path.with_file_name(format!("{}{}", file_name, suffix))
+ } else {
+ path.with_file_name(suffix)
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -730,4 +758,44 @@ mod tests {
assert_eq!(result, expected);
}
}
+
+ #[test]
+ fn test_path_with_stem_suffix() {
+ assert_eq!(
+ path_with_stem_suffix(&PathBuf::from("/"), "_2"),
+ PathBuf::from("/_2")
+ );
+ assert_eq!(
+ path_with_stem_suffix(&PathBuf::from("/test"), "_2"),
+ PathBuf::from("/test_2")
+ );
+ assert_eq!(
+ path_with_stem_suffix(&PathBuf::from("/test.txt"), "_2"),
+ PathBuf::from("/test_2.txt")
+ );
+ assert_eq!(
+ path_with_stem_suffix(&PathBuf::from("/test/subdir"), "_2"),
+ PathBuf::from("/test/subdir_2")
+ );
+ assert_eq!(
+ path_with_stem_suffix(&PathBuf::from("/test/subdir.other.txt"), "_2"),
+ PathBuf::from("/test/subdir.other_2.txt")
+ );
+ assert_eq!(
+ path_with_stem_suffix(&PathBuf::from("/test.d.ts"), "_2"),
+ PathBuf::from("/test_2.d.ts")
+ );
+ assert_eq!(
+ path_with_stem_suffix(&PathBuf::from("/test.D.TS"), "_2"),
+ PathBuf::from("/test_2.D.TS")
+ );
+ assert_eq!(
+ path_with_stem_suffix(&PathBuf::from("/test.d.mts"), "_2"),
+ PathBuf::from("/test_2.d.mts")
+ );
+ assert_eq!(
+ path_with_stem_suffix(&PathBuf::from("/test.d.cts"), "_2"),
+ PathBuf::from("/test_2.d.cts")
+ );
+ }
}
diff --git a/cli/main.rs b/cli/main.rs
index ca6b36f0a..f8c5d69df 100644
--- a/cli/main.rs
+++ b/cli/main.rs
@@ -58,6 +58,7 @@ use crate::flags::RunFlags;
use crate::flags::TestFlags;
use crate::flags::UninstallFlags;
use crate::flags::UpgradeFlags;
+use crate::flags::VendorFlags;
use crate::fmt_errors::PrettyJsError;
use crate::graph_util::graph_lock_or_exit;
use crate::graph_util::graph_valid;
@@ -1290,6 +1291,15 @@ async fn upgrade_command(
Ok(0)
}
+async fn vendor_command(
+ flags: Flags,
+ vendor_flags: VendorFlags,
+) -> Result<i32, AnyError> {
+ let ps = ProcState::build(Arc::new(flags)).await?;
+ tools::vendor::vendor(ps, vendor_flags).await?;
+ Ok(0)
+}
+
fn init_v8_flags(v8_flags: &[String]) {
let v8_flags_includes_help = v8_flags
.iter()
@@ -1368,6 +1378,9 @@ fn get_subcommand(
DenoSubcommand::Upgrade(upgrade_flags) => {
upgrade_command(flags, upgrade_flags).boxed_local()
}
+ DenoSubcommand::Vendor(vendor_flags) => {
+ vendor_command(flags, vendor_flags).boxed_local()
+ }
}
}
diff --git a/cli/tests/integration/mod.rs b/cli/tests/integration/mod.rs
index cc6770472..2a1e69bd1 100644
--- a/cli/tests/integration/mod.rs
+++ b/cli/tests/integration/mod.rs
@@ -84,6 +84,8 @@ mod run;
mod test;
#[path = "upgrade_tests.rs"]
mod upgrade;
+#[path = "vendor_tests.rs"]
+mod vendor;
#[path = "watcher_tests.rs"]
mod watcher;
#[path = "worker_tests.rs"]
diff --git a/cli/tests/integration/vendor_tests.rs b/cli/tests/integration/vendor_tests.rs
new file mode 100644
index 000000000..4aa883a7e
--- /dev/null
+++ b/cli/tests/integration/vendor_tests.rs
@@ -0,0 +1,372 @@
+// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
+
+use deno_core::serde_json;
+use deno_core::serde_json::json;
+use pretty_assertions::assert_eq;
+use std::fs;
+use std::path::PathBuf;
+use std::process::Stdio;
+use tempfile::TempDir;
+use test_util as util;
+use util::http_server;
+
+#[test]
+fn output_dir_exists() {
+ let t = TempDir::new().unwrap();
+ let vendor_dir = t.path().join("vendor");
+ fs::write(t.path().join("mod.ts"), "").unwrap();
+ fs::create_dir_all(&vendor_dir).unwrap();
+ fs::write(vendor_dir.join("mod.ts"), "").unwrap();
+
+ let deno = util::deno_cmd()
+ .current_dir(t.path())
+ .env("NO_COLOR", "1")
+ .arg("vendor")
+ .arg("mod.ts")
+ .stderr(Stdio::piped())
+ .spawn()
+ .unwrap();
+ let output = deno.wait_with_output().unwrap();
+ assert_eq!(
+ String::from_utf8_lossy(&output.stderr).trim(),
+ concat!(
+ "error: Output directory was not empty. Please specify an empty ",
+ "directory or use --force to ignore this error and potentially ",
+ "overwrite its contents.",
+ ),
+ );
+ assert!(!output.status.success());
+
+ // ensure it errors when using the `--output` arg too
+ let deno = util::deno_cmd()
+ .current_dir(t.path())
+ .env("NO_COLOR", "1")
+ .arg("vendor")
+ .arg("--output")
+ .arg("vendor")
+ .arg("mod.ts")
+ .stderr(Stdio::piped())
+ .spawn()
+ .unwrap();
+ let output = deno.wait_with_output().unwrap();
+ assert_eq!(
+ String::from_utf8_lossy(&output.stderr).trim(),
+ concat!(
+ "error: Output directory was not empty. Please specify an empty ",
+ "directory or use --force to ignore this error and potentially ",
+ "overwrite its contents.",
+ ),
+ );
+ assert!(!output.status.success());
+
+ // now use `--force`
+ let status = util::deno_cmd()
+ .current_dir(t.path())
+ .env("NO_COLOR", "1")
+ .arg("vendor")
+ .arg("mod.ts")
+ .arg("--force")
+ .spawn()
+ .unwrap()
+ .wait()
+ .unwrap();
+ assert!(status.success());
+}
+
+#[test]
+fn import_map_output_dir() {
+ let t = TempDir::new().unwrap();
+ let vendor_dir = t.path().join("vendor");
+ fs::write(t.path().join("mod.ts"), "").unwrap();
+ fs::create_dir_all(&vendor_dir).unwrap();
+ let import_map_path = vendor_dir.join("import_map.json");
+ fs::write(
+ &import_map_path,
+ "{ \"imports\": { \"https://localhost/\": \"./localhost/\" }}",
+ )
+ .unwrap();
+
+ let deno = util::deno_cmd()
+ .current_dir(t.path())
+ .env("NO_COLOR", "1")
+ .arg("vendor")
+ .arg("--force")
+ .arg("--import-map")
+ .arg(import_map_path)
+ .arg("mod.ts")
+ .stdout(Stdio::piped())
+ .stderr(Stdio::piped())
+ .spawn()
+ .unwrap();
+ let output = deno.wait_with_output().unwrap();
+ assert_eq!(
+ String::from_utf8_lossy(&output.stderr).trim(),
+ "error: Using an import map found in the output directory is not supported.",
+ );
+ assert!(!output.status.success());
+}
+
+#[test]
+fn standard_test() {
+ let _server = http_server();
+ let t = TempDir::new().unwrap();
+ let vendor_dir = t.path().join("vendor2");
+ fs::write(
+ t.path().join("my_app.ts"),
+ "import {Logger} from 'http://localhost:4545/vendor/query_reexport.ts?testing'; new Logger().log('outputted');",
+ ).unwrap();
+
+ let deno = util::deno_cmd()
+ .current_dir(t.path())
+ .arg("vendor")
+ .arg("my_app.ts")
+ .arg("--output")
+ .arg("vendor2")
+ .env("NO_COLOR", "1")
+ .stdout(Stdio::piped())
+ .stderr(Stdio::piped())
+ .spawn()
+ .unwrap();
+ let output = deno.wait_with_output().unwrap();
+ assert_eq!(
+ String::from_utf8_lossy(&output.stderr).trim(),
+ format!(
+ concat!(
+ "Download http://localhost:4545/vendor/query_reexport.ts?testing\n",
+ "Download http://localhost:4545/vendor/logger.ts?test\n",
+ "{}",
+ ),
+ success_text("2 modules", "vendor2", "my_app.ts"),
+ )
+ );
+ assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "");
+ assert!(output.status.success());
+
+ assert!(vendor_dir.exists());
+ assert!(!t.path().join("vendor").exists());
+ let import_map: serde_json::Value = serde_json::from_str(
+ &fs::read_to_string(vendor_dir.join("import_map.json")).unwrap(),
+ )
+ .unwrap();
+ assert_eq!(
+ import_map,
+ json!({
+ "imports": {
+ "http://localhost:4545/": "./localhost_4545/",
+ "http://localhost:4545/vendor/query_reexport.ts?testing": "./localhost_4545/vendor/query_reexport.ts",
+ },
+ "scopes": {
+ "./localhost_4545/": {
+ "./localhost_4545/vendor/logger.ts?test": "./localhost_4545/vendor/logger.ts"
+ }
+ }
+ }),
+ );
+
+ // try running the output with `--no-remote`
+ let deno = util::deno_cmd()
+ .current_dir(t.path())
+ .env("NO_COLOR", "1")
+ .arg("run")
+ .arg("--no-remote")
+ .arg("--no-check")
+ .arg("--import-map")
+ .arg("vendor2/import_map.json")
+ .arg("my_app.ts")
+ .stdout(Stdio::piped())
+ .stderr(Stdio::piped())
+ .spawn()
+ .unwrap();
+ let output = deno.wait_with_output().unwrap();
+ assert_eq!(String::from_utf8_lossy(&output.stderr).trim(), "");
+ assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "outputted");
+ assert!(output.status.success());
+}
+
+#[test]
+fn remote_module_test() {
+ let _server = http_server();
+ let t = TempDir::new().unwrap();
+ let vendor_dir = t.path().join("vendor");
+
+ let deno = util::deno_cmd()
+ .current_dir(t.path())
+ .env("NO_COLOR", "1")
+ .arg("vendor")
+ .arg("http://localhost:4545/vendor/query_reexport.ts")
+ .stdout(Stdio::piped())
+ .stderr(Stdio::piped())
+ .spawn()
+ .unwrap();
+ let output = deno.wait_with_output().unwrap();
+ assert_eq!(
+ String::from_utf8_lossy(&output.stderr).trim(),
+ format!(
+ concat!(
+ "Download http://localhost:4545/vendor/query_reexport.ts\n",
+ "Download http://localhost:4545/vendor/logger.ts?test\n",
+ "{}",
+ ),
+ success_text("2 modules", "vendor/", "main.ts"),
+ )
+ );
+ assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "");
+ assert!(output.status.success());
+ assert!(vendor_dir.exists());
+ assert!(vendor_dir
+ .join("localhost_4545/vendor/query_reexport.ts")
+ .exists());
+ assert!(vendor_dir.join("localhost_4545/vendor/logger.ts").exists());
+ let import_map: serde_json::Value = serde_json::from_str(
+ &fs::read_to_string(vendor_dir.join("import_map.json")).unwrap(),
+ )
+ .unwrap();
+ assert_eq!(
+ import_map,
+ json!({
+ "imports": {
+ "http://localhost:4545/": "./localhost_4545/",
+ },
+ "scopes": {
+ "./localhost_4545/": {
+ "./localhost_4545/vendor/logger.ts?test": "./localhost_4545/vendor/logger.ts"
+ }
+ }
+ }),
+ );
+}
+
+#[test]
+fn existing_import_map() {
+ let _server = http_server();
+ let t = TempDir::new().unwrap();
+ let vendor_dir = t.path().join("vendor");
+ fs::write(
+ t.path().join("mod.ts"),
+ "import {Logger} from 'http://localhost:4545/vendor/logger.ts';",
+ )
+ .unwrap();
+ fs::write(
+ t.path().join("imports.json"),
+ r#"{ "imports": { "http://localhost:4545/vendor/": "./logger/" } }"#,
+ )
+ .unwrap();
+ fs::create_dir(t.path().join("logger")).unwrap();
+ fs::write(t.path().join("logger/logger.ts"), "export class Logger {}")
+ .unwrap();
+
+ let status = util::deno_cmd()
+ .current_dir(t.path())
+ .arg("vendor")
+ .arg("mod.ts")
+ .arg("--import-map")
+ .arg("imports.json")
+ .spawn()
+ .unwrap()
+ .wait()
+ .unwrap();
+ assert!(status.success());
+ // it should not have found any remote dependencies because
+ // the provided import map mapped it to a local directory
+ assert!(!vendor_dir.join("import_map.json").exists());
+}
+
+#[test]
+fn dynamic_import() {
+ let _server = http_server();
+ let t = TempDir::new().unwrap();
+ let vendor_dir = t.path().join("vendor");
+ fs::write(
+ t.path().join("mod.ts"),
+ "import {Logger} from 'http://localhost:4545/vendor/dynamic.ts'; new Logger().log('outputted');",
+ ).unwrap();
+
+ let status = util::deno_cmd()
+ .current_dir(t.path())
+ .arg("vendor")
+ .arg("mod.ts")
+ .spawn()
+ .unwrap()
+ .wait()
+ .unwrap();
+ assert!(status.success());
+ let import_map: serde_json::Value = serde_json::from_str(
+ &fs::read_to_string(vendor_dir.join("import_map.json")).unwrap(),
+ )
+ .unwrap();
+ assert_eq!(
+ import_map,
+ json!({
+ "imports": {
+ "http://localhost:4545/": "./localhost_4545/",
+ }
+ }),
+ );
+
+ // try running the output with `--no-remote`
+ let deno = util::deno_cmd()
+ .current_dir(t.path())
+ .env("NO_COLOR", "1")
+ .arg("run")
+ .arg("--allow-read=.")
+ .arg("--no-remote")
+ .arg("--no-check")
+ .arg("--import-map")
+ .arg("vendor/import_map.json")
+ .arg("mod.ts")
+ .stdout(Stdio::piped())
+ .stderr(Stdio::piped())
+ .spawn()
+ .unwrap();
+ let output = deno.wait_with_output().unwrap();
+ assert_eq!(String::from_utf8_lossy(&output.stderr).trim(), "");
+ assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "outputted");
+ assert!(output.status.success());
+}
+
+#[test]
+fn dynamic_non_analyzable_import() {
+ let _server = http_server();
+ let t = TempDir::new().unwrap();
+ fs::write(
+ t.path().join("mod.ts"),
+ "import {Logger} from 'http://localhost:4545/vendor/dynamic_non_analyzable.ts'; new Logger().log('outputted');",
+ ).unwrap();
+
+ let deno = util::deno_cmd()
+ .current_dir(t.path())
+ .env("NO_COLOR", "1")
+ .arg("vendor")
+ .arg("--reload")
+ .arg("mod.ts")
+ .stdout(Stdio::piped())
+ .stderr(Stdio::piped())
+ .spawn()
+ .unwrap();
+ let output = deno.wait_with_output().unwrap();
+ // todo(https://github.com/denoland/deno_graph/issues/138): it should warn about
+ // how it couldn't analyze the dynamic import
+ assert_eq!(
+ String::from_utf8_lossy(&output.stderr).trim(),
+ format!(
+ "Download http://localhost:4545/vendor/dynamic_non_analyzable.ts\n{}",
+ success_text("1 module", "vendor/", "mod.ts"),
+ )
+ );
+ assert_eq!(String::from_utf8_lossy(&output.stdout).trim(), "");
+ assert!(output.status.success());
+}
+
+fn success_text(module_count: &str, dir: &str, entry_point: &str) -> String {
+ format!(
+ concat!(
+ "Vendored {} into {} directory.\n\n",
+ "To use vendored modules, specify the `--import-map` flag when invoking deno subcommands:\n",
+ " deno run -A --import-map {} {}"
+ ),
+ module_count,
+ dir,
+ PathBuf::from(dir).join("import_map.json").display(),
+ entry_point,
+ )
+}
diff --git a/cli/tests/testdata/vendor/dynamic.ts b/cli/tests/testdata/vendor/dynamic.ts
new file mode 100644
index 000000000..e2cbb0e59
--- /dev/null
+++ b/cli/tests/testdata/vendor/dynamic.ts
@@ -0,0 +1,3 @@
+const { Logger } = await import("./logger.ts");
+
+export { Logger };
diff --git a/cli/tests/testdata/vendor/dynamic_non_analyzable.ts b/cli/tests/testdata/vendor/dynamic_non_analyzable.ts
new file mode 100644
index 000000000..1847939f6
--- /dev/null
+++ b/cli/tests/testdata/vendor/dynamic_non_analyzable.ts
@@ -0,0 +1,4 @@
+const value = (() => "./logger.ts")();
+const { Logger } = await import(value);
+
+export { Logger };
diff --git a/cli/tests/testdata/vendor/logger.ts b/cli/tests/testdata/vendor/logger.ts
new file mode 100644
index 000000000..97f603a48
--- /dev/null
+++ b/cli/tests/testdata/vendor/logger.ts
@@ -0,0 +1,5 @@
+export class Logger {
+ log(text: string) {
+ console.log(text);
+ }
+}
diff --git a/cli/tests/testdata/vendor/query_reexport.ts b/cli/tests/testdata/vendor/query_reexport.ts
new file mode 100644
index 000000000..5dfafb532
--- /dev/null
+++ b/cli/tests/testdata/vendor/query_reexport.ts
@@ -0,0 +1 @@
+export * from "./logger.ts?test";
diff --git a/cli/tools/mod.rs b/cli/tools/mod.rs
index 83c501dde..ffea76e1d 100644
--- a/cli/tools/mod.rs
+++ b/cli/tools/mod.rs
@@ -9,3 +9,4 @@ pub mod repl;
pub mod standalone;
pub mod test;
pub mod upgrade;
+pub mod vendor;
diff --git a/cli/tools/vendor/analyze.rs b/cli/tools/vendor/analyze.rs
new file mode 100644
index 000000000..0639c0487
--- /dev/null
+++ b/cli/tools/vendor/analyze.rs
@@ -0,0 +1,113 @@
+// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
+
+use deno_ast::swc::ast::ExportDefaultDecl;
+use deno_ast::swc::ast::ExportSpecifier;
+use deno_ast::swc::ast::ModuleExportName;
+use deno_ast::swc::ast::NamedExport;
+use deno_ast::swc::ast::Program;
+use deno_ast::swc::visit::noop_visit_type;
+use deno_ast::swc::visit::Visit;
+use deno_ast::swc::visit::VisitWith;
+use deno_ast::ParsedSource;
+
+/// Gets if the parsed source has a default export.
+pub fn has_default_export(source: &ParsedSource) -> bool {
+ let mut visitor = DefaultExportFinder {
+ has_default_export: false,
+ };
+ let program = source.program();
+ let program: &Program = &program;
+ program.visit_with(&mut visitor);
+ visitor.has_default_export
+}
+
+struct DefaultExportFinder {
+ has_default_export: bool,
+}
+
+impl<'a> Visit for DefaultExportFinder {
+ noop_visit_type!();
+
+ fn visit_export_default_decl(&mut self, _: &ExportDefaultDecl) {
+ self.has_default_export = true;
+ }
+
+ fn visit_named_export(&mut self, named_export: &NamedExport) {
+ if named_export
+ .specifiers
+ .iter()
+ .any(export_specifier_has_default)
+ {
+ self.has_default_export = true;
+ }
+ }
+}
+
+fn export_specifier_has_default(s: &ExportSpecifier) -> bool {
+ match s {
+ ExportSpecifier::Default(_) => true,
+ ExportSpecifier::Namespace(_) => false,
+ ExportSpecifier::Named(named) => {
+ let export_name = named.exported.as_ref().unwrap_or(&named.orig);
+
+ match export_name {
+ ModuleExportName::Str(_) => false,
+ ModuleExportName::Ident(ident) => &*ident.sym == "default",
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod test {
+ use deno_ast::MediaType;
+ use deno_ast::ParseParams;
+ use deno_ast::ParsedSource;
+ use deno_ast::SourceTextInfo;
+
+ use super::has_default_export;
+
+ #[test]
+ fn has_default_when_export_default_decl() {
+ let parsed_source = parse_module("export default class Class {}");
+ assert!(has_default_export(&parsed_source));
+ }
+
+ #[test]
+ fn has_default_when_named_export() {
+ let parsed_source = parse_module("export {default} from './test.ts';");
+ assert!(has_default_export(&parsed_source));
+ }
+
+ #[test]
+ fn has_default_when_named_export_alias() {
+ let parsed_source =
+ parse_module("export {test as default} from './test.ts';");
+ assert!(has_default_export(&parsed_source));
+ }
+
+ #[test]
+ fn not_has_default_when_named_export_not_exported() {
+ let parsed_source =
+ parse_module("export {default as test} from './test.ts';");
+ assert!(!has_default_export(&parsed_source));
+ }
+
+ #[test]
+ fn not_has_default_when_not() {
+ let parsed_source = parse_module("export {test} from './test.ts'; export class Test{} export * from './test';");
+ assert!(!has_default_export(&parsed_source));
+ }
+
+ fn parse_module(text: &str) -> ParsedSource {
+ deno_ast::parse_module(ParseParams {
+ specifier: "file:///mod.ts".to_string(),
+ capture_tokens: false,
+ maybe_syntax: None,
+ media_type: MediaType::TypeScript,
+ scope_analysis: false,
+ source: SourceTextInfo::from_string(text.to_string()),
+ })
+ .unwrap()
+ }
+}
diff --git a/cli/tools/vendor/build.rs b/cli/tools/vendor/build.rs
new file mode 100644
index 000000000..58f351dd8
--- /dev/null
+++ b/cli/tools/vendor/build.rs
@@ -0,0 +1,577 @@
+// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
+
+use std::path::Path;
+
+use deno_core::error::AnyError;
+use deno_graph::Module;
+use deno_graph::ModuleGraph;
+use deno_graph::ModuleKind;
+
+use super::analyze::has_default_export;
+use super::import_map::build_import_map;
+use super::mappings::Mappings;
+use super::mappings::ProxiedModule;
+use super::specifiers::is_remote_specifier;
+
+/// Allows substituting the environment for testing purposes.
+pub trait VendorEnvironment {
+ fn create_dir_all(&self, dir_path: &Path) -> Result<(), AnyError>;
+ fn write_file(&self, file_path: &Path, text: &str) -> Result<(), AnyError>;
+}
+
+pub struct RealVendorEnvironment;
+
+impl VendorEnvironment for RealVendorEnvironment {
+ fn create_dir_all(&self, dir_path: &Path) -> Result<(), AnyError> {
+ Ok(std::fs::create_dir_all(dir_path)?)
+ }
+
+ fn write_file(&self, file_path: &Path, text: &str) -> Result<(), AnyError> {
+ Ok(std::fs::write(file_path, text)?)
+ }
+}
+
+/// Vendors remote modules and returns how many were vendored.
+pub fn build(
+ graph: &ModuleGraph,
+ output_dir: &Path,
+ environment: &impl VendorEnvironment,
+) -> Result<usize, AnyError> {
+ assert!(output_dir.is_absolute());
+ let all_modules = graph.modules();
+ let remote_modules = all_modules
+ .iter()
+ .filter(|m| is_remote_specifier(&m.specifier))
+ .copied()
+ .collect::<Vec<_>>();
+ let mappings =
+ Mappings::from_remote_modules(graph, &remote_modules, output_dir)?;
+
+ // write out all the files
+ for module in &remote_modules {
+ let source = match &module.maybe_source {
+ Some(source) => source,
+ None => continue,
+ };
+ let local_path = mappings
+ .proxied_path(&module.specifier)
+ .unwrap_or_else(|| mappings.local_path(&module.specifier));
+ if !matches!(module.kind, ModuleKind::Esm | ModuleKind::Asserted) {
+ log::warn!(
+ "Unsupported module kind {:?} for {}",
+ module.kind,
+ module.specifier
+ );
+ continue;
+ }
+ environment.create_dir_all(local_path.parent().unwrap())?;
+ environment.write_file(&local_path, source)?;
+ }
+
+ // write out the proxies
+ for (specifier, proxied_module) in mappings.proxied_modules() {
+ let proxy_path = mappings.local_path(specifier);
+ let module = graph.get(specifier).unwrap();
+ let text = build_proxy_module_source(module, proxied_module);
+
+ environment.write_file(&proxy_path, &text)?;
+ }
+
+ // create the import map
+ if !mappings.base_specifiers().is_empty() {
+ let import_map_text = build_import_map(graph, &all_modules, &mappings);
+ environment
+ .write_file(&output_dir.join("import_map.json"), &import_map_text)?;
+ }
+
+ Ok(remote_modules.len())
+}
+
+fn build_proxy_module_source(
+ module: &Module,
+ proxied_module: &ProxiedModule,
+) -> String {
+ let mut text = format!(
+ "// @deno-types=\"{}\"\n",
+ proxied_module.declaration_specifier
+ );
+ let relative_specifier = format!(
+ "./{}",
+ proxied_module
+ .output_path
+ .file_name()
+ .unwrap()
+ .to_string_lossy()
+ );
+
+ // for simplicity, always include the `export *` statement as it won't error
+ // even when the module does not contain a named export
+ text.push_str(&format!("export * from \"{}\";\n", relative_specifier));
+
+ // add a default export if one exists in the module
+ if let Some(parsed_source) = module.maybe_parsed_source.as_ref() {
+ if has_default_export(parsed_source) {
+ text.push_str(&format!(
+ "export {{ default }} from \"{}\";\n",
+ relative_specifier
+ ));
+ }
+ }
+
+ text
+}
+
+#[cfg(test)]
+mod test {
+ use crate::tools::vendor::test::VendorTestBuilder;
+ use deno_core::serde_json::json;
+ use pretty_assertions::assert_eq;
+
+ #[tokio::test]
+ async fn no_remote_modules() {
+ let mut builder = VendorTestBuilder::with_default_setup();
+ let output = builder
+ .with_loader(|loader| {
+ loader.add("/mod.ts", "");
+ })
+ .build()
+ .await
+ .unwrap();
+
+ assert_eq!(output.import_map, None,);
+ assert_eq!(output.files, vec![],);
+ }
+
+ #[tokio::test]
+ async fn local_specifiers_to_remote() {
+ let mut builder = VendorTestBuilder::with_default_setup();
+ let output = builder
+ .with_loader(|loader| {
+ loader
+ .add(
+ "/mod.ts",
+ concat!(
+ r#"import "https://localhost/mod.ts";"#,
+ r#"import "https://localhost/other.ts?test";"#,
+ r#"import "https://localhost/redirect.ts";"#,
+ ),
+ )
+ .add("https://localhost/mod.ts", "export class Mod {}")
+ .add("https://localhost/other.ts?test", "export class Other {}")
+ .add_redirect(
+ "https://localhost/redirect.ts",
+ "https://localhost/mod.ts",
+ );
+ })
+ .build()
+ .await
+ .unwrap();
+
+ assert_eq!(
+ output.import_map,
+ Some(json!({
+ "imports": {
+ "https://localhost/": "./localhost/",
+ "https://localhost/other.ts?test": "./localhost/other.ts",
+ "https://localhost/redirect.ts": "./localhost/mod.ts",
+ }
+ }))
+ );
+ assert_eq!(
+ output.files,
+ to_file_vec(&[
+ ("/vendor/localhost/mod.ts", "export class Mod {}"),
+ ("/vendor/localhost/other.ts", "export class Other {}"),
+ ]),
+ );
+ }
+
+ #[tokio::test]
+ async fn remote_specifiers() {
+ let mut builder = VendorTestBuilder::with_default_setup();
+ let output = builder
+ .with_loader(|loader| {
+ loader
+ .add(
+ "/mod.ts",
+ concat!(
+ r#"import "https://localhost/mod.ts";"#,
+ r#"import "https://other/mod.ts";"#,
+ ),
+ )
+ .add(
+ "https://localhost/mod.ts",
+ concat!(
+ "export * from './other.ts';",
+ "export * from './redirect.ts';",
+ "export * from '/absolute.ts';",
+ ),
+ )
+ .add("https://localhost/other.ts", "export class Other {}")
+ .add_redirect(
+ "https://localhost/redirect.ts",
+ "https://localhost/other.ts",
+ )
+ .add("https://localhost/absolute.ts", "export class Absolute {}")
+ .add("https://other/mod.ts", "export * from './sub/mod.ts';")
+ .add(
+ "https://other/sub/mod.ts",
+ concat!(
+ "export * from '../sub2/mod.ts';",
+ "export * from '../sub2/other?asdf';",
+ // reference a path on a different origin
+ "export * from 'https://localhost/other.ts';",
+ "export * from 'https://localhost/redirect.ts';",
+ ),
+ )
+ .add("https://other/sub2/mod.ts", "export class Mod {}")
+ .add_with_headers(
+ "https://other/sub2/other?asdf",
+ "export class Other {}",
+ &[("content-type", "application/javascript")],
+ );
+ })
+ .build()
+ .await
+ .unwrap();
+
+ assert_eq!(
+ output.import_map,
+ Some(json!({
+ "imports": {
+ "https://localhost/": "./localhost/",
+ "https://localhost/redirect.ts": "./localhost/other.ts",
+ "https://other/": "./other/"
+ },
+ "scopes": {
+ "./localhost/": {
+ "./localhost/redirect.ts": "./localhost/other.ts",
+ "/absolute.ts": "./localhost/absolute.ts",
+ },
+ "./other/": {
+ "./other/sub2/other?asdf": "./other/sub2/other.js"
+ }
+ }
+ }))
+ );
+ assert_eq!(
+ output.files,
+ to_file_vec(&[
+ ("/vendor/localhost/absolute.ts", "export class Absolute {}"),
+ (
+ "/vendor/localhost/mod.ts",
+ concat!(
+ "export * from './other.ts';",
+ "export * from './redirect.ts';",
+ "export * from '/absolute.ts';",
+ )
+ ),
+ ("/vendor/localhost/other.ts", "export class Other {}"),
+ ("/vendor/other/mod.ts", "export * from './sub/mod.ts';"),
+ (
+ "/vendor/other/sub/mod.ts",
+ concat!(
+ "export * from '../sub2/mod.ts';",
+ "export * from '../sub2/other?asdf';",
+ "export * from 'https://localhost/other.ts';",
+ "export * from 'https://localhost/redirect.ts';",
+ )
+ ),
+ ("/vendor/other/sub2/mod.ts", "export class Mod {}"),
+ ("/vendor/other/sub2/other.js", "export class Other {}"),
+ ]),
+ );
+ }
+
+ #[tokio::test]
+ async fn same_target_filename_specifiers() {
+ let mut builder = VendorTestBuilder::with_default_setup();
+ let output = builder
+ .with_loader(|loader| {
+ loader
+ .add(
+ "/mod.ts",
+ concat!(
+ r#"import "https://localhost/MOD.TS";"#,
+ r#"import "https://localhost/mod.TS";"#,
+ r#"import "https://localhost/mod.ts";"#,
+ r#"import "https://localhost/mod.ts?test";"#,
+ r#"import "https://localhost/CAPS.TS";"#,
+ ),
+ )
+ .add("https://localhost/MOD.TS", "export class Mod {}")
+ .add("https://localhost/mod.TS", "export class Mod2 {}")
+ .add("https://localhost/mod.ts", "export class Mod3 {}")
+ .add("https://localhost/mod.ts?test", "export class Mod4 {}")
+ .add("https://localhost/CAPS.TS", "export class Caps {}");
+ })
+ .build()
+ .await
+ .unwrap();
+
+ assert_eq!(
+ output.import_map,
+ Some(json!({
+ "imports": {
+ "https://localhost/": "./localhost/",
+ "https://localhost/mod.TS": "./localhost/mod_2.TS",
+ "https://localhost/mod.ts": "./localhost/mod_3.ts",
+ "https://localhost/mod.ts?test": "./localhost/mod_4.ts",
+ }
+ }))
+ );
+ assert_eq!(
+ output.files,
+ to_file_vec(&[
+ ("/vendor/localhost/CAPS.TS", "export class Caps {}"),
+ ("/vendor/localhost/MOD.TS", "export class Mod {}"),
+ ("/vendor/localhost/mod_2.TS", "export class Mod2 {}"),
+ ("/vendor/localhost/mod_3.ts", "export class Mod3 {}"),
+ ("/vendor/localhost/mod_4.ts", "export class Mod4 {}"),
+ ]),
+ );
+ }
+
+ #[tokio::test]
+ async fn multiple_entrypoints() {
+ let mut builder = VendorTestBuilder::with_default_setup();
+ let output = builder
+ .add_entry_point("/test.deps.ts")
+ .with_loader(|loader| {
+ loader
+ .add("/mod.ts", r#"import "https://localhost/mod.ts";"#)
+ .add(
+ "/test.deps.ts",
+ r#"export * from "https://localhost/test.ts";"#,
+ )
+ .add("https://localhost/mod.ts", "export class Mod {}")
+ .add("https://localhost/test.ts", "export class Test {}");
+ })
+ .build()
+ .await
+ .unwrap();
+
+ assert_eq!(
+ output.import_map,
+ Some(json!({
+ "imports": {
+ "https://localhost/": "./localhost/",
+ }
+ }))
+ );
+ assert_eq!(
+ output.files,
+ to_file_vec(&[
+ ("/vendor/localhost/mod.ts", "export class Mod {}"),
+ ("/vendor/localhost/test.ts", "export class Test {}"),
+ ]),
+ );
+ }
+
+ #[tokio::test]
+ async fn json_module() {
+ let mut builder = VendorTestBuilder::with_default_setup();
+ let output = builder
+ .with_loader(|loader| {
+ loader
+ .add(
+ "/mod.ts",
+ r#"import data from "https://localhost/data.json" assert { type: "json" };"#,
+ )
+ .add("https://localhost/data.json", "{}");
+ })
+ .build()
+ .await
+ .unwrap();
+
+ assert_eq!(
+ output.import_map,
+ Some(json!({
+ "imports": {
+ "https://localhost/": "./localhost/"
+ }
+ }))
+ );
+ assert_eq!(
+ output.files,
+ to_file_vec(&[("/vendor/localhost/data.json", "{}"),]),
+ );
+ }
+
+ #[tokio::test]
+ async fn data_urls() {
+ let mut builder = VendorTestBuilder::with_default_setup();
+
+ let mod_file_text = r#"import * as b from "data:application/typescript,export%20*%20from%20%22https://localhost/mod.ts%22;";"#;
+
+ let output = builder
+ .with_loader(|loader| {
+ loader
+ .add("/mod.ts", &mod_file_text)
+ .add("https://localhost/mod.ts", "export class Example {}");
+ })
+ .build()
+ .await
+ .unwrap();
+
+ assert_eq!(
+ output.import_map,
+ Some(json!({
+ "imports": {
+ "https://localhost/": "./localhost/"
+ }
+ }))
+ );
+ assert_eq!(
+ output.files,
+ to_file_vec(&[("/vendor/localhost/mod.ts", "export class Example {}"),]),
+ );
+ }
+
+ #[tokio::test]
+ async fn x_typescript_types_no_default() {
+ let mut builder = VendorTestBuilder::with_default_setup();
+ let output = builder
+ .with_loader(|loader| {
+ loader
+ .add("/mod.ts", r#"import "https://localhost/mod.js";"#)
+ .add_with_headers(
+ "https://localhost/mod.js",
+ "export class Mod {}",
+ &[("x-typescript-types", "https://localhost/mod.d.ts")],
+ )
+ .add("https://localhost/mod.d.ts", "export class Mod {}");
+ })
+ .build()
+ .await
+ .unwrap();
+
+ assert_eq!(
+ output.import_map,
+ Some(json!({
+ "imports": {
+ "https://localhost/": "./localhost/"
+ }
+ }))
+ );
+ assert_eq!(
+ output.files,
+ to_file_vec(&[
+ ("/vendor/localhost/mod.d.ts", "export class Mod {}"),
+ (
+ "/vendor/localhost/mod.js",
+ concat!(
+ "// @deno-types=\"https://localhost/mod.d.ts\"\n",
+ "export * from \"./mod.proxied.js\";\n"
+ )
+ ),
+ ("/vendor/localhost/mod.proxied.js", "export class Mod {}"),
+ ]),
+ );
+ }
+
+ #[tokio::test]
+ async fn x_typescript_types_default_export() {
+ let mut builder = VendorTestBuilder::with_default_setup();
+ let output = builder
+ .with_loader(|loader| {
+ loader
+ .add("/mod.ts", r#"import "https://localhost/mod.js";"#)
+ .add_with_headers(
+ "https://localhost/mod.js",
+ "export default class Mod {}",
+ &[("x-typescript-types", "https://localhost/mod.d.ts")],
+ )
+ .add("https://localhost/mod.d.ts", "export default class Mod {}");
+ })
+ .build()
+ .await
+ .unwrap();
+
+ assert_eq!(
+ output.import_map,
+ Some(json!({
+ "imports": {
+ "https://localhost/": "./localhost/"
+ }
+ }))
+ );
+ assert_eq!(
+ output.files,
+ to_file_vec(&[
+ ("/vendor/localhost/mod.d.ts", "export default class Mod {}"),
+ (
+ "/vendor/localhost/mod.js",
+ concat!(
+ "// @deno-types=\"https://localhost/mod.d.ts\"\n",
+ "export * from \"./mod.proxied.js\";\n",
+ "export { default } from \"./mod.proxied.js\";\n",
+ )
+ ),
+ (
+ "/vendor/localhost/mod.proxied.js",
+ "export default class Mod {}"
+ ),
+ ]),
+ );
+ }
+
+ #[tokio::test]
+ async fn subdir() {
+ let mut builder = VendorTestBuilder::with_default_setup();
+ let output = builder
+ .with_loader(|loader| {
+ loader
+ .add(
+ "/mod.ts",
+ r#"import "http://localhost:4545/sub/logger/mod.ts?testing";"#,
+ )
+ .add(
+ "http://localhost:4545/sub/logger/mod.ts?testing",
+ "export * from './logger.ts?test';",
+ )
+ .add(
+ "http://localhost:4545/sub/logger/logger.ts?test",
+ "export class Logger {}",
+ );
+ })
+ .build()
+ .await
+ .unwrap();
+
+ assert_eq!(
+ output.import_map,
+ Some(json!({
+ "imports": {
+ "http://localhost:4545/": "./localhost_4545/",
+ "http://localhost:4545/sub/logger/mod.ts?testing": "./localhost_4545/sub/logger/mod.ts",
+ },
+ "scopes": {
+ "./localhost_4545/": {
+ "./localhost_4545/sub/logger/logger.ts?test": "./localhost_4545/sub/logger/logger.ts"
+ }
+ }
+ }))
+ );
+ assert_eq!(
+ output.files,
+ to_file_vec(&[
+ (
+ "/vendor/localhost_4545/sub/logger/logger.ts",
+ "export class Logger {}",
+ ),
+ (
+ "/vendor/localhost_4545/sub/logger/mod.ts",
+ "export * from './logger.ts?test';"
+ ),
+ ]),
+ );
+ }
+
+ fn to_file_vec(items: &[(&str, &str)]) -> Vec<(String, String)> {
+ items
+ .iter()
+ .map(|(f, t)| (f.to_string(), t.to_string()))
+ .collect()
+ }
+}
diff --git a/cli/tools/vendor/import_map.rs b/cli/tools/vendor/import_map.rs
new file mode 100644
index 000000000..7e18d56aa
--- /dev/null
+++ b/cli/tools/vendor/import_map.rs
@@ -0,0 +1,285 @@
+// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
+
+use std::collections::BTreeMap;
+
+use deno_ast::LineAndColumnIndex;
+use deno_ast::ModuleSpecifier;
+use deno_ast::SourceTextInfo;
+use deno_core::serde_json;
+use deno_graph::Module;
+use deno_graph::ModuleGraph;
+use deno_graph::Position;
+use deno_graph::Range;
+use deno_graph::Resolved;
+use serde::Serialize;
+
+use super::mappings::Mappings;
+use super::specifiers::is_remote_specifier;
+use super::specifiers::is_remote_specifier_text;
+
+#[derive(Serialize)]
+struct SerializableImportMap {
+ imports: BTreeMap<String, String>,
+ #[serde(skip_serializing_if = "BTreeMap::is_empty")]
+ scopes: BTreeMap<String, BTreeMap<String, String>>,
+}
+
+struct ImportMapBuilder<'a> {
+ mappings: &'a Mappings,
+ imports: ImportsBuilder<'a>,
+ scopes: BTreeMap<String, ImportsBuilder<'a>>,
+}
+
+impl<'a> ImportMapBuilder<'a> {
+ pub fn new(mappings: &'a Mappings) -> Self {
+ ImportMapBuilder {
+ mappings,
+ imports: ImportsBuilder::new(mappings),
+ scopes: Default::default(),
+ }
+ }
+
+ pub fn scope(
+ &mut self,
+ base_specifier: &ModuleSpecifier,
+ ) -> &mut ImportsBuilder<'a> {
+ self
+ .scopes
+ .entry(
+ self
+ .mappings
+ .relative_specifier_text(self.mappings.output_dir(), base_specifier),
+ )
+ .or_insert_with(|| ImportsBuilder::new(self.mappings))
+ }
+
+ pub fn into_serializable(self) -> SerializableImportMap {
+ SerializableImportMap {
+ imports: self.imports.imports,
+ scopes: self
+ .scopes
+ .into_iter()
+ .map(|(key, value)| (key, value.imports))
+ .collect(),
+ }
+ }
+
+ pub fn into_file_text(self) -> String {
+ let mut text =
+ serde_json::to_string_pretty(&self.into_serializable()).unwrap();
+ text.push('\n');
+ text
+ }
+}
+
+struct ImportsBuilder<'a> {
+ mappings: &'a Mappings,
+ imports: BTreeMap<String, String>,
+}
+
+impl<'a> ImportsBuilder<'a> {
+ pub fn new(mappings: &'a Mappings) -> Self {
+ Self {
+ mappings,
+ imports: Default::default(),
+ }
+ }
+
+ pub fn add(&mut self, key: String, specifier: &ModuleSpecifier) {
+ self.imports.insert(
+ key,
+ self
+ .mappings
+ .relative_specifier_text(self.mappings.output_dir(), specifier),
+ );
+ }
+}
+
+pub fn build_import_map(
+ graph: &ModuleGraph,
+ modules: &[&Module],
+ mappings: &Mappings,
+) -> String {
+ let mut import_map = ImportMapBuilder::new(mappings);
+ visit_modules(graph, modules, mappings, &mut import_map);
+
+ for base_specifier in mappings.base_specifiers() {
+ import_map
+ .imports
+ .add(base_specifier.to_string(), base_specifier);
+ }
+
+ import_map.into_file_text()
+}
+
+fn visit_modules(
+ graph: &ModuleGraph,
+ modules: &[&Module],
+ mappings: &Mappings,
+ import_map: &mut ImportMapBuilder,
+) {
+ for module in modules {
+ let text_info = match &module.maybe_parsed_source {
+ Some(source) => source.source(),
+ None => continue,
+ };
+ let source_text = match &module.maybe_source {
+ Some(source) => source,
+ None => continue,
+ };
+
+ for dep in module.dependencies.values() {
+ visit_maybe_resolved(
+ &dep.maybe_code,
+ graph,
+ import_map,
+ &module.specifier,
+ mappings,
+ text_info,
+ source_text,
+ );
+ visit_maybe_resolved(
+ &dep.maybe_type,
+ graph,
+ import_map,
+ &module.specifier,
+ mappings,
+ text_info,
+ source_text,
+ );
+ }
+
+ if let Some((_, maybe_resolved)) = &module.maybe_types_dependency {
+ visit_maybe_resolved(
+ maybe_resolved,
+ graph,
+ import_map,
+ &module.specifier,
+ mappings,
+ text_info,
+ source_text,
+ );
+ }
+ }
+}
+
+fn visit_maybe_resolved(
+ maybe_resolved: &Resolved,
+ graph: &ModuleGraph,
+ import_map: &mut ImportMapBuilder,
+ referrer: &ModuleSpecifier,
+ mappings: &Mappings,
+ text_info: &SourceTextInfo,
+ source_text: &str,
+) {
+ if let Resolved::Ok {
+ specifier, range, ..
+ } = maybe_resolved
+ {
+ let text = text_from_range(text_info, source_text, range);
+ // if the text is empty then it's probably an x-TypeScript-types
+ if !text.is_empty() {
+ handle_dep_specifier(
+ text, specifier, graph, import_map, referrer, mappings,
+ );
+ }
+ }
+}
+
+fn handle_dep_specifier(
+ text: &str,
+ unresolved_specifier: &ModuleSpecifier,
+ graph: &ModuleGraph,
+ import_map: &mut ImportMapBuilder,
+ referrer: &ModuleSpecifier,
+ mappings: &Mappings,
+) {
+ let specifier = graph.resolve(unresolved_specifier);
+ // do not handle specifiers pointing at local modules
+ if !is_remote_specifier(&specifier) {
+ return;
+ }
+
+ let base_specifier = mappings.base_specifier(&specifier);
+ if is_remote_specifier_text(text) {
+ if !text.starts_with(base_specifier.as_str()) {
+ panic!("Expected {} to start with {}", text, base_specifier);
+ }
+
+ let sub_path = &text[base_specifier.as_str().len()..];
+ let expected_relative_specifier_text =
+ mappings.relative_path(base_specifier, &specifier);
+ if expected_relative_specifier_text == sub_path {
+ return;
+ }
+
+ if referrer.origin() == specifier.origin() {
+ let imports = import_map.scope(base_specifier);
+ imports.add(sub_path.to_string(), &specifier);
+ } else {
+ import_map.imports.add(text.to_string(), &specifier);
+ }
+ } else {
+ let expected_relative_specifier_text =
+ mappings.relative_specifier_text(referrer, &specifier);
+ if expected_relative_specifier_text == text {
+ return;
+ }
+
+ let key = if text.starts_with("./") || text.starts_with("../") {
+ // resolve relative specifier key
+ let mut local_base_specifier = mappings.local_uri(base_specifier);
+ local_base_specifier.set_query(unresolved_specifier.query());
+ local_base_specifier = local_base_specifier
+ .join(&unresolved_specifier.path()[1..])
+ .unwrap_or_else(|_| {
+ panic!(
+ "Error joining {} to {}",
+ unresolved_specifier.path(),
+ local_base_specifier
+ )
+ });
+ local_base_specifier.set_query(unresolved_specifier.query());
+ mappings
+ .relative_specifier_text(mappings.output_dir(), &local_base_specifier)
+ } else {
+ // absolute (`/`) or bare specifier should be left as-is
+ text.to_string()
+ };
+ let imports = import_map.scope(base_specifier);
+ imports.add(key, &specifier);
+ }
+}
+
+fn text_from_range<'a>(
+ text_info: &SourceTextInfo,
+ text: &'a str,
+ range: &Range,
+) -> &'a str {
+ let result = &text[byte_range(text_info, range)];
+ if result.starts_with('"') || result.starts_with('\'') {
+ // remove the quotes
+ &result[1..result.len() - 1]
+ } else {
+ result
+ }
+}
+
+fn byte_range(
+ text_info: &SourceTextInfo,
+ range: &Range,
+) -> std::ops::Range<usize> {
+ let start = byte_index(text_info, &range.start);
+ let end = byte_index(text_info, &range.end);
+ start..end
+}
+
+fn byte_index(text_info: &SourceTextInfo, pos: &Position) -> usize {
+ // todo(https://github.com/denoland/deno_graph/issues/79): use byte indexes all the way down
+ text_info
+ .byte_index(LineAndColumnIndex {
+ line_index: pos.line,
+ column_index: pos.character,
+ })
+ .0 as usize
+}
diff --git a/cli/tools/vendor/mappings.rs b/cli/tools/vendor/mappings.rs
new file mode 100644
index 000000000..2e85445dc
--- /dev/null
+++ b/cli/tools/vendor/mappings.rs
@@ -0,0 +1,286 @@
+// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
+
+use std::collections::HashMap;
+use std::collections::HashSet;
+use std::path::Path;
+use std::path::PathBuf;
+
+use deno_ast::MediaType;
+use deno_ast::ModuleSpecifier;
+use deno_core::error::AnyError;
+use deno_graph::Module;
+use deno_graph::ModuleGraph;
+use deno_graph::Position;
+use deno_graph::Resolved;
+
+use crate::fs_util::path_with_stem_suffix;
+
+use super::specifiers::dir_name_for_root;
+use super::specifiers::get_unique_path;
+use super::specifiers::make_url_relative;
+use super::specifiers::partition_by_root_specifiers;
+use super::specifiers::sanitize_filepath;
+
+pub struct ProxiedModule {
+ pub output_path: PathBuf,
+ pub declaration_specifier: ModuleSpecifier,
+}
+
+/// Constructs and holds the remote specifier to local path mappings.
+pub struct Mappings {
+ output_dir: ModuleSpecifier,
+ mappings: HashMap<ModuleSpecifier, PathBuf>,
+ base_specifiers: Vec<ModuleSpecifier>,
+ proxies: HashMap<ModuleSpecifier, ProxiedModule>,
+}
+
+impl Mappings {
+ pub fn from_remote_modules(
+ graph: &ModuleGraph,
+ remote_modules: &[&Module],
+ output_dir: &Path,
+ ) -> Result<Self, AnyError> {
+ let partitioned_specifiers =
+ partition_by_root_specifiers(remote_modules.iter().map(|m| &m.specifier));
+ let mut mapped_paths = HashSet::new();
+ let mut mappings = HashMap::new();
+ let mut proxies = HashMap::new();
+ let mut base_specifiers = Vec::new();
+
+ for (root, specifiers) in partitioned_specifiers.into_iter() {
+ let base_dir = get_unique_path(
+ output_dir.join(dir_name_for_root(&root)),
+ &mut mapped_paths,
+ );
+ for specifier in specifiers {
+ let media_type = graph.get(&specifier).unwrap().media_type;
+ let sub_path = sanitize_filepath(&make_url_relative(&root, &{
+ let mut specifier = specifier.clone();
+ specifier.set_query(None);
+ specifier
+ })?);
+ let new_path = path_with_extension(
+ &base_dir.join(if cfg!(windows) {
+ sub_path.replace('/', "\\")
+ } else {
+ sub_path
+ }),
+ &media_type.as_ts_extension()[1..],
+ );
+ mappings
+ .insert(specifier, get_unique_path(new_path, &mut mapped_paths));
+ }
+ base_specifiers.push(root.clone());
+ mappings.insert(root, base_dir);
+ }
+
+ // resolve all the "proxy" paths to use for when an x-typescript-types header is specified
+ for module in remote_modules {
+ if let Some((
+ _,
+ Resolved::Ok {
+ specifier, range, ..
+ },
+ )) = &module.maybe_types_dependency
+ {
+ // hack to tell if it's an x-typescript-types header
+ let is_ts_types_header =
+ range.start == Position::zeroed() && range.end == Position::zeroed();
+ if is_ts_types_header {
+ let module_path = mappings.get(&module.specifier).unwrap();
+ let proxied_path = get_unique_path(
+ path_with_stem_suffix(module_path, ".proxied"),
+ &mut mapped_paths,
+ );
+ proxies.insert(
+ module.specifier.clone(),
+ ProxiedModule {
+ output_path: proxied_path,
+ declaration_specifier: specifier.clone(),
+ },
+ );
+ }
+ }
+ }
+
+ Ok(Self {
+ output_dir: ModuleSpecifier::from_directory_path(output_dir).unwrap(),
+ mappings,
+ base_specifiers,
+ proxies,
+ })
+ }
+
+ pub fn output_dir(&self) -> &ModuleSpecifier {
+ &self.output_dir
+ }
+
+ pub fn local_uri(&self, specifier: &ModuleSpecifier) -> ModuleSpecifier {
+ if specifier.scheme() == "file" {
+ specifier.clone()
+ } else {
+ let local_path = self.local_path(specifier);
+ if specifier.path().ends_with('/') {
+ ModuleSpecifier::from_directory_path(&local_path)
+ } else {
+ ModuleSpecifier::from_file_path(&local_path)
+ }
+ .unwrap_or_else(|_| {
+ panic!("Could not convert {} to uri.", local_path.display())
+ })
+ }
+ }
+
+ pub fn local_path(&self, specifier: &ModuleSpecifier) -> PathBuf {
+ if specifier.scheme() == "file" {
+ specifier.to_file_path().unwrap()
+ } else {
+ self
+ .mappings
+ .get(specifier)
+ .as_ref()
+ .unwrap_or_else(|| {
+ panic!("Could not find local path for {}", specifier)
+ })
+ .to_path_buf()
+ }
+ }
+
+ pub fn relative_path(
+ &self,
+ from: &ModuleSpecifier,
+ to: &ModuleSpecifier,
+ ) -> String {
+ let mut from = self.local_uri(from);
+ let to = self.local_uri(to);
+
+ // workaround using parent directory until https://github.com/servo/rust-url/pull/754 is merged
+ if !from.path().ends_with('/') {
+ let local_path = self.local_path(&from);
+ from = ModuleSpecifier::from_directory_path(local_path.parent().unwrap())
+ .unwrap();
+ }
+
+ // workaround for url crate not adding a trailing slash for a directory
+ // it seems to be fixed once a version greater than 2.2.2 is released
+ let is_dir = to.path().ends_with('/');
+ let mut text = from.make_relative(&to).unwrap();
+ if is_dir && !text.ends_with('/') && to.query().is_none() {
+ text.push('/');
+ }
+ text
+ }
+
+ pub fn relative_specifier_text(
+ &self,
+ from: &ModuleSpecifier,
+ to: &ModuleSpecifier,
+ ) -> String {
+ let relative_path = self.relative_path(from, to);
+
+ if relative_path.starts_with("../") || relative_path.starts_with("./") {
+ relative_path
+ } else {
+ format!("./{}", relative_path)
+ }
+ }
+
+ pub fn base_specifiers(&self) -> &Vec<ModuleSpecifier> {
+ &self.base_specifiers
+ }
+
+ pub fn base_specifier(
+ &self,
+ child_specifier: &ModuleSpecifier,
+ ) -> &ModuleSpecifier {
+ self
+ .base_specifiers
+ .iter()
+ .find(|s| child_specifier.as_str().starts_with(s.as_str()))
+ .unwrap_or_else(|| {
+ panic!("Could not find base specifier for {}", child_specifier)
+ })
+ }
+
+ pub fn proxied_path(&self, specifier: &ModuleSpecifier) -> Option<PathBuf> {
+ self.proxies.get(specifier).map(|s| s.output_path.clone())
+ }
+
+ pub fn proxied_modules(
+ &self,
+ ) -> std::collections::hash_map::Iter<'_, ModuleSpecifier, ProxiedModule> {
+ self.proxies.iter()
+ }
+}
+
+fn path_with_extension(path: &Path, new_ext: &str) -> PathBuf {
+ if let Some(file_stem) = path.file_stem().map(|f| f.to_string_lossy()) {
+ if let Some(old_ext) = path.extension().map(|f| f.to_string_lossy()) {
+ if file_stem.to_lowercase().ends_with(".d") {
+ if new_ext.to_lowercase() == format!("d.{}", old_ext.to_lowercase()) {
+ // maintain casing
+ return path.to_path_buf();
+ }
+ return path.with_file_name(format!(
+ "{}.{}",
+ &file_stem[..file_stem.len() - ".d".len()],
+ new_ext
+ ));
+ }
+ if new_ext.to_lowercase() == old_ext.to_lowercase() {
+ // maintain casing
+ return path.to_path_buf();
+ }
+ let media_type: MediaType = path.into();
+ if media_type == MediaType::Unknown {
+ return path.with_file_name(format!(
+ "{}.{}",
+ path.file_name().unwrap().to_string_lossy(),
+ new_ext
+ ));
+ }
+ }
+ }
+ path.with_extension(new_ext)
+}
+
+#[cfg(test)]
+mod test {
+ use pretty_assertions::assert_eq;
+
+ use super::*;
+
+ #[test]
+ fn test_path_with_extension() {
+ assert_eq!(
+ path_with_extension(&PathBuf::from("/test.D.TS"), "ts"),
+ PathBuf::from("/test.ts")
+ );
+ assert_eq!(
+ path_with_extension(&PathBuf::from("/test.D.MTS"), "js"),
+ PathBuf::from("/test.js")
+ );
+ assert_eq!(
+ path_with_extension(&PathBuf::from("/test.D.TS"), "d.ts"),
+ // maintains casing
+ PathBuf::from("/test.D.TS"),
+ );
+ assert_eq!(
+ path_with_extension(&PathBuf::from("/test.TS"), "ts"),
+ // maintains casing
+ PathBuf::from("/test.TS"),
+ );
+ assert_eq!(
+ path_with_extension(&PathBuf::from("/test.ts"), "js"),
+ PathBuf::from("/test.js")
+ );
+ assert_eq!(
+ path_with_extension(&PathBuf::from("/test.js"), "js"),
+ PathBuf::from("/test.js")
+ );
+ assert_eq!(
+ path_with_extension(&PathBuf::from("/chai@1.2.3"), "js"),
+ PathBuf::from("/chai@1.2.3.js")
+ );
+ }
+}
diff --git a/cli/tools/vendor/mod.rs b/cli/tools/vendor/mod.rs
new file mode 100644
index 000000000..eb9c91071
--- /dev/null
+++ b/cli/tools/vendor/mod.rs
@@ -0,0 +1,172 @@
+// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
+
+use std::path::Path;
+use std::path::PathBuf;
+
+use deno_core::anyhow::bail;
+use deno_core::anyhow::Context;
+use deno_core::error::AnyError;
+use deno_core::resolve_url_or_path;
+use deno_runtime::permissions::Permissions;
+
+use crate::flags::VendorFlags;
+use crate::fs_util;
+use crate::lockfile;
+use crate::proc_state::ProcState;
+use crate::resolver::ImportMapResolver;
+use crate::resolver::JsxResolver;
+use crate::tools::vendor::specifiers::is_remote_specifier_text;
+
+mod analyze;
+mod build;
+mod import_map;
+mod mappings;
+mod specifiers;
+#[cfg(test)]
+mod test;
+
+pub async fn vendor(ps: ProcState, flags: VendorFlags) -> Result<(), AnyError> {
+ let raw_output_dir = match &flags.output_path {
+ Some(output_path) => output_path.to_owned(),
+ None => PathBuf::from("vendor/"),
+ };
+ let output_dir = fs_util::resolve_from_cwd(&raw_output_dir)?;
+ validate_output_dir(&output_dir, &flags, &ps)?;
+ let graph = create_graph(&ps, &flags).await?;
+ let vendored_count =
+ build::build(&graph, &output_dir, &build::RealVendorEnvironment)?;
+
+ eprintln!(
+ r#"Vendored {} {} into {} directory.
+
+To use vendored modules, specify the `--import-map` flag when invoking deno subcommands:
+ deno run -A --import-map {} {}"#,
+ vendored_count,
+ if vendored_count == 1 {
+ "module"
+ } else {
+ "modules"
+ },
+ raw_output_dir.display(),
+ raw_output_dir.join("import_map.json").display(),
+ flags
+ .specifiers
+ .iter()
+ .map(|s| s.as_str())
+ .find(|s| !is_remote_specifier_text(s))
+ .unwrap_or("main.ts"),
+ );
+
+ Ok(())
+}
+
+fn validate_output_dir(
+ output_dir: &Path,
+ flags: &VendorFlags,
+ ps: &ProcState,
+) -> Result<(), AnyError> {
+ if !flags.force && !is_dir_empty(output_dir)? {
+ bail!(concat!(
+ "Output directory was not empty. Please specify an empty directory or use ",
+ "--force to ignore this error and potentially overwrite its contents.",
+ ));
+ }
+
+ // check the import map
+ if let Some(import_map_path) = ps
+ .maybe_import_map
+ .as_ref()
+ .and_then(|m| m.base_url().to_file_path().ok())
+ .and_then(|p| fs_util::canonicalize_path(&p).ok())
+ {
+ // make the output directory in order to canonicalize it for the check below
+ std::fs::create_dir_all(&output_dir)?;
+ let output_dir =
+ fs_util::canonicalize_path(output_dir).with_context(|| {
+ format!("Failed to canonicalize: {}", output_dir.display())
+ })?;
+
+ if import_map_path.starts_with(&output_dir) {
+ // We don't allow using the output directory to help generate the new state
+ // of itself because supporting this scenario adds a lot of complexity.
+ bail!(
+ "Using an import map found in the output directory is not supported."
+ );
+ }
+ }
+
+ Ok(())
+}
+
+fn is_dir_empty(dir_path: &Path) -> Result<bool, AnyError> {
+ match std::fs::read_dir(&dir_path) {
+ Ok(mut dir) => Ok(dir.next().is_none()),
+ Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(true),
+ Err(err) => {
+ bail!("Error reading directory {}: {}", dir_path.display(), err)
+ }
+ }
+}
+
+async fn create_graph(
+ ps: &ProcState,
+ flags: &VendorFlags,
+) -> Result<deno_graph::ModuleGraph, AnyError> {
+ let entry_points = flags
+ .specifiers
+ .iter()
+ .map(|p| {
+ let url = resolve_url_or_path(p)?;
+ Ok((url, deno_graph::ModuleKind::Esm))
+ })
+ .collect::<Result<Vec<_>, AnyError>>()?;
+
+ // todo(dsherret): there is a lot of copy and paste here from
+ // other parts of the codebase. We should consolidate this.
+ let mut cache = crate::cache::FetchCacher::new(
+ ps.dir.gen_cache.clone(),
+ ps.file_fetcher.clone(),
+ Permissions::allow_all(),
+ Permissions::allow_all(),
+ );
+ let maybe_locker = lockfile::as_maybe_locker(ps.lockfile.clone());
+ let maybe_imports = if let Some(config_file) = &ps.maybe_config_file {
+ config_file.to_maybe_imports()?
+ } else {
+ None
+ };
+ let maybe_import_map_resolver =
+ ps.maybe_import_map.clone().map(ImportMapResolver::new);
+ let maybe_jsx_resolver = ps
+ .maybe_config_file
+ .as_ref()
+ .map(|cf| {
+ cf.to_maybe_jsx_import_source_module()
+ .map(|im| JsxResolver::new(im, maybe_import_map_resolver.clone()))
+ })
+ .flatten();
+ let maybe_resolver = if maybe_jsx_resolver.is_some() {
+ maybe_jsx_resolver.as_ref().map(|jr| jr.as_resolver())
+ } else {
+ maybe_import_map_resolver
+ .as_ref()
+ .map(|im| im.as_resolver())
+ };
+
+ let graph = deno_graph::create_graph(
+ entry_points,
+ false,
+ maybe_imports,
+ &mut cache,
+ maybe_resolver,
+ maybe_locker,
+ None,
+ None,
+ )
+ .await;
+
+ graph.lock()?;
+ graph.valid()?;
+
+ Ok(graph)
+}
diff --git a/cli/tools/vendor/specifiers.rs b/cli/tools/vendor/specifiers.rs
new file mode 100644
index 000000000..b869e989c
--- /dev/null
+++ b/cli/tools/vendor/specifiers.rs
@@ -0,0 +1,251 @@
+// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
+
+use std::collections::BTreeMap;
+use std::collections::HashSet;
+use std::path::PathBuf;
+
+use deno_ast::ModuleSpecifier;
+use deno_core::anyhow::anyhow;
+use deno_core::error::AnyError;
+
+use crate::fs_util::path_with_stem_suffix;
+
+/// Partitions the provided specifiers by the non-path and non-query parts of a specifier.
+pub fn partition_by_root_specifiers<'a>(
+ specifiers: impl Iterator<Item = &'a ModuleSpecifier>,
+) -> BTreeMap<ModuleSpecifier, Vec<ModuleSpecifier>> {
+ let mut root_specifiers: BTreeMap<ModuleSpecifier, Vec<ModuleSpecifier>> =
+ Default::default();
+ for remote_specifier in specifiers {
+ let mut root_specifier = remote_specifier.clone();
+ root_specifier.set_query(None);
+ root_specifier.set_path("/");
+
+ let specifiers = root_specifiers.entry(root_specifier).or_default();
+ specifiers.push(remote_specifier.clone());
+ }
+ root_specifiers
+}
+
+/// Gets the directory name to use for the provided root.
+pub fn dir_name_for_root(root: &ModuleSpecifier) -> PathBuf {
+ let mut result = String::new();
+ if let Some(domain) = root.domain() {
+ result.push_str(&sanitize_segment(domain));
+ }
+ if let Some(port) = root.port() {
+ if !result.is_empty() {
+ result.push('_');
+ }
+ result.push_str(&port.to_string());
+ }
+ let mut result = PathBuf::from(result);
+ if let Some(segments) = root.path_segments() {
+ for segment in segments.filter(|s| !s.is_empty()) {
+ result = result.join(sanitize_segment(segment));
+ }
+ }
+
+ result
+}
+
+/// Gets a unique file path given the provided file path
+/// and the set of existing file paths. Inserts to the
+/// set when finding a unique path.
+pub fn get_unique_path(
+ mut path: PathBuf,
+ unique_set: &mut HashSet<String>,
+) -> PathBuf {
+ let original_path = path.clone();
+ let mut count = 2;
+ // case insensitive comparison so the output works on case insensitive file systems
+ while !unique_set.insert(path.to_string_lossy().to_lowercase()) {
+ path = path_with_stem_suffix(&original_path, &format!("_{}", count));
+ count += 1;
+ }
+ path
+}
+
+pub fn make_url_relative(
+ root: &ModuleSpecifier,
+ url: &ModuleSpecifier,
+) -> Result<String, AnyError> {
+ root.make_relative(url).ok_or_else(|| {
+ anyhow!(
+ "Error making url ({}) relative to root: {}",
+ url.to_string(),
+ root.to_string()
+ )
+ })
+}
+
+pub fn is_remote_specifier(specifier: &ModuleSpecifier) -> bool {
+ specifier.scheme().to_lowercase().starts_with("http")
+}
+
+pub fn is_remote_specifier_text(text: &str) -> bool {
+ text.trim_start().to_lowercase().starts_with("http")
+}
+
+pub fn sanitize_filepath(text: &str) -> String {
+ text
+ .chars()
+ .map(|c| if is_banned_path_char(c) { '_' } else { c })
+ .collect()
+}
+
+fn is_banned_path_char(c: char) -> bool {
+ matches!(c, '<' | '>' | ':' | '"' | '|' | '?' | '*')
+}
+
+fn sanitize_segment(text: &str) -> String {
+ text
+ .chars()
+ .map(|c| if is_banned_segment_char(c) { '_' } else { c })
+ .collect()
+}
+
+fn is_banned_segment_char(c: char) -> bool {
+ matches!(c, '/' | '\\') || is_banned_path_char(c)
+}
+
+#[cfg(test)]
+mod test {
+ use super::*;
+ use pretty_assertions::assert_eq;
+
+ #[test]
+ fn partition_by_root_specifiers_same_sub_folder() {
+ run_partition_by_root_specifiers_test(
+ vec![
+ "https://deno.land/x/mod/A.ts",
+ "https://deno.land/x/mod/other/A.ts",
+ ],
+ vec![(
+ "https://deno.land/",
+ vec![
+ "https://deno.land/x/mod/A.ts",
+ "https://deno.land/x/mod/other/A.ts",
+ ],
+ )],
+ );
+ }
+
+ #[test]
+ fn partition_by_root_specifiers_different_sub_folder() {
+ run_partition_by_root_specifiers_test(
+ vec![
+ "https://deno.land/x/mod/A.ts",
+ "https://deno.land/x/other/A.ts",
+ ],
+ vec![(
+ "https://deno.land/",
+ vec![
+ "https://deno.land/x/mod/A.ts",
+ "https://deno.land/x/other/A.ts",
+ ],
+ )],
+ );
+ }
+
+ #[test]
+ fn partition_by_root_specifiers_different_hosts() {
+ run_partition_by_root_specifiers_test(
+ vec![
+ "https://deno.land/mod/A.ts",
+ "http://deno.land/B.ts",
+ "https://deno.land:8080/C.ts",
+ "https://localhost/mod/A.ts",
+ "https://other/A.ts",
+ ],
+ vec![
+ ("http://deno.land/", vec!["http://deno.land/B.ts"]),
+ ("https://deno.land/", vec!["https://deno.land/mod/A.ts"]),
+ (
+ "https://deno.land:8080/",
+ vec!["https://deno.land:8080/C.ts"],
+ ),
+ ("https://localhost/", vec!["https://localhost/mod/A.ts"]),
+ ("https://other/", vec!["https://other/A.ts"]),
+ ],
+ );
+ }
+
+ fn run_partition_by_root_specifiers_test(
+ input: Vec<&str>,
+ expected: Vec<(&str, Vec<&str>)>,
+ ) {
+ let input = input
+ .iter()
+ .map(|s| ModuleSpecifier::parse(s).unwrap())
+ .collect::<Vec<_>>();
+ let output = partition_by_root_specifiers(input.iter());
+ // the assertion is much easier to compare when everything is strings
+ let output = output
+ .into_iter()
+ .map(|(s, vec)| {
+ (
+ s.to_string(),
+ vec.into_iter().map(|s| s.to_string()).collect::<Vec<_>>(),
+ )
+ })
+ .collect::<Vec<_>>();
+ let expected = expected
+ .into_iter()
+ .map(|(s, vec)| {
+ (
+ s.to_string(),
+ vec.into_iter().map(|s| s.to_string()).collect::<Vec<_>>(),
+ )
+ })
+ .collect::<Vec<_>>();
+ assert_eq!(output, expected);
+ }
+
+ #[test]
+ fn should_get_dir_name_root() {
+ run_test("http://deno.land/x/test", "deno.land/x/test");
+ run_test("http://localhost", "localhost");
+ run_test("http://localhost/test%20:test", "localhost/test%20_test");
+
+ fn run_test(specifier: &str, expected: &str) {
+ assert_eq!(
+ dir_name_for_root(&ModuleSpecifier::parse(specifier).unwrap()),
+ PathBuf::from(expected)
+ );
+ }
+ }
+
+ #[test]
+ fn test_unique_path() {
+ let mut paths = HashSet::new();
+ assert_eq!(
+ get_unique_path(PathBuf::from("/test"), &mut paths),
+ PathBuf::from("/test")
+ );
+ assert_eq!(
+ get_unique_path(PathBuf::from("/test"), &mut paths),
+ PathBuf::from("/test_2")
+ );
+ assert_eq!(
+ get_unique_path(PathBuf::from("/test"), &mut paths),
+ PathBuf::from("/test_3")
+ );
+ assert_eq!(
+ get_unique_path(PathBuf::from("/TEST"), &mut paths),
+ PathBuf::from("/TEST_4")
+ );
+ assert_eq!(
+ get_unique_path(PathBuf::from("/test.txt"), &mut paths),
+ PathBuf::from("/test.txt")
+ );
+ assert_eq!(
+ get_unique_path(PathBuf::from("/test.txt"), &mut paths),
+ PathBuf::from("/test_2.txt")
+ );
+ assert_eq!(
+ get_unique_path(PathBuf::from("/TEST.TXT"), &mut paths),
+ PathBuf::from("/TEST_3.TXT")
+ );
+ }
+}
diff --git a/cli/tools/vendor/test.rs b/cli/tools/vendor/test.rs
new file mode 100644
index 000000000..b37e2b3b0
--- /dev/null
+++ b/cli/tools/vendor/test.rs
@@ -0,0 +1,240 @@
+// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
+
+use std::cell::RefCell;
+use std::collections::HashMap;
+use std::collections::HashSet;
+use std::path::Path;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use deno_ast::ModuleSpecifier;
+use deno_core::anyhow::anyhow;
+use deno_core::anyhow::bail;
+use deno_core::error::AnyError;
+use deno_core::futures;
+use deno_core::serde_json;
+use deno_graph::source::LoadFuture;
+use deno_graph::source::LoadResponse;
+use deno_graph::source::Loader;
+use deno_graph::ModuleGraph;
+
+use super::build::VendorEnvironment;
+
+// Utilities that help `deno vendor` get tested in memory.
+
+type RemoteFileText = String;
+type RemoteFileHeaders = Option<HashMap<String, String>>;
+type RemoteFileResult = Result<(RemoteFileText, RemoteFileHeaders), String>;
+
+#[derive(Clone, Default)]
+pub struct TestLoader {
+ files: HashMap<ModuleSpecifier, RemoteFileResult>,
+ redirects: HashMap<ModuleSpecifier, ModuleSpecifier>,
+}
+
+impl TestLoader {
+ pub fn add(
+ &mut self,
+ path_or_specifier: impl AsRef<str>,
+ text: impl AsRef<str>,
+ ) -> &mut Self {
+ if path_or_specifier
+ .as_ref()
+ .to_lowercase()
+ .starts_with("http")
+ {
+ self.files.insert(
+ ModuleSpecifier::parse(path_or_specifier.as_ref()).unwrap(),
+ Ok((text.as_ref().to_string(), None)),
+ );
+ } else {
+ let path = make_path(path_or_specifier.as_ref());
+ let specifier = ModuleSpecifier::from_file_path(path).unwrap();
+ self
+ .files
+ .insert(specifier, Ok((text.as_ref().to_string(), None)));
+ }
+ self
+ }
+
+ pub fn add_with_headers(
+ &mut self,
+ specifier: impl AsRef<str>,
+ text: impl AsRef<str>,
+ headers: &[(&str, &str)],
+ ) -> &mut Self {
+ let headers = headers
+ .iter()
+ .map(|(key, value)| (key.to_string(), value.to_string()))
+ .collect();
+ self.files.insert(
+ ModuleSpecifier::parse(specifier.as_ref()).unwrap(),
+ Ok((text.as_ref().to_string(), Some(headers))),
+ );
+ self
+ }
+
+ pub fn add_redirect(
+ &mut self,
+ from: impl AsRef<str>,
+ to: impl AsRef<str>,
+ ) -> &mut Self {
+ self.redirects.insert(
+ ModuleSpecifier::parse(from.as_ref()).unwrap(),
+ ModuleSpecifier::parse(to.as_ref()).unwrap(),
+ );
+ self
+ }
+}
+
+impl Loader for TestLoader {
+ fn load(
+ &mut self,
+ specifier: &ModuleSpecifier,
+ _is_dynamic: bool,
+ ) -> LoadFuture {
+ let specifier = self.redirects.get(specifier).unwrap_or(specifier);
+ let result = self.files.get(specifier).map(|result| match result {
+ Ok(result) => Ok(LoadResponse::Module {
+ specifier: specifier.clone(),
+ content: Arc::new(result.0.clone()),
+ maybe_headers: result.1.clone(),
+ }),
+ Err(err) => Err(err),
+ });
+ let result = match result {
+ Some(Ok(result)) => Ok(Some(result)),
+ Some(Err(err)) => Err(anyhow!("{}", err)),
+ None if specifier.scheme() == "data" => {
+ deno_graph::source::load_data_url(specifier)
+ }
+ None => Ok(None),
+ };
+ Box::pin(futures::future::ready(result))
+ }
+}
+
+#[derive(Default)]
+struct TestVendorEnvironment {
+ directories: RefCell<HashSet<PathBuf>>,
+ files: RefCell<HashMap<PathBuf, String>>,
+}
+
+impl VendorEnvironment for TestVendorEnvironment {
+ fn create_dir_all(&self, dir_path: &Path) -> Result<(), AnyError> {
+ let mut directories = self.directories.borrow_mut();
+ for path in dir_path.ancestors() {
+ if !directories.insert(path.to_path_buf()) {
+ break;
+ }
+ }
+ Ok(())
+ }
+
+ fn write_file(&self, file_path: &Path, text: &str) -> Result<(), AnyError> {
+ let parent = file_path.parent().unwrap();
+ if !self.directories.borrow().contains(parent) {
+ bail!("Directory not found: {}", parent.display());
+ }
+ self
+ .files
+ .borrow_mut()
+ .insert(file_path.to_path_buf(), text.to_string());
+ Ok(())
+ }
+}
+
+pub struct VendorOutput {
+ pub files: Vec<(String, String)>,
+ pub import_map: Option<serde_json::Value>,
+}
+
+#[derive(Default)]
+pub struct VendorTestBuilder {
+ entry_points: Vec<ModuleSpecifier>,
+ loader: TestLoader,
+}
+
+impl VendorTestBuilder {
+ pub fn with_default_setup() -> Self {
+ let mut builder = VendorTestBuilder::default();
+ builder.add_entry_point("/mod.ts");
+ builder
+ }
+
+ pub fn add_entry_point(&mut self, entry_point: impl AsRef<str>) -> &mut Self {
+ let entry_point = make_path(entry_point.as_ref());
+ self
+ .entry_points
+ .push(ModuleSpecifier::from_file_path(entry_point).unwrap());
+ self
+ }
+
+ pub async fn build(&mut self) -> Result<VendorOutput, AnyError> {
+ let graph = self.build_graph().await;
+ let output_dir = make_path("/vendor");
+ let environment = TestVendorEnvironment::default();
+ super::build::build(&graph, &output_dir, &environment)?;
+ let mut files = environment.files.borrow_mut();
+ let import_map = files.remove(&output_dir.join("import_map.json"));
+ let mut files = files
+ .iter()
+ .map(|(path, text)| (path_to_string(path), text.clone()))
+ .collect::<Vec<_>>();
+
+ files.sort_by(|a, b| a.0.cmp(&b.0));
+
+ Ok(VendorOutput {
+ import_map: import_map.map(|text| serde_json::from_str(&text).unwrap()),
+ files,
+ })
+ }
+
+ pub fn with_loader(&mut self, action: impl Fn(&mut TestLoader)) -> &mut Self {
+ action(&mut self.loader);
+ self
+ }
+
+ async fn build_graph(&mut self) -> ModuleGraph {
+ let graph = deno_graph::create_graph(
+ self
+ .entry_points
+ .iter()
+ .map(|s| (s.to_owned(), deno_graph::ModuleKind::Esm))
+ .collect(),
+ false,
+ None,
+ &mut self.loader,
+ None,
+ None,
+ None,
+ None,
+ )
+ .await;
+ graph.lock().unwrap();
+ graph.valid().unwrap();
+ graph
+ }
+}
+
+fn make_path(text: &str) -> PathBuf {
+ // This should work all in memory. We're waiting on
+ // https://github.com/servo/rust-url/issues/730 to provide
+ // a cross platform path here
+ assert!(text.starts_with('/'));
+ if cfg!(windows) {
+ PathBuf::from(format!("C:{}", text.replace("/", "\\")))
+ } else {
+ PathBuf::from(text)
+ }
+}
+
+fn path_to_string(path: &Path) -> String {
+ // inverse of the function above
+ let path = path.to_string_lossy();
+ if cfg!(windows) {
+ path.replace("C:\\", "\\").replace('\\', "/")
+ } else {
+ path.to_string()
+ }
+}
diff --git a/ext/http/lib.rs b/ext/http/lib.rs
index e11d42da1..312942303 100644
--- a/ext/http/lib.rs
+++ b/ext/http/lib.rs
@@ -39,7 +39,6 @@ use hyper::service::Service;
use hyper::Body;
use hyper::Request;
use hyper::Response;
-use percent_encoding::percent_encode;
use serde::Deserialize;
use serde::Serialize;
use std::borrow::Cow;
@@ -428,7 +427,7 @@ fn req_url(
// httpie uses http+unix://[percent_encoding_of_path]/ which we follow
#[cfg(unix)]
HttpSocketAddr::UnixSocket(addr) => Cow::Owned(
- percent_encode(
+ percent_encoding::percent_encode(
addr
.as_pathname()
.and_then(|x| x.to_str())
diff --git a/runtime/ops/http.rs b/runtime/ops/http.rs
index 53a99bd47..5b8acb881 100644
--- a/runtime/ops/http.rs
+++ b/runtime/ops/http.rs
@@ -8,7 +8,6 @@ use deno_core::OpState;
use deno_core::ResourceId;
use deno_http::http_create_conn_resource;
use deno_net::io::TcpStreamResource;
-use deno_net::io::UnixStreamResource;
use deno_net::ops_tls::TlsStreamResource;
pub fn init() -> Extension {
@@ -49,7 +48,7 @@ fn op_http_start(
#[cfg(unix)]
if let Ok(resource_rc) = state
.resource_table
- .take::<UnixStreamResource>(tcp_stream_rid)
+ .take::<deno_net::io::UnixStreamResource>(tcp_stream_rid)
{
super::check_unstable(state, "Deno.serveHttp");