summaryrefslogtreecommitdiff
path: root/cli/tools
diff options
context:
space:
mode:
Diffstat (limited to 'cli/tools')
-rw-r--r--cli/tools/mod.rs1
-rw-r--r--cli/tools/vendor/analyze.rs113
-rw-r--r--cli/tools/vendor/build.rs577
-rw-r--r--cli/tools/vendor/import_map.rs285
-rw-r--r--cli/tools/vendor/mappings.rs286
-rw-r--r--cli/tools/vendor/mod.rs172
-rw-r--r--cli/tools/vendor/specifiers.rs251
-rw-r--r--cli/tools/vendor/test.rs240
8 files changed, 1925 insertions, 0 deletions
diff --git a/cli/tools/mod.rs b/cli/tools/mod.rs
index 83c501dde..ffea76e1d 100644
--- a/cli/tools/mod.rs
+++ b/cli/tools/mod.rs
@@ -9,3 +9,4 @@ pub mod repl;
pub mod standalone;
pub mod test;
pub mod upgrade;
+pub mod vendor;
diff --git a/cli/tools/vendor/analyze.rs b/cli/tools/vendor/analyze.rs
new file mode 100644
index 000000000..0639c0487
--- /dev/null
+++ b/cli/tools/vendor/analyze.rs
@@ -0,0 +1,113 @@
+// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
+
+use deno_ast::swc::ast::ExportDefaultDecl;
+use deno_ast::swc::ast::ExportSpecifier;
+use deno_ast::swc::ast::ModuleExportName;
+use deno_ast::swc::ast::NamedExport;
+use deno_ast::swc::ast::Program;
+use deno_ast::swc::visit::noop_visit_type;
+use deno_ast::swc::visit::Visit;
+use deno_ast::swc::visit::VisitWith;
+use deno_ast::ParsedSource;
+
+/// Gets if the parsed source has a default export.
+pub fn has_default_export(source: &ParsedSource) -> bool {
+ let mut visitor = DefaultExportFinder {
+ has_default_export: false,
+ };
+ let program = source.program();
+ let program: &Program = &program;
+ program.visit_with(&mut visitor);
+ visitor.has_default_export
+}
+
+struct DefaultExportFinder {
+ has_default_export: bool,
+}
+
+impl<'a> Visit for DefaultExportFinder {
+ noop_visit_type!();
+
+ fn visit_export_default_decl(&mut self, _: &ExportDefaultDecl) {
+ self.has_default_export = true;
+ }
+
+ fn visit_named_export(&mut self, named_export: &NamedExport) {
+ if named_export
+ .specifiers
+ .iter()
+ .any(export_specifier_has_default)
+ {
+ self.has_default_export = true;
+ }
+ }
+}
+
+fn export_specifier_has_default(s: &ExportSpecifier) -> bool {
+ match s {
+ ExportSpecifier::Default(_) => true,
+ ExportSpecifier::Namespace(_) => false,
+ ExportSpecifier::Named(named) => {
+ let export_name = named.exported.as_ref().unwrap_or(&named.orig);
+
+ match export_name {
+ ModuleExportName::Str(_) => false,
+ ModuleExportName::Ident(ident) => &*ident.sym == "default",
+ }
+ }
+ }
+}
+
+#[cfg(test)]
+mod test {
+ use deno_ast::MediaType;
+ use deno_ast::ParseParams;
+ use deno_ast::ParsedSource;
+ use deno_ast::SourceTextInfo;
+
+ use super::has_default_export;
+
+ #[test]
+ fn has_default_when_export_default_decl() {
+ let parsed_source = parse_module("export default class Class {}");
+ assert!(has_default_export(&parsed_source));
+ }
+
+ #[test]
+ fn has_default_when_named_export() {
+ let parsed_source = parse_module("export {default} from './test.ts';");
+ assert!(has_default_export(&parsed_source));
+ }
+
+ #[test]
+ fn has_default_when_named_export_alias() {
+ let parsed_source =
+ parse_module("export {test as default} from './test.ts';");
+ assert!(has_default_export(&parsed_source));
+ }
+
+ #[test]
+ fn not_has_default_when_named_export_not_exported() {
+ let parsed_source =
+ parse_module("export {default as test} from './test.ts';");
+ assert!(!has_default_export(&parsed_source));
+ }
+
+ #[test]
+ fn not_has_default_when_not() {
+ let parsed_source = parse_module("export {test} from './test.ts'; export class Test{} export * from './test';");
+ assert!(!has_default_export(&parsed_source));
+ }
+
+ fn parse_module(text: &str) -> ParsedSource {
+ deno_ast::parse_module(ParseParams {
+ specifier: "file:///mod.ts".to_string(),
+ capture_tokens: false,
+ maybe_syntax: None,
+ media_type: MediaType::TypeScript,
+ scope_analysis: false,
+ source: SourceTextInfo::from_string(text.to_string()),
+ })
+ .unwrap()
+ }
+}
diff --git a/cli/tools/vendor/build.rs b/cli/tools/vendor/build.rs
new file mode 100644
index 000000000..58f351dd8
--- /dev/null
+++ b/cli/tools/vendor/build.rs
@@ -0,0 +1,577 @@
+// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
+
+use std::path::Path;
+
+use deno_core::error::AnyError;
+use deno_graph::Module;
+use deno_graph::ModuleGraph;
+use deno_graph::ModuleKind;
+
+use super::analyze::has_default_export;
+use super::import_map::build_import_map;
+use super::mappings::Mappings;
+use super::mappings::ProxiedModule;
+use super::specifiers::is_remote_specifier;
+
+/// Allows substituting the environment for testing purposes.
+pub trait VendorEnvironment {
+ fn create_dir_all(&self, dir_path: &Path) -> Result<(), AnyError>;
+ fn write_file(&self, file_path: &Path, text: &str) -> Result<(), AnyError>;
+}
+
+pub struct RealVendorEnvironment;
+
+impl VendorEnvironment for RealVendorEnvironment {
+ fn create_dir_all(&self, dir_path: &Path) -> Result<(), AnyError> {
+ Ok(std::fs::create_dir_all(dir_path)?)
+ }
+
+ fn write_file(&self, file_path: &Path, text: &str) -> Result<(), AnyError> {
+ Ok(std::fs::write(file_path, text)?)
+ }
+}
+
+/// Vendors remote modules and returns how many were vendored.
+pub fn build(
+ graph: &ModuleGraph,
+ output_dir: &Path,
+ environment: &impl VendorEnvironment,
+) -> Result<usize, AnyError> {
+ assert!(output_dir.is_absolute());
+ let all_modules = graph.modules();
+ let remote_modules = all_modules
+ .iter()
+ .filter(|m| is_remote_specifier(&m.specifier))
+ .copied()
+ .collect::<Vec<_>>();
+ let mappings =
+ Mappings::from_remote_modules(graph, &remote_modules, output_dir)?;
+
+ // write out all the files
+ for module in &remote_modules {
+ let source = match &module.maybe_source {
+ Some(source) => source,
+ None => continue,
+ };
+ let local_path = mappings
+ .proxied_path(&module.specifier)
+ .unwrap_or_else(|| mappings.local_path(&module.specifier));
+ if !matches!(module.kind, ModuleKind::Esm | ModuleKind::Asserted) {
+ log::warn!(
+ "Unsupported module kind {:?} for {}",
+ module.kind,
+ module.specifier
+ );
+ continue;
+ }
+ environment.create_dir_all(local_path.parent().unwrap())?;
+ environment.write_file(&local_path, source)?;
+ }
+
+ // write out the proxies
+ for (specifier, proxied_module) in mappings.proxied_modules() {
+ let proxy_path = mappings.local_path(specifier);
+ let module = graph.get(specifier).unwrap();
+ let text = build_proxy_module_source(module, proxied_module);
+
+ environment.write_file(&proxy_path, &text)?;
+ }
+
+ // create the import map
+ if !mappings.base_specifiers().is_empty() {
+ let import_map_text = build_import_map(graph, &all_modules, &mappings);
+ environment
+ .write_file(&output_dir.join("import_map.json"), &import_map_text)?;
+ }
+
+ Ok(remote_modules.len())
+}
+
+fn build_proxy_module_source(
+ module: &Module,
+ proxied_module: &ProxiedModule,
+) -> String {
+ let mut text = format!(
+ "// @deno-types=\"{}\"\n",
+ proxied_module.declaration_specifier
+ );
+ let relative_specifier = format!(
+ "./{}",
+ proxied_module
+ .output_path
+ .file_name()
+ .unwrap()
+ .to_string_lossy()
+ );
+
+ // for simplicity, always include the `export *` statement as it won't error
+ // even when the module does not contain a named export
+ text.push_str(&format!("export * from \"{}\";\n", relative_specifier));
+
+ // add a default export if one exists in the module
+ if let Some(parsed_source) = module.maybe_parsed_source.as_ref() {
+ if has_default_export(parsed_source) {
+ text.push_str(&format!(
+ "export {{ default }} from \"{}\";\n",
+ relative_specifier
+ ));
+ }
+ }
+
+ text
+}
+
+#[cfg(test)]
+mod test {
+ use crate::tools::vendor::test::VendorTestBuilder;
+ use deno_core::serde_json::json;
+ use pretty_assertions::assert_eq;
+
+ #[tokio::test]
+ async fn no_remote_modules() {
+ let mut builder = VendorTestBuilder::with_default_setup();
+ let output = builder
+ .with_loader(|loader| {
+ loader.add("/mod.ts", "");
+ })
+ .build()
+ .await
+ .unwrap();
+
+ assert_eq!(output.import_map, None,);
+ assert_eq!(output.files, vec![],);
+ }
+
+ #[tokio::test]
+ async fn local_specifiers_to_remote() {
+ let mut builder = VendorTestBuilder::with_default_setup();
+ let output = builder
+ .with_loader(|loader| {
+ loader
+ .add(
+ "/mod.ts",
+ concat!(
+ r#"import "https://localhost/mod.ts";"#,
+ r#"import "https://localhost/other.ts?test";"#,
+ r#"import "https://localhost/redirect.ts";"#,
+ ),
+ )
+ .add("https://localhost/mod.ts", "export class Mod {}")
+ .add("https://localhost/other.ts?test", "export class Other {}")
+ .add_redirect(
+ "https://localhost/redirect.ts",
+ "https://localhost/mod.ts",
+ );
+ })
+ .build()
+ .await
+ .unwrap();
+
+ assert_eq!(
+ output.import_map,
+ Some(json!({
+ "imports": {
+ "https://localhost/": "./localhost/",
+ "https://localhost/other.ts?test": "./localhost/other.ts",
+ "https://localhost/redirect.ts": "./localhost/mod.ts",
+ }
+ }))
+ );
+ assert_eq!(
+ output.files,
+ to_file_vec(&[
+ ("/vendor/localhost/mod.ts", "export class Mod {}"),
+ ("/vendor/localhost/other.ts", "export class Other {}"),
+ ]),
+ );
+ }
+
+ #[tokio::test]
+ async fn remote_specifiers() {
+ let mut builder = VendorTestBuilder::with_default_setup();
+ let output = builder
+ .with_loader(|loader| {
+ loader
+ .add(
+ "/mod.ts",
+ concat!(
+ r#"import "https://localhost/mod.ts";"#,
+ r#"import "https://other/mod.ts";"#,
+ ),
+ )
+ .add(
+ "https://localhost/mod.ts",
+ concat!(
+ "export * from './other.ts';",
+ "export * from './redirect.ts';",
+ "export * from '/absolute.ts';",
+ ),
+ )
+ .add("https://localhost/other.ts", "export class Other {}")
+ .add_redirect(
+ "https://localhost/redirect.ts",
+ "https://localhost/other.ts",
+ )
+ .add("https://localhost/absolute.ts", "export class Absolute {}")
+ .add("https://other/mod.ts", "export * from './sub/mod.ts';")
+ .add(
+ "https://other/sub/mod.ts",
+ concat!(
+ "export * from '../sub2/mod.ts';",
+ "export * from '../sub2/other?asdf';",
+ // reference a path on a different origin
+ "export * from 'https://localhost/other.ts';",
+ "export * from 'https://localhost/redirect.ts';",
+ ),
+ )
+ .add("https://other/sub2/mod.ts", "export class Mod {}")
+ .add_with_headers(
+ "https://other/sub2/other?asdf",
+ "export class Other {}",
+ &[("content-type", "application/javascript")],
+ );
+ })
+ .build()
+ .await
+ .unwrap();
+
+ assert_eq!(
+ output.import_map,
+ Some(json!({
+ "imports": {
+ "https://localhost/": "./localhost/",
+ "https://localhost/redirect.ts": "./localhost/other.ts",
+ "https://other/": "./other/"
+ },
+ "scopes": {
+ "./localhost/": {
+ "./localhost/redirect.ts": "./localhost/other.ts",
+ "/absolute.ts": "./localhost/absolute.ts",
+ },
+ "./other/": {
+ "./other/sub2/other?asdf": "./other/sub2/other.js"
+ }
+ }
+ }))
+ );
+ assert_eq!(
+ output.files,
+ to_file_vec(&[
+ ("/vendor/localhost/absolute.ts", "export class Absolute {}"),
+ (
+ "/vendor/localhost/mod.ts",
+ concat!(
+ "export * from './other.ts';",
+ "export * from './redirect.ts';",
+ "export * from '/absolute.ts';",
+ )
+ ),
+ ("/vendor/localhost/other.ts", "export class Other {}"),
+ ("/vendor/other/mod.ts", "export * from './sub/mod.ts';"),
+ (
+ "/vendor/other/sub/mod.ts",
+ concat!(
+ "export * from '../sub2/mod.ts';",
+ "export * from '../sub2/other?asdf';",
+ "export * from 'https://localhost/other.ts';",
+ "export * from 'https://localhost/redirect.ts';",
+ )
+ ),
+ ("/vendor/other/sub2/mod.ts", "export class Mod {}"),
+ ("/vendor/other/sub2/other.js", "export class Other {}"),
+ ]),
+ );
+ }
+
+ #[tokio::test]
+ async fn same_target_filename_specifiers() {
+ let mut builder = VendorTestBuilder::with_default_setup();
+ let output = builder
+ .with_loader(|loader| {
+ loader
+ .add(
+ "/mod.ts",
+ concat!(
+ r#"import "https://localhost/MOD.TS";"#,
+ r#"import "https://localhost/mod.TS";"#,
+ r#"import "https://localhost/mod.ts";"#,
+ r#"import "https://localhost/mod.ts?test";"#,
+ r#"import "https://localhost/CAPS.TS";"#,
+ ),
+ )
+ .add("https://localhost/MOD.TS", "export class Mod {}")
+ .add("https://localhost/mod.TS", "export class Mod2 {}")
+ .add("https://localhost/mod.ts", "export class Mod3 {}")
+ .add("https://localhost/mod.ts?test", "export class Mod4 {}")
+ .add("https://localhost/CAPS.TS", "export class Caps {}");
+ })
+ .build()
+ .await
+ .unwrap();
+
+ assert_eq!(
+ output.import_map,
+ Some(json!({
+ "imports": {
+ "https://localhost/": "./localhost/",
+ "https://localhost/mod.TS": "./localhost/mod_2.TS",
+ "https://localhost/mod.ts": "./localhost/mod_3.ts",
+ "https://localhost/mod.ts?test": "./localhost/mod_4.ts",
+ }
+ }))
+ );
+ assert_eq!(
+ output.files,
+ to_file_vec(&[
+ ("/vendor/localhost/CAPS.TS", "export class Caps {}"),
+ ("/vendor/localhost/MOD.TS", "export class Mod {}"),
+ ("/vendor/localhost/mod_2.TS", "export class Mod2 {}"),
+ ("/vendor/localhost/mod_3.ts", "export class Mod3 {}"),
+ ("/vendor/localhost/mod_4.ts", "export class Mod4 {}"),
+ ]),
+ );
+ }
+
+ #[tokio::test]
+ async fn multiple_entrypoints() {
+ let mut builder = VendorTestBuilder::with_default_setup();
+ let output = builder
+ .add_entry_point("/test.deps.ts")
+ .with_loader(|loader| {
+ loader
+ .add("/mod.ts", r#"import "https://localhost/mod.ts";"#)
+ .add(
+ "/test.deps.ts",
+ r#"export * from "https://localhost/test.ts";"#,
+ )
+ .add("https://localhost/mod.ts", "export class Mod {}")
+ .add("https://localhost/test.ts", "export class Test {}");
+ })
+ .build()
+ .await
+ .unwrap();
+
+ assert_eq!(
+ output.import_map,
+ Some(json!({
+ "imports": {
+ "https://localhost/": "./localhost/",
+ }
+ }))
+ );
+ assert_eq!(
+ output.files,
+ to_file_vec(&[
+ ("/vendor/localhost/mod.ts", "export class Mod {}"),
+ ("/vendor/localhost/test.ts", "export class Test {}"),
+ ]),
+ );
+ }
+
+ #[tokio::test]
+ async fn json_module() {
+ let mut builder = VendorTestBuilder::with_default_setup();
+ let output = builder
+ .with_loader(|loader| {
+ loader
+ .add(
+ "/mod.ts",
+ r#"import data from "https://localhost/data.json" assert { type: "json" };"#,
+ )
+ .add("https://localhost/data.json", "{}");
+ })
+ .build()
+ .await
+ .unwrap();
+
+ assert_eq!(
+ output.import_map,
+ Some(json!({
+ "imports": {
+ "https://localhost/": "./localhost/"
+ }
+ }))
+ );
+ assert_eq!(
+ output.files,
+ to_file_vec(&[("/vendor/localhost/data.json", "{}"),]),
+ );
+ }
+
+ #[tokio::test]
+ async fn data_urls() {
+ let mut builder = VendorTestBuilder::with_default_setup();
+
+ let mod_file_text = r#"import * as b from "data:application/typescript,export%20*%20from%20%22https://localhost/mod.ts%22;";"#;
+
+ let output = builder
+ .with_loader(|loader| {
+ loader
+ .add("/mod.ts", &mod_file_text)
+ .add("https://localhost/mod.ts", "export class Example {}");
+ })
+ .build()
+ .await
+ .unwrap();
+
+ assert_eq!(
+ output.import_map,
+ Some(json!({
+ "imports": {
+ "https://localhost/": "./localhost/"
+ }
+ }))
+ );
+ assert_eq!(
+ output.files,
+ to_file_vec(&[("/vendor/localhost/mod.ts", "export class Example {}"),]),
+ );
+ }
+
+ #[tokio::test]
+ async fn x_typescript_types_no_default() {
+ let mut builder = VendorTestBuilder::with_default_setup();
+ let output = builder
+ .with_loader(|loader| {
+ loader
+ .add("/mod.ts", r#"import "https://localhost/mod.js";"#)
+ .add_with_headers(
+ "https://localhost/mod.js",
+ "export class Mod {}",
+ &[("x-typescript-types", "https://localhost/mod.d.ts")],
+ )
+ .add("https://localhost/mod.d.ts", "export class Mod {}");
+ })
+ .build()
+ .await
+ .unwrap();
+
+ assert_eq!(
+ output.import_map,
+ Some(json!({
+ "imports": {
+ "https://localhost/": "./localhost/"
+ }
+ }))
+ );
+ assert_eq!(
+ output.files,
+ to_file_vec(&[
+ ("/vendor/localhost/mod.d.ts", "export class Mod {}"),
+ (
+ "/vendor/localhost/mod.js",
+ concat!(
+ "// @deno-types=\"https://localhost/mod.d.ts\"\n",
+ "export * from \"./mod.proxied.js\";\n"
+ )
+ ),
+ ("/vendor/localhost/mod.proxied.js", "export class Mod {}"),
+ ]),
+ );
+ }
+
+ #[tokio::test]
+ async fn x_typescript_types_default_export() {
+ let mut builder = VendorTestBuilder::with_default_setup();
+ let output = builder
+ .with_loader(|loader| {
+ loader
+ .add("/mod.ts", r#"import "https://localhost/mod.js";"#)
+ .add_with_headers(
+ "https://localhost/mod.js",
+ "export default class Mod {}",
+ &[("x-typescript-types", "https://localhost/mod.d.ts")],
+ )
+ .add("https://localhost/mod.d.ts", "export default class Mod {}");
+ })
+ .build()
+ .await
+ .unwrap();
+
+ assert_eq!(
+ output.import_map,
+ Some(json!({
+ "imports": {
+ "https://localhost/": "./localhost/"
+ }
+ }))
+ );
+ assert_eq!(
+ output.files,
+ to_file_vec(&[
+ ("/vendor/localhost/mod.d.ts", "export default class Mod {}"),
+ (
+ "/vendor/localhost/mod.js",
+ concat!(
+ "// @deno-types=\"https://localhost/mod.d.ts\"\n",
+ "export * from \"./mod.proxied.js\";\n",
+ "export { default } from \"./mod.proxied.js\";\n",
+ )
+ ),
+ (
+ "/vendor/localhost/mod.proxied.js",
+ "export default class Mod {}"
+ ),
+ ]),
+ );
+ }
+
+ #[tokio::test]
+ async fn subdir() {
+ let mut builder = VendorTestBuilder::with_default_setup();
+ let output = builder
+ .with_loader(|loader| {
+ loader
+ .add(
+ "/mod.ts",
+ r#"import "http://localhost:4545/sub/logger/mod.ts?testing";"#,
+ )
+ .add(
+ "http://localhost:4545/sub/logger/mod.ts?testing",
+ "export * from './logger.ts?test';",
+ )
+ .add(
+ "http://localhost:4545/sub/logger/logger.ts?test",
+ "export class Logger {}",
+ );
+ })
+ .build()
+ .await
+ .unwrap();
+
+ assert_eq!(
+ output.import_map,
+ Some(json!({
+ "imports": {
+ "http://localhost:4545/": "./localhost_4545/",
+ "http://localhost:4545/sub/logger/mod.ts?testing": "./localhost_4545/sub/logger/mod.ts",
+ },
+ "scopes": {
+ "./localhost_4545/": {
+ "./localhost_4545/sub/logger/logger.ts?test": "./localhost_4545/sub/logger/logger.ts"
+ }
+ }
+ }))
+ );
+ assert_eq!(
+ output.files,
+ to_file_vec(&[
+ (
+ "/vendor/localhost_4545/sub/logger/logger.ts",
+ "export class Logger {}",
+ ),
+ (
+ "/vendor/localhost_4545/sub/logger/mod.ts",
+ "export * from './logger.ts?test';"
+ ),
+ ]),
+ );
+ }
+
+ fn to_file_vec(items: &[(&str, &str)]) -> Vec<(String, String)> {
+ items
+ .iter()
+ .map(|(f, t)| (f.to_string(), t.to_string()))
+ .collect()
+ }
+}
diff --git a/cli/tools/vendor/import_map.rs b/cli/tools/vendor/import_map.rs
new file mode 100644
index 000000000..7e18d56aa
--- /dev/null
+++ b/cli/tools/vendor/import_map.rs
@@ -0,0 +1,285 @@
+// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
+
+use std::collections::BTreeMap;
+
+use deno_ast::LineAndColumnIndex;
+use deno_ast::ModuleSpecifier;
+use deno_ast::SourceTextInfo;
+use deno_core::serde_json;
+use deno_graph::Module;
+use deno_graph::ModuleGraph;
+use deno_graph::Position;
+use deno_graph::Range;
+use deno_graph::Resolved;
+use serde::Serialize;
+
+use super::mappings::Mappings;
+use super::specifiers::is_remote_specifier;
+use super::specifiers::is_remote_specifier_text;
+
+#[derive(Serialize)]
+struct SerializableImportMap {
+ imports: BTreeMap<String, String>,
+ #[serde(skip_serializing_if = "BTreeMap::is_empty")]
+ scopes: BTreeMap<String, BTreeMap<String, String>>,
+}
+
+struct ImportMapBuilder<'a> {
+ mappings: &'a Mappings,
+ imports: ImportsBuilder<'a>,
+ scopes: BTreeMap<String, ImportsBuilder<'a>>,
+}
+
+impl<'a> ImportMapBuilder<'a> {
+ pub fn new(mappings: &'a Mappings) -> Self {
+ ImportMapBuilder {
+ mappings,
+ imports: ImportsBuilder::new(mappings),
+ scopes: Default::default(),
+ }
+ }
+
+ pub fn scope(
+ &mut self,
+ base_specifier: &ModuleSpecifier,
+ ) -> &mut ImportsBuilder<'a> {
+ self
+ .scopes
+ .entry(
+ self
+ .mappings
+ .relative_specifier_text(self.mappings.output_dir(), base_specifier),
+ )
+ .or_insert_with(|| ImportsBuilder::new(self.mappings))
+ }
+
+ pub fn into_serializable(self) -> SerializableImportMap {
+ SerializableImportMap {
+ imports: self.imports.imports,
+ scopes: self
+ .scopes
+ .into_iter()
+ .map(|(key, value)| (key, value.imports))
+ .collect(),
+ }
+ }
+
+ pub fn into_file_text(self) -> String {
+ let mut text =
+ serde_json::to_string_pretty(&self.into_serializable()).unwrap();
+ text.push('\n');
+ text
+ }
+}
+
+struct ImportsBuilder<'a> {
+ mappings: &'a Mappings,
+ imports: BTreeMap<String, String>,
+}
+
+impl<'a> ImportsBuilder<'a> {
+ pub fn new(mappings: &'a Mappings) -> Self {
+ Self {
+ mappings,
+ imports: Default::default(),
+ }
+ }
+
+ pub fn add(&mut self, key: String, specifier: &ModuleSpecifier) {
+ self.imports.insert(
+ key,
+ self
+ .mappings
+ .relative_specifier_text(self.mappings.output_dir(), specifier),
+ );
+ }
+}
+
+pub fn build_import_map(
+ graph: &ModuleGraph,
+ modules: &[&Module],
+ mappings: &Mappings,
+) -> String {
+ let mut import_map = ImportMapBuilder::new(mappings);
+ visit_modules(graph, modules, mappings, &mut import_map);
+
+ for base_specifier in mappings.base_specifiers() {
+ import_map
+ .imports
+ .add(base_specifier.to_string(), base_specifier);
+ }
+
+ import_map.into_file_text()
+}
+
+fn visit_modules(
+ graph: &ModuleGraph,
+ modules: &[&Module],
+ mappings: &Mappings,
+ import_map: &mut ImportMapBuilder,
+) {
+ for module in modules {
+ let text_info = match &module.maybe_parsed_source {
+ Some(source) => source.source(),
+ None => continue,
+ };
+ let source_text = match &module.maybe_source {
+ Some(source) => source,
+ None => continue,
+ };
+
+ for dep in module.dependencies.values() {
+ visit_maybe_resolved(
+ &dep.maybe_code,
+ graph,
+ import_map,
+ &module.specifier,
+ mappings,
+ text_info,
+ source_text,
+ );
+ visit_maybe_resolved(
+ &dep.maybe_type,
+ graph,
+ import_map,
+ &module.specifier,
+ mappings,
+ text_info,
+ source_text,
+ );
+ }
+
+ if let Some((_, maybe_resolved)) = &module.maybe_types_dependency {
+ visit_maybe_resolved(
+ maybe_resolved,
+ graph,
+ import_map,
+ &module.specifier,
+ mappings,
+ text_info,
+ source_text,
+ );
+ }
+ }
+}
+
+fn visit_maybe_resolved(
+ maybe_resolved: &Resolved,
+ graph: &ModuleGraph,
+ import_map: &mut ImportMapBuilder,
+ referrer: &ModuleSpecifier,
+ mappings: &Mappings,
+ text_info: &SourceTextInfo,
+ source_text: &str,
+) {
+ if let Resolved::Ok {
+ specifier, range, ..
+ } = maybe_resolved
+ {
+ let text = text_from_range(text_info, source_text, range);
+ // if the text is empty then it's probably an x-TypeScript-types
+ if !text.is_empty() {
+ handle_dep_specifier(
+ text, specifier, graph, import_map, referrer, mappings,
+ );
+ }
+ }
+}
+
+fn handle_dep_specifier(
+ text: &str,
+ unresolved_specifier: &ModuleSpecifier,
+ graph: &ModuleGraph,
+ import_map: &mut ImportMapBuilder,
+ referrer: &ModuleSpecifier,
+ mappings: &Mappings,
+) {
+ let specifier = graph.resolve(unresolved_specifier);
+ // do not handle specifiers pointing at local modules
+ if !is_remote_specifier(&specifier) {
+ return;
+ }
+
+ let base_specifier = mappings.base_specifier(&specifier);
+ if is_remote_specifier_text(text) {
+ if !text.starts_with(base_specifier.as_str()) {
+ panic!("Expected {} to start with {}", text, base_specifier);
+ }
+
+ let sub_path = &text[base_specifier.as_str().len()..];
+ let expected_relative_specifier_text =
+ mappings.relative_path(base_specifier, &specifier);
+ if expected_relative_specifier_text == sub_path {
+ return;
+ }
+
+ if referrer.origin() == specifier.origin() {
+ let imports = import_map.scope(base_specifier);
+ imports.add(sub_path.to_string(), &specifier);
+ } else {
+ import_map.imports.add(text.to_string(), &specifier);
+ }
+ } else {
+ let expected_relative_specifier_text =
+ mappings.relative_specifier_text(referrer, &specifier);
+ if expected_relative_specifier_text == text {
+ return;
+ }
+
+ let key = if text.starts_with("./") || text.starts_with("../") {
+ // resolve relative specifier key
+ let mut local_base_specifier = mappings.local_uri(base_specifier);
+ local_base_specifier.set_query(unresolved_specifier.query());
+ local_base_specifier = local_base_specifier
+ .join(&unresolved_specifier.path()[1..])
+ .unwrap_or_else(|_| {
+ panic!(
+ "Error joining {} to {}",
+ unresolved_specifier.path(),
+ local_base_specifier
+ )
+ });
+ local_base_specifier.set_query(unresolved_specifier.query());
+ mappings
+ .relative_specifier_text(mappings.output_dir(), &local_base_specifier)
+ } else {
+ // absolute (`/`) or bare specifier should be left as-is
+ text.to_string()
+ };
+ let imports = import_map.scope(base_specifier);
+ imports.add(key, &specifier);
+ }
+}
+
+fn text_from_range<'a>(
+ text_info: &SourceTextInfo,
+ text: &'a str,
+ range: &Range,
+) -> &'a str {
+ let result = &text[byte_range(text_info, range)];
+ if result.starts_with('"') || result.starts_with('\'') {
+ // remove the quotes
+ &result[1..result.len() - 1]
+ } else {
+ result
+ }
+}
+
+fn byte_range(
+ text_info: &SourceTextInfo,
+ range: &Range,
+) -> std::ops::Range<usize> {
+ let start = byte_index(text_info, &range.start);
+ let end = byte_index(text_info, &range.end);
+ start..end
+}
+
+fn byte_index(text_info: &SourceTextInfo, pos: &Position) -> usize {
+ // todo(https://github.com/denoland/deno_graph/issues/79): use byte indexes all the way down
+ text_info
+ .byte_index(LineAndColumnIndex {
+ line_index: pos.line,
+ column_index: pos.character,
+ })
+ .0 as usize
+}
diff --git a/cli/tools/vendor/mappings.rs b/cli/tools/vendor/mappings.rs
new file mode 100644
index 000000000..2e85445dc
--- /dev/null
+++ b/cli/tools/vendor/mappings.rs
@@ -0,0 +1,286 @@
+// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
+
+use std::collections::HashMap;
+use std::collections::HashSet;
+use std::path::Path;
+use std::path::PathBuf;
+
+use deno_ast::MediaType;
+use deno_ast::ModuleSpecifier;
+use deno_core::error::AnyError;
+use deno_graph::Module;
+use deno_graph::ModuleGraph;
+use deno_graph::Position;
+use deno_graph::Resolved;
+
+use crate::fs_util::path_with_stem_suffix;
+
+use super::specifiers::dir_name_for_root;
+use super::specifiers::get_unique_path;
+use super::specifiers::make_url_relative;
+use super::specifiers::partition_by_root_specifiers;
+use super::specifiers::sanitize_filepath;
+
+pub struct ProxiedModule {
+ pub output_path: PathBuf,
+ pub declaration_specifier: ModuleSpecifier,
+}
+
+/// Constructs and holds the remote specifier to local path mappings.
+pub struct Mappings {
+ output_dir: ModuleSpecifier,
+ mappings: HashMap<ModuleSpecifier, PathBuf>,
+ base_specifiers: Vec<ModuleSpecifier>,
+ proxies: HashMap<ModuleSpecifier, ProxiedModule>,
+}
+
+impl Mappings {
+ pub fn from_remote_modules(
+ graph: &ModuleGraph,
+ remote_modules: &[&Module],
+ output_dir: &Path,
+ ) -> Result<Self, AnyError> {
+ let partitioned_specifiers =
+ partition_by_root_specifiers(remote_modules.iter().map(|m| &m.specifier));
+ let mut mapped_paths = HashSet::new();
+ let mut mappings = HashMap::new();
+ let mut proxies = HashMap::new();
+ let mut base_specifiers = Vec::new();
+
+ for (root, specifiers) in partitioned_specifiers.into_iter() {
+ let base_dir = get_unique_path(
+ output_dir.join(dir_name_for_root(&root)),
+ &mut mapped_paths,
+ );
+ for specifier in specifiers {
+ let media_type = graph.get(&specifier).unwrap().media_type;
+ let sub_path = sanitize_filepath(&make_url_relative(&root, &{
+ let mut specifier = specifier.clone();
+ specifier.set_query(None);
+ specifier
+ })?);
+ let new_path = path_with_extension(
+ &base_dir.join(if cfg!(windows) {
+ sub_path.replace('/', "\\")
+ } else {
+ sub_path
+ }),
+ &media_type.as_ts_extension()[1..],
+ );
+ mappings
+ .insert(specifier, get_unique_path(new_path, &mut mapped_paths));
+ }
+ base_specifiers.push(root.clone());
+ mappings.insert(root, base_dir);
+ }
+
+ // resolve all the "proxy" paths to use for when an x-typescript-types header is specified
+ for module in remote_modules {
+ if let Some((
+ _,
+ Resolved::Ok {
+ specifier, range, ..
+ },
+ )) = &module.maybe_types_dependency
+ {
+ // hack to tell if it's an x-typescript-types header
+ let is_ts_types_header =
+ range.start == Position::zeroed() && range.end == Position::zeroed();
+ if is_ts_types_header {
+ let module_path = mappings.get(&module.specifier).unwrap();
+ let proxied_path = get_unique_path(
+ path_with_stem_suffix(module_path, ".proxied"),
+ &mut mapped_paths,
+ );
+ proxies.insert(
+ module.specifier.clone(),
+ ProxiedModule {
+ output_path: proxied_path,
+ declaration_specifier: specifier.clone(),
+ },
+ );
+ }
+ }
+ }
+
+ Ok(Self {
+ output_dir: ModuleSpecifier::from_directory_path(output_dir).unwrap(),
+ mappings,
+ base_specifiers,
+ proxies,
+ })
+ }
+
+ pub fn output_dir(&self) -> &ModuleSpecifier {
+ &self.output_dir
+ }
+
+ pub fn local_uri(&self, specifier: &ModuleSpecifier) -> ModuleSpecifier {
+ if specifier.scheme() == "file" {
+ specifier.clone()
+ } else {
+ let local_path = self.local_path(specifier);
+ if specifier.path().ends_with('/') {
+ ModuleSpecifier::from_directory_path(&local_path)
+ } else {
+ ModuleSpecifier::from_file_path(&local_path)
+ }
+ .unwrap_or_else(|_| {
+ panic!("Could not convert {} to uri.", local_path.display())
+ })
+ }
+ }
+
+ pub fn local_path(&self, specifier: &ModuleSpecifier) -> PathBuf {
+ if specifier.scheme() == "file" {
+ specifier.to_file_path().unwrap()
+ } else {
+ self
+ .mappings
+ .get(specifier)
+ .as_ref()
+ .unwrap_or_else(|| {
+ panic!("Could not find local path for {}", specifier)
+ })
+ .to_path_buf()
+ }
+ }
+
+ pub fn relative_path(
+ &self,
+ from: &ModuleSpecifier,
+ to: &ModuleSpecifier,
+ ) -> String {
+ let mut from = self.local_uri(from);
+ let to = self.local_uri(to);
+
+ // workaround using parent directory until https://github.com/servo/rust-url/pull/754 is merged
+ if !from.path().ends_with('/') {
+ let local_path = self.local_path(&from);
+ from = ModuleSpecifier::from_directory_path(local_path.parent().unwrap())
+ .unwrap();
+ }
+
+ // workaround for url crate not adding a trailing slash for a directory
+ // it seems to be fixed once a version greater than 2.2.2 is released
+ let is_dir = to.path().ends_with('/');
+ let mut text = from.make_relative(&to).unwrap();
+ if is_dir && !text.ends_with('/') && to.query().is_none() {
+ text.push('/');
+ }
+ text
+ }
+
+ pub fn relative_specifier_text(
+ &self,
+ from: &ModuleSpecifier,
+ to: &ModuleSpecifier,
+ ) -> String {
+ let relative_path = self.relative_path(from, to);
+
+ if relative_path.starts_with("../") || relative_path.starts_with("./") {
+ relative_path
+ } else {
+ format!("./{}", relative_path)
+ }
+ }
+
+ pub fn base_specifiers(&self) -> &Vec<ModuleSpecifier> {
+ &self.base_specifiers
+ }
+
+ pub fn base_specifier(
+ &self,
+ child_specifier: &ModuleSpecifier,
+ ) -> &ModuleSpecifier {
+ self
+ .base_specifiers
+ .iter()
+ .find(|s| child_specifier.as_str().starts_with(s.as_str()))
+ .unwrap_or_else(|| {
+ panic!("Could not find base specifier for {}", child_specifier)
+ })
+ }
+
+ pub fn proxied_path(&self, specifier: &ModuleSpecifier) -> Option<PathBuf> {
+ self.proxies.get(specifier).map(|s| s.output_path.clone())
+ }
+
+ pub fn proxied_modules(
+ &self,
+ ) -> std::collections::hash_map::Iter<'_, ModuleSpecifier, ProxiedModule> {
+ self.proxies.iter()
+ }
+}
+
+fn path_with_extension(path: &Path, new_ext: &str) -> PathBuf {
+ if let Some(file_stem) = path.file_stem().map(|f| f.to_string_lossy()) {
+ if let Some(old_ext) = path.extension().map(|f| f.to_string_lossy()) {
+ if file_stem.to_lowercase().ends_with(".d") {
+ if new_ext.to_lowercase() == format!("d.{}", old_ext.to_lowercase()) {
+ // maintain casing
+ return path.to_path_buf();
+ }
+ return path.with_file_name(format!(
+ "{}.{}",
+ &file_stem[..file_stem.len() - ".d".len()],
+ new_ext
+ ));
+ }
+ if new_ext.to_lowercase() == old_ext.to_lowercase() {
+ // maintain casing
+ return path.to_path_buf();
+ }
+ let media_type: MediaType = path.into();
+ if media_type == MediaType::Unknown {
+ return path.with_file_name(format!(
+ "{}.{}",
+ path.file_name().unwrap().to_string_lossy(),
+ new_ext
+ ));
+ }
+ }
+ }
+ path.with_extension(new_ext)
+}
+
+#[cfg(test)]
+mod test {
+ use pretty_assertions::assert_eq;
+
+ use super::*;
+
+ #[test]
+ fn test_path_with_extension() {
+ assert_eq!(
+ path_with_extension(&PathBuf::from("/test.D.TS"), "ts"),
+ PathBuf::from("/test.ts")
+ );
+ assert_eq!(
+ path_with_extension(&PathBuf::from("/test.D.MTS"), "js"),
+ PathBuf::from("/test.js")
+ );
+ assert_eq!(
+ path_with_extension(&PathBuf::from("/test.D.TS"), "d.ts"),
+ // maintains casing
+ PathBuf::from("/test.D.TS"),
+ );
+ assert_eq!(
+ path_with_extension(&PathBuf::from("/test.TS"), "ts"),
+ // maintains casing
+ PathBuf::from("/test.TS"),
+ );
+ assert_eq!(
+ path_with_extension(&PathBuf::from("/test.ts"), "js"),
+ PathBuf::from("/test.js")
+ );
+ assert_eq!(
+ path_with_extension(&PathBuf::from("/test.js"), "js"),
+ PathBuf::from("/test.js")
+ );
+ assert_eq!(
+ path_with_extension(&PathBuf::from("/chai@1.2.3"), "js"),
+ PathBuf::from("/chai@1.2.3.js")
+ );
+ }
+}
diff --git a/cli/tools/vendor/mod.rs b/cli/tools/vendor/mod.rs
new file mode 100644
index 000000000..eb9c91071
--- /dev/null
+++ b/cli/tools/vendor/mod.rs
@@ -0,0 +1,172 @@
+// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
+
+use std::path::Path;
+use std::path::PathBuf;
+
+use deno_core::anyhow::bail;
+use deno_core::anyhow::Context;
+use deno_core::error::AnyError;
+use deno_core::resolve_url_or_path;
+use deno_runtime::permissions::Permissions;
+
+use crate::flags::VendorFlags;
+use crate::fs_util;
+use crate::lockfile;
+use crate::proc_state::ProcState;
+use crate::resolver::ImportMapResolver;
+use crate::resolver::JsxResolver;
+use crate::tools::vendor::specifiers::is_remote_specifier_text;
+
+mod analyze;
+mod build;
+mod import_map;
+mod mappings;
+mod specifiers;
+#[cfg(test)]
+mod test;
+
+pub async fn vendor(ps: ProcState, flags: VendorFlags) -> Result<(), AnyError> {
+ let raw_output_dir = match &flags.output_path {
+ Some(output_path) => output_path.to_owned(),
+ None => PathBuf::from("vendor/"),
+ };
+ let output_dir = fs_util::resolve_from_cwd(&raw_output_dir)?;
+ validate_output_dir(&output_dir, &flags, &ps)?;
+ let graph = create_graph(&ps, &flags).await?;
+ let vendored_count =
+ build::build(&graph, &output_dir, &build::RealVendorEnvironment)?;
+
+ eprintln!(
+ r#"Vendored {} {} into {} directory.
+
+To use vendored modules, specify the `--import-map` flag when invoking deno subcommands:
+ deno run -A --import-map {} {}"#,
+ vendored_count,
+ if vendored_count == 1 {
+ "module"
+ } else {
+ "modules"
+ },
+ raw_output_dir.display(),
+ raw_output_dir.join("import_map.json").display(),
+ flags
+ .specifiers
+ .iter()
+ .map(|s| s.as_str())
+ .find(|s| !is_remote_specifier_text(s))
+ .unwrap_or("main.ts"),
+ );
+
+ Ok(())
+}
+
+fn validate_output_dir(
+ output_dir: &Path,
+ flags: &VendorFlags,
+ ps: &ProcState,
+) -> Result<(), AnyError> {
+ if !flags.force && !is_dir_empty(output_dir)? {
+ bail!(concat!(
+ "Output directory was not empty. Please specify an empty directory or use ",
+ "--force to ignore this error and potentially overwrite its contents.",
+ ));
+ }
+
+ // check the import map
+ if let Some(import_map_path) = ps
+ .maybe_import_map
+ .as_ref()
+ .and_then(|m| m.base_url().to_file_path().ok())
+ .and_then(|p| fs_util::canonicalize_path(&p).ok())
+ {
+ // make the output directory in order to canonicalize it for the check below
+ std::fs::create_dir_all(&output_dir)?;
+ let output_dir =
+ fs_util::canonicalize_path(output_dir).with_context(|| {
+ format!("Failed to canonicalize: {}", output_dir.display())
+ })?;
+
+ if import_map_path.starts_with(&output_dir) {
+ // We don't allow using the output directory to help generate the new state
+ // of itself because supporting this scenario adds a lot of complexity.
+ bail!(
+ "Using an import map found in the output directory is not supported."
+ );
+ }
+ }
+
+ Ok(())
+}
+
+fn is_dir_empty(dir_path: &Path) -> Result<bool, AnyError> {
+ match std::fs::read_dir(&dir_path) {
+ Ok(mut dir) => Ok(dir.next().is_none()),
+ Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(true),
+ Err(err) => {
+ bail!("Error reading directory {}: {}", dir_path.display(), err)
+ }
+ }
+}
+
+async fn create_graph(
+ ps: &ProcState,
+ flags: &VendorFlags,
+) -> Result<deno_graph::ModuleGraph, AnyError> {
+ let entry_points = flags
+ .specifiers
+ .iter()
+ .map(|p| {
+ let url = resolve_url_or_path(p)?;
+ Ok((url, deno_graph::ModuleKind::Esm))
+ })
+ .collect::<Result<Vec<_>, AnyError>>()?;
+
+ // todo(dsherret): there is a lot of copy and paste here from
+ // other parts of the codebase. We should consolidate this.
+ let mut cache = crate::cache::FetchCacher::new(
+ ps.dir.gen_cache.clone(),
+ ps.file_fetcher.clone(),
+ Permissions::allow_all(),
+ Permissions::allow_all(),
+ );
+ let maybe_locker = lockfile::as_maybe_locker(ps.lockfile.clone());
+ let maybe_imports = if let Some(config_file) = &ps.maybe_config_file {
+ config_file.to_maybe_imports()?
+ } else {
+ None
+ };
+ let maybe_import_map_resolver =
+ ps.maybe_import_map.clone().map(ImportMapResolver::new);
+ let maybe_jsx_resolver = ps
+ .maybe_config_file
+ .as_ref()
+ .map(|cf| {
+ cf.to_maybe_jsx_import_source_module()
+ .map(|im| JsxResolver::new(im, maybe_import_map_resolver.clone()))
+ })
+ .flatten();
+ let maybe_resolver = if maybe_jsx_resolver.is_some() {
+ maybe_jsx_resolver.as_ref().map(|jr| jr.as_resolver())
+ } else {
+ maybe_import_map_resolver
+ .as_ref()
+ .map(|im| im.as_resolver())
+ };
+
+ let graph = deno_graph::create_graph(
+ entry_points,
+ false,
+ maybe_imports,
+ &mut cache,
+ maybe_resolver,
+ maybe_locker,
+ None,
+ None,
+ )
+ .await;
+
+ graph.lock()?;
+ graph.valid()?;
+
+ Ok(graph)
+}
diff --git a/cli/tools/vendor/specifiers.rs b/cli/tools/vendor/specifiers.rs
new file mode 100644
index 000000000..b869e989c
--- /dev/null
+++ b/cli/tools/vendor/specifiers.rs
@@ -0,0 +1,251 @@
+// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
+
+use std::collections::BTreeMap;
+use std::collections::HashSet;
+use std::path::PathBuf;
+
+use deno_ast::ModuleSpecifier;
+use deno_core::anyhow::anyhow;
+use deno_core::error::AnyError;
+
+use crate::fs_util::path_with_stem_suffix;
+
+/// Partitions the provided specifiers by the non-path and non-query parts of a specifier.
+pub fn partition_by_root_specifiers<'a>(
+ specifiers: impl Iterator<Item = &'a ModuleSpecifier>,
+) -> BTreeMap<ModuleSpecifier, Vec<ModuleSpecifier>> {
+ let mut root_specifiers: BTreeMap<ModuleSpecifier, Vec<ModuleSpecifier>> =
+ Default::default();
+ for remote_specifier in specifiers {
+ let mut root_specifier = remote_specifier.clone();
+ root_specifier.set_query(None);
+ root_specifier.set_path("/");
+
+ let specifiers = root_specifiers.entry(root_specifier).or_default();
+ specifiers.push(remote_specifier.clone());
+ }
+ root_specifiers
+}
+
+/// Gets the directory name to use for the provided root.
+pub fn dir_name_for_root(root: &ModuleSpecifier) -> PathBuf {
+ let mut result = String::new();
+ if let Some(domain) = root.domain() {
+ result.push_str(&sanitize_segment(domain));
+ }
+ if let Some(port) = root.port() {
+ if !result.is_empty() {
+ result.push('_');
+ }
+ result.push_str(&port.to_string());
+ }
+ let mut result = PathBuf::from(result);
+ if let Some(segments) = root.path_segments() {
+ for segment in segments.filter(|s| !s.is_empty()) {
+ result = result.join(sanitize_segment(segment));
+ }
+ }
+
+ result
+}
+
+/// Gets a unique file path given the provided file path
+/// and the set of existing file paths. Inserts to the
+/// set when finding a unique path.
+pub fn get_unique_path(
+ mut path: PathBuf,
+ unique_set: &mut HashSet<String>,
+) -> PathBuf {
+ let original_path = path.clone();
+ let mut count = 2;
+ // case insensitive comparison so the output works on case insensitive file systems
+ while !unique_set.insert(path.to_string_lossy().to_lowercase()) {
+ path = path_with_stem_suffix(&original_path, &format!("_{}", count));
+ count += 1;
+ }
+ path
+}
+
+pub fn make_url_relative(
+ root: &ModuleSpecifier,
+ url: &ModuleSpecifier,
+) -> Result<String, AnyError> {
+ root.make_relative(url).ok_or_else(|| {
+ anyhow!(
+ "Error making url ({}) relative to root: {}",
+ url.to_string(),
+ root.to_string()
+ )
+ })
+}
+
+pub fn is_remote_specifier(specifier: &ModuleSpecifier) -> bool {
+ specifier.scheme().to_lowercase().starts_with("http")
+}
+
+pub fn is_remote_specifier_text(text: &str) -> bool {
+ text.trim_start().to_lowercase().starts_with("http")
+}
+
+pub fn sanitize_filepath(text: &str) -> String {
+ text
+ .chars()
+ .map(|c| if is_banned_path_char(c) { '_' } else { c })
+ .collect()
+}
+
+fn is_banned_path_char(c: char) -> bool {
+ matches!(c, '<' | '>' | ':' | '"' | '|' | '?' | '*')
+}
+
+fn sanitize_segment(text: &str) -> String {
+ text
+ .chars()
+ .map(|c| if is_banned_segment_char(c) { '_' } else { c })
+ .collect()
+}
+
+fn is_banned_segment_char(c: char) -> bool {
+ matches!(c, '/' | '\\') || is_banned_path_char(c)
+}
+
+#[cfg(test)]
+mod test {
+ use super::*;
+ use pretty_assertions::assert_eq;
+
+ #[test]
+ fn partition_by_root_specifiers_same_sub_folder() {
+ run_partition_by_root_specifiers_test(
+ vec![
+ "https://deno.land/x/mod/A.ts",
+ "https://deno.land/x/mod/other/A.ts",
+ ],
+ vec![(
+ "https://deno.land/",
+ vec![
+ "https://deno.land/x/mod/A.ts",
+ "https://deno.land/x/mod/other/A.ts",
+ ],
+ )],
+ );
+ }
+
+ #[test]
+ fn partition_by_root_specifiers_different_sub_folder() {
+ run_partition_by_root_specifiers_test(
+ vec![
+ "https://deno.land/x/mod/A.ts",
+ "https://deno.land/x/other/A.ts",
+ ],
+ vec![(
+ "https://deno.land/",
+ vec![
+ "https://deno.land/x/mod/A.ts",
+ "https://deno.land/x/other/A.ts",
+ ],
+ )],
+ );
+ }
+
+ #[test]
+ fn partition_by_root_specifiers_different_hosts() {
+ run_partition_by_root_specifiers_test(
+ vec![
+ "https://deno.land/mod/A.ts",
+ "http://deno.land/B.ts",
+ "https://deno.land:8080/C.ts",
+ "https://localhost/mod/A.ts",
+ "https://other/A.ts",
+ ],
+ vec![
+ ("http://deno.land/", vec!["http://deno.land/B.ts"]),
+ ("https://deno.land/", vec!["https://deno.land/mod/A.ts"]),
+ (
+ "https://deno.land:8080/",
+ vec!["https://deno.land:8080/C.ts"],
+ ),
+ ("https://localhost/", vec!["https://localhost/mod/A.ts"]),
+ ("https://other/", vec!["https://other/A.ts"]),
+ ],
+ );
+ }
+
+ fn run_partition_by_root_specifiers_test(
+ input: Vec<&str>,
+ expected: Vec<(&str, Vec<&str>)>,
+ ) {
+ let input = input
+ .iter()
+ .map(|s| ModuleSpecifier::parse(s).unwrap())
+ .collect::<Vec<_>>();
+ let output = partition_by_root_specifiers(input.iter());
+ // the assertion is much easier to compare when everything is strings
+ let output = output
+ .into_iter()
+ .map(|(s, vec)| {
+ (
+ s.to_string(),
+ vec.into_iter().map(|s| s.to_string()).collect::<Vec<_>>(),
+ )
+ })
+ .collect::<Vec<_>>();
+ let expected = expected
+ .into_iter()
+ .map(|(s, vec)| {
+ (
+ s.to_string(),
+ vec.into_iter().map(|s| s.to_string()).collect::<Vec<_>>(),
+ )
+ })
+ .collect::<Vec<_>>();
+ assert_eq!(output, expected);
+ }
+
+ #[test]
+ fn should_get_dir_name_root() {
+ run_test("http://deno.land/x/test", "deno.land/x/test");
+ run_test("http://localhost", "localhost");
+ run_test("http://localhost/test%20:test", "localhost/test%20_test");
+
+ fn run_test(specifier: &str, expected: &str) {
+ assert_eq!(
+ dir_name_for_root(&ModuleSpecifier::parse(specifier).unwrap()),
+ PathBuf::from(expected)
+ );
+ }
+ }
+
+ #[test]
+ fn test_unique_path() {
+ let mut paths = HashSet::new();
+ assert_eq!(
+ get_unique_path(PathBuf::from("/test"), &mut paths),
+ PathBuf::from("/test")
+ );
+ assert_eq!(
+ get_unique_path(PathBuf::from("/test"), &mut paths),
+ PathBuf::from("/test_2")
+ );
+ assert_eq!(
+ get_unique_path(PathBuf::from("/test"), &mut paths),
+ PathBuf::from("/test_3")
+ );
+ assert_eq!(
+ get_unique_path(PathBuf::from("/TEST"), &mut paths),
+ PathBuf::from("/TEST_4")
+ );
+ assert_eq!(
+ get_unique_path(PathBuf::from("/test.txt"), &mut paths),
+ PathBuf::from("/test.txt")
+ );
+ assert_eq!(
+ get_unique_path(PathBuf::from("/test.txt"), &mut paths),
+ PathBuf::from("/test_2.txt")
+ );
+ assert_eq!(
+ get_unique_path(PathBuf::from("/TEST.TXT"), &mut paths),
+ PathBuf::from("/TEST_3.TXT")
+ );
+ }
+}
diff --git a/cli/tools/vendor/test.rs b/cli/tools/vendor/test.rs
new file mode 100644
index 000000000..b37e2b3b0
--- /dev/null
+++ b/cli/tools/vendor/test.rs
@@ -0,0 +1,240 @@
+// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
+
+use std::cell::RefCell;
+use std::collections::HashMap;
+use std::collections::HashSet;
+use std::path::Path;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use deno_ast::ModuleSpecifier;
+use deno_core::anyhow::anyhow;
+use deno_core::anyhow::bail;
+use deno_core::error::AnyError;
+use deno_core::futures;
+use deno_core::serde_json;
+use deno_graph::source::LoadFuture;
+use deno_graph::source::LoadResponse;
+use deno_graph::source::Loader;
+use deno_graph::ModuleGraph;
+
+use super::build::VendorEnvironment;
+
+// Utilities that help `deno vendor` get tested in memory.
+
+type RemoteFileText = String;
+type RemoteFileHeaders = Option<HashMap<String, String>>;
+type RemoteFileResult = Result<(RemoteFileText, RemoteFileHeaders), String>;
+
+#[derive(Clone, Default)]
+pub struct TestLoader {
+ files: HashMap<ModuleSpecifier, RemoteFileResult>,
+ redirects: HashMap<ModuleSpecifier, ModuleSpecifier>,
+}
+
+impl TestLoader {
+ pub fn add(
+ &mut self,
+ path_or_specifier: impl AsRef<str>,
+ text: impl AsRef<str>,
+ ) -> &mut Self {
+ if path_or_specifier
+ .as_ref()
+ .to_lowercase()
+ .starts_with("http")
+ {
+ self.files.insert(
+ ModuleSpecifier::parse(path_or_specifier.as_ref()).unwrap(),
+ Ok((text.as_ref().to_string(), None)),
+ );
+ } else {
+ let path = make_path(path_or_specifier.as_ref());
+ let specifier = ModuleSpecifier::from_file_path(path).unwrap();
+ self
+ .files
+ .insert(specifier, Ok((text.as_ref().to_string(), None)));
+ }
+ self
+ }
+
+ pub fn add_with_headers(
+ &mut self,
+ specifier: impl AsRef<str>,
+ text: impl AsRef<str>,
+ headers: &[(&str, &str)],
+ ) -> &mut Self {
+ let headers = headers
+ .iter()
+ .map(|(key, value)| (key.to_string(), value.to_string()))
+ .collect();
+ self.files.insert(
+ ModuleSpecifier::parse(specifier.as_ref()).unwrap(),
+ Ok((text.as_ref().to_string(), Some(headers))),
+ );
+ self
+ }
+
+ pub fn add_redirect(
+ &mut self,
+ from: impl AsRef<str>,
+ to: impl AsRef<str>,
+ ) -> &mut Self {
+ self.redirects.insert(
+ ModuleSpecifier::parse(from.as_ref()).unwrap(),
+ ModuleSpecifier::parse(to.as_ref()).unwrap(),
+ );
+ self
+ }
+}
+
+impl Loader for TestLoader {
+ fn load(
+ &mut self,
+ specifier: &ModuleSpecifier,
+ _is_dynamic: bool,
+ ) -> LoadFuture {
+ let specifier = self.redirects.get(specifier).unwrap_or(specifier);
+ let result = self.files.get(specifier).map(|result| match result {
+ Ok(result) => Ok(LoadResponse::Module {
+ specifier: specifier.clone(),
+ content: Arc::new(result.0.clone()),
+ maybe_headers: result.1.clone(),
+ }),
+ Err(err) => Err(err),
+ });
+ let result = match result {
+ Some(Ok(result)) => Ok(Some(result)),
+ Some(Err(err)) => Err(anyhow!("{}", err)),
+ None if specifier.scheme() == "data" => {
+ deno_graph::source::load_data_url(specifier)
+ }
+ None => Ok(None),
+ };
+ Box::pin(futures::future::ready(result))
+ }
+}
+
+#[derive(Default)]
+struct TestVendorEnvironment {
+ directories: RefCell<HashSet<PathBuf>>,
+ files: RefCell<HashMap<PathBuf, String>>,
+}
+
+impl VendorEnvironment for TestVendorEnvironment {
+ fn create_dir_all(&self, dir_path: &Path) -> Result<(), AnyError> {
+ let mut directories = self.directories.borrow_mut();
+ for path in dir_path.ancestors() {
+ if !directories.insert(path.to_path_buf()) {
+ break;
+ }
+ }
+ Ok(())
+ }
+
+ fn write_file(&self, file_path: &Path, text: &str) -> Result<(), AnyError> {
+ let parent = file_path.parent().unwrap();
+ if !self.directories.borrow().contains(parent) {
+ bail!("Directory not found: {}", parent.display());
+ }
+ self
+ .files
+ .borrow_mut()
+ .insert(file_path.to_path_buf(), text.to_string());
+ Ok(())
+ }
+}
+
+pub struct VendorOutput {
+ pub files: Vec<(String, String)>,
+ pub import_map: Option<serde_json::Value>,
+}
+
+#[derive(Default)]
+pub struct VendorTestBuilder {
+ entry_points: Vec<ModuleSpecifier>,
+ loader: TestLoader,
+}
+
+impl VendorTestBuilder {
+ pub fn with_default_setup() -> Self {
+ let mut builder = VendorTestBuilder::default();
+ builder.add_entry_point("/mod.ts");
+ builder
+ }
+
+ pub fn add_entry_point(&mut self, entry_point: impl AsRef<str>) -> &mut Self {
+ let entry_point = make_path(entry_point.as_ref());
+ self
+ .entry_points
+ .push(ModuleSpecifier::from_file_path(entry_point).unwrap());
+ self
+ }
+
+ pub async fn build(&mut self) -> Result<VendorOutput, AnyError> {
+ let graph = self.build_graph().await;
+ let output_dir = make_path("/vendor");
+ let environment = TestVendorEnvironment::default();
+ super::build::build(&graph, &output_dir, &environment)?;
+ let mut files = environment.files.borrow_mut();
+ let import_map = files.remove(&output_dir.join("import_map.json"));
+ let mut files = files
+ .iter()
+ .map(|(path, text)| (path_to_string(path), text.clone()))
+ .collect::<Vec<_>>();
+
+ files.sort_by(|a, b| a.0.cmp(&b.0));
+
+ Ok(VendorOutput {
+ import_map: import_map.map(|text| serde_json::from_str(&text).unwrap()),
+ files,
+ })
+ }
+
+ pub fn with_loader(&mut self, action: impl Fn(&mut TestLoader)) -> &mut Self {
+ action(&mut self.loader);
+ self
+ }
+
+ async fn build_graph(&mut self) -> ModuleGraph {
+ let graph = deno_graph::create_graph(
+ self
+ .entry_points
+ .iter()
+ .map(|s| (s.to_owned(), deno_graph::ModuleKind::Esm))
+ .collect(),
+ false,
+ None,
+ &mut self.loader,
+ None,
+ None,
+ None,
+ None,
+ )
+ .await;
+ graph.lock().unwrap();
+ graph.valid().unwrap();
+ graph
+ }
+}
+
+fn make_path(text: &str) -> PathBuf {
+ // This should work all in memory. We're waiting on
+ // https://github.com/servo/rust-url/issues/730 to provide
+ // a cross platform path here
+ assert!(text.starts_with('/'));
+ if cfg!(windows) {
+ PathBuf::from(format!("C:{}", text.replace("/", "\\")))
+ } else {
+ PathBuf::from(text)
+ }
+}
+
+fn path_to_string(path: &Path) -> String {
+ // inverse of the function above
+ let path = path.to_string_lossy();
+ if cfg!(windows) {
+ path.replace("C:\\", "\\").replace('\\', "/")
+ } else {
+ path.to_string()
+ }
+}
generated by cgit v1.2.3 (git 2.25.1) at 2025-06-27 12:00:18 +0000