refactor: use deno_graph for npm specifiers (#17858)

This changes npm specifiers to be handled by deno_graph and resolved to
an npm package name and version when the specifier is encountered. It
also slightly changes how npm specifier resolution occurs—previously it
would collect all the npm specifiers and resolve them all at once, but
now it resolves them on the fly as they are encountered in the module
graph.

https://github.com/denoland/deno_graph/pull/232

---------

Co-authored-by: Bartek Iwańczuk <biwanczuk@gmail.com>

4 files changed, 77 insertions, 65 deletions

diff --git a/cli/tools/vendor/build.rs b/cli/tools/vendor/build.rs
index 89130f391..3bee843fd 100644
--- a/cli/tools/vendor/build.rs
+++ b/cli/tools/vendor/build.rs
@@ -10,9 +10,9 @@ use deno_core::anyhow::bail;
 use deno_core::anyhow::Context;
 use deno_core::error::AnyError;
 use deno_core::parking_lot::Mutex;
+use deno_graph::EsmModule;
 use deno_graph::Module;
 use deno_graph::ModuleGraph;
-use deno_graph::ModuleKind;
 use import_map::ImportMap;
 use import_map::SpecifierMap;
 
@@ -93,7 +93,7 @@ pub fn build(
   let all_modules = graph.modules().collect::<Vec<_>>();
   let remote_modules = all_modules
     .iter()
-    .filter(|m| is_remote_specifier(&m.specifier))
+    .filter(|m| is_remote_specifier(m.specifier()))
     .copied()
     .collect::<Vec<_>>();
   let mappings =
@@ -101,21 +101,16 @@ pub fn build(
 
   // write out all the files
   for module in &remote_modules {
-    let source = match &module.maybe_source {
-      Some(source) => source,
-      None => continue,
+    let source = match module {
+      Module::Esm(module) => &module.source,
+      Module::Json(module) => &module.source,
+      Module::Node(_) | Module::Npm(_) | Module::External(_) => continue,
     };
+    let specifier = module.specifier();
     let local_path = mappings
-      .proxied_path(&module.specifier)
-      .unwrap_or_else(|| mappings.local_path(&module.specifier));
-    if !matches!(module.kind, ModuleKind::Esm | ModuleKind::Asserted) {
-      log::warn!(
-        "Unsupported module kind {:?} for {}",
-        module.kind,
-        module.specifier
-      );
-      continue;
-    }
+      .proxied_path(specifier)
+      .unwrap_or_else(|| mappings.local_path(specifier));
+
     environment.create_dir_all(local_path.parent().unwrap())?;
     environment.write_file(&local_path, source)?;
   }
@@ -123,7 +118,7 @@ pub fn build(
   // write out the proxies
   for (specifier, proxied_module) in mappings.proxied_modules() {
     let proxy_path = mappings.local_path(specifier);
-    let module = graph.get(specifier).unwrap();
+    let module = graph.get(specifier).unwrap().esm().unwrap();
     let text =
       build_proxy_module_source(module, proxied_module, parsed_source_cache)?;
 
@@ -185,7 +180,7 @@ fn validate_original_import_map(
 }
 
 fn build_proxy_module_source(
-  module: &Module,
+  module: &EsmModule,
   proxied_module: &ProxiedModule,
   parsed_source_cache: &ParsedSourceCache,
 ) -> Result<String, AnyError> {
@@ -211,13 +206,11 @@ fn build_proxy_module_source(
   writeln!(text, "export * from \"{relative_specifier}\";").unwrap();
 
   // add a default export if one exists in the module
-  if let Some(parsed_source) =
-    parsed_source_cache.get_parsed_source_from_module(module)?
-  {
-    if has_default_export(&parsed_source) {
-      writeln!(text, "export {{ default }} from \"{relative_specifier}\";")
-        .unwrap();
-    }
+  let parsed_source =
+    parsed_source_cache.get_parsed_source_from_esm_module(module)?;
+  if has_default_export(&parsed_source) {
+    writeln!(text, "export {{ default }} from \"{relative_specifier}\";")
+      .unwrap();
   }
 
   Ok(text)
diff --git a/cli/tools/vendor/import_map.rs b/cli/tools/vendor/import_map.rs
index 3d2c1efd9..916eb55c5 100644
--- a/cli/tools/vendor/import_map.rs
+++ b/cli/tools/vendor/import_map.rs
@@ -4,7 +4,6 @@ use deno_ast::LineAndColumnIndex;
 use deno_ast::ModuleSpecifier;
 use deno_ast::SourceTextInfo;
 use deno_core::error::AnyError;
-use deno_graph::MediaType;
 use deno_graph::Module;
 use deno_graph::ModuleGraph;
 use deno_graph::Position;
@@ -205,21 +204,20 @@ fn visit_modules(
   parsed_source_cache: &ParsedSourceCache,
 ) -> Result<(), AnyError> {
   for module in modules {
-    if module.media_type == MediaType::Json {
+    let module = match module {
+      Module::Esm(module) => module,
       // skip visiting Json modules as they are leaves
-      continue;
-    }
-
-    let text_info =
-      match parsed_source_cache.get_parsed_source_from_module(module)? {
-        Some(source) => source.text_info().clone(),
-        None => continue,
-      };
-    let source_text = match &module.maybe_source {
-      Some(source) => source,
-      None => continue,
+      Module::Json(_)
+      | Module::Npm(_)
+      | Module::Node(_)
+      | Module::External(_) => continue,
     };
 
+    let parsed_source =
+      parsed_source_cache.get_parsed_source_from_esm_module(module)?;
+    let text_info = parsed_source.text_info().clone();
+    let source_text = &module.source;
+
     for dep in module.dependencies.values() {
       visit_resolution(
         &dep.maybe_code,
@@ -291,7 +289,7 @@ fn handle_dep_specifier(
   mappings: &Mappings,
 ) {
   let specifier = match graph.get(unresolved_specifier) {
-    Some(module) => module.specifier.clone(),
+    Some(module) => module.specifier().clone(),
     // Ignore when None. The graph was previous validated so this is a
     // dynamic import that was missing and is ignored for vendoring
     None => return,
diff --git a/cli/tools/vendor/mappings.rs b/cli/tools/vendor/mappings.rs
index 399002ea3..1ecc14edf 100644
--- a/cli/tools/vendor/mappings.rs
+++ b/cli/tools/vendor/mappings.rs
@@ -39,8 +39,9 @@ impl Mappings {
     remote_modules: &[&Module],
     output_dir: &Path,
   ) -> Result<Self, AnyError> {
-    let partitioned_specifiers =
-      partition_by_root_specifiers(remote_modules.iter().map(|m| &m.specifier));
+    let partitioned_specifiers = partition_by_root_specifiers(
+      remote_modules.iter().map(|m| m.specifier()),
+    );
     let mut mapped_paths = HashSet::new();
     let mut mappings = HashMap::new();
     let mut proxies = HashMap::new();
@@ -52,7 +53,12 @@ impl Mappings {
         &mut mapped_paths,
       );
       for specifier in specifiers {
-        let media_type = graph.get(&specifier).unwrap().media_type;
+        let module = graph.get(&specifier).unwrap();
+        let media_type = match module {
+          Module::Esm(module) => module.media_type,
+          Module::Json(_) => MediaType::Json,
+          Module::Node(_) | Module::Npm(_) | Module::External(_) => continue,
+        };
         let sub_path = sanitize_filepath(&make_url_relative(&root, &{
           let mut specifier = specifier.clone();
           specifier.set_query(None);
@@ -75,28 +81,30 @@ impl Mappings {
 
     // resolve all the "proxy" paths to use for when an x-typescript-types header is specified
     for module in remote_modules {
-      if let Some(resolved) = &module
-        .maybe_types_dependency
-        .as_ref()
-        .and_then(|d| d.dependency.ok())
-      {
-        let range = &resolved.range;
-        // hack to tell if it's an x-typescript-types header
-        let is_ts_types_header =
-          range.start == Position::zeroed() && range.end == Position::zeroed();
-        if is_ts_types_header {
-          let module_path = mappings.get(&module.specifier).unwrap();
-          let proxied_path = get_unique_path(
-            path_with_stem_suffix(module_path, ".proxied"),
-            &mut mapped_paths,
-          );
-          proxies.insert(
-            module.specifier.clone(),
-            ProxiedModule {
-              output_path: proxied_path,
-              declaration_specifier: resolved.specifier.clone(),
-            },
-          );
+      if let Some(module) = module.esm() {
+        if let Some(resolved) = &module
+          .maybe_types_dependency
+          .as_ref()
+          .and_then(|d| d.dependency.ok())
+        {
+          let range = &resolved.range;
+          // hack to tell if it's an x-typescript-types header
+          let is_ts_types_header = range.start == Position::zeroed()
+            && range.end == Position::zeroed();
+          if is_ts_types_header {
+            let module_path = mappings.get(&module.specifier).unwrap();
+            let proxied_path = get_unique_path(
+              path_with_stem_suffix(module_path, ".proxied"),
+              &mut mapped_paths,
+            );
+            proxies.insert(
+              module.specifier.clone(),
+              ProxiedModule {
+                output_path: proxied_path,
+                declaration_specifier: resolved.specifier.clone(),
+              },
+            );
+          }
         }
       }
     }
diff --git a/cli/tools/vendor/test.rs b/cli/tools/vendor/test.rs
index 874b329da..aed2a852c 100644
--- a/cli/tools/vendor/test.rs
+++ b/cli/tools/vendor/test.rs
@@ -20,6 +20,8 @@ use deno_graph::ModuleGraph;
 use import_map::ImportMap;
 
 use crate::cache::ParsedSourceCache;
+use crate::npm::NpmRegistryApi;
+use crate::npm::NpmResolution;
 use crate::resolver::CliGraphResolver;
 
 use super::build::VendorEnvironment;
@@ -260,8 +262,19 @@ async fn build_test_graph(
   mut loader: TestLoader,
   analyzer: &dyn deno_graph::ModuleAnalyzer,
 ) -> ModuleGraph {
-  let resolver = original_import_map
-    .map(|m| CliGraphResolver::new(None, Some(Arc::new(m)), None));
+  let resolver = original_import_map.map(|m| {
+    let npm_registry_api = NpmRegistryApi::new_uninitialized();
+    let npm_resolution =
+      NpmResolution::new(npm_registry_api.clone(), None, None);
+    CliGraphResolver::new(
+      None,
+      Some(Arc::new(m)),
+      false,
+      npm_registry_api,
+      npm_resolution,
+      None,
+    )
+  });
   let mut graph = ModuleGraph::default();
   graph
     .build(