feat(compile): unstable npm and node specifier support (#19005)

This is the initial support for npm and node specifiers in `deno
compile`. The npm packages are included in the binary and read from it via
a virtual file system. This also supports the `--node-modules-dir` flag,
dependencies specified in a package.json, and npm binary commands (ex.
`deno compile --unstable npm:cowsay`)

Closes #16632

1 files changed, 47 insertions, 29 deletions

diff --git a/cli/module_loader.rs b/cli/module_loader.rs
index 0ed84a20f..5465ad1b8 100644
--- a/cli/module_loader.rs
+++ b/cli/module_loader.rs
@@ -46,11 +46,13 @@ use deno_graph::JsonModule;
 use deno_graph::Module;
 use deno_graph::Resolution;
 use deno_lockfile::Lockfile;
+use deno_runtime::deno_fs;
 use deno_runtime::deno_node;
 use deno_runtime::deno_node::NodeResolution;
 use deno_runtime::deno_node::NodeResolutionMode;
 use deno_runtime::deno_node::NodeResolver;
 use deno_runtime::permissions::PermissionsContainer;
+use deno_semver::npm::NpmPackageNvReference;
 use deno_semver::npm::NpmPackageReqReference;
 use std::borrow::Cow;
 use std::cell::RefCell;
@@ -417,13 +419,12 @@ impl CliModuleLoader {
     } else {
       &self.root_permissions
     };
-    let code_source = if let Some(code_source) =
-      self.shared.npm_module_loader.load_sync(
-        specifier,
-        maybe_referrer,
-        permissions,
-      )? {
-      code_source
+    let code_source = if let Some(result) = self
+      .shared
+      .npm_module_loader
+      .load_sync_if_in_npm_package(specifier, maybe_referrer, permissions)
+    {
+      result?
     } else {
       self
         .shared
@@ -494,7 +495,7 @@ impl ModuleLoader for CliModuleLoader {
             Some(Module::Npm(module)) => self
               .shared
               .npm_module_loader
-              .resolve_npm_module(module, permissions),
+              .resolve_nv_ref(&module.nv_reference, permissions),
             Some(Module::Node(module)) => {
               deno_node::resolve_builtin_node_module(&module.module_name)
             }
@@ -547,7 +548,7 @@ impl ModuleLoader for CliModuleLoader {
           return self
             .shared
             .npm_module_loader
-            .resolve_for_repl(&reference, permissions);
+            .resolve_req_reference(&reference, permissions);
         }
       }
     }
@@ -652,6 +653,7 @@ impl SourceMapGetter for CliSourceMapGetter {
 pub struct NpmModuleLoader {
   cjs_resolutions: Arc<CjsResolutionStore>,
   node_code_translator: Arc<CliNodeCodeTranslator>,
+  fs: Arc<dyn deno_fs::FileSystem>,
   node_resolver: Arc<NodeResolver>,
 }
 
@@ -659,11 +661,13 @@ impl NpmModuleLoader {
   pub fn new(
     cjs_resolutions: Arc<CjsResolutionStore>,
     node_code_translator: Arc<CliNodeCodeTranslator>,
+    fs: Arc<dyn deno_fs::FileSystem>,
     node_resolver: Arc<NodeResolver>,
   ) -> Self {
     Self {
       cjs_resolutions,
       node_code_translator,
+      fs,
       node_resolver,
     }
   }
@@ -693,21 +697,21 @@ impl NpmModuleLoader {
     }
   }
 
-  pub fn resolve_npm_module(
+  pub fn resolve_nv_ref(
     &self,
-    module: &deno_graph::NpmModule,
+    nv_ref: &NpmPackageNvReference,
     permissions: &PermissionsContainer,
   ) -> Result<ModuleSpecifier, AnyError> {
     self
       .handle_node_resolve_result(self.node_resolver.resolve_npm_reference(
-        &module.nv_reference,
+        nv_ref,
         NodeResolutionMode::Execution,
         permissions,
       ))
-      .with_context(|| format!("Could not resolve '{}'.", module.nv_reference))
+      .with_context(|| format!("Could not resolve '{}'.", nv_ref))
   }
 
-  pub fn resolve_for_repl(
+  pub fn resolve_req_reference(
     &self,
     reference: &NpmPackageReqReference,
     permissions: &PermissionsContainer,
@@ -733,25 +737,39 @@ impl NpmModuleLoader {
     }
   }
 
-  pub fn load_sync(
+  pub fn load_sync_if_in_npm_package(
     &self,
     specifier: &ModuleSpecifier,
     maybe_referrer: Option<&ModuleSpecifier>,
     permissions: &PermissionsContainer,
-  ) -> Result<Option<ModuleCodeSource>, AnyError> {
-    if !self.node_resolver.in_npm_package(specifier) {
-      return Ok(None);
+  ) -> Option<Result<ModuleCodeSource, AnyError>> {
+    if self.node_resolver.in_npm_package(specifier) {
+      Some(self.load_sync(specifier, maybe_referrer, permissions))
+    } else {
+      None
     }
+  }
+
+  fn load_sync(
+    &self,
+    specifier: &ModuleSpecifier,
+    maybe_referrer: Option<&ModuleSpecifier>,
+    permissions: &PermissionsContainer,
+  ) -> Result<ModuleCodeSource, AnyError> {
     let file_path = specifier.to_file_path().unwrap();
-    let code = std::fs::read_to_string(&file_path).with_context(|| {
-      let mut msg = "Unable to load ".to_string();
-      msg.push_str(&file_path.to_string_lossy());
-      if let Some(referrer) = &maybe_referrer {
-        msg.push_str(" imported from ");
-        msg.push_str(referrer.as_str());
-      }
-      msg
-    })?;
+    let code = self
+      .fs
+      .read_to_string(&file_path)
+      .map_err(AnyError::from)
+      .with_context(|| {
+        let mut msg = "Unable to load ".to_string();
+        msg.push_str(&file_path.to_string_lossy());
+        if let Some(referrer) = &maybe_referrer {
+          msg.push_str(" imported from ");
+          msg.push_str(referrer.as_str());
+        }
+        msg
+      })?;
 
     let code = if self.cjs_resolutions.contains(specifier) {
       // translate cjs to esm if it's cjs and inject node globals
@@ -766,11 +784,11 @@ impl NpmModuleLoader {
         .node_code_translator
         .esm_code_with_node_globals(specifier, &code)?
     };
-    Ok(Some(ModuleCodeSource {
+    Ok(ModuleCodeSource {
       code: code.into(),
       found_url: specifier.clone(),
       media_type: MediaType::from_specifier(specifier),
-    }))
+    })
   }
 
   fn handle_node_resolve_result(