feat(compile): support "bring your own node_modules" in deno compile (#21377)

Not tested thoroughly. This is a good start.

Closes #21350

3 files changed, 178 insertions, 84 deletions

diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs
index 38ef3648e..d1a5863ee 100644
--- a/cli/standalone/binary.rs
+++ b/cli/standalone/binary.rs
@@ -123,6 +123,18 @@ impl SerializablePackageJsonDeps {
 }
 
 #[derive(Deserialize, Serialize)]
+pub enum NodeModules {
+  Managed {
+    /// Whether this uses a node_modules directory (true) or the global cache (false).
+    node_modules_dir: bool,
+    package_json_deps: Option<SerializablePackageJsonDeps>,
+  },
+  Byonm {
+    package_json_deps: Option<SerializablePackageJsonDeps>,
+  },
+}
+
+#[derive(Deserialize, Serialize)]
 pub struct Metadata {
   pub argv: Vec<String>,
   pub unstable: bool,
@@ -136,9 +148,7 @@ pub struct Metadata {
   pub unsafely_ignore_certificate_errors: Option<Vec<String>>,
   pub maybe_import_map: Option<(Url, String)>,
   pub entrypoint: ModuleSpecifier,
-  /// Whether this uses a node_modules directory (true) or the global cache (false).
-  pub node_modules_dir: bool,
-  pub package_json_deps: Option<SerializablePackageJsonDeps>,
+  pub node_modules: Option<NodeModules>,
 }
 
 pub fn load_npm_vfs(root_dir_path: PathBuf) -> Result<FileBackedVfs, AnyError> {
@@ -490,23 +500,44 @@ impl<'a> DenoCompileBinaryWriter<'a> {
       .resolve_import_map(self.file_fetcher)
       .await?
       .map(|import_map| (import_map.base_url().clone(), import_map.to_json()));
-    let (npm_vfs, npm_files) = match self.npm_resolver.as_inner() {
-      InnerCliNpmResolverRef::Managed(managed) => {
-        let snapshot =
-          managed.serialized_valid_snapshot_for_system(&self.npm_system_info);
-        if !snapshot.as_serialized().packages.is_empty() {
+    let (npm_vfs, npm_files, node_modules) =
+      match self.npm_resolver.as_inner() {
+        InnerCliNpmResolverRef::Managed(managed) => {
+          let snapshot =
+            managed.serialized_valid_snapshot_for_system(&self.npm_system_info);
+          if !snapshot.as_serialized().packages.is_empty() {
+            let (root_dir, files) = self.build_vfs()?.into_dir_and_files();
+            eszip.add_npm_snapshot(snapshot);
+            (
+              Some(root_dir),
+              files,
+              Some(NodeModules::Managed {
+                node_modules_dir: self
+                  .npm_resolver
+                  .root_node_modules_path()
+                  .is_some(),
+                package_json_deps: self.package_json_deps_provider.deps().map(
+                  |deps| SerializablePackageJsonDeps::from_deps(deps.clone()),
+                ),
+              }),
+            )
+          } else {
+            (None, Vec::new(), None)
+          }
+        }
+        InnerCliNpmResolverRef::Byonm(_) => {
           let (root_dir, files) = self.build_vfs()?.into_dir_and_files();
-          eszip.add_npm_snapshot(snapshot);
-          (Some(root_dir), files)
-        } else {
-          (None, Vec::new())
+          (
+            Some(root_dir),
+            files,
+            Some(NodeModules::Byonm {
+              package_json_deps: self.package_json_deps_provider.deps().map(
+                |deps| SerializablePackageJsonDeps::from_deps(deps.clone()),
+              ),
+            }),
+          )
         }
-      }
-      InnerCliNpmResolverRef::Byonm(_) => {
-        let (root_dir, files) = self.build_vfs()?.into_dir_and_files();
-        (Some(root_dir), files)
-      }
-    };
+      };
 
     let metadata = Metadata {
       argv: compile_flags.args.clone(),
@@ -523,11 +554,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
       ca_data,
       entrypoint: entrypoint.clone(),
       maybe_import_map,
-      node_modules_dir: self.npm_resolver.root_node_modules_path().is_some(),
-      package_json_deps: self
-        .package_json_deps_provider
-        .deps()
-        .map(|deps| SerializablePackageJsonDeps::from_deps(deps.clone())),
+      node_modules,
     };
 
     write_binary_bytes(
@@ -545,7 +572,7 @@ impl<'a> DenoCompileBinaryWriter<'a> {
       InnerCliNpmResolverRef::Managed(npm_resolver) => {
         if let Some(node_modules_path) = npm_resolver.root_node_modules_path() {
           let mut builder = VfsBuilder::new(node_modules_path.clone())?;
-          builder.add_dir_recursive(&node_modules_path)?;
+          builder.add_dir_recursive(node_modules_path)?;
           Ok(builder)
         } else {
           // DO NOT include the user's registry url as it may contain credentials,
@@ -565,9 +592,19 @@ impl<'a> DenoCompileBinaryWriter<'a> {
           Ok(builder)
         }
       }
-      InnerCliNpmResolverRef::Byonm(_) => {
-        // todo(#18967): should use the node_modules directory
-        todo!()
+      InnerCliNpmResolverRef::Byonm(npm_resolver) => {
+        // the root_node_modules directory will always exist for byonm
+        let node_modules_path = npm_resolver.root_node_modules_path().unwrap();
+        let parent_path = node_modules_path.parent().unwrap();
+        let mut builder = VfsBuilder::new(parent_path.to_path_buf())?;
+        let package_json_path = parent_path.join("package.json");
+        if package_json_path.exists() {
+          builder.add_file_at_path(&package_json_path)?;
+        }
+        if node_modules_path.exists() {
+          builder.add_dir_recursive(node_modules_path)?;
+        }
+        Ok(builder)
       }
     }
   }
diff --git a/cli/standalone/mod.rs b/cli/standalone/mod.rs
index 13123a8d6..a6bf12862 100644
--- a/cli/standalone/mod.rs
+++ b/cli/standalone/mod.rs
@@ -16,6 +16,7 @@ use crate::module_loader::CliNodeResolver;
 use crate::module_loader::NpmModuleLoader;
 use crate::node::CliCjsCodeAnalyzer;
 use crate::npm::create_cli_npm_resolver;
+use crate::npm::CliNpmResolverByonmCreateOptions;
 use crate::npm::CliNpmResolverCreateOptions;
 use crate::npm::CliNpmResolverManagedCreateOptions;
 use crate::npm::CliNpmResolverManagedPackageJsonInstallerOption;
@@ -311,61 +312,113 @@ pub async fn run(
     .join("node_modules");
   let npm_cache_dir = NpmCacheDir::new(root_path.clone());
   let npm_global_cache_dir = npm_cache_dir.get_cache_location();
-  let (fs, vfs_root, maybe_node_modules_path, maybe_snapshot) =
-    if let Some(snapshot) = eszip.take_npm_snapshot() {
-      let vfs_root_dir_path = if metadata.node_modules_dir {
-        root_path
-      } else {
-        npm_cache_dir.registry_folder(&npm_registry_url)
-      };
-      let vfs = load_npm_vfs(vfs_root_dir_path.clone())
-        .context("Failed to load npm vfs.")?;
-      let node_modules_path = if metadata.node_modules_dir {
-        Some(vfs.root().to_path_buf())
-      } else {
-        None
-      };
-      (
-        Arc::new(DenoCompileFileSystem::new(vfs))
-          as Arc<dyn deno_fs::FileSystem>,
-        Some(vfs_root_dir_path),
-        node_modules_path,
-        Some(snapshot),
-      )
-    } else {
-      (
-        Arc::new(deno_fs::RealFs) as Arc<dyn deno_fs::FileSystem>,
-        None,
-        None,
-        None,
-      )
+  let cache_setting = CacheSetting::Only;
+  let (package_json_deps_provider, fs, npm_resolver, maybe_vfs_root) =
+    match metadata.node_modules {
+      Some(binary::NodeModules::Managed {
+        node_modules_dir,
+        package_json_deps,
+      }) => {
+        // this will always have a snapshot
+        let snapshot = eszip.take_npm_snapshot().unwrap();
+        let vfs_root_dir_path = if node_modules_dir {
+          root_path
+        } else {
+          npm_cache_dir.registry_folder(&npm_registry_url)
+        };
+        let vfs = load_npm_vfs(vfs_root_dir_path.clone())
+          .context("Failed to load npm vfs.")?;
+        let maybe_node_modules_path = if node_modules_dir {
+          Some(vfs.root().to_path_buf())
+        } else {
+          None
+        };
+        let package_json_deps_provider =
+          Arc::new(PackageJsonDepsProvider::new(
+            package_json_deps.map(|serialized| serialized.into_deps()),
+          ));
+        let fs = Arc::new(DenoCompileFileSystem::new(vfs))
+          as Arc<dyn deno_fs::FileSystem>;
+        let npm_resolver = create_cli_npm_resolver(
+          CliNpmResolverCreateOptions::Managed(CliNpmResolverManagedCreateOptions {
+            snapshot: CliNpmResolverManagedSnapshotOption::Specified(Some(snapshot)),
+            maybe_lockfile: None,
+            fs: fs.clone(),
+            http_client: http_client.clone(),
+            npm_global_cache_dir,
+            cache_setting,
+            text_only_progress_bar: progress_bar,
+            maybe_node_modules_path,
+            package_json_installer:
+              CliNpmResolverManagedPackageJsonInstallerOption::ConditionalInstall(
+                package_json_deps_provider.clone(),
+              ),
+            npm_registry_url,
+            npm_system_info: Default::default(),
+          }),
+        )
+        .await?;
+        (
+          package_json_deps_provider,
+          fs,
+          npm_resolver,
+          Some(vfs_root_dir_path),
+        )
+      }
+      Some(binary::NodeModules::Byonm { package_json_deps }) => {
+        let vfs_root_dir_path = root_path;
+        let vfs = load_npm_vfs(vfs_root_dir_path.clone())
+          .context("Failed to load npm vfs.")?;
+        let node_modules_path = vfs.root().join("node_modules");
+        let package_json_deps_provider =
+          Arc::new(PackageJsonDepsProvider::new(
+            package_json_deps.map(|serialized| serialized.into_deps()),
+          ));
+        let fs = Arc::new(DenoCompileFileSystem::new(vfs))
+          as Arc<dyn deno_fs::FileSystem>;
+        let npm_resolver =
+          create_cli_npm_resolver(CliNpmResolverCreateOptions::Byonm(
+            CliNpmResolverByonmCreateOptions {
+              fs: fs.clone(),
+              root_node_modules_dir: node_modules_path,
+            },
+          ))
+          .await?;
+        (
+          package_json_deps_provider,
+          fs,
+          npm_resolver,
+          Some(vfs_root_dir_path),
+        )
+      }
+      None => {
+        let package_json_deps_provider =
+          Arc::new(PackageJsonDepsProvider::new(None));
+        let fs = Arc::new(deno_fs::RealFs) as Arc<dyn deno_fs::FileSystem>;
+        let npm_resolver = create_cli_npm_resolver(
+          CliNpmResolverCreateOptions::Managed(CliNpmResolverManagedCreateOptions {
+            snapshot: CliNpmResolverManagedSnapshotOption::Specified(None),
+            maybe_lockfile: None,
+            fs: fs.clone(),
+            http_client: http_client.clone(),
+            npm_global_cache_dir,
+            cache_setting,
+            text_only_progress_bar: progress_bar,
+            maybe_node_modules_path: None,
+            package_json_installer:
+              CliNpmResolverManagedPackageJsonInstallerOption::ConditionalInstall(
+                package_json_deps_provider.clone(),
+              ),
+            npm_registry_url,
+            npm_system_info: Default::default(),
+          }),
+        )
+        .await?;
+        (package_json_deps_provider, fs, npm_resolver, None)
+      }
     };
 
-  let has_node_modules_dir = maybe_node_modules_path.is_some();
-  let package_json_deps_provider = Arc::new(PackageJsonDepsProvider::new(
-    metadata
-      .package_json_deps
-      .map(|serialized| serialized.into_deps()),
-  ));
-  let npm_resolver = create_cli_npm_resolver(
-    CliNpmResolverCreateOptions::Managed(CliNpmResolverManagedCreateOptions {
-      snapshot: CliNpmResolverManagedSnapshotOption::Specified(maybe_snapshot),
-      maybe_lockfile: None,
-      fs: fs.clone(),
-      http_client: http_client.clone(),
-      npm_global_cache_dir,
-      cache_setting: CacheSetting::Only,
-      text_only_progress_bar: progress_bar,
-      maybe_node_modules_path,
-      package_json_installer:
-        CliNpmResolverManagedPackageJsonInstallerOption::ConditionalInstall(
-          package_json_deps_provider.clone(),
-        ),
-      npm_registry_url,
-      npm_system_info: Default::default(),
-    }),
-  )
-  .await?;
+  let has_node_modules_dir = npm_resolver.root_node_modules_path().is_some();
   let node_resolver = Arc::new(NodeResolver::new(
     fs.clone(),
     npm_resolver.clone().into_npm_resolver(),
@@ -409,7 +462,7 @@ pub async fn run(
   let permissions = {
     let mut permissions = metadata.permissions;
     // if running with an npm vfs, grant read access to it
-    if let Some(vfs_root) = vfs_root {
+    if let Some(vfs_root) = maybe_vfs_root {
       match &mut permissions.allow_read {
         Some(vec) if vec.is_empty() => {
           // do nothing, already granted
diff --git a/cli/standalone/virtual_fs.rs b/cli/standalone/virtual_fs.rs
index c96aed143..ee870611b 100644
--- a/cli/standalone/virtual_fs.rs
+++ b/cli/standalone/virtual_fs.rs
@@ -92,9 +92,7 @@ impl VfsBuilder {
       if file_type.is_dir() {
         self.add_dir_recursive_internal(&path)?;
       } else if file_type.is_file() {
-        let file_bytes = std::fs::read(&path)
-          .with_context(|| format!("Reading {}", path.display()))?;
-        self.add_file(&path, file_bytes)?;
+        self.add_file_at_path(&path)?;
       } else if file_type.is_symlink() {
         let target = util::fs::canonicalize_path(&path)
           .with_context(|| format!("Reading symlink {}", path.display()))?;
@@ -163,6 +161,12 @@ impl VfsBuilder {
     Ok(current_dir)
   }
 
+  pub fn add_file_at_path(&mut self, path: &Path) -> Result<(), AnyError> {
+    let file_bytes = std::fs::read(path)
+      .with_context(|| format!("Reading {}", path.display()))?;
+    self.add_file(path, file_bytes)
+  }
+
   fn add_file(&mut self, path: &Path, data: Vec<u8>) -> Result<(), AnyError> {
     log::debug!("Adding file '{}'", path.display());
     let checksum = util::checksum::gen(&[&data]);