feat(unstable): ability to `npm install` then `deno run main.ts` (#20967)

This PR adds a new unstable "bring your own node_modules" (BYONM)
functionality currently behind a `--unstable-byonm` flag (`"unstable":
["byonm"]` in a deno.json).

This enables users to run a separate install command (ex. `npm install`,
`pnpm install`) then run `deno run main.ts` and Deno will respect the
layout of the node_modules directory as setup by the separate install
command. It also works with npm/yarn/pnpm workspaces.

For this PR, the behaviour is opted into by specifying
`--unstable-byonm`/`"unstable": ["byonm"]`, but in the future we may
make this the default behaviour as outlined in
https://github.com/denoland/deno/issues/18967#issuecomment-1761248941

This is an extremely rough initial implementation. Errors are
terrible in this and the LSP requires frequent restarts. Improvements
will be done in follow up PRs.

3 files changed, 75 insertions, 46 deletions

diff --git a/cli/lsp/analysis.rs b/cli/lsp/analysis.rs
index 6515e7dc0..f8ace060a 100644
--- a/cli/lsp/analysis.rs
+++ b/cli/lsp/analysis.rs
@@ -253,7 +253,7 @@ impl<'a> TsResponseImportMapper<'a> {
     let root_folder = self
       .npm_resolver
       .as_ref()
-      .and_then(|r| r.resolve_pkg_folder_from_specifier(specifier).ok())
+      .and_then(|r| r.resolve_package_folder_from_path(specifier).ok())
       .flatten()?;
     let package_json_path = root_folder.join("package.json");
     let package_json_text = std::fs::read_to_string(&package_json_path).ok()?;
diff --git a/cli/lsp/documents.rs b/cli/lsp/documents.rs
index 57957780b..e29ad785e 100644
--- a/cli/lsp/documents.rs
+++ b/cli/lsp/documents.rs
@@ -37,9 +37,11 @@ use deno_core::ModuleSpecifier;
 use deno_graph::source::ResolutionMode;
 use deno_graph::GraphImport;
 use deno_graph::Resolution;
+use deno_runtime::deno_fs::RealFs;
 use deno_runtime::deno_node;
 use deno_runtime::deno_node::NodeResolution;
 use deno_runtime::deno_node::NodeResolutionMode;
+use deno_runtime::deno_node::NodeResolver;
 use deno_runtime::deno_node::PackageJson;
 use deno_runtime::permissions::PermissionsContainer;
 use deno_semver::npm::NpmPackageReqReference;
@@ -899,6 +901,7 @@ pub struct UpdateDocumentConfigOptions<'a> {
   pub maybe_import_map: Option<Arc<import_map::ImportMap>>,
   pub maybe_config_file: Option<&'a ConfigFile>,
   pub maybe_package_json: Option<&'a PackageJson>,
+  pub node_resolver: Option<Arc<NodeResolver>>,
   pub npm_resolver: Option<Arc<dyn CliNpmResolver>>,
 }
 
@@ -955,16 +958,17 @@ impl Documents {
       file_system_docs: Default::default(),
       resolver_config_hash: 0,
       imports: Default::default(),
-      resolver: Arc::new(CliGraphResolver::new(
-        None,
-        Arc::new(PackageJsonDepsProvider::default()),
-        CliGraphResolverOptions {
-          maybe_jsx_import_source_config: None,
-          maybe_import_map: None,
-          maybe_vendor_dir: None,
-          bare_node_builtins_enabled: false,
-        },
-      )),
+      resolver: Arc::new(CliGraphResolver::new(CliGraphResolverOptions {
+        fs: Arc::new(RealFs),
+        node_resolver: None,
+        npm_resolver: None,
+        cjs_resolutions: None,
+        package_json_deps_provider: Arc::new(PackageJsonDepsProvider::default()),
+        maybe_jsx_import_source_config: None,
+        maybe_import_map: None,
+        maybe_vendor_dir: None,
+        bare_node_builtins_enabled: false,
+      })),
       npm_specifier_reqs: Default::default(),
       has_injected_types_node_package: false,
       specifier_resolver: Arc::new(SpecifierResolver::new(cache)),
@@ -1329,7 +1333,7 @@ impl Documents {
       if let Some(package_json_deps) = &maybe_package_json_deps {
         // We need to ensure the hashing is deterministic so explicitly type
         // this in order to catch if the type of package_json_deps ever changes
-        // from a sorted/deterministic IndexMap to something else.
+        // from a deterministic IndexMap to something else.
         let package_json_deps: &IndexMap<_, _> = *package_json_deps;
         for (key, value) in package_json_deps {
           hasher.write_hashable(key);
@@ -1364,27 +1368,28 @@ impl Documents {
     );
     let deps_provider =
       Arc::new(PackageJsonDepsProvider::new(maybe_package_json_deps));
-    self.resolver = Arc::new(CliGraphResolver::new(
-      options.npm_resolver,
-      deps_provider,
-      CliGraphResolverOptions {
-        maybe_jsx_import_source_config: maybe_jsx_config,
-        maybe_import_map: options.maybe_import_map,
-        maybe_vendor_dir: options
-          .maybe_config_file
-          .and_then(|c| c.vendor_dir_path())
-          .as_ref(),
-        bare_node_builtins_enabled: options
-          .maybe_config_file
-          .map(|config| {
-            config
-              .json
-              .unstable
-              .contains(&"bare-node-builtins".to_string())
-          })
-          .unwrap_or(false),
-      },
-    ));
+    self.resolver = Arc::new(CliGraphResolver::new(CliGraphResolverOptions {
+      fs: Arc::new(RealFs),
+      node_resolver: options.node_resolver,
+      npm_resolver: options.npm_resolver,
+      cjs_resolutions: None, // only used for runtime
+      package_json_deps_provider: deps_provider,
+      maybe_jsx_import_source_config: maybe_jsx_config,
+      maybe_import_map: options.maybe_import_map,
+      maybe_vendor_dir: options
+        .maybe_config_file
+        .and_then(|c| c.vendor_dir_path())
+        .as_ref(),
+      bare_node_builtins_enabled: options
+        .maybe_config_file
+        .map(|config| {
+          config
+            .json
+            .unstable
+            .contains(&"bare-node-builtins".to_string())
+        })
+        .unwrap_or(false),
+    }));
     self.imports = Arc::new(
       if let Some(Ok(imports)) =
         options.maybe_config_file.map(|cf| cf.to_maybe_imports())
@@ -2194,6 +2199,7 @@ console.log(b, "hello deno");
         maybe_import_map: Some(Arc::new(import_map)),
         maybe_config_file: None,
         maybe_package_json: None,
+        node_resolver: None,
         npm_resolver: None,
       });
 
@@ -2235,6 +2241,7 @@ console.log(b, "hello deno");
         maybe_import_map: Some(Arc::new(import_map)),
         maybe_config_file: None,
         maybe_package_json: None,
+        node_resolver: None,
         npm_resolver: None,
       });
 
diff --git a/cli/lsp/language_server.rs b/cli/lsp/language_server.rs
index 8afcc7ffc..8b275a650 100644
--- a/cli/lsp/language_server.rs
+++ b/cli/lsp/language_server.rs
@@ -105,6 +105,7 @@ use crate::lsp::tsc::file_text_changes_to_workspace_edit;
 use crate::lsp::urls::LspUrlKind;
 use crate::npm::create_cli_npm_resolver_for_lsp;
 use crate::npm::CliNpmResolver;
+use crate::npm::CliNpmResolverByonmCreateOptions;
 use crate::npm::CliNpmResolverCreateOptions;
 use crate::npm::CliNpmResolverManagedCreateOptions;
 use crate::npm::CliNpmResolverManagedPackageJsonInstallerOption;
@@ -131,6 +132,8 @@ struct LspNpmServices {
   config_hash: LspNpmConfigHash,
   /// Npm's search api.
   search_api: CliNpmSearchApi,
+  /// Node resolver.
+  node_resolver: Option<Arc<NodeResolver>>,
   /// Resolver for npm packages.
   resolver: Option<Arc<dyn CliNpmResolver>>,
 }
@@ -495,6 +498,7 @@ impl Inner {
       npm: LspNpmServices {
         config_hash: LspNpmConfigHash(0), // this will be updated in initialize
         search_api: npm_search_api,
+        node_resolver: None,
         resolver: None,
       },
       performance,
@@ -815,15 +819,19 @@ impl Inner {
       return; // no need to do anything
     }
 
-    self.npm.resolver = Some(
-      create_npm_resolver(
-        &deno_dir,
-        &self.http_client,
-        self.config.maybe_lockfile(),
-        self.config.maybe_node_modules_dir_path().cloned(),
-      )
-      .await,
-    );
+    let npm_resolver = create_npm_resolver(
+      &deno_dir,
+      &self.http_client,
+      self.config.maybe_config_file(),
+      self.config.maybe_lockfile(),
+      self.config.maybe_node_modules_dir_path().cloned(),
+    )
+    .await;
+    self.npm.node_resolver = Some(Arc::new(NodeResolver::new(
+      Arc::new(deno_fs::RealFs),
+      npm_resolver.clone().into_npm_resolver(),
+    )));
+    self.npm.resolver = Some(npm_resolver);
 
     // update the hash
     self.npm.config_hash = config_hash;
@@ -1059,11 +1067,24 @@ impl Inner {
 async fn create_npm_resolver(
   deno_dir: &DenoDir,
   http_client: &Arc<HttpClient>,
+  maybe_config_file: Option<&ConfigFile>,
   maybe_lockfile: Option<&Arc<Mutex<Lockfile>>>,
   maybe_node_modules_dir_path: Option<PathBuf>,
 ) -> Arc<dyn CliNpmResolver> {
-  create_cli_npm_resolver_for_lsp(CliNpmResolverCreateOptions::Managed(
-    CliNpmResolverManagedCreateOptions {
+  let is_byonm = std::env::var("DENO_UNSTABLE_BYONM").as_deref() == Ok("1")
+    || maybe_config_file
+      .as_ref()
+      .map(|c| c.json.unstable.iter().any(|c| c == "byonm"))
+      .unwrap_or(false);
+  create_cli_npm_resolver_for_lsp(if is_byonm {
+    CliNpmResolverCreateOptions::Byonm(CliNpmResolverByonmCreateOptions {
+      fs: Arc::new(deno_fs::RealFs),
+      root_node_modules_dir: std::env::current_dir()
+        .unwrap()
+        .join("node_modules"),
+    })
+  } else {
+    CliNpmResolverCreateOptions::Managed(CliNpmResolverManagedCreateOptions {
       http_client: http_client.clone(),
       snapshot: match maybe_lockfile {
         Some(lockfile) => {
@@ -1090,8 +1111,8 @@ async fn create_npm_resolver(
         CliNpmResolverManagedPackageJsonInstallerOption::NoInstall,
       npm_registry_url: crate::args::npm_registry_default_url().to_owned(),
       npm_system_info: NpmSystemInfo::default(),
-    },
-  ))
+    })
+  })
   .await
 }
 
@@ -1250,6 +1271,7 @@ impl Inner {
       maybe_import_map: self.maybe_import_map.clone(),
       maybe_config_file: self.config.maybe_config_file(),
       maybe_package_json: self.maybe_package_json.as_ref(),
+      node_resolver: self.npm.node_resolver.clone(),
       npm_resolver: self.npm.resolver.clone(),
     });