feat: npm workspace and better Deno workspace support (#24334)

Adds much better support for the unstable Deno workspaces as well as
support for npm workspaces. npm workspaces is still lacking in that we
only install packages into the root node_modules folder. We'll make it
smarter over time in order for it to figure out when to add node_modules
folders within packages.

This includes a breaking change in config file resolution where we stop
searching for config files on the first found package.json unless it's
in a workspace. For the previous behaviour, the root deno.json needs to
be updated to be a workspace by adding `"workspace":
["./path-to-pkg-json-folder-goes-here"]`. See details in
https://github.com/denoland/deno_config/pull/66

Closes #24340
Closes #24159
Closes #24161
Closes #22020
Closes #18546
Closes #16106
Closes #24160

6 files changed, 58 insertions, 44 deletions

diff --git a/cli/lsp/config.rs b/cli/lsp/config.rs
index e1f3e3207..4b96511c0 100644
--- a/cli/lsp/config.rs
+++ b/cli/lsp/config.rs
@@ -16,7 +16,6 @@ use crate::util::fs::canonicalize_path_maybe_not_exists;
 use deno_ast::MediaType;
 use deno_config::FmtOptionsConfig;
 use deno_config::TsConfig;
-use deno_core::anyhow::anyhow;
 use deno_core::normalize_path;
 use deno_core::serde::de::DeserializeOwned;
 use deno_core::serde::Deserialize;
@@ -27,6 +26,8 @@ use deno_core::serde_json::Value;
 use deno_core::ModuleSpecifier;
 use deno_lint::linter::LintConfig;
 use deno_npm::npm_rc::ResolvedNpmRc;
+use deno_runtime::deno_fs::DenoConfigFsAdapter;
+use deno_runtime::deno_fs::RealFs;
 use deno_runtime::deno_node::PackageJson;
 use deno_runtime::deno_permissions::PermissionsContainer;
 use deno_runtime::fs_util::specifier_to_file_path;
@@ -935,7 +936,7 @@ impl Config {
   pub fn specifier_enabled(&self, specifier: &ModuleSpecifier) -> bool {
     let config_file = self.tree.config_file_for_specifier(specifier);
     if let Some(cf) = config_file {
-      if let Ok(files) = cf.to_files_config() {
+      if let Ok(files) = cf.to_exclude_files_config() {
         if !files.matches_specifier(specifier) {
           return false;
         }
@@ -952,7 +953,7 @@ impl Config {
     specifier: &ModuleSpecifier,
   ) -> bool {
     if let Some(cf) = self.tree.config_file_for_specifier(specifier) {
-      if let Some(options) = cf.to_test_config().ok().flatten() {
+      if let Ok(options) = cf.to_test_config() {
         if !options.files.matches_specifier(specifier) {
           return false;
         }
@@ -1135,8 +1136,9 @@ impl ConfigData {
   ) -> Self {
     if let Some(specifier) = config_file_specifier {
       match ConfigFile::from_specifier(
+        &DenoConfigFsAdapter::new(&RealFs),
         specifier.clone(),
-        &deno_config::ParseOptions::default(),
+        &deno_config::ConfigParseOptions::default(),
       ) {
         Ok(config_file) => {
           lsp_log!(
@@ -1230,13 +1232,7 @@ impl ConfigData {
         .and_then(|config_file| {
           config_file
             .to_fmt_config()
-            .and_then(|o| {
-              let base_path = config_file
-                .specifier
-                .to_file_path()
-                .map_err(|_| anyhow!("Invalid base path."))?;
-              FmtOptions::resolve(o, None, &base_path)
-            })
+            .and_then(|o| FmtOptions::resolve(o, &Default::default(), None))
             .inspect_err(|err| {
               lsp_warn!("  Couldn't read formatter configuration: {}", err)
             })
@@ -1264,13 +1260,7 @@ impl ConfigData {
         .and_then(|config_file| {
           config_file
             .to_lint_config()
-            .and_then(|o| {
-              let base_path = config_file
-                .specifier
-                .to_file_path()
-                .map_err(|_| anyhow!("Invalid base path."))?;
-              LintOptions::resolve(o, None, &base_path)
-            })
+            .and_then(|o| LintOptions::resolve(o, Default::default(), None))
             .inspect_err(|err| {
               lsp_warn!("  Couldn't read lint configuration: {}", err)
             })
@@ -2115,7 +2105,7 @@ mod tests {
         ConfigFile::new(
           "{}",
           root_uri.join("deno.json").unwrap(),
-          &deno_config::ParseOptions::default(),
+          &deno_config::ConfigParseOptions::default(),
         )
         .unwrap(),
       )
@@ -2173,7 +2163,7 @@ mod tests {
           })
           .to_string(),
           root_uri.join("deno.json").unwrap(),
-          &deno_config::ParseOptions::default(),
+          &deno_config::ConfigParseOptions::default(),
         )
         .unwrap(),
       )
@@ -2199,7 +2189,7 @@ mod tests {
           })
           .to_string(),
           root_uri.join("deno.json").unwrap(),
-          &deno_config::ParseOptions::default(),
+          &deno_config::ConfigParseOptions::default(),
         )
         .unwrap(),
       )
@@ -2217,7 +2207,7 @@ mod tests {
           })
           .to_string(),
           root_uri.join("deno.json").unwrap(),
-          &deno_config::ParseOptions::default(),
+          &deno_config::ConfigParseOptions::default(),
         )
         .unwrap(),
       )
diff --git a/cli/lsp/diagnostics.rs b/cli/lsp/diagnostics.rs
index 27983867a..9b500567d 100644
--- a/cli/lsp/diagnostics.rs
+++ b/cli/lsp/diagnostics.rs
@@ -1655,7 +1655,7 @@ mod tests {
       let config_file = ConfigFile::new(
         json_string,
         base_url,
-        &deno_config::ParseOptions::default(),
+        &deno_config::ConfigParseOptions::default(),
       )
       .unwrap();
       config.tree.inject_config_file(config_file).await;
diff --git a/cli/lsp/documents.rs b/cli/lsp/documents.rs
index 0d9cd4fbb..48cfebfcc 100644
--- a/cli/lsp/documents.rs
+++ b/cli/lsp/documents.rs
@@ -1751,7 +1751,7 @@ console.log(b, "hello deno");
             })
             .to_string(),
             config.root_uri().unwrap().join("deno.json").unwrap(),
-            &deno_config::ParseOptions::default(),
+            &deno_config::ConfigParseOptions::default(),
           )
           .unwrap(),
         )
@@ -1795,7 +1795,7 @@ console.log(b, "hello deno");
             })
             .to_string(),
             config.root_uri().unwrap().join("deno.json").unwrap(),
-            &deno_config::ParseOptions::default(),
+            &deno_config::ConfigParseOptions::default(),
           )
           .unwrap(),
         )
diff --git a/cli/lsp/language_server.rs b/cli/lsp/language_server.rs
index 25782b95c..cfc58439d 100644
--- a/cli/lsp/language_server.rs
+++ b/cli/lsp/language_server.rs
@@ -2,6 +2,8 @@
 
 use base64::Engine;
 use deno_ast::MediaType;
+use deno_config::workspace::Workspace;
+use deno_config::workspace::WorkspaceDiscoverOptions;
 use deno_core::anyhow::anyhow;
 use deno_core::error::AnyError;
 use deno_core::resolve_url;
@@ -13,6 +15,7 @@ use deno_core::url;
 use deno_core::ModuleSpecifier;
 use deno_graph::GraphKind;
 use deno_graph::Resolution;
+use deno_runtime::deno_fs::DenoConfigFsAdapter;
 use deno_runtime::deno_tls::rustls::RootCertStore;
 use deno_runtime::deno_tls::RootCertStoreProvider;
 use deno_semver::jsr::JsrPackageReqReference;
@@ -3549,6 +3552,24 @@ impl Inner {
     }
 
     let workspace_settings = self.config.workspace_settings();
+    let initial_cwd = config_data
+      .and_then(|d| d.scope.to_file_path().ok())
+      .unwrap_or_else(|| self.initial_cwd.clone());
+    // todo: we need a way to convert config data to a Workspace
+    let workspace = Arc::new(Workspace::discover(
+      deno_config::workspace::WorkspaceDiscoverStart::Dirs(&[
+        initial_cwd.clone()
+      ]),
+      &WorkspaceDiscoverOptions {
+        fs: &DenoConfigFsAdapter::new(&deno_runtime::deno_fs::RealFs),
+        pkg_json_cache: None,
+        config_parse_options: deno_config::ConfigParseOptions {
+          include_task_comments: false,
+        },
+        additional_config_file_names: &[],
+        discover_pkg_json: true,
+      },
+    )?);
     let cli_options = CliOptions::new(
       Flags {
         cache_path: Some(self.cache.deno_dir().root.clone()),
@@ -3572,13 +3593,12 @@ impl Inner {
         type_check_mode: crate::args::TypeCheckMode::Local,
         ..Default::default()
       },
-      self.initial_cwd.clone(),
-      config_data.and_then(|d| d.config_file.as_deref().cloned()),
+      initial_cwd,
       config_data.and_then(|d| d.lockfile.clone()),
-      config_data.and_then(|d| d.package_json.clone()),
       config_data
         .and_then(|d| d.npmrc.clone())
         .unwrap_or_else(create_default_npmrc),
+      workspace,
       force_global_cache,
     )?;
 
diff --git a/cli/lsp/resolver.rs b/cli/lsp/resolver.rs
index 5cf7f82b1..18d22afad 100644
--- a/cli/lsp/resolver.rs
+++ b/cli/lsp/resolver.rs
@@ -1,9 +1,9 @@
 // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
 
 use crate::args::create_default_npmrc;
-use crate::args::package_json;
 use crate::args::CacheSetting;
 use crate::args::CliLockfile;
+use crate::args::PackageJsonInstallDepsProvider;
 use crate::graph_util::CliJsrUrlProvider;
 use crate::http_util::HttpClientProvider;
 use crate::lsp::config::Config;
@@ -26,6 +26,8 @@ use crate::util::progress_bar::ProgressBarStyle;
 use dashmap::DashMap;
 use deno_ast::MediaType;
 use deno_cache_dir::HttpCache;
+use deno_config::workspace::PackageJsonDepResolution;
+use deno_config::workspace::WorkspaceResolver;
 use deno_core::error::AnyError;
 use deno_core::url::Url;
 use deno_graph::source::Resolver;
@@ -43,7 +45,6 @@ use deno_semver::npm::NpmPackageReqReference;
 use deno_semver::package::PackageNv;
 use deno_semver::package::PackageReq;
 use indexmap::IndexMap;
-use package_json::PackageJsonDepsProvider;
 use std::borrow::Cow;
 use std::collections::BTreeMap;
 use std::collections::BTreeSet;
@@ -460,13 +461,10 @@ async fn create_npm_resolver(
       text_only_progress_bar: ProgressBar::new(ProgressBarStyle::TextOnly),
       maybe_node_modules_path: config_data
         .and_then(|d| d.node_modules_dir.clone()),
-      package_json_deps_provider: Arc::new(PackageJsonDepsProvider::new(
-        config_data
-          .and_then(|d| d.package_json.as_ref())
-          .map(|package_json| {
-            package_json.resolve_local_package_json_version_reqs()
-          }),
-      )),
+      // only used for top level install, so we can ignore this
+      package_json_deps_provider: Arc::new(
+        PackageJsonInstallDepsProvider::empty(),
+      ),
       npmrc: config_data
         .and_then(|d| d.npmrc.clone())
         .unwrap_or_else(create_default_npmrc),
@@ -504,16 +502,22 @@ fn create_graph_resolver(
   Arc::new(CliGraphResolver::new(CliGraphResolverOptions {
     node_resolver: node_resolver.cloned(),
     npm_resolver: npm_resolver.cloned(),
-    package_json_deps_provider: Arc::new(PackageJsonDepsProvider::new(
+    workspace_resolver: Arc::new(WorkspaceResolver::new_raw(
+      config_data.and_then(|d| d.import_map.as_ref().map(|i| (**i).clone())),
       config_data
-        .and_then(|d| d.package_json.as_ref())
-        .map(|package_json| {
-          package_json.resolve_local_package_json_version_reqs()
-        }),
+        .and_then(|d| d.package_json.clone())
+        .into_iter()
+        .collect(),
+      if config_data.map(|d| d.byonm).unwrap_or(false) {
+        PackageJsonDepResolution::Disabled
+      } else {
+        // todo(dsherret): this should also be disabled for when using
+        // auto-install with a node_modules directory
+        PackageJsonDepResolution::Enabled
+      },
     )),
     maybe_jsx_import_source_config: config_file
       .and_then(|cf| cf.to_maybe_jsx_import_source_config().ok().flatten()),
-    maybe_import_map: config_data.and_then(|d| d.import_map.clone()),
     maybe_vendor_dir: config_data.and_then(|d| d.vendor_dir.as_ref()),
     bare_node_builtins_enabled: config_file
       .map(|cf| cf.has_unstable("bare-node-builtins"))
diff --git a/cli/lsp/tsc.rs b/cli/lsp/tsc.rs
index bab9766eb..cc88a0811 100644
--- a/cli/lsp/tsc.rs
+++ b/cli/lsp/tsc.rs
@@ -5405,7 +5405,7 @@ mod tests {
           })
           .to_string(),
           resolve_url("file:///deno.json").unwrap(),
-          &deno_config::ParseOptions::default(),
+          &deno_config::ConfigParseOptions::default(),
         )
         .unwrap(),
       )