feat: npm workspace and better Deno workspace support (#24334)

Adds much better support for the unstable Deno workspaces as well as
support for npm workspaces. npm workspaces is still lacking in that we
only install packages into the root node_modules folder. We'll make it
smarter over time in order for it to figure out when to add node_modules
folders within packages.

This includes a breaking change in config file resolution where we stop
searching for config files on the first found package.json unless it's
in a workspace. For the previous behaviour, the root deno.json needs to
be updated to be a workspace by adding `"workspace":
["./path-to-pkg-json-folder-goes-here"]`. See details in
https://github.com/denoland/deno_config/pull/66

Closes #24340
Closes #24159
Closes #24161
Closes #22020
Closes #18546
Closes #16106
Closes #24160

4 files changed, 125 insertions, 102 deletions

diff --git a/cli/tools/registry/mod.rs b/cli/tools/registry/mod.rs
index d300e5eaf..134a973f7 100644
--- a/cli/tools/registry/mod.rs
+++ b/cli/tools/registry/mod.rs
@@ -11,9 +11,8 @@ use std::sync::Arc;
 use base64::prelude::BASE64_STANDARD;
 use base64::Engine;
 use deno_ast::ModuleSpecifier;
-use deno_config::glob::FilePatterns;
-use deno_config::ConfigFile;
-use deno_config::WorkspaceMemberConfig;
+use deno_config::workspace::JsrPackageConfig;
+use deno_config::workspace::WorkspaceResolver;
 use deno_core::anyhow::bail;
 use deno_core::anyhow::Context;
 use deno_core::error::AnyError;
@@ -27,7 +26,6 @@ use deno_core::serde_json::Value;
 use deno_runtime::deno_fetch::reqwest;
 use deno_runtime::deno_fs::FileSystem;
 use deno_terminal::colors;
-use import_map::ImportMap;
 use lsp_types::Url;
 use serde::Deserialize;
 use serde::Serialize;
@@ -44,7 +42,6 @@ use crate::cache::ParsedSourceCache;
 use crate::factory::CliFactory;
 use crate::graph_util::ModuleGraphCreator;
 use crate::http_util::HttpClient;
-use crate::resolver::MappedSpecifierResolver;
 use crate::resolver::SloppyImportsResolver;
 use crate::tools::check::CheckOptions;
 use crate::tools::lint::no_slow_types;
@@ -84,27 +81,28 @@ pub async fn publish(
   let auth_method =
     get_auth_method(publish_flags.token, publish_flags.dry_run)?;
 
-  let import_map = cli_factory
-    .maybe_import_map()
-    .await?
-    .clone()
-    .unwrap_or_else(|| {
-      Arc::new(ImportMap::new(Url::parse("file:///dev/null").unwrap()))
-    });
+  let workspace_resolver = cli_factory.workspace_resolver().await?.clone();
 
   let directory_path = cli_factory.cli_options().initial_cwd();
-
-  let mapped_resolver = Arc::new(MappedSpecifierResolver::new(
-    Some(import_map),
-    cli_factory.package_json_deps_provider().clone(),
-  ));
   let cli_options = cli_factory.cli_options();
-  let Some(config_file) = cli_options.maybe_config_file() else {
-    bail!(
-      "Couldn't find a deno.json, deno.jsonc, jsr.json or jsr.jsonc configuration file in {}.",
-      directory_path.display()
-    );
-  };
+  let publish_configs = cli_options.workspace.jsr_packages_for_publish();
+  if publish_configs.is_empty() {
+    match cli_options.workspace.resolve_start_ctx().maybe_deno_json() {
+      Some(deno_json) => {
+        debug_assert!(!deno_json.is_package());
+        bail!(
+          "Missing 'name', 'version' and 'exports' field in '{}'.",
+          deno_json.specifier
+        );
+      }
+      None => {
+        bail!(
+          "Couldn't find a deno.json, deno.jsonc, jsr.json or jsr.jsonc configuration file in {}.",
+          directory_path.display()
+        );
+      }
+    }
+  }
 
   let diagnostics_collector = PublishDiagnosticsCollector::default();
   let publish_preparer = PublishPreparer::new(
@@ -114,14 +112,14 @@ pub async fn publish(
     cli_factory.type_checker().await?.clone(),
     cli_factory.fs().clone(),
     cli_factory.cli_options().clone(),
-    mapped_resolver,
+    workspace_resolver,
   );
 
   let prepared_data = publish_preparer
     .prepare_packages_for_publishing(
       publish_flags.allow_slow_types,
       &diagnostics_collector,
-      config_file.clone(),
+      publish_configs,
     )
     .await?;
 
@@ -193,8 +191,8 @@ struct PublishPreparer {
   source_cache: Arc<ParsedSourceCache>,
   type_checker: Arc<TypeChecker>,
   cli_options: Arc<CliOptions>,
-  mapped_resolver: Arc<MappedSpecifierResolver>,
   sloppy_imports_resolver: Option<Arc<SloppyImportsResolver>>,
+  workspace_resolver: Arc<WorkspaceResolver>,
 }
 
 impl PublishPreparer {
@@ -205,7 +203,7 @@ impl PublishPreparer {
     type_checker: Arc<TypeChecker>,
     fs: Arc<dyn FileSystem>,
     cli_options: Arc<CliOptions>,
-    mapped_resolver: Arc<MappedSpecifierResolver>,
+    workspace_resolver: Arc<WorkspaceResolver>,
   ) -> Self {
     let sloppy_imports_resolver = if cli_options.unstable_sloppy_imports() {
       Some(Arc::new(SloppyImportsResolver::new(fs.clone())))
@@ -218,8 +216,8 @@ impl PublishPreparer {
       source_cache,
       type_checker,
       cli_options,
-      mapped_resolver,
       sloppy_imports_resolver,
+      workspace_resolver,
     }
   }
 
@@ -227,11 +225,9 @@ impl PublishPreparer {
     &self,
     allow_slow_types: bool,
     diagnostics_collector: &PublishDiagnosticsCollector,
-    deno_json: ConfigFile,
+    publish_configs: Vec<JsrPackageConfig>,
   ) -> Result<PreparePackagesData, AnyError> {
-    let members = deno_json.to_workspace_members()?;
-
-    if members.len() > 1 {
+    if publish_configs.len() > 1 {
       log::info!("Publishing a workspace...");
     }
 
@@ -240,31 +236,24 @@ impl PublishPreparer {
       .build_and_check_graph_for_publish(
         allow_slow_types,
         diagnostics_collector,
-        &members,
+        &publish_configs,
       )
       .await?;
 
-    let mut package_by_name = HashMap::with_capacity(members.len());
+    let mut package_by_name = HashMap::with_capacity(publish_configs.len());
     let publish_order_graph =
-      publish_order::build_publish_order_graph(&graph, &members)?;
+      publish_order::build_publish_order_graph(&graph, &publish_configs)?;
 
-    let results = members
+    let results = publish_configs
       .into_iter()
       .map(|member| {
         let graph = graph.clone();
         async move {
           let package = self
-            .prepare_publish(
-              &member.package_name,
-              &member.config_file,
-              graph,
-              diagnostics_collector,
-            )
+            .prepare_publish(&member, graph, diagnostics_collector)
             .await
-            .with_context(|| {
-              format!("Failed preparing '{}'.", member.package_name)
-            })?;
-          Ok::<_, AnyError>((member.package_name, package))
+            .with_context(|| format!("Failed preparing '{}'.", member.name))?;
+          Ok::<_, AnyError>((member.name, package))
         }
         .boxed()
       })
@@ -284,12 +273,15 @@ impl PublishPreparer {
     &self,
     allow_slow_types: bool,
     diagnostics_collector: &PublishDiagnosticsCollector,
-    packages: &[WorkspaceMemberConfig],
+    package_configs: &[JsrPackageConfig],
   ) -> Result<Arc<deno_graph::ModuleGraph>, deno_core::anyhow::Error> {
     let build_fast_check_graph = !allow_slow_types;
     let graph = self
       .module_graph_creator
-      .create_and_validate_publish_graph(packages, build_fast_check_graph)
+      .create_and_validate_publish_graph(
+        package_configs,
+        build_fast_check_graph,
+      )
       .await?;
 
     // todo(dsherret): move to lint rule
@@ -335,7 +327,7 @@ impl PublishPreparer {
     } else {
       log::info!("Checking for slow types in the public API...");
       let mut any_pkg_had_diagnostics = false;
-      for package in packages {
+      for package in package_configs {
         let export_urls = package.config_file.resolve_export_value_urls()?;
         let diagnostics =
           no_slow_types::collect_no_slow_type_diagnostics(&export_urls, &graph);
@@ -389,14 +381,14 @@ impl PublishPreparer {
   #[allow(clippy::too_many_arguments)]
   async fn prepare_publish(
     &self,
-    package_name: &str,
-    deno_json: &ConfigFile,
+    package: &JsrPackageConfig,
     graph: Arc<deno_graph::ModuleGraph>,
     diagnostics_collector: &PublishDiagnosticsCollector,
   ) -> Result<Rc<PreparedPublishPackage>, AnyError> {
     static SUGGESTED_ENTRYPOINTS: [&str; 4] =
       ["mod.ts", "mod.js", "index.ts", "index.js"];
 
+    let deno_json = &package.config_file;
     let config_path = deno_json.specifier.to_file_path().unwrap();
     let root_dir = config_path.parent().unwrap().to_path_buf();
     let Some(version) = deno_json.json.version.clone() else {
@@ -418,32 +410,29 @@ impl PublishPreparer {
   "version": "{}",
   "exports": "{}"
 }}"#,
-        package_name,
+        package.name,
         version,
         suggested_entrypoint.unwrap_or("<path_to_entrypoint>")
       );
 
       bail!(
       "You did not specify an entrypoint to \"{}\" package in {}. Add `exports` mapping in the configuration file, eg:\n{}",
-      package_name,
+      package.name,
       deno_json.specifier,
       exports_content
     );
     }
-    let Some(name_no_at) = package_name.strip_prefix('@') else {
+    let Some(name_no_at) = package.name.strip_prefix('@') else {
       bail!("Invalid package name, use '@<scope_name>/<package_name> format");
     };
     let Some((scope, name_no_scope)) = name_no_at.split_once('/') else {
       bail!("Invalid package name, use '@<scope_name>/<package_name> format");
     };
-    let file_patterns = deno_json
-      .to_publish_config()?
-      .map(|c| c.files)
-      .unwrap_or_else(|| FilePatterns::new_with_base(root_dir.to_path_buf()));
+    let file_patterns = package.member_ctx.to_publish_config()?.files;
 
     let tarball = deno_core::unsync::spawn_blocking({
       let diagnostics_collector = diagnostics_collector.clone();
-      let mapped_resolver = self.mapped_resolver.clone();
+      let workspace_resolver = self.workspace_resolver.clone();
       let sloppy_imports_resolver = self.sloppy_imports_resolver.clone();
       let cli_options = self.cli_options.clone();
       let source_cache = self.source_cache.clone();
@@ -451,8 +440,8 @@ impl PublishPreparer {
       move || {
         let bare_node_builtins = cli_options.unstable_bare_node_builtins();
         let unfurler = SpecifierUnfurler::new(
-          &mapped_resolver,
           sloppy_imports_resolver.as_deref(),
+          &workspace_resolver,
           bare_node_builtins,
         );
         let root_specifier =
@@ -482,7 +471,7 @@ impl PublishPreparer {
     })
     .await??;
 
-    log::debug!("Tarball size ({}): {}", package_name, tarball.bytes.len());
+    log::debug!("Tarball size ({}): {}", package.name, tarball.bytes.len());
 
     Ok(Rc::new(PreparedPublishPackage {
       scope: scope.to_string(),
diff --git a/cli/tools/registry/pm.rs b/cli/tools/registry/pm.rs
index 4fdc02550..e3e2f1b55 100644
--- a/cli/tools/registry/pm.rs
+++ b/cli/tools/registry/pm.rs
@@ -49,7 +49,7 @@ impl DenoConfigFormat {
 }
 
 enum DenoOrPackageJson {
-  Deno(deno_config::ConfigFile, DenoConfigFormat),
+  Deno(Arc<deno_config::ConfigFile>, DenoConfigFormat),
   Npm(Arc<deno_node::PackageJson>, Option<FmtOptionsConfig>),
 }
 
@@ -87,7 +87,6 @@ impl DenoOrPackageJson {
       DenoOrPackageJson::Deno(deno, ..) => deno
         .to_fmt_config()
         .ok()
-        .flatten()
         .map(|f| f.options)
         .unwrap_or_default(),
       DenoOrPackageJson::Npm(_, config) => config.clone().unwrap_or_default(),
@@ -122,9 +121,10 @@ impl DenoOrPackageJson {
   /// the new config
   fn from_flags(flags: Flags) -> Result<(Self, CliFactory), AnyError> {
     let factory = CliFactory::from_flags(flags.clone())?;
-    let options = factory.cli_options().clone();
+    let options = factory.cli_options();
+    let start_ctx = options.workspace.resolve_start_ctx();
 
-    match (options.maybe_config_file(), options.maybe_package_json()) {
+    match (start_ctx.maybe_deno_json(), start_ctx.maybe_pkg_json()) {
       // when both are present, for now,
       // default to deno.json
       (Some(deno), Some(_) | None) => Ok((
@@ -141,20 +141,17 @@ impl DenoOrPackageJson {
         std::fs::write(options.initial_cwd().join("deno.json"), "{}\n")
           .context("Failed to create deno.json file")?;
         log::info!("Created deno.json configuration file.");
-        let new_factory = CliFactory::from_flags(flags.clone())?;
-        let new_options = new_factory.cli_options().clone();
+        let factory = CliFactory::from_flags(flags.clone())?;
+        let options = factory.cli_options().clone();
+        let start_ctx = options.workspace.resolve_start_ctx();
         Ok((
           DenoOrPackageJson::Deno(
-            new_options
-              .maybe_config_file()
-              .as_ref()
-              .ok_or_else(|| {
-                anyhow!("config not found, but it was just created")
-              })?
-              .clone(),
+            start_ctx.maybe_deno_json().cloned().ok_or_else(|| {
+              anyhow!("config not found, but it was just created")
+            })?,
             DenoConfigFormat::Json,
           ),
-          new_factory,
+          factory,
         ))
       }
     }
diff --git a/cli/tools/registry/publish_order.rs b/cli/tools/registry/publish_order.rs
index ad0f72272..ad77a56bb 100644
--- a/cli/tools/registry/publish_order.rs
+++ b/cli/tools/registry/publish_order.rs
@@ -5,7 +5,7 @@ use std::collections::HashSet;
 use std::collections::VecDeque;
 
 use deno_ast::ModuleSpecifier;
-use deno_config::WorkspaceMemberConfig;
+use deno_config::workspace::JsrPackageConfig;
 use deno_core::anyhow::bail;
 use deno_core::error::AnyError;
 use deno_graph::ModuleGraph;
@@ -114,7 +114,7 @@ impl PublishOrderGraph {
 
 pub fn build_publish_order_graph(
   graph: &ModuleGraph,
-  roots: &[WorkspaceMemberConfig],
+  roots: &[JsrPackageConfig],
 ) -> Result<PublishOrderGraph, AnyError> {
   let packages = build_pkg_deps(graph, roots)?;
   Ok(build_publish_order_graph_from_pkgs_deps(packages))
@@ -122,18 +122,23 @@ pub fn build_publish_order_graph(
 
 fn build_pkg_deps(
   graph: &deno_graph::ModuleGraph,
-  roots: &[WorkspaceMemberConfig],
+  roots: &[JsrPackageConfig],
 ) -> Result<HashMap<String, HashSet<String>>, AnyError> {
   let mut members = HashMap::with_capacity(roots.len());
   let mut seen_modules = HashSet::with_capacity(graph.modules().count());
   let roots = roots
     .iter()
-    .map(|r| (ModuleSpecifier::from_file_path(&r.dir_path).unwrap(), r))
+    .map(|r| {
+      (
+        ModuleSpecifier::from_directory_path(r.config_file.dir_path()).unwrap(),
+        r,
+      )
+    })
     .collect::<Vec<_>>();
-  for (root_dir_url, root) in &roots {
+  for (root_dir_url, pkg_config) in &roots {
     let mut deps = HashSet::new();
     let mut pending = VecDeque::new();
-    pending.extend(root.config_file.resolve_export_value_urls()?);
+    pending.extend(pkg_config.config_file.resolve_export_value_urls()?);
     while let Some(specifier) = pending.pop_front() {
       let Some(module) = graph.get(&specifier).and_then(|m| m.js()) else {
         continue;
@@ -168,12 +173,12 @@ fn build_pkg_deps(
             specifier.as_str().starts_with(dir_url.as_str())
           });
           if let Some(root) = found_root {
-            deps.insert(root.1.package_name.clone());
+            deps.insert(root.1.name.clone());
           }
         }
       }
     }
-    members.insert(root.package_name.clone(), deps);
+    members.insert(pkg_config.name.clone(), deps);
   }
   Ok(members)
 }
diff --git a/cli/tools/registry/unfurl.rs b/cli/tools/registry/unfurl.rs
index 36bff64bb..147b59f30 100644
--- a/cli/tools/registry/unfurl.rs
+++ b/cli/tools/registry/unfurl.rs
@@ -3,6 +3,9 @@
 use deno_ast::ParsedSource;
 use deno_ast::SourceRange;
 use deno_ast::SourceTextInfo;
+use deno_config::package_json::PackageJsonDepValue;
+use deno_config::workspace::MappedResolution;
+use deno_config::workspace::WorkspaceResolver;
 use deno_core::ModuleSpecifier;
 use deno_graph::DependencyDescriptor;
 use deno_graph::DynamicTemplatePart;
@@ -10,7 +13,6 @@ use deno_graph::ParserModuleAnalyzer;
 use deno_graph::TypeScriptReference;
 use deno_runtime::deno_node::is_builtin_node_module;
 
-use crate::resolver::MappedSpecifierResolver;
 use crate::resolver::SloppyImportsResolver;
 
 #[derive(Debug, Clone)]
@@ -39,20 +41,20 @@ impl SpecifierUnfurlerDiagnostic {
 }
 
 pub struct SpecifierUnfurler<'a> {
-  mapped_resolver: &'a MappedSpecifierResolver,
   sloppy_imports_resolver: Option<&'a SloppyImportsResolver>,
+  workspace_resolver: &'a WorkspaceResolver,
   bare_node_builtins: bool,
 }
 
 impl<'a> SpecifierUnfurler<'a> {
   pub fn new(
-    mapped_resolver: &'a MappedSpecifierResolver,
     sloppy_imports_resolver: Option<&'a SloppyImportsResolver>,
+    workspace_resolver: &'a WorkspaceResolver,
     bare_node_builtins: bool,
   ) -> Self {
     Self {
-      mapped_resolver,
       sloppy_imports_resolver,
+      workspace_resolver,
       bare_node_builtins,
     }
   }
@@ -62,12 +64,46 @@ impl<'a> SpecifierUnfurler<'a> {
     referrer: &ModuleSpecifier,
     specifier: &str,
   ) -> Option<String> {
-    let resolved =
-      if let Ok(resolved) = self.mapped_resolver.resolve(specifier, referrer) {
-        resolved.into_specifier()
-      } else {
-        None
-      };
+    let resolved = if let Ok(resolved) =
+      self.workspace_resolver.resolve(specifier, referrer)
+    {
+      match resolved {
+        MappedResolution::Normal(specifier)
+        | MappedResolution::ImportMap(specifier) => Some(specifier),
+        MappedResolution::PackageJson {
+          sub_path,
+          dep_result,
+          ..
+        } => match dep_result {
+          Ok(dep) => match dep {
+            PackageJsonDepValue::Req(req) => ModuleSpecifier::parse(&format!(
+              "npm:{}{}",
+              req,
+              sub_path
+                .as_ref()
+                .map(|s| format!("/{}", s))
+                .unwrap_or_default()
+            ))
+            .ok(),
+            PackageJsonDepValue::Workspace(_) => {
+              log::warn!(
+                "package.json workspace entries are not implemented yet for publishing."
+              );
+              None
+            }
+          },
+          Err(err) => {
+            log::warn!(
+              "Ignoring failed to resolve package.json dependency. {:#}",
+              err
+            );
+            None
+          }
+        },
+      }
+    } else {
+      None
+    };
     let resolved = match resolved {
       Some(resolved) => resolved,
       None if self.bare_node_builtins && is_builtin_node_module(specifier) => {
@@ -305,8 +341,6 @@ fn to_range(
 mod tests {
   use std::sync::Arc;
 
-  use crate::args::PackageJsonDepsProvider;
-
   use super::*;
   use deno_ast::MediaType;
   use deno_ast::ModuleSpecifier;
@@ -355,19 +389,17 @@ mod tests {
         }
       }),
     );
-    let mapped_resolver = MappedSpecifierResolver::new(
-      Some(Arc::new(import_map)),
-      Arc::new(PackageJsonDepsProvider::new(Some(
-        package_json.resolve_local_package_json_version_reqs(),
-      ))),
+    let workspace_resolver = WorkspaceResolver::new_raw(
+      Some(import_map),
+      vec![Arc::new(package_json)],
+      deno_config::workspace::PackageJsonDepResolution::Enabled,
     );
-
     let fs = Arc::new(RealFs);
     let sloppy_imports_resolver = SloppyImportsResolver::new(fs);
 
     let unfurler = SpecifierUnfurler::new(
-      &mapped_resolver,
       Some(&sloppy_imports_resolver),
+      &workspace_resolver,
       true,
     );