feat: npm workspace and better Deno workspace support (#24334)

Adds much better support for the unstable Deno workspaces as well as
support for npm workspaces. npm workspaces is still lacking in that we
only install packages into the root node_modules folder. We'll make it
smarter over time in order for it to figure out when to add node_modules
folders within packages.

This includes a breaking change in config file resolution where we stop
searching for config files on the first found package.json unless it's
in a workspace. For the previous behaviour, the root deno.json needs to
be updated to be a workspace by adding `"workspace":
["./path-to-pkg-json-folder-goes-here"]`. See details in
https://github.com/denoland/deno_config/pull/66

Closes #24340
Closes #24159
Closes #24161
Closes #22020
Closes #18546
Closes #16106
Closes #24160

4 files changed, 85 insertions, 35 deletions

diff --git a/tests/integration/compile_tests.rs b/tests/integration/compile_tests.rs
index 0d94d4367..c902adfb2 100644
--- a/tests/integration/compile_tests.rs
+++ b/tests/integration/compile_tests.rs
@@ -822,7 +822,7 @@ testing[WILDCARD]this
     .args("compile --output binary main.ts")
     .run()
     .assert_exit_code(0)
-    .assert_matches_text("Check file:///[WILDCARD]/main.ts\nCompile file:///[WILDCARD]/main.ts to binary[WILDCARD]\n");
+    .assert_matches_text("Check file:///[WILDLINE]/main.ts\nCompile file:///[WILDLINE]/main.ts to binary[WILDLINE]\n");
 
   context
     .new_command()
@@ -835,6 +835,7 @@ testing[WILDCARD]this
 fn compile_npm_file_system() {
   run_npm_bin_compile_test(RunNpmBinCompileOptions {
     input_specifier: "compile/npm_fs/main.ts",
+    copy_temp_dir: Some("compile/npm_fs"),
     compile_args: vec!["-A"],
     run_args: vec![],
     output_file: "compile/npm_fs/main.out",
@@ -849,6 +850,7 @@ fn compile_npm_file_system() {
 fn compile_npm_bin_esm() {
   run_npm_bin_compile_test(RunNpmBinCompileOptions {
     input_specifier: "npm:@denotest/bin/cli-esm",
+    copy_temp_dir: None,
     compile_args: vec![],
     run_args: vec!["this", "is", "a", "test"],
     output_file: "npm/deno_run_esm.out",
@@ -863,6 +865,7 @@ fn compile_npm_bin_esm() {
 fn compile_npm_bin_cjs() {
   run_npm_bin_compile_test(RunNpmBinCompileOptions {
     input_specifier: "npm:@denotest/bin/cli-cjs",
+    copy_temp_dir: None,
     compile_args: vec![],
     run_args: vec!["this", "is", "a", "test"],
     output_file: "npm/deno_run_cjs.out",
@@ -877,6 +880,7 @@ fn compile_npm_bin_cjs() {
 fn compile_npm_cowsay_main() {
   run_npm_bin_compile_test(RunNpmBinCompileOptions {
     input_specifier: "npm:cowsay@1.5.0",
+    copy_temp_dir: None,
     compile_args: vec!["--allow-read"],
     run_args: vec!["Hello"],
     output_file: "npm/deno_run_cowsay.out",
@@ -891,6 +895,7 @@ fn compile_npm_cowsay_main() {
 fn compile_npm_vfs_implicit_read_permissions() {
   run_npm_bin_compile_test(RunNpmBinCompileOptions {
     input_specifier: "compile/vfs_implicit_read_permission/main.ts",
+    copy_temp_dir: Some("compile/vfs_implicit_read_permission"),
     compile_args: vec![],
     run_args: vec![],
     output_file: "compile/vfs_implicit_read_permission/main.out",
@@ -905,6 +910,7 @@ fn compile_npm_vfs_implicit_read_permissions() {
 fn compile_npm_no_permissions() {
   run_npm_bin_compile_test(RunNpmBinCompileOptions {
     input_specifier: "npm:cowsay@1.5.0",
+    copy_temp_dir: None,
     compile_args: vec![],
     run_args: vec!["Hello"],
     output_file: "npm/deno_run_cowsay_no_permissions.out",
@@ -919,6 +925,7 @@ fn compile_npm_no_permissions() {
 fn compile_npm_cowsay_explicit() {
   run_npm_bin_compile_test(RunNpmBinCompileOptions {
     input_specifier: "npm:cowsay@1.5.0/cowsay",
+    copy_temp_dir: None,
     compile_args: vec!["--allow-read"],
     run_args: vec!["Hello"],
     output_file: "npm/deno_run_cowsay.out",
@@ -933,6 +940,7 @@ fn compile_npm_cowsay_explicit() {
 fn compile_npm_cowthink() {
   run_npm_bin_compile_test(RunNpmBinCompileOptions {
     input_specifier: "npm:cowsay@1.5.0/cowthink",
+    copy_temp_dir: None,
     compile_args: vec!["--allow-read"],
     run_args: vec!["Hello"],
     output_file: "npm/deno_run_cowthink.out",
@@ -945,6 +953,7 @@ fn compile_npm_cowthink() {
 
 struct RunNpmBinCompileOptions<'a> {
   input_specifier: &'a str,
+  copy_temp_dir: Option<&'a str>,
   node_modules_dir: bool,
   output_file: &'a str,
   input_name: Option<&'a str>,
@@ -955,15 +964,13 @@ struct RunNpmBinCompileOptions<'a> {
 }
 
 fn run_npm_bin_compile_test(opts: RunNpmBinCompileOptions) {
-  let context = TestContextBuilder::for_npm().use_temp_cwd().build();
-
-  let temp_dir = context.temp_dir();
-  let main_specifier = if opts.input_specifier.starts_with("npm:") {
-    opts.input_specifier.to_string()
-  } else {
-    testdata_path().join(opts.input_specifier).to_string()
+  let builder = TestContextBuilder::for_npm();
+  let context = match opts.copy_temp_dir {
+    Some(copy_temp_dir) => builder.use_copy_temp_dir(copy_temp_dir).build(),
+    None => builder.use_temp_cwd().build(),
   };
 
+  let temp_dir = context.temp_dir();
   let mut args = vec!["compile".to_string()];
 
   args.extend(opts.compile_args.iter().map(|s| s.to_string()));
@@ -977,7 +984,7 @@ fn run_npm_bin_compile_test(opts: RunNpmBinCompileOptions) {
     args.push(bin_name.to_string());
   }
 
-  args.push(main_specifier);
+  args.push(opts.input_specifier.to_string());
 
   // compile
   let output = context.new_command().args_vec(args).run();
@@ -1004,7 +1011,13 @@ fn run_npm_bin_compile_test(opts: RunNpmBinCompileOptions) {
 
 #[test]
 fn compile_node_modules_symlink_outside() {
+  // this code is using a canonicalized temp dir because otherwise
+  // it fails on the Windows CI because Deno makes the root directory
+  // a common ancestor of the symlinked temp dir and the canonicalized
+  // temp dir, which causes the warnings to not be surfaced
+  #[allow(deprecated)]
   let context = TestContextBuilder::for_npm()
+    .use_canonicalized_temp_dir()
     .use_copy_temp_dir("compile/node_modules_symlink_outside")
     .cwd("compile/node_modules_symlink_outside")
     .build();
@@ -1014,15 +1027,15 @@ fn compile_node_modules_symlink_outside() {
     .path()
     .join("compile")
     .join("node_modules_symlink_outside");
-  temp_dir.create_dir_all(project_dir.join("node_modules"));
-  temp_dir.create_dir_all(project_dir.join("some_folder"));
-  temp_dir.write(project_dir.join("test.txt"), "5");
-
-  // create a symlink in the node_modules directory that points to a folder in the cwd
-  temp_dir.symlink_dir(
-    project_dir.join("some_folder"),
-    project_dir.join("node_modules").join("some_folder"),
-  );
+  let symlink_target_dir = temp_dir.path().join("some_folder");
+  project_dir.join("node_modules").create_dir_all();
+  symlink_target_dir.create_dir_all();
+  let symlink_target_file = temp_dir.path().join("target.txt");
+  symlink_target_file.write("5");
+  let symlink_dir = project_dir.join("node_modules").join("symlink_dir");
+
+  // create a symlink in the node_modules directory that points to a folder outside the project
+  temp_dir.symlink_dir(&symlink_target_dir, &symlink_dir);
   // compile folder
   let output = context
     .new_command()
@@ -1032,16 +1045,16 @@ fn compile_node_modules_symlink_outside() {
   output.assert_matches_file(
     "compile/node_modules_symlink_outside/main_compile_folder.out",
   );
-  assert!(project_dir.join("node_modules/some_folder").exists());
+  assert!(symlink_dir.exists());
 
   // Cleanup and remove the folder. The folder test is done separately from
   // the file symlink test because different systems would traverse
   // the directory items in different order.
-  temp_dir.remove_dir_all(project_dir.join("node_modules/some_folder"));
+  symlink_dir.remove_dir_all();
 
   // create a symlink in the node_modules directory that points to a file in the cwd
   temp_dir.symlink_file(
-    project_dir.join("test.txt"),
+    &symlink_target_file,
     project_dir.join("node_modules").join("test.txt"),
   );
   assert!(project_dir.join("node_modules/test.txt").exists());
@@ -1154,8 +1167,11 @@ fn granular_unstable_features() {
 
 #[test]
 fn granular_unstable_features_config_file() {
-  let context = TestContextBuilder::new().build();
+  let context = TestContextBuilder::new().use_temp_cwd().build();
   let dir = context.temp_dir();
+  testdata_path()
+    .join("compile/unstable_features.ts")
+    .copy(&dir.path().join("unstable_features.ts"));
   let exe = if cfg!(windows) {
     dir.path().join("app.exe")
   } else {
@@ -1176,7 +1192,7 @@ fn granular_unstable_features_config_file() {
       &dir.path().join("deno.json").to_string(),
       "--output",
       &exe.to_string_lossy(),
-      "./compile/unstable_features.ts",
+      "./unstable_features.ts",
     ])
     .run();
   output.assert_exit_code(0);
diff --git a/tests/integration/lsp_tests.rs b/tests/integration/lsp_tests.rs
index d0df5e6e8..f66fc97be 100644
--- a/tests/integration/lsp_tests.rs
+++ b/tests/integration/lsp_tests.rs
@@ -13051,7 +13051,7 @@ fn lsp_deno_json_workspace_fmt_config() {
   temp_dir.write(
     "deno.json",
     json!({
-      "workspaces": ["project1", "project2"],
+      "workspace": ["project1", "project2"],
       "fmt": {
         "semiColons": false,
       },
@@ -13174,7 +13174,7 @@ fn lsp_deno_json_workspace_lint_config() {
   temp_dir.write(
     "deno.json",
     json!({
-      "workspaces": ["project1", "project2"],
+      "workspace": ["project1", "project2"],
       "lint": {
         "rules": {
           "include": ["camelcase"],
@@ -13315,7 +13315,7 @@ fn lsp_deno_json_workspace_import_map() {
   temp_dir.write(
     "project1/deno.json",
     json!({
-      "workspaces": ["project2"],
+      "workspace": ["project2"],
       "imports": {
         "foo": "./foo1.ts",
       },
@@ -13376,7 +13376,7 @@ fn lsp_deno_json_workspace_jsr_resolution() {
   temp_dir.write(
     "deno.json",
     json!({
-      "workspaces": ["project1"],
+      "workspace": ["project1"],
     })
     .to_string(),
   );
diff --git a/tests/integration/run_tests.rs b/tests/integration/run_tests.rs
index b8263be29..d4d1fea2e 100644
--- a/tests/integration/run_tests.rs
+++ b/tests/integration/run_tests.rs
@@ -1113,7 +1113,9 @@ fn lock_deno_json_package_json_deps_workspace() {
 
   // deno.json
   let deno_json = temp_dir.join("deno.json");
-  deno_json.write_json(&json!({}));
+  deno_json.write_json(&json!({
+    "nodeModulesDir": true
+  }));
 
   // package.json
   let package_json = temp_dir.join("package.json");
@@ -1147,16 +1149,23 @@ fn lock_deno_json_package_json_deps_workspace() {
   let lockfile = temp_dir.join("deno.lock");
   let esm_basic_integrity =
     get_lockfile_npm_package_integrity(&lockfile, "@denotest/esm-basic@1.0.0");
+  let cjs_default_export_integrity = get_lockfile_npm_package_integrity(
+    &lockfile,
+    "@denotest/cjs-default-export@1.0.0",
+  );
 
-  // no "workspace" because deno isn't smart enough to figure this out yet
-  // since it discovered the package.json in a folder different from the lockfile
   lockfile.assert_matches_json(json!({
     "version": "3",
     "packages": {
       "specifiers": {
+        "npm:@denotest/cjs-default-export@1": "npm:@denotest/cjs-default-export@1.0.0",
         "npm:@denotest/esm-basic@1": "npm:@denotest/esm-basic@1.0.0"
       },
       "npm": {
+        "@denotest/cjs-default-export@1.0.0": {
+          "integrity": cjs_default_export_integrity,
+          "dependencies": {}
+        },
         "@denotest/esm-basic@1.0.0": {
           "integrity": esm_basic_integrity,
           "dependencies": {}
@@ -1164,6 +1173,22 @@ fn lock_deno_json_package_json_deps_workspace() {
       }
     },
     "remote": {},
+    "workspace": {
+      "packageJson": {
+        "dependencies": [
+          "npm:@denotest/cjs-default-export@1"
+        ]
+      },
+      "members": {
+        "package-a": {
+          "packageJson": {
+            "dependencies": [
+              "npm:@denotest/esm-basic@1"
+            ]
+          }
+        }
+      }
+    }
   }));
 
   // run a command that causes discovery of the root package.json beside the lockfile
@@ -1201,6 +1226,15 @@ fn lock_deno_json_package_json_deps_workspace() {
         "dependencies": [
           "npm:@denotest/cjs-default-export@1"
         ]
+      },
+      "members": {
+        "package-a": {
+          "packageJson": {
+            "dependencies": [
+              "npm:@denotest/esm-basic@1"
+            ]
+          }
+        }
       }
     }
   });
diff --git a/tests/integration/watcher_tests.rs b/tests/integration/watcher_tests.rs
index 252411627..7864938f8 100644
--- a/tests/integration/watcher_tests.rs
+++ b/tests/integration/watcher_tests.rs
@@ -1547,7 +1547,7 @@ async fn run_watch_dynamic_imports() {
     .unwrap();
   let (mut stdout_lines, mut stderr_lines) = child_lines(&mut child);
   wait_contains("Process started", &mut stderr_lines).await;
-  wait_contains("No package.json file found", &mut stderr_lines).await;
+  wait_contains("Finished config loading.", &mut stderr_lines).await;
 
   wait_contains(
     "Hopefully dynamic import will be watched...",
@@ -1714,7 +1714,7 @@ console.log("Listening...")
     .unwrap();
   let (mut stdout_lines, mut stderr_lines) = child_lines(&mut child);
   wait_contains("Process started", &mut stderr_lines).await;
-  wait_contains("No package.json file found", &mut stderr_lines).await;
+  wait_contains("Finished config loading.", &mut stderr_lines).await;
 
   wait_for_watcher("file_to_watch.js", &mut stderr_lines).await;
   wait_contains("Listening...", &mut stdout_lines).await;
@@ -1787,7 +1787,7 @@ export function foo() {
     .unwrap();
   let (mut stdout_lines, mut stderr_lines) = child_lines(&mut child);
   wait_contains("Process started", &mut stderr_lines).await;
-  wait_contains("No package.json file found", &mut stderr_lines).await;
+  wait_contains("Finished config loading.", &mut stderr_lines).await;
 
   wait_for_watcher("file_to_watch.js", &mut stderr_lines).await;
   wait_contains("5 <h1>Hello</h1>", &mut stdout_lines).await;
@@ -1846,7 +1846,7 @@ export function foo() {
     .unwrap();
   let (mut stdout_lines, mut stderr_lines) = child_lines(&mut child);
   wait_contains("Process started", &mut stderr_lines).await;
-  wait_contains("No package.json file found", &mut stderr_lines).await;
+  wait_contains("Finished config loading.", &mut stderr_lines).await;
 
   wait_for_watcher("file_to_watch.js", &mut stderr_lines).await;
   wait_contains("<h1>asd1</h1>", &mut stdout_lines).await;
@@ -1912,7 +1912,7 @@ export function foo() {
     .unwrap();
   let (mut stdout_lines, mut stderr_lines) = child_lines(&mut child);
   wait_contains("Process started", &mut stderr_lines).await;
-  wait_contains("No package.json file found", &mut stderr_lines).await;
+  wait_contains("Finished config loading.", &mut stderr_lines).await;
 
   wait_for_watcher("file_to_watch.js", &mut stderr_lines).await;
   wait_contains("2 <h1>asd1</h1>", &mut stdout_lines).await;