feat(unstable): ability to `npm install` then `deno run main.ts` (#20967)

This PR adds a new unstable "bring your own node_modules" (BYONM)
functionality currently behind a `--unstable-byonm` flag (`"unstable":
["byonm"]` in a deno.json).

This enables users to run a separate install command (ex. `npm install`,
`pnpm install`) then run `deno run main.ts` and Deno will respect the
layout of the node_modules directory as setup by the separate install
command. It also works with npm/yarn/pnpm workspaces.

For this PR, the behaviour is opted into by specifying
`--unstable-byonm`/`"unstable": ["byonm"]`, but in the future we may
make this the default behaviour as outlined in
https://github.com/denoland/deno/issues/18967#issuecomment-1761248941

This is an extremely rough initial implementation. Errors are
terrible in this and the LSP requires frequent restarts. Improvements
will be done in follow up PRs.

17 files changed, 288 insertions, 16 deletions

diff --git a/cli/tests/integration/cert_tests.rs b/cli/tests/integration/cert_tests.rs
index ffd4b449d..20bf4d80d 100644
--- a/cli/tests/integration/cert_tests.rs
+++ b/cli/tests/integration/cert_tests.rs
@@ -118,7 +118,7 @@ fn cafile_compile() {
 
   context
     .new_command()
-    .command_name(output_exe)
+    .name(output_exe)
     .run()
     .assert_matches_text("[WILDCARD]\nHello\n");
 }
diff --git a/cli/tests/integration/compile_tests.rs b/cli/tests/integration/compile_tests.rs
index 657d17d7d..3ca57a35f 100644
--- a/cli/tests/integration/compile_tests.rs
+++ b/cli/tests/integration/compile_tests.rs
@@ -29,7 +29,7 @@ fn compile_basic() {
       .run();
     output.assert_exit_code(0);
     output.skip_output_check();
-    let output = context.new_command().command_name(&exe).run();
+    let output = context.new_command().name(&exe).run();
     output.assert_matches_text("Welcome to Deno!\n");
   }
 
@@ -42,7 +42,7 @@ fn compile_basic() {
     .new_command()
     // it should fail creating this, but still work
     .env("DENO_DIR", readonly_sub_dir)
-    .command_name(exe)
+    .name(exe)
     .run();
   output.assert_matches_text("Welcome to Deno!\n");
 }
@@ -688,11 +688,7 @@ fn workers_not_in_module_map() {
   output.assert_exit_code(0);
   output.skip_output_check();
 
-  let output = context
-    .new_command()
-    .command_name(exe)
-    .env("NO_COLOR", "")
-    .run();
+  let output = context.new_command().name(exe).env("NO_COLOR", "").run();
   output.assert_exit_code(1);
   output.assert_matches_text(concat!(
     "error: Uncaught (in worker \"\") Module not found: [WILDCARD]",
@@ -825,7 +821,7 @@ fn compile_npm_specifiers() {
     output.assert_exit_code(0);
     output.skip_output_check();
 
-    let output = context.new_command().command_name(&binary_path).run();
+    let output = context.new_command().name(&binary_path).run();
     output.assert_matches_text(
       r#"Node esm importing node cjs
 ===========================
@@ -881,7 +877,7 @@ testing[WILDCARD]this
   output.assert_exit_code(0);
   output.skip_output_check();
 
-  let output = context.new_command().command_name(binary_path).run();
+  let output = context.new_command().name(binary_path).run();
   output.assert_matches_text("2\n");
 }
 
@@ -1050,7 +1046,7 @@ fn run_npm_bin_compile_test(opts: RunNpmBinCompileOptions) {
   };
   let output = context
     .new_command()
-    .command_name(binary_path)
+    .name(binary_path)
     .args_vec(opts.run_args)
     .run();
   output.assert_matches_file(opts.output_file);
@@ -1114,6 +1110,6 @@ fn compile_node_modules_symlink_outside() {
   // run
   let binary_path =
     project_dir.join(if cfg!(windows) { "bin.exe" } else { "bin" });
-  let output = context.new_command().command_name(binary_path).run();
+  let output = context.new_command().name(binary_path).run();
   output.assert_matches_file("compile/node_modules_symlink_outside/main.out");
 }
diff --git a/cli/tests/integration/npm_tests.rs b/cli/tests/integration/npm_tests.rs
index ddbc79cec..c3fe2949c 100644
--- a/cli/tests/integration/npm_tests.rs
+++ b/cli/tests/integration/npm_tests.rs
@@ -2219,3 +2219,240 @@ itest!(require_resolve_url_paths {
   cwd: Some("npm/require_resolve_url/"),
   copy_temp_dir: Some("npm/require_resolve_url/"),
 });
+
+#[test]
+pub fn byonm_cjs_esm_packages() {
+  let test_context = TestContextBuilder::for_npm()
+    .env("DENO_UNSTABLE_BYONM", "1")
+    .use_temp_cwd()
+    .build();
+  let dir = test_context.temp_dir();
+  let run_npm = |args: &str| {
+    test_context
+      .new_command()
+      .name("npm")
+      .args(args)
+      .run()
+      .skip_output_check();
+  };
+
+  run_npm("init -y");
+  run_npm("install @denotest/esm-basic @denotest/cjs-default-export @denotest/dual-cjs-esm chalk@4 chai@4.3");
+
+  dir.write(
+    "main.ts",
+    r#"
+import { getValue, setValue } from "@denotest/esm-basic";
+
+setValue(2);
+console.log(getValue());
+
+import cjsDefault from "@denotest/cjs-default-export";
+console.log(cjsDefault.default());
+console.log(cjsDefault.named());
+
+import { getKind } from "@denotest/dual-cjs-esm";
+console.log(getKind());
+
+
+"#,
+  );
+  let output = test_context.new_command().args("run --check main.ts").run();
+  output
+    .assert_matches_text("Check file:///[WILDCARD]/main.ts\n2\n1\n2\nesm\n");
+
+  // should not have created the .deno directory
+  assert!(!dir.path().join("node_modules/.deno").exists());
+
+  // try chai
+  dir.write(
+    "chai.ts",
+    r#"import { expect } from "chai";
+
+    const timeout = setTimeout(() => {}, 0);
+    expect(timeout).to.be.a("number");
+    clearTimeout(timeout);"#,
+  );
+  test_context.new_command().args("run chai.ts").run();
+
+  // try chalk cjs
+  dir.write(
+    "chalk.ts",
+    "import chalk from 'chalk'; console.log(chalk.green('chalk cjs loads'));",
+  );
+  let output = test_context
+    .new_command()
+    .args("run --allow-read chalk.ts")
+    .run();
+  output.assert_matches_text("chalk cjs loads\n");
+
+  // try using an npm specifier for chalk that matches the version we installed
+  dir.write(
+    "chalk.ts",
+    "import chalk from 'npm:chalk@4'; console.log(chalk.green('chalk cjs loads'));",
+  );
+  let output = test_context
+    .new_command()
+    .args("run --allow-read chalk.ts")
+    .run();
+  output.assert_matches_text("chalk cjs loads\n");
+
+  // try with one that doesn't match the package.json
+  dir.write(
+    "chalk.ts",
+    "import chalk from 'npm:chalk@5'; console.log(chalk.green('chalk cjs loads'));",
+  );
+  let output = test_context
+    .new_command()
+    .args("run --allow-read chalk.ts")
+    .run();
+  output.assert_matches_text(
+    r#"error: Could not find a matching package for 'npm:chalk@5' in '[WILDCARD]package.json'. You must specify this as a package.json dependency when the node_modules folder is not managed by Deno.
+    at file:///[WILDCARD]chalk.ts:1:19
+"#);
+  output.assert_exit_code(1);
+}
+
+#[test]
+pub fn byonm_package_npm_specifier_not_installed_and_invalid_subpath() {
+  let test_context = TestContextBuilder::for_npm()
+    .env("DENO_UNSTABLE_BYONM", "1")
+    .use_temp_cwd()
+    .build();
+  let dir = test_context.temp_dir();
+  dir.path().join("package.json").write_json(&json!({
+    "dependencies": {
+      "chalk": "4",
+      "@denotest/conditional-exports-strict": "1"
+    }
+  }));
+  dir.write(
+    "main.ts",
+    "import chalk from 'npm:chalk'; console.log(chalk.green('hi'));",
+  );
+
+  // no npm install has been run, so this should give an informative error
+  let output = test_context.new_command().args("run main.ts").run();
+  output.assert_matches_text(
+    r#"error: Could not find '[WILDCARD]package.json'. Maybe run `npm install`?
+    at file:///[WILDCARD]/main.ts:1:19
+"#,
+  );
+  output.assert_exit_code(1);
+
+  // now test for an invalid sub path after doing an npm install
+  dir.write(
+    "main.ts",
+    "import 'npm:@denotest/conditional-exports-strict/test';",
+  );
+
+  test_context
+    .new_command()
+    .name("npm")
+    .args("install")
+    .run()
+    .skip_output_check();
+
+  let output = test_context.new_command().args("run main.ts").run();
+  output.assert_matches_text(
+    r#"error: Failed resolving package subpath './test' for '[WILDCARD]package.json'
+    at file:///[WILDCARD]/main.ts:1:8
+"#,
+  );
+  output.assert_exit_code(1);
+}
+
+#[test]
+pub fn byonm_npm_workspaces() {
+  let test_context = TestContextBuilder::for_npm().use_temp_cwd().build();
+  let dir = test_context.temp_dir();
+  dir.write(
+    "deno.json",
+    r#"{
+    "unstable": [
+      "byonm"
+    ]
+  }"#,
+  );
+
+  dir.write(
+    "package.json",
+    r#"{
+  "name": "my-workspace",
+  "workspaces": [
+    "project-a",
+    "project-b"
+  ]
+}
+"#,
+  );
+
+  let project_a_dir = dir.path().join("project-a");
+  project_a_dir.create_dir_all();
+  project_a_dir.join("package.json").write_json(&json!({
+    "name": "project-a",
+    "version": "1.0.0",
+    "main": "./index.js",
+    "type": "module",
+    "dependencies": {
+      "chai": "^4.2",
+      "project-b": "^1"
+    }
+  }));
+  project_a_dir.join("index.js").write(
+    r#"
+import { expect } from "chai";
+
+const timeout = setTimeout(() => {}, 0);
+expect(timeout).to.be.a("number");
+clearTimeout(timeout);
+
+export function add(a, b) {
+  return a + b;
+}
+"#,
+  );
+  project_a_dir
+    .join("index.d.ts")
+    .write("export function add(a: number, b: number): number;");
+
+  let project_b_dir = dir.path().join("project-b");
+  project_b_dir.create_dir_all();
+  project_b_dir.join("package.json").write_json(&json!({
+    "name": "project-b",
+    "version": "1.0.0",
+    "type": "module",
+    "dependencies": {
+      "@denotest/esm-basic": "^1.0",
+    }
+  }));
+  project_b_dir.join("main.ts").write(
+    r#"
+import { getValue, setValue } from "@denotest/esm-basic";
+
+setValue(5);
+console.log(getValue());
+
+import { add } from "project-a";
+console.log(add(1, 2));
+"#,
+  );
+
+  test_context
+    .new_command()
+    .name("npm")
+    .args("install")
+    .run()
+    .skip_output_check();
+
+  let output = test_context
+    .new_command()
+    .args("run ./project-b/main.ts")
+    .run();
+  output.assert_matches_text("5\n3\n");
+  let output = test_context
+    .new_command()
+    .args("check ./project-b/main.ts")
+    .run();
+  output.assert_matches_text("Check file:///[WILDCARD]/project-b/main.ts\n");
+}
diff --git a/cli/tests/testdata/npm/registry/@denotest/conditional-exports-strict/1.0.0/cjs/index.cjs b/cli/tests/testdata/npm/registry/@denotest/conditional-exports-strict/1.0.0/cjs/index.cjs
new file mode 100644
index 000000000..16895e48c
--- /dev/null
+++ b/cli/tests/testdata/npm/registry/@denotest/conditional-exports-strict/1.0.0/cjs/index.cjs
@@ -0,0 +1,3 @@
+module.exports = {
+    hello: "from cjs"
+};
\ No newline at end of file
diff --git a/cli/tests/testdata/npm/registry/@denotest/conditional-exports-strict/1.0.0/esm/client/bar.js b/cli/tests/testdata/npm/registry/@denotest/conditional-exports-strict/1.0.0/esm/client/bar.js
new file mode 100644
index 000000000..1474f5d29
--- /dev/null
+++ b/cli/tests/testdata/npm/registry/@denotest/conditional-exports-strict/1.0.0/esm/client/bar.js
@@ -0,0 +1,3 @@
+export default {
+    hello: "from esm client bar",
+}
diff --git a/cli/tests/testdata/npm/registry/@denotest/conditional-exports-strict/1.0.0/esm/client/foo.js b/cli/tests/testdata/npm/registry/@denotest/conditional-exports-strict/1.0.0/esm/client/foo.js
new file mode 100644
index 000000000..bb5284b15
--- /dev/null
+++ b/cli/tests/testdata/npm/registry/@denotest/conditional-exports-strict/1.0.0/esm/client/foo.js
@@ -0,0 +1,3 @@
+export default {
+    hello: "from esm client foo",
+}
diff --git a/cli/tests/testdata/npm/registry/@denotest/conditional-exports-strict/1.0.0/esm/client/index.js b/cli/tests/testdata/npm/registry/@denotest/conditional-exports-strict/1.0.0/esm/client/index.js
new file mode 100644
index 000000000..dc1ec197d
--- /dev/null
+++ b/cli/tests/testdata/npm/registry/@denotest/conditional-exports-strict/1.0.0/esm/client/index.js
@@ -0,0 +1,3 @@
+export default {
+    hello: "from esm client",
+}
diff --git a/cli/tests/testdata/npm/registry/@denotest/conditional-exports-strict/1.0.0/esm/client/m.js b/cli/tests/testdata/npm/registry/@denotest/conditional-exports-strict/1.0.0/esm/client/m.js
new file mode 100644
index 000000000..fec6807ac
--- /dev/null
+++ b/cli/tests/testdata/npm/registry/@denotest/conditional-exports-strict/1.0.0/esm/client/m.js
@@ -0,0 +1,3 @@
+export default {
+    hello: "from esm client m",
+}
diff --git a/cli/tests/testdata/npm/registry/@denotest/conditional-exports-strict/1.0.0/esm/index.js b/cli/tests/testdata/npm/registry/@denotest/conditional-exports-strict/1.0.0/esm/index.js
new file mode 100644
index 000000000..38dae7d93
--- /dev/null
+++ b/cli/tests/testdata/npm/registry/@denotest/conditional-exports-strict/1.0.0/esm/index.js
@@ -0,0 +1,3 @@
+export default {
+    hello: "from esm",
+}
\ No newline at end of file
diff --git a/cli/tests/testdata/npm/registry/@denotest/conditional-exports-strict/1.0.0/foo.js b/cli/tests/testdata/npm/registry/@denotest/conditional-exports-strict/1.0.0/foo.js
new file mode 100644
index 000000000..6060c8a67
--- /dev/null
+++ b/cli/tests/testdata/npm/registry/@denotest/conditional-exports-strict/1.0.0/foo.js
@@ -0,0 +1,3 @@
+export default {
+  hello: "from foo",
+}
diff --git a/cli/tests/testdata/npm/registry/@denotest/conditional-exports-strict/1.0.0/package.json b/cli/tests/testdata/npm/registry/@denotest/conditional-exports-strict/1.0.0/package.json
new file mode 100644
index 000000000..3576e48f8
--- /dev/null
+++ b/cli/tests/testdata/npm/registry/@denotest/conditional-exports-strict/1.0.0/package.json
@@ -0,0 +1,16 @@
+{
+  "name": "@denotest/conditional-exports-strict",
+  "version": "1.0.0",
+  "type": "module",
+  "exports": {
+    ".": {
+      "types": "./types/src/index.d.ts",
+      "require": "./cjs/index.cjs",
+      "import": "./esm/index.js"
+    },
+    "./client": {
+      "types": "./types/src/client/index.d.ts",
+      "import": "./esm/client/index.js"
+    }
+  }
+}
diff --git a/cli/tests/testdata/npm/registry/@denotest/conditional-exports/1.0.0/esm/client/bar.js b/cli/tests/testdata/npm/registry/@denotest/conditional-exports/1.0.0/esm/client/bar.js
index 12352639d..1474f5d29 100644
--- a/cli/tests/testdata/npm/registry/@denotest/conditional-exports/1.0.0/esm/client/bar.js
+++ b/cli/tests/testdata/npm/registry/@denotest/conditional-exports/1.0.0/esm/client/bar.js
@@ -1,3 +1,3 @@
 export default {
     hello: "from esm client bar",
-}
\ No newline at end of file
+}
diff --git a/cli/tests/testdata/npm/registry/@denotest/conditional-exports/1.0.0/esm/client/foo.js b/cli/tests/testdata/npm/registry/@denotest/conditional-exports/1.0.0/esm/client/foo.js
index 1ab5baf1b..bb5284b15 100644
--- a/cli/tests/testdata/npm/registry/@denotest/conditional-exports/1.0.0/esm/client/foo.js
+++ b/cli/tests/testdata/npm/registry/@denotest/conditional-exports/1.0.0/esm/client/foo.js
@@ -1,3 +1,3 @@
 export default {
     hello: "from esm client foo",
-}
\ No newline at end of file
+}
diff --git a/cli/tests/testdata/npm/registry/@denotest/conditional-exports/1.0.0/esm/client/index.js b/cli/tests/testdata/npm/registry/@denotest/conditional-exports/1.0.0/esm/client/index.js
index 86f246be4..dc1ec197d 100644
--- a/cli/tests/testdata/npm/registry/@denotest/conditional-exports/1.0.0/esm/client/index.js
+++ b/cli/tests/testdata/npm/registry/@denotest/conditional-exports/1.0.0/esm/client/index.js
@@ -1,3 +1,3 @@
 export default {
     hello: "from esm client",
-}
\ No newline at end of file
+}
diff --git a/cli/tests/testdata/npm/registry/@denotest/conditional-exports/1.0.0/esm/client/m.js b/cli/tests/testdata/npm/registry/@denotest/conditional-exports/1.0.0/esm/client/m.js
index 40e769031..fec6807ac 100644
--- a/cli/tests/testdata/npm/registry/@denotest/conditional-exports/1.0.0/esm/client/m.js
+++ b/cli/tests/testdata/npm/registry/@denotest/conditional-exports/1.0.0/esm/client/m.js
@@ -1,3 +1,3 @@
 export default {
     hello: "from esm client m",
-}
\ No newline at end of file
+}
diff --git a/cli/tests/testdata/npm/registry/@denotest/dual-cjs-esm/1.0.0/main.d.cts b/cli/tests/testdata/npm/registry/@denotest/dual-cjs-esm/1.0.0/main.d.cts
new file mode 100644
index 000000000..f969ba996
--- /dev/null
+++ b/cli/tests/testdata/npm/registry/@denotest/dual-cjs-esm/1.0.0/main.d.cts
@@ -0,0 +1 @@
+export function getKind(): string;
diff --git a/cli/tests/testdata/npm/registry/@denotest/dual-cjs-esm/1.0.0/main.d.mts b/cli/tests/testdata/npm/registry/@denotest/dual-cjs-esm/1.0.0/main.d.mts
new file mode 100644
index 000000000..f969ba996
--- /dev/null
+++ b/cli/tests/testdata/npm/registry/@denotest/dual-cjs-esm/1.0.0/main.d.mts
@@ -0,0 +1 @@
+export function getKind(): string;