feat(unstable): ability to `npm install` then `deno run main.ts` (#20967)

This PR adds a new unstable "bring your own node_modules" (BYONM)
functionality currently behind a `--unstable-byonm` flag (`"unstable":
["byonm"]` in a deno.json).

This enables users to run a separate install command (ex. `npm install`,
`pnpm install`) then run `deno run main.ts` and Deno will respect the
layout of the node_modules directory as setup by the separate install
command. It also works with npm/yarn/pnpm workspaces.

For this PR, the behaviour is opted into by specifying
`--unstable-byonm`/`"unstable": ["byonm"]`, but in the future we may
make this the default behaviour as outlined in
https://github.com/denoland/deno/issues/18967#issuecomment-1761248941

This is an extremely rough initial implementation. Errors are
terrible in this and the LSP requires frequent restarts. Improvements
will be done in follow up PRs.

3 files changed, 35 insertions, 11 deletions

diff --git a/test_util/src/builders.rs b/test_util/src/builders.rs
index 17871baa1..39771e088 100644
--- a/test_util/src/builders.rs
+++ b/test_util/src/builders.rs
@@ -227,7 +227,7 @@ pub struct TestCommandBuilder {
 }
 
 impl TestCommandBuilder {
-  pub fn command_name(mut self, name: impl AsRef<OsStr>) -> Self {
+  pub fn name(mut self, name: impl AsRef<OsStr>) -> Self {
     self.command_name = name.as_ref().to_string_lossy().to_string();
     self
   }
@@ -306,7 +306,11 @@ impl TestCommandBuilder {
   }
 
   fn build_command_path(&self) -> PathRef {
-    let command_name = &self.command_name;
+    let command_name = if cfg!(windows) && self.command_name == "npm" {
+      "npm.cmd"
+    } else {
+      &self.command_name
+    };
     if command_name == "deno" {
       deno_exe_path()
     } else {
@@ -407,11 +411,11 @@ impl TestCommandBuilder {
       command.env_clear();
     }
     command.env("DENO_DIR", self.context.deno_dir.path());
-    let envs = self.build_envs();
+    let mut envs = self.build_envs();
     if !envs.contains_key("NPM_CONFIG_REGISTRY") {
-      command.env("NPM_CONFIG_REGISTRY", npm_registry_unset_url());
+      envs.insert("NPM_CONFIG_REGISTRY".to_string(), npm_registry_unset_url());
     }
-    command.envs(self.build_envs());
+    command.envs(envs);
     command.current_dir(cwd);
     command.stdin(Stdio::piped());
 
@@ -527,6 +531,7 @@ impl Drop for TestCommandOutput {
 
     // now ensure the exit code was asserted
     if !*self.asserted_exit_code.borrow() && self.exit_code != Some(0) {
+      self.print_output();
       panic!(
         "The non-zero exit code of the command was not asserted: {:?}",
         self.exit_code,
diff --git a/test_util/src/lib.rs b/test_util/src/lib.rs
index 07ed55822..9f764007c 100644
--- a/test_util/src/lib.rs
+++ b/test_util/src/lib.rs
@@ -37,7 +37,9 @@ use std::env;
 use std::io;
 use std::io::Write;
 use std::mem::replace;
+use std::net::Ipv6Addr;
 use std::net::SocketAddr;
+use std::net::SocketAddrV6;
 use std::ops::Deref;
 use std::ops::DerefMut;
 use std::path::Path;
@@ -1316,15 +1318,18 @@ async fn main_server(
     }
     _ => {
       let mut file_path = testdata_path().to_path_buf();
-      file_path.push(&req.uri().path()[1..]);
+      file_path.push(&req.uri().path()[1..].replace("%2f", "/"));
       if let Ok(file) = tokio::fs::read(&file_path).await {
         let file_resp = custom_headers(req.uri().path(), file);
         return Ok(file_resp);
       }
 
       // serve npm registry files
-      if let Some(suffix) =
-        req.uri().path().strip_prefix("/npm/registry/@denotest/")
+      if let Some(suffix) = req
+        .uri()
+        .path()
+        .strip_prefix("/npm/registry/@denotest/")
+        .or_else(|| req.uri().path().strip_prefix("/npm/registry/@denotest%2f"))
       {
         // serve all requests to /npm/registry/@deno using the file system
         // at that path
@@ -1571,10 +1576,22 @@ async fn wrap_abs_redirect_server() {
 }
 
 async fn wrap_main_server() {
+  let main_server_addr = SocketAddr::from(([127, 0, 0, 1], PORT));
+  wrap_main_server_for_addr(&main_server_addr).await
+}
+
+// necessary because on Windows the npm binary will resolve localhost to ::1
+async fn wrap_main_ipv6_server() {
+  let ipv6_loopback = Ipv6Addr::new(0, 0, 0, 0, 0, 0, 0, 1);
+  let main_server_addr =
+    SocketAddr::V6(SocketAddrV6::new(ipv6_loopback, PORT, 0, 0));
+  wrap_main_server_for_addr(&main_server_addr).await
+}
+
+async fn wrap_main_server_for_addr(main_server_addr: &SocketAddr) {
   let main_server_svc =
     make_service_fn(|_| async { Ok::<_, Infallible>(service_fn(main_server)) });
-  let main_server_addr = SocketAddr::from(([127, 0, 0, 1], PORT));
-  let main_server = Server::bind(&main_server_addr).serve(main_server_svc);
+  let main_server = Server::bind(main_server_addr).serve(main_server_svc);
   if let Err(e) = main_server.await {
     eprintln!("HTTP server error: {e:?}");
   }
@@ -1922,6 +1939,7 @@ pub async fn run_all_servers() {
   let tls_client_auth_server_fut = run_tls_client_auth_server();
   let client_auth_server_https_fut = wrap_client_auth_https_server();
   let main_server_fut = wrap_main_server();
+  let main_server_ipv6_fut = wrap_main_ipv6_server();
   let main_server_https_fut = wrap_main_https_server();
   let h1_only_server_tls_fut = wrap_https_h1_only_tls_server();
   let h2_only_server_tls_fut = wrap_https_h2_only_tls_server();
@@ -1945,6 +1963,7 @@ pub async fn run_all_servers() {
       double_redirects_server_fut,
       abs_redirect_server_fut,
       main_server_fut,
+      main_server_ipv6_fut,
       main_server_https_fut,
       client_auth_server_https_fut,
       h1_only_server_tls_fut,
diff --git a/test_util/src/npm.rs b/test_util/src/npm.rs
index 98308ae21..9cbadad5c 100644
--- a/test_util/src/npm.rs
+++ b/test_util/src/npm.rs
@@ -104,7 +104,7 @@ fn get_npm_package(package_name: &str) -> Result<Option<CustomNpmPackage>> {
     let mut hash_ctx = Context::new(&SHA512);
     hash_ctx.update(&tarball_bytes);
     let digest = hash_ctx.finish();
-    let tarball_checksum = base64::encode(digest.as_ref()).to_lowercase();
+    let tarball_checksum = base64::encode(digest.as_ref());
 
     // create the registry file JSON for this version
     let mut dist = serde_json::Map::new();