Merge branch 'denoland:main' into main

5 files changed, 307 insertions, 19 deletions

diff --git a/tests/util/server/src/servers/hyper_utils.rs b/tests/util/server/src/servers/hyper_utils.rs
index c2db7ea66..8e01151ed 100644
--- a/tests/util/server/src/servers/hyper_utils.rs
+++ b/tests/util/server/src/servers/hyper_utils.rs
@@ -42,7 +42,10 @@ where
   let fut: Pin<Box<dyn Future<Output = Result<(), anyhow::Error>>>> =
     async move {
       let listener = TcpListener::bind(options.addr).await?;
-      println!("ready: {}", options.addr);
+      #[allow(clippy::print_stdout)]
+      {
+        println!("ready: {}", options.addr);
+      }
       loop {
         let (stream, _) = listener.accept().await?;
         let io = TokioIo::new(stream);
@@ -58,6 +61,7 @@ where
 
   if let Err(e) = fut.await {
     let err_str = e.to_string();
+    #[allow(clippy::print_stderr)]
     if !err_str.contains("early eof") {
       eprintln!("{}: {:?}", options.error_msg, e);
     }
@@ -89,6 +93,7 @@ pub async fn run_server_with_acceptor<'a, A, F, S>(
 
   if let Err(e) = fut.await {
     let err_str = e.to_string();
+    #[allow(clippy::print_stderr)]
     if !err_str.contains("early eof") {
       eprintln!("{}: {:?}", error_msg, e);
     }
@@ -135,6 +140,7 @@ async fn hyper_serve_connection<I, F, S>(
 
   if let Err(e) = result {
     let err_str = e.to_string();
+    #[allow(clippy::print_stderr)]
     if !err_str.contains("early eof") {
       eprintln!("{}: {:?}", error_msg, e);
     }
diff --git a/tests/util/server/src/servers/mod.rs b/tests/util/server/src/servers/mod.rs
index 3e18aafce..0b1d99aeb 100644
--- a/tests/util/server/src/servers/mod.rs
+++ b/tests/util/server/src/servers/mod.rs
@@ -39,6 +39,7 @@ use tokio::net::TcpStream;
 mod grpc;
 mod hyper_utils;
 mod jsr_registry;
+mod nodejs_org_mirror;
 mod npm_registry;
 mod ws;
 
@@ -86,11 +87,13 @@ const WS_CLOSE_PORT: u16 = 4244;
 const WS_PING_PORT: u16 = 4245;
 const H2_GRPC_PORT: u16 = 4246;
 const H2S_GRPC_PORT: u16 = 4247;
-const JSR_REGISTRY_SERVER_PORT: u16 = 4250;
-const PROVENANCE_MOCK_SERVER_PORT: u16 = 4251;
+pub(crate) const JSR_REGISTRY_SERVER_PORT: u16 = 4250;
+pub(crate) const PROVENANCE_MOCK_SERVER_PORT: u16 = 4251;
+pub(crate) const NODEJS_ORG_MIRROR_SERVER_PORT: u16 = 4252;
 pub(crate) const PUBLIC_NPM_REGISTRY_PORT: u16 = 4260;
 pub(crate) const PRIVATE_NPM_REGISTRY_1_PORT: u16 = 4261;
 pub(crate) const PRIVATE_NPM_REGISTRY_2_PORT: u16 = 4262;
+pub(crate) const PRIVATE_NPM_REGISTRY_3_PORT: u16 = 4263;
 
 // Use the single-threaded scheduler. The hyper server is used as a point of
 // comparison for the (single-threaded!) benchmarks in cli/bench. We're not
@@ -143,6 +146,12 @@ pub async fn run_all_servers() {
     npm_registry::private_npm_registry1(PRIVATE_NPM_REGISTRY_1_PORT);
   let private_npm_registry_2_server_futs =
     npm_registry::private_npm_registry2(PRIVATE_NPM_REGISTRY_2_PORT);
+  let private_npm_registry_3_server_futs =
+    npm_registry::private_npm_registry3(PRIVATE_NPM_REGISTRY_3_PORT);
+
+  // for serving node header files to node-gyp in tests
+  let node_js_mirror_server_fut =
+    nodejs_org_mirror::nodejs_org_mirror(NODEJS_ORG_MIRROR_SERVER_PORT);
 
   let mut futures = vec![
     redirect_server_fut.boxed_local(),
@@ -169,10 +178,12 @@ pub async fn run_all_servers() {
     h2_grpc_server_fut.boxed_local(),
     registry_server_fut.boxed_local(),
     provenance_mock_server_fut.boxed_local(),
+    node_js_mirror_server_fut.boxed_local(),
   ];
   futures.extend(npm_registry_server_futs);
   futures.extend(private_npm_registry_1_server_futs);
   futures.extend(private_npm_registry_2_server_futs);
+  futures.extend(private_npm_registry_3_server_futs);
 
   assert_eq!(futures.len(), TEST_SERVERS_COUNT);
 
@@ -194,7 +205,6 @@ fn json_body(value: serde_json::Value) -> UnsyncBoxBody<Bytes, Infallible> {
 
 /// Benchmark server that just serves "hello world" responses.
 async fn hyper_hello(port: u16) {
-  println!("hyper hello");
   let addr = SocketAddr::from(([127, 0, 0, 1], port));
   let handler = move |_: Request<hyper::body::Incoming>| async move {
     Ok::<_, anyhow::Error>(Response::new(UnsyncBoxBody::new(
@@ -338,7 +348,10 @@ async fn get_tcp_listener_stream(
     .collect::<Vec<_>>();
 
   // Eye catcher for HttpServerCount
-  println!("ready: {name} on {:?}", addresses);
+  #[allow(clippy::print_stdout)]
+  {
+    println!("ready: {name} on {:?}", addresses);
+  }
 
   futures::stream::select_all(listeners)
 }
@@ -354,7 +367,10 @@ async fn run_tls_client_auth_server(port: u16) {
   while let Some(Ok(mut tls_stream)) = tls.next().await {
     tokio::spawn(async move {
       let Ok(handshake) = tls_stream.handshake().await else {
-        eprintln!("Failed to handshake");
+        #[allow(clippy::print_stderr)]
+        {
+          eprintln!("Failed to handshake");
+        }
         return;
       };
       // We only need to check for the presence of client certificates
@@ -401,7 +417,6 @@ async fn absolute_redirect(
       .collect();
 
     if let Some(url) = query_params.get("redirect_to") {
-      println!("URL: {url:?}");
       let redirect = redirect_resp(url.to_owned());
       return Ok(redirect);
     }
@@ -409,7 +424,6 @@ async fn absolute_redirect(
 
   if path.starts_with("/REDIRECT") {
     let url = &req.uri().path()[9..];
-    println!("URL: {url:?}");
     let redirect = redirect_resp(url.to_string());
     return Ok(redirect);
   }
@@ -793,17 +807,17 @@ async fn main_server(
     (_, "/jsx/jsx-runtime") | (_, "/jsx/jsx-dev-runtime") => {
       let mut res = Response::new(string_body(
         r#"export function jsx(
-          _type,
-          _props,
-          _key,
-          _source,
-          _self,
-        ) {}
-        export const jsxs = jsx;
-        export const jsxDEV = jsx;
-        export const Fragment = Symbol("Fragment");
-        console.log("imported", import.meta.url);
-        "#,
+  _type,
+  _props,
+  _key,
+  _source,
+  _self,
+) {}
+export const jsxs = jsx;
+export const jsxDEV = jsx;
+export const Fragment = Symbol("Fragment");
+console.log("imported", import.meta.url);
+"#,
       ));
       res.headers_mut().insert(
         "Content-type",
@@ -1353,6 +1367,7 @@ async fn wrap_client_auth_https_server(port: u16) {
       // here. Rusttls ensures that they are valid and signed by the CA.
       match handshake.has_peer_certificates {
         true => { yield Ok(tls); },
+        #[allow(clippy::print_stderr)]
         false => { eprintln!("https_client_auth: no valid client certificate"); },
       };
     }
diff --git a/tests/util/server/src/servers/nodejs_org_mirror.rs b/tests/util/server/src/servers/nodejs_org_mirror.rs
new file mode 100644
index 000000000..521e79d3c
--- /dev/null
+++ b/tests/util/server/src/servers/nodejs_org_mirror.rs
@@ -0,0 +1,245 @@
+// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
+
+//! Server for NodeJS header tarballs, used by `node-gyp` in tests to download headers
+//!
+//! Loads from `testdata/assets`, if we update our node version in `process.versions` we'll need to
+//! update the header tarball there.
+
+#![allow(clippy::print_stderr)]
+
+use std::collections::HashMap;
+use std::convert::Infallible;
+use std::net::SocketAddr;
+use std::sync::LazyLock;
+
+use bytes::Bytes;
+use http::Response;
+use http::StatusCode;
+use http_body_util::combinators::UnsyncBoxBody;
+use http_body_util::Full;
+use parking_lot::Mutex;
+
+use crate::servers::hyper_utils::run_server;
+use crate::servers::hyper_utils::ServerKind;
+use crate::servers::hyper_utils::ServerOptions;
+use crate::servers::string_body;
+use crate::testdata_path;
+use crate::PathRef;
+
+/// a little helper extension trait to log errors but convert to option
+trait OkWarn<T, E> {
+  fn ok_warn(self) -> Option<T>;
+}
+
+impl<T, E> OkWarn<T, E> for Result<T, E>
+where
+  E: std::fmt::Display,
+{
+  fn ok_warn(self) -> Option<T> {
+    self
+      .inspect_err(|err| {
+        eprintln!(
+          "test_server warning: error occurred in nodejs_org_mirror.rs: {err}"
+        )
+      })
+      .ok()
+  }
+}
+
+pub static NODEJS_MIRROR: LazyLock<NodeJsMirror> =
+  LazyLock::new(NodeJsMirror::default);
+
+#[derive(Default)]
+pub struct NodeJsMirror {
+  cache: Mutex<HashMap<String, Bytes>>,
+  checksum_cache: Mutex<HashMap<String, String>>,
+}
+
+fn asset_file_path(file: &str) -> PathRef {
+  testdata_path().join("assets").join("node-gyp").join(file)
+}
+
+impl NodeJsMirror {
+  pub fn get_header_bytes(&self, file: &str) -> Option<Bytes> {
+    let mut cache = self.cache.lock();
+    let entry = cache.entry(file.to_owned());
+    match entry {
+      std::collections::hash_map::Entry::Occupied(occupied) => {
+        Some(occupied.get().clone())
+      }
+      std::collections::hash_map::Entry::Vacant(vacant) => {
+        let contents = asset_file_path(file);
+        let contents = contents
+          .read_to_bytes_if_exists()
+          .ok_warn()
+          .map(Bytes::from)?;
+        vacant.insert(contents.clone());
+        Some(contents)
+      }
+    }
+  }
+
+  fn get_checksum(&self, file: &str, bytes: Bytes) -> String {
+    use sha2::Digest;
+    if let Some(checksum) = self.checksum_cache.lock().get(file).cloned() {
+      return checksum;
+    }
+    let mut hasher = sha2::Sha256::new();
+    hasher.update(&bytes);
+    let checksum = faster_hex::hex_string(hasher.finalize().as_ref());
+    self
+      .checksum_cache
+      .lock()
+      .insert(file.to_owned(), checksum.clone());
+    checksum
+  }
+
+  pub fn get_checksum_file(&self, version: &str) -> Option<String> {
+    let mut entries = Vec::with_capacity(2);
+
+    let header_file = header_tar_name(version);
+    let header_bytes = self.get_header_bytes(&header_file)?;
+    let header_checksum = self.get_checksum(&header_file, header_bytes);
+    entries.push((header_file, header_checksum));
+
+    if cfg!(windows) {
+      if !cfg!(target_arch = "x86_64") {
+        panic!("unsupported target arch on windows, only support x86_64");
+      }
+      let Some(bytes) = self.get_node_lib_bytes(version, "win-x64") else {
+        eprintln!("test server failed to get node lib");
+        return None;
+      };
+      {
+        let file = format!("{version}/win-x64/node.lib");
+        let checksum = self.get_checksum(&file, bytes);
+        let filename_for_checksum =
+          file.trim_start_matches(&format!("{version}/"));
+        entries.push((filename_for_checksum.to_owned(), checksum));
+      }
+    }
+
+    Some(
+      entries
+        .into_iter()
+        .map(|(file, checksum)| format!("{checksum}  {file}"))
+        .collect::<Vec<_>>()
+        .join("\n"),
+    )
+  }
+
+  pub fn get_node_lib_bytes(
+    &self,
+    version: &str,
+    platform: &str,
+  ) -> Option<Bytes> {
+    let mut cache = self.cache.lock();
+    let file_name = format!("{version}/{platform}/node.lib");
+    let entry = cache.entry(file_name);
+    match entry {
+      std::collections::hash_map::Entry::Occupied(occupied) => {
+        Some(occupied.get().clone())
+      }
+      std::collections::hash_map::Entry::Vacant(vacant) => {
+        let tarball_filename =
+          format!("{version}__{platform}__node.lib.tar.gz");
+        let contents = asset_file_path(&tarball_filename);
+        let contents = contents.read_to_bytes_if_exists().ok_warn()?;
+        let extracted = Bytes::from(extract_tarball(&contents)?);
+        vacant.insert(extracted.clone());
+        Some(extracted)
+      }
+    }
+  }
+}
+
+fn header_tar_name(version: &str) -> String {
+  format!("node-{version}-headers.tar.gz")
+}
+
+fn extract_tarball(compressed: &[u8]) -> Option<Vec<u8>> {
+  let mut out = Vec::with_capacity(compressed.len());
+  let decoder = flate2::read::GzDecoder::new(compressed);
+  let mut archive = tar::Archive::new(decoder);
+  for file in archive.entries().ok_warn()? {
+    let mut file = file.ok_warn()?;
+
+    std::io::copy(&mut file, &mut out).ok_warn()?;
+  }
+  Some(out)
+}
+
+/// Server for node JS header tarballs, used by `node-gyp` in tests
+pub async fn nodejs_org_mirror(port: u16) {
+  let addr = SocketAddr::from(([127, 0, 0, 1], port));
+
+  run_server(
+    ServerOptions {
+      addr,
+      error_msg: "nodejs mirror server error",
+      kind: ServerKind::Auto,
+    },
+    |req| async move {
+      let path = req.uri().path();
+      if path.contains("-headers.tar.gz")
+        || path.contains("SHASUMS256.txt")
+        || path.contains("node.lib")
+      {
+        let mut parts = path.split('/');
+        let _ = parts.next(); // empty
+        let Some(version) = parts.next() else {
+          return not_found(format!("missing node version in path: {path}"));
+        };
+        let Some(file) = parts.next() else {
+          return not_found(format!("missing file version in path: {path}"));
+        };
+        if file == "SHASUMS256.txt" {
+          let Some(checksum_file) = NODEJS_MIRROR.get_checksum_file(version)
+          else {
+            return not_found(format!("failed to get header checksum: {path}"));
+          };
+          return Ok(Response::new(string_body(&checksum_file)));
+        } else if !file.contains("headers") {
+          let platform = file;
+          let Some(file) = parts.next() else {
+            return not_found("expected file");
+          };
+          if file != "node.lib" {
+            return not_found(format!(
+              "unexpected file name, expected node.lib, got: {file}"
+            ));
+          }
+          let Some(bytes) = NODEJS_MIRROR.get_node_lib_bytes(version, platform)
+          else {
+            return not_found("expected node lib bytes");
+          };
+
+          return Ok(Response::new(UnsyncBoxBody::new(Full::new(bytes))));
+        }
+
+        let Some(bytes) = NODEJS_MIRROR.get_header_bytes(file) else {
+          return not_found(format!(
+            "couldn't find headers for version {version}, missing file: {file}"
+          ));
+        };
+        Ok(Response::new(UnsyncBoxBody::new(Full::new(bytes))))
+      } else {
+        not_found(format!("unexpected request path: {path}"))
+      }
+    },
+  )
+  .await
+}
+
+fn not_found(
+  msg: impl AsRef<str>,
+) -> Result<Response<UnsyncBoxBody<Bytes, Infallible>>, anyhow::Error> {
+  let msg = msg.as_ref();
+  eprintln!(
+    "test_server warning: error likely occurred in nodejs_org_mirror.rs: {msg}"
+  );
+  Response::builder()
+    .status(StatusCode::NOT_FOUND)
+    .body(string_body(msg))
+    .map_err(|e| e.into())
+}
diff --git a/tests/util/server/src/servers/npm_registry.rs b/tests/util/server/src/servers/npm_registry.rs
index acbd9cab4..4ada468fa 100644
--- a/tests/util/server/src/servers/npm_registry.rs
+++ b/tests/util/server/src/servers/npm_registry.rs
@@ -56,6 +56,14 @@ pub fn private_npm_registry2(port: u16) -> Vec<LocalBoxFuture<'static, ()>> {
   )
 }
 
+pub fn private_npm_registry3(port: u16) -> Vec<LocalBoxFuture<'static, ()>> {
+  run_npm_server(
+    port,
+    "npm private registry server error",
+    private_npm_registry3_handler,
+  )
+}
+
 fn run_npm_server<F, S>(
   port: u16,
   error_msg: &'static str,
@@ -141,6 +149,13 @@ async fn private_npm_registry2_handler(
   handle_req_for_registry(req, &npm::PRIVATE_TEST_NPM_REGISTRY_2).await
 }
 
+async fn private_npm_registry3_handler(
+  req: Request<hyper::body::Incoming>,
+) -> Result<Response<UnsyncBoxBody<Bytes, Infallible>>, anyhow::Error> {
+  // No auth for this registry
+  handle_req_for_registry(req, &npm::PRIVATE_TEST_NPM_REGISTRY_3).await
+}
+
 async fn handle_req_for_registry(
   req: Request<Incoming>,
   test_npm_registry: &npm::TestNpmRegistry,
diff --git a/tests/util/server/src/servers/ws.rs b/tests/util/server/src/servers/ws.rs
index 815119b6a..dd4efbf65 100644
--- a/tests/util/server/src/servers/ws.rs
+++ b/tests/util/server/src/servers/ws.rs
@@ -76,6 +76,7 @@ pub async fn run_wss2_server(port: u16) {
       let server: Handshake<_, Bytes> = h2.handshake(tls);
       let mut server = match server.await {
         Ok(server) => server,
+        #[allow(clippy::print_stdout)]
         Err(e) => {
           println!("Failed to handshake h2: {e:?}");
           return;
@@ -87,6 +88,7 @@ pub async fn run_wss2_server(port: u16) {
         };
         let (recv, send) = match conn {
           Ok(conn) => conn,
+          #[allow(clippy::print_stdout)]
           Err(e) => {
             println!("Failed to accept a connection: {e:?}");
             break;
@@ -137,6 +139,7 @@ where
           .map_err(|e| anyhow!("Error upgrading websocket connection: {}", e))
           .unwrap();
 
+        #[allow(clippy::print_stderr)]
         if let Err(e) = handler(ws).await {
           eprintln!("Error in websocket connection: {}", e);
         }
@@ -152,6 +155,7 @@ where
       .serve_connection(io, service)
       .with_upgrades();
 
+    #[allow(clippy::print_stderr)]
     if let Err(e) = conn.await {
       eprintln!("websocket server error: {e:?}");
     }
@@ -162,16 +166,19 @@ async fn handle_wss_stream(
   recv: Request<RecvStream>,
   mut send: SendResponse<Bytes>,
 ) -> Result<(), h2::Error> {
+  #[allow(clippy::print_stderr)]
   if recv.method() != Method::CONNECT {
     eprintln!("wss2: refusing non-CONNECT stream");
     send.send_reset(Reason::REFUSED_STREAM);
     return Ok(());
   }
+  #[allow(clippy::print_stderr)]
   let Some(protocol) = recv.extensions().get::<h2::ext::Protocol>() else {
     eprintln!("wss2: refusing no-:protocol stream");
     send.send_reset(Reason::REFUSED_STREAM);
     return Ok(());
   };
+  #[allow(clippy::print_stderr)]
   if protocol.as_str() != "websocket" && protocol.as_str() != "WebSocket" {
     eprintln!("wss2: refusing non-websocket stream");
     send.send_reset(Reason::REFUSED_STREAM);