From 473e3069de4bf5877a6f1140aa0462e05f745536 Mon Sep 17 00:00:00 2001 From: Leo Kettmeir Date: Sat, 19 Oct 2024 14:59:39 -0700 Subject: chore: update nix crate (#26422) Dedupes nix dependency, since `rustyline` depends on a newer version that what we currently use --- tests/util/server/src/pty.rs | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'tests/util/server/src') diff --git a/tests/util/server/src/pty.rs b/tests/util/server/src/pty.rs index 5d8049fee..8d42fed78 100644 --- a/tests/util/server/src/pty.rs +++ b/tests/util/server/src/pty.rs @@ -297,10 +297,12 @@ fn setup_pty(fd: i32) { use nix::sys::termios::tcsetattr; use nix::sys::termios::SetArg; - let mut term = tcgetattr(fd).unwrap(); + // SAFETY: Nix crate requires value to implement the AsFd trait + let as_fd = unsafe { std::os::fd::BorrowedFd::borrow_raw(fd) }; + let mut term = tcgetattr(as_fd).unwrap(); // disable cooked mode term.local_flags.remove(termios::LocalFlags::ICANON); - tcsetattr(fd, SetArg::TCSANOW, &term).unwrap(); + tcsetattr(as_fd, SetArg::TCSANOW, &term).unwrap(); // turn on non-blocking mode so we get timeouts let flags = fcntl(fd, FcntlArg::F_GETFL).unwrap(); -- cgit v1.2.3 From 9fe2bf42dc584779cc43f0ec15a5a3d6dddca283 Mon Sep 17 00:00:00 2001 From: Nayeem Rahman Date: Mon, 21 Oct 2024 17:15:52 +0100 Subject: feat(lsp): interactive inlay hints (#26382) --- tests/util/server/src/lsp.rs | 28 ---------------------------- 1 file changed, 28 deletions(-) (limited to 'tests/util/server/src') diff --git a/tests/util/server/src/lsp.rs b/tests/util/server/src/lsp.rs index ffe72b88a..4e75cfadb 100644 --- a/tests/util/server/src/lsp.rs +++ b/tests/util/server/src/lsp.rs @@ -308,34 +308,6 @@ impl InitializeParamsBuilder { self } - pub fn enable_inlay_hints(&mut self) -> &mut Self { - let options = self.initialization_options_mut(); - options.insert( - "inlayHints".to_string(), - json!({ - "parameterNames": { - "enabled": "all" - }, - "parameterTypes": { - "enabled": true - }, - "variableTypes": { - "enabled": true - }, - "propertyDeclarationTypes": { - "enabled": true - }, - "functionLikeReturnTypes": { - "enabled": true - }, - "enumMemberValues": { - "enabled": true - } - }), - ); - self - } - pub fn disable_testing_api(&mut self) -> &mut Self { let obj = self .params -- cgit v1.2.3 From 5f0bb3c6f4328003012e98ba70ce18e4e2e842de Mon Sep 17 00:00:00 2001 From: Marvin Hagemeister Date: Thu, 24 Oct 2024 20:03:56 +0200 Subject: fix: `.npmrc` settings not being passed to install/add command (#26473) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We weren't passing the resolved npmrc settings to the install commands. This lead us to always fall back to the default registry url instead of using the one from npmrc. Fixes https://github.com/denoland/deno/issues/26139 Fixes https://github.com/denoland/deno/issues/26033 Fixes https://github.com/denoland/deno/issues/25924 Fixes https://github.com/denoland/deno/issues/25822 Fixes https://github.com/denoland/deno/issues/26152 --------- Co-authored-by: Bartek Iwańczuk --- tests/util/server/src/lib.rs | 3 ++- tests/util/server/src/npm.rs | 32 +++++++++++++++++++++++++++ tests/util/server/src/servers/mod.rs | 4 ++++ tests/util/server/src/servers/npm_registry.rs | 15 +++++++++++++ 4 files changed, 53 insertions(+), 1 deletion(-) (limited to 'tests/util/server/src') diff --git a/tests/util/server/src/lib.rs b/tests/util/server/src/lib.rs index 88e8287e0..e4a2cc02e 100644 --- a/tests/util/server/src/lib.rs +++ b/tests/util/server/src/lib.rs @@ -307,7 +307,7 @@ async fn get_tcp_listener_stream( futures::stream::select_all(listeners) } -pub const TEST_SERVERS_COUNT: usize = 30; +pub const TEST_SERVERS_COUNT: usize = 32; #[derive(Default)] struct HttpServerCount { @@ -360,6 +360,7 @@ impl Default for HttpServerStarter { let mut ready_count = 0; for maybe_line in lines { if let Ok(line) = maybe_line { + eprintln!("LINE: {}", line); if line.starts_with("ready:") { ready_count += 1; } diff --git a/tests/util/server/src/npm.rs b/tests/util/server/src/npm.rs index f1c341738..4b17b95f7 100644 --- a/tests/util/server/src/npm.rs +++ b/tests/util/server/src/npm.rs @@ -18,6 +18,7 @@ use crate::PathRef; pub const DENOTEST_SCOPE_NAME: &str = "@denotest"; pub const DENOTEST2_SCOPE_NAME: &str = "@denotest2"; +pub const DENOTEST3_SCOPE_NAME: &str = "@denotest3"; pub static PUBLIC_TEST_NPM_REGISTRY: Lazy = Lazy::new(|| { TestNpmRegistry::new( @@ -54,6 +55,18 @@ pub static PRIVATE_TEST_NPM_REGISTRY_2: Lazy = ) }); +pub static PRIVATE_TEST_NPM_REGISTRY_3: Lazy = + Lazy::new(|| { + TestNpmRegistry::new( + NpmRegistryKind::Private, + &format!( + "http://localhost:{}", + crate::servers::PRIVATE_NPM_REGISTRY_3_PORT + ), + "npm-private3", + ) + }); + pub enum NpmRegistryKind { Public, Private, @@ -90,6 +103,7 @@ impl TestNpmRegistry { } pub fn root_dir(&self) -> PathRef { + eprintln!("root {}", self.local_path); tests_path().join("registry").join(&self.local_path) } @@ -106,6 +120,7 @@ impl TestNpmRegistry { } pub fn registry_file(&self, name: &str) -> Result>> { + eprintln!("registry file {}", name); self.get_package_property(name, |p| p.registry_file.as_bytes().to_vec()) } @@ -123,6 +138,7 @@ impl TestNpmRegistry { package_name: &str, func: impl FnOnce(&CustomNpmPackage) -> TResult, ) -> Result> { + eprintln!("get package property {}", package_name); // it's ok if multiple threads race here as they will do the same work twice if !self.cache.lock().contains_key(package_name) { match get_npm_package(&self.hostname, &self.local_path, package_name)? { @@ -139,6 +155,7 @@ impl TestNpmRegistry { &self, uri_path: &'s str, ) -> Option<(&'s str, &'s str)> { + eprintln!("GEETT {}", uri_path); let prefix1 = format!("/{}/", DENOTEST_SCOPE_NAME); let prefix2 = format!("/{}%2f", DENOTEST_SCOPE_NAME); @@ -161,6 +178,17 @@ impl TestNpmRegistry { return Some((DENOTEST2_SCOPE_NAME, package_name_with_path)); } + let prefix1 = format!("/{}/", DENOTEST3_SCOPE_NAME); + let prefix2 = format!("/{}%2f", DENOTEST3_SCOPE_NAME); + + let maybe_package_name_with_path = uri_path + .strip_prefix(&prefix1) + .or_else(|| uri_path.strip_prefix(&prefix2)); + + if let Some(package_name_with_path) = maybe_package_name_with_path { + return Some((DENOTEST3_SCOPE_NAME, package_name_with_path)); + } + None } } @@ -170,6 +198,10 @@ fn get_npm_package( local_path: &str, package_name: &str, ) -> Result> { + eprintln!( + "get npm package {} {} {}", + registry_hostname, local_path, package_name + ); let registry_hostname = if package_name == "@denotest/tarballs-privateserver2" { "http://localhost:4262" diff --git a/tests/util/server/src/servers/mod.rs b/tests/util/server/src/servers/mod.rs index 3e18aafce..3a9c44010 100644 --- a/tests/util/server/src/servers/mod.rs +++ b/tests/util/server/src/servers/mod.rs @@ -91,6 +91,7 @@ const PROVENANCE_MOCK_SERVER_PORT: u16 = 4251; pub(crate) const PUBLIC_NPM_REGISTRY_PORT: u16 = 4260; pub(crate) const PRIVATE_NPM_REGISTRY_1_PORT: u16 = 4261; pub(crate) const PRIVATE_NPM_REGISTRY_2_PORT: u16 = 4262; +pub(crate) const PRIVATE_NPM_REGISTRY_3_PORT: u16 = 4263; // Use the single-threaded scheduler. The hyper server is used as a point of // comparison for the (single-threaded!) benchmarks in cli/bench. We're not @@ -143,6 +144,8 @@ pub async fn run_all_servers() { npm_registry::private_npm_registry1(PRIVATE_NPM_REGISTRY_1_PORT); let private_npm_registry_2_server_futs = npm_registry::private_npm_registry2(PRIVATE_NPM_REGISTRY_2_PORT); + let private_npm_registry_3_server_futs = + npm_registry::private_npm_registry3(PRIVATE_NPM_REGISTRY_3_PORT); let mut futures = vec![ redirect_server_fut.boxed_local(), @@ -173,6 +176,7 @@ pub async fn run_all_servers() { futures.extend(npm_registry_server_futs); futures.extend(private_npm_registry_1_server_futs); futures.extend(private_npm_registry_2_server_futs); + futures.extend(private_npm_registry_3_server_futs); assert_eq!(futures.len(), TEST_SERVERS_COUNT); diff --git a/tests/util/server/src/servers/npm_registry.rs b/tests/util/server/src/servers/npm_registry.rs index acbd9cab4..4ada468fa 100644 --- a/tests/util/server/src/servers/npm_registry.rs +++ b/tests/util/server/src/servers/npm_registry.rs @@ -56,6 +56,14 @@ pub fn private_npm_registry2(port: u16) -> Vec> { ) } +pub fn private_npm_registry3(port: u16) -> Vec> { + run_npm_server( + port, + "npm private registry server error", + private_npm_registry3_handler, + ) +} + fn run_npm_server( port: u16, error_msg: &'static str, @@ -141,6 +149,13 @@ async fn private_npm_registry2_handler( handle_req_for_registry(req, &npm::PRIVATE_TEST_NPM_REGISTRY_2).await } +async fn private_npm_registry3_handler( + req: Request, +) -> Result>, anyhow::Error> { + // No auth for this registry + handle_req_for_registry(req, &npm::PRIVATE_TEST_NPM_REGISTRY_3).await +} + async fn handle_req_for_registry( req: Request, test_npm_registry: &npm::TestNpmRegistry, -- cgit v1.2.3 From e16230624747eccea7636b20b952df5899719336 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 24 Oct 2024 18:06:17 -0400 Subject: chore: remove print debugging from test server (#26529) Accidentally added in https://github.com/denoland/deno/pull/26473/files --- tests/util/server/src/builders.rs | 1 + tests/util/server/src/fs.rs | 5 ++++- tests/util/server/src/lib.rs | 18 ++++++++++++------ tests/util/server/src/lsp.rs | 18 +++++++++++++----- tests/util/server/src/npm.rs | 8 -------- tests/util/server/src/pty.rs | 18 ++++++++++++------ tests/util/server/src/servers/hyper_utils.rs | 8 +++++++- tests/util/server/src/servers/mod.rs | 14 +++++++++----- tests/util/server/src/servers/ws.rs | 7 +++++++ 9 files changed, 65 insertions(+), 32 deletions(-) (limited to 'tests/util/server/src') diff --git a/tests/util/server/src/builders.rs b/tests/util/server/src/builders.rs index 4a4b6a761..3bf5c704a 100644 --- a/tests/util/server/src/builders.rs +++ b/tests/util/server/src/builders.rs @@ -78,6 +78,7 @@ impl DiagnosticLogger { logger.write_all(text.as_ref().as_bytes()).unwrap(); logger.write_all(b"\n").unwrap(); } + #[allow(clippy::print_stderr)] None => eprintln!("{}", text.as_ref()), } } diff --git a/tests/util/server/src/fs.rs b/tests/util/server/src/fs.rs index 47d0d61fa..7feb0799a 100644 --- a/tests/util/server/src/fs.rs +++ b/tests/util/server/src/fs.rs @@ -285,7 +285,10 @@ impl PathRef { #[track_caller] pub fn assert_matches_file(&self, wildcard_file: impl AsRef) -> &Self { let wildcard_file = testdata_path().join(wildcard_file); - println!("output path {}", wildcard_file); + #[allow(clippy::print_stdout)] + { + println!("output path {}", wildcard_file); + } let expected_text = wildcard_file.read_to_string(); self.assert_matches_text(&expected_text) } diff --git a/tests/util/server/src/lib.rs b/tests/util/server/src/lib.rs index e4a2cc02e..67dbcd820 100644 --- a/tests/util/server/src/lib.rs +++ b/tests/util/server/src/lib.rs @@ -1,8 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -#![allow(clippy::print_stdout)] -#![allow(clippy::print_stderr)] - use std::collections::HashMap; use std::env; use std::io::Write; @@ -302,7 +299,10 @@ async fn get_tcp_listener_stream( .collect::>(); // Eye catcher for HttpServerCount - println!("ready: {name} on {:?}", addresses); + #[allow(clippy::print_stdout)] + { + println!("ready: {name} on {:?}", addresses); + } futures::stream::select_all(listeners) } @@ -345,7 +345,10 @@ struct HttpServerStarter { impl Default for HttpServerStarter { fn default() -> Self { - println!("test_server starting..."); + #[allow(clippy::print_stdout)] + { + println!("test_server starting..."); + } let mut test_server = Command::new(test_server_path()) .current_dir(testdata_path()) .stdout(Stdio::piped()) @@ -360,7 +363,6 @@ impl Default for HttpServerStarter { let mut ready_count = 0; for maybe_line in lines { if let Ok(line) = maybe_line { - eprintln!("LINE: {}", line); if line.starts_with("ready:") { ready_count += 1; } @@ -480,6 +482,7 @@ pub fn run_collect( } = prog.wait_with_output().expect("failed to wait on child"); let stdout = String::from_utf8(stdout).unwrap(); let stderr = String::from_utf8(stderr).unwrap(); + #[allow(clippy::print_stderr)] if expect_success != status.success() { eprintln!("stdout: <<<{stdout}>>>"); eprintln!("stderr: <<<{stderr}>>>"); @@ -540,6 +543,7 @@ pub fn run_and_collect_output_with_args( } = deno.wait_with_output().expect("failed to wait on child"); let stdout = String::from_utf8(stdout).unwrap(); let stderr = String::from_utf8(stderr).unwrap(); + #[allow(clippy::print_stderr)] if expect_success != status.success() { eprintln!("stdout: <<<{stdout}>>>"); eprintln!("stderr: <<<{stderr}>>>"); @@ -564,6 +568,7 @@ pub fn deno_cmd_with_deno_dir(deno_dir: &TempDir) -> TestCommandBuilder { .env("JSR_URL", jsr_registry_unset_url()) } +#[allow(clippy::print_stdout)] pub fn run_powershell_script_file( script_file_path: &str, args: Vec<&str>, @@ -655,6 +660,7 @@ impl<'a> CheckOutputIntegrationTest<'a> { } pub fn wildcard_match(pattern: &str, text: &str) -> bool { + #[allow(clippy::print_stderr)] match wildcard_match_detailed(pattern, text) { WildcardMatchResult::Success => true, WildcardMatchResult::Fail(debug_output) => { diff --git a/tests/util/server/src/lsp.rs b/tests/util/server/src/lsp.rs index 4e75cfadb..d34deb216 100644 --- a/tests/util/server/src/lsp.rs +++ b/tests/util/server/src/lsp.rs @@ -157,6 +157,7 @@ impl LspStdoutReader { self.pending_messages.0.lock().len() } + #[allow(clippy::print_stderr)] pub fn output_pending_messages(&self) { let messages = self.pending_messages.0.lock(); eprintln!("{:?}", messages); @@ -573,6 +574,7 @@ impl LspClientBuilder { for line in stderr.lines() { match line { Ok(line) => { + #[allow(clippy::print_stderr)] if print_stderr { eprintln!("{}", line); } @@ -587,7 +589,10 @@ impl LspClientBuilder { continue; } Err(err) => { - eprintln!("failed to parse perf record: {:#}", err); + #[allow(clippy::print_stderr)] + { + eprintln!("failed to parse perf record: {:#}", err); + } } } } @@ -782,11 +787,14 @@ impl LspClient { std::thread::sleep(Duration::from_millis(20)); } - eprintln!("==== STDERR OUTPUT ===="); - for line in found_lines { - eprintln!("{}", line) + #[allow(clippy::print_stderr)] + { + eprintln!("==== STDERR OUTPUT ===="); + for line in found_lines { + eprintln!("{}", line) + } + eprintln!("== END STDERR OUTPUT =="); } - eprintln!("== END STDERR OUTPUT =="); panic!("Timed out waiting on condition.") } diff --git a/tests/util/server/src/npm.rs b/tests/util/server/src/npm.rs index 4b17b95f7..31686fa85 100644 --- a/tests/util/server/src/npm.rs +++ b/tests/util/server/src/npm.rs @@ -103,7 +103,6 @@ impl TestNpmRegistry { } pub fn root_dir(&self) -> PathRef { - eprintln!("root {}", self.local_path); tests_path().join("registry").join(&self.local_path) } @@ -120,7 +119,6 @@ impl TestNpmRegistry { } pub fn registry_file(&self, name: &str) -> Result>> { - eprintln!("registry file {}", name); self.get_package_property(name, |p| p.registry_file.as_bytes().to_vec()) } @@ -138,7 +136,6 @@ impl TestNpmRegistry { package_name: &str, func: impl FnOnce(&CustomNpmPackage) -> TResult, ) -> Result> { - eprintln!("get package property {}", package_name); // it's ok if multiple threads race here as they will do the same work twice if !self.cache.lock().contains_key(package_name) { match get_npm_package(&self.hostname, &self.local_path, package_name)? { @@ -155,7 +152,6 @@ impl TestNpmRegistry { &self, uri_path: &'s str, ) -> Option<(&'s str, &'s str)> { - eprintln!("GEETT {}", uri_path); let prefix1 = format!("/{}/", DENOTEST_SCOPE_NAME); let prefix2 = format!("/{}%2f", DENOTEST_SCOPE_NAME); @@ -198,10 +194,6 @@ fn get_npm_package( local_path: &str, package_name: &str, ) -> Result> { - eprintln!( - "get npm package {} {} {}", - registry_hostname, local_path, package_name - ); let registry_hostname = if package_name == "@denotest/tarballs-privateserver2" { "http://localhost:4262" diff --git a/tests/util/server/src/pty.rs b/tests/util/server/src/pty.rs index 8d42fed78..07659262c 100644 --- a/tests/util/server/src/pty.rs +++ b/tests/util/server/src/pty.rs @@ -61,7 +61,10 @@ impl Pty { if is_windows && *IS_CI { // the pty tests don't really start up on the windows CI for some reason // so ignore them for now - eprintln!("Ignoring windows CI."); + #[allow(clippy::print_stderr)] + { + eprintln!("Ignoring windows CI."); + } false } else { true @@ -250,11 +253,14 @@ impl Pty { } let text = self.next_text(); - eprintln!( - "------ Start Full Text ------\n{:?}\n------- End Full Text -------", - String::from_utf8_lossy(&self.read_bytes) - ); - eprintln!("Next text: {:?}", text); + #[allow(clippy::print_stderr)] + { + eprintln!( + "------ Start Full Text ------\n{:?}\n------- End Full Text -------", + String::from_utf8_lossy(&self.read_bytes) + ); + eprintln!("Next text: {:?}", text); + } false } diff --git a/tests/util/server/src/servers/hyper_utils.rs b/tests/util/server/src/servers/hyper_utils.rs index c2db7ea66..8e01151ed 100644 --- a/tests/util/server/src/servers/hyper_utils.rs +++ b/tests/util/server/src/servers/hyper_utils.rs @@ -42,7 +42,10 @@ where let fut: Pin>>> = async move { let listener = TcpListener::bind(options.addr).await?; - println!("ready: {}", options.addr); + #[allow(clippy::print_stdout)] + { + println!("ready: {}", options.addr); + } loop { let (stream, _) = listener.accept().await?; let io = TokioIo::new(stream); @@ -58,6 +61,7 @@ where if let Err(e) = fut.await { let err_str = e.to_string(); + #[allow(clippy::print_stderr)] if !err_str.contains("early eof") { eprintln!("{}: {:?}", options.error_msg, e); } @@ -89,6 +93,7 @@ pub async fn run_server_with_acceptor<'a, A, F, S>( if let Err(e) = fut.await { let err_str = e.to_string(); + #[allow(clippy::print_stderr)] if !err_str.contains("early eof") { eprintln!("{}: {:?}", error_msg, e); } @@ -135,6 +140,7 @@ async fn hyper_serve_connection( if let Err(e) = result { let err_str = e.to_string(); + #[allow(clippy::print_stderr)] if !err_str.contains("early eof") { eprintln!("{}: {:?}", error_msg, e); } diff --git a/tests/util/server/src/servers/mod.rs b/tests/util/server/src/servers/mod.rs index 3a9c44010..6c960ce52 100644 --- a/tests/util/server/src/servers/mod.rs +++ b/tests/util/server/src/servers/mod.rs @@ -198,7 +198,6 @@ fn json_body(value: serde_json::Value) -> UnsyncBoxBody { /// Benchmark server that just serves "hello world" responses. async fn hyper_hello(port: u16) { - println!("hyper hello"); let addr = SocketAddr::from(([127, 0, 0, 1], port)); let handler = move |_: Request| async move { Ok::<_, anyhow::Error>(Response::new(UnsyncBoxBody::new( @@ -342,7 +341,10 @@ async fn get_tcp_listener_stream( .collect::>(); // Eye catcher for HttpServerCount - println!("ready: {name} on {:?}", addresses); + #[allow(clippy::print_stdout)] + { + println!("ready: {name} on {:?}", addresses); + } futures::stream::select_all(listeners) } @@ -358,7 +360,10 @@ async fn run_tls_client_auth_server(port: u16) { while let Some(Ok(mut tls_stream)) = tls.next().await { tokio::spawn(async move { let Ok(handshake) = tls_stream.handshake().await else { - eprintln!("Failed to handshake"); + #[allow(clippy::print_stderr)] + { + eprintln!("Failed to handshake"); + } return; }; // We only need to check for the presence of client certificates @@ -405,7 +410,6 @@ async fn absolute_redirect( .collect(); if let Some(url) = query_params.get("redirect_to") { - println!("URL: {url:?}"); let redirect = redirect_resp(url.to_owned()); return Ok(redirect); } @@ -413,7 +417,6 @@ async fn absolute_redirect( if path.starts_with("/REDIRECT") { let url = &req.uri().path()[9..]; - println!("URL: {url:?}"); let redirect = redirect_resp(url.to_string()); return Ok(redirect); } @@ -1357,6 +1360,7 @@ async fn wrap_client_auth_https_server(port: u16) { // here. Rusttls ensures that they are valid and signed by the CA. match handshake.has_peer_certificates { true => { yield Ok(tls); }, + #[allow(clippy::print_stderr)] false => { eprintln!("https_client_auth: no valid client certificate"); }, }; } diff --git a/tests/util/server/src/servers/ws.rs b/tests/util/server/src/servers/ws.rs index 815119b6a..dd4efbf65 100644 --- a/tests/util/server/src/servers/ws.rs +++ b/tests/util/server/src/servers/ws.rs @@ -76,6 +76,7 @@ pub async fn run_wss2_server(port: u16) { let server: Handshake<_, Bytes> = h2.handshake(tls); let mut server = match server.await { Ok(server) => server, + #[allow(clippy::print_stdout)] Err(e) => { println!("Failed to handshake h2: {e:?}"); return; @@ -87,6 +88,7 @@ pub async fn run_wss2_server(port: u16) { }; let (recv, send) = match conn { Ok(conn) => conn, + #[allow(clippy::print_stdout)] Err(e) => { println!("Failed to accept a connection: {e:?}"); break; @@ -137,6 +139,7 @@ where .map_err(|e| anyhow!("Error upgrading websocket connection: {}", e)) .unwrap(); + #[allow(clippy::print_stderr)] if let Err(e) = handler(ws).await { eprintln!("Error in websocket connection: {}", e); } @@ -152,6 +155,7 @@ where .serve_connection(io, service) .with_upgrades(); + #[allow(clippy::print_stderr)] if let Err(e) = conn.await { eprintln!("websocket server error: {e:?}"); } @@ -162,16 +166,19 @@ async fn handle_wss_stream( recv: Request, mut send: SendResponse, ) -> Result<(), h2::Error> { + #[allow(clippy::print_stderr)] if recv.method() != Method::CONNECT { eprintln!("wss2: refusing non-CONNECT stream"); send.send_reset(Reason::REFUSED_STREAM); return Ok(()); } + #[allow(clippy::print_stderr)] let Some(protocol) = recv.extensions().get::() else { eprintln!("wss2: refusing no-:protocol stream"); send.send_reset(Reason::REFUSED_STREAM); return Ok(()); }; + #[allow(clippy::print_stderr)] if protocol.as_str() != "websocket" && protocol.as_str() != "WebSocket" { eprintln!("wss2: refusing non-websocket stream"); send.send_reset(Reason::REFUSED_STREAM); -- cgit v1.2.3 From 742744d4985548a948bc90e78673c0c22d607d8a Mon Sep 17 00:00:00 2001 From: Nathan Whitaker <17734409+nathanwhit@users.noreply.github.com> Date: Wed, 6 Nov 2024 19:52:46 -0800 Subject: chore: serve node headers from a test server to fix flaky `node-gyp` test (#26749) Fixes https://github.com/denoland/deno/issues/24749 Runs a server that just returns the header tarball and checksum, and sets the `NODEJS_ORG_MIRROR` env var so that `node-gyp` uses it instead of `nodejs.org` --- tests/util/server/src/builders.rs | 7 + tests/util/server/src/lib.rs | 29 ++- tests/util/server/src/servers/mod.rs | 11 +- tests/util/server/src/servers/nodejs_org_mirror.rs | 245 +++++++++++++++++++++ 4 files changed, 284 insertions(+), 8 deletions(-) create mode 100644 tests/util/server/src/servers/nodejs_org_mirror.rs (limited to 'tests/util/server/src') diff --git a/tests/util/server/src/builders.rs b/tests/util/server/src/builders.rs index 3bf5c704a..4a1510ce4 100644 --- a/tests/util/server/src/builders.rs +++ b/tests/util/server/src/builders.rs @@ -28,6 +28,7 @@ use crate::fs::PathRef; use crate::http_server; use crate::jsr_registry_unset_url; use crate::lsp::LspClientBuilder; +use crate::nodejs_org_mirror_unset_url; use crate::npm_registry_unset_url; use crate::pty::Pty; use crate::strip_ansi_codes; @@ -843,6 +844,12 @@ impl TestCommandBuilder { if !envs.contains_key("JSR_URL") { envs.insert("JSR_URL".to_string(), jsr_registry_unset_url()); } + if !envs.contains_key("NODEJS_ORG_MIRROR") { + envs.insert( + "NODEJS_ORG_MIRROR".to_string(), + nodejs_org_mirror_unset_url(), + ); + } for key in &self.envs_remove { envs.remove(key); } diff --git a/tests/util/server/src/lib.rs b/tests/util/server/src/lib.rs index 67dbcd820..89dc1ffc3 100644 --- a/tests/util/server/src/lib.rs +++ b/tests/util/server/src/lib.rs @@ -52,6 +52,7 @@ static GUARD: Lazy> = Lazy::new(Default::default); pub fn env_vars_for_npm_tests() -> Vec<(String, String)> { vec![ ("NPM_CONFIG_REGISTRY".to_string(), npm_registry_url()), + ("NODEJS_ORG_MIRROR".to_string(), nodejs_org_mirror_url()), ("NO_COLOR".to_string(), "1".to_string()), ] } @@ -130,6 +131,7 @@ pub fn env_vars_for_jsr_npm_tests() -> Vec<(String, String)> { ), ("DISABLE_JSR_PROVENANCE".to_string(), "true".to_string()), ("NO_COLOR".to_string(), "1".to_string()), + ("NODEJS_ORG_MIRROR".to_string(), nodejs_org_mirror_url()), ] } @@ -175,27 +177,41 @@ pub fn deno_config_path() -> PathRef { /// Test server registry url. pub fn npm_registry_url() -> String { - "http://localhost:4260/".to_string() + format!("http://localhost:{}/", servers::PUBLIC_NPM_REGISTRY_PORT) } pub fn npm_registry_unset_url() -> String { "http://NPM_CONFIG_REGISTRY.is.unset".to_string() } +pub fn nodejs_org_mirror_url() -> String { + format!( + "http://127.0.0.1:{}/", + servers::NODEJS_ORG_MIRROR_SERVER_PORT + ) +} + +pub fn nodejs_org_mirror_unset_url() -> String { + "http://NODEJS_ORG_MIRROR.is.unset".to_string() +} + pub fn jsr_registry_url() -> String { - "http://127.0.0.1:4250/".to_string() + format!("http://127.0.0.1:{}/", servers::JSR_REGISTRY_SERVER_PORT) } pub fn rekor_url() -> String { - "http://127.0.0.1:4251".to_string() + format!("http://127.0.0.1:{}", servers::PROVENANCE_MOCK_SERVER_PORT) } pub fn fulcio_url() -> String { - "http://127.0.0.1:4251".to_string() + format!("http://127.0.0.1:{}", servers::PROVENANCE_MOCK_SERVER_PORT) } pub fn gha_token_url() -> String { - "http://127.0.0.1:4251/gha_oidc?test=true".to_string() + format!( + "http://127.0.0.1:{}/gha_oidc?test=true", + servers::PROVENANCE_MOCK_SERVER_PORT + ) } pub fn jsr_registry_unset_url() -> String { @@ -307,7 +323,7 @@ async fn get_tcp_listener_stream( futures::stream::select_all(listeners) } -pub const TEST_SERVERS_COUNT: usize = 32; +pub const TEST_SERVERS_COUNT: usize = 33; #[derive(Default)] struct HttpServerCount { @@ -565,6 +581,7 @@ pub fn deno_cmd_with_deno_dir(deno_dir: &TempDir) -> TestCommandBuilder { TestCommandBuilder::new(deno_dir.clone()) .env("DENO_DIR", deno_dir.path()) .env("NPM_CONFIG_REGISTRY", npm_registry_unset_url()) + .env("NODEJS_ORG_MIRROR", nodejs_org_mirror_unset_url()) .env("JSR_URL", jsr_registry_unset_url()) } diff --git a/tests/util/server/src/servers/mod.rs b/tests/util/server/src/servers/mod.rs index 6c960ce52..d9adde542 100644 --- a/tests/util/server/src/servers/mod.rs +++ b/tests/util/server/src/servers/mod.rs @@ -39,6 +39,7 @@ use tokio::net::TcpStream; mod grpc; mod hyper_utils; mod jsr_registry; +mod nodejs_org_mirror; mod npm_registry; mod ws; @@ -86,8 +87,9 @@ const WS_CLOSE_PORT: u16 = 4244; const WS_PING_PORT: u16 = 4245; const H2_GRPC_PORT: u16 = 4246; const H2S_GRPC_PORT: u16 = 4247; -const JSR_REGISTRY_SERVER_PORT: u16 = 4250; -const PROVENANCE_MOCK_SERVER_PORT: u16 = 4251; +pub(crate) const JSR_REGISTRY_SERVER_PORT: u16 = 4250; +pub(crate) const PROVENANCE_MOCK_SERVER_PORT: u16 = 4251; +pub(crate) const NODEJS_ORG_MIRROR_SERVER_PORT: u16 = 4252; pub(crate) const PUBLIC_NPM_REGISTRY_PORT: u16 = 4260; pub(crate) const PRIVATE_NPM_REGISTRY_1_PORT: u16 = 4261; pub(crate) const PRIVATE_NPM_REGISTRY_2_PORT: u16 = 4262; @@ -147,6 +149,10 @@ pub async fn run_all_servers() { let private_npm_registry_3_server_futs = npm_registry::private_npm_registry3(PRIVATE_NPM_REGISTRY_3_PORT); + // for serving node header files to node-gyp in tests + let node_js_mirror_server_fut = + nodejs_org_mirror::nodejs_org_mirror(NODEJS_ORG_MIRROR_SERVER_PORT); + let mut futures = vec![ redirect_server_fut.boxed_local(), ws_server_fut.boxed_local(), @@ -172,6 +178,7 @@ pub async fn run_all_servers() { h2_grpc_server_fut.boxed_local(), registry_server_fut.boxed_local(), provenance_mock_server_fut.boxed_local(), + node_js_mirror_server_fut.boxed_local(), ]; futures.extend(npm_registry_server_futs); futures.extend(private_npm_registry_1_server_futs); diff --git a/tests/util/server/src/servers/nodejs_org_mirror.rs b/tests/util/server/src/servers/nodejs_org_mirror.rs new file mode 100644 index 000000000..521e79d3c --- /dev/null +++ b/tests/util/server/src/servers/nodejs_org_mirror.rs @@ -0,0 +1,245 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +//! Server for NodeJS header tarballs, used by `node-gyp` in tests to download headers +//! +//! Loads from `testdata/assets`, if we update our node version in `process.versions` we'll need to +//! update the header tarball there. + +#![allow(clippy::print_stderr)] + +use std::collections::HashMap; +use std::convert::Infallible; +use std::net::SocketAddr; +use std::sync::LazyLock; + +use bytes::Bytes; +use http::Response; +use http::StatusCode; +use http_body_util::combinators::UnsyncBoxBody; +use http_body_util::Full; +use parking_lot::Mutex; + +use crate::servers::hyper_utils::run_server; +use crate::servers::hyper_utils::ServerKind; +use crate::servers::hyper_utils::ServerOptions; +use crate::servers::string_body; +use crate::testdata_path; +use crate::PathRef; + +/// a little helper extension trait to log errors but convert to option +trait OkWarn { + fn ok_warn(self) -> Option; +} + +impl OkWarn for Result +where + E: std::fmt::Display, +{ + fn ok_warn(self) -> Option { + self + .inspect_err(|err| { + eprintln!( + "test_server warning: error occurred in nodejs_org_mirror.rs: {err}" + ) + }) + .ok() + } +} + +pub static NODEJS_MIRROR: LazyLock = + LazyLock::new(NodeJsMirror::default); + +#[derive(Default)] +pub struct NodeJsMirror { + cache: Mutex>, + checksum_cache: Mutex>, +} + +fn asset_file_path(file: &str) -> PathRef { + testdata_path().join("assets").join("node-gyp").join(file) +} + +impl NodeJsMirror { + pub fn get_header_bytes(&self, file: &str) -> Option { + let mut cache = self.cache.lock(); + let entry = cache.entry(file.to_owned()); + match entry { + std::collections::hash_map::Entry::Occupied(occupied) => { + Some(occupied.get().clone()) + } + std::collections::hash_map::Entry::Vacant(vacant) => { + let contents = asset_file_path(file); + let contents = contents + .read_to_bytes_if_exists() + .ok_warn() + .map(Bytes::from)?; + vacant.insert(contents.clone()); + Some(contents) + } + } + } + + fn get_checksum(&self, file: &str, bytes: Bytes) -> String { + use sha2::Digest; + if let Some(checksum) = self.checksum_cache.lock().get(file).cloned() { + return checksum; + } + let mut hasher = sha2::Sha256::new(); + hasher.update(&bytes); + let checksum = faster_hex::hex_string(hasher.finalize().as_ref()); + self + .checksum_cache + .lock() + .insert(file.to_owned(), checksum.clone()); + checksum + } + + pub fn get_checksum_file(&self, version: &str) -> Option { + let mut entries = Vec::with_capacity(2); + + let header_file = header_tar_name(version); + let header_bytes = self.get_header_bytes(&header_file)?; + let header_checksum = self.get_checksum(&header_file, header_bytes); + entries.push((header_file, header_checksum)); + + if cfg!(windows) { + if !cfg!(target_arch = "x86_64") { + panic!("unsupported target arch on windows, only support x86_64"); + } + let Some(bytes) = self.get_node_lib_bytes(version, "win-x64") else { + eprintln!("test server failed to get node lib"); + return None; + }; + { + let file = format!("{version}/win-x64/node.lib"); + let checksum = self.get_checksum(&file, bytes); + let filename_for_checksum = + file.trim_start_matches(&format!("{version}/")); + entries.push((filename_for_checksum.to_owned(), checksum)); + } + } + + Some( + entries + .into_iter() + .map(|(file, checksum)| format!("{checksum} {file}")) + .collect::>() + .join("\n"), + ) + } + + pub fn get_node_lib_bytes( + &self, + version: &str, + platform: &str, + ) -> Option { + let mut cache = self.cache.lock(); + let file_name = format!("{version}/{platform}/node.lib"); + let entry = cache.entry(file_name); + match entry { + std::collections::hash_map::Entry::Occupied(occupied) => { + Some(occupied.get().clone()) + } + std::collections::hash_map::Entry::Vacant(vacant) => { + let tarball_filename = + format!("{version}__{platform}__node.lib.tar.gz"); + let contents = asset_file_path(&tarball_filename); + let contents = contents.read_to_bytes_if_exists().ok_warn()?; + let extracted = Bytes::from(extract_tarball(&contents)?); + vacant.insert(extracted.clone()); + Some(extracted) + } + } + } +} + +fn header_tar_name(version: &str) -> String { + format!("node-{version}-headers.tar.gz") +} + +fn extract_tarball(compressed: &[u8]) -> Option> { + let mut out = Vec::with_capacity(compressed.len()); + let decoder = flate2::read::GzDecoder::new(compressed); + let mut archive = tar::Archive::new(decoder); + for file in archive.entries().ok_warn()? { + let mut file = file.ok_warn()?; + + std::io::copy(&mut file, &mut out).ok_warn()?; + } + Some(out) +} + +/// Server for node JS header tarballs, used by `node-gyp` in tests +pub async fn nodejs_org_mirror(port: u16) { + let addr = SocketAddr::from(([127, 0, 0, 1], port)); + + run_server( + ServerOptions { + addr, + error_msg: "nodejs mirror server error", + kind: ServerKind::Auto, + }, + |req| async move { + let path = req.uri().path(); + if path.contains("-headers.tar.gz") + || path.contains("SHASUMS256.txt") + || path.contains("node.lib") + { + let mut parts = path.split('/'); + let _ = parts.next(); // empty + let Some(version) = parts.next() else { + return not_found(format!("missing node version in path: {path}")); + }; + let Some(file) = parts.next() else { + return not_found(format!("missing file version in path: {path}")); + }; + if file == "SHASUMS256.txt" { + let Some(checksum_file) = NODEJS_MIRROR.get_checksum_file(version) + else { + return not_found(format!("failed to get header checksum: {path}")); + }; + return Ok(Response::new(string_body(&checksum_file))); + } else if !file.contains("headers") { + let platform = file; + let Some(file) = parts.next() else { + return not_found("expected file"); + }; + if file != "node.lib" { + return not_found(format!( + "unexpected file name, expected node.lib, got: {file}" + )); + } + let Some(bytes) = NODEJS_MIRROR.get_node_lib_bytes(version, platform) + else { + return not_found("expected node lib bytes"); + }; + + return Ok(Response::new(UnsyncBoxBody::new(Full::new(bytes)))); + } + + let Some(bytes) = NODEJS_MIRROR.get_header_bytes(file) else { + return not_found(format!( + "couldn't find headers for version {version}, missing file: {file}" + )); + }; + Ok(Response::new(UnsyncBoxBody::new(Full::new(bytes)))) + } else { + not_found(format!("unexpected request path: {path}")) + } + }, + ) + .await +} + +fn not_found( + msg: impl AsRef, +) -> Result>, anyhow::Error> { + let msg = msg.as_ref(); + eprintln!( + "test_server warning: error likely occurred in nodejs_org_mirror.rs: {msg}" + ); + Response::builder() + .status(StatusCode::NOT_FOUND) + .body(string_body(msg)) + .map_err(|e| e.into()) +} -- cgit v1.2.3 From f091d1ad69b4e5217ae3272b641171781a372c4f Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 13 Nov 2024 10:10:09 -0500 Subject: feat(node): stabilize detecting if CJS via `"type": "commonjs"` in a package.json (#26439) This will respect `"type": "commonjs"` in a package.json to determine if `.js`/`.jsx`/`.ts`/.tsx` files are CJS or ESM. If the file is found to be ESM it will be loaded as ESM though. --- tests/util/server/src/servers/mod.rs | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'tests/util/server/src') diff --git a/tests/util/server/src/servers/mod.rs b/tests/util/server/src/servers/mod.rs index d9adde542..0b1d99aeb 100644 --- a/tests/util/server/src/servers/mod.rs +++ b/tests/util/server/src/servers/mod.rs @@ -807,17 +807,17 @@ async fn main_server( (_, "/jsx/jsx-runtime") | (_, "/jsx/jsx-dev-runtime") => { let mut res = Response::new(string_body( r#"export function jsx( - _type, - _props, - _key, - _source, - _self, - ) {} - export const jsxs = jsx; - export const jsxDEV = jsx; - export const Fragment = Symbol("Fragment"); - console.log("imported", import.meta.url); - "#, + _type, + _props, + _key, + _source, + _self, +) {} +export const jsxs = jsx; +export const jsxDEV = jsx; +export const Fragment = Symbol("Fragment"); +console.log("imported", import.meta.url); +"#, )); res.headers_mut().insert( "Content-type", -- cgit v1.2.3 From 4e899d48cffa95617266dd8f9aef54603a87ad82 Mon Sep 17 00:00:00 2001 From: snek Date: Thu, 14 Nov 2024 13:16:28 +0100 Subject: fix: otel resiliency (#26857) Improving the breadth of collected data, and ensuring that the collected data is more likely to be successfully reported. - Use `log` crate in more places - Hook up `log` crate to otel - Switch to process-wide otel processors - Handle places that use `process::exit` Also adds a more robust testing framework, with a deterministic tracing setting. Refs: https://github.com/denoland/deno/issues/26852 --- tests/util/server/src/macros.rs | 1 + 1 file changed, 1 insertion(+) (limited to 'tests/util/server/src') diff --git a/tests/util/server/src/macros.rs b/tests/util/server/src/macros.rs index fdbb977e9..e076583f1 100644 --- a/tests/util/server/src/macros.rs +++ b/tests/util/server/src/macros.rs @@ -33,6 +33,7 @@ macro_rules! timeout { use std::io::Write; eprintln!("Test {function} timed out after {timeout} seconds, aborting"); _ = std::io::stderr().flush(); + #[allow(clippy::disallowed_methods)] ::std::process::exit(1); } }); -- cgit v1.2.3