diff options
| -rw-r--r-- | Cargo.lock | 51 | ||||
| -rw-r--r-- | cli/http_util.rs | 14 | ||||
| -rw-r--r-- | cli/tests/053_import_compression/brotli.header | 3 | ||||
| -rw-r--r-- | cli/tests/053_import_compression/gziped.header | 3 | ||||
| -rw-r--r-- | cli/tests/integration_tests.rs | 3 | ||||
| -rw-r--r-- | cli/tests/unit/README.md | 6 | ||||
| -rw-r--r-- | cli/tests/unit/body_test.ts | 2 | ||||
| -rw-r--r-- | cli/tests/unit/fetch_test.ts | 50 | ||||
| -rw-r--r-- | cli/tests/x_deno_warning.js.header | 2 | ||||
| -rw-r--r-- | test_util/Cargo.toml | 8 | ||||
| -rw-r--r-- | test_util/src/lib.rs | 375 | ||||
| -rw-r--r-- | test_util/src/test_server.rs | 3 | ||||
| -rwxr-xr-x | tools/benchmark.py | 11 | ||||
| -rwxr-xr-x | tools/benchmark_test.py | 2 | ||||
| -rwxr-xr-x | tools/http_server.py | 420 | ||||
| -rwxr-xr-x | tools/throughput_benchmark.py | 2 |
16 files changed, 457 insertions, 498 deletions
diff --git a/Cargo.lock b/Cargo.lock index 608d25c1c..c7f8b1e09 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -123,6 +123,15 @@ checksum = "f8aac770f1885fd7e387acedd76065302551364496e46b3dd00860b2f8359b9d" [[package]] name = "base64" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0b25d992356d2eb0ed82172f5248873db5560c4721f564b13cb5193bda5e668e" +dependencies = [ + "byteorder", +] + +[[package]] +name = "base64" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" @@ -408,7 +417,7 @@ dependencies = [ "termcolor", "test_util", "tokio", - "tokio-rustls", + "tokio-rustls 0.13.1", "tokio-tungstenite", "url", "utime", @@ -941,9 +950,9 @@ dependencies = [ "futures-util", "hyper", "log 0.4.8", - "rustls", + "rustls 0.17.0", "tokio", - "tokio-rustls", + "tokio-rustls 0.13.1", "webpki", ] @@ -1815,11 +1824,11 @@ dependencies = [ "mime_guess 2.0.3", "percent-encoding", "pin-project-lite", - "rustls", + "rustls 0.17.0", "serde", "serde_urlencoded", "tokio", - "tokio-rustls", + "tokio-rustls 0.13.1", "url", "wasm-bindgen", "wasm-bindgen-futures", @@ -1854,6 +1863,19 @@ dependencies = [ [[package]] name = "rustls" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b25a18b1bf7387f0145e7f8324e700805aade3842dd3db2e74e4cdeb4677c09e" +dependencies = [ + "base64 0.10.1", + "log 0.4.8", + "ring", + "sct", + "webpki", +] + +[[package]] +name = "rustls" version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0d4a31f5d68413404705d6982529b0e11a9aacd4839d1d6222ee3b8cb4015e1" @@ -2349,10 +2371,14 @@ dependencies = [ name = "test_util" version = "0.1.0" dependencies = [ + "bytes 0.5.5", + "futures 0.3.5", "lazy_static", "os_pipe", "regex", "tempfile", + "tokio", + "warp", ] [[package]] @@ -2457,12 +2483,24 @@ dependencies = [ [[package]] name = "tokio-rustls" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3068d891551949b37681724d6b73666787cc63fa8e255c812a41d2513aff9775" +dependencies = [ + "futures-core", + "rustls 0.16.0", + "tokio", + "webpki", +] + +[[package]] +name = "tokio-rustls" version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "15cb62a0d2770787abc96e99c1cd98fcf17f94959f3af63ca85bdfb203f051b4" dependencies = [ "futures-core", - "rustls", + "rustls 0.17.0", "tokio", "webpki", ] @@ -2721,6 +2759,7 @@ dependencies = [ "serde_json", "serde_urlencoded", "tokio", + "tokio-rustls 0.12.3", "tokio-tungstenite", "tower-service", "urlencoding", diff --git a/cli/http_util.rs b/cli/http_util.rs index 5b434a276..f0109ca36 100644 --- a/cli/http_util.rs +++ b/cli/http_util.rs @@ -249,7 +249,7 @@ mod tests { #[tokio::test] async fn test_fetch_string() { let http_server_guard = test_util::http_server(); - // Relies on external http server. See tools/http_server.py + // Relies on external http server. See target/debug/test_server let url = Url::parse("http://127.0.0.1:4545/cli/tests/fixture.json").unwrap(); let client = create_http_client(None).unwrap(); @@ -268,7 +268,7 @@ mod tests { #[tokio::test] async fn test_fetch_gzip() { let http_server_guard = test_util::http_server(); - // Relies on external http server. See tools/http_server.py + // Relies on external http server. See target/debug/test_server let url = Url::parse( "http://127.0.0.1:4545/cli/tests/053_import_compression/gziped", ) @@ -317,7 +317,7 @@ mod tests { #[tokio::test] async fn test_fetch_brotli() { let http_server_guard = test_util::http_server(); - // Relies on external http server. See tools/http_server.py + // Relies on external http server. See target/debug/test_server let url = Url::parse( "http://127.0.0.1:4545/cli/tests/053_import_compression/brotli", ) @@ -342,7 +342,7 @@ mod tests { #[tokio::test] async fn test_fetch_once_with_redirect() { let http_server_guard = test_util::http_server(); - // Relies on external http server. See tools/http_server.py + // Relies on external http server. See target/debug/test_server let url = Url::parse("http://127.0.0.1:4546/cli/tests/fixture.json").unwrap(); // Dns resolver substitutes `127.0.0.1` with `localhost` @@ -399,7 +399,7 @@ mod tests { #[tokio::test] async fn test_fetch_with_cafile_string() { let http_server_guard = test_util::http_server(); - // Relies on external http server. See tools/http_server.py + // Relies on external http server. See target/debug/test_server let url = Url::parse("https://localhost:5545/cli/tests/fixture.json").unwrap(); @@ -425,7 +425,7 @@ mod tests { #[tokio::test] async fn test_fetch_with_cafile_gzip() { let http_server_guard = test_util::http_server(); - // Relies on external http server. See tools/http_server.py + // Relies on external http server. See target/debug/test_server let url = Url::parse( "https://localhost:5545/cli/tests/053_import_compression/gziped", ) @@ -487,7 +487,7 @@ mod tests { #[tokio::test] async fn test_fetch_with_cafile_brotli() { let http_server_guard = test_util::http_server(); - // Relies on external http server. See tools/http_server.py + // Relies on external http server. See target/debug/test_server let url = Url::parse( "https://localhost:5545/cli/tests/053_import_compression/brotli", ) diff --git a/cli/tests/053_import_compression/brotli.header b/cli/tests/053_import_compression/brotli.header deleted file mode 100644 index 6047a3993..000000000 --- a/cli/tests/053_import_compression/brotli.header +++ /dev/null @@ -1,3 +0,0 @@ -Content-Encoding: br -Content-Type: application/javascript -Content-Length: 26
\ No newline at end of file diff --git a/cli/tests/053_import_compression/gziped.header b/cli/tests/053_import_compression/gziped.header deleted file mode 100644 index fda818af6..000000000 --- a/cli/tests/053_import_compression/gziped.header +++ /dev/null @@ -1,3 +0,0 @@ -Content-Encoding: gzip -Content-Type: application/javascript -Content-Length: 39
\ No newline at end of file diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs index 6855df707..616849520 100644 --- a/cli/tests/integration_tests.rs +++ b/cli/tests/integration_tests.rs @@ -2600,7 +2600,8 @@ fn test_permissions_net_listen_allow_localhost_4555_fail() { #[test] fn test_permissions_net_listen_allow_localhost() { - // Port 4600 is chosen to not colide with those used by tools/http_server.py + // Port 4600 is chosen to not colide with those used by + // target/debug/test_server let (_, err) = util::run_and_collect_output( true, "run --allow-net=localhost complex_permissions_test.ts netListen localhost:4600", diff --git a/cli/tests/unit/README.md b/cli/tests/unit/README.md index c333b0046..1546038c4 100644 --- a/cli/tests/unit/README.md +++ b/cli/tests/unit/README.md @@ -75,6 +75,6 @@ RUST_BACKTRACE=1 cargo run -- run --unstable --allow-read --allow-write cli/test ### Http server -`tools/http_server.py` is required to run when one's running unit tests. During -CI it's spawned automatically, but if you want to run tests manually make sure -that server is spawned otherwise there'll be cascade of test failures. +`target/debug/test_server` is required to run when one's running unit tests. +During CI it's spawned automatically, but if you want to run tests manually make +sure that server is spawned otherwise there'll be cascade of test failures. diff --git a/cli/tests/unit/body_test.ts b/cli/tests/unit/body_test.ts index d0423f7de..3b01190a6 100644 --- a/cli/tests/unit/body_test.ts +++ b/cli/tests/unit/body_test.ts @@ -36,7 +36,7 @@ unitTest( { perms: { net: true } }, async function bodyMultipartFormData(): Promise<void> { const response = await fetch( - "http://localhost:4545/cli/tests/subdir/multipart_form_data.txt" + "http://localhost:4545/multipart_form_data.txt" ); const text = await response.text(); diff --git a/cli/tests/unit/fetch_test.ts b/cli/tests/unit/fetch_test.ts index a52a7809a..1dab8f7f9 100644 --- a/cli/tests/unit/fetch_test.ts +++ b/cli/tests/unit/fetch_test.ts @@ -67,7 +67,6 @@ unitTest({ perms: { net: true } }, async function fetchHeaders(): Promise< const response = await fetch("http://localhost:4545/cli/tests/fixture.json"); const headers = response.headers; assertEquals(headers.get("Content-Type"), "application/json"); - assert(headers.get("Server")!.startsWith("SimpleHTTP")); const _json = await response.json(); }); @@ -162,13 +161,10 @@ unitTest( { perms: { net: true } }, async function fetchBodyReaderBigBody(): Promise<void> { const data = "a".repeat(10 << 10); // 10mb - const response = await fetch( - "http://localhost:4545/cli/tests/echo_server", - { - method: "POST", - body: data, - } - ); + const response = await fetch("http://localhost:4545/echo_server", { + method: "POST", + body: data, + }); assert(response.body !== null); const reader = await response.body.getReader(); let total = 0; @@ -210,7 +206,7 @@ unitTest( { perms: { net: true } }, async function fetchMultipartFormDataSuccess(): Promise<void> { const response = await fetch( - "http://localhost:4545/cli/tests/subdir/multipart_form_data.txt" + "http://localhost:4545/multipart_form_data.txt" ); const formData = await response.formData(); assert(formData.has("field_1")); @@ -315,12 +311,12 @@ unitTest( perms: { net: true }, }, async function fetchWithRedirection(): Promise<void> { - const response = await fetch("http://localhost:4546/"); // will redirect to http://localhost:4545/ + const response = await fetch("http://localhost:4546/README.md"); assertEquals(response.status, 200); assertEquals(response.statusText, "OK"); - assertEquals(response.url, "http://localhost:4545/"); + assertEquals(response.url, "http://localhost:4545/README.md"); const body = await response.text(); - assert(body.includes("<title>Directory listing for /</title>")); + assert(body.includes("Deno")); } ); @@ -329,11 +325,13 @@ unitTest( perms: { net: true }, }, async function fetchWithRelativeRedirection(): Promise<void> { - const response = await fetch("http://localhost:4545/cli/tests"); // will redirect to /cli/tests/ + const response = await fetch( + "http://localhost:4545/cli/tests/001_hello.js" + ); assertEquals(response.status, 200); assertEquals(response.statusText, "OK"); const body = await response.text(); - assert(body.includes("<title>Directory listing for /cli/tests/</title>")); + assert(body.includes("Hello")); } ); @@ -769,13 +767,10 @@ unitTest( { perms: { net: true } }, async function fetchBodyReaderWithCancelAndNewReader(): Promise<void> { const data = "a".repeat(1 << 10); - const response = await fetch( - "http://localhost:4545/cli/tests/echo_server", - { - method: "POST", - body: data, - } - ); + const response = await fetch("http://localhost:4545/echo_server", { + method: "POST", + body: data, + }); assert(response.body !== null); const firstReader = await response.body.getReader(); @@ -801,13 +796,10 @@ unitTest( async function fetchBodyReaderWithReadCancelAndNewReader(): Promise<void> { const data = "a".repeat(1 << 10); - const response = await fetch( - "http://localhost:4545/cli/tests/echo_server", - { - method: "POST", - body: data, - } - ); + const response = await fetch("http://localhost:4545/echo_server", { + method: "POST", + body: data, + }); assert(response.body !== null); const firstReader = await response.body.getReader(); @@ -848,7 +840,7 @@ unitTest( for (const status of nullBodyStatus) { const headers = new Headers([["x-status", String(status)]]); - const res = await fetch("http://localhost:4545/cli/tests/echo_server", { + const res = await fetch("http://localhost:4545/echo_server", { body: "deno", method: "POST", headers, diff --git a/cli/tests/x_deno_warning.js.header b/cli/tests/x_deno_warning.js.header deleted file mode 100644 index 9a8ba0190..000000000 --- a/cli/tests/x_deno_warning.js.header +++ /dev/null @@ -1,2 +0,0 @@ -Content-Type: application/javascript -X-Deno-Warning: foobar
\ No newline at end of file diff --git a/test_util/Cargo.toml b/test_util/Cargo.toml index e65d2c224..b82d61b43 100644 --- a/test_util/Cargo.toml +++ b/test_util/Cargo.toml @@ -5,10 +5,16 @@ authors = ["the Deno authors"] edition = "2018" publish = false -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html +[[bin]] +name = "test_server" +path = "src/test_server.rs" [dependencies] +tokio = { version = "0.2.21", features = ["rt-core", "tcp", "udp", "uds", "process", "fs", "blocking", "sync", "io-std", "macros", "time"] } +futures = { version = "0.3.5", features = ["compat", "io-compat"] } +bytes = "0.5.5" lazy_static = "1.4.0" os_pipe = "0.9.2" regex = "1.3.9" tempfile = "3.1.0" +warp = { version = "0.2.3", features = ["tls"] } diff --git a/test_util/src/lib.rs b/test_util/src/lib.rs index f63ed71a7..67ced971a 100644 --- a/test_util/src/lib.rs +++ b/test_util/src/lib.rs @@ -3,10 +3,12 @@ #[macro_use] extern crate lazy_static; +use futures::future::{self, FutureExt}; use os_pipe::pipe; use regex::Regex; use std::io::Read; use std::io::Write; +use std::mem::replace; use std::path::PathBuf; use std::process::Child; use std::process::Command; @@ -15,6 +17,20 @@ use std::process::Stdio; use std::sync::Mutex; use std::sync::MutexGuard; use tempfile::TempDir; +use warp::http::Uri; +use warp::http::{HeaderValue, Response, StatusCode}; +use warp::hyper::Body; +use warp::reply::with_header; +use warp::reply::Reply; +use warp::Filter; + +const PORT: u16 = 4545; +const REDIRECT_PORT: u16 = 4546; +const ANOTHER_REDIRECT_PORT: u16 = 4547; +const DOUBLE_REDIRECTS_PORT: u16 = 4548; +const INF_REDIRECTS_PORT: u16 = 4549; +const REDIRECT_ABSOLUTE_PORT: u16 = 4550; +const HTTPS_PORT: u16 = 5545; pub const PERMISSION_VARIANTS: [&str; 5] = ["read", "write", "env", "net", "run"]; @@ -54,27 +70,358 @@ pub fn deno_exe_path() -> PathBuf { p } +pub fn test_server_path() -> PathBuf { + let mut p = target_dir().join("test_server"); + if cfg!(windows) { + p.set_extension("exe"); + } + p +} + +#[tokio::main] +pub async fn run_all_servers() { + let routes = warp::path::full().map(|path: warp::path::FullPath| { + let p = path.as_str(); + assert_eq!(&p[0..1], "/"); + let url = format!("http://localhost:{}{}", PORT, p); + let u = url.parse::<Uri>().unwrap(); + warp::redirect(u) + }); + let redirect_server_fut = + warp::serve(routes).bind(([127, 0, 0, 1], REDIRECT_PORT)); + + let routes = warp::path::full().map(|path: warp::path::FullPath| { + let p = path.as_str(); + assert_eq!(&p[0..1], "/"); + let url = format!("http://localhost:{}/cli/tests/subdir{}", PORT, p); + let u = url.parse::<Uri>().unwrap(); + warp::redirect(u) + }); + let another_redirect_server_fut = + warp::serve(routes).bind(([127, 0, 0, 1], ANOTHER_REDIRECT_PORT)); + + let routes = warp::path::full().map(|path: warp::path::FullPath| { + let p = path.as_str(); + assert_eq!(&p[0..1], "/"); + let url = format!("http://localhost:{}{}", REDIRECT_PORT, p); + let u = url.parse::<Uri>().unwrap(); + warp::redirect(u) + }); + let double_redirect_server_fut = + warp::serve(routes).bind(([127, 0, 0, 1], DOUBLE_REDIRECTS_PORT)); + + let routes = warp::path::full().map(|path: warp::path::FullPath| { + let p = path.as_str(); + assert_eq!(&p[0..1], "/"); + let url = format!("http://localhost:{}{}", INF_REDIRECTS_PORT, p); + let u = url.parse::<Uri>().unwrap(); + warp::redirect(u) + }); + let inf_redirect_server_fut = + warp::serve(routes).bind(([127, 0, 0, 1], INF_REDIRECTS_PORT)); + + // redirect server that redirect to absolute paths under same host + // redirects /REDIRECT/file_name to /file_name + let routes = warp::path("REDIRECT") + .and(warp::path::peek()) + .map(|path: warp::path::Peek| { + let p = path.as_str(); + let url = format!("/{}", p); + let u = url.parse::<Uri>().unwrap(); + warp::redirect(u) + }) + .or( + warp::any() + .and(warp::path::peek()) + .and(warp::fs::dir(root_path())) + .map(custom_headers), + ); + let absolute_redirect_server_fut = + warp::serve(routes).bind(([127, 0, 0, 1], REDIRECT_ABSOLUTE_PORT)); + + let echo_server = warp::path("echo_server") + .and(warp::post()) + .and(warp::body::bytes()) + .and(warp::header::optional::<String>("x-status")) + .and(warp::header::optional::<String>("content-type")) + .and(warp::header::optional::<String>("user-agent")) + .map( + |bytes: bytes::Bytes, + status: Option<String>, + content_type: Option<String>, + user_agent: Option<String>| + -> Box<dyn Reply> { + let mut res = Response::new(Body::from(bytes)); + if let Some(v) = status { + *res.status_mut() = StatusCode::from_bytes(v.as_bytes()).unwrap(); + } + let h = res.headers_mut(); + if let Some(v) = content_type { + h.insert("content-type", HeaderValue::from_str(&v).unwrap()); + } + if let Some(v) = user_agent { + h.insert("user-agent", HeaderValue::from_str(&v).unwrap()); + } + Box::new(res) + }, + ); + let echo_multipart_file = warp::path("echo_multipart_file") + .and(warp::post()) + .and(warp::body::bytes()) + .map(|bytes: bytes::Bytes| -> Box<dyn Reply> { + let start = b"--boundary\t \r\n\ + Content-Disposition: form-data; name=\"field_1\"\r\n\ + \r\n\ + value_1 \r\n\ + \r\n--boundary\r\n\ + Content-Disposition: form-data; name=\"file\"; \ + filename=\"file.bin\"\r\n\ + Content-Type: application/octet-stream\r\n\ + \r\n"; + let end = b"\r\n--boundary--\r\n"; + let b = [start as &[u8], &bytes, end].concat(); + + let mut res = Response::new(Body::from(b)); + let h = res.headers_mut(); + h.insert( + "content-type", + HeaderValue::from_static("multipart/form-data;boundary=boundary"), + ); + Box::new(res) + }); + let multipart_form_data = + warp::path("multipart_form_data.txt").map(|| -> Box<dyn Reply> { + let b = "Preamble\r\n\ + --boundary\t \r\n\ + Content-Disposition: form-data; name=\"field_1\"\r\n\ + \r\n\ + value_1 \r\n\ + \r\n--boundary\r\n\ + Content-Disposition: form-data; name=\"field_2\";\ + filename=\"file.js\"\r\n\ + Content-Type: text/javascript\r\n\ + \r\n\ + console.log(\"Hi\")\ + \r\n--boundary--\r\n\ + Epilogue"; + let mut res = Response::new(Body::from(b)); + res.headers_mut().insert( + "content-type", + HeaderValue::from_static("multipart/form-data;boundary=boundary"), + ); + Box::new(res) + }); + + let etag_script = warp::path!("etag_script.ts") + .and(warp::header::optional::<String>("if-none-match")) + .map(|if_none_match| -> Box<dyn Reply> { + if if_none_match == Some("33a64df551425fcc55e".to_string()) { + let r = + warp::reply::with_status(warp::reply(), StatusCode::NOT_MODIFIED); + let r = with_header(r, "Content-type", "application/typescript"); + let r = with_header(r, "ETag", "33a64df551425fcc55e"); + Box::new(r) + } else { + let mut res = Response::new(Body::from("console.log('etag')")); + let h = res.headers_mut(); + h.insert( + "Content-type", + HeaderValue::from_static("application/typescript"), + ); + h.insert("ETag", HeaderValue::from_static("33a64df551425fcc55e")); + Box::new(res) + } + }); + let xtypescripttypes = warp::path!("xTypeScriptTypes.js") + .map(|| { + let mut res = Response::new(Body::from("export const foo = 'foo';")); + let h = res.headers_mut(); + h.insert( + "Content-type", + HeaderValue::from_static("application/javascript"), + ); + h.insert( + "X-TypeScript-Types", + HeaderValue::from_static("./xTypeScriptTypes.d.ts"), + ); + res + }) + .or(warp::path!("xTypeScriptTypes.d.ts").map(|| { + let mut res = Response::new(Body::from("export const foo: 'foo';")); + res.headers_mut().insert( + "Content-type", + HeaderValue::from_static("application/typescript"), + ); + res + })) + .or(warp::path!("type_directives_redirect.js").map(|| { + let mut res = Response::new(Body::from("export const foo = 'foo';")); + let h = res.headers_mut(); + h.insert( + "Content-type", + HeaderValue::from_static("application/javascript"), + ); + h.insert( + "X-TypeScript-Types", + HeaderValue::from_static( + "http://localhost:4547/xTypeScriptTypesRedirect.d.ts", + ), + ); + res + })) + .or(warp::path!("cli"/"tests"/"subdir"/"xTypeScriptTypesRedirect.d.ts").map(|| { + let mut res = Response::new(Body::from( + "import './xTypeScriptTypesRedirected.d.ts';", + )); + let h = res.headers_mut(); + h.insert( + "Content-type", + HeaderValue::from_static("application/typescript"), + ); + res + })) + .or(warp::path!("cli"/"tests"/"subdir"/"xTypeScriptTypesRedirected.d.ts").map(|| { + let mut res = Response::new(Body::from("export const foo: 'foo';")); + let h = res.headers_mut(); + h.insert( + "Content-type", + HeaderValue::from_static("application/typescript"), + ); + res + })) + .or(warp::path!("referenceTypes.js").map(|| { + let mut res = Response::new(Body::from("/// <reference types=\"./xTypeScriptTypes.d.ts\" />\r\nexport const foo = \"foo\";\r\n")); + let h = res.headers_mut(); + h.insert( + "Content-type", + HeaderValue::from_static("application/javascript"), + ); + res + })); + + let content_type_handler = warp::any() + .and(warp::path::peek()) + .and(warp::fs::dir(root_path())) + .map(custom_headers) + .or(etag_script) + .or(xtypescripttypes) + .or(echo_server) + .or(echo_multipart_file) + .or(multipart_form_data); + + let http_fut = + warp::serve(content_type_handler.clone()).bind(([127, 0, 0, 1], PORT)); + + let https_fut = warp::serve(content_type_handler.clone()) + .tls() + .cert_path("std/http/testdata/tls/localhost.crt") + .key_path("std/http/testdata/tls/localhost.key") + .bind(([127, 0, 0, 1], HTTPS_PORT)); + + let mut server_fut = async { + futures::join!( + http_fut, + https_fut, + redirect_server_fut, + another_redirect_server_fut, + inf_redirect_server_fut, + double_redirect_server_fut, + absolute_redirect_server_fut, + ) + } + .boxed(); + + let mut did_print_ready = false; + future::poll_fn(move |cx| { + let poll_result = server_fut.poll_unpin(cx); + if !replace(&mut did_print_ready, true) { + println!("ready"); + } + poll_result + }) + .await; +} + +fn custom_headers(path: warp::path::Peek, f: warp::fs::File) -> Box<dyn Reply> { + let p = path.as_str(); + + if p.ends_with("cli/tests/x_deno_warning.js") { + let f = with_header(f, "Content-Type", "application/javascript"); + let f = with_header(f, "X-Deno-Warning", "foobar"); + return Box::new(f); + } + if p.ends_with("cli/tests/053_import_compression/brotli") { + let f = with_header(f, "Content-Encoding", "br"); + let f = with_header(f, "Content-Type", "application/javascript"); + let f = with_header(f, "Content-Length", "26"); + return Box::new(f); + } + if p.ends_with("cli/tests/053_import_compression/gziped") { + let f = with_header(f, "Content-Encoding", "gzip"); + let f = with_header(f, "Content-Type", "application/javascript"); + let f = with_header(f, "Content-Length", "39"); + return Box::new(f); + } + + let content_type = if p.contains(".t1.") { + Some("text/typescript") + } else if p.contains(".t2.") { + Some("video/vnd.dlna.mpeg-tts") + } else if p.contains(".t3.") { + Some("video/mp2t") + } else if p.contains(".t4.") { + Some("application/x-typescript") + } else if p.contains(".j1.") { + Some("text/javascript") + } else if p.contains(".j2.") { + Some("application/ecmascript") + } else if p.contains(".j3.") { + Some("text/ecmascript") + } else if p.contains(".j4.") { + Some("application/x-javascript") + } else if p.contains("form_urlencoded") { + Some("application/x-www-form-urlencoded") + } else if p.contains("unknown_ext") || p.contains("no_ext") { + Some("text/typescript") + } else if p.contains("mismatch_ext") { + Some("text/javascript") + } else if p.ends_with(".ts") || p.ends_with(".tsx") { + Some("application/typescript") + } else if p.ends_with(".js") || p.ends_with(".jsx") { + Some("application/javascript") + } else if p.ends_with(".json") { + Some("application/json") + } else { + None + }; + + if let Some(t) = content_type { + Box::new(with_header(f, "Content-Type", t)) + } else { + Box::new(f) + } +} + pub struct HttpServerGuard<'a> { #[allow(dead_code)] g: MutexGuard<'a, ()>, - child: Child, + test_server: Child, } impl<'a> Drop for HttpServerGuard<'a> { fn drop(&mut self) { - match self.child.try_wait() { + match self.test_server.try_wait() { Ok(None) => { - self.child.kill().expect("failed to kill http_server.py"); - } - Ok(Some(status)) => { - panic!("http_server.py exited unexpectedly {}", status) + self.test_server.kill().expect("failed to kill test_server"); + let _ = self.test_server.wait(); } - Err(e) => panic!("http_server.py err {}", e), + Ok(Some(status)) => panic!("test_server exited unexpectedly {}", status), + Err(e) => panic!("test_server error: {}", e), } } } -/// Starts tools/http_server.py when the returned guard is dropped, the server +/// Starts target/debug/test_server when the returned guard is dropped, the server /// will be killed. pub fn http_server<'a>() -> HttpServerGuard<'a> { // TODO(bartlomieju) Allow tests to use the http server in parallel. @@ -86,18 +433,16 @@ pub fn http_server<'a>() -> HttpServerGuard<'a> { r.unwrap() }; - println!("tools/http_server.py starting..."); - let mut child = Command::new("python") + println!("test_server starting..."); + let mut test_server = Command::new(test_server_path()) .current_dir(root_path()) - .args(&["-u", "tools/http_server.py"]) .stdout(Stdio::piped()) .spawn() - .expect("failed to execute child"); + .expect("failed to execute test_server"); - let stdout = child.stdout.as_mut().unwrap(); + let stdout = test_server.stdout.as_mut().unwrap(); use std::io::{BufRead, BufReader}; let lines = BufReader::new(stdout).lines(); - // Wait for "ready" on stdout. See tools/http_server.py for maybe_line in lines { if let Ok(line) = maybe_line { if line.starts_with("ready") { @@ -108,7 +453,7 @@ pub fn http_server<'a>() -> HttpServerGuard<'a> { } } - HttpServerGuard { child, g } + HttpServerGuard { test_server, g } } /// Helper function to strip ansi codes. diff --git a/test_util/src/test_server.rs b/test_util/src/test_server.rs new file mode 100644 index 000000000..ed958a94a --- /dev/null +++ b/test_util/src/test_server.rs @@ -0,0 +1,3 @@ +fn main() { + test_util::run_all_servers(); +} diff --git a/tools/benchmark.py b/tools/benchmark.py index c5200a281..47f9fd821 100755 --- a/tools/benchmark.py +++ b/tools/benchmark.py @@ -3,7 +3,7 @@ # Performs benchmark and append data to //website/data.json. # If //website/data.json doesn't exist, this script tries to import it from # gh-pages branch. -# To view the results locally run ./tools/http_server.py and visit +# To view the results locally run target/debug/test_server and visit # http://localhost:4545/website import os @@ -12,11 +12,11 @@ import json import time import tempfile import subprocess -from util import build_path, executable_suffix, root_path, run, run_output +from util import (build_path, executable_suffix, root_path, run, run_output, + build_mode) import third_party from http_benchmark import http_benchmark import throughput_benchmark -import http_server # The list of the tuples of the benchmark name, arguments and return code exec_time_benchmarks = [ @@ -239,7 +239,6 @@ def main(): build_dir = build_path() sha1 = run_output(["git", "rev-parse", "HEAD"], exit_on_fail=True).out.strip() - http_server.spawn() deno_exe = os.path.join(build_dir, "deno") @@ -253,7 +252,11 @@ def main(): # TODO(ry) The "benchmark" benchmark should actually be called "exec_time". # When this is changed, the historical data in gh-pages branch needs to be # changed too. + server_cmd = os.path.join("target", build_mode(), "test_server") + p = subprocess.Popen([server_cmd]) new_data["benchmark"] = run_exec_time(deno_exe, build_dir) + p.kill() + p.wait() new_data["binary_size"] = get_binary_sizes(build_dir) new_data["bundle_size"] = bundle_benchmark(deno_exe) diff --git a/tools/benchmark_test.py b/tools/benchmark_test.py index 2c1f384da..82d8c4fd8 100755 --- a/tools/benchmark_test.py +++ b/tools/benchmark_test.py @@ -65,6 +65,4 @@ class TestBenchmark(DenoTestCase): if __name__ == '__main__': - # FIME this doesn't appear to be the case. - # This test assumes tools/http_server.py is running in the background. run_tests() diff --git a/tools/http_server.py b/tools/http_server.py deleted file mode 100755 index d143f0ba8..000000000 --- a/tools/http_server.py +++ /dev/null @@ -1,420 +0,0 @@ -#!/usr/bin/env python -# Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -# Many tests expect there to be an http server on port 4545 servering the deno -# root directory. -from collections import namedtuple -from contextlib import contextmanager -import os -import SimpleHTTPServer -import SocketServer -import socket -import sys -from time import sleep -from threading import Thread -from util import root_path -import ssl -import getopt -import argparse - -PORT = 4545 -REDIRECT_PORT = 4546 -ANOTHER_REDIRECT_PORT = 4547 -DOUBLE_REDIRECTS_PORT = 4548 -INF_REDIRECTS_PORT = 4549 -REDIRECT_ABSOLUTE_PORT = 4550 -HTTPS_PORT = 5545 - - -def create_http_arg_parser(): - parser = argparse.ArgumentParser() - parser.add_argument('--verbose', '-v', action='store_true') - return parser - - -HttpArgParser = create_http_arg_parser() - -args, unknown = HttpArgParser.parse_known_args(sys.argv[1:]) -CERT_FILE = os.path.join(root_path, "std/http/testdata/tls/localhost.crt") -KEY_FILE = os.path.join(root_path, "std/http/testdata/tls/localhost.key") -QUIET = not args.verbose - - -class SSLTCPServer(SocketServer.TCPServer): - def __init__(self, - server_address, - request_handler, - certfile, - keyfile, - ssl_version=ssl.PROTOCOL_TLSv1_2, - bind_and_activate=True): - SocketServer.TCPServer.__init__(self, server_address, request_handler, - bind_and_activate) - self.certfile = certfile - self.keyfile = keyfile - self.ssl_version = ssl_version - - def get_request(self): - newsocket, fromaddr = self.socket.accept() - connstream = ssl.wrap_socket( - newsocket, - server_side=True, - certfile=self.certfile, - keyfile=self.keyfile, - ssl_version=self.ssl_version) - return connstream, fromaddr - - -class SSLThreadingTCPServer(SocketServer.ThreadingMixIn, SSLTCPServer): - pass - - -class QuietSimpleHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): - def log_request(self, code='-', size='-'): - if not QUIET: - SimpleHTTPServer.SimpleHTTPRequestHandler.log_request( - self, code, size) - - -class ContentTypeHandler(QuietSimpleHTTPRequestHandler): - def do_GET(self): - - # Check if there is a custom header configuration ending - # with ".header" before sending the file - maybe_header_file_path = "./" + self.path + ".header" - if os.path.exists(maybe_header_file_path): - self.protocol_version = 'HTTP/1.1' - self.send_response(200, 'OK') - - f = open(maybe_header_file_path) - for line in f: - kv = line.split(": ") - self.send_header(kv[0].strip(), kv[1].strip()) - f.close() - self.end_headers() - - body = open("./" + self.path) - self.wfile.write(body.read()) - body.close() - return - - if "etag_script.ts" in self.path: - self.protocol_version = 'HTTP/1.1' - if_not_match = self.headers.getheader('if-none-match') - if if_not_match == "33a64df551425fcc55e": - self.send_response(304, 'Not Modified') - self.send_header('Content-type', 'application/typescript') - self.send_header('ETag', '33a64df551425fcc55e') - self.end_headers() - else: - self.send_response(200, 'OK') - self.send_header('Content-type', 'application/typescript') - self.send_header('ETag', '33a64df551425fcc55e') - self.end_headers() - self.wfile.write(bytes("console.log('etag')")) - return - - if "xTypeScriptTypes.js" in self.path: - self.protocol_version = "HTTP/1.1" - self.send_response(200, 'OK') - self.send_header('Content-type', 'application/javascript') - self.send_header('X-TypeScript-Types', './xTypeScriptTypes.d.ts') - self.end_headers() - self.wfile.write(bytes("export const foo = 'foo';")) - return - - if "type_directives_redirect.js" in self.path: - self.protocol_version = "HTTP/1.1" - self.send_response(200, 'OK') - self.send_header('Content-type', 'application/javascript') - self.send_header( - 'X-TypeScript-Types', - 'http://localhost:4547/xTypeScriptTypesRedirect.d.ts') - self.end_headers() - self.wfile.write(bytes("export const foo = 'foo';")) - return - - if "xTypeScriptTypesRedirect.d.ts" in self.path: - self.protocol_version = "HTTP/1.1" - self.send_response(200, 'OK') - self.send_header('Content-type', 'application/typescript') - self.end_headers() - self.wfile.write( - bytes("import './xTypeScriptTypesRedirected.d.ts';")) - return - - if "xTypeScriptTypesRedirected.d.ts" in self.path: - self.protocol_version = "HTTP/1.1" - self.send_response(200, 'OK') - self.send_header('Content-type', 'application/typescript') - self.end_headers() - self.wfile.write(bytes("export const foo: 'foo';")) - return - - if "xTypeScriptTypes.d.ts" in self.path: - self.protocol_version = "HTTP/1.1" - self.send_response(200, 'OK') - self.send_header('Content-type', 'application/typescript') - self.end_headers() - self.wfile.write(bytes("export const foo: 'foo';")) - return - - if "referenceTypes.js" in self.path: - self.protocol_version = "HTTP/1.1" - self.send_response(200, 'OK') - self.send_header('Content-type', 'application/javascript') - self.end_headers() - self.wfile.write( - bytes('/// <reference types="./xTypeScriptTypes.d.ts" />\r\n' - 'export const foo = "foo";\r\n')) - return - - if "multipart_form_data.txt" in self.path: - self.protocol_version = 'HTTP/1.1' - self.send_response(200, 'OK') - self.send_header('Content-type', - 'multipart/form-data;boundary=boundary') - self.end_headers() - self.wfile.write( - bytes('Preamble\r\n' - '--boundary\t \r\n' - 'Content-Disposition: form-data; name="field_1"\r\n' - '\r\n' - 'value_1 \r\n' - '\r\n--boundary\r\n' - 'Content-Disposition: form-data; name="field_2"; ' - 'filename="file.js"\r\n' - 'Content-Type: text/javascript\r\n' - '\r\n' - 'console.log("Hi")' - '\r\n--boundary--\r\n' - 'Epilogue')) - return - return SimpleHTTPServer.SimpleHTTPRequestHandler.do_GET(self) - - def do_POST(self): - # Simple echo server for request reflection - if "echo_server" in self.path: - status = int(self.headers.getheader('x-status', "200")) - self.protocol_version = 'HTTP/1.1' - self.send_response(status, 'OK') - if self.headers.has_key('content-type'): - self.send_header('content-type', - self.headers.getheader('content-type')) - if self.headers.has_key('user-agent'): - self.send_header('user-agent', - self.headers.getheader('user-agent')) - self.end_headers() - data_string = self.rfile.read(int(self.headers['Content-Length'])) - self.wfile.write(bytes(data_string)) - return - if "echo_multipart_file" in self.path: - self.protocol_version = 'HTTP/1.1' - self.send_response(200, 'OK') - self.send_header('Content-type', - 'multipart/form-data;boundary=boundary') - self.end_headers() - file_content = self.rfile.read(int(self.headers['Content-Length'])) - self.wfile.write( - bytes('--boundary\t \r\n' - 'Content-Disposition: form-data; name="field_1"\r\n' - '\r\n' - 'value_1 \r\n' - '\r\n--boundary\r\n' - 'Content-Disposition: form-data; name="file"; ' - 'filename="file.bin"\r\n' - 'Content-Type: application/octet-stream\r\n' - '\r\n') + bytes(file_content) + - bytes('\r\n--boundary--\r\n')) - return - self.protocol_version = 'HTTP/1.1' - self.send_response(501) - self.send_header('content-type', 'text/plain') - self.end_headers() - self.wfile.write(bytes('Server does not support this operation')) - - def guess_type(self, path): - if ".t1." in path: - return "text/typescript" - if ".t2." in path: - return "video/vnd.dlna.mpeg-tts" - if ".t3." in path: - return "video/mp2t" - if ".t4." in path: - return "application/x-typescript" - if ".j1." in path: - return "text/javascript" - if ".j2." in path: - return "application/ecmascript" - if ".j3." in path: - return "text/ecmascript" - if ".j4." in path: - return "application/x-javascript" - if "form_urlencoded" in path: - return "application/x-www-form-urlencoded" - if "no_ext" in path: - return "text/typescript" - if "unknown_ext" in path: - return "text/typescript" - if "mismatch_ext" in path: - return "text/javascript" - return SimpleHTTPServer.SimpleHTTPRequestHandler.guess_type(self, path) - - -RunningServer = namedtuple("RunningServer", ["server", "thread"]) - - -def get_socket(port, handler, use_https): - SocketServer.TCPServer.allow_reuse_address = True - if os.name != "nt": - # We use AF_INET6 to avoid flaky test issue, particularly with - # the test 019_media_types. It's not well understood why this fixes the - # flaky tests, but it does appear to... - # See https://github.com/denoland/deno/issues/3332 - SocketServer.TCPServer.address_family = socket.AF_INET6 - - if use_https: - return SSLThreadingTCPServer(("", port), handler, CERT_FILE, KEY_FILE) - return SocketServer.TCPServer(("", port), handler) - - -def server(): - os.chdir(root_path) # Hopefully the main thread doesn't also chdir. - Handler = ContentTypeHandler - Handler.extensions_map.update({ - ".ts": "application/typescript", - ".js": "application/javascript", - ".tsx": "application/typescript", - ".jsx": "application/javascript", - ".json": "application/json", - }) - s = get_socket(PORT, Handler, False) - if not QUIET: - print "Deno test server http://localhost:%d/" % PORT - return RunningServer(s, start(s)) - - -def base_redirect_server(host_port, target_port, extra_path_segment=""): - os.chdir(root_path) - target_host = "http://localhost:%d" % target_port - - class RedirectHandler(QuietSimpleHTTPRequestHandler): - def do_GET(self): - self.send_response(301) - self.send_header('Location', - target_host + extra_path_segment + self.path) - self.end_headers() - - s = get_socket(host_port, RedirectHandler, False) - if not QUIET: - print "redirect server http://localhost:%d/ -> http://localhost:%d/" % ( - host_port, target_port) - return RunningServer(s, start(s)) - - -# redirect server -def redirect_server(): - return base_redirect_server(REDIRECT_PORT, PORT) - - -# another redirect server pointing to the same port as the one above -# BUT with an extra subdir path -def another_redirect_server(): - return base_redirect_server( - ANOTHER_REDIRECT_PORT, PORT, extra_path_segment="/cli/tests/subdir") - - -# redirect server that points to another redirect server -def double_redirects_server(): - return base_redirect_server(DOUBLE_REDIRECTS_PORT, REDIRECT_PORT) - - -# redirect server that points to itself -def inf_redirects_server(): - return base_redirect_server(INF_REDIRECTS_PORT, INF_REDIRECTS_PORT) - - -# redirect server that redirect to absolute paths under same host -# redirects /REDIRECT/file_name to /file_name -def absolute_redirect_server(): - os.chdir(root_path) - - class AbsoluteRedirectHandler(ContentTypeHandler): - def do_GET(self): - print(self.path) - if (self.path.startswith("/REDIRECT/")): - self.send_response(302) - self.send_header('Location', - self.path.split('/REDIRECT', 1)[1]) - self.end_headers() - else: - ContentTypeHandler.do_GET(self) - - s = get_socket(REDIRECT_ABSOLUTE_PORT, AbsoluteRedirectHandler, False) - if not QUIET: - print("absolute redirect server http://localhost:%d/" % - REDIRECT_ABSOLUTE_PORT) - return RunningServer(s, start(s)) - - -def https_server(): - os.chdir(root_path) # Hopefully the main thread doesn't also chdir. - Handler = ContentTypeHandler - Handler.extensions_map.update({ - ".ts": "application/typescript", - ".js": "application/javascript", - ".tsx": "application/typescript", - ".jsx": "application/javascript", - ".json": "application/json", - }) - s = get_socket(HTTPS_PORT, Handler, True) - if not QUIET: - print "Deno https test server https://localhost:%d/" % HTTPS_PORT - return RunningServer(s, start(s)) - - -def start(s): - thread = Thread(target=s.serve_forever, kwargs={"poll_interval": 0.05}) - thread.daemon = True - thread.start() - return thread - - -@contextmanager -def spawn(): - servers = (server(), redirect_server(), another_redirect_server(), - double_redirects_server(), https_server(), - absolute_redirect_server(), inf_redirects_server()) - # In order to wait for each of the servers to be ready, we try connecting to - # them with a tcp socket. - for running_server in servers: - client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - port = running_server.server.server_address[1] - client.connect(("127.0.0.1", port)) - print "connected", port - client.close() - assert running_server.thread.is_alive() - # The following output "ready" is specificly looked for in cli/test_util.rs - # to prevent race conditions. - print "ready" - try: - yield servers - finally: - for s in servers: - # Make sure all servers still running, - # if not assume there was an error - assert s.thread.is_alive() - s.server.shutdown() - - -def main(): - with spawn() as servers: - try: - while all(s.thread.is_alive() for s in servers): - sleep(1) - except KeyboardInterrupt: - pass - sys.exit(1) - - -if __name__ == '__main__': - main() diff --git a/tools/throughput_benchmark.py b/tools/throughput_benchmark.py index 8b1de13d4..f5b9f2db2 100755 --- a/tools/throughput_benchmark.py +++ b/tools/throughput_benchmark.py @@ -3,7 +3,7 @@ # Performs benchmark and append data to //website/data.json. # If //website/data.json doesn't exist, this script tries to import it from # gh-pages branch. -# To view the results locally run ./tools/http_server.py and visit +# To view the results locally run target/debug/test_server and visit # http://localhost:4545/website import os |