diff options
| -rw-r--r-- | Cargo.lock | 71 | ||||
| -rw-r--r-- | cli/Cargo.toml | 3 | ||||
| -rw-r--r-- | cli/file_fetcher.rs | 105 | ||||
| -rw-r--r-- | cli/http_util.rs | 156 | ||||
| -rw-r--r-- | cli/tests/053_import_compression.out | 3 | ||||
| -rw-r--r-- | cli/tests/053_import_compression/brotli | 2 | ||||
| -rw-r--r-- | cli/tests/053_import_compression/brotli.header | 3 | ||||
| -rw-r--r-- | cli/tests/053_import_compression/gziped | bin | 0 -> 39 bytes | |||
| -rw-r--r-- | cli/tests/053_import_compression/gziped.header | 3 | ||||
| -rw-r--r-- | cli/tests/053_import_compression/main.ts | 8 | ||||
| -rw-r--r-- | cli/tests/integration_tests.rs | 6 | ||||
| -rwxr-xr-x | tools/http_server.py | 36 |
12 files changed, 373 insertions, 23 deletions
diff --git a/Cargo.lock b/Cargo.lock index c3a77b5ff..9aa2aed46 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1,6 +1,11 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. [[package]] +name = "adler32" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" + +[[package]] name = "aho-corasick" version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -37,6 +42,18 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] +name = "async-compression" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "bytes 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", + "flate2 1.0.13 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-core 0.3.1 (registry+https://github.com/rust-lang/crates.io-index)", + "memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)", + "pin-project-lite 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] name = "atty" version = "0.2.13" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -99,6 +116,24 @@ dependencies = [ ] [[package]] +name = "brotli-sys" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cc 1.0.48 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "brotli2" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "brotli-sys 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] name = "bumpalo" version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -187,6 +222,14 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" [[package]] +name = "crc32fast" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] name = "crossbeam-utils" version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -210,6 +253,7 @@ dependencies = [ "ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)", "base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", + "brotli2 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)", "bytes 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", "clap 2.33.0 (registry+https://github.com/rust-lang/crates.io-index)", @@ -356,6 +400,17 @@ dependencies = [ ] [[package]] +name = "flate2" +version = "1.0.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "cfg-if 0.1.10 (registry+https://github.com/rust-lang/crates.io-index)", + "crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc 0.2.66 (registry+https://github.com/rust-lang/crates.io-index)", + "miniz_oxide 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] name = "fnv" version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -684,6 +739,14 @@ dependencies = [ ] [[package]] +name = "miniz_oxide" +version = "0.3.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +dependencies = [ + "adler32 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] name = "mio" version = "0.6.21" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1009,6 +1072,7 @@ name = "reqwest" version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" dependencies = [ + "async-compression 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)", "base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)", "bytes 0.5.3 (registry+https://github.com/rust-lang/crates.io-index)", "encoding_rs 0.8.22 (registry+https://github.com/rust-lang/crates.io-index)", @@ -1738,12 +1802,14 @@ dependencies = [ ] [metadata] +"checksum adler32 1.0.4 (registry+https://github.com/rust-lang/crates.io-index)" = "5d2e7343e7fc9de883d1b0341e0b13970f764c14101234857d2ddafa1cb1cac2" "checksum aho-corasick 0.7.6 (registry+https://github.com/rust-lang/crates.io-index)" = "58fb5e95d83b38284460a5fda7d6470aa0b8844d283a0b614b8535e880800d2d" "checksum ansi_term 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ee49baf6cb617b853aa8d93bf420db2383fab46d314482ca2803b40d5fde979b" "checksum anyhow 1.0.26 (registry+https://github.com/rust-lang/crates.io-index)" = "7825f6833612eb2414095684fcf6c635becf3ce97fe48cf6421321e93bfbd53c" "checksum arc-swap 0.4.4 (registry+https://github.com/rust-lang/crates.io-index)" = "d7b8a9123b8027467bce0099fe556c628a53c8d83df0507084c31e9ba2e39aff" "checksum arrayref 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "0d382e583f07208808f6b1249e60848879ba3543f57c32277bf52d69c2f0f0ee" "checksum arrayvec 0.5.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cff77d8686867eceff3105329d4698d96c2391c176d5d03adc90c7389162b5b8" +"checksum async-compression 0.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "2c5c52622726d68ec35fec88edfb4ccb862d4f3b3bfa4af2f45142e69ef9b220" "checksum atty 0.2.13 (registry+https://github.com/rust-lang/crates.io-index)" = "1803c647a3ec87095e7ae7acfca019e98de5ec9a7d01343f611cf3152ed71a90" "checksum autocfg 0.1.7 (registry+https://github.com/rust-lang/crates.io-index)" = "1d49d90015b3c36167a20fe2810c5cd875ad504b39cff3d4eae7977e6b7c1cb2" "checksum backtrace 0.3.40 (registry+https://github.com/rust-lang/crates.io-index)" = "924c76597f0d9ca25d762c25a4d369d51267536465dc5064bdf0eb073ed477ea" @@ -1752,6 +1818,8 @@ dependencies = [ "checksum base64 0.11.0 (registry+https://github.com/rust-lang/crates.io-index)" = "b41b7ea54a0c9d92199de89e20e58d49f02f8e699814ef3fdf266f6f748d15c7" "checksum bitflags 1.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" "checksum blake2b_simd 0.5.10 (registry+https://github.com/rust-lang/crates.io-index)" = "d8fb2d74254a3a0b5cac33ac9f8ed0e44aa50378d9dbb2e5d83bd21ed1dc2c8a" +"checksum brotli-sys 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "4445dea95f4c2b41cde57cc9fee236ae4dbae88d8fcbdb4750fc1bb5d86aaecd" +"checksum brotli2 0.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "0cb036c3eade309815c15ddbacec5b22c4d1f3983a774ab2eac2e3e9ea85568e" "checksum bumpalo 2.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ad807f2fc2bf185eeb98ff3a901bd46dc5ad58163d0fa4577ba0d25674d71708" "checksum byteorder 1.3.2 (registry+https://github.com/rust-lang/crates.io-index)" = "a7c3dd8985a7111efc5c80b44e23ecdd8c007de8ade3b96595387e812b957cf5" "checksum bytes 0.4.12 (registry+https://github.com/rust-lang/crates.io-index)" = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" @@ -1765,6 +1833,7 @@ dependencies = [ "checksum constant_time_eq 0.1.4 (registry+https://github.com/rust-lang/crates.io-index)" = "995a44c877f9212528ccc74b21a232f66ad69001e40ede5bcee2ac9ef2657120" "checksum core-foundation 0.6.4 (registry+https://github.com/rust-lang/crates.io-index)" = "25b9e03f145fd4f2bf705e07b900cd41fc636598fe5dc452fd0db1441c3f496d" "checksum core-foundation-sys 0.6.2 (registry+https://github.com/rust-lang/crates.io-index)" = "e7ca8a5221364ef15ce201e8ed2f609fc312682a8f4e0e3d4aa5879764e0fa3b" +"checksum crc32fast 1.2.0 (registry+https://github.com/rust-lang/crates.io-index)" = "ba125de2af0df55319f41944744ad91c71113bf74a4646efff39afe1f6842db1" "checksum crossbeam-utils 0.6.6 (registry+https://github.com/rust-lang/crates.io-index)" = "04973fa96e96579258a5091af6003abde64af786b860f18622b82e026cca60e6" "checksum ct-logs 0.6.0 (registry+https://github.com/rust-lang/crates.io-index)" = "4d3686f5fa27dbc1d76c751300376e167c5a43387f44bb451fd1c24776e49113" "checksum dirs 2.0.2 (registry+https://github.com/rust-lang/crates.io-index)" = "13aea89a5c93364a98e9b37b2fa237effbb694d5cfe01c5b70941f7eb087d5e3" @@ -1776,6 +1845,7 @@ dependencies = [ "checksum encoding_rs 0.8.22 (registry+https://github.com/rust-lang/crates.io-index)" = "cd8d03faa7fe0c1431609dfad7bbe827af30f82e1e2ae6f7ee4fca6bd764bc28" "checksum failure 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f8273f13c977665c5db7eb2b99ae520952fe5ac831ae4cd09d80c4c7042b5ed9" "checksum failure_derive 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "0bc225b78e0391e4b8683440bf2e63c2deeeb2ce5189eab46e2b68c6d3725d08" +"checksum flate2 1.0.13 (registry+https://github.com/rust-lang/crates.io-index)" = "6bd6d6f4752952feb71363cffc9ebac9411b75b87c6ab6058c40c8900cf43c0f" "checksum fnv 1.0.6 (registry+https://github.com/rust-lang/crates.io-index)" = "2fad85553e09a6f881f739c29f0b00b0f01357c743266d478b68951ce23285f3" "checksum fuchsia-cprng 0.1.1 (registry+https://github.com/rust-lang/crates.io-index)" = "a06f77d526c1a601b7c4cdd98f54b5eaabffc14d5f2f0296febdc7f357c6d3ba" "checksum fuchsia-zircon 0.3.3 (registry+https://github.com/rust-lang/crates.io-index)" = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" @@ -1813,6 +1883,7 @@ dependencies = [ "checksum memchr 2.2.1 (registry+https://github.com/rust-lang/crates.io-index)" = "88579771288728879b57485cc7d6b07d648c9f0141eb955f8ab7f9d45394468e" "checksum mime 0.3.14 (registry+https://github.com/rust-lang/crates.io-index)" = "dd1d63acd1b78403cc0c325605908475dd9b9a3acbf65ed8bcab97e27014afcf" "checksum mime_guess 2.0.1 (registry+https://github.com/rust-lang/crates.io-index)" = "1a0ed03949aef72dbdf3116a383d7b38b4768e6f960528cd6a6044aa9ed68599" +"checksum miniz_oxide 0.3.5 (registry+https://github.com/rust-lang/crates.io-index)" = "6f3f74f726ae935c3f514300cc6773a0c9492abc5e972d42ba0c0ebb88757625" "checksum mio 0.6.21 (registry+https://github.com/rust-lang/crates.io-index)" = "302dec22bcf6bae6dfb69c647187f4b4d0fb6f535521f7bc022430ce8e12008f" "checksum mio-named-pipes 0.1.6 (registry+https://github.com/rust-lang/crates.io-index)" = "f5e374eff525ce1c5b7687c4cef63943e7686524a387933ad27ca7ec43779cb3" "checksum mio-uds 0.6.7 (registry+https://github.com/rust-lang/crates.io-index)" = "966257a94e196b11bb43aca423754d87429960a768de9414f3691d6957abf125" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 2a5e2d414..15c46d108 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -30,6 +30,7 @@ atty = "0.2.13" base64 = "0.11.0" bytes = "0.5.3" byteorder = "1.3.2" +brotli2 = "0.3.2" clap = "2.33.0" dirs = "2.0.2" dlopen = "0.1.8" @@ -44,7 +45,7 @@ log = "0.4.8" rand = "0.7.2" regex = "1.3.1" remove_dir_all = "0.5.2" -reqwest = { version = "0.10.0", default-features = false, features = ["rustls-tls", "stream"] } +reqwest = { version = "0.10.0", default-features = false, features = ["rustls-tls", "stream", "gzip"] } ring = "0.16.9" rustyline = "5.0.6" serde = { version = "1.0.104", features = ["derive"] } diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs index 8fe27b2c3..471027e39 100644 --- a/cli/file_fetcher.rs +++ b/cli/file_fetcher.rs @@ -393,10 +393,21 @@ impl SourceFileFetcher { let download_job = self.progress.add("Download", &module_url.to_string()); let dir = self.clone(); let module_url = module_url.clone(); + let headers = self.get_source_code_headers(&module_url); + let module_etag = headers.etag; // Single pass fetch, either yields code or yields redirect. let f = async move { - match http_util::fetch_string_once(&module_url).await? { + match http_util::fetch_string_once(&module_url, module_etag).await? { + FetchOnceResult::NotModified => { + let source_file = + dir.fetch_cached_remote_source(&module_url)?.unwrap(); + + // Explicit drop to keep reference alive until future completes. + drop(download_job); + + Ok(source_file) + } FetchOnceResult::Redirect(new_module_url) => { // If redirects, update module_name and filename for next looped call. dir @@ -404,6 +415,7 @@ impl SourceFileFetcher { &module_url, None, Some(new_module_url.to_string()), + None, ) .unwrap(); @@ -420,13 +432,14 @@ impl SourceFileFetcher { ) .await } - FetchOnceResult::Code(source, maybe_content_type) => { + FetchOnceResult::Code(source, maybe_content_type, etag) => { // We land on the code. dir .save_source_code_headers( &module_url, maybe_content_type.clone(), None, + etag, ) .unwrap(); @@ -501,6 +514,7 @@ impl SourceFileFetcher { url: &Url, mime_type: Option<String>, redirect_to: Option<String>, + etag: Option<String>, ) -> std::io::Result<()> { let cache_key = self .deps_cache @@ -513,6 +527,7 @@ impl SourceFileFetcher { let headers = SourceCodeHeaders { mime_type, redirect_to, + etag, }; let cache_filename = self.deps_cache.get_cache_filename(url); @@ -634,10 +649,13 @@ pub struct SourceCodeHeaders { /// Where should we actually look for source code. /// This should be an absolute path! pub redirect_to: Option<String>, + /// ETag of the remote source file + pub etag: Option<String>, } static MIME_TYPE: &str = "mime_type"; static REDIRECT_TO: &str = "redirect_to"; +static ETAG: &str = "etag"; impl SourceCodeHeaders { pub fn from_json_string(headers_string: String) -> Self { @@ -648,10 +666,12 @@ impl SourceCodeHeaders { if let Ok(headers_json) = maybe_headers_json { let mime_type = headers_json[MIME_TYPE].as_str().map(String::from); let redirect_to = headers_json[REDIRECT_TO].as_str().map(String::from); + let etag = headers_json[ETAG].as_str().map(String::from); return SourceCodeHeaders { mime_type, redirect_to, + etag, }; } @@ -688,6 +708,10 @@ impl SourceCodeHeaders { value_map.insert(REDIRECT_TO.to_string(), json!(redirect_to)); } + if let Some(etag) = &self.etag { + value_map.insert(ETAG.to_string(), json!(etag)); + } + if value_map.is_empty() { return Ok(None); } @@ -808,21 +832,27 @@ mod tests { let _ = deno_fs::write_file( headers_filepath.as_path(), "{\"mime_type\":\"text/javascript\",\"redirect_to\":\"http://example.com/a.js\"}", - 0o666 + 0o666, ); let headers = fetcher.get_source_code_headers(&url); assert_eq!(headers.mime_type.clone().unwrap(), "text/javascript"); assert_eq!(headers.redirect_to.unwrap(), "http://example.com/a.js"); + assert_eq!(headers.etag, None); let _ = fetcher.save_source_code_headers( &url, Some("text/typescript".to_owned()), Some("http://deno.land/a.js".to_owned()), + Some("W/\"04572f4749af993f4961a7e5daa1e4d5\"".to_owned()), ); let headers2 = fetcher.get_source_code_headers(&url); assert_eq!(headers2.mime_type.clone().unwrap(), "text/typescript"); assert_eq!(headers2.redirect_to.unwrap(), "http://deno.land/a.js"); + assert_eq!( + headers2.etag.unwrap(), + "W/\"04572f4749af993f4961a7e5daa1e4d5\"" + ); } #[test] @@ -901,6 +931,7 @@ mod tests { &module_url_1, Some("application/json".to_owned()), None, + None, ); fetcher_2.get_source_file_async(&module_url_1, true, false, false) }) @@ -976,6 +1007,7 @@ mod tests { &module_url, Some("text/typescript".to_owned()), None, + None, ); fetcher.get_source_file_async(&module_url, true, false, false) }) @@ -1344,6 +1376,7 @@ mod tests { &module_url, Some("text/javascript".to_owned()), None, + None, ); let result2 = fetcher.fetch_cached_remote_source(&module_url); assert!(result2.is_ok()); @@ -1386,6 +1419,7 @@ mod tests { &module_url, Some("text/javascript".to_owned()), None, + None, ); let result2 = fetcher.fetch_cached_remote_source(&module_url); assert!(result2.is_ok()); @@ -1683,28 +1717,28 @@ mod tests { assert_eq!( map_content_type( Path::new("foo/bar.tsx"), - Some("application/typescript") + Some("application/typescript"), ), msg::MediaType::TSX ); assert_eq!( map_content_type( Path::new("foo/bar.tsx"), - Some("application/javascript") + Some("application/javascript"), ), msg::MediaType::TSX ); assert_eq!( map_content_type( Path::new("foo/bar.tsx"), - Some("application/x-typescript") + Some("application/x-typescript"), ), msg::MediaType::TSX ); assert_eq!( map_content_type( Path::new("foo/bar.tsx"), - Some("video/vnd.dlna.mpeg-tts") + Some("video/vnd.dlna.mpeg-tts"), ), msg::MediaType::TSX ); @@ -1715,21 +1749,21 @@ mod tests { assert_eq!( map_content_type( Path::new("foo/bar.jsx"), - Some("application/javascript") + Some("application/javascript"), ), msg::MediaType::JSX ); assert_eq!( map_content_type( Path::new("foo/bar.jsx"), - Some("application/x-typescript") + Some("application/x-typescript"), ), msg::MediaType::JSX ); assert_eq!( map_content_type( Path::new("foo/bar.jsx"), - Some("application/ecmascript") + Some("application/ecmascript"), ), msg::MediaType::JSX ); @@ -1740,7 +1774,7 @@ mod tests { assert_eq!( map_content_type( Path::new("foo/bar.jsx"), - Some("application/x-javascript") + Some("application/x-javascript"), ), msg::MediaType::JSX ); @@ -1758,4 +1792,53 @@ mod tests { .to_owned(); assert_eq!(filter_shebang(code), "\nconsole.log('hello');\n".as_bytes()); } + + #[test] + fn test_fetch_with_etag() { + let http_server_guard = crate::test_util::http_server(); + let (_temp_dir, fetcher) = test_setup(); + let module_url = + Url::parse("http://127.0.0.1:4545/etag_script.ts").unwrap(); + + let fut = async move { + let source = fetcher + .fetch_remote_source_async(&module_url, false, false, 1) + .await; + assert!(source.is_ok()); + let source = source.unwrap(); + assert_eq!(source.source_code, b"console.log('etag')"); + assert_eq!(&(source.media_type), &msg::MediaType::JavaScript); + + let headers = fetcher.get_source_code_headers(&module_url); + assert_eq!(headers.etag, Some("33a64df551425fcc55e".to_string())); + + let header_path = fetcher.deps_cache.location.join( + fetcher + .deps_cache + .get_cache_filename_with_extension(&module_url, "headers.json"), + ); + + let modified1 = header_path.metadata().unwrap().modified().unwrap(); + + // Forcibly change the contents of the cache file and request + // it again with the cache parameters turned off. + // If the fetched content changes, the cached content is used. + fetcher + .save_source_code(&module_url, "changed content") + .unwrap(); + let cached_source = fetcher + .fetch_remote_source_async(&module_url, false, false, 1) + .await + .unwrap(); + assert_eq!(cached_source.source_code, b"changed content"); + + let modified2 = header_path.metadata().unwrap().modified().unwrap(); + + // Assert that the file has not been modified + assert_eq!(modified1, modified2); + }; + + tokio_util::run(fut); + drop(http_server_guard); + } } diff --git a/cli/http_util.rs b/cli/http_util.rs index 8176e88ae..f9dc5933e 100644 --- a/cli/http_util.rs +++ b/cli/http_util.rs @@ -2,20 +2,27 @@ use crate::deno_error; use crate::deno_error::DenoError; use crate::version; +use brotli2::read::BrotliDecoder; use bytes::Bytes; use deno_core::ErrBox; use futures::future::FutureExt; use reqwest; -use reqwest::header::HeaderMap; +use reqwest::header::ACCEPT_ENCODING; +use reqwest::header::CONTENT_ENCODING; use reqwest::header::CONTENT_TYPE; +use reqwest::header::ETAG; +use reqwest::header::IF_NONE_MATCH; use reqwest::header::LOCATION; use reqwest::header::USER_AGENT; +use reqwest::header::{HeaderMap, HeaderValue}; use reqwest::redirect::Policy; use reqwest::Client; use reqwest::Response; +use reqwest::StatusCode; use std::cmp::min; use std::future::Future; use std::io; +use std::io::Read; use std::pin::Pin; use std::task::Context; use std::task::Poll; @@ -29,10 +36,15 @@ lazy_static! { USER_AGENT, format!("Deno/{}", version::DENO).parse().unwrap(), ); + // todo support brotli for fetch ops + headers.insert( + ACCEPT_ENCODING, HeaderValue::from_static("gzip") + ); Client::builder() .redirect(Policy::none()) .default_headers(headers) .use_rustls_tls() + .gzip(true) .build() .unwrap() }; @@ -41,7 +53,7 @@ lazy_static! { /// Get instance of async reqwest::Client. This client supports /// proxies and doesn't follow redirects. pub fn get_client() -> &'static Client { - &HTTP_CLIENT + &HTTP_CLIENT as &Client } /// Construct the next uri based on base uri and location header fragment @@ -73,8 +85,9 @@ fn resolve_url_from_location(base_url: &Url, location: &str) -> Url { #[derive(Debug, PartialEq)] pub enum FetchOnceResult { - // (code, maybe_content_type) - Code(String, Option<String>), + // (code, maybe_content_type, etag) + Code(String, Option<String>, Option<String>), + NotModified, Redirect(Url), } @@ -85,12 +98,25 @@ pub enum FetchOnceResult { /// yields Redirect(url). pub fn fetch_string_once( url: &Url, + cached_etag: Option<String>, ) -> impl Future<Output = Result<FetchOnceResult, ErrBox>> { let url = url.clone(); - let client = get_client(); + let client: &Client = get_client(); let fut = async move { - let response = client.get(url.clone()).send().await?; + let mut request = client + .get(url.clone()) + .header(ACCEPT_ENCODING, HeaderValue::from_static("gzip, br")); + + if let Some(etag) = cached_etag { + let if_none_match_val = HeaderValue::from_str(&etag).unwrap(); + request = request.header(IF_NONE_MATCH, if_none_match_val); + } + let response = request.send().await?; + + if response.status() == StatusCode::NOT_MODIFIED { + return Ok(FetchOnceResult::NotModified); + } if response.status().is_redirection() { let location_string = response @@ -120,8 +146,33 @@ pub fn fetch_string_once( .get(CONTENT_TYPE) .map(|content_type| content_type.to_str().unwrap().to_owned()); - let body = response.text().await?; - return Ok(FetchOnceResult::Code(body, content_type)); + let etag = response + .headers() + .get(ETAG) + .map(|etag| etag.to_str().unwrap().to_owned()); + + let content_encoding = response + .headers() + .get(CONTENT_ENCODING) + .map(|content_encoding| content_encoding.to_str().unwrap().to_owned()); + + let body; + if let Some(content_encoding) = content_encoding { + body = match content_encoding { + _ if content_encoding == "br" => { + let full_bytes = response.bytes().await?; + let mut decoder = BrotliDecoder::new(full_bytes.as_ref()); + let mut body = String::new(); + decoder.read_to_string(&mut body)?; + body + } + _ => response.text().await?, + } + } else { + body = response.text().await?; + } + + return Ok(FetchOnceResult::Code(body, content_type, etag)); }; fut.boxed() @@ -215,10 +266,93 @@ mod tests { let url = Url::parse("http://127.0.0.1:4545/cli/tests/fixture.json").unwrap(); - let fut = fetch_string_once(&url).map(|result| match result { - Ok(FetchOnceResult::Code(code, maybe_content_type)) => { + let fut = fetch_string_once(&url, None).map(|result| match result { + Ok(FetchOnceResult::Code(code, maybe_content_type, etag)) => { assert!(!code.is_empty()); assert_eq!(maybe_content_type, Some("application/json".to_string())); + assert_eq!(etag, None) + } + _ => panic!(), + }); + + tokio_util::run(fut); + drop(http_server_guard); + } + + #[test] + fn test_fetch_gzip() { + let http_server_guard = crate::test_util::http_server(); + // Relies on external http server. See tools/http_server.py + let url = Url::parse( + "http://127.0.0.1:4545/cli/tests/053_import_compression/gziped", + ) + .unwrap(); + + let fut = fetch_string_once(&url, None).map(|result| match result { + Ok(FetchOnceResult::Code(code, maybe_content_type, etag)) => { + assert!(!code.is_empty()); + assert_eq!(code, "console.log('gzip')"); + assert_eq!( + maybe_content_type, + Some("application/javascript".to_string()) + ); + assert_eq!(etag, None); + } + _ => panic!(), + }); + + tokio_util::run(fut); + drop(http_server_guard); + } + + #[test] + fn test_fetch_with_etag() { + let http_server_guard = crate::test_util::http_server(); + let url = Url::parse("http://127.0.0.1:4545/etag_script.ts").unwrap(); + + let fut = async move { + fetch_string_once(&url, None) + .map(|result| match result { + Ok(FetchOnceResult::Code(code, maybe_content_type, etag)) => { + assert!(!code.is_empty()); + assert_eq!(code, "console.log('etag')"); + assert_eq!( + maybe_content_type, + Some("application/javascript".to_string()) + ); + assert_eq!(etag, Some("33a64df551425fcc55e".to_string())); + } + _ => panic!(), + }) + .await; + + let res = + fetch_string_once(&url, Some("33a64df551425fcc55e".to_string())).await; + assert_eq!(res.unwrap(), FetchOnceResult::NotModified); + }; + + tokio_util::run(fut); + drop(http_server_guard); + } + + #[test] + fn test_fetch_brotli() { + let http_server_guard = crate::test_util::http_server(); + // Relies on external http server. See tools/http_server.py + let url = Url::parse( + "http://127.0.0.1:4545/cli/tests/053_import_compression/brotli", + ) + .unwrap(); + + let fut = fetch_string_once(&url, None).map(|result| match result { + Ok(FetchOnceResult::Code(code, maybe_content_type, etag)) => { + assert!(!code.is_empty()); + assert_eq!(code, "console.log('brotli');"); + assert_eq!( + maybe_content_type, + Some("application/javascript".to_string()) + ); + assert_eq!(etag, None); } _ => panic!(), }); @@ -236,7 +370,7 @@ mod tests { // Dns resolver substitutes `127.0.0.1` with `localhost` let target_url = Url::parse("http://localhost:4545/cli/tests/fixture.json").unwrap(); - let fut = fetch_string_once(&url).map(move |result| match result { + let fut = fetch_string_once(&url, None).map(move |result| match result { Ok(FetchOnceResult::Redirect(url)) => { assert_eq!(url, target_url); } diff --git a/cli/tests/053_import_compression.out b/cli/tests/053_import_compression.out new file mode 100644 index 000000000..5815b8ae2 --- /dev/null +++ b/cli/tests/053_import_compression.out @@ -0,0 +1,3 @@ +gzip +brotli +console.log('gzip') diff --git a/cli/tests/053_import_compression/brotli b/cli/tests/053_import_compression/brotli new file mode 100644 index 000000000..65f679d57 --- /dev/null +++ b/cli/tests/053_import_compression/brotli @@ -0,0 +1,2 @@ +‹ +€console.log('brotli');
\ No newline at end of file diff --git a/cli/tests/053_import_compression/brotli.header b/cli/tests/053_import_compression/brotli.header new file mode 100644 index 000000000..6047a3993 --- /dev/null +++ b/cli/tests/053_import_compression/brotli.header @@ -0,0 +1,3 @@ +Content-Encoding: br +Content-Type: application/javascript +Content-Length: 26
\ No newline at end of file diff --git a/cli/tests/053_import_compression/gziped b/cli/tests/053_import_compression/gziped Binary files differnew file mode 100644 index 000000000..9f9a7bc69 --- /dev/null +++ b/cli/tests/053_import_compression/gziped diff --git a/cli/tests/053_import_compression/gziped.header b/cli/tests/053_import_compression/gziped.header new file mode 100644 index 000000000..fda818af6 --- /dev/null +++ b/cli/tests/053_import_compression/gziped.header @@ -0,0 +1,3 @@ +Content-Encoding: gzip +Content-Type: application/javascript +Content-Length: 39
\ No newline at end of file diff --git a/cli/tests/053_import_compression/main.ts b/cli/tests/053_import_compression/main.ts new file mode 100644 index 000000000..eb19cc75d --- /dev/null +++ b/cli/tests/053_import_compression/main.ts @@ -0,0 +1,8 @@ +import "http://127.0.0.1:4545/cli/tests/053_import_compression/gziped"; +import "http://127.0.0.1:4545/cli/tests/053_import_compression/brotli"; + +console.log( + await fetch( + "http://127.0.0.1:4545/cli/tests/053_import_compression/gziped" + ).then(res => res.text()) +); diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs index 70ad55e56..91c30a51d 100644 --- a/cli/tests/integration_tests.rs +++ b/cli/tests/integration_tests.rs @@ -661,6 +661,12 @@ itest!(top_level_for_await_ts { output: "top_level_for_await.out", }); +itest!(_053_import_compression { + args: "run --reload --allow-net 053_import_compression/main.ts", + output: "053_import_compression.out", + http_server: true, +}); + mod util { use deno::colors::strip_ansi_codes; pub use deno::test_util::*; diff --git a/tools/http_server.py b/tools/http_server.py index 47fbf959a..3152a8982 100755 --- a/tools/http_server.py +++ b/tools/http_server.py @@ -31,6 +31,42 @@ class QuietSimpleHTTPRequestHandler(SimpleHTTPServer.SimpleHTTPRequestHandler): class ContentTypeHandler(QuietSimpleHTTPRequestHandler): def do_GET(self): + + # Check if there is a custom header configuration ending + # with ".header" before sending the file + maybe_header_file_path = "./" + self.path + ".header" + if os.path.exists(maybe_header_file_path): + self.protocol_version = 'HTTP/1.1' + self.send_response(200, 'OK') + + f = open(maybe_header_file_path) + for line in f: + kv = line.split(": ") + self.send_header(kv[0].strip(), kv[1].strip()) + f.close() + self.end_headers() + + body = open("./" + self.path) + self.wfile.write(body.read()) + body.close() + return + + if "etag_script.ts" in self.path: + self.protocol_version = 'HTTP/1.1' + if_not_match = self.headers.getheader('if-none-match') + if if_not_match == "33a64df551425fcc55e": + self.send_response(304, 'Not Modified') + self.send_header('Content-type', 'application/javascript') + self.send_header('ETag', '33a64df551425fcc55e') + self.end_headers() + else: + self.send_response(200, 'OK') + self.send_header('Content-type', 'application/javascript') + self.send_header('ETag', '33a64df551425fcc55e') + self.end_headers() + self.wfile.write(bytes("console.log('etag')")) + return + if "multipart_form_data.txt" in self.path: self.protocol_version = 'HTTP/1.1' self.send_response(200, 'OK') |