summaryrefslogtreecommitdiff
path: root/cli
diff options
context:
space:
mode:
Diffstat (limited to 'cli')
-rw-r--r--cli/Cargo.toml3
-rw-r--r--cli/file_fetcher.rs105
-rw-r--r--cli/http_util.rs156
-rw-r--r--cli/tests/053_import_compression.out3
-rw-r--r--cli/tests/053_import_compression/brotli2
-rw-r--r--cli/tests/053_import_compression/brotli.header3
-rw-r--r--cli/tests/053_import_compression/gzipedbin0 -> 39 bytes
-rw-r--r--cli/tests/053_import_compression/gziped.header3
-rw-r--r--cli/tests/053_import_compression/main.ts8
-rw-r--r--cli/tests/integration_tests.rs6
10 files changed, 266 insertions, 23 deletions
diff --git a/cli/Cargo.toml b/cli/Cargo.toml
index 2a5e2d414..15c46d108 100644
--- a/cli/Cargo.toml
+++ b/cli/Cargo.toml
@@ -30,6 +30,7 @@ atty = "0.2.13"
base64 = "0.11.0"
bytes = "0.5.3"
byteorder = "1.3.2"
+brotli2 = "0.3.2"
clap = "2.33.0"
dirs = "2.0.2"
dlopen = "0.1.8"
@@ -44,7 +45,7 @@ log = "0.4.8"
rand = "0.7.2"
regex = "1.3.1"
remove_dir_all = "0.5.2"
-reqwest = { version = "0.10.0", default-features = false, features = ["rustls-tls", "stream"] }
+reqwest = { version = "0.10.0", default-features = false, features = ["rustls-tls", "stream", "gzip"] }
ring = "0.16.9"
rustyline = "5.0.6"
serde = { version = "1.0.104", features = ["derive"] }
diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs
index 8fe27b2c3..471027e39 100644
--- a/cli/file_fetcher.rs
+++ b/cli/file_fetcher.rs
@@ -393,10 +393,21 @@ impl SourceFileFetcher {
let download_job = self.progress.add("Download", &module_url.to_string());
let dir = self.clone();
let module_url = module_url.clone();
+ let headers = self.get_source_code_headers(&module_url);
+ let module_etag = headers.etag;
// Single pass fetch, either yields code or yields redirect.
let f = async move {
- match http_util::fetch_string_once(&module_url).await? {
+ match http_util::fetch_string_once(&module_url, module_etag).await? {
+ FetchOnceResult::NotModified => {
+ let source_file =
+ dir.fetch_cached_remote_source(&module_url)?.unwrap();
+
+ // Explicit drop to keep reference alive until future completes.
+ drop(download_job);
+
+ Ok(source_file)
+ }
FetchOnceResult::Redirect(new_module_url) => {
// If redirects, update module_name and filename for next looped call.
dir
@@ -404,6 +415,7 @@ impl SourceFileFetcher {
&module_url,
None,
Some(new_module_url.to_string()),
+ None,
)
.unwrap();
@@ -420,13 +432,14 @@ impl SourceFileFetcher {
)
.await
}
- FetchOnceResult::Code(source, maybe_content_type) => {
+ FetchOnceResult::Code(source, maybe_content_type, etag) => {
// We land on the code.
dir
.save_source_code_headers(
&module_url,
maybe_content_type.clone(),
None,
+ etag,
)
.unwrap();
@@ -501,6 +514,7 @@ impl SourceFileFetcher {
url: &Url,
mime_type: Option<String>,
redirect_to: Option<String>,
+ etag: Option<String>,
) -> std::io::Result<()> {
let cache_key = self
.deps_cache
@@ -513,6 +527,7 @@ impl SourceFileFetcher {
let headers = SourceCodeHeaders {
mime_type,
redirect_to,
+ etag,
};
let cache_filename = self.deps_cache.get_cache_filename(url);
@@ -634,10 +649,13 @@ pub struct SourceCodeHeaders {
/// Where should we actually look for source code.
/// This should be an absolute path!
pub redirect_to: Option<String>,
+ /// ETag of the remote source file
+ pub etag: Option<String>,
}
static MIME_TYPE: &str = "mime_type";
static REDIRECT_TO: &str = "redirect_to";
+static ETAG: &str = "etag";
impl SourceCodeHeaders {
pub fn from_json_string(headers_string: String) -> Self {
@@ -648,10 +666,12 @@ impl SourceCodeHeaders {
if let Ok(headers_json) = maybe_headers_json {
let mime_type = headers_json[MIME_TYPE].as_str().map(String::from);
let redirect_to = headers_json[REDIRECT_TO].as_str().map(String::from);
+ let etag = headers_json[ETAG].as_str().map(String::from);
return SourceCodeHeaders {
mime_type,
redirect_to,
+ etag,
};
}
@@ -688,6 +708,10 @@ impl SourceCodeHeaders {
value_map.insert(REDIRECT_TO.to_string(), json!(redirect_to));
}
+ if let Some(etag) = &self.etag {
+ value_map.insert(ETAG.to_string(), json!(etag));
+ }
+
if value_map.is_empty() {
return Ok(None);
}
@@ -808,21 +832,27 @@ mod tests {
let _ = deno_fs::write_file(
headers_filepath.as_path(),
"{\"mime_type\":\"text/javascript\",\"redirect_to\":\"http://example.com/a.js\"}",
- 0o666
+ 0o666,
);
let headers = fetcher.get_source_code_headers(&url);
assert_eq!(headers.mime_type.clone().unwrap(), "text/javascript");
assert_eq!(headers.redirect_to.unwrap(), "http://example.com/a.js");
+ assert_eq!(headers.etag, None);
let _ = fetcher.save_source_code_headers(
&url,
Some("text/typescript".to_owned()),
Some("http://deno.land/a.js".to_owned()),
+ Some("W/\"04572f4749af993f4961a7e5daa1e4d5\"".to_owned()),
);
let headers2 = fetcher.get_source_code_headers(&url);
assert_eq!(headers2.mime_type.clone().unwrap(), "text/typescript");
assert_eq!(headers2.redirect_to.unwrap(), "http://deno.land/a.js");
+ assert_eq!(
+ headers2.etag.unwrap(),
+ "W/\"04572f4749af993f4961a7e5daa1e4d5\""
+ );
}
#[test]
@@ -901,6 +931,7 @@ mod tests {
&module_url_1,
Some("application/json".to_owned()),
None,
+ None,
);
fetcher_2.get_source_file_async(&module_url_1, true, false, false)
})
@@ -976,6 +1007,7 @@ mod tests {
&module_url,
Some("text/typescript".to_owned()),
None,
+ None,
);
fetcher.get_source_file_async(&module_url, true, false, false)
})
@@ -1344,6 +1376,7 @@ mod tests {
&module_url,
Some("text/javascript".to_owned()),
None,
+ None,
);
let result2 = fetcher.fetch_cached_remote_source(&module_url);
assert!(result2.is_ok());
@@ -1386,6 +1419,7 @@ mod tests {
&module_url,
Some("text/javascript".to_owned()),
None,
+ None,
);
let result2 = fetcher.fetch_cached_remote_source(&module_url);
assert!(result2.is_ok());
@@ -1683,28 +1717,28 @@ mod tests {
assert_eq!(
map_content_type(
Path::new("foo/bar.tsx"),
- Some("application/typescript")
+ Some("application/typescript"),
),
msg::MediaType::TSX
);
assert_eq!(
map_content_type(
Path::new("foo/bar.tsx"),
- Some("application/javascript")
+ Some("application/javascript"),
),
msg::MediaType::TSX
);
assert_eq!(
map_content_type(
Path::new("foo/bar.tsx"),
- Some("application/x-typescript")
+ Some("application/x-typescript"),
),
msg::MediaType::TSX
);
assert_eq!(
map_content_type(
Path::new("foo/bar.tsx"),
- Some("video/vnd.dlna.mpeg-tts")
+ Some("video/vnd.dlna.mpeg-tts"),
),
msg::MediaType::TSX
);
@@ -1715,21 +1749,21 @@ mod tests {
assert_eq!(
map_content_type(
Path::new("foo/bar.jsx"),
- Some("application/javascript")
+ Some("application/javascript"),
),
msg::MediaType::JSX
);
assert_eq!(
map_content_type(
Path::new("foo/bar.jsx"),
- Some("application/x-typescript")
+ Some("application/x-typescript"),
),
msg::MediaType::JSX
);
assert_eq!(
map_content_type(
Path::new("foo/bar.jsx"),
- Some("application/ecmascript")
+ Some("application/ecmascript"),
),
msg::MediaType::JSX
);
@@ -1740,7 +1774,7 @@ mod tests {
assert_eq!(
map_content_type(
Path::new("foo/bar.jsx"),
- Some("application/x-javascript")
+ Some("application/x-javascript"),
),
msg::MediaType::JSX
);
@@ -1758,4 +1792,53 @@ mod tests {
.to_owned();
assert_eq!(filter_shebang(code), "\nconsole.log('hello');\n".as_bytes());
}
+
+ #[test]
+ fn test_fetch_with_etag() {
+ let http_server_guard = crate::test_util::http_server();
+ let (_temp_dir, fetcher) = test_setup();
+ let module_url =
+ Url::parse("http://127.0.0.1:4545/etag_script.ts").unwrap();
+
+ let fut = async move {
+ let source = fetcher
+ .fetch_remote_source_async(&module_url, false, false, 1)
+ .await;
+ assert!(source.is_ok());
+ let source = source.unwrap();
+ assert_eq!(source.source_code, b"console.log('etag')");
+ assert_eq!(&(source.media_type), &msg::MediaType::JavaScript);
+
+ let headers = fetcher.get_source_code_headers(&module_url);
+ assert_eq!(headers.etag, Some("33a64df551425fcc55e".to_string()));
+
+ let header_path = fetcher.deps_cache.location.join(
+ fetcher
+ .deps_cache
+ .get_cache_filename_with_extension(&module_url, "headers.json"),
+ );
+
+ let modified1 = header_path.metadata().unwrap().modified().unwrap();
+
+ // Forcibly change the contents of the cache file and request
+ // it again with the cache parameters turned off.
+ // If the fetched content changes, the cached content is used.
+ fetcher
+ .save_source_code(&module_url, "changed content")
+ .unwrap();
+ let cached_source = fetcher
+ .fetch_remote_source_async(&module_url, false, false, 1)
+ .await
+ .unwrap();
+ assert_eq!(cached_source.source_code, b"changed content");
+
+ let modified2 = header_path.metadata().unwrap().modified().unwrap();
+
+ // Assert that the file has not been modified
+ assert_eq!(modified1, modified2);
+ };
+
+ tokio_util::run(fut);
+ drop(http_server_guard);
+ }
}
diff --git a/cli/http_util.rs b/cli/http_util.rs
index 8176e88ae..f9dc5933e 100644
--- a/cli/http_util.rs
+++ b/cli/http_util.rs
@@ -2,20 +2,27 @@
use crate::deno_error;
use crate::deno_error::DenoError;
use crate::version;
+use brotli2::read::BrotliDecoder;
use bytes::Bytes;
use deno_core::ErrBox;
use futures::future::FutureExt;
use reqwest;
-use reqwest::header::HeaderMap;
+use reqwest::header::ACCEPT_ENCODING;
+use reqwest::header::CONTENT_ENCODING;
use reqwest::header::CONTENT_TYPE;
+use reqwest::header::ETAG;
+use reqwest::header::IF_NONE_MATCH;
use reqwest::header::LOCATION;
use reqwest::header::USER_AGENT;
+use reqwest::header::{HeaderMap, HeaderValue};
use reqwest::redirect::Policy;
use reqwest::Client;
use reqwest::Response;
+use reqwest::StatusCode;
use std::cmp::min;
use std::future::Future;
use std::io;
+use std::io::Read;
use std::pin::Pin;
use std::task::Context;
use std::task::Poll;
@@ -29,10 +36,15 @@ lazy_static! {
USER_AGENT,
format!("Deno/{}", version::DENO).parse().unwrap(),
);
+ // todo support brotli for fetch ops
+ headers.insert(
+ ACCEPT_ENCODING, HeaderValue::from_static("gzip")
+ );
Client::builder()
.redirect(Policy::none())
.default_headers(headers)
.use_rustls_tls()
+ .gzip(true)
.build()
.unwrap()
};
@@ -41,7 +53,7 @@ lazy_static! {
/// Get instance of async reqwest::Client. This client supports
/// proxies and doesn't follow redirects.
pub fn get_client() -> &'static Client {
- &HTTP_CLIENT
+ &HTTP_CLIENT as &Client
}
/// Construct the next uri based on base uri and location header fragment
@@ -73,8 +85,9 @@ fn resolve_url_from_location(base_url: &Url, location: &str) -> Url {
#[derive(Debug, PartialEq)]
pub enum FetchOnceResult {
- // (code, maybe_content_type)
- Code(String, Option<String>),
+ // (code, maybe_content_type, etag)
+ Code(String, Option<String>, Option<String>),
+ NotModified,
Redirect(Url),
}
@@ -85,12 +98,25 @@ pub enum FetchOnceResult {
/// yields Redirect(url).
pub fn fetch_string_once(
url: &Url,
+ cached_etag: Option<String>,
) -> impl Future<Output = Result<FetchOnceResult, ErrBox>> {
let url = url.clone();
- let client = get_client();
+ let client: &Client = get_client();
let fut = async move {
- let response = client.get(url.clone()).send().await?;
+ let mut request = client
+ .get(url.clone())
+ .header(ACCEPT_ENCODING, HeaderValue::from_static("gzip, br"));
+
+ if let Some(etag) = cached_etag {
+ let if_none_match_val = HeaderValue::from_str(&etag).unwrap();
+ request = request.header(IF_NONE_MATCH, if_none_match_val);
+ }
+ let response = request.send().await?;
+
+ if response.status() == StatusCode::NOT_MODIFIED {
+ return Ok(FetchOnceResult::NotModified);
+ }
if response.status().is_redirection() {
let location_string = response
@@ -120,8 +146,33 @@ pub fn fetch_string_once(
.get(CONTENT_TYPE)
.map(|content_type| content_type.to_str().unwrap().to_owned());
- let body = response.text().await?;
- return Ok(FetchOnceResult::Code(body, content_type));
+ let etag = response
+ .headers()
+ .get(ETAG)
+ .map(|etag| etag.to_str().unwrap().to_owned());
+
+ let content_encoding = response
+ .headers()
+ .get(CONTENT_ENCODING)
+ .map(|content_encoding| content_encoding.to_str().unwrap().to_owned());
+
+ let body;
+ if let Some(content_encoding) = content_encoding {
+ body = match content_encoding {
+ _ if content_encoding == "br" => {
+ let full_bytes = response.bytes().await?;
+ let mut decoder = BrotliDecoder::new(full_bytes.as_ref());
+ let mut body = String::new();
+ decoder.read_to_string(&mut body)?;
+ body
+ }
+ _ => response.text().await?,
+ }
+ } else {
+ body = response.text().await?;
+ }
+
+ return Ok(FetchOnceResult::Code(body, content_type, etag));
};
fut.boxed()
@@ -215,10 +266,93 @@ mod tests {
let url =
Url::parse("http://127.0.0.1:4545/cli/tests/fixture.json").unwrap();
- let fut = fetch_string_once(&url).map(|result| match result {
- Ok(FetchOnceResult::Code(code, maybe_content_type)) => {
+ let fut = fetch_string_once(&url, None).map(|result| match result {
+ Ok(FetchOnceResult::Code(code, maybe_content_type, etag)) => {
assert!(!code.is_empty());
assert_eq!(maybe_content_type, Some("application/json".to_string()));
+ assert_eq!(etag, None)
+ }
+ _ => panic!(),
+ });
+
+ tokio_util::run(fut);
+ drop(http_server_guard);
+ }
+
+ #[test]
+ fn test_fetch_gzip() {
+ let http_server_guard = crate::test_util::http_server();
+ // Relies on external http server. See tools/http_server.py
+ let url = Url::parse(
+ "http://127.0.0.1:4545/cli/tests/053_import_compression/gziped",
+ )
+ .unwrap();
+
+ let fut = fetch_string_once(&url, None).map(|result| match result {
+ Ok(FetchOnceResult::Code(code, maybe_content_type, etag)) => {
+ assert!(!code.is_empty());
+ assert_eq!(code, "console.log('gzip')");
+ assert_eq!(
+ maybe_content_type,
+ Some("application/javascript".to_string())
+ );
+ assert_eq!(etag, None);
+ }
+ _ => panic!(),
+ });
+
+ tokio_util::run(fut);
+ drop(http_server_guard);
+ }
+
+ #[test]
+ fn test_fetch_with_etag() {
+ let http_server_guard = crate::test_util::http_server();
+ let url = Url::parse("http://127.0.0.1:4545/etag_script.ts").unwrap();
+
+ let fut = async move {
+ fetch_string_once(&url, None)
+ .map(|result| match result {
+ Ok(FetchOnceResult::Code(code, maybe_content_type, etag)) => {
+ assert!(!code.is_empty());
+ assert_eq!(code, "console.log('etag')");
+ assert_eq!(
+ maybe_content_type,
+ Some("application/javascript".to_string())
+ );
+ assert_eq!(etag, Some("33a64df551425fcc55e".to_string()));
+ }
+ _ => panic!(),
+ })
+ .await;
+
+ let res =
+ fetch_string_once(&url, Some("33a64df551425fcc55e".to_string())).await;
+ assert_eq!(res.unwrap(), FetchOnceResult::NotModified);
+ };
+
+ tokio_util::run(fut);
+ drop(http_server_guard);
+ }
+
+ #[test]
+ fn test_fetch_brotli() {
+ let http_server_guard = crate::test_util::http_server();
+ // Relies on external http server. See tools/http_server.py
+ let url = Url::parse(
+ "http://127.0.0.1:4545/cli/tests/053_import_compression/brotli",
+ )
+ .unwrap();
+
+ let fut = fetch_string_once(&url, None).map(|result| match result {
+ Ok(FetchOnceResult::Code(code, maybe_content_type, etag)) => {
+ assert!(!code.is_empty());
+ assert_eq!(code, "console.log('brotli');");
+ assert_eq!(
+ maybe_content_type,
+ Some("application/javascript".to_string())
+ );
+ assert_eq!(etag, None);
}
_ => panic!(),
});
@@ -236,7 +370,7 @@ mod tests {
// Dns resolver substitutes `127.0.0.1` with `localhost`
let target_url =
Url::parse("http://localhost:4545/cli/tests/fixture.json").unwrap();
- let fut = fetch_string_once(&url).map(move |result| match result {
+ let fut = fetch_string_once(&url, None).map(move |result| match result {
Ok(FetchOnceResult::Redirect(url)) => {
assert_eq!(url, target_url);
}
diff --git a/cli/tests/053_import_compression.out b/cli/tests/053_import_compression.out
new file mode 100644
index 000000000..5815b8ae2
--- /dev/null
+++ b/cli/tests/053_import_compression.out
@@ -0,0 +1,3 @@
+gzip
+brotli
+console.log('gzip')
diff --git a/cli/tests/053_import_compression/brotli b/cli/tests/053_import_compression/brotli
new file mode 100644
index 000000000..65f679d57
--- /dev/null
+++ b/cli/tests/053_import_compression/brotli
@@ -0,0 +1,2 @@
+‹
+€console.log('brotli'); \ No newline at end of file
diff --git a/cli/tests/053_import_compression/brotli.header b/cli/tests/053_import_compression/brotli.header
new file mode 100644
index 000000000..6047a3993
--- /dev/null
+++ b/cli/tests/053_import_compression/brotli.header
@@ -0,0 +1,3 @@
+Content-Encoding: br
+Content-Type: application/javascript
+Content-Length: 26 \ No newline at end of file
diff --git a/cli/tests/053_import_compression/gziped b/cli/tests/053_import_compression/gziped
new file mode 100644
index 000000000..9f9a7bc69
--- /dev/null
+++ b/cli/tests/053_import_compression/gziped
Binary files differ
diff --git a/cli/tests/053_import_compression/gziped.header b/cli/tests/053_import_compression/gziped.header
new file mode 100644
index 000000000..fda818af6
--- /dev/null
+++ b/cli/tests/053_import_compression/gziped.header
@@ -0,0 +1,3 @@
+Content-Encoding: gzip
+Content-Type: application/javascript
+Content-Length: 39 \ No newline at end of file
diff --git a/cli/tests/053_import_compression/main.ts b/cli/tests/053_import_compression/main.ts
new file mode 100644
index 000000000..eb19cc75d
--- /dev/null
+++ b/cli/tests/053_import_compression/main.ts
@@ -0,0 +1,8 @@
+import "http://127.0.0.1:4545/cli/tests/053_import_compression/gziped";
+import "http://127.0.0.1:4545/cli/tests/053_import_compression/brotli";
+
+console.log(
+ await fetch(
+ "http://127.0.0.1:4545/cli/tests/053_import_compression/gziped"
+ ).then(res => res.text())
+);
diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs
index 70ad55e56..91c30a51d 100644
--- a/cli/tests/integration_tests.rs
+++ b/cli/tests/integration_tests.rs
@@ -661,6 +661,12 @@ itest!(top_level_for_await_ts {
output: "top_level_for_await.out",
});
+itest!(_053_import_compression {
+ args: "run --reload --allow-net 053_import_compression/main.ts",
+ output: "053_import_compression.out",
+ http_server: true,
+});
+
mod util {
use deno::colors::strip_ansi_codes;
pub use deno::test_util::*;
generated by cgit v1.2.3 (git 2.25.1) at 2025-08-06 14:32:54 +0000