summaryrefslogtreecommitdiff
path: root/cli/file_fetcher.rs
diff options
context:
space:
mode:
Diffstat (limited to 'cli/file_fetcher.rs')
-rw-r--r--cli/file_fetcher.rs618
1 files changed, 140 insertions, 478 deletions
diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs
index 41f31fb49..de3a76eec 100644
--- a/cli/file_fetcher.rs
+++ b/cli/file_fetcher.rs
@@ -3,11 +3,10 @@ use crate::colors;
use crate::deno_error::DenoError;
use crate::deno_error::ErrorKind;
use crate::deno_error::GetErrorKind;
-use crate::disk_cache::DiskCache;
+use crate::http_cache::HttpCache;
use crate::http_util;
use crate::http_util::create_http_client;
use crate::http_util::FetchOnceResult;
-use crate::http_util::ResultPayload;
use crate::msg;
use deno_core::ErrBox;
use deno_core::ModuleSpecifier;
@@ -15,11 +14,11 @@ use futures::future::Either;
use futures::future::FutureExt;
use regex::Regex;
use reqwest;
-use serde_json;
use std;
use std::collections::HashMap;
use std::fs;
use std::future::Future;
+use std::io::Read;
use std::path::Path;
use std::path::PathBuf;
use std::pin::Pin;
@@ -30,28 +29,6 @@ use std::sync::Mutex;
use url;
use url::Url;
-pub fn source_header_cache_failed_error(
- module_name: &str,
- reason: &str,
-) -> ErrBox {
- DenoError::new(
- ErrorKind::Other,
- format!(
- "Source code header cache failed for '{}': {}",
- module_name, reason
- ),
- )
- .into()
-}
-
-pub fn source_cache_failed_error(module_name: &str, reason: &str) -> ErrBox {
- DenoError::new(
- ErrorKind::Other,
- format!("Source code cache failed for '{}': {}", module_name, reason),
- )
- .into()
-}
-
/// Structure representing local or remote file.
///
/// In case of remote file `url` might be different than originally requested URL, if so
@@ -90,22 +67,21 @@ impl SourceFileCache {
const SUPPORTED_URL_SCHEMES: [&str; 3] = ["http", "https", "file"];
-/// `DenoDir` serves as coordinator for multiple `DiskCache`s containing them
-/// in single directory that can be controlled with `$DENO_DIR` env variable.
#[derive(Clone)]
pub struct SourceFileFetcher {
- deps_cache: DiskCache,
source_file_cache: SourceFileCache,
cache_blacklist: Vec<String>,
use_disk_cache: bool,
no_remote: bool,
cached_only: bool,
http_client: reqwest::Client,
+ // This field is public only to expose it's location
+ pub http_cache: HttpCache,
}
impl SourceFileFetcher {
pub fn new(
- deps_cache: DiskCache,
+ http_cache: HttpCache,
use_disk_cache: bool,
cache_blacklist: Vec<String>,
no_remote: bool,
@@ -113,7 +89,7 @@ impl SourceFileFetcher {
ca_file: Option<String>,
) -> Result<Self, ErrBox> {
let file_fetcher = Self {
- deps_cache,
+ http_cache,
source_file_cache: SourceFileCache::default(),
cache_blacklist,
use_disk_cache,
@@ -333,54 +309,45 @@ impl SourceFileFetcher {
&self,
module_url: &Url,
) -> Result<Option<SourceFile>, ErrBox> {
- let source_code_headers = self.get_source_code_headers(&module_url);
- // If source code headers says that it would redirect elsewhere,
- // (meaning that the source file might not exist; only .headers.json is present)
- // Abort reading attempts to the cached source file and and follow the redirect.
- if let Some(redirect_to) = source_code_headers.redirect_to {
- // E.g.
- // module_name https://import-meta.now.sh/redirect.js
- // filename /Users/kun/Library/Caches/deno/deps/https/import-meta.now.sh/redirect.js
- // redirect_to https://import-meta.now.sh/sub/final1.js
- // real_filename /Users/kun/Library/Caches/deno/deps/https/import-meta.now.sh/sub/final1.js
- // real_module_name = https://import-meta.now.sh/sub/final1.js
- let redirect_url = Url::parse(&redirect_to).expect("Should be valid URL");
-
- // Recurse.
- // TODO(bartlomieju): I'm pretty sure we should call `fetch_remote_source_async` here.
- // Should we expect that all redirects are cached?
- return self.fetch_cached_remote_source(&redirect_url);
- }
-
- // No redirect needed or end of redirects.
- // We can try read the file
- let filepath = self
- .deps_cache
- .location
- .join(self.deps_cache.get_cache_filename(&module_url));
- let source_code = match fs::read(filepath.clone()) {
+ let result = self.http_cache.get(&module_url);
+ let result = match result {
Err(e) => {
- if e.kind() == std::io::ErrorKind::NotFound {
- return Ok(None);
- } else {
- return Err(e.into());
+ if let Some(e) = e.downcast_ref::<std::io::Error>() {
+ if e.kind() == std::io::ErrorKind::NotFound {
+ return Ok(None);
+ }
}
+ return Err(e);
}
Ok(c) => c,
};
- let media_type =
- map_content_type(&filepath, source_code_headers.mime_type.as_deref());
+
+ let (mut source_file, headers) = result;
+ if let Some(redirect_to) = headers.get("location") {
+ let redirect_url = Url::parse(redirect_to).expect("Should be valid URL");
+ return self.fetch_cached_remote_source(&redirect_url);
+ }
+
+ let mut source_code = Vec::new();
+ source_file.read_to_end(&mut source_code)?;
+
+ let cache_filename = self.http_cache.get_cache_filename(module_url);
+ let fake_filepath = PathBuf::from(module_url.path());
+ let media_type = map_content_type(
+ &fake_filepath,
+ headers.get("content-type").map(|e| e.as_str()),
+ );
let types_url = match media_type {
msg::MediaType::JavaScript | msg::MediaType::JSX => get_types_url(
&module_url,
&source_code,
- source_code_headers.x_typescript_types.as_deref(),
+ headers.get("x-typescript-types").map(|e| e.as_str()),
),
_ => None,
};
Ok(Some(SourceFile {
url: module_url.clone(),
- filename: filepath,
+ filename: cache_filename,
media_type,
source_code,
types_url,
@@ -440,8 +407,10 @@ impl SourceFileFetcher {
);
let dir = self.clone();
let module_url = module_url.clone();
- let headers = self.get_source_code_headers(&module_url);
- let module_etag = headers.etag;
+ let module_etag = match self.http_cache.get(&module_url) {
+ Ok((_, headers)) => headers.get("etag").map(String::from),
+ Err(_) => None,
+ };
let http_client = self.http_client.clone();
// Single pass fetch, either yields code or yields redirect.
let f = async move {
@@ -453,20 +422,9 @@ impl SourceFileFetcher {
Ok(source_file)
}
- FetchOnceResult::Redirect(new_module_url) => {
+ FetchOnceResult::Redirect(new_module_url, headers) => {
// If redirects, update module_name and filename for next looped call.
- if let Err(e) = dir.save_source_code_headers(
- &module_url,
- None,
- Some(new_module_url.to_string()),
- None,
- None,
- ) {
- return Err(source_header_cache_failed_error(
- module_url.as_str(),
- &e.to_string(),
- ));
- }
+ dir.http_cache.set(&module_url, headers, &[])?;
// Recurse
dir
@@ -478,51 +436,30 @@ impl SourceFileFetcher {
)
.await
}
- FetchOnceResult::Code(ResultPayload {
- body: source,
- content_type: maybe_content_type,
- etag,
- x_typescript_types,
- }) => {
+ FetchOnceResult::Code(source, headers) => {
// We land on the code.
- if let Err(e) = dir.save_source_code_headers(
- &module_url,
- maybe_content_type.clone(),
- None,
- etag,
- x_typescript_types.clone(),
- ) {
- return Err(source_header_cache_failed_error(
- module_url.as_str(),
- &e.to_string(),
- ));
- }
-
- if let Err(e) = dir.save_source_code(&module_url, &source) {
- return Err(source_cache_failed_error(
- module_url.as_str(),
- &e.to_string(),
- ));
- }
+ dir.http_cache.set(&module_url, headers.clone(), &source)?;
- let filepath = dir
- .deps_cache
- .location
- .join(dir.deps_cache.get_cache_filename(&module_url));
-
- let media_type =
- map_content_type(&filepath, maybe_content_type.as_deref());
+ let cache_filepath = dir.http_cache.get_cache_filename(&module_url);
+ // Used to sniff out content type from file extension - probably to be removed
+ let fake_filepath = PathBuf::from(module_url.path());
+ let media_type = map_content_type(
+ &fake_filepath,
+ headers.get("content-type").map(String::as_str),
+ );
let types_url = match media_type {
- msg::MediaType::JavaScript | msg::MediaType::JSX => {
- get_types_url(&module_url, &source, x_typescript_types.as_deref())
- }
+ msg::MediaType::JavaScript | msg::MediaType::JSX => get_types_url(
+ &module_url,
+ &source,
+ headers.get("x-typescript-types").map(String::as_str),
+ ),
_ => None,
};
let source_file = SourceFile {
url: module_url.clone(),
- filename: filepath,
+ filename: cache_filepath,
media_type,
source_code: source,
types_url,
@@ -535,75 +472,6 @@ impl SourceFileFetcher {
f.boxed()
}
-
- /// Get header metadata associated with a remote file.
- ///
- /// NOTE: chances are that the source file was downloaded due to redirects.
- /// In this case, the headers file provides info about where we should go and get
- /// the file that redirect eventually points to.
- fn get_source_code_headers(&self, url: &Url) -> SourceCodeHeaders {
- let cache_key = self
- .deps_cache
- .get_cache_filename_with_extension(url, "headers.json");
-
- if let Ok(bytes) = self.deps_cache.get(&cache_key) {
- if let Ok(json_string) = std::str::from_utf8(&bytes) {
- return SourceCodeHeaders::from_json_string(json_string.to_string());
- }
- }
-
- SourceCodeHeaders::default()
- }
-
- /// Save contents of downloaded remote file in on-disk cache for subsequent access.
- fn save_source_code(&self, url: &Url, source: &[u8]) -> std::io::Result<()> {
- let cache_key = self.deps_cache.get_cache_filename(url);
-
- // May not exist. DON'T unwrap.
- let _ = self.deps_cache.remove(&cache_key);
-
- self.deps_cache.set(&cache_key, source)
- }
-
- /// Save headers related to source file to {filename}.headers.json file,
- /// only when there is actually something necessary to save.
- ///
- /// For example, if the extension ".js" already mean JS file and we have
- /// content type of "text/javascript", then we would not save the mime type.
- ///
- /// If nothing needs to be saved, the headers file is not created.
- fn save_source_code_headers(
- &self,
- url: &Url,
- mime_type: Option<String>,
- redirect_to: Option<String>,
- etag: Option<String>,
- x_typescript_types: Option<String>,
- ) -> std::io::Result<()> {
- let cache_key = self
- .deps_cache
- .get_cache_filename_with_extension(url, "headers.json");
-
- // Remove possibly existing stale .headers.json file.
- // May not exist. DON'T unwrap.
- let _ = self.deps_cache.remove(&cache_key);
-
- let headers = SourceCodeHeaders {
- mime_type,
- redirect_to,
- etag,
- x_typescript_types,
- };
-
- let cache_filename = self.deps_cache.get_cache_filename(url);
- if let Ok(maybe_json_string) = headers.to_json_string(&cache_filename) {
- if let Some(json_string) = maybe_json_string {
- return self.deps_cache.set(&cache_key, json_string.as_bytes());
- }
- }
-
- Ok(())
- }
}
fn map_file_extension(path: &Path) -> msg::MediaType {
@@ -755,92 +623,14 @@ pub struct SourceCodeHeaders {
pub x_typescript_types: Option<String>,
}
-static MIME_TYPE: &str = "mime_type";
-static REDIRECT_TO: &str = "redirect_to";
-static ETAG: &str = "etag";
-static X_TYPESCRIPT_TYPES: &str = "x_typescript_types";
-
-impl SourceCodeHeaders {
- pub fn from_json_string(headers_string: String) -> Self {
- // TODO: use serde for deserialization
- let maybe_headers_json: serde_json::Result<serde_json::Value> =
- serde_json::from_str(&headers_string);
-
- if let Ok(headers_json) = maybe_headers_json {
- let mime_type = headers_json[MIME_TYPE].as_str().map(String::from);
- let redirect_to = headers_json[REDIRECT_TO].as_str().map(String::from);
- let etag = headers_json[ETAG].as_str().map(String::from);
- let x_typescript_types =
- headers_json[X_TYPESCRIPT_TYPES].as_str().map(String::from);
-
- return SourceCodeHeaders {
- mime_type,
- redirect_to,
- etag,
- x_typescript_types,
- };
- }
-
- SourceCodeHeaders::default()
- }
-
- // TODO: remove this nonsense `cache_filename` param, this should be
- // done when instantiating SourceCodeHeaders
- pub fn to_json_string(
- &self,
- cache_filename: &Path,
- ) -> Result<Option<String>, serde_json::Error> {
- // TODO(kevinkassimo): consider introduce serde::Deserialize to make things simpler.
- // This is super ugly at this moment...
- // Had trouble to make serde_derive work: I'm unable to build proc-macro2.
- let mut value_map = serde_json::map::Map::new();
-
- if let Some(mime_type) = &self.mime_type {
- let resolved_mime_type =
- map_content_type(Path::new(""), Some(mime_type.clone().as_str()));
-
- // TODO: fix this
- let ext_based_mime_type = map_file_extension(cache_filename);
-
- // Add mime to headers only when content type is different from extension.
- if ext_based_mime_type == msg::MediaType::Unknown
- || resolved_mime_type != ext_based_mime_type
- {
- value_map.insert(MIME_TYPE.to_string(), json!(mime_type));
- }
- }
-
- if let Some(redirect_to) = &self.redirect_to {
- value_map.insert(REDIRECT_TO.to_string(), json!(redirect_to));
- }
-
- if let Some(etag) = &self.etag {
- value_map.insert(ETAG.to_string(), json!(etag));
- }
-
- if let Some(x_typescript_types) = &self.x_typescript_types {
- value_map
- .insert(X_TYPESCRIPT_TYPES.to_string(), json!(x_typescript_types));
- }
-
- if value_map.is_empty() {
- return Ok(None);
- }
-
- serde_json::to_string(&value_map)
- .and_then(|serialized| Ok(Some(serialized)))
- }
-}
-
#[cfg(test)]
mod tests {
use super::*;
- use crate::fs as deno_fs;
use tempfile::TempDir;
fn setup_file_fetcher(dir_path: &Path) -> SourceFileFetcher {
SourceFileFetcher::new(
- DiskCache::new(&dir_path.to_path_buf().join("deps")),
+ HttpCache::new(&dir_path.to_path_buf().join("deps")).unwrap(),
true,
vec![],
false,
@@ -926,49 +716,6 @@ mod tests {
}
#[test]
- fn test_source_code_headers_get_and_save() {
- let (_temp_dir, fetcher) = test_setup();
- let url = Url::parse("http://example.com/f.js").unwrap();
- let headers_filepath = fetcher.deps_cache.location.join(
- fetcher
- .deps_cache
- .get_cache_filename_with_extension(&url, "headers.json"),
- );
-
- if let Some(ref parent) = headers_filepath.parent() {
- fs::create_dir_all(parent).unwrap();
- };
-
- let _ = deno_fs::write_file(
- headers_filepath.as_path(),
- "{\"mime_type\":\"text/javascript\",\"redirect_to\":\"http://example.com/a.js\"}",
- 0o666,
- );
- let headers = fetcher.get_source_code_headers(&url);
-
- assert_eq!(headers.mime_type.clone().unwrap(), "text/javascript");
- assert_eq!(headers.redirect_to.unwrap(), "http://example.com/a.js");
- assert_eq!(headers.etag, None);
- assert_eq!(headers.x_typescript_types, None);
-
- let _ = fetcher.save_source_code_headers(
- &url,
- Some("text/typescript".to_owned()),
- Some("http://deno.land/a.js".to_owned()),
- Some("W/\"04572f4749af993f4961a7e5daa1e4d5\"".to_owned()),
- Some("./a.d.ts".to_owned()),
- );
- let headers2 = fetcher.get_source_code_headers(&url);
- assert_eq!(headers2.mime_type.clone().unwrap(), "text/typescript");
- assert_eq!(headers2.redirect_to.unwrap(), "http://deno.land/a.js");
- assert_eq!(
- headers2.etag.unwrap(),
- "W/\"04572f4749af993f4961a7e5daa1e4d5\""
- );
- assert_eq!(headers2.x_typescript_types.unwrap(), "./a.d.ts")
- }
-
- #[test]
fn test_fetch_local_file_no_panic() {
let (_temp_dir, fetcher) = test_setup();
if cfg!(windows) {
@@ -992,14 +739,12 @@ mod tests {
Url::parse("http://localhost:4545/cli/tests/subdir/mod2.ts").unwrap();
let module_url_1 = module_url.clone();
let module_url_2 = module_url.clone();
- let headers_file_name = fetcher.deps_cache.location.join(
- fetcher
- .deps_cache
- .get_cache_filename_with_extension(&module_url, "headers.json"),
- );
+ let headers_file_name = fetcher
+ .http_cache
+ .get_cache_filename(&module_url)
+ .with_extension("headers.json");
let headers_file_name_1 = headers_file_name.clone();
let headers_file_name_2 = headers_file_name.clone();
- let headers_file_name_3 = headers_file_name;
let result = fetcher
.get_source_file_async(&module_url, true, false, false)
@@ -1011,13 +756,12 @@ mod tests {
&b"export { printHello } from \"./print_hello.ts\";\n"[..]
);
assert_eq!(&(r.media_type), &msg::MediaType::TypeScript);
- // Should not create .headers.json file due to matching ext
- assert!(fs::read_to_string(&headers_file_name_1).is_err());
+ assert!(fs::read_to_string(&headers_file_name_1).is_ok());
- // Modify .headers.json, write using fs write and read using save_source_code_headers
+ // Modify .headers.json, write using fs write
let _ = fs::write(
&headers_file_name_1,
- "{ \"mime_type\": \"text/javascript\" }",
+ "{ \"content-type\": \"text/javascript\" }",
);
let result2 = fetcher_1
.get_source_file_async(&module_url, true, false, false)
@@ -1031,21 +775,14 @@ mod tests {
// If get_source_file_async does not call remote, this should be JavaScript
// as we modified before! (we do not overwrite .headers.json due to no http fetch)
assert_eq!(&(r2.media_type), &msg::MediaType::JavaScript);
- assert_eq!(
- fetcher_2
- .get_source_code_headers(&module_url_1)
- .mime_type
- .unwrap(),
- "text/javascript"
- );
+ let (_, headers) = fetcher_2.http_cache.get(&module_url_1).unwrap();
+
+ assert_eq!(headers.get("content-type").unwrap(), "text/javascript");
// Modify .headers.json again, but the other way around
- let _ = fetcher_2.save_source_code_headers(
- &module_url_1,
- Some("application/json".to_owned()),
- None,
- None,
- None,
+ let _ = fs::write(
+ &headers_file_name_1,
+ "{ \"content-type\": \"application/json\" }",
);
let result3 = fetcher_2
.get_source_file_async(&module_url_1, true, false, false)
@@ -1073,9 +810,8 @@ mod tests {
let r4 = result4.unwrap();
let expected4 = &b"export { printHello } from \"./print_hello.ts\";\n"[..];
assert_eq!(r4.source_code, expected4);
- // Now the old .headers.json file should have gone! Resolved back to TypeScript
+ // Resolved back to TypeScript
assert_eq!(&(r4.media_type), &msg::MediaType::TypeScript);
- assert!(fs::read_to_string(&headers_file_name_3).is_err());
drop(http_server_guard);
}
@@ -1084,17 +820,14 @@ mod tests {
async fn test_get_source_code_2() {
let http_server_guard = crate::test_util::http_server();
let (temp_dir, fetcher) = test_setup();
- let fetcher_1 = fetcher.clone();
let module_url =
Url::parse("http://localhost:4545/cli/tests/subdir/mismatch_ext.ts")
.unwrap();
let module_url_1 = module_url.clone();
- let module_url_2 = module_url.clone();
- let headers_file_name = fetcher.deps_cache.location.join(
- fetcher
- .deps_cache
- .get_cache_filename_with_extension(&module_url, "headers.json"),
- );
+ let headers_file_name = fetcher
+ .http_cache
+ .get_cache_filename(&module_url)
+ .with_extension("headers.json");
let result = fetcher
.get_source_file_async(&module_url, true, false, false)
@@ -1103,23 +836,14 @@ mod tests {
let r = result.unwrap();
let expected = b"export const loaded = true;\n";
assert_eq!(r.source_code, expected);
- // Mismatch ext with content type, create .headers.json
assert_eq!(&(r.media_type), &msg::MediaType::JavaScript);
- assert_eq!(
- fetcher
- .get_source_code_headers(&module_url)
- .mime_type
- .unwrap(),
- "text/javascript"
- );
+ let (_, headers) = fetcher.http_cache.get(&module_url).unwrap();
+ assert_eq!(headers.get("content-type").unwrap(), "text/javascript");
// Modify .headers.json
- let _ = fetcher.save_source_code_headers(
- &module_url,
- Some("text/typescript".to_owned()),
- None,
- None,
- None,
+ let _ = fs::write(
+ &headers_file_name,
+ "{ \"content-type\": \"text/typescript\" }",
);
let result2 = fetcher
.get_source_file_async(&module_url, true, false, false)
@@ -1132,9 +856,12 @@ mod tests {
// as we modified before! (we do not overwrite .headers.json due to no http
// fetch)
assert_eq!(&(r2.media_type), &msg::MediaType::TypeScript);
- assert!(fs::read_to_string(&headers_file_name).is_err());
+ assert_eq!(
+ fs::read_to_string(&headers_file_name).unwrap(),
+ "{ \"content-type\": \"text/typescript\" }",
+ );
- // let's create fresh instance of DenoDir (simulating another freshh Deno
+ // let's create fresh instance of DenoDir (simulating another fresh Deno
// process) and don't use cache
let fetcher = setup_file_fetcher(temp_dir.path());
let result3 = fetcher
@@ -1147,13 +874,8 @@ mod tests {
// Now the old .headers.json file should be overwritten back to JavaScript!
// (due to http fetch)
assert_eq!(&(r3.media_type), &msg::MediaType::JavaScript);
- assert_eq!(
- fetcher_1
- .get_source_code_headers(&module_url_2)
- .mime_type
- .unwrap(),
- "text/javascript"
- );
+ let (_, headers) = fetcher.http_cache.get(&module_url).unwrap();
+ assert_eq!(headers.get("content-type").unwrap(), "text/javascript");
drop(http_server_guard);
}
@@ -1166,11 +888,10 @@ mod tests {
"http://localhost:4545/cli/tests/subdir/mismatch_ext.ts",
)
.unwrap();
- let headers_file_name = fetcher.deps_cache.location.join(
- fetcher
- .deps_cache
- .get_cache_filename_with_extension(specifier.as_url(), "headers.json"),
- );
+ let headers_file_name = fetcher
+ .http_cache
+ .get_cache_filename(specifier.as_url())
+ .with_extension("headers.json");
// first download
let r = fetcher.fetch_source_file_async(&specifier, None).await;
@@ -1210,20 +931,16 @@ mod tests {
"http://localhost:4546/cli/tests/subdir/redirects/redirect1.js",
)
.unwrap();
- let redirect_source_filepath = fetcher
- .deps_cache
- .location
- .join("http/localhost_PORT4546/cli/tests/subdir/redirects/redirect1.js");
+ let redirect_source_filepath =
+ fetcher.http_cache.get_cache_filename(&redirect_module_url);
let redirect_source_filename =
redirect_source_filepath.to_str().unwrap().to_string();
let target_module_url = Url::parse(
"http://localhost:4545/cli/tests/subdir/redirects/redirect1.js",
)
.unwrap();
- let redirect_target_filepath = fetcher
- .deps_cache
- .location
- .join("http/localhost_PORT4545/cli/tests/subdir/redirects/redirect1.js");
+ let redirect_target_filepath =
+ fetcher.http_cache.get_cache_filename(&target_module_url);
let redirect_target_filename =
redirect_target_filepath.to_str().unwrap().to_string();
@@ -1233,13 +950,11 @@ mod tests {
.await;
assert!(result.is_ok());
let mod_meta = result.unwrap();
- // File that requires redirection is not downloaded.
- assert!(fs::read_to_string(&redirect_source_filename).is_err());
- // ... but its .headers.json is created.
- let redirect_source_headers =
- fetcher.get_source_code_headers(&redirect_module_url);
+ // File that requires redirection should be empty file.
+ assert_eq!(fs::read_to_string(&redirect_source_filename).unwrap(), "");
+ let (_, headers) = fetcher.http_cache.get(&redirect_module_url).unwrap();
assert_eq!(
- redirect_source_headers.redirect_to.unwrap(),
+ headers.get("location").unwrap(),
"http://localhost:4545/cli/tests/subdir/redirects/redirect1.js"
);
// The target of redirection is downloaded instead.
@@ -1247,10 +962,8 @@ mod tests {
fs::read_to_string(&redirect_target_filename).unwrap(),
"export const redirect = 1;\n"
);
- let redirect_target_headers =
- fetcher.get_source_code_headers(&target_module_url);
- assert!(redirect_target_headers.redirect_to.is_none());
-
+ let (_, headers) = fetcher.http_cache.get(&target_module_url).unwrap();
+ assert!(headers.get("location").is_none());
// Examine the meta result.
assert_eq!(mod_meta.url, target_module_url);
@@ -1265,28 +978,20 @@ mod tests {
"http://localhost:4548/cli/tests/subdir/redirects/redirect1.js",
)
.unwrap();
- let double_redirect_path = fetcher
- .deps_cache
- .location
- .join("http/localhost_PORT4548/cli/tests/subdir/redirects/redirect1.js");
+ let double_redirect_path =
+ fetcher.http_cache.get_cache_filename(&double_redirect_url);
let redirect_url = Url::parse(
"http://localhost:4546/cli/tests/subdir/redirects/redirect1.js",
)
.unwrap();
- let redirect_path = fetcher
- .deps_cache
- .location
- .join("http/localhost_PORT4546/cli/tests/subdir/redirects/redirect1.js");
+ let redirect_path = fetcher.http_cache.get_cache_filename(&redirect_url);
let target_url = Url::parse(
"http://localhost:4545/cli/tests/subdir/redirects/redirect1.js",
)
.unwrap();
- let target_path = fetcher
- .deps_cache
- .location
- .join("http/localhost_PORT4545/cli/tests/subdir/redirects/redirect1.js");
+ let target_path = fetcher.http_cache.get_cache_filename(&target_url);
// Test double redirects and headers recording
let result = fetcher
@@ -1294,28 +999,22 @@ mod tests {
.await;
assert!(result.is_ok());
let mod_meta = result.unwrap();
- assert!(fs::read_to_string(&double_redirect_path).is_err());
- assert!(fs::read_to_string(&redirect_path).is_err());
+ assert_eq!(fs::read_to_string(&double_redirect_path).unwrap(), "");
+ assert_eq!(fs::read_to_string(&redirect_path).unwrap(), "");
- let double_redirect_headers =
- fetcher.get_source_code_headers(&double_redirect_url);
- assert_eq!(
- double_redirect_headers.redirect_to.unwrap(),
- redirect_url.to_string()
- );
- let redirect_headers = fetcher.get_source_code_headers(&redirect_url);
- assert_eq!(
- redirect_headers.redirect_to.unwrap(),
- target_url.to_string()
- );
+ let (_, headers) = fetcher.http_cache.get(&double_redirect_url).unwrap();
+ assert_eq!(headers.get("location").unwrap(), &redirect_url.to_string());
+
+ let (_, headers) = fetcher.http_cache.get(&redirect_url).unwrap();
+ assert_eq!(headers.get("location").unwrap(), &target_url.to_string());
// The target of redirection is downloaded instead.
assert_eq!(
fs::read_to_string(&target_path).unwrap(),
"export const redirect = 1;\n"
);
- let redirect_target_headers = fetcher.get_source_code_headers(&target_url);
- assert!(redirect_target_headers.redirect_to.is_none());
+ let (_, headers) = fetcher.http_cache.get(&target_url).unwrap();
+ assert!(headers.get("location").is_none());
// Examine the meta result.
assert_eq!(mod_meta.url, target_url);
@@ -1338,10 +1037,7 @@ mod tests {
)
.unwrap();
- let target_path = fetcher
- .deps_cache
- .location
- .join("http/localhost_PORT4545/cli/tests/subdir/redirects/redirect1.js");
+ let target_path = fetcher.http_cache.get_cache_filename(&redirect_url);
let target_path_ = target_path.clone();
// Test that redirect target is not downloaded twice for different redirect source.
@@ -1459,12 +1155,10 @@ mod tests {
let module_url =
Url::parse("http://127.0.0.1:4545/cli/tests/subdir/mt_video_mp2t.t3.ts")
.unwrap();
- let headers_file_name = fetcher.deps_cache.location.join(
- fetcher
- .deps_cache
- .get_cache_filename_with_extension(&module_url, "headers.json"),
- );
-
+ let headers_file_name = fetcher
+ .http_cache
+ .get_cache_filename(&module_url)
+ .with_extension("headers.json");
let result = fetcher
.fetch_remote_source_async(&module_url, false, false, 10)
.await;
@@ -1472,15 +1166,10 @@ mod tests {
let r = result.unwrap();
assert_eq!(r.source_code, b"export const loaded = true;\n");
assert_eq!(&(r.media_type), &msg::MediaType::TypeScript);
- // matching ext, no .headers.json file created
- assert!(fs::read_to_string(&headers_file_name).is_err());
// Modify .headers.json, make sure read from local
- let _ = fetcher.save_source_code_headers(
- &module_url,
- Some("text/javascript".to_owned()),
- None,
- None,
- None,
+ let _ = fs::write(
+ &headers_file_name,
+ "{ \"content-type\": \"text/javascript\" }",
);
let result2 = fetcher.fetch_cached_remote_source(&module_url);
assert!(result2.is_ok());
@@ -1500,11 +1189,6 @@ mod tests {
let module_url =
Url::parse("http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts")
.unwrap();
- let headers_file_name = fetcher.deps_cache.location.join(
- fetcher
- .deps_cache
- .get_cache_filename_with_extension(&module_url, "headers.json"),
- );
let result = fetcher
.fetch_remote_source_async(&module_url, false, false, 10)
@@ -1513,16 +1197,14 @@ mod tests {
let r = result.unwrap();
assert_eq!(r.source_code, b"export const loaded = true;\n");
assert_eq!(&(r.media_type), &msg::MediaType::TypeScript);
- // matching ext, no .headers.json file created
- assert!(fs::read_to_string(&headers_file_name).is_err());
-
// Modify .headers.json, make sure read from local
- let _ = fetcher.save_source_code_headers(
- &module_url,
- Some("text/javascript".to_owned()),
- None,
- None,
- None,
+ let headers_file_name = fetcher
+ .http_cache
+ .get_cache_filename(&module_url)
+ .with_extension("headers.json");
+ let _ = fs::write(
+ &headers_file_name,
+ "{ \"content-type\": \"text/javascript\" }",
);
let result2 = fetcher.fetch_cached_remote_source(&module_url);
assert!(result2.is_ok());
@@ -1540,7 +1222,6 @@ mod tests {
let (_temp_dir, fetcher) = test_setup();
let fetcher_1 = fetcher.clone();
let fetcher_2 = fetcher.clone();
- let fetcher_3 = fetcher.clone();
let module_url =
Url::parse("http://localhost:4545/cli/tests/subdir/no_ext").unwrap();
let module_url_2 =
@@ -1559,14 +1240,8 @@ mod tests {
let r = result.unwrap();
assert_eq!(r.source_code, b"export const loaded = true;\n");
assert_eq!(&(r.media_type), &msg::MediaType::TypeScript);
- // no ext, should create .headers.json file
- assert_eq!(
- fetcher_1
- .get_source_code_headers(&module_url)
- .mime_type
- .unwrap(),
- "text/typescript"
- );
+ let (_, headers) = fetcher.http_cache.get(&module_url).unwrap();
+ assert_eq!(headers.get("content-type").unwrap(), "text/typescript");
let result = fetcher_1
.fetch_remote_source_async(&module_url_2, false, false, 10)
.await;
@@ -1574,14 +1249,9 @@ mod tests {
let r2 = result.unwrap();
assert_eq!(r2.source_code, b"export const loaded = true;\n");
assert_eq!(&(r2.media_type), &msg::MediaType::JavaScript);
- // mismatch ext, should create .headers.json file
- assert_eq!(
- fetcher_2
- .get_source_code_headers(&module_url_2_)
- .mime_type
- .unwrap(),
- "text/javascript"
- );
+ let (_, headers) = fetcher.http_cache.get(&module_url_2_).unwrap();
+ assert_eq!(headers.get("content-type").unwrap(), "text/javascript");
+
// test unknown extension
let result = fetcher_2
.fetch_remote_source_async(&module_url_3, false, false, 10)
@@ -1590,14 +1260,8 @@ mod tests {
let r3 = result.unwrap();
assert_eq!(r3.source_code, b"export const loaded = true;\n");
assert_eq!(&(r3.media_type), &msg::MediaType::TypeScript);
- // unknown ext, should create .headers.json file
- assert_eq!(
- fetcher_3
- .get_source_code_headers(&module_url_3_)
- .mime_type
- .unwrap(),
- "text/typescript"
- );
+ let (_, headers) = fetcher.http_cache.get(&module_url_3_).unwrap();
+ assert_eq!(headers.get("content-type").unwrap(), "text/typescript");
drop(http_server_guard);
}
@@ -1904,23 +1568,21 @@ mod tests {
assert_eq!(source.source_code, b"console.log('etag')");
assert_eq!(&(source.media_type), &msg::MediaType::TypeScript);
- let headers = fetcher.get_source_code_headers(&module_url);
- assert_eq!(headers.etag, Some("33a64df551425fcc55e".to_string()));
+ let (_, headers) = fetcher.http_cache.get(&module_url).unwrap();
+ assert_eq!(headers.get("etag").unwrap(), "33a64df551425fcc55e");
- let header_path = fetcher.deps_cache.location.join(
- fetcher
- .deps_cache
- .get_cache_filename_with_extension(&module_url, "headers.json"),
- );
+ let header_path = fetcher
+ .http_cache
+ .get_cache_filename(&module_url)
+ .with_extension("headers.json");
let modified1 = header_path.metadata().unwrap().modified().unwrap();
// Forcibly change the contents of the cache file and request
// it again with the cache parameters turned off.
// If the fetched content changes, the cached content is used.
- fetcher
- .save_source_code(&module_url, b"changed content")
- .unwrap();
+ let file_name = fetcher.http_cache.get_cache_filename(&module_url);
+ let _ = fs::write(&file_name, "changed content");
let cached_source = fetcher
.fetch_remote_source_async(&module_url, false, false, 1)
.await
generated by cgit v1.2.3 (git 2.25.1) at 2025-06-27 07:59:56 +0000