diff options
Diffstat (limited to 'cli/file_fetcher.rs')
| -rw-r--r-- | cli/file_fetcher.rs | 342 |
1 files changed, 141 insertions, 201 deletions
diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs index cebb22b20..864c59cb5 100644 --- a/cli/file_fetcher.rs +++ b/cli/file_fetcher.rs @@ -170,7 +170,7 @@ pub struct FileFetcher { allow_remote: bool, cache: FileCache, cache_setting: CacheSetting, - pub http_cache: HttpCache, + http_cache: Arc<dyn HttpCache>, http_client: Arc<HttpClient>, blob_store: Arc<BlobStore>, download_log_level: log::Level, @@ -179,7 +179,7 @@ pub struct FileFetcher { impl FileFetcher { pub fn new( - http_cache: HttpCache, + http_cache: Arc<dyn HttpCache>, cache_setting: CacheSetting, allow_remote: bool, http_client: Arc<HttpClient>, @@ -245,8 +245,8 @@ impl FileFetcher { return Err(custom_error("Http", "Too many redirects.")); } - let cache_item = self.http_cache.get(specifier)?; - let Some(metadata) = cache_item.read_metadata()? else { + let cache_key = self.http_cache.cache_item_key(specifier)?; // compute this once + let Some(metadata) = self.http_cache.read_metadata(&cache_key)? else { return Ok(None); }; let headers = metadata.headers; @@ -255,7 +255,7 @@ impl FileFetcher { deno_core::resolve_import(redirect_to, specifier.as_str())?; return self.fetch_cached(&redirect, redirect_limit - 1); } - let Some(bytes) = cache_item.read_to_bytes()? else { + let Some(bytes) = self.http_cache.read_file_bytes(&cache_key)? else { return Ok(None); }; let file = self.build_remote_file(specifier, bytes, &headers)?; @@ -375,9 +375,9 @@ impl FileFetcher { let maybe_etag = self .http_cache - .get(specifier) + .cache_item_key(specifier) .ok() - .and_then(|item| item.read_metadata().ok()?) + .and_then(|key| self.http_cache.read_metadata(&key).ok().flatten()) .and_then(|metadata| metadata.headers.get("etag").cloned()); let maybe_auth_token = self.auth_tokens.get(specifier); let specifier = specifier.clone(); @@ -473,10 +473,10 @@ impl FileFetcher { CacheSetting::ReloadAll => false, CacheSetting::Use | CacheSetting::Only => true, CacheSetting::RespectHeaders => { - let Ok(item) = self.http_cache.get(specifier) else { + let Ok(cache_key) = self.http_cache.cache_item_key(specifier) else { return false; }; - let Ok(Some(metadata)) = item.read_metadata() else { + let Ok(Some(metadata)) = self.http_cache.read_metadata(&cache_key) else { return false; }; let cache_semantics = CacheSemantics::new( @@ -556,33 +556,6 @@ impl FileFetcher { } } - // DEPRECATED: Where the file is stored and how it's stored should be an implementation - // detail of the cache. - // - // todo(dsheret): remove once implementing - // * https://github.com/denoland/deno/issues/17707 - // * https://github.com/denoland/deno/issues/17703 - #[deprecated( - note = "There should not be a way to do this because the file may not be cached at a local path in the future." - )] - pub fn get_local_path(&self, specifier: &ModuleSpecifier) -> Option<PathBuf> { - // TODO(@kitsonk) fix when deno_graph does not query cache for synthetic - // modules - if specifier.scheme() == "flags" { - None - } else if specifier.scheme() == "file" { - specifier.to_file_path().ok() - } else { - #[allow(deprecated)] - self.http_cache.get_cache_filepath(specifier).ok() - } - } - - /// Get the location of the current HTTP cache associated with the fetcher. - pub fn get_http_cache_location(&self) -> &PathBuf { - &self.http_cache.location - } - /// A synchronous way to retrieve a source file, where if the file has already /// been cached in memory it will be returned, otherwise for local files will /// be read from disk. @@ -722,6 +695,7 @@ async fn fetch_once<'a>( #[cfg(test)] mod tests { + use crate::cache::GlobalHttpCache; use crate::http_util::HttpClient; use crate::version; @@ -753,7 +727,7 @@ mod tests { let location = temp_dir.path().join("deps").to_path_buf(); let blob_store: Arc<BlobStore> = Default::default(); let file_fetcher = FileFetcher::new( - HttpCache::new(location), + Arc::new(GlobalHttpCache::new(location)), cache_setting, true, Arc::new(HttpClient::new(None, None)), @@ -790,10 +764,15 @@ mod tests { let result: Result<File, AnyError> = file_fetcher .fetch_remote(specifier, PermissionsContainer::allow_all(), 1, None) .await; - let cache_item = file_fetcher.http_cache.get(specifier).unwrap(); + let cache_key = file_fetcher.http_cache.cache_item_key(specifier).unwrap(); ( result.unwrap(), - cache_item.read_metadata().unwrap().unwrap().headers, + file_fetcher + .http_cache + .read_metadata(&cache_key) + .unwrap() + .unwrap() + .headers, ) } @@ -1063,14 +1042,6 @@ mod tests { ); } - #[test] - fn test_get_http_cache_location() { - let (file_fetcher, temp_dir) = setup(CacheSetting::Use, None); - let expected = temp_dir.path().join("deps").to_path_buf(); - let actual = file_fetcher.get_http_cache_location().to_path_buf(); - assert_eq!(actual, expected); - } - #[tokio::test] async fn test_fetch_data_url() { let (file_fetcher, _) = setup(CacheSetting::Use, None); @@ -1142,11 +1113,11 @@ mod tests { ); assert_eq!(file.media_type, MediaType::TypeScript); + let cache_item_key = + file_fetcher.http_cache.cache_item_key(&specifier).unwrap(); let mut metadata = file_fetcher .http_cache - .get(&specifier) - .unwrap() - .read_metadata() + .read_metadata(&cache_item_key) .unwrap() .unwrap(); metadata.headers = HashMap::new(); @@ -1155,7 +1126,7 @@ mod tests { .insert("content-type".to_string(), "text/javascript".to_string()); file_fetcher .http_cache - .write_metadata(&specifier, &metadata) + .set(&specifier, metadata.headers.clone(), file.source.as_bytes()) .unwrap(); let result = file_fetcher_01 @@ -1173,9 +1144,7 @@ mod tests { let headers = file_fetcher_02 .http_cache - .get(&specifier) - .unwrap() - .read_metadata() + .read_metadata(&cache_item_key) .unwrap() .unwrap() .headers; @@ -1186,7 +1155,7 @@ mod tests { .insert("content-type".to_string(), "application/json".to_string()); file_fetcher_02 .http_cache - .write_metadata(&specifier, &metadata) + .set(&specifier, metadata.headers.clone(), file.source.as_bytes()) .unwrap(); let result = file_fetcher_02 @@ -1204,7 +1173,7 @@ mod tests { // invocation and indicates to "cache bust". let location = temp_dir.path().join("deps").to_path_buf(); let file_fetcher = FileFetcher::new( - HttpCache::new(location), + Arc::new(GlobalHttpCache::new(location)), CacheSetting::ReloadAll, true, Arc::new(HttpClient::new(None, None)), @@ -1233,7 +1202,7 @@ mod tests { let file_modified_01 = { let file_fetcher = FileFetcher::new( - HttpCache::new(location.clone()), + Arc::new(GlobalHttpCache::new(location.clone())), CacheSetting::Use, true, Arc::new(HttpClient::new(None, None)), @@ -1245,16 +1214,16 @@ mod tests { .fetch(&specifier, PermissionsContainer::allow_all()) .await; assert!(result.is_ok()); + let cache_key = + file_fetcher.http_cache.cache_item_key(&specifier).unwrap(); ( file_fetcher .http_cache - .get_modified_time(&specifier) + .read_modified_time(&cache_key) .unwrap(), file_fetcher .http_cache - .get(&specifier) - .unwrap() - .read_metadata() + .read_metadata(&cache_key) .unwrap() .unwrap(), ) @@ -1262,7 +1231,7 @@ mod tests { let file_modified_02 = { let file_fetcher = FileFetcher::new( - HttpCache::new(location), + Arc::new(GlobalHttpCache::new(location)), CacheSetting::Use, true, Arc::new(HttpClient::new(None, None)), @@ -1274,16 +1243,16 @@ mod tests { .await; assert!(result.is_ok()); + let cache_key = + file_fetcher.http_cache.cache_item_key(&specifier).unwrap(); ( file_fetcher .http_cache - .get_modified_time(&specifier) + .read_modified_time(&cache_key) .unwrap(), file_fetcher .http_cache - .get(&specifier) - .unwrap() - .read_metadata() + .read_metadata(&cache_key) .unwrap() .unwrap(), ) @@ -1310,40 +1279,24 @@ mod tests { let file = result.unwrap(); assert_eq!(file.specifier, redirected_specifier); - { - let cache_item = file_fetcher.http_cache.get(&specifier).unwrap(); - assert_eq!( - cache_item.read_to_string().unwrap().unwrap(), - "", - "redirected files should have empty cached contents" - ); - assert_eq!( - cache_item - .read_metadata() - .unwrap() - .unwrap() - .headers - .get("location") - .unwrap(), - "http://localhost:4545/subdir/redirects/redirect1.js" - ); - } + assert_eq!( + get_text_from_cache(&file_fetcher, &specifier), + "", + "redirected files should have empty cached contents" + ); + assert_eq!( + get_location_header_from_cache(&file_fetcher, &specifier), + Some("http://localhost:4545/subdir/redirects/redirect1.js".to_string()), + ); - { - let cache_item = - file_fetcher.http_cache.get(&redirected_specifier).unwrap(); - assert_eq!( - cache_item.read_to_string().unwrap().unwrap(), - "export const redirect = 1;\n" - ); - assert!(cache_item - .read_metadata() - .unwrap() - .unwrap() - .headers - .get("location") - .is_none()); - } + assert_eq!( + get_text_from_cache(&file_fetcher, &redirected_specifier), + "export const redirect = 1;\n" + ); + assert_eq!( + get_location_header_from_cache(&file_fetcher, &redirected_specifier), + None, + ); } #[tokio::test] @@ -1367,64 +1320,34 @@ mod tests { let file = result.unwrap(); assert_eq!(file.specifier, redirected_02_specifier); - { - let cache_item = file_fetcher.http_cache.get(&specifier).unwrap(); - assert_eq!( - cache_item.read_to_string().unwrap().unwrap(), - "", - "redirected files should have empty cached contents" - ); - assert_eq!( - cache_item - .read_metadata() - .unwrap() - .unwrap() - .headers - .get("location") - .unwrap(), - "http://localhost:4546/subdir/redirects/redirect1.js" - ); - } + assert_eq!( + get_text_from_cache(&file_fetcher, &specifier), + "", + "redirected files should have empty cached contents" + ); + assert_eq!( + get_location_header_from_cache(&file_fetcher, &specifier), + Some("http://localhost:4546/subdir/redirects/redirect1.js".to_string()), + ); - { - let cache_item = file_fetcher - .http_cache - .get(&redirected_01_specifier) - .unwrap(); - assert_eq!( - cache_item.read_to_string().unwrap().unwrap(), - "", - "redirected files should have empty cached contents" - ); - assert_eq!( - cache_item - .read_metadata() - .unwrap() - .unwrap() - .headers - .get("location") - .unwrap(), - "http://localhost:4545/subdir/redirects/redirect1.js" - ); - } + assert_eq!( + get_text_from_cache(&file_fetcher, &redirected_01_specifier), + "", + "redirected files should have empty cached contents" + ); + assert_eq!( + get_location_header_from_cache(&file_fetcher, &redirected_01_specifier), + Some("http://localhost:4545/subdir/redirects/redirect1.js".to_string()), + ); - { - let cache_item = file_fetcher - .http_cache - .get(&redirected_02_specifier) - .unwrap(); - assert_eq!( - cache_item.read_to_string().unwrap().unwrap(), - "export const redirect = 1;\n" - ); - assert!(cache_item - .read_metadata() - .unwrap() - .unwrap() - .headers - .get("location") - .is_none()); - } + assert_eq!( + get_text_from_cache(&file_fetcher, &redirected_02_specifier), + "export const redirect = 1;\n" + ); + assert_eq!( + get_location_header_from_cache(&file_fetcher, &redirected_02_specifier), + None, + ); } #[tokio::test] @@ -1439,7 +1362,7 @@ mod tests { let metadata_file_modified_01 = { let file_fetcher = FileFetcher::new( - HttpCache::new(location.clone()), + Arc::new(GlobalHttpCache::new(location.clone())), CacheSetting::Use, true, Arc::new(HttpClient::new(None, None)), @@ -1452,16 +1375,18 @@ mod tests { .await; assert!(result.is_ok()); + let cache_key = file_fetcher + .http_cache + .cache_item_key(&redirected_specifier) + .unwrap(); ( file_fetcher .http_cache - .get_modified_time(&redirected_specifier) + .read_modified_time(&cache_key) .unwrap(), file_fetcher .http_cache - .get(&redirected_specifier) - .unwrap() - .read_metadata() + .read_metadata(&cache_key) .unwrap() .unwrap(), ) @@ -1469,7 +1394,7 @@ mod tests { let metadata_file_modified_02 = { let file_fetcher = FileFetcher::new( - HttpCache::new(location), + Arc::new(GlobalHttpCache::new(location)), CacheSetting::Use, true, Arc::new(HttpClient::new(None, None)), @@ -1481,16 +1406,18 @@ mod tests { .await; assert!(result.is_ok()); + let cache_key = file_fetcher + .http_cache + .cache_item_key(&redirected_specifier) + .unwrap(); ( file_fetcher .http_cache - .get_modified_time(&redirected_specifier) + .read_modified_time(&cache_key) .unwrap(), file_fetcher .http_cache - .get(&redirected_specifier) - .unwrap() - .read_metadata() + .read_metadata(&cache_key) .unwrap() .unwrap(), ) @@ -1543,40 +1470,24 @@ mod tests { let file = result.unwrap(); assert_eq!(file.specifier, redirected_specifier); - { - let cache_item = file_fetcher.http_cache.get(&specifier).unwrap(); - assert_eq!( - cache_item.read_to_string().unwrap().unwrap(), - "", - "redirected files should have empty cached contents" - ); - assert_eq!( - cache_item - .read_metadata() - .unwrap() - .unwrap() - .headers - .get("location") - .unwrap(), - "/subdir/redirects/redirect1.js" - ); - } + assert_eq!( + get_text_from_cache(&file_fetcher, &specifier), + "", + "redirected files should have empty cached contents" + ); + assert_eq!( + get_location_header_from_cache(&file_fetcher, &specifier), + Some("/subdir/redirects/redirect1.js".to_string()), + ); - { - let cache_item = - file_fetcher.http_cache.get(&redirected_specifier).unwrap(); - assert_eq!( - cache_item.read_to_string().unwrap().unwrap(), - "export const redirect = 1;\n" - ); - assert!(cache_item - .read_metadata() - .unwrap() - .unwrap() - .headers - .get("location") - .is_none()); - } + assert_eq!( + get_text_from_cache(&file_fetcher, &redirected_specifier), + "export const redirect = 1;\n" + ); + assert_eq!( + get_location_header_from_cache(&file_fetcher, &redirected_specifier), + None + ); } #[tokio::test] @@ -1585,7 +1496,7 @@ mod tests { let temp_dir = TempDir::new(); let location = temp_dir.path().join("deps").to_path_buf(); let file_fetcher = FileFetcher::new( - HttpCache::new(location), + Arc::new(GlobalHttpCache::new(location)), CacheSetting::Use, false, Arc::new(HttpClient::new(None, None)), @@ -1610,7 +1521,7 @@ mod tests { let temp_dir = TempDir::new(); let location = temp_dir.path().join("deps").to_path_buf(); let file_fetcher_01 = FileFetcher::new( - HttpCache::new(location.clone()), + Arc::new(GlobalHttpCache::new(location.clone())), CacheSetting::Only, true, Arc::new(HttpClient::new(None, None)), @@ -1618,7 +1529,7 @@ mod tests { None, ); let file_fetcher_02 = FileFetcher::new( - HttpCache::new(location), + Arc::new(GlobalHttpCache::new(location)), CacheSetting::Use, true, Arc::new(HttpClient::new(None, None)), @@ -2331,4 +2242,33 @@ mod tests { assert!(matches!(result, Ok(FetchOnceResult::RequestError(_)))); } + + #[track_caller] + fn get_text_from_cache( + file_fetcher: &FileFetcher, + url: &ModuleSpecifier, + ) -> String { + let cache_key = file_fetcher.http_cache.cache_item_key(url).unwrap(); + let bytes = file_fetcher + .http_cache + .read_file_bytes(&cache_key) + .unwrap() + .unwrap(); + String::from_utf8(bytes).unwrap() + } + + #[track_caller] + fn get_location_header_from_cache( + file_fetcher: &FileFetcher, + url: &ModuleSpecifier, + ) -> Option<String> { + let cache_key = file_fetcher.http_cache.cache_item_key(url).unwrap(); + file_fetcher + .http_cache + .read_metadata(&cache_key) + .unwrap() + .unwrap() + .headers + .remove("location") + } } |