summaryrefslogtreecommitdiff
path: root/cli/cache
diff options
context:
space:
mode:
Diffstat (limited to 'cli/cache')
-rw-r--r--cli/cache/cache_db.rs6
-rw-r--r--cli/cache/code_cache.rs10
-rw-r--r--cli/cache/emit.rs27
-rw-r--r--cli/cache/mod.rs53
-rw-r--r--cli/cache/module_info.rs130
-rw-r--r--cli/cache/parsed_source.rs20
6 files changed, 159 insertions, 87 deletions
diff --git a/cli/cache/cache_db.rs b/cli/cache/cache_db.rs
index b24078f29..329ed2d97 100644
--- a/cli/cache/cache_db.rs
+++ b/cli/cache/cache_db.rs
@@ -57,7 +57,7 @@ impl rusqlite::types::FromSql for CacheDBHash {
}
/// What should the cache should do on failure?
-#[derive(Default)]
+#[derive(Debug, Default)]
pub enum CacheFailure {
/// Return errors if failure mode otherwise unspecified.
#[default]
@@ -69,6 +69,7 @@ pub enum CacheFailure {
}
/// Configuration SQL and other parameters for a [`CacheDB`].
+#[derive(Debug)]
pub struct CacheDBConfiguration {
/// SQL to run for a new database.
pub table_initializer: &'static str,
@@ -98,6 +99,7 @@ impl CacheDBConfiguration {
}
}
+#[derive(Debug)]
enum ConnectionState {
Connected(Connection),
Blackhole,
@@ -106,7 +108,7 @@ enum ConnectionState {
/// A cache database that eagerly initializes itself off-thread, preventing initialization operations
/// from blocking the main thread.
-#[derive(Clone)]
+#[derive(Debug, Clone)]
pub struct CacheDB {
// TODO(mmastrac): We can probably simplify our thread-safe implementation here
conn: Arc<Mutex<OnceCell<ConnectionState>>>,
diff --git a/cli/cache/code_cache.rs b/cli/cache/code_cache.rs
index abcd0d46a..b1d9ae757 100644
--- a/cli/cache/code_cache.rs
+++ b/cli/cache/code_cache.rs
@@ -1,10 +1,14 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
+use std::sync::Arc;
+
use deno_ast::ModuleSpecifier;
use deno_core::error::AnyError;
use deno_runtime::code_cache;
use deno_runtime::deno_webstorage::rusqlite::params;
+use crate::worker::CliCodeCache;
+
use super::cache_db::CacheDB;
use super::cache_db::CacheDBConfiguration;
use super::cache_db::CacheDBHash;
@@ -82,6 +86,12 @@ impl CodeCache {
}
}
+impl CliCodeCache for CodeCache {
+ fn as_code_cache(self: Arc<Self>) -> Arc<dyn code_cache::CodeCache> {
+ self
+ }
+}
+
impl code_cache::CodeCache for CodeCache {
fn get_sync(
&self,
diff --git a/cli/cache/emit.rs b/cli/cache/emit.rs
index 6807f06c1..3c9eecfcb 100644
--- a/cli/cache/emit.rs
+++ b/cli/cache/emit.rs
@@ -10,6 +10,7 @@ use deno_core::unsync::sync::AtomicFlag;
use super::DiskCache;
/// The cache that stores previously emitted files.
+#[derive(Debug)]
pub struct EmitCache {
disk_cache: DiskCache,
emit_failed_flag: AtomicFlag,
@@ -39,7 +40,7 @@ impl EmitCache {
&self,
specifier: &ModuleSpecifier,
expected_source_hash: u64,
- ) -> Option<Vec<u8>> {
+ ) -> Option<String> {
let emit_filename = self.get_emit_filename(specifier)?;
let bytes = self.disk_cache.get(&emit_filename).ok()?;
self
@@ -91,6 +92,7 @@ impl EmitCache {
const LAST_LINE_PREFIX: &str = "\n// denoCacheMetadata=";
+#[derive(Debug)]
struct EmitFileSerializer {
cli_version: &'static str,
}
@@ -100,7 +102,7 @@ impl EmitFileSerializer {
&self,
mut bytes: Vec<u8>,
expected_source_hash: u64,
- ) -> Option<Vec<u8>> {
+ ) -> Option<String> {
let last_newline_index = bytes.iter().rposition(|&b| b == b'\n')?;
let (content, last_line) = bytes.split_at(last_newline_index);
let hashes = last_line.strip_prefix(LAST_LINE_PREFIX.as_bytes())?;
@@ -120,7 +122,7 @@ impl EmitFileSerializer {
// everything looks good, truncate and return it
bytes.truncate(content.len());
- Some(bytes)
+ String::from_utf8(bytes).ok()
}
pub fn serialize(&self, code: &[u8], source_hash: u64) -> Vec<u8> {
@@ -170,8 +172,6 @@ mod test {
},
emit_failed_flag: Default::default(),
};
- let to_string =
- |bytes: Vec<u8>| -> String { String::from_utf8(bytes).unwrap() };
let specifier1 =
ModuleSpecifier::from_file_path(temp_dir.path().join("file1.ts"))
@@ -188,13 +188,10 @@ mod test {
assert_eq!(cache.get_emit_code(&specifier1, 5), None);
// providing the correct source hash
assert_eq!(
- cache.get_emit_code(&specifier1, 10).map(to_string),
+ cache.get_emit_code(&specifier1, 10),
Some(emit_code1.clone()),
);
- assert_eq!(
- cache.get_emit_code(&specifier2, 2).map(to_string),
- Some(emit_code2)
- );
+ assert_eq!(cache.get_emit_code(&specifier2, 2), Some(emit_code2));
// try changing the cli version (should not load previous ones)
let cache = EmitCache {
@@ -215,18 +212,12 @@ mod test {
},
emit_failed_flag: Default::default(),
};
- assert_eq!(
- cache.get_emit_code(&specifier1, 5).map(to_string),
- Some(emit_code1)
- );
+ assert_eq!(cache.get_emit_code(&specifier1, 5), Some(emit_code1));
// adding when already exists should not cause issue
let emit_code3 = "asdf".to_string();
cache.set_emit_code(&specifier1, 20, emit_code3.as_bytes());
assert_eq!(cache.get_emit_code(&specifier1, 5), None);
- assert_eq!(
- cache.get_emit_code(&specifier1, 20).map(to_string),
- Some(emit_code3)
- );
+ assert_eq!(cache.get_emit_code(&specifier1, 20), Some(emit_code3));
}
}
diff --git a/cli/cache/mod.rs b/cli/cache/mod.rs
index 628502c50..50fc135dd 100644
--- a/cli/cache/mod.rs
+++ b/cli/cache/mod.rs
@@ -8,11 +8,9 @@ use crate::file_fetcher::FetchOptions;
use crate::file_fetcher::FetchPermissionsOptionRef;
use crate::file_fetcher::FileFetcher;
use crate::file_fetcher::FileOrRedirect;
-use crate::npm::CliNpmResolver;
use crate::util::fs::atomic_write_file_with_retries;
use crate::util::fs::atomic_write_file_with_retries_and_fs;
use crate::util::fs::AtomicWriteFileFsAdapter;
-use crate::util::path::specifier_has_extension;
use deno_ast::MediaType;
use deno_core::futures;
@@ -22,7 +20,9 @@ use deno_graph::source::CacheInfo;
use deno_graph::source::LoadFuture;
use deno_graph::source::LoadResponse;
use deno_graph::source::Loader;
+use deno_runtime::deno_fs;
use deno_runtime::deno_permissions::PermissionsContainer;
+use node_resolver::InNpmPackageChecker;
use std::collections::HashMap;
use std::path::Path;
use std::path::PathBuf;
@@ -182,28 +182,31 @@ pub struct FetchCacherOptions {
/// A "wrapper" for the FileFetcher and DiskCache for the Deno CLI that provides
/// a concise interface to the DENO_DIR when building module graphs.
pub struct FetchCacher {
- file_fetcher: Arc<FileFetcher>,
pub file_header_overrides: HashMap<ModuleSpecifier, HashMap<String, String>>,
+ file_fetcher: Arc<FileFetcher>,
+ fs: Arc<dyn deno_fs::FileSystem>,
global_http_cache: Arc<GlobalHttpCache>,
- npm_resolver: Arc<dyn CliNpmResolver>,
+ in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
module_info_cache: Arc<ModuleInfoCache>,
permissions: PermissionsContainer,
- cache_info_enabled: bool,
is_deno_publish: bool,
+ cache_info_enabled: bool,
}
impl FetchCacher {
pub fn new(
file_fetcher: Arc<FileFetcher>,
+ fs: Arc<dyn deno_fs::FileSystem>,
global_http_cache: Arc<GlobalHttpCache>,
- npm_resolver: Arc<dyn CliNpmResolver>,
+ in_npm_pkg_checker: Arc<dyn InNpmPackageChecker>,
module_info_cache: Arc<ModuleInfoCache>,
options: FetchCacherOptions,
) -> Self {
Self {
file_fetcher,
+ fs,
global_http_cache,
- npm_resolver,
+ in_npm_pkg_checker,
module_info_cache,
file_header_overrides: options.file_header_overrides,
permissions: options.permissions,
@@ -258,28 +261,21 @@ impl Loader for FetchCacher {
) -> LoadFuture {
use deno_graph::source::CacheSetting as LoaderCacheSetting;
- if specifier.scheme() == "file" {
- if specifier.path().contains("/node_modules/") {
- // The specifier might be in a completely different symlinked tree than
- // what the node_modules url is in (ex. `/my-project-1/node_modules`
- // symlinked to `/my-project-2/node_modules`), so first we checked if the path
- // is in a node_modules dir to avoid needlessly canonicalizing, then now compare
- // against the canonicalized specifier.
- let specifier =
- crate::node::resolve_specifier_into_node_modules(specifier);
- if self.npm_resolver.in_npm_package(&specifier) {
- return Box::pin(futures::future::ready(Ok(Some(
- LoadResponse::External { specifier },
- ))));
- }
- }
-
- // make local CJS modules external to the graph
- if specifier_has_extension(specifier, "cjs") {
+ if specifier.scheme() == "file"
+ && specifier.path().contains("/node_modules/")
+ {
+ // The specifier might be in a completely different symlinked tree than
+ // what the node_modules url is in (ex. `/my-project-1/node_modules`
+ // symlinked to `/my-project-2/node_modules`), so first we checked if the path
+ // is in a node_modules dir to avoid needlessly canonicalizing, then now compare
+ // against the canonicalized specifier.
+ let specifier = crate::node::resolve_specifier_into_node_modules(
+ specifier,
+ self.fs.as_ref(),
+ );
+ if self.in_npm_pkg_checker.in_npm_package(&specifier) {
return Box::pin(futures::future::ready(Ok(Some(
- LoadResponse::External {
- specifier: specifier.clone(),
- },
+ LoadResponse::External { specifier },
))));
}
}
@@ -325,6 +321,7 @@ impl Loader for FetchCacher {
} else {
FetchPermissionsOptionRef::DynamicContainer(&permissions)
},
+ maybe_auth: None,
maybe_accept: None,
maybe_cache_setting: maybe_cache_setting.as_ref(),
},
diff --git a/cli/cache/module_info.rs b/cli/cache/module_info.rs
index 4dbb01c37..060a6f4f0 100644
--- a/cli/cache/module_info.rs
+++ b/cli/cache/module_info.rs
@@ -44,18 +44,32 @@ pub static MODULE_INFO_CACHE_DB: CacheDBConfiguration = CacheDBConfiguration {
/// A cache of `deno_graph::ModuleInfo` objects. Using this leads to a considerable
/// performance improvement because when it exists we can skip parsing a module for
/// deno_graph.
+#[derive(Debug)]
pub struct ModuleInfoCache {
conn: CacheDB,
+ parsed_source_cache: Arc<ParsedSourceCache>,
}
impl ModuleInfoCache {
#[cfg(test)]
- pub fn new_in_memory(version: &'static str) -> Self {
- Self::new(CacheDB::in_memory(&MODULE_INFO_CACHE_DB, version))
+ pub fn new_in_memory(
+ version: &'static str,
+ parsed_source_cache: Arc<ParsedSourceCache>,
+ ) -> Self {
+ Self::new(
+ CacheDB::in_memory(&MODULE_INFO_CACHE_DB, version),
+ parsed_source_cache,
+ )
}
- pub fn new(conn: CacheDB) -> Self {
- Self { conn }
+ pub fn new(
+ conn: CacheDB,
+ parsed_source_cache: Arc<ParsedSourceCache>,
+ ) -> Self {
+ Self {
+ conn,
+ parsed_source_cache,
+ }
}
/// Useful for testing: re-create this cache DB with a different current version.
@@ -63,6 +77,7 @@ impl ModuleInfoCache {
pub(crate) fn recreate_with_version(self, version: &'static str) -> Self {
Self {
conn: self.conn.recreate_with_version(version),
+ parsed_source_cache: self.parsed_source_cache,
}
}
@@ -113,13 +128,10 @@ impl ModuleInfoCache {
Ok(())
}
- pub fn as_module_analyzer<'a>(
- &'a self,
- parsed_source_cache: &'a Arc<ParsedSourceCache>,
- ) -> ModuleInfoCacheModuleAnalyzer<'a> {
+ pub fn as_module_analyzer(&self) -> ModuleInfoCacheModuleAnalyzer {
ModuleInfoCacheModuleAnalyzer {
module_info_cache: self,
- parsed_source_cache,
+ parsed_source_cache: &self.parsed_source_cache,
}
}
}
@@ -129,31 +141,99 @@ pub struct ModuleInfoCacheModuleAnalyzer<'a> {
parsed_source_cache: &'a Arc<ParsedSourceCache>,
}
-#[async_trait::async_trait(?Send)]
-impl<'a> deno_graph::ModuleAnalyzer for ModuleInfoCacheModuleAnalyzer<'a> {
- async fn analyze(
+impl<'a> ModuleInfoCacheModuleAnalyzer<'a> {
+ fn load_cached_module_info(
&self,
specifier: &ModuleSpecifier,
- source: Arc<str>,
media_type: MediaType,
- ) -> Result<ModuleInfo, deno_ast::ParseDiagnostic> {
- // attempt to load from the cache
- let source_hash = CacheDBHash::from_source(&source);
+ source_hash: CacheDBHash,
+ ) -> Option<ModuleInfo> {
match self.module_info_cache.get_module_info(
specifier,
media_type,
source_hash,
) {
- Ok(Some(info)) => return Ok(info),
- Ok(None) => {}
+ Ok(Some(info)) => Some(info),
+ Ok(None) => None,
Err(err) => {
log::debug!(
"Error loading module cache info for {}. {:#}",
specifier,
err
);
+ None
}
}
+ }
+
+ fn save_module_info_to_cache(
+ &self,
+ specifier: &ModuleSpecifier,
+ media_type: MediaType,
+ source_hash: CacheDBHash,
+ module_info: &ModuleInfo,
+ ) {
+ if let Err(err) = self.module_info_cache.set_module_info(
+ specifier,
+ media_type,
+ source_hash,
+ module_info,
+ ) {
+ log::debug!(
+ "Error saving module cache info for {}. {:#}",
+ specifier,
+ err
+ );
+ }
+ }
+
+ pub fn analyze_sync(
+ &self,
+ specifier: &ModuleSpecifier,
+ media_type: MediaType,
+ source: &Arc<str>,
+ ) -> Result<ModuleInfo, deno_ast::ParseDiagnostic> {
+ // attempt to load from the cache
+ let source_hash = CacheDBHash::from_source(source);
+ if let Some(info) =
+ self.load_cached_module_info(specifier, media_type, source_hash)
+ {
+ return Ok(info);
+ }
+
+ // otherwise, get the module info from the parsed source cache
+ let parser = self.parsed_source_cache.as_capturing_parser();
+ let analyzer = ParserModuleAnalyzer::new(&parser);
+ let module_info =
+ analyzer.analyze_sync(specifier, source.clone(), media_type)?;
+
+ // then attempt to cache it
+ self.save_module_info_to_cache(
+ specifier,
+ media_type,
+ source_hash,
+ &module_info,
+ );
+
+ Ok(module_info)
+ }
+}
+
+#[async_trait::async_trait(?Send)]
+impl<'a> deno_graph::ModuleAnalyzer for ModuleInfoCacheModuleAnalyzer<'a> {
+ async fn analyze(
+ &self,
+ specifier: &ModuleSpecifier,
+ source: Arc<str>,
+ media_type: MediaType,
+ ) -> Result<ModuleInfo, deno_ast::ParseDiagnostic> {
+ // attempt to load from the cache
+ let source_hash = CacheDBHash::from_source(&source);
+ if let Some(info) =
+ self.load_cached_module_info(specifier, media_type, source_hash)
+ {
+ return Ok(info);
+ }
// otherwise, get the module info from the parsed source cache
let module_info = deno_core::unsync::spawn_blocking({
@@ -169,18 +249,12 @@ impl<'a> deno_graph::ModuleAnalyzer for ModuleInfoCacheModuleAnalyzer<'a> {
.unwrap()?;
// then attempt to cache it
- if let Err(err) = self.module_info_cache.set_module_info(
+ self.save_module_info_to_cache(
specifier,
media_type,
source_hash,
&module_info,
- ) {
- log::debug!(
- "Error saving module cache info for {}. {:#}",
- specifier,
- err
- );
- }
+ );
Ok(module_info)
}
@@ -202,7 +276,7 @@ fn serialize_media_type(media_type: MediaType) -> i64 {
Tsx => 11,
Json => 12,
Wasm => 13,
- TsBuildInfo => 14,
+ Css => 14,
SourceMap => 15,
Unknown => 16,
}
@@ -217,7 +291,7 @@ mod test {
#[test]
pub fn module_info_cache_general_use() {
- let cache = ModuleInfoCache::new_in_memory("1.0.0");
+ let cache = ModuleInfoCache::new_in_memory("1.0.0", Default::default());
let specifier1 =
ModuleSpecifier::parse("https://localhost/mod.ts").unwrap();
let specifier2 =
diff --git a/cli/cache/parsed_source.rs b/cli/cache/parsed_source.rs
index e956361f4..7e819ae99 100644
--- a/cli/cache/parsed_source.rs
+++ b/cli/cache/parsed_source.rs
@@ -7,9 +7,9 @@ use deno_ast::MediaType;
use deno_ast::ModuleSpecifier;
use deno_ast::ParsedSource;
use deno_core::parking_lot::Mutex;
-use deno_graph::CapturingModuleParser;
-use deno_graph::DefaultModuleParser;
-use deno_graph::ModuleParser;
+use deno_graph::CapturingEsParser;
+use deno_graph::DefaultEsParser;
+use deno_graph::EsParser;
use deno_graph::ParseOptions;
use deno_graph::ParsedSourceStore;
@@ -46,7 +46,7 @@ impl<'a> LazyGraphSourceParser<'a> {
}
}
-#[derive(Default)]
+#[derive(Debug, Default)]
pub struct ParsedSourceCache {
sources: Mutex<HashMap<ModuleSpecifier, ParsedSource>>,
}
@@ -57,12 +57,11 @@ impl ParsedSourceCache {
module: &deno_graph::JsModule,
) -> Result<ParsedSource, deno_ast::ParseDiagnostic> {
let parser = self.as_capturing_parser();
- // this will conditionally parse because it's using a CapturingModuleParser
- parser.parse_module(ParseOptions {
+ // this will conditionally parse because it's using a CapturingEsParser
+ parser.parse_program(ParseOptions {
specifier: &module.specifier,
source: module.source.clone(),
media_type: module.media_type,
- // don't bother enabling because this method is currently only used for vendoring
scope_analysis: false,
})
}
@@ -86,10 +85,9 @@ impl ParsedSourceCache {
specifier,
source,
media_type,
- // don't bother enabling because this method is currently only used for emitting
scope_analysis: false,
};
- DefaultModuleParser.parse_module(options)
+ DefaultEsParser.parse_program(options)
}
/// Frees the parsed source from memory.
@@ -99,8 +97,8 @@ impl ParsedSourceCache {
/// Creates a parser that will reuse a ParsedSource from the store
/// if it exists, or else parse.
- pub fn as_capturing_parser(&self) -> CapturingModuleParser {
- CapturingModuleParser::new(None, self)
+ pub fn as_capturing_parser(&self) -> CapturingEsParser {
+ CapturingEsParser::new(None, self)
}
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-06-26 19:12:54 +0000