summaryrefslogtreecommitdiff
path: root/cli/cache
diff options
context:
space:
mode:
Diffstat (limited to 'cli/cache')
-rw-r--r--cli/cache/caches.rs4
-rw-r--r--cli/cache/mod.rs11
-rw-r--r--cli/cache/module_info.rs291
-rw-r--r--cli/cache/parsed_source.rs333
4 files changed, 321 insertions, 318 deletions
diff --git a/cli/cache/caches.rs b/cli/cache/caches.rs
index f630a82ff..b91c81a15 100644
--- a/cli/cache/caches.rs
+++ b/cli/cache/caches.rs
@@ -10,8 +10,8 @@ use super::cache_db::CacheDBConfiguration;
use super::check::TYPE_CHECK_CACHE_DB;
use super::deno_dir::DenoDirProvider;
use super::incremental::INCREMENTAL_CACHE_DB;
+use super::module_info::MODULE_INFO_CACHE_DB;
use super::node::NODE_ANALYSIS_CACHE_DB;
-use super::parsed_source::PARSED_SOURCE_CACHE_DB;
pub struct Caches {
dir_provider: Arc<DenoDirProvider>,
@@ -77,7 +77,7 @@ impl Caches {
pub fn dep_analysis_db(&self) -> CacheDB {
Self::make_db(
&self.dep_analysis_db,
- &PARSED_SOURCE_CACHE_DB,
+ &MODULE_INFO_CACHE_DB,
self
.dir_provider
.get_or_create()
diff --git a/cli/cache/mod.rs b/cli/cache/mod.rs
index 5cc91f50f..526236ace 100644
--- a/cli/cache/mod.rs
+++ b/cli/cache/mod.rs
@@ -32,6 +32,7 @@ mod deno_dir;
mod disk_cache;
mod emit;
mod incremental;
+mod module_info;
mod node;
mod parsed_source;
@@ -43,6 +44,8 @@ pub use deno_dir::DenoDirProvider;
pub use disk_cache::DiskCache;
pub use emit::EmitCache;
pub use incremental::IncrementalCache;
+pub use module_info::ModuleInfoCache;
+pub use module_info::ModuleInfoCacheModuleAnalyzer;
pub use node::NodeAnalysisCache;
pub use parsed_source::ParsedSourceCache;
@@ -103,7 +106,7 @@ pub struct FetchCacher {
file_header_overrides: HashMap<ModuleSpecifier, HashMap<String, String>>,
global_http_cache: Arc<GlobalHttpCache>,
npm_resolver: Arc<dyn CliNpmResolver>,
- parsed_source_cache: Arc<ParsedSourceCache>,
+ module_info_cache: Arc<ModuleInfoCache>,
permissions: PermissionsContainer,
cache_info_enabled: bool,
}
@@ -115,7 +118,7 @@ impl FetchCacher {
file_header_overrides: HashMap<ModuleSpecifier, HashMap<String, String>>,
global_http_cache: Arc<GlobalHttpCache>,
npm_resolver: Arc<dyn CliNpmResolver>,
- parsed_source_cache: Arc<ParsedSourceCache>,
+ module_info_cache: Arc<ModuleInfoCache>,
permissions: PermissionsContainer,
) -> Self {
Self {
@@ -124,7 +127,7 @@ impl FetchCacher {
file_header_overrides,
global_http_cache,
npm_resolver,
- parsed_source_cache,
+ module_info_cache,
permissions,
cache_info_enabled: false,
}
@@ -297,7 +300,7 @@ impl Loader for FetchCacher {
source: &str,
module_info: &deno_graph::ModuleInfo,
) {
- let result = self.parsed_source_cache.cache_module_info(
+ let result = self.module_info_cache.set_module_info(
specifier,
MediaType::from_specifier(specifier),
source,
diff --git a/cli/cache/module_info.rs b/cli/cache/module_info.rs
new file mode 100644
index 000000000..afdb8349c
--- /dev/null
+++ b/cli/cache/module_info.rs
@@ -0,0 +1,291 @@
+// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
+
+use std::sync::Arc;
+
+use deno_ast::MediaType;
+use deno_ast::ModuleSpecifier;
+use deno_core::error::AnyError;
+use deno_core::serde_json;
+use deno_graph::CapturingModuleParser;
+use deno_graph::DefaultModuleAnalyzer;
+use deno_graph::ModuleInfo;
+use deno_graph::ModuleParser;
+use deno_graph::ParsedSourceStore;
+use deno_runtime::deno_webstorage::rusqlite::params;
+
+use super::cache_db::CacheDB;
+use super::cache_db::CacheDBConfiguration;
+use super::cache_db::CacheFailure;
+use super::FastInsecureHasher;
+
+const SELECT_MODULE_INFO: &str = "
+SELECT
+ module_info
+FROM
+ moduleinfocache
+WHERE
+ specifier=?1
+ AND media_type=?2
+ AND source_hash=?3
+LIMIT 1";
+
+pub static MODULE_INFO_CACHE_DB: CacheDBConfiguration = CacheDBConfiguration {
+ table_initializer: "CREATE TABLE IF NOT EXISTS moduleinfocache (
+ specifier TEXT PRIMARY KEY,
+ media_type TEXT NOT NULL,
+ source_hash TEXT NOT NULL,
+ module_info TEXT NOT NULL
+ );",
+ on_version_change: "DELETE FROM moduleinfocache;",
+ preheat_queries: &[SELECT_MODULE_INFO],
+ on_failure: CacheFailure::InMemory,
+};
+
+/// A cache of `deno_graph::ModuleInfo` objects. Using this leads to a considerable
+/// performance improvement because when it exists we can skip parsing a module for
+/// deno_graph.
+pub struct ModuleInfoCache {
+ conn: CacheDB,
+}
+
+impl ModuleInfoCache {
+ #[cfg(test)]
+ pub fn new_in_memory(version: &'static str) -> Self {
+ Self::new(CacheDB::in_memory(&MODULE_INFO_CACHE_DB, version))
+ }
+
+ pub fn new(conn: CacheDB) -> Self {
+ Self { conn }
+ }
+
+ /// Useful for testing: re-create this cache DB with a different current version.
+ #[cfg(test)]
+ pub(crate) fn recreate_with_version(self, version: &'static str) -> Self {
+ Self {
+ conn: self.conn.recreate_with_version(version),
+ }
+ }
+
+ pub fn get_module_info(
+ &self,
+ specifier: &ModuleSpecifier,
+ media_type: MediaType,
+ expected_source_hash: &str,
+ ) -> Result<Option<ModuleInfo>, AnyError> {
+ let query = SELECT_MODULE_INFO;
+ let res = self.conn.query_row(
+ query,
+ params![
+ &specifier.as_str(),
+ serialize_media_type(media_type),
+ &expected_source_hash,
+ ],
+ |row| {
+ let module_info: String = row.get(0)?;
+ let module_info = serde_json::from_str(&module_info)?;
+ Ok(module_info)
+ },
+ )?;
+ Ok(res)
+ }
+
+ pub fn set_module_info(
+ &self,
+ specifier: &ModuleSpecifier,
+ media_type: MediaType,
+ source_hash: &str,
+ module_info: &ModuleInfo,
+ ) -> Result<(), AnyError> {
+ let sql = "
+ INSERT OR REPLACE INTO
+ moduleinfocache (specifier, media_type, source_hash, module_info)
+ VALUES
+ (?1, ?2, ?3, ?4)";
+ self.conn.execute(
+ sql,
+ params![
+ specifier.as_str(),
+ serialize_media_type(media_type),
+ &source_hash,
+ &serde_json::to_string(&module_info)?,
+ ],
+ )?;
+ Ok(())
+ }
+
+ pub fn as_module_analyzer<'a>(
+ &'a self,
+ parser: Option<&'a dyn ModuleParser>,
+ store: &'a dyn ParsedSourceStore,
+ ) -> ModuleInfoCacheModuleAnalyzer<'a> {
+ ModuleInfoCacheModuleAnalyzer {
+ module_info_cache: self,
+ parser: CapturingModuleParser::new(parser, store),
+ }
+ }
+}
+
+pub struct ModuleInfoCacheModuleAnalyzer<'a> {
+ module_info_cache: &'a ModuleInfoCache,
+ parser: CapturingModuleParser<'a>,
+}
+
+impl<'a> deno_graph::ModuleAnalyzer for ModuleInfoCacheModuleAnalyzer<'a> {
+ fn analyze(
+ &self,
+ specifier: &ModuleSpecifier,
+ source: Arc<str>,
+ media_type: MediaType,
+ ) -> Result<ModuleInfo, deno_ast::Diagnostic> {
+ // attempt to load from the cache
+ let source_hash = FastInsecureHasher::hash(source.as_bytes()).to_string();
+ match self.module_info_cache.get_module_info(
+ specifier,
+ media_type,
+ &source_hash,
+ ) {
+ Ok(Some(info)) => return Ok(info),
+ Ok(None) => {}
+ Err(err) => {
+ log::debug!(
+ "Error loading module cache info for {}. {:#}",
+ specifier,
+ err
+ );
+ }
+ }
+
+ // otherwise, get the module info from the parsed source cache
+ let analyzer = DefaultModuleAnalyzer::new(&self.parser);
+ let module_info = analyzer.analyze(specifier, source, media_type)?;
+
+ // then attempt to cache it
+ if let Err(err) = self.module_info_cache.set_module_info(
+ specifier,
+ media_type,
+ &source_hash,
+ &module_info,
+ ) {
+ log::debug!(
+ "Error saving module cache info for {}. {:#}",
+ specifier,
+ err
+ );
+ }
+
+ Ok(module_info)
+ }
+}
+
+// todo(dsherret): change this to be stored as an integer next time
+// the cache version is bumped
+fn serialize_media_type(media_type: MediaType) -> &'static str {
+ use MediaType::*;
+ match media_type {
+ JavaScript => "1",
+ Jsx => "2",
+ Mjs => "3",
+ Cjs => "4",
+ TypeScript => "5",
+ Mts => "6",
+ Cts => "7",
+ Dts => "8",
+ Dmts => "9",
+ Dcts => "10",
+ Tsx => "11",
+ Json => "12",
+ Wasm => "13",
+ TsBuildInfo => "14",
+ SourceMap => "15",
+ Unknown => "16",
+ }
+}
+
+#[cfg(test)]
+mod test {
+ use deno_graph::PositionRange;
+ use deno_graph::SpecifierWithRange;
+
+ use super::*;
+
+ #[test]
+ pub fn module_info_cache_general_use() {
+ let cache = ModuleInfoCache::new_in_memory("1.0.0");
+ let specifier1 =
+ ModuleSpecifier::parse("https://localhost/mod.ts").unwrap();
+ let specifier2 =
+ ModuleSpecifier::parse("https://localhost/mod2.ts").unwrap();
+ assert_eq!(
+ cache
+ .get_module_info(&specifier1, MediaType::JavaScript, "1")
+ .unwrap(),
+ None
+ );
+
+ let mut module_info = ModuleInfo::default();
+ module_info.jsdoc_imports.push(SpecifierWithRange {
+ range: PositionRange {
+ start: deno_graph::Position {
+ line: 0,
+ character: 3,
+ },
+ end: deno_graph::Position {
+ line: 1,
+ character: 2,
+ },
+ },
+ text: "test".to_string(),
+ });
+ cache
+ .set_module_info(&specifier1, MediaType::JavaScript, "1", &module_info)
+ .unwrap();
+ assert_eq!(
+ cache
+ .get_module_info(&specifier1, MediaType::JavaScript, "1")
+ .unwrap(),
+ Some(module_info.clone())
+ );
+ assert_eq!(
+ cache
+ .get_module_info(&specifier2, MediaType::JavaScript, "1")
+ .unwrap(),
+ None,
+ );
+ // different media type
+ assert_eq!(
+ cache
+ .get_module_info(&specifier1, MediaType::TypeScript, "1")
+ .unwrap(),
+ None,
+ );
+ // different source hash
+ assert_eq!(
+ cache
+ .get_module_info(&specifier1, MediaType::JavaScript, "2")
+ .unwrap(),
+ None,
+ );
+
+ // try recreating with the same version
+ let cache = cache.recreate_with_version("1.0.0");
+
+ // should get it
+ assert_eq!(
+ cache
+ .get_module_info(&specifier1, MediaType::JavaScript, "1")
+ .unwrap(),
+ Some(module_info)
+ );
+
+ // try recreating with a different version
+ let cache = cache.recreate_with_version("1.0.1");
+
+ // should no longer exist
+ assert_eq!(
+ cache
+ .get_module_info(&specifier1, MediaType::JavaScript, "1")
+ .unwrap(),
+ None,
+ );
+ }
+}
diff --git a/cli/cache/parsed_source.rs b/cli/cache/parsed_source.rs
index 68503e6aa..8ca3d80dd 100644
--- a/cli/cache/parsed_source.rs
+++ b/cli/cache/parsed_source.rs
@@ -6,94 +6,16 @@ use std::sync::Arc;
use deno_ast::MediaType;
use deno_ast::ModuleSpecifier;
use deno_ast::ParsedSource;
-use deno_core::error::AnyError;
use deno_core::parking_lot::Mutex;
-use deno_core::serde_json;
use deno_graph::CapturingModuleParser;
-use deno_graph::DefaultModuleAnalyzer;
-use deno_graph::ModuleInfo;
use deno_graph::ModuleParser;
-use deno_runtime::deno_webstorage::rusqlite::params;
-use super::cache_db::CacheDB;
-use super::cache_db::CacheDBConfiguration;
-use super::cache_db::CacheFailure;
-use super::FastInsecureHasher;
-
-const SELECT_MODULE_INFO: &str = "
-SELECT
- module_info
-FROM
- moduleinfocache
-WHERE
- specifier=?1
- AND media_type=?2
- AND source_hash=?3
-LIMIT 1";
-
-pub static PARSED_SOURCE_CACHE_DB: CacheDBConfiguration =
- CacheDBConfiguration {
- table_initializer: "CREATE TABLE IF NOT EXISTS moduleinfocache (
- specifier TEXT PRIMARY KEY,
- media_type TEXT NOT NULL,
- source_hash TEXT NOT NULL,
- module_info TEXT NOT NULL
- );",
- on_version_change: "DELETE FROM moduleinfocache;",
- preheat_queries: &[SELECT_MODULE_INFO],
- on_failure: CacheFailure::InMemory,
- };
-
-#[derive(Clone, Default)]
-struct ParsedSourceCacheSources(
- Arc<Mutex<HashMap<ModuleSpecifier, ParsedSource>>>,
-);
-
-/// It's ok that this is racy since in non-LSP situations
-/// this will only ever store one form of a parsed source
-/// and in LSP settings the concurrency will be enforced
-/// at a higher level to ensure this will have the latest
-/// parsed source.
-impl deno_graph::ParsedSourceStore for ParsedSourceCacheSources {
- fn set_parsed_source(
- &self,
- specifier: deno_graph::ModuleSpecifier,
- parsed_source: ParsedSource,
- ) -> Option<ParsedSource> {
- self.0.lock().insert(specifier, parsed_source)
- }
-
- fn get_parsed_source(
- &self,
- specifier: &deno_graph::ModuleSpecifier,
- ) -> Option<ParsedSource> {
- self.0.lock().get(specifier).cloned()
- }
-}
-
-/// A cache of `ParsedSource`s, which may be used with `deno_graph`
-/// for cached dependency analysis.
+#[derive(Default)]
pub struct ParsedSourceCache {
- db: CacheDB,
- sources: ParsedSourceCacheSources,
+ sources: Mutex<HashMap<ModuleSpecifier, ParsedSource>>,
}
impl ParsedSourceCache {
- #[cfg(test)]
- pub fn new_in_memory() -> Self {
- Self {
- db: CacheDB::in_memory(&PARSED_SOURCE_CACHE_DB, crate::version::deno()),
- sources: Default::default(),
- }
- }
-
- pub fn new(db: CacheDB) -> Self {
- Self {
- db,
- sources: Default::default(),
- }
- }
-
pub fn get_parsed_source_from_esm_module(
&self,
module: &deno_graph::EsmModule,
@@ -120,251 +42,38 @@ impl ParsedSourceCache {
/// Frees the parsed source from memory.
pub fn free(&self, specifier: &ModuleSpecifier) {
- self.sources.0.lock().remove(specifier);
- }
-
- pub fn as_analyzer(&self) -> Box<dyn deno_graph::ModuleAnalyzer> {
- Box::new(ParsedSourceCacheModuleAnalyzer::new(
- self.db.clone(),
- self.sources.clone(),
- ))
+ self.sources.lock().remove(specifier);
}
/// Creates a parser that will reuse a ParsedSource from the store
/// if it exists, or else parse.
pub fn as_capturing_parser(&self) -> CapturingModuleParser {
- CapturingModuleParser::new(None, &self.sources)
+ CapturingModuleParser::new(None, self)
}
- pub fn cache_module_info(
- &self,
- specifier: &ModuleSpecifier,
- media_type: MediaType,
- source: &str,
- module_info: &ModuleInfo,
- ) -> Result<(), AnyError> {
- let source_hash = compute_source_hash(source.as_bytes());
- ParsedSourceCacheModuleAnalyzer::new(self.db.clone(), self.sources.clone())
- .set_module_info(specifier, media_type, &source_hash, module_info)
+ pub fn as_store(self: &Arc<Self>) -> Arc<dyn deno_graph::ParsedSourceStore> {
+ self.clone()
}
}
-struct ParsedSourceCacheModuleAnalyzer {
- conn: CacheDB,
- sources: ParsedSourceCacheSources,
-}
-
-impl ParsedSourceCacheModuleAnalyzer {
- pub fn new(conn: CacheDB, sources: ParsedSourceCacheSources) -> Self {
- Self { conn, sources }
- }
-
- pub fn get_module_info(
- &self,
- specifier: &ModuleSpecifier,
- media_type: MediaType,
- expected_source_hash: &str,
- ) -> Result<Option<ModuleInfo>, AnyError> {
- let query = SELECT_MODULE_INFO;
- let res = self.conn.query_row(
- query,
- params![
- &specifier.as_str(),
- serialize_media_type(media_type),
- &expected_source_hash,
- ],
- |row| {
- let module_info: String = row.get(0)?;
- let module_info = serde_json::from_str(&module_info)?;
- Ok(module_info)
- },
- )?;
- Ok(res)
- }
-
- pub fn set_module_info(
+/// It's ok that this is racy since in non-LSP situations
+/// this will only ever store one form of a parsed source
+/// and in LSP settings the concurrency will be enforced
+/// at a higher level to ensure this will have the latest
+/// parsed source.
+impl deno_graph::ParsedSourceStore for ParsedSourceCache {
+ fn set_parsed_source(
&self,
- specifier: &ModuleSpecifier,
- media_type: MediaType,
- source_hash: &str,
- module_info: &ModuleInfo,
- ) -> Result<(), AnyError> {
- let sql = "
- INSERT OR REPLACE INTO
- moduleinfocache (specifier, media_type, source_hash, module_info)
- VALUES
- (?1, ?2, ?3, ?4)";
- self.conn.execute(
- sql,
- params![
- specifier.as_str(),
- serialize_media_type(media_type),
- &source_hash,
- &serde_json::to_string(&module_info)?,
- ],
- )?;
- Ok(())
- }
-}
-
-// todo(dsherret): change this to be stored as an integer next time
-// the cache version is bumped
-fn serialize_media_type(media_type: MediaType) -> &'static str {
- use MediaType::*;
- match media_type {
- JavaScript => "1",
- Jsx => "2",
- Mjs => "3",
- Cjs => "4",
- TypeScript => "5",
- Mts => "6",
- Cts => "7",
- Dts => "8",
- Dmts => "9",
- Dcts => "10",
- Tsx => "11",
- Json => "12",
- Wasm => "13",
- TsBuildInfo => "14",
- SourceMap => "15",
- Unknown => "16",
+ specifier: deno_graph::ModuleSpecifier,
+ parsed_source: ParsedSource,
+ ) -> Option<ParsedSource> {
+ self.sources.lock().insert(specifier, parsed_source)
}
-}
-impl deno_graph::ModuleAnalyzer for ParsedSourceCacheModuleAnalyzer {
- fn analyze(
+ fn get_parsed_source(
&self,
- specifier: &ModuleSpecifier,
- source: Arc<str>,
- media_type: MediaType,
- ) -> Result<ModuleInfo, deno_ast::Diagnostic> {
- // attempt to load from the cache
- let source_hash = compute_source_hash(source.as_bytes());
- match self.get_module_info(specifier, media_type, &source_hash) {
- Ok(Some(info)) => return Ok(info),
- Ok(None) => {}
- Err(err) => {
- log::debug!(
- "Error loading module cache info for {}. {:#}",
- specifier,
- err
- );
- }
- }
-
- // otherwise, get the module info from the parsed source cache
- let parser = CapturingModuleParser::new(None, &self.sources);
- let analyzer = DefaultModuleAnalyzer::new(&parser);
-
- let module_info = analyzer.analyze(specifier, source, media_type)?;
-
- // then attempt to cache it
- if let Err(err) =
- self.set_module_info(specifier, media_type, &source_hash, &module_info)
- {
- log::debug!(
- "Error saving module cache info for {}. {:#}",
- specifier,
- err
- );
- }
-
- Ok(module_info)
- }
-}
-
-fn compute_source_hash(bytes: &[u8]) -> String {
- FastInsecureHasher::hash(bytes).to_string()
-}
-
-#[cfg(test)]
-mod test {
- use deno_graph::PositionRange;
- use deno_graph::SpecifierWithRange;
-
- use super::*;
-
- #[test]
- pub fn parsed_source_cache_module_analyzer_general_use() {
- let conn = CacheDB::in_memory(&PARSED_SOURCE_CACHE_DB, "1.0.0");
- let cache = ParsedSourceCacheModuleAnalyzer::new(conn, Default::default());
- let specifier1 =
- ModuleSpecifier::parse("https://localhost/mod.ts").unwrap();
- let specifier2 =
- ModuleSpecifier::parse("https://localhost/mod2.ts").unwrap();
- assert_eq!(
- cache
- .get_module_info(&specifier1, MediaType::JavaScript, "1")
- .unwrap(),
- None
- );
-
- let mut module_info = ModuleInfo::default();
- module_info.jsdoc_imports.push(SpecifierWithRange {
- range: PositionRange {
- start: deno_graph::Position {
- line: 0,
- character: 3,
- },
- end: deno_graph::Position {
- line: 1,
- character: 2,
- },
- },
- text: "test".to_string(),
- });
- cache
- .set_module_info(&specifier1, MediaType::JavaScript, "1", &module_info)
- .unwrap();
- assert_eq!(
- cache
- .get_module_info(&specifier1, MediaType::JavaScript, "1")
- .unwrap(),
- Some(module_info.clone())
- );
- assert_eq!(
- cache
- .get_module_info(&specifier2, MediaType::JavaScript, "1")
- .unwrap(),
- None,
- );
- // different media type
- assert_eq!(
- cache
- .get_module_info(&specifier1, MediaType::TypeScript, "1")
- .unwrap(),
- None,
- );
- // different source hash
- assert_eq!(
- cache
- .get_module_info(&specifier1, MediaType::JavaScript, "2")
- .unwrap(),
- None,
- );
-
- // try recreating with the same version
- let conn = cache.conn.recreate_with_version("1.0.0");
- let cache = ParsedSourceCacheModuleAnalyzer::new(conn, Default::default());
-
- // should get it
- assert_eq!(
- cache
- .get_module_info(&specifier1, MediaType::JavaScript, "1")
- .unwrap(),
- Some(module_info)
- );
-
- // try recreating with a different version
- let conn = cache.conn.recreate_with_version("1.0.1");
- let cache = ParsedSourceCacheModuleAnalyzer::new(conn, Default::default());
-
- // should no longer exist
- assert_eq!(
- cache
- .get_module_info(&specifier1, MediaType::JavaScript, "1")
- .unwrap(),
- None,
- );
+ specifier: &deno_graph::ModuleSpecifier,
+ ) -> Option<ParsedSource> {
+ self.sources.lock().get(specifier).cloned()
}
}
generated by cgit v1.2.3 (git 2.25.1) at 2025-06-27 04:59:43 +0000