From 1a0cb5b5312941521ab021cfe9eaed498f35900b Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 14 Oct 2024 20:48:39 -0400 Subject: feat(unstable): `--unstable-detect-cjs` for respecting explicit `"type": "commonjs"` (#26149) When using the `--unstable-detect-cjs` flag or adding `"unstable": ["detect-cjs"]` to a deno.json, it will make a JS file CJS if the closest package.json contains `"type": "commonjs"` and the file is not an ESM module (no TLA, no `import.meta`, no `import`/`export`). --- cli/cache/mod.rs | 57 ++++++++++++++++++++++++++++++++++++++++++++-- cli/cache/parsed_source.rs | 40 ++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+), 2 deletions(-) (limited to 'cli/cache') diff --git a/cli/cache/mod.rs b/cli/cache/mod.rs index 628502c50..ded163b4e 100644 --- a/cli/cache/mod.rs +++ b/cli/cache/mod.rs @@ -9,10 +9,13 @@ use crate::file_fetcher::FetchPermissionsOptionRef; use crate::file_fetcher::FileFetcher; use crate::file_fetcher::FileOrRedirect; use crate::npm::CliNpmResolver; +use crate::resolver::CliNodeResolver; use crate::util::fs::atomic_write_file_with_retries; use crate::util::fs::atomic_write_file_with_retries_and_fs; use crate::util::fs::AtomicWriteFileFsAdapter; use crate::util::path::specifier_has_extension; +use crate::util::text_encoding::arc_str_to_bytes; +use crate::util::text_encoding::from_utf8_lossy_owned; use deno_ast::MediaType; use deno_core::futures; @@ -57,6 +60,7 @@ pub use fast_check::FastCheckCache; pub use incremental::IncrementalCache; pub use module_info::ModuleInfoCache; pub use node::NodeAnalysisCache; +pub use parsed_source::EsmOrCjsChecker; pub use parsed_source::LazyGraphSourceParser; pub use parsed_source::ParsedSourceCache; @@ -177,37 +181,46 @@ pub struct FetchCacherOptions { pub permissions: PermissionsContainer, /// If we're publishing for `deno publish`. pub is_deno_publish: bool, + pub unstable_detect_cjs: bool, } /// A "wrapper" for the FileFetcher and DiskCache for the Deno CLI that provides /// a concise interface to the DENO_DIR when building module graphs. pub struct FetchCacher { - file_fetcher: Arc, pub file_header_overrides: HashMap>, + esm_or_cjs_checker: Arc, + file_fetcher: Arc, global_http_cache: Arc, + node_resolver: Arc, npm_resolver: Arc, module_info_cache: Arc, permissions: PermissionsContainer, - cache_info_enabled: bool, is_deno_publish: bool, + unstable_detect_cjs: bool, + cache_info_enabled: bool, } impl FetchCacher { pub fn new( + esm_or_cjs_checker: Arc, file_fetcher: Arc, global_http_cache: Arc, + node_resolver: Arc, npm_resolver: Arc, module_info_cache: Arc, options: FetchCacherOptions, ) -> Self { Self { file_fetcher, + esm_or_cjs_checker, global_http_cache, + node_resolver, npm_resolver, module_info_cache, file_header_overrides: options.file_header_overrides, permissions: options.permissions, is_deno_publish: options.is_deno_publish, + unstable_detect_cjs: options.unstable_detect_cjs, cache_info_enabled: false, } } @@ -282,6 +295,46 @@ impl Loader for FetchCacher { }, )))); } + + if self.unstable_detect_cjs && specifier_has_extension(specifier, "js") { + if let Ok(Some(pkg_json)) = + self.node_resolver.get_closest_package_json(specifier) + { + if pkg_json.typ == "commonjs" { + if let Ok(path) = specifier.to_file_path() { + if let Ok(bytes) = std::fs::read(&path) { + let text: Arc = from_utf8_lossy_owned(bytes).into(); + let is_es_module = match self.esm_or_cjs_checker.is_esm( + specifier, + text.clone(), + MediaType::JavaScript, + ) { + Ok(value) => value, + Err(err) => { + return Box::pin(futures::future::ready(Err(err.into()))); + } + }; + if !is_es_module { + self.node_resolver.mark_cjs_resolution(specifier.clone()); + return Box::pin(futures::future::ready(Ok(Some( + LoadResponse::External { + specifier: specifier.clone(), + }, + )))); + } else { + return Box::pin(futures::future::ready(Ok(Some( + LoadResponse::Module { + specifier: specifier.clone(), + content: arc_str_to_bytes(text), + maybe_headers: None, + }, + )))); + } + } + } + } + } + } } if self.is_deno_publish diff --git a/cli/cache/parsed_source.rs b/cli/cache/parsed_source.rs index e956361f4..df6e45c35 100644 --- a/cli/cache/parsed_source.rs +++ b/cli/cache/parsed_source.rs @@ -5,6 +5,7 @@ use std::sync::Arc; use deno_ast::MediaType; use deno_ast::ModuleSpecifier; +use deno_ast::ParseDiagnostic; use deno_ast::ParsedSource; use deno_core::parking_lot::Mutex; use deno_graph::CapturingModuleParser; @@ -149,3 +150,42 @@ impl deno_graph::ParsedSourceStore for ParsedSourceCache { } } } + +pub struct EsmOrCjsChecker { + parsed_source_cache: Arc, +} + +impl EsmOrCjsChecker { + pub fn new(parsed_source_cache: Arc) -> Self { + Self { + parsed_source_cache, + } + } + + pub fn is_esm( + &self, + specifier: &ModuleSpecifier, + source: Arc, + media_type: MediaType, + ) -> Result { + // todo(dsherret): add a file cache here to avoid parsing with swc on each run + let source = match self.parsed_source_cache.get_parsed_source(specifier) { + Some(source) => source.clone(), + None => { + let source = deno_ast::parse_program(deno_ast::ParseParams { + specifier: specifier.clone(), + text: source, + media_type, + capture_tokens: true, // capture because it's used for cjs export analysis + scope_analysis: false, + maybe_syntax: None, + })?; + self + .parsed_source_cache + .set_parsed_source(specifier.clone(), source.clone()); + source + } + }; + Ok(source.is_module()) + } +} -- cgit v1.2.3 From 5f0bb3c6f4328003012e98ba70ce18e4e2e842de Mon Sep 17 00:00:00 2001 From: Marvin Hagemeister Date: Thu, 24 Oct 2024 20:03:56 +0200 Subject: fix: `.npmrc` settings not being passed to install/add command (#26473) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We weren't passing the resolved npmrc settings to the install commands. This lead us to always fall back to the default registry url instead of using the one from npmrc. Fixes https://github.com/denoland/deno/issues/26139 Fixes https://github.com/denoland/deno/issues/26033 Fixes https://github.com/denoland/deno/issues/25924 Fixes https://github.com/denoland/deno/issues/25822 Fixes https://github.com/denoland/deno/issues/26152 --------- Co-authored-by: Bartek Iwańczuk --- cli/cache/mod.rs | 1 + 1 file changed, 1 insertion(+) (limited to 'cli/cache') diff --git a/cli/cache/mod.rs b/cli/cache/mod.rs index ded163b4e..bf8f1b1f0 100644 --- a/cli/cache/mod.rs +++ b/cli/cache/mod.rs @@ -378,6 +378,7 @@ impl Loader for FetchCacher { } else { FetchPermissionsOptionRef::DynamicContainer(&permissions) }, + maybe_auth: None, maybe_accept: None, maybe_cache_setting: maybe_cache_setting.as_ref(), }, -- cgit v1.2.3 From f0f476e58453d502ab5c118fc91d1475e6829967 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sat, 26 Oct 2024 13:41:09 -0400 Subject: perf: pass transpiled module to deno_core as known string (#26555) --- cli/cache/emit.rs | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) (limited to 'cli/cache') diff --git a/cli/cache/emit.rs b/cli/cache/emit.rs index 6807f06c1..74e1c1101 100644 --- a/cli/cache/emit.rs +++ b/cli/cache/emit.rs @@ -39,7 +39,7 @@ impl EmitCache { &self, specifier: &ModuleSpecifier, expected_source_hash: u64, - ) -> Option> { + ) -> Option { let emit_filename = self.get_emit_filename(specifier)?; let bytes = self.disk_cache.get(&emit_filename).ok()?; self @@ -100,7 +100,7 @@ impl EmitFileSerializer { &self, mut bytes: Vec, expected_source_hash: u64, - ) -> Option> { + ) -> Option { let last_newline_index = bytes.iter().rposition(|&b| b == b'\n')?; let (content, last_line) = bytes.split_at(last_newline_index); let hashes = last_line.strip_prefix(LAST_LINE_PREFIX.as_bytes())?; @@ -120,7 +120,7 @@ impl EmitFileSerializer { // everything looks good, truncate and return it bytes.truncate(content.len()); - Some(bytes) + String::from_utf8(bytes).ok() } pub fn serialize(&self, code: &[u8], source_hash: u64) -> Vec { @@ -170,8 +170,6 @@ mod test { }, emit_failed_flag: Default::default(), }; - let to_string = - |bytes: Vec| -> String { String::from_utf8(bytes).unwrap() }; let specifier1 = ModuleSpecifier::from_file_path(temp_dir.path().join("file1.ts")) @@ -188,13 +186,10 @@ mod test { assert_eq!(cache.get_emit_code(&specifier1, 5), None); // providing the correct source hash assert_eq!( - cache.get_emit_code(&specifier1, 10).map(to_string), + cache.get_emit_code(&specifier1, 10), Some(emit_code1.clone()), ); - assert_eq!( - cache.get_emit_code(&specifier2, 2).map(to_string), - Some(emit_code2) - ); + assert_eq!(cache.get_emit_code(&specifier2, 2), Some(emit_code2)); // try changing the cli version (should not load previous ones) let cache = EmitCache { @@ -215,18 +210,12 @@ mod test { }, emit_failed_flag: Default::default(), }; - assert_eq!( - cache.get_emit_code(&specifier1, 5).map(to_string), - Some(emit_code1) - ); + assert_eq!(cache.get_emit_code(&specifier1, 5), Some(emit_code1)); // adding when already exists should not cause issue let emit_code3 = "asdf".to_string(); cache.set_emit_code(&specifier1, 20, emit_code3.as_bytes()); assert_eq!(cache.get_emit_code(&specifier1, 5), None); - assert_eq!( - cache.get_emit_code(&specifier1, 20).map(to_string), - Some(emit_code3) - ); + assert_eq!(cache.get_emit_code(&specifier1, 20), Some(emit_code3)); } } -- cgit v1.2.3 From 826e42a5b5880c974ae019a7a21aade6a718062c Mon Sep 17 00:00:00 2001 From: David Sherret Date: Fri, 1 Nov 2024 12:27:00 -0400 Subject: fix: improved support for cjs and cts modules (#26558) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * cts support * better cjs/cts type checking * deno compile cjs/cts support * More efficient detect cjs (going towards stabilization) * Determination of whether .js, .ts, .jsx, or .tsx is cjs or esm is only done after loading * Support `import x = require(...);` Co-authored-by: Bartek Iwańczuk --- cli/cache/cache_db.rs | 6 ++- cli/cache/emit.rs | 2 + cli/cache/mod.rs | 101 ++++++++--------------------------- cli/cache/module_info.rs | 130 +++++++++++++++++++++++++++++++++++---------- cli/cache/parsed_source.rs | 60 ++++----------------- 5 files changed, 139 insertions(+), 160 deletions(-) (limited to 'cli/cache') diff --git a/cli/cache/cache_db.rs b/cli/cache/cache_db.rs index b24078f29..329ed2d97 100644 --- a/cli/cache/cache_db.rs +++ b/cli/cache/cache_db.rs @@ -57,7 +57,7 @@ impl rusqlite::types::FromSql for CacheDBHash { } /// What should the cache should do on failure? -#[derive(Default)] +#[derive(Debug, Default)] pub enum CacheFailure { /// Return errors if failure mode otherwise unspecified. #[default] @@ -69,6 +69,7 @@ pub enum CacheFailure { } /// Configuration SQL and other parameters for a [`CacheDB`]. +#[derive(Debug)] pub struct CacheDBConfiguration { /// SQL to run for a new database. pub table_initializer: &'static str, @@ -98,6 +99,7 @@ impl CacheDBConfiguration { } } +#[derive(Debug)] enum ConnectionState { Connected(Connection), Blackhole, @@ -106,7 +108,7 @@ enum ConnectionState { /// A cache database that eagerly initializes itself off-thread, preventing initialization operations /// from blocking the main thread. -#[derive(Clone)] +#[derive(Debug, Clone)] pub struct CacheDB { // TODO(mmastrac): We can probably simplify our thread-safe implementation here conn: Arc>>, diff --git a/cli/cache/emit.rs b/cli/cache/emit.rs index 74e1c1101..3c9eecfcb 100644 --- a/cli/cache/emit.rs +++ b/cli/cache/emit.rs @@ -10,6 +10,7 @@ use deno_core::unsync::sync::AtomicFlag; use super::DiskCache; /// The cache that stores previously emitted files. +#[derive(Debug)] pub struct EmitCache { disk_cache: DiskCache, emit_failed_flag: AtomicFlag, @@ -91,6 +92,7 @@ impl EmitCache { const LAST_LINE_PREFIX: &str = "\n// denoCacheMetadata="; +#[derive(Debug)] struct EmitFileSerializer { cli_version: &'static str, } diff --git a/cli/cache/mod.rs b/cli/cache/mod.rs index bf8f1b1f0..50fc135dd 100644 --- a/cli/cache/mod.rs +++ b/cli/cache/mod.rs @@ -8,14 +8,9 @@ use crate::file_fetcher::FetchOptions; use crate::file_fetcher::FetchPermissionsOptionRef; use crate::file_fetcher::FileFetcher; use crate::file_fetcher::FileOrRedirect; -use crate::npm::CliNpmResolver; -use crate::resolver::CliNodeResolver; use crate::util::fs::atomic_write_file_with_retries; use crate::util::fs::atomic_write_file_with_retries_and_fs; use crate::util::fs::AtomicWriteFileFsAdapter; -use crate::util::path::specifier_has_extension; -use crate::util::text_encoding::arc_str_to_bytes; -use crate::util::text_encoding::from_utf8_lossy_owned; use deno_ast::MediaType; use deno_core::futures; @@ -25,7 +20,9 @@ use deno_graph::source::CacheInfo; use deno_graph::source::LoadFuture; use deno_graph::source::LoadResponse; use deno_graph::source::Loader; +use deno_runtime::deno_fs; use deno_runtime::deno_permissions::PermissionsContainer; +use node_resolver::InNpmPackageChecker; use std::collections::HashMap; use std::path::Path; use std::path::PathBuf; @@ -60,7 +57,6 @@ pub use fast_check::FastCheckCache; pub use incremental::IncrementalCache; pub use module_info::ModuleInfoCache; pub use node::NodeAnalysisCache; -pub use parsed_source::EsmOrCjsChecker; pub use parsed_source::LazyGraphSourceParser; pub use parsed_source::ParsedSourceCache; @@ -181,46 +177,40 @@ pub struct FetchCacherOptions { pub permissions: PermissionsContainer, /// If we're publishing for `deno publish`. pub is_deno_publish: bool, - pub unstable_detect_cjs: bool, } /// A "wrapper" for the FileFetcher and DiskCache for the Deno CLI that provides /// a concise interface to the DENO_DIR when building module graphs. pub struct FetchCacher { pub file_header_overrides: HashMap>, - esm_or_cjs_checker: Arc, file_fetcher: Arc, + fs: Arc, global_http_cache: Arc, - node_resolver: Arc, - npm_resolver: Arc, + in_npm_pkg_checker: Arc, module_info_cache: Arc, permissions: PermissionsContainer, is_deno_publish: bool, - unstable_detect_cjs: bool, cache_info_enabled: bool, } impl FetchCacher { pub fn new( - esm_or_cjs_checker: Arc, file_fetcher: Arc, + fs: Arc, global_http_cache: Arc, - node_resolver: Arc, - npm_resolver: Arc, + in_npm_pkg_checker: Arc, module_info_cache: Arc, options: FetchCacherOptions, ) -> Self { Self { file_fetcher, - esm_or_cjs_checker, + fs, global_http_cache, - node_resolver, - npm_resolver, + in_npm_pkg_checker, module_info_cache, file_header_overrides: options.file_header_overrides, permissions: options.permissions, is_deno_publish: options.is_deno_publish, - unstable_detect_cjs: options.unstable_detect_cjs, cache_info_enabled: false, } } @@ -271,70 +261,23 @@ impl Loader for FetchCacher { ) -> LoadFuture { use deno_graph::source::CacheSetting as LoaderCacheSetting; - if specifier.scheme() == "file" { - if specifier.path().contains("/node_modules/") { - // The specifier might be in a completely different symlinked tree than - // what the node_modules url is in (ex. `/my-project-1/node_modules` - // symlinked to `/my-project-2/node_modules`), so first we checked if the path - // is in a node_modules dir to avoid needlessly canonicalizing, then now compare - // against the canonicalized specifier. - let specifier = - crate::node::resolve_specifier_into_node_modules(specifier); - if self.npm_resolver.in_npm_package(&specifier) { - return Box::pin(futures::future::ready(Ok(Some( - LoadResponse::External { specifier }, - )))); - } - } - - // make local CJS modules external to the graph - if specifier_has_extension(specifier, "cjs") { + if specifier.scheme() == "file" + && specifier.path().contains("/node_modules/") + { + // The specifier might be in a completely different symlinked tree than + // what the node_modules url is in (ex. `/my-project-1/node_modules` + // symlinked to `/my-project-2/node_modules`), so first we checked if the path + // is in a node_modules dir to avoid needlessly canonicalizing, then now compare + // against the canonicalized specifier. + let specifier = crate::node::resolve_specifier_into_node_modules( + specifier, + self.fs.as_ref(), + ); + if self.in_npm_pkg_checker.in_npm_package(&specifier) { return Box::pin(futures::future::ready(Ok(Some( - LoadResponse::External { - specifier: specifier.clone(), - }, + LoadResponse::External { specifier }, )))); } - - if self.unstable_detect_cjs && specifier_has_extension(specifier, "js") { - if let Ok(Some(pkg_json)) = - self.node_resolver.get_closest_package_json(specifier) - { - if pkg_json.typ == "commonjs" { - if let Ok(path) = specifier.to_file_path() { - if let Ok(bytes) = std::fs::read(&path) { - let text: Arc = from_utf8_lossy_owned(bytes).into(); - let is_es_module = match self.esm_or_cjs_checker.is_esm( - specifier, - text.clone(), - MediaType::JavaScript, - ) { - Ok(value) => value, - Err(err) => { - return Box::pin(futures::future::ready(Err(err.into()))); - } - }; - if !is_es_module { - self.node_resolver.mark_cjs_resolution(specifier.clone()); - return Box::pin(futures::future::ready(Ok(Some( - LoadResponse::External { - specifier: specifier.clone(), - }, - )))); - } else { - return Box::pin(futures::future::ready(Ok(Some( - LoadResponse::Module { - specifier: specifier.clone(), - content: arc_str_to_bytes(text), - maybe_headers: None, - }, - )))); - } - } - } - } - } - } } if self.is_deno_publish diff --git a/cli/cache/module_info.rs b/cli/cache/module_info.rs index 4dbb01c37..060a6f4f0 100644 --- a/cli/cache/module_info.rs +++ b/cli/cache/module_info.rs @@ -44,18 +44,32 @@ pub static MODULE_INFO_CACHE_DB: CacheDBConfiguration = CacheDBConfiguration { /// A cache of `deno_graph::ModuleInfo` objects. Using this leads to a considerable /// performance improvement because when it exists we can skip parsing a module for /// deno_graph. +#[derive(Debug)] pub struct ModuleInfoCache { conn: CacheDB, + parsed_source_cache: Arc, } impl ModuleInfoCache { #[cfg(test)] - pub fn new_in_memory(version: &'static str) -> Self { - Self::new(CacheDB::in_memory(&MODULE_INFO_CACHE_DB, version)) + pub fn new_in_memory( + version: &'static str, + parsed_source_cache: Arc, + ) -> Self { + Self::new( + CacheDB::in_memory(&MODULE_INFO_CACHE_DB, version), + parsed_source_cache, + ) } - pub fn new(conn: CacheDB) -> Self { - Self { conn } + pub fn new( + conn: CacheDB, + parsed_source_cache: Arc, + ) -> Self { + Self { + conn, + parsed_source_cache, + } } /// Useful for testing: re-create this cache DB with a different current version. @@ -63,6 +77,7 @@ impl ModuleInfoCache { pub(crate) fn recreate_with_version(self, version: &'static str) -> Self { Self { conn: self.conn.recreate_with_version(version), + parsed_source_cache: self.parsed_source_cache, } } @@ -113,13 +128,10 @@ impl ModuleInfoCache { Ok(()) } - pub fn as_module_analyzer<'a>( - &'a self, - parsed_source_cache: &'a Arc, - ) -> ModuleInfoCacheModuleAnalyzer<'a> { + pub fn as_module_analyzer(&self) -> ModuleInfoCacheModuleAnalyzer { ModuleInfoCacheModuleAnalyzer { module_info_cache: self, - parsed_source_cache, + parsed_source_cache: &self.parsed_source_cache, } } } @@ -129,31 +141,99 @@ pub struct ModuleInfoCacheModuleAnalyzer<'a> { parsed_source_cache: &'a Arc, } -#[async_trait::async_trait(?Send)] -impl<'a> deno_graph::ModuleAnalyzer for ModuleInfoCacheModuleAnalyzer<'a> { - async fn analyze( +impl<'a> ModuleInfoCacheModuleAnalyzer<'a> { + fn load_cached_module_info( &self, specifier: &ModuleSpecifier, - source: Arc, media_type: MediaType, - ) -> Result { - // attempt to load from the cache - let source_hash = CacheDBHash::from_source(&source); + source_hash: CacheDBHash, + ) -> Option { match self.module_info_cache.get_module_info( specifier, media_type, source_hash, ) { - Ok(Some(info)) => return Ok(info), - Ok(None) => {} + Ok(Some(info)) => Some(info), + Ok(None) => None, Err(err) => { log::debug!( "Error loading module cache info for {}. {:#}", specifier, err ); + None } } + } + + fn save_module_info_to_cache( + &self, + specifier: &ModuleSpecifier, + media_type: MediaType, + source_hash: CacheDBHash, + module_info: &ModuleInfo, + ) { + if let Err(err) = self.module_info_cache.set_module_info( + specifier, + media_type, + source_hash, + module_info, + ) { + log::debug!( + "Error saving module cache info for {}. {:#}", + specifier, + err + ); + } + } + + pub fn analyze_sync( + &self, + specifier: &ModuleSpecifier, + media_type: MediaType, + source: &Arc, + ) -> Result { + // attempt to load from the cache + let source_hash = CacheDBHash::from_source(source); + if let Some(info) = + self.load_cached_module_info(specifier, media_type, source_hash) + { + return Ok(info); + } + + // otherwise, get the module info from the parsed source cache + let parser = self.parsed_source_cache.as_capturing_parser(); + let analyzer = ParserModuleAnalyzer::new(&parser); + let module_info = + analyzer.analyze_sync(specifier, source.clone(), media_type)?; + + // then attempt to cache it + self.save_module_info_to_cache( + specifier, + media_type, + source_hash, + &module_info, + ); + + Ok(module_info) + } +} + +#[async_trait::async_trait(?Send)] +impl<'a> deno_graph::ModuleAnalyzer for ModuleInfoCacheModuleAnalyzer<'a> { + async fn analyze( + &self, + specifier: &ModuleSpecifier, + source: Arc, + media_type: MediaType, + ) -> Result { + // attempt to load from the cache + let source_hash = CacheDBHash::from_source(&source); + if let Some(info) = + self.load_cached_module_info(specifier, media_type, source_hash) + { + return Ok(info); + } // otherwise, get the module info from the parsed source cache let module_info = deno_core::unsync::spawn_blocking({ @@ -169,18 +249,12 @@ impl<'a> deno_graph::ModuleAnalyzer for ModuleInfoCacheModuleAnalyzer<'a> { .unwrap()?; // then attempt to cache it - if let Err(err) = self.module_info_cache.set_module_info( + self.save_module_info_to_cache( specifier, media_type, source_hash, &module_info, - ) { - log::debug!( - "Error saving module cache info for {}. {:#}", - specifier, - err - ); - } + ); Ok(module_info) } @@ -202,7 +276,7 @@ fn serialize_media_type(media_type: MediaType) -> i64 { Tsx => 11, Json => 12, Wasm => 13, - TsBuildInfo => 14, + Css => 14, SourceMap => 15, Unknown => 16, } @@ -217,7 +291,7 @@ mod test { #[test] pub fn module_info_cache_general_use() { - let cache = ModuleInfoCache::new_in_memory("1.0.0"); + let cache = ModuleInfoCache::new_in_memory("1.0.0", Default::default()); let specifier1 = ModuleSpecifier::parse("https://localhost/mod.ts").unwrap(); let specifier2 = diff --git a/cli/cache/parsed_source.rs b/cli/cache/parsed_source.rs index df6e45c35..7e819ae99 100644 --- a/cli/cache/parsed_source.rs +++ b/cli/cache/parsed_source.rs @@ -5,12 +5,11 @@ use std::sync::Arc; use deno_ast::MediaType; use deno_ast::ModuleSpecifier; -use deno_ast::ParseDiagnostic; use deno_ast::ParsedSource; use deno_core::parking_lot::Mutex; -use deno_graph::CapturingModuleParser; -use deno_graph::DefaultModuleParser; -use deno_graph::ModuleParser; +use deno_graph::CapturingEsParser; +use deno_graph::DefaultEsParser; +use deno_graph::EsParser; use deno_graph::ParseOptions; use deno_graph::ParsedSourceStore; @@ -47,7 +46,7 @@ impl<'a> LazyGraphSourceParser<'a> { } } -#[derive(Default)] +#[derive(Debug, Default)] pub struct ParsedSourceCache { sources: Mutex>, } @@ -58,12 +57,11 @@ impl ParsedSourceCache { module: &deno_graph::JsModule, ) -> Result { let parser = self.as_capturing_parser(); - // this will conditionally parse because it's using a CapturingModuleParser - parser.parse_module(ParseOptions { + // this will conditionally parse because it's using a CapturingEsParser + parser.parse_program(ParseOptions { specifier: &module.specifier, source: module.source.clone(), media_type: module.media_type, - // don't bother enabling because this method is currently only used for vendoring scope_analysis: false, }) } @@ -87,10 +85,9 @@ impl ParsedSourceCache { specifier, source, media_type, - // don't bother enabling because this method is currently only used for emitting scope_analysis: false, }; - DefaultModuleParser.parse_module(options) + DefaultEsParser.parse_program(options) } /// Frees the parsed source from memory. @@ -100,8 +97,8 @@ impl ParsedSourceCache { /// Creates a parser that will reuse a ParsedSource from the store /// if it exists, or else parse. - pub fn as_capturing_parser(&self) -> CapturingModuleParser { - CapturingModuleParser::new(None, self) + pub fn as_capturing_parser(&self) -> CapturingEsParser { + CapturingEsParser::new(None, self) } } @@ -150,42 +147,3 @@ impl deno_graph::ParsedSourceStore for ParsedSourceCache { } } } - -pub struct EsmOrCjsChecker { - parsed_source_cache: Arc, -} - -impl EsmOrCjsChecker { - pub fn new(parsed_source_cache: Arc) -> Self { - Self { - parsed_source_cache, - } - } - - pub fn is_esm( - &self, - specifier: &ModuleSpecifier, - source: Arc, - media_type: MediaType, - ) -> Result { - // todo(dsherret): add a file cache here to avoid parsing with swc on each run - let source = match self.parsed_source_cache.get_parsed_source(specifier) { - Some(source) => source.clone(), - None => { - let source = deno_ast::parse_program(deno_ast::ParseParams { - specifier: specifier.clone(), - text: source, - media_type, - capture_tokens: true, // capture because it's used for cjs export analysis - scope_analysis: false, - maybe_syntax: None, - })?; - self - .parsed_source_cache - .set_parsed_source(specifier.clone(), source.clone()); - source - } - }; - Ok(source.is_module()) - } -} -- cgit v1.2.3 From dd4570ed85888d9659a2eec98437dbd6de4a5799 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 18 Nov 2024 15:09:28 -0500 Subject: perf(compile): code cache (#26528) Adds a lazily created code cache to `deno compile` by default. The code cache is created on first run to a single file in the temp directory and is only written once. After it's been written, the code cache becomes read only on subsequent runs. Only the modules loaded during startup are cached (dynamic imports are not code cached). The code cache can be disabled by compiling with `--no-code-cache`. --- cli/cache/code_cache.rs | 10 ++++++++++ 1 file changed, 10 insertions(+) (limited to 'cli/cache') diff --git a/cli/cache/code_cache.rs b/cli/cache/code_cache.rs index abcd0d46a..b1d9ae757 100644 --- a/cli/cache/code_cache.rs +++ b/cli/cache/code_cache.rs @@ -1,10 +1,14 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use std::sync::Arc; + use deno_ast::ModuleSpecifier; use deno_core::error::AnyError; use deno_runtime::code_cache; use deno_runtime::deno_webstorage::rusqlite::params; +use crate::worker::CliCodeCache; + use super::cache_db::CacheDB; use super::cache_db::CacheDBConfiguration; use super::cache_db::CacheDBHash; @@ -82,6 +86,12 @@ impl CodeCache { } } +impl CliCodeCache for CodeCache { + fn as_code_cache(self: Arc) -> Arc { + self + } +} + impl code_cache::CodeCache for CodeCache { fn get_sync( &self, -- cgit v1.2.3