From eedf243b5ea98d22649bb0445444719a2fc12c59 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 24 Oct 2024 15:48:48 -0400 Subject: perf(compile): pass module source data from binary directly to v8 (#26494) This changes denort to pass a static reference of the moude source bytes found in the binary to v8 instead of copying it. --- cli/emit.rs | 1 + 1 file changed, 1 insertion(+) (limited to 'cli/emit.rs') diff --git a/cli/emit.rs b/cli/emit.rs index b3f4a4477..ad200af05 100644 --- a/cli/emit.rs +++ b/cli/emit.rs @@ -60,6 +60,7 @@ impl Emitter { continue; }; + // todo(https://github.com/denoland/deno_media_type/pull/12): use is_emittable() let is_emittable = matches!( module.media_type, MediaType::TypeScript -- cgit v1.2.3 From f0f476e58453d502ab5c118fc91d1475e6829967 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sat, 26 Oct 2024 13:41:09 -0400 Subject: perf: pass transpiled module to deno_core as known string (#26555) --- cli/emit.rs | 55 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 29 insertions(+), 26 deletions(-) (limited to 'cli/emit.rs') diff --git a/cli/emit.rs b/cli/emit.rs index ad200af05..8e93092e6 100644 --- a/cli/emit.rs +++ b/cli/emit.rs @@ -13,7 +13,6 @@ use deno_core::error::AnyError; use deno_core::futures::stream::FuturesUnordered; use deno_core::futures::FutureExt; use deno_core::futures::StreamExt; -use deno_core::ModuleCodeBytes; use deno_core::ModuleSpecifier; use deno_graph::MediaType; use deno_graph::Module; @@ -94,7 +93,7 @@ impl Emitter { &self, specifier: &ModuleSpecifier, source: &str, - ) -> Option> { + ) -> Option { let source_hash = self.get_source_hash(source); self.emit_cache.get_emit_code(specifier, source_hash) } @@ -104,7 +103,7 @@ impl Emitter { specifier: &ModuleSpecifier, media_type: MediaType, source: &Arc, - ) -> Result { + ) -> Result { // Note: keep this in sync with the sync version below let helper = EmitParsedSourceHelper(self); match helper.pre_emit_parsed_source(specifier, source) { @@ -113,7 +112,7 @@ impl Emitter { let parsed_source_cache = self.parsed_source_cache.clone(); let transpile_and_emit_options = self.transpile_and_emit_options.clone(); - let transpile_result = deno_core::unsync::spawn_blocking({ + let transpiled_source = deno_core::unsync::spawn_blocking({ let specifier = specifier.clone(); let source = source.clone(); move || -> Result<_, AnyError> { @@ -129,11 +128,12 @@ impl Emitter { }) .await .unwrap()?; - Ok(helper.post_emit_parsed_source( + helper.post_emit_parsed_source( specifier, - transpile_result, + &transpiled_source, source_hash, - )) + ); + Ok(transpiled_source) } } } @@ -143,13 +143,13 @@ impl Emitter { specifier: &ModuleSpecifier, media_type: MediaType, source: &Arc, - ) -> Result { + ) -> Result { // Note: keep this in sync with the async version above let helper = EmitParsedSourceHelper(self); match helper.pre_emit_parsed_source(specifier, source) { PreEmitResult::Cached(emitted_text) => Ok(emitted_text), PreEmitResult::NotCached { source_hash } => { - let transpile_result = EmitParsedSourceHelper::transpile( + let transpiled_source = EmitParsedSourceHelper::transpile( &self.parsed_source_cache, specifier, source.clone(), @@ -157,11 +157,12 @@ impl Emitter { &self.transpile_and_emit_options.0, &self.transpile_and_emit_options.1, )?; - Ok(helper.post_emit_parsed_source( + helper.post_emit_parsed_source( specifier, - transpile_result, + &transpiled_source, source_hash, - )) + ); + Ok(transpiled_source) } } } @@ -227,7 +228,7 @@ impl Emitter { } enum PreEmitResult { - Cached(ModuleCodeBytes), + Cached(String), NotCached { source_hash: u64 }, } @@ -245,7 +246,7 @@ impl<'a> EmitParsedSourceHelper<'a> { if let Some(emit_code) = self.0.emit_cache.get_emit_code(specifier, source_hash) { - PreEmitResult::Cached(emit_code.into_boxed_slice().into()) + PreEmitResult::Cached(emit_code) } else { PreEmitResult::NotCached { source_hash } } @@ -258,21 +259,14 @@ impl<'a> EmitParsedSourceHelper<'a> { media_type: MediaType, transpile_options: &deno_ast::TranspileOptions, emit_options: &deno_ast::EmitOptions, - ) -> Result { + ) -> Result { // nothing else needs the parsed source at this point, so remove from // the cache in order to not transpile owned let parsed_source = parsed_source_cache .remove_or_parse_module(specifier, source, media_type)?; ensure_no_import_assertion(&parsed_source)?; - Ok(parsed_source.transpile(transpile_options, emit_options)?) - } - - pub fn post_emit_parsed_source( - &self, - specifier: &ModuleSpecifier, - transpile_result: TranspileResult, - source_hash: u64, - ) -> ModuleCodeBytes { + let transpile_result = + parsed_source.transpile(transpile_options, emit_options)?; let transpiled_source = match transpile_result { TranspileResult::Owned(source) => source, TranspileResult::Cloned(source) => { @@ -281,12 +275,21 @@ impl<'a> EmitParsedSourceHelper<'a> { } }; debug_assert!(transpiled_source.source_map.is_none()); + let text = String::from_utf8(transpiled_source.source)?; + Ok(text) + } + + pub fn post_emit_parsed_source( + &self, + specifier: &ModuleSpecifier, + transpiled_source: &str, + source_hash: u64, + ) { self.0.emit_cache.set_emit_code( specifier, source_hash, - &transpiled_source.source, + transpiled_source.as_bytes(), ); - transpiled_source.source.into_boxed_slice().into() } } -- cgit v1.2.3 From 826e42a5b5880c974ae019a7a21aade6a718062c Mon Sep 17 00:00:00 2001 From: David Sherret Date: Fri, 1 Nov 2024 12:27:00 -0400 Subject: fix: improved support for cjs and cts modules (#26558) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * cts support * better cjs/cts type checking * deno compile cjs/cts support * More efficient detect cjs (going towards stabilization) * Determination of whether .js, .ts, .jsx, or .tsx is cjs or esm is only done after loading * Support `import x = require(...);` Co-authored-by: Bartek Iwańczuk --- cli/emit.rs | 77 +++++++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 50 insertions(+), 27 deletions(-) (limited to 'cli/emit.rs') diff --git a/cli/emit.rs b/cli/emit.rs index 8e93092e6..8c4f2091c 100644 --- a/cli/emit.rs +++ b/cli/emit.rs @@ -3,11 +3,14 @@ use crate::cache::EmitCache; use crate::cache::FastInsecureHasher; use crate::cache::ParsedSourceCache; +use crate::resolver::CjsTracker; +use deno_ast::ModuleKind; use deno_ast::SourceMapOption; use deno_ast::SourceRange; use deno_ast::SourceRanged; use deno_ast::SourceRangedForSpanned; +use deno_ast::TranspileModuleOptions; use deno_ast::TranspileResult; use deno_core::error::AnyError; use deno_core::futures::stream::FuturesUnordered; @@ -19,7 +22,9 @@ use deno_graph::Module; use deno_graph::ModuleGraph; use std::sync::Arc; +#[derive(Debug)] pub struct Emitter { + cjs_tracker: Arc, emit_cache: Arc, parsed_source_cache: Arc, transpile_and_emit_options: @@ -30,6 +35,7 @@ pub struct Emitter { impl Emitter { pub fn new( + cjs_tracker: Arc, emit_cache: Arc, parsed_source_cache: Arc, transpile_options: deno_ast::TranspileOptions, @@ -42,6 +48,7 @@ impl Emitter { hasher.finish() }; Self { + cjs_tracker, emit_cache, parsed_source_cache, transpile_and_emit_options: Arc::new((transpile_options, emit_options)), @@ -59,21 +66,19 @@ impl Emitter { continue; }; - // todo(https://github.com/denoland/deno_media_type/pull/12): use is_emittable() - let is_emittable = matches!( - module.media_type, - MediaType::TypeScript - | MediaType::Mts - | MediaType::Cts - | MediaType::Jsx - | MediaType::Tsx - ); - if is_emittable { + if module.media_type.is_emittable() { futures.push( self .emit_parsed_source( &module.specifier, module.media_type, + ModuleKind::from_is_cjs( + self.cjs_tracker.is_cjs_with_known_is_script( + &module.specifier, + module.media_type, + module.is_script, + )?, + ), &module.source, ) .boxed_local(), @@ -92,9 +97,10 @@ impl Emitter { pub fn maybe_cached_emit( &self, specifier: &ModuleSpecifier, + module_kind: deno_ast::ModuleKind, source: &str, ) -> Option { - let source_hash = self.get_source_hash(source); + let source_hash = self.get_source_hash(module_kind, source); self.emit_cache.get_emit_code(specifier, source_hash) } @@ -102,11 +108,12 @@ impl Emitter { &self, specifier: &ModuleSpecifier, media_type: MediaType, + module_kind: deno_ast::ModuleKind, source: &Arc, ) -> Result { // Note: keep this in sync with the sync version below let helper = EmitParsedSourceHelper(self); - match helper.pre_emit_parsed_source(specifier, source) { + match helper.pre_emit_parsed_source(specifier, module_kind, source) { PreEmitResult::Cached(emitted_text) => Ok(emitted_text), PreEmitResult::NotCached { source_hash } => { let parsed_source_cache = self.parsed_source_cache.clone(); @@ -119,8 +126,9 @@ impl Emitter { EmitParsedSourceHelper::transpile( &parsed_source_cache, &specifier, - source.clone(), media_type, + module_kind, + source.clone(), &transpile_and_emit_options.0, &transpile_and_emit_options.1, ) @@ -142,18 +150,20 @@ impl Emitter { &self, specifier: &ModuleSpecifier, media_type: MediaType, + module_kind: deno_ast::ModuleKind, source: &Arc, ) -> Result { // Note: keep this in sync with the async version above let helper = EmitParsedSourceHelper(self); - match helper.pre_emit_parsed_source(specifier, source) { + match helper.pre_emit_parsed_source(specifier, module_kind, source) { PreEmitResult::Cached(emitted_text) => Ok(emitted_text), PreEmitResult::NotCached { source_hash } => { let transpiled_source = EmitParsedSourceHelper::transpile( &self.parsed_source_cache, specifier, - source.clone(), media_type, + module_kind, + source.clone(), &self.transpile_and_emit_options.0, &self.transpile_and_emit_options.1, )?; @@ -171,6 +181,7 @@ impl Emitter { pub async fn load_and_emit_for_hmr( &self, specifier: &ModuleSpecifier, + module_kind: deno_ast::ModuleKind, ) -> Result { let media_type = MediaType::from_specifier(specifier); let source_code = tokio::fs::read_to_string( @@ -193,9 +204,14 @@ impl Emitter { let mut options = self.transpile_and_emit_options.1.clone(); options.source_map = SourceMapOption::None; let transpiled_source = parsed_source - .transpile(&self.transpile_and_emit_options.0, &options)? - .into_source() - .into_string()?; + .transpile( + &self.transpile_and_emit_options.0, + &deno_ast::TranspileModuleOptions { + module_kind: Some(module_kind), + }, + &options, + )? + .into_source(); Ok(transpiled_source.text) } MediaType::JavaScript @@ -206,7 +222,7 @@ impl Emitter { | MediaType::Dcts | MediaType::Json | MediaType::Wasm - | MediaType::TsBuildInfo + | MediaType::Css | MediaType::SourceMap | MediaType::Unknown => { // clear this specifier from the parsed source cache as it's now out of date @@ -219,10 +235,11 @@ impl Emitter { /// A hashing function that takes the source code and uses the global emit /// options then generates a string hash which can be stored to /// determine if the cached emit is valid or not. - fn get_source_hash(&self, source_text: &str) -> u64 { + fn get_source_hash(&self, module_kind: ModuleKind, source_text: &str) -> u64 { FastInsecureHasher::new_without_deno_version() // stored in the transpile_and_emit_options_hash .write_str(source_text) .write_u64(self.transpile_and_emit_options_hash) + .write_hashable(module_kind) .finish() } } @@ -239,9 +256,10 @@ impl<'a> EmitParsedSourceHelper<'a> { pub fn pre_emit_parsed_source( &self, specifier: &ModuleSpecifier, + module_kind: deno_ast::ModuleKind, source: &Arc, ) -> PreEmitResult { - let source_hash = self.0.get_source_hash(source); + let source_hash = self.0.get_source_hash(module_kind, source); if let Some(emit_code) = self.0.emit_cache.get_emit_code(specifier, source_hash) @@ -255,8 +273,9 @@ impl<'a> EmitParsedSourceHelper<'a> { pub fn transpile( parsed_source_cache: &ParsedSourceCache, specifier: &ModuleSpecifier, - source: Arc, media_type: MediaType, + module_kind: deno_ast::ModuleKind, + source: Arc, transpile_options: &deno_ast::TranspileOptions, emit_options: &deno_ast::EmitOptions, ) -> Result { @@ -265,8 +284,13 @@ impl<'a> EmitParsedSourceHelper<'a> { let parsed_source = parsed_source_cache .remove_or_parse_module(specifier, source, media_type)?; ensure_no_import_assertion(&parsed_source)?; - let transpile_result = - parsed_source.transpile(transpile_options, emit_options)?; + let transpile_result = parsed_source.transpile( + transpile_options, + &TranspileModuleOptions { + module_kind: Some(module_kind), + }, + emit_options, + )?; let transpiled_source = match transpile_result { TranspileResult::Owned(source) => source, TranspileResult::Cloned(source) => { @@ -275,8 +299,7 @@ impl<'a> EmitParsedSourceHelper<'a> { } }; debug_assert!(transpiled_source.source_map.is_none()); - let text = String::from_utf8(transpiled_source.source)?; - Ok(text) + Ok(transpiled_source.text) } pub fn post_emit_parsed_source( @@ -321,7 +344,7 @@ fn ensure_no_import_assertion( deno_core::anyhow::anyhow!("{}", msg) } - let Some(module) = parsed_source.program_ref().as_module() else { + let deno_ast::ProgramRef::Module(module) = parsed_source.program_ref() else { return Ok(()); }; -- cgit v1.2.3 From f091d1ad69b4e5217ae3272b641171781a372c4f Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 13 Nov 2024 10:10:09 -0500 Subject: feat(node): stabilize detecting if CJS via `"type": "commonjs"` in a package.json (#26439) This will respect `"type": "commonjs"` in a package.json to determine if `.js`/`.jsx`/`.ts`/.tsx` files are CJS or ESM. If the file is found to be ESM it will be loaded as ESM though. --- cli/emit.rs | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'cli/emit.rs') diff --git a/cli/emit.rs b/cli/emit.rs index 8c4f2091c..3cd23b7ab 100644 --- a/cli/emit.rs +++ b/cli/emit.rs @@ -181,7 +181,6 @@ impl Emitter { pub async fn load_and_emit_for_hmr( &self, specifier: &ModuleSpecifier, - module_kind: deno_ast::ModuleKind, ) -> Result { let media_type = MediaType::from_specifier(specifier); let source_code = tokio::fs::read_to_string( @@ -203,11 +202,16 @@ impl Emitter { // this statement is probably wrong) let mut options = self.transpile_and_emit_options.1.clone(); options.source_map = SourceMapOption::None; + let is_cjs = self.cjs_tracker.is_cjs_with_known_is_script( + specifier, + media_type, + parsed_source.compute_is_script(), + )?; let transpiled_source = parsed_source .transpile( &self.transpile_and_emit_options.0, &deno_ast::TranspileModuleOptions { - module_kind: Some(module_kind), + module_kind: Some(ModuleKind::from_is_cjs(is_cjs)), }, &options, )? -- cgit v1.2.3