From 1a0cb5b5312941521ab021cfe9eaed498f35900b Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 14 Oct 2024 20:48:39 -0400 Subject: feat(unstable): `--unstable-detect-cjs` for respecting explicit `"type": "commonjs"` (#26149) When using the `--unstable-detect-cjs` flag or adding `"unstable": ["detect-cjs"]` to a deno.json, it will make a JS file CJS if the closest package.json contains `"type": "commonjs"` and the file is not an ESM module (no TLA, no `import.meta`, no `import`/`export`). --- cli/node.rs | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) (limited to 'cli/node.rs') diff --git a/cli/node.rs b/cli/node.rs index a3cee8dde..733d5f871 100644 --- a/cli/node.rs +++ b/cli/node.rs @@ -5,6 +5,7 @@ use std::sync::Arc; use deno_ast::MediaType; use deno_ast::ModuleSpecifier; use deno_core::error::AnyError; +use deno_graph::ParsedSourceStore; use deno_runtime::deno_fs; use deno_runtime::deno_node::DenoFsNodeResolverEnv; use node_resolver::analyze::CjsAnalysis as ExtNodeCjsAnalysis; @@ -16,6 +17,7 @@ use serde::Serialize; use crate::cache::CacheDBHash; use crate::cache::NodeAnalysisCache; +use crate::cache::ParsedSourceCache; use crate::resolver::CliNodeResolver; use crate::util::fs::canonicalize_path_maybe_not_exists; @@ -56,6 +58,7 @@ pub struct CliCjsCodeAnalyzer { cache: NodeAnalysisCache, fs: deno_fs::FileSystemRc, node_resolver: Arc, + parsed_source_cache: Option>, } impl CliCjsCodeAnalyzer { @@ -63,11 +66,13 @@ impl CliCjsCodeAnalyzer { cache: NodeAnalysisCache, fs: deno_fs::FileSystemRc, node_resolver: Arc, + parsed_source_cache: Option>, ) -> Self { Self { cache, fs, node_resolver, + parsed_source_cache, } } @@ -107,18 +112,26 @@ impl CliCjsCodeAnalyzer { } } + let maybe_parsed_source = self + .parsed_source_cache + .as_ref() + .and_then(|c| c.remove_parsed_source(specifier)); + let analysis = deno_core::unsync::spawn_blocking({ let specifier = specifier.clone(); let source: Arc = source.into(); move || -> Result<_, deno_ast::ParseDiagnostic> { - let parsed_source = deno_ast::parse_program(deno_ast::ParseParams { - specifier, - text: source, - media_type, - capture_tokens: true, - scope_analysis: false, - maybe_syntax: None, - })?; + let parsed_source = + maybe_parsed_source.map(Ok).unwrap_or_else(|| { + deno_ast::parse_program(deno_ast::ParseParams { + specifier, + text: source, + media_type, + capture_tokens: true, + scope_analysis: false, + maybe_syntax: None, + }) + })?; if parsed_source.is_script() { let analysis = parsed_source.analyze_cjs(); Ok(CliCjsAnalysis::Cjs { -- cgit v1.2.3 From 826e42a5b5880c974ae019a7a21aade6a718062c Mon Sep 17 00:00:00 2001 From: David Sherret Date: Fri, 1 Nov 2024 12:27:00 -0400 Subject: fix: improved support for cjs and cts modules (#26558) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * cts support * better cjs/cts type checking * deno compile cjs/cts support * More efficient detect cjs (going towards stabilization) * Determination of whether .js, .ts, .jsx, or .tsx is cjs or esm is only done after loading * Support `import x = require(...);` Co-authored-by: Bartek Iwańczuk --- cli/node.rs | 136 ++++++++++++++++++++++++++++++------------------------------ 1 file changed, 68 insertions(+), 68 deletions(-) (limited to 'cli/node.rs') diff --git a/cli/node.rs b/cli/node.rs index 733d5f871..1d410a726 100644 --- a/cli/node.rs +++ b/cli/node.rs @@ -1,11 +1,14 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use std::borrow::Cow; use std::sync::Arc; use deno_ast::MediaType; use deno_ast::ModuleSpecifier; use deno_core::error::AnyError; use deno_graph::ParsedSourceStore; +use deno_path_util::url_from_file_path; +use deno_path_util::url_to_file_path; use deno_runtime::deno_fs; use deno_runtime::deno_node::DenoFsNodeResolverEnv; use node_resolver::analyze::CjsAnalysis as ExtNodeCjsAnalysis; @@ -18,8 +21,8 @@ use serde::Serialize; use crate::cache::CacheDBHash; use crate::cache::NodeAnalysisCache; use crate::cache::ParsedSourceCache; -use crate::resolver::CliNodeResolver; -use crate::util::fs::canonicalize_path_maybe_not_exists; +use crate::resolver::CjsTracker; +use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs; pub type CliNodeCodeTranslator = NodeCodeTranslator; @@ -32,14 +35,14 @@ pub type CliNodeCodeTranslator = /// because the node_modules folder might not exist at that time. pub fn resolve_specifier_into_node_modules( specifier: &ModuleSpecifier, + fs: &dyn deno_fs::FileSystem, ) -> ModuleSpecifier { - specifier - .to_file_path() + url_to_file_path(specifier) .ok() // this path might not exist at the time the graph is being created // because the node_modules folder might not yet exist - .and_then(|path| canonicalize_path_maybe_not_exists(&path).ok()) - .and_then(|path| ModuleSpecifier::from_file_path(path).ok()) + .and_then(|path| canonicalize_path_maybe_not_exists_with_fs(&path, fs).ok()) + .and_then(|path| url_from_file_path(&path).ok()) .unwrap_or_else(|| specifier.clone()) } @@ -56,23 +59,29 @@ pub enum CliCjsAnalysis { pub struct CliCjsCodeAnalyzer { cache: NodeAnalysisCache, + cjs_tracker: Arc, fs: deno_fs::FileSystemRc, - node_resolver: Arc, parsed_source_cache: Option>, + // todo(dsherret): hack, remove in https://github.com/denoland/deno/pull/26439 + // For example, this does not properly handle if cjs analysis was already done + // and has been cached. + is_npm_main: bool, } impl CliCjsCodeAnalyzer { pub fn new( cache: NodeAnalysisCache, + cjs_tracker: Arc, fs: deno_fs::FileSystemRc, - node_resolver: Arc, parsed_source_cache: Option>, + is_npm_main: bool, ) -> Self { Self { cache, + cjs_tracker, fs, - node_resolver, parsed_source_cache, + is_npm_main, } } @@ -88,7 +97,7 @@ impl CliCjsCodeAnalyzer { return Ok(analysis); } - let mut media_type = MediaType::from_specifier(specifier); + let media_type = MediaType::from_specifier(specifier); if media_type == MediaType::Json { return Ok(CliCjsAnalysis::Cjs { exports: vec![], @@ -96,62 +105,53 @@ impl CliCjsCodeAnalyzer { }); } - if media_type == MediaType::JavaScript { - if let Some(package_json) = - self.node_resolver.get_closest_package_json(specifier)? - { - match package_json.typ.as_str() { - "commonjs" => { - media_type = MediaType::Cjs; - } - "module" => { - media_type = MediaType::Mjs; - } - _ => {} - } - } - } - - let maybe_parsed_source = self - .parsed_source_cache - .as_ref() - .and_then(|c| c.remove_parsed_source(specifier)); - - let analysis = deno_core::unsync::spawn_blocking({ - let specifier = specifier.clone(); - let source: Arc = source.into(); - move || -> Result<_, deno_ast::ParseDiagnostic> { - let parsed_source = - maybe_parsed_source.map(Ok).unwrap_or_else(|| { - deno_ast::parse_program(deno_ast::ParseParams { - specifier, - text: source, - media_type, - capture_tokens: true, - scope_analysis: false, - maybe_syntax: None, + let cjs_tracker = self.cjs_tracker.clone(); + let is_npm_main = self.is_npm_main; + let is_maybe_cjs = + cjs_tracker.is_maybe_cjs(specifier, media_type)? || is_npm_main; + let analysis = if is_maybe_cjs { + let maybe_parsed_source = self + .parsed_source_cache + .as_ref() + .and_then(|c| c.remove_parsed_source(specifier)); + + deno_core::unsync::spawn_blocking({ + let specifier = specifier.clone(); + let source: Arc = source.into(); + move || -> Result<_, AnyError> { + let parsed_source = + maybe_parsed_source.map(Ok).unwrap_or_else(|| { + deno_ast::parse_program(deno_ast::ParseParams { + specifier, + text: source, + media_type, + capture_tokens: true, + scope_analysis: false, + maybe_syntax: None, + }) + })?; + let is_script = parsed_source.compute_is_script(); + let is_cjs = cjs_tracker.is_cjs_with_known_is_script( + parsed_source.specifier(), + media_type, + is_script, + )? || is_script && is_npm_main; + if is_cjs { + let analysis = parsed_source.analyze_cjs(); + Ok(CliCjsAnalysis::Cjs { + exports: analysis.exports, + reexports: analysis.reexports, }) - })?; - if parsed_source.is_script() { - let analysis = parsed_source.analyze_cjs(); - Ok(CliCjsAnalysis::Cjs { - exports: analysis.exports, - reexports: analysis.reexports, - }) - } else if media_type == MediaType::Cjs { - // FIXME: `deno_ast` should internally handle MediaType::Cjs implying that - // the result must never be Esm - Ok(CliCjsAnalysis::Cjs { - exports: vec![], - reexports: vec![], - }) - } else { - Ok(CliCjsAnalysis::Esm) + } else { + Ok(CliCjsAnalysis::Esm) + } } - } - }) - .await - .unwrap()?; + }) + .await + .unwrap()? + } else { + CliCjsAnalysis::Esm + }; self .cache @@ -163,11 +163,11 @@ impl CliCjsCodeAnalyzer { #[async_trait::async_trait(?Send)] impl CjsCodeAnalyzer for CliCjsCodeAnalyzer { - async fn analyze_cjs( + async fn analyze_cjs<'a>( &self, specifier: &ModuleSpecifier, - source: Option, - ) -> Result { + source: Option>, + ) -> Result, AnyError> { let source = match source { Some(source) => source, None => { @@ -175,7 +175,7 @@ impl CjsCodeAnalyzer for CliCjsCodeAnalyzer { if let Ok(source_from_file) = self.fs.read_text_file_lossy_async(path, None).await { - source_from_file + Cow::Owned(source_from_file) } else { return Ok(ExtNodeCjsAnalysis::Cjs(CjsAnalysisExports { exports: vec![], -- cgit v1.2.3 From f091d1ad69b4e5217ae3272b641171781a372c4f Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 13 Nov 2024 10:10:09 -0500 Subject: feat(node): stabilize detecting if CJS via `"type": "commonjs"` in a package.json (#26439) This will respect `"type": "commonjs"` in a package.json to determine if `.js`/`.jsx`/`.ts`/.tsx` files are CJS or ESM. If the file is found to be ESM it will be loaded as ESM though. --- cli/node.rs | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) (limited to 'cli/node.rs') diff --git a/cli/node.rs b/cli/node.rs index 1d410a726..8235745a9 100644 --- a/cli/node.rs +++ b/cli/node.rs @@ -62,10 +62,6 @@ pub struct CliCjsCodeAnalyzer { cjs_tracker: Arc, fs: deno_fs::FileSystemRc, parsed_source_cache: Option>, - // todo(dsherret): hack, remove in https://github.com/denoland/deno/pull/26439 - // For example, this does not properly handle if cjs analysis was already done - // and has been cached. - is_npm_main: bool, } impl CliCjsCodeAnalyzer { @@ -74,14 +70,12 @@ impl CliCjsCodeAnalyzer { cjs_tracker: Arc, fs: deno_fs::FileSystemRc, parsed_source_cache: Option>, - is_npm_main: bool, ) -> Self { Self { cache, cjs_tracker, fs, parsed_source_cache, - is_npm_main, } } @@ -106,9 +100,7 @@ impl CliCjsCodeAnalyzer { } let cjs_tracker = self.cjs_tracker.clone(); - let is_npm_main = self.is_npm_main; - let is_maybe_cjs = - cjs_tracker.is_maybe_cjs(specifier, media_type)? || is_npm_main; + let is_maybe_cjs = cjs_tracker.is_maybe_cjs(specifier, media_type)?; let analysis = if is_maybe_cjs { let maybe_parsed_source = self .parsed_source_cache @@ -135,7 +127,7 @@ impl CliCjsCodeAnalyzer { parsed_source.specifier(), media_type, is_script, - )? || is_script && is_npm_main; + )?; if is_cjs { let analysis = parsed_source.analyze_cjs(); Ok(CliCjsAnalysis::Cjs { -- cgit v1.2.3 From 617350e79c58b6e01984e3d7c7436d243d0e5cff Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 14 Nov 2024 15:24:25 -0500 Subject: refactor(resolver): move more resolution code into deno_resolver (#26873) Follow-up to cjs refactor. This moves most of the resolution code into the deno_resolver crate. Still pending is the npm resolution code. --- cli/node.rs | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) (limited to 'cli/node.rs') diff --git a/cli/node.rs b/cli/node.rs index 8235745a9..bc39cdbde 100644 --- a/cli/node.rs +++ b/cli/node.rs @@ -7,8 +7,6 @@ use deno_ast::MediaType; use deno_ast::ModuleSpecifier; use deno_core::error::AnyError; use deno_graph::ParsedSourceStore; -use deno_path_util::url_from_file_path; -use deno_path_util::url_to_file_path; use deno_runtime::deno_fs; use deno_runtime::deno_node::DenoFsNodeResolverEnv; use node_resolver::analyze::CjsAnalysis as ExtNodeCjsAnalysis; @@ -22,7 +20,6 @@ use crate::cache::CacheDBHash; use crate::cache::NodeAnalysisCache; use crate::cache::ParsedSourceCache; use crate::resolver::CjsTracker; -use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs; pub type CliNodeCodeTranslator = NodeCodeTranslator; @@ -37,13 +34,9 @@ pub fn resolve_specifier_into_node_modules( specifier: &ModuleSpecifier, fs: &dyn deno_fs::FileSystem, ) -> ModuleSpecifier { - url_to_file_path(specifier) - .ok() - // this path might not exist at the time the graph is being created - // because the node_modules folder might not yet exist - .and_then(|path| canonicalize_path_maybe_not_exists_with_fs(&path, fs).ok()) - .and_then(|path| url_from_file_path(&path).ok()) - .unwrap_or_else(|| specifier.clone()) + node_resolver::resolve_specifier_into_node_modules(specifier, &|path| { + fs.realpath_sync(path).map_err(|err| err.into_io_error()) + }) } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] -- cgit v1.2.3