fix: improved support for cjs and cts modules (#26558)

* cts support
* better cjs/cts type checking
* deno compile cjs/cts support
* More efficient detect cjs (going towards stabilization)
* Determination of whether .js, .ts, .jsx, or .tsx is cjs or esm is only
done after loading
* Support `import x = require(...);`

Co-authored-by: Bartek Iwańczuk <biwanczuk@gmail.com>

9 files changed, 51 insertions, 36 deletions

diff --git a/cli/tools/bench/mod.rs b/cli/tools/bench/mod.rs
index be5d0ad0e..272d06335 100644
--- a/cli/tools/bench/mod.rs
+++ b/cli/tools/bench/mod.rs
@@ -193,7 +193,7 @@ async fn bench_specifier_inner(
     .await?;
 
   // We execute the main module as a side module so that import.meta.main is not set.
-  worker.execute_side_module_possibly_with_npm().await?;
+  worker.execute_side_module().await?;
 
   let mut worker = worker.into_main_worker();
 
diff --git a/cli/tools/check.rs b/cli/tools/check.rs
index 7edb392d4..d88027888 100644
--- a/cli/tools/check.rs
+++ b/cli/tools/check.rs
@@ -32,6 +32,7 @@ use crate::graph_util::ModuleGraphBuilder;
 use crate::npm::CliNpmResolver;
 use crate::tsc;
 use crate::tsc::Diagnostics;
+use crate::tsc::TypeCheckingCjsTracker;
 use crate::util::extract;
 use crate::util::path::to_percent_decoded_str;
 
@@ -99,6 +100,7 @@ pub struct CheckOptions {
 
 pub struct TypeChecker {
   caches: Arc<Caches>,
+  cjs_tracker: Arc<TypeCheckingCjsTracker>,
   cli_options: Arc<CliOptions>,
   module_graph_builder: Arc<ModuleGraphBuilder>,
   node_resolver: Arc<NodeResolver>,
@@ -108,6 +110,7 @@ pub struct TypeChecker {
 impl TypeChecker {
   pub fn new(
     caches: Arc<Caches>,
+    cjs_tracker: Arc<TypeCheckingCjsTracker>,
     cli_options: Arc<CliOptions>,
     module_graph_builder: Arc<ModuleGraphBuilder>,
     node_resolver: Arc<NodeResolver>,
@@ -115,6 +118,7 @@ impl TypeChecker {
   ) -> Self {
     Self {
       caches,
+      cjs_tracker,
       cli_options,
       module_graph_builder,
       node_resolver,
@@ -244,6 +248,7 @@ impl TypeChecker {
       graph: graph.clone(),
       hash_data,
       maybe_npm: Some(tsc::RequestNpmState {
+        cjs_tracker: self.cjs_tracker.clone(),
         node_resolver: self.node_resolver.clone(),
         npm_resolver: self.npm_resolver.clone(),
       }),
@@ -346,7 +351,7 @@ fn get_check_hash(
             }
           }
           MediaType::Json
-          | MediaType::TsBuildInfo
+          | MediaType::Css
           | MediaType::SourceMap
           | MediaType::Wasm
           | MediaType::Unknown => continue,
@@ -428,7 +433,7 @@ fn get_tsc_roots(
         }
         MediaType::Json
         | MediaType::Wasm
-        | MediaType::TsBuildInfo
+        | MediaType::Css
         | MediaType::SourceMap
         | MediaType::Unknown => None,
       },
@@ -536,7 +541,7 @@ fn has_ts_check(media_type: MediaType, file_text: &str) -> bool {
     | MediaType::Tsx
     | MediaType::Json
     | MediaType::Wasm
-    | MediaType::TsBuildInfo
+    | MediaType::Css
     | MediaType::SourceMap
     | MediaType::Unknown => false,
   }
diff --git a/cli/tools/compile.rs b/cli/tools/compile.rs
index 5a4a938bb..b3e999337 100644
--- a/cli/tools/compile.rs
+++ b/cli/tools/compile.rs
@@ -53,16 +53,6 @@ pub async fn compile(
     );
   }
 
-  if cli_options.unstable_detect_cjs() {
-    log::warn!(
-      concat!(
-        "{} --unstable-detect-cjs is not properly supported in deno compile. ",
-        "The compiled executable may encounter runtime errors.",
-      ),
-      crate::colors::yellow("Warning"),
-    );
-  }
-
   let output_path = resolve_compile_executable_output_path(
     http_client,
     &compile_flags,
diff --git a/cli/tools/coverage/mod.rs b/cli/tools/coverage/mod.rs
index 48922f144..f59333247 100644
--- a/cli/tools/coverage/mod.rs
+++ b/cli/tools/coverage/mod.rs
@@ -6,12 +6,12 @@ use crate::args::FileFlags;
 use crate::args::Flags;
 use crate::cdp;
 use crate::factory::CliFactory;
-use crate::npm::CliNpmResolver;
 use crate::tools::fmt::format_json;
 use crate::tools::test::is_supported_test_path;
 use crate::util::text_encoding::source_map_from_code;
 
 use deno_ast::MediaType;
+use deno_ast::ModuleKind;
 use deno_ast::ModuleSpecifier;
 use deno_config::glob::FileCollector;
 use deno_config::glob::FilePatterns;
@@ -25,6 +25,7 @@ use deno_core::serde_json;
 use deno_core::sourcemap::SourceMap;
 use deno_core::url::Url;
 use deno_core::LocalInspectorSession;
+use node_resolver::InNpmPackageChecker;
 use regex::Regex;
 use std::fs;
 use std::fs::File;
@@ -461,7 +462,7 @@ fn filter_coverages(
   coverages: Vec<cdp::ScriptCoverage>,
   include: Vec<String>,
   exclude: Vec<String>,
-  npm_resolver: &dyn CliNpmResolver,
+  in_npm_pkg_checker: &dyn InNpmPackageChecker,
 ) -> Vec<cdp::ScriptCoverage> {
   let include: Vec<Regex> =
     include.iter().map(|e| Regex::new(e).unwrap()).collect();
@@ -485,7 +486,7 @@ fn filter_coverages(
         || doc_test_re.is_match(e.url.as_str())
         || Url::parse(&e.url)
           .ok()
-          .map(|url| npm_resolver.in_npm_package(&url))
+          .map(|url| in_npm_pkg_checker.in_npm_package(&url))
           .unwrap_or(false);
 
       let is_included = include.iter().any(|p| p.is_match(&e.url));
@@ -496,7 +497,7 @@ fn filter_coverages(
     .collect::<Vec<cdp::ScriptCoverage>>()
 }
 
-pub async fn cover_files(
+pub fn cover_files(
   flags: Arc<Flags>,
   coverage_flags: CoverageFlags,
 ) -> Result<(), AnyError> {
@@ -506,9 +507,10 @@ pub async fn cover_files(
 
   let factory = CliFactory::from_flags(flags);
   let cli_options = factory.cli_options()?;
-  let npm_resolver = factory.npm_resolver().await?;
+  let in_npm_pkg_checker = factory.in_npm_pkg_checker()?;
   let file_fetcher = factory.file_fetcher()?;
   let emitter = factory.emitter()?;
+  let cjs_tracker = factory.cjs_tracker()?;
 
   assert!(!coverage_flags.files.include.is_empty());
 
@@ -528,7 +530,7 @@ pub async fn cover_files(
     script_coverages,
     coverage_flags.include,
     coverage_flags.exclude,
-    npm_resolver.as_ref(),
+    in_npm_pkg_checker.as_ref(),
   );
   if script_coverages.is_empty() {
     return Err(generic_error("No covered files included in the report"));
@@ -585,6 +587,8 @@ pub async fn cover_files(
     let transpiled_code = match file.media_type {
       MediaType::JavaScript
       | MediaType::Unknown
+      | MediaType::Css
+      | MediaType::Wasm
       | MediaType::Cjs
       | MediaType::Mjs
       | MediaType::Json => None,
@@ -594,7 +598,10 @@ pub async fn cover_files(
       | MediaType::Mts
       | MediaType::Cts
       | MediaType::Tsx => {
-        Some(match emitter.maybe_cached_emit(&file.specifier, &file.source) {
+        let module_kind = ModuleKind::from_is_cjs(
+          cjs_tracker.is_maybe_cjs(&file.specifier, file.media_type)?,
+        );
+        Some(match emitter.maybe_cached_emit(&file.specifier, module_kind, &file.source) {
           Some(code) => code,
           None => {
             return Err(anyhow!(
@@ -605,7 +612,7 @@ pub async fn cover_files(
           }
         })
       }
-      MediaType::Wasm | MediaType::TsBuildInfo | MediaType::SourceMap => {
+      MediaType::SourceMap => {
         unreachable!()
       }
     };
diff --git a/cli/tools/doc.rs b/cli/tools/doc.rs
index 5e18546a2..e33da4efb 100644
--- a/cli/tools/doc.rs
+++ b/cli/tools/doc.rs
@@ -22,9 +22,9 @@ use deno_core::serde_json;
 use deno_doc as doc;
 use deno_doc::html::UrlResolveKind;
 use deno_graph::source::NullFileSystem;
+use deno_graph::EsParser;
 use deno_graph::GraphKind;
 use deno_graph::ModuleAnalyzer;
-use deno_graph::ModuleParser;
 use deno_graph::ModuleSpecifier;
 use doc::html::ShortPath;
 use doc::DocDiagnostic;
@@ -37,7 +37,7 @@ const JSON_SCHEMA_VERSION: u8 = 1;
 
 async fn generate_doc_nodes_for_builtin_types(
   doc_flags: DocFlags,
-  parser: &dyn ModuleParser,
+  parser: &dyn EsParser,
   analyzer: &dyn ModuleAnalyzer,
 ) -> Result<IndexMap<ModuleSpecifier, Vec<doc::DocNode>>, AnyError> {
   let source_file_specifier =
@@ -96,7 +96,7 @@ pub async fn doc(
   let module_info_cache = factory.module_info_cache()?;
   let parsed_source_cache = factory.parsed_source_cache();
   let capturing_parser = parsed_source_cache.as_capturing_parser();
-  let analyzer = module_info_cache.as_module_analyzer(parsed_source_cache);
+  let analyzer = module_info_cache.as_module_analyzer();
 
   let doc_nodes_by_url = match doc_flags.source_files {
     DocSourceFileFlag::Builtin => {
diff --git a/cli/tools/registry/tar.rs b/cli/tools/registry/tar.rs
index aca125e00..6d1801ce6 100644
--- a/cli/tools/registry/tar.rs
+++ b/cli/tools/registry/tar.rs
@@ -120,7 +120,7 @@ fn resolve_content_maybe_unfurling(
         | MediaType::Unknown
         | MediaType::Json
         | MediaType::Wasm
-        | MediaType::TsBuildInfo => {
+        | MediaType::Css => {
           // not unfurlable data
           return Ok(data);
         }
diff --git a/cli/tools/repl/session.rs b/cli/tools/repl/session.rs
index 484664dae..23b0f11ac 100644
--- a/cli/tools/repl/session.rs
+++ b/cli/tools/repl/session.rs
@@ -25,6 +25,7 @@ use deno_ast::swc::visit::noop_visit_type;
 use deno_ast::swc::visit::Visit;
 use deno_ast::swc::visit::VisitWith;
 use deno_ast::ImportsNotUsedAsValues;
+use deno_ast::ModuleKind;
 use deno_ast::ModuleSpecifier;
 use deno_ast::ParseDiagnosticsError;
 use deno_ast::ParsedSource;
@@ -641,6 +642,10 @@ impl ReplSession {
           jsx_fragment_factory: self.jsx.frag_factory.clone(),
           jsx_import_source: self.jsx.import_source.clone(),
           var_decl_imports: true,
+          verbatim_module_syntax: false,
+        },
+        &deno_ast::TranspileModuleOptions {
+          module_kind: Some(ModuleKind::Esm),
         },
         &deno_ast::EmitOptions {
           source_map: deno_ast::SourceMapOption::None,
@@ -651,7 +656,6 @@ impl ReplSession {
         },
       )?
       .into_source()
-      .into_string()?
       .text;
 
     let value = self
diff --git a/cli/tools/run/hmr.rs b/cli/tools/run/hmr.rs
index 6ccf8e344..6cebedd01 100644
--- a/cli/tools/run/hmr.rs
+++ b/cli/tools/run/hmr.rs
@@ -1,9 +1,11 @@
 // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
 
-use crate::cdp;
-use crate::emit::Emitter;
-use crate::util::file_watcher::WatcherCommunicator;
-use crate::util::file_watcher::WatcherRestartMode;
+use std::collections::HashMap;
+use std::path::PathBuf;
+use std::sync::Arc;
+
+use deno_ast::MediaType;
+use deno_ast::ModuleKind;
 use deno_core::error::generic_error;
 use deno_core::error::AnyError;
 use deno_core::futures::StreamExt;
@@ -12,11 +14,14 @@ use deno_core::serde_json::{self};
 use deno_core::url::Url;
 use deno_core::LocalInspectorSession;
 use deno_terminal::colors;
-use std::collections::HashMap;
-use std::path::PathBuf;
-use std::sync::Arc;
 use tokio::select;
 
+use crate::cdp;
+use crate::emit::Emitter;
+use crate::resolver::CjsTracker;
+use crate::util::file_watcher::WatcherCommunicator;
+use crate::util::file_watcher::WatcherRestartMode;
+
 fn explain(status: &cdp::Status) -> &'static str {
   match status {
     cdp::Status::Ok => "OK",
@@ -58,6 +63,7 @@ pub struct HmrRunner {
   session: LocalInspectorSession,
   watcher_communicator: Arc<WatcherCommunicator>,
   script_ids: HashMap<String, String>,
+  cjs_tracker: Arc<CjsTracker>,
   emitter: Arc<Emitter>,
 }
 
@@ -139,7 +145,8 @@ impl crate::worker::HmrRunner for HmrRunner {
             };
 
             let source_code = self.emitter.load_and_emit_for_hmr(
-              &module_url
+              &module_url,
+              ModuleKind::from_is_cjs(self.cjs_tracker.is_maybe_cjs(&module_url, MediaType::from_specifier(&module_url))?),
             ).await?;
 
             let mut tries = 1;
@@ -172,12 +179,14 @@ impl crate::worker::HmrRunner for HmrRunner {
 
 impl HmrRunner {
   pub fn new(
+    cjs_tracker: Arc<CjsTracker>,
     emitter: Arc<Emitter>,
     session: LocalInspectorSession,
     watcher_communicator: Arc<WatcherCommunicator>,
   ) -> Self {
     Self {
       session,
+      cjs_tracker,
       emitter,
       watcher_communicator,
       script_ids: HashMap::new(),
diff --git a/cli/tools/test/mod.rs b/cli/tools/test/mod.rs
index e81abad0b..fa849614f 100644
--- a/cli/tools/test/mod.rs
+++ b/cli/tools/test/mod.rs
@@ -631,7 +631,7 @@ async fn configure_main_worker(
       "Deno[Deno.internal].core.setLeakTracingEnabled(true);",
     )?;
   }
-  let res = worker.execute_side_module_possibly_with_npm().await;
+  let res = worker.execute_side_module().await;
   let mut worker = worker.into_main_worker();
   match res {
     Ok(()) => Ok(()),