fix: improved support for cjs and cts modules (#26558)

* cts support
* better cjs/cts type checking
* deno compile cjs/cts support
* More efficient detect cjs (going towards stabilization)
* Determination of whether .js, .ts, .jsx, or .tsx is cjs or esm is only
done after loading
* Support `import x = require(...);`

Co-authored-by: Bartek Iwańczuk <biwanczuk@gmail.com>

7 files changed, 112 insertions, 83 deletions

diff --git a/cli/npm/byonm.rs b/cli/npm/byonm.rs
index fc095ab16..4535b07fc 100644
--- a/cli/npm/byonm.rs
+++ b/cli/npm/byonm.rs
@@ -10,8 +10,8 @@ use deno_core::serde_json;
 use deno_core::url::Url;
 use deno_resolver::npm::ByonmNpmResolver;
 use deno_resolver::npm::ByonmNpmResolverCreateOptions;
+use deno_runtime::deno_node::DenoFsNodeResolverEnv;
 use deno_runtime::deno_node::NodePermissions;
-use deno_runtime::deno_node::NodeRequireResolver;
 use deno_runtime::ops::process::NpmProcessStateProvider;
 use deno_semver::package::PackageReq;
 use node_resolver::NpmResolver;
@@ -25,30 +25,14 @@ use super::InnerCliNpmResolverRef;
 use super::ResolvePkgFolderFromDenoReqError;
 
 pub type CliByonmNpmResolverCreateOptions =
-  ByonmNpmResolverCreateOptions<CliDenoResolverFs>;
-pub type CliByonmNpmResolver = ByonmNpmResolver<CliDenoResolverFs>;
+  ByonmNpmResolverCreateOptions<CliDenoResolverFs, DenoFsNodeResolverEnv>;
+pub type CliByonmNpmResolver =
+  ByonmNpmResolver<CliDenoResolverFs, DenoFsNodeResolverEnv>;
 
 // todo(dsherret): the services hanging off `CliNpmResolver` doesn't seem ideal. We should probably decouple.
 #[derive(Debug)]
 struct CliByonmWrapper(Arc<CliByonmNpmResolver>);
 
-impl NodeRequireResolver for CliByonmWrapper {
-  fn ensure_read_permission<'a>(
-    &self,
-    permissions: &mut dyn NodePermissions,
-    path: &'a Path,
-  ) -> Result<Cow<'a, Path>, AnyError> {
-    if !path
-      .components()
-      .any(|c| c.as_os_str().to_ascii_lowercase() == "node_modules")
-    {
-      permissions.check_read_path(path)
-    } else {
-      Ok(Cow::Borrowed(path))
-    }
-  }
-}
-
 impl NpmProcessStateProvider for CliByonmWrapper {
   fn get_npm_process_state(&self) -> String {
     serde_json::to_string(&NpmProcessState {
@@ -67,10 +51,6 @@ impl CliNpmResolver for CliByonmNpmResolver {
     self
   }
 
-  fn into_require_resolver(self: Arc<Self>) -> Arc<dyn NodeRequireResolver> {
-    Arc::new(CliByonmWrapper(self))
-  }
-
   fn into_process_state_provider(
     self: Arc<Self>,
   ) -> Arc<dyn NpmProcessStateProvider> {
@@ -100,6 +80,21 @@ impl CliNpmResolver for CliByonmNpmResolver {
     .map_err(ResolvePkgFolderFromDenoReqError::Byonm)
   }
 
+  fn ensure_read_permission<'a>(
+    &self,
+    permissions: &mut dyn NodePermissions,
+    path: &'a Path,
+  ) -> Result<Cow<'a, Path>, AnyError> {
+    if !path
+      .components()
+      .any(|c| c.as_os_str().to_ascii_lowercase() == "node_modules")
+    {
+      permissions.check_read_path(path)
+    } else {
+      Ok(Cow::Borrowed(path))
+    }
+  }
+
   fn check_state_hash(&self) -> Option<u64> {
     // it is very difficult to determine the check state hash for byonm
     // so we just return None to signify check caching is not supported
diff --git a/cli/npm/managed/cache/mod.rs b/cli/npm/managed/cache/mod.rs
index aaec48668..8ae99f41e 100644
--- a/cli/npm/managed/cache/mod.rs
+++ b/cli/npm/managed/cache/mod.rs
@@ -36,7 +36,7 @@ pub use tarball::TarballCache;
 /// Stores a single copy of npm packages in a cache.
 #[derive(Debug)]
 pub struct NpmCache {
-  cache_dir: NpmCacheDir,
+  cache_dir: Arc<NpmCacheDir>,
   cache_setting: CacheSetting,
   npmrc: Arc<ResolvedNpmRc>,
   /// ensures a package is only downloaded once per run
@@ -45,7 +45,7 @@ pub struct NpmCache {
 
 impl NpmCache {
   pub fn new(
-    cache_dir: NpmCacheDir,
+    cache_dir: Arc<NpmCacheDir>,
     cache_setting: CacheSetting,
     npmrc: Arc<ResolvedNpmRc>,
   ) -> Self {
@@ -61,6 +61,10 @@ impl NpmCache {
     &self.cache_setting
   }
 
+  pub fn root_dir_path(&self) -> &Path {
+    self.cache_dir.root_dir()
+  }
+
   pub fn root_dir_url(&self) -> &Url {
     self.cache_dir.root_dir_url()
   }
@@ -152,10 +156,6 @@ impl NpmCache {
     self.cache_dir.package_name_folder(name, registry_url)
   }
 
-  pub fn root_folder(&self) -> PathBuf {
-    self.cache_dir.root_dir().to_owned()
-  }
-
   pub fn resolve_package_folder_id_from_specifier(
     &self,
     specifier: &ModuleSpecifier,
diff --git a/cli/npm/managed/mod.rs b/cli/npm/managed/mod.rs
index a0754812b..4a91bc347 100644
--- a/cli/npm/managed/mod.rs
+++ b/cli/npm/managed/mod.rs
@@ -12,6 +12,7 @@ use deno_cache_dir::npm::NpmCacheDir;
 use deno_core::anyhow::Context;
 use deno_core::error::AnyError;
 use deno_core::serde_json;
+use deno_core::url::Url;
 use deno_npm::npm_rc::ResolvedNpmRc;
 use deno_npm::registry::NpmPackageInfo;
 use deno_npm::registry::NpmRegistryApi;
@@ -24,12 +25,12 @@ use deno_npm::NpmSystemInfo;
 use deno_runtime::colors;
 use deno_runtime::deno_fs::FileSystem;
 use deno_runtime::deno_node::NodePermissions;
-use deno_runtime::deno_node::NodeRequireResolver;
 use deno_runtime::ops::process::NpmProcessStateProvider;
 use deno_semver::package::PackageNv;
 use deno_semver::package::PackageReq;
 use node_resolver::errors::PackageFolderResolveError;
 use node_resolver::errors::PackageFolderResolveIoError;
+use node_resolver::InNpmPackageChecker;
 use node_resolver::NpmResolver;
 use resolution::AddPkgReqsResult;
 
@@ -39,7 +40,6 @@ use crate::args::NpmInstallDepsProvider;
 use crate::args::NpmProcessState;
 use crate::args::NpmProcessStateKind;
 use crate::args::PackageJsonDepValueParseWithLocationError;
-use crate::cache::DenoCacheEnvFsAdapter;
 use crate::cache::FastInsecureHasher;
 use crate::http_util::HttpClientProvider;
 use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs;
@@ -66,12 +66,12 @@ pub enum CliNpmResolverManagedSnapshotOption {
   Specified(Option<ValidSerializedNpmResolutionSnapshot>),
 }
 
-pub struct CliNpmResolverManagedCreateOptions {
+pub struct CliManagedNpmResolverCreateOptions {
   pub snapshot: CliNpmResolverManagedSnapshotOption,
   pub maybe_lockfile: Option<Arc<CliLockfile>>,
   pub fs: Arc<dyn deno_runtime::deno_fs::FileSystem>,
   pub http_client_provider: Arc<crate::http_util::HttpClientProvider>,
-  pub npm_global_cache_dir: PathBuf,
+  pub npm_cache_dir: Arc<NpmCacheDir>,
   pub cache_setting: crate::args::CacheSetting,
   pub text_only_progress_bar: crate::util::progress_bar::ProgressBar,
   pub maybe_node_modules_path: Option<PathBuf>,
@@ -82,7 +82,7 @@ pub struct CliNpmResolverManagedCreateOptions {
 }
 
 pub async fn create_managed_npm_resolver_for_lsp(
-  options: CliNpmResolverManagedCreateOptions,
+  options: CliManagedNpmResolverCreateOptions,
 ) -> Arc<dyn CliNpmResolver> {
   let npm_cache = create_cache(&options);
   let npm_api = create_api(&options, npm_cache.clone());
@@ -115,7 +115,7 @@ pub async fn create_managed_npm_resolver_for_lsp(
 }
 
 pub async fn create_managed_npm_resolver(
-  options: CliNpmResolverManagedCreateOptions,
+  options: CliManagedNpmResolverCreateOptions,
 ) -> Result<Arc<dyn CliNpmResolver>, AnyError> {
   let npm_cache = create_cache(&options);
   let npm_api = create_api(&options, npm_cache.clone());
@@ -189,20 +189,16 @@ fn create_inner(
   ))
 }
 
-fn create_cache(options: &CliNpmResolverManagedCreateOptions) -> Arc<NpmCache> {
+fn create_cache(options: &CliManagedNpmResolverCreateOptions) -> Arc<NpmCache> {
   Arc::new(NpmCache::new(
-    NpmCacheDir::new(
-      &DenoCacheEnvFsAdapter(options.fs.as_ref()),
-      options.npm_global_cache_dir.clone(),
-      options.npmrc.get_all_known_registries_urls(),
-    ),
+    options.npm_cache_dir.clone(),
     options.cache_setting.clone(),
     options.npmrc.clone(),
   ))
 }
 
 fn create_api(
-  options: &CliNpmResolverManagedCreateOptions,
+  options: &CliManagedNpmResolverCreateOptions,
   npm_cache: Arc<NpmCache>,
 ) -> Arc<CliNpmRegistryApi> {
   Arc::new(CliNpmRegistryApi::new(
@@ -259,6 +255,35 @@ async fn snapshot_from_lockfile(
   Ok(snapshot)
 }
 
+#[derive(Debug)]
+struct ManagedInNpmPackageChecker {
+  root_dir: Url,
+}
+
+impl InNpmPackageChecker for ManagedInNpmPackageChecker {
+  fn in_npm_package(&self, specifier: &Url) -> bool {
+    specifier.as_ref().starts_with(self.root_dir.as_str())
+  }
+}
+
+pub struct CliManagedInNpmPkgCheckerCreateOptions<'a> {
+  pub root_cache_dir_url: &'a Url,
+  pub maybe_node_modules_path: Option<&'a Path>,
+}
+
+pub fn create_managed_in_npm_pkg_checker(
+  options: CliManagedInNpmPkgCheckerCreateOptions,
+) -> Arc<dyn InNpmPackageChecker> {
+  let root_dir = match options.maybe_node_modules_path {
+    Some(node_modules_folder) => {
+      deno_path_util::url_from_directory_path(node_modules_folder).unwrap()
+    }
+    None => options.root_cache_dir_url.clone(),
+  };
+  debug_assert!(root_dir.as_str().ends_with('/'));
+  Arc::new(ManagedInNpmPackageChecker { root_dir })
+}
+
 /// An npm resolver where the resolution is managed by Deno rather than
 /// the user bringing their own node_modules (BYONM) on the file system.
 pub struct ManagedCliNpmResolver {
@@ -555,8 +580,16 @@ impl ManagedCliNpmResolver {
       .map_err(|err| err.into())
   }
 
-  pub fn global_cache_root_folder(&self) -> PathBuf {
-    self.npm_cache.root_folder()
+  pub fn maybe_node_modules_path(&self) -> Option<&Path> {
+    self.fs_resolver.node_modules_path()
+  }
+
+  pub fn global_cache_root_path(&self) -> &Path {
+    self.npm_cache.root_dir_path()
+  }
+
+  pub fn global_cache_root_url(&self) -> &Url {
+    self.npm_cache.root_dir_url()
   }
 }
 
@@ -591,22 +624,6 @@ impl NpmResolver for ManagedCliNpmResolver {
     log::debug!("Resolved {} from {} to {}", name, referrer, path.display());
     Ok(path)
   }
-
-  fn in_npm_package(&self, specifier: &ModuleSpecifier) -> bool {
-    let root_dir_url = self.fs_resolver.root_dir_url();
-    debug_assert!(root_dir_url.as_str().ends_with('/'));
-    specifier.as_ref().starts_with(root_dir_url.as_str())
-  }
-}
-
-impl NodeRequireResolver for ManagedCliNpmResolver {
-  fn ensure_read_permission<'a>(
-    &self,
-    permissions: &mut dyn NodePermissions,
-    path: &'a Path,
-  ) -> Result<Cow<'a, Path>, AnyError> {
-    self.fs_resolver.ensure_read_permission(permissions, path)
-  }
 }
 
 impl NpmProcessStateProvider for ManagedCliNpmResolver {
@@ -623,10 +640,6 @@ impl CliNpmResolver for ManagedCliNpmResolver {
     self
   }
 
-  fn into_require_resolver(self: Arc<Self>) -> Arc<dyn NodeRequireResolver> {
-    self
-  }
-
   fn into_process_state_provider(
     self: Arc<Self>,
   ) -> Arc<dyn NpmProcessStateProvider> {
@@ -687,6 +700,14 @@ impl CliNpmResolver for ManagedCliNpmResolver {
       .map_err(ResolvePkgFolderFromDenoReqError::Managed)
   }
 
+  fn ensure_read_permission<'a>(
+    &self,
+    permissions: &mut dyn NodePermissions,
+    path: &'a Path,
+  ) -> Result<Cow<'a, Path>, AnyError> {
+    self.fs_resolver.ensure_read_permission(permissions, path)
+  }
+
   fn check_state_hash(&self) -> Option<u64> {
     // We could go further and check all the individual
     // npm packages, but that's probably overkill.
diff --git a/cli/npm/managed/resolvers/common.rs b/cli/npm/managed/resolvers/common.rs
index 867bb4168..590f8fb25 100644
--- a/cli/npm/managed/resolvers/common.rs
+++ b/cli/npm/managed/resolvers/common.rs
@@ -17,7 +17,6 @@ use deno_core::anyhow::Context;
 use deno_core::error::AnyError;
 use deno_core::futures;
 use deno_core::futures::StreamExt;
-use deno_core::url::Url;
 use deno_npm::NpmPackageCacheFolderId;
 use deno_npm::NpmPackageId;
 use deno_npm::NpmResolutionPackage;
@@ -30,9 +29,6 @@ use crate::npm::managed::cache::TarballCache;
 /// Part of the resolution that interacts with the file system.
 #[async_trait(?Send)]
 pub trait NpmPackageFsResolver: Send + Sync {
-  /// Specifier for the root directory.
-  fn root_dir_url(&self) -> &Url;
-
   /// The local node_modules folder if it is applicable to the implementation.
   fn node_modules_path(&self) -> Option<&Path>;
 
diff --git a/cli/npm/managed/resolvers/global.rs b/cli/npm/managed/resolvers/global.rs
index 5be315e99..f0193e78e 100644
--- a/cli/npm/managed/resolvers/global.rs
+++ b/cli/npm/managed/resolvers/global.rs
@@ -11,7 +11,6 @@ use crate::colors;
 use async_trait::async_trait;
 use deno_ast::ModuleSpecifier;
 use deno_core::error::AnyError;
-use deno_core::url::Url;
 use deno_npm::NpmPackageCacheFolderId;
 use deno_npm::NpmPackageId;
 use deno_npm::NpmResolutionPackage;
@@ -56,7 +55,7 @@ impl GlobalNpmPackageResolver {
     Self {
       registry_read_permission_checker: RegistryReadPermissionChecker::new(
         fs,
-        cache.root_folder(),
+        cache.root_dir_path().to_path_buf(),
       ),
       cache,
       tarball_cache,
@@ -69,10 +68,6 @@ impl GlobalNpmPackageResolver {
 
 #[async_trait(?Send)]
 impl NpmPackageFsResolver for GlobalNpmPackageResolver {
-  fn root_dir_url(&self) -> &Url {
-    self.cache.root_dir_url()
-  }
-
   fn node_modules_path(&self) -> Option<&Path> {
     None
   }
diff --git a/cli/npm/managed/resolvers/local.rs b/cli/npm/managed/resolvers/local.rs
index 54f7576ad..0968be8a7 100644
--- a/cli/npm/managed/resolvers/local.rs
+++ b/cli/npm/managed/resolvers/local.rs
@@ -155,10 +155,6 @@ impl LocalNpmPackageResolver {
 
 #[async_trait(?Send)]
 impl NpmPackageFsResolver for LocalNpmPackageResolver {
-  fn root_dir_url(&self) -> &Url {
-    &self.root_node_modules_url
-  }
-
   fn node_modules_path(&self) -> Option<&Path> {
     Some(self.root_node_modules_path.as_ref())
   }
diff --git a/cli/npm/mod.rs b/cli/npm/mod.rs
index 3971b1a0b..0d434ca27 100644
--- a/cli/npm/mod.rs
+++ b/cli/npm/mod.rs
@@ -4,6 +4,7 @@ mod byonm;
 mod common;
 mod managed;
 
+use std::borrow::Cow;
 use std::path::Path;
 use std::path::PathBuf;
 use std::sync::Arc;
@@ -15,13 +16,16 @@ use deno_core::error::AnyError;
 use deno_core::serde_json;
 use deno_npm::npm_rc::ResolvedNpmRc;
 use deno_npm::registry::NpmPackageInfo;
+use deno_resolver::npm::ByonmInNpmPackageChecker;
 use deno_resolver::npm::ByonmNpmResolver;
 use deno_resolver::npm::ByonmResolvePkgFolderFromDenoReqError;
-use deno_runtime::deno_node::NodeRequireResolver;
+use deno_runtime::deno_node::NodePermissions;
 use deno_runtime::ops::process::NpmProcessStateProvider;
 use deno_semver::package::PackageNv;
 use deno_semver::package::PackageReq;
 use managed::cache::registry_info::get_package_url;
+use managed::create_managed_in_npm_pkg_checker;
+use node_resolver::InNpmPackageChecker;
 use node_resolver::NpmResolver;
 use thiserror::Error;
 
@@ -29,7 +33,8 @@ use crate::file_fetcher::FileFetcher;
 
 pub use self::byonm::CliByonmNpmResolver;
 pub use self::byonm::CliByonmNpmResolverCreateOptions;
-pub use self::managed::CliNpmResolverManagedCreateOptions;
+pub use self::managed::CliManagedInNpmPkgCheckerCreateOptions;
+pub use self::managed::CliManagedNpmResolverCreateOptions;
 pub use self::managed::CliNpmResolverManagedSnapshotOption;
 pub use self::managed::ManagedCliNpmResolver;
 
@@ -42,7 +47,7 @@ pub enum ResolvePkgFolderFromDenoReqError {
 }
 
 pub enum CliNpmResolverCreateOptions {
-  Managed(CliNpmResolverManagedCreateOptions),
+  Managed(CliManagedNpmResolverCreateOptions),
   Byonm(CliByonmNpmResolverCreateOptions),
 }
 
@@ -68,6 +73,22 @@ pub async fn create_cli_npm_resolver(
   }
 }
 
+pub enum CreateInNpmPkgCheckerOptions<'a> {
+  Managed(CliManagedInNpmPkgCheckerCreateOptions<'a>),
+  Byonm,
+}
+
+pub fn create_in_npm_pkg_checker(
+  options: CreateInNpmPkgCheckerOptions,
+) -> Arc<dyn InNpmPackageChecker> {
+  match options {
+    CreateInNpmPkgCheckerOptions::Managed(options) => {
+      create_managed_in_npm_pkg_checker(options)
+    }
+    CreateInNpmPkgCheckerOptions::Byonm => Arc::new(ByonmInNpmPackageChecker),
+  }
+}
+
 pub enum InnerCliNpmResolverRef<'a> {
   Managed(&'a ManagedCliNpmResolver),
   #[allow(dead_code)]
@@ -76,7 +97,6 @@ pub enum InnerCliNpmResolverRef<'a> {
 
 pub trait CliNpmResolver: NpmResolver {
   fn into_npm_resolver(self: Arc<Self>) -> Arc<dyn NpmResolver>;
-  fn into_require_resolver(self: Arc<Self>) -> Arc<dyn NodeRequireResolver>;
   fn into_process_state_provider(
     self: Arc<Self>,
   ) -> Arc<dyn NpmProcessStateProvider>;
@@ -107,6 +127,12 @@ pub trait CliNpmResolver: NpmResolver {
     referrer: &ModuleSpecifier,
   ) -> Result<PathBuf, ResolvePkgFolderFromDenoReqError>;
 
+  fn ensure_read_permission<'a>(
+    &self,
+    permissions: &mut dyn NodePermissions,
+    path: &'a Path,
+  ) -> Result<Cow<'a, Path>, AnyError>;
+
   /// Returns a hash returning the state of the npm resolver
   /// or `None` if the state currently can't be determined.
   fn check_state_hash(&self) -> Option<u64>;