diff options
author | David Sherret <dsherret@users.noreply.github.com> | 2024-09-30 09:33:32 -0400 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-09-30 13:33:32 +0000 |
commit | 69ab72002550b5797185b7651de28c700b220bb2 (patch) | |
tree | ce224c0631f42e3ad73e6bd848d1a597d19f1a9f | |
parent | c8f692057b256dac57342867b7606a74309449fc (diff) |
refactor: move ByonmNpmResolver to deno_resolver (#25937)
Some more slow progress on moving all the resolution code into
deno_resolver.
30 files changed, 629 insertions, 555 deletions
diff --git a/.dprint.json b/.dprint.json index 6ee3db3c5..082eb3005 100644 --- a/.dprint.json +++ b/.dprint.json @@ -71,7 +71,7 @@ "https://plugins.dprint.dev/typescript-0.93.0.wasm", "https://plugins.dprint.dev/json-0.19.3.wasm", "https://plugins.dprint.dev/markdown-0.17.8.wasm", - "https://plugins.dprint.dev/toml-0.6.2.wasm", + "https://plugins.dprint.dev/toml-0.6.3.wasm", "https://plugins.dprint.dev/exec-0.5.0.json@8d9972eee71fa1590e04873540421f3eda7674d0f1aae3d7c788615e7b7413d0", "https://plugins.dprint.dev/g-plane/pretty_yaml-v0.5.0.wasm" ] diff --git a/Cargo.lock b/Cargo.lock index b4dafacdd..2263351fc 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1923,9 +1923,9 @@ dependencies = [ [[package]] name = "deno_path_util" -version = "0.1.1" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "625b490bc4734ff778926d7034650a2ddcef2fd42b71e30bb722249156d5a144" +checksum = "4889646c1ce8437a6fde3acb057fd7e2d039e62c61f5063fc125ed1ede114dc6" dependencies = [ "percent-encoding", "thiserror", @@ -1953,8 +1953,13 @@ dependencies = [ name = "deno_resolver" version = "0.0.1" dependencies = [ + "anyhow", + "base32", "deno_media_type", + "deno_package_json", "deno_path_util", + "deno_semver", + "node_resolver", "test_server", "url", ] @@ -4557,6 +4562,7 @@ dependencies = [ "async-trait", "deno_media_type", "deno_package_json", + "deno_path_util", "futures", "lazy-regex", "once_cell", diff --git a/Cargo.toml b/Cargo.toml index 2f53c8999..ef9a4576c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -52,7 +52,7 @@ deno_bench_util = { version = "0.162.0", path = "./bench_util" } deno_lockfile = "=0.23.1" deno_media_type = { version = "0.1.4", features = ["module_specifier"] } deno_npm = "=0.25.2" -deno_path_util = "=0.1.1" +deno_path_util = "=0.2.0" deno_permissions = { version = "0.28.0", path = "./runtime/permissions" } deno_runtime = { version = "0.177.0", path = "./runtime" } deno_semver = "=0.5.13" diff --git a/cli/factory.rs b/cli/factory.rs index ffe7f8d2f..2cef87599 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -32,12 +32,13 @@ use crate::module_loader::ModuleLoadPreparer; use crate::node::CliCjsCodeAnalyzer; use crate::node::CliNodeCodeTranslator; use crate::npm::create_cli_npm_resolver; +use crate::npm::CliByonmNpmResolverCreateOptions; use crate::npm::CliNpmResolver; -use crate::npm::CliNpmResolverByonmCreateOptions; use crate::npm::CliNpmResolverCreateOptions; use crate::npm::CliNpmResolverManagedCreateOptions; use crate::npm::CliNpmResolverManagedSnapshotOption; use crate::resolver::CjsResolutionStore; +use crate::resolver::CliDenoResolverFs; use crate::resolver::CliGraphResolver; use crate::resolver::CliGraphResolverOptions; use crate::resolver::CliNodeResolver; @@ -361,8 +362,8 @@ impl CliFactory { let cli_options = self.cli_options()?; // For `deno install` we want to force the managed resolver so it can set up `node_modules/` directory. create_cli_npm_resolver(if cli_options.use_byonm() && !matches!(cli_options.sub_command(), DenoSubcommand::Install(_) | DenoSubcommand::Add(_) | DenoSubcommand::Remove(_)) { - CliNpmResolverCreateOptions::Byonm(CliNpmResolverByonmCreateOptions { - fs: fs.clone(), + CliNpmResolverCreateOptions::Byonm(CliByonmNpmResolverCreateOptions { + fs: CliDenoResolverFs(fs.clone()), root_node_modules_dir: Some(match cli_options.node_modules_dir_path() { Some(node_modules_path) => node_modules_path.to_path_buf(), // path needs to be canonicalized for node resolution diff --git a/cli/lsp/resolver.rs b/cli/lsp/resolver.rs index f98b23a3f..a5dfd25a6 100644 --- a/cli/lsp/resolver.rs +++ b/cli/lsp/resolver.rs @@ -42,13 +42,14 @@ use crate::lsp::config::Config; use crate::lsp::config::ConfigData; use crate::lsp::logging::lsp_warn; use crate::npm::create_cli_npm_resolver_for_lsp; +use crate::npm::CliByonmNpmResolverCreateOptions; use crate::npm::CliNpmResolver; -use crate::npm::CliNpmResolverByonmCreateOptions; use crate::npm::CliNpmResolverCreateOptions; use crate::npm::CliNpmResolverManagedCreateOptions; use crate::npm::CliNpmResolverManagedSnapshotOption; use crate::npm::ManagedCliNpmResolver; use crate::resolver::CjsResolutionStore; +use crate::resolver::CliDenoResolverFs; use crate::resolver::CliGraphResolver; use crate::resolver::CliGraphResolverOptions; use crate::resolver::CliNodeResolver; @@ -439,8 +440,8 @@ async fn create_npm_resolver( ) -> Option<Arc<dyn CliNpmResolver>> { let enable_byonm = config_data.map(|d| d.byonm).unwrap_or(false); let options = if enable_byonm { - CliNpmResolverCreateOptions::Byonm(CliNpmResolverByonmCreateOptions { - fs: Arc::new(deno_fs::RealFs), + CliNpmResolverCreateOptions::Byonm(CliByonmNpmResolverCreateOptions { + fs: CliDenoResolverFs(Arc::new(deno_fs::RealFs)), root_node_modules_dir: config_data.and_then(|config_data| { config_data.node_modules_dir.clone().or_else(|| { url_to_file_path(&config_data.scope) diff --git a/cli/npm/byonm.rs b/cli/npm/byonm.rs index 83c406765..ceef68135 100644 --- a/cli/npm/byonm.rs +++ b/cli/npm/byonm.rs @@ -1,276 +1,36 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. -use std::borrow::Cow; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; -use deno_ast::ModuleSpecifier; -use deno_core::anyhow::bail; use deno_core::error::AnyError; use deno_core::serde_json; -use deno_package_json::PackageJsonDepValue; -use deno_path_util::url_to_file_path; -use deno_runtime::deno_fs::FileSystem; -use deno_runtime::deno_node::DenoPkgJsonFsAdapter; +use deno_core::url::Url; +use deno_resolver::npm::ByonmNpmResolver; +use deno_resolver::npm::ByonmNpmResolverCreateOptions; use deno_runtime::deno_node::NodePermissions; use deno_runtime::deno_node::NodeRequireResolver; -use deno_runtime::deno_node::PackageJson; use deno_runtime::ops::process::NpmProcessStateProvider; use deno_semver::package::PackageReq; -use deno_semver::Version; -use node_resolver::errors::PackageFolderResolveError; -use node_resolver::errors::PackageFolderResolveIoError; -use node_resolver::errors::PackageJsonLoadError; -use node_resolver::errors::PackageNotFoundError; -use node_resolver::load_pkg_json; use node_resolver::NpmResolver; use crate::args::NpmProcessState; use crate::args::NpmProcessStateKind; -use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs; +use crate::resolver::CliDenoResolverFs; -use super::managed::normalize_pkg_name_for_node_modules_deno_folder; use super::CliNpmResolver; use super::InnerCliNpmResolverRef; -pub struct CliNpmResolverByonmCreateOptions { - pub fs: Arc<dyn FileSystem>, - // todo(dsherret): investigate removing this - pub root_node_modules_dir: Option<PathBuf>, -} - -pub fn create_byonm_npm_resolver( - options: CliNpmResolverByonmCreateOptions, -) -> Arc<dyn CliNpmResolver> { - Arc::new(ByonmCliNpmResolver { - fs: options.fs, - root_node_modules_dir: options.root_node_modules_dir, - }) -} +pub type CliByonmNpmResolverCreateOptions = + ByonmNpmResolverCreateOptions<CliDenoResolverFs>; +pub type CliByonmNpmResolver = ByonmNpmResolver<CliDenoResolverFs>; +// todo(dsherret): the services hanging off `CliNpmResolver` doesn't seem ideal. We should probably decouple. #[derive(Debug)] -pub struct ByonmCliNpmResolver { - fs: Arc<dyn FileSystem>, - root_node_modules_dir: Option<PathBuf>, -} - -impl ByonmCliNpmResolver { - fn load_pkg_json( - &self, - path: &Path, - ) -> Result<Option<Arc<PackageJson>>, PackageJsonLoadError> { - load_pkg_json(&DenoPkgJsonFsAdapter(self.fs.as_ref()), path) - } - - /// Finds the ancestor package.json that contains the specified dependency. - pub fn find_ancestor_package_json_with_dep( - &self, - dep_name: &str, - referrer: &ModuleSpecifier, - ) -> Option<Arc<PackageJson>> { - let referrer_path = referrer.to_file_path().ok()?; - let mut current_folder = referrer_path.parent()?; - loop { - let pkg_json_path = current_folder.join("package.json"); - if let Ok(Some(pkg_json)) = self.load_pkg_json(&pkg_json_path) { - if let Some(deps) = &pkg_json.dependencies { - if deps.contains_key(dep_name) { - return Some(pkg_json); - } - } - if let Some(deps) = &pkg_json.dev_dependencies { - if deps.contains_key(dep_name) { - return Some(pkg_json); - } - } - } - - if let Some(parent) = current_folder.parent() { - current_folder = parent; - } else { - return None; - } - } - } - - fn resolve_pkg_json_and_alias_for_req( - &self, - req: &PackageReq, - referrer: &ModuleSpecifier, - ) -> Result<Option<(Arc<PackageJson>, String)>, AnyError> { - fn resolve_alias_from_pkg_json( - req: &PackageReq, - pkg_json: &PackageJson, - ) -> Option<String> { - let deps = pkg_json.resolve_local_package_json_deps(); - for (key, value) in deps { - if let Ok(value) = value { - match value { - PackageJsonDepValue::Req(dep_req) => { - if dep_req.name == req.name - && dep_req.version_req.intersects(&req.version_req) - { - return Some(key); - } - } - PackageJsonDepValue::Workspace(_workspace) => { - if key == req.name && req.version_req.tag() == Some("workspace") { - return Some(key); - } - } - } - } - } - None - } - - // attempt to resolve the npm specifier from the referrer's package.json, - if let Ok(file_path) = url_to_file_path(referrer) { - let mut current_path = file_path.as_path(); - while let Some(dir_path) = current_path.parent() { - let package_json_path = dir_path.join("package.json"); - if let Some(pkg_json) = self.load_pkg_json(&package_json_path)? { - if let Some(alias) = - resolve_alias_from_pkg_json(req, pkg_json.as_ref()) - { - return Ok(Some((pkg_json, alias))); - } - } - current_path = dir_path; - } - } - - // otherwise, fall fallback to the project's package.json - if let Some(root_node_modules_dir) = &self.root_node_modules_dir { - let root_pkg_json_path = - root_node_modules_dir.parent().unwrap().join("package.json"); - if let Some(pkg_json) = self.load_pkg_json(&root_pkg_json_path)? { - if let Some(alias) = resolve_alias_from_pkg_json(req, pkg_json.as_ref()) - { - return Ok(Some((pkg_json, alias))); - } - } - } - - Ok(None) - } - - fn resolve_folder_in_root_node_modules( - &self, - req: &PackageReq, - ) -> Option<PathBuf> { - // now check if node_modules/.deno/ matches this constraint - let root_node_modules_dir = self.root_node_modules_dir.as_ref()?; - let node_modules_deno_dir = root_node_modules_dir.join(".deno"); - let Ok(entries) = self.fs.read_dir_sync(&node_modules_deno_dir) else { - return None; - }; - let search_prefix = format!( - "{}@", - normalize_pkg_name_for_node_modules_deno_folder(&req.name) - ); - let mut best_version = None; - - // example entries: - // - @denotest+add@1.0.0 - // - @denotest+add@1.0.0_1 - for entry in entries { - if !entry.is_directory { - continue; - } - let Some(version_and_copy_idx) = entry.name.strip_prefix(&search_prefix) - else { - continue; - }; - let version = version_and_copy_idx - .rsplit_once('_') - .map(|(v, _)| v) - .unwrap_or(version_and_copy_idx); - let Ok(version) = Version::parse_from_npm(version) else { - continue; - }; - if req.version_req.matches(&version) { - if let Some((best_version_version, _)) = &best_version { - if version > *best_version_version { - best_version = Some((version, entry.name)); - } - } else { - best_version = Some((version, entry.name)); - } - } - } - - best_version.map(|(_version, entry_name)| { - join_package_name( - &node_modules_deno_dir.join(entry_name).join("node_modules"), - &req.name, - ) - }) - } -} - -impl NpmResolver for ByonmCliNpmResolver { - fn resolve_package_folder_from_package( - &self, - name: &str, - referrer: &ModuleSpecifier, - ) -> Result<PathBuf, PackageFolderResolveError> { - fn inner( - fs: &dyn FileSystem, - name: &str, - referrer: &ModuleSpecifier, - ) -> Result<PathBuf, PackageFolderResolveError> { - let maybe_referrer_file = url_to_file_path(referrer).ok(); - let maybe_start_folder = - maybe_referrer_file.as_ref().and_then(|f| f.parent()); - if let Some(start_folder) = maybe_start_folder { - for current_folder in start_folder.ancestors() { - let node_modules_folder = if current_folder.ends_with("node_modules") - { - Cow::Borrowed(current_folder) - } else { - Cow::Owned(current_folder.join("node_modules")) - }; +struct CliByonmWrapper(Arc<CliByonmNpmResolver>); - let sub_dir = join_package_name(&node_modules_folder, name); - if fs.is_dir_sync(&sub_dir) { - return Ok(sub_dir); - } - } - } - - Err( - PackageNotFoundError { - package_name: name.to_string(), - referrer: referrer.clone(), - referrer_extra: None, - } - .into(), - ) - } - - let path = inner(&*self.fs, name, referrer)?; - self.fs.realpath_sync(&path).map_err(|err| { - PackageFolderResolveIoError { - package_name: name.to_string(), - referrer: referrer.clone(), - source: err.into_io_error(), - } - .into() - }) - } - - fn in_npm_package(&self, specifier: &ModuleSpecifier) -> bool { - specifier.scheme() == "file" - && specifier - .path() - .to_ascii_lowercase() - .contains("/node_modules/") - } -} - -impl NodeRequireResolver for ByonmCliNpmResolver { +impl NodeRequireResolver for CliByonmWrapper { fn ensure_read_permission( &self, permissions: &mut dyn NodePermissions, @@ -286,110 +46,54 @@ impl NodeRequireResolver for ByonmCliNpmResolver { } } -impl NpmProcessStateProvider for ByonmCliNpmResolver { +impl NpmProcessStateProvider for CliByonmWrapper { fn get_npm_process_state(&self) -> String { serde_json::to_string(&NpmProcessState { kind: NpmProcessStateKind::Byonm, local_node_modules_path: self - .root_node_modules_dir - .as_ref() + .0 + .root_node_modules_dir() .map(|p| p.to_string_lossy().to_string()), }) .unwrap() } } -impl CliNpmResolver for ByonmCliNpmResolver { +impl CliNpmResolver for CliByonmNpmResolver { fn into_npm_resolver(self: Arc<Self>) -> Arc<dyn NpmResolver> { self } fn into_require_resolver(self: Arc<Self>) -> Arc<dyn NodeRequireResolver> { - self + Arc::new(CliByonmWrapper(self)) } fn into_process_state_provider( self: Arc<Self>, ) -> Arc<dyn NpmProcessStateProvider> { - self + Arc::new(CliByonmWrapper(self)) } fn clone_snapshotted(&self) -> Arc<dyn CliNpmResolver> { - Arc::new(Self { - fs: self.fs.clone(), - root_node_modules_dir: self.root_node_modules_dir.clone(), - }) + Arc::new(self.clone()) } fn as_inner(&self) -> InnerCliNpmResolverRef { InnerCliNpmResolverRef::Byonm(self) } - fn root_node_modules_path(&self) -> Option<&PathBuf> { - self.root_node_modules_dir.as_ref() + fn root_node_modules_path(&self) -> Option<&Path> { + self.root_node_modules_dir() } fn resolve_pkg_folder_from_deno_module_req( &self, req: &PackageReq, - referrer: &ModuleSpecifier, + referrer: &Url, ) -> Result<PathBuf, AnyError> { - fn node_resolve_dir( - fs: &dyn FileSystem, - alias: &str, - start_dir: &Path, - ) -> Result<Option<PathBuf>, AnyError> { - for ancestor in start_dir.ancestors() { - let node_modules_folder = ancestor.join("node_modules"); - let sub_dir = join_package_name(&node_modules_folder, alias); - if fs.is_dir_sync(&sub_dir) { - return Ok(Some(canonicalize_path_maybe_not_exists_with_fs( - &sub_dir, fs, - )?)); - } - } - Ok(None) - } - - // now attempt to resolve if it's found in any package.json - let maybe_pkg_json_and_alias = - self.resolve_pkg_json_and_alias_for_req(req, referrer)?; - match maybe_pkg_json_and_alias { - Some((pkg_json, alias)) => { - // now try node resolution - if let Some(resolved) = - node_resolve_dir(self.fs.as_ref(), &alias, pkg_json.dir_path())? - { - return Ok(resolved); - } - - bail!( - concat!( - "Could not find \"{}\" in a node_modules folder. ", - "Deno expects the node_modules/ directory to be up to date. ", - "Did you forget to run `deno install`?" - ), - alias, - ); - } - None => { - // now check if node_modules/.deno/ matches this constraint - if let Some(folder) = self.resolve_folder_in_root_node_modules(req) { - return Ok(folder); - } - - bail!( - concat!( - "Could not find a matching package for 'npm:{}' in the node_modules ", - "directory. Ensure you have all your JSR and npm dependencies listed ", - "in your deno.json or package.json, then run `deno install`. Alternatively, ", - r#"turn on auto-install by specifying `"nodeModulesDir": "auto"` in your "#, - "deno.json file." - ), - req, - ); - } - } + ByonmNpmResolver::resolve_pkg_folder_from_deno_module_req( + self, req, referrer, + ) } fn check_state_hash(&self) -> Option<u64> { @@ -398,12 +102,3 @@ impl CliNpmResolver for ByonmCliNpmResolver { None } } - -fn join_package_name(path: &Path, package_name: &str) -> PathBuf { - let mut path = path.to_path_buf(); - // ensure backslashes are used on windows - for part in package_name.split('/') { - path = path.join(part); - } - path -} diff --git a/cli/npm/managed/mod.rs b/cli/npm/managed/mod.rs index e3ac5e1af..62af3e4aa 100644 --- a/cli/npm/managed/mod.rs +++ b/cli/npm/managed/mod.rs @@ -47,7 +47,6 @@ use self::cache::NpmCache; use self::registry::CliNpmRegistryApi; use self::resolution::NpmResolution; use self::resolvers::create_npm_fs_resolver; -pub use self::resolvers::normalize_pkg_name_for_node_modules_deno_folder; use self::resolvers::NpmPackageFsResolver; use super::CliNpmResolver; @@ -575,7 +574,7 @@ impl NpmProcessStateProvider for ManagedCliNpmResolver { fn get_npm_process_state(&self) -> String { npm_process_state( self.resolution.serialized_valid_snapshot(), - self.fs_resolver.node_modules_path().map(|p| p.as_path()), + self.fs_resolver.node_modules_path(), ) } } @@ -632,7 +631,7 @@ impl CliNpmResolver for ManagedCliNpmResolver { InnerCliNpmResolverRef::Managed(self) } - fn root_node_modules_path(&self) -> Option<&PathBuf> { + fn root_node_modules_path(&self) -> Option<&Path> { self.fs_resolver.node_modules_path() } diff --git a/cli/npm/managed/resolvers/common.rs b/cli/npm/managed/resolvers/common.rs index 620daf4b3..8df4debc5 100644 --- a/cli/npm/managed/resolvers/common.rs +++ b/cli/npm/managed/resolvers/common.rs @@ -33,7 +33,7 @@ pub trait NpmPackageFsResolver: Send + Sync { fn root_dir_url(&self) -> &Url; /// The local node_modules folder if it is applicable to the implementation. - fn node_modules_path(&self) -> Option<&PathBuf>; + fn node_modules_path(&self) -> Option<&Path>; fn maybe_package_folder(&self, package_id: &NpmPackageId) -> Option<PathBuf>; diff --git a/cli/npm/managed/resolvers/global.rs b/cli/npm/managed/resolvers/global.rs index 187e6b277..ca5722c86 100644 --- a/cli/npm/managed/resolvers/global.rs +++ b/cli/npm/managed/resolvers/global.rs @@ -73,7 +73,7 @@ impl NpmPackageFsResolver for GlobalNpmPackageResolver { self.cache.root_dir_url() } - fn node_modules_path(&self) -> Option<&PathBuf> { + fn node_modules_path(&self) -> Option<&Path> { None } diff --git a/cli/npm/managed/resolvers/local.rs b/cli/npm/managed/resolvers/local.rs index 297fcab23..59ba27d05 100644 --- a/cli/npm/managed/resolvers/local.rs +++ b/cli/npm/managed/resolvers/local.rs @@ -20,7 +20,6 @@ use crate::colors; use async_trait::async_trait; use deno_ast::ModuleSpecifier; use deno_cache_dir::npm::mixed_case_package_name_decode; -use deno_cache_dir::npm::mixed_case_package_name_encode; use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::futures::stream::FuturesUnordered; @@ -32,6 +31,7 @@ use deno_npm::NpmPackageCacheFolderId; use deno_npm::NpmPackageId; use deno_npm::NpmResolutionPackage; use deno_npm::NpmSystemInfo; +use deno_resolver::npm::normalize_pkg_name_for_node_modules_deno_folder; use deno_runtime::deno_fs; use deno_runtime::deno_node::NodePermissions; use deno_semver::package::PackageNv; @@ -159,8 +159,8 @@ impl NpmPackageFsResolver for LocalNpmPackageResolver { &self.root_node_modules_url } - fn node_modules_path(&self) -> Option<&PathBuf> { - Some(&self.root_node_modules_path) + fn node_modules_path(&self) -> Option<&Path> { + Some(self.root_node_modules_path.as_ref()) } fn maybe_package_folder(&self, id: &NpmPackageId) -> Option<PathBuf> { @@ -920,20 +920,6 @@ impl SetupCache { } } -/// Normalizes a package name for use at `node_modules/.deno/<pkg-name>@<version>[_<copy_index>]` -pub fn normalize_pkg_name_for_node_modules_deno_folder(name: &str) -> Cow<str> { - let name = if name.to_lowercase() == name { - Cow::Borrowed(name) - } else { - Cow::Owned(format!("_{}", mixed_case_package_name_encode(name))) - }; - if name.starts_with('@') { - name.replace('/', "+").into() - } else { - name - } -} - fn get_package_folder_id_folder_name( folder_id: &NpmPackageCacheFolderId, ) -> String { diff --git a/cli/npm/managed/resolvers/mod.rs b/cli/npm/managed/resolvers/mod.rs index 234a6e4db..36d795ee7 100644 --- a/cli/npm/managed/resolvers/mod.rs +++ b/cli/npm/managed/resolvers/mod.rs @@ -15,7 +15,6 @@ use crate::args::NpmInstallDepsProvider; use crate::util::progress_bar::ProgressBar; pub use self::common::NpmPackageFsResolver; -pub use self::local::normalize_pkg_name_for_node_modules_deno_folder; use self::global::GlobalNpmPackageResolver; use self::local::LocalNpmPackageResolver; diff --git a/cli/npm/mod.rs b/cli/npm/mod.rs index 2c9ee20bc..1e3c752ae 100644 --- a/cli/npm/mod.rs +++ b/cli/npm/mod.rs @@ -4,6 +4,7 @@ mod byonm; mod common; mod managed; +use std::path::Path; use std::path::PathBuf; use std::sync::Arc; @@ -12,6 +13,7 @@ use deno_ast::ModuleSpecifier; use deno_core::error::AnyError; use deno_core::serde_json; use deno_npm::registry::NpmPackageInfo; +use deno_resolver::npm::ByonmNpmResolver; use deno_runtime::deno_node::NodeRequireResolver; use deno_runtime::ops::process::NpmProcessStateProvider; use deno_semver::package::PackageNv; @@ -21,15 +23,15 @@ use node_resolver::NpmResolver; use crate::args::npm_registry_url; use crate::file_fetcher::FileFetcher; -pub use self::byonm::ByonmCliNpmResolver; -pub use self::byonm::CliNpmResolverByonmCreateOptions; +pub use self::byonm::CliByonmNpmResolver; +pub use self::byonm::CliByonmNpmResolverCreateOptions; pub use self::managed::CliNpmResolverManagedCreateOptions; pub use self::managed::CliNpmResolverManagedSnapshotOption; pub use self::managed::ManagedCliNpmResolver; pub enum CliNpmResolverCreateOptions { Managed(CliNpmResolverManagedCreateOptions), - Byonm(CliNpmResolverByonmCreateOptions), + Byonm(CliByonmNpmResolverCreateOptions), } pub async fn create_cli_npm_resolver_for_lsp( @@ -40,7 +42,7 @@ pub async fn create_cli_npm_resolver_for_lsp( Managed(options) => { managed::create_managed_npm_resolver_for_lsp(options).await } - Byonm(options) => byonm::create_byonm_npm_resolver(options), + Byonm(options) => Arc::new(ByonmNpmResolver::new(options)), } } @@ -50,14 +52,14 @@ pub async fn create_cli_npm_resolver( use CliNpmResolverCreateOptions::*; match options { Managed(options) => managed::create_managed_npm_resolver(options).await, - Byonm(options) => Ok(byonm::create_byonm_npm_resolver(options)), + Byonm(options) => Ok(Arc::new(ByonmNpmResolver::new(options))), } } pub enum InnerCliNpmResolverRef<'a> { Managed(&'a ManagedCliNpmResolver), #[allow(dead_code)] - Byonm(&'a ByonmCliNpmResolver), + Byonm(&'a CliByonmNpmResolver), } pub trait CliNpmResolver: NpmResolver { @@ -78,14 +80,14 @@ pub trait CliNpmResolver: NpmResolver { } } - fn as_byonm(&self) -> Option<&ByonmCliNpmResolver> { + fn as_byonm(&self) -> Option<&CliByonmNpmResolver> { match self.as_inner() { InnerCliNpmResolverRef::Managed(_) => None, InnerCliNpmResolverRef::Byonm(inner) => Some(inner), } } - fn root_node_modules_path(&self) -> Option<&PathBuf>; + fn root_node_modules_path(&self) -> Option<&Path>; fn resolve_pkg_folder_from_deno_module_req( &self, diff --git a/cli/resolver.rs b/cli/resolver.rs index 211f8aba1..7804261b8 100644 --- a/cli/resolver.rs +++ b/cli/resolver.rs @@ -61,6 +61,46 @@ pub struct ModuleCodeStringSource { pub media_type: MediaType, } +#[derive(Debug, Clone)] +pub struct CliDenoResolverFs(pub Arc<dyn FileSystem>); + +impl deno_resolver::fs::DenoResolverFs for CliDenoResolverFs { + fn read_to_string_lossy(&self, path: &Path) -> std::io::Result<String> { + self + .0 + .read_text_file_lossy_sync(path, None) + .map_err(|e| e.into_io_error()) + } + + fn realpath_sync(&self, path: &Path) -> std::io::Result<PathBuf> { + self.0.realpath_sync(path).map_err(|e| e.into_io_error()) + } + + fn is_dir_sync(&self, path: &Path) -> bool { + self.0.is_dir_sync(path) + } + + fn read_dir_sync( + &self, + dir_path: &Path, + ) -> std::io::Result<Vec<deno_resolver::fs::DirEntry>> { + self + .0 + .read_dir_sync(dir_path) + .map(|entries| { + entries + .into_iter() + .map(|e| deno_resolver::fs::DirEntry { + name: e.name, + is_file: e.is_file, + is_directory: e.is_directory, + }) + .collect::<Vec<_>>() + }) + .map_err(|err| err.into_io_error()) + } +} + #[derive(Debug)] pub struct CliNodeResolver { cjs_resolutions: Arc<CjsResolutionStore>, diff --git a/cli/standalone/mod.rs b/cli/standalone/mod.rs index 56f76ffb9..258de0dad 100644 --- a/cli/standalone/mod.rs +++ b/cli/standalone/mod.rs @@ -60,11 +60,12 @@ use crate::cache::RealDenoCacheEnv; use crate::http_util::HttpClientProvider; use crate::node::CliCjsCodeAnalyzer; use crate::npm::create_cli_npm_resolver; -use crate::npm::CliNpmResolverByonmCreateOptions; +use crate::npm::CliByonmNpmResolverCreateOptions; use crate::npm::CliNpmResolverCreateOptions; use crate::npm::CliNpmResolverManagedCreateOptions; use crate::npm::CliNpmResolverManagedSnapshotOption; use crate::resolver::CjsResolutionStore; +use crate::resolver::CliDenoResolverFs; use crate::resolver::CliNodeResolver; use crate::resolver::NpmModuleLoader; use crate::util::progress_bar::ProgressBar; @@ -530,8 +531,8 @@ pub async fn run( let fs = Arc::new(DenoCompileFileSystem::new(vfs)) as Arc<dyn deno_fs::FileSystem>; let npm_resolver = create_cli_npm_resolver( - CliNpmResolverCreateOptions::Byonm(CliNpmResolverByonmCreateOptions { - fs: fs.clone(), + CliNpmResolverCreateOptions::Byonm(CliByonmNpmResolverCreateOptions { + fs: CliDenoResolverFs(fs.clone()), root_node_modules_dir, }), ) diff --git a/cli/tools/task.rs b/cli/tools/task.rs index a5a5027d0..464b65d98 100644 --- a/cli/tools/task.rs +++ b/cli/tools/task.rs @@ -190,9 +190,7 @@ async fn run_task(opts: RunTaskOptions<'_>) -> Result<i32, AnyError> { custom_commands, init_cwd: opts.cli_options.initial_cwd(), argv: cli_options.argv(), - root_node_modules_dir: npm_resolver - .root_node_modules_path() - .map(|p| p.as_path()), + root_node_modules_dir: npm_resolver.root_node_modules_path(), }) .await } diff --git a/cli/util/fs.rs b/cli/util/fs.rs index 9734d417e..a021ec19c 100644 --- a/cli/util/fs.rs +++ b/cli/util/fs.rs @@ -20,7 +20,6 @@ use deno_core::error::AnyError; use deno_core::unsync::spawn_blocking; use deno_core::ModuleSpecifier; use deno_runtime::deno_fs::FileSystem; -use deno_runtime::deno_node::PathClean; use crate::util::path::get_atomic_file_path; use crate::util::progress_bar::ProgressBar; @@ -290,48 +289,18 @@ pub fn canonicalize_path(path: &Path) -> Result<PathBuf, Error> { pub fn canonicalize_path_maybe_not_exists( path: &Path, ) -> Result<PathBuf, Error> { - canonicalize_path_maybe_not_exists_with_custom_fn(path, canonicalize_path) + deno_path_util::canonicalize_path_maybe_not_exists(path, &canonicalize_path) } pub fn canonicalize_path_maybe_not_exists_with_fs( path: &Path, fs: &dyn FileSystem, ) -> Result<PathBuf, Error> { - canonicalize_path_maybe_not_exists_with_custom_fn(path, |path| { + deno_path_util::canonicalize_path_maybe_not_exists(path, &|path| { fs.realpath_sync(path).map_err(|err| err.into_io_error()) }) } -fn canonicalize_path_maybe_not_exists_with_custom_fn( - path: &Path, - canonicalize: impl Fn(&Path) -> Result<PathBuf, Error>, -) -> Result<PathBuf, Error> { - let path = path.to_path_buf().clean(); - let mut path = path.as_path(); - let mut names_stack = Vec::new(); - loop { - match canonicalize(path) { - Ok(mut canonicalized_path) => { - for name in names_stack.into_iter().rev() { - canonicalized_path = canonicalized_path.join(name); - } - return Ok(canonicalized_path); - } - Err(err) if err.kind() == ErrorKind::NotFound => { - names_stack.push(match path.file_name() { - Some(name) => name.to_owned(), - None => return Err(err), - }); - path = match path.parent() { - Some(parent) => parent, - None => return Err(err), - }; - } - Err(err) => return Err(err), - } - } -} - /// Collects module specifiers that satisfy the given predicate as a file path, by recursively walking `include`. /// Specifiers that start with http and https are left intact. /// Note: This ignores all .git and node_modules folders. diff --git a/resolvers/deno/Cargo.toml b/resolvers/deno/Cargo.toml index 23c43810a..a14635c38 100644 --- a/resolvers/deno/Cargo.toml +++ b/resolvers/deno/Cargo.toml @@ -16,8 +16,13 @@ path = "lib.rs" [features] [dependencies] +anyhow.workspace = true +base32.workspace = true deno_media_type.workspace = true +deno_package_json.workspace = true deno_path_util.workspace = true +deno_semver.workspace = true +node_resolver.workspace = true url.workspace = true [dev-dependencies] diff --git a/resolvers/deno/clippy.toml b/resolvers/deno/clippy.toml new file mode 100644 index 000000000..733ac83da --- /dev/null +++ b/resolvers/deno/clippy.toml @@ -0,0 +1,52 @@ +disallowed-methods = [ + { path = "std::env::current_dir", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::canonicalize", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::is_dir", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::is_file", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::is_symlink", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::metadata", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::read_dir", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::read_link", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::symlink_metadata", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::try_exists", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::PathBuf::exists", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::PathBuf::canonicalize", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::PathBuf::is_dir", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::PathBuf::is_file", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::PathBuf::is_symlink", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::PathBuf::metadata", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::PathBuf::read_dir", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::PathBuf::read_link", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::PathBuf::symlink_metadata", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::PathBuf::try_exists", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::env::set_current_dir", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::env::temp_dir", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::canonicalize", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::copy", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::create_dir_all", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::create_dir", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::DirBuilder::new", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::hard_link", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::metadata", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::OpenOptions::new", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::read_dir", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::read_link", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::read_to_string", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::read", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::remove_dir_all", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::remove_dir", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::remove_file", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::rename", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::set_permissions", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::symlink_metadata", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::fs::write", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::canonicalize", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "std::path::Path::exists", reason = "File system operations should be done using DenoResolverFs trait" }, + { path = "url::Url::to_file_path", reason = "Use deno_path_util instead so it works in Wasm" }, + { path = "url::Url::from_file_path", reason = "Use deno_path_util instead so it works in Wasm" }, + { path = "url::Url::from_directory_path", reason = "Use deno_path_util instead so it works in Wasm" }, +] +disallowed-types = [ + # todo(dsherret): consider for the future + # { path = "std::sync::Arc", reason = "use crate::sync::MaybeArc instead" }, +] diff --git a/resolvers/deno/fs.rs b/resolvers/deno/fs.rs new file mode 100644 index 000000000..b08be3798 --- /dev/null +++ b/resolvers/deno/fs.rs @@ -0,0 +1,27 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use std::path::Path; +use std::path::PathBuf; + +pub struct DirEntry { + pub name: String, + pub is_file: bool, + pub is_directory: bool, +} + +pub trait DenoResolverFs { + fn read_to_string_lossy(&self, path: &Path) -> std::io::Result<String>; + fn realpath_sync(&self, path: &Path) -> std::io::Result<PathBuf>; + fn is_dir_sync(&self, path: &Path) -> bool; + fn read_dir_sync(&self, dir_path: &Path) -> std::io::Result<Vec<DirEntry>>; +} + +pub(crate) struct DenoPkgJsonFsAdapter<'a, Fs: DenoResolverFs>(pub &'a Fs); + +impl<'a, Fs: DenoResolverFs> deno_package_json::fs::DenoPkgJsonFs + for DenoPkgJsonFsAdapter<'a, Fs> +{ + fn read_to_string_lossy(&self, path: &Path) -> std::io::Result<String> { + self.0.read_to_string_lossy(path) + } +} diff --git a/resolvers/deno/lib.rs b/resolvers/deno/lib.rs index 7d7796d77..57fa67512 100644 --- a/resolvers/deno/lib.rs +++ b/resolvers/deno/lib.rs @@ -1,3 +1,5 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +pub mod fs; +pub mod npm; pub mod sloppy_imports; diff --git a/resolvers/deno/npm/byonm.rs b/resolvers/deno/npm/byonm.rs new file mode 100644 index 000000000..c847cee0f --- /dev/null +++ b/resolvers/deno/npm/byonm.rs @@ -0,0 +1,348 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use std::borrow::Cow; +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; + +use anyhow::bail; +use anyhow::Error as AnyError; +use deno_package_json::PackageJson; +use deno_package_json::PackageJsonDepValue; +use deno_path_util::url_to_file_path; +use deno_semver::package::PackageReq; +use deno_semver::Version; +use node_resolver::errors::PackageFolderResolveError; +use node_resolver::errors::PackageFolderResolveIoError; +use node_resolver::errors::PackageJsonLoadError; +use node_resolver::errors::PackageNotFoundError; +use node_resolver::load_pkg_json; +use node_resolver::NpmResolver; +use url::Url; + +use crate::fs::DenoPkgJsonFsAdapter; +use crate::fs::DenoResolverFs; + +use super::local::normalize_pkg_name_for_node_modules_deno_folder; + +pub struct ByonmNpmResolverCreateOptions<Fs: DenoResolverFs> { + pub fs: Fs, + // todo(dsherret): investigate removing this + pub root_node_modules_dir: Option<PathBuf>, +} + +#[derive(Debug)] +pub struct ByonmNpmResolver<Fs: DenoResolverFs> { + fs: Fs, + root_node_modules_dir: Option<PathBuf>, +} + +impl<Fs: DenoResolverFs + Clone> Clone for ByonmNpmResolver<Fs> { + fn clone(&self) -> Self { + Self { + fs: self.fs.clone(), + root_node_modules_dir: self.root_node_modules_dir.clone(), + } + } +} + +impl<Fs: DenoResolverFs> ByonmNpmResolver<Fs> { + pub fn new(options: ByonmNpmResolverCreateOptions<Fs>) -> Self { + Self { + fs: options.fs, + root_node_modules_dir: options.root_node_modules_dir, + } + } + + pub fn root_node_modules_dir(&self) -> Option<&Path> { + self.root_node_modules_dir.as_deref() + } + + fn load_pkg_json( + &self, + path: &Path, + ) -> Result<Option<Arc<PackageJson>>, PackageJsonLoadError> { + load_pkg_json(&DenoPkgJsonFsAdapter(&self.fs), path) + } + + /// Finds the ancestor package.json that contains the specified dependency. + pub fn find_ancestor_package_json_with_dep( + &self, + dep_name: &str, + referrer: &Url, + ) -> Option<Arc<PackageJson>> { + let referrer_path = url_to_file_path(referrer).ok()?; + let mut current_folder = referrer_path.parent()?; + loop { + let pkg_json_path = current_folder.join("package.json"); + if let Ok(Some(pkg_json)) = self.load_pkg_json(&pkg_json_path) { + if let Some(deps) = &pkg_json.dependencies { + if deps.contains_key(dep_name) { + return Some(pkg_json); + } + } + if let Some(deps) = &pkg_json.dev_dependencies { + if deps.contains_key(dep_name) { + return Some(pkg_json); + } + } + } + + if let Some(parent) = current_folder.parent() { + current_folder = parent; + } else { + return None; + } + } + } + + pub fn resolve_pkg_folder_from_deno_module_req( + &self, + req: &PackageReq, + referrer: &Url, + ) -> Result<PathBuf, AnyError> { + fn node_resolve_dir<Fs: DenoResolverFs>( + fs: &Fs, + alias: &str, + start_dir: &Path, + ) -> Result<Option<PathBuf>, AnyError> { + for ancestor in start_dir.ancestors() { + let node_modules_folder = ancestor.join("node_modules"); + let sub_dir = join_package_name(&node_modules_folder, alias); + if fs.is_dir_sync(&sub_dir) { + return Ok(Some(deno_path_util::canonicalize_path_maybe_not_exists( + &sub_dir, + &|path| fs.realpath_sync(path), + )?)); + } + } + Ok(None) + } + + // now attempt to resolve if it's found in any package.json + let maybe_pkg_json_and_alias = + self.resolve_pkg_json_and_alias_for_req(req, referrer)?; + match maybe_pkg_json_and_alias { + Some((pkg_json, alias)) => { + // now try node resolution + if let Some(resolved) = + node_resolve_dir(&self.fs, &alias, pkg_json.dir_path())? + { + return Ok(resolved); + } + + bail!( + concat!( + "Could not find \"{}\" in a node_modules folder. ", + "Deno expects the node_modules/ directory to be up to date. ", + "Did you forget to run `deno install`?" + ), + alias, + ); + } + None => { + // now check if node_modules/.deno/ matches this constraint + if let Some(folder) = self.resolve_folder_in_root_node_modules(req) { + return Ok(folder); + } + + bail!( + concat!( + "Could not find a matching package for 'npm:{}' in the node_modules ", + "directory. Ensure you have all your JSR and npm dependencies listed ", + "in your deno.json or package.json, then run `deno install`. Alternatively, ", + r#"turn on auto-install by specifying `"nodeModulesDir": "auto"` in your "#, + "deno.json file." + ), + req, + ); + } + } + } + + fn resolve_pkg_json_and_alias_for_req( + &self, + req: &PackageReq, + referrer: &Url, + ) -> Result<Option<(Arc<PackageJson>, String)>, AnyError> { + fn resolve_alias_from_pkg_json( + req: &PackageReq, + pkg_json: &PackageJson, + ) -> Option<String> { + let deps = pkg_json.resolve_local_package_json_deps(); + for (key, value) in deps { + if let Ok(value) = value { + match value { + PackageJsonDepValue::Req(dep_req) => { + if dep_req.name == req.name + && dep_req.version_req.intersects(&req.version_req) + { + return Some(key); + } + } + PackageJsonDepValue::Workspace(_workspace) => { + if key == req.name && req.version_req.tag() == Some("workspace") { + return Some(key); + } + } + } + } + } + None + } + + // attempt to resolve the npm specifier from the referrer's package.json, + if let Ok(file_path) = url_to_file_path(referrer) { + let mut current_path = file_path.as_path(); + while let Some(dir_path) = current_path.parent() { + let package_json_path = dir_path.join("package.json"); + if let Some(pkg_json) = self.load_pkg_json(&package_json_path)? { + if let Some(alias) = + resolve_alias_from_pkg_json(req, pkg_json.as_ref()) + { + return Ok(Some((pkg_json, alias))); + } + } + current_path = dir_path; + } + } + + // otherwise, fall fallback to the project's package.json + if let Some(root_node_modules_dir) = &self.root_node_modules_dir { + let root_pkg_json_path = + root_node_modules_dir.parent().unwrap().join("package.json"); + if let Some(pkg_json) = self.load_pkg_json(&root_pkg_json_path)? { + if let Some(alias) = resolve_alias_from_pkg_json(req, pkg_json.as_ref()) + { + return Ok(Some((pkg_json, alias))); + } + } + } + + Ok(None) + } + + fn resolve_folder_in_root_node_modules( + &self, + req: &PackageReq, + ) -> Option<PathBuf> { + // now check if node_modules/.deno/ matches this constraint + let root_node_modules_dir = self.root_node_modules_dir.as_ref()?; + let node_modules_deno_dir = root_node_modules_dir.join(".deno"); + let Ok(entries) = self.fs.read_dir_sync(&node_modules_deno_dir) else { + return None; + }; + let search_prefix = format!( + "{}@", + normalize_pkg_name_for_node_modules_deno_folder(&req.name) + ); + let mut best_version = None; + + // example entries: + // - @denotest+add@1.0.0 + // - @denotest+add@1.0.0_1 + for entry in entries { + if !entry.is_directory { + continue; + } + let Some(version_and_copy_idx) = entry.name.strip_prefix(&search_prefix) + else { + continue; + }; + let version = version_and_copy_idx + .rsplit_once('_') + .map(|(v, _)| v) + .unwrap_or(version_and_copy_idx); + let Ok(version) = Version::parse_from_npm(version) else { + continue; + }; + if req.version_req.matches(&version) { + if let Some((best_version_version, _)) = &best_version { + if version > *best_version_version { + best_version = Some((version, entry.name)); + } + } else { + best_version = Some((version, entry.name)); + } + } + } + + best_version.map(|(_version, entry_name)| { + join_package_name( + &node_modules_deno_dir.join(entry_name).join("node_modules"), + &req.name, + ) + }) + } +} + +impl<Fs: DenoResolverFs + Send + Sync + std::fmt::Debug> NpmResolver + for ByonmNpmResolver<Fs> +{ + fn resolve_package_folder_from_package( + &self, + name: &str, + referrer: &Url, + ) -> Result<PathBuf, PackageFolderResolveError> { + fn inner<Fs: DenoResolverFs>( + fs: &Fs, + name: &str, + referrer: &Url, + ) -> Result<PathBuf, PackageFolderResolveError> { + let maybe_referrer_file = url_to_file_path(referrer).ok(); + let maybe_start_folder = + maybe_referrer_file.as_ref().and_then(|f| f.parent()); + if let Some(start_folder) = maybe_start_folder { + for current_folder in start_folder.ancestors() { + let node_modules_folder = if current_folder.ends_with("node_modules") + { + Cow::Borrowed(current_folder) + } else { + Cow::Owned(current_folder.join("node_modules")) + }; + + let sub_dir = join_package_name(&node_modules_folder, name); + if fs.is_dir_sync(&sub_dir) { + return Ok(sub_dir); + } + } + } + + Err( + PackageNotFoundError { + package_name: name.to_string(), + referrer: referrer.clone(), + referrer_extra: None, + } + .into(), + ) + } + + let path = inner(&self.fs, name, referrer)?; + self.fs.realpath_sync(&path).map_err(|err| { + PackageFolderResolveIoError { + package_name: name.to_string(), + referrer: referrer.clone(), + source: err, + } + .into() + }) + } + + fn in_npm_package(&self, specifier: &Url) -> bool { + specifier.scheme() == "file" + && specifier + .path() + .to_ascii_lowercase() + .contains("/node_modules/") + } +} + +fn join_package_name(path: &Path, package_name: &str) -> PathBuf { + let mut path = path.to_path_buf(); + // ensure backslashes are used on windows + for part in package_name.split('/') { + path = path.join(part); + } + path +} diff --git a/resolvers/deno/npm/local.rs b/resolvers/deno/npm/local.rs new file mode 100644 index 000000000..aef476ad9 --- /dev/null +++ b/resolvers/deno/npm/local.rs @@ -0,0 +1,27 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use std::borrow::Cow; + +/// Normalizes a package name for use at `node_modules/.deno/<pkg-name>@<version>[_<copy_index>]` +pub fn normalize_pkg_name_for_node_modules_deno_folder(name: &str) -> Cow<str> { + let name = if name.to_lowercase() == name { + Cow::Borrowed(name) + } else { + Cow::Owned(format!("_{}", mixed_case_package_name_encode(name))) + }; + if name.starts_with('@') { + name.replace('/', "+").into() + } else { + name + } +} + +fn mixed_case_package_name_encode(name: &str) -> String { + // use base32 encoding because it's reversible and the character set + // only includes the characters within 0-9 and A-Z so it can be lower cased + base32::encode( + base32::Alphabet::Rfc4648Lower { padding: false }, + name.as_bytes(), + ) + .to_lowercase() +} diff --git a/resolvers/deno/npm/mod.rs b/resolvers/deno/npm/mod.rs new file mode 100644 index 000000000..2e24144cd --- /dev/null +++ b/resolvers/deno/npm/mod.rs @@ -0,0 +1,8 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +mod byonm; +mod local; + +pub use byonm::ByonmNpmResolver; +pub use byonm::ByonmNpmResolverCreateOptions; +pub use local::normalize_pkg_name_for_node_modules_deno_folder; diff --git a/resolvers/deno/sloppy_imports.rs b/resolvers/deno/sloppy_imports.rs index e4d0898e5..e215e8768 100644 --- a/resolvers/deno/sloppy_imports.rs +++ b/resolvers/deno/sloppy_imports.rs @@ -5,6 +5,7 @@ use std::path::Path; use std::path::PathBuf; use deno_media_type::MediaType; +use deno_path_util::url_from_file_path; use deno_path_util::url_to_file_path; use url::Url; @@ -343,7 +344,7 @@ impl<Fs: SloppyImportResolverFs> SloppyImportsResolver<Fs> { for (probe_path, reason) in probe_paths { if self.fs.is_file(&probe_path) { - if let Ok(specifier) = Url::from_file_path(probe_path) { + if let Ok(specifier) = url_from_file_path(&probe_path) { match reason { SloppyImportsResolutionReason::JsToTs => { return Some(SloppyImportsResolution::JsToTs(specifier)); @@ -386,6 +387,7 @@ mod test { struct RealSloppyImportsResolverFs; impl SloppyImportResolverFs for RealSloppyImportsResolverFs { fn stat_sync(&self, path: &Path) -> Option<SloppyImportsFsEntry> { + #[allow(clippy::disallowed_methods)] let stat = std::fs::metadata(path).ok()?; if stat.is_dir() { Some(SloppyImportsFsEntry::Dir) diff --git a/resolvers/node/Cargo.toml b/resolvers/node/Cargo.toml index 104204569..c2f2f1cc1 100644 --- a/resolvers/node/Cargo.toml +++ b/resolvers/node/Cargo.toml @@ -21,6 +21,7 @@ anyhow.workspace = true async-trait.workspace = true deno_media_type.workspace = true deno_package_json.workspace = true +deno_path_util.workspace = true futures.workspace = true lazy-regex.workspace = true once_cell.workspace = true diff --git a/resolvers/node/analyze.rs b/resolvers/node/analyze.rs index deb56d064..009296006 100644 --- a/resolvers/node/analyze.rs +++ b/resolvers/node/analyze.rs @@ -6,6 +6,8 @@ use std::collections::HashSet; use std::path::Path; use std::path::PathBuf; +use deno_path_util::url_from_file_path; +use deno_path_util::url_to_file_path; use futures::future::LocalBoxFuture; use futures::stream::FuturesUnordered; use futures::FutureExt; @@ -18,7 +20,6 @@ use url::Url; use crate::env::NodeResolverEnv; use crate::package_json::load_pkg_json; -use crate::path::to_file_specifier; use crate::resolution::NodeResolverRc; use crate::NodeModuleKind; use crate::NodeResolutionMode; @@ -135,8 +136,7 @@ impl<TCjsCodeAnalyzer: CjsCodeAnalyzer, TNodeResolverEnv: NodeResolverEnv> source.push(format!( "const mod = require(\"{}\");", - entry_specifier - .to_file_path() + url_to_file_path(entry_specifier) .unwrap() .to_str() .unwrap() @@ -297,15 +297,13 @@ impl<TCjsCodeAnalyzer: CjsCodeAnalyzer, TNodeResolverEnv: NodeResolverEnv> todo!(); } - let referrer_path = referrer.to_file_path().unwrap(); + let referrer_path = url_to_file_path(referrer).unwrap(); if specifier.starts_with("./") || specifier.starts_with("../") { if let Some(parent) = referrer_path.parent() { - return Some( - self - .file_extension_probe(parent.join(specifier), &referrer_path) - .map(|p| to_file_specifier(&p)), - ) - .transpose(); + return self + .file_extension_probe(parent.join(specifier), &referrer_path) + .and_then(|p| url_from_file_path(&p).map_err(AnyError::from)) + .map(Some); } else { todo!(); } @@ -362,24 +360,22 @@ impl<TCjsCodeAnalyzer: CjsCodeAnalyzer, TNodeResolverEnv: NodeResolverEnv> load_pkg_json(self.env.pkg_json_fs(), &package_json_path)?; if let Some(package_json) = maybe_package_json { if let Some(main) = package_json.main(NodeModuleKind::Cjs) { - return Ok(Some(to_file_specifier(&d.join(main).clean()))); + return Ok(Some(url_from_file_path(&d.join(main).clean())?)); } } - return Ok(Some(to_file_specifier(&d.join("index.js").clean()))); + return Ok(Some(url_from_file_path(&d.join("index.js").clean())?)); } - return Some( - self - .file_extension_probe(d, &referrer_path) - .map(|p| to_file_specifier(&p)), - ) - .transpose(); + return self + .file_extension_probe(d, &referrer_path) + .and_then(|p| url_from_file_path(&p).map_err(AnyError::from)) + .map(Some); } else if let Some(main) = package_json.main(NodeModuleKind::Cjs) { - return Ok(Some(to_file_specifier(&module_dir.join(main).clean()))); + return Ok(Some(url_from_file_path(&module_dir.join(main).clean())?)); } else { - return Ok(Some(to_file_specifier( + return Ok(Some(url_from_file_path( &module_dir.join("index.js").clean(), - ))); + )?)); } } @@ -395,7 +391,7 @@ impl<TCjsCodeAnalyzer: CjsCodeAnalyzer, TNodeResolverEnv: NodeResolverEnv> parent.join("node_modules").join(specifier) }; if let Ok(path) = self.file_extension_probe(path, &referrer_path) { - return Ok(Some(to_file_specifier(&path))); + return Ok(Some(url_from_file_path(&path)?)); } last = parent; } diff --git a/resolvers/node/clippy.toml b/resolvers/node/clippy.toml index 86150781b..90eaba3fa 100644 --- a/resolvers/node/clippy.toml +++ b/resolvers/node/clippy.toml @@ -42,6 +42,9 @@ disallowed-methods = [ { path = "std::fs::write", reason = "File system operations should be done using NodeResolverFs trait" }, { path = "std::path::Path::canonicalize", reason = "File system operations should be done using NodeResolverFs trait" }, { path = "std::path::Path::exists", reason = "File system operations should be done using NodeResolverFs trait" }, + { path = "url::Url::to_file_path", reason = "Use deno_path_util instead so it works in Wasm" }, + { path = "url::Url::from_file_path", reason = "Use deno_path_util instead so it works in Wasm" }, + { path = "url::Url::from_directory_path", reason = "Use deno_path_util instead so it works in Wasm" }, ] disallowed-types = [ { path = "std::sync::Arc", reason = "use crate::sync::MaybeArc instead" }, diff --git a/resolvers/node/npm.rs b/resolvers/node/npm.rs index 77df57c48..6b5f21db6 100644 --- a/resolvers/node/npm.rs +++ b/resolvers/node/npm.rs @@ -3,6 +3,8 @@ use std::path::Path; use std::path::PathBuf; +use deno_path_util::url_from_directory_path; +use deno_path_util::url_from_file_path; use url::Url; use crate::errors; @@ -24,7 +26,7 @@ pub trait NpmResolver: std::fmt::Debug + MaybeSend + MaybeSync { fn in_npm_package(&self, specifier: &Url) -> bool; fn in_npm_package_at_dir_path(&self, path: &Path) -> bool { - let specifier = match Url::from_directory_path(path.to_path_buf().clean()) { + let specifier = match url_from_directory_path(&path.to_path_buf().clean()) { Ok(p) => p, Err(_) => return false, }; @@ -32,7 +34,7 @@ pub trait NpmResolver: std::fmt::Debug + MaybeSend + MaybeSync { } fn in_npm_package_at_file_path(&self, path: &Path) -> bool { - let specifier = match Url::from_file_path(path.to_path_buf().clean()) { + let specifier = match url_from_file_path(&path.to_path_buf().clean()) { Ok(p) => p, Err(_) => return false, }; diff --git a/resolvers/node/path.rs b/resolvers/node/path.rs index ece270cd9..8c2d35fad 100644 --- a/resolvers/node/path.rs +++ b/resolvers/node/path.rs @@ -4,8 +4,6 @@ use std::path::Component; use std::path::Path; use std::path::PathBuf; -use url::Url; - /// Extension to path_clean::PathClean pub trait PathClean<T> { fn clean(&self) -> T; @@ -65,65 +63,6 @@ impl PathClean<PathBuf> for PathBuf { } } -pub(crate) fn to_file_specifier(path: &Path) -> Url { - match Url::from_file_path(path) { - Ok(url) => url, - Err(_) => panic!("Invalid path: {}", path.display()), - } -} - -// todo(dsherret): we have the below code also in deno_core and it -// would be good to somehow re-use it in both places (we don't want -// to create a dependency on deno_core here) - -#[cfg(not(windows))] -#[inline] -pub fn strip_unc_prefix(path: PathBuf) -> PathBuf { - path -} - -/// Strips the unc prefix (ex. \\?\) from Windows paths. -#[cfg(windows)] -pub fn strip_unc_prefix(path: PathBuf) -> PathBuf { - use std::path::Component; - use std::path::Prefix; - - let mut components = path.components(); - match components.next() { - Some(Component::Prefix(prefix)) => { - match prefix.kind() { - // \\?\device - Prefix::Verbatim(device) => { - let mut path = PathBuf::new(); - path.push(format!(r"\\{}\", device.to_string_lossy())); - path.extend(components.filter(|c| !matches!(c, Component::RootDir))); - path - } - // \\?\c:\path - Prefix::VerbatimDisk(_) => { - let mut path = PathBuf::new(); - path.push(prefix.as_os_str().to_string_lossy().replace(r"\\?\", "")); - path.extend(components); - path - } - // \\?\UNC\hostname\share_name\path - Prefix::VerbatimUNC(hostname, share_name) => { - let mut path = PathBuf::new(); - path.push(format!( - r"\\{}\{}\", - hostname.to_string_lossy(), - share_name.to_string_lossy() - )); - path.extend(components.filter(|c| !matches!(c, Component::RootDir))); - path - } - _ => path, - } - } - _ => path, - } -} - #[cfg(test)] mod test { #[cfg(windows)] @@ -139,41 +78,4 @@ mod test { assert_eq!(PathBuf::from(input).clean(), PathBuf::from(expected)); } } - - #[cfg(windows)] - #[test] - fn test_strip_unc_prefix() { - use std::path::PathBuf; - - run_test(r"C:\", r"C:\"); - run_test(r"C:\test\file.txt", r"C:\test\file.txt"); - - run_test(r"\\?\C:\", r"C:\"); - run_test(r"\\?\C:\test\file.txt", r"C:\test\file.txt"); - - run_test(r"\\.\C:\", r"\\.\C:\"); - run_test(r"\\.\C:\Test\file.txt", r"\\.\C:\Test\file.txt"); - - run_test(r"\\?\UNC\localhost\", r"\\localhost"); - run_test(r"\\?\UNC\localhost\c$\", r"\\localhost\c$"); - run_test( - r"\\?\UNC\localhost\c$\Windows\file.txt", - r"\\localhost\c$\Windows\file.txt", - ); - run_test(r"\\?\UNC\wsl$\deno.json", r"\\wsl$\deno.json"); - - run_test(r"\\?\server1", r"\\server1"); - run_test(r"\\?\server1\e$\", r"\\server1\e$\"); - run_test( - r"\\?\server1\e$\test\file.txt", - r"\\server1\e$\test\file.txt", - ); - - fn run_test(input: &str, expected: &str) { - assert_eq!( - super::strip_unc_prefix(PathBuf::from(input)), - PathBuf::from(expected) - ); - } - } } diff --git a/resolvers/node/resolution.rs b/resolvers/node/resolution.rs index ad9dbb710..811583a5e 100644 --- a/resolvers/node/resolution.rs +++ b/resolvers/node/resolution.rs @@ -8,6 +8,8 @@ use anyhow::bail; use anyhow::Error as AnyError; use deno_media_type::MediaType; use deno_package_json::PackageJsonRc; +use deno_path_util::strip_unc_prefix; +use deno_path_util::url_from_file_path; use serde_json::Map; use serde_json::Value; use url::Url; @@ -47,8 +49,6 @@ use crate::errors::TypesNotFoundErrorData; use crate::errors::UnsupportedDirImportError; use crate::errors::UnsupportedEsmUrlSchemeError; use crate::errors::UrlToNodeResolutionError; -use crate::path::strip_unc_prefix; -use crate::path::to_file_specifier; use crate::NpmResolverRc; use crate::PathClean; use deno_package_json::PackageJson; @@ -394,7 +394,7 @@ impl<TEnv: NodeResolverEnv> NodeResolver<TEnv> { message: err.to_string(), } })?; - let url = to_file_specifier(&package_folder.join(bin_entry)); + let url = url_from_file_path(&package_folder.join(bin_entry)).unwrap(); let resolve_response = self.url_to_node_resolution(url)?; // TODO(bartlomieju): skipped checking errors for commonJS resolution and @@ -485,12 +485,12 @@ impl<TEnv: NodeResolverEnv> NodeResolver<TEnv> { || lowercase_path.ends_with(".d.cts") || lowercase_path.ends_with(".d.mts") { - return Ok(to_file_specifier(path)); + return Ok(url_from_file_path(path).unwrap()); } if let Some(path) = probe_extensions(&self.env, path, &lowercase_path, referrer_kind) { - return Ok(to_file_specifier(&path)); + return Ok(url_from_file_path(&path).unwrap()); } if self.env.is_dir_sync(path) { let resolution_result = self.resolve_package_dir_subpath( @@ -514,15 +514,15 @@ impl<TEnv: NodeResolverEnv> NodeResolver<TEnv> { &index_path.to_string_lossy().to_lowercase(), referrer_kind, ) { - return Ok(to_file_specifier(&path)); + return Ok(url_from_file_path(&path).unwrap()); } } // allow resolving .css files for types resolution if lowercase_path.ends_with(".css") { - return Ok(to_file_specifier(path)); + return Ok(url_from_file_path(path).unwrap()); } Err(TypesNotFoundError(Box::new(TypesNotFoundErrorData { - code_specifier: to_file_specifier(path), + code_specifier: url_from_file_path(path).unwrap(), maybe_referrer: maybe_referrer.cloned(), }))) } @@ -673,7 +673,8 @@ impl<TEnv: NodeResolverEnv> NodeResolver<TEnv> { } else { format!("{target}{subpath}") }; - let package_json_url = to_file_specifier(package_json_path); + let package_json_url = + url_from_file_path(package_json_path).unwrap(); let result = match self.package_resolve( &export_target, &package_json_url, @@ -760,7 +761,7 @@ impl<TEnv: NodeResolverEnv> NodeResolver<TEnv> { ); } if subpath.is_empty() { - return Ok(to_file_specifier(&resolved_path)); + return Ok(url_from_file_path(&resolved_path).unwrap()); } if invalid_segment_re.is_match(subpath) { let request = if pattern { @@ -782,9 +783,11 @@ impl<TEnv: NodeResolverEnv> NodeResolver<TEnv> { let resolved_path_str = resolved_path.to_string_lossy(); let replaced = pattern_re .replace(&resolved_path_str, |_caps: ®ex::Captures| subpath); - return Ok(to_file_specifier(&PathBuf::from(replaced.to_string()))); + return Ok( + url_from_file_path(&PathBuf::from(replaced.to_string())).unwrap(), + ); } - Ok(to_file_specifier(&resolved_path.join(subpath).clean())) + Ok(url_from_file_path(&resolved_path.join(subpath).clean()).unwrap()) } #[allow(clippy::too_many_arguments)] @@ -871,7 +874,7 @@ impl<TEnv: NodeResolverEnv> NodeResolver<TEnv> { mode, )?; if mode.is_types() && url.scheme() == "file" { - let path = url.to_file_path().unwrap(); + let path = deno_path_util::url_to_file_path(&url).unwrap(); return Ok(Some(self.path_to_declaration_url( &path, maybe_referrer, @@ -1307,7 +1310,7 @@ impl<TEnv: NodeResolverEnv> NodeResolver<TEnv> { if mode.is_types() { Ok(self.path_to_declaration_url(&file_path, referrer, referrer_kind)?) } else { - Ok(to_file_specifier(&file_path)) + Ok(url_from_file_path(&file_path).unwrap()) } } @@ -1338,7 +1341,7 @@ impl<TEnv: NodeResolverEnv> NodeResolver<TEnv> { &self, url: &Url, ) -> Result<Option<PackageJsonRc>, ClosestPkgJsonError> { - let Ok(file_path) = url.to_file_path() else { + let Ok(file_path) = deno_path_util::url_to_file_path(url) else { return Ok(None); }; self.get_closest_package_json_from_path(&file_path) @@ -1433,7 +1436,7 @@ impl<TEnv: NodeResolverEnv> NodeResolver<TEnv> { if let Some(main) = maybe_main { let guess = package_json.path.parent().unwrap().join(main).clean(); if self.env.is_file_sync(&guess) { - return Ok(to_file_specifier(&guess)); + return Ok(url_from_file_path(&guess).unwrap()); } // todo(dsherret): investigate exactly how node and typescript handles this @@ -1463,7 +1466,7 @@ impl<TEnv: NodeResolverEnv> NodeResolver<TEnv> { .clean(); if self.env.is_file_sync(&guess) { // TODO(bartlomieju): emitLegacyIndexDeprecation() - return Ok(to_file_specifier(&guess)); + return Ok(url_from_file_path(&guess).unwrap()); } } } @@ -1496,14 +1499,15 @@ impl<TEnv: NodeResolverEnv> NodeResolver<TEnv> { let guess = directory.join(index_file_name).clean(); if self.env.is_file_sync(&guess) { // TODO(bartlomieju): emitLegacyIndexDeprecation() - return Ok(to_file_specifier(&guess)); + return Ok(url_from_file_path(&guess).unwrap()); } } if mode.is_types() { Err( TypesNotFoundError(Box::new(TypesNotFoundErrorData { - code_specifier: to_file_specifier(&directory.join("index.js")), + code_specifier: url_from_file_path(&directory.join("index.js")) + .unwrap(), maybe_referrer: maybe_referrer.cloned(), })) .into(), @@ -1511,7 +1515,7 @@ impl<TEnv: NodeResolverEnv> NodeResolver<TEnv> { } else { Err( ModuleNotFoundError { - specifier: to_file_specifier(&directory.join("index.js")), + specifier: url_from_file_path(&directory.join("index.js")).unwrap(), typ: "module", maybe_referrer: maybe_referrer.cloned(), } @@ -1611,9 +1615,7 @@ fn resolve_bin_entry_value<'a>( } fn to_file_path(url: &Url) -> PathBuf { - url - .to_file_path() - .unwrap_or_else(|_| panic!("Provided URL was not file:// URL: {url}")) + deno_path_util::url_to_file_path(url).unwrap() } fn to_file_path_string(url: &Url) -> String { @@ -1692,7 +1694,7 @@ fn with_known_extension(path: &Path, ext: &str) -> PathBuf { } fn to_specifier_display_string(url: &Url) -> String { - if let Ok(path) = url.to_file_path() { + if let Ok(path) = deno_path_util::url_to_file_path(url) { path.display().to_string() } else { url.to_string() |