diff options
Diffstat (limited to 'cli/npm/resolvers')
-rw-r--r-- | cli/npm/resolvers/common.rs | 54 | ||||
-rw-r--r-- | cli/npm/resolvers/global.rs | 64 | ||||
-rw-r--r-- | cli/npm/resolvers/local.rs | 321 | ||||
-rw-r--r-- | cli/npm/resolvers/mod.rs | 91 |
4 files changed, 434 insertions, 96 deletions
diff --git a/cli/npm/resolvers/common.rs b/cli/npm/resolvers/common.rs index f0231859a..cc590e2ad 100644 --- a/cli/npm/resolvers/common.rs +++ b/cli/npm/resolvers/common.rs @@ -1,3 +1,6 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +use std::io::ErrorKind; use std::path::Path; use std::path::PathBuf; @@ -9,34 +12,25 @@ use deno_core::futures::future::BoxFuture; use deno_core::url::Url; use crate::npm::NpmCache; -use crate::npm::NpmPackageId; use crate::npm::NpmPackageReq; use crate::npm::NpmResolutionPackage; -/// Information about the local npm package. -pub struct LocalNpmPackageInfo { - /// Unique identifier. - pub id: NpmPackageId, - /// Local folder path of the npm package. - pub folder_path: PathBuf, -} - pub trait InnerNpmPackageResolver: Send + Sync { - fn resolve_package_from_deno_module( + fn resolve_package_folder_from_deno_module( &self, pkg_req: &NpmPackageReq, - ) -> Result<LocalNpmPackageInfo, AnyError>; + ) -> Result<PathBuf, AnyError>; - fn resolve_package_from_package( + fn resolve_package_folder_from_package( &self, name: &str, referrer: &ModuleSpecifier, - ) -> Result<LocalNpmPackageInfo, AnyError>; + ) -> Result<PathBuf, AnyError>; - fn resolve_package_from_specifier( + fn resolve_package_folder_from_specifier( &self, specifier: &ModuleSpecifier, - ) -> Result<LocalNpmPackageInfo, AnyError>; + ) -> Result<PathBuf, AnyError>; fn has_packages(&self) -> bool; @@ -87,3 +81,33 @@ pub async fn cache_packages( } Ok(()) } + +pub fn ensure_registry_read_permission( + registry_path: &Path, + path: &Path, +) -> Result<(), AnyError> { + // allow reading if it's in the node_modules + if path.starts_with(®istry_path) + && path + .components() + .all(|c| !matches!(c, std::path::Component::ParentDir)) + { + // todo(dsherret): cache this? + if let Ok(registry_path) = std::fs::canonicalize(registry_path) { + match std::fs::canonicalize(path) { + Ok(path) if path.starts_with(registry_path) => { + return Ok(()); + } + Err(e) if e.kind() == ErrorKind::NotFound => { + return Ok(()); + } + _ => {} // ignore + } + } + } + + Err(deno_core::error::custom_error( + "PermissionDenied", + format!("Reading {} is not allowed", path.display()), + )) +} diff --git a/cli/npm/resolvers/global.rs b/cli/npm/resolvers/global.rs index 259d9b9a0..94b963898 100644 --- a/cli/npm/resolvers/global.rs +++ b/cli/npm/resolvers/global.rs @@ -1,7 +1,9 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. -use std::io::ErrorKind; +//! Code for global npm cache resolution. + use std::path::Path; +use std::path::PathBuf; use std::sync::Arc; use deno_ast::ModuleSpecifier; @@ -17,9 +19,10 @@ use crate::npm::NpmPackageId; use crate::npm::NpmPackageReq; use crate::npm::NpmRegistryApi; +use super::common::ensure_registry_read_permission; use super::common::InnerNpmPackageResolver; -use super::common::LocalNpmPackageInfo; +/// Resolves packages from the global npm cache. #[derive(Debug, Clone)] pub struct GlobalNpmPackageResolver { cache: NpmCache, @@ -39,45 +42,42 @@ impl GlobalNpmPackageResolver { } } - fn local_package_info(&self, id: &NpmPackageId) -> LocalNpmPackageInfo { - LocalNpmPackageInfo { - folder_path: self.cache.package_folder(id, &self.registry_url), - id: id.clone(), - } + fn package_folder(&self, id: &NpmPackageId) -> PathBuf { + self.cache.package_folder(id, &self.registry_url) } } impl InnerNpmPackageResolver for GlobalNpmPackageResolver { - fn resolve_package_from_deno_module( + fn resolve_package_folder_from_deno_module( &self, pkg_req: &NpmPackageReq, - ) -> Result<LocalNpmPackageInfo, AnyError> { + ) -> Result<PathBuf, AnyError> { let pkg = self.resolution.resolve_package_from_deno_module(pkg_req)?; - Ok(self.local_package_info(&pkg.id)) + Ok(self.package_folder(&pkg.id)) } - fn resolve_package_from_package( + fn resolve_package_folder_from_package( &self, name: &str, referrer: &ModuleSpecifier, - ) -> Result<LocalNpmPackageInfo, AnyError> { + ) -> Result<PathBuf, AnyError> { let referrer_pkg_id = self .cache .resolve_package_id_from_specifier(referrer, &self.registry_url)?; let pkg = self .resolution .resolve_package_from_package(name, &referrer_pkg_id)?; - Ok(self.local_package_info(&pkg.id)) + Ok(self.package_folder(&pkg.id)) } - fn resolve_package_from_specifier( + fn resolve_package_folder_from_specifier( &self, specifier: &ModuleSpecifier, - ) -> Result<LocalNpmPackageInfo, AnyError> { + ) -> Result<PathBuf, AnyError> { let pkg_id = self .cache .resolve_package_id_from_specifier(specifier, &self.registry_url)?; - Ok(self.local_package_info(&pkg_id)) + Ok(self.package_folder(&pkg_id)) } fn has_packages(&self) -> bool { @@ -103,36 +103,6 @@ impl InnerNpmPackageResolver for GlobalNpmPackageResolver { fn ensure_read_permission(&self, path: &Path) -> Result<(), AnyError> { let registry_path = self.cache.registry_folder(&self.registry_url); - ensure_read_permission(®istry_path, path) + ensure_registry_read_permission(®istry_path, path) } } - -fn ensure_read_permission( - registry_path: &Path, - path: &Path, -) -> Result<(), AnyError> { - // allow reading if it's in the deno_dir node modules - if path.starts_with(®istry_path) - && path - .components() - .all(|c| !matches!(c, std::path::Component::ParentDir)) - { - // todo(dsherret): cache this? - if let Ok(registry_path) = std::fs::canonicalize(registry_path) { - match std::fs::canonicalize(path) { - Ok(path) if path.starts_with(registry_path) => { - return Ok(()); - } - Err(e) if e.kind() == ErrorKind::NotFound => { - return Ok(()); - } - _ => {} // ignore - } - } - } - - Err(deno_core::error::custom_error( - "PermissionDenied", - format!("Reading {} is not allowed", path.display()), - )) -} diff --git a/cli/npm/resolvers/local.rs b/cli/npm/resolvers/local.rs new file mode 100644 index 000000000..d92ffb84d --- /dev/null +++ b/cli/npm/resolvers/local.rs @@ -0,0 +1,321 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +//! Code for local node_modules resolution. + +use std::collections::HashSet; +use std::collections::VecDeque; +use std::fs; +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; + +use deno_ast::ModuleSpecifier; +use deno_core::anyhow::bail; +use deno_core::anyhow::Context; +use deno_core::error::AnyError; +use deno_core::futures::future::BoxFuture; +use deno_core::futures::FutureExt; +use deno_core::url::Url; + +use crate::fs_util; +use crate::npm::resolution::NpmResolution; +use crate::npm::resolution::NpmResolutionSnapshot; +use crate::npm::NpmCache; +use crate::npm::NpmPackageId; +use crate::npm::NpmPackageReq; +use crate::npm::NpmRegistryApi; + +use super::common::cache_packages; +use super::common::ensure_registry_read_permission; +use super::common::InnerNpmPackageResolver; + +/// Resolver that creates a local node_modules directory +/// and resolves packages from it. +#[derive(Debug, Clone)] +pub struct LocalNpmPackageResolver { + cache: NpmCache, + resolution: Arc<NpmResolution>, + registry_url: Url, + root_node_modules_path: PathBuf, + root_node_modules_specifier: ModuleSpecifier, +} + +impl LocalNpmPackageResolver { + pub fn new( + cache: NpmCache, + api: NpmRegistryApi, + node_modules_folder: PathBuf, + ) -> Self { + let registry_url = api.base_url().to_owned(); + let resolution = Arc::new(NpmResolution::new(api)); + + Self { + cache, + resolution, + registry_url, + root_node_modules_specifier: ModuleSpecifier::from_directory_path( + &node_modules_folder, + ) + .unwrap(), + root_node_modules_path: node_modules_folder, + } + } + + fn resolve_package_root(&self, path: &Path) -> PathBuf { + let mut last_found = path; + loop { + let parent = last_found.parent().unwrap(); + if parent.file_name().unwrap() == "node_modules" { + return last_found.to_path_buf(); + } else { + last_found = parent; + } + } + } + + fn resolve_folder_for_specifier( + &self, + specifier: &ModuleSpecifier, + ) -> Result<PathBuf, AnyError> { + match self.maybe_resolve_folder_for_specifier(specifier) { + Some(path) => Ok(path), + None => bail!("could not find npm package for '{}'", specifier), + } + } + + fn maybe_resolve_folder_for_specifier( + &self, + specifier: &ModuleSpecifier, + ) -> Option<PathBuf> { + let relative_url = + self.root_node_modules_specifier.make_relative(specifier)?; + if relative_url.starts_with("../") { + return None; + } + // it's within the directory, so use it + specifier.to_file_path().ok() + } +} + +impl InnerNpmPackageResolver for LocalNpmPackageResolver { + fn resolve_package_folder_from_deno_module( + &self, + pkg_req: &NpmPackageReq, + ) -> Result<PathBuf, AnyError> { + let resolved_package = + self.resolution.resolve_package_from_deno_module(pkg_req)?; + + // it might be at the full path if there are duplicate names + let fully_resolved_folder_path = join_package_name( + &self.root_node_modules_path, + &resolved_package.id.to_string(), + ); + Ok(if fully_resolved_folder_path.exists() { + fully_resolved_folder_path + } else { + join_package_name(&self.root_node_modules_path, &resolved_package.id.name) + }) + } + + fn resolve_package_folder_from_package( + &self, + name: &str, + referrer: &ModuleSpecifier, + ) -> Result<PathBuf, AnyError> { + let local_path = self.resolve_folder_for_specifier(referrer)?; + let package_root_path = self.resolve_package_root(&local_path); + let mut current_folder = package_root_path.as_path(); + loop { + current_folder = get_next_node_modules_ancestor(current_folder); + let sub_dir = join_package_name(current_folder, name); + if sub_dir.is_dir() { + return Ok(sub_dir); + } + if current_folder == self.root_node_modules_path { + bail!( + "could not find package '{}' from referrer '{}'.", + name, + referrer + ); + } + } + } + + fn resolve_package_folder_from_specifier( + &self, + specifier: &ModuleSpecifier, + ) -> Result<PathBuf, AnyError> { + let local_path = self.resolve_folder_for_specifier(specifier)?; + let package_root_path = self.resolve_package_root(&local_path); + Ok(package_root_path) + } + + fn has_packages(&self) -> bool { + self.resolution.has_packages() + } + + fn add_package_reqs( + &self, + packages: Vec<NpmPackageReq>, + ) -> BoxFuture<'static, Result<(), AnyError>> { + let resolver = self.clone(); + async move { + resolver.resolution.add_package_reqs(packages).await?; + cache_packages( + resolver.resolution.all_packages(), + &resolver.cache, + &resolver.registry_url, + ) + .await?; + + sync_resolution_with_fs( + &resolver.resolution.snapshot(), + &resolver.cache, + &resolver.registry_url, + &resolver.root_node_modules_path, + )?; + + Ok(()) + } + .boxed() + } + + fn ensure_read_permission(&self, path: &Path) -> Result<(), AnyError> { + ensure_registry_read_permission(&self.root_node_modules_path, path) + } +} + +/// Creates a pnpm style folder structure. +fn sync_resolution_with_fs( + snapshot: &NpmResolutionSnapshot, + cache: &NpmCache, + registry_url: &Url, + root_node_modules_dir_path: &Path, +) -> Result<(), AnyError> { + fn get_package_folder_name(package_id: &NpmPackageId) -> String { + package_id.to_string().replace('/', "+") + } + + let deno_local_registry_dir = root_node_modules_dir_path.join(".deno"); + fs::create_dir_all(&deno_local_registry_dir).with_context(|| { + format!("Creating '{}'", deno_local_registry_dir.display()) + })?; + + // 1. Write all the packages out the .deno directory. + // + // Copy (hardlink in future) <global_registry_cache>/<package_id>/ to + // node_modules/.deno/<package_id>/node_modules/<package_name> + let all_packages = snapshot.all_packages(); + for package in &all_packages { + let folder_name = get_package_folder_name(&package.id); + let folder_path = deno_local_registry_dir.join(&folder_name); + let initialized_file = folder_path.join("deno_initialized"); + if !initialized_file.exists() { + let sub_node_modules = folder_path.join("node_modules"); + let package_path = join_package_name(&sub_node_modules, &package.id.name); + fs::create_dir_all(&package_path) + .with_context(|| format!("Creating '{}'", folder_path.display()))?; + let cache_folder = cache.package_folder(&package.id, registry_url); + // for now copy, but in the future consider hard linking + fs_util::copy_dir_recursive(&cache_folder, &package_path)?; + // write out a file that indicates this folder has been initialized + fs::write(initialized_file, "")?; + } + } + + // 2. Symlink all the dependencies into the .deno directory. + // + // Symlink node_modules/.deno/<package_id>/node_modules/<dep_name> to + // node_modules/.deno/<dep_id>/node_modules/<dep_package_name> + for package in &all_packages { + let sub_node_modules = deno_local_registry_dir + .join(&get_package_folder_name(&package.id)) + .join("node_modules"); + for (name, dep_id) in &package.dependencies { + let dep_folder_name = get_package_folder_name(dep_id); + let dep_folder_path = join_package_name( + &deno_local_registry_dir + .join(dep_folder_name) + .join("node_modules"), + &dep_id.name, + ); + symlink_package_dir( + &dep_folder_path, + &join_package_name(&sub_node_modules, name), + )?; + } + } + + // 3. Create all the packages in the node_modules folder, which are symlinks. + // + // Symlink node_modules/<package_name> to + // node_modules/.deno/<package_id>/node_modules/<package_name> + let mut found_names = HashSet::new(); + let mut pending_packages = VecDeque::new(); + pending_packages.extend( + snapshot + .top_level_packages() + .into_iter() + .map(|id| (id, true)), + ); + while let Some((package_id, is_top_level)) = pending_packages.pop_front() { + let root_folder_name = if found_names.insert(package_id.name.clone()) { + package_id.name.clone() + } else if is_top_level { + package_id.to_string() + } else { + continue; // skip, already handled + }; + let local_registry_package_path = deno_local_registry_dir + .join(&get_package_folder_name(&package_id)) + .join("node_modules") + .join(&package_id.name); + + symlink_package_dir( + &local_registry_package_path, + &join_package_name(root_node_modules_dir_path, &root_folder_name), + )?; + if let Some(package) = snapshot.package_from_id(&package_id) { + for id in package.dependencies.values() { + pending_packages.push_back((id.clone(), false)); + } + } + } + + Ok(()) +} + +fn symlink_package_dir( + old_path: &Path, + new_path: &Path, +) -> Result<(), AnyError> { + let new_parent = new_path.parent().unwrap(); + if new_parent.file_name().unwrap() != "node_modules" { + // create the parent folder that will contain the symlink + fs::create_dir_all(new_parent) + .with_context(|| format!("Creating '{}'", new_parent.display()))?; + } + + // need to delete the previous symlink before creating a new one + let _ignore = fs::remove_dir_all(new_path); + fs_util::symlink_dir(old_path, new_path) +} + +fn join_package_name(path: &Path, package_name: &str) -> PathBuf { + let mut path = path.to_path_buf(); + // ensure backslashes are used on windows + for part in package_name.split('/') { + path = path.join(part); + } + path +} + +fn get_next_node_modules_ancestor(mut path: &Path) -> &Path { + loop { + path = path.parent().unwrap(); + let file_name = path.file_name().unwrap().to_string_lossy(); + if file_name == "node_modules" { + return path; + } + } +} diff --git a/cli/npm/resolvers/mod.rs b/cli/npm/resolvers/mod.rs index 02e5be983..3a40340f0 100644 --- a/cli/npm/resolvers/mod.rs +++ b/cli/npm/resolvers/mod.rs @@ -2,9 +2,13 @@ mod common; mod global; +mod local; +use deno_ast::ModuleSpecifier; use deno_core::anyhow::bail; use deno_core::error::custom_error; +use deno_core::error::AnyError; +use deno_runtime::deno_node::PathClean; use deno_runtime::deno_node::RequireNpmResolver; use global::GlobalNpmPackageResolver; @@ -12,16 +16,14 @@ use std::path::Path; use std::path::PathBuf; use std::sync::Arc; -use deno_ast::ModuleSpecifier; -use deno_core::error::AnyError; +use crate::fs_util; use self::common::InnerNpmPackageResolver; +use self::local::LocalNpmPackageResolver; use super::NpmCache; use super::NpmPackageReq; use super::NpmRegistryApi; -pub use self::common::LocalNpmPackageInfo; - #[derive(Clone)] pub struct NpmPackageResolver { unstable: bool, @@ -35,10 +37,17 @@ impl NpmPackageResolver { api: NpmRegistryApi, unstable: bool, no_npm: bool, + local_node_modules_path: Option<PathBuf>, ) -> Self { - // For now, always create a GlobalNpmPackageResolver, but in the future - // this might be a local node_modules folder - let inner = Arc::new(GlobalNpmPackageResolver::new(cache, api)); + let inner: Arc<dyn InnerNpmPackageResolver> = match local_node_modules_path + { + Some(node_modules_folder) => Arc::new(LocalNpmPackageResolver::new( + cache, + api, + node_modules_folder, + )), + None => Arc::new(GlobalNpmPackageResolver::new(cache, api)), + }; Self { unstable, no_npm, @@ -46,36 +55,51 @@ impl NpmPackageResolver { } } - /// Resolves an npm package from a Deno module. - pub fn resolve_package_from_deno_module( + /// Resolves an npm package folder path from a Deno module. + pub fn resolve_package_folder_from_deno_module( &self, pkg_req: &NpmPackageReq, - ) -> Result<LocalNpmPackageInfo, AnyError> { - self.inner.resolve_package_from_deno_module(pkg_req) + ) -> Result<PathBuf, AnyError> { + let path = self + .inner + .resolve_package_folder_from_deno_module(pkg_req)?; + let path = fs_util::canonicalize_path_maybe_not_exists(&path)?; + log::debug!("Resolved {} to {}", pkg_req, path.display()); + Ok(path) } - /// Resolves an npm package from an npm package referrer. - pub fn resolve_package_from_package( + /// Resolves an npm package folder path from an npm package referrer. + pub fn resolve_package_folder_from_package( &self, name: &str, referrer: &ModuleSpecifier, - ) -> Result<LocalNpmPackageInfo, AnyError> { - self.inner.resolve_package_from_package(name, referrer) + ) -> Result<PathBuf, AnyError> { + let path = self + .inner + .resolve_package_folder_from_package(name, referrer)?; + log::debug!("Resolved {} from {} to {}", name, referrer, path.display()); + Ok(path) } /// Resolve the root folder of the package the provided specifier is in. /// /// This will error when the provided specifier is not in an npm package. - pub fn resolve_package_from_specifier( + pub fn resolve_package_folder_from_specifier( &self, specifier: &ModuleSpecifier, - ) -> Result<LocalNpmPackageInfo, AnyError> { - self.inner.resolve_package_from_specifier(specifier) + ) -> Result<PathBuf, AnyError> { + let path = self + .inner + .resolve_package_folder_from_specifier(specifier)?; + log::debug!("Resolved {} to {}", specifier, path.display()); + Ok(path) } /// Gets if the provided specifier is in an npm package. pub fn in_npm_package(&self, specifier: &ModuleSpecifier) -> bool { - self.resolve_package_from_specifier(specifier).is_ok() + self + .resolve_package_folder_from_specifier(specifier) + .is_ok() } /// If the resolver has resolved any npm packages. @@ -121,28 +145,27 @@ impl RequireNpmResolver for NpmPackageResolver { specifier: &str, referrer: &std::path::Path, ) -> Result<PathBuf, AnyError> { - let referrer = specifier_to_path(referrer)?; - self - .resolve_package_from_package(specifier, &referrer) - .map(|p| p.folder_path) + let referrer = path_to_specifier(referrer)?; + self.resolve_package_folder_from_package(specifier, &referrer) } fn resolve_package_folder_from_path( &self, path: &Path, ) -> Result<PathBuf, AnyError> { - let specifier = specifier_to_path(path)?; - self - .resolve_package_from_specifier(&specifier) - .map(|p| p.folder_path) + let specifier = path_to_specifier(path)?; + self.resolve_package_folder_from_specifier(&specifier) } fn in_npm_package(&self, path: &Path) -> bool { - let specifier = match ModuleSpecifier::from_file_path(path) { - Ok(p) => p, - Err(_) => return false, - }; - self.resolve_package_from_specifier(&specifier).is_ok() + let specifier = + match ModuleSpecifier::from_file_path(&path.to_path_buf().clean()) { + Ok(p) => p, + Err(_) => return false, + }; + self + .resolve_package_folder_from_specifier(&specifier) + .is_ok() } fn ensure_read_permission(&self, path: &Path) -> Result<(), AnyError> { @@ -150,8 +173,8 @@ impl RequireNpmResolver for NpmPackageResolver { } } -fn specifier_to_path(path: &Path) -> Result<ModuleSpecifier, AnyError> { - match ModuleSpecifier::from_file_path(&path) { +fn path_to_specifier(path: &Path) -> Result<ModuleSpecifier, AnyError> { + match ModuleSpecifier::from_file_path(&path.to_path_buf().clean()) { Ok(specifier) => Ok(specifier), Err(()) => bail!("Could not convert '{}' to url.", path.display()), } |