diff options
Diffstat (limited to 'cli/npm/resolvers')
| -rw-r--r-- | cli/npm/resolvers/common.rs | 8 | ||||
| -rw-r--r-- | cli/npm/resolvers/global.rs | 43 | ||||
| -rw-r--r-- | cli/npm/resolvers/local.rs | 174 | ||||
| -rw-r--r-- | cli/npm/resolvers/mod.rs | 18 |
4 files changed, 172 insertions, 71 deletions
diff --git a/cli/npm/resolvers/common.rs b/cli/npm/resolvers/common.rs index 07996c4e1..32b8293cd 100644 --- a/cli/npm/resolvers/common.rs +++ b/cli/npm/resolvers/common.rs @@ -70,13 +70,19 @@ pub async fn cache_packages( // and we want the output to be deterministic packages.sort_by(|a, b| a.id.cmp(&b.id)); } + let mut handles = Vec::with_capacity(packages.len()); for package in packages { + assert_eq!(package.copy_index, 0); // the caller should not provide any of these let cache = cache.clone(); let registry_url = registry_url.clone(); let handle = tokio::task::spawn(async move { cache - .ensure_package(&package.id, &package.dist, ®istry_url) + .ensure_package( + (package.id.name.as_str(), &package.id.version), + &package.dist, + ®istry_url, + ) .await }); if sync_download { diff --git a/cli/npm/resolvers/global.rs b/cli/npm/resolvers/global.rs index 42090415a..474cb55d6 100644 --- a/cli/npm/resolvers/global.rs +++ b/cli/npm/resolvers/global.rs @@ -23,7 +23,7 @@ use crate::npm::resolvers::common::cache_packages; use crate::npm::NpmCache; use crate::npm::NpmPackageId; use crate::npm::NpmPackageReq; -use crate::npm::NpmRegistryApi; +use crate::npm::RealNpmRegistryApi; use super::common::ensure_registry_read_permission; use super::common::InnerNpmPackageResolver; @@ -39,7 +39,7 @@ pub struct GlobalNpmPackageResolver { impl GlobalNpmPackageResolver { pub fn new( cache: NpmCache, - api: NpmRegistryApi, + api: RealNpmRegistryApi, initial_snapshot: Option<NpmResolutionSnapshot>, ) -> Self { let registry_url = api.base_url().to_owned(); @@ -53,7 +53,13 @@ impl GlobalNpmPackageResolver { } fn package_folder(&self, id: &NpmPackageId) -> PathBuf { - self.cache.package_folder(id, &self.registry_url) + let folder_id = self + .resolution + .resolve_package_cache_folder_id_from_id(id) + .unwrap(); + self + .cache + .package_folder_for_id(&folder_id, &self.registry_url) } } @@ -74,7 +80,7 @@ impl InnerNpmPackageResolver for GlobalNpmPackageResolver { ) -> Result<PathBuf, AnyError> { let referrer_pkg_id = self .cache - .resolve_package_id_from_specifier(referrer, &self.registry_url)?; + .resolve_package_folder_id_from_specifier(referrer, &self.registry_url)?; let pkg_result = self .resolution .resolve_package_from_package(name, &referrer_pkg_id); @@ -105,10 +111,15 @@ impl InnerNpmPackageResolver for GlobalNpmPackageResolver { &self, specifier: &ModuleSpecifier, ) -> Result<PathBuf, AnyError> { - let pkg_id = self - .cache - .resolve_package_id_from_specifier(specifier, &self.registry_url)?; - Ok(self.package_folder(&pkg_id)) + let pkg_folder_id = self.cache.resolve_package_folder_id_from_specifier( + specifier, + &self.registry_url, + )?; + Ok( + self + .cache + .package_folder_for_id(&pkg_folder_id, &self.registry_url), + ) } fn package_size(&self, package_id: &NpmPackageId) -> Result<u64, AnyError> { @@ -162,10 +173,22 @@ impl InnerNpmPackageResolver for GlobalNpmPackageResolver { async fn cache_packages_in_resolver( resolver: &GlobalNpmPackageResolver, ) -> Result<(), AnyError> { + let package_partitions = resolver.resolution.all_packages_partitioned(); + cache_packages( - resolver.resolution.all_packages(), + package_partitions.packages, &resolver.cache, &resolver.registry_url, ) - .await + .await?; + + // create the copy package folders + for copy in package_partitions.copy_packages { + resolver.cache.ensure_copy_package( + ©.get_package_cache_folder_id(), + &resolver.registry_url, + )?; + } + + Ok(()) } diff --git a/cli/npm/resolvers/local.rs b/cli/npm/resolvers/local.rs index cad940d56..678f776f3 100644 --- a/cli/npm/resolvers/local.rs +++ b/cli/npm/resolvers/local.rs @@ -24,12 +24,14 @@ use tokio::task::JoinHandle; use crate::fs_util; use crate::lockfile::Lockfile; use crate::npm::cache::should_sync_download; +use crate::npm::cache::NpmPackageCacheFolderId; use crate::npm::resolution::NpmResolution; use crate::npm::resolution::NpmResolutionSnapshot; use crate::npm::NpmCache; use crate::npm::NpmPackageId; use crate::npm::NpmPackageReq; -use crate::npm::NpmRegistryApi; +use crate::npm::NpmResolutionPackage; +use crate::npm::RealNpmRegistryApi; use super::common::ensure_registry_read_permission; use super::common::InnerNpmPackageResolver; @@ -48,7 +50,7 @@ pub struct LocalNpmPackageResolver { impl LocalNpmPackageResolver { pub fn new( cache: NpmCache, - api: NpmRegistryApi, + api: RealNpmRegistryApi, node_modules_folder: PathBuf, initial_snapshot: Option<NpmResolutionSnapshot>, ) -> Self { @@ -101,6 +103,35 @@ impl LocalNpmPackageResolver { // it's within the directory, so use it specifier.to_file_path().ok() } + + fn get_package_id_folder( + &self, + package_id: &NpmPackageId, + ) -> Result<PathBuf, AnyError> { + match self.resolution.resolve_package_from_id(package_id) { + Some(package) => Ok(self.get_package_id_folder_from_package(&package)), + None => bail!( + "Could not find package information for '{}'", + package_id.as_serialized() + ), + } + } + + fn get_package_id_folder_from_package( + &self, + package: &NpmResolutionPackage, + ) -> PathBuf { + // package is stored at: + // node_modules/.deno/<package_cache_folder_id_folder_name>/node_modules/<package_name> + self + .root_node_modules_path + .join(".deno") + .join(get_package_folder_id_folder_name( + &package.get_package_cache_folder_id(), + )) + .join("node_modules") + .join(&package.id.name) + } } impl InnerNpmPackageResolver for LocalNpmPackageResolver { @@ -108,19 +139,8 @@ impl InnerNpmPackageResolver for LocalNpmPackageResolver { &self, pkg_req: &NpmPackageReq, ) -> Result<PathBuf, AnyError> { - let resolved_package = - self.resolution.resolve_package_from_deno_module(pkg_req)?; - - // it might be at the full path if there are duplicate names - let fully_resolved_folder_path = join_package_name( - &self.root_node_modules_path, - &resolved_package.id.to_string(), - ); - Ok(if fully_resolved_folder_path.exists() { - fully_resolved_folder_path - } else { - join_package_name(&self.root_node_modules_path, &resolved_package.id.name) - }) + let package = self.resolution.resolve_package_from_deno_module(pkg_req)?; + Ok(self.get_package_id_folder_from_package(&package)) } fn resolve_package_folder_from_package( @@ -178,19 +198,9 @@ impl InnerNpmPackageResolver for LocalNpmPackageResolver { } fn package_size(&self, package_id: &NpmPackageId) -> Result<u64, AnyError> { - match self.resolution.resolve_package_from_id(package_id) { - Some(package) => Ok(fs_util::dir_size( - // package is stored at: - // node_modules/.deno/<package_id>/node_modules/<package_name> - &self - .root_node_modules_path - .join(".deno") - .join(package.id.to_string()) - .join("node_modules") - .join(package.id.name), - )?), - None => bail!("Could not find package folder for '{}'", package_id), - } + let package_folder_path = self.get_package_id_folder(package_id)?; + + Ok(fs_util::dir_size(&package_folder_path)?) } fn has_packages(&self) -> bool { @@ -255,10 +265,6 @@ async fn sync_resolution_with_fs( registry_url: &Url, root_node_modules_dir_path: &Path, ) -> Result<(), AnyError> { - fn get_package_folder_name(package_id: &NpmPackageId) -> String { - package_id.to_string().replace('/', "+") - } - let deno_local_registry_dir = root_node_modules_dir_path.join(".deno"); fs::create_dir_all(&deno_local_registry_dir).with_context(|| { format!("Creating '{}'", deno_local_registry_dir.display()) @@ -267,34 +273,45 @@ async fn sync_resolution_with_fs( // 1. Write all the packages out the .deno directory. // // Copy (hardlink in future) <global_registry_cache>/<package_id>/ to - // node_modules/.deno/<package_id>/node_modules/<package_name> + // node_modules/.deno/<package_folder_id_folder_name>/node_modules/<package_name> let sync_download = should_sync_download(); - let mut all_packages = snapshot.all_packages(); + let mut package_partitions = snapshot.all_packages_partitioned(); if sync_download { // we're running the tests not with --quiet // and we want the output to be deterministic - all_packages.sort_by(|a, b| a.id.cmp(&b.id)); + package_partitions.packages.sort_by(|a, b| a.id.cmp(&b.id)); } let mut handles: Vec<JoinHandle<Result<(), AnyError>>> = - Vec::with_capacity(all_packages.len()); - for package in &all_packages { - let folder_name = get_package_folder_name(&package.id); + Vec::with_capacity(package_partitions.packages.len()); + for package in &package_partitions.packages { + let folder_name = + get_package_folder_id_folder_name(&package.get_package_cache_folder_id()); let folder_path = deno_local_registry_dir.join(&folder_name); - let initialized_file = folder_path.join("deno_initialized"); - if !initialized_file.exists() { + let initialized_file = folder_path.join(".initialized"); + if !cache.should_use_cache_for_npm_package(&package.id.name) + || !initialized_file.exists() + { let cache = cache.clone(); let registry_url = registry_url.clone(); let package = package.clone(); let handle = tokio::task::spawn(async move { cache - .ensure_package(&package.id, &package.dist, ®istry_url) + .ensure_package( + (&package.id.name, &package.id.version), + &package.dist, + ®istry_url, + ) .await?; let sub_node_modules = folder_path.join("node_modules"); let package_path = join_package_name(&sub_node_modules, &package.id.name); fs::create_dir_all(&package_path) .with_context(|| format!("Creating '{}'", folder_path.display()))?; - let cache_folder = cache.package_folder(&package.id, ®istry_url); + let cache_folder = cache.package_folder_for_name_and_version( + &package.id.name, + &package.id.version, + ®istry_url, + ); // for now copy, but in the future consider hard linking fs_util::copy_dir_recursive(&cache_folder, &package_path)?; // write out a file that indicates this folder has been initialized @@ -314,16 +331,51 @@ async fn sync_resolution_with_fs( result??; // surface the first error } - // 2. Symlink all the dependencies into the .deno directory. + // 2. Create any "copy" packages, which are used for peer dependencies + for package in &package_partitions.copy_packages { + let package_cache_folder_id = package.get_package_cache_folder_id(); + let destination_path = deno_local_registry_dir + .join(&get_package_folder_id_folder_name(&package_cache_folder_id)); + let initialized_file = destination_path.join(".initialized"); + if !initialized_file.exists() { + let sub_node_modules = destination_path.join("node_modules"); + let package_path = join_package_name(&sub_node_modules, &package.id.name); + fs::create_dir_all(&package_path).with_context(|| { + format!("Creating '{}'", destination_path.display()) + })?; + let source_path = join_package_name( + &deno_local_registry_dir + .join(&get_package_folder_id_folder_name( + &package_cache_folder_id.with_no_count(), + )) + .join("node_modules"), + &package.id.name, + ); + fs_util::hard_link_dir_recursive(&source_path, &package_path)?; + // write out a file that indicates this folder has been initialized + fs::write(initialized_file, "")?; + } + } + + let all_packages = package_partitions.into_all(); + + // 3. Symlink all the dependencies into the .deno directory. // // Symlink node_modules/.deno/<package_id>/node_modules/<dep_name> to // node_modules/.deno/<dep_id>/node_modules/<dep_package_name> for package in &all_packages { let sub_node_modules = deno_local_registry_dir - .join(&get_package_folder_name(&package.id)) + .join(&get_package_folder_id_folder_name( + &package.get_package_cache_folder_id(), + )) .join("node_modules"); for (name, dep_id) in &package.dependencies { - let dep_folder_name = get_package_folder_name(dep_id); + let dep_cache_folder_id = snapshot + .package_from_id(dep_id) + .unwrap() + .get_package_cache_folder_id(); + let dep_folder_name = + get_package_folder_id_folder_name(&dep_cache_folder_id); let dep_folder_path = join_package_name( &deno_local_registry_dir .join(dep_folder_name) @@ -337,7 +389,7 @@ async fn sync_resolution_with_fs( } } - // 3. Create all the packages in the node_modules folder, which are symlinks. + // 4. Create all the packages in the node_modules folder, which are symlinks. // // Symlink node_modules/<package_name> to // node_modules/.deno/<package_id>/node_modules/<package_name> @@ -353,29 +405,41 @@ async fn sync_resolution_with_fs( let root_folder_name = if found_names.insert(package_id.name.clone()) { package_id.name.clone() } else if is_top_level { - package_id.to_string() + package_id.display() } else { continue; // skip, already handled }; - let local_registry_package_path = deno_local_registry_dir - .join(&get_package_folder_name(&package_id)) - .join("node_modules") - .join(&package_id.name); + let package = snapshot.package_from_id(&package_id).unwrap(); + let local_registry_package_path = join_package_name( + &deno_local_registry_dir + .join(&get_package_folder_id_folder_name( + &package.get_package_cache_folder_id(), + )) + .join("node_modules"), + &package_id.name, + ); symlink_package_dir( &local_registry_package_path, &join_package_name(root_node_modules_dir_path, &root_folder_name), )?; - if let Some(package) = snapshot.package_from_id(&package_id) { - for id in package.dependencies.values() { - pending_packages.push_back((id.clone(), false)); - } + for id in package.dependencies.values() { + pending_packages.push_back((id.clone(), false)); } } Ok(()) } +fn get_package_folder_id_folder_name(id: &NpmPackageCacheFolderId) -> String { + let copy_str = if id.copy_index == 0 { + "".to_string() + } else { + format!("_{}", id.copy_index) + }; + format!("{}@{}{}", id.name, id.version, copy_str).replace('/', "+") +} + fn symlink_package_dir( old_path: &Path, new_path: &Path, diff --git a/cli/npm/resolvers/mod.rs b/cli/npm/resolvers/mod.rs index 71c2abc00..6cd40594b 100644 --- a/cli/npm/resolvers/mod.rs +++ b/cli/npm/resolvers/mod.rs @@ -6,6 +6,7 @@ mod local; use deno_ast::ModuleSpecifier; use deno_core::anyhow::bail; +use deno_core::anyhow::Context; use deno_core::error::custom_error; use deno_core::error::AnyError; use deno_core::parking_lot::Mutex; @@ -29,8 +30,8 @@ use self::local::LocalNpmPackageResolver; use super::NpmCache; use super::NpmPackageId; use super::NpmPackageReq; -use super::NpmRegistryApi; use super::NpmResolutionSnapshot; +use super::RealNpmRegistryApi; const RESOLUTION_STATE_ENV_VAR_NAME: &str = "DENO_DONT_USE_INTERNAL_NODE_COMPAT_STATE"; @@ -71,7 +72,7 @@ pub struct NpmPackageResolver { no_npm: bool, inner: Arc<dyn InnerNpmPackageResolver>, local_node_modules_path: Option<PathBuf>, - api: NpmRegistryApi, + api: RealNpmRegistryApi, cache: NpmCache, maybe_lockfile: Option<Arc<Mutex<Lockfile>>>, } @@ -90,7 +91,7 @@ impl std::fmt::Debug for NpmPackageResolver { impl NpmPackageResolver { pub fn new( cache: NpmCache, - api: NpmRegistryApi, + api: RealNpmRegistryApi, unstable: bool, no_npm: bool, local_node_modules_path: Option<PathBuf>, @@ -112,7 +113,14 @@ impl NpmPackageResolver { lockfile: Arc<Mutex<Lockfile>>, ) -> Result<(), AnyError> { let snapshot = - NpmResolutionSnapshot::from_lockfile(lockfile.clone(), &self.api).await?; + NpmResolutionSnapshot::from_lockfile(lockfile.clone(), &self.api) + .await + .with_context(|| { + format!( + "failed reading lockfile '{}'", + lockfile.lock().filename.display() + ) + })?; self.maybe_lockfile = Some(lockfile); if let Some(node_modules_folder) = &self.local_node_modules_path { self.inner = Arc::new(LocalNpmPackageResolver::new( @@ -133,7 +141,7 @@ impl NpmPackageResolver { fn new_with_maybe_snapshot( cache: NpmCache, - api: NpmRegistryApi, + api: RealNpmRegistryApi, unstable: bool, no_npm: bool, local_node_modules_path: Option<PathBuf>, |