diff options
Diffstat (limited to 'cli/npm/managed/resolution.rs')
| -rw-r--r-- | cli/npm/managed/resolution.rs | 433 |
1 files changed, 433 insertions, 0 deletions
diff --git a/cli/npm/managed/resolution.rs b/cli/npm/managed/resolution.rs new file mode 100644 index 000000000..05c1227a7 --- /dev/null +++ b/cli/npm/managed/resolution.rs @@ -0,0 +1,433 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +use std::collections::HashMap; +use std::collections::HashSet; +use std::sync::Arc; + +use deno_core::error::AnyError; +use deno_core::parking_lot::Mutex; +use deno_core::parking_lot::RwLock; +use deno_lockfile::NpmPackageDependencyLockfileInfo; +use deno_lockfile::NpmPackageLockfileInfo; +use deno_npm::registry::NpmPackageInfo; +use deno_npm::registry::NpmPackageVersionDistInfoIntegrity; +use deno_npm::registry::NpmRegistryApi; +use deno_npm::resolution::NpmPackageVersionResolutionError; +use deno_npm::resolution::NpmPackagesPartitioned; +use deno_npm::resolution::NpmResolutionError; +use deno_npm::resolution::NpmResolutionSnapshot; +use deno_npm::resolution::NpmResolutionSnapshotPendingResolver; +use deno_npm::resolution::NpmResolutionSnapshotPendingResolverOptions; +use deno_npm::resolution::PackageCacheFolderIdNotFoundError; +use deno_npm::resolution::PackageNotFoundFromReferrerError; +use deno_npm::resolution::PackageNvNotFoundError; +use deno_npm::resolution::PackageReqNotFoundError; +use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; +use deno_npm::NpmPackageCacheFolderId; +use deno_npm::NpmPackageId; +use deno_npm::NpmResolutionPackage; +use deno_npm::NpmSystemInfo; +use deno_semver::package::PackageNv; +use deno_semver::package::PackageReq; +use deno_semver::VersionReq; + +use crate::args::Lockfile; +use crate::util::sync::TaskQueue; + +use super::super::registry::CliNpmRegistryApi; + +/// Handles updating and storing npm resolution in memory where the underlying +/// snapshot can be updated concurrently. Additionally handles updating the lockfile +/// based on changes to the resolution. +/// +/// This does not interact with the file system. +pub struct NpmResolution { + api: Arc<CliNpmRegistryApi>, + snapshot: RwLock<NpmResolutionSnapshot>, + update_queue: TaskQueue, + maybe_lockfile: Option<Arc<Mutex<Lockfile>>>, +} + +impl std::fmt::Debug for NpmResolution { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let snapshot = self.snapshot.read(); + f.debug_struct("NpmResolution") + .field("snapshot", &snapshot.as_valid_serialized().as_serialized()) + .finish() + } +} + +impl NpmResolution { + pub fn from_serialized( + api: Arc<CliNpmRegistryApi>, + initial_snapshot: Option<ValidSerializedNpmResolutionSnapshot>, + maybe_lockfile: Option<Arc<Mutex<Lockfile>>>, + ) -> Self { + let snapshot = + NpmResolutionSnapshot::new(initial_snapshot.unwrap_or_default()); + Self::new(api, snapshot, maybe_lockfile) + } + + pub fn new( + api: Arc<CliNpmRegistryApi>, + initial_snapshot: NpmResolutionSnapshot, + maybe_lockfile: Option<Arc<Mutex<Lockfile>>>, + ) -> Self { + Self { + api, + snapshot: RwLock::new(initial_snapshot), + update_queue: Default::default(), + maybe_lockfile, + } + } + + pub async fn add_package_reqs( + &self, + package_reqs: &[PackageReq], + ) -> Result<(), AnyError> { + // only allow one thread in here at a time + let _permit = self.update_queue.acquire().await; + let snapshot = add_package_reqs_to_snapshot( + &self.api, + package_reqs, + self.maybe_lockfile.clone(), + || self.snapshot.read().clone(), + ) + .await?; + + *self.snapshot.write() = snapshot; + Ok(()) + } + + pub async fn set_package_reqs( + &self, + package_reqs: &[PackageReq], + ) -> Result<(), AnyError> { + // only allow one thread in here at a time + let _permit = self.update_queue.acquire().await; + + let reqs_set = package_reqs.iter().collect::<HashSet<_>>(); + let snapshot = add_package_reqs_to_snapshot( + &self.api, + package_reqs, + self.maybe_lockfile.clone(), + || { + let snapshot = self.snapshot.read().clone(); + let has_removed_package = !snapshot + .package_reqs() + .keys() + .all(|req| reqs_set.contains(req)); + // if any packages were removed, we need to completely recreate the npm resolution snapshot + if has_removed_package { + snapshot.into_empty() + } else { + snapshot + } + }, + ) + .await?; + + *self.snapshot.write() = snapshot; + + Ok(()) + } + + pub async fn resolve_pending(&self) -> Result<(), AnyError> { + // only allow one thread in here at a time + let _permit = self.update_queue.acquire().await; + + let snapshot = add_package_reqs_to_snapshot( + &self.api, + &Vec::new(), + self.maybe_lockfile.clone(), + || self.snapshot.read().clone(), + ) + .await?; + + *self.snapshot.write() = snapshot; + + Ok(()) + } + + pub fn resolve_pkg_cache_folder_id_from_pkg_id( + &self, + id: &NpmPackageId, + ) -> Option<NpmPackageCacheFolderId> { + self + .snapshot + .read() + .package_from_id(id) + .map(|p| p.get_package_cache_folder_id()) + } + + pub fn resolve_pkg_id_from_pkg_cache_folder_id( + &self, + id: &NpmPackageCacheFolderId, + ) -> Result<NpmPackageId, PackageCacheFolderIdNotFoundError> { + self + .snapshot + .read() + .resolve_pkg_from_pkg_cache_folder_id(id) + .map(|pkg| pkg.id.clone()) + } + + pub fn resolve_package_from_package( + &self, + name: &str, + referrer: &NpmPackageCacheFolderId, + ) -> Result<NpmResolutionPackage, Box<PackageNotFoundFromReferrerError>> { + self + .snapshot + .read() + .resolve_package_from_package(name, referrer) + .cloned() + } + + /// Resolve a node package from a deno module. + pub fn resolve_pkg_id_from_pkg_req( + &self, + req: &PackageReq, + ) -> Result<NpmPackageId, PackageReqNotFoundError> { + self + .snapshot + .read() + .resolve_pkg_from_pkg_req(req) + .map(|pkg| pkg.id.clone()) + } + + pub fn resolve_pkg_reqs_from_pkg_id( + &self, + id: &NpmPackageId, + ) -> Vec<PackageReq> { + let snapshot = self.snapshot.read(); + let mut pkg_reqs = snapshot + .package_reqs() + .iter() + .filter(|(_, nv)| *nv == &id.nv) + .map(|(req, _)| req.clone()) + .collect::<Vec<_>>(); + pkg_reqs.sort(); // be deterministic + pkg_reqs + } + + pub fn resolve_pkg_id_from_deno_module( + &self, + id: &PackageNv, + ) -> Result<NpmPackageId, PackageNvNotFoundError> { + self + .snapshot + .read() + .resolve_package_from_deno_module(id) + .map(|pkg| pkg.id.clone()) + } + + // todo: NEXT + + /// Resolves a package requirement for deno graph. This should only be + /// called by deno_graph's NpmResolver or for resolving packages in + /// a package.json + pub fn resolve_pkg_req_as_pending( + &self, + pkg_req: &PackageReq, + ) -> Result<PackageNv, NpmPackageVersionResolutionError> { + // we should always have this because it should have been cached before here + let package_info = self.api.get_cached_package_info(&pkg_req.name).unwrap(); + self.resolve_pkg_req_as_pending_with_info(pkg_req, &package_info) + } + + /// Resolves a package requirement for deno graph. This should only be + /// called by deno_graph's NpmResolver or for resolving packages in + /// a package.json + pub fn resolve_pkg_req_as_pending_with_info( + &self, + pkg_req: &PackageReq, + package_info: &NpmPackageInfo, + ) -> Result<PackageNv, NpmPackageVersionResolutionError> { + debug_assert_eq!(pkg_req.name, package_info.name); + let mut snapshot = self.snapshot.write(); + let pending_resolver = get_npm_pending_resolver(&self.api); + let nv = pending_resolver.resolve_package_req_as_pending( + &mut snapshot, + pkg_req, + package_info, + )?; + Ok(nv) + } + + pub fn package_reqs(&self) -> HashMap<PackageReq, PackageNv> { + self.snapshot.read().package_reqs().clone() + } + + pub fn all_system_packages( + &self, + system_info: &NpmSystemInfo, + ) -> Vec<NpmResolutionPackage> { + self.snapshot.read().all_system_packages(system_info) + } + + pub fn all_system_packages_partitioned( + &self, + system_info: &NpmSystemInfo, + ) -> NpmPackagesPartitioned { + self + .snapshot + .read() + .all_system_packages_partitioned(system_info) + } + + // todo: NEXT + + pub fn has_packages(&self) -> bool { + !self.snapshot.read().is_empty() + } + + // todo: NEXT + + pub fn snapshot(&self) -> NpmResolutionSnapshot { + self.snapshot.read().clone() + } + + pub fn serialized_valid_snapshot( + &self, + ) -> ValidSerializedNpmResolutionSnapshot { + self.snapshot.read().as_valid_serialized() + } + + // todo: NEXT + + pub fn serialized_valid_snapshot_for_system( + &self, + system_info: &NpmSystemInfo, + ) -> ValidSerializedNpmResolutionSnapshot { + self + .snapshot + .read() + .as_valid_serialized_for_system(system_info) + } + + pub fn lock(&self, lockfile: &mut Lockfile) -> Result<(), AnyError> { + let snapshot = self.snapshot.read(); + populate_lockfile_from_snapshot(lockfile, &snapshot) + } +} + +async fn add_package_reqs_to_snapshot( + api: &CliNpmRegistryApi, + package_reqs: &[PackageReq], + maybe_lockfile: Option<Arc<Mutex<Lockfile>>>, + get_new_snapshot: impl Fn() -> NpmResolutionSnapshot, +) -> Result<NpmResolutionSnapshot, AnyError> { + let snapshot = get_new_snapshot(); + let snapshot = if !snapshot.has_pending() + && package_reqs + .iter() + .all(|req| snapshot.package_reqs().contains_key(req)) + { + log::debug!("Snapshot already up to date. Skipping pending resolution."); + snapshot + } else { + let pending_resolver = get_npm_pending_resolver(api); + let result = pending_resolver + .resolve_pending(snapshot, package_reqs) + .await; + api.clear_memory_cache(); + match result { + Ok(snapshot) => snapshot, + Err(NpmResolutionError::Resolution(err)) if api.mark_force_reload() => { + log::debug!("{err:#}"); + log::debug!("npm resolution failed. Trying again..."); + + // try again + let snapshot = get_new_snapshot(); + let result = pending_resolver + .resolve_pending(snapshot, package_reqs) + .await; + api.clear_memory_cache(); + // now surface the result after clearing the cache + result? + } + Err(err) => return Err(err.into()), + } + }; + + if let Some(lockfile_mutex) = maybe_lockfile { + let mut lockfile = lockfile_mutex.lock(); + populate_lockfile_from_snapshot(&mut lockfile, &snapshot)?; + } + + Ok(snapshot) +} + +fn get_npm_pending_resolver( + api: &CliNpmRegistryApi, +) -> NpmResolutionSnapshotPendingResolver<CliNpmRegistryApi> { + NpmResolutionSnapshotPendingResolver::new( + NpmResolutionSnapshotPendingResolverOptions { + api, + // WARNING: When bumping this version, check if anything needs to be + // updated in the `setNodeOnlyGlobalNames` call in 99_main_compiler.js + types_node_version_req: Some( + VersionReq::parse_from_npm("18.0.0 - 18.16.19").unwrap(), + ), + }, + ) +} + +fn populate_lockfile_from_snapshot( + lockfile: &mut Lockfile, + snapshot: &NpmResolutionSnapshot, +) -> Result<(), AnyError> { + for (package_req, nv) in snapshot.package_reqs() { + lockfile.insert_package_specifier( + format!("npm:{}", package_req), + format!( + "npm:{}", + snapshot + .resolve_package_from_deno_module(nv) + .unwrap() + .id + .as_serialized() + ), + ); + } + for package in snapshot.all_packages_for_every_system() { + lockfile + .check_or_insert_npm_package(npm_package_to_lockfile_info(package))?; + } + Ok(()) +} + +fn npm_package_to_lockfile_info( + pkg: &NpmResolutionPackage, +) -> NpmPackageLockfileInfo { + fn integrity_for_lockfile( + integrity: NpmPackageVersionDistInfoIntegrity, + ) -> String { + match integrity { + NpmPackageVersionDistInfoIntegrity::Integrity { + algorithm, + base64_hash, + } => format!("{}-{}", algorithm, base64_hash), + NpmPackageVersionDistInfoIntegrity::UnknownIntegrity(integrity) => { + integrity.to_string() + } + NpmPackageVersionDistInfoIntegrity::LegacySha1Hex(hex) => hex.to_string(), + } + } + + let dependencies = pkg + .dependencies + .iter() + .map(|(name, id)| NpmPackageDependencyLockfileInfo { + name: name.clone(), + id: id.as_serialized(), + }) + .collect(); + + NpmPackageLockfileInfo { + display_id: pkg.id.nv.to_string(), + serialized_id: pkg.id.as_serialized(), + integrity: integrity_for_lockfile(pkg.dist.integrity()), + dependencies, + } +} |