diff options
author | David Sherret <dsherret@users.noreply.github.com> | 2023-09-30 12:06:38 -0400 |
---|---|---|
committer | GitHub <noreply@github.com> | 2023-09-30 12:06:38 -0400 |
commit | 8d24be1a59714761665516e0d78d25059608c29b (patch) | |
tree | aed0140b63441008cb9b549d44948f7a36a4f5f1 /cli/npm/managed | |
parent | 1cda3840ff673512f7c6d58fa8402c35c760bc3b (diff) |
refactor(npm): create `cli::npm::managed` module (#20740)
Creates the `cli::npm::managed` module and starts moving more
functionality into it.
Diffstat (limited to 'cli/npm/managed')
-rw-r--r-- | cli/npm/managed/installer.rs | 122 | ||||
-rw-r--r-- | cli/npm/managed/mod.rs | 394 | ||||
-rw-r--r-- | cli/npm/managed/resolution.rs | 433 | ||||
-rw-r--r-- | cli/npm/managed/resolvers/common.rs | 173 | ||||
-rw-r--r-- | cli/npm/managed/resolvers/global.rs | 184 | ||||
-rw-r--r-- | cli/npm/managed/resolvers/local.rs | 780 | ||||
-rw-r--r-- | cli/npm/managed/resolvers/mod.rs | 50 |
7 files changed, 2136 insertions, 0 deletions
diff --git a/cli/npm/managed/installer.rs b/cli/npm/managed/installer.rs new file mode 100644 index 000000000..21285c3d7 --- /dev/null +++ b/cli/npm/managed/installer.rs @@ -0,0 +1,122 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +use std::future::Future; +use std::sync::Arc; + +use deno_core::error::AnyError; +use deno_core::futures::stream::FuturesOrdered; +use deno_core::futures::StreamExt; +use deno_npm::registry::NpmRegistryApi; +use deno_npm::registry::NpmRegistryPackageInfoLoadError; +use deno_semver::package::PackageReq; + +use crate::args::PackageJsonDepsProvider; +use crate::util::sync::AtomicFlag; + +use super::super::CliNpmRegistryApi; +use super::NpmResolution; + +#[derive(Debug)] +struct PackageJsonDepsInstallerInner { + deps_provider: Arc<PackageJsonDepsProvider>, + has_installed_flag: AtomicFlag, + npm_registry_api: Arc<CliNpmRegistryApi>, + npm_resolution: Arc<NpmResolution>, +} + +impl PackageJsonDepsInstallerInner { + pub fn reqs_with_info_futures<'a>( + &self, + reqs: &'a [&'a PackageReq], + ) -> FuturesOrdered< + impl Future< + Output = Result< + (&'a PackageReq, Arc<deno_npm::registry::NpmPackageInfo>), + NpmRegistryPackageInfoLoadError, + >, + >, + > { + FuturesOrdered::from_iter(reqs.iter().map(|req| { + let api = self.npm_registry_api.clone(); + async move { + let info = api.package_info(&req.name).await?; + Ok::<_, NpmRegistryPackageInfoLoadError>((*req, info)) + } + })) + } +} + +/// Holds and controls installing dependencies from package.json. +#[derive(Debug, Default)] +pub struct PackageJsonDepsInstaller(Option<PackageJsonDepsInstallerInner>); + +impl PackageJsonDepsInstaller { + pub fn new( + deps_provider: Arc<PackageJsonDepsProvider>, + npm_registry_api: Arc<CliNpmRegistryApi>, + npm_resolution: Arc<NpmResolution>, + ) -> Self { + Self(Some(PackageJsonDepsInstallerInner { + deps_provider, + has_installed_flag: Default::default(), + npm_registry_api, + npm_resolution, + })) + } + + /// Creates an installer that never installs local packages during + /// resolution. A top level install will be a no-op. + pub fn no_op() -> Self { + Self(None) + } + + /// Installs the top level dependencies in the package.json file + /// without going through and resolving the descendant dependencies yet. + pub async fn ensure_top_level_install(&self) -> Result<(), AnyError> { + let inner = match &self.0 { + Some(inner) => inner, + None => return Ok(()), + }; + + if !inner.has_installed_flag.raise() { + return Ok(()); // already installed by something else + } + + let package_reqs = inner.deps_provider.reqs(); + + // check if something needs resolving before bothering to load all + // the package information (which is slow) + if package_reqs.iter().all(|req| { + inner + .npm_resolution + .resolve_pkg_id_from_pkg_req(req) + .is_ok() + }) { + log::debug!( + "All package.json deps resolvable. Skipping top level install." + ); + return Ok(()); // everything is already resolvable + } + + let mut reqs_with_info_futures = + inner.reqs_with_info_futures(&package_reqs); + + while let Some(result) = reqs_with_info_futures.next().await { + let (req, info) = result?; + let result = inner + .npm_resolution + .resolve_pkg_req_as_pending_with_info(req, &info); + if let Err(err) = result { + if inner.npm_registry_api.mark_force_reload() { + log::debug!("Failed to resolve package. Retrying. Error: {err:#}"); + // re-initialize + reqs_with_info_futures = inner.reqs_with_info_futures(&package_reqs); + } else { + return Err(err.into()); + } + } + } + + Ok(()) + } +} diff --git a/cli/npm/managed/mod.rs b/cli/npm/managed/mod.rs new file mode 100644 index 000000000..c5ba3d3af --- /dev/null +++ b/cli/npm/managed/mod.rs @@ -0,0 +1,394 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +use std::collections::HashMap; +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; + +use deno_ast::ModuleSpecifier; +use deno_core::error::AnyError; +use deno_core::parking_lot::Mutex; +use deno_core::serde_json; +use deno_core::url::Url; +use deno_graph::NpmPackageReqResolution; +use deno_npm::registry::NpmRegistryApi; +use deno_npm::resolution::NpmResolutionSnapshot; +use deno_npm::resolution::PackageReqNotFoundError; +use deno_npm::resolution::SerializedNpmResolutionSnapshot; +use deno_npm::NpmPackageId; +use deno_npm::NpmResolutionPackage; +use deno_npm::NpmSystemInfo; +use deno_runtime::deno_fs::FileSystem; +use deno_runtime::deno_node::NodePermissions; +use deno_runtime::deno_node::NodeResolutionMode; +use deno_runtime::deno_node::NpmResolver; +use deno_semver::npm::NpmPackageNvReference; +use deno_semver::npm::NpmPackageReqReference; +use deno_semver::package::PackageNv; +use deno_semver::package::PackageNvReference; +use deno_semver::package::PackageReq; +use serde::Deserialize; +use serde::Serialize; + +use crate::args::Lockfile; +use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs; + +use super::CliNpmRegistryApi; +use super::CliNpmResolver; +use super::InnerCliNpmResolverRef; + +pub use self::installer::PackageJsonDepsInstaller; +pub use self::resolution::NpmResolution; +pub use self::resolvers::create_npm_fs_resolver; +pub use self::resolvers::NpmPackageFsResolver; + +mod installer; +mod resolution; +mod resolvers; + +/// State provided to the process via an environment variable. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct NpmProcessState { + pub snapshot: SerializedNpmResolutionSnapshot, + pub local_node_modules_path: Option<String>, +} + +/// An npm resolver where the resolution is managed by Deno rather than +/// the user bringing their own node_modules (BYONM) on the file system. +pub struct ManagedCliNpmResolver { + api: Arc<CliNpmRegistryApi>, + fs: Arc<dyn FileSystem>, + fs_resolver: Arc<dyn NpmPackageFsResolver>, + resolution: Arc<NpmResolution>, + maybe_lockfile: Option<Arc<Mutex<Lockfile>>>, + package_json_deps_installer: Arc<PackageJsonDepsInstaller>, +} + +impl std::fmt::Debug for ManagedCliNpmResolver { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("ManagedNpmResolver") + .field("api", &"<omitted>") + .field("fs", &"<omitted>") + .field("fs_resolver", &"<omitted>") + .field("resolution", &"<omitted>") + .field("maybe_lockfile", &"<omitted>") + .field("package_json_deps_installer", &"<omitted>") + .finish() + } +} + +impl ManagedCliNpmResolver { + pub fn new( + api: Arc<CliNpmRegistryApi>, + fs: Arc<dyn FileSystem>, + resolution: Arc<NpmResolution>, + fs_resolver: Arc<dyn NpmPackageFsResolver>, + maybe_lockfile: Option<Arc<Mutex<Lockfile>>>, + package_json_deps_installer: Arc<PackageJsonDepsInstaller>, + ) -> Self { + Self { + api, + fs, + fs_resolver, + resolution, + maybe_lockfile, + package_json_deps_installer, + } + } + + pub fn resolve_pkg_folder_from_pkg_id( + &self, + pkg_id: &NpmPackageId, + ) -> Result<PathBuf, AnyError> { + let path = self.fs_resolver.package_folder(pkg_id)?; + let path = canonicalize_path_maybe_not_exists_with_fs(&path, |path| { + self + .fs + .realpath_sync(path) + .map_err(|err| err.into_io_error()) + })?; + log::debug!( + "Resolved package folder of {} to {}", + pkg_id.as_serialized(), + path.display() + ); + Ok(path) + } + + /// Resolves the package nv from the provided specifier. + pub fn resolve_pkg_id_from_specifier( + &self, + specifier: &ModuleSpecifier, + ) -> Result<Option<NpmPackageId>, AnyError> { + let Some(cache_folder_id) = self + .fs_resolver + .resolve_package_cache_folder_id_from_specifier(specifier)? + else { + return Ok(None); + }; + Ok(Some( + self + .resolution + .resolve_pkg_id_from_pkg_cache_folder_id(&cache_folder_id)?, + )) + } + + pub fn resolve_pkg_reqs_from_pkg_id( + &self, + id: &NpmPackageId, + ) -> Vec<PackageReq> { + self.resolution.resolve_pkg_reqs_from_pkg_id(id) + } + + /// Attempts to get the package size in bytes. + pub fn package_size( + &self, + package_id: &NpmPackageId, + ) -> Result<u64, AnyError> { + let package_folder = self.fs_resolver.package_folder(package_id)?; + Ok(crate::util::fs::dir_size(&package_folder)?) + } + + pub fn all_system_packages( + &self, + system_info: &NpmSystemInfo, + ) -> Vec<NpmResolutionPackage> { + self.resolution.all_system_packages(system_info) + } + + /// Adds package requirements to the resolver and ensures everything is setup. + pub async fn add_package_reqs( + &self, + packages: &[PackageReq], + ) -> Result<(), AnyError> { + if packages.is_empty() { + return Ok(()); + } + + self.resolution.add_package_reqs(packages).await?; + self.fs_resolver.cache_packages().await?; + + // If there's a lock file, update it with all discovered npm packages + if let Some(lockfile_mutex) = &self.maybe_lockfile { + let mut lockfile = lockfile_mutex.lock(); + self.lock(&mut lockfile)?; + } + + Ok(()) + } + + /// Sets package requirements to the resolver, removing old requirements and adding new ones. + /// + /// This will retrieve and resolve package information, but not cache any package files. + pub async fn set_package_reqs( + &self, + packages: &[PackageReq], + ) -> Result<(), AnyError> { + self.resolution.set_package_reqs(packages).await + } + + pub fn snapshot(&self) -> NpmResolutionSnapshot { + self.resolution.snapshot() + } + + pub fn lock(&self, lockfile: &mut Lockfile) -> Result<(), AnyError> { + self.resolution.lock(lockfile) + } + + pub async fn inject_synthetic_types_node_package( + &self, + ) -> Result<(), AnyError> { + // add and ensure this isn't added to the lockfile + let package_reqs = vec![PackageReq::from_str("@types/node").unwrap()]; + self.resolution.add_package_reqs(&package_reqs).await?; + self.fs_resolver.cache_packages().await?; + + Ok(()) + } + + pub async fn resolve_pending(&self) -> Result<(), AnyError> { + self.resolution.resolve_pending().await?; + self.fs_resolver.cache_packages().await?; + Ok(()) + } + + fn resolve_pkg_id_from_pkg_req( + &self, + req: &PackageReq, + ) -> Result<NpmPackageId, PackageReqNotFoundError> { + self.resolution.resolve_pkg_id_from_pkg_req(req) + } + + pub async fn ensure_top_level_package_json_install( + &self, + ) -> Result<(), AnyError> { + self + .package_json_deps_installer + .ensure_top_level_install() + .await + } + + pub async fn cache_package_info( + &self, + package_name: &str, + ) -> Result<(), AnyError> { + // this will internally cache the package information + self + .api + .package_info(package_name) + .await + .map(|_| ()) + .map_err(|err| err.into()) + } +} + +impl NpmResolver for ManagedCliNpmResolver { + fn resolve_package_folder_from_package( + &self, + name: &str, + referrer: &ModuleSpecifier, + mode: NodeResolutionMode, + ) -> Result<PathBuf, AnyError> { + let path = self + .fs_resolver + .resolve_package_folder_from_package(name, referrer, mode)?; + log::debug!("Resolved {} from {} to {}", name, referrer, path.display()); + Ok(path) + } + + fn resolve_package_folder_from_path( + &self, + specifier: &ModuleSpecifier, + ) -> Result<Option<PathBuf>, AnyError> { + self.resolve_pkg_folder_from_specifier(specifier) + } + + fn in_npm_package(&self, specifier: &ModuleSpecifier) -> bool { + let root_dir_url = self.fs_resolver.root_dir_url(); + debug_assert!(root_dir_url.as_str().ends_with('/')); + specifier.as_ref().starts_with(root_dir_url.as_str()) + } + + fn ensure_read_permission( + &self, + permissions: &dyn NodePermissions, + path: &Path, + ) -> Result<(), AnyError> { + self.fs_resolver.ensure_read_permission(permissions, path) + } +} + +impl CliNpmResolver for ManagedCliNpmResolver { + fn into_npm_resolver(self: Arc<Self>) -> Arc<dyn NpmResolver> { + self + } + + fn root_dir_url(&self) -> &Url { + self.fs_resolver.root_dir_url() + } + + fn as_inner(&self) -> InnerCliNpmResolverRef { + InnerCliNpmResolverRef::Managed(self) + } + + fn node_modules_path(&self) -> Option<PathBuf> { + self.fs_resolver.node_modules_path() + } + + /// Checks if the provided package req's folder is cached. + fn is_pkg_req_folder_cached(&self, req: &PackageReq) -> bool { + self + .resolve_pkg_id_from_pkg_req(req) + .ok() + .and_then(|id| self.fs_resolver.package_folder(&id).ok()) + .map(|folder| folder.exists()) + .unwrap_or(false) + } + + fn resolve_npm_for_deno_graph( + &self, + pkg_req: &PackageReq, + ) -> NpmPackageReqResolution { + let result = self.resolution.resolve_pkg_req_as_pending(pkg_req); + match result { + Ok(nv) => NpmPackageReqResolution::Ok(nv), + Err(err) => { + if self.api.mark_force_reload() { + log::debug!("Restarting npm specifier resolution to check for new registry information. Error: {:#}", err); + NpmPackageReqResolution::ReloadRegistryInfo(err.into()) + } else { + NpmPackageReqResolution::Err(err.into()) + } + } + } + } + + fn resolve_pkg_nv_ref_from_pkg_req_ref( + &self, + req_ref: &NpmPackageReqReference, + ) -> Result<NpmPackageNvReference, PackageReqNotFoundError> { + let pkg_nv = self + .resolve_pkg_id_from_pkg_req(req_ref.req()) + .map(|id| id.nv)?; + Ok(NpmPackageNvReference::new(PackageNvReference { + nv: pkg_nv, + sub_path: req_ref.sub_path().map(|s| s.to_string()), + })) + } + + /// Resolve the root folder of the package the provided specifier is in. + /// + /// This will error when the provided specifier is not in an npm package. + fn resolve_pkg_folder_from_specifier( + &self, + specifier: &ModuleSpecifier, + ) -> Result<Option<PathBuf>, AnyError> { + let Some(path) = self + .fs_resolver + .resolve_package_folder_from_specifier(specifier)? + else { + return Ok(None); + }; + log::debug!( + "Resolved package folder of {} to {}", + specifier, + path.display() + ); + Ok(Some(path)) + } + + fn resolve_pkg_folder_from_deno_module_req( + &self, + req: &PackageReq, + ) -> Result<PathBuf, AnyError> { + let pkg_id = self.resolve_pkg_id_from_pkg_req(req)?; + self.resolve_pkg_folder_from_pkg_id(&pkg_id) + } + + fn resolve_pkg_folder_from_deno_module( + &self, + nv: &PackageNv, + ) -> Result<PathBuf, AnyError> { + let pkg_id = self.resolution.resolve_pkg_id_from_deno_module(nv)?; + self.resolve_pkg_folder_from_pkg_id(&pkg_id) + } + + /// Gets the state of npm for the process. + fn get_npm_process_state(&self) -> String { + serde_json::to_string(&NpmProcessState { + snapshot: self + .resolution + .serialized_valid_snapshot() + .into_serialized(), + local_node_modules_path: self + .fs_resolver + .node_modules_path() + .map(|p| p.to_string_lossy().to_string()), + }) + .unwrap() + } + + fn package_reqs(&self) -> HashMap<PackageReq, PackageNv> { + self.resolution.package_reqs() + } +} diff --git a/cli/npm/managed/resolution.rs b/cli/npm/managed/resolution.rs new file mode 100644 index 000000000..05c1227a7 --- /dev/null +++ b/cli/npm/managed/resolution.rs @@ -0,0 +1,433 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +use std::collections::HashMap; +use std::collections::HashSet; +use std::sync::Arc; + +use deno_core::error::AnyError; +use deno_core::parking_lot::Mutex; +use deno_core::parking_lot::RwLock; +use deno_lockfile::NpmPackageDependencyLockfileInfo; +use deno_lockfile::NpmPackageLockfileInfo; +use deno_npm::registry::NpmPackageInfo; +use deno_npm::registry::NpmPackageVersionDistInfoIntegrity; +use deno_npm::registry::NpmRegistryApi; +use deno_npm::resolution::NpmPackageVersionResolutionError; +use deno_npm::resolution::NpmPackagesPartitioned; +use deno_npm::resolution::NpmResolutionError; +use deno_npm::resolution::NpmResolutionSnapshot; +use deno_npm::resolution::NpmResolutionSnapshotPendingResolver; +use deno_npm::resolution::NpmResolutionSnapshotPendingResolverOptions; +use deno_npm::resolution::PackageCacheFolderIdNotFoundError; +use deno_npm::resolution::PackageNotFoundFromReferrerError; +use deno_npm::resolution::PackageNvNotFoundError; +use deno_npm::resolution::PackageReqNotFoundError; +use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; +use deno_npm::NpmPackageCacheFolderId; +use deno_npm::NpmPackageId; +use deno_npm::NpmResolutionPackage; +use deno_npm::NpmSystemInfo; +use deno_semver::package::PackageNv; +use deno_semver::package::PackageReq; +use deno_semver::VersionReq; + +use crate::args::Lockfile; +use crate::util::sync::TaskQueue; + +use super::super::registry::CliNpmRegistryApi; + +/// Handles updating and storing npm resolution in memory where the underlying +/// snapshot can be updated concurrently. Additionally handles updating the lockfile +/// based on changes to the resolution. +/// +/// This does not interact with the file system. +pub struct NpmResolution { + api: Arc<CliNpmRegistryApi>, + snapshot: RwLock<NpmResolutionSnapshot>, + update_queue: TaskQueue, + maybe_lockfile: Option<Arc<Mutex<Lockfile>>>, +} + +impl std::fmt::Debug for NpmResolution { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let snapshot = self.snapshot.read(); + f.debug_struct("NpmResolution") + .field("snapshot", &snapshot.as_valid_serialized().as_serialized()) + .finish() + } +} + +impl NpmResolution { + pub fn from_serialized( + api: Arc<CliNpmRegistryApi>, + initial_snapshot: Option<ValidSerializedNpmResolutionSnapshot>, + maybe_lockfile: Option<Arc<Mutex<Lockfile>>>, + ) -> Self { + let snapshot = + NpmResolutionSnapshot::new(initial_snapshot.unwrap_or_default()); + Self::new(api, snapshot, maybe_lockfile) + } + + pub fn new( + api: Arc<CliNpmRegistryApi>, + initial_snapshot: NpmResolutionSnapshot, + maybe_lockfile: Option<Arc<Mutex<Lockfile>>>, + ) -> Self { + Self { + api, + snapshot: RwLock::new(initial_snapshot), + update_queue: Default::default(), + maybe_lockfile, + } + } + + pub async fn add_package_reqs( + &self, + package_reqs: &[PackageReq], + ) -> Result<(), AnyError> { + // only allow one thread in here at a time + let _permit = self.update_queue.acquire().await; + let snapshot = add_package_reqs_to_snapshot( + &self.api, + package_reqs, + self.maybe_lockfile.clone(), + || self.snapshot.read().clone(), + ) + .await?; + + *self.snapshot.write() = snapshot; + Ok(()) + } + + pub async fn set_package_reqs( + &self, + package_reqs: &[PackageReq], + ) -> Result<(), AnyError> { + // only allow one thread in here at a time + let _permit = self.update_queue.acquire().await; + + let reqs_set = package_reqs.iter().collect::<HashSet<_>>(); + let snapshot = add_package_reqs_to_snapshot( + &self.api, + package_reqs, + self.maybe_lockfile.clone(), + || { + let snapshot = self.snapshot.read().clone(); + let has_removed_package = !snapshot + .package_reqs() + .keys() + .all(|req| reqs_set.contains(req)); + // if any packages were removed, we need to completely recreate the npm resolution snapshot + if has_removed_package { + snapshot.into_empty() + } else { + snapshot + } + }, + ) + .await?; + + *self.snapshot.write() = snapshot; + + Ok(()) + } + + pub async fn resolve_pending(&self) -> Result<(), AnyError> { + // only allow one thread in here at a time + let _permit = self.update_queue.acquire().await; + + let snapshot = add_package_reqs_to_snapshot( + &self.api, + &Vec::new(), + self.maybe_lockfile.clone(), + || self.snapshot.read().clone(), + ) + .await?; + + *self.snapshot.write() = snapshot; + + Ok(()) + } + + pub fn resolve_pkg_cache_folder_id_from_pkg_id( + &self, + id: &NpmPackageId, + ) -> Option<NpmPackageCacheFolderId> { + self + .snapshot + .read() + .package_from_id(id) + .map(|p| p.get_package_cache_folder_id()) + } + + pub fn resolve_pkg_id_from_pkg_cache_folder_id( + &self, + id: &NpmPackageCacheFolderId, + ) -> Result<NpmPackageId, PackageCacheFolderIdNotFoundError> { + self + .snapshot + .read() + .resolve_pkg_from_pkg_cache_folder_id(id) + .map(|pkg| pkg.id.clone()) + } + + pub fn resolve_package_from_package( + &self, + name: &str, + referrer: &NpmPackageCacheFolderId, + ) -> Result<NpmResolutionPackage, Box<PackageNotFoundFromReferrerError>> { + self + .snapshot + .read() + .resolve_package_from_package(name, referrer) + .cloned() + } + + /// Resolve a node package from a deno module. + pub fn resolve_pkg_id_from_pkg_req( + &self, + req: &PackageReq, + ) -> Result<NpmPackageId, PackageReqNotFoundError> { + self + .snapshot + .read() + .resolve_pkg_from_pkg_req(req) + .map(|pkg| pkg.id.clone()) + } + + pub fn resolve_pkg_reqs_from_pkg_id( + &self, + id: &NpmPackageId, + ) -> Vec<PackageReq> { + let snapshot = self.snapshot.read(); + let mut pkg_reqs = snapshot + .package_reqs() + .iter() + .filter(|(_, nv)| *nv == &id.nv) + .map(|(req, _)| req.clone()) + .collect::<Vec<_>>(); + pkg_reqs.sort(); // be deterministic + pkg_reqs + } + + pub fn resolve_pkg_id_from_deno_module( + &self, + id: &PackageNv, + ) -> Result<NpmPackageId, PackageNvNotFoundError> { + self + .snapshot + .read() + .resolve_package_from_deno_module(id) + .map(|pkg| pkg.id.clone()) + } + + // todo: NEXT + + /// Resolves a package requirement for deno graph. This should only be + /// called by deno_graph's NpmResolver or for resolving packages in + /// a package.json + pub fn resolve_pkg_req_as_pending( + &self, + pkg_req: &PackageReq, + ) -> Result<PackageNv, NpmPackageVersionResolutionError> { + // we should always have this because it should have been cached before here + let package_info = self.api.get_cached_package_info(&pkg_req.name).unwrap(); + self.resolve_pkg_req_as_pending_with_info(pkg_req, &package_info) + } + + /// Resolves a package requirement for deno graph. This should only be + /// called by deno_graph's NpmResolver or for resolving packages in + /// a package.json + pub fn resolve_pkg_req_as_pending_with_info( + &self, + pkg_req: &PackageReq, + package_info: &NpmPackageInfo, + ) -> Result<PackageNv, NpmPackageVersionResolutionError> { + debug_assert_eq!(pkg_req.name, package_info.name); + let mut snapshot = self.snapshot.write(); + let pending_resolver = get_npm_pending_resolver(&self.api); + let nv = pending_resolver.resolve_package_req_as_pending( + &mut snapshot, + pkg_req, + package_info, + )?; + Ok(nv) + } + + pub fn package_reqs(&self) -> HashMap<PackageReq, PackageNv> { + self.snapshot.read().package_reqs().clone() + } + + pub fn all_system_packages( + &self, + system_info: &NpmSystemInfo, + ) -> Vec<NpmResolutionPackage> { + self.snapshot.read().all_system_packages(system_info) + } + + pub fn all_system_packages_partitioned( + &self, + system_info: &NpmSystemInfo, + ) -> NpmPackagesPartitioned { + self + .snapshot + .read() + .all_system_packages_partitioned(system_info) + } + + // todo: NEXT + + pub fn has_packages(&self) -> bool { + !self.snapshot.read().is_empty() + } + + // todo: NEXT + + pub fn snapshot(&self) -> NpmResolutionSnapshot { + self.snapshot.read().clone() + } + + pub fn serialized_valid_snapshot( + &self, + ) -> ValidSerializedNpmResolutionSnapshot { + self.snapshot.read().as_valid_serialized() + } + + // todo: NEXT + + pub fn serialized_valid_snapshot_for_system( + &self, + system_info: &NpmSystemInfo, + ) -> ValidSerializedNpmResolutionSnapshot { + self + .snapshot + .read() + .as_valid_serialized_for_system(system_info) + } + + pub fn lock(&self, lockfile: &mut Lockfile) -> Result<(), AnyError> { + let snapshot = self.snapshot.read(); + populate_lockfile_from_snapshot(lockfile, &snapshot) + } +} + +async fn add_package_reqs_to_snapshot( + api: &CliNpmRegistryApi, + package_reqs: &[PackageReq], + maybe_lockfile: Option<Arc<Mutex<Lockfile>>>, + get_new_snapshot: impl Fn() -> NpmResolutionSnapshot, +) -> Result<NpmResolutionSnapshot, AnyError> { + let snapshot = get_new_snapshot(); + let snapshot = if !snapshot.has_pending() + && package_reqs + .iter() + .all(|req| snapshot.package_reqs().contains_key(req)) + { + log::debug!("Snapshot already up to date. Skipping pending resolution."); + snapshot + } else { + let pending_resolver = get_npm_pending_resolver(api); + let result = pending_resolver + .resolve_pending(snapshot, package_reqs) + .await; + api.clear_memory_cache(); + match result { + Ok(snapshot) => snapshot, + Err(NpmResolutionError::Resolution(err)) if api.mark_force_reload() => { + log::debug!("{err:#}"); + log::debug!("npm resolution failed. Trying again..."); + + // try again + let snapshot = get_new_snapshot(); + let result = pending_resolver + .resolve_pending(snapshot, package_reqs) + .await; + api.clear_memory_cache(); + // now surface the result after clearing the cache + result? + } + Err(err) => return Err(err.into()), + } + }; + + if let Some(lockfile_mutex) = maybe_lockfile { + let mut lockfile = lockfile_mutex.lock(); + populate_lockfile_from_snapshot(&mut lockfile, &snapshot)?; + } + + Ok(snapshot) +} + +fn get_npm_pending_resolver( + api: &CliNpmRegistryApi, +) -> NpmResolutionSnapshotPendingResolver<CliNpmRegistryApi> { + NpmResolutionSnapshotPendingResolver::new( + NpmResolutionSnapshotPendingResolverOptions { + api, + // WARNING: When bumping this version, check if anything needs to be + // updated in the `setNodeOnlyGlobalNames` call in 99_main_compiler.js + types_node_version_req: Some( + VersionReq::parse_from_npm("18.0.0 - 18.16.19").unwrap(), + ), + }, + ) +} + +fn populate_lockfile_from_snapshot( + lockfile: &mut Lockfile, + snapshot: &NpmResolutionSnapshot, +) -> Result<(), AnyError> { + for (package_req, nv) in snapshot.package_reqs() { + lockfile.insert_package_specifier( + format!("npm:{}", package_req), + format!( + "npm:{}", + snapshot + .resolve_package_from_deno_module(nv) + .unwrap() + .id + .as_serialized() + ), + ); + } + for package in snapshot.all_packages_for_every_system() { + lockfile + .check_or_insert_npm_package(npm_package_to_lockfile_info(package))?; + } + Ok(()) +} + +fn npm_package_to_lockfile_info( + pkg: &NpmResolutionPackage, +) -> NpmPackageLockfileInfo { + fn integrity_for_lockfile( + integrity: NpmPackageVersionDistInfoIntegrity, + ) -> String { + match integrity { + NpmPackageVersionDistInfoIntegrity::Integrity { + algorithm, + base64_hash, + } => format!("{}-{}", algorithm, base64_hash), + NpmPackageVersionDistInfoIntegrity::UnknownIntegrity(integrity) => { + integrity.to_string() + } + NpmPackageVersionDistInfoIntegrity::LegacySha1Hex(hex) => hex.to_string(), + } + } + + let dependencies = pkg + .dependencies + .iter() + .map(|(name, id)| NpmPackageDependencyLockfileInfo { + name: name.clone(), + id: id.as_serialized(), + }) + .collect(); + + NpmPackageLockfileInfo { + display_id: pkg.id.nv.to_string(), + serialized_id: pkg.id.as_serialized(), + integrity: integrity_for_lockfile(pkg.dist.integrity()), + dependencies, + } +} diff --git a/cli/npm/managed/resolvers/common.rs b/cli/npm/managed/resolvers/common.rs new file mode 100644 index 000000000..4076579bf --- /dev/null +++ b/cli/npm/managed/resolvers/common.rs @@ -0,0 +1,173 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +use std::collections::HashMap; +use std::io::ErrorKind; +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; +use std::sync::Mutex; + +use async_trait::async_trait; +use deno_ast::ModuleSpecifier; +use deno_core::error::AnyError; +use deno_core::futures; +use deno_core::unsync::spawn; +use deno_core::url::Url; +use deno_npm::NpmPackageCacheFolderId; +use deno_npm::NpmPackageId; +use deno_npm::NpmResolutionPackage; +use deno_runtime::deno_fs::FileSystem; +use deno_runtime::deno_node::NodePermissions; +use deno_runtime::deno_node::NodeResolutionMode; + +use crate::npm::NpmCache; + +/// Part of the resolution that interacts with the file system. +#[async_trait] +pub trait NpmPackageFsResolver: Send + Sync { + /// Specifier for the root directory. + fn root_dir_url(&self) -> &Url; + + /// The local node_modules folder if it is applicable to the implementation. + fn node_modules_path(&self) -> Option<PathBuf>; + + fn package_folder( + &self, + package_id: &NpmPackageId, + ) -> Result<PathBuf, AnyError>; + + fn resolve_package_folder_from_package( + &self, + name: &str, + referrer: &ModuleSpecifier, + mode: NodeResolutionMode, + ) -> Result<PathBuf, AnyError>; + + fn resolve_package_folder_from_specifier( + &self, + specifier: &ModuleSpecifier, + ) -> Result<Option<PathBuf>, AnyError>; + + fn resolve_package_cache_folder_id_from_specifier( + &self, + specifier: &ModuleSpecifier, + ) -> Result<Option<NpmPackageCacheFolderId>, AnyError>; + + async fn cache_packages(&self) -> Result<(), AnyError>; + + fn ensure_read_permission( + &self, + permissions: &dyn NodePermissions, + path: &Path, + ) -> Result<(), AnyError>; +} + +#[derive(Debug)] +pub struct RegistryReadPermissionChecker { + fs: Arc<dyn FileSystem>, + cache: Mutex<HashMap<PathBuf, PathBuf>>, + registry_path: PathBuf, +} + +impl RegistryReadPermissionChecker { + pub fn new(fs: Arc<dyn FileSystem>, registry_path: PathBuf) -> Self { + Self { + fs, + registry_path, + cache: Default::default(), + } + } + + pub fn ensure_registry_read_permission( + &self, + permissions: &dyn NodePermissions, + path: &Path, + ) -> Result<(), AnyError> { + // allow reading if it's in the node_modules + let is_path_in_node_modules = path.starts_with(&self.registry_path) + && path + .components() + .all(|c| !matches!(c, std::path::Component::ParentDir)); + + if is_path_in_node_modules { + let mut cache = self.cache.lock().unwrap(); + let registry_path_canon = match cache.get(&self.registry_path) { + Some(canon) => canon.clone(), + None => { + let canon = self.fs.realpath_sync(&self.registry_path)?; + cache.insert(self.registry_path.to_path_buf(), canon.clone()); + canon + } + }; + + let path_canon = match cache.get(path) { + Some(canon) => canon.clone(), + None => { + let canon = self.fs.realpath_sync(path); + if let Err(e) = &canon { + if e.kind() == ErrorKind::NotFound { + return Ok(()); + } + } + + let canon = canon?; + cache.insert(path.to_path_buf(), canon.clone()); + canon + } + }; + + if path_canon.starts_with(registry_path_canon) { + return Ok(()); + } + } + + permissions.check_read(path) + } +} + +/// Caches all the packages in parallel. +pub async fn cache_packages( + packages: Vec<NpmResolutionPackage>, + cache: &Arc<NpmCache>, + registry_url: &Url, +) -> Result<(), AnyError> { + let mut handles = Vec::with_capacity(packages.len()); + for package in packages { + let cache = cache.clone(); + let registry_url = registry_url.clone(); + let handle = spawn(async move { + cache + .ensure_package(&package.id.nv, &package.dist, ®istry_url) + .await + }); + handles.push(handle); + } + let results = futures::future::join_all(handles).await; + for result in results { + // surface the first error + result??; + } + Ok(()) +} + +/// Gets the corresponding @types package for the provided package name. +pub fn types_package_name(package_name: &str) -> String { + debug_assert!(!package_name.starts_with("@types/")); + // Scoped packages will get two underscores for each slash + // https://github.com/DefinitelyTyped/DefinitelyTyped/tree/15f1ece08f7b498f4b9a2147c2a46e94416ca777#what-about-scoped-packages + format!("@types/{}", package_name.replace('/', "__")) +} + +#[cfg(test)] +mod test { + use super::types_package_name; + + #[test] + fn test_types_package_name() { + assert_eq!(types_package_name("name"), "@types/name"); + assert_eq!( + types_package_name("@scoped/package"), + "@types/@scoped__package" + ); + } +} diff --git a/cli/npm/managed/resolvers/global.rs b/cli/npm/managed/resolvers/global.rs new file mode 100644 index 000000000..25db62f73 --- /dev/null +++ b/cli/npm/managed/resolvers/global.rs @@ -0,0 +1,184 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +//! Code for global npm cache resolution. + +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; + +use async_trait::async_trait; +use deno_ast::ModuleSpecifier; +use deno_core::anyhow::bail; +use deno_core::error::AnyError; +use deno_core::url::Url; +use deno_npm::resolution::PackageNotFoundFromReferrerError; +use deno_npm::NpmPackageCacheFolderId; +use deno_npm::NpmPackageId; +use deno_npm::NpmResolutionPackage; +use deno_npm::NpmSystemInfo; +use deno_runtime::deno_fs::FileSystem; +use deno_runtime::deno_node::NodePermissions; +use deno_runtime::deno_node::NodeResolutionMode; + +use crate::npm::NpmCache; + +use super::super::resolution::NpmResolution; +use super::common::cache_packages; +use super::common::types_package_name; +use super::common::NpmPackageFsResolver; +use super::common::RegistryReadPermissionChecker; + +/// Resolves packages from the global npm cache. +#[derive(Debug)] +pub struct GlobalNpmPackageResolver { + cache: Arc<NpmCache>, + resolution: Arc<NpmResolution>, + registry_url: Url, + system_info: NpmSystemInfo, + registry_read_permission_checker: RegistryReadPermissionChecker, +} + +impl GlobalNpmPackageResolver { + pub fn new( + fs: Arc<dyn FileSystem>, + cache: Arc<NpmCache>, + registry_url: Url, + resolution: Arc<NpmResolution>, + system_info: NpmSystemInfo, + ) -> Self { + Self { + cache: cache.clone(), + resolution, + registry_url: registry_url.clone(), + system_info, + registry_read_permission_checker: RegistryReadPermissionChecker::new( + fs, + cache.registry_folder(®istry_url), + ), + } + } + + fn resolve_types_package( + &self, + package_name: &str, + referrer_pkg_id: &NpmPackageCacheFolderId, + ) -> Result<NpmResolutionPackage, Box<PackageNotFoundFromReferrerError>> { + let types_name = types_package_name(package_name); + self + .resolution + .resolve_package_from_package(&types_name, referrer_pkg_id) + } +} + +#[async_trait] +impl NpmPackageFsResolver for GlobalNpmPackageResolver { + fn root_dir_url(&self) -> &Url { + self.cache.root_dir_url() + } + + fn node_modules_path(&self) -> Option<PathBuf> { + None + } + + fn package_folder(&self, id: &NpmPackageId) -> Result<PathBuf, AnyError> { + let folder_id = self + .resolution + .resolve_pkg_cache_folder_id_from_pkg_id(id) + .unwrap(); + Ok( + self + .cache + .package_folder_for_id(&folder_id, &self.registry_url), + ) + } + + fn resolve_package_folder_from_package( + &self, + name: &str, + referrer: &ModuleSpecifier, + mode: NodeResolutionMode, + ) -> Result<PathBuf, AnyError> { + let Some(referrer_pkg_id) = self + .cache + .resolve_package_folder_id_from_specifier(referrer, &self.registry_url) + else { + bail!("could not find npm package for '{}'", referrer); + }; + let pkg = if mode.is_types() && !name.starts_with("@types/") { + // attempt to resolve the types package first, then fallback to the regular package + match self.resolve_types_package(name, &referrer_pkg_id) { + Ok(pkg) => pkg, + Err(_) => self + .resolution + .resolve_package_from_package(name, &referrer_pkg_id)?, + } + } else { + self + .resolution + .resolve_package_from_package(name, &referrer_pkg_id)? + }; + self.package_folder(&pkg.id) + } + + fn resolve_package_folder_from_specifier( + &self, + specifier: &ModuleSpecifier, + ) -> Result<Option<PathBuf>, AnyError> { + let Some(pkg_folder_id) = self + .cache + .resolve_package_folder_id_from_specifier(specifier, &self.registry_url) + else { + return Ok(None); + }; + Ok(Some( + self + .cache + .package_folder_for_id(&pkg_folder_id, &self.registry_url), + )) + } + + fn resolve_package_cache_folder_id_from_specifier( + &self, + specifier: &ModuleSpecifier, + ) -> Result<Option<NpmPackageCacheFolderId>, AnyError> { + Ok( + self.cache.resolve_package_folder_id_from_specifier( + specifier, + &self.registry_url, + ), + ) + } + + async fn cache_packages(&self) -> Result<(), AnyError> { + let package_partitions = self + .resolution + .all_system_packages_partitioned(&self.system_info); + + cache_packages( + package_partitions.packages, + &self.cache, + &self.registry_url, + ) + .await?; + + // create the copy package folders + for copy in package_partitions.copy_packages { + self.cache.ensure_copy_package( + ©.get_package_cache_folder_id(), + &self.registry_url, + )?; + } + + Ok(()) + } + + fn ensure_read_permission( + &self, + permissions: &dyn NodePermissions, + path: &Path, + ) -> Result<(), AnyError> { + self + .registry_read_permission_checker + .ensure_registry_read_permission(permissions, path) + } +} diff --git a/cli/npm/managed/resolvers/local.rs b/cli/npm/managed/resolvers/local.rs new file mode 100644 index 000000000..57170eccd --- /dev/null +++ b/cli/npm/managed/resolvers/local.rs @@ -0,0 +1,780 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +//! Code for local node_modules resolution. + +use std::borrow::Cow; +use std::cmp::Ordering; +use std::collections::HashMap; +use std::collections::HashSet; +use std::fs; +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; + +use crate::cache::CACHE_PERM; +use crate::npm::cache::mixed_case_package_name_decode; +use crate::util::fs::atomic_write_file; +use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs; +use crate::util::fs::symlink_dir; +use crate::util::fs::LaxSingleProcessFsFlag; +use crate::util::progress_bar::ProgressBar; +use crate::util::progress_bar::ProgressMessagePrompt; +use async_trait::async_trait; +use deno_ast::ModuleSpecifier; +use deno_core::anyhow::bail; +use deno_core::anyhow::Context; +use deno_core::error::AnyError; +use deno_core::unsync::spawn; +use deno_core::unsync::JoinHandle; +use deno_core::url::Url; +use deno_npm::resolution::NpmResolutionSnapshot; +use deno_npm::NpmPackageCacheFolderId; +use deno_npm::NpmPackageId; +use deno_npm::NpmResolutionPackage; +use deno_npm::NpmSystemInfo; +use deno_runtime::deno_core::futures; +use deno_runtime::deno_fs; +use deno_runtime::deno_node::NodePermissions; +use deno_runtime::deno_node::NodeResolutionMode; +use deno_runtime::deno_node::PackageJson; +use deno_semver::package::PackageNv; +use serde::Deserialize; +use serde::Serialize; + +use crate::npm::cache::mixed_case_package_name_encode; +use crate::npm::NpmCache; +use crate::util::fs::copy_dir_recursive; +use crate::util::fs::hard_link_dir_recursive; + +use super::super::resolution::NpmResolution; +use super::common::types_package_name; +use super::common::NpmPackageFsResolver; +use super::common::RegistryReadPermissionChecker; + +/// Resolver that creates a local node_modules directory +/// and resolves packages from it. +#[derive(Debug)] +pub struct LocalNpmPackageResolver { + fs: Arc<dyn deno_fs::FileSystem>, + cache: Arc<NpmCache>, + progress_bar: ProgressBar, + resolution: Arc<NpmResolution>, + registry_url: Url, + root_node_modules_path: PathBuf, + root_node_modules_url: Url, + system_info: NpmSystemInfo, + registry_read_permission_checker: RegistryReadPermissionChecker, +} + +impl LocalNpmPackageResolver { + pub fn new( + fs: Arc<dyn deno_fs::FileSystem>, + cache: Arc<NpmCache>, + progress_bar: ProgressBar, + registry_url: Url, + node_modules_folder: PathBuf, + resolution: Arc<NpmResolution>, + system_info: NpmSystemInfo, + ) -> Self { + Self { + fs: fs.clone(), + cache, + progress_bar, + resolution, + registry_url, + root_node_modules_url: Url::from_directory_path(&node_modules_folder) + .unwrap(), + root_node_modules_path: node_modules_folder.clone(), + system_info, + registry_read_permission_checker: RegistryReadPermissionChecker::new( + fs, + node_modules_folder, + ), + } + } + + fn resolve_package_root(&self, path: &Path) -> PathBuf { + let mut last_found = path; + loop { + let parent = last_found.parent().unwrap(); + if parent.file_name().unwrap() == "node_modules" { + return last_found.to_path_buf(); + } else { + last_found = parent; + } + } + } + + fn resolve_folder_for_specifier( + &self, + specifier: &ModuleSpecifier, + ) -> Result<Option<PathBuf>, AnyError> { + let Some(relative_url) = + self.root_node_modules_url.make_relative(specifier) + else { + return Ok(None); + }; + if relative_url.starts_with("../") { + return Ok(None); + } + // it's within the directory, so use it + let Some(path) = specifier.to_file_path().ok() else { + return Ok(None); + }; + // Canonicalize the path so it's not pointing to the symlinked directory + // in `node_modules` directory of the referrer. + canonicalize_path_maybe_not_exists_with_fs(&path, |path| { + self + .fs + .realpath_sync(path) + .map_err(|err| err.into_io_error()) + }) + .map(Some) + .map_err(|err| err.into()) + } +} + +#[async_trait] +impl NpmPackageFsResolver for LocalNpmPackageResolver { + fn root_dir_url(&self) -> &Url { + &self.root_node_modules_url + } + + fn node_modules_path(&self) -> Option<PathBuf> { + Some(self.root_node_modules_path.clone()) + } + + fn package_folder(&self, id: &NpmPackageId) -> Result<PathBuf, AnyError> { + match self.resolution.resolve_pkg_cache_folder_id_from_pkg_id(id) { + // package is stored at: + // node_modules/.deno/<package_cache_folder_id_folder_name>/node_modules/<package_name> + Some(cache_folder_id) => Ok( + self + .root_node_modules_path + .join(".deno") + .join(get_package_folder_id_folder_name(&cache_folder_id)) + .join("node_modules") + .join(&cache_folder_id.nv.name), + ), + None => bail!( + "Could not find package information for '{}'", + id.as_serialized() + ), + } + } + + fn resolve_package_folder_from_package( + &self, + name: &str, + referrer: &ModuleSpecifier, + mode: NodeResolutionMode, + ) -> Result<PathBuf, AnyError> { + let Some(local_path) = self.resolve_folder_for_specifier(referrer)? else { + bail!("could not find npm package for '{}'", referrer); + }; + let package_root_path = self.resolve_package_root(&local_path); + let mut current_folder = package_root_path.as_path(); + loop { + current_folder = current_folder.parent().unwrap(); + let node_modules_folder = if current_folder.ends_with("node_modules") { + Cow::Borrowed(current_folder) + } else { + Cow::Owned(current_folder.join("node_modules")) + }; + let sub_dir = join_package_name(&node_modules_folder, name); + if self.fs.is_dir_sync(&sub_dir) { + // if doing types resolution, only resolve the package if it specifies a types property + if mode.is_types() && !name.starts_with("@types/") { + let package_json = PackageJson::load_skip_read_permission( + &*self.fs, + sub_dir.join("package.json"), + )?; + if package_json.types.is_some() { + return Ok(sub_dir); + } + } else { + return Ok(sub_dir); + } + } + + // if doing type resolution, check for the existence of a @types package + if mode.is_types() && !name.starts_with("@types/") { + let sub_dir = + join_package_name(&node_modules_folder, &types_package_name(name)); + if self.fs.is_dir_sync(&sub_dir) { + return Ok(sub_dir); + } + } + + if current_folder == self.root_node_modules_path { + bail!( + "could not find package '{}' from referrer '{}'.", + name, + referrer + ); + } + } + } + + fn resolve_package_folder_from_specifier( + &self, + specifier: &ModuleSpecifier, + ) -> Result<Option<PathBuf>, AnyError> { + let Some(local_path) = self.resolve_folder_for_specifier(specifier)? else { + return Ok(None); + }; + let package_root_path = self.resolve_package_root(&local_path); + Ok(Some(package_root_path)) + } + + fn resolve_package_cache_folder_id_from_specifier( + &self, + specifier: &ModuleSpecifier, + ) -> Result<Option<NpmPackageCacheFolderId>, AnyError> { + let Some(folder_path) = + self.resolve_package_folder_from_specifier(specifier)? + else { + return Ok(None); + }; + let folder_name = folder_path.parent().unwrap().to_string_lossy(); + Ok(get_package_folder_id_from_folder_name(&folder_name)) + } + + async fn cache_packages(&self) -> Result<(), AnyError> { + sync_resolution_with_fs( + &self.resolution.snapshot(), + &self.cache, + &self.progress_bar, + &self.registry_url, + &self.root_node_modules_path, + &self.system_info, + ) + .await + } + + fn ensure_read_permission( + &self, + permissions: &dyn NodePermissions, + path: &Path, + ) -> Result<(), AnyError> { + self + .registry_read_permission_checker + .ensure_registry_read_permission(permissions, path) + } +} + +/// Creates a pnpm style folder structure. +async fn sync_resolution_with_fs( + snapshot: &NpmResolutionSnapshot, + cache: &Arc<NpmCache>, + progress_bar: &ProgressBar, + registry_url: &Url, + root_node_modules_dir_path: &Path, + system_info: &NpmSystemInfo, +) -> Result<(), AnyError> { + if snapshot.is_empty() { + return Ok(()); // don't create the directory + } + + let deno_local_registry_dir = root_node_modules_dir_path.join(".deno"); + let deno_node_modules_dir = deno_local_registry_dir.join("node_modules"); + fs::create_dir_all(&deno_node_modules_dir).with_context(|| { + format!("Creating '{}'", deno_local_registry_dir.display()) + })?; + + let single_process_lock = LaxSingleProcessFsFlag::lock( + deno_local_registry_dir.join(".deno.lock"), + // similar message used by cargo build + "waiting for file lock on node_modules directory", + ) + .await; + + // load this after we get the directory lock + let mut setup_cache = + SetupCache::load(deno_local_registry_dir.join(".setup-cache.bin")); + + let pb_clear_guard = progress_bar.clear_guard(); // prevent flickering + + // 1. Write all the packages out the .deno directory. + // + // Copy (hardlink in future) <global_registry_cache>/<package_id>/ to + // node_modules/.deno/<package_folder_id_folder_name>/node_modules/<package_name> + let package_partitions = + snapshot.all_system_packages_partitioned(system_info); + let mut handles: Vec<JoinHandle<Result<(), AnyError>>> = + Vec::with_capacity(package_partitions.packages.len()); + let mut newest_packages_by_name: HashMap<&String, &NpmResolutionPackage> = + HashMap::with_capacity(package_partitions.packages.len()); + for package in &package_partitions.packages { + if let Some(current_pkg) = + newest_packages_by_name.get_mut(&package.id.nv.name) + { + if current_pkg.id.nv.cmp(&package.id.nv) == Ordering::Less { + *current_pkg = package; + } + } else { + newest_packages_by_name.insert(&package.id.nv.name, package); + }; + + let package_folder_name = + get_package_folder_id_folder_name(&package.get_package_cache_folder_id()); + let folder_path = deno_local_registry_dir.join(&package_folder_name); + let initialized_file = folder_path.join(".initialized"); + if !cache + .cache_setting() + .should_use_for_npm_package(&package.id.nv.name) + || !initialized_file.exists() + { + // cache bust the dep from the dep setup cache so the symlinks + // are forced to be recreated + setup_cache.remove_dep(&package_folder_name); + + let pb = progress_bar.clone(); + let cache = cache.clone(); + let registry_url = registry_url.clone(); + let package = package.clone(); + let handle = spawn(async move { + cache + .ensure_package(&package.id.nv, &package.dist, ®istry_url) + .await?; + let pb_guard = pb.update_with_prompt( + ProgressMessagePrompt::Initialize, + &package.id.nv.to_string(), + ); + let sub_node_modules = folder_path.join("node_modules"); + let package_path = + join_package_name(&sub_node_modules, &package.id.nv.name); + fs::create_dir_all(&package_path) + .with_context(|| format!("Creating '{}'", folder_path.display()))?; + let cache_folder = cache + .package_folder_for_name_and_version(&package.id.nv, ®istry_url); + // for now copy, but in the future consider hard linking + copy_dir_recursive(&cache_folder, &package_path)?; + // write out a file that indicates this folder has been initialized + fs::write(initialized_file, "")?; + // finally stop showing the progress bar + drop(pb_guard); // explicit for clarity + Ok(()) + }); + handles.push(handle); + } + } + + let results = futures::future::join_all(handles).await; + for result in results { + result??; // surface the first error + } + + // 2. Create any "copy" packages, which are used for peer dependencies + for package in &package_partitions.copy_packages { + let package_cache_folder_id = package.get_package_cache_folder_id(); + let destination_path = deno_local_registry_dir + .join(get_package_folder_id_folder_name(&package_cache_folder_id)); + let initialized_file = destination_path.join(".initialized"); + if !initialized_file.exists() { + let sub_node_modules = destination_path.join("node_modules"); + let package_path = + join_package_name(&sub_node_modules, &package.id.nv.name); + fs::create_dir_all(&package_path).with_context(|| { + format!("Creating '{}'", destination_path.display()) + })?; + let source_path = join_package_name( + &deno_local_registry_dir + .join(get_package_folder_id_folder_name( + &package_cache_folder_id.with_no_count(), + )) + .join("node_modules"), + &package.id.nv.name, + ); + hard_link_dir_recursive(&source_path, &package_path)?; + // write out a file that indicates this folder has been initialized + fs::write(initialized_file, "")?; + } + } + + // 3. Symlink all the dependencies into the .deno directory. + // + // Symlink node_modules/.deno/<package_id>/node_modules/<dep_name> to + // node_modules/.deno/<dep_id>/node_modules/<dep_package_name> + for package in package_partitions.iter_all() { + let package_folder_name = + get_package_folder_id_folder_name(&package.get_package_cache_folder_id()); + let sub_node_modules = deno_local_registry_dir + .join(&package_folder_name) + .join("node_modules"); + let mut dep_setup_cache = setup_cache.with_dep(&package_folder_name); + for (name, dep_id) in &package.dependencies { + let dep_cache_folder_id = snapshot + .package_from_id(dep_id) + .unwrap() + .get_package_cache_folder_id(); + let dep_folder_name = + get_package_folder_id_folder_name(&dep_cache_folder_id); + if dep_setup_cache.insert(name, &dep_folder_name) { + let dep_folder_path = join_package_name( + &deno_local_registry_dir + .join(dep_folder_name) + .join("node_modules"), + &dep_id.nv.name, + ); + symlink_package_dir( + &dep_folder_path, + &join_package_name(&sub_node_modules, name), + )?; + } + } + } + + // 4. Create all the top level packages in the node_modules folder, which are symlinks. + // + // Symlink node_modules/<package_name> to + // node_modules/.deno/<package_id>/node_modules/<package_name> + let mut found_names = HashSet::new(); + let mut ids = snapshot.top_level_packages().collect::<Vec<_>>(); + ids.sort_by(|a, b| b.cmp(a)); // create determinism and only include the latest version + for id in ids { + if !found_names.insert(&id.nv.name) { + continue; // skip, already handled + } + let package = snapshot.package_from_id(id).unwrap(); + let target_folder_name = + get_package_folder_id_folder_name(&package.get_package_cache_folder_id()); + if setup_cache.insert_root_symlink(&id.nv.name, &target_folder_name) { + let local_registry_package_path = join_package_name( + &deno_local_registry_dir + .join(target_folder_name) + .join("node_modules"), + &id.nv.name, + ); + + symlink_package_dir( + &local_registry_package_path, + &join_package_name(root_node_modules_dir_path, &id.nv.name), + )?; + } + } + + // 5. Create a node_modules/.deno/node_modules/<package-name> directory with + // the remaining packages + for package in newest_packages_by_name.values() { + if !found_names.insert(&package.id.nv.name) { + continue; // skip, already handled + } + + let target_folder_name = + get_package_folder_id_folder_name(&package.get_package_cache_folder_id()); + if setup_cache.insert_deno_symlink(&package.id.nv.name, &target_folder_name) + { + let local_registry_package_path = join_package_name( + &deno_local_registry_dir + .join(target_folder_name) + .join("node_modules"), + &package.id.nv.name, + ); + + symlink_package_dir( + &local_registry_package_path, + &join_package_name(&deno_node_modules_dir, &package.id.nv.name), + )?; + } + } + + setup_cache.save(); + drop(single_process_lock); + drop(pb_clear_guard); + + Ok(()) +} + +/// Represents a dependency at `node_modules/.deno/<package_id>/` +struct SetupCacheDep<'a> { + previous: Option<&'a HashMap<String, String>>, + current: &'a mut HashMap<String, String>, +} + +impl<'a> SetupCacheDep<'a> { + pub fn insert(&mut self, name: &str, target_folder_name: &str) -> bool { + self + .current + .insert(name.to_string(), target_folder_name.to_string()); + if let Some(previous_target) = self.previous.and_then(|p| p.get(name)) { + previous_target != target_folder_name + } else { + true + } + } +} + +#[derive(Debug, Default, Serialize, Deserialize, PartialEq, Eq)] +struct SetupCacheData { + root_symlinks: HashMap<String, String>, + deno_symlinks: HashMap<String, String>, + dep_symlinks: HashMap<String, HashMap<String, String>>, +} + +/// It is very slow to try to re-setup the symlinks each time, so this will +/// cache what we've setup on the last run and only update what is necessary. +/// Obviously this could lead to issues if the cache gets out of date with the +/// file system, such as if the user manually deletes a symlink. +struct SetupCache { + file_path: PathBuf, + previous: Option<SetupCacheData>, + current: SetupCacheData, +} + +impl SetupCache { + pub fn load(file_path: PathBuf) -> Self { + let previous = std::fs::read(&file_path) + .ok() + .and_then(|data| bincode::deserialize(&data).ok()); + Self { + file_path, + previous, + current: Default::default(), + } + } + + pub fn save(&self) -> bool { + if let Some(previous) = &self.previous { + if previous == &self.current { + return false; // nothing to save + } + } + + bincode::serialize(&self.current).ok().and_then(|data| { + atomic_write_file(&self.file_path, data, CACHE_PERM).ok() + }); + true + } + + /// Inserts and checks for the existence of a root symlink + /// at `node_modules/<package_name>` pointing to + /// `node_modules/.deno/<package_id>/` + pub fn insert_root_symlink( + &mut self, + name: &str, + target_folder_name: &str, + ) -> bool { + self + .current + .root_symlinks + .insert(name.to_string(), target_folder_name.to_string()); + if let Some(previous_target) = self + .previous + .as_ref() + .and_then(|p| p.root_symlinks.get(name)) + { + previous_target != target_folder_name + } else { + true + } + } + + /// Inserts and checks for the existence of a symlink at + /// `node_modules/.deno/node_modules/<package_name>` pointing to + /// `node_modules/.deno/<package_id>/` + pub fn insert_deno_symlink( + &mut self, + name: &str, + target_folder_name: &str, + ) -> bool { + self + .current + .deno_symlinks + .insert(name.to_string(), target_folder_name.to_string()); + if let Some(previous_target) = self + .previous + .as_ref() + .and_then(|p| p.deno_symlinks.get(name)) + { + previous_target != target_folder_name + } else { + true + } + } + + pub fn remove_dep(&mut self, parent_name: &str) { + if let Some(previous) = &mut self.previous { + previous.dep_symlinks.remove(parent_name); + } + } + + pub fn with_dep(&mut self, parent_name: &str) -> SetupCacheDep<'_> { + SetupCacheDep { + previous: self + .previous + .as_ref() + .and_then(|p| p.dep_symlinks.get(parent_name)), + current: self + .current + .dep_symlinks + .entry(parent_name.to_string()) + .or_default(), + } + } +} + +fn get_package_folder_id_folder_name( + folder_id: &NpmPackageCacheFolderId, +) -> String { + let copy_str = if folder_id.copy_index == 0 { + "".to_string() + } else { + format!("_{}", folder_id.copy_index) + }; + let nv = &folder_id.nv; + let name = if nv.name.to_lowercase() == nv.name { + Cow::Borrowed(&nv.name) + } else { + Cow::Owned(format!("_{}", mixed_case_package_name_encode(&nv.name))) + }; + format!("{}@{}{}", name, nv.version, copy_str).replace('/', "+") +} + +fn get_package_folder_id_from_folder_name( + folder_name: &str, +) -> Option<NpmPackageCacheFolderId> { + let folder_name = folder_name.replace('+', "/"); + let (name, ending) = folder_name.rsplit_once('@')?; + let name = if let Some(encoded_name) = name.strip_prefix('_') { + mixed_case_package_name_decode(encoded_name)? + } else { + name.to_string() + }; + let (raw_version, copy_index) = match ending.split_once('_') { + Some((raw_version, copy_index)) => { + let copy_index = copy_index.parse::<u8>().ok()?; + (raw_version, copy_index) + } + None => (ending, 0), + }; + let version = deno_semver::Version::parse_from_npm(raw_version).ok()?; + Some(NpmPackageCacheFolderId { + nv: PackageNv { name, version }, + copy_index, + }) +} + +fn symlink_package_dir( + old_path: &Path, + new_path: &Path, +) -> Result<(), AnyError> { + let new_parent = new_path.parent().unwrap(); + if new_parent.file_name().unwrap() != "node_modules" { + // create the parent folder that will contain the symlink + fs::create_dir_all(new_parent) + .with_context(|| format!("Creating '{}'", new_parent.display()))?; + } + + // need to delete the previous symlink before creating a new one + let _ignore = fs::remove_dir_all(new_path); + + #[cfg(windows)] + return junction_or_symlink_dir(old_path, new_path); + #[cfg(not(windows))] + symlink_dir(old_path, new_path) +} + +#[cfg(windows)] +fn junction_or_symlink_dir( + old_path: &Path, + new_path: &Path, +) -> Result<(), AnyError> { + // Use junctions because they're supported on ntfs file systems without + // needing to elevate privileges on Windows + + match junction::create(old_path, new_path) { + Ok(()) => Ok(()), + Err(junction_err) => { + if cfg!(debug) { + // When running the tests, junctions should be created, but if not then + // surface this error. + log::warn!("Error creating junction. {:#}", junction_err); + } + + match symlink_dir(old_path, new_path) { + Ok(()) => Ok(()), + Err(symlink_err) => bail!( + concat!( + "Failed creating junction and fallback symlink in node_modules folder.\n\n", + "{:#}\n\n{:#}", + ), + junction_err, + symlink_err, + ), + } + } + } +} + +fn join_package_name(path: &Path, package_name: &str) -> PathBuf { + let mut path = path.to_path_buf(); + // ensure backslashes are used on windows + for part in package_name.split('/') { + path = path.join(part); + } + path +} + +#[cfg(test)] +mod test { + use deno_npm::NpmPackageCacheFolderId; + use deno_semver::package::PackageNv; + use test_util::TempDir; + + use super::*; + + #[test] + fn test_get_package_folder_id_folder_name() { + let cases = vec![ + ( + NpmPackageCacheFolderId { + nv: PackageNv::from_str("@types/foo@1.2.3").unwrap(), + copy_index: 1, + }, + "@types+foo@1.2.3_1".to_string(), + ), + ( + NpmPackageCacheFolderId { + nv: PackageNv::from_str("JSON@3.2.1").unwrap(), + copy_index: 0, + }, + "_jjju6tq@3.2.1".to_string(), + ), + ]; + for (input, output) in cases { + assert_eq!(get_package_folder_id_folder_name(&input), output); + let folder_id = get_package_folder_id_from_folder_name(&output).unwrap(); + assert_eq!(folder_id, input); + } + } + + #[test] + fn test_setup_cache() { + let temp_dir = TempDir::new(); + let cache_bin_path = temp_dir.path().join("cache.bin").to_path_buf(); + let mut cache = SetupCache::load(cache_bin_path.clone()); + assert!(cache.insert_deno_symlink("package-a", "package-a@1.0.0")); + assert!(cache.insert_root_symlink("package-a", "package-a@1.0.0")); + assert!(cache + .with_dep("package-a") + .insert("package-b", "package-b@1.0.0")); + assert!(cache.save()); + + let mut cache = SetupCache::load(cache_bin_path.clone()); + assert!(!cache.insert_deno_symlink("package-a", "package-a@1.0.0")); + assert!(!cache.insert_root_symlink("package-a", "package-a@1.0.0")); + assert!(!cache + .with_dep("package-a") + .insert("package-b", "package-b@1.0.0")); + assert!(!cache.save()); + assert!(cache.insert_root_symlink("package-b", "package-b@0.2.0")); + assert!(cache.save()); + + let mut cache = SetupCache::load(cache_bin_path); + cache.remove_dep("package-a"); + assert!(cache + .with_dep("package-a") + .insert("package-b", "package-b@1.0.0")); + } +} diff --git a/cli/npm/managed/resolvers/mod.rs b/cli/npm/managed/resolvers/mod.rs new file mode 100644 index 000000000..b6d96c4af --- /dev/null +++ b/cli/npm/managed/resolvers/mod.rs @@ -0,0 +1,50 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +mod common; +mod global; +mod local; + +use std::path::PathBuf; +use std::sync::Arc; + +use deno_core::url::Url; +use deno_npm::NpmSystemInfo; +use deno_runtime::deno_fs::FileSystem; + +use crate::npm::NpmCache; +use crate::util::progress_bar::ProgressBar; + +pub use self::common::NpmPackageFsResolver; +use self::global::GlobalNpmPackageResolver; +use self::local::LocalNpmPackageResolver; + +use super::NpmResolution; + +pub fn create_npm_fs_resolver( + fs: Arc<dyn FileSystem>, + cache: Arc<NpmCache>, + progress_bar: &ProgressBar, + registry_url: Url, + resolution: Arc<NpmResolution>, + maybe_node_modules_path: Option<PathBuf>, + system_info: NpmSystemInfo, +) -> Arc<dyn NpmPackageFsResolver> { + match maybe_node_modules_path { + Some(node_modules_folder) => Arc::new(LocalNpmPackageResolver::new( + fs, + cache, + progress_bar.clone(), + registry_url, + node_modules_folder, + resolution, + system_info, + )), + None => Arc::new(GlobalNpmPackageResolver::new( + fs, + cache, + registry_url, + resolution, + system_info, + )), + } +} |