diff options
Diffstat (limited to 'cli/npm/resolvers')
-rw-r--r-- | cli/npm/resolvers/common.rs | 173 | ||||
-rw-r--r-- | cli/npm/resolvers/global.rs | 184 | ||||
-rw-r--r-- | cli/npm/resolvers/local.rs | 780 | ||||
-rw-r--r-- | cli/npm/resolvers/mod.rs | 416 |
4 files changed, 0 insertions, 1553 deletions
diff --git a/cli/npm/resolvers/common.rs b/cli/npm/resolvers/common.rs deleted file mode 100644 index 4076579bf..000000000 --- a/cli/npm/resolvers/common.rs +++ /dev/null @@ -1,173 +0,0 @@ -// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. - -use std::collections::HashMap; -use std::io::ErrorKind; -use std::path::Path; -use std::path::PathBuf; -use std::sync::Arc; -use std::sync::Mutex; - -use async_trait::async_trait; -use deno_ast::ModuleSpecifier; -use deno_core::error::AnyError; -use deno_core::futures; -use deno_core::unsync::spawn; -use deno_core::url::Url; -use deno_npm::NpmPackageCacheFolderId; -use deno_npm::NpmPackageId; -use deno_npm::NpmResolutionPackage; -use deno_runtime::deno_fs::FileSystem; -use deno_runtime::deno_node::NodePermissions; -use deno_runtime::deno_node::NodeResolutionMode; - -use crate::npm::NpmCache; - -/// Part of the resolution that interacts with the file system. -#[async_trait] -pub trait NpmPackageFsResolver: Send + Sync { - /// Specifier for the root directory. - fn root_dir_url(&self) -> &Url; - - /// The local node_modules folder if it is applicable to the implementation. - fn node_modules_path(&self) -> Option<PathBuf>; - - fn package_folder( - &self, - package_id: &NpmPackageId, - ) -> Result<PathBuf, AnyError>; - - fn resolve_package_folder_from_package( - &self, - name: &str, - referrer: &ModuleSpecifier, - mode: NodeResolutionMode, - ) -> Result<PathBuf, AnyError>; - - fn resolve_package_folder_from_specifier( - &self, - specifier: &ModuleSpecifier, - ) -> Result<Option<PathBuf>, AnyError>; - - fn resolve_package_cache_folder_id_from_specifier( - &self, - specifier: &ModuleSpecifier, - ) -> Result<Option<NpmPackageCacheFolderId>, AnyError>; - - async fn cache_packages(&self) -> Result<(), AnyError>; - - fn ensure_read_permission( - &self, - permissions: &dyn NodePermissions, - path: &Path, - ) -> Result<(), AnyError>; -} - -#[derive(Debug)] -pub struct RegistryReadPermissionChecker { - fs: Arc<dyn FileSystem>, - cache: Mutex<HashMap<PathBuf, PathBuf>>, - registry_path: PathBuf, -} - -impl RegistryReadPermissionChecker { - pub fn new(fs: Arc<dyn FileSystem>, registry_path: PathBuf) -> Self { - Self { - fs, - registry_path, - cache: Default::default(), - } - } - - pub fn ensure_registry_read_permission( - &self, - permissions: &dyn NodePermissions, - path: &Path, - ) -> Result<(), AnyError> { - // allow reading if it's in the node_modules - let is_path_in_node_modules = path.starts_with(&self.registry_path) - && path - .components() - .all(|c| !matches!(c, std::path::Component::ParentDir)); - - if is_path_in_node_modules { - let mut cache = self.cache.lock().unwrap(); - let registry_path_canon = match cache.get(&self.registry_path) { - Some(canon) => canon.clone(), - None => { - let canon = self.fs.realpath_sync(&self.registry_path)?; - cache.insert(self.registry_path.to_path_buf(), canon.clone()); - canon - } - }; - - let path_canon = match cache.get(path) { - Some(canon) => canon.clone(), - None => { - let canon = self.fs.realpath_sync(path); - if let Err(e) = &canon { - if e.kind() == ErrorKind::NotFound { - return Ok(()); - } - } - - let canon = canon?; - cache.insert(path.to_path_buf(), canon.clone()); - canon - } - }; - - if path_canon.starts_with(registry_path_canon) { - return Ok(()); - } - } - - permissions.check_read(path) - } -} - -/// Caches all the packages in parallel. -pub async fn cache_packages( - packages: Vec<NpmResolutionPackage>, - cache: &Arc<NpmCache>, - registry_url: &Url, -) -> Result<(), AnyError> { - let mut handles = Vec::with_capacity(packages.len()); - for package in packages { - let cache = cache.clone(); - let registry_url = registry_url.clone(); - let handle = spawn(async move { - cache - .ensure_package(&package.id.nv, &package.dist, ®istry_url) - .await - }); - handles.push(handle); - } - let results = futures::future::join_all(handles).await; - for result in results { - // surface the first error - result??; - } - Ok(()) -} - -/// Gets the corresponding @types package for the provided package name. -pub fn types_package_name(package_name: &str) -> String { - debug_assert!(!package_name.starts_with("@types/")); - // Scoped packages will get two underscores for each slash - // https://github.com/DefinitelyTyped/DefinitelyTyped/tree/15f1ece08f7b498f4b9a2147c2a46e94416ca777#what-about-scoped-packages - format!("@types/{}", package_name.replace('/', "__")) -} - -#[cfg(test)] -mod test { - use super::types_package_name; - - #[test] - fn test_types_package_name() { - assert_eq!(types_package_name("name"), "@types/name"); - assert_eq!( - types_package_name("@scoped/package"), - "@types/@scoped__package" - ); - } -} diff --git a/cli/npm/resolvers/global.rs b/cli/npm/resolvers/global.rs deleted file mode 100644 index e1144f610..000000000 --- a/cli/npm/resolvers/global.rs +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. - -//! Code for global npm cache resolution. - -use std::path::Path; -use std::path::PathBuf; -use std::sync::Arc; - -use async_trait::async_trait; -use deno_ast::ModuleSpecifier; -use deno_core::anyhow::bail; -use deno_core::error::AnyError; -use deno_core::url::Url; -use deno_npm::resolution::PackageNotFoundFromReferrerError; -use deno_npm::NpmPackageCacheFolderId; -use deno_npm::NpmPackageId; -use deno_npm::NpmResolutionPackage; -use deno_npm::NpmSystemInfo; -use deno_runtime::deno_fs::FileSystem; -use deno_runtime::deno_node::NodePermissions; -use deno_runtime::deno_node::NodeResolutionMode; - -use crate::npm::resolution::NpmResolution; -use crate::npm::resolvers::common::cache_packages; -use crate::npm::NpmCache; - -use super::common::types_package_name; -use super::common::NpmPackageFsResolver; -use super::common::RegistryReadPermissionChecker; - -/// Resolves packages from the global npm cache. -#[derive(Debug)] -pub struct GlobalNpmPackageResolver { - cache: Arc<NpmCache>, - resolution: Arc<NpmResolution>, - registry_url: Url, - system_info: NpmSystemInfo, - registry_read_permission_checker: RegistryReadPermissionChecker, -} - -impl GlobalNpmPackageResolver { - pub fn new( - fs: Arc<dyn FileSystem>, - cache: Arc<NpmCache>, - registry_url: Url, - resolution: Arc<NpmResolution>, - system_info: NpmSystemInfo, - ) -> Self { - Self { - cache: cache.clone(), - resolution, - registry_url: registry_url.clone(), - system_info, - registry_read_permission_checker: RegistryReadPermissionChecker::new( - fs, - cache.registry_folder(®istry_url), - ), - } - } - - fn resolve_types_package( - &self, - package_name: &str, - referrer_pkg_id: &NpmPackageCacheFolderId, - ) -> Result<NpmResolutionPackage, Box<PackageNotFoundFromReferrerError>> { - let types_name = types_package_name(package_name); - self - .resolution - .resolve_package_from_package(&types_name, referrer_pkg_id) - } -} - -#[async_trait] -impl NpmPackageFsResolver for GlobalNpmPackageResolver { - fn root_dir_url(&self) -> &Url { - self.cache.root_dir_url() - } - - fn node_modules_path(&self) -> Option<PathBuf> { - None - } - - fn package_folder(&self, id: &NpmPackageId) -> Result<PathBuf, AnyError> { - let folder_id = self - .resolution - .resolve_pkg_cache_folder_id_from_pkg_id(id) - .unwrap(); - Ok( - self - .cache - .package_folder_for_id(&folder_id, &self.registry_url), - ) - } - - fn resolve_package_folder_from_package( - &self, - name: &str, - referrer: &ModuleSpecifier, - mode: NodeResolutionMode, - ) -> Result<PathBuf, AnyError> { - let Some(referrer_pkg_id) = self - .cache - .resolve_package_folder_id_from_specifier(referrer, &self.registry_url) - else { - bail!("could not find npm package for '{}'", referrer); - }; - let pkg = if mode.is_types() && !name.starts_with("@types/") { - // attempt to resolve the types package first, then fallback to the regular package - match self.resolve_types_package(name, &referrer_pkg_id) { - Ok(pkg) => pkg, - Err(_) => self - .resolution - .resolve_package_from_package(name, &referrer_pkg_id)?, - } - } else { - self - .resolution - .resolve_package_from_package(name, &referrer_pkg_id)? - }; - self.package_folder(&pkg.id) - } - - fn resolve_package_folder_from_specifier( - &self, - specifier: &ModuleSpecifier, - ) -> Result<Option<PathBuf>, AnyError> { - let Some(pkg_folder_id) = self - .cache - .resolve_package_folder_id_from_specifier(specifier, &self.registry_url) - else { - return Ok(None); - }; - Ok(Some( - self - .cache - .package_folder_for_id(&pkg_folder_id, &self.registry_url), - )) - } - - fn resolve_package_cache_folder_id_from_specifier( - &self, - specifier: &ModuleSpecifier, - ) -> Result<Option<NpmPackageCacheFolderId>, AnyError> { - Ok( - self.cache.resolve_package_folder_id_from_specifier( - specifier, - &self.registry_url, - ), - ) - } - - async fn cache_packages(&self) -> Result<(), AnyError> { - let package_partitions = self - .resolution - .all_system_packages_partitioned(&self.system_info); - - cache_packages( - package_partitions.packages, - &self.cache, - &self.registry_url, - ) - .await?; - - // create the copy package folders - for copy in package_partitions.copy_packages { - self.cache.ensure_copy_package( - ©.get_package_cache_folder_id(), - &self.registry_url, - )?; - } - - Ok(()) - } - - fn ensure_read_permission( - &self, - permissions: &dyn NodePermissions, - path: &Path, - ) -> Result<(), AnyError> { - self - .registry_read_permission_checker - .ensure_registry_read_permission(permissions, path) - } -} diff --git a/cli/npm/resolvers/local.rs b/cli/npm/resolvers/local.rs deleted file mode 100644 index afa95e756..000000000 --- a/cli/npm/resolvers/local.rs +++ /dev/null @@ -1,780 +0,0 @@ -// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. - -//! Code for local node_modules resolution. - -use std::borrow::Cow; -use std::cmp::Ordering; -use std::collections::HashMap; -use std::collections::HashSet; -use std::fs; -use std::path::Path; -use std::path::PathBuf; -use std::sync::Arc; - -use crate::cache::CACHE_PERM; -use crate::npm::cache::mixed_case_package_name_decode; -use crate::util::fs::atomic_write_file; -use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs; -use crate::util::fs::symlink_dir; -use crate::util::fs::LaxSingleProcessFsFlag; -use crate::util::progress_bar::ProgressBar; -use crate::util::progress_bar::ProgressMessagePrompt; -use async_trait::async_trait; -use deno_ast::ModuleSpecifier; -use deno_core::anyhow::bail; -use deno_core::anyhow::Context; -use deno_core::error::AnyError; -use deno_core::unsync::spawn; -use deno_core::unsync::JoinHandle; -use deno_core::url::Url; -use deno_npm::resolution::NpmResolutionSnapshot; -use deno_npm::NpmPackageCacheFolderId; -use deno_npm::NpmPackageId; -use deno_npm::NpmResolutionPackage; -use deno_npm::NpmSystemInfo; -use deno_runtime::deno_core::futures; -use deno_runtime::deno_fs; -use deno_runtime::deno_node::NodePermissions; -use deno_runtime::deno_node::NodeResolutionMode; -use deno_runtime::deno_node::PackageJson; -use deno_semver::package::PackageNv; -use serde::Deserialize; -use serde::Serialize; - -use crate::npm::cache::mixed_case_package_name_encode; -use crate::npm::resolution::NpmResolution; -use crate::npm::NpmCache; -use crate::util::fs::copy_dir_recursive; -use crate::util::fs::hard_link_dir_recursive; - -use super::common::types_package_name; -use super::common::NpmPackageFsResolver; -use super::common::RegistryReadPermissionChecker; - -/// Resolver that creates a local node_modules directory -/// and resolves packages from it. -#[derive(Debug)] -pub struct LocalNpmPackageResolver { - fs: Arc<dyn deno_fs::FileSystem>, - cache: Arc<NpmCache>, - progress_bar: ProgressBar, - resolution: Arc<NpmResolution>, - registry_url: Url, - root_node_modules_path: PathBuf, - root_node_modules_url: Url, - system_info: NpmSystemInfo, - registry_read_permission_checker: RegistryReadPermissionChecker, -} - -impl LocalNpmPackageResolver { - pub fn new( - fs: Arc<dyn deno_fs::FileSystem>, - cache: Arc<NpmCache>, - progress_bar: ProgressBar, - registry_url: Url, - node_modules_folder: PathBuf, - resolution: Arc<NpmResolution>, - system_info: NpmSystemInfo, - ) -> Self { - Self { - fs: fs.clone(), - cache, - progress_bar, - resolution, - registry_url, - root_node_modules_url: Url::from_directory_path(&node_modules_folder) - .unwrap(), - root_node_modules_path: node_modules_folder.clone(), - system_info, - registry_read_permission_checker: RegistryReadPermissionChecker::new( - fs, - node_modules_folder, - ), - } - } - - fn resolve_package_root(&self, path: &Path) -> PathBuf { - let mut last_found = path; - loop { - let parent = last_found.parent().unwrap(); - if parent.file_name().unwrap() == "node_modules" { - return last_found.to_path_buf(); - } else { - last_found = parent; - } - } - } - - fn resolve_folder_for_specifier( - &self, - specifier: &ModuleSpecifier, - ) -> Result<Option<PathBuf>, AnyError> { - let Some(relative_url) = - self.root_node_modules_url.make_relative(specifier) - else { - return Ok(None); - }; - if relative_url.starts_with("../") { - return Ok(None); - } - // it's within the directory, so use it - let Some(path) = specifier.to_file_path().ok() else { - return Ok(None); - }; - // Canonicalize the path so it's not pointing to the symlinked directory - // in `node_modules` directory of the referrer. - canonicalize_path_maybe_not_exists_with_fs(&path, |path| { - self - .fs - .realpath_sync(path) - .map_err(|err| err.into_io_error()) - }) - .map(Some) - .map_err(|err| err.into()) - } -} - -#[async_trait] -impl NpmPackageFsResolver for LocalNpmPackageResolver { - fn root_dir_url(&self) -> &Url { - &self.root_node_modules_url - } - - fn node_modules_path(&self) -> Option<PathBuf> { - Some(self.root_node_modules_path.clone()) - } - - fn package_folder(&self, id: &NpmPackageId) -> Result<PathBuf, AnyError> { - match self.resolution.resolve_pkg_cache_folder_id_from_pkg_id(id) { - // package is stored at: - // node_modules/.deno/<package_cache_folder_id_folder_name>/node_modules/<package_name> - Some(cache_folder_id) => Ok( - self - .root_node_modules_path - .join(".deno") - .join(get_package_folder_id_folder_name(&cache_folder_id)) - .join("node_modules") - .join(&cache_folder_id.nv.name), - ), - None => bail!( - "Could not find package information for '{}'", - id.as_serialized() - ), - } - } - - fn resolve_package_folder_from_package( - &self, - name: &str, - referrer: &ModuleSpecifier, - mode: NodeResolutionMode, - ) -> Result<PathBuf, AnyError> { - let Some(local_path) = self.resolve_folder_for_specifier(referrer)? else { - bail!("could not find npm package for '{}'", referrer); - }; - let package_root_path = self.resolve_package_root(&local_path); - let mut current_folder = package_root_path.as_path(); - loop { - current_folder = current_folder.parent().unwrap(); - let node_modules_folder = if current_folder.ends_with("node_modules") { - Cow::Borrowed(current_folder) - } else { - Cow::Owned(current_folder.join("node_modules")) - }; - let sub_dir = join_package_name(&node_modules_folder, name); - if self.fs.is_dir_sync(&sub_dir) { - // if doing types resolution, only resolve the package if it specifies a types property - if mode.is_types() && !name.starts_with("@types/") { - let package_json = PackageJson::load_skip_read_permission( - &*self.fs, - sub_dir.join("package.json"), - )?; - if package_json.types.is_some() { - return Ok(sub_dir); - } - } else { - return Ok(sub_dir); - } - } - - // if doing type resolution, check for the existence of a @types package - if mode.is_types() && !name.starts_with("@types/") { - let sub_dir = - join_package_name(&node_modules_folder, &types_package_name(name)); - if self.fs.is_dir_sync(&sub_dir) { - return Ok(sub_dir); - } - } - - if current_folder == self.root_node_modules_path { - bail!( - "could not find package '{}' from referrer '{}'.", - name, - referrer - ); - } - } - } - - fn resolve_package_folder_from_specifier( - &self, - specifier: &ModuleSpecifier, - ) -> Result<Option<PathBuf>, AnyError> { - let Some(local_path) = self.resolve_folder_for_specifier(specifier)? else { - return Ok(None); - }; - let package_root_path = self.resolve_package_root(&local_path); - Ok(Some(package_root_path)) - } - - fn resolve_package_cache_folder_id_from_specifier( - &self, - specifier: &ModuleSpecifier, - ) -> Result<Option<NpmPackageCacheFolderId>, AnyError> { - let Some(folder_path) = - self.resolve_package_folder_from_specifier(specifier)? - else { - return Ok(None); - }; - let folder_name = folder_path.parent().unwrap().to_string_lossy(); - Ok(get_package_folder_id_from_folder_name(&folder_name)) - } - - async fn cache_packages(&self) -> Result<(), AnyError> { - sync_resolution_with_fs( - &self.resolution.snapshot(), - &self.cache, - &self.progress_bar, - &self.registry_url, - &self.root_node_modules_path, - &self.system_info, - ) - .await - } - - fn ensure_read_permission( - &self, - permissions: &dyn NodePermissions, - path: &Path, - ) -> Result<(), AnyError> { - self - .registry_read_permission_checker - .ensure_registry_read_permission(permissions, path) - } -} - -/// Creates a pnpm style folder structure. -async fn sync_resolution_with_fs( - snapshot: &NpmResolutionSnapshot, - cache: &Arc<NpmCache>, - progress_bar: &ProgressBar, - registry_url: &Url, - root_node_modules_dir_path: &Path, - system_info: &NpmSystemInfo, -) -> Result<(), AnyError> { - if snapshot.is_empty() { - return Ok(()); // don't create the directory - } - - let deno_local_registry_dir = root_node_modules_dir_path.join(".deno"); - let deno_node_modules_dir = deno_local_registry_dir.join("node_modules"); - fs::create_dir_all(&deno_node_modules_dir).with_context(|| { - format!("Creating '{}'", deno_local_registry_dir.display()) - })?; - - let single_process_lock = LaxSingleProcessFsFlag::lock( - deno_local_registry_dir.join(".deno.lock"), - // similar message used by cargo build - "waiting for file lock on node_modules directory", - ) - .await; - - // load this after we get the directory lock - let mut setup_cache = - SetupCache::load(deno_local_registry_dir.join(".setup-cache.bin")); - - let pb_clear_guard = progress_bar.clear_guard(); // prevent flickering - - // 1. Write all the packages out the .deno directory. - // - // Copy (hardlink in future) <global_registry_cache>/<package_id>/ to - // node_modules/.deno/<package_folder_id_folder_name>/node_modules/<package_name> - let package_partitions = - snapshot.all_system_packages_partitioned(system_info); - let mut handles: Vec<JoinHandle<Result<(), AnyError>>> = - Vec::with_capacity(package_partitions.packages.len()); - let mut newest_packages_by_name: HashMap<&String, &NpmResolutionPackage> = - HashMap::with_capacity(package_partitions.packages.len()); - for package in &package_partitions.packages { - if let Some(current_pkg) = - newest_packages_by_name.get_mut(&package.id.nv.name) - { - if current_pkg.id.nv.cmp(&package.id.nv) == Ordering::Less { - *current_pkg = package; - } - } else { - newest_packages_by_name.insert(&package.id.nv.name, package); - }; - - let package_folder_name = - get_package_folder_id_folder_name(&package.get_package_cache_folder_id()); - let folder_path = deno_local_registry_dir.join(&package_folder_name); - let initialized_file = folder_path.join(".initialized"); - if !cache - .cache_setting() - .should_use_for_npm_package(&package.id.nv.name) - || !initialized_file.exists() - { - // cache bust the dep from the dep setup cache so the symlinks - // are forced to be recreated - setup_cache.remove_dep(&package_folder_name); - - let pb = progress_bar.clone(); - let cache = cache.clone(); - let registry_url = registry_url.clone(); - let package = package.clone(); - let handle = spawn(async move { - cache - .ensure_package(&package.id.nv, &package.dist, ®istry_url) - .await?; - let pb_guard = pb.update_with_prompt( - ProgressMessagePrompt::Initialize, - &package.id.nv.to_string(), - ); - let sub_node_modules = folder_path.join("node_modules"); - let package_path = - join_package_name(&sub_node_modules, &package.id.nv.name); - fs::create_dir_all(&package_path) - .with_context(|| format!("Creating '{}'", folder_path.display()))?; - let cache_folder = cache - .package_folder_for_name_and_version(&package.id.nv, ®istry_url); - // for now copy, but in the future consider hard linking - copy_dir_recursive(&cache_folder, &package_path)?; - // write out a file that indicates this folder has been initialized - fs::write(initialized_file, "")?; - // finally stop showing the progress bar - drop(pb_guard); // explicit for clarity - Ok(()) - }); - handles.push(handle); - } - } - - let results = futures::future::join_all(handles).await; - for result in results { - result??; // surface the first error - } - - // 2. Create any "copy" packages, which are used for peer dependencies - for package in &package_partitions.copy_packages { - let package_cache_folder_id = package.get_package_cache_folder_id(); - let destination_path = deno_local_registry_dir - .join(get_package_folder_id_folder_name(&package_cache_folder_id)); - let initialized_file = destination_path.join(".initialized"); - if !initialized_file.exists() { - let sub_node_modules = destination_path.join("node_modules"); - let package_path = - join_package_name(&sub_node_modules, &package.id.nv.name); - fs::create_dir_all(&package_path).with_context(|| { - format!("Creating '{}'", destination_path.display()) - })?; - let source_path = join_package_name( - &deno_local_registry_dir - .join(get_package_folder_id_folder_name( - &package_cache_folder_id.with_no_count(), - )) - .join("node_modules"), - &package.id.nv.name, - ); - hard_link_dir_recursive(&source_path, &package_path)?; - // write out a file that indicates this folder has been initialized - fs::write(initialized_file, "")?; - } - } - - // 3. Symlink all the dependencies into the .deno directory. - // - // Symlink node_modules/.deno/<package_id>/node_modules/<dep_name> to - // node_modules/.deno/<dep_id>/node_modules/<dep_package_name> - for package in package_partitions.iter_all() { - let package_folder_name = - get_package_folder_id_folder_name(&package.get_package_cache_folder_id()); - let sub_node_modules = deno_local_registry_dir - .join(&package_folder_name) - .join("node_modules"); - let mut dep_setup_cache = setup_cache.with_dep(&package_folder_name); - for (name, dep_id) in &package.dependencies { - let dep_cache_folder_id = snapshot - .package_from_id(dep_id) - .unwrap() - .get_package_cache_folder_id(); - let dep_folder_name = - get_package_folder_id_folder_name(&dep_cache_folder_id); - if dep_setup_cache.insert(name, &dep_folder_name) { - let dep_folder_path = join_package_name( - &deno_local_registry_dir - .join(dep_folder_name) - .join("node_modules"), - &dep_id.nv.name, - ); - symlink_package_dir( - &dep_folder_path, - &join_package_name(&sub_node_modules, name), - )?; - } - } - } - - // 4. Create all the top level packages in the node_modules folder, which are symlinks. - // - // Symlink node_modules/<package_name> to - // node_modules/.deno/<package_id>/node_modules/<package_name> - let mut found_names = HashSet::new(); - let mut ids = snapshot.top_level_packages().collect::<Vec<_>>(); - ids.sort_by(|a, b| b.cmp(a)); // create determinism and only include the latest version - for id in ids { - if !found_names.insert(&id.nv.name) { - continue; // skip, already handled - } - let package = snapshot.package_from_id(id).unwrap(); - let target_folder_name = - get_package_folder_id_folder_name(&package.get_package_cache_folder_id()); - if setup_cache.insert_root_symlink(&id.nv.name, &target_folder_name) { - let local_registry_package_path = join_package_name( - &deno_local_registry_dir - .join(target_folder_name) - .join("node_modules"), - &id.nv.name, - ); - - symlink_package_dir( - &local_registry_package_path, - &join_package_name(root_node_modules_dir_path, &id.nv.name), - )?; - } - } - - // 5. Create a node_modules/.deno/node_modules/<package-name> directory with - // the remaining packages - for package in newest_packages_by_name.values() { - if !found_names.insert(&package.id.nv.name) { - continue; // skip, already handled - } - - let target_folder_name = - get_package_folder_id_folder_name(&package.get_package_cache_folder_id()); - if setup_cache.insert_deno_symlink(&package.id.nv.name, &target_folder_name) - { - let local_registry_package_path = join_package_name( - &deno_local_registry_dir - .join(target_folder_name) - .join("node_modules"), - &package.id.nv.name, - ); - - symlink_package_dir( - &local_registry_package_path, - &join_package_name(&deno_node_modules_dir, &package.id.nv.name), - )?; - } - } - - setup_cache.save(); - drop(single_process_lock); - drop(pb_clear_guard); - - Ok(()) -} - -/// Represents a dependency at `node_modules/.deno/<package_id>/` -struct SetupCacheDep<'a> { - previous: Option<&'a HashMap<String, String>>, - current: &'a mut HashMap<String, String>, -} - -impl<'a> SetupCacheDep<'a> { - pub fn insert(&mut self, name: &str, target_folder_name: &str) -> bool { - self - .current - .insert(name.to_string(), target_folder_name.to_string()); - if let Some(previous_target) = self.previous.and_then(|p| p.get(name)) { - previous_target != target_folder_name - } else { - true - } - } -} - -#[derive(Debug, Default, Serialize, Deserialize, PartialEq, Eq)] -struct SetupCacheData { - root_symlinks: HashMap<String, String>, - deno_symlinks: HashMap<String, String>, - dep_symlinks: HashMap<String, HashMap<String, String>>, -} - -/// It is very slow to try to re-setup the symlinks each time, so this will -/// cache what we've setup on the last run and only update what is necessary. -/// Obviously this could lead to issues if the cache gets out of date with the -/// file system, such as if the user manually deletes a symlink. -struct SetupCache { - file_path: PathBuf, - previous: Option<SetupCacheData>, - current: SetupCacheData, -} - -impl SetupCache { - pub fn load(file_path: PathBuf) -> Self { - let previous = std::fs::read(&file_path) - .ok() - .and_then(|data| bincode::deserialize(&data).ok()); - Self { - file_path, - previous, - current: Default::default(), - } - } - - pub fn save(&self) -> bool { - if let Some(previous) = &self.previous { - if previous == &self.current { - return false; // nothing to save - } - } - - bincode::serialize(&self.current).ok().and_then(|data| { - atomic_write_file(&self.file_path, data, CACHE_PERM).ok() - }); - true - } - - /// Inserts and checks for the existence of a root symlink - /// at `node_modules/<package_name>` pointing to - /// `node_modules/.deno/<package_id>/` - pub fn insert_root_symlink( - &mut self, - name: &str, - target_folder_name: &str, - ) -> bool { - self - .current - .root_symlinks - .insert(name.to_string(), target_folder_name.to_string()); - if let Some(previous_target) = self - .previous - .as_ref() - .and_then(|p| p.root_symlinks.get(name)) - { - previous_target != target_folder_name - } else { - true - } - } - - /// Inserts and checks for the existence of a symlink at - /// `node_modules/.deno/node_modules/<package_name>` pointing to - /// `node_modules/.deno/<package_id>/` - pub fn insert_deno_symlink( - &mut self, - name: &str, - target_folder_name: &str, - ) -> bool { - self - .current - .deno_symlinks - .insert(name.to_string(), target_folder_name.to_string()); - if let Some(previous_target) = self - .previous - .as_ref() - .and_then(|p| p.deno_symlinks.get(name)) - { - previous_target != target_folder_name - } else { - true - } - } - - pub fn remove_dep(&mut self, parent_name: &str) { - if let Some(previous) = &mut self.previous { - previous.dep_symlinks.remove(parent_name); - } - } - - pub fn with_dep(&mut self, parent_name: &str) -> SetupCacheDep<'_> { - SetupCacheDep { - previous: self - .previous - .as_ref() - .and_then(|p| p.dep_symlinks.get(parent_name)), - current: self - .current - .dep_symlinks - .entry(parent_name.to_string()) - .or_default(), - } - } -} - -fn get_package_folder_id_folder_name( - folder_id: &NpmPackageCacheFolderId, -) -> String { - let copy_str = if folder_id.copy_index == 0 { - "".to_string() - } else { - format!("_{}", folder_id.copy_index) - }; - let nv = &folder_id.nv; - let name = if nv.name.to_lowercase() == nv.name { - Cow::Borrowed(&nv.name) - } else { - Cow::Owned(format!("_{}", mixed_case_package_name_encode(&nv.name))) - }; - format!("{}@{}{}", name, nv.version, copy_str).replace('/', "+") -} - -fn get_package_folder_id_from_folder_name( - folder_name: &str, -) -> Option<NpmPackageCacheFolderId> { - let folder_name = folder_name.replace('+', "/"); - let (name, ending) = folder_name.rsplit_once('@')?; - let name = if let Some(encoded_name) = name.strip_prefix('_') { - mixed_case_package_name_decode(encoded_name)? - } else { - name.to_string() - }; - let (raw_version, copy_index) = match ending.split_once('_') { - Some((raw_version, copy_index)) => { - let copy_index = copy_index.parse::<u8>().ok()?; - (raw_version, copy_index) - } - None => (ending, 0), - }; - let version = deno_semver::Version::parse_from_npm(raw_version).ok()?; - Some(NpmPackageCacheFolderId { - nv: PackageNv { name, version }, - copy_index, - }) -} - -fn symlink_package_dir( - old_path: &Path, - new_path: &Path, -) -> Result<(), AnyError> { - let new_parent = new_path.parent().unwrap(); - if new_parent.file_name().unwrap() != "node_modules" { - // create the parent folder that will contain the symlink - fs::create_dir_all(new_parent) - .with_context(|| format!("Creating '{}'", new_parent.display()))?; - } - - // need to delete the previous symlink before creating a new one - let _ignore = fs::remove_dir_all(new_path); - - #[cfg(windows)] - return junction_or_symlink_dir(old_path, new_path); - #[cfg(not(windows))] - symlink_dir(old_path, new_path) -} - -#[cfg(windows)] -fn junction_or_symlink_dir( - old_path: &Path, - new_path: &Path, -) -> Result<(), AnyError> { - // Use junctions because they're supported on ntfs file systems without - // needing to elevate privileges on Windows - - match junction::create(old_path, new_path) { - Ok(()) => Ok(()), - Err(junction_err) => { - if cfg!(debug) { - // When running the tests, junctions should be created, but if not then - // surface this error. - log::warn!("Error creating junction. {:#}", junction_err); - } - - match symlink_dir(old_path, new_path) { - Ok(()) => Ok(()), - Err(symlink_err) => bail!( - concat!( - "Failed creating junction and fallback symlink in node_modules folder.\n\n", - "{:#}\n\n{:#}", - ), - junction_err, - symlink_err, - ), - } - } - } -} - -fn join_package_name(path: &Path, package_name: &str) -> PathBuf { - let mut path = path.to_path_buf(); - // ensure backslashes are used on windows - for part in package_name.split('/') { - path = path.join(part); - } - path -} - -#[cfg(test)] -mod test { - use deno_npm::NpmPackageCacheFolderId; - use deno_semver::package::PackageNv; - use test_util::TempDir; - - use super::*; - - #[test] - fn test_get_package_folder_id_folder_name() { - let cases = vec![ - ( - NpmPackageCacheFolderId { - nv: PackageNv::from_str("@types/foo@1.2.3").unwrap(), - copy_index: 1, - }, - "@types+foo@1.2.3_1".to_string(), - ), - ( - NpmPackageCacheFolderId { - nv: PackageNv::from_str("JSON@3.2.1").unwrap(), - copy_index: 0, - }, - "_jjju6tq@3.2.1".to_string(), - ), - ]; - for (input, output) in cases { - assert_eq!(get_package_folder_id_folder_name(&input), output); - let folder_id = get_package_folder_id_from_folder_name(&output).unwrap(); - assert_eq!(folder_id, input); - } - } - - #[test] - fn test_setup_cache() { - let temp_dir = TempDir::new(); - let cache_bin_path = temp_dir.path().join("cache.bin").to_path_buf(); - let mut cache = SetupCache::load(cache_bin_path.clone()); - assert!(cache.insert_deno_symlink("package-a", "package-a@1.0.0")); - assert!(cache.insert_root_symlink("package-a", "package-a@1.0.0")); - assert!(cache - .with_dep("package-a") - .insert("package-b", "package-b@1.0.0")); - assert!(cache.save()); - - let mut cache = SetupCache::load(cache_bin_path.clone()); - assert!(!cache.insert_deno_symlink("package-a", "package-a@1.0.0")); - assert!(!cache.insert_root_symlink("package-a", "package-a@1.0.0")); - assert!(!cache - .with_dep("package-a") - .insert("package-b", "package-b@1.0.0")); - assert!(!cache.save()); - assert!(cache.insert_root_symlink("package-b", "package-b@0.2.0")); - assert!(cache.save()); - - let mut cache = SetupCache::load(cache_bin_path); - cache.remove_dep("package-a"); - assert!(cache - .with_dep("package-a") - .insert("package-b", "package-b@1.0.0")); - } -} diff --git a/cli/npm/resolvers/mod.rs b/cli/npm/resolvers/mod.rs deleted file mode 100644 index 07a122a3e..000000000 --- a/cli/npm/resolvers/mod.rs +++ /dev/null @@ -1,416 +0,0 @@ -// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. - -mod common; -mod global; -mod local; - -use std::collections::HashMap; -use std::path::Path; -use std::path::PathBuf; -use std::sync::Arc; - -use deno_ast::ModuleSpecifier; -use deno_core::error::AnyError; -use deno_core::parking_lot::Mutex; -use deno_core::serde_json; -use deno_core::url::Url; -use deno_npm::resolution::NpmResolutionSnapshot; -use deno_npm::resolution::PackageReqNotFoundError; -use deno_npm::resolution::SerializedNpmResolutionSnapshot; -use deno_npm::NpmPackageId; -use deno_npm::NpmSystemInfo; -use deno_runtime::deno_fs::FileSystem; -use deno_runtime::deno_node::NodePermissions; -use deno_runtime::deno_node::NodeResolutionMode; -use deno_runtime::deno_node::NpmResolver; -use deno_semver::npm::NpmPackageNvReference; -use deno_semver::npm::NpmPackageReqReference; -use deno_semver::package::PackageNv; -use deno_semver::package::PackageNvReference; -use deno_semver::package::PackageReq; -use global::GlobalNpmPackageResolver; -use serde::Deserialize; -use serde::Serialize; - -use crate::args::Lockfile; -use crate::util::fs::canonicalize_path_maybe_not_exists_with_fs; -use crate::util::progress_bar::ProgressBar; - -use self::local::LocalNpmPackageResolver; -use super::resolution::NpmResolution; -use super::NpmCache; - -pub use self::common::NpmPackageFsResolver; - -/// State provided to the process via an environment variable. -#[derive(Clone, Debug, Serialize, Deserialize)] -pub struct NpmProcessState { - pub snapshot: SerializedNpmResolutionSnapshot, - pub local_node_modules_path: Option<String>, -} - -pub enum InnerCliNpmResolverRef<'a> { - Managed(&'a ManagedCliNpmResolver), - #[allow(dead_code)] - Byonm(&'a ByonmCliNpmResolver), -} - -pub trait CliNpmResolver: NpmResolver { - fn into_npm_resolver(self: Arc<Self>) -> Arc<dyn NpmResolver>; - - fn root_dir_url(&self) -> &Url; - - fn as_inner(&self) -> InnerCliNpmResolverRef; - - fn as_managed(&self) -> Option<&ManagedCliNpmResolver> { - match self.as_inner() { - InnerCliNpmResolverRef::Managed(inner) => Some(inner), - InnerCliNpmResolverRef::Byonm(_) => None, - } - } - - fn node_modules_path(&self) -> Option<PathBuf>; - - /// Checks if the provided package req's folder is cached. - fn is_pkg_req_folder_cached(&self, req: &PackageReq) -> bool; - - fn resolve_pkg_nv_ref_from_pkg_req_ref( - &self, - req_ref: &NpmPackageReqReference, - ) -> Result<NpmPackageNvReference, PackageReqNotFoundError>; - - /// Resolve the root folder of the package the provided specifier is in. - /// - /// This will error when the provided specifier is not in an npm package. - fn resolve_pkg_folder_from_specifier( - &self, - specifier: &ModuleSpecifier, - ) -> Result<Option<PathBuf>, AnyError>; - - fn resolve_pkg_folder_from_deno_module_req( - &self, - req: &PackageReq, - ) -> Result<PathBuf, AnyError>; - - fn resolve_pkg_folder_from_deno_module( - &self, - nv: &PackageNv, - ) -> Result<PathBuf, AnyError>; - - /// Gets the state of npm for the process. - fn get_npm_process_state(&self) -> String; - - // todo(#18967): should instead return a hash state of the resolver - // or perhaps this could be non-BYONM only and byonm always runs deno check - fn package_reqs(&self) -> HashMap<PackageReq, PackageNv>; -} - -// todo(dsherret): implement this -pub struct ByonmCliNpmResolver; - -/// An npm resolver where the resolution is managed by Deno rather than -/// the user bringing their own node_modules (BYONM) on the file system. -pub struct ManagedCliNpmResolver { - fs: Arc<dyn FileSystem>, - fs_resolver: Arc<dyn NpmPackageFsResolver>, - resolution: Arc<NpmResolution>, - maybe_lockfile: Option<Arc<Mutex<Lockfile>>>, -} - -impl std::fmt::Debug for ManagedCliNpmResolver { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.debug_struct("ManagedNpmResolver") - .field("fs", &"<omitted>") - .field("fs_resolver", &"<omitted>") - .field("resolution", &"<omitted>") - .field("maybe_lockfile", &"<omitted>") - .finish() - } -} - -impl ManagedCliNpmResolver { - pub fn new( - fs: Arc<dyn FileSystem>, - resolution: Arc<NpmResolution>, - fs_resolver: Arc<dyn NpmPackageFsResolver>, - maybe_lockfile: Option<Arc<Mutex<Lockfile>>>, - ) -> Self { - Self { - fs, - fs_resolver, - resolution, - maybe_lockfile, - } - } - - pub fn resolve_pkg_folder_from_pkg_id( - &self, - pkg_id: &NpmPackageId, - ) -> Result<PathBuf, AnyError> { - let path = self.fs_resolver.package_folder(pkg_id)?; - let path = canonicalize_path_maybe_not_exists_with_fs(&path, |path| { - self - .fs - .realpath_sync(path) - .map_err(|err| err.into_io_error()) - })?; - log::debug!( - "Resolved package folder of {} to {}", - pkg_id.as_serialized(), - path.display() - ); - Ok(path) - } - - /// Resolves the package nv from the provided specifier. - pub fn resolve_pkg_id_from_specifier( - &self, - specifier: &ModuleSpecifier, - ) -> Result<Option<NpmPackageId>, AnyError> { - let Some(cache_folder_id) = self - .fs_resolver - .resolve_package_cache_folder_id_from_specifier(specifier)? - else { - return Ok(None); - }; - Ok(Some( - self - .resolution - .resolve_pkg_id_from_pkg_cache_folder_id(&cache_folder_id)?, - )) - } - - /// Attempts to get the package size in bytes. - pub fn package_size( - &self, - package_id: &NpmPackageId, - ) -> Result<u64, AnyError> { - let package_folder = self.fs_resolver.package_folder(package_id)?; - Ok(crate::util::fs::dir_size(&package_folder)?) - } - - /// Adds package requirements to the resolver and ensures everything is setup. - pub async fn add_package_reqs( - &self, - packages: &[PackageReq], - ) -> Result<(), AnyError> { - if packages.is_empty() { - return Ok(()); - } - - self.resolution.add_package_reqs(packages).await?; - self.fs_resolver.cache_packages().await?; - - // If there's a lock file, update it with all discovered npm packages - if let Some(lockfile_mutex) = &self.maybe_lockfile { - let mut lockfile = lockfile_mutex.lock(); - self.lock(&mut lockfile)?; - } - - Ok(()) - } - - /// Sets package requirements to the resolver, removing old requirements and adding new ones. - /// - /// This will retrieve and resolve package information, but not cache any package files. - pub async fn set_package_reqs( - &self, - packages: &[PackageReq], - ) -> Result<(), AnyError> { - self.resolution.set_package_reqs(packages).await - } - - pub fn snapshot(&self) -> NpmResolutionSnapshot { - self.resolution.snapshot() - } - - pub fn lock(&self, lockfile: &mut Lockfile) -> Result<(), AnyError> { - self.resolution.lock(lockfile) - } - - pub async fn inject_synthetic_types_node_package( - &self, - ) -> Result<(), AnyError> { - // add and ensure this isn't added to the lockfile - let package_reqs = vec![PackageReq::from_str("@types/node").unwrap()]; - self.resolution.add_package_reqs(&package_reqs).await?; - self.fs_resolver.cache_packages().await?; - - Ok(()) - } - - pub async fn resolve_pending(&self) -> Result<(), AnyError> { - self.resolution.resolve_pending().await?; - self.fs_resolver.cache_packages().await?; - Ok(()) - } - - fn resolve_pkg_id_from_pkg_req( - &self, - req: &PackageReq, - ) -> Result<NpmPackageId, PackageReqNotFoundError> { - self.resolution.resolve_pkg_id_from_pkg_req(req) - } -} - -impl NpmResolver for ManagedCliNpmResolver { - fn resolve_package_folder_from_package( - &self, - name: &str, - referrer: &ModuleSpecifier, - mode: NodeResolutionMode, - ) -> Result<PathBuf, AnyError> { - let path = self - .fs_resolver - .resolve_package_folder_from_package(name, referrer, mode)?; - log::debug!("Resolved {} from {} to {}", name, referrer, path.display()); - Ok(path) - } - - fn resolve_package_folder_from_path( - &self, - specifier: &ModuleSpecifier, - ) -> Result<Option<PathBuf>, AnyError> { - self.resolve_pkg_folder_from_specifier(specifier) - } - - fn in_npm_package(&self, specifier: &ModuleSpecifier) -> bool { - let root_dir_url = self.fs_resolver.root_dir_url(); - debug_assert!(root_dir_url.as_str().ends_with('/')); - specifier.as_ref().starts_with(root_dir_url.as_str()) - } - - fn ensure_read_permission( - &self, - permissions: &dyn NodePermissions, - path: &Path, - ) -> Result<(), AnyError> { - self.fs_resolver.ensure_read_permission(permissions, path) - } -} - -impl CliNpmResolver for ManagedCliNpmResolver { - fn into_npm_resolver(self: Arc<Self>) -> Arc<dyn NpmResolver> { - self - } - - fn root_dir_url(&self) -> &Url { - self.fs_resolver.root_dir_url() - } - - fn as_inner(&self) -> InnerCliNpmResolverRef { - InnerCliNpmResolverRef::Managed(self) - } - - fn node_modules_path(&self) -> Option<PathBuf> { - self.fs_resolver.node_modules_path() - } - - /// Checks if the provided package req's folder is cached. - fn is_pkg_req_folder_cached(&self, req: &PackageReq) -> bool { - self - .resolve_pkg_id_from_pkg_req(req) - .ok() - .and_then(|id| self.fs_resolver.package_folder(&id).ok()) - .map(|folder| folder.exists()) - .unwrap_or(false) - } - - fn resolve_pkg_nv_ref_from_pkg_req_ref( - &self, - req_ref: &NpmPackageReqReference, - ) -> Result<NpmPackageNvReference, PackageReqNotFoundError> { - let pkg_nv = self - .resolve_pkg_id_from_pkg_req(req_ref.req()) - .map(|id| id.nv)?; - Ok(NpmPackageNvReference::new(PackageNvReference { - nv: pkg_nv, - sub_path: req_ref.sub_path().map(|s| s.to_string()), - })) - } - - /// Resolve the root folder of the package the provided specifier is in. - /// - /// This will error when the provided specifier is not in an npm package. - fn resolve_pkg_folder_from_specifier( - &self, - specifier: &ModuleSpecifier, - ) -> Result<Option<PathBuf>, AnyError> { - let Some(path) = self - .fs_resolver - .resolve_package_folder_from_specifier(specifier)? - else { - return Ok(None); - }; - log::debug!( - "Resolved package folder of {} to {}", - specifier, - path.display() - ); - Ok(Some(path)) - } - - fn resolve_pkg_folder_from_deno_module_req( - &self, - req: &PackageReq, - ) -> Result<PathBuf, AnyError> { - let pkg_id = self.resolve_pkg_id_from_pkg_req(req)?; - self.resolve_pkg_folder_from_pkg_id(&pkg_id) - } - - fn resolve_pkg_folder_from_deno_module( - &self, - nv: &PackageNv, - ) -> Result<PathBuf, AnyError> { - let pkg_id = self.resolution.resolve_pkg_id_from_deno_module(nv)?; - self.resolve_pkg_folder_from_pkg_id(&pkg_id) - } - - /// Gets the state of npm for the process. - fn get_npm_process_state(&self) -> String { - serde_json::to_string(&NpmProcessState { - snapshot: self - .resolution - .serialized_valid_snapshot() - .into_serialized(), - local_node_modules_path: self - .fs_resolver - .node_modules_path() - .map(|p| p.to_string_lossy().to_string()), - }) - .unwrap() - } - - fn package_reqs(&self) -> HashMap<PackageReq, PackageNv> { - self.resolution.package_reqs() - } -} - -pub fn create_npm_fs_resolver( - fs: Arc<dyn FileSystem>, - cache: Arc<NpmCache>, - progress_bar: &ProgressBar, - registry_url: Url, - resolution: Arc<NpmResolution>, - maybe_node_modules_path: Option<PathBuf>, - system_info: NpmSystemInfo, -) -> Arc<dyn NpmPackageFsResolver> { - match maybe_node_modules_path { - Some(node_modules_folder) => Arc::new(LocalNpmPackageResolver::new( - fs, - cache, - progress_bar.clone(), - registry_url, - node_modules_folder, - resolution, - system_info, - )), - None => Arc::new(GlobalNpmPackageResolver::new( - fs, - cache, - registry_url, - resolution, - system_info, - )), - } -} |