From 7c3c13cecf48219cdcb90dc0b5019686cdd88626 Mon Sep 17 00:00:00 2001 From: Nathan Whitaker <17734409+nathanwhit@users.noreply.github.com> Date: Tue, 15 Oct 2024 16:46:42 -0700 Subject: fix(install): retry downloads of registry info / tarballs (#26278) Fixes #26085. Adds a basic retry utility with some defaults, starts off with a 100ms wait, then 250ms, then 500ms I've applied the retry in the http client, reusing an existing function, so this also applies to retrying downloads of deno binaries in `upgrade` and `compile`. I can make a separate function that doesn't retry so this doesn't affect `upgrade` and `compile`, but it seemed desirable to have retries there too, so I left it in. --- cli/npm/managed/cache/registry_info.rs | 11 +++++++---- cli/npm/managed/cache/tarball.rs | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) (limited to 'cli/npm/managed/cache') diff --git a/cli/npm/managed/cache/registry_info.rs b/cli/npm/managed/cache/registry_info.rs index 28b19373e..597a2283f 100644 --- a/cli/npm/managed/cache/registry_info.rs +++ b/cli/npm/managed/cache/registry_info.rs @@ -202,10 +202,13 @@ impl RegistryInfoDownloader { let guard = self.progress_bar.update(package_url.as_str()); let name = name.to_string(); async move { - let maybe_bytes = downloader - .http_client_provider - .get_or_create()? - .download_with_progress(package_url, maybe_auth_header, &guard) + let client = downloader.http_client_provider.get_or_create()?; + let maybe_bytes = client + .download_with_progress_and_retries( + package_url, + maybe_auth_header, + &guard, + ) .await?; match maybe_bytes { Some(bytes) => { diff --git a/cli/npm/managed/cache/tarball.rs b/cli/npm/managed/cache/tarball.rs index 4bcee38ea..7cf88d6d6 100644 --- a/cli/npm/managed/cache/tarball.rs +++ b/cli/npm/managed/cache/tarball.rs @@ -172,7 +172,7 @@ impl TarballCache { let guard = tarball_cache.progress_bar.update(&dist.tarball); let result = tarball_cache.http_client_provider .get_or_create()? - .download_with_progress(tarball_uri, maybe_auth_header, &guard) + .download_with_progress_and_retries(tarball_uri, maybe_auth_header, &guard) .await; let maybe_bytes = match result { Ok(maybe_bytes) => maybe_bytes, -- cgit v1.2.3 From 4b99cde504854baabdcf912f2fce3a7448119653 Mon Sep 17 00:00:00 2001 From: Marvin Hagemeister Date: Fri, 18 Oct 2024 20:38:57 +0200 Subject: fix(npm): ensure scoped package name is encoded in URLs (#26390) Fixes https://github.com/denoland/deno/issues/26385 --- cli/npm/managed/cache/registry_info.rs | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) (limited to 'cli/npm/managed/cache') diff --git a/cli/npm/managed/cache/registry_info.rs b/cli/npm/managed/cache/registry_info.rs index 597a2283f..6c4a7503b 100644 --- a/cli/npm/managed/cache/registry_info.rs +++ b/cli/npm/managed/cache/registry_info.rs @@ -242,6 +242,14 @@ impl RegistryInfoDownloader { fn get_package_url(&self, name: &str) -> Url { let registry_url = self.npmrc.get_registry_url(name); + // The '/' character in scoped package names "@scope/name" must be + // encoded for older third party registries. Newer registries and + // npm itself support both ways + // - encoded: https://registry.npmjs.org/@rollup%2fplugin-json + // - non-ecoded: https://registry.npmjs.org/@rollup/plugin-json + // To support as many third party registries as possible we'll + // always encode the '/' character. + // list of all characters used in npm packages: // !, ', (, ), *, -, ., /, [0-9], @, [A-Za-z], _, ~ const ASCII_SET: percent_encoding::AsciiSet = @@ -253,11 +261,14 @@ impl RegistryInfoDownloader { .remove(b'*') .remove(b'-') .remove(b'.') - .remove(b'/') .remove(b'@') .remove(b'_') .remove(b'~'); let name = percent_encoding::utf8_percent_encode(name, &ASCII_SET); - registry_url.join(&name.to_string()).unwrap() + registry_url + // Ensure that scoped package name percent encoding is lower cased + // to match npm. + .join(&name.to_string().replace("%2F", "%2f")) + .unwrap() } } -- cgit v1.2.3 From 5f0bb3c6f4328003012e98ba70ce18e4e2e842de Mon Sep 17 00:00:00 2001 From: Marvin Hagemeister Date: Thu, 24 Oct 2024 20:03:56 +0200 Subject: fix: `.npmrc` settings not being passed to install/add command (#26473) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We weren't passing the resolved npmrc settings to the install commands. This lead us to always fall back to the default registry url instead of using the one from npmrc. Fixes https://github.com/denoland/deno/issues/26139 Fixes https://github.com/denoland/deno/issues/26033 Fixes https://github.com/denoland/deno/issues/25924 Fixes https://github.com/denoland/deno/issues/25822 Fixes https://github.com/denoland/deno/issues/26152 --------- Co-authored-by: Bartek Iwańczuk --- cli/npm/managed/cache/mod.rs | 2 +- cli/npm/managed/cache/registry_info.rs | 64 +++++++++++++++++----------------- 2 files changed, 33 insertions(+), 33 deletions(-) (limited to 'cli/npm/managed/cache') diff --git a/cli/npm/managed/cache/mod.rs b/cli/npm/managed/cache/mod.rs index fa0e8c8a5..aaec48668 100644 --- a/cli/npm/managed/cache/mod.rs +++ b/cli/npm/managed/cache/mod.rs @@ -26,7 +26,7 @@ use crate::cache::CACHE_PERM; use crate::util::fs::atomic_write_file_with_retries; use crate::util::fs::hard_link_dir_recursive; -mod registry_info; +pub mod registry_info; mod tarball; mod tarball_extract; diff --git a/cli/npm/managed/cache/registry_info.rs b/cli/npm/managed/cache/registry_info.rs index 6c4a7503b..6d39d3c13 100644 --- a/cli/npm/managed/cache/registry_info.rs +++ b/cli/npm/managed/cache/registry_info.rs @@ -84,7 +84,7 @@ impl RegistryInfoDownloader { self.load_package_info_inner(name).await.with_context(|| { format!( "Error getting response at {} for package \"{}\"", - self.get_package_url(name), + get_package_url(&self.npmrc, name), name ) }) @@ -190,7 +190,7 @@ impl RegistryInfoDownloader { fn create_load_future(self: &Arc, name: &str) -> LoadFuture { let downloader = self.clone(); - let package_url = self.get_package_url(name); + let package_url = get_package_url(&self.npmrc, name); let registry_config = self.npmrc.get_registry_config(name); let maybe_auth_header = match maybe_auth_header_for_npm_registry(registry_config) { @@ -239,36 +239,36 @@ impl RegistryInfoDownloader { .map(|r| r.map_err(Arc::new)) .boxed_local() } +} - fn get_package_url(&self, name: &str) -> Url { - let registry_url = self.npmrc.get_registry_url(name); - // The '/' character in scoped package names "@scope/name" must be - // encoded for older third party registries. Newer registries and - // npm itself support both ways - // - encoded: https://registry.npmjs.org/@rollup%2fplugin-json - // - non-ecoded: https://registry.npmjs.org/@rollup/plugin-json - // To support as many third party registries as possible we'll - // always encode the '/' character. +pub fn get_package_url(npmrc: &ResolvedNpmRc, name: &str) -> Url { + let registry_url = npmrc.get_registry_url(name); + // The '/' character in scoped package names "@scope/name" must be + // encoded for older third party registries. Newer registries and + // npm itself support both ways + // - encoded: https://registry.npmjs.org/@rollup%2fplugin-json + // - non-ecoded: https://registry.npmjs.org/@rollup/plugin-json + // To support as many third party registries as possible we'll + // always encode the '/' character. - // list of all characters used in npm packages: - // !, ', (, ), *, -, ., /, [0-9], @, [A-Za-z], _, ~ - const ASCII_SET: percent_encoding::AsciiSet = - percent_encoding::NON_ALPHANUMERIC - .remove(b'!') - .remove(b'\'') - .remove(b'(') - .remove(b')') - .remove(b'*') - .remove(b'-') - .remove(b'.') - .remove(b'@') - .remove(b'_') - .remove(b'~'); - let name = percent_encoding::utf8_percent_encode(name, &ASCII_SET); - registry_url - // Ensure that scoped package name percent encoding is lower cased - // to match npm. - .join(&name.to_string().replace("%2F", "%2f")) - .unwrap() - } + // list of all characters used in npm packages: + // !, ', (, ), *, -, ., /, [0-9], @, [A-Za-z], _, ~ + const ASCII_SET: percent_encoding::AsciiSet = + percent_encoding::NON_ALPHANUMERIC + .remove(b'!') + .remove(b'\'') + .remove(b'(') + .remove(b')') + .remove(b'*') + .remove(b'-') + .remove(b'.') + .remove(b'@') + .remove(b'_') + .remove(b'~'); + let name = percent_encoding::utf8_percent_encode(name, &ASCII_SET); + registry_url + // Ensure that scoped package name percent encoding is lower cased + // to match npm. + .join(&name.to_string().replace("%2F", "%2f")) + .unwrap() } -- cgit v1.2.3 From 826e42a5b5880c974ae019a7a21aade6a718062c Mon Sep 17 00:00:00 2001 From: David Sherret Date: Fri, 1 Nov 2024 12:27:00 -0400 Subject: fix: improved support for cjs and cts modules (#26558) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * cts support * better cjs/cts type checking * deno compile cjs/cts support * More efficient detect cjs (going towards stabilization) * Determination of whether .js, .ts, .jsx, or .tsx is cjs or esm is only done after loading * Support `import x = require(...);` Co-authored-by: Bartek Iwańczuk --- cli/npm/managed/cache/mod.rs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'cli/npm/managed/cache') diff --git a/cli/npm/managed/cache/mod.rs b/cli/npm/managed/cache/mod.rs index aaec48668..8ae99f41e 100644 --- a/cli/npm/managed/cache/mod.rs +++ b/cli/npm/managed/cache/mod.rs @@ -36,7 +36,7 @@ pub use tarball::TarballCache; /// Stores a single copy of npm packages in a cache. #[derive(Debug)] pub struct NpmCache { - cache_dir: NpmCacheDir, + cache_dir: Arc, cache_setting: CacheSetting, npmrc: Arc, /// ensures a package is only downloaded once per run @@ -45,7 +45,7 @@ pub struct NpmCache { impl NpmCache { pub fn new( - cache_dir: NpmCacheDir, + cache_dir: Arc, cache_setting: CacheSetting, npmrc: Arc, ) -> Self { @@ -61,6 +61,10 @@ impl NpmCache { &self.cache_setting } + pub fn root_dir_path(&self) -> &Path { + self.cache_dir.root_dir() + } + pub fn root_dir_url(&self) -> &Url { self.cache_dir.root_dir_url() } @@ -152,10 +156,6 @@ impl NpmCache { self.cache_dir.package_name_folder(name, registry_url) } - pub fn root_folder(&self) -> PathBuf { - self.cache_dir.root_dir().to_owned() - } - pub fn resolve_package_folder_id_from_specifier( &self, specifier: &ModuleSpecifier, -- cgit v1.2.3