diff options
author | David Sherret <dsherret@users.noreply.github.com> | 2024-05-28 14:58:43 -0400 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-05-28 14:58:43 -0400 |
commit | 448fe67b7a2142f62332b651f9d215534dceb1f5 (patch) | |
tree | 3cfc763f39bf275a537e6228767b3e43866f5d0f /cli | |
parent | cd8f5f53f7616e4c74de0f1ff5eadd6ef024118a (diff) |
feat(vendor): support modifying remote files in vendor folder without checksum errors (#23979)
Includes:
* https://github.com/denoland/deno_graph/pull/486
* https://github.com/denoland/deno_graph/pull/488
* https://github.com/denoland/deno_lockfile/pull/25
* https://github.com/denoland/deno_lockfile/pull/22
* https://github.com/denoland/deno_graph/pull/483
* https://github.com/denoland/deno_graph/pull/470
Diffstat (limited to 'cli')
-rw-r--r-- | cli/Cargo.toml | 10 | ||||
-rw-r--r-- | cli/args/lockfile.rs | 34 | ||||
-rw-r--r-- | cli/args/mod.rs | 3 | ||||
-rw-r--r-- | cli/cache/mod.rs | 2 | ||||
-rw-r--r-- | cli/cache/module_info.rs | 6 | ||||
-rw-r--r-- | cli/errors.rs | 55 | ||||
-rw-r--r-- | cli/factory.rs | 1 | ||||
-rw-r--r-- | cli/file_fetcher.rs | 23 | ||||
-rw-r--r-- | cli/graph_util.rs | 356 | ||||
-rw-r--r-- | cli/lsp/config.rs | 3 | ||||
-rw-r--r-- | cli/lsp/documents.rs | 26 | ||||
-rw-r--r-- | cli/lsp/language_server.rs | 8 | ||||
-rw-r--r-- | cli/main.rs | 5 | ||||
-rw-r--r-- | cli/module_loader.rs | 10 | ||||
-rw-r--r-- | cli/npm/managed/mod.rs | 7 | ||||
-rw-r--r-- | cli/npm/managed/resolution.rs | 11 | ||||
-rw-r--r-- | cli/standalone/mod.rs | 2 | ||||
-rw-r--r-- | cli/tools/doc.rs | 7 | ||||
-rw-r--r-- | cli/tools/info.rs | 25 | ||||
-rw-r--r-- | cli/tools/installer.rs | 3 | ||||
-rw-r--r-- | cli/tools/registry/graph.rs | 4 | ||||
-rw-r--r-- | cli/tools/vendor/build.rs | 14 | ||||
-rw-r--r-- | cli/tools/vendor/mod.rs | 1 | ||||
-rw-r--r-- | cli/tools/vendor/test.rs | 1 | ||||
-rw-r--r-- | cli/worker.rs | 7 |
25 files changed, 395 insertions, 229 deletions
diff --git a/cli/Cargo.toml b/cli/Cargo.toml index ec02a801c..bdb1bbfb1 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -67,17 +67,17 @@ deno_ast = { workspace = true, features = ["bundler", "cjs", "codegen", "proposa deno_cache_dir = { workspace = true } deno_config = "=0.16.3" deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] } -deno_doc = { version = "=0.135.0", features = ["html", "syntect"] } -deno_emit = "=0.40.3" -deno_graph = { version = "=0.75.2", features = ["tokio_executor"] } +deno_doc = { version = "=0.137.0", features = ["html", "syntect"] } +deno_emit = "=0.41.0" +deno_graph = { version = "=0.77.0", features = ["tokio_executor"] } deno_lint = { version = "=0.58.4", features = ["docs"] } deno_lockfile.workspace = true -deno_npm = "=0.20.2" +deno_npm = "=0.21.0" deno_runtime = { workspace = true, features = ["include_js_files_for_snapshotting"] } deno_semver = "=0.5.4" deno_task_shell = "=0.16.1" deno_terminal.workspace = true -eszip = "=0.69.0" +eszip = "=0.70.1" napi_sym.workspace = true async-trait.workspace = true diff --git a/cli/args/lockfile.rs b/cli/args/lockfile.rs index d5ab14432..79d139487 100644 --- a/cli/args/lockfile.rs +++ b/cli/args/lockfile.rs @@ -2,10 +2,13 @@ use std::path::PathBuf; +use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_runtime::deno_node::PackageJson; use crate::args::ConfigFile; +use crate::cache; +use crate::util::fs::atomic_write_file; use crate::Flags; use super::DenoSubcommand; @@ -13,7 +16,6 @@ use super::InstallFlags; use super::InstallKind; pub use deno_lockfile::Lockfile; -pub use deno_lockfile::LockfileError; pub fn discover( flags: &Flags, @@ -54,6 +56,34 @@ pub fn discover( }, }; - let lockfile = Lockfile::new(filename, flags.lock_write)?; + let lockfile = if flags.lock_write { + Lockfile::new_empty(filename, true) + } else { + read_lockfile_at_path(filename)? + }; Ok(Some(lockfile)) } + +pub fn read_lockfile_at_path(filename: PathBuf) -> Result<Lockfile, AnyError> { + match std::fs::read_to_string(&filename) { + Ok(text) => Ok(Lockfile::with_lockfile_content(filename, &text, false)?), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + Ok(Lockfile::new_empty(filename, false)) + } + Err(err) => Err(err).with_context(|| { + format!("Failed reading lockfile '{}'", filename.display()) + }), + } +} + +pub fn write_lockfile_if_has_changes( + lockfile: &Lockfile, +) -> Result<(), AnyError> { + let Some(bytes) = lockfile.resolve_write_bytes() else { + return Ok(()); // nothing to do + }; + // do an atomic write to reduce the chance of multiple deno + // processes corrupting the file + atomic_write_file(&lockfile.filename, bytes, cache::CACHE_PERM) + .context("Failed writing lockfile.") +} diff --git a/cli/args/mod.rs b/cli/args/mod.rs index a1ca6d9a7..69adee970 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -34,8 +34,9 @@ pub use deno_config::TsConfigType; pub use deno_config::TsTypeLib; pub use deno_config::WorkspaceConfig; pub use flags::*; +pub use lockfile::read_lockfile_at_path; +pub use lockfile::write_lockfile_if_has_changes; pub use lockfile::Lockfile; -pub use lockfile::LockfileError; pub use package_json::PackageJsonDepsProvider; use deno_ast::ModuleSpecifier; diff --git a/cli/cache/mod.rs b/cli/cache/mod.rs index cee93bac5..bb18b1a13 100644 --- a/cli/cache/mod.rs +++ b/cli/cache/mod.rs @@ -228,7 +228,7 @@ impl Loader for FetchCacher { LoaderCacheSetting::Reload => { if matches!(file_fetcher.cache_setting(), CacheSetting::Only) { return Err(deno_core::anyhow::anyhow!( - "Failed to resolve version constraint. Try running again without --cached-only" + "Could not resolve version constraint using only cached data. Try running again without --cached-only" )); } Some(CacheSetting::ReloadAll) diff --git a/cli/cache/module_info.rs b/cli/cache/module_info.rs index 0e7a97678..e34b8d2bb 100644 --- a/cli/cache/module_info.rs +++ b/cli/cache/module_info.rs @@ -150,8 +150,9 @@ pub struct ModuleInfoCacheModuleAnalyzer<'a> { parser: &'a dyn ModuleParser, } +#[async_trait::async_trait(?Send)] impl<'a> deno_graph::ModuleAnalyzer for ModuleInfoCacheModuleAnalyzer<'a> { - fn analyze( + async fn analyze( &self, specifier: &ModuleSpecifier, source: Arc<str>, @@ -176,8 +177,9 @@ impl<'a> deno_graph::ModuleAnalyzer for ModuleInfoCacheModuleAnalyzer<'a> { } // otherwise, get the module info from the parsed source cache + // todo(23858): take advantage of this being async let analyzer = ParserModuleAnalyzer::new(self.parser); - let module_info = analyzer.analyze(specifier, source, media_type)?; + let module_info = analyzer.analyze(specifier, source, media_type).await?; // then attempt to cache it if let Err(err) = self.module_info_cache.set_module_info( diff --git a/cli/errors.rs b/cli/errors.rs index fce286f15..9fe433f18 100644 --- a/cli/errors.rs +++ b/cli/errors.rs @@ -14,6 +14,7 @@ use deno_core::error::AnyError; use deno_graph::source::ResolveError; use deno_graph::ModuleError; use deno_graph::ModuleGraphError; +use deno_graph::ModuleLoadError; use deno_graph::ResolutionError; use import_map::ImportMapError; use std::fmt::Write; @@ -27,24 +28,56 @@ fn get_diagnostic_class(_: &ParseDiagnostic) -> &'static str { } fn get_module_graph_error_class(err: &ModuleGraphError) -> &'static str { + use deno_graph::JsrLoadError; + use deno_graph::NpmLoadError; + use deno_graph::WorkspaceLoadError; + match err { + ModuleGraphError::ResolutionError(err) + | ModuleGraphError::TypesResolutionError(err) => { + get_resolution_error_class(err) + } ModuleGraphError::ModuleError(err) => match err { - ModuleError::LoadingErr(_, _, err) => get_error_class_name(err.as_ref()), ModuleError::InvalidTypeAssertion { .. } => "SyntaxError", ModuleError::ParseErr(_, diagnostic) => get_diagnostic_class(diagnostic), ModuleError::UnsupportedMediaType { .. } | ModuleError::UnsupportedImportAttributeType { .. } => "TypeError", - ModuleError::Missing(_, _) - | ModuleError::MissingDynamic(_, _) - | ModuleError::MissingWorkspaceMemberExports { .. } - | ModuleError::UnknownExport { .. } - | ModuleError::UnknownPackage { .. } - | ModuleError::UnknownPackageReq { .. } => "NotFound", + ModuleError::Missing(_, _) | ModuleError::MissingDynamic(_, _) => { + "NotFound" + } + ModuleError::LoadingErr(_, _, err) => match err { + ModuleLoadError::Loader(err) => get_error_class_name(err.as_ref()), + ModuleLoadError::HttpsChecksumIntegrity(_) + | ModuleLoadError::TooManyRedirects => "Error", + ModuleLoadError::NodeUnknownBuiltinModule(_) => "NotFound", + ModuleLoadError::Decode(_) => "TypeError", + ModuleLoadError::Npm(err) => match err { + NpmLoadError::NotSupportedEnvironment + | NpmLoadError::PackageReqResolution(_) + | NpmLoadError::RegistryInfo(_) => "Error", + NpmLoadError::PackageReqReferenceParse(_) => "TypeError", + }, + ModuleLoadError::Jsr(err) => match err { + JsrLoadError::UnsupportedManifestChecksum + | JsrLoadError::PackageFormat(_) => "TypeError", + JsrLoadError::ContentLoadExternalSpecifier + | JsrLoadError::ContentLoad(_) + | JsrLoadError::ContentChecksumIntegrity(_) + | JsrLoadError::PackageManifestLoad(_, _) + | JsrLoadError::PackageVersionManifestChecksumIntegrity(..) + | JsrLoadError::PackageVersionManifestLoad(_, _) + | JsrLoadError::RedirectInPackage(_) => "Error", + JsrLoadError::PackageNotFound(_) + | JsrLoadError::PackageReqNotFound(_) + | JsrLoadError::PackageVersionNotFound(_) + | JsrLoadError::UnknownExport { .. } => "NotFound", + }, + ModuleLoadError::Workspace(err) => match err { + WorkspaceLoadError::MemberInvalidExportPath { .. } => "TypeError", + WorkspaceLoadError::MissingMemberExports { .. } => "NotFound", + }, + }, }, - ModuleGraphError::ResolutionError(err) - | ModuleGraphError::TypesResolutionError(err) => { - get_resolution_error_class(err) - } } } diff --git a/cli/factory.rs b/cli/factory.rs index 6cdbff9bb..ce9736e68 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -665,7 +665,6 @@ impl CliFactory { self.options.clone(), self.npm_resolver().await?.clone(), self.module_graph_builder().await?.clone(), - self.maybe_lockfile().clone(), self.type_checker().await?.clone(), ))) }) diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs index eda579b2b..a8d835d0e 100644 --- a/cli/file_fetcher.rs +++ b/cli/file_fetcher.rs @@ -253,15 +253,30 @@ impl FileFetcher { deno_core::resolve_import(redirect_to, specifier.as_str())?; return Ok(Some(FileOrRedirect::Redirect(redirect))); } - let Some(bytes) = self.http_cache.read_file_bytes( + let result = self.http_cache.read_file_bytes( &cache_key, maybe_checksum .as_ref() .map(|c| deno_cache_dir::Checksum::new(c.as_str())), deno_cache_dir::GlobalToLocalCopy::Allow, - )? - else { - return Ok(None); + ); + let bytes = match result { + Ok(Some(bytes)) => bytes, + Ok(None) => return Ok(None), + Err(err) => match err { + deno_cache_dir::CacheReadFileError::Io(err) => return Err(err.into()), + deno_cache_dir::CacheReadFileError::ChecksumIntegrity(err) => { + // convert to the equivalent deno_graph error so that it + // enhances it if this is passed to deno_graph + return Err( + deno_graph::source::ChecksumIntegrityError { + actual: err.actual, + expected: err.expected, + } + .into(), + ); + } + }, }; Ok(Some(FileOrRedirect::File(File { diff --git a/cli/graph_util.rs b/cli/graph_util.rs index ed56cf9f7..81ae21d0d 100644 --- a/cli/graph_util.rs +++ b/cli/graph_util.rs @@ -18,6 +18,9 @@ use crate::tools::check; use crate::tools::check::TypeChecker; use crate::util::file_watcher::WatcherCommunicator; use crate::util::fs::canonicalize_path; +use deno_emit::LoaderChecksum; +use deno_graph::JsrLoadError; +use deno_graph::ModuleLoadError; use deno_runtime::fs_util::specifier_to_file_path; use deno_config::WorkspaceMemberConfig; @@ -51,6 +54,9 @@ pub struct GraphValidOptions { pub check_js: bool, pub follow_type_only: bool, pub is_vendoring: bool, + /// Whether to exit the process for lockfile errors. + /// Otherwise, surfaces lockfile errors as errors. + pub exit_lockfile_errors: bool, } /// Check if `roots` and their deps are available. Returns `Ok(())` if @@ -62,10 +68,14 @@ pub struct GraphValidOptions { /// for the CLI. pub fn graph_valid( graph: &ModuleGraph, - fs: Arc<dyn FileSystem>, + fs: &Arc<dyn FileSystem>, roots: &[ModuleSpecifier], options: GraphValidOptions, ) -> Result<(), AnyError> { + if options.exit_lockfile_errors { + graph_exit_lock_errors(graph); + } + let mut errors = graph .walk( roots, @@ -95,8 +105,10 @@ pub fn graph_valid( enhanced_resolution_error_message(resolution_error) ) } - ModuleGraphError::ModuleError(e) => { - enhanced_module_error_message(fs.clone(), e) + ModuleGraphError::ModuleError(error) => { + enhanced_lockfile_error_message(error) + .or_else(|| enhanced_sloppy_imports_error_message(fs, error)) + .unwrap_or_else(|| format!("{}", error)) } }; @@ -152,40 +164,16 @@ pub fn graph_valid( } } -/// Checks the lockfile against the graph and exits on errors. -pub fn graph_lock_or_exit(graph: &ModuleGraph, lockfile: &mut Lockfile) { - for module in graph.modules() { - let source = match module { - Module::Js(module) if module.media_type.is_declaration() => continue, // skip declaration files - Module::Js(module) => &module.source, - Module::Json(module) => &module.source, - Module::Node(_) | Module::Npm(_) | Module::External(_) => continue, - }; - - // skip over any specifiers in JSR packages because those - // are enforced via the integrity - if deno_graph::source::recommended_registry_package_url_to_nv( - jsr_url(), - module.specifier(), - ) - .is_some() - { - continue; - } +pub fn graph_exit_lock_errors(graph: &ModuleGraph) { + for error in graph.module_errors() { + exit_for_lockfile_error(error); + } +} - if !lockfile.check_or_insert_remote(module.specifier().as_str(), source) { - let err = format!( - concat!( - "The source code is invalid, as it does not match the expected hash in the lock file.\n", - " Specifier: {}\n", - " Lock file: {}", - ), - module.specifier(), - lockfile.filename.display(), - ); - log::error!("{} {}", colors::red("error:"), err); - std::process::exit(10); - } +fn exit_for_lockfile_error(err: &ModuleError) { + if let Some(err_message) = enhanced_lockfile_error_message(err) { + log::error!("{} {}", colors::red("error:"), err_message); + std::process::exit(10); } } @@ -201,7 +189,6 @@ pub struct ModuleGraphCreator { options: Arc<CliOptions>, npm_resolver: Arc<dyn CliNpmResolver>, module_graph_builder: Arc<ModuleGraphBuilder>, - lockfile: Option<Arc<Mutex<Lockfile>>>, type_checker: Arc<TypeChecker>, } @@ -210,13 +197,11 @@ impl ModuleGraphCreator { options: Arc<CliOptions>, npm_resolver: Arc<dyn CliNpmResolver>, module_graph_builder: Arc<ModuleGraphBuilder>, - lockfile: Option<Arc<Mutex<Lockfile>>>, type_checker: Arc<TypeChecker>, ) -> Self { Self { options, npm_resolver, - lockfile, module_graph_builder, type_checker, } @@ -317,9 +302,6 @@ impl ModuleGraphCreator { .await?; self.graph_valid(&graph)?; - if let Some(lockfile) = &self.lockfile { - graph_lock_or_exit(&graph, &mut lockfile.lock()); - } if self.options.type_check_mode().is_true() { // provide the graph to the type checker, then get it back after it's done @@ -426,6 +408,67 @@ impl ModuleGraphBuilder { } } + struct LockfileLocker<'a>(&'a Mutex<Lockfile>); + + impl<'a> deno_graph::source::Locker for LockfileLocker<'a> { + fn get_remote_checksum( + &self, + specifier: &deno_ast::ModuleSpecifier, + ) -> Option<LoaderChecksum> { + self + .0 + .lock() + .remote() + .get(specifier.as_str()) + .map(|s| LoaderChecksum::new(s.clone())) + } + + fn has_remote_checksum( + &self, + specifier: &deno_ast::ModuleSpecifier, + ) -> bool { + self.0.lock().remote().contains_key(specifier.as_str()) + } + + fn set_remote_checksum( + &mut self, + specifier: &deno_ast::ModuleSpecifier, + checksum: LoaderChecksum, + ) { + self + .0 + .lock() + .insert_remote(specifier.to_string(), checksum.into_string()) + } + + fn get_pkg_manifest_checksum( + &self, + package_nv: &PackageNv, + ) -> Option<LoaderChecksum> { + self + .0 + .lock() + .content + .packages + .jsr + .get(&package_nv.to_string()) + .map(|s| LoaderChecksum::new(s.integrity.clone())) + } + + fn set_pkg_manifest_checksum( + &mut self, + package_nv: &PackageNv, + checksum: LoaderChecksum, + ) { + // a value would only exist in here if two workers raced + // to insert the same package manifest checksum + self + .0 + .lock() + .insert_package(package_nv.to_string(), checksum.into_string()); + } + } + let maybe_imports = self.options.to_maybe_imports()?; let parser = self.parsed_source_cache.as_capturing_parser(); let analyzer = self.module_info_cache.as_module_analyzer(&parser); @@ -442,6 +485,10 @@ impl ModuleGraphBuilder { .map(|r| r.as_reporter()); let workspace_members = self.options.resolve_deno_graph_workspace_members()?; + let mut locker = self + .lockfile + .as_ref() + .map(|lockfile| LockfileLocker(lockfile)); self .build_graph_with_npm_resolution_and_build_options( graph, @@ -459,6 +506,7 @@ impl ModuleGraphBuilder { module_analyzer: &analyzer, reporter: maybe_file_watcher_reporter, resolver: Some(graph_resolver), + locker: locker.as_mut().map(|l| l as _), }, ) .await @@ -468,7 +516,7 @@ impl ModuleGraphBuilder { &self, graph: &mut ModuleGraph, roots: Vec<ModuleSpecifier>, - loader: &mut dyn deno_graph::source::Loader, + loader: &'a mut dyn deno_graph::source::Loader, options: deno_graph::BuildOptions<'a>, ) -> Result<(), AnyError> { // ensure an "npm install" is done if the user has explicitly @@ -479,8 +527,10 @@ impl ModuleGraphBuilder { } } - // add the lockfile redirects to the graph if it's the first time executing - if graph.redirects.is_empty() { + // fill the graph with the information from the lockfile + let is_first_execution = graph.roots.is_empty(); + if is_first_execution { + // populate the information from the lockfile if let Some(lockfile) = &self.lockfile { let lockfile = lockfile.lock(); for (from, to) in &lockfile.content.redirects { @@ -492,13 +542,6 @@ impl ModuleGraphBuilder { } } } - } - } - - // add the jsr specifiers to the graph if it's the first time executing - if graph.packages.is_empty() { - if let Some(lockfile) = &self.lockfile { - let lockfile = lockfile.lock(); for (key, value) in &lockfile.content.packages.specifiers { if let Some(key) = key .strip_prefix("jsr:") @@ -512,59 +555,15 @@ impl ModuleGraphBuilder { } } } - for (nv, value) in &lockfile.content.packages.jsr { - if let Ok(nv) = PackageNv::from_str(nv) { - graph - .packages - .add_manifest_checksum(nv, value.integrity.clone()) - .map_err(|err| deno_lockfile::IntegrityCheckFailedError { - package_display_id: format!("jsr:{}", err.nv), - actual: err.actual, - expected: err.expected, - filename: lockfile.filename.display().to_string(), - })?; - } - } } } - graph.build(roots, loader, options).await; - - // add the redirects in the graph to the lockfile - if !graph.redirects.is_empty() { - if let Some(lockfile) = &self.lockfile { - let graph_redirects = graph.redirects.iter().filter(|(from, _)| { - !matches!(from.scheme(), "npm" | "file" | "deno") - }); - let mut lockfile = lockfile.lock(); - for (from, to) in graph_redirects { - lockfile.insert_redirect(from.to_string(), to.to_string()); - } - } - } + let initial_redirects_len = graph.redirects.len(); + let initial_package_deps_len = graph.packages.package_deps_sum(); + let initial_package_mappings_len = graph.packages.mappings().len(); + let initial_npm_packages = graph.npm_packages.len(); - // add the jsr specifiers in the graph to the lockfile - if !graph.packages.is_empty() { - if let Some(lockfile) = &self.lockfile { - let mappings = graph.packages.mappings(); - let mut lockfile = lockfile.lock(); - for (from, to) in mappings { - lockfile.insert_package_specifier( - format!("jsr:{}", from), - format!("jsr:{}", to), - ); - } - for (name, checksum, deps) in - graph.packages.packages_with_checksum_and_deps() - { - lockfile.insert_package( - name.to_string(), - checksum.clone(), - deps.map(|s| s.to_string()), - ); - } - } - } + graph.build(roots, loader, options).await; if let Some(npm_resolver) = self.npm_resolver.as_managed() { // ensure that the top level package.json is installed if a @@ -578,6 +577,53 @@ impl ModuleGraphBuilder { npm_resolver.resolve_pending().await?; } + let has_redirects_changed = graph.redirects.len() != initial_redirects_len; + let has_jsr_package_deps_changed = + graph.packages.package_deps_sum() != initial_package_deps_len; + let has_jsr_package_mappings_changed = + graph.packages.mappings().len() != initial_package_mappings_len; + let has_npm_packages_changed = + graph.npm_packages.len() != initial_npm_packages; + + if has_redirects_changed + || has_jsr_package_deps_changed + || has_jsr_package_mappings_changed + || has_npm_packages_changed + { + if let Some(lockfile) = &self.lockfile { + let mut lockfile = lockfile.lock(); + // https redirects + if has_redirects_changed { + let graph_redirects = graph.redirects.iter().filter(|(from, _)| { + !matches!(from.scheme(), "npm" | "file" | "deno") + }); + for (from, to) in graph_redirects { + lockfile.insert_redirect(from.to_string(), to.to_string()); + } + } + // jsr package mappings + if has_jsr_package_mappings_changed { + for (from, to) in graph.packages.mappings() { + lockfile.insert_package_specifier( + format!("jsr:{}", from), + format!("jsr:{}", to), + ); + } + } + // jsr packages + if has_jsr_package_deps_changed { + for (name, deps) in graph.packages.packages_with_deps() { + lockfile + .add_package_deps(&name.to_string(), deps.map(|s| s.to_string())); + } + } + // npm packages + if has_npm_packages_changed { + self.npm_resolver.as_managed().unwrap().lock(&mut lockfile); + } + } + } + Ok(()) } @@ -658,12 +704,13 @@ impl ModuleGraphBuilder { ) -> Result<(), AnyError> { graph_valid( graph, - self.fs.clone(), + &self.fs, roots, GraphValidOptions { is_vendoring: false, follow_type_only: self.options.type_check_mode().is_true(), check_js: self.options.check_js(), + exit_lockfile_errors: true, }, ) } @@ -701,28 +748,99 @@ pub fn enhanced_resolution_error_message(error: &ResolutionError) -> String { message } -pub fn enhanced_module_error_message( - fs: Arc<dyn FileSystem>, +fn enhanced_sloppy_imports_error_message( + fs: &Arc<dyn FileSystem>, error: &ModuleError, -) -> String { - let additional_message = match error { - ModuleError::LoadingErr(specifier, _, _) // ex. "Is a directory" error +) -> Option<String> { + match error { + ModuleError::LoadingErr(specifier, _, ModuleLoadError::Loader(_)) // ex. "Is a directory" error | ModuleError::Missing(specifier, _) => { - SloppyImportsResolver::new(fs).resolve( + let additional_message = SloppyImportsResolver::new(fs.clone()) + .resolve(specifier, ResolutionMode::Execution) + .as_suggestion_message()?; + Some(format!( + "{} {} or run with --unstable-sloppy-imports", + error, + additional_message, + )) + } + _ => None, + } +} + +fn enhanced_lockfile_error_message(err: &ModuleError) -> Option<String> { + match err { + ModuleError::LoadingErr( + specifier, + _, + ModuleLoadError::Jsr(JsrLoadError::ContentChecksumIntegrity( + checksum_err, + )), + ) => { + Some(format!( + concat!( + "Integrity check failed in package. The package may have been tampered with.\n\n", + " Specifier: {}\n", + " Actual: {}\n", + " Expected: {}\n\n", + "If you modified your global cache, run again with the --reload flag to restore ", + "its state. If you want to modify dependencies locally run again with the ", + "--vendor flag or specify `\"vendor\": true` in a deno.json then modify the contents ", + "of the vendor/ folder." + ), specifier, - ResolutionMode::Execution, - ) - .as_suggestion_message() + checksum_err.actual, + checksum_err.expected, + )) + } + ModuleError::LoadingErr( + _specifier, + _, + ModuleLoadError::Jsr( + JsrLoadError::PackageVersionManifestChecksumIntegrity( + package_nv, + checksum_err, + ), + ), + ) => { + Some(format!( + concat!( + "Integrity check failed for package. The source code is invalid, as it does not match the expected hash in the lock file.\n\n", + " Package: {}\n", + " Actual: {}\n", + " Expected: {}\n\n", + "This could be caused by:\n", + " * the lock file may be corrupt\n", + " * the source itself may be corrupt\n\n", + "Use the --lock-write flag to regenerate the lockfile or --reload to reload the source code from the server." + ), + package_nv, + checksum_err.actual, + checksum_err.expected, + )) + } + ModuleError::LoadingErr( + specifier, + _, + ModuleLoadError::HttpsChecksumIntegrity(checksum_err), + ) => { + Some(format!( + concat!( + "Integrity check failed for remote specifier. The source code is invalid, as it does not match the expected hash in the lock file.\n\n", + " Specifier: {}\n", + " Actual: {}\n", + " Expected: {}\n\n", + "This could be caused by:\n", + " * the lock file may be corrupt\n", + " * the source itself may be corrupt\n\n", + "Use the --lock-write flag to regenerate the lockfile or --reload to reload the source code from the server." + ), + specifier, + checksum_err.actual, + checksum_err.expected, + )) } _ => None, - }; - if let Some(message) = additional_message { - format!( - "{} {} or run with --unstable-sloppy-imports", - error, message - ) - } else { - format!("{}", error) } } diff --git a/cli/lsp/config.rs b/cli/lsp/config.rs index 89ecb84d8..543200ad1 100644 --- a/cli/lsp/config.rs +++ b/cli/lsp/config.rs @@ -1,6 +1,7 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use super::logging::lsp_log; +use crate::args::read_lockfile_at_path; use crate::args::ConfigFile; use crate::args::FmtOptions; use crate::args::LintOptions; @@ -1685,7 +1686,7 @@ fn resolve_node_modules_dir( } fn resolve_lockfile_from_path(lockfile_path: PathBuf) -> Option<Lockfile> { - match Lockfile::new(lockfile_path, false) { + match read_lockfile_at_path(lockfile_path) { Ok(value) => { if let Ok(specifier) = ModuleSpecifier::from_file_path(&value.filename) { lsp_log!(" Resolved lockfile: \"{}\"", specifier); diff --git a/cli/lsp/documents.rs b/cli/lsp/documents.rs index ab053336f..b30217bcb 100644 --- a/cli/lsp/documents.rs +++ b/cli/lsp/documents.rs @@ -310,7 +310,7 @@ impl Document { let (maybe_parsed_source, maybe_module) = if media_type_is_diagnosable(media_type) { parse_and_analyze_module( - &specifier, + specifier.clone(), text_info.clone(), maybe_headers.as_ref(), media_type, @@ -370,10 +370,13 @@ impl Document { let maybe_parsed_source; let maybe_test_module_fut; if media_type != self.media_type { - let parsed_source_result = - parse_source(&self.specifier, self.text_info.clone(), media_type); + let parsed_source_result = parse_source( + self.specifier.clone(), + self.text_info.clone(), + media_type, + ); let maybe_module = analyze_module( - &self.specifier, + self.specifier.clone(), &parsed_source_result, self.maybe_headers.as_ref(), self.file_referrer.as_ref(), @@ -481,7 +484,7 @@ impl Document { .unwrap_or(false) { parse_and_analyze_module( - &self.specifier, + self.specifier.clone(), text_info.clone(), self.maybe_headers.as_ref(), media_type, @@ -1455,14 +1458,15 @@ impl<'a> deno_graph::source::Loader for OpenDocumentsGraphLoader<'a> { } fn parse_and_analyze_module( - specifier: &ModuleSpecifier, + specifier: ModuleSpecifier, text_info: SourceTextInfo, maybe_headers: Option<&HashMap<String, String>>, media_type: MediaType, file_referrer: Option<&ModuleSpecifier>, resolver: &LspResolver, ) -> (Option<ParsedSourceResult>, Option<ModuleResult>) { - let parsed_source_result = parse_source(specifier, text_info, media_type); + let parsed_source_result = + parse_source(specifier.clone(), text_info, media_type); let module_result = analyze_module( specifier, &parsed_source_result, @@ -1474,12 +1478,12 @@ fn parse_and_analyze_module( } fn parse_source( - specifier: &ModuleSpecifier, + specifier: ModuleSpecifier, text_info: SourceTextInfo, media_type: MediaType, ) -> ParsedSourceResult { deno_ast::parse_module(deno_ast::ParseParams { - specifier: specifier.clone(), + specifier, text_info, media_type, capture_tokens: true, @@ -1489,7 +1493,7 @@ fn parse_source( } fn analyze_module( - specifier: &ModuleSpecifier, + specifier: ModuleSpecifier, parsed_source_result: &ParsedSourceResult, maybe_headers: Option<&HashMap<String, String>>, file_referrer: Option<&ModuleSpecifier>, @@ -1511,7 +1515,7 @@ fn analyze_module( }, )), Err(err) => Err(deno_graph::ModuleGraphError::ModuleError( - deno_graph::ModuleError::ParseErr(specifier.clone(), err.clone()), + deno_graph::ModuleError::ParseErr(specifier, err.clone()), )), } } diff --git a/cli/lsp/language_server.rs b/cli/lsp/language_server.rs index 8f37d8b9c..621f45f96 100644 --- a/cli/lsp/language_server.rs +++ b/cli/lsp/language_server.rs @@ -85,6 +85,7 @@ use super::tsc::TsServer; use super::urls; use crate::args::create_default_npmrc; use crate::args::get_root_cert_store; +use crate::args::write_lockfile_if_has_changes; use crate::args::CaData; use crate::args::CacheSetting; use crate::args::CliOptions; @@ -247,12 +248,13 @@ impl LanguageServer { .await?; graph_util::graph_valid( &graph, - factory.fs().clone(), + factory.fs(), &roots, graph_util::GraphValidOptions { is_vendoring: false, follow_type_only: true, check_js: false, + exit_lockfile_errors: false, }, )?; @@ -260,8 +262,8 @@ impl LanguageServer { // found after caching if let Some(lockfile) = cli_options.maybe_lockfile() { let lockfile = lockfile.lock(); - if let Err(err) = lockfile.write() { - lsp_warn!("Error writing lockfile: {:#}", err); + if let Err(err) = write_lockfile_if_has_changes(&lockfile) { + lsp_warn!("{:#}", err); } } diff --git a/cli/main.rs b/cli/main.rs index 586c2ff10..e8bea9f54 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -280,11 +280,6 @@ fn exit_for_error(error: AnyError) -> ! { if let Some(e) = error.downcast_ref::<JsError>() { error_string = format_js_error(e); - } else if let Some(args::LockfileError::IntegrityCheckFailed(e)) = - error.downcast_ref::<args::LockfileError>() - { - error_string = e.to_string(); - error_code = 10; } else if let Some(SnapshotFromLockfileError::IntegrityCheckFailed(e)) = error.downcast_ref::<SnapshotFromLockfileError>() { diff --git a/cli/module_loader.rs b/cli/module_loader.rs index 6d8d3e92b..9be1ceff2 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -9,6 +9,7 @@ use std::str; use std::sync::Arc; use crate::args::jsr_url; +use crate::args::write_lockfile_if_has_changes; use crate::args::CliOptions; use crate::args::DenoSubcommand; use crate::args::TsTypeLib; @@ -20,7 +21,6 @@ use crate::factory::CliFactory; use crate::graph_container::MainModuleGraphContainer; use crate::graph_container::ModuleGraphContainer; use crate::graph_container::ModuleGraphUpdatePermit; -use crate::graph_util::graph_lock_or_exit; use crate::graph_util::CreateGraphOptions; use crate::graph_util::ModuleGraphBuilder; use crate::node; @@ -173,13 +173,9 @@ impl ModuleLoadPreparer { self.module_graph_builder.graph_roots_valid(graph, roots)?; - // If there is a lockfile... + // write the lockfile if there is one if let Some(lockfile) = &self.lockfile { - let mut lockfile = lockfile.lock(); - // validate the integrity of all the modules - graph_lock_or_exit(graph, &mut lockfile); - // update it with anything new - lockfile.write().context("Failed writing lockfile.")?; + write_lockfile_if_has_changes(&lockfile.lock())?; } drop(_pb_clear_guard); diff --git a/cli/npm/managed/mod.rs b/cli/npm/managed/mod.rs index 3a2657cfb..938c16d9a 100644 --- a/cli/npm/managed/mod.rs +++ b/cli/npm/managed/mod.rs @@ -370,9 +370,8 @@ impl ManagedCliNpmResolver { self.fs_resolver.cache_packages().await?; // If there's a lock file, update it with all discovered npm packages - if let Some(lockfile_mutex) = &self.maybe_lockfile { - let mut lockfile = lockfile_mutex.lock(); - self.lock(&mut lockfile)?; + if let Some(lockfile) = &self.maybe_lockfile { + self.lock(&mut lockfile.lock()); } Ok(()) @@ -401,7 +400,7 @@ impl ManagedCliNpmResolver { .serialized_valid_snapshot_for_system(system_info) } - pub fn lock(&self, lockfile: &mut Lockfile) -> Result<(), AnyError> { + pub fn lock(&self, lockfile: &mut Lockfile) { self.resolution.lock(lockfile) } diff --git a/cli/npm/managed/resolution.rs b/cli/npm/managed/resolution.rs index 1903d339b..9cea5d305 100644 --- a/cli/npm/managed/resolution.rs +++ b/cli/npm/managed/resolution.rs @@ -292,7 +292,7 @@ impl NpmResolution { .as_valid_serialized_for_system(system_info) } - pub fn lock(&self, lockfile: &mut Lockfile) -> Result<(), AnyError> { + pub fn lock(&self, lockfile: &mut Lockfile) { let snapshot = self.snapshot.read(); populate_lockfile_from_snapshot(lockfile, &snapshot) } @@ -345,7 +345,7 @@ async fn add_package_reqs_to_snapshot( if let Some(lockfile_mutex) = maybe_lockfile { let mut lockfile = lockfile_mutex.lock(); - populate_lockfile_from_snapshot(&mut lockfile, &snapshot)?; + populate_lockfile_from_snapshot(&mut lockfile, &snapshot); } Ok(snapshot) @@ -369,7 +369,7 @@ fn get_npm_pending_resolver( fn populate_lockfile_from_snapshot( lockfile: &mut Lockfile, snapshot: &NpmResolutionSnapshot, -) -> Result<(), AnyError> { +) { for (package_req, nv) in snapshot.package_reqs() { lockfile.insert_package_specifier( format!("npm:{}", package_req), @@ -384,10 +384,8 @@ fn populate_lockfile_from_snapshot( ); } for package in snapshot.all_packages_for_every_system() { - lockfile - .check_or_insert_npm_package(npm_package_to_lockfile_info(package))?; + lockfile.insert_npm_package(npm_package_to_lockfile_info(package)); } - Ok(()) } fn npm_package_to_lockfile_info( @@ -403,7 +401,6 @@ fn npm_package_to_lockfile_info( .collect(); NpmPackageLockfileInfo { - display_id: pkg.id.nv.to_string(), serialized_id: pkg.id.as_serialized(), integrity: pkg.dist.integrity().for_lockfile(), dependencies, diff --git a/cli/standalone/mod.rs b/cli/standalone/mod.rs index 8f8a78de0..8c268d928 100644 --- a/cli/standalone/mod.rs +++ b/cli/standalone/mod.rs @@ -180,7 +180,7 @@ impl ModuleLoader for EmbeddedModuleLoader { if original_specifier.scheme() == "data" { let data_url_text = match deno_graph::source::RawDataUrl::parse(original_specifier) - .and_then(|url| url.decode().map_err(|err| err.into())) + .and_then(|url| url.decode()) { Ok(response) => response, Err(err) => { diff --git a/cli/tools/doc.rs b/cli/tools/doc.rs index 0ae9a8483..696823a91 100644 --- a/cli/tools/doc.rs +++ b/cli/tools/doc.rs @@ -8,7 +8,7 @@ use crate::colors; use crate::display::write_json_to_stdout; use crate::display::write_to_stdout_ignore_sigpipe; use crate::factory::CliFactory; -use crate::graph_util::graph_lock_or_exit; +use crate::graph_util::graph_exit_lock_errors; use crate::tsc::get_types_declaration_file_text; use crate::util::fs::collect_specifiers; use deno_ast::diagnostics::Diagnostic; @@ -62,6 +62,7 @@ async fn generate_doc_nodes_for_builtin_types( executor: Default::default(), file_system: &NullFileSystem, jsr_url_provider: Default::default(), + locker: None, module_analyzer: analyzer, npm_resolver: None, reporter: None, @@ -121,8 +122,8 @@ pub async fn doc(flags: Flags, doc_flags: DocFlags) -> Result<(), AnyError> { .create_graph(GraphKind::TypesOnly, module_specifiers.clone()) .await?; - if let Some(lockfile) = maybe_lockfile { - graph_lock_or_exit(&graph, &mut lockfile.lock()); + if maybe_lockfile.is_some() { + graph_exit_lock_errors(&graph); } let doc_parser = doc::DocParser::new( diff --git a/cli/tools/info.rs b/cli/tools/info.rs index 19975571b..b023970f8 100644 --- a/cli/tools/info.rs +++ b/cli/tools/info.rs @@ -7,7 +7,6 @@ use std::fmt::Write; use deno_ast::ModuleSpecifier; use deno_core::anyhow::bail; -use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::resolve_url_or_path; use deno_core::serde_json; @@ -26,11 +25,12 @@ use deno_semver::npm::NpmPackageReqReference; use deno_semver::package::PackageNv; use deno_terminal::colors; +use crate::args::write_lockfile_if_has_changes; use crate::args::Flags; use crate::args::InfoFlags; use crate::display; use crate::factory::CliFactory; -use crate::graph_util::graph_lock_or_exit; +use crate::graph_util::graph_exit_lock_errors; use crate::npm::CliNpmResolver; use crate::npm::ManagedCliNpmResolver; use crate::util::checksum; @@ -68,13 +68,10 @@ pub async fn info(flags: Flags, info_flags: InfoFlags) -> Result<(), AnyError> { .create_graph_with_loader(GraphKind::All, vec![specifier], &mut loader) .await?; - // If there is a lockfile... + // write out the lockfile if there is one if let Some(lockfile) = &maybe_lockfile { - let mut lockfile = lockfile.lock(); - // validate the integrity of all the modules - graph_lock_or_exit(&graph, &mut lockfile); - // update it with anything new - lockfile.write().context("Failed writing lockfile.")?; + graph_exit_lock_errors(&graph); + write_lockfile_if_has_changes(&lockfile.lock())?; } if info_flags.json { @@ -669,18 +666,6 @@ impl<'a> GraphDisplayContext<'a> { ModuleError::Missing(_, _) | ModuleError::MissingDynamic(_, _) => { self.build_error_msg(specifier, "(missing)") } - ModuleError::MissingWorkspaceMemberExports { .. } => { - self.build_error_msg(specifier, "(missing exports)") - } - ModuleError::UnknownExport { .. } => { - self.build_error_msg(specifier, "(unknown export)") - } - ModuleError::UnknownPackage { .. } => { - self.build_error_msg(specifier, "(unknown package)") - } - ModuleError::UnknownPackageReq { .. } => { - self.build_error_msg(specifier, "(unknown package constraint)") - } } } diff --git a/cli/tools/installer.rs b/cli/tools/installer.rs index e0cb5a222..2b518f46f 100644 --- a/cli/tools/installer.rs +++ b/cli/tools/installer.rs @@ -1,6 +1,7 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use crate::args::resolve_no_prompt; +use crate::args::write_lockfile_if_has_changes; use crate::args::AddFlags; use crate::args::CaData; use crate::args::Flags; @@ -266,7 +267,7 @@ async fn install_local( crate::module_loader::load_top_level_deps(&factory).await?; if let Some(lockfile) = factory.cli_options().maybe_lockfile() { - lockfile.lock().write()?; + write_lockfile_if_has_changes(&lockfile.lock())?; } Ok(()) diff --git a/cli/tools/registry/graph.rs b/cli/tools/registry/graph.rs index d1356df9e..b363efdb4 100644 --- a/cli/tools/registry/graph.rs +++ b/cli/tools/registry/graph.rs @@ -59,8 +59,8 @@ impl GraphDiagnosticsCollector { let maybe_version = graph .packages .mappings() - .find(|(req, _)| *req == jsr_req_ref.req()) - .map(|(_, nv)| nv.version.clone()); + .get(jsr_req_ref.req()) + .map(|nv| nv.version.clone()); diagnostics_collector.push( PublishDiagnostic::MissingConstraint { specifier: resolution.specifier.clone(), diff --git a/cli/tools/vendor/build.rs b/cli/tools/vendor/build.rs index 5435a0035..fd7401f77 100644 --- a/cli/tools/vendor/build.rs +++ b/cli/tools/vendor/build.rs @@ -9,7 +9,6 @@ use deno_core::anyhow::bail; use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::futures::future::LocalBoxFuture; -use deno_core::parking_lot::Mutex; use deno_graph::source::ResolutionMode; use deno_graph::JsModule; use deno_graph::Module; @@ -19,10 +18,8 @@ use import_map::ImportMap; use import_map::SpecifierMap; use crate::args::JsxImportSourceConfig; -use crate::args::Lockfile; use crate::cache::ParsedSourceCache; use crate::graph_util; -use crate::graph_util::graph_lock_or_exit; use crate::tools::vendor::import_map::BuildImportMapInput; use super::analyze::has_default_export; @@ -62,7 +59,6 @@ pub struct BuildInput< pub parsed_source_cache: &'a ParsedSourceCache, pub output_dir: &'a Path, pub maybe_original_import_map: Option<&'a ImportMap>, - pub maybe_lockfile: Option<Arc<Mutex<Lockfile>>>, pub maybe_jsx_import_source: Option<&'a JsxImportSourceConfig>, pub resolver: &'a dyn deno_graph::source::Resolver, pub environment: &'a TEnvironment, @@ -86,7 +82,6 @@ pub async fn build< parsed_source_cache, output_dir, maybe_original_import_map: original_import_map, - maybe_lockfile, maybe_jsx_import_source: jsx_import_source, resolver, environment, @@ -118,20 +113,17 @@ pub async fn build< let graph = build_graph(entry_points).await?; - // check the lockfile - if let Some(lockfile) = maybe_lockfile { - graph_lock_or_exit(&graph, &mut lockfile.lock()); - } - // surface any errors + let real_fs = Arc::new(deno_fs::RealFs) as Arc<dyn deno_fs::FileSystem>; graph_util::graph_valid( &graph, - Arc::new(deno_fs::RealFs), + &real_fs, &graph.roots, graph_util::GraphValidOptions { is_vendoring: true, check_js: true, follow_type_only: true, + exit_lockfile_errors: true, }, )?; diff --git a/cli/tools/vendor/mod.rs b/cli/tools/vendor/mod.rs index cf10b77c7..a8d8000d8 100644 --- a/cli/tools/vendor/mod.rs +++ b/cli/tools/vendor/mod.rs @@ -65,7 +65,6 @@ pub async fn vendor( parsed_source_cache: factory.parsed_source_cache(), output_dir: &output_dir, maybe_original_import_map: factory.maybe_import_map().await?.as_deref(), - maybe_lockfile: factory.maybe_lockfile().clone(), maybe_jsx_import_source: jsx_import_source.as_ref(), resolver: factory.resolver().await?.as_graph_resolver(), environment: &build::RealVendorEnvironment, diff --git a/cli/tools/vendor/test.rs b/cli/tools/vendor/test.rs index b4993565d..830d5f8f0 100644 --- a/cli/tools/vendor/test.rs +++ b/cli/tools/vendor/test.rs @@ -258,7 +258,6 @@ impl VendorTestBuilder { parsed_source_cache: &parsed_source_cache, output_dir: &output_dir, maybe_original_import_map: self.original_import_map.as_ref(), - maybe_lockfile: None, maybe_jsx_import_source: self.jsx_import_source_config.as_ref(), resolver: resolver.as_graph_resolver(), environment: &self.environment, diff --git a/cli/worker.rs b/cli/worker.rs index bfdeb3568..833f3d543 100644 --- a/cli/worker.rs +++ b/cli/worker.rs @@ -7,7 +7,6 @@ use std::sync::Arc; use deno_ast::ModuleSpecifier; use deno_core::anyhow::bail; -use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::futures::FutureExt; use deno_core::parking_lot::Mutex; @@ -48,6 +47,7 @@ use deno_terminal::colors; use tokio::select; use crate::args::package_json::PackageJsonDeps; +use crate::args::write_lockfile_if_has_changes; use crate::args::DenoSubcommand; use crate::args::StorageKeyResolver; use crate::errors; @@ -533,10 +533,7 @@ impl CliMainWorkerFactory { // For npm binary commands, ensure that the lockfile gets updated // so that we can re-use the npm resolution the next time it runs // for better performance - lockfile - .lock() - .write() - .context("Failed writing lockfile.")?; + write_lockfile_if_has_changes(&lockfile.lock())?; } (node_resolution.into_url(), is_main_cjs) |