From 0cc90d9246ff2c392457632d5030eaca2ca1ca6f Mon Sep 17 00:00:00 2001 From: David Sherret Date: Fri, 25 Nov 2022 17:00:28 -0500 Subject: refactor: move lockfile.rs to args module (#16818) This should be in the `args` folder as it's similar to `config_file`. --- cli/args/lockfile.rs | 630 +++++++++++++++++++++++++++++++++++++++++ cli/args/mod.rs | 11 +- cli/emit.rs | 2 +- cli/lockfile.rs | 630 ----------------------------------------- cli/main.rs | 6 +- cli/npm/resolution/mod.rs | 2 +- cli/npm/resolution/snapshot.rs | 2 +- cli/npm/resolvers/common.rs | 2 +- cli/npm/resolvers/global.rs | 2 +- cli/npm/resolvers/local.rs | 2 +- cli/npm/resolvers/mod.rs | 2 +- cli/proc_state.rs | 7 +- cli/resolver.rs | 2 +- 13 files changed, 651 insertions(+), 649 deletions(-) create mode 100644 cli/args/lockfile.rs delete mode 100644 cli/lockfile.rs (limited to 'cli') diff --git a/cli/args/lockfile.rs b/cli/args/lockfile.rs new file mode 100644 index 000000000..f99d2f570 --- /dev/null +++ b/cli/args/lockfile.rs @@ -0,0 +1,630 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +use deno_core::anyhow::Context; +use deno_core::error::AnyError; +use deno_core::parking_lot::Mutex; +use deno_core::serde::Deserialize; +use deno_core::serde::Serialize; +use deno_core::serde_json; +use deno_core::ModuleSpecifier; +use log::debug; +use std::cell::RefCell; +use std::collections::BTreeMap; +use std::io::Write; +use std::path::PathBuf; +use std::rc::Rc; +use std::sync::Arc; + +use crate::args::ConfigFile; +use crate::npm::NpmPackageId; +use crate::npm::NpmPackageReq; +use crate::npm::NpmResolutionPackage; +use crate::tools::fmt::format_json; +use crate::Flags; + +#[derive(Debug)] +pub struct LockfileError(String); + +impl std::fmt::Display for LockfileError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for LockfileError {} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NpmPackageInfo { + pub integrity: String, + pub dependencies: BTreeMap, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct NpmContent { + /// Mapping between requests for npm packages and resolved packages, eg. + /// { + /// "chalk": "chalk@5.0.0" + /// "react@17": "react@17.0.1" + /// "foo@latest": "foo@1.0.0" + /// } + pub specifiers: BTreeMap, + /// Mapping between resolved npm specifiers and their associated info, eg. + /// { + /// "chalk@5.0.0": { + /// "integrity": "sha512-...", + /// "dependencies": { + /// "ansi-styles": "ansi-styles@4.1.0", + /// } + /// } + /// } + pub packages: BTreeMap, +} + +impl NpmContent { + fn is_empty(&self) -> bool { + self.specifiers.is_empty() && self.packages.is_empty() + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LockfileContent { + version: String, + // Mapping between URLs and their checksums for "http:" and "https:" deps + remote: BTreeMap, + #[serde(skip_serializing_if = "NpmContent::is_empty")] + #[serde(default)] + pub npm: NpmContent, +} + +impl LockfileContent { + fn empty() -> Self { + Self { + version: "2".to_string(), + remote: BTreeMap::new(), + npm: NpmContent::default(), + } + } +} + +#[derive(Debug, Clone)] +pub struct Lockfile { + pub overwrite: bool, + pub has_content_changed: bool, + pub content: LockfileContent, + pub filename: PathBuf, +} + +impl Lockfile { + pub fn as_maybe_locker( + lockfile: Option>>, + ) -> Option>> { + lockfile.as_ref().map(|lf| { + Rc::new(RefCell::new(Locker(Some(lf.clone())))) + as Rc> + }) + } + + pub fn discover( + flags: &Flags, + maybe_config_file: Option<&ConfigFile>, + ) -> Result, AnyError> { + if flags.no_lock { + return Ok(None); + } + + let filename = match flags.lock { + Some(ref lock) => PathBuf::from(lock), + None => match maybe_config_file { + Some(config_file) => { + if config_file.specifier.scheme() == "file" { + let mut path = config_file.specifier.to_file_path().unwrap(); + path.set_file_name("deno.lock"); + path + } else { + return Ok(None); + } + } + None => return Ok(None), + }, + }; + + let lockfile = Self::new(filename, flags.lock_write)?; + Ok(Some(lockfile)) + } + + pub fn new(filename: PathBuf, overwrite: bool) -> Result { + // Writing a lock file always uses the new format. + if overwrite { + return Ok(Lockfile { + overwrite, + has_content_changed: false, + content: LockfileContent::empty(), + filename, + }); + } + + let result = match std::fs::read_to_string(&filename) { + Ok(content) => Ok(content), + Err(e) => { + if e.kind() == std::io::ErrorKind::NotFound { + return Ok(Lockfile { + overwrite, + has_content_changed: false, + content: LockfileContent::empty(), + filename, + }); + } else { + Err(e) + } + } + }; + + let s = result.with_context(|| { + format!("Unable to read lockfile: \"{}\"", filename.display()) + })?; + let value: serde_json::Value = + serde_json::from_str(&s).with_context(|| { + format!( + "Unable to parse contents of the lockfile \"{}\"", + filename.display() + ) + })?; + let version = value.get("version").and_then(|v| v.as_str()); + let content = if version == Some("2") { + serde_json::from_value::(value).with_context(|| { + format!( + "Unable to parse contents of the lockfile \"{}\"", + filename.display() + ) + })? + } else { + // If there's no version field, we assume that user is using the old + // version of the lockfile. We'll migrate it in-place into v2 and it + // will be writte in v2 if user uses `--lock-write` flag. + let remote: BTreeMap = serde_json::from_value(value) + .with_context(|| { + format!( + "Unable to parse contents of the lockfile \"{}\"", + filename.display() + ) + })?; + LockfileContent { + version: "2".to_string(), + remote, + npm: NpmContent::default(), + } + }; + + Ok(Lockfile { + overwrite, + has_content_changed: false, + content, + filename, + }) + } + + // Synchronize lock file to disk - noop if --lock-write file is not specified. + pub fn write(&self) -> Result<(), AnyError> { + if !self.has_content_changed && !self.overwrite { + return Ok(()); + } + + let json_string = serde_json::to_string(&self.content).unwrap(); + let format_s = format_json(&json_string, &Default::default()) + .ok() + .flatten() + .unwrap_or(json_string); + let mut f = std::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(&self.filename)?; + f.write_all(format_s.as_bytes())?; + debug!("lockfile write {}", self.filename.display()); + Ok(()) + } + + // TODO(bartlomieju): this function should return an error instead of a bool, + // but it requires changes to `deno_graph`'s `Locker`. + pub fn check_or_insert_remote( + &mut self, + specifier: &str, + code: &str, + ) -> bool { + if !(specifier.starts_with("http:") || specifier.starts_with("https:")) { + return true; + } + if self.overwrite { + // In case --lock-write is specified check always passes + self.insert(specifier, code); + true + } else { + self.check_or_insert(specifier, code) + } + } + + pub fn check_or_insert_npm_package( + &mut self, + package: &NpmResolutionPackage, + ) -> Result<(), LockfileError> { + if self.overwrite { + // In case --lock-write is specified check always passes + self.insert_npm(package); + Ok(()) + } else { + self.check_or_insert_npm(package) + } + } + + /// Checks the given module is included, if so verify the checksum. If module + /// is not included, insert it. + fn check_or_insert(&mut self, specifier: &str, code: &str) -> bool { + if let Some(lockfile_checksum) = self.content.remote.get(specifier) { + let compiled_checksum = crate::checksum::gen(&[code.as_bytes()]); + lockfile_checksum == &compiled_checksum + } else { + self.insert(specifier, code); + true + } + } + + fn insert(&mut self, specifier: &str, code: &str) { + let checksum = crate::checksum::gen(&[code.as_bytes()]); + self.content.remote.insert(specifier.to_string(), checksum); + self.has_content_changed = true; + } + + fn check_or_insert_npm( + &mut self, + package: &NpmResolutionPackage, + ) -> Result<(), LockfileError> { + let specifier = package.id.as_serialized(); + if let Some(package_info) = self.content.npm.packages.get(&specifier) { + let integrity = package + .dist + .integrity + .as_ref() + .unwrap_or(&package.dist.shasum); + if &package_info.integrity != integrity { + return Err(LockfileError(format!( + "Integrity check failed for npm package: \"{}\". Unable to verify that the package +is the same as when the lockfile was generated. + +This could be caused by: + * the lock file may be corrupt + * the source itself may be corrupt + +Use \"--lock-write\" flag to regenerate the lockfile at \"{}\".", + package.id.display(), self.filename.display() + ))); + } + } else { + self.insert_npm(package); + } + + Ok(()) + } + + fn insert_npm(&mut self, package: &NpmResolutionPackage) { + let dependencies = package + .dependencies + .iter() + .map(|(name, id)| (name.to_string(), id.as_serialized())) + .collect::>(); + + let integrity = package + .dist + .integrity + .as_ref() + .unwrap_or(&package.dist.shasum); + self.content.npm.packages.insert( + package.id.as_serialized(), + NpmPackageInfo { + integrity: integrity.to_string(), + dependencies, + }, + ); + self.has_content_changed = true; + } + + pub fn insert_npm_specifier( + &mut self, + package_req: &NpmPackageReq, + package_id: &NpmPackageId, + ) { + self + .content + .npm + .specifiers + .insert(package_req.to_string(), package_id.as_serialized()); + self.has_content_changed = true; + } +} + +#[derive(Debug)] +pub struct Locker(Option>>); + +impl deno_graph::source::Locker for Locker { + fn check_or_insert( + &mut self, + specifier: &ModuleSpecifier, + source: &str, + ) -> bool { + if let Some(lock_file) = &self.0 { + let mut lock_file = lock_file.lock(); + lock_file.check_or_insert_remote(specifier.as_str(), source) + } else { + true + } + } + + fn get_checksum(&self, content: &str) -> String { + crate::checksum::gen(&[content.as_bytes()]) + } + + fn get_filename(&self) -> Option { + let lock_file = self.0.as_ref()?.lock(); + lock_file.filename.to_str().map(|s| s.to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::npm::NpmPackageId; + use crate::npm::NpmPackageVersionDistInfo; + use crate::npm::NpmVersion; + use deno_core::serde_json; + use deno_core::serde_json::json; + use std::collections::HashMap; + use std::fs::File; + use std::io::prelude::*; + use std::io::Write; + use test_util::TempDir; + + fn setup(temp_dir: &TempDir) -> PathBuf { + let file_path = temp_dir.path().join("valid_lockfile.json"); + let mut file = File::create(file_path).expect("write file fail"); + + let value: serde_json::Value = json!({ + "version": "2", + "remote": { + "https://deno.land/std@0.71.0/textproto/mod.ts": "3118d7a42c03c242c5a49c2ad91c8396110e14acca1324e7aaefd31a999b71a4", + "https://deno.land/std@0.71.0/async/delay.ts": "35957d585a6e3dd87706858fb1d6b551cb278271b03f52c5a2cb70e65e00c26a" + }, + "npm": { + "specifiers": {}, + "packages": { + "nanoid@3.3.4": { + "integrity": "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==", + "dependencies": {} + }, + "picocolors@1.0.0": { + "integrity": "sha512-foobar", + "dependencies": {} + }, + } + } + }); + + file.write_all(value.to_string().as_bytes()).unwrap(); + + temp_dir.path().join("valid_lockfile.json") + } + + #[test] + fn create_lockfile_for_nonexistent_path() { + let file_path = PathBuf::from("nonexistent_lock_file.json"); + assert!(Lockfile::new(file_path, false).is_ok()); + } + + #[test] + fn new_valid_lockfile() { + let temp_dir = TempDir::new(); + let file_path = setup(&temp_dir); + + let result = Lockfile::new(file_path, false).unwrap(); + + let remote = result.content.remote; + let keys: Vec = remote.keys().cloned().collect(); + let expected_keys = vec![ + String::from("https://deno.land/std@0.71.0/async/delay.ts"), + String::from("https://deno.land/std@0.71.0/textproto/mod.ts"), + ]; + + assert_eq!(keys.len(), 2); + assert_eq!(keys, expected_keys); + } + + #[test] + fn new_lockfile_from_file_and_insert() { + let temp_dir = TempDir::new(); + let file_path = setup(&temp_dir); + + let mut lockfile = Lockfile::new(file_path, false).unwrap(); + + lockfile.insert( + "https://deno.land/std@0.71.0/io/util.ts", + "Here is some source code", + ); + + let remote = lockfile.content.remote; + let keys: Vec = remote.keys().cloned().collect(); + let expected_keys = vec![ + String::from("https://deno.land/std@0.71.0/async/delay.ts"), + String::from("https://deno.land/std@0.71.0/io/util.ts"), + String::from("https://deno.land/std@0.71.0/textproto/mod.ts"), + ]; + assert_eq!(keys.len(), 3); + assert_eq!(keys, expected_keys); + } + + #[test] + fn new_lockfile_and_write() { + let temp_dir = TempDir::new(); + let file_path = setup(&temp_dir); + + let mut lockfile = Lockfile::new(file_path, true).unwrap(); + + lockfile.insert( + "https://deno.land/std@0.71.0/textproto/mod.ts", + "Here is some source code", + ); + lockfile.insert( + "https://deno.land/std@0.71.0/io/util.ts", + "more source code here", + ); + lockfile.insert( + "https://deno.land/std@0.71.0/async/delay.ts", + "this source is really exciting", + ); + + lockfile.write().expect("unable to write"); + + let file_path_buf = temp_dir.path().join("valid_lockfile.json"); + let file_path = file_path_buf.to_str().expect("file path fail").to_string(); + + // read the file contents back into a string and check + let mut checkfile = File::open(file_path).expect("Unable to open the file"); + let mut contents = String::new(); + checkfile + .read_to_string(&mut contents) + .expect("Unable to read the file"); + + let contents_json = + serde_json::from_str::(&contents).unwrap(); + let object = contents_json["remote"].as_object().unwrap(); + + assert_eq!( + object + .get("https://deno.land/std@0.71.0/textproto/mod.ts") + .and_then(|v| v.as_str()), + // sha-256 hash of the source 'Here is some source code' + Some("fedebba9bb82cce293196f54b21875b649e457f0eaf55556f1e318204947a28f") + ); + + // confirm that keys are sorted alphabetically + let mut keys = object.keys().map(|k| k.as_str()); + assert_eq!( + keys.next(), + Some("https://deno.land/std@0.71.0/async/delay.ts") + ); + assert_eq!(keys.next(), Some("https://deno.land/std@0.71.0/io/util.ts")); + assert_eq!( + keys.next(), + Some("https://deno.land/std@0.71.0/textproto/mod.ts") + ); + assert!(keys.next().is_none()); + } + + #[test] + fn check_or_insert_lockfile() { + let temp_dir = TempDir::new(); + let file_path = setup(&temp_dir); + + let mut lockfile = Lockfile::new(file_path, false).unwrap(); + + lockfile.insert( + "https://deno.land/std@0.71.0/textproto/mod.ts", + "Here is some source code", + ); + + let check_true = lockfile.check_or_insert_remote( + "https://deno.land/std@0.71.0/textproto/mod.ts", + "Here is some source code", + ); + assert!(check_true); + + let check_false = lockfile.check_or_insert_remote( + "https://deno.land/std@0.71.0/textproto/mod.ts", + "Here is some NEW source code", + ); + assert!(!check_false); + + // Not present in lockfile yet, should be inserted and check passed. + let check_true = lockfile.check_or_insert_remote( + "https://deno.land/std@0.71.0/http/file_server.ts", + "This is new Source code", + ); + assert!(check_true); + } + + #[test] + fn check_or_insert_lockfile_npm() { + let temp_dir = TempDir::new(); + let file_path = setup(&temp_dir); + + let mut lockfile = Lockfile::new(file_path, false).unwrap(); + + let npm_package = NpmResolutionPackage { + id: NpmPackageId { + name: "nanoid".to_string(), + version: NpmVersion::parse("3.3.4").unwrap(), + peer_dependencies: Vec::new(), + }, + copy_index: 0, + dist: NpmPackageVersionDistInfo { + tarball: "foo".to_string(), + shasum: "foo".to_string(), + integrity: Some("sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==".to_string()) + }, + dependencies: HashMap::new(), + }; + let check_ok = lockfile.check_or_insert_npm_package(&npm_package); + assert!(check_ok.is_ok()); + + let npm_package = NpmResolutionPackage { + id: NpmPackageId { + name: "picocolors".to_string(), + version: NpmVersion::parse("1.0.0").unwrap(), + peer_dependencies: Vec::new(), + }, + copy_index: 0, + dist: NpmPackageVersionDistInfo { + tarball: "foo".to_string(), + shasum: "foo".to_string(), + integrity: Some("sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==".to_string()) + }, + dependencies: HashMap::new(), + }; + // Integrity is borked in the loaded lockfile + let check_err = lockfile.check_or_insert_npm_package(&npm_package); + assert!(check_err.is_err()); + + let npm_package = NpmResolutionPackage { + id: NpmPackageId { + name: "source-map-js".to_string(), + version: NpmVersion::parse("1.0.2").unwrap(), + peer_dependencies: Vec::new(), + }, + copy_index: 0, + dist: NpmPackageVersionDistInfo { + tarball: "foo".to_string(), + shasum: "foo".to_string(), + integrity: Some("sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==".to_string()) + }, + dependencies: HashMap::new(), + }; + // Not present in lockfile yet, should be inserted and check passed. + let check_ok = lockfile.check_or_insert_npm_package(&npm_package); + assert!(check_ok.is_ok()); + + let npm_package = NpmResolutionPackage { + id: NpmPackageId { + name: "source-map-js".to_string(), + version: NpmVersion::parse("1.0.2").unwrap(), + peer_dependencies: Vec::new(), + }, + copy_index: 0, + dist: NpmPackageVersionDistInfo { + tarball: "foo".to_string(), + shasum: "foo".to_string(), + integrity: Some("sha512-foobar".to_string()), + }, + dependencies: HashMap::new(), + }; + // Now present in lockfile, should file due to borked integrity + let check_err = lockfile.check_or_insert_npm_package(&npm_package); + assert!(check_err.is_err()); + } +} diff --git a/cli/args/mod.rs b/cli/args/mod.rs index 64755a494..ff9a6f73c 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -1,7 +1,8 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. -pub mod config_file; -pub mod flags; +mod config_file; +mod flags; +mod lockfile; mod flags_allow_net; @@ -10,6 +11,8 @@ pub use config_file::ConfigFile; pub use config_file::EmitConfigOptions; pub use config_file::FmtConfig; pub use config_file::FmtOptionsConfig; +pub use config_file::IgnoredCompilerOptions; +pub use config_file::JsxImportSourceConfig; pub use config_file::LintConfig; pub use config_file::LintRulesConfig; pub use config_file::MaybeImportsResult; @@ -17,6 +20,8 @@ pub use config_file::ProseWrap; pub use config_file::TestConfig; pub use config_file::TsConfig; pub use flags::*; +pub use lockfile::Lockfile; +pub use lockfile::LockfileError; use deno_ast::ModuleSpecifier; use deno_core::anyhow::anyhow; @@ -36,7 +41,6 @@ use std::net::SocketAddr; use std::path::PathBuf; use std::sync::Arc; -use crate::args::config_file::JsxImportSourceConfig; use crate::deno_dir::DenoDir; use crate::emit::get_ts_config_for_emit; use crate::emit::TsConfigType; @@ -45,7 +49,6 @@ use crate::emit::TsTypeLib; use crate::file_fetcher::get_root_cert_store; use crate::file_fetcher::CacheSetting; use crate::fs_util; -use crate::lockfile::Lockfile; use crate::version; /// Overrides for the options below that when set will diff --git a/cli/emit.rs b/cli/emit.rs index f2d890adc..766524187 100644 --- a/cli/emit.rs +++ b/cli/emit.rs @@ -4,9 +4,9 @@ //! populate a cache, emit files, and transform a graph into the structures for //! loading into an isolate. -use crate::args::config_file::IgnoredCompilerOptions; use crate::args::ConfigFile; use crate::args::EmitConfigOptions; +use crate::args::IgnoredCompilerOptions; use crate::args::TsConfig; use crate::cache::EmitCache; use crate::cache::FastInsecureHasher; diff --git a/cli/lockfile.rs b/cli/lockfile.rs deleted file mode 100644 index b1f0c777f..000000000 --- a/cli/lockfile.rs +++ /dev/null @@ -1,630 +0,0 @@ -// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. - -use deno_core::anyhow::Context; -use deno_core::error::AnyError; -use deno_core::parking_lot::Mutex; -use deno_core::serde::Deserialize; -use deno_core::serde::Serialize; -use deno_core::serde_json; -use deno_core::ModuleSpecifier; -use log::debug; -use std::cell::RefCell; -use std::collections::BTreeMap; -use std::io::Write; -use std::path::PathBuf; -use std::rc::Rc; -use std::sync::Arc; - -use crate::args::ConfigFile; -use crate::npm::NpmPackageId; -use crate::npm::NpmPackageReq; -use crate::npm::NpmResolutionPackage; -use crate::tools::fmt::format_json; -use crate::Flags; - -#[derive(Debug)] -pub struct LockfileError(String); - -impl std::fmt::Display for LockfileError { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - f.write_str(&self.0) - } -} - -impl std::error::Error for LockfileError {} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct NpmPackageInfo { - pub integrity: String, - pub dependencies: BTreeMap, -} - -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -pub struct NpmContent { - /// Mapping between requests for npm packages and resolved packages, eg. - /// { - /// "chalk": "chalk@5.0.0" - /// "react@17": "react@17.0.1" - /// "foo@latest": "foo@1.0.0" - /// } - pub specifiers: BTreeMap, - /// Mapping between resolved npm specifiers and their associated info, eg. - /// { - /// "chalk@5.0.0": { - /// "integrity": "sha512-...", - /// "dependencies": { - /// "ansi-styles": "ansi-styles@4.1.0", - /// } - /// } - /// } - pub packages: BTreeMap, -} - -impl NpmContent { - fn is_empty(&self) -> bool { - self.specifiers.is_empty() && self.packages.is_empty() - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct LockfileContent { - version: String, - // Mapping between URLs and their checksums for "http:" and "https:" deps - remote: BTreeMap, - #[serde(skip_serializing_if = "NpmContent::is_empty")] - #[serde(default)] - pub npm: NpmContent, -} - -impl LockfileContent { - fn empty() -> Self { - Self { - version: "2".to_string(), - remote: BTreeMap::new(), - npm: NpmContent::default(), - } - } -} - -#[derive(Debug, Clone)] -pub struct Lockfile { - pub overwrite: bool, - pub has_content_changed: bool, - pub content: LockfileContent, - pub filename: PathBuf, -} - -impl Lockfile { - pub fn discover( - flags: &Flags, - maybe_config_file: Option<&ConfigFile>, - ) -> Result, AnyError> { - if flags.no_lock { - return Ok(None); - } - - let filename = match flags.lock { - Some(ref lock) => PathBuf::from(lock), - None => match maybe_config_file { - Some(config_file) => { - if config_file.specifier.scheme() == "file" { - let mut path = config_file.specifier.to_file_path().unwrap(); - path.set_file_name("deno.lock"); - path - } else { - return Ok(None); - } - } - None => return Ok(None), - }, - }; - - let lockfile = Self::new(filename, flags.lock_write)?; - Ok(Some(lockfile)) - } - - pub fn new(filename: PathBuf, overwrite: bool) -> Result { - // Writing a lock file always uses the new format. - if overwrite { - return Ok(Lockfile { - overwrite, - has_content_changed: false, - content: LockfileContent::empty(), - filename, - }); - } - - let result = match std::fs::read_to_string(&filename) { - Ok(content) => Ok(content), - Err(e) => { - if e.kind() == std::io::ErrorKind::NotFound { - return Ok(Lockfile { - overwrite, - has_content_changed: false, - content: LockfileContent::empty(), - filename, - }); - } else { - Err(e) - } - } - }; - - let s = result.with_context(|| { - format!("Unable to read lockfile: \"{}\"", filename.display()) - })?; - let value: serde_json::Value = - serde_json::from_str(&s).with_context(|| { - format!( - "Unable to parse contents of the lockfile \"{}\"", - filename.display() - ) - })?; - let version = value.get("version").and_then(|v| v.as_str()); - let content = if version == Some("2") { - serde_json::from_value::(value).with_context(|| { - format!( - "Unable to parse contents of the lockfile \"{}\"", - filename.display() - ) - })? - } else { - // If there's no version field, we assume that user is using the old - // version of the lockfile. We'll migrate it in-place into v2 and it - // will be writte in v2 if user uses `--lock-write` flag. - let remote: BTreeMap = serde_json::from_value(value) - .with_context(|| { - format!( - "Unable to parse contents of the lockfile \"{}\"", - filename.display() - ) - })?; - LockfileContent { - version: "2".to_string(), - remote, - npm: NpmContent::default(), - } - }; - - Ok(Lockfile { - overwrite, - has_content_changed: false, - content, - filename, - }) - } - - // Synchronize lock file to disk - noop if --lock-write file is not specified. - pub fn write(&self) -> Result<(), AnyError> { - if !self.has_content_changed && !self.overwrite { - return Ok(()); - } - - let json_string = serde_json::to_string(&self.content).unwrap(); - let format_s = format_json(&json_string, &Default::default()) - .ok() - .flatten() - .unwrap_or(json_string); - let mut f = std::fs::OpenOptions::new() - .write(true) - .create(true) - .truncate(true) - .open(&self.filename)?; - f.write_all(format_s.as_bytes())?; - debug!("lockfile write {}", self.filename.display()); - Ok(()) - } - - // TODO(bartlomieju): this function should return an error instead of a bool, - // but it requires changes to `deno_graph`'s `Locker`. - pub fn check_or_insert_remote( - &mut self, - specifier: &str, - code: &str, - ) -> bool { - if !(specifier.starts_with("http:") || specifier.starts_with("https:")) { - return true; - } - if self.overwrite { - // In case --lock-write is specified check always passes - self.insert(specifier, code); - true - } else { - self.check_or_insert(specifier, code) - } - } - - pub fn check_or_insert_npm_package( - &mut self, - package: &NpmResolutionPackage, - ) -> Result<(), LockfileError> { - if self.overwrite { - // In case --lock-write is specified check always passes - self.insert_npm(package); - Ok(()) - } else { - self.check_or_insert_npm(package) - } - } - - /// Checks the given module is included, if so verify the checksum. If module - /// is not included, insert it. - fn check_or_insert(&mut self, specifier: &str, code: &str) -> bool { - if let Some(lockfile_checksum) = self.content.remote.get(specifier) { - let compiled_checksum = crate::checksum::gen(&[code.as_bytes()]); - lockfile_checksum == &compiled_checksum - } else { - self.insert(specifier, code); - true - } - } - - fn insert(&mut self, specifier: &str, code: &str) { - let checksum = crate::checksum::gen(&[code.as_bytes()]); - self.content.remote.insert(specifier.to_string(), checksum); - self.has_content_changed = true; - } - - fn check_or_insert_npm( - &mut self, - package: &NpmResolutionPackage, - ) -> Result<(), LockfileError> { - let specifier = package.id.as_serialized(); - if let Some(package_info) = self.content.npm.packages.get(&specifier) { - let integrity = package - .dist - .integrity - .as_ref() - .unwrap_or(&package.dist.shasum); - if &package_info.integrity != integrity { - return Err(LockfileError(format!( - "Integrity check failed for npm package: \"{}\". Unable to verify that the package -is the same as when the lockfile was generated. - -This could be caused by: - * the lock file may be corrupt - * the source itself may be corrupt - -Use \"--lock-write\" flag to regenerate the lockfile at \"{}\".", - package.id.display(), self.filename.display() - ))); - } - } else { - self.insert_npm(package); - } - - Ok(()) - } - - fn insert_npm(&mut self, package: &NpmResolutionPackage) { - let dependencies = package - .dependencies - .iter() - .map(|(name, id)| (name.to_string(), id.as_serialized())) - .collect::>(); - - let integrity = package - .dist - .integrity - .as_ref() - .unwrap_or(&package.dist.shasum); - self.content.npm.packages.insert( - package.id.as_serialized(), - NpmPackageInfo { - integrity: integrity.to_string(), - dependencies, - }, - ); - self.has_content_changed = true; - } - - pub fn insert_npm_specifier( - &mut self, - package_req: &NpmPackageReq, - package_id: &NpmPackageId, - ) { - self - .content - .npm - .specifiers - .insert(package_req.to_string(), package_id.as_serialized()); - self.has_content_changed = true; - } -} - -#[derive(Debug)] -pub struct Locker(Option>>); - -impl deno_graph::source::Locker for Locker { - fn check_or_insert( - &mut self, - specifier: &ModuleSpecifier, - source: &str, - ) -> bool { - if let Some(lock_file) = &self.0 { - let mut lock_file = lock_file.lock(); - lock_file.check_or_insert_remote(specifier.as_str(), source) - } else { - true - } - } - - fn get_checksum(&self, content: &str) -> String { - crate::checksum::gen(&[content.as_bytes()]) - } - - fn get_filename(&self) -> Option { - let lock_file = self.0.as_ref()?.lock(); - lock_file.filename.to_str().map(|s| s.to_string()) - } -} - -pub fn as_maybe_locker( - lockfile: Option>>, -) -> Option>> { - lockfile.as_ref().map(|lf| { - Rc::new(RefCell::new(Locker(Some(lf.clone())))) - as Rc> - }) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::npm::NpmPackageId; - use crate::npm::NpmPackageVersionDistInfo; - use crate::npm::NpmVersion; - use deno_core::serde_json; - use deno_core::serde_json::json; - use std::collections::HashMap; - use std::fs::File; - use std::io::prelude::*; - use std::io::Write; - use test_util::TempDir; - - fn setup(temp_dir: &TempDir) -> PathBuf { - let file_path = temp_dir.path().join("valid_lockfile.json"); - let mut file = File::create(file_path).expect("write file fail"); - - let value: serde_json::Value = json!({ - "version": "2", - "remote": { - "https://deno.land/std@0.71.0/textproto/mod.ts": "3118d7a42c03c242c5a49c2ad91c8396110e14acca1324e7aaefd31a999b71a4", - "https://deno.land/std@0.71.0/async/delay.ts": "35957d585a6e3dd87706858fb1d6b551cb278271b03f52c5a2cb70e65e00c26a" - }, - "npm": { - "specifiers": {}, - "packages": { - "nanoid@3.3.4": { - "integrity": "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==", - "dependencies": {} - }, - "picocolors@1.0.0": { - "integrity": "sha512-foobar", - "dependencies": {} - }, - } - } - }); - - file.write_all(value.to_string().as_bytes()).unwrap(); - - temp_dir.path().join("valid_lockfile.json") - } - - #[test] - fn create_lockfile_for_nonexistent_path() { - let file_path = PathBuf::from("nonexistent_lock_file.json"); - assert!(Lockfile::new(file_path, false).is_ok()); - } - - #[test] - fn new_valid_lockfile() { - let temp_dir = TempDir::new(); - let file_path = setup(&temp_dir); - - let result = Lockfile::new(file_path, false).unwrap(); - - let remote = result.content.remote; - let keys: Vec = remote.keys().cloned().collect(); - let expected_keys = vec![ - String::from("https://deno.land/std@0.71.0/async/delay.ts"), - String::from("https://deno.land/std@0.71.0/textproto/mod.ts"), - ]; - - assert_eq!(keys.len(), 2); - assert_eq!(keys, expected_keys); - } - - #[test] - fn new_lockfile_from_file_and_insert() { - let temp_dir = TempDir::new(); - let file_path = setup(&temp_dir); - - let mut lockfile = Lockfile::new(file_path, false).unwrap(); - - lockfile.insert( - "https://deno.land/std@0.71.0/io/util.ts", - "Here is some source code", - ); - - let remote = lockfile.content.remote; - let keys: Vec = remote.keys().cloned().collect(); - let expected_keys = vec![ - String::from("https://deno.land/std@0.71.0/async/delay.ts"), - String::from("https://deno.land/std@0.71.0/io/util.ts"), - String::from("https://deno.land/std@0.71.0/textproto/mod.ts"), - ]; - assert_eq!(keys.len(), 3); - assert_eq!(keys, expected_keys); - } - - #[test] - fn new_lockfile_and_write() { - let temp_dir = TempDir::new(); - let file_path = setup(&temp_dir); - - let mut lockfile = Lockfile::new(file_path, true).unwrap(); - - lockfile.insert( - "https://deno.land/std@0.71.0/textproto/mod.ts", - "Here is some source code", - ); - lockfile.insert( - "https://deno.land/std@0.71.0/io/util.ts", - "more source code here", - ); - lockfile.insert( - "https://deno.land/std@0.71.0/async/delay.ts", - "this source is really exciting", - ); - - lockfile.write().expect("unable to write"); - - let file_path_buf = temp_dir.path().join("valid_lockfile.json"); - let file_path = file_path_buf.to_str().expect("file path fail").to_string(); - - // read the file contents back into a string and check - let mut checkfile = File::open(file_path).expect("Unable to open the file"); - let mut contents = String::new(); - checkfile - .read_to_string(&mut contents) - .expect("Unable to read the file"); - - let contents_json = - serde_json::from_str::(&contents).unwrap(); - let object = contents_json["remote"].as_object().unwrap(); - - assert_eq!( - object - .get("https://deno.land/std@0.71.0/textproto/mod.ts") - .and_then(|v| v.as_str()), - // sha-256 hash of the source 'Here is some source code' - Some("fedebba9bb82cce293196f54b21875b649e457f0eaf55556f1e318204947a28f") - ); - - // confirm that keys are sorted alphabetically - let mut keys = object.keys().map(|k| k.as_str()); - assert_eq!( - keys.next(), - Some("https://deno.land/std@0.71.0/async/delay.ts") - ); - assert_eq!(keys.next(), Some("https://deno.land/std@0.71.0/io/util.ts")); - assert_eq!( - keys.next(), - Some("https://deno.land/std@0.71.0/textproto/mod.ts") - ); - assert!(keys.next().is_none()); - } - - #[test] - fn check_or_insert_lockfile() { - let temp_dir = TempDir::new(); - let file_path = setup(&temp_dir); - - let mut lockfile = Lockfile::new(file_path, false).unwrap(); - - lockfile.insert( - "https://deno.land/std@0.71.0/textproto/mod.ts", - "Here is some source code", - ); - - let check_true = lockfile.check_or_insert_remote( - "https://deno.land/std@0.71.0/textproto/mod.ts", - "Here is some source code", - ); - assert!(check_true); - - let check_false = lockfile.check_or_insert_remote( - "https://deno.land/std@0.71.0/textproto/mod.ts", - "Here is some NEW source code", - ); - assert!(!check_false); - - // Not present in lockfile yet, should be inserted and check passed. - let check_true = lockfile.check_or_insert_remote( - "https://deno.land/std@0.71.0/http/file_server.ts", - "This is new Source code", - ); - assert!(check_true); - } - - #[test] - fn check_or_insert_lockfile_npm() { - let temp_dir = TempDir::new(); - let file_path = setup(&temp_dir); - - let mut lockfile = Lockfile::new(file_path, false).unwrap(); - - let npm_package = NpmResolutionPackage { - id: NpmPackageId { - name: "nanoid".to_string(), - version: NpmVersion::parse("3.3.4").unwrap(), - peer_dependencies: Vec::new(), - }, - copy_index: 0, - dist: NpmPackageVersionDistInfo { - tarball: "foo".to_string(), - shasum: "foo".to_string(), - integrity: Some("sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==".to_string()) - }, - dependencies: HashMap::new(), - }; - let check_ok = lockfile.check_or_insert_npm_package(&npm_package); - assert!(check_ok.is_ok()); - - let npm_package = NpmResolutionPackage { - id: NpmPackageId { - name: "picocolors".to_string(), - version: NpmVersion::parse("1.0.0").unwrap(), - peer_dependencies: Vec::new(), - }, - copy_index: 0, - dist: NpmPackageVersionDistInfo { - tarball: "foo".to_string(), - shasum: "foo".to_string(), - integrity: Some("sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==".to_string()) - }, - dependencies: HashMap::new(), - }; - // Integrity is borked in the loaded lockfile - let check_err = lockfile.check_or_insert_npm_package(&npm_package); - assert!(check_err.is_err()); - - let npm_package = NpmResolutionPackage { - id: NpmPackageId { - name: "source-map-js".to_string(), - version: NpmVersion::parse("1.0.2").unwrap(), - peer_dependencies: Vec::new(), - }, - copy_index: 0, - dist: NpmPackageVersionDistInfo { - tarball: "foo".to_string(), - shasum: "foo".to_string(), - integrity: Some("sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==".to_string()) - }, - dependencies: HashMap::new(), - }; - // Not present in lockfile yet, should be inserted and check passed. - let check_ok = lockfile.check_or_insert_npm_package(&npm_package); - assert!(check_ok.is_ok()); - - let npm_package = NpmResolutionPackage { - id: NpmPackageId { - name: "source-map-js".to_string(), - version: NpmVersion::parse("1.0.2").unwrap(), - peer_dependencies: Vec::new(), - }, - copy_index: 0, - dist: NpmPackageVersionDistInfo { - tarball: "foo".to_string(), - shasum: "foo".to_string(), - integrity: Some("sha512-foobar".to_string()), - }, - dependencies: HashMap::new(), - }; - // Now present in lockfile, should file due to borked integrity - let check_err = lockfile.check_or_insert_npm_package(&npm_package); - assert!(check_err.is_err()); - } -} diff --git a/cli/main.rs b/cli/main.rs index f18c3c976..93f2c501d 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -19,7 +19,6 @@ mod graph_util; mod http_cache; mod http_util; mod js; -mod lockfile; mod logger; mod lsp; mod module_loader; @@ -74,6 +73,7 @@ use crate::resolver::CliResolver; use crate::tools::check; use args::CliOptions; +use args::Lockfile; use deno_ast::MediaType; use deno_core::anyhow::bail; use deno_core::error::generic_error; @@ -327,7 +327,7 @@ async fn create_graph_and_maybe_check( Permissions::allow_all(), Permissions::allow_all(), ); - let maybe_locker = lockfile::as_maybe_locker(ps.lockfile.clone()); + let maybe_locker = Lockfile::as_maybe_locker(ps.lockfile.clone()); let maybe_imports = ps.options.to_maybe_imports()?; let maybe_cli_resolver = CliResolver::maybe_new( ps.options.to_maybe_jsx_import_source_config(), @@ -937,7 +937,7 @@ fn unwrap_or_exit(result: Result) -> T { if let Some(e) = error.downcast_ref::() { error_string = format_js_error(e); - } else if let Some(e) = error.downcast_ref::() { + } else if let Some(e) = error.downcast_ref::() { error_string = e.to_string(); error_code = 10; } diff --git a/cli/npm/resolution/mod.rs b/cli/npm/resolution/mod.rs index 4a8c2e8e1..15de4ceff 100644 --- a/cli/npm/resolution/mod.rs +++ b/cli/npm/resolution/mod.rs @@ -10,7 +10,7 @@ use deno_core::parking_lot::RwLock; use serde::Deserialize; use serde::Serialize; -use crate::lockfile::Lockfile; +use crate::args::Lockfile; use self::graph::GraphDependencyResolver; use self::snapshot::NpmPackagesPartitioned; diff --git a/cli/npm/resolution/snapshot.rs b/cli/npm/resolution/snapshot.rs index d76ba8b1a..738b68d21 100644 --- a/cli/npm/resolution/snapshot.rs +++ b/cli/npm/resolution/snapshot.rs @@ -13,7 +13,7 @@ use deno_core::parking_lot::Mutex; use serde::Deserialize; use serde::Serialize; -use crate::lockfile::Lockfile; +use crate::args::Lockfile; use crate::npm::cache::should_sync_download; use crate::npm::cache::NpmPackageCacheFolderId; use crate::npm::registry::NpmPackageVersionDistInfo; diff --git a/cli/npm/resolvers/common.rs b/cli/npm/resolvers/common.rs index b160697d1..e3acef3f5 100644 --- a/cli/npm/resolvers/common.rs +++ b/cli/npm/resolvers/common.rs @@ -11,7 +11,7 @@ use deno_core::futures; use deno_core::futures::future::BoxFuture; use deno_core::url::Url; -use crate::lockfile::Lockfile; +use crate::args::Lockfile; use crate::npm::cache::should_sync_download; use crate::npm::resolution::NpmResolutionSnapshot; use crate::npm::NpmCache; diff --git a/cli/npm/resolvers/global.rs b/cli/npm/resolvers/global.rs index 3fad9f2d9..044c889d8 100644 --- a/cli/npm/resolvers/global.rs +++ b/cli/npm/resolvers/global.rs @@ -15,8 +15,8 @@ use deno_core::url::Url; use deno_runtime::deno_node::PackageJson; use deno_runtime::deno_node::TYPES_CONDITIONS; +use crate::args::Lockfile; use crate::fs_util; -use crate::lockfile::Lockfile; use crate::npm::resolution::NpmResolution; use crate::npm::resolution::NpmResolutionSnapshot; use crate::npm::resolvers::common::cache_packages; diff --git a/cli/npm/resolvers/local.rs b/cli/npm/resolvers/local.rs index 0a47a7ff1..a6df641d1 100644 --- a/cli/npm/resolvers/local.rs +++ b/cli/npm/resolvers/local.rs @@ -22,8 +22,8 @@ use deno_runtime::deno_node::PackageJson; use deno_runtime::deno_node::TYPES_CONDITIONS; use tokio::task::JoinHandle; +use crate::args::Lockfile; use crate::fs_util; -use crate::lockfile::Lockfile; use crate::npm::cache::mixed_case_package_name_encode; use crate::npm::cache::should_sync_download; use crate::npm::cache::NpmPackageCacheFolderId; diff --git a/cli/npm/resolvers/mod.rs b/cli/npm/resolvers/mod.rs index 23cbde5d9..869874c8b 100644 --- a/cli/npm/resolvers/mod.rs +++ b/cli/npm/resolvers/mod.rs @@ -22,8 +22,8 @@ use std::path::Path; use std::path::PathBuf; use std::sync::Arc; +use crate::args::Lockfile; use crate::fs_util; -use crate::lockfile::Lockfile; use self::common::InnerNpmPackageResolver; use self::local::LocalNpmPackageResolver; diff --git a/cli/proc_state.rs b/cli/proc_state.rs index 2064d3851..32a2d9d42 100644 --- a/cli/proc_state.rs +++ b/cli/proc_state.rs @@ -3,6 +3,7 @@ use crate::args::CliOptions; use crate::args::DenoSubcommand; use crate::args::Flags; +use crate::args::Lockfile; use crate::args::TypeCheckMode; use crate::cache; use crate::cache::EmitCache; @@ -20,8 +21,6 @@ use crate::graph_util::GraphData; use crate::graph_util::ModuleEntry; use crate::http_cache; use crate::http_util::HttpClient; -use crate::lockfile::as_maybe_locker; -use crate::lockfile::Lockfile; use crate::node; use crate::node::NodeResolution; use crate::npm::resolve_npm_package_reqs; @@ -330,7 +329,7 @@ impl ProcState { root_permissions.clone(), dynamic_permissions.clone(), ); - let maybe_locker = as_maybe_locker(self.lockfile.clone()); + let maybe_locker = Lockfile::as_maybe_locker(self.lockfile.clone()); let maybe_imports = self.options.to_maybe_imports()?; let maybe_resolver = self.maybe_resolver.as_ref().map(|r| r.as_graph_resolver()); @@ -640,7 +639,7 @@ impl ProcState { roots: Vec<(ModuleSpecifier, ModuleKind)>, loader: &mut dyn Loader, ) -> Result { - let maybe_locker = as_maybe_locker(self.lockfile.clone()); + let maybe_locker = Lockfile::as_maybe_locker(self.lockfile.clone()); let maybe_imports = self.options.to_maybe_imports()?; let maybe_cli_resolver = CliResolver::maybe_new( diff --git a/cli/resolver.rs b/cli/resolver.rs index c28d9df75..a4c4439ab 100644 --- a/cli/resolver.rs +++ b/cli/resolver.rs @@ -8,7 +8,7 @@ use deno_graph::source::DEFAULT_JSX_IMPORT_SOURCE_MODULE; use import_map::ImportMap; use std::sync::Arc; -use crate::args::config_file::JsxImportSourceConfig; +use crate::args::JsxImportSourceConfig; /// A resolver that takes care of resolution, taking into account loaded /// import map, JSX settings. -- cgit v1.2.3