From bf237c6241f53122e37341a0dda65ef9e3b51a49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Mon, 23 Jan 2023 23:41:02 +0100 Subject: refactor: Move lockfile to a separate crate (#17503) Moves the lockfile implementation to a separate crate so other projects like Deploy can use it as well. --- Cargo.lock | 12 + Cargo.toml | 2 + cli/Cargo.toml | 1 + cli/args/lockfile.rs | 646 ++++--------------------------------- cli/args/mod.rs | 2 +- cli/npm/registry.rs | 13 - cli/npm/resolution/mod.rs | 7 +- cli/tests/integration/npm_tests.rs | 35 +- lockfile/Cargo.toml | 20 ++ lockfile/README.md | 3 + lockfile/lib.rs | 530 ++++++++++++++++++++++++++++++ 11 files changed, 663 insertions(+), 608 deletions(-) create mode 100644 lockfile/Cargo.toml create mode 100644 lockfile/README.md create mode 100644 lockfile/lib.rs diff --git a/Cargo.lock b/Cargo.lock index b26cf7f52..7d2e2c01b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -775,6 +775,7 @@ dependencies = [ "deno_emit", "deno_graph", "deno_lint", + "deno_lockfile", "deno_runtime", "deno_task_shell", "dissimilar", @@ -1112,6 +1113,17 @@ dependencies = [ "serde_json", ] +[[package]] +name = "deno_lockfile" +version = "0.1.0" +dependencies = [ + "anyhow", + "ring", + "serde", + "serde_json", + "test_util", +] + [[package]] name = "deno_napi" version = "0.15.0" diff --git a/Cargo.toml b/Cargo.toml index 00ecf554c..94f084a5e 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,6 +30,7 @@ members = [ "ext/websocket", "ext/webstorage", "ext/napi", + "lockfile", ] exclude = ["test_util/std/hash/_wasm"] @@ -50,6 +51,7 @@ deno_runtime = { version = "0.93.0", path = "./runtime" } napi_sym = { version = "0.15.0", path = "./cli/napi/sym" } deno_bench_util = { version = "0.79.0", path = "./bench_util" } test_util = { path = "./test_util" } +deno_lockfile = { version = "0.1.0", path = "./lockfile" } # exts deno_broadcast_channel = { version = "0.79.0", path = "./ext/broadcast_channel" } diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 954c2d13e..a9b546224 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -48,6 +48,7 @@ deno_doc = "0.52.0" deno_emit = "0.13.0" deno_graph = "0.41.0" deno_lint = { version = "0.37.0", features = ["docs"] } +deno_lockfile.workspace = true deno_runtime.workspace = true deno_task_shell = "0.8.1" napi_sym.workspace = true diff --git a/cli/args/lockfile.rs b/cli/args/lockfile.rs index db9ace515..f93027430 100644 --- a/cli/args/lockfile.rs +++ b/cli/args/lockfile.rs @@ -1,602 +1,84 @@ // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. -use std::collections::BTreeMap; -use std::io::Write; -use std::path::PathBuf; - -use deno_core::anyhow::Context; use deno_core::error::AnyError; -use deno_core::serde::Deserialize; -use deno_core::serde::Serialize; -use deno_core::serde_json; -use log::debug; +use std::path::PathBuf; use crate::args::config_file::LockConfig; use crate::args::ConfigFile; -use crate::npm::NpmPackageId; -use crate::npm::NpmPackageReq; use crate::npm::NpmResolutionPackage; -use crate::tools::fmt::format_json; -use crate::util; use crate::Flags; use super::DenoSubcommand; -#[derive(Debug)] -pub struct LockfileError(String); - -impl std::fmt::Display for LockfileError { - fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { - f.write_str(&self.0) - } -} - -impl std::error::Error for LockfileError {} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct NpmPackageInfo { - pub integrity: String, - pub dependencies: BTreeMap, -} - -#[derive(Clone, Debug, Default, Serialize, Deserialize)] -pub struct NpmContent { - /// Mapping between requests for npm packages and resolved packages, eg. - /// { - /// "chalk": "chalk@5.0.0" - /// "react@17": "react@17.0.1" - /// "foo@latest": "foo@1.0.0" - /// } - pub specifiers: BTreeMap, - /// Mapping between resolved npm specifiers and their associated info, eg. - /// { - /// "chalk@5.0.0": { - /// "integrity": "sha512-...", - /// "dependencies": { - /// "ansi-styles": "ansi-styles@4.1.0", - /// } - /// } - /// } - pub packages: BTreeMap, -} - -impl NpmContent { - fn is_empty(&self) -> bool { - self.specifiers.is_empty() && self.packages.is_empty() - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct LockfileContent { - version: String, - // Mapping between URLs and their checksums for "http:" and "https:" deps - remote: BTreeMap, - #[serde(skip_serializing_if = "NpmContent::is_empty")] - #[serde(default)] - pub npm: NpmContent, -} - -impl LockfileContent { - fn empty() -> Self { - Self { - version: "2".to_string(), - remote: BTreeMap::new(), - npm: NpmContent::default(), - } - } -} - -#[derive(Debug, Clone)] -pub struct Lockfile { - pub overwrite: bool, - pub has_content_changed: bool, - pub content: LockfileContent, - pub filename: PathBuf, -} - -impl Lockfile { - pub fn discover( - flags: &Flags, - maybe_config_file: Option<&ConfigFile>, - ) -> Result, AnyError> { - if flags.no_lock - || matches!( - flags.subcommand, - DenoSubcommand::Install(_) | DenoSubcommand::Uninstall(_) - ) - { - return Ok(None); - } - - let filename = match flags.lock { - Some(ref lock) => PathBuf::from(lock), - None => match maybe_config_file { - Some(config_file) => { - if config_file.specifier.scheme() == "file" { - match config_file.to_lock_config()? { - Some(LockConfig::Bool(lock)) if !lock => { - return Ok(None); - } - Some(LockConfig::PathBuf(lock)) => config_file - .specifier - .to_file_path() - .unwrap() - .parent() - .unwrap() - .join(lock), - _ => { - let mut path = config_file.specifier.to_file_path().unwrap(); - path.set_file_name("deno.lock"); - path - } +pub use deno_lockfile::Lockfile; +pub use deno_lockfile::LockfileError; +use deno_lockfile::NpmPackageDependencyLockfileInfo; +use deno_lockfile::NpmPackageLockfileInfo; + +pub fn discover( + flags: &Flags, + maybe_config_file: Option<&ConfigFile>, +) -> Result, AnyError> { + if flags.no_lock + || matches!( + flags.subcommand, + DenoSubcommand::Install(_) | DenoSubcommand::Uninstall(_) + ) + { + return Ok(None); + } + + let filename = match flags.lock { + Some(ref lock) => PathBuf::from(lock), + None => match maybe_config_file { + Some(config_file) => { + if config_file.specifier.scheme() == "file" { + match config_file.to_lock_config()? { + Some(LockConfig::Bool(lock)) if !lock => { + return Ok(None); + } + Some(LockConfig::PathBuf(lock)) => config_file + .specifier + .to_file_path() + .unwrap() + .parent() + .unwrap() + .join(lock), + _ => { + let mut path = config_file.specifier.to_file_path().unwrap(); + path.set_file_name("deno.lock"); + path } - } else { - return Ok(None); } - } - None => return Ok(None), - }, - }; - - let lockfile = Self::new(filename, flags.lock_write)?; - Ok(Some(lockfile)) - } - - pub fn new(filename: PathBuf, overwrite: bool) -> Result { - // Writing a lock file always uses the new format. - if overwrite { - return Ok(Lockfile { - overwrite, - has_content_changed: false, - content: LockfileContent::empty(), - filename, - }); - } - - let result = match std::fs::read_to_string(&filename) { - Ok(content) => Ok(content), - Err(e) => { - if e.kind() == std::io::ErrorKind::NotFound { - return Ok(Lockfile { - overwrite, - has_content_changed: false, - content: LockfileContent::empty(), - filename, - }); } else { - Err(e) + return Ok(None); } } - }; - - let s = result.with_context(|| { - format!("Unable to read lockfile: \"{}\"", filename.display()) - })?; - let value: serde_json::Value = - serde_json::from_str(&s).with_context(|| { - format!( - "Unable to parse contents of the lockfile \"{}\"", - filename.display() - ) - })?; - let version = value.get("version").and_then(|v| v.as_str()); - let content = if version == Some("2") { - serde_json::from_value::(value).with_context(|| { - format!( - "Unable to parse contents of the lockfile \"{}\"", - filename.display() - ) - })? - } else { - // If there's no version field, we assume that user is using the old - // version of the lockfile. We'll migrate it in-place into v2 and it - // will be writte in v2 if user uses `--lock-write` flag. - let remote: BTreeMap = serde_json::from_value(value) - .with_context(|| { - format!( - "Unable to parse contents of the lockfile \"{}\"", - filename.display() - ) - })?; - LockfileContent { - version: "2".to_string(), - remote, - npm: NpmContent::default(), - } - }; - - Ok(Lockfile { - overwrite, - has_content_changed: false, - content, - filename, - }) - } - - // Synchronize lock file to disk - noop if --lock-write file is not specified. - pub fn write(&self) -> Result<(), AnyError> { - if !self.has_content_changed && !self.overwrite { - return Ok(()); - } - - let json_string = serde_json::to_string(&self.content).unwrap(); - let format_s = format_json(&json_string, &Default::default()) - .ok() - .flatten() - .unwrap_or(json_string); - let mut f = std::fs::OpenOptions::new() - .write(true) - .create(true) - .truncate(true) - .open(&self.filename)?; - f.write_all(format_s.as_bytes())?; - debug!("lockfile write {}", self.filename.display()); - Ok(()) - } - - // TODO(bartlomieju): this function should return an error instead of a bool, - // but it requires changes to `deno_graph`'s `Locker`. - pub fn check_or_insert_remote( - &mut self, - specifier: &str, - code: &str, - ) -> bool { - if !(specifier.starts_with("http:") || specifier.starts_with("https:")) { - return true; - } - if self.overwrite { - // In case --lock-write is specified check always passes - self.insert(specifier, code); - true - } else { - self.check_or_insert(specifier, code) - } - } - - pub fn check_or_insert_npm_package( - &mut self, - package: &NpmResolutionPackage, - ) -> Result<(), LockfileError> { - if self.overwrite { - // In case --lock-write is specified check always passes - self.insert_npm(package); - Ok(()) - } else { - self.check_or_insert_npm(package) - } - } - - /// Checks the given module is included, if so verify the checksum. If module - /// is not included, insert it. - fn check_or_insert(&mut self, specifier: &str, code: &str) -> bool { - if let Some(lockfile_checksum) = self.content.remote.get(specifier) { - let compiled_checksum = util::checksum::gen(&[code.as_bytes()]); - lockfile_checksum == &compiled_checksum - } else { - self.insert(specifier, code); - true - } - } - - fn insert(&mut self, specifier: &str, code: &str) { - let checksum = util::checksum::gen(&[code.as_bytes()]); - self.content.remote.insert(specifier.to_string(), checksum); - self.has_content_changed = true; - } - - fn check_or_insert_npm( - &mut self, - package: &NpmResolutionPackage, - ) -> Result<(), LockfileError> { - let specifier = package.id.as_serialized(); - if let Some(package_info) = self.content.npm.packages.get(&specifier) { - if package_info.integrity.as_str() != package.dist.integrity().as_str() { - return Err(LockfileError(format!( - "Integrity check failed for npm package: \"{}\". Unable to verify that the package -is the same as when the lockfile was generated. - -This could be caused by: - * the lock file may be corrupt - * the source itself may be corrupt - -Use \"--lock-write\" flag to regenerate the lockfile at \"{}\".", - package.id.display(), self.filename.display() - ))); - } - } else { - self.insert_npm(package); - } - - Ok(()) - } - - fn insert_npm(&mut self, package: &NpmResolutionPackage) { - let dependencies = package - .dependencies - .iter() - .map(|(name, id)| (name.to_string(), id.as_serialized())) - .collect::>(); + None => return Ok(None), + }, + }; - self.content.npm.packages.insert( - package.id.as_serialized(), - NpmPackageInfo { - integrity: package.dist.integrity().to_string(), - dependencies, - }, - ); - self.has_content_changed = true; - } - - pub fn insert_npm_specifier( - &mut self, - package_req: &NpmPackageReq, - package_id: &NpmPackageId, - ) { - self - .content - .npm - .specifiers - .insert(package_req.to_string(), package_id.as_serialized()); - self.has_content_changed = true; - } + let lockfile = Lockfile::new(filename, flags.lock_write)?; + Ok(Some(lockfile)) } -#[cfg(test)] -mod tests { - use super::*; - use crate::npm::NpmPackageId; - use crate::npm::NpmPackageVersionDistInfo; - use crate::npm::NpmVersion; - use deno_core::serde_json; - use deno_core::serde_json::json; - use std::collections::HashMap; - use std::fs::File; - use std::io::prelude::*; - use std::io::Write; - use test_util::TempDir; - - fn setup(temp_dir: &TempDir) -> PathBuf { - let file_path = temp_dir.path().join("valid_lockfile.json"); - let mut file = File::create(file_path).expect("write file fail"); - - let value: serde_json::Value = json!({ - "version": "2", - "remote": { - "https://deno.land/std@0.71.0/textproto/mod.ts": "3118d7a42c03c242c5a49c2ad91c8396110e14acca1324e7aaefd31a999b71a4", - "https://deno.land/std@0.71.0/async/delay.ts": "35957d585a6e3dd87706858fb1d6b551cb278271b03f52c5a2cb70e65e00c26a" - }, - "npm": { - "specifiers": {}, - "packages": { - "nanoid@3.3.4": { - "integrity": "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==", - "dependencies": {} - }, - "picocolors@1.0.0": { - "integrity": "sha512-foobar", - "dependencies": {} - }, - } - } - }); - - file.write_all(value.to_string().as_bytes()).unwrap(); - - temp_dir.path().join("valid_lockfile.json") - } - - #[test] - fn create_lockfile_for_nonexistent_path() { - let file_path = PathBuf::from("nonexistent_lock_file.json"); - assert!(Lockfile::new(file_path, false).is_ok()); - } - - #[test] - fn new_valid_lockfile() { - let temp_dir = TempDir::new(); - let file_path = setup(&temp_dir); - - let result = Lockfile::new(file_path, false).unwrap(); - - let remote = result.content.remote; - let keys: Vec = remote.keys().cloned().collect(); - let expected_keys = vec![ - String::from("https://deno.land/std@0.71.0/async/delay.ts"), - String::from("https://deno.land/std@0.71.0/textproto/mod.ts"), - ]; - - assert_eq!(keys.len(), 2); - assert_eq!(keys, expected_keys); - } - - #[test] - fn new_lockfile_from_file_and_insert() { - let temp_dir = TempDir::new(); - let file_path = setup(&temp_dir); - - let mut lockfile = Lockfile::new(file_path, false).unwrap(); - - lockfile.insert( - "https://deno.land/std@0.71.0/io/util.ts", - "Here is some source code", - ); - - let remote = lockfile.content.remote; - let keys: Vec = remote.keys().cloned().collect(); - let expected_keys = vec![ - String::from("https://deno.land/std@0.71.0/async/delay.ts"), - String::from("https://deno.land/std@0.71.0/io/util.ts"), - String::from("https://deno.land/std@0.71.0/textproto/mod.ts"), - ]; - assert_eq!(keys.len(), 3); - assert_eq!(keys, expected_keys); - } - - #[test] - fn new_lockfile_and_write() { - let temp_dir = TempDir::new(); - let file_path = setup(&temp_dir); - - let mut lockfile = Lockfile::new(file_path, true).unwrap(); - - lockfile.insert( - "https://deno.land/std@0.71.0/textproto/mod.ts", - "Here is some source code", - ); - lockfile.insert( - "https://deno.land/std@0.71.0/io/util.ts", - "more source code here", - ); - lockfile.insert( - "https://deno.land/std@0.71.0/async/delay.ts", - "this source is really exciting", - ); - - lockfile.write().expect("unable to write"); - - let file_path_buf = temp_dir.path().join("valid_lockfile.json"); - let file_path = file_path_buf.to_str().expect("file path fail").to_string(); - - // read the file contents back into a string and check - let mut checkfile = File::open(file_path).expect("Unable to open the file"); - let mut contents = String::new(); - checkfile - .read_to_string(&mut contents) - .expect("Unable to read the file"); - - let contents_json = - serde_json::from_str::(&contents).unwrap(); - let object = contents_json["remote"].as_object().unwrap(); - - assert_eq!( - object - .get("https://deno.land/std@0.71.0/textproto/mod.ts") - .and_then(|v| v.as_str()), - // sha-256 hash of the source 'Here is some source code' - Some("fedebba9bb82cce293196f54b21875b649e457f0eaf55556f1e318204947a28f") - ); - - // confirm that keys are sorted alphabetically - let mut keys = object.keys().map(|k| k.as_str()); - assert_eq!( - keys.next(), - Some("https://deno.land/std@0.71.0/async/delay.ts") - ); - assert_eq!(keys.next(), Some("https://deno.land/std@0.71.0/io/util.ts")); - assert_eq!( - keys.next(), - Some("https://deno.land/std@0.71.0/textproto/mod.ts") - ); - assert!(keys.next().is_none()); - } - - #[test] - fn check_or_insert_lockfile() { - let temp_dir = TempDir::new(); - let file_path = setup(&temp_dir); - - let mut lockfile = Lockfile::new(file_path, false).unwrap(); - - lockfile.insert( - "https://deno.land/std@0.71.0/textproto/mod.ts", - "Here is some source code", - ); - - let check_true = lockfile.check_or_insert_remote( - "https://deno.land/std@0.71.0/textproto/mod.ts", - "Here is some source code", - ); - assert!(check_true); - - let check_false = lockfile.check_or_insert_remote( - "https://deno.land/std@0.71.0/textproto/mod.ts", - "Here is some NEW source code", - ); - assert!(!check_false); - - // Not present in lockfile yet, should be inserted and check passed. - let check_true = lockfile.check_or_insert_remote( - "https://deno.land/std@0.71.0/http/file_server.ts", - "This is new Source code", - ); - assert!(check_true); - } - - #[test] - fn check_or_insert_lockfile_npm() { - let temp_dir = TempDir::new(); - let file_path = setup(&temp_dir); - - let mut lockfile = Lockfile::new(file_path, false).unwrap(); - - let npm_package = NpmResolutionPackage { - id: NpmPackageId { - name: "nanoid".to_string(), - version: NpmVersion::parse("3.3.4").unwrap(), - peer_dependencies: Vec::new(), - }, - copy_index: 0, - dist: NpmPackageVersionDistInfo::new( - "foo".to_string(), - "shasum".to_string(), - Some("sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==".to_string()), - ), - dependencies: HashMap::new(), - }; - let check_ok = lockfile.check_or_insert_npm_package(&npm_package); - assert!(check_ok.is_ok()); - - let npm_package = NpmResolutionPackage { - id: NpmPackageId { - name: "picocolors".to_string(), - version: NpmVersion::parse("1.0.0").unwrap(), - peer_dependencies: Vec::new(), - }, - copy_index: 0, - dist: NpmPackageVersionDistInfo::new( - "foo".to_string(), - "shasum".to_string(), - Some("sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==".to_string()), - ), - dependencies: HashMap::new(), - }; - // Integrity is borked in the loaded lockfile - let check_err = lockfile.check_or_insert_npm_package(&npm_package); - assert!(check_err.is_err()); - - let npm_package = NpmResolutionPackage { - id: NpmPackageId { - name: "source-map-js".to_string(), - version: NpmVersion::parse("1.0.2").unwrap(), - peer_dependencies: Vec::new(), - }, - copy_index: 0, - dist: NpmPackageVersionDistInfo::new( - "foo".to_string(), - "foo".to_string(), - Some("sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==".to_string()), - ), - dependencies: HashMap::new(), - }; - // Not present in lockfile yet, should be inserted and check passed. - let check_ok = lockfile.check_or_insert_npm_package(&npm_package); - assert!(check_ok.is_ok()); - - let npm_package = NpmResolutionPackage { - id: NpmPackageId { - name: "source-map-js".to_string(), - version: NpmVersion::parse("1.0.2").unwrap(), - peer_dependencies: Vec::new(), - }, - copy_index: 0, - dist: NpmPackageVersionDistInfo::new( - "foo".to_string(), - "foo".to_string(), - Some("sha512-foobar".to_string()), - ), - dependencies: HashMap::new(), - }; - // Now present in lockfile, should file due to borked integrity - let check_err = lockfile.check_or_insert_npm_package(&npm_package); - assert!(check_err.is_err()); +// NOTE(bartlomieju): we don't want a reverse mapping to be possible. +#[allow(clippy::from_over_into)] +impl Into for NpmResolutionPackage { + fn into(self) -> NpmPackageLockfileInfo { + let dependencies = self + .dependencies + .into_iter() + .map(|(name, id)| NpmPackageDependencyLockfileInfo { + name, + id: id.as_serialized(), + }) + .collect(); + + NpmPackageLockfileInfo { + display_id: self.id.display(), + serialized_id: self.id.as_serialized(), + integrity: self.dist.integrity().to_string(), + dependencies, + } } } diff --git a/cli/args/mod.rs b/cli/args/mod.rs index b604d3ab5..a81c84b59 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -502,7 +502,7 @@ impl CliOptions { pub fn from_flags(flags: Flags) -> Result { let maybe_config_file = ConfigFile::discover(&flags)?; let maybe_lock_file = - Lockfile::discover(&flags, maybe_config_file.as_ref())?; + lockfile::discover(&flags, maybe_config_file.as_ref())?; Ok(Self::new(flags, maybe_config_file, maybe_lock_file)) } diff --git a/cli/npm/registry.rs b/cli/npm/registry.rs index 0b35079de..97397350d 100644 --- a/cli/npm/registry.rs +++ b/cli/npm/registry.rs @@ -184,19 +184,6 @@ pub struct NpmPackageVersionDistInfo { } impl NpmPackageVersionDistInfo { - #[cfg(test)] - pub fn new( - tarball: String, - shasum: String, - integrity: Option, - ) -> Self { - Self { - tarball, - shasum, - integrity, - } - } - pub fn integrity(&self) -> Cow { self .integrity diff --git a/cli/npm/resolution/mod.rs b/cli/npm/resolution/mod.rs index c4d05598c..f10678cbe 100644 --- a/cli/npm/resolution/mod.rs +++ b/cli/npm/resolution/mod.rs @@ -397,10 +397,13 @@ impl NpmResolution { pub fn lock(&self, lockfile: &mut Lockfile) -> Result<(), AnyError> { let snapshot = self.snapshot.read(); for (package_req, package_id) in snapshot.package_reqs.iter() { - lockfile.insert_npm_specifier(package_req, package_id); + lockfile.insert_npm_specifier( + package_req.to_string(), + package_id.as_serialized(), + ); } for package in snapshot.all_packages() { - lockfile.check_or_insert_npm_package(&package)?; + lockfile.check_or_insert_npm_package(package.into())?; } Ok(()) } diff --git a/cli/tests/integration/npm_tests.rs b/cli/tests/integration/npm_tests.rs index f2bbbcd3c..36559b706 100644 --- a/cli/tests/integration/npm_tests.rs +++ b/cli/tests/integration/npm_tests.rs @@ -1116,7 +1116,9 @@ fn lock_file_lock_write() { "version": "2", "remote": {}, "npm": { - "specifiers": { "cowsay@1.5.0": "cowsay@1.5.0" }, + "specifiers": { + "cowsay@1.5.0": "cowsay@1.5.0" + }, "packages": { "ansi-regex@3.0.1": { "integrity": "sha512-+O9Jct8wf++lXxxFc4hc8LsjaSq0HFzzL7cVsw8pRDIPdjKD2mT4ytDZlLuSBZ4cLKZFXIrMGO7DbQCtMJJMKw==", @@ -1128,7 +1130,9 @@ fn lock_file_lock_write() { }, "ansi-styles@4.3.0": { "integrity": "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg==", - "dependencies": { "color-convert": "color-convert@2.0.1" } + "dependencies": { + "color-convert": "color-convert@2.0.1" + } }, "camelcase@5.3.1": { "integrity": "sha512-L28STB170nwWS63UjtlEOE3dldQApaJXZkOI1uMFfzf3rRuPegHaHesyee+YxQ+W6SvRDQV6UrdOdRiR153wJg==", @@ -1144,7 +1148,9 @@ fn lock_file_lock_write() { }, "color-convert@2.0.1": { "integrity": "sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ==", - "dependencies": { "color-name": "color-name@1.1.4" } + "dependencies": { + "color-name": "color-name@1.1.4" + } }, "color-name@1.1.4": { "integrity": "sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA==", @@ -1192,15 +1198,21 @@ fn lock_file_lock_write() { }, "locate-path@5.0.0": { "integrity": "sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==", - "dependencies": { "p-locate": "p-locate@4.1.0" } + "dependencies": { + "p-locate": "p-locate@4.1.0" + } }, "p-limit@2.3.0": { "integrity": "sha512-//88mFWSJx8lxCzwdAABTJL2MyWB12+eIY7MDL2SqLmAkeKU9qxRvWuSyTjm3FUmpBEMuFfckAIqEaVGUDxb6w==", - "dependencies": { "p-try": "p-try@2.2.0" } + "dependencies": { + "p-try": "p-try@2.2.0" + } }, "p-locate@4.1.0": { "integrity": "sha512-R79ZZ/0wAxKGu3oYMlz8jy/kbhsNrS7SKZ7PxEHBgJ5+F2mtFW2fK2cOtBh1cHYkQsbzFV7I+EoRKe6Yt0oK7A==", - "dependencies": { "p-limit": "p-limit@2.3.0" } + "dependencies": { + "p-limit": "p-limit@2.3.0" + } }, "p-try@2.2.0": { "integrity": "sha512-R4nPAVTAU0B9D35/Gk3uJf/7XYbQcyohSKdvAxIRSNghFl4e71hVoGnBNQz9cWaXxO2I10KTC+3jMdvvoKw6dQ==", @@ -1239,11 +1251,15 @@ fn lock_file_lock_write() { }, "strip-ansi@4.0.0": { "integrity": "sha512-4XaJ2zQdCzROZDivEVIDPkcQn8LMFSa8kj8Gxb/Lnwzv9A8VctNZ+lfivC/sV3ivW8ElJTERXZoPBRrZKkNKow==", - "dependencies": { "ansi-regex": "ansi-regex@3.0.1" } + "dependencies": { + "ansi-regex": "ansi-regex@3.0.1" + } }, "strip-ansi@6.0.1": { "integrity": "sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A==", - "dependencies": { "ansi-regex": "ansi-regex@5.0.1" } + "dependencies": { + "ansi-regex": "ansi-regex@5.0.1" + } }, "strip-final-newline@2.0.0": { "integrity": "sha512-BrpvfNAE3dcvq7ll3xVumzjKjZQ5tI1sEUIKr3Uoks0XUl45St3FlatVqef9prk4jRDzhW6WZg+3bk93y6pLjA==", @@ -1290,8 +1306,7 @@ fn lock_file_lock_write() { } } } -} -"#; +}"#; temp_dir.write("deno.lock", lock_file_content); let deno = util::deno_cmd_with_deno_dir(&deno_dir) diff --git a/lockfile/Cargo.toml b/lockfile/Cargo.toml new file mode 100644 index 000000000..8e6843285 --- /dev/null +++ b/lockfile/Cargo.toml @@ -0,0 +1,20 @@ +# Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +[package] +name = "deno_lockfile" +version = "0.1.0" +edition = "2021" +license = "MIT" +description = "An implementation of a lockfile used in Deno" + +[lib] +path = "lib.rs" + +[dependencies] +anyhow.workspace = true +ring.workspace = true +serde.workspace = true +serde_json.workspace = true + +[dev-dependencies] +test_util.workspace = true diff --git a/lockfile/README.md b/lockfile/README.md new file mode 100644 index 000000000..54d113206 --- /dev/null +++ b/lockfile/README.md @@ -0,0 +1,3 @@ +# `deno_lockfile` + +This crate implements the lockfile format used by Deno. diff --git a/lockfile/lib.rs b/lockfile/lib.rs new file mode 100644 index 000000000..eb1009e8a --- /dev/null +++ b/lockfile/lib.rs @@ -0,0 +1,530 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +use std::collections::BTreeMap; +use std::io::Write; + +use anyhow::Context; +use anyhow::Error as AnyError; +use ring::digest; +use serde::Deserialize; +use serde::Serialize; +use std::path::PathBuf; + +pub struct NpmPackageLockfileInfo { + pub display_id: String, + pub serialized_id: String, + pub integrity: String, + pub dependencies: Vec, +} + +pub struct NpmPackageDependencyLockfileInfo { + pub name: String, + pub id: String, +} + +fn gen_checksum(v: &[impl AsRef<[u8]>]) -> String { + let mut ctx = digest::Context::new(&digest::SHA256); + for src in v { + ctx.update(src.as_ref()); + } + let digest = ctx.finish(); + let out: Vec = digest + .as_ref() + .iter() + .map(|byte| format!("{:02x}", byte)) + .collect(); + out.join("") +} + +#[derive(Debug)] +pub struct LockfileError(String); + +impl std::fmt::Display for LockfileError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for LockfileError {} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NpmPackageInfo { + pub integrity: String, + pub dependencies: BTreeMap, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct NpmContent { + /// Mapping between requests for npm packages and resolved packages, eg. + /// { + /// "chalk": "chalk@5.0.0" + /// "react@17": "react@17.0.1" + /// "foo@latest": "foo@1.0.0" + /// } + pub specifiers: BTreeMap, + /// Mapping between resolved npm specifiers and their associated info, eg. + /// { + /// "chalk@5.0.0": { + /// "integrity": "sha512-...", + /// "dependencies": { + /// "ansi-styles": "ansi-styles@4.1.0", + /// } + /// } + /// } + pub packages: BTreeMap, +} + +impl NpmContent { + fn is_empty(&self) -> bool { + self.specifiers.is_empty() && self.packages.is_empty() + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LockfileContent { + version: String, + // Mapping between URLs and their checksums for "http:" and "https:" deps + remote: BTreeMap, + #[serde(skip_serializing_if = "NpmContent::is_empty")] + #[serde(default)] + pub npm: NpmContent, +} + +impl LockfileContent { + fn empty() -> Self { + Self { + version: "2".to_string(), + remote: BTreeMap::new(), + npm: NpmContent::default(), + } + } +} + +#[derive(Debug, Clone)] +pub struct Lockfile { + pub overwrite: bool, + pub has_content_changed: bool, + pub content: LockfileContent, + pub filename: PathBuf, +} + +impl Lockfile { + pub fn new(filename: PathBuf, overwrite: bool) -> Result { + // Writing a lock file always uses the new format. + if overwrite { + return Ok(Lockfile { + overwrite, + has_content_changed: false, + content: LockfileContent::empty(), + filename, + }); + } + + let result = match std::fs::read_to_string(&filename) { + Ok(content) => Ok(content), + Err(e) => { + if e.kind() == std::io::ErrorKind::NotFound { + return Ok(Lockfile { + overwrite, + has_content_changed: false, + content: LockfileContent::empty(), + filename, + }); + } else { + Err(e) + } + } + }; + + let s = result.with_context(|| { + format!("Unable to read lockfile: \"{}\"", filename.display()) + })?; + let value: serde_json::Value = + serde_json::from_str(&s).with_context(|| { + format!( + "Unable to parse contents of the lockfile \"{}\"", + filename.display() + ) + })?; + let version = value.get("version").and_then(|v| v.as_str()); + let content = if version == Some("2") { + serde_json::from_value::(value).with_context(|| { + format!( + "Unable to parse contents of the lockfile \"{}\"", + filename.display() + ) + })? + } else { + // If there's no version field, we assume that user is using the old + // version of the lockfile. We'll migrate it in-place into v2 and it + // will be writte in v2 if user uses `--lock-write` flag. + let remote: BTreeMap = serde_json::from_value(value) + .with_context(|| { + format!( + "Unable to parse contents of the lockfile \"{}\"", + filename.display() + ) + })?; + LockfileContent { + version: "2".to_string(), + remote, + npm: NpmContent::default(), + } + }; + + Ok(Lockfile { + overwrite, + has_content_changed: false, + content, + filename, + }) + } + + // Synchronize lock file to disk - noop if --lock-write file is not specified. + pub fn write(&self) -> Result<(), AnyError> { + if !self.has_content_changed && !self.overwrite { + return Ok(()); + } + + let json_string = serde_json::to_string_pretty(&self.content).unwrap(); + let mut f = std::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(&self.filename)?; + f.write_all(json_string.as_bytes())?; + Ok(()) + } + + // TODO(bartlomieju): this function should return an error instead of a bool, + // but it requires changes to `deno_graph`'s `Locker`. + pub fn check_or_insert_remote( + &mut self, + specifier: &str, + code: &str, + ) -> bool { + if !(specifier.starts_with("http:") || specifier.starts_with("https:")) { + return true; + } + if self.overwrite { + // In case --lock-write is specified check always passes + self.insert(specifier, code); + true + } else { + self.check_or_insert(specifier, code) + } + } + + pub fn check_or_insert_npm_package( + &mut self, + package_info: NpmPackageLockfileInfo, + ) -> Result<(), LockfileError> { + if self.overwrite { + // In case --lock-write is specified check always passes + self.insert_npm(package_info); + Ok(()) + } else { + self.check_or_insert_npm(package_info) + } + } + + /// Checks the given module is included, if so verify the checksum. If module + /// is not included, insert it. + fn check_or_insert(&mut self, specifier: &str, code: &str) -> bool { + if let Some(lockfile_checksum) = self.content.remote.get(specifier) { + let compiled_checksum = gen_checksum(&[code.as_bytes()]); + lockfile_checksum == &compiled_checksum + } else { + self.insert(specifier, code); + true + } + } + + fn insert(&mut self, specifier: &str, code: &str) { + let checksum = gen_checksum(&[code.as_bytes()]); + self.content.remote.insert(specifier.to_string(), checksum); + self.has_content_changed = true; + } + + fn check_or_insert_npm( + &mut self, + package: NpmPackageLockfileInfo, + ) -> Result<(), LockfileError> { + if let Some(package_info) = + self.content.npm.packages.get(&package.serialized_id) + { + if package_info.integrity.as_str() != package.integrity { + return Err(LockfileError(format!( + "Integrity check failed for npm package: \"{}\". Unable to verify that the package +is the same as when the lockfile was generated. + +This could be caused by: + * the lock file may be corrupt + * the source itself may be corrupt + +Use \"--lock-write\" flag to regenerate the lockfile at \"{}\".", + package.display_id, self.filename.display() + ))); + } + } else { + self.insert_npm(package); + } + + Ok(()) + } + + fn insert_npm(&mut self, package_info: NpmPackageLockfileInfo) { + let dependencies = package_info + .dependencies + .iter() + .map(|dep| (dep.name.to_string(), dep.id.to_string())) + .collect::>(); + + self.content.npm.packages.insert( + package_info.serialized_id.to_string(), + NpmPackageInfo { + integrity: package_info.integrity, + dependencies, + }, + ); + self.has_content_changed = true; + } + + pub fn insert_npm_specifier( + &mut self, + serialized_package_req: String, + serialized_package_id: String, + ) { + self + .content + .npm + .specifiers + .insert(serialized_package_req, serialized_package_id); + self.has_content_changed = true; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use std::fs::File; + use std::io::prelude::*; + use std::io::Write; + use test_util::TempDir; + + fn setup(temp_dir: &TempDir) -> PathBuf { + let file_path = temp_dir.path().join("valid_lockfile.json"); + let mut file = File::create(file_path).expect("write file fail"); + + let value: serde_json::Value = json!({ + "version": "2", + "remote": { + "https://deno.land/std@0.71.0/textproto/mod.ts": "3118d7a42c03c242c5a49c2ad91c8396110e14acca1324e7aaefd31a999b71a4", + "https://deno.land/std@0.71.0/async/delay.ts": "35957d585a6e3dd87706858fb1d6b551cb278271b03f52c5a2cb70e65e00c26a" + }, + "npm": { + "specifiers": {}, + "packages": { + "nanoid@3.3.4": { + "integrity": "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==", + "dependencies": {} + }, + "picocolors@1.0.0": { + "integrity": "sha512-foobar", + "dependencies": {} + }, + } + } + }); + + file.write_all(value.to_string().as_bytes()).unwrap(); + + temp_dir.path().join("valid_lockfile.json") + } + + #[test] + fn create_lockfile_for_nonexistent_path() { + let file_path = PathBuf::from("nonexistent_lock_file.json"); + assert!(Lockfile::new(file_path, false).is_ok()); + } + + #[test] + fn new_valid_lockfile() { + let temp_dir = TempDir::new(); + let file_path = setup(&temp_dir); + + let result = Lockfile::new(file_path, false).unwrap(); + + let remote = result.content.remote; + let keys: Vec = remote.keys().cloned().collect(); + let expected_keys = vec![ + String::from("https://deno.land/std@0.71.0/async/delay.ts"), + String::from("https://deno.land/std@0.71.0/textproto/mod.ts"), + ]; + + assert_eq!(keys.len(), 2); + assert_eq!(keys, expected_keys); + } + + #[test] + fn new_lockfile_from_file_and_insert() { + let temp_dir = TempDir::new(); + let file_path = setup(&temp_dir); + + let mut lockfile = Lockfile::new(file_path, false).unwrap(); + + lockfile.insert( + "https://deno.land/std@0.71.0/io/util.ts", + "Here is some source code", + ); + + let remote = lockfile.content.remote; + let keys: Vec = remote.keys().cloned().collect(); + let expected_keys = vec![ + String::from("https://deno.land/std@0.71.0/async/delay.ts"), + String::from("https://deno.land/std@0.71.0/io/util.ts"), + String::from("https://deno.land/std@0.71.0/textproto/mod.ts"), + ]; + assert_eq!(keys.len(), 3); + assert_eq!(keys, expected_keys); + } + + #[test] + fn new_lockfile_and_write() { + let temp_dir = TempDir::new(); + let file_path = setup(&temp_dir); + + let mut lockfile = Lockfile::new(file_path, true).unwrap(); + + lockfile.insert( + "https://deno.land/std@0.71.0/textproto/mod.ts", + "Here is some source code", + ); + lockfile.insert( + "https://deno.land/std@0.71.0/io/util.ts", + "more source code here", + ); + lockfile.insert( + "https://deno.land/std@0.71.0/async/delay.ts", + "this source is really exciting", + ); + + lockfile.write().expect("unable to write"); + + let file_path_buf = temp_dir.path().join("valid_lockfile.json"); + let file_path = file_path_buf.to_str().expect("file path fail").to_string(); + + // read the file contents back into a string and check + let mut checkfile = File::open(file_path).expect("Unable to open the file"); + let mut contents = String::new(); + checkfile + .read_to_string(&mut contents) + .expect("Unable to read the file"); + + let contents_json = + serde_json::from_str::(&contents).unwrap(); + let object = contents_json["remote"].as_object().unwrap(); + + assert_eq!( + object + .get("https://deno.land/std@0.71.0/textproto/mod.ts") + .and_then(|v| v.as_str()), + // sha-256 hash of the source 'Here is some source code' + Some("fedebba9bb82cce293196f54b21875b649e457f0eaf55556f1e318204947a28f") + ); + + // confirm that keys are sorted alphabetically + let mut keys = object.keys().map(|k| k.as_str()); + assert_eq!( + keys.next(), + Some("https://deno.land/std@0.71.0/async/delay.ts") + ); + assert_eq!(keys.next(), Some("https://deno.land/std@0.71.0/io/util.ts")); + assert_eq!( + keys.next(), + Some("https://deno.land/std@0.71.0/textproto/mod.ts") + ); + assert!(keys.next().is_none()); + } + + #[test] + fn check_or_insert_lockfile() { + let temp_dir = TempDir::new(); + let file_path = setup(&temp_dir); + + let mut lockfile = Lockfile::new(file_path, false).unwrap(); + + lockfile.insert( + "https://deno.land/std@0.71.0/textproto/mod.ts", + "Here is some source code", + ); + + let check_true = lockfile.check_or_insert_remote( + "https://deno.land/std@0.71.0/textproto/mod.ts", + "Here is some source code", + ); + assert!(check_true); + + let check_false = lockfile.check_or_insert_remote( + "https://deno.land/std@0.71.0/textproto/mod.ts", + "Here is some NEW source code", + ); + assert!(!check_false); + + // Not present in lockfile yet, should be inserted and check passed. + let check_true = lockfile.check_or_insert_remote( + "https://deno.land/std@0.71.0/http/file_server.ts", + "This is new Source code", + ); + assert!(check_true); + } + + #[test] + fn check_or_insert_lockfile_npm() { + let temp_dir = TempDir::new(); + let file_path = setup(&temp_dir); + + let mut lockfile = Lockfile::new(file_path, false).unwrap(); + + let npm_package = NpmPackageLockfileInfo { + display_id: "nanoid@3.3.4".to_string(), + serialized_id: "nanoid@3.3.4".to_string(), + integrity: "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==".to_string(), + dependencies: vec![], + }; + let check_ok = lockfile.check_or_insert_npm_package(npm_package); + assert!(check_ok.is_ok()); + + let npm_package = NpmPackageLockfileInfo { + display_id: "picocolors@1.0.0".to_string(), + serialized_id: "picocolors@1.0.0".to_string(), + integrity: "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==".to_string(), + dependencies: vec![], + }; + // Integrity is borked in the loaded lockfile + let check_err = lockfile.check_or_insert_npm_package(npm_package); + assert!(check_err.is_err()); + + let npm_package = NpmPackageLockfileInfo { + display_id: "source-map-js@1.0.2".to_string(), + serialized_id: "source-map-js@1.0.2".to_string(), + integrity: "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==".to_string(), + dependencies: vec![], + }; + // Not present in lockfile yet, should be inserted and check passed. + let check_ok = lockfile.check_or_insert_npm_package(npm_package); + assert!(check_ok.is_ok()); + + let npm_package = NpmPackageLockfileInfo { + display_id: "source-map-js@1.0.2".to_string(), + serialized_id: "source-map-js@1.0.2".to_string(), + integrity: "sha512-foobar".to_string(), + dependencies: vec![], + }; + // Now present in lockfile, should file due to borked integrity + let check_err = lockfile.check_or_insert_npm_package(npm_package); + assert!(check_err.is_err()); + } +} -- cgit v1.2.3