From bf237c6241f53122e37341a0dda65ef9e3b51a49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Mon, 23 Jan 2023 23:41:02 +0100 Subject: refactor: Move lockfile to a separate crate (#17503) Moves the lockfile implementation to a separate crate so other projects like Deploy can use it as well. --- lockfile/lib.rs | 530 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 530 insertions(+) create mode 100644 lockfile/lib.rs (limited to 'lockfile/lib.rs') diff --git a/lockfile/lib.rs b/lockfile/lib.rs new file mode 100644 index 000000000..eb1009e8a --- /dev/null +++ b/lockfile/lib.rs @@ -0,0 +1,530 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +use std::collections::BTreeMap; +use std::io::Write; + +use anyhow::Context; +use anyhow::Error as AnyError; +use ring::digest; +use serde::Deserialize; +use serde::Serialize; +use std::path::PathBuf; + +pub struct NpmPackageLockfileInfo { + pub display_id: String, + pub serialized_id: String, + pub integrity: String, + pub dependencies: Vec, +} + +pub struct NpmPackageDependencyLockfileInfo { + pub name: String, + pub id: String, +} + +fn gen_checksum(v: &[impl AsRef<[u8]>]) -> String { + let mut ctx = digest::Context::new(&digest::SHA256); + for src in v { + ctx.update(src.as_ref()); + } + let digest = ctx.finish(); + let out: Vec = digest + .as_ref() + .iter() + .map(|byte| format!("{:02x}", byte)) + .collect(); + out.join("") +} + +#[derive(Debug)] +pub struct LockfileError(String); + +impl std::fmt::Display for LockfileError { + fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str(&self.0) + } +} + +impl std::error::Error for LockfileError {} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NpmPackageInfo { + pub integrity: String, + pub dependencies: BTreeMap, +} + +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct NpmContent { + /// Mapping between requests for npm packages and resolved packages, eg. + /// { + /// "chalk": "chalk@5.0.0" + /// "react@17": "react@17.0.1" + /// "foo@latest": "foo@1.0.0" + /// } + pub specifiers: BTreeMap, + /// Mapping between resolved npm specifiers and their associated info, eg. + /// { + /// "chalk@5.0.0": { + /// "integrity": "sha512-...", + /// "dependencies": { + /// "ansi-styles": "ansi-styles@4.1.0", + /// } + /// } + /// } + pub packages: BTreeMap, +} + +impl NpmContent { + fn is_empty(&self) -> bool { + self.specifiers.is_empty() && self.packages.is_empty() + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LockfileContent { + version: String, + // Mapping between URLs and their checksums for "http:" and "https:" deps + remote: BTreeMap, + #[serde(skip_serializing_if = "NpmContent::is_empty")] + #[serde(default)] + pub npm: NpmContent, +} + +impl LockfileContent { + fn empty() -> Self { + Self { + version: "2".to_string(), + remote: BTreeMap::new(), + npm: NpmContent::default(), + } + } +} + +#[derive(Debug, Clone)] +pub struct Lockfile { + pub overwrite: bool, + pub has_content_changed: bool, + pub content: LockfileContent, + pub filename: PathBuf, +} + +impl Lockfile { + pub fn new(filename: PathBuf, overwrite: bool) -> Result { + // Writing a lock file always uses the new format. + if overwrite { + return Ok(Lockfile { + overwrite, + has_content_changed: false, + content: LockfileContent::empty(), + filename, + }); + } + + let result = match std::fs::read_to_string(&filename) { + Ok(content) => Ok(content), + Err(e) => { + if e.kind() == std::io::ErrorKind::NotFound { + return Ok(Lockfile { + overwrite, + has_content_changed: false, + content: LockfileContent::empty(), + filename, + }); + } else { + Err(e) + } + } + }; + + let s = result.with_context(|| { + format!("Unable to read lockfile: \"{}\"", filename.display()) + })?; + let value: serde_json::Value = + serde_json::from_str(&s).with_context(|| { + format!( + "Unable to parse contents of the lockfile \"{}\"", + filename.display() + ) + })?; + let version = value.get("version").and_then(|v| v.as_str()); + let content = if version == Some("2") { + serde_json::from_value::(value).with_context(|| { + format!( + "Unable to parse contents of the lockfile \"{}\"", + filename.display() + ) + })? + } else { + // If there's no version field, we assume that user is using the old + // version of the lockfile. We'll migrate it in-place into v2 and it + // will be writte in v2 if user uses `--lock-write` flag. + let remote: BTreeMap = serde_json::from_value(value) + .with_context(|| { + format!( + "Unable to parse contents of the lockfile \"{}\"", + filename.display() + ) + })?; + LockfileContent { + version: "2".to_string(), + remote, + npm: NpmContent::default(), + } + }; + + Ok(Lockfile { + overwrite, + has_content_changed: false, + content, + filename, + }) + } + + // Synchronize lock file to disk - noop if --lock-write file is not specified. + pub fn write(&self) -> Result<(), AnyError> { + if !self.has_content_changed && !self.overwrite { + return Ok(()); + } + + let json_string = serde_json::to_string_pretty(&self.content).unwrap(); + let mut f = std::fs::OpenOptions::new() + .write(true) + .create(true) + .truncate(true) + .open(&self.filename)?; + f.write_all(json_string.as_bytes())?; + Ok(()) + } + + // TODO(bartlomieju): this function should return an error instead of a bool, + // but it requires changes to `deno_graph`'s `Locker`. + pub fn check_or_insert_remote( + &mut self, + specifier: &str, + code: &str, + ) -> bool { + if !(specifier.starts_with("http:") || specifier.starts_with("https:")) { + return true; + } + if self.overwrite { + // In case --lock-write is specified check always passes + self.insert(specifier, code); + true + } else { + self.check_or_insert(specifier, code) + } + } + + pub fn check_or_insert_npm_package( + &mut self, + package_info: NpmPackageLockfileInfo, + ) -> Result<(), LockfileError> { + if self.overwrite { + // In case --lock-write is specified check always passes + self.insert_npm(package_info); + Ok(()) + } else { + self.check_or_insert_npm(package_info) + } + } + + /// Checks the given module is included, if so verify the checksum. If module + /// is not included, insert it. + fn check_or_insert(&mut self, specifier: &str, code: &str) -> bool { + if let Some(lockfile_checksum) = self.content.remote.get(specifier) { + let compiled_checksum = gen_checksum(&[code.as_bytes()]); + lockfile_checksum == &compiled_checksum + } else { + self.insert(specifier, code); + true + } + } + + fn insert(&mut self, specifier: &str, code: &str) { + let checksum = gen_checksum(&[code.as_bytes()]); + self.content.remote.insert(specifier.to_string(), checksum); + self.has_content_changed = true; + } + + fn check_or_insert_npm( + &mut self, + package: NpmPackageLockfileInfo, + ) -> Result<(), LockfileError> { + if let Some(package_info) = + self.content.npm.packages.get(&package.serialized_id) + { + if package_info.integrity.as_str() != package.integrity { + return Err(LockfileError(format!( + "Integrity check failed for npm package: \"{}\". Unable to verify that the package +is the same as when the lockfile was generated. + +This could be caused by: + * the lock file may be corrupt + * the source itself may be corrupt + +Use \"--lock-write\" flag to regenerate the lockfile at \"{}\".", + package.display_id, self.filename.display() + ))); + } + } else { + self.insert_npm(package); + } + + Ok(()) + } + + fn insert_npm(&mut self, package_info: NpmPackageLockfileInfo) { + let dependencies = package_info + .dependencies + .iter() + .map(|dep| (dep.name.to_string(), dep.id.to_string())) + .collect::>(); + + self.content.npm.packages.insert( + package_info.serialized_id.to_string(), + NpmPackageInfo { + integrity: package_info.integrity, + dependencies, + }, + ); + self.has_content_changed = true; + } + + pub fn insert_npm_specifier( + &mut self, + serialized_package_req: String, + serialized_package_id: String, + ) { + self + .content + .npm + .specifiers + .insert(serialized_package_req, serialized_package_id); + self.has_content_changed = true; + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + use std::fs::File; + use std::io::prelude::*; + use std::io::Write; + use test_util::TempDir; + + fn setup(temp_dir: &TempDir) -> PathBuf { + let file_path = temp_dir.path().join("valid_lockfile.json"); + let mut file = File::create(file_path).expect("write file fail"); + + let value: serde_json::Value = json!({ + "version": "2", + "remote": { + "https://deno.land/std@0.71.0/textproto/mod.ts": "3118d7a42c03c242c5a49c2ad91c8396110e14acca1324e7aaefd31a999b71a4", + "https://deno.land/std@0.71.0/async/delay.ts": "35957d585a6e3dd87706858fb1d6b551cb278271b03f52c5a2cb70e65e00c26a" + }, + "npm": { + "specifiers": {}, + "packages": { + "nanoid@3.3.4": { + "integrity": "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==", + "dependencies": {} + }, + "picocolors@1.0.0": { + "integrity": "sha512-foobar", + "dependencies": {} + }, + } + } + }); + + file.write_all(value.to_string().as_bytes()).unwrap(); + + temp_dir.path().join("valid_lockfile.json") + } + + #[test] + fn create_lockfile_for_nonexistent_path() { + let file_path = PathBuf::from("nonexistent_lock_file.json"); + assert!(Lockfile::new(file_path, false).is_ok()); + } + + #[test] + fn new_valid_lockfile() { + let temp_dir = TempDir::new(); + let file_path = setup(&temp_dir); + + let result = Lockfile::new(file_path, false).unwrap(); + + let remote = result.content.remote; + let keys: Vec = remote.keys().cloned().collect(); + let expected_keys = vec![ + String::from("https://deno.land/std@0.71.0/async/delay.ts"), + String::from("https://deno.land/std@0.71.0/textproto/mod.ts"), + ]; + + assert_eq!(keys.len(), 2); + assert_eq!(keys, expected_keys); + } + + #[test] + fn new_lockfile_from_file_and_insert() { + let temp_dir = TempDir::new(); + let file_path = setup(&temp_dir); + + let mut lockfile = Lockfile::new(file_path, false).unwrap(); + + lockfile.insert( + "https://deno.land/std@0.71.0/io/util.ts", + "Here is some source code", + ); + + let remote = lockfile.content.remote; + let keys: Vec = remote.keys().cloned().collect(); + let expected_keys = vec![ + String::from("https://deno.land/std@0.71.0/async/delay.ts"), + String::from("https://deno.land/std@0.71.0/io/util.ts"), + String::from("https://deno.land/std@0.71.0/textproto/mod.ts"), + ]; + assert_eq!(keys.len(), 3); + assert_eq!(keys, expected_keys); + } + + #[test] + fn new_lockfile_and_write() { + let temp_dir = TempDir::new(); + let file_path = setup(&temp_dir); + + let mut lockfile = Lockfile::new(file_path, true).unwrap(); + + lockfile.insert( + "https://deno.land/std@0.71.0/textproto/mod.ts", + "Here is some source code", + ); + lockfile.insert( + "https://deno.land/std@0.71.0/io/util.ts", + "more source code here", + ); + lockfile.insert( + "https://deno.land/std@0.71.0/async/delay.ts", + "this source is really exciting", + ); + + lockfile.write().expect("unable to write"); + + let file_path_buf = temp_dir.path().join("valid_lockfile.json"); + let file_path = file_path_buf.to_str().expect("file path fail").to_string(); + + // read the file contents back into a string and check + let mut checkfile = File::open(file_path).expect("Unable to open the file"); + let mut contents = String::new(); + checkfile + .read_to_string(&mut contents) + .expect("Unable to read the file"); + + let contents_json = + serde_json::from_str::(&contents).unwrap(); + let object = contents_json["remote"].as_object().unwrap(); + + assert_eq!( + object + .get("https://deno.land/std@0.71.0/textproto/mod.ts") + .and_then(|v| v.as_str()), + // sha-256 hash of the source 'Here is some source code' + Some("fedebba9bb82cce293196f54b21875b649e457f0eaf55556f1e318204947a28f") + ); + + // confirm that keys are sorted alphabetically + let mut keys = object.keys().map(|k| k.as_str()); + assert_eq!( + keys.next(), + Some("https://deno.land/std@0.71.0/async/delay.ts") + ); + assert_eq!(keys.next(), Some("https://deno.land/std@0.71.0/io/util.ts")); + assert_eq!( + keys.next(), + Some("https://deno.land/std@0.71.0/textproto/mod.ts") + ); + assert!(keys.next().is_none()); + } + + #[test] + fn check_or_insert_lockfile() { + let temp_dir = TempDir::new(); + let file_path = setup(&temp_dir); + + let mut lockfile = Lockfile::new(file_path, false).unwrap(); + + lockfile.insert( + "https://deno.land/std@0.71.0/textproto/mod.ts", + "Here is some source code", + ); + + let check_true = lockfile.check_or_insert_remote( + "https://deno.land/std@0.71.0/textproto/mod.ts", + "Here is some source code", + ); + assert!(check_true); + + let check_false = lockfile.check_or_insert_remote( + "https://deno.land/std@0.71.0/textproto/mod.ts", + "Here is some NEW source code", + ); + assert!(!check_false); + + // Not present in lockfile yet, should be inserted and check passed. + let check_true = lockfile.check_or_insert_remote( + "https://deno.land/std@0.71.0/http/file_server.ts", + "This is new Source code", + ); + assert!(check_true); + } + + #[test] + fn check_or_insert_lockfile_npm() { + let temp_dir = TempDir::new(); + let file_path = setup(&temp_dir); + + let mut lockfile = Lockfile::new(file_path, false).unwrap(); + + let npm_package = NpmPackageLockfileInfo { + display_id: "nanoid@3.3.4".to_string(), + serialized_id: "nanoid@3.3.4".to_string(), + integrity: "sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==".to_string(), + dependencies: vec![], + }; + let check_ok = lockfile.check_or_insert_npm_package(npm_package); + assert!(check_ok.is_ok()); + + let npm_package = NpmPackageLockfileInfo { + display_id: "picocolors@1.0.0".to_string(), + serialized_id: "picocolors@1.0.0".to_string(), + integrity: "sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==".to_string(), + dependencies: vec![], + }; + // Integrity is borked in the loaded lockfile + let check_err = lockfile.check_or_insert_npm_package(npm_package); + assert!(check_err.is_err()); + + let npm_package = NpmPackageLockfileInfo { + display_id: "source-map-js@1.0.2".to_string(), + serialized_id: "source-map-js@1.0.2".to_string(), + integrity: "sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==".to_string(), + dependencies: vec![], + }; + // Not present in lockfile yet, should be inserted and check passed. + let check_ok = lockfile.check_or_insert_npm_package(npm_package); + assert!(check_ok.is_ok()); + + let npm_package = NpmPackageLockfileInfo { + display_id: "source-map-js@1.0.2".to_string(), + serialized_id: "source-map-js@1.0.2".to_string(), + integrity: "sha512-foobar".to_string(), + dependencies: vec![], + }; + // Now present in lockfile, should file due to borked integrity + let check_err = lockfile.check_or_insert_npm_package(npm_package); + assert!(check_err.is_err()); + } +} -- cgit v1.2.3