feat(unstable): single checksum per JSR package in the lockfile (#22421)

This changes the lockfile to not store JSR specifiers in the "remote"
section. Instead a single JSR integrity is stored per package in the
lockfile, which is a hash of the version's `x.x.x_meta.json` file, which
contains hashes for every file in the package. The hashes in this file
are then compared against when loading.

Additionally, when using `{ "vendor": true }` in a deno.json, the files
can be modified without causing lockfile errors—the checksum is only
checked when copying into the vendor folder and not afterwards
(eventually we should add this behaviour for non-jsr specifiers as
well). As part of this change, the `vendor` folder creation is not
always automatic in the LSP and running an explicit cache command is
necessary. The code required to track checksums in the LSP would have
been too complex for this PR, so that all goes through deno_graph now.
The vendoring is still automatic when running from the CLI.

3 files changed, 139 insertions, 7 deletions

diff --git a/test_util/src/builders.rs b/test_util/src/builders.rs
index 9bbe6693f..d8c209dd7 100644
--- a/test_util/src/builders.rs
+++ b/test_util/src/builders.rs
@@ -280,6 +280,34 @@ impl TestContext {
       .run()
       .skip_output_check();
   }
+
+  pub fn get_jsr_package_integrity(&self, sub_path: &str) -> String {
+    fn get_checksum(bytes: &[u8]) -> String {
+      use sha2::Digest;
+      let mut hasher = sha2::Sha256::new();
+      hasher.update(bytes);
+      format!("{:x}", hasher.finalize())
+    }
+
+    let url = url::Url::parse(self.envs.get("JSR_URL").unwrap()).unwrap();
+    let url = url.join(&format!("{}_meta.json", sub_path)).unwrap();
+    let bytes = sync_fetch(url);
+    get_checksum(&bytes)
+  }
+}
+
+fn sync_fetch(url: url::Url) -> bytes::Bytes {
+  let runtime = tokio::runtime::Builder::new_current_thread()
+    .enable_io()
+    .enable_time()
+    .build()
+    .unwrap();
+  runtime.block_on(async move {
+    let client = reqwest::Client::new();
+    let response = client.get(url).send().await.unwrap();
+    assert!(response.status().is_success());
+    response.bytes().await.unwrap()
+  })
 }
 
 /// We can't clone an stdio, so if someone clones a DenoCmd,
diff --git a/test_util/src/npm.rs b/test_util/src/npm.rs
index 04207b0ee..7469e9b9e 100644
--- a/test_util/src/npm.rs
+++ b/test_util/src/npm.rs
@@ -64,9 +64,6 @@ impl CustomNpmPackageCache {
 }
 
 fn get_npm_package(package_name: &str) -> Result<Option<CustomNpmPackage>> {
-  use ring::digest::Context;
-  use ring::digest::SHA512;
-
   let package_folder = testdata_path().join("npm/registry").join(package_name);
   if !package_folder.exists() {
     return Ok(None);
@@ -103,10 +100,7 @@ fn get_npm_package(package_name: &str) -> Result<Option<CustomNpmPackage>> {
     }
 
     // get tarball hash
-    let mut hash_ctx = Context::new(&SHA512);
-    hash_ctx.update(&tarball_bytes);
-    let digest = hash_ctx.finish();
-    let tarball_checksum = BASE64_STANDARD.encode(digest.as_ref());
+    let tarball_checksum = get_tarball_checksum(&tarball_bytes);
 
     // create the registry file JSON for this version
     let mut dist = serde_json::Map::new();
@@ -176,3 +170,10 @@ fn get_npm_package(package_name: &str) -> Result<Option<CustomNpmPackage>> {
     tarballs,
   }))
 }
+
+fn get_tarball_checksum(bytes: &[u8]) -> String {
+  use sha2::Digest;
+  let mut hasher = sha2::Sha512::new();
+  hasher.update(bytes);
+  BASE64_STANDARD.encode(hasher.finalize())
+}
diff --git a/test_util/src/servers/registry.rs b/test_util/src/servers/registry.rs
index 69728f706..0efe06217 100644
--- a/test_util/src/servers/registry.rs
+++ b/test_util/src/servers/registry.rs
@@ -13,9 +13,14 @@ use hyper::body::Incoming;
 use hyper::Request;
 use hyper::Response;
 use hyper::StatusCode;
+use once_cell::sync::Lazy;
 use serde_json::json;
+use std::collections::BTreeMap;
+use std::collections::HashMap;
 use std::convert::Infallible;
 use std::net::SocketAddr;
+use std::path::Path;
+use std::sync::Mutex;
 
 pub async fn registry_server(port: u16) {
   let registry_server_addr = SocketAddr::from(([127, 0, 0, 1], port));
@@ -66,6 +71,27 @@ async fn registry_server_handler(
     testdata_path().to_path_buf().join("jsr").join("registry");
   file_path.push(&req.uri().path()[1..].replace("%2f", "/"));
   if let Ok(body) = tokio::fs::read(&file_path).await {
+    let body = if let Some(version) = file_path
+      .file_name()
+      .unwrap()
+      .to_string_lossy()
+      .strip_suffix("_meta.json")
+    {
+      // fill the manifest with checksums found in the directory so that
+      // we don't need to maintain them manually in the testdata directory
+      let mut meta: serde_json::Value = serde_json::from_slice(&body)?;
+      let mut manifest =
+        manifest_sorted(meta.get("manifest").cloned().unwrap_or(json!({})));
+      let version_dir = file_path.parent().unwrap().join(version);
+      fill_manifest_at_dir(&mut manifest, &version_dir);
+      meta
+        .as_object_mut()
+        .unwrap()
+        .insert("manifest".to_string(), json!(manifest));
+      serde_json::to_string(&meta).unwrap().into_bytes()
+    } else {
+      body
+    };
     return Ok(Response::new(UnsyncBoxBody::new(
       http_body_util::Full::new(Bytes::from(body)),
     )));
@@ -77,3 +103,80 @@ async fn registry_server_handler(
     .body(empty_body)?;
   Ok(res)
 }
+
+fn manifest_sorted(
+  meta: serde_json::Value,
+) -> BTreeMap<String, serde_json::Value> {
+  let mut manifest = BTreeMap::new();
+  if let serde_json::Value::Object(files) = meta {
+    for (file, checksum) in files {
+      manifest.insert(file.clone(), checksum.clone());
+    }
+  }
+  manifest
+}
+
+fn fill_manifest_at_dir(
+  manifest: &mut BTreeMap<String, serde_json::Value>,
+  dir: &Path,
+) {
+  let file_system_manifest = get_manifest_entries_for_dir(dir);
+  for (file_path, value) in file_system_manifest {
+    manifest.entry(file_path).or_insert(value);
+  }
+}
+
+static DIR_MANIFEST_CACHE: Lazy<
+  Mutex<HashMap<String, BTreeMap<String, serde_json::Value>>>,
+> = Lazy::new(Default::default);
+
+fn get_manifest_entries_for_dir(
+  dir: &Path,
+) -> BTreeMap<String, serde_json::Value> {
+  fn inner_fill(
+    root_dir: &Path,
+    dir: &Path,
+    manifest: &mut BTreeMap<String, serde_json::Value>,
+  ) {
+    for entry in std::fs::read_dir(dir).unwrap() {
+      let entry = entry.unwrap();
+      let path = entry.path();
+      if path.is_file() {
+        let file_bytes = std::fs::read(&path).unwrap();
+        let checksum = format!("sha256-{}", get_checksum(&file_bytes));
+        let relative_path = path
+          .to_string_lossy()
+          .strip_prefix(&root_dir.to_string_lossy().to_string())
+          .unwrap()
+          .replace('\\', "/");
+        manifest.insert(
+          relative_path,
+          json!({
+            "size": file_bytes.len(),
+            "checksum": checksum,
+          }),
+        );
+      } else if path.is_dir() {
+        inner_fill(root_dir, &path, manifest);
+      }
+    }
+  }
+
+  DIR_MANIFEST_CACHE
+    .lock()
+    .unwrap()
+    .entry(dir.to_string_lossy().to_string())
+    .or_insert_with(|| {
+      let mut manifest = BTreeMap::new();
+      inner_fill(dir, dir, &mut manifest);
+      manifest
+    })
+    .clone()
+}
+
+fn get_checksum(bytes: &[u8]) -> String {
+  use sha2::Digest;
+  let mut hasher = sha2::Sha256::new();
+  hasher.update(bytes);
+  format!("{:x}", hasher.finalize())
+}