diff options
author | David Sherret <dsherret@users.noreply.github.com> | 2024-02-15 14:49:35 -0500 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-02-15 14:49:35 -0500 |
commit | 4f80d83774ce5402a2b10503529fe422c998b841 (patch) | |
tree | d99c2e0bdc13e36727c62800130ebcab3b85dae7 | |
parent | 052b7d8bbdb43eedcdaae1a3094a5f2c70bba279 (diff) |
feat(unstable): single checksum per JSR package in the lockfile (#22421)
This changes the lockfile to not store JSR specifiers in the "remote"
section. Instead a single JSR integrity is stored per package in the
lockfile, which is a hash of the version's `x.x.x_meta.json` file, which
contains hashes for every file in the package. The hashes in this file
are then compared against when loading.
Additionally, when using `{ "vendor": true }` in a deno.json, the files
can be modified without causing lockfile errors—the checksum is only
checked when copying into the vendor folder and not afterwards
(eventually we should add this behaviour for non-jsr specifiers as
well). As part of this change, the `vendor` folder creation is not
always automatic in the LSP and running an explicit cache command is
necessary. The code required to track checksums in the LSP would have
been too complex for this PR, so that all goes through deno_graph now.
The vendoring is still automatic when running from the CLI.
27 files changed, 767 insertions, 243 deletions
diff --git a/Cargo.lock b/Cargo.lock index aa07029b1..89cc00b2d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -129,6 +129,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" [[package]] +name = "ammonia" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64e6d1c7838db705c9b756557ee27c384ce695a1c51a6fe528784cb1c6840170" +dependencies = [ + "html5ever", + "maplit", + "once_cell", + "tendril", + "url", +] + +[[package]] name = "android_system_properties" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1061,7 +1074,7 @@ dependencies = [ "open", "os_pipe", "percent-encoding", - "phf", + "phf 0.11.2", "pin-project", "pretty_assertions", "quick-junit", @@ -1097,9 +1110,9 @@ dependencies = [ [[package]] name = "deno_ast" -version = "0.33.2" +version = "0.33.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fdafff817ae3ad89672d54cd8daebc86dc352065ccc18691605043e6b845d00" +checksum = "7f61944e781d268799bf65857e664d3c09a37590043d4b0ed10facefc9bea473" dependencies = [ "anyhow", "base64", @@ -1172,9 +1185,9 @@ dependencies = [ [[package]] name = "deno_cache_dir" -version = "0.6.1" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bbb245d9a3719b5eb2b5195aaaa25108c3c93d1762b181a20fb1af1c7703eaf" +checksum = "6cf517bddfd22d79d0f284500318e3f9aea193536c2b61cbf6ce7b50a85f1b6a" dependencies = [ "anyhow", "deno_media_type", @@ -1182,9 +1195,9 @@ dependencies = [ "log", "once_cell", "parking_lot 0.12.1", - "ring", "serde", "serde_json", + "sha2", "thiserror", "url", ] @@ -1310,10 +1323,11 @@ dependencies = [ [[package]] name = "deno_doc" -version = "0.103.0" +version = "0.107.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73fe6bd8144456ca3f01b8d1cd1b668b974c84dc94cb642936c0938348b17017" +checksum = "f82478f27de7958eb6a1e48e447b8cb030a1294097ef510eec190d29e81f330f" dependencies = [ + "ammonia", "anyhow", "cfg-if", "comrak", @@ -1328,15 +1342,14 @@ dependencies = [ "regex", "serde", "serde_json", - "syntect", "termcolor", ] [[package]] name = "deno_emit" -version = "0.36.0" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5002f2c25489fb993132dc0cb0dabd41bae70a8629168db4bd726ee2e296ac" +checksum = "a670c56f233f85f18f1d4a3288c5241505d8aea559fe3870b45e00d4c0e731dc" dependencies = [ "anyhow", "base64", @@ -1404,9 +1417,9 @@ dependencies = [ [[package]] name = "deno_graph" -version = "0.65.3" +version = "0.66.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d12c87f92df950ad0eed3ea8951f30bf9c54f69b3a903b805950d6761b35002" +checksum = "8c67c7c05d70e43560b1dfa38ee385d2d0153ccd4ea16fdc6a706881fd60f3c5" dependencies = [ "anyhow", "async-trait", @@ -1424,6 +1437,7 @@ dependencies = [ "regex", "serde", "serde_json", + "sha2", "thiserror", "url", ] @@ -1455,7 +1469,7 @@ dependencies = [ "mime", "once_cell", "percent-encoding", - "phf", + "phf 0.11.2", "pin-project", "rand", "ring", @@ -1531,13 +1545,13 @@ dependencies = [ [[package]] name = "deno_lockfile" -version = "0.18.2" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f348633cc4425b2a9011436e256b1ae8f6c8026ec2705d852baee8643dc5562" +checksum = "8835418ae924f25ab20f508bf6240193b22d893519d44432b670a27b8fb1efeb" dependencies = [ - "ring", "serde", "serde_json", + "sha2", "thiserror", ] @@ -1659,9 +1673,9 @@ dependencies = [ [[package]] name = "deno_npm" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "376262760b173ff01f8f5d05d58a64f6d863472396afb5582590fa0949342854" +checksum = "53a333104d3fb6aa52e499384e523aefc09d3ac8ecd05ca7f65f856044fbcb09" dependencies = [ "anyhow", "async-trait", @@ -2414,9 +2428,9 @@ dependencies = [ [[package]] name = "eszip" -version = "0.62.0" +version = "0.63.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a26aa6791e6021e9e3ffc6bc8ab00ff2d0d748c64a75b7333076d973ce32f6b" +checksum = "731a0e44e886cb8efbbd63b8121341d505e9dab855fe487249d70c362a6bd774" dependencies = [ "anyhow", "base64", @@ -2649,6 +2663,16 @@ dependencies = [ ] [[package]] +name = "futf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843" +dependencies = [ + "mac", + "new_debug_unreachable", +] + +[[package]] name = "futures" version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3059,7 +3083,7 @@ checksum = "de90d3db62411eb62eddabe402d706ac4970f7ac8d088c05f11069cad9be9857" dependencies = [ "new_debug_unreachable", "once_cell", - "phf", + "phf 0.11.2", "rustc-hash", "smallvec", ] @@ -3074,6 +3098,20 @@ dependencies = [ ] [[package]] +name = "html5ever" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bea68cab48b8459f17cf1c944c67ddc572d272d9f2b274140f223ecb1da4a3b7" +dependencies = [ + "log", + "mac", + "markup5ever", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] name = "http" version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3732,6 +3770,12 @@ dependencies = [ ] [[package]] +name = "mac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" + +[[package]] name = "malloc_buf" version = "0.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3741,6 +3785,26 @@ dependencies = [ ] [[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + +[[package]] +name = "markup5ever" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2629bb1404f3d34c2e921f21fd34ba00b206124c81f65c50b43b6aaefeb016" +dependencies = [ + "log", + "phf 0.10.1", + "phf_codegen", + "string_cache", + "string_cache_codegen", + "tendril", +] + +[[package]] name = "match_cfg" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -4148,28 +4212,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] -name = "onig" -version = "6.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f" -dependencies = [ - "bitflags 1.3.2", - "libc", - "once_cell", - "onig_sys", -] - -[[package]] -name = "onig_sys" -version = "69.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7" -dependencies = [ - "cc", - "pkg-config", -] - -[[package]] name = "opaque-debug" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -4435,12 +4477,41 @@ dependencies = [ [[package]] name = "phf" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259" +dependencies = [ + "phf_shared 0.10.0", +] + +[[package]] +name = "phf" version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" dependencies = [ "phf_macros", - "phf_shared", + "phf_shared 0.11.2", +] + +[[package]] +name = "phf_codegen" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb1c3a8bc4dd4e5cfce29b44ffc14bedd2ee294559a294e2a4d4c9e9a6a13cd" +dependencies = [ + "phf_generator 0.10.0", + "phf_shared 0.10.0", +] + +[[package]] +name = "phf_generator" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6" +dependencies = [ + "phf_shared 0.10.0", + "rand", ] [[package]] @@ -4449,7 +4520,7 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" dependencies = [ - "phf_shared", + "phf_shared 0.11.2", "rand", ] @@ -4459,8 +4530,8 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" dependencies = [ - "phf_generator", - "phf_shared", + "phf_generator 0.11.2", + "phf_shared 0.11.2", "proc-macro2", "quote", "syn 2.0.48", @@ -4468,6 +4539,15 @@ dependencies = [ [[package]] name = "phf_shared" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" +dependencies = [ + "siphasher", +] + +[[package]] +name = "phf_shared" version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" @@ -4589,6 +4669,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + +[[package]] name = "presser" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -4923,7 +5009,7 @@ dependencies = [ "aho-corasick", "memchr", "regex-automata", - "regex-syntax 0.8.2", + "regex-syntax", ] [[package]] @@ -4934,17 +5020,11 @@ checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.2", + "regex-syntax", ] [[package]] name = "regex-syntax" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" - -[[package]] -name = "regex-syntax" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" @@ -5704,6 +5784,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] +name = "string_cache" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b" +dependencies = [ + "new_debug_unreachable", + "once_cell", + "parking_lot 0.12.1", + "phf_shared 0.10.0", + "precomputed-hash", + "serde", +] + +[[package]] +name = "string_cache_codegen" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988" +dependencies = [ + "phf_generator 0.10.0", + "phf_shared 0.10.0", + "proc-macro2", + "quote", +] + +[[package]] name = "string_enum" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -5772,9 +5878,9 @@ dependencies = [ [[package]] name = "swc_bundler" -version = "0.225.3" +version = "0.225.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26491762e84ae1d0a2e179fe48066072834777a1b12e8e88a7f07c8f92cc0188" +checksum = "feb8b6f3ad184a5ae21544411491bf136635237fc097d7c93ccd915449ebb2ba" dependencies = [ "anyhow", "crc", @@ -5875,7 +5981,7 @@ dependencies = [ "bitflags 2.4.1", "is-macro", "num-bigint", - "phf", + "phf 0.11.2", "scoped-tls", "serde", "string_enum", @@ -5939,7 +6045,7 @@ dependencies = [ "new_debug_unreachable", "num-bigint", "num-traits", - "phf", + "phf 0.11.2", "serde", "smallvec", "smartstring", @@ -5953,15 +6059,15 @@ dependencies = [ [[package]] name = "swc_ecma_transforms_base" -version = "0.137.3" +version = "0.137.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfd47dd9ccb73a1f5d8d7eff9518554b752b1733b56503af090e78859abb42dd" +checksum = "803bb435fdd532d5c931f0d487e48dbc94750d26c9336d79a6f1c04c62f08d93" dependencies = [ "better_scoped_tls", "bitflags 2.4.1", "indexmap", "once_cell", - "phf", + "phf 0.11.2", "rustc-hash", "serde", "smallvec", @@ -5976,9 +6082,9 @@ dependencies = [ [[package]] name = "swc_ecma_transforms_classes" -version = "0.126.3" +version = "0.126.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb31417e0d415d7f0ff026f1e7c909427e386b7d0af9a2a78678507e4d9d79" +checksum = "486479e75907547d4c65ca6deed8faa465a2e9475cc9605be36a0e5eb609f578" dependencies = [ "swc_atoms", "swc_common", @@ -6002,9 +6108,9 @@ dependencies = [ [[package]] name = "swc_ecma_transforms_optimization" -version = "0.198.3" +version = "0.198.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3920268ac8972b494067d0b7c088964b21d08f5d1f58d7151bd1eb7054a137b0" +checksum = "f3c6fffe4d6e3609fdd6c768cc063dbc9f5101f9be0db1168ec76ace979cf616" dependencies = [ "dashmap", "indexmap", @@ -6026,9 +6132,9 @@ dependencies = [ [[package]] name = "swc_ecma_transforms_proposal" -version = "0.171.3" +version = "0.171.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "448c40c2a2b224cb5101cc6cdee81837c281a34f2a2aa6dd18d6d5cd8d492e60" +checksum = "e2822bc6c28bb1a96090a3b2caa28f45efbaecb333714d30868a2390eaae943f" dependencies = [ "either", "rustc-hash", @@ -6046,9 +6152,9 @@ dependencies = [ [[package]] name = "swc_ecma_transforms_react" -version = "0.183.3" +version = "0.183.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2394dc3abceada246feeb709b8c4d23392973f49a24fcc59b2ee21737cb6c8" +checksum = "8984ebb8955116c426457a0c70b0aa9a08a06656e245781ff617a9cd1e289697" dependencies = [ "base64", "dashmap", @@ -6070,9 +6176,9 @@ dependencies = [ [[package]] name = "swc_ecma_transforms_typescript" -version = "0.188.3" +version = "0.188.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cff231437173e041e5a3be9b8c782fd297ffcb53ed16d805f853e4a68315c45" +checksum = "a2e898fbab993abb60fb67009521908f2317f1d33f804e5b38d769f52e58572e" dependencies = [ "ryu-js", "serde", @@ -6087,9 +6193,9 @@ dependencies = [ [[package]] name = "swc_ecma_utils" -version = "0.127.3" +version = "0.127.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd185161161dfc65ee0d6f3044c901b766c3abb4efcd0b35c9e76c833724896" +checksum = "8ff9e77ea18468895d26bd38656885860fede2acd24d1687f64363aaf8910441" dependencies = [ "indexmap", "num_cpus", @@ -6223,25 +6329,6 @@ dependencies = [ ] [[package]] -name = "syntect" -version = "5.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e02b4b303bf8d08bfeb0445cba5068a3d306b6baece1d5582171a9bf49188f91" -dependencies = [ - "bincode", - "bitflags 1.3.2", - "flate2", - "fnv", - "once_cell", - "onig", - "regex-syntax 0.7.5", - "serde", - "serde_json", - "thiserror", - "walkdir", -] - -[[package]] name = "tar" version = "0.4.40" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -6266,6 +6353,17 @@ dependencies = [ ] [[package]] +name = "tendril" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0" +dependencies = [ + "futf", + "mac", + "utf-8", +] + +[[package]] name = "termcolor" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -6324,12 +6422,12 @@ dependencies = [ "prost-build", "regex", "reqwest", - "ring", "rustls-pemfile", "rustls-tokio-stream", "semver 1.0.14", "serde", "serde_json", + "sha2", "tar", "tempfile", "termcolor", diff --git a/Cargo.toml b/Cargo.toml index e54176c90..7b0686432 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,7 +46,7 @@ deno_ast = { version = "0.33.2", features = ["transpiling"] } deno_core = { version = "0.262.0" } deno_bench_util = { version = "0.132.0", path = "./bench_util" } -deno_lockfile = "0.18.2" +deno_lockfile = "0.19.0" deno_media_type = { version = "0.1.1", features = ["module_specifier"] } deno_runtime = { version = "0.146.0", path = "./runtime" } deno_terminal = "0.1.1" @@ -97,7 +97,7 @@ chrono = { version = "0.4", default-features = false, features = ["std", "serde" console_static_text = "=0.8.1" data-encoding = "2.3.3" data-url = "=0.3.0" -deno_cache_dir = "=0.6.1" +deno_cache_dir = "=0.7.1" dlopen2 = "0.6.1" ecb = "=0.1.2" elliptic-curve = { version = "0.13.4", features = ["alloc", "arithmetic", "ecdh", "std", "pem"] } diff --git a/cli/Cargo.toml b/cli/Cargo.toml index ca00774ea..3898fb7bb 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -66,17 +66,17 @@ deno_ast = { workspace = true, features = ["bundler", "cjs", "codegen", "proposa deno_cache_dir = { workspace = true } deno_config = "=0.9.2" deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] } -deno_doc = { version = "=0.103.0", features = ["html"] } -deno_emit = "=0.36.0" -deno_graph = "=0.65.3" +deno_doc = { version = "=0.107.0", features = ["html"] } +deno_emit = "=0.37.0" +deno_graph = "=0.66.0" deno_lint = { version = "=0.56.0", features = ["docs"] } deno_lockfile.workspace = true -deno_npm = "=0.16.0" +deno_npm = "=0.17.0" deno_runtime = { workspace = true, features = ["include_js_files_for_snapshotting"] } deno_semver = "=0.5.4" deno_task_shell = "=0.14.3" deno_terminal.workspace = true -eszip = "=0.62.0" +eszip = "=0.63.0" napi_sym.workspace = true async-trait.workspace = true diff --git a/cli/cache/mod.rs b/cli/cache/mod.rs index 6c48799ce..e9d225146 100644 --- a/cli/cache/mod.rs +++ b/cli/cache/mod.rs @@ -196,8 +196,7 @@ impl Loader for FetchCacher { fn load( &mut self, specifier: &ModuleSpecifier, - _is_dynamic: bool, - cache_setting: deno_graph::source::CacheSetting, + options: deno_graph::source::LoadOptions, ) -> LoadFuture { use deno_graph::source::CacheSetting as LoaderCacheSetting; @@ -222,7 +221,7 @@ impl Loader for FetchCacher { let specifier = specifier.clone(); async move { - let maybe_cache_setting = match cache_setting { + let maybe_cache_setting = match options.cache_setting { LoaderCacheSetting::Use => None, LoaderCacheSetting::Reload => { if matches!(file_fetcher.cache_setting(), CacheSetting::Only) { @@ -240,6 +239,7 @@ impl Loader for FetchCacher { permissions, maybe_accept: None, maybe_cache_setting: maybe_cache_setting.as_ref(), + maybe_checksum: options.maybe_checksum, }) .await .map(|file| { @@ -269,7 +269,7 @@ impl Loader for FetchCacher { let error_class_name = get_error_class_name(&err); match error_class_name { "NotFound" => Ok(None), - "NotCached" if cache_setting == LoaderCacheSetting::Only => Ok(None), + "NotCached" if options.cache_setting == LoaderCacheSetting::Only => Ok(None), _ => Err(err), } }) diff --git a/cli/factory.rs b/cli/factory.rs index 2236e6090..cccbecbf1 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -66,7 +66,6 @@ use deno_runtime::deno_node::NodeResolver; use deno_runtime::deno_tls::RootCertStoreProvider; use deno_runtime::deno_web::BlobStore; use deno_runtime::inspector_server::InspectorServer; -use deno_semver::package::PackageNv; use import_map::ImportMap; use log::warn; use std::future::Future; @@ -381,16 +380,6 @@ impl CliFactory { no_npm, no_config: self.options.no_config(), config, - nv_to_jsr_url: |nv| { - let nv = PackageNv::from_str(nv).ok()?; - Some( - deno_graph::source::recommended_registry_package_url( - crate::args::jsr_url(), - &nv, - ) - .to_string(), - ) - }, }, ); } diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs index 8f4d3feab..a74a14a3f 100644 --- a/cli/file_fetcher.rs +++ b/cli/file_fetcher.rs @@ -24,6 +24,7 @@ use deno_core::futures::future::FutureExt; use deno_core::parking_lot::Mutex; use deno_core::url::Url; use deno_core::ModuleSpecifier; +use deno_graph::source::LoaderChecksum; use deno_runtime::deno_fetch::reqwest::header::HeaderValue; use deno_runtime::deno_fetch::reqwest::header::ACCEPT; use deno_runtime::deno_fetch::reqwest::header::AUTHORIZATION; @@ -146,6 +147,7 @@ pub struct FetchOptions<'a> { pub permissions: PermissionsContainer, pub maybe_accept: Option<&'a str>, pub maybe_cache_setting: Option<&'a CacheSetting>, + pub maybe_checksum: Option<LoaderChecksum>, } /// A structure for resolving, fetching and caching source files. @@ -199,6 +201,7 @@ impl FileFetcher { pub fn fetch_cached( &self, specifier: &ModuleSpecifier, + maybe_checksum: Option<LoaderChecksum>, redirect_limit: i64, ) -> Result<Option<File>, AnyError> { debug!("FileFetcher::fetch_cached - specifier: {}", specifier); @@ -207,16 +210,22 @@ impl FileFetcher { } let cache_key = self.http_cache.cache_item_key(specifier)?; // compute this once - let Some(metadata) = self.http_cache.read_metadata(&cache_key)? else { + let Some(headers) = self.http_cache.read_headers(&cache_key)? else { return Ok(None); }; - let headers = metadata.headers; if let Some(redirect_to) = headers.get("location") { let redirect = deno_core::resolve_import(redirect_to, specifier.as_str())?; - return self.fetch_cached(&redirect, redirect_limit - 1); + return self.fetch_cached(&redirect, maybe_checksum, redirect_limit - 1); } - let Some(bytes) = self.http_cache.read_file_bytes(&cache_key)? else { + let Some(bytes) = self.http_cache.read_file_bytes( + &cache_key, + maybe_checksum + .as_ref() + .map(|c| deno_cache_dir::Checksum::new(c.as_str())), + deno_cache_dir::GlobalToLocalCopy::Allow, + )? + else { return Ok(None); }; @@ -282,6 +291,7 @@ impl FileFetcher { redirect_limit: i64, maybe_accept: Option<String>, cache_setting: &CacheSetting, + maybe_checksum: Option<LoaderChecksum>, ) -> Pin<Box<dyn Future<Output = Result<File, AnyError>> + Send>> { debug!("FileFetcher::fetch_remote() - specifier: {}", specifier); if redirect_limit < 0 { @@ -294,7 +304,8 @@ impl FileFetcher { } if self.should_use_cache(specifier, cache_setting) { - match self.fetch_cached(specifier, redirect_limit) { + match self.fetch_cached(specifier, maybe_checksum.clone(), redirect_limit) + { Ok(Some(file)) => { return futures::future::ok(file).boxed(); } @@ -331,8 +342,8 @@ impl FileFetcher { .http_cache .cache_item_key(specifier) .ok() - .and_then(|key| self.http_cache.read_metadata(&key).ok().flatten()) - .and_then(|metadata| metadata.headers.get("etag").cloned()); + .and_then(|key| self.http_cache.read_headers(&key).ok().flatten()) + .and_then(|headers| headers.get("etag").cloned()); let maybe_auth_token = self.auth_tokens.get(specifier); let specifier = specifier.clone(); let client = self.http_client.clone(); @@ -376,7 +387,9 @@ impl FileFetcher { .await? { FetchOnceResult::NotModified => { - let file = file_fetcher.fetch_cached(&specifier, 10)?.unwrap(); + let file = file_fetcher + .fetch_cached(&specifier, maybe_checksum, 10)? + .unwrap(); Ok(file) } FetchOnceResult::Redirect(redirect_url, headers) => { @@ -388,6 +401,7 @@ impl FileFetcher { redirect_limit - 1, maybe_accept, &cache_setting, + maybe_checksum, ) .await } @@ -395,6 +409,9 @@ impl FileFetcher { file_fetcher .http_cache .set(&specifier, headers.clone(), &bytes)?; + if let Some(checksum) = &maybe_checksum { + checksum.check_source(&bytes)?; + } Ok(File { specifier, maybe_headers: Some(headers), @@ -438,15 +455,16 @@ impl FileFetcher { let Ok(cache_key) = self.http_cache.cache_item_key(specifier) else { return false; }; - let Ok(Some(metadata)) = self.http_cache.read_metadata(&cache_key) + let Ok(Some(headers)) = self.http_cache.read_headers(&cache_key) else { + return false; + }; + let Ok(Some(download_time)) = + self.http_cache.read_download_time(&cache_key) else { return false; }; - let cache_semantics = CacheSemantics::new( - metadata.headers, - metadata.time, - SystemTime::now(), - ); + let cache_semantics = + CacheSemantics::new(headers, download_time, SystemTime::now()); cache_semantics.should_use() } CacheSetting::ReloadSome(list) => { @@ -482,6 +500,7 @@ impl FileFetcher { permissions, maybe_accept: None, maybe_cache_setting: None, + maybe_checksum: None, }) .await } @@ -517,6 +536,7 @@ impl FileFetcher { 10, options.maybe_accept.map(String::from), options.maybe_cache_setting.unwrap_or(&self.cache_setting), + options.maybe_checksum, ) .await } @@ -728,6 +748,7 @@ mod tests { 1, None, &file_fetcher.cache_setting, + None, ) .await; let cache_key = file_fetcher.http_cache.cache_item_key(specifier).unwrap(); @@ -735,10 +756,9 @@ mod tests { result.unwrap(), file_fetcher .http_cache - .read_metadata(&cache_key) - .unwrap() + .read_headers(&cache_key) .unwrap() - .headers, + .unwrap(), ) } @@ -899,18 +919,11 @@ mod tests { let cache_item_key = file_fetcher.http_cache.cache_item_key(&specifier).unwrap(); - let mut metadata = file_fetcher - .http_cache - .read_metadata(&cache_item_key) - .unwrap() - .unwrap(); - metadata.headers = HashMap::new(); - metadata - .headers - .insert("content-type".to_string(), "text/javascript".to_string()); + let mut headers = HashMap::new(); + headers.insert("content-type".to_string(), "text/javascript".to_string()); file_fetcher .http_cache - .set(&specifier, metadata.headers.clone(), file.source.as_bytes()) + .set(&specifier, headers.clone(), file.source.as_bytes()) .unwrap(); let result = file_fetcher_01 @@ -926,20 +939,17 @@ mod tests { // the value above. assert_eq!(file.media_type, MediaType::JavaScript); - let headers = file_fetcher_02 + let headers2 = file_fetcher_02 .http_cache - .read_metadata(&cache_item_key) + .read_headers(&cache_item_key) .unwrap() - .unwrap() - .headers; - assert_eq!(headers.get("content-type").unwrap(), "text/javascript"); - metadata.headers = HashMap::new(); - metadata - .headers - .insert("content-type".to_string(), "application/json".to_string()); + .unwrap(); + assert_eq!(headers2.get("content-type").unwrap(), "text/javascript"); + headers = HashMap::new(); + headers.insert("content-type".to_string(), "application/json".to_string()); file_fetcher_02 .http_cache - .set(&specifier, metadata.headers.clone(), file.source.as_bytes()) + .set(&specifier, headers.clone(), file.source.as_bytes()) .unwrap(); let result = file_fetcher_02 @@ -1013,7 +1023,12 @@ mod tests { .unwrap(), file_fetcher .http_cache - .read_metadata(&cache_key) + .read_headers(&cache_key) + .unwrap() + .unwrap(), + file_fetcher + .http_cache + .read_download_time(&cache_key) .unwrap() .unwrap(), ) @@ -1045,7 +1060,12 @@ mod tests { .unwrap(), file_fetcher .http_cache - .read_metadata(&cache_key) + .read_headers(&cache_key) + .unwrap() + .unwrap(), + file_fetcher + .http_cache + .read_download_time(&cache_key) .unwrap() .unwrap(), ) @@ -1182,7 +1202,12 @@ mod tests { .unwrap(), file_fetcher .http_cache - .read_metadata(&cache_key) + .read_headers(&cache_key) + .unwrap() + .unwrap(), + file_fetcher + .http_cache + .read_download_time(&cache_key) .unwrap() .unwrap(), ) @@ -1216,7 +1241,12 @@ mod tests { .unwrap(), file_fetcher .http_cache - .read_metadata(&cache_key) + .read_headers(&cache_key) + .unwrap() + .unwrap(), + file_fetcher + .http_cache + .read_download_time(&cache_key) .unwrap() .unwrap(), ) @@ -1240,6 +1270,7 @@ mod tests { 2, None, &file_fetcher.cache_setting, + None, ) .await; assert!(result.is_ok()); @@ -1251,14 +1282,15 @@ mod tests { 1, None, &file_fetcher.cache_setting, + None, ) .await; assert!(result.is_err()); - let result = file_fetcher.fetch_cached(&specifier, 2); + let result = file_fetcher.fetch_cached(&specifier, None, 2); assert!(result.is_ok()); - let result = file_fetcher.fetch_cached(&specifier, 1); + let result = file_fetcher.fetch_cached(&specifier, None, 1); assert!(result.is_err()); } @@ -2072,7 +2104,11 @@ mod tests { let cache_key = file_fetcher.http_cache.cache_item_key(url).unwrap(); let bytes = file_fetcher .http_cache - .read_file_bytes(&cache_key) + .read_file_bytes( + &cache_key, + None, + deno_cache_dir::GlobalToLocalCopy::Allow, + ) .unwrap() .unwrap(); String::from_utf8(bytes).unwrap() @@ -2086,10 +2122,9 @@ mod tests { let cache_key = file_fetcher.http_cache.cache_item_key(url).unwrap(); file_fetcher .http_cache - .read_metadata(&cache_key) + .read_headers(&cache_key) .unwrap() .unwrap() - .headers .remove("location") } } diff --git a/cli/graph_util.rs b/cli/graph_util.rs index 3633784b8..09f0db9e6 100644 --- a/cli/graph_util.rs +++ b/cli/graph_util.rs @@ -1,5 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use crate::args::jsr_url; use crate::args::CliOptions; use crate::args::Lockfile; use crate::args::TsTypeLib; @@ -174,6 +175,18 @@ pub fn graph_lock_or_exit(graph: &ModuleGraph, lockfile: &mut Lockfile) { Module::Json(module) => &module.source, Module::Node(_) | Module::Npm(_) | Module::External(_) => continue, }; + + // skip over any specifiers in JSR packages because those + // are enforced via the integrity + if deno_graph::source::recommended_registry_package_url_to_nv( + jsr_url(), + module.specifier(), + ) + .is_some() + { + continue; + } + if !lockfile.check_or_insert_remote(module.specifier().as_str(), source) { let err = format!( concat!( @@ -475,6 +488,19 @@ impl ModuleGraphBuilder { } } } + for (nv, value) in &lockfile.content.packages.jsr { + if let Ok(nv) = PackageNv::from_str(nv) { + graph + .packages + .add_manifest_checksum(nv, value.integrity.clone()) + .map_err(|err| deno_lockfile::IntegrityCheckFailedError { + package_display_id: format!("jsr:{}", err.nv), + actual: err.actual, + expected: err.expected, + filename: lockfile.filename.display().to_string(), + })?; + } + } } } @@ -504,9 +530,14 @@ impl ModuleGraphBuilder { format!("jsr:{}", to), ); } - for (name, deps) in graph.packages.package_deps() { - lockfile - .insert_package_deps(name.to_string(), deps.map(|s| s.to_string())); + for (name, checksum, deps) in + graph.packages.packages_with_checksum_and_deps() + { + lockfile.insert_package( + name.to_string(), + checksum.clone(), + deps.map(|s| s.to_string()), + ); } } } diff --git a/cli/lsp/cache.rs b/cli/lsp/cache.rs index eec6433a2..e0034207d 100644 --- a/cli/lsp/cache.rs +++ b/cli/lsp/cache.rs @@ -11,6 +11,16 @@ use std::path::Path; use std::sync::Arc; use std::time::SystemTime; +/// In the LSP, we disallow the cache from automatically copying from +/// the global cache to the local cache for technical reasons. +/// +/// 1. We need to verify the checksums from the lockfile are correct when +/// moving from the global to the local cache. +/// 2. We need to verify the checksums for JSR https specifiers match what +/// is found in the package's manifest. +pub const LSP_DISALLOW_GLOBAL_TO_LOCAL_COPY: deno_cache_dir::GlobalToLocalCopy = + deno_cache_dir::GlobalToLocalCopy::Disallow; + pub fn calculate_fs_version( cache: &Arc<dyn HttpCache>, specifier: &ModuleSpecifier, @@ -123,8 +133,8 @@ impl CacheMetadata { return None; } let cache_key = self.cache.cache_item_key(specifier).ok()?; - let specifier_metadata = self.cache.read_metadata(&cache_key).ok()??; - let values = Arc::new(parse_metadata(&specifier_metadata.headers)); + let headers = self.cache.read_headers(&cache_key).ok()??; + let values = Arc::new(parse_metadata(&headers)); let version = calculate_fs_version_in_cache(&self.cache, specifier); let mut metadata_map = self.metadata.lock(); let metadata = Metadata { values, version }; diff --git a/cli/lsp/documents.rs b/cli/lsp/documents.rs index c58a392d5..125307757 100644 --- a/cli/lsp/documents.rs +++ b/cli/lsp/documents.rs @@ -2,6 +2,7 @@ use super::cache::calculate_fs_version; use super::cache::calculate_fs_version_at_path; +use super::cache::LSP_DISALLOW_GLOBAL_TO_LOCAL_COPY; use super::jsr_resolver::JsrResolver; use super::language_server::StateNpmSnapshot; use super::text::LineIndex; @@ -736,12 +737,7 @@ impl RedirectResolver { ) -> Option<ModuleSpecifier> { if redirect_limit > 0 { let cache_key = self.cache.cache_item_key(specifier).ok()?; - let headers = self - .cache - .read_metadata(&cache_key) - .ok() - .flatten() - .map(|m| m.headers)?; + let headers = self.cache.read_headers(&cache_key).ok().flatten()?; if let Some(location) = headers.get("location") { let redirect = deno_core::resolve_import(location, specifier.as_str()).ok()?; @@ -822,12 +818,14 @@ impl FileSystemDocuments { } else { let fs_version = calculate_fs_version(cache, specifier)?; let cache_key = cache.cache_item_key(specifier).ok()?; - let bytes = cache.read_file_bytes(&cache_key).ok()??; - let specifier_metadata = cache.read_metadata(&cache_key).ok()??; + let bytes = cache + .read_file_bytes(&cache_key, None, LSP_DISALLOW_GLOBAL_TO_LOCAL_COPY) + .ok()??; + let specifier_headers = cache.read_headers(&cache_key).ok()??; let (_, maybe_charset) = deno_graph::source::resolve_media_type_and_charset_from_headers( specifier, - Some(&specifier_metadata.headers), + Some(&specifier_headers), ); let content = deno_graph::source::decode_owned_source( specifier, @@ -835,7 +833,7 @@ impl FileSystemDocuments { maybe_charset, ) .ok()?; - let maybe_headers = Some(specifier_metadata.headers); + let maybe_headers = Some(specifier_headers); Document::new( specifier.clone(), fs_version, @@ -1826,8 +1824,7 @@ impl<'a> deno_graph::source::Loader for OpenDocumentsGraphLoader<'a> { fn load( &mut self, specifier: &ModuleSpecifier, - is_dynamic: bool, - cache_setting: deno_graph::source::CacheSetting, + options: deno_graph::source::LoadOptions, ) -> deno_graph::source::LoadFuture { let specifier = if self.unstable_sloppy_imports { self @@ -1839,9 +1836,7 @@ impl<'a> deno_graph::source::Loader for OpenDocumentsGraphLoader<'a> { match self.load_from_docs(&specifier) { Some(fut) => fut, - None => self - .inner_loader - .load(&specifier, is_dynamic, cache_setting), + None => self.inner_loader.load(&specifier, options), } } diff --git a/cli/lsp/jsr_resolver.rs b/cli/lsp/jsr_resolver.rs index 8243bb0f2..be7bdc0f5 100644 --- a/cli/lsp/jsr_resolver.rs +++ b/cli/lsp/jsr_resolver.rs @@ -15,6 +15,8 @@ use deno_semver::package::PackageReq; use std::borrow::Cow; use std::sync::Arc; +use super::cache::LSP_DISALLOW_GLOBAL_TO_LOCAL_COPY; + #[derive(Debug)] pub struct JsrResolver { nv_by_req: DashMap<PackageReq, Option<PackageNv>>, @@ -111,7 +113,13 @@ fn read_cached_package_info( ) -> Option<JsrPackageInfo> { let meta_url = jsr_url().join(&format!("{}/meta.json", name)).ok()?; let meta_cache_item_key = cache.cache_item_key(&meta_url).ok()?; - let meta_bytes = cache.read_file_bytes(&meta_cache_item_key).ok()??; + let meta_bytes = cache + .read_file_bytes( + &meta_cache_item_key, + None, + LSP_DISALLOW_GLOBAL_TO_LOCAL_COPY, + ) + .ok()??; serde_json::from_slice::<JsrPackageInfo>(&meta_bytes).ok() } @@ -123,12 +131,19 @@ fn read_cached_package_version_info( .join(&format!("{}/{}_meta.json", &nv.name, &nv.version)) .ok()?; let meta_cache_item_key = cache.cache_item_key(&meta_url).ok()?; - let meta_bytes = cache.read_file_bytes(&meta_cache_item_key).ok()??; + let meta_bytes = cache + .read_file_bytes( + &meta_cache_item_key, + None, + LSP_DISALLOW_GLOBAL_TO_LOCAL_COPY, + ) + .ok()??; // This is a roundabout way of deserializing `JsrPackageVersionInfo`, // because we only want the `exports` field and `module_graph` is large. let mut info = serde_json::from_slice::<serde_json::Value>(&meta_bytes).ok()?; Some(JsrPackageVersionInfo { + manifest: Default::default(), // not used by the LSP (only caching checks this in deno_graph) exports: info.as_object_mut()?.remove("exports")?, module_graph: None, }) diff --git a/cli/lsp/registries.rs b/cli/lsp/registries.rs index f4a64c7ee..2b0cae7d2 100644 --- a/cli/lsp/registries.rs +++ b/cli/lsp/registries.rs @@ -515,6 +515,7 @@ impl ModuleRegistry { permissions: PermissionsContainer::allow_all(), maybe_accept: Some("application/vnd.deno.reg.v2+json, application/vnd.deno.reg.v1+json;q=0.9, application/json;q=0.8"), maybe_cache_setting: None, + maybe_checksum: None, }) .await; // if there is an error fetching, we will cache an empty file, so that diff --git a/cli/tools/coverage/mod.rs b/cli/tools/coverage/mod.rs index aafef292f..5cc705741 100644 --- a/cli/tools/coverage/mod.rs +++ b/cli/tools/coverage/mod.rs @@ -523,7 +523,7 @@ pub async fn cover_files( file_fetcher.get_source(&module_specifier) } else { file_fetcher - .fetch_cached(&module_specifier, 10) + .fetch_cached(&module_specifier, None, 10) .with_context(|| { format!("Failed to fetch \"{module_specifier}\" from cache.") })? diff --git a/cli/tools/doc.rs b/cli/tools/doc.rs index d2cd0c2a2..5044e73d3 100644 --- a/cli/tools/doc.rs +++ b/cli/tools/doc.rs @@ -211,9 +211,9 @@ impl deno_doc::html::HrefResolver for DocResolver { fn resolve_usage( &self, _current_specifier: &ModuleSpecifier, - current_file: &str, + current_file: Option<&str>, ) -> Option<String> { - Some(current_file.to_string()) + current_file.map(|f| f.to_string()) } fn resolve_source(&self, location: &deno_doc::Location) -> Option<String> { diff --git a/cli/tools/vendor/test.rs b/cli/tools/vendor/test.rs index 7910dcf22..6a960c302 100644 --- a/cli/tools/vendor/test.rs +++ b/cli/tools/vendor/test.rs @@ -115,8 +115,7 @@ impl Loader for TestLoader { fn load( &mut self, specifier: &ModuleSpecifier, - _is_dynamic: bool, - _cache_setting: deno_graph::source::CacheSetting, + _options: deno_graph::source::LoadOptions, ) -> LoadFuture { let specifier = self.redirects.get(specifier).unwrap_or(specifier); let result = self.files.get(specifier).map(|result| match result { diff --git a/cli/tsc/mod.rs b/cli/tsc/mod.rs index 18316b750..f3f974690 100644 --- a/cli/tsc/mod.rs +++ b/cli/tsc/mod.rs @@ -904,8 +904,7 @@ mod tests { fn load( &mut self, specifier: &ModuleSpecifier, - _is_dynamic: bool, - _cache_setting: deno_graph::source::CacheSetting, + _options: deno_graph::source::LoadOptions, ) -> deno_graph::source::LoadFuture { let specifier_text = specifier .to_string() diff --git a/test_util/Cargo.toml b/test_util/Cargo.toml index b2ff10277..e46c308b6 100644 --- a/test_util/Cargo.toml +++ b/test_util/Cargo.toml @@ -42,12 +42,12 @@ pretty_assertions.workspace = true prost.workspace = true regex.workspace = true reqwest.workspace = true -ring.workspace = true rustls-pemfile.workspace = true rustls-tokio-stream.workspace = true semver = "=1.0.14" serde.workspace = true serde_json.workspace = true +sha2.workspace = true tar.workspace = true tempfile.workspace = true termcolor.workspace = true diff --git a/test_util/src/builders.rs b/test_util/src/builders.rs index 9bbe6693f..d8c209dd7 100644 --- a/test_util/src/builders.rs +++ b/test_util/src/builders.rs @@ -280,6 +280,34 @@ impl TestContext { .run() .skip_output_check(); } + + pub fn get_jsr_package_integrity(&self, sub_path: &str) -> String { + fn get_checksum(bytes: &[u8]) -> String { + use sha2::Digest; + let mut hasher = sha2::Sha256::new(); + hasher.update(bytes); + format!("{:x}", hasher.finalize()) + } + + let url = url::Url::parse(self.envs.get("JSR_URL").unwrap()).unwrap(); + let url = url.join(&format!("{}_meta.json", sub_path)).unwrap(); + let bytes = sync_fetch(url); + get_checksum(&bytes) + } +} + +fn sync_fetch(url: url::Url) -> bytes::Bytes { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_io() + .enable_time() + .build() + .unwrap(); + runtime.block_on(async move { + let client = reqwest::Client::new(); + let response = client.get(url).send().await.unwrap(); + assert!(response.status().is_success()); + response.bytes().await.unwrap() + }) } /// We can't clone an stdio, so if someone clones a DenoCmd, diff --git a/test_util/src/npm.rs b/test_util/src/npm.rs index 04207b0ee..7469e9b9e 100644 --- a/test_util/src/npm.rs +++ b/test_util/src/npm.rs @@ -64,9 +64,6 @@ impl CustomNpmPackageCache { } fn get_npm_package(package_name: &str) -> Result<Option<CustomNpmPackage>> { - use ring::digest::Context; - use ring::digest::SHA512; - let package_folder = testdata_path().join("npm/registry").join(package_name); if !package_folder.exists() { return Ok(None); @@ -103,10 +100,7 @@ fn get_npm_package(package_name: &str) -> Result<Option<CustomNpmPackage>> { } // get tarball hash - let mut hash_ctx = Context::new(&SHA512); - hash_ctx.update(&tarball_bytes); - let digest = hash_ctx.finish(); - let tarball_checksum = BASE64_STANDARD.encode(digest.as_ref()); + let tarball_checksum = get_tarball_checksum(&tarball_bytes); // create the registry file JSON for this version let mut dist = serde_json::Map::new(); @@ -176,3 +170,10 @@ fn get_npm_package(package_name: &str) -> Result<Option<CustomNpmPackage>> { tarballs, })) } + +fn get_tarball_checksum(bytes: &[u8]) -> String { + use sha2::Digest; + let mut hasher = sha2::Sha512::new(); + hasher.update(bytes); + BASE64_STANDARD.encode(hasher.finalize()) +} diff --git a/test_util/src/servers/registry.rs b/test_util/src/servers/registry.rs index 69728f706..0efe06217 100644 --- a/test_util/src/servers/registry.rs +++ b/test_util/src/servers/registry.rs @@ -13,9 +13,14 @@ use hyper::body::Incoming; use hyper::Request; use hyper::Response; use hyper::StatusCode; +use once_cell::sync::Lazy; use serde_json::json; +use std::collections::BTreeMap; +use std::collections::HashMap; use std::convert::Infallible; use std::net::SocketAddr; +use std::path::Path; +use std::sync::Mutex; pub async fn registry_server(port: u16) { let registry_server_addr = SocketAddr::from(([127, 0, 0, 1], port)); @@ -66,6 +71,27 @@ async fn registry_server_handler( testdata_path().to_path_buf().join("jsr").join("registry"); file_path.push(&req.uri().path()[1..].replace("%2f", "/")); if let Ok(body) = tokio::fs::read(&file_path).await { + let body = if let Some(version) = file_path + .file_name() + .unwrap() + .to_string_lossy() + .strip_suffix("_meta.json") + { + // fill the manifest with checksums found in the directory so that + // we don't need to maintain them manually in the testdata directory + let mut meta: serde_json::Value = serde_json::from_slice(&body)?; + let mut manifest = + manifest_sorted(meta.get("manifest").cloned().unwrap_or(json!({}))); + let version_dir = file_path.parent().unwrap().join(version); + fill_manifest_at_dir(&mut manifest, &version_dir); + meta + .as_object_mut() + .unwrap() + .insert("manifest".to_string(), json!(manifest)); + serde_json::to_string(&meta).unwrap().into_bytes() + } else { + body + }; return Ok(Response::new(UnsyncBoxBody::new( http_body_util::Full::new(Bytes::from(body)), ))); @@ -77,3 +103,80 @@ async fn registry_server_handler( .body(empty_body)?; Ok(res) } + +fn manifest_sorted( + meta: serde_json::Value, +) -> BTreeMap<String, serde_json::Value> { + let mut manifest = BTreeMap::new(); + if let serde_json::Value::Object(files) = meta { + for (file, checksum) in files { + manifest.insert(file.clone(), checksum.clone()); + } + } + manifest +} + +fn fill_manifest_at_dir( + manifest: &mut BTreeMap<String, serde_json::Value>, + dir: &Path, +) { + let file_system_manifest = get_manifest_entries_for_dir(dir); + for (file_path, value) in file_system_manifest { + manifest.entry(file_path).or_insert(value); + } +} + +static DIR_MANIFEST_CACHE: Lazy< + Mutex<HashMap<String, BTreeMap<String, serde_json::Value>>>, +> = Lazy::new(Default::default); + +fn get_manifest_entries_for_dir( + dir: &Path, +) -> BTreeMap<String, serde_json::Value> { + fn inner_fill( + root_dir: &Path, + dir: &Path, + manifest: &mut BTreeMap<String, serde_json::Value>, + ) { + for entry in std::fs::read_dir(dir).unwrap() { + let entry = entry.unwrap(); + let path = entry.path(); + if path.is_file() { + let file_bytes = std::fs::read(&path).unwrap(); + let checksum = format!("sha256-{}", get_checksum(&file_bytes)); + let relative_path = path + .to_string_lossy() + .strip_prefix(&root_dir.to_string_lossy().to_string()) + .unwrap() + .replace('\\', "/"); + manifest.insert( + relative_path, + json!({ + "size": file_bytes.len(), + "checksum": checksum, + }), + ); + } else if path.is_dir() { + inner_fill(root_dir, &path, manifest); + } + } + } + + DIR_MANIFEST_CACHE + .lock() + .unwrap() + .entry(dir.to_string_lossy().to_string()) + .or_insert_with(|| { + let mut manifest = BTreeMap::new(); + inner_fill(dir, dir, &mut manifest); + manifest + }) + .clone() +} + +fn get_checksum(bytes: &[u8]) -> String { + use sha2::Digest; + let mut hasher = sha2::Sha256::new(); + hasher.update(bytes); + format!("{:x}", hasher.finalize()) +} diff --git a/tests/integration/jsr_tests.rs b/tests/integration/jsr_tests.rs index 51cfcfaac..b6e4d8a4f 100644 --- a/tests/integration/jsr_tests.rs +++ b/tests/integration/jsr_tests.rs @@ -1,5 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use deno_core::serde_json::json; use deno_core::serde_json::Value; use deno_lockfile::Lockfile; use test_util as util; @@ -182,3 +183,167 @@ fn reload_info_not_found_cache_but_exists_remote() { )) .assert_exit_code(0); } + +#[test] +fn lockfile_bad_package_integrity() { + let test_context = TestContextBuilder::for_jsr().use_temp_cwd().build(); + let temp_dir = test_context.temp_dir(); + + temp_dir.write( + "main.ts", + r#"import version from "jsr:@denotest/no_module_graph@0.1"; + +console.log(version);"#, + ); + temp_dir.write("deno.json", "{}"); // to automatically create a lockfile + + test_context + .new_command() + .args("run --quiet main.ts") + .run() + .assert_matches_text("0.1.1\n"); + + let lockfile_path = temp_dir.path().join("deno.lock"); + let mut lockfile = Lockfile::new(lockfile_path.to_path_buf(), false).unwrap(); + let pkg_name = "@denotest/no_module_graph@0.1.1"; + let original_integrity = get_lockfile_pkg_integrity(&lockfile, pkg_name); + set_lockfile_pkg_integrity(&mut lockfile, pkg_name, "bad_integrity"); + lockfile_path.write(lockfile.as_json_string()); + + let actual_integrity = + test_context.get_jsr_package_integrity("@denotest/no_module_graph/0.1.1"); + let integrity_check_failed_msg = format!("error: Integrity check failed for http://127.0.0.1:4250/@denotest/no_module_graph/0.1.1_meta.json + +Actual: {} +Expected: bad_integrity + at file:///[WILDCARD]/main.ts:1:21 +", actual_integrity); + test_context + .new_command() + .args("run --quiet main.ts") + .run() + .assert_matches_text(&integrity_check_failed_msg) + .assert_exit_code(1); + + // now try with a vendor folder + temp_dir + .path() + .join("deno.json") + .write_json(&json!({ "vendor": true })); + + // should fail again + test_context + .new_command() + .args("run --quiet main.ts") + .run() + .assert_matches_text(&integrity_check_failed_msg) + .assert_exit_code(1); + + // now update to the correct integrity + set_lockfile_pkg_integrity(&mut lockfile, pkg_name, &original_integrity); + lockfile_path.write(lockfile.as_json_string()); + + // should pass now + test_context + .new_command() + .args("run --quiet main.ts") + .run() + .assert_matches_text("0.1.1\n") + .assert_exit_code(0); + + // now update to a bad integrity again + set_lockfile_pkg_integrity(&mut lockfile, pkg_name, "bad_integrity"); + lockfile_path.write(lockfile.as_json_string()); + + // shouldn't matter because we have a vendor folder + test_context + .new_command() + .args("run --quiet main.ts") + .run() + .assert_matches_text("0.1.1\n") + .assert_exit_code(0); + + // now remove the vendor dir and it should fail again + temp_dir.path().join("vendor").remove_dir_all(); + + test_context + .new_command() + .args("run --quiet main.ts") + .run() + .assert_matches_text(&integrity_check_failed_msg) + .assert_exit_code(1); +} + +#[test] +fn bad_manifest_checksum() { + let test_context = TestContextBuilder::for_jsr().use_temp_cwd().build(); + let temp_dir = test_context.temp_dir(); + + temp_dir.write( + "main.ts", + r#"import { add } from "jsr:@denotest/bad-manifest-checksum@1.0.0"; +console.log(add);"#, + ); + + // test it properly checks the checksum on download + test_context + .new_command() + .args("run main.ts") + .run() + .assert_matches_text( + "Download http://127.0.0.1:4250/@denotest/bad-manifest-checksum/meta.json +Download http://127.0.0.1:4250/@denotest/bad-manifest-checksum/1.0.0_meta.json +Download http://127.0.0.1:4250/@denotest/bad-manifest-checksum/1.0.0/mod.ts +error: Integrity check failed. + +Actual: 9a30ac96b5d5c1b67eca69e1e2cf0798817d9578c8d7d904a81a67b983b35cba +Expected: bad-checksum + at file:///[WILDCARD]main.ts:1:21 +", + ) + .assert_exit_code(1); + + // test it properly checks the checksum when loading from the cache + test_context + .new_command() + .args("run main.ts") + .run() + .assert_matches_text( + // ideally the two error messages would be the same... this one comes from + // deno_cache and the one above comes from deno_graph. The thing is, in deno_cache + // (source of this error) it makes sense to include the url in the error message + // because it's not always used in the context of deno_graph + "error: Integrity check failed for http://127.0.0.1:4250/@denotest/bad-manifest-checksum/1.0.0/mod.ts + +Actual: 9a30ac96b5d5c1b67eca69e1e2cf0798817d9578c8d7d904a81a67b983b35cba +Expected: bad-checksum + at file:///[WILDCARD]main.ts:1:21 +", + ) + .assert_exit_code(1); +} + +fn get_lockfile_pkg_integrity(lockfile: &Lockfile, pkg_name: &str) -> String { + lockfile + .content + .packages + .jsr + .get(pkg_name) + .unwrap() + .integrity + .clone() +} + +fn set_lockfile_pkg_integrity( + lockfile: &mut Lockfile, + pkg_name: &str, + integrity: &str, +) { + lockfile + .content + .packages + .jsr + .get_mut(pkg_name) + .unwrap() + .integrity = integrity.to_string(); +} diff --git a/tests/integration/lsp_tests.rs b/tests/integration/lsp_tests.rs index 749af95c4..97e9215df 100644 --- a/tests/integration/lsp_tests.rs +++ b/tests/integration/lsp_tests.rs @@ -693,11 +693,19 @@ fn lsp_format_vendor_path() { .use_http_server() .use_temp_cwd() .build(); + + // put this dependency in the global cache + context + .new_command() + .args("cache http://localhost:4545/run/002_hello.ts") + .run() + .skip_output_check(); + let temp_dir = context.temp_dir(); temp_dir.write("deno.json", json!({ "vendor": true }).to_string()); let mut client = context.new_lsp_command().build(); client.initialize_default(); - client.did_open(json!({ + let diagnostics = client.did_open(json!({ "textDocument": { "uri": "file:///a/file.ts", "languageId": "typescript", @@ -705,6 +713,18 @@ fn lsp_format_vendor_path() { "text": r#"import "http://localhost:4545/run/002_hello.ts";"#, }, })); + // copying from the global cache to the local cache requires explicitly + // running the cache command so that the checksums can be verified + assert_eq!( + diagnostics + .all() + .iter() + .map(|d| d.message.as_str()) + .collect::<Vec<_>>(), + vec![ + "Uncached or missing remote URL: http://localhost:4545/run/002_hello.ts" + ] + ); client.write_request( "workspace/executeCommand", json!({ @@ -4358,7 +4378,8 @@ fn lsp_code_actions() { }]) ); let res = client - .write_request( "codeAction/resolve", + .write_request( + "codeAction/resolve", json!({ "title": "Add all missing 'async' modifiers", "kind": "quickfix", @@ -4378,8 +4399,7 @@ fn lsp_code_actions() { "fixId": "fixAwaitInSyncFunction" } }), - ) - ; + ); assert_eq!( res, json!({ @@ -4762,26 +4782,27 @@ fn lsp_code_actions_deno_cache_jsr() { #[test] fn lsp_jsr_lockfile() { - let context = TestContextBuilder::new() - .use_http_server() - .use_temp_cwd() - .build(); + let context = TestContextBuilder::for_jsr().use_temp_cwd().build(); let temp_dir = context.temp_dir(); temp_dir.write("./deno.json", json!({}).to_string()); - temp_dir.write( - "./deno.lock", - json!({ - "version": "3", - "packages": { - "specifiers": { - // This is an old version of the package which exports `sum()` instead - // of `add()`. - "jsr:@denotest/add": "jsr:@denotest/add@0.2.0", - }, - }, - }) - .to_string(), - ); + let lockfile = temp_dir.path().join("deno.lock"); + let integrity = context.get_jsr_package_integrity("@denotest/add/0.2.0"); + lockfile.write_json(&json!({ + "version": "3", + "packages": { + "specifiers": { + // This is an old version of the package which exports `sum()` instead + // of `add()`. + "jsr:@denotest/add": "jsr:@denotest/add@0.2.0", + }, + "jsr": { + "@denotest/add@0.2.0": { + "integrity": integrity + } + } + }, + "remote": {}, + })); let mut client = context.new_lsp_command().build(); client.initialize_default(); client.did_open(json!({ @@ -4790,8 +4811,8 @@ fn lsp_jsr_lockfile() { "languageId": "typescript", "version": 1, "text": r#" - import { add } from "jsr:@denotest/add"; - console.log(add(1, 2)); + import { sum } from "jsr:@denotest/add"; + console.log(sum(1, 2)); "#, }, })); @@ -10672,9 +10693,27 @@ fn lsp_vendor_dir() { refresh_config(&mut client); let diagnostics = client.read_diagnostics(); - assert_eq!(diagnostics.all().len(), 0, "{:#?}", diagnostics); // cached + // won't be cached until a manual cache occurs + assert_eq!( + diagnostics + .all() + .iter() + .map(|d| d.message.as_str()) + .collect::<Vec<_>>(), + vec![ + "Uncached or missing remote URL: http://localhost:4545/subdir/mod1.ts" + ] + ); + + assert!(!temp_dir + .path() + .join("vendor/http_localhost_4545/subdir/mod1.ts") + .exists()); - // no caching necessary because it was already cached. It should exist now + // now cache + cache(&mut client); + let diagnostics = client.read_diagnostics(); + assert_eq!(diagnostics.all().len(), 0, "{:#?}", diagnostics); // cached assert!(temp_dir .path() .join("vendor/http_localhost_4545/subdir/mod1.ts") diff --git a/tests/integration/npm_tests.rs b/tests/integration/npm_tests.rs index 3777bfe8a..33e331fc3 100644 --- a/tests/integration/npm_tests.rs +++ b/tests/integration/npm_tests.rs @@ -1549,7 +1549,7 @@ fn auto_discover_lock_file() { output .assert_matches_text( r#"Download http://localhost:4545/npm/registry/@denotest/bin -error: Integrity check failed for npm package: "@denotest/bin@1.0.0". Unable to verify that the package +error: Integrity check failed for package: "npm:@denotest/bin@1.0.0". Unable to verify that the package is the same as when the lockfile was generated. Actual: sha512-[WILDCARD] diff --git a/tests/integration/run_tests.rs b/tests/integration/run_tests.rs index 6f108f739..68b72ffed 100644 --- a/tests/integration/run_tests.rs +++ b/tests/integration/run_tests.rs @@ -1053,7 +1053,9 @@ fn lock_deno_json_package_json_deps() { "npm:@denotest/esm-basic": "npm:@denotest/esm-basic@1.0.0" }, "jsr": { - "@denotest/module_graph@1.4.0": {} + "@denotest/module_graph@1.4.0": { + "integrity": "555bbe259f55a4a2e7a39e8bf4bcbf25da4c874a313c3e98771eddceedac050b" + } }, "npm": { "@denotest/esm-basic@1.0.0": { @@ -1062,10 +1064,7 @@ fn lock_deno_json_package_json_deps() { } } }, - "remote": { - "http://127.0.0.1:4250/@denotest/module_graph/1.4.0/mod.ts": "5b0ce36e08d759118200d8b4627627b5a89b6261fbb0598e6961a6b287abb699", - "http://127.0.0.1:4250/@denotest/module_graph/1.4.0/other.ts": "9ce27ca439cb0e218b6e1ec26c043dbc0b54c9babc4cb432df478dd1721faade" - }, + "remote": {}, "workspace": { "dependencies": [ "jsr:@denotest/module_graph@1.4", @@ -1106,7 +1105,9 @@ fn lock_deno_json_package_json_deps() { "npm:@denotest/esm-basic": "npm:@denotest/esm-basic@1.0.0" }, "jsr": { - "@denotest/module_graph@1.4.0": {} + "@denotest/module_graph@1.4.0": { + "integrity": "555bbe259f55a4a2e7a39e8bf4bcbf25da4c874a313c3e98771eddceedac050b" + } }, "npm": { "@denotest/esm-basic@1.0.0": { @@ -1115,10 +1116,7 @@ fn lock_deno_json_package_json_deps() { } } }, - "remote": { - "http://127.0.0.1:4250/@denotest/module_graph/1.4.0/mod.ts": "5b0ce36e08d759118200d8b4627627b5a89b6261fbb0598e6961a6b287abb699", - "http://127.0.0.1:4250/@denotest/module_graph/1.4.0/other.ts": "9ce27ca439cb0e218b6e1ec26c043dbc0b54c9babc4cb432df478dd1721faade" - }, + "remote": {}, "workspace": { "dependencies": [ "jsr:@denotest/module_graph@1.4" @@ -1147,13 +1145,12 @@ fn lock_deno_json_package_json_deps() { "jsr:@denotest/module_graph@1.4": "jsr:@denotest/module_graph@1.4.0", }, "jsr": { - "@denotest/module_graph@1.4.0": {} + "@denotest/module_graph@1.4.0": { + "integrity": "555bbe259f55a4a2e7a39e8bf4bcbf25da4c874a313c3e98771eddceedac050b" + } } }, - "remote": { - "http://127.0.0.1:4250/@denotest/module_graph/1.4.0/mod.ts": "5b0ce36e08d759118200d8b4627627b5a89b6261fbb0598e6961a6b287abb699", - "http://127.0.0.1:4250/@denotest/module_graph/1.4.0/other.ts": "9ce27ca439cb0e218b6e1ec26c043dbc0b54c9babc4cb432df478dd1721faade" - }, + "remote": {}, "workspace": { "dependencies": [ "jsr:@denotest/module_graph@1.4" diff --git a/tests/testdata/jsr/registry/@denotest/bad-manifest-checksum/1.0.0/mod.ts b/tests/testdata/jsr/registry/@denotest/bad-manifest-checksum/1.0.0/mod.ts new file mode 100644 index 000000000..8d9b8a22a --- /dev/null +++ b/tests/testdata/jsr/registry/@denotest/bad-manifest-checksum/1.0.0/mod.ts @@ -0,0 +1,3 @@ +export function add(a: number, b: number): number { + return a + b; +} diff --git a/tests/testdata/jsr/registry/@denotest/bad-manifest-checksum/1.0.0_meta.json b/tests/testdata/jsr/registry/@denotest/bad-manifest-checksum/1.0.0_meta.json new file mode 100644 index 000000000..8ef8ab3c3 --- /dev/null +++ b/tests/testdata/jsr/registry/@denotest/bad-manifest-checksum/1.0.0_meta.json @@ -0,0 +1,11 @@ +{ + "exports": { + ".": "./mod.ts" + }, + "manifest": { + "/mod.ts": { + "size": 0, + "checksum": "sha256-bad-checksum" + } + } +} diff --git a/tests/testdata/jsr/registry/@denotest/bad-manifest-checksum/meta.json b/tests/testdata/jsr/registry/@denotest/bad-manifest-checksum/meta.json new file mode 100644 index 000000000..02601e4d0 --- /dev/null +++ b/tests/testdata/jsr/registry/@denotest/bad-manifest-checksum/meta.json @@ -0,0 +1,5 @@ +{ + "versions": { + "1.0.0": {} + } +} diff --git a/tests/testdata/npm/lock_file/main.out b/tests/testdata/npm/lock_file/main.out index 65e881be6..dead1a623 100644 --- a/tests/testdata/npm/lock_file/main.out +++ b/tests/testdata/npm/lock_file/main.out @@ -1,5 +1,5 @@ Download [WILDCARD] -error: Integrity check failed for npm package: "@babel/parser@7.19.0". Unable to verify that the package +error: Integrity check failed for package: "npm:@babel/parser@7.19.0". Unable to verify that the package is the same as when the lockfile was generated. Actual: sha512-74bEXKX2h+8rrfQUfsBfuZZHzsEs6Eql4pqy/T4Nn6Y9wNPggQOqD6z6pn5Bl8ZfysKouFZT/UXEH94ummEeQw== |