diff options
27 files changed, 767 insertions, 243 deletions
diff --git a/Cargo.lock b/Cargo.lock index aa07029b1..89cc00b2d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -129,6 +129,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0942ffc6dcaadf03badf6e6a2d0228460359d5e34b57ccdc720b7382dfbd5ec5" [[package]] +name = "ammonia" +version = "3.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "64e6d1c7838db705c9b756557ee27c384ce695a1c51a6fe528784cb1c6840170" +dependencies = [ + "html5ever", + "maplit", + "once_cell", + "tendril", + "url", +] + +[[package]] name = "android_system_properties" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -1061,7 +1074,7 @@ dependencies = [ "open", "os_pipe", "percent-encoding", - "phf", + "phf 0.11.2", "pin-project", "pretty_assertions", "quick-junit", @@ -1097,9 +1110,9 @@ dependencies = [ [[package]] name = "deno_ast" -version = "0.33.2" +version = "0.33.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fdafff817ae3ad89672d54cd8daebc86dc352065ccc18691605043e6b845d00" +checksum = "7f61944e781d268799bf65857e664d3c09a37590043d4b0ed10facefc9bea473" dependencies = [ "anyhow", "base64", @@ -1172,9 +1185,9 @@ dependencies = [ [[package]] name = "deno_cache_dir" -version = "0.6.1" +version = "0.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2bbb245d9a3719b5eb2b5195aaaa25108c3c93d1762b181a20fb1af1c7703eaf" +checksum = "6cf517bddfd22d79d0f284500318e3f9aea193536c2b61cbf6ce7b50a85f1b6a" dependencies = [ "anyhow", "deno_media_type", @@ -1182,9 +1195,9 @@ dependencies = [ "log", "once_cell", "parking_lot 0.12.1", - "ring", "serde", "serde_json", + "sha2", "thiserror", "url", ] @@ -1310,10 +1323,11 @@ dependencies = [ [[package]] name = "deno_doc" -version = "0.103.0" +version = "0.107.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73fe6bd8144456ca3f01b8d1cd1b668b974c84dc94cb642936c0938348b17017" +checksum = "f82478f27de7958eb6a1e48e447b8cb030a1294097ef510eec190d29e81f330f" dependencies = [ + "ammonia", "anyhow", "cfg-if", "comrak", @@ -1328,15 +1342,14 @@ dependencies = [ "regex", "serde", "serde_json", - "syntect", "termcolor", ] [[package]] name = "deno_emit" -version = "0.36.0" +version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c5002f2c25489fb993132dc0cb0dabd41bae70a8629168db4bd726ee2e296ac" +checksum = "a670c56f233f85f18f1d4a3288c5241505d8aea559fe3870b45e00d4c0e731dc" dependencies = [ "anyhow", "base64", @@ -1404,9 +1417,9 @@ dependencies = [ [[package]] name = "deno_graph" -version = "0.65.3" +version = "0.66.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4d12c87f92df950ad0eed3ea8951f30bf9c54f69b3a903b805950d6761b35002" +checksum = "8c67c7c05d70e43560b1dfa38ee385d2d0153ccd4ea16fdc6a706881fd60f3c5" dependencies = [ "anyhow", "async-trait", @@ -1424,6 +1437,7 @@ dependencies = [ "regex", "serde", "serde_json", + "sha2", "thiserror", "url", ] @@ -1455,7 +1469,7 @@ dependencies = [ "mime", "once_cell", "percent-encoding", - "phf", + "phf 0.11.2", "pin-project", "rand", "ring", @@ -1531,13 +1545,13 @@ dependencies = [ [[package]] name = "deno_lockfile" -version = "0.18.2" +version = "0.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9f348633cc4425b2a9011436e256b1ae8f6c8026ec2705d852baee8643dc5562" +checksum = "8835418ae924f25ab20f508bf6240193b22d893519d44432b670a27b8fb1efeb" dependencies = [ - "ring", "serde", "serde_json", + "sha2", "thiserror", ] @@ -1659,9 +1673,9 @@ dependencies = [ [[package]] name = "deno_npm" -version = "0.16.0" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "376262760b173ff01f8f5d05d58a64f6d863472396afb5582590fa0949342854" +checksum = "53a333104d3fb6aa52e499384e523aefc09d3ac8ecd05ca7f65f856044fbcb09" dependencies = [ "anyhow", "async-trait", @@ -2414,9 +2428,9 @@ dependencies = [ [[package]] name = "eszip" -version = "0.62.0" +version = "0.63.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a26aa6791e6021e9e3ffc6bc8ab00ff2d0d748c64a75b7333076d973ce32f6b" +checksum = "731a0e44e886cb8efbbd63b8121341d505e9dab855fe487249d70c362a6bd774" dependencies = [ "anyhow", "base64", @@ -2649,6 +2663,16 @@ dependencies = [ ] [[package]] +name = "futf" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df420e2e84819663797d1ec6544b13c5be84629e7bb00dc960d6917db2987843" +dependencies = [ + "mac", + "new_debug_unreachable", +] + +[[package]] name = "futures" version = "0.3.29" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3059,7 +3083,7 @@ checksum = "de90d3db62411eb62eddabe402d706ac4970f7ac8d088c05f11069cad9be9857" dependencies = [ "new_debug_unreachable", "once_cell", - "phf", + "phf 0.11.2", "rustc-hash", "smallvec", ] @@ -3074,6 +3098,20 @@ dependencies = [ ] [[package]] +name = "html5ever" +version = "0.26.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bea68cab48b8459f17cf1c944c67ddc572d272d9f2b274140f223ecb1da4a3b7" +dependencies = [ + "log", + "mac", + "markup5ever", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] name = "http" version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3732,6 +3770,12 @@ dependencies = [ ] [[package]] +name = "mac" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c41e0c4fef86961ac6d6f8a82609f55f31b05e4fce149ac5710e439df7619ba4" + +[[package]] name = "malloc_buf" version = "0.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -3741,6 +3785,26 @@ dependencies = [ ] [[package]] +name = "maplit" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e2e65a1a2e43cfcb47a895c4c8b10d1f4a61097f9f254f183aee60cad9c651d" + +[[package]] +name = "markup5ever" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a2629bb1404f3d34c2e921f21fd34ba00b206124c81f65c50b43b6aaefeb016" +dependencies = [ + "log", + "phf 0.10.1", + "phf_codegen", + "string_cache", + "string_cache_codegen", + "tendril", +] + +[[package]] name = "match_cfg" version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -4148,28 +4212,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" [[package]] -name = "onig" -version = "6.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c4b31c8722ad9171c6d77d3557db078cab2bd50afcc9d09c8b315c59df8ca4f" -dependencies = [ - "bitflags 1.3.2", - "libc", - "once_cell", - "onig_sys", -] - -[[package]] -name = "onig_sys" -version = "69.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b829e3d7e9cc74c7e315ee8edb185bf4190da5acde74afd7fc59c35b1f086e7" -dependencies = [ - "cc", - "pkg-config", -] - -[[package]] name = "opaque-debug" version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -4435,12 +4477,41 @@ dependencies = [ [[package]] name = "phf" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fabbf1ead8a5bcbc20f5f8b939ee3f5b0f6f281b6ad3468b84656b658b455259" +dependencies = [ + "phf_shared 0.10.0", +] + +[[package]] +name = "phf" version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ade2d8b8f33c7333b51bcf0428d37e217e9f32192ae4772156f65063b8ce03dc" dependencies = [ "phf_macros", - "phf_shared", + "phf_shared 0.11.2", +] + +[[package]] +name = "phf_codegen" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fb1c3a8bc4dd4e5cfce29b44ffc14bedd2ee294559a294e2a4d4c9e9a6a13cd" +dependencies = [ + "phf_generator 0.10.0", + "phf_shared 0.10.0", +] + +[[package]] +name = "phf_generator" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d5285893bb5eb82e6aaf5d59ee909a06a16737a8970984dd7746ba9283498d6" +dependencies = [ + "phf_shared 0.10.0", + "rand", ] [[package]] @@ -4449,7 +4520,7 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "48e4cc64c2ad9ebe670cb8fd69dd50ae301650392e81c05f9bfcb2d5bdbc24b0" dependencies = [ - "phf_shared", + "phf_shared 0.11.2", "rand", ] @@ -4459,8 +4530,8 @@ version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3444646e286606587e49f3bcf1679b8cef1dc2c5ecc29ddacaffc305180d464b" dependencies = [ - "phf_generator", - "phf_shared", + "phf_generator 0.11.2", + "phf_shared 0.11.2", "proc-macro2", "quote", "syn 2.0.48", @@ -4468,6 +4539,15 @@ dependencies = [ [[package]] name = "phf_shared" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6796ad771acdc0123d2a88dc428b5e38ef24456743ddb1744ed628f9815c096" +dependencies = [ + "siphasher", +] + +[[package]] +name = "phf_shared" version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "90fcb95eef784c2ac79119d1dd819e162b5da872ce6f3c3abe1e8ca1c082f72b" @@ -4589,6 +4669,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5b40af805b3121feab8a3c29f04d8ad262fa8e0561883e7653e024ae4479e6de" [[package]] +name = "precomputed-hash" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "925383efa346730478fb4838dbe9137d2a47675ad789c546d150a6e1dd4ab31c" + +[[package]] name = "presser" version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -4923,7 +5009,7 @@ dependencies = [ "aho-corasick", "memchr", "regex-automata", - "regex-syntax 0.8.2", + "regex-syntax", ] [[package]] @@ -4934,17 +5020,11 @@ checksum = "5f804c7828047e88b2d32e2d7fe5a105da8ee3264f01902f796c8e067dc2483f" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.2", + "regex-syntax", ] [[package]] name = "regex-syntax" -version = "0.7.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbb5fb1acd8a1a18b3dd5be62d25485eb770e05afb408a9627d14d451bae12da" - -[[package]] -name = "regex-syntax" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c08c74e62047bb2de4ff487b251e4a92e24f48745648451635cec7d591162d9f" @@ -5704,6 +5784,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] +name = "string_cache" +version = "0.8.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f91138e76242f575eb1d3b38b4f1362f10d3a43f47d182a5b359af488a02293b" +dependencies = [ + "new_debug_unreachable", + "once_cell", + "parking_lot 0.12.1", + "phf_shared 0.10.0", + "precomputed-hash", + "serde", +] + +[[package]] +name = "string_cache_codegen" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6bb30289b722be4ff74a408c3cc27edeaad656e06cb1fe8fa9231fa59c728988" +dependencies = [ + "phf_generator 0.10.0", + "phf_shared 0.10.0", + "proc-macro2", + "quote", +] + +[[package]] name = "string_enum" version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -5772,9 +5878,9 @@ dependencies = [ [[package]] name = "swc_bundler" -version = "0.225.3" +version = "0.225.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26491762e84ae1d0a2e179fe48066072834777a1b12e8e88a7f07c8f92cc0188" +checksum = "feb8b6f3ad184a5ae21544411491bf136635237fc097d7c93ccd915449ebb2ba" dependencies = [ "anyhow", "crc", @@ -5875,7 +5981,7 @@ dependencies = [ "bitflags 2.4.1", "is-macro", "num-bigint", - "phf", + "phf 0.11.2", "scoped-tls", "serde", "string_enum", @@ -5939,7 +6045,7 @@ dependencies = [ "new_debug_unreachable", "num-bigint", "num-traits", - "phf", + "phf 0.11.2", "serde", "smallvec", "smartstring", @@ -5953,15 +6059,15 @@ dependencies = [ [[package]] name = "swc_ecma_transforms_base" -version = "0.137.3" +version = "0.137.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfd47dd9ccb73a1f5d8d7eff9518554b752b1733b56503af090e78859abb42dd" +checksum = "803bb435fdd532d5c931f0d487e48dbc94750d26c9336d79a6f1c04c62f08d93" dependencies = [ "better_scoped_tls", "bitflags 2.4.1", "indexmap", "once_cell", - "phf", + "phf 0.11.2", "rustc-hash", "serde", "smallvec", @@ -5976,9 +6082,9 @@ dependencies = [ [[package]] name = "swc_ecma_transforms_classes" -version = "0.126.3" +version = "0.126.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ecb31417e0d415d7f0ff026f1e7c909427e386b7d0af9a2a78678507e4d9d79" +checksum = "486479e75907547d4c65ca6deed8faa465a2e9475cc9605be36a0e5eb609f578" dependencies = [ "swc_atoms", "swc_common", @@ -6002,9 +6108,9 @@ dependencies = [ [[package]] name = "swc_ecma_transforms_optimization" -version = "0.198.3" +version = "0.198.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3920268ac8972b494067d0b7c088964b21d08f5d1f58d7151bd1eb7054a137b0" +checksum = "f3c6fffe4d6e3609fdd6c768cc063dbc9f5101f9be0db1168ec76ace979cf616" dependencies = [ "dashmap", "indexmap", @@ -6026,9 +6132,9 @@ dependencies = [ [[package]] name = "swc_ecma_transforms_proposal" -version = "0.171.3" +version = "0.171.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "448c40c2a2b224cb5101cc6cdee81837c281a34f2a2aa6dd18d6d5cd8d492e60" +checksum = "e2822bc6c28bb1a96090a3b2caa28f45efbaecb333714d30868a2390eaae943f" dependencies = [ "either", "rustc-hash", @@ -6046,9 +6152,9 @@ dependencies = [ [[package]] name = "swc_ecma_transforms_react" -version = "0.183.3" +version = "0.183.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee2394dc3abceada246feeb709b8c4d23392973f49a24fcc59b2ee21737cb6c8" +checksum = "8984ebb8955116c426457a0c70b0aa9a08a06656e245781ff617a9cd1e289697" dependencies = [ "base64", "dashmap", @@ -6070,9 +6176,9 @@ dependencies = [ [[package]] name = "swc_ecma_transforms_typescript" -version = "0.188.3" +version = "0.188.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cff231437173e041e5a3be9b8c782fd297ffcb53ed16d805f853e4a68315c45" +checksum = "a2e898fbab993abb60fb67009521908f2317f1d33f804e5b38d769f52e58572e" dependencies = [ "ryu-js", "serde", @@ -6087,9 +6193,9 @@ dependencies = [ [[package]] name = "swc_ecma_utils" -version = "0.127.3" +version = "0.127.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cd185161161dfc65ee0d6f3044c901b766c3abb4efcd0b35c9e76c833724896" +checksum = "8ff9e77ea18468895d26bd38656885860fede2acd24d1687f64363aaf8910441" dependencies = [ "indexmap", "num_cpus", @@ -6223,25 +6329,6 @@ dependencies = [ ] [[package]] -name = "syntect" -version = "5.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e02b4b303bf8d08bfeb0445cba5068a3d306b6baece1d5582171a9bf49188f91" -dependencies = [ - "bincode", - "bitflags 1.3.2", - "flate2", - "fnv", - "once_cell", - "onig", - "regex-syntax 0.7.5", - "serde", - "serde_json", - "thiserror", - "walkdir", -] - -[[package]] name = "tar" version = "0.4.40" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -6266,6 +6353,17 @@ dependencies = [ ] [[package]] +name = "tendril" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d24a120c5fc464a3458240ee02c299ebcb9d67b5249c8848b09d639dca8d7bb0" +dependencies = [ + "futf", + "mac", + "utf-8", +] + +[[package]] name = "termcolor" version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" @@ -6324,12 +6422,12 @@ dependencies = [ "prost-build", "regex", "reqwest", - "ring", "rustls-pemfile", "rustls-tokio-stream", "semver 1.0.14", "serde", "serde_json", + "sha2", "tar", "tempfile", "termcolor", diff --git a/Cargo.toml b/Cargo.toml index e54176c90..7b0686432 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -46,7 +46,7 @@ deno_ast = { version = "0.33.2", features = ["transpiling"] } deno_core = { version = "0.262.0" } deno_bench_util = { version = "0.132.0", path = "./bench_util" } -deno_lockfile = "0.18.2" +deno_lockfile = "0.19.0" deno_media_type = { version = "0.1.1", features = ["module_specifier"] } deno_runtime = { version = "0.146.0", path = "./runtime" } deno_terminal = "0.1.1" @@ -97,7 +97,7 @@ chrono = { version = "0.4", default-features = false, features = ["std", "serde" console_static_text = "=0.8.1" data-encoding = "2.3.3" data-url = "=0.3.0" -deno_cache_dir = "=0.6.1" +deno_cache_dir = "=0.7.1" dlopen2 = "0.6.1" ecb = "=0.1.2" elliptic-curve = { version = "0.13.4", features = ["alloc", "arithmetic", "ecdh", "std", "pem"] } diff --git a/cli/Cargo.toml b/cli/Cargo.toml index ca00774ea..3898fb7bb 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -66,17 +66,17 @@ deno_ast = { workspace = true, features = ["bundler", "cjs", "codegen", "proposa deno_cache_dir = { workspace = true } deno_config = "=0.9.2" deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] } -deno_doc = { version = "=0.103.0", features = ["html"] } -deno_emit = "=0.36.0" -deno_graph = "=0.65.3" +deno_doc = { version = "=0.107.0", features = ["html"] } +deno_emit = "=0.37.0" +deno_graph = "=0.66.0" deno_lint = { version = "=0.56.0", features = ["docs"] } deno_lockfile.workspace = true -deno_npm = "=0.16.0" +deno_npm = "=0.17.0" deno_runtime = { workspace = true, features = ["include_js_files_for_snapshotting"] } deno_semver = "=0.5.4" deno_task_shell = "=0.14.3" deno_terminal.workspace = true -eszip = "=0.62.0" +eszip = "=0.63.0" napi_sym.workspace = true async-trait.workspace = true diff --git a/cli/cache/mod.rs b/cli/cache/mod.rs index 6c48799ce..e9d225146 100644 --- a/cli/cache/mod.rs +++ b/cli/cache/mod.rs @@ -196,8 +196,7 @@ impl Loader for FetchCacher { fn load( &mut self, specifier: &ModuleSpecifier, - _is_dynamic: bool, - cache_setting: deno_graph::source::CacheSetting, + options: deno_graph::source::LoadOptions, ) -> LoadFuture { use deno_graph::source::CacheSetting as LoaderCacheSetting; @@ -222,7 +221,7 @@ impl Loader for FetchCacher { let specifier = specifier.clone(); async move { - let maybe_cache_setting = match cache_setting { + let maybe_cache_setting = match options.cache_setting { LoaderCacheSetting::Use => None, LoaderCacheSetting::Reload => { if matches!(file_fetcher.cache_setting(), CacheSetting::Only) { @@ -240,6 +239,7 @@ impl Loader for FetchCacher { permissions, maybe_accept: None, maybe_cache_setting: maybe_cache_setting.as_ref(), + maybe_checksum: options.maybe_checksum, }) .await .map(|file| { @@ -269,7 +269,7 @@ impl Loader for FetchCacher { let error_class_name = get_error_class_name(&err); match error_class_name { "NotFound" => Ok(None), - "NotCached" if cache_setting == LoaderCacheSetting::Only => Ok(None), + "NotCached" if options.cache_setting == LoaderCacheSetting::Only => Ok(None), _ => Err(err), } }) diff --git a/cli/factory.rs b/cli/factory.rs index 2236e6090..cccbecbf1 100644 --- a/cli/factory.rs +++ b/cli/factory.rs @@ -66,7 +66,6 @@ use deno_runtime::deno_node::NodeResolver; use deno_runtime::deno_tls::RootCertStoreProvider; use deno_runtime::deno_web::BlobStore; use deno_runtime::inspector_server::InspectorServer; -use deno_semver::package::PackageNv; use import_map::ImportMap; use log::warn; use std::future::Future; @@ -381,16 +380,6 @@ impl CliFactory { no_npm, no_config: self.options.no_config(), config, - nv_to_jsr_url: |nv| { - let nv = PackageNv::from_str(nv).ok()?; - Some( - deno_graph::source::recommended_registry_package_url( - crate::args::jsr_url(), - &nv, - ) - .to_string(), - ) - }, }, ); } diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs index 8f4d3feab..a74a14a3f 100644 --- a/cli/file_fetcher.rs +++ b/cli/file_fetcher.rs @@ -24,6 +24,7 @@ use deno_core::futures::future::FutureExt; use deno_core::parking_lot::Mutex; use deno_core::url::Url; use deno_core::ModuleSpecifier; +use deno_graph::source::LoaderChecksum; use deno_runtime::deno_fetch::reqwest::header::HeaderValue; use deno_runtime::deno_fetch::reqwest::header::ACCEPT; use deno_runtime::deno_fetch::reqwest::header::AUTHORIZATION; @@ -146,6 +147,7 @@ pub struct FetchOptions<'a> { pub permissions: PermissionsContainer, pub maybe_accept: Option<&'a str>, pub maybe_cache_setting: Option<&'a CacheSetting>, + pub maybe_checksum: Option<LoaderChecksum>, } /// A structure for resolving, fetching and caching source files. @@ -199,6 +201,7 @@ impl FileFetcher { pub fn fetch_cached( &self, specifier: &ModuleSpecifier, + maybe_checksum: Option<LoaderChecksum>, redirect_limit: i64, ) -> Result<Option<File>, AnyError> { debug!("FileFetcher::fetch_cached - specifier: {}", specifier); @@ -207,16 +210,22 @@ impl FileFetcher { } let cache_key = self.http_cache.cache_item_key(specifier)?; // compute this once - let Some(metadata) = self.http_cache.read_metadata(&cache_key)? else { + let Some(headers) = self.http_cache.read_headers(&cache_key)? else { return Ok(None); }; - let headers = metadata.headers; if let Some(redirect_to) = headers.get("location") { let redirect = deno_core::resolve_import(redirect_to, specifier.as_str())?; - return self.fetch_cached(&redirect, redirect_limit - 1); + return self.fetch_cached(&redirect, maybe_checksum, redirect_limit - 1); } - let Some(bytes) = self.http_cache.read_file_bytes(&cache_key)? else { + let Some(bytes) = self.http_cache.read_file_bytes( + &cache_key, + maybe_checksum + .as_ref() + .map(|c| deno_cache_dir::Checksum::new(c.as_str())), + deno_cache_dir::GlobalToLocalCopy::Allow, + )? + else { return Ok(None); }; @@ -282,6 +291,7 @@ impl FileFetcher { redirect_limit: i64, maybe_accept: Option<String>, cache_setting: &CacheSetting, + maybe_checksum: Option<LoaderChecksum>, ) -> Pin<Box<dyn Future<Output = Result<File, AnyError>> + Send>> { debug!("FileFetcher::fetch_remote() - specifier: {}", specifier); if redirect_limit < 0 { @@ -294,7 +304,8 @@ impl FileFetcher { } if self.should_use_cache(specifier, cache_setting) { - match self.fetch_cached(specifier, redirect_limit) { + match self.fetch_cached(specifier, maybe_checksum.clone(), redirect_limit) + { Ok(Some(file)) => { return futures::future::ok(file).boxed(); } @@ -331,8 +342,8 @@ impl FileFetcher { .http_cache .cache_item_key(specifier) .ok() - .and_then(|key| self.http_cache.read_metadata(&key).ok().flatten()) - .and_then(|metadata| metadata.headers.get("etag").cloned()); + .and_then(|key| self.http_cache.read_headers(&key).ok().flatten()) + .and_then(|headers| headers.get("etag").cloned()); let maybe_auth_token = self.auth_tokens.get(specifier); let specifier = specifier.clone(); let client = self.http_client.clone(); @@ -376,7 +387,9 @@ impl FileFetcher { .await? { FetchOnceResult::NotModified => { - let file = file_fetcher.fetch_cached(&specifier, 10)?.unwrap(); + let file = file_fetcher + .fetch_cached(&specifier, maybe_checksum, 10)? + .unwrap(); Ok(file) } FetchOnceResult::Redirect(redirect_url, headers) => { @@ -388,6 +401,7 @@ impl FileFetcher { redirect_limit - 1, maybe_accept, &cache_setting, + maybe_checksum, ) .await } @@ -395,6 +409,9 @@ impl FileFetcher { file_fetcher .http_cache .set(&specifier, headers.clone(), &bytes)?; + if let Some(checksum) = &maybe_checksum { + checksum.check_source(&bytes)?; + } Ok(File { specifier, maybe_headers: Some(headers), @@ -438,15 +455,16 @@ impl FileFetcher { let Ok(cache_key) = self.http_cache.cache_item_key(specifier) else { return false; }; - let Ok(Some(metadata)) = self.http_cache.read_metadata(&cache_key) + let Ok(Some(headers)) = self.http_cache.read_headers(&cache_key) else { + return false; + }; + let Ok(Some(download_time)) = + self.http_cache.read_download_time(&cache_key) else { return false; }; - let cache_semantics = CacheSemantics::new( - metadata.headers, - metadata.time, - SystemTime::now(), - ); + let cache_semantics = + CacheSemantics::new(headers, download_time, SystemTime::now()); cache_semantics.should_use() } CacheSetting::ReloadSome(list) => { @@ -482,6 +500,7 @@ impl FileFetcher { permissions, maybe_accept: None, maybe_cache_setting: None, + maybe_checksum: None, }) .await } @@ -517,6 +536,7 @@ impl FileFetcher { 10, options.maybe_accept.map(String::from), options.maybe_cache_setting.unwrap_or(&self.cache_setting), + options.maybe_checksum, ) .await } @@ -728,6 +748,7 @@ mod tests { 1, None, &file_fetcher.cache_setting, + None, ) .await; let cache_key = file_fetcher.http_cache.cache_item_key(specifier).unwrap(); @@ -735,10 +756,9 @@ mod tests { result.unwrap(), file_fetcher .http_cache - .read_metadata(&cache_key) - .unwrap() + .read_headers(&cache_key) .unwrap() - .headers, + .unwrap(), ) } @@ -899,18 +919,11 @@ mod tests { let cache_item_key = file_fetcher.http_cache.cache_item_key(&specifier).unwrap(); - let mut metadata = file_fetcher - .http_cache - .read_metadata(&cache_item_key) - .unwrap() - .unwrap(); - metadata.headers = HashMap::new(); - metadata - .headers - .insert("content-type".to_string(), "text/javascript".to_string()); + let mut headers = HashMap::new(); + headers.insert("content-type".to_string(), "text/javascript".to_string()); file_fetcher .http_cache - .set(&specifier, metadata.headers.clone(), file.source.as_bytes()) + .set(&specifier, headers.clone(), file.source.as_bytes()) .unwrap(); let result = file_fetcher_01 @@ -926,20 +939,17 @@ mod tests { // the value above. assert_eq!(file.media_type, MediaType::JavaScript); - let headers = file_fetcher_02 + let headers2 = file_fetcher_02 .http_cache - .read_metadata(&cache_item_key) + .read_headers(&cache_item_key) .unwrap() - .unwrap() - .headers; - assert_eq!(headers.get("content-type").unwrap(), "text/javascript"); - metadata.headers = HashMap::new(); - metadata - .headers - .insert("content-type".to_string(), "application/json".to_string()); + .unwrap(); + assert_eq!(headers2.get("content-type").unwrap(), "text/javascript"); + headers = HashMap::new(); + headers.insert("content-type".to_string(), "application/json".to_string()); file_fetcher_02 .http_cache - .set(&specifier, metadata.headers.clone(), file.source.as_bytes()) + .set(&specifier, headers.clone(), file.source.as_bytes()) .unwrap(); let result = file_fetcher_02 @@ -1013,7 +1023,12 @@ mod tests { .unwrap(), file_fetcher .http_cache - .read_metadata(&cache_key) + .read_headers(&cache_key) + .unwrap() + .unwrap(), + file_fetcher + .http_cache + .read_download_time(&cache_key) .unwrap() .unwrap(), ) @@ -1045,7 +1060,12 @@ mod tests { .unwrap(), file_fetcher .http_cache - .read_metadata(&cache_key) + .read_headers(&cache_key) + .unwrap() + .unwrap(), + file_fetcher + .http_cache + .read_download_time(&cache_key) .unwrap() .unwrap(), ) @@ -1182,7 +1202,12 @@ mod tests { .unwrap(), file_fetcher .http_cache - .read_metadata(&cache_key) + .read_headers(&cache_key) + .unwrap() + .unwrap(), + file_fetcher + .http_cache + .read_download_time(&cache_key) .unwrap() .unwrap(), ) @@ -1216,7 +1241,12 @@ mod tests { .unwrap(), file_fetcher .http_cache - .read_metadata(&cache_key) + .read_headers(&cache_key) + .unwrap() + .unwrap(), + file_fetcher + .http_cache + .read_download_time(&cache_key) .unwrap() .unwrap(), ) @@ -1240,6 +1270,7 @@ mod tests { 2, None, &file_fetcher.cache_setting, + None, ) .await; assert!(result.is_ok()); @@ -1251,14 +1282,15 @@ mod tests { 1, None, &file_fetcher.cache_setting, + None, ) .await; assert!(result.is_err()); - let result = file_fetcher.fetch_cached(&specifier, 2); + let result = file_fetcher.fetch_cached(&specifier, None, 2); assert!(result.is_ok()); - let result = file_fetcher.fetch_cached(&specifier, 1); + let result = file_fetcher.fetch_cached(&specifier, None, 1); assert!(result.is_err()); } @@ -2072,7 +2104,11 @@ mod tests { let cache_key = file_fetcher.http_cache.cache_item_key(url).unwrap(); let bytes = file_fetcher .http_cache - .read_file_bytes(&cache_key) + .read_file_bytes( + &cache_key, + None, + deno_cache_dir::GlobalToLocalCopy::Allow, + ) .unwrap() .unwrap(); String::from_utf8(bytes).unwrap() @@ -2086,10 +2122,9 @@ mod tests { let cache_key = file_fetcher.http_cache.cache_item_key(url).unwrap(); file_fetcher .http_cache - .read_metadata(&cache_key) + .read_headers(&cache_key) .unwrap() .unwrap() - .headers .remove("location") } } diff --git a/cli/graph_util.rs b/cli/graph_util.rs index 3633784b8..09f0db9e6 100644 --- a/cli/graph_util.rs +++ b/cli/graph_util.rs @@ -1,5 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use crate::args::jsr_url; use crate::args::CliOptions; use crate::args::Lockfile; use crate::args::TsTypeLib; @@ -174,6 +175,18 @@ pub fn graph_lock_or_exit(graph: &ModuleGraph, lockfile: &mut Lockfile) { Module::Json(module) => &module.source, Module::Node(_) | Module::Npm(_) | Module::External(_) => continue, }; + + // skip over any specifiers in JSR packages because those + // are enforced via the integrity + if deno_graph::source::recommended_registry_package_url_to_nv( + jsr_url(), + module.specifier(), + ) + .is_some() + { + continue; + } + if !lockfile.check_or_insert_remote(module.specifier().as_str(), source) { let err = format!( concat!( @@ -475,6 +488,19 @@ impl ModuleGraphBuilder { } } } + for (nv, value) in &lockfile.content.packages.jsr { + if let Ok(nv) = PackageNv::from_str(nv) { + graph + .packages + .add_manifest_checksum(nv, value.integrity.clone()) + .map_err(|err| deno_lockfile::IntegrityCheckFailedError { + package_display_id: format!("jsr:{}", err.nv), + actual: err.actual, + expected: err.expected, + filename: lockfile.filename.display().to_string(), + })?; + } + } } } @@ -504,9 +530,14 @@ impl ModuleGraphBuilder { format!("jsr:{}", to), ); } - for (name, deps) in graph.packages.package_deps() { - lockfile - .insert_package_deps(name.to_string(), deps.map(|s| s.to_string())); + for (name, checksum, deps) in + graph.packages.packages_with_checksum_and_deps() + { + lockfile.insert_package( + name.to_string(), + checksum.clone(), + deps.map(|s| s.to_string()), + ); } } } diff --git a/cli/lsp/cache.rs b/cli/lsp/cache.rs index eec6433a2..e0034207d 100644 --- a/cli/lsp/cache.rs +++ b/cli/lsp/cache.rs @@ -11,6 +11,16 @@ use std::path::Path; use std::sync::Arc; use std::time::SystemTime; +/// In the LSP, we disallow the cache from automatically copying from +/// the global cache to the local cache for technical reasons. +/// +/// 1. We need to verify the checksums from the lockfile are correct when +/// moving from the global to the local cache. +/// 2. We need to verify the checksums for JSR https specifiers match what +/// is found in the package's manifest. +pub const LSP_DISALLOW_GLOBAL_TO_LOCAL_COPY: deno_cache_dir::GlobalToLocalCopy = + deno_cache_dir::GlobalToLocalCopy::Disallow; + pub fn calculate_fs_version( cache: &Arc<dyn HttpCache>, specifier: &ModuleSpecifier, @@ -123,8 +133,8 @@ impl CacheMetadata { return None; } let cache_key = self.cache.cache_item_key(specifier).ok()?; - let specifier_metadata = self.cache.read_metadata(&cache_key).ok()??; - let values = Arc::new(parse_metadata(&specifier_metadata.headers)); + let headers = self.cache.read_headers(&cache_key).ok()??; + let values = Arc::new(parse_metadata(&headers)); let version = calculate_fs_version_in_cache(&self.cache, specifier); let mut metadata_map = self.metadata.lock(); let metadata = Metadata { values, version }; diff --git a/cli/lsp/documents.rs b/cli/lsp/documents.rs index c58a392d5..125307757 100644 --- a/cli/lsp/documents.rs +++ b/cli/lsp/documents.rs @@ -2,6 +2,7 @@ use super::cache::calculate_fs_version; use super::cache::calculate_fs_version_at_path; +use super::cache::LSP_DISALLOW_GLOBAL_TO_LOCAL_COPY; use super::jsr_resolver::JsrResolver; use super::language_server::StateNpmSnapshot; use super::text::LineIndex; @@ -736,12 +737,7 @@ impl RedirectResolver { ) -> Option<ModuleSpecifier> { if redirect_limit > 0 { let cache_key = self.cache.cache_item_key(specifier).ok()?; - let headers = self - .cache - .read_metadata(&cache_key) - .ok() - .flatten() - .map(|m| m.headers)?; + let headers = self.cache.read_headers(&cache_key).ok().flatten()?; if let Some(location) = headers.get("location") { let redirect = deno_core::resolve_import(location, specifier.as_str()).ok()?; @@ -822,12 +818,14 @@ impl FileSystemDocuments { } else { let fs_version = calculate_fs_version(cache, specifier)?; let cache_key = cache.cache_item_key(specifier).ok()?; - let bytes = cache.read_file_bytes(&cache_key).ok()??; - let specifier_metadata = cache.read_metadata(&cache_key).ok()??; + let bytes = cache + .read_file_bytes(&cache_key, None, LSP_DISALLOW_GLOBAL_TO_LOCAL_COPY) + .ok()??; + let specifier_headers = cache.read_headers(&cache_key).ok()??; let (_, maybe_charset) = deno_graph::source::resolve_media_type_and_charset_from_headers( specifier, - Some(&specifier_metadata.headers), + Some(&specifier_headers), ); let content = deno_graph::source::decode_owned_source( specifier, @@ -835,7 +833,7 @@ impl FileSystemDocuments { maybe_charset, ) .ok()?; - let maybe_headers = Some(specifier_metadata.headers); + let maybe_headers = Some(specifier_headers); Document::new( specifier.clone(), fs_version, @@ -1826,8 +1824,7 @@ impl<'a> deno_graph::source::Loader for OpenDocumentsGraphLoader<'a> { fn load( &mut self, specifier: &ModuleSpecifier, - is_dynamic: bool, - cache_setting: deno_graph::source::CacheSetting, + options: deno_graph::source::LoadOptions, ) -> deno_graph::source::LoadFuture { let specifier = if self.unstable_sloppy_imports { self @@ -1839,9 +1836,7 @@ impl<'a> deno_graph::source::Loader for OpenDocumentsGraphLoader<'a> { match self.load_from_docs(&specifier) { Some(fut) => fut, - None => self - .inner_loader - .load(&specifier, is_dynamic, cache_setting), + None => self.inner_loader.load(&specifier, options), } } diff --git a/cli/lsp/jsr_resolver.rs b/cli/lsp/jsr_resolver.rs index 8243bb0f2..be7bdc0f5 100644 --- a/cli/lsp/jsr_resolver.rs +++ b/cli/lsp/jsr_resolver.rs @@ -15,6 +15,8 @@ use deno_semver::package::PackageReq; use std::borrow::Cow; use std::sync::Arc; +use super::cache::LSP_DISALLOW_GLOBAL_TO_LOCAL_COPY; + #[derive(Debug)] pub struct JsrResolver { nv_by_req: DashMap<PackageReq, Option<PackageNv>>, @@ -111,7 +113,13 @@ fn read_cached_package_info( ) -> Option<JsrPackageInfo> { let meta_url = jsr_url().join(&format!("{}/meta.json", name)).ok()?; let meta_cache_item_key = cache.cache_item_key(&meta_url).ok()?; - let meta_bytes = cache.read_file_bytes(&meta_cache_item_key).ok()??; + let meta_bytes = cache + .read_file_bytes( + &meta_cache_item_key, + None, + LSP_DISALLOW_GLOBAL_TO_LOCAL_COPY, + ) + .ok()??; serde_json::from_slice::<JsrPackageInfo>(&meta_bytes).ok() } @@ -123,12 +131,19 @@ fn read_cached_package_version_info( .join(&format!("{}/{}_meta.json", &nv.name, &nv.version)) .ok()?; let meta_cache_item_key = cache.cache_item_key(&meta_url).ok()?; - let meta_bytes = cache.read_file_bytes(&meta_cache_item_key).ok()??; + let meta_bytes = cache + .read_file_bytes( + &meta_cache_item_key, + None, + LSP_DISALLOW_GLOBAL_TO_LOCAL_COPY, + ) + .ok()??; // This is a roundabout way of deserializing `JsrPackageVersionInfo`, // because we only want the `exports` field and `module_graph` is large. let mut info = serde_json::from_slice::<serde_json::Value>(&meta_bytes).ok()?; Some(JsrPackageVersionInfo { + manifest: Default::default(), // not used by the LSP (only caching checks this in deno_graph) exports: info.as_object_mut()?.remove("exports")?, module_graph: None, }) diff --git a/cli/lsp/registries.rs b/cli/lsp/registries.rs index f4a64c7ee..2b0cae7d2 100644 --- a/cli/lsp/registries.rs +++ b/cli/lsp/registries.rs @@ -515,6 +515,7 @@ impl ModuleRegistry { permissions: PermissionsContainer::allow_all(), maybe_accept: Some("application/vnd.deno.reg.v2+json, application/vnd.deno.reg.v1+json;q=0.9, application/json;q=0.8"), maybe_cache_setting: None, + maybe_checksum: None, }) .await; // if there is an error fetching, we will cache an empty file, so that diff --git a/cli/tools/coverage/mod.rs b/cli/tools/coverage/mod.rs index aafef292f..5cc705741 100644 --- a/cli/tools/coverage/mod.rs +++ b/cli/tools/coverage/mod.rs @@ -523,7 +523,7 @@ pub async fn cover_files( file_fetcher.get_source(&module_specifier) } else { file_fetcher - .fetch_cached(&module_specifier, 10) + .fetch_cached(&module_specifier, None, 10) .with_context(|| { format!("Failed to fetch \"{module_specifier}\" from cache.") })? diff --git a/cli/tools/doc.rs b/cli/tools/doc.rs index d2cd0c2a2..5044e73d3 100644 --- a/cli/tools/doc.rs +++ b/cli/tools/doc.rs @@ -211,9 +211,9 @@ impl deno_doc::html::HrefResolver for DocResolver { fn resolve_usage( &self, _current_specifier: &ModuleSpecifier, - current_file: &str, + current_file: Option<&str>, ) -> Option<String> { - Some(current_file.to_string()) + current_file.map(|f| f.to_string()) } fn resolve_source(&self, location: &deno_doc::Location) -> Option<String> { diff --git a/cli/tools/vendor/test.rs b/cli/tools/vendor/test.rs index 7910dcf22..6a960c302 100644 --- a/cli/tools/vendor/test.rs +++ b/cli/tools/vendor/test.rs @@ -115,8 +115,7 @@ impl Loader for TestLoader { fn load( &mut self, specifier: &ModuleSpecifier, - _is_dynamic: bool, - _cache_setting: deno_graph::source::CacheSetting, + _options: deno_graph::source::LoadOptions, ) -> LoadFuture { let specifier = self.redirects.get(specifier).unwrap_or(specifier); let result = self.files.get(specifier).map(|result| match result { diff --git a/cli/tsc/mod.rs b/cli/tsc/mod.rs index 18316b750..f3f974690 100644 --- a/cli/tsc/mod.rs +++ b/cli/tsc/mod.rs @@ -904,8 +904,7 @@ mod tests { fn load( &mut self, specifier: &ModuleSpecifier, - _is_dynamic: bool, - _cache_setting: deno_graph::source::CacheSetting, + _options: deno_graph::source::LoadOptions, ) -> deno_graph::source::LoadFuture { let specifier_text = specifier .to_string() diff --git a/test_util/Cargo.toml b/test_util/Cargo.toml index b2ff10277..e46c308b6 100644 --- a/test_util/Cargo.toml +++ b/test_util/Cargo.toml @@ -42,12 +42,12 @@ pretty_assertions.workspace = true prost.workspace = true regex.workspace = true reqwest.workspace = true -ring.workspace = true rustls-pemfile.workspace = true rustls-tokio-stream.workspace = true semver = "=1.0.14" serde.workspace = true serde_json.workspace = true +sha2.workspace = true tar.workspace = true tempfile.workspace = true termcolor.workspace = true diff --git a/test_util/src/builders.rs b/test_util/src/builders.rs index 9bbe6693f..d8c209dd7 100644 --- a/test_util/src/builders.rs +++ b/test_util/src/builders.rs @@ -280,6 +280,34 @@ impl TestContext { .run() .skip_output_check(); } + + pub fn get_jsr_package_integrity(&self, sub_path: &str) -> String { + fn get_checksum(bytes: &[u8]) -> String { + use sha2::Digest; + let mut hasher = sha2::Sha256::new(); + hasher.update(bytes); + format!("{:x}", hasher.finalize()) + } + + let url = url::Url::parse(self.envs.get("JSR_URL").unwrap()).unwrap(); + let url = url.join(&format!("{}_meta.json", sub_path)).unwrap(); + let bytes = sync_fetch(url); + get_checksum(&bytes) + } +} + +fn sync_fetch(url: url::Url) -> bytes::Bytes { + let runtime = tokio::runtime::Builder::new_current_thread() + .enable_io() + .enable_time() + .build() + .unwrap(); + runtime.block_on(async move { + let client = reqwest::Client::new(); + let response = client.get(url).send().await.unwrap(); + assert!(response.status().is_success()); + response.bytes().await.unwrap() + }) } /// We can't clone an stdio, so if someone clones a DenoCmd, diff --git a/test_util/src/npm.rs b/test_util/src/npm.rs index 04207b0ee..7469e9b9e 100644 --- a/test_util/src/npm.rs +++ b/test_util/src/npm.rs @@ -64,9 +64,6 @@ impl CustomNpmPackageCache { } fn get_npm_package(package_name: &str) -> Result<Option<CustomNpmPackage>> { - use ring::digest::Context; - use ring::digest::SHA512; - let package_folder = testdata_path().join("npm/registry").join(package_name); if !package_folder.exists() { return Ok(None); @@ -103,10 +100,7 @@ fn get_npm_package(package_name: &str) -> Result<Option<CustomNpmPackage>> { } // get tarball hash - let mut hash_ctx = Context::new(&SHA512); - hash_ctx.update(&tarball_bytes); - let digest = hash_ctx.finish(); - let tarball_checksum = BASE64_STANDARD.encode(digest.as_ref()); + let tarball_checksum = get_tarball_checksum(&tarball_bytes); // create the registry file JSON for this version let mut dist = serde_json::Map::new(); @@ -176,3 +170,10 @@ fn get_npm_package(package_name: &str) -> Result<Option<CustomNpmPackage>> { tarballs, })) } + +fn get_tarball_checksum(bytes: &[u8]) -> String { + use sha2::Digest; + let mut hasher = sha2::Sha512::new(); + hasher.update(bytes); + BASE64_STANDARD.encode(hasher.finalize()) +} diff --git a/test_util/src/servers/registry.rs b/test_util/src/servers/registry.rs index 69728f706..0efe06217 100644 --- a/test_util/src/servers/registry.rs +++ b/test_util/src/servers/registry.rs @@ -13,9 +13,14 @@ use hyper::body::Incoming; use hyper::Request; use hyper::Response; use hyper::StatusCode; +use once_cell::sync::Lazy; use serde_json::json; +use std::collections::BTreeMap; +use std::collections::HashMap; use std::convert::Infallible; use std::net::SocketAddr; +use std::path::Path; +use std::sync::Mutex; pub async fn registry_server(port: u16) { let registry_server_addr = SocketAddr::from(([127, 0, 0, 1], port)); @@ -66,6 +71,27 @@ async fn registry_server_handler( testdata_path().to_path_buf().join("jsr").join("registry"); file_path.push(&req.uri().path()[1..].replace("%2f", "/")); if let Ok(body) = tokio::fs::read(&file_path).await { + let body = if let Some(version) = file_path + .file_name() + .unwrap() + .to_string_lossy() + .strip_suffix("_meta.json") + { + // fill the manifest with checksums found in the directory so that + // we don't need to maintain them manually in the testdata directory + let mut meta: serde_json::Value = serde_json::from_slice(&body)?; + let mut manifest = + manifest_sorted(meta.get("manifest").cloned().unwrap_or(json!({}))); + let version_dir = file_path.parent().unwrap().join(version); + fill_manifest_at_dir(&mut manifest, &version_dir); + meta + .as_object_mut() + .unwrap() + .insert("manifest".to_string(), json!(manifest)); + serde_json::to_string(&meta).unwrap().into_bytes() + } else { + body + }; return Ok(Response::new(UnsyncBoxBody::new( http_body_util::Full::new(Bytes::from(body)), ))); @@ -77,3 +103,80 @@ async fn registry_server_handler( .body(empty_body)?; Ok(res) } + +fn manifest_sorted( + meta: serde_json::Value, +) -> BTreeMap<String, serde_json::Value> { + let mut manifest = BTreeMap::new(); + if let serde_json::Value::Object(files) = meta { + for (file, checksum) in files { + manifest.insert(file.clone(), checksum.clone()); + } + } + manifest +} + +fn fill_manifest_at_dir( + manifest: &mut BTreeMap<String, serde_json::Value>, + dir: &Path, +) { + let file_system_manifest = get_manifest_entries_for_dir(dir); + for (file_path, value) in file_system_manifest { + manifest.entry(file_path).or_insert(value); + } +} + +static DIR_MANIFEST_CACHE: Lazy< + Mutex<HashMap<String, BTreeMap<String, serde_json::Value>>>, +> = Lazy::new(Default::default); + +fn get_manifest_entries_for_dir( + dir: &Path, +) -> BTreeMap<String, serde_json::Value> { + fn inner_fill( + root_dir: &Path, + dir: &Path, + manifest: &mut BTreeMap<String, serde_json::Value>, + ) { + for entry in std::fs::read_dir(dir).unwrap() { + let entry = entry.unwrap(); + let path = entry.path(); + if path.is_file() { + let file_bytes = std::fs::read(&path).unwrap(); + let checksum = format!("sha256-{}", get_checksum(&file_bytes)); + let relative_path = path + .to_string_lossy() + .strip_prefix(&root_dir.to_string_lossy().to_string()) + .unwrap() + .replace('\\', "/"); + manifest.insert( + relative_path, + json!({ + "size": file_bytes.len(), + "checksum": checksum, + }), + ); + } else if path.is_dir() { + inner_fill(root_dir, &path, manifest); + } + } + } + + DIR_MANIFEST_CACHE + .lock() + .unwrap() + .entry(dir.to_string_lossy().to_string()) + .or_insert_with(|| { + let mut manifest = BTreeMap::new(); + inner_fill(dir, dir, &mut manifest); + manifest + }) + .clone() +} + +fn get_checksum(bytes: &[u8]) -> String { + use sha2::Digest; + let mut hasher = sha2::Sha256::new(); + hasher.update(bytes); + format!("{:x}", hasher.finalize()) +} diff --git a/tests/integration/jsr_tests.rs b/tests/integration/jsr_tests.rs index 51cfcfaac..b6e4d8a4f 100644 --- a/tests/integration/jsr_tests.rs +++ b/tests/integration/jsr_tests.rs @@ -1,5 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use deno_core::serde_json::json; use deno_core::serde_json::Value; use deno_lockfile::Lockfile; use test_util as util; @@ -182,3 +183,167 @@ fn reload_info_not_found_cache_but_exists_remote() { )) .assert_exit_code(0); } + +#[test] +fn lockfile_bad_package_integrity() { + let test_context = TestContextBuilder::for_jsr().use_temp_cwd().build(); + let temp_dir = test_context.temp_dir(); + + temp_dir.write( + "main.ts", + r#"import version from "jsr:@denotest/no_module_graph@0.1"; + +console.log(version);"#, + ); + temp_dir.write("deno.json", "{}"); // to automatically create a lockfile + + test_context + .new_command() + .args("run --quiet main.ts") + .run() + .assert_matches_text("0.1.1\n"); + + let lockfile_path = temp_dir.path().join("deno.lock"); + let mut lockfile = Lockfile::new(lockfile_path.to_path_buf(), false).unwrap(); + let pkg_name = "@denotest/no_module_graph@0.1.1"; + let original_integrity = get_lockfile_pkg_integrity(&lockfile, pkg_name); + set_lockfile_pkg_integrity(&mut lockfile, pkg_name, "bad_integrity"); + lockfile_path.write(lockfile.as_json_string()); + + let actual_integrity = + test_context.get_jsr_package_integrity("@denotest/no_module_graph/0.1.1"); + let integrity_check_failed_msg = format!("error: Integrity check failed for http://127.0.0.1:4250/@denotest/no_module_graph/0.1.1_meta.json + +Actual: {} +Expected: bad_integrity + at file:///[WILDCARD]/main.ts:1:21 +", actual_integrity); + test_context + .new_command() + .args("run --quiet main.ts") + .run() + .assert_matches_text(&integrity_check_failed_msg) + .assert_exit_code(1); + + // now try with a vendor folder + temp_dir + .path() + .join("deno.json") + .write_json(&json!({ "vendor": true })); + + // should fail again + test_context + .new_command() + .args("run --quiet main.ts") + .run() + .assert_matches_text(&integrity_check_failed_msg) + .assert_exit_code(1); + + // now update to the correct integrity + set_lockfile_pkg_integrity(&mut lockfile, pkg_name, &original_integrity); + lockfile_path.write(lockfile.as_json_string()); + + // should pass now + test_context + .new_command() + .args("run --quiet main.ts") + .run() + .assert_matches_text("0.1.1\n") + .assert_exit_code(0); + + // now update to a bad integrity again + set_lockfile_pkg_integrity(&mut lockfile, pkg_name, "bad_integrity"); + lockfile_path.write(lockfile.as_json_string()); + + // shouldn't matter because we have a vendor folder + test_context + .new_command() + .args("run --quiet main.ts") + .run() + .assert_matches_text("0.1.1\n") + .assert_exit_code(0); + + // now remove the vendor dir and it should fail again + temp_dir.path().join("vendor").remove_dir_all(); + + test_context + .new_command() + .args("run --quiet main.ts") + .run() + .assert_matches_text(&integrity_check_failed_msg) + .assert_exit_code(1); +} + +#[test] +fn bad_manifest_checksum() { + let test_context = TestContextBuilder::for_jsr().use_temp_cwd().build(); + let temp_dir = test_context.temp_dir(); + + temp_dir.write( + "main.ts", + r#"import { add } from "jsr:@denotest/bad-manifest-checksum@1.0.0"; +console.log(add);"#, + ); + + // test it properly checks the checksum on download + test_context + .new_command() + .args("run main.ts") + .run() + .assert_matches_text( + "Download http://127.0.0.1:4250/@denotest/bad-manifest-checksum/meta.json +Download http://127.0.0.1:4250/@denotest/bad-manifest-checksum/1.0.0_meta.json +Download http://127.0.0.1:4250/@denotest/bad-manifest-checksum/1.0.0/mod.ts +error: Integrity check failed. + +Actual: 9a30ac96b5d5c1b67eca69e1e2cf0798817d9578c8d7d904a81a67b983b35cba +Expected: bad-checksum + at file:///[WILDCARD]main.ts:1:21 +", + ) + .assert_exit_code(1); + + // test it properly checks the checksum when loading from the cache + test_context + .new_command() + .args("run main.ts") + .run() + .assert_matches_text( + // ideally the two error messages would be the same... this one comes from + // deno_cache and the one above comes from deno_graph. The thing is, in deno_cache + // (source of this error) it makes sense to include the url in the error message + // because it's not always used in the context of deno_graph + "error: Integrity check failed for http://127.0.0.1:4250/@denotest/bad-manifest-checksum/1.0.0/mod.ts + +Actual: 9a30ac96b5d5c1b67eca69e1e2cf0798817d9578c8d7d904a81a67b983b35cba +Expected: bad-checksum + at file:///[WILDCARD]main.ts:1:21 +", + ) + .assert_exit_code(1); +} + +fn get_lockfile_pkg_integrity(lockfile: &Lockfile, pkg_name: &str) -> String { + lockfile + .content + .packages + .jsr + .get(pkg_name) + .unwrap() + .integrity + .clone() +} + +fn set_lockfile_pkg_integrity( + lockfile: &mut Lockfile, + pkg_name: &str, + integrity: &str, +) { + lockfile + .content + .packages + .jsr + .get_mut(pkg_name) + .unwrap() + .integrity = integrity.to_string(); +} diff --git a/tests/integration/lsp_tests.rs b/tests/integration/lsp_tests.rs index 749af95c4..97e9215df 100644 --- a/tests/integration/lsp_tests.rs +++ b/tests/integration/lsp_tests.rs @@ -693,11 +693,19 @@ fn lsp_format_vendor_path() { .use_http_server() .use_temp_cwd() .build(); + + // put this dependency in the global cache + context + .new_command() + .args("cache http://localhost:4545/run/002_hello.ts") + .run() + .skip_output_check(); + let temp_dir = context.temp_dir(); temp_dir.write("deno.json", json!({ "vendor": true }).to_string()); let mut client = context.new_lsp_command().build(); client.initialize_default(); - client.did_open(json!({ + let diagnostics = client.did_open(json!({ "textDocument": { "uri": "file:///a/file.ts", "languageId": "typescript", @@ -705,6 +713,18 @@ fn lsp_format_vendor_path() { "text": r#"import "http://localhost:4545/run/002_hello.ts";"#, }, })); + // copying from the global cache to the local cache requires explicitly + // running the cache command so that the checksums can be verified + assert_eq!( + diagnostics + .all() + .iter() + .map(|d| d.message.as_str()) + .collect::<Vec<_>>(), + vec![ + "Uncached or missing remote URL: http://localhost:4545/run/002_hello.ts" + ] + ); client.write_request( "workspace/executeCommand", json!({ @@ -4358,7 +4378,8 @@ fn lsp_code_actions() { }]) ); let res = client - .write_request( "codeAction/resolve", + .write_request( + "codeAction/resolve", json!({ "title": "Add all missing 'async' modifiers", "kind": "quickfix", @@ -4378,8 +4399,7 @@ fn lsp_code_actions() { "fixId": "fixAwaitInSyncFunction" } }), - ) - ; + ); assert_eq!( res, json!({ @@ -4762,26 +4782,27 @@ fn lsp_code_actions_deno_cache_jsr() { #[test] fn lsp_jsr_lockfile() { - let context = TestContextBuilder::new() - .use_http_server() - .use_temp_cwd() - .build(); + let context = TestContextBuilder::for_jsr().use_temp_cwd().build(); let temp_dir = context.temp_dir(); temp_dir.write("./deno.json", json!({}).to_string()); - temp_dir.write( - "./deno.lock", - json!({ - "version": "3", - "packages": { - "specifiers": { - // This is an old version of the package which exports `sum()` instead - // of `add()`. - "jsr:@denotest/add": "jsr:@denotest/add@0.2.0", - }, - }, - }) - .to_string(), - ); + let lockfile = temp_dir.path().join("deno.lock"); + let integrity = context.get_jsr_package_integrity("@denotest/add/0.2.0"); + lockfile.write_json(&json!({ + "version": "3", + "packages": { + "specifiers": { + // This is an old version of the package which exports `sum()` instead + // of `add()`. + "jsr:@denotest/add": "jsr:@denotest/add@0.2.0", + }, + "jsr": { + "@denotest/add@0.2.0": { + "integrity": integrity + } + } + }, + "remote": {}, + })); let mut client = context.new_lsp_command().build(); client.initialize_default(); client.did_open(json!({ @@ -4790,8 +4811,8 @@ fn lsp_jsr_lockfile() { "languageId": "typescript", "version": 1, "text": r#" - import { add } from "jsr:@denotest/add"; - console.log(add(1, 2)); + import { sum } from "jsr:@denotest/add"; + console.log(sum(1, 2)); "#, }, })); @@ -10672,9 +10693,27 @@ fn lsp_vendor_dir() { refresh_config(&mut client); let diagnostics = client.read_diagnostics(); - assert_eq!(diagnostics.all().len(), 0, "{:#?}", diagnostics); // cached + // won't be cached until a manual cache occurs + assert_eq!( + diagnostics + .all() + .iter() + .map(|d| d.message.as_str()) + .collect::<Vec<_>>(), + vec![ + "Uncached or missing remote URL: http://localhost:4545/subdir/mod1.ts" + ] + ); + + assert!(!temp_dir + .path() + .join("vendor/http_localhost_4545/subdir/mod1.ts") + .exists()); - // no caching necessary because it was already cached. It should exist now + // now cache + cache(&mut client); + let diagnostics = client.read_diagnostics(); + assert_eq!(diagnostics.all().len(), 0, "{:#?}", diagnostics); // cached assert!(temp_dir .path() .join("vendor/http_localhost_4545/subdir/mod1.ts") diff --git a/tests/integration/npm_tests.rs b/tests/integration/npm_tests.rs index 3777bfe8a..33e331fc3 100644 --- a/tests/integration/npm_tests.rs +++ b/tests/integration/npm_tests.rs @@ -1549,7 +1549,7 @@ fn auto_discover_lock_file() { output .assert_matches_text( r#"Download http://localhost:4545/npm/registry/@denotest/bin -error: Integrity check failed for npm package: "@denotest/bin@1.0.0". Unable to verify that the package +error: Integrity check failed for package: "npm:@denotest/bin@1.0.0". Unable to verify that the package is the same as when the lockfile was generated. Actual: sha512-[WILDCARD] diff --git a/tests/integration/run_tests.rs b/tests/integration/run_tests.rs index 6f108f739..68b72ffed 100644 --- a/tests/integration/run_tests.rs +++ b/tests/integration/run_tests.rs @@ -1053,7 +1053,9 @@ fn lock_deno_json_package_json_deps() { "npm:@denotest/esm-basic": "npm:@denotest/esm-basic@1.0.0" }, "jsr": { - "@denotest/module_graph@1.4.0": {} + "@denotest/module_graph@1.4.0": { + "integrity": "555bbe259f55a4a2e7a39e8bf4bcbf25da4c874a313c3e98771eddceedac050b" + } }, "npm": { "@denotest/esm-basic@1.0.0": { @@ -1062,10 +1064,7 @@ fn lock_deno_json_package_json_deps() { } } }, - "remote": { - "http://127.0.0.1:4250/@denotest/module_graph/1.4.0/mod.ts": "5b0ce36e08d759118200d8b4627627b5a89b6261fbb0598e6961a6b287abb699", - "http://127.0.0.1:4250/@denotest/module_graph/1.4.0/other.ts": "9ce27ca439cb0e218b6e1ec26c043dbc0b54c9babc4cb432df478dd1721faade" - }, + "remote": {}, "workspace": { "dependencies": [ "jsr:@denotest/module_graph@1.4", @@ -1106,7 +1105,9 @@ fn lock_deno_json_package_json_deps() { "npm:@denotest/esm-basic": "npm:@denotest/esm-basic@1.0.0" }, "jsr": { - "@denotest/module_graph@1.4.0": {} + "@denotest/module_graph@1.4.0": { + "integrity": "555bbe259f55a4a2e7a39e8bf4bcbf25da4c874a313c3e98771eddceedac050b" + } }, "npm": { "@denotest/esm-basic@1.0.0": { @@ -1115,10 +1116,7 @@ fn lock_deno_json_package_json_deps() { } } }, - "remote": { - "http://127.0.0.1:4250/@denotest/module_graph/1.4.0/mod.ts": "5b0ce36e08d759118200d8b4627627b5a89b6261fbb0598e6961a6b287abb699", - "http://127.0.0.1:4250/@denotest/module_graph/1.4.0/other.ts": "9ce27ca439cb0e218b6e1ec26c043dbc0b54c9babc4cb432df478dd1721faade" - }, + "remote": {}, "workspace": { "dependencies": [ "jsr:@denotest/module_graph@1.4" @@ -1147,13 +1145,12 @@ fn lock_deno_json_package_json_deps() { "jsr:@denotest/module_graph@1.4": "jsr:@denotest/module_graph@1.4.0", }, "jsr": { - "@denotest/module_graph@1.4.0": {} + "@denotest/module_graph@1.4.0": { + "integrity": "555bbe259f55a4a2e7a39e8bf4bcbf25da4c874a313c3e98771eddceedac050b" + } } }, - "remote": { - "http://127.0.0.1:4250/@denotest/module_graph/1.4.0/mod.ts": "5b0ce36e08d759118200d8b4627627b5a89b6261fbb0598e6961a6b287abb699", - "http://127.0.0.1:4250/@denotest/module_graph/1.4.0/other.ts": "9ce27ca439cb0e218b6e1ec26c043dbc0b54c9babc4cb432df478dd1721faade" - }, + "remote": {}, "workspace": { "dependencies": [ "jsr:@denotest/module_graph@1.4" diff --git a/tests/testdata/jsr/registry/@denotest/bad-manifest-checksum/1.0.0/mod.ts b/tests/testdata/jsr/registry/@denotest/bad-manifest-checksum/1.0.0/mod.ts new file mode 100644 index 000000000..8d9b8a22a --- /dev/null +++ b/tests/testdata/jsr/registry/@denotest/bad-manifest-checksum/1.0.0/mod.ts @@ -0,0 +1,3 @@ +export function add(a: number, b: number): number { + return a + b; +} diff --git a/tests/testdata/jsr/registry/@denotest/bad-manifest-checksum/1.0.0_meta.json b/tests/testdata/jsr/registry/@denotest/bad-manifest-checksum/1.0.0_meta.json new file mode 100644 index 000000000..8ef8ab3c3 --- /dev/null +++ b/tests/testdata/jsr/registry/@denotest/bad-manifest-checksum/1.0.0_meta.json @@ -0,0 +1,11 @@ +{ + "exports": { + ".": "./mod.ts" + }, + "manifest": { + "/mod.ts": { + "size": 0, + "checksum": "sha256-bad-checksum" + } + } +} diff --git a/tests/testdata/jsr/registry/@denotest/bad-manifest-checksum/meta.json b/tests/testdata/jsr/registry/@denotest/bad-manifest-checksum/meta.json new file mode 100644 index 000000000..02601e4d0 --- /dev/null +++ b/tests/testdata/jsr/registry/@denotest/bad-manifest-checksum/meta.json @@ -0,0 +1,5 @@ +{ + "versions": { + "1.0.0": {} + } +} diff --git a/tests/testdata/npm/lock_file/main.out b/tests/testdata/npm/lock_file/main.out index 65e881be6..dead1a623 100644 --- a/tests/testdata/npm/lock_file/main.out +++ b/tests/testdata/npm/lock_file/main.out @@ -1,5 +1,5 @@ Download [WILDCARD] -error: Integrity check failed for npm package: "@babel/parser@7.19.0". Unable to verify that the package +error: Integrity check failed for package: "npm:@babel/parser@7.19.0". Unable to verify that the package is the same as when the lockfile was generated. Actual: sha512-74bEXKX2h+8rrfQUfsBfuZZHzsEs6Eql4pqy/T4Nn6Y9wNPggQOqD6z6pn5Bl8ZfysKouFZT/UXEH94ummEeQw== |