fix(ext/node): lossy UTF-8 read node_modules files (#24140)

Previously various reads of files in `node_modules` would error on
invalid UTF-8. These were cases involving:

- reading package.json from Rust
- reading package.json from JS
- reading CommonJS files from JS
- reading CommonJS files from Rust (for ESM translation)
- reading ESM files from Rust

19 files changed, 78 insertions, 2 deletions

diff --git a/tests/registry/npm/@denotest/lossy-utf8-module/1.0.0/index.js b/tests/registry/npm/@denotest/lossy-utf8-module/1.0.0/index.js
new file mode 100644
index 000000000..411f3c372
--- /dev/null
+++ b/tests/registry/npm/@denotest/lossy-utf8-module/1.0.0/index.js
@@ -0,0 +1 @@
+export default 'þþÿÿ';
\ No newline at end of file
diff --git a/tests/registry/npm/@denotest/lossy-utf8-module/1.0.0/package.json b/tests/registry/npm/@denotest/lossy-utf8-module/1.0.0/package.json
new file mode 100644
index 000000000..1260655fb
--- /dev/null
+++ b/tests/registry/npm/@denotest/lossy-utf8-module/1.0.0/package.json
@@ -0,0 +1,6 @@
+{
+  "name": "@denotest/lossy-utf8-script",
+  "version": "1.0.0",
+  "type": "module",
+  "dependencies": {}
+}
diff --git a/tests/registry/npm/@denotest/lossy-utf8-package-json/1.0.0/index.js b/tests/registry/npm/@denotest/lossy-utf8-package-json/1.0.0/index.js
new file mode 100644
index 000000000..34b58e6e1
--- /dev/null
+++ b/tests/registry/npm/@denotest/lossy-utf8-package-json/1.0.0/index.js
@@ -0,0 +1 @@
+export default "hello";
diff --git a/tests/registry/npm/@denotest/lossy-utf8-package-json/1.0.0/package.json b/tests/registry/npm/@denotest/lossy-utf8-package-json/1.0.0/package.json
new file mode 100644
index 000000000..5ca42d0d1
--- /dev/null
+++ b/tests/registry/npm/@denotest/lossy-utf8-package-json/1.0.0/package.json
@@ -0,0 +1,7 @@
+{
+  "name": "@denotest/lossy-utf8-package-json",
+  "version": "1.0.0",
+  "type": "module",
+  "dependencies": {},
+  "files": ["þþÿÿ"]
+}
\ No newline at end of file
diff --git a/tests/registry/npm/@denotest/lossy-utf8-script/1.0.0/index.js b/tests/registry/npm/@denotest/lossy-utf8-script/1.0.0/index.js
new file mode 100644
index 000000000..adbfd382a
--- /dev/null
+++ b/tests/registry/npm/@denotest/lossy-utf8-script/1.0.0/index.js
@@ -0,0 +1 @@
+module.exports = 'þþÿÿ';
\ No newline at end of file
diff --git a/tests/registry/npm/@denotest/lossy-utf8-script/1.0.0/package.json b/tests/registry/npm/@denotest/lossy-utf8-script/1.0.0/package.json
new file mode 100644
index 000000000..e23605945
--- /dev/null
+++ b/tests/registry/npm/@denotest/lossy-utf8-script/1.0.0/package.json
@@ -0,0 +1,6 @@
+{
+  "name": "@denotest/lossy-utf8-script",
+  "version": "1.0.0",
+  "type": "commonjs",
+  "dependencies": {}
+}
diff --git a/tests/specs/npm/lossy_utf8_module/__test__.jsonc b/tests/specs/npm/lossy_utf8_module/__test__.jsonc
new file mode 100644
index 000000000..1ee7d2d6c
--- /dev/null
+++ b/tests/specs/npm/lossy_utf8_module/__test__.jsonc
@@ -0,0 +1,5 @@
+{
+  "args": "run main.mjs",
+  "output": "main.out",
+  "exitCode": 0
+}
diff --git a/tests/specs/npm/lossy_utf8_module/main.mjs b/tests/specs/npm/lossy_utf8_module/main.mjs
new file mode 100644
index 000000000..54cfbb16a
--- /dev/null
+++ b/tests/specs/npm/lossy_utf8_module/main.mjs
@@ -0,0 +1,3 @@
+import mod from "npm:@denotest/lossy-utf8-module@1.0.0";
+
+console.log(mod);
diff --git a/tests/specs/npm/lossy_utf8_module/main.out b/tests/specs/npm/lossy_utf8_module/main.out
new file mode 100644
index 000000000..0e96f9ebb
--- /dev/null
+++ b/tests/specs/npm/lossy_utf8_module/main.out
@@ -0,0 +1,3 @@
+Download http://localhost:4260/@denotest/lossy-utf8-module
+Download http://localhost:4260/@denotest/lossy-utf8-module/1.0.0.tgz
+����
diff --git a/tests/specs/npm/lossy_utf8_package_json/__test__.jsonc b/tests/specs/npm/lossy_utf8_package_json/__test__.jsonc
new file mode 100644
index 000000000..1ee7d2d6c
--- /dev/null
+++ b/tests/specs/npm/lossy_utf8_package_json/__test__.jsonc
@@ -0,0 +1,5 @@
+{
+  "args": "run main.mjs",
+  "output": "main.out",
+  "exitCode": 0
+}
diff --git a/tests/specs/npm/lossy_utf8_package_json/main.mjs b/tests/specs/npm/lossy_utf8_package_json/main.mjs
new file mode 100644
index 000000000..9a63eb604
--- /dev/null
+++ b/tests/specs/npm/lossy_utf8_package_json/main.mjs
@@ -0,0 +1,3 @@
+import mod from "npm:@denotest/lossy-utf8-package-json@1.0.0";
+
+console.log(mod);
diff --git a/tests/specs/npm/lossy_utf8_package_json/main.out b/tests/specs/npm/lossy_utf8_package_json/main.out
new file mode 100644
index 000000000..99aa5ab61
--- /dev/null
+++ b/tests/specs/npm/lossy_utf8_package_json/main.out
@@ -0,0 +1,3 @@
+Download http://localhost:4260/@denotest/lossy-utf8-package-json
+Download http://localhost:4260/@denotest/lossy-utf8-package-json/1.0.0.tgz
+hello
diff --git a/tests/specs/npm/lossy_utf8_script/__test__.jsonc b/tests/specs/npm/lossy_utf8_script/__test__.jsonc
new file mode 100644
index 000000000..1ee7d2d6c
--- /dev/null
+++ b/tests/specs/npm/lossy_utf8_script/__test__.jsonc
@@ -0,0 +1,5 @@
+{
+  "args": "run main.mjs",
+  "output": "main.out",
+  "exitCode": 0
+}
diff --git a/tests/specs/npm/lossy_utf8_script/main.mjs b/tests/specs/npm/lossy_utf8_script/main.mjs
new file mode 100644
index 000000000..59fe555aa
--- /dev/null
+++ b/tests/specs/npm/lossy_utf8_script/main.mjs
@@ -0,0 +1,3 @@
+import mod from "npm:@denotest/lossy-utf8-script@1.0.0";
+
+console.log(mod);
diff --git a/tests/specs/npm/lossy_utf8_script/main.out b/tests/specs/npm/lossy_utf8_script/main.out
new file mode 100644
index 000000000..180ecdf1c
--- /dev/null
+++ b/tests/specs/npm/lossy_utf8_script/main.out
@@ -0,0 +1,3 @@
+Download http://localhost:4260/@denotest/lossy-utf8-script
+Download http://localhost:4260/@denotest/lossy-utf8-script/1.0.0.tgz
+����
diff --git a/tests/specs/npm/lossy_utf8_script_from_cjs/__test__.jsonc b/tests/specs/npm/lossy_utf8_script_from_cjs/__test__.jsonc
new file mode 100644
index 000000000..c8d353de0
--- /dev/null
+++ b/tests/specs/npm/lossy_utf8_script_from_cjs/__test__.jsonc
@@ -0,0 +1,6 @@
+{
+  "args": "run --node-modules-dir --allow-read main.mjs",
+  "output": "main.out",
+  "exitCode": 0,
+  "tempDir": true
+}
diff --git a/tests/specs/npm/lossy_utf8_script_from_cjs/main.mjs b/tests/specs/npm/lossy_utf8_script_from_cjs/main.mjs
new file mode 100644
index 000000000..a9e70adfb
--- /dev/null
+++ b/tests/specs/npm/lossy_utf8_script_from_cjs/main.mjs
@@ -0,0 +1,10 @@
+import { createRequire } from "node:module";
+
+// Import this so that deno_graph knows to download this file.
+if (false) import("npm:@denotest/lossy-utf8-script@1.0.0");
+
+const require = createRequire(import.meta.url);
+
+const mod = require("@denotest/lossy-utf8-script");
+
+console.log(mod);
diff --git a/tests/specs/npm/lossy_utf8_script_from_cjs/main.out b/tests/specs/npm/lossy_utf8_script_from_cjs/main.out
new file mode 100644
index 000000000..4f062a2ae
--- /dev/null
+++ b/tests/specs/npm/lossy_utf8_script_from_cjs/main.out
@@ -0,0 +1,4 @@
+Download http://localhost:4260/@denotest/lossy-utf8-script
+Download http://localhost:4260/@denotest/lossy-utf8-script/1.0.0.tgz
+Initialize @denotest/lossy-utf8-script@1.0.0
+����
diff --git a/tests/util/server/src/npm.rs b/tests/util/server/src/npm.rs
index 66b7bddcd..e7d8d96ab 100644
--- a/tests/util/server/src/npm.rs
+++ b/tests/util/server/src/npm.rs
@@ -226,10 +226,11 @@ fn get_npm_package(
 
     tarballs.insert(version.clone(), tarball_bytes);
     let package_json_path = version_folder.join("package.json");
-    let package_json_text = fs::read_to_string(&package_json_path)
-      .with_context(|| {
+    let package_json_bytes =
+      fs::read(&package_json_path).with_context(|| {
         format!("Error reading package.json at {}", package_json_path)
       })?;
+    let package_json_text = String::from_utf8_lossy(&package_json_bytes);
     let mut version_info: serde_json::Map<String, serde_json::Value> =
       serde_json::from_str(&package_json_text)?;
     version_info.insert("dist".to_string(), dist.into());