feat: wire up ext/node to the Node compatibility layer (#17785)

This PR changes Node.js/npm compatibility layer to use polyfills for
built-in Node.js
embedded in the snapshot (that are coming from "ext/node" extension).

As a result loading `std/node`, either from
"https://deno.land/std@<latest>/" or
from "DENO_NODE_COMPAT_URL" env variable were removed. All code that is
imported via "npm:" specifiers now uses code embedded in the snapshot.

Several fixes were applied to various modules in "ext/node" to make
tests pass.

---------

Co-authored-by: Yoshiya Hinosawa <stibium121@gmail.com>
Co-authored-by: Divy Srivastava <dj.srivastava23@gmail.com>

10 files changed, 121 insertions, 142 deletions

diff --git a/ext/node/polyfills/_next_tick.ts b/ext/node/polyfills/_next_tick.ts
index 7cbff0ea4..d5aa88218 100644
--- a/ext/node/polyfills/_next_tick.ts
+++ b/ext/node/polyfills/_next_tick.ts
@@ -1,8 +1,6 @@
 // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
 // Copyright Joyent, Inc. and other Node contributors.
 
-// deno-lint-ignore-file no-inner-declarations
-
 import { core } from "internal:deno_node/polyfills/_core.ts";
 import { validateFunction } from "internal:deno_node/polyfills/internal/validators.mjs";
 import { _exiting } from "internal:deno_node/polyfills/_process/exiting.ts";
@@ -15,9 +13,6 @@ interface Tock {
 
 const queue = new FixedQueue();
 
-// deno-lint-ignore no-explicit-any
-let _nextTick: any;
-
 export function processTicksAndRejections() {
   let tock;
   do {
@@ -68,92 +63,21 @@ export function processTicksAndRejections() {
   // setHasRejectionToWarn(false);
 }
 
-if (typeof core.setNextTickCallback !== "undefined") {
-  function runNextTicks() {
-    // FIXME(bartlomieju): Deno currently doesn't unhandled rejections
-    // if (!hasTickScheduled() && !hasRejectionToWarn())
-    //   runMicrotasks();
-    // if (!hasTickScheduled() && !hasRejectionToWarn())
-    //   return;
-    if (!core.hasTickScheduled()) {
-      core.runMicrotasks();
-    }
-    if (!core.hasTickScheduled()) {
-      return true;
-    }
-
-    processTicksAndRejections();
-    return true;
-  }
-
-  core.setNextTickCallback(processTicksAndRejections);
-  core.setMacrotaskCallback(runNextTicks);
-
-  function __nextTickNative<T extends Array<unknown>>(
-    this: unknown,
-    callback: (...args: T) => void,
-    ...args: T
-  ) {
-    validateFunction(callback, "callback");
-
-    if (_exiting) {
-      return;
-    }
-
-    // TODO(bartlomieju): seems superfluous if we don't depend on `arguments`
-    let args_;
-    switch (args.length) {
-      case 0:
-        break;
-      case 1:
-        args_ = [args[0]];
-        break;
-      case 2:
-        args_ = [args[0], args[1]];
-        break;
-      case 3:
-        args_ = [args[0], args[1], args[2]];
-        break;
-      default:
-        args_ = new Array(args.length);
-        for (let i = 0; i < args.length; i++) {
-          args_[i] = args[i];
-        }
-    }
-
-    if (queue.isEmpty()) {
-      core.setHasTickScheduled(true);
-    }
-    // FIXME(bartlomieju): Deno currently doesn't support async hooks
-    // const asyncId = newAsyncId();
-    // const triggerAsyncId = getDefaultTriggerAsyncId();
-    const tickObject = {
-      // FIXME(bartlomieju): Deno currently doesn't support async hooks
-      // [async_id_symbol]: asyncId,
-      // [trigger_async_id_symbol]: triggerAsyncId,
-      callback,
-      args: args_,
-    };
-    // FIXME(bartlomieju): Deno currently doesn't support async hooks
-    // if (initHooksExist())
-    //   emitInit(asyncId, 'TickObject', triggerAsyncId, tickObject);
-    queue.push(tickObject);
+export function runNextTicks() {
+  // FIXME(bartlomieju): Deno currently doesn't unhandled rejections
+  // if (!hasTickScheduled() && !hasRejectionToWarn())
+  //   runMicrotasks();
+  // if (!hasTickScheduled() && !hasRejectionToWarn())
+  //   return;
+  if (!core.hasTickScheduled()) {
+    core.runMicrotasks();
   }
-  _nextTick = __nextTickNative;
-} else {
-  function __nextTickQueueMicrotask<T extends Array<unknown>>(
-    this: unknown,
-    callback: (...args: T) => void,
-    ...args: T
-  ) {
-    if (args) {
-      queueMicrotask(() => callback.call(this, ...args));
-    } else {
-      queueMicrotask(callback);
-    }
+  if (!core.hasTickScheduled()) {
+    return true;
   }
 
-  _nextTick = __nextTickQueueMicrotask;
+  processTicksAndRejections();
+  return true;
 }
 
 // `nextTick()` will not enqueue any callback when the process is about to
@@ -169,5 +93,48 @@ export function nextTick<T extends Array<unknown>>(
   callback: (...args: T) => void,
   ...args: T
 ) {
-  _nextTick(callback, ...args);
+  validateFunction(callback, "callback");
+
+  if (_exiting) {
+    return;
+  }
+
+  // TODO(bartlomieju): seems superfluous if we don't depend on `arguments`
+  let args_;
+  switch (args.length) {
+    case 0:
+      break;
+    case 1:
+      args_ = [args[0]];
+      break;
+    case 2:
+      args_ = [args[0], args[1]];
+      break;
+    case 3:
+      args_ = [args[0], args[1], args[2]];
+      break;
+    default:
+      args_ = new Array(args.length);
+      for (let i = 0; i < args.length; i++) {
+        args_[i] = args[i];
+      }
+  }
+
+  if (queue.isEmpty()) {
+    core.setHasTickScheduled(true);
+  }
+  // FIXME(bartlomieju): Deno currently doesn't support async hooks
+  // const asyncId = newAsyncId();
+  // const triggerAsyncId = getDefaultTriggerAsyncId();
+  const tickObject = {
+    // FIXME(bartlomieju): Deno currently doesn't support async hooks
+    // [async_id_symbol]: asyncId,
+    // [trigger_async_id_symbol]: triggerAsyncId,
+    callback,
+    args: args_,
+  };
+  // FIXME(bartlomieju): Deno currently doesn't support async hooks
+  // if (initHooksExist())
+  //   emitInit(asyncId, 'TickObject', triggerAsyncId, tickObject);
+  queue.push(tickObject);
 }
diff --git a/ext/node/polyfills/_process/process.ts b/ext/node/polyfills/_process/process.ts
index 7ba44e431..48b2e4620 100644
--- a/ext/node/polyfills/_process/process.ts
+++ b/ext/node/polyfills/_process/process.ts
@@ -7,6 +7,7 @@
 import { build } from "internal:runtime/js/01_build.js";
 import { nextTick as _nextTick } from "internal:deno_node/polyfills/_next_tick.ts";
 import { _exiting } from "internal:deno_node/polyfills/_process/exiting.ts";
+import * as fs from "internal:runtime/js/30_fs.js";
 
 /** Returns the operating system CPU architecture for which the Deno binary was compiled */
 export function arch(): string {
@@ -20,10 +21,10 @@ export function arch(): string {
 }
 
 /** https://nodejs.org/api/process.html#process_process_chdir_directory */
-export const chdir = Deno.chdir;
+export const chdir = fs.chdir;
 
 /** https://nodejs.org/api/process.html#process_process_cwd */
-export const cwd = Deno.cwd;
+export const cwd = fs.cwd;
 
 /** https://nodejs.org/api/process.html#process_process_nexttick_callback_args */
 export const nextTick = _nextTick;
diff --git a/ext/node/polyfills/_process/streams.mjs b/ext/node/polyfills/_process/streams.mjs
index 30811e673..46213a4ed 100644
--- a/ext/node/polyfills/_process/streams.mjs
+++ b/ext/node/polyfills/_process/streams.mjs
@@ -10,7 +10,9 @@ import {
 } from "internal:deno_node/polyfills/internal/readline/callbacks.mjs";
 import { Duplex, Readable, Writable } from "internal:deno_node/polyfills/stream.ts";
 import { stdio } from "internal:deno_node/polyfills/_process/stdio.mjs";
+import { isWindows } from "internal:deno_node/polyfills/_util/os.ts";
 import { fs as fsConstants } from "internal:deno_node/polyfills/internal_binding/constants.ts";
+import * as files from "internal:runtime/js/40_files.js";
 
 // https://github.com/nodejs/node/blob/00738314828074243c9a52a228ab4c68b04259ef/lib/internal/bootstrap/switches/is_main_thread.js#L41
 function createWritableStdioStream(writer, name) {
@@ -92,13 +94,13 @@ function createWritableStdioStream(writer, name) {
 
 /** https://nodejs.org/api/process.html#process_process_stderr */
 export const stderr = stdio.stderr = createWritableStdioStream(
-  Deno.stderr,
+  files.stderr,
   "stderr",
 );
 
 /** https://nodejs.org/api/process.html#process_process_stdout */
 export const stdout = stdio.stdout = createWritableStdioStream(
-  Deno.stdout,
+  files.stdout,
   "stdout",
 );
 
@@ -113,7 +115,7 @@ function _guessStdinType(fd) {
     const fileInfo = Deno.fstatSync?.(fd);
 
     // https://github.com/nodejs/node/blob/v18.12.1/deps/uv/src/unix/tty.c#L333
-    if (Deno.build.os !== "windows") {
+    if (!isWindows) {
       switch (fileInfo.mode & fsConstants.S_IFMT) {
         case fsConstants.S_IFREG:
         case fsConstants.S_IFCHR:
@@ -143,7 +145,7 @@ function _guessStdinType(fd) {
     // TODO(PolarETech): Need a better way to identify a character file on Windows.
     // "EISDIR" error occurs when stdin is "null" on Windows,
     // so use the error as a workaround.
-    if (Deno.build.os === "windows" && e.code === "EISDIR") return "FILE";
+    if (isWindows && e.code === "EISDIR") return "FILE";
   }
 
   return "UNKNOWN";
@@ -151,7 +153,7 @@ function _guessStdinType(fd) {
 
 const _read = function (size) {
   const p = Buffer.alloc(size || 16 * 1024);
-  Deno.stdin?.read(p).then((length) => {
+  files.stdin?.read(p).then((length) => {
     this.push(length === null ? null : p.slice(0, length));
   }, (error) => {
     this.destroy(error);
@@ -161,7 +163,7 @@ const _read = function (size) {
 /** https://nodejs.org/api/process.html#process_process_stdin */
 // https://github.com/nodejs/node/blob/v18.12.1/lib/internal/bootstrap/switches/is_main_thread.js#L189
 export const stdin = stdio.stdin = (() => {
-  const fd = Deno.stdin?.rid;
+  const fd = files.stdin?.rid;
   let _stdin;
   const stdinType = _guessStdinType(fd);
 
@@ -222,8 +224,8 @@ export const stdin = stdio.stdin = (() => {
 
   return _stdin;
 })();
-stdin.on("close", () => Deno.stdin?.close());
-stdin.fd = Deno.stdin?.rid ?? -1;
+stdin.on("close", () => files.stdin?.close());
+stdin.fd = files.stdin?.rid ?? -1;
 Object.defineProperty(stdin, "isTTY", {
   enumerable: true,
   configurable: true,
@@ -233,7 +235,7 @@ Object.defineProperty(stdin, "isTTY", {
 });
 stdin._isRawMode = false;
 stdin.setRawMode = (enable) => {
-  Deno.stdin?.setRaw?.(enable);
+  files.stdin?.setRaw?.(enable);
   stdin._isRawMode = enable;
   return stdin;
 };
diff --git a/ext/node/polyfills/_util/os.ts b/ext/node/polyfills/_util/os.ts
index 66d18534c..a3cb396bd 100644
--- a/ext/node/polyfills/_util/os.ts
+++ b/ext/node/polyfills/_util/os.ts
@@ -1,22 +1,10 @@
 // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
 
-export type OSType = "windows" | "linux" | "darwin" | "freebsd";
-
-export const osType: OSType = (() => {
-  // deno-lint-ignore no-explicit-any
-  const { Deno } = globalThis as any;
-  if (typeof Deno?.build?.os === "string") {
-    return Deno.build.os;
-  }
+const { ops } = globalThis.__bootstrap.core;
 
-  // deno-lint-ignore no-explicit-any
-  const { navigator } = globalThis as any;
-  if (navigator?.appVersion?.includes?.("Win")) {
-    return "windows";
-  }
+export type OSType = "windows" | "linux" | "darwin" | "freebsd";
 
-  return "linux";
-})();
+export const osType: OSType = ops.op_node_build_os();
 
 export const isWindows = osType === "windows";
 export const isLinux = osType === "linux";
diff --git a/ext/node/polyfills/console.ts b/ext/node/polyfills/console.ts
index bfc9be051..f811f1a86 100644
--- a/ext/node/polyfills/console.ts
+++ b/ext/node/polyfills/console.ts
@@ -1,6 +1,10 @@
 // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
 
 import { Console } from "internal:deno_node/polyfills/internal/console/constructor.mjs";
+import { windowOrWorkerGlobalScope } from "internal:runtime/js/98_global_scope.js";
+// Don't rely on global `console` because during bootstrapping, it is pointing
+// to native `console` object provided by V8.
+const console = windowOrWorkerGlobalScope.console.value;
 
 export default Object.assign({}, console, { Console });
 
diff --git a/ext/node/polyfills/internal/child_process.ts b/ext/node/polyfills/internal/child_process.ts
index 92aa8d4fa..81a404c14 100644
--- a/ext/node/polyfills/internal/child_process.ts
+++ b/ext/node/polyfills/internal/child_process.ts
@@ -67,10 +67,6 @@ export function mapValues<T, O>(
 type NodeStdio = "pipe" | "overlapped" | "ignore" | "inherit" | "ipc";
 type DenoStdio = "inherit" | "piped" | "null";
 
-// @ts-ignore Deno[Deno.internal] is used on purpose here
-const DenoCommand = Deno[Deno.internal]?.nodeUnstable?.Command ||
-  Deno.Command;
-
 export function stdioStringToArray(
   stdio: NodeStdio,
   channel: NodeStdio | number,
@@ -183,9 +179,8 @@ export class ChildProcess extends EventEmitter {
     this.spawnargs = [cmd, ...cmdArgs];
 
     const stringEnv = mapValues(env, (value) => value.toString());
-
     try {
-      this.#process = new DenoCommand(cmd, {
+      this.#process = new Deno.Command(cmd, {
         args: cmdArgs,
         cwd,
         env: stringEnv,
@@ -804,7 +799,7 @@ export function spawnSync(
 
   const result: SpawnSyncResult = {};
   try {
-    const output = new DenoCommand(command, {
+    const output = new Deno.Command(command, {
       args,
       cwd,
       env,
diff --git a/ext/node/polyfills/internal/crypto/hash.ts b/ext/node/polyfills/internal/crypto/hash.ts
index 7995e5f8c..e6e2409a2 100644
--- a/ext/node/polyfills/internal/crypto/hash.ts
+++ b/ext/node/polyfills/internal/crypto/hash.ts
@@ -107,7 +107,7 @@ export class Hash extends Transform {
    * Supported encodings are currently 'hex', 'binary', 'base64', 'base64url'.
    */
   digest(encoding?: string): Buffer | string {
-    const digest = this.#context.digest(undefined);
+    const digest = ops.op_node_hash_digest(this.#context);
     if (encoding === undefined) {
       return Buffer.from(digest);
     }
diff --git a/ext/node/polyfills/internal/timers.mjs b/ext/node/polyfills/internal/timers.mjs
index 648fb1bc1..6796885ce 100644
--- a/ext/node/polyfills/internal/timers.mjs
+++ b/ext/node/polyfills/internal/timers.mjs
@@ -5,10 +5,11 @@ import { inspect } from "internal:deno_node/polyfills/internal/util/inspect.mjs"
 import { validateFunction, validateNumber } from "internal:deno_node/polyfills/internal/validators.mjs";
 import { ERR_OUT_OF_RANGE } from "internal:deno_node/polyfills/internal/errors.ts";
 import { emitWarning } from "internal:deno_node/polyfills/process.ts";
-
-const setTimeout_ = globalThis.setTimeout;
-const clearTimeout_ = globalThis.clearTimeout;
-const setInterval_ = globalThis.setInterval;
+import {
+  setTimeout as setTimeout_,
+  clearTimeout as clearTimeout_,
+  setInterval as setInterval_,
+} from "internal:deno_web/02_timers.js";
 
 // Timeout values > TIMEOUT_MAX are set to 1.
 export const TIMEOUT_MAX = 2 ** 31 - 1;
diff --git a/ext/node/polyfills/module_all.ts b/ext/node/polyfills/module_all.ts
index 4a5f73b0c..989ce55a8 100644
--- a/ext/node/polyfills/module_all.ts
+++ b/ext/node/polyfills/module_all.ts
@@ -1,4 +1,5 @@
 // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
+const internals = globalThis.__bootstrap.internals;
 import _httpAgent from "internal:deno_node/polyfills/_http_agent.mjs";
 import _httpOutgoing from "internal:deno_node/polyfills/_http_outgoing.ts";
 import _streamDuplex from "internal:deno_node/polyfills/internal/streams/duplex.mjs";
@@ -88,7 +89,7 @@ import wasi from "internal:deno_node/polyfills/wasi.ts";
 import zlib from "internal:deno_node/polyfills/zlib.ts";
 
 // Canonical mapping of supported modules
-export default {
+const moduleAll = {
   "_http_agent": _httpAgent,
   "_http_outgoing": _httpOutgoing,
   "_stream_duplex": _streamDuplex,
@@ -185,3 +186,6 @@ export default {
   worker_threads: workerThreads,
   zlib,
 } as Record<string, unknown>;
+
+internals.nodeModuleAll = moduleAll;
+export default moduleAll;
diff --git a/ext/node/polyfills/process.ts b/ext/node/polyfills/process.ts
index 9e7c29be7..828b4c660 100644
--- a/ext/node/polyfills/process.ts
+++ b/ext/node/polyfills/process.ts
@@ -2,6 +2,7 @@
 // Copyright Joyent, Inc. and Node.js contributors. All rights reserved. MIT license.
 
 const internals = globalThis.__bootstrap.internals;
+import { core } from "internal:deno_node/polyfills/_core.ts";
 import {
   notImplemented,
   warnNotImplemented,
@@ -32,8 +33,11 @@ import {
   stdin as stdin_,
   stdout as stdout_,
 } from "internal:deno_node/polyfills/_process/streams.mjs";
-import { core } from "internal:deno_node/polyfills/_core.ts";
-import { processTicksAndRejections } from "internal:deno_node/polyfills/_next_tick.ts";
+import {
+  processTicksAndRejections,
+  runNextTicks,
+} from "internal:deno_node/polyfills/_next_tick.ts";
+import { isWindows } from "internal:deno_node/polyfills/_util/os.ts";
 
 // TODO(kt3k): This should be set at start up time
 export let arch = "";
@@ -71,10 +75,19 @@ const notImplementedEvents = [
   "worker",
 ];
 
-export const argv = [];
+export const argv: string[] = [];
 
 // Overwrites the 1st item with getter.
-Object.defineProperty(argv, "0", { get: Deno.execPath });
+// TODO(bartlomieju): added "configurable: true" to make this work for binary
+// commands, but that is probably a wrong solution
+// TODO(bartlomieju): move the configuration for all "argv" to
+// "internals.__bootstrapNodeProcess"
+Object.defineProperty(argv, "0", {
+  get: () => {
+    return Deno.execPath();
+  },
+  configurable: true,
+});
 // Overwrites the 2st item with getter.
 Object.defineProperty(argv, "1", {
   get: () => {
@@ -86,13 +99,6 @@ Object.defineProperty(argv, "1", {
   },
 });
 
-// TODO(kt3k): Set the rest of args at start up time instead of defining
-// random number of getters.
-for (let i = 0; i < 30; i++) {
-  const j = i;
-  Object.defineProperty(argv, j + 2, { get: () => Deno.args[j] });
-}
-
 /** https://nodejs.org/api/process.html#process_process_exit_code */
 export const exit = (code?: number | string) => {
   if (code || code === 0) {
@@ -681,9 +687,18 @@ addReadOnlyProcessAlias("throwDeprecation", "--throw-deprecation");
 export const removeListener = process.removeListener;
 export const removeAllListeners = process.removeAllListeners;
 
-// FIXME(bartlomieju): currently it's not called
-// only call this from runtime's main.js
-internals.__bootstrapNodeProcess = function () {
+// Should be called only once, in `runtime/js/99_main.js` when the runtime is
+// bootstrapped.
+internals.__bootstrapNodeProcess = function (args: string[]) {
+  for (let i = 0; i < args.length; i++) {
+    argv[i + 2] = args[i];
+  }
+
+  core.setNextTickCallback(processTicksAndRejections);
+  core.setMacrotaskCallback(runNextTicks);
+
+  // TODO(bartlomieju): this is buggy, see https://github.com/denoland/deno/issues/16928
+  // We should use a specialized API in 99_main.js instead
   globalThis.addEventListener("unhandledrejection", (event) => {
     if (process.listenerCount("unhandledRejection") === 0) {
       // The Node.js default behavior is to raise an uncaught exception if
@@ -724,6 +739,8 @@ internals.__bootstrapNodeProcess = function () {
       process.emit("exit", process.exitCode || 0);
     }
   });
+
+  delete internals.__bootstrapNodeProcess;
 };
 
 export default process;