feat(test): run test modules in parallel (#9815)

This commit adds support for running test in parallel.

Entire test runner functionality has been rewritten
from JavaScript to Rust and a set of ops was added to support reporting in Rust.

A new "--jobs" flag was added to "deno test" that allows to configure 
how many threads will be used. When given no value it defaults to 2.

1 files changed, 68 insertions, 217 deletions

diff --git a/runtime/js/40_testing.js b/runtime/js/40_testing.js
index 4a97f6437..f835a0cf7 100644
--- a/runtime/js/40_testing.js
+++ b/runtime/js/40_testing.js
@@ -3,29 +3,12 @@
 
 ((window) => {
   const core = window.Deno.core;
-  const colors = window.__bootstrap.colors;
   const { parsePermissions } = window.__bootstrap.worker;
   const { setExitHandler, exit } = window.__bootstrap.os;
   const { Console, inspectArgs } = window.__bootstrap.console;
-  const { stdout } = window.__bootstrap.files;
   const { metrics } = window.__bootstrap.metrics;
   const { assert } = window.__bootstrap.util;
 
-  const disabledConsole = new Console(() => {});
-
-  function delay(ms) {
-    return new Promise((resolve) => {
-      setTimeout(resolve, ms);
-    });
-  }
-
-  function formatDuration(time = 0) {
-    const gray = colors.maybeColor(colors.gray);
-    const italic = colors.maybeColor(colors.italic);
-    const timeStr = `(${time}ms)`;
-    return gray(italic(timeStr));
-  }
-
   // Wrap test function in additional assertion that makes sure
   // the test case does not leak async "ops" - ie. number of async
   // completed ops after the test is the same as number of dispatched
@@ -40,8 +23,9 @@
         // Defer until next event loop turn - that way timeouts and intervals
         // cleared can actually be removed from resource table, otherwise
         // false positives may occur (https://github.com/denoland/deno/issues/4591)
-        await delay(0);
+        await new Promise((resolve) => setTimeout(resolve, 0));
       }
+
       const post = metrics();
       // We're checking diff because one might spawn HTTP server in the background
       // that will be a pending async op before test starts.
@@ -107,7 +91,7 @@ finishing test case.`;
     };
   }
 
-  const TEST_REGISTRY = [];
+  const tests = [];
 
   // Main test function provided by Deno, as you can see it merely
   // creates a new object with "name" and "fn" fields.
@@ -155,77 +139,26 @@ finishing test case.`;
       testDef.fn = assertExit(testDef.fn);
     }
 
-    TEST_REGISTRY.push(testDef);
+    tests.push(testDef);
   }
 
-  const encoder = new TextEncoder();
-
-  function log(msg, noNewLine = false) {
-    if (!noNewLine) {
-      msg += "\n";
-    }
-
-    // Using `stdout` here because it doesn't force new lines
-    // compared to `console.log`; `core.print` on the other hand
-    // is line-buffered and doesn't output message without newline
-    stdout.writeSync(encoder.encode(msg));
+  function postTestMessage(kind, data) {
+    return core.opSync("op_post_test_message", { message: { kind, data } });
   }
 
-  function reportToConsole(message) {
-    const green = colors.maybeColor(colors.green);
-    const red = colors.maybeColor(colors.red);
-    const yellow = colors.maybeColor(colors.yellow);
-    const redFailed = red("FAILED");
-    const greenOk = green("ok");
-    const yellowIgnored = yellow("ignored");
-    if (message.start != null) {
-      log(`running ${message.start.tests.length} tests`);
-    } else if (message.testStart != null) {
-      const { name } = message.testStart;
-
-      log(`test ${name} ... `, true);
-      return;
-    } else if (message.testEnd != null) {
-      switch (message.testEnd.status) {
-        case "passed":
-          log(`${greenOk} ${formatDuration(message.testEnd.duration)}`);
-          break;
-        case "failed":
-          log(`${redFailed} ${formatDuration(message.testEnd.duration)}`);
-          break;
-        case "ignored":
-          log(`${yellowIgnored} ${formatDuration(message.testEnd.duration)}`);
-          break;
-      }
-    } else if (message.end != null) {
-      const failures = message.end.results.filter((m) => m.error != null);
-      if (failures.length > 0) {
-        log(`\nfailures:\n`);
-
-        for (const { name, error } of failures) {
-          log(name);
-          log(inspectArgs([error]));
-          log("");
+  function createTestFilter(filter) {
+    return (def) => {
+      if (filter) {
+        if (filter.startsWith("/") && filter.endsWith("/")) {
+          const regex = new RegExp(filter.slice(1, filter.length - 1));
+          return regex.test(def.name);
         }
 
-        log(`failures:\n`);
-
-        for (const { name } of failures) {
-          log(`\t${name}`);
-        }
+        return def.name.includes(filter);
       }
-      log(
-        `\ntest result: ${message.end.failed ? redFailed : greenOk}. ` +
-          `${message.end.passed} passed; ${message.end.failed} failed; ` +
-          `${message.end.ignored} ignored; ${message.end.measured} measured; ` +
-          `${message.end.filtered} filtered out ` +
-          `${formatDuration(message.end.duration)}\n`,
-      );
 
-      if (message.end.usedOnly && message.end.failed == 0) {
-        log(`${redFailed} because the "only" option was used\n`);
-      }
-    }
+      return true;
+    };
   }
 
   function pledgeTestPermissions(permissions) {
@@ -239,167 +172,85 @@ finishing test case.`;
     core.opSync("op_restore_test_permissions", token);
   }
 
-  // TODO(bartlomieju): already implements AsyncGenerator<RunTestsMessage>, but add as "implements to class"
-  // TODO(bartlomieju): implements PromiseLike<RunTestsEndResult>
-  class TestRunner {
-    #usedOnly = false;
-
-    constructor(
-      tests,
-      filterFn,
-      failFast,
-    ) {
-      this.stats = {
-        filtered: 0,
-        ignored: 0,
-        measured: 0,
-        passed: 0,
-        failed: 0,
-      };
-      this.filterFn = filterFn;
-      this.failFast = failFast;
-      const onlyTests = tests.filter(({ only }) => only);
-      this.#usedOnly = onlyTests.length > 0;
-      const unfilteredTests = this.#usedOnly ? onlyTests : tests;
-      this.testsToRun = unfilteredTests.filter(filterFn);
-      this.stats.filtered = unfilteredTests.length - this.testsToRun.length;
-    }
-
-    async *[Symbol.asyncIterator]() {
-      yield { start: { tests: this.testsToRun } };
-
-      const results = [];
-      const suiteStart = +new Date();
-
-      for (const test of this.testsToRun) {
-        const endMessage = {
-          name: test.name,
-          duration: 0,
-        };
-        yield { testStart: { ...test } };
-        if (test.ignore) {
-          endMessage.status = "ignored";
-          this.stats.ignored++;
-        } else {
-          const start = +new Date();
-
-          let token;
-          try {
-            if (test.permissions) {
-              token = pledgeTestPermissions(test.permissions);
-            }
-
-            await test.fn();
-
-            endMessage.status = "passed";
-            this.stats.passed++;
-          } catch (err) {
-            endMessage.status = "failed";
-            endMessage.error = err;
-            this.stats.failed++;
-          } finally {
-            // Permissions must always be restored for a clean environment,
-            // otherwise the process can end up dropping permissions
-            // until there are none left.
-            if (token) {
-              restoreTestPermissions(token);
-            }
-          }
-
-          endMessage.duration = +new Date() - start;
-        }
-        results.push(endMessage);
-        yield { testEnd: endMessage };
-        if (this.failFast && endMessage.error != null) {
-          break;
-        }
-      }
+  async function runTest({ name, ignore, fn, permissions }) {
+    let token = null;
+    const time = Date.now();
 
-      const duration = +new Date() - suiteStart;
+    try {
+      postTestMessage("wait", {
+        name,
+      });
 
-      yield {
-        end: { ...this.stats, usedOnly: this.#usedOnly, duration, results },
-      };
-    }
-  }
+      if (permissions) {
+        token = pledgeTestPermissions(permissions);
+      }
 
-  function createFilterFn(
-    filter,
-    skip,
-  ) {
-    return (def) => {
-      let passes = true;
+      if (ignore) {
+        const duration = Date.now() - time;
+        postTestMessage("result", {
+          name,
+          duration,
+          result: "ignored",
+        });
 
-      if (filter) {
-        if (filter instanceof RegExp) {
-          passes = passes && filter.test(def.name);
-        } else if (filter.startsWith("/") && filter.endsWith("/")) {
-          const filterAsRegex = new RegExp(filter.slice(1, filter.length - 1));
-          passes = passes && filterAsRegex.test(def.name);
-        } else {
-          passes = passes && def.name.includes(filter);
-        }
+        return;
       }
 
-      if (skip) {
-        if (skip instanceof RegExp) {
-          passes = passes && !skip.test(def.name);
-        } else {
-          passes = passes && !def.name.includes(skip);
-        }
-      }
+      await fn();
 
-      return passes;
-    };
+      const duration = Date.now() - time;
+      postTestMessage("result", {
+        name,
+        duration,
+        result: "ok",
+      });
+    } catch (error) {
+      const duration = Date.now() - time;
+
+      postTestMessage("result", {
+        name,
+        duration,
+        result: {
+          "failed": inspectArgs([error]),
+        },
+      });
+    } finally {
+      if (token) {
+        restoreTestPermissions(token);
+      }
+    }
   }
 
   async function runTests({
-    exitOnFail = true,
-    failFast = false,
-    filter = undefined,
-    skip = undefined,
     disableLog = false,
-    reportToConsole: reportToConsole_ = true,
-    onMessage = undefined,
+    filter = null,
   } = {}) {
-    const filterFn = createFilterFn(filter, skip);
-    const testRunner = new TestRunner(TEST_REGISTRY, filterFn, failFast);
-
     const originalConsole = globalThis.console;
-
     if (disableLog) {
-      globalThis.console = disabledConsole;
+      globalThis.console = new Console(() => {});
     }
 
-    let endMsg;
+    const only = tests.filter((test) => test.only);
+    const pending = (only.length > 0 ? only : tests).filter(
+      createTestFilter(filter),
+    );
+    postTestMessage("plan", {
+      filtered: tests.length - pending.length,
+      pending: pending.length,
+      only: only.length > 0,
+    });
 
-    for await (const message of testRunner) {
-      if (onMessage != null) {
-        await onMessage(message);
-      }
-      if (reportToConsole_) {
-        reportToConsole(message);
-      }
-      if (message.end != null) {
-        endMsg = message.end;
-      }
+    for (const test of pending) {
+      await runTest(test);
     }
 
     if (disableLog) {
       globalThis.console = originalConsole;
     }
-
-    if ((endMsg.failed > 0 || endMsg?.usedOnly) && exitOnFail) {
-      exit(1);
-    }
-
-    return endMsg;
   }
 
   window.__bootstrap.internals = {
     ...window.__bootstrap.internals ?? {},
-    reportToConsole,
-    createFilterFn,
     runTests,
   };