refactor: unit test runner communicates using TCP socket (#4336)

Rewrites "cli/js/unit_test_runner.ts" to communicate with spawned subprocesses 
using TCP socket.

* Rewrite "Deno.runTests()" by factoring out testing logic to private "TestApi" 
  class. "TestApi" implements "AsyncIterator" that yields "TestEvent"s, 
  which is an interface for different types of event occuring during running
  tests.

* Add "reporter" argument to "Deno.runTests()" to allow users to provide custom
  reporting mechanism for tests. It's represented by "TestReporter" interface,
  that implements hook functions for each type of "TestEvent". If "reporter"
  is not provided then default console reporting is used (via 
  "ConsoleReporter").

* Change how "unit_test_runner" communicates with spawned suprocesses. Instead
  of parsing text data from child's stdout, a TCP socket is created and used
  for communication. "unit_test_runner" can run in either "master" or "worker"
  mode. Former is responsible for test discovery and establishing needed
  permission combinations; while latter (that is spawned by "master") executes
  tests that match given permission set.

* Use "SocketReporter" that implements "TestReporter" interface to send output
  of tests to "master" process. Data is sent as stringified JSON and then
  parsed by "master" as structured data. "master" applies it's own reporting 
  logic to output tests to console (by reusing default "ConsoleReporter").

1 files changed, 200 insertions, 99 deletions

diff --git a/cli/js/testing.ts b/cli/js/testing.ts
index f1318f0ce..a2944aff4 100644
--- a/cli/js/testing.ts
+++ b/cli/js/testing.ts
@@ -3,17 +3,16 @@ import { red, green, bgRed, gray, italic } from "./colors.ts";
 import { exit } from "./ops/os.ts";
 import { Console } from "./web/console.ts";
 
+const RED_FAILED = red("FAILED");
+const GREEN_OK = green("OK");
+const RED_BG_FAIL = bgRed(" FAIL ");
+const disabledConsole = new Console((_x: string, _isErr?: boolean): void => {});
+
 function formatDuration(time = 0): string {
   const timeStr = `(${time}ms)`;
   return gray(italic(timeStr));
 }
 
-function defer(n: number): Promise<void> {
-  return new Promise((resolve: () => void, _) => {
-    setTimeout(resolve, n);
-  });
-}
-
 export type TestFunction = () => void | Promise<void>;
 
 export interface TestDefinition {
@@ -70,27 +69,137 @@ interface TestStats {
   failed: number;
 }
 
-interface TestCase {
-  name: string;
-  fn: TestFunction;
-  timeElapsed?: number;
-  error?: Error;
-}
-
 export interface RunTestsOptions {
   exitOnFail?: boolean;
   failFast?: boolean;
   only?: string | RegExp;
   skip?: string | RegExp;
   disableLog?: boolean;
+  reporter?: TestReporter;
+}
+
+interface TestResult {
+  passed: boolean;
+  name: string;
+  skipped: boolean;
+  hasRun: boolean;
+  duration: number;
+  error?: Error;
+}
+
+interface TestCase {
+  result: TestResult;
+  fn: TestFunction;
+}
+
+export enum TestEvent {
+  Start = "start",
+  Result = "result",
+  End = "end"
+}
+
+interface TestEventStart {
+  kind: TestEvent.Start;
+  tests: number;
+}
+
+interface TestEventResult {
+  kind: TestEvent.Result;
+  result: TestResult;
+}
+
+interface TestEventEnd {
+  kind: TestEvent.End;
+  stats: TestStats;
+  duration: number;
+  results: TestResult[];
+}
+
+function testDefinitionToTestCase(def: TestDefinition): TestCase {
+  return {
+    fn: def.fn,
+    result: {
+      name: def.name,
+      passed: false,
+      skipped: false,
+      hasRun: false,
+      duration: 0
+    }
+  };
+}
+
+// TODO: already implements AsyncGenerator<RunTestsMessage>, but add as "implements to class"
+// TODO: implements PromiseLike<TestsResult>
+class TestApi {
+  readonly testsToRun: TestDefinition[];
+  readonly testCases: TestCase[];
+  readonly stats: TestStats = {
+    filtered: 0,
+    ignored: 0,
+    measured: 0,
+    passed: 0,
+    failed: 0
+  };
+
+  constructor(
+    public tests: TestDefinition[],
+    public filterFn: (def: TestDefinition) => boolean,
+    public failFast: boolean
+  ) {
+    this.testsToRun = tests.filter(filterFn);
+    this.stats.filtered = tests.length - this.testsToRun.length;
+    this.testCases = this.testsToRun.map(testDefinitionToTestCase);
+  }
+
+  async *[Symbol.asyncIterator](): AsyncIterator<
+    TestEventStart | TestEventResult | TestEventEnd
+  > {
+    yield {
+      kind: TestEvent.Start,
+      tests: this.testsToRun.length
+    };
+
+    const suiteStart = +new Date();
+    for (const testCase of this.testCases) {
+      const { fn, result } = testCase;
+      let shouldBreak = false;
+      try {
+        const start = +new Date();
+        await fn();
+        result.duration = +new Date() - start;
+        result.passed = true;
+        this.stats.passed++;
+      } catch (err) {
+        result.passed = false;
+        result.error = err;
+        this.stats.failed++;
+        shouldBreak = this.failFast;
+      } finally {
+        result.hasRun = true;
+        yield { kind: TestEvent.Result, result };
+        if (shouldBreak) {
+          break;
+        }
+      }
+    }
+
+    const duration = +new Date() - suiteStart;
+    const results = this.testCases.map(r => r.result);
+
+    yield {
+      kind: TestEvent.End,
+      stats: this.stats,
+      results,
+      duration
+    };
+  }
 }
 
-function filterTests(
-  tests: TestDefinition[],
+function createFilterFn(
   only: undefined | string | RegExp,
   skip: undefined | string | RegExp
-): TestDefinition[] {
-  return tests.filter((def: TestDefinition): boolean => {
+): (def: TestDefinition) => boolean {
+  return (def: TestDefinition): boolean => {
     let passes = true;
 
     if (only) {
@@ -110,7 +219,49 @@ function filterTests(
     }
 
     return passes;
-  });
+  };
+}
+
+interface TestReporter {
+  start(msg: TestEventStart): Promise<void>;
+  result(msg: TestEventResult): Promise<void>;
+  end(msg: TestEventEnd): Promise<void>;
+}
+
+export class ConsoleTestReporter implements TestReporter {
+  private console: Console;
+  constructor() {
+    this.console = globalThis.console as Console;
+  }
+
+  async start(event: TestEventStart): Promise<void> {
+    this.console.log(`running ${event.tests} tests`);
+  }
+
+  async result(event: TestEventResult): Promise<void> {
+    const { result } = event;
+
+    if (result.passed) {
+      this.console.log(
+        `${GREEN_OK}     ${result.name} ${formatDuration(result.duration)}`
+      );
+    } else {
+      this.console.log(`${RED_FAILED} ${result.name}`);
+      this.console.log(result.error!);
+    }
+  }
+
+  async end(event: TestEventEnd): Promise<void> {
+    const { stats, duration } = event;
+    // Attempting to match the output of Rust's test runner.
+    this.console.log(
+      `\ntest result: ${stats.failed ? RED_BG_FAIL : GREEN_OK} ` +
+        `${stats.passed} passed; ${stats.failed} failed; ` +
+        `${stats.ignored} ignored; ${stats.measured} measured; ` +
+        `${stats.filtered} filtered out ` +
+        `${formatDuration(duration)}\n`
+    );
+  }
 }
 
 export async function runTests({
@@ -118,104 +269,54 @@ export async function runTests({
   failFast = false,
   only = undefined,
   skip = undefined,
-  disableLog = false
-}: RunTestsOptions = {}): Promise<void> {
-  const testsToRun = filterTests(TEST_REGISTRY, only, skip);
+  disableLog = false,
+  reporter = undefined
+}: RunTestsOptions = {}): Promise<{
+  results: TestResult[];
+  stats: TestStats;
+  duration: number;
+}> {
+  const filterFn = createFilterFn(only, skip);
+  const testApi = new TestApi(TEST_REGISTRY, filterFn, failFast);
 
-  const stats: TestStats = {
-    measured: 0,
-    ignored: 0,
-    filtered: 0,
-    passed: 0,
-    failed: 0
-  };
-
-  const testCases = testsToRun.map(
-    ({ name, fn }): TestCase => {
-      return {
-        name,
-        fn,
-        timeElapsed: 0,
-        error: undefined
-      };
-    }
-  );
+  if (!reporter) {
+    reporter = new ConsoleTestReporter();
+  }
 
   // @ts-ignore
   const originalConsole = globalThis.console;
-  // TODO(bartlomieju): add option to capture output of test
-  // cases and display it if test fails (like --nopcature in Rust)
-  const disabledConsole = new Console(
-    (_x: string, _isErr?: boolean): void => {}
-  );
 
   if (disableLog) {
     // @ts-ignore
     globalThis.console = disabledConsole;
   }
 
-  const RED_FAILED = red("FAILED");
-  const GREEN_OK = green("OK");
-  const RED_BG_FAIL = bgRed(" FAIL ");
-
-  originalConsole.log(`running ${testsToRun.length} tests`);
-  const suiteStart = +new Date();
-
-  for (const testCase of testCases) {
-    try {
-      const start = +new Date();
-      await testCase.fn();
-      testCase.timeElapsed = +new Date() - start;
-      originalConsole.log(
-        `${GREEN_OK}     ${testCase.name} ${formatDuration(
-          testCase.timeElapsed
-        )}`
-      );
-      stats.passed++;
-    } catch (err) {
-      testCase.error = err;
-      originalConsole.log(`${RED_FAILED} ${testCase.name}`);
-      originalConsole.log(err.stack);
-      stats.failed++;
-      if (failFast) {
-        break;
-      }
+  let endMsg: TestEventEnd;
+
+  for await (const testMsg of testApi) {
+    switch (testMsg.kind) {
+      case TestEvent.Start:
+        await reporter.start(testMsg);
+        continue;
+      case TestEvent.Result:
+        await reporter.result(testMsg);
+        continue;
+      case TestEvent.End:
+        endMsg = testMsg;
+        delete endMsg!.kind;
+        await reporter.end(testMsg);
+        continue;
     }
   }
 
-  const suiteDuration = +new Date() - suiteStart;
-
   if (disableLog) {
     // @ts-ignore
     globalThis.console = originalConsole;
   }
 
-  // Attempting to match the output of Rust's test runner.
-  originalConsole.log(
-    `\ntest result: ${stats.failed ? RED_BG_FAIL : GREEN_OK} ` +
-      `${stats.passed} passed; ${stats.failed} failed; ` +
-      `${stats.ignored} ignored; ${stats.measured} measured; ` +
-      `${stats.filtered} filtered out ` +
-      `${formatDuration(suiteDuration)}\n`
-  );
-
-  // TODO(bartlomieju): is `defer` really needed? Shouldn't unhandled
-  // promise rejection be handled per test case?
-  // Use defer to avoid the error being ignored due to unhandled
-  // promise rejections being swallowed.
-  await defer(0);
-
-  if (stats.failed > 0) {
-    originalConsole.error(`There were ${stats.failed} test failures.`);
-    testCases
-      .filter(testCase => !!testCase.error)
-      .forEach(testCase => {
-        originalConsole.error(`${RED_BG_FAIL} ${red(testCase.name)}`);
-        originalConsole.error(testCase.error);
-      });
-
-    if (exitOnFail) {
-      exit(1);
-    }
+  if (endMsg!.stats.failed > 0 && exitOnFail) {
+    exit(1);
   }
+
+  return endMsg!;
 }