refactor: unit test runner communicates using TCP socket (#4336)

Rewrites "cli/js/unit_test_runner.ts" to communicate with spawned subprocesses 
using TCP socket.

* Rewrite "Deno.runTests()" by factoring out testing logic to private "TestApi" 
  class. "TestApi" implements "AsyncIterator" that yields "TestEvent"s, 
  which is an interface for different types of event occuring during running
  tests.

* Add "reporter" argument to "Deno.runTests()" to allow users to provide custom
  reporting mechanism for tests. It's represented by "TestReporter" interface,
  that implements hook functions for each type of "TestEvent". If "reporter"
  is not provided then default console reporting is used (via 
  "ConsoleReporter").

* Change how "unit_test_runner" communicates with spawned suprocesses. Instead
  of parsing text data from child's stdout, a TCP socket is created and used
  for communication. "unit_test_runner" can run in either "master" or "worker"
  mode. Former is responsible for test discovery and establishing needed
  permission combinations; while latter (that is spawned by "master") executes
  tests that match given permission set.

* Use "SocketReporter" that implements "TestReporter" interface to send output
  of tests to "master" process. Data is sent as stringified JSON and then
  parsed by "master" as structured data. "master" applies it's own reporting 
  logic to output tests to console (by reusing default "ConsoleReporter").

1 files changed, 97 insertions, 105 deletions

diff --git a/cli/js/tests/test_util.ts b/cli/js/tests/test_util.ts
index c8f28437d..66edd6681 100644
--- a/cli/js/tests/test_util.ts
+++ b/cli/js/tests/test_util.ts
@@ -1,13 +1,5 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
-//
-// We want to test many ops in deno which have different behavior depending on
-// the permissions set. These tests can specify which permissions they expect,
-// which appends a special string like "permW1N0" to the end of the test name.
-// Here we run several copies of deno with different permissions, filtering the
-// tests by the special string. permW1N0 means allow-write but not allow-net.
-// See tools/unit_tests.py for more details.
-
-import { readLines } from "../../../std/io/bufio.ts";
+
 import { assert, assertEquals } from "../../../std/testing/asserts.ts";
 export {
   assert,
@@ -20,16 +12,7 @@ export {
   unreachable,
   fail
 } from "../../../std/testing/asserts.ts";
-
-interface TestPermissions {
-  read?: boolean;
-  write?: boolean;
-  net?: boolean;
-  env?: boolean;
-  run?: boolean;
-  plugin?: boolean;
-  hrtime?: boolean;
-}
+export { readLines } from "../../../std/io/bufio.ts";
 
 export interface Permissions {
   read: boolean;
@@ -41,10 +24,22 @@ export interface Permissions {
   hrtime: boolean;
 }
 
+export function fmtPerms(perms: Permissions): string {
+  const p = Object.keys(perms)
+    .filter((e): boolean => perms[e as keyof Permissions] === true)
+    .map(key => `--allow-${key}`);
+
+  if (p.length) {
+    return p.join(" ");
+  }
+
+  return "<no permissions>";
+}
+
 const isGranted = async (name: Deno.PermissionName): Promise<boolean> =>
   (await Deno.permissions.query({ name })).state === "granted";
 
-async function getProcessPermissions(): Promise<Permissions> {
+export async function getProcessPermissions(): Promise<Permissions> {
   return {
     run: await isGranted("run"),
     read: await isGranted("read"),
@@ -56,9 +51,7 @@ async function getProcessPermissions(): Promise<Permissions> {
   };
 }
 
-const processPerms = await getProcessPermissions();
-
-function permissionsMatch(
+export function permissionsMatch(
   processPerms: Permissions,
   requiredPerms: Permissions
 ): boolean {
@@ -94,7 +87,23 @@ function registerPermCombination(perms: Permissions): void {
   }
 }
 
-function normalizeTestPermissions(perms: TestPermissions): Permissions {
+export async function registerUnitTests(): Promise<void> {
+  const processPerms = await getProcessPermissions();
+
+  for (const unitTestDefinition of REGISTERED_UNIT_TESTS) {
+    if (unitTestDefinition.skip) {
+      continue;
+    }
+
+    if (!permissionsMatch(processPerms, unitTestDefinition.perms)) {
+      continue;
+    }
+
+    Deno.test(unitTestDefinition);
+  }
+}
+
+function normalizeTestPermissions(perms: UnitTestPermissions): Permissions {
   return {
     read: !!perms.read,
     write: !!perms.write,
@@ -147,11 +156,30 @@ function assertResources(fn: Deno.TestFunction): Deno.TestFunction {
   };
 }
 
+interface UnitTestPermissions {
+  read?: boolean;
+  write?: boolean;
+  net?: boolean;
+  env?: boolean;
+  run?: boolean;
+  plugin?: boolean;
+  hrtime?: boolean;
+}
+
 interface UnitTestOptions {
   skip?: boolean;
-  perms?: TestPermissions;
+  perms?: UnitTestPermissions;
 }
 
+interface UnitTestDefinition {
+  name: string;
+  fn: Deno.TestFunction;
+  skip?: boolean;
+  perms: Permissions;
+}
+
+export const REGISTERED_UNIT_TESTS: UnitTestDefinition[] = [];
+
 export function unitTest(fn: Deno.TestFunction): void;
 export function unitTest(options: UnitTestOptions, fn: Deno.TestFunction): void;
 export function unitTest(
@@ -187,53 +215,15 @@ export function unitTest(
 
   const normalizedPerms = normalizeTestPermissions(options.perms || {});
   registerPermCombination(normalizedPerms);
-  if (!permissionsMatch(processPerms, normalizedPerms)) {
-    return;
-  }
 
-  const testDefinition: Deno.TestDefinition = {
+  const unitTestDefinition: UnitTestDefinition = {
     name,
-    fn: assertResources(assertOps(fn))
+    fn: assertResources(assertOps(fn)),
+    skip: !!options.skip,
+    perms: normalizedPerms
   };
-  Deno.test(testDefinition);
-}
 
-function extractNumber(re: RegExp, str: string): number | undefined {
-  const match = str.match(re);
-
-  if (match) {
-    return Number.parseInt(match[1]);
-  }
-}
-
-export async function parseUnitTestOutput(
-  reader: Deno.Reader,
-  print: boolean
-): Promise<{ actual?: number; expected?: number; resultOutput?: string }> {
-  let expected, actual, result;
-
-  for await (const line of readLines(reader)) {
-    if (!expected) {
-      // expect "running 30 tests"
-      expected = extractNumber(/running (\d+) tests/, line);
-    } else if (line.indexOf("test result:") !== -1) {
-      result = line;
-    }
-
-    if (print) {
-      console.log(line);
-    }
-  }
-
-  // Check that the number of expected tests equals what was reported at the
-  // bottom.
-  if (result) {
-    // result should be a string like this:
-    // "test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; ..."
-    actual = extractNumber(/(\d+) passed/, result);
-  }
-
-  return { actual, expected, resultOutput: result };
+  REGISTERED_UNIT_TESTS.push(unitTestDefinition);
 }
 
 export interface ResolvableMethods<T> {
@@ -254,6 +244,45 @@ export function createResolvable<T>(): Resolvable<T> {
   return Object.assign(promise, methods!) as Resolvable<T>;
 }
 
+export class SocketReporter implements Deno.TestReporter {
+  private encoder: TextEncoder;
+
+  constructor(private conn: Deno.Conn) {
+    this.encoder = new TextEncoder();
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  async write(msg: any): Promise<void> {
+    const encodedMsg = this.encoder.encode(`${JSON.stringify(msg)}\n`);
+    await Deno.writeAll(this.conn, encodedMsg);
+  }
+
+  async start(msg: Deno.TestEventStart): Promise<void> {
+    await this.write(msg);
+  }
+
+  async result(msg: Deno.TestEventResult): Promise<void> {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const serializedMsg: any = { ...msg };
+
+    // Error is a JS object, so we need to turn it into string to
+    // send over socket.
+    if (serializedMsg.result.error) {
+      serializedMsg.result.error = String(serializedMsg.result.error.stack);
+    }
+
+    await this.write(serializedMsg);
+  }
+
+  async end(msg: Deno.TestEventEnd): Promise<void> {
+    await this.write(msg);
+  }
+
+  close(): void {
+    this.conn.close();
+  }
+}
+
 unitTest(function permissionsMatches(): void {
   assert(
     permissionsMatch(
@@ -341,43 +370,6 @@ unitTest(function permissionsMatches(): void {
   );
 });
 
-unitTest(
-  { perms: { read: true } },
-  async function parsingUnitTestOutput(): Promise<void> {
-    const cwd = Deno.cwd();
-    const testDataPath = `${cwd}/tools/testdata/`;
-
-    let result;
-
-    // This is an example of a successful unit test output.
-    const f1 = await Deno.open(`${testDataPath}/unit_test_output1.txt`);
-    result = await parseUnitTestOutput(f1, false);
-    assertEquals(result.actual, 96);
-    assertEquals(result.expected, 96);
-    f1.close();
-
-    // This is an example of a silently dying unit test.
-    const f2 = await Deno.open(`${testDataPath}/unit_test_output2.txt`);
-    result = await parseUnitTestOutput(f2, false);
-    assertEquals(result.actual, undefined);
-    assertEquals(result.expected, 96);
-    f2.close();
-
-    // This is an example of compiling before successful unit tests.
-    const f3 = await Deno.open(`${testDataPath}/unit_test_output3.txt`);
-    result = await parseUnitTestOutput(f3, false);
-    assertEquals(result.actual, 96);
-    assertEquals(result.expected, 96);
-    f3.close();
-
-    // Check what happens on empty output.
-    const f = new Deno.Buffer(new TextEncoder().encode("\n\n\n"));
-    result = await parseUnitTestOutput(f, false);
-    assertEquals(result.actual, undefined);
-    assertEquals(result.expected, undefined);
-  }
-);
-
 /*
  * Ensure all unit test files (e.g. xxx_test.ts) are present as imports in
  * cli/js/tests/unit_tests.ts as it is easy to miss this out