refactor: unit test runner communicates using TCP socket (#4336)

Rewrites "cli/js/unit_test_runner.ts" to communicate with spawned subprocesses 
using TCP socket.

* Rewrite "Deno.runTests()" by factoring out testing logic to private "TestApi" 
  class. "TestApi" implements "AsyncIterator" that yields "TestEvent"s, 
  which is an interface for different types of event occuring during running
  tests.

* Add "reporter" argument to "Deno.runTests()" to allow users to provide custom
  reporting mechanism for tests. It's represented by "TestReporter" interface,
  that implements hook functions for each type of "TestEvent". If "reporter"
  is not provided then default console reporting is used (via 
  "ConsoleReporter").

* Change how "unit_test_runner" communicates with spawned suprocesses. Instead
  of parsing text data from child's stdout, a TCP socket is created and used
  for communication. "unit_test_runner" can run in either "master" or "worker"
  mode. Former is responsible for test discovery and establishing needed
  permission combinations; while latter (that is spawned by "master") executes
  tests that match given permission set.

* Use "SocketReporter" that implements "TestReporter" interface to send output
  of tests to "master" process. Data is sent as stringified JSON and then
  parsed by "master" as structured data. "master" applies it's own reporting 
  logic to output tests to console (by reusing default "ConsoleReporter").

9 files changed, 563 insertions, 282 deletions

diff --git a/cli/js/deno.ts b/cli/js/deno.ts
index b22f076ef..6a493faf8 100644
--- a/cli/js/deno.ts
+++ b/cli/js/deno.ts
@@ -118,7 +118,7 @@ export { utimeSync, utime } from "./ops/fs/utime.ts";
 export { version } from "./version.ts";
 export { writeFileSync, writeFile, WriteFileOptions } from "./write_file.ts";
 export const args: string[] = [];
-export { test, runTests } from "./testing.ts";
+export { test, runTests, TestEvent, ConsoleTestReporter } from "./testing.ts";
 
 // These are internal Deno APIs.  We are marking them as internal so they do not
 // appear in the runtime type library.
diff --git a/cli/js/lib.deno.ns.d.ts b/cli/js/lib.deno.ns.d.ts
index bf10049bb..751e4452b 100644
--- a/cli/js/lib.deno.ns.d.ts
+++ b/cli/js/lib.deno.ns.d.ts
@@ -32,6 +32,59 @@ declare namespace Deno {
    * when `Deno.runTests` is used */
   export function test(name: string, fn: TestFunction): void;
 
+  interface TestResult {
+    passed: boolean;
+    name: string;
+    skipped: boolean;
+    hasRun: boolean;
+    duration: number;
+    error?: Error;
+  }
+
+  interface TestStats {
+    filtered: number;
+    ignored: number;
+    measured: number;
+    passed: number;
+    failed: number;
+  }
+
+  export enum TestEvent {
+    Start = "start",
+    Result = "result",
+    End = "end"
+  }
+
+  interface TestEventStart {
+    kind: TestEvent.Start;
+    tests: number;
+  }
+
+  interface TestEventResult {
+    kind: TestEvent.Result;
+    result: TestResult;
+  }
+
+  interface TestEventEnd {
+    kind: TestEvent.End;
+    stats: TestStats;
+    duration: number;
+    results: TestResult[];
+  }
+
+  interface TestReporter {
+    start(event: TestEventStart): Promise<void>;
+    result(event: TestEventResult): Promise<void>;
+    end(event: TestEventEnd): Promise<void>;
+  }
+
+  export class ConsoleTestReporter implements TestReporter {
+    constructor();
+    start(event: TestEventStart): Promise<void>;
+    result(event: TestEventResult): Promise<void>;
+    end(event: TestEventEnd): Promise<void>;
+  }
+
   export interface RunTestsOptions {
     /** If `true`, Deno will exit with status code 1 if there was
      * test failure. Defaults to `true`. */
@@ -46,11 +99,19 @@ declare namespace Deno {
     skip?: string | RegExp;
     /** Disable logging of the results. Defaults to `false`. */
     disableLog?: boolean;
+    /** Custom reporter class. If not provided uses console reporter. */
+    reporter?: TestReporter;
   }
 
   /** Run any tests which have been registered. Always resolves
    * asynchronously. */
-  export function runTests(opts?: RunTestsOptions): Promise<void>;
+  export function runTests(
+    opts?: RunTestsOptions
+  ): Promise<{
+    results: TestResult[];
+    stats: TestStats;
+    duration: number;
+  }>;
 
   /** Get the `loadavg`. Requires `allow-env` permission.
    *
diff --git a/cli/js/testing.ts b/cli/js/testing.ts
index f1318f0ce..a2944aff4 100644
--- a/cli/js/testing.ts
+++ b/cli/js/testing.ts
@@ -3,17 +3,16 @@ import { red, green, bgRed, gray, italic } from "./colors.ts";
 import { exit } from "./ops/os.ts";
 import { Console } from "./web/console.ts";
 
+const RED_FAILED = red("FAILED");
+const GREEN_OK = green("OK");
+const RED_BG_FAIL = bgRed(" FAIL ");
+const disabledConsole = new Console((_x: string, _isErr?: boolean): void => {});
+
 function formatDuration(time = 0): string {
   const timeStr = `(${time}ms)`;
   return gray(italic(timeStr));
 }
 
-function defer(n: number): Promise<void> {
-  return new Promise((resolve: () => void, _) => {
-    setTimeout(resolve, n);
-  });
-}
-
 export type TestFunction = () => void | Promise<void>;
 
 export interface TestDefinition {
@@ -70,27 +69,137 @@ interface TestStats {
   failed: number;
 }
 
-interface TestCase {
-  name: string;
-  fn: TestFunction;
-  timeElapsed?: number;
-  error?: Error;
-}
-
 export interface RunTestsOptions {
   exitOnFail?: boolean;
   failFast?: boolean;
   only?: string | RegExp;
   skip?: string | RegExp;
   disableLog?: boolean;
+  reporter?: TestReporter;
+}
+
+interface TestResult {
+  passed: boolean;
+  name: string;
+  skipped: boolean;
+  hasRun: boolean;
+  duration: number;
+  error?: Error;
+}
+
+interface TestCase {
+  result: TestResult;
+  fn: TestFunction;
+}
+
+export enum TestEvent {
+  Start = "start",
+  Result = "result",
+  End = "end"
+}
+
+interface TestEventStart {
+  kind: TestEvent.Start;
+  tests: number;
+}
+
+interface TestEventResult {
+  kind: TestEvent.Result;
+  result: TestResult;
+}
+
+interface TestEventEnd {
+  kind: TestEvent.End;
+  stats: TestStats;
+  duration: number;
+  results: TestResult[];
+}
+
+function testDefinitionToTestCase(def: TestDefinition): TestCase {
+  return {
+    fn: def.fn,
+    result: {
+      name: def.name,
+      passed: false,
+      skipped: false,
+      hasRun: false,
+      duration: 0
+    }
+  };
+}
+
+// TODO: already implements AsyncGenerator<RunTestsMessage>, but add as "implements to class"
+// TODO: implements PromiseLike<TestsResult>
+class TestApi {
+  readonly testsToRun: TestDefinition[];
+  readonly testCases: TestCase[];
+  readonly stats: TestStats = {
+    filtered: 0,
+    ignored: 0,
+    measured: 0,
+    passed: 0,
+    failed: 0
+  };
+
+  constructor(
+    public tests: TestDefinition[],
+    public filterFn: (def: TestDefinition) => boolean,
+    public failFast: boolean
+  ) {
+    this.testsToRun = tests.filter(filterFn);
+    this.stats.filtered = tests.length - this.testsToRun.length;
+    this.testCases = this.testsToRun.map(testDefinitionToTestCase);
+  }
+
+  async *[Symbol.asyncIterator](): AsyncIterator<
+    TestEventStart | TestEventResult | TestEventEnd
+  > {
+    yield {
+      kind: TestEvent.Start,
+      tests: this.testsToRun.length
+    };
+
+    const suiteStart = +new Date();
+    for (const testCase of this.testCases) {
+      const { fn, result } = testCase;
+      let shouldBreak = false;
+      try {
+        const start = +new Date();
+        await fn();
+        result.duration = +new Date() - start;
+        result.passed = true;
+        this.stats.passed++;
+      } catch (err) {
+        result.passed = false;
+        result.error = err;
+        this.stats.failed++;
+        shouldBreak = this.failFast;
+      } finally {
+        result.hasRun = true;
+        yield { kind: TestEvent.Result, result };
+        if (shouldBreak) {
+          break;
+        }
+      }
+    }
+
+    const duration = +new Date() - suiteStart;
+    const results = this.testCases.map(r => r.result);
+
+    yield {
+      kind: TestEvent.End,
+      stats: this.stats,
+      results,
+      duration
+    };
+  }
 }
 
-function filterTests(
-  tests: TestDefinition[],
+function createFilterFn(
   only: undefined | string | RegExp,
   skip: undefined | string | RegExp
-): TestDefinition[] {
-  return tests.filter((def: TestDefinition): boolean => {
+): (def: TestDefinition) => boolean {
+  return (def: TestDefinition): boolean => {
     let passes = true;
 
     if (only) {
@@ -110,7 +219,49 @@ function filterTests(
     }
 
     return passes;
-  });
+  };
+}
+
+interface TestReporter {
+  start(msg: TestEventStart): Promise<void>;
+  result(msg: TestEventResult): Promise<void>;
+  end(msg: TestEventEnd): Promise<void>;
+}
+
+export class ConsoleTestReporter implements TestReporter {
+  private console: Console;
+  constructor() {
+    this.console = globalThis.console as Console;
+  }
+
+  async start(event: TestEventStart): Promise<void> {
+    this.console.log(`running ${event.tests} tests`);
+  }
+
+  async result(event: TestEventResult): Promise<void> {
+    const { result } = event;
+
+    if (result.passed) {
+      this.console.log(
+        `${GREEN_OK}     ${result.name} ${formatDuration(result.duration)}`
+      );
+    } else {
+      this.console.log(`${RED_FAILED} ${result.name}`);
+      this.console.log(result.error!);
+    }
+  }
+
+  async end(event: TestEventEnd): Promise<void> {
+    const { stats, duration } = event;
+    // Attempting to match the output of Rust's test runner.
+    this.console.log(
+      `\ntest result: ${stats.failed ? RED_BG_FAIL : GREEN_OK} ` +
+        `${stats.passed} passed; ${stats.failed} failed; ` +
+        `${stats.ignored} ignored; ${stats.measured} measured; ` +
+        `${stats.filtered} filtered out ` +
+        `${formatDuration(duration)}\n`
+    );
+  }
 }
 
 export async function runTests({
@@ -118,104 +269,54 @@ export async function runTests({
   failFast = false,
   only = undefined,
   skip = undefined,
-  disableLog = false
-}: RunTestsOptions = {}): Promise<void> {
-  const testsToRun = filterTests(TEST_REGISTRY, only, skip);
+  disableLog = false,
+  reporter = undefined
+}: RunTestsOptions = {}): Promise<{
+  results: TestResult[];
+  stats: TestStats;
+  duration: number;
+}> {
+  const filterFn = createFilterFn(only, skip);
+  const testApi = new TestApi(TEST_REGISTRY, filterFn, failFast);
 
-  const stats: TestStats = {
-    measured: 0,
-    ignored: 0,
-    filtered: 0,
-    passed: 0,
-    failed: 0
-  };
-
-  const testCases = testsToRun.map(
-    ({ name, fn }): TestCase => {
-      return {
-        name,
-        fn,
-        timeElapsed: 0,
-        error: undefined
-      };
-    }
-  );
+  if (!reporter) {
+    reporter = new ConsoleTestReporter();
+  }
 
   // @ts-ignore
   const originalConsole = globalThis.console;
-  // TODO(bartlomieju): add option to capture output of test
-  // cases and display it if test fails (like --nopcature in Rust)
-  const disabledConsole = new Console(
-    (_x: string, _isErr?: boolean): void => {}
-  );
 
   if (disableLog) {
     // @ts-ignore
     globalThis.console = disabledConsole;
   }
 
-  const RED_FAILED = red("FAILED");
-  const GREEN_OK = green("OK");
-  const RED_BG_FAIL = bgRed(" FAIL ");
-
-  originalConsole.log(`running ${testsToRun.length} tests`);
-  const suiteStart = +new Date();
-
-  for (const testCase of testCases) {
-    try {
-      const start = +new Date();
-      await testCase.fn();
-      testCase.timeElapsed = +new Date() - start;
-      originalConsole.log(
-        `${GREEN_OK}     ${testCase.name} ${formatDuration(
-          testCase.timeElapsed
-        )}`
-      );
-      stats.passed++;
-    } catch (err) {
-      testCase.error = err;
-      originalConsole.log(`${RED_FAILED} ${testCase.name}`);
-      originalConsole.log(err.stack);
-      stats.failed++;
-      if (failFast) {
-        break;
-      }
+  let endMsg: TestEventEnd;
+
+  for await (const testMsg of testApi) {
+    switch (testMsg.kind) {
+      case TestEvent.Start:
+        await reporter.start(testMsg);
+        continue;
+      case TestEvent.Result:
+        await reporter.result(testMsg);
+        continue;
+      case TestEvent.End:
+        endMsg = testMsg;
+        delete endMsg!.kind;
+        await reporter.end(testMsg);
+        continue;
     }
   }
 
-  const suiteDuration = +new Date() - suiteStart;
-
   if (disableLog) {
     // @ts-ignore
     globalThis.console = originalConsole;
   }
 
-  // Attempting to match the output of Rust's test runner.
-  originalConsole.log(
-    `\ntest result: ${stats.failed ? RED_BG_FAIL : GREEN_OK} ` +
-      `${stats.passed} passed; ${stats.failed} failed; ` +
-      `${stats.ignored} ignored; ${stats.measured} measured; ` +
-      `${stats.filtered} filtered out ` +
-      `${formatDuration(suiteDuration)}\n`
-  );
-
-  // TODO(bartlomieju): is `defer` really needed? Shouldn't unhandled
-  // promise rejection be handled per test case?
-  // Use defer to avoid the error being ignored due to unhandled
-  // promise rejections being swallowed.
-  await defer(0);
-
-  if (stats.failed > 0) {
-    originalConsole.error(`There were ${stats.failed} test failures.`);
-    testCases
-      .filter(testCase => !!testCase.error)
-      .forEach(testCase => {
-        originalConsole.error(`${RED_BG_FAIL} ${red(testCase.name)}`);
-        originalConsole.error(testCase.error);
-      });
-
-    if (exitOnFail) {
-      exit(1);
-    }
+  if (endMsg!.stats.failed > 0 && exitOnFail) {
+    exit(1);
   }
+
+  return endMsg!;
 }
diff --git a/cli/js/tests/location_test.ts b/cli/js/tests/location_test.ts
index 78ecb55b3..2d2faf0c2 100644
--- a/cli/js/tests/location_test.ts
+++ b/cli/js/tests/location_test.ts
@@ -3,5 +3,5 @@ import { unitTest, assert } from "./test_util.ts";
 
 unitTest(function locationBasic(): void {
   // location example: file:///Users/rld/src/deno/js/unit_tests.ts
-  assert(window.location.toString().endsWith("unit_tests.ts"));
+  assert(window.location.toString().endsWith("unit_test_runner.ts"));
 });
diff --git a/cli/js/tests/resources_test.ts b/cli/js/tests/resources_test.ts
index 84b713a6d..680fac8b7 100644
--- a/cli/js/tests/resources_test.ts
+++ b/cli/js/tests/resources_test.ts
@@ -1,5 +1,5 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
-import { unitTest, assertEquals } from "./test_util.ts";
+import { unitTest, assertEquals, assert } from "./test_util.ts";
 
 unitTest(function resourcesStdio(): void {
   const res = Deno.resources();
@@ -21,10 +21,10 @@ unitTest({ perms: { net: true } }, async function resourcesNet(): Promise<
     Object.values(res).filter((r): boolean => r === "tcpListener").length,
     1
   );
-  assertEquals(
-    Object.values(res).filter((r): boolean => r === "tcpStream").length,
-    2
+  const tcpStreams = Object.values(res).filter(
+    (r): boolean => r === "tcpStream"
   );
+  assert(tcpStreams.length >= 2);
 
   listenerConn.close();
   dialerConn.close();
diff --git a/cli/js/tests/test_util.ts b/cli/js/tests/test_util.ts
index c8f28437d..66edd6681 100644
--- a/cli/js/tests/test_util.ts
+++ b/cli/js/tests/test_util.ts
@@ -1,13 +1,5 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
-//
-// We want to test many ops in deno which have different behavior depending on
-// the permissions set. These tests can specify which permissions they expect,
-// which appends a special string like "permW1N0" to the end of the test name.
-// Here we run several copies of deno with different permissions, filtering the
-// tests by the special string. permW1N0 means allow-write but not allow-net.
-// See tools/unit_tests.py for more details.
-
-import { readLines } from "../../../std/io/bufio.ts";
+
 import { assert, assertEquals } from "../../../std/testing/asserts.ts";
 export {
   assert,
@@ -20,16 +12,7 @@ export {
   unreachable,
   fail
 } from "../../../std/testing/asserts.ts";
-
-interface TestPermissions {
-  read?: boolean;
-  write?: boolean;
-  net?: boolean;
-  env?: boolean;
-  run?: boolean;
-  plugin?: boolean;
-  hrtime?: boolean;
-}
+export { readLines } from "../../../std/io/bufio.ts";
 
 export interface Permissions {
   read: boolean;
@@ -41,10 +24,22 @@ export interface Permissions {
   hrtime: boolean;
 }
 
+export function fmtPerms(perms: Permissions): string {
+  const p = Object.keys(perms)
+    .filter((e): boolean => perms[e as keyof Permissions] === true)
+    .map(key => `--allow-${key}`);
+
+  if (p.length) {
+    return p.join(" ");
+  }
+
+  return "<no permissions>";
+}
+
 const isGranted = async (name: Deno.PermissionName): Promise<boolean> =>
   (await Deno.permissions.query({ name })).state === "granted";
 
-async function getProcessPermissions(): Promise<Permissions> {
+export async function getProcessPermissions(): Promise<Permissions> {
   return {
     run: await isGranted("run"),
     read: await isGranted("read"),
@@ -56,9 +51,7 @@ async function getProcessPermissions(): Promise<Permissions> {
   };
 }
 
-const processPerms = await getProcessPermissions();
-
-function permissionsMatch(
+export function permissionsMatch(
   processPerms: Permissions,
   requiredPerms: Permissions
 ): boolean {
@@ -94,7 +87,23 @@ function registerPermCombination(perms: Permissions): void {
   }
 }
 
-function normalizeTestPermissions(perms: TestPermissions): Permissions {
+export async function registerUnitTests(): Promise<void> {
+  const processPerms = await getProcessPermissions();
+
+  for (const unitTestDefinition of REGISTERED_UNIT_TESTS) {
+    if (unitTestDefinition.skip) {
+      continue;
+    }
+
+    if (!permissionsMatch(processPerms, unitTestDefinition.perms)) {
+      continue;
+    }
+
+    Deno.test(unitTestDefinition);
+  }
+}
+
+function normalizeTestPermissions(perms: UnitTestPermissions): Permissions {
   return {
     read: !!perms.read,
     write: !!perms.write,
@@ -147,11 +156,30 @@ function assertResources(fn: Deno.TestFunction): Deno.TestFunction {
   };
 }
 
+interface UnitTestPermissions {
+  read?: boolean;
+  write?: boolean;
+  net?: boolean;
+  env?: boolean;
+  run?: boolean;
+  plugin?: boolean;
+  hrtime?: boolean;
+}
+
 interface UnitTestOptions {
   skip?: boolean;
-  perms?: TestPermissions;
+  perms?: UnitTestPermissions;
 }
 
+interface UnitTestDefinition {
+  name: string;
+  fn: Deno.TestFunction;
+  skip?: boolean;
+  perms: Permissions;
+}
+
+export const REGISTERED_UNIT_TESTS: UnitTestDefinition[] = [];
+
 export function unitTest(fn: Deno.TestFunction): void;
 export function unitTest(options: UnitTestOptions, fn: Deno.TestFunction): void;
 export function unitTest(
@@ -187,53 +215,15 @@ export function unitTest(
 
   const normalizedPerms = normalizeTestPermissions(options.perms || {});
   registerPermCombination(normalizedPerms);
-  if (!permissionsMatch(processPerms, normalizedPerms)) {
-    return;
-  }
 
-  const testDefinition: Deno.TestDefinition = {
+  const unitTestDefinition: UnitTestDefinition = {
     name,
-    fn: assertResources(assertOps(fn))
+    fn: assertResources(assertOps(fn)),
+    skip: !!options.skip,
+    perms: normalizedPerms
   };
-  Deno.test(testDefinition);
-}
 
-function extractNumber(re: RegExp, str: string): number | undefined {
-  const match = str.match(re);
-
-  if (match) {
-    return Number.parseInt(match[1]);
-  }
-}
-
-export async function parseUnitTestOutput(
-  reader: Deno.Reader,
-  print: boolean
-): Promise<{ actual?: number; expected?: number; resultOutput?: string }> {
-  let expected, actual, result;
-
-  for await (const line of readLines(reader)) {
-    if (!expected) {
-      // expect "running 30 tests"
-      expected = extractNumber(/running (\d+) tests/, line);
-    } else if (line.indexOf("test result:") !== -1) {
-      result = line;
-    }
-
-    if (print) {
-      console.log(line);
-    }
-  }
-
-  // Check that the number of expected tests equals what was reported at the
-  // bottom.
-  if (result) {
-    // result should be a string like this:
-    // "test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; ..."
-    actual = extractNumber(/(\d+) passed/, result);
-  }
-
-  return { actual, expected, resultOutput: result };
+  REGISTERED_UNIT_TESTS.push(unitTestDefinition);
 }
 
 export interface ResolvableMethods<T> {
@@ -254,6 +244,45 @@ export function createResolvable<T>(): Resolvable<T> {
   return Object.assign(promise, methods!) as Resolvable<T>;
 }
 
+export class SocketReporter implements Deno.TestReporter {
+  private encoder: TextEncoder;
+
+  constructor(private conn: Deno.Conn) {
+    this.encoder = new TextEncoder();
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  async write(msg: any): Promise<void> {
+    const encodedMsg = this.encoder.encode(`${JSON.stringify(msg)}\n`);
+    await Deno.writeAll(this.conn, encodedMsg);
+  }
+
+  async start(msg: Deno.TestEventStart): Promise<void> {
+    await this.write(msg);
+  }
+
+  async result(msg: Deno.TestEventResult): Promise<void> {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const serializedMsg: any = { ...msg };
+
+    // Error is a JS object, so we need to turn it into string to
+    // send over socket.
+    if (serializedMsg.result.error) {
+      serializedMsg.result.error = String(serializedMsg.result.error.stack);
+    }
+
+    await this.write(serializedMsg);
+  }
+
+  async end(msg: Deno.TestEventEnd): Promise<void> {
+    await this.write(msg);
+  }
+
+  close(): void {
+    this.conn.close();
+  }
+}
+
 unitTest(function permissionsMatches(): void {
   assert(
     permissionsMatch(
@@ -341,43 +370,6 @@ unitTest(function permissionsMatches(): void {
   );
 });
 
-unitTest(
-  { perms: { read: true } },
-  async function parsingUnitTestOutput(): Promise<void> {
-    const cwd = Deno.cwd();
-    const testDataPath = `${cwd}/tools/testdata/`;
-
-    let result;
-
-    // This is an example of a successful unit test output.
-    const f1 = await Deno.open(`${testDataPath}/unit_test_output1.txt`);
-    result = await parseUnitTestOutput(f1, false);
-    assertEquals(result.actual, 96);
-    assertEquals(result.expected, 96);
-    f1.close();
-
-    // This is an example of a silently dying unit test.
-    const f2 = await Deno.open(`${testDataPath}/unit_test_output2.txt`);
-    result = await parseUnitTestOutput(f2, false);
-    assertEquals(result.actual, undefined);
-    assertEquals(result.expected, 96);
-    f2.close();
-
-    // This is an example of compiling before successful unit tests.
-    const f3 = await Deno.open(`${testDataPath}/unit_test_output3.txt`);
-    result = await parseUnitTestOutput(f3, false);
-    assertEquals(result.actual, 96);
-    assertEquals(result.expected, 96);
-    f3.close();
-
-    // Check what happens on empty output.
-    const f = new Deno.Buffer(new TextEncoder().encode("\n\n\n"));
-    result = await parseUnitTestOutput(f, false);
-    assertEquals(result.actual, undefined);
-    assertEquals(result.expected, undefined);
-  }
-);
-
 /*
  * Ensure all unit test files (e.g. xxx_test.ts) are present as imports in
  * cli/js/tests/unit_tests.ts as it is easy to miss this out
diff --git a/cli/js/tests/unit_test_runner.ts b/cli/js/tests/unit_test_runner.ts
index a5b7c3a48..f018fb59e 100755
--- a/cli/js/tests/unit_test_runner.ts
+++ b/cli/js/tests/unit_test_runner.ts
@@ -2,42 +2,187 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
 import "./unit_tests.ts";
 import {
+  assert,
+  readLines,
   permissionCombinations,
-  parseUnitTestOutput,
-  Permissions
+  Permissions,
+  registerUnitTests,
+  SocketReporter,
+  fmtPerms
 } from "./test_util.ts";
 
-interface TestResult {
-  perms: string;
-  output?: string;
-  result: number;
+interface PermissionSetTestResult {
+  perms: Permissions;
+  passed: boolean;
+  stats: Deno.TestStats;
+  permsStr: string;
+  duration: number;
 }
 
-function permsToCliFlags(perms: Permissions): string[] {
-  return Object.keys(perms)
-    .map(key => {
-      if (!perms[key as keyof Permissions]) return "";
+const PERMISSIONS: Deno.PermissionName[] = [
+  "read",
+  "write",
+  "net",
+  "env",
+  "run",
+  "plugin",
+  "hrtime"
+];
+
+/**
+ * Take a list of permissions and revoke missing permissions.
+ */
+async function dropWorkerPermissions(
+  requiredPermissions: Deno.PermissionName[]
+): Promise<void> {
+  const permsToDrop = PERMISSIONS.filter((p): boolean => {
+    return !requiredPermissions.includes(p);
+  });
+
+  for (const perm of permsToDrop) {
+    await Deno.permissions.revoke({ name: perm });
+  }
+}
+
+async function workerRunnerMain(args: string[]): Promise<void> {
+  const addrArg = args.find(e => e.includes("--addr"));
+  assert(typeof addrArg === "string", "Missing --addr argument");
+  const addrStr = addrArg.split("=")[1];
+  const [hostname, port] = addrStr.split(":");
+  const addr = { hostname, port: Number(port) };
+
+  let perms: Deno.PermissionName[] = [];
+  const permsArg = args.find(e => e.includes("--perms"));
+  assert(typeof permsArg === "string", "Missing --perms argument");
+  const permsStr = permsArg.split("=")[1];
+  if (permsStr.length > 0) {
+    perms = permsStr.split(",") as Deno.PermissionName[];
+  }
+  // Setup reporter
+  const conn = await Deno.connect(addr);
+  const socketReporter = new SocketReporter(conn);
+  // Drop current process permissions to requested set
+  await dropWorkerPermissions(perms);
+  // Register unit tests that match process permissions
+  await registerUnitTests();
+  // Execute tests
+  await Deno.runTests({
+    failFast: false,
+    exitOnFail: false,
+    reporter: socketReporter
+  });
+  // Notify parent process we're done
+  socketReporter.close();
+}
 
-      const cliFlag = key.replace(
-        /\.?([A-Z])/g,
-        (x, y): string => `-${y.toLowerCase()}`
-      );
-      return `--allow-${cliFlag}`;
+function spawnWorkerRunner(addr: string, perms: Permissions): Deno.Process {
+  // run subsequent tests using same deno executable
+  const permStr = Object.keys(perms)
+    .filter((permName): boolean => {
+      return perms[permName as Deno.PermissionName] === true;
     })
-    .filter((e): boolean => e.length > 0);
+    .join(",");
+
+  const args = [
+    Deno.execPath(),
+    "run",
+    "-A",
+    "cli/js/tests/unit_test_runner.ts",
+    "--",
+    "--worker",
+    `--addr=${addr}`,
+    `--perms=${permStr}`
+  ];
+
+  const p = Deno.run({
+    args,
+    stdin: "null",
+    stdout: "piped",
+    stderr: "null"
+  });
+
+  return p;
 }
 
-function fmtPerms(perms: Permissions): string {
-  let fmt = permsToCliFlags(perms).join(" ");
+async function runTestsForPermissionSet(
+  reporter: Deno.ConsoleTestReporter,
+  perms: Permissions
+): Promise<PermissionSetTestResult> {
+  const permsFmt = fmtPerms(perms);
+  console.log(`Running tests for: ${permsFmt}`);
+  const addr = { hostname: "127.0.0.1", port: 4510 };
+  const addrStr = `${addr.hostname}:${addr.port}`;
+  const workerListener = Deno.listen(addr);
+
+  const workerProcess = spawnWorkerRunner(addrStr, perms);
+
+  // Wait for worker subprocess to go online
+  const conn = await workerListener.accept();
+
+  let err;
+  let hasThrown = false;
+  let expectedPassedTests;
+  let endEvent;
+
+  try {
+    for await (const line of readLines(conn)) {
+      const msg = JSON.parse(line);
+
+      if (msg.kind === Deno.TestEvent.Start) {
+        expectedPassedTests = msg.tests;
+        await reporter.start(msg);
+        continue;
+      }
+
+      if (msg.kind === Deno.TestEvent.Result) {
+        await reporter.result(msg);
+        continue;
+      }
+
+      endEvent = msg;
+      await reporter.end(msg);
+      break;
+    }
+  } catch (e) {
+    hasThrown = true;
+    err = e;
+  } finally {
+    workerListener.close();
+  }
+
+  if (hasThrown) {
+    throw err;
+  }
 
-  if (!fmt) {
-    fmt = "<no permissions>";
+  if (typeof expectedPassedTests === "undefined") {
+    throw new Error("Worker runner didn't report start");
   }
 
-  return fmt;
+  if (typeof endEvent === "undefined") {
+    throw new Error("Worker runner didn't report end");
+  }
+
+  const workerStatus = await workerProcess.status();
+  if (!workerStatus.success) {
+    throw new Error(
+      `Worker runner exited with status code: ${workerStatus.code}`
+    );
+  }
+
+  workerProcess.close();
+
+  const passed = expectedPassedTests === endEvent.stats.passed;
+
+  return {
+    perms,
+    passed,
+    permsStr: permsFmt,
+    duration: endEvent.duration,
+    stats: endEvent.stats
+  };
 }
 
-async function main(): Promise<void> {
+async function masterRunnerMain(): Promise<void> {
   console.log(
     "Discovered permission combinations for tests:",
     permissionCombinations.size
@@ -47,57 +192,31 @@ async function main(): Promise<void> {
     console.log("\t" + fmtPerms(perms));
   }
 
-  const testResults = new Set<TestResult>();
+  const testResults = new Set<PermissionSetTestResult>();
+  const consoleReporter = new Deno.ConsoleTestReporter();
 
   for (const perms of permissionCombinations.values()) {
-    const permsFmt = fmtPerms(perms);
-    console.log(`Running tests for: ${permsFmt}`);
-    const cliPerms = permsToCliFlags(perms);
-    // run subsequent tests using same deno executable
-    const args = [
-      Deno.execPath(),
-      "run",
-      ...cliPerms,
-      "cli/js/tests/unit_tests.ts"
-    ];
-
-    const p = Deno.run({
-      args,
-      stdout: "piped"
-    });
-
-    const { actual, expected, resultOutput } = await parseUnitTestOutput(
-      p.stdout!,
-      true
-    );
-
-    let result = 0;
-
-    if (!actual && !expected) {
-      console.error("Bad cli/js/tests/unit_test.ts output");
-      result = 1;
-    } else if (expected !== actual) {
-      result = 1;
-    }
-
-    testResults.add({
-      perms: permsFmt,
-      output: resultOutput,
-      result
-    });
+    const result = await runTestsForPermissionSet(consoleReporter, perms);
+    testResults.add(result);
   }
 
   // if any run tests returned non-zero status then whole test
   // run should fail
-  let testsFailed = false;
+  let testsPassed = true;
 
   for (const testResult of testResults) {
-    console.log(`Summary for ${testResult.perms}`);
-    console.log(testResult.output + "\n");
-    testsFailed = testsFailed || Boolean(testResult.result);
+    const { permsStr, stats, duration } = testResult;
+    console.log(`Summary for ${permsStr}`);
+    await consoleReporter.end({
+      kind: Deno.TestEvent.End,
+      stats,
+      duration,
+      results: []
+    });
+    testsPassed = testsPassed && testResult.passed;
   }
 
-  if (testsFailed) {
+  if (!testsPassed) {
     console.error("Unit tests failed");
     Deno.exit(1);
   }
@@ -105,4 +224,16 @@ async function main(): Promise<void> {
   console.log("Unit tests passed");
 }
 
+async function main(): Promise<void> {
+  const args = Deno.args;
+
+  const isWorker = args.includes("--worker");
+
+  if (isWorker) {
+    return await workerRunnerMain(args);
+  }
+
+  return await masterRunnerMain();
+}
+
 main();
diff --git a/cli/js/tests/unit_tests.ts b/cli/js/tests/unit_tests.ts
index 9c80859d6..4cff3d1d8 100644
--- a/cli/js/tests/unit_tests.ts
+++ b/cli/js/tests/unit_tests.ts
@@ -1,7 +1,8 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
 
-// This test is executed as part of tools/test.py
-// But it can also be run manually: ./target/debug/deno cli/js/tests/unit_tests.ts
+// This test is executed as part of unit test suite.
+//
+// Test runner automatically spawns subprocesses for each required permissions combination.
 
 import "./blob_test.ts";
 import "./body_test.ts";
@@ -63,7 +64,3 @@ import "./utime_test.ts";
 import "./write_file_test.ts";
 import "./performance_test.ts";
 import "./version_test.ts";
-
-if (import.meta.main) {
-  await Deno.runTests();
-}
diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs
index ce08c6b61..f90e434da 100644
--- a/cli/tests/integration_tests.rs
+++ b/cli/tests/integration_tests.rs
@@ -272,8 +272,7 @@ fn js_unit_tests() {
     .current_dir(util::root_path())
     .arg("run")
     .arg("--reload")
-    .arg("--allow-run")
-    .arg("--allow-env")
+    .arg("-A")
     .arg("cli/js/tests/unit_test_runner.ts")
     .spawn()
     .expect("failed to spawn script");