5 files changed, 299 insertions, 179 deletions

diff --git a/cli/js/tests/location_test.ts b/cli/js/tests/location_test.ts
index 78ecb55b3..2d2faf0c2 100644
--- a/cli/js/tests/location_test.ts
+++ b/cli/js/tests/location_test.ts
@@ -3,5 +3,5 @@ import { unitTest, assert } from "./test_util.ts";
 
 unitTest(function locationBasic(): void {
   // location example: file:///Users/rld/src/deno/js/unit_tests.ts
-  assert(window.location.toString().endsWith("unit_tests.ts"));
+  assert(window.location.toString().endsWith("unit_test_runner.ts"));
 });
diff --git a/cli/js/tests/resources_test.ts b/cli/js/tests/resources_test.ts
index 84b713a6d..680fac8b7 100644
--- a/cli/js/tests/resources_test.ts
+++ b/cli/js/tests/resources_test.ts
@@ -1,5 +1,5 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
-import { unitTest, assertEquals } from "./test_util.ts";
+import { unitTest, assertEquals, assert } from "./test_util.ts";
 
 unitTest(function resourcesStdio(): void {
   const res = Deno.resources();
@@ -21,10 +21,10 @@ unitTest({ perms: { net: true } }, async function resourcesNet(): Promise<
     Object.values(res).filter((r): boolean => r === "tcpListener").length,
     1
   );
-  assertEquals(
-    Object.values(res).filter((r): boolean => r === "tcpStream").length,
-    2
+  const tcpStreams = Object.values(res).filter(
+    (r): boolean => r === "tcpStream"
   );
+  assert(tcpStreams.length >= 2);
 
   listenerConn.close();
   dialerConn.close();
diff --git a/cli/js/tests/test_util.ts b/cli/js/tests/test_util.ts
index c8f28437d..66edd6681 100644
--- a/cli/js/tests/test_util.ts
+++ b/cli/js/tests/test_util.ts
@@ -1,13 +1,5 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
-//
-// We want to test many ops in deno which have different behavior depending on
-// the permissions set. These tests can specify which permissions they expect,
-// which appends a special string like "permW1N0" to the end of the test name.
-// Here we run several copies of deno with different permissions, filtering the
-// tests by the special string. permW1N0 means allow-write but not allow-net.
-// See tools/unit_tests.py for more details.
-
-import { readLines } from "../../../std/io/bufio.ts";
+
 import { assert, assertEquals } from "../../../std/testing/asserts.ts";
 export {
   assert,
@@ -20,16 +12,7 @@ export {
   unreachable,
   fail
 } from "../../../std/testing/asserts.ts";
-
-interface TestPermissions {
-  read?: boolean;
-  write?: boolean;
-  net?: boolean;
-  env?: boolean;
-  run?: boolean;
-  plugin?: boolean;
-  hrtime?: boolean;
-}
+export { readLines } from "../../../std/io/bufio.ts";
 
 export interface Permissions {
   read: boolean;
@@ -41,10 +24,22 @@ export interface Permissions {
   hrtime: boolean;
 }
 
+export function fmtPerms(perms: Permissions): string {
+  const p = Object.keys(perms)
+    .filter((e): boolean => perms[e as keyof Permissions] === true)
+    .map(key => `--allow-${key}`);
+
+  if (p.length) {
+    return p.join(" ");
+  }
+
+  return "<no permissions>";
+}
+
 const isGranted = async (name: Deno.PermissionName): Promise<boolean> =>
   (await Deno.permissions.query({ name })).state === "granted";
 
-async function getProcessPermissions(): Promise<Permissions> {
+export async function getProcessPermissions(): Promise<Permissions> {
   return {
     run: await isGranted("run"),
     read: await isGranted("read"),
@@ -56,9 +51,7 @@ async function getProcessPermissions(): Promise<Permissions> {
   };
 }
 
-const processPerms = await getProcessPermissions();
-
-function permissionsMatch(
+export function permissionsMatch(
   processPerms: Permissions,
   requiredPerms: Permissions
 ): boolean {
@@ -94,7 +87,23 @@ function registerPermCombination(perms: Permissions): void {
   }
 }
 
-function normalizeTestPermissions(perms: TestPermissions): Permissions {
+export async function registerUnitTests(): Promise<void> {
+  const processPerms = await getProcessPermissions();
+
+  for (const unitTestDefinition of REGISTERED_UNIT_TESTS) {
+    if (unitTestDefinition.skip) {
+      continue;
+    }
+
+    if (!permissionsMatch(processPerms, unitTestDefinition.perms)) {
+      continue;
+    }
+
+    Deno.test(unitTestDefinition);
+  }
+}
+
+function normalizeTestPermissions(perms: UnitTestPermissions): Permissions {
   return {
     read: !!perms.read,
     write: !!perms.write,
@@ -147,11 +156,30 @@ function assertResources(fn: Deno.TestFunction): Deno.TestFunction {
   };
 }
 
+interface UnitTestPermissions {
+  read?: boolean;
+  write?: boolean;
+  net?: boolean;
+  env?: boolean;
+  run?: boolean;
+  plugin?: boolean;
+  hrtime?: boolean;
+}
+
 interface UnitTestOptions {
   skip?: boolean;
-  perms?: TestPermissions;
+  perms?: UnitTestPermissions;
 }
 
+interface UnitTestDefinition {
+  name: string;
+  fn: Deno.TestFunction;
+  skip?: boolean;
+  perms: Permissions;
+}
+
+export const REGISTERED_UNIT_TESTS: UnitTestDefinition[] = [];
+
 export function unitTest(fn: Deno.TestFunction): void;
 export function unitTest(options: UnitTestOptions, fn: Deno.TestFunction): void;
 export function unitTest(
@@ -187,53 +215,15 @@ export function unitTest(
 
   const normalizedPerms = normalizeTestPermissions(options.perms || {});
   registerPermCombination(normalizedPerms);
-  if (!permissionsMatch(processPerms, normalizedPerms)) {
-    return;
-  }
 
-  const testDefinition: Deno.TestDefinition = {
+  const unitTestDefinition: UnitTestDefinition = {
     name,
-    fn: assertResources(assertOps(fn))
+    fn: assertResources(assertOps(fn)),
+    skip: !!options.skip,
+    perms: normalizedPerms
   };
-  Deno.test(testDefinition);
-}
 
-function extractNumber(re: RegExp, str: string): number | undefined {
-  const match = str.match(re);
-
-  if (match) {
-    return Number.parseInt(match[1]);
-  }
-}
-
-export async function parseUnitTestOutput(
-  reader: Deno.Reader,
-  print: boolean
-): Promise<{ actual?: number; expected?: number; resultOutput?: string }> {
-  let expected, actual, result;
-
-  for await (const line of readLines(reader)) {
-    if (!expected) {
-      // expect "running 30 tests"
-      expected = extractNumber(/running (\d+) tests/, line);
-    } else if (line.indexOf("test result:") !== -1) {
-      result = line;
-    }
-
-    if (print) {
-      console.log(line);
-    }
-  }
-
-  // Check that the number of expected tests equals what was reported at the
-  // bottom.
-  if (result) {
-    // result should be a string like this:
-    // "test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; ..."
-    actual = extractNumber(/(\d+) passed/, result);
-  }
-
-  return { actual, expected, resultOutput: result };
+  REGISTERED_UNIT_TESTS.push(unitTestDefinition);
 }
 
 export interface ResolvableMethods<T> {
@@ -254,6 +244,45 @@ export function createResolvable<T>(): Resolvable<T> {
   return Object.assign(promise, methods!) as Resolvable<T>;
 }
 
+export class SocketReporter implements Deno.TestReporter {
+  private encoder: TextEncoder;
+
+  constructor(private conn: Deno.Conn) {
+    this.encoder = new TextEncoder();
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  async write(msg: any): Promise<void> {
+    const encodedMsg = this.encoder.encode(`${JSON.stringify(msg)}\n`);
+    await Deno.writeAll(this.conn, encodedMsg);
+  }
+
+  async start(msg: Deno.TestEventStart): Promise<void> {
+    await this.write(msg);
+  }
+
+  async result(msg: Deno.TestEventResult): Promise<void> {
+    // eslint-disable-next-line @typescript-eslint/no-explicit-any
+    const serializedMsg: any = { ...msg };
+
+    // Error is a JS object, so we need to turn it into string to
+    // send over socket.
+    if (serializedMsg.result.error) {
+      serializedMsg.result.error = String(serializedMsg.result.error.stack);
+    }
+
+    await this.write(serializedMsg);
+  }
+
+  async end(msg: Deno.TestEventEnd): Promise<void> {
+    await this.write(msg);
+  }
+
+  close(): void {
+    this.conn.close();
+  }
+}
+
 unitTest(function permissionsMatches(): void {
   assert(
     permissionsMatch(
@@ -341,43 +370,6 @@ unitTest(function permissionsMatches(): void {
   );
 });
 
-unitTest(
-  { perms: { read: true } },
-  async function parsingUnitTestOutput(): Promise<void> {
-    const cwd = Deno.cwd();
-    const testDataPath = `${cwd}/tools/testdata/`;
-
-    let result;
-
-    // This is an example of a successful unit test output.
-    const f1 = await Deno.open(`${testDataPath}/unit_test_output1.txt`);
-    result = await parseUnitTestOutput(f1, false);
-    assertEquals(result.actual, 96);
-    assertEquals(result.expected, 96);
-    f1.close();
-
-    // This is an example of a silently dying unit test.
-    const f2 = await Deno.open(`${testDataPath}/unit_test_output2.txt`);
-    result = await parseUnitTestOutput(f2, false);
-    assertEquals(result.actual, undefined);
-    assertEquals(result.expected, 96);
-    f2.close();
-
-    // This is an example of compiling before successful unit tests.
-    const f3 = await Deno.open(`${testDataPath}/unit_test_output3.txt`);
-    result = await parseUnitTestOutput(f3, false);
-    assertEquals(result.actual, 96);
-    assertEquals(result.expected, 96);
-    f3.close();
-
-    // Check what happens on empty output.
-    const f = new Deno.Buffer(new TextEncoder().encode("\n\n\n"));
-    result = await parseUnitTestOutput(f, false);
-    assertEquals(result.actual, undefined);
-    assertEquals(result.expected, undefined);
-  }
-);
-
 /*
  * Ensure all unit test files (e.g. xxx_test.ts) are present as imports in
  * cli/js/tests/unit_tests.ts as it is easy to miss this out
diff --git a/cli/js/tests/unit_test_runner.ts b/cli/js/tests/unit_test_runner.ts
index a5b7c3a48..f018fb59e 100755
--- a/cli/js/tests/unit_test_runner.ts
+++ b/cli/js/tests/unit_test_runner.ts
@@ -2,42 +2,187 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
 import "./unit_tests.ts";
 import {
+  assert,
+  readLines,
   permissionCombinations,
-  parseUnitTestOutput,
-  Permissions
+  Permissions,
+  registerUnitTests,
+  SocketReporter,
+  fmtPerms
 } from "./test_util.ts";
 
-interface TestResult {
-  perms: string;
-  output?: string;
-  result: number;
+interface PermissionSetTestResult {
+  perms: Permissions;
+  passed: boolean;
+  stats: Deno.TestStats;
+  permsStr: string;
+  duration: number;
 }
 
-function permsToCliFlags(perms: Permissions): string[] {
-  return Object.keys(perms)
-    .map(key => {
-      if (!perms[key as keyof Permissions]) return "";
+const PERMISSIONS: Deno.PermissionName[] = [
+  "read",
+  "write",
+  "net",
+  "env",
+  "run",
+  "plugin",
+  "hrtime"
+];
+
+/**
+ * Take a list of permissions and revoke missing permissions.
+ */
+async function dropWorkerPermissions(
+  requiredPermissions: Deno.PermissionName[]
+): Promise<void> {
+  const permsToDrop = PERMISSIONS.filter((p): boolean => {
+    return !requiredPermissions.includes(p);
+  });
+
+  for (const perm of permsToDrop) {
+    await Deno.permissions.revoke({ name: perm });
+  }
+}
+
+async function workerRunnerMain(args: string[]): Promise<void> {
+  const addrArg = args.find(e => e.includes("--addr"));
+  assert(typeof addrArg === "string", "Missing --addr argument");
+  const addrStr = addrArg.split("=")[1];
+  const [hostname, port] = addrStr.split(":");
+  const addr = { hostname, port: Number(port) };
+
+  let perms: Deno.PermissionName[] = [];
+  const permsArg = args.find(e => e.includes("--perms"));
+  assert(typeof permsArg === "string", "Missing --perms argument");
+  const permsStr = permsArg.split("=")[1];
+  if (permsStr.length > 0) {
+    perms = permsStr.split(",") as Deno.PermissionName[];
+  }
+  // Setup reporter
+  const conn = await Deno.connect(addr);
+  const socketReporter = new SocketReporter(conn);
+  // Drop current process permissions to requested set
+  await dropWorkerPermissions(perms);
+  // Register unit tests that match process permissions
+  await registerUnitTests();
+  // Execute tests
+  await Deno.runTests({
+    failFast: false,
+    exitOnFail: false,
+    reporter: socketReporter
+  });
+  // Notify parent process we're done
+  socketReporter.close();
+}
 
-      const cliFlag = key.replace(
-        /\.?([A-Z])/g,
-        (x, y): string => `-${y.toLowerCase()}`
-      );
-      return `--allow-${cliFlag}`;
+function spawnWorkerRunner(addr: string, perms: Permissions): Deno.Process {
+  // run subsequent tests using same deno executable
+  const permStr = Object.keys(perms)
+    .filter((permName): boolean => {
+      return perms[permName as Deno.PermissionName] === true;
     })
-    .filter((e): boolean => e.length > 0);
+    .join(",");
+
+  const args = [
+    Deno.execPath(),
+    "run",
+    "-A",
+    "cli/js/tests/unit_test_runner.ts",
+    "--",
+    "--worker",
+    `--addr=${addr}`,
+    `--perms=${permStr}`
+  ];
+
+  const p = Deno.run({
+    args,
+    stdin: "null",
+    stdout: "piped",
+    stderr: "null"
+  });
+
+  return p;
 }
 
-function fmtPerms(perms: Permissions): string {
-  let fmt = permsToCliFlags(perms).join(" ");
+async function runTestsForPermissionSet(
+  reporter: Deno.ConsoleTestReporter,
+  perms: Permissions
+): Promise<PermissionSetTestResult> {
+  const permsFmt = fmtPerms(perms);
+  console.log(`Running tests for: ${permsFmt}`);
+  const addr = { hostname: "127.0.0.1", port: 4510 };
+  const addrStr = `${addr.hostname}:${addr.port}`;
+  const workerListener = Deno.listen(addr);
+
+  const workerProcess = spawnWorkerRunner(addrStr, perms);
+
+  // Wait for worker subprocess to go online
+  const conn = await workerListener.accept();
+
+  let err;
+  let hasThrown = false;
+  let expectedPassedTests;
+  let endEvent;
+
+  try {
+    for await (const line of readLines(conn)) {
+      const msg = JSON.parse(line);
+
+      if (msg.kind === Deno.TestEvent.Start) {
+        expectedPassedTests = msg.tests;
+        await reporter.start(msg);
+        continue;
+      }
+
+      if (msg.kind === Deno.TestEvent.Result) {
+        await reporter.result(msg);
+        continue;
+      }
+
+      endEvent = msg;
+      await reporter.end(msg);
+      break;
+    }
+  } catch (e) {
+    hasThrown = true;
+    err = e;
+  } finally {
+    workerListener.close();
+  }
+
+  if (hasThrown) {
+    throw err;
+  }
 
-  if (!fmt) {
-    fmt = "<no permissions>";
+  if (typeof expectedPassedTests === "undefined") {
+    throw new Error("Worker runner didn't report start");
   }
 
-  return fmt;
+  if (typeof endEvent === "undefined") {
+    throw new Error("Worker runner didn't report end");
+  }
+
+  const workerStatus = await workerProcess.status();
+  if (!workerStatus.success) {
+    throw new Error(
+      `Worker runner exited with status code: ${workerStatus.code}`
+    );
+  }
+
+  workerProcess.close();
+
+  const passed = expectedPassedTests === endEvent.stats.passed;
+
+  return {
+    perms,
+    passed,
+    permsStr: permsFmt,
+    duration: endEvent.duration,
+    stats: endEvent.stats
+  };
 }
 
-async function main(): Promise<void> {
+async function masterRunnerMain(): Promise<void> {
   console.log(
     "Discovered permission combinations for tests:",
     permissionCombinations.size
@@ -47,57 +192,31 @@ async function main(): Promise<void> {
     console.log("\t" + fmtPerms(perms));
   }
 
-  const testResults = new Set<TestResult>();
+  const testResults = new Set<PermissionSetTestResult>();
+  const consoleReporter = new Deno.ConsoleTestReporter();
 
   for (const perms of permissionCombinations.values()) {
-    const permsFmt = fmtPerms(perms);
-    console.log(`Running tests for: ${permsFmt}`);
-    const cliPerms = permsToCliFlags(perms);
-    // run subsequent tests using same deno executable
-    const args = [
-      Deno.execPath(),
-      "run",
-      ...cliPerms,
-      "cli/js/tests/unit_tests.ts"
-    ];
-
-    const p = Deno.run({
-      args,
-      stdout: "piped"
-    });
-
-    const { actual, expected, resultOutput } = await parseUnitTestOutput(
-      p.stdout!,
-      true
-    );
-
-    let result = 0;
-
-    if (!actual && !expected) {
-      console.error("Bad cli/js/tests/unit_test.ts output");
-      result = 1;
-    } else if (expected !== actual) {
-      result = 1;
-    }
-
-    testResults.add({
-      perms: permsFmt,
-      output: resultOutput,
-      result
-    });
+    const result = await runTestsForPermissionSet(consoleReporter, perms);
+    testResults.add(result);
   }
 
   // if any run tests returned non-zero status then whole test
   // run should fail
-  let testsFailed = false;
+  let testsPassed = true;
 
   for (const testResult of testResults) {
-    console.log(`Summary for ${testResult.perms}`);
-    console.log(testResult.output + "\n");
-    testsFailed = testsFailed || Boolean(testResult.result);
+    const { permsStr, stats, duration } = testResult;
+    console.log(`Summary for ${permsStr}`);
+    await consoleReporter.end({
+      kind: Deno.TestEvent.End,
+      stats,
+      duration,
+      results: []
+    });
+    testsPassed = testsPassed && testResult.passed;
   }
 
-  if (testsFailed) {
+  if (!testsPassed) {
     console.error("Unit tests failed");
     Deno.exit(1);
   }
@@ -105,4 +224,16 @@ async function main(): Promise<void> {
   console.log("Unit tests passed");
 }
 
+async function main(): Promise<void> {
+  const args = Deno.args;
+
+  const isWorker = args.includes("--worker");
+
+  if (isWorker) {
+    return await workerRunnerMain(args);
+  }
+
+  return await masterRunnerMain();
+}
+
 main();
diff --git a/cli/js/tests/unit_tests.ts b/cli/js/tests/unit_tests.ts
index 9c80859d6..4cff3d1d8 100644
--- a/cli/js/tests/unit_tests.ts
+++ b/cli/js/tests/unit_tests.ts
@@ -1,7 +1,8 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
 
-// This test is executed as part of tools/test.py
-// But it can also be run manually: ./target/debug/deno cli/js/tests/unit_tests.ts
+// This test is executed as part of unit test suite.
+//
+// Test runner automatically spawns subprocesses for each required permissions combination.
 
 import "./blob_test.ts";
 import "./body_test.ts";
@@ -63,7 +64,3 @@ import "./utime_test.ts";
 import "./write_file_test.ts";
 import "./performance_test.ts";
 import "./version_test.ts";
-
-if (import.meta.main) {
-  await Deno.runTests();
-}