feat(unstable/test): imperative test steps API (#12190)

13 files changed, 588 insertions, 2 deletions

diff --git a/cli/tests/integration/test_tests.rs b/cli/tests/integration/test_tests.rs
index 24ceeefb4..3ea8186b8 100644
--- a/cli/tests/integration/test_tests.rs
+++ b/cli/tests/integration/test_tests.rs
@@ -186,3 +186,39 @@ itest!(aggregate_error {
   exit_code: 1,
   output: "test/aggregate_error.out",
 });
+
+itest!(steps_passing_steps {
+  args: "test --unstable test/steps/passing_steps.ts",
+  exit_code: 0,
+  output: "test/steps/passing_steps.out",
+});
+
+itest!(steps_passing_steps_concurrent {
+  args: "test --unstable --jobs=2 test/steps/passing_steps.ts",
+  exit_code: 0,
+  output: "test/steps/passing_steps.out",
+});
+
+itest!(steps_failing_steps {
+  args: "test --unstable test/steps/failing_steps.ts",
+  exit_code: 1,
+  output: "test/steps/failing_steps.out",
+});
+
+itest!(steps_ignored_steps {
+  args: "test --unstable test/steps/ignored_steps.ts",
+  exit_code: 0,
+  output: "test/steps/ignored_steps.out",
+});
+
+itest!(steps_invalid_usage {
+  args: "test --unstable test/steps/invalid_usage.ts",
+  exit_code: 1,
+  output: "test/steps/invalid_usage.out",
+});
+
+itest!(steps_no_unstable_flag {
+  args: "test test/steps/no_unstable_flag.ts",
+  exit_code: 1,
+  output: "test/steps/no_unstable_flag.out",
+});
diff --git a/cli/tests/testdata/test/steps/failing_steps.out b/cli/tests/testdata/test/steps/failing_steps.out
new file mode 100644
index 000000000..1c5e2e591
--- /dev/null
+++ b/cli/tests/testdata/test/steps/failing_steps.out
@@ -0,0 +1,53 @@
+[WILDCARD]
+running 3 tests from [WILDCARD]/failing_steps.ts
+test nested failure ...
+  test step 1 ...
+    test inner 1 ... FAILED ([WILDCARD])
+      Error: Failed.
+          at [WILDCARD]/failing_steps.ts:[WILDCARD]
+          [WILDCARD]
+    test inner 2 ... ok ([WILDCARD])
+  FAILED ([WILDCARD])
+FAILED ([WILDCARD])
+test multiple test step failures ...
+  test step 1 ... FAILED ([WILDCARD])
+    Error: Fail.
+        [WILDCARD]
+  test step 2 ... FAILED ([WILDCARD])
+    Error: Fail.
+        at [WILDCARD]/failing_steps.ts:[WILDCARD]
+        [WILDCARD]
+FAILED ([WILDCARD])
+test failing step in failing test ...
+  test step 1 ... FAILED ([WILDCARD])
+    Error: Fail.
+        at [WILDCARD]/failing_steps.ts:[WILDCARD]
+        at [WILDCARD]
+FAILED ([WILDCARD])
+
+failures:
+
+nested failure
+Error: 1 test step failed.
+    at runTest (deno:runtime/js/40_testing.js:[WILDCARD])
+    at async Object.runTests (deno:runtime/js/40_testing.js:[WILDCARD])
+
+multiple test step failures
+Error: 2 test steps failed.
+    at runTest (deno:runtime/js/40_testing.js:[WILDCARD])
+    at async Object.runTests (deno:runtime/js/40_testing.js:[WILDCARD])
+
+failing step in failing test
+Error: Fail test.
+    at [WILDCARD]/failing_steps.ts:[WILDCARD]
+    at [WILDCARD]
+
+failures:
+
+	nested failure
+	multiple test step failures
+	failing step in failing test
+
+test result: FAILED. 0 passed; 3 failed; 0 ignored; 0 measured; 0 filtered out ([WILDCARD])
+
+error: Test failed
diff --git a/cli/tests/testdata/test/steps/failing_steps.ts b/cli/tests/testdata/test/steps/failing_steps.ts
new file mode 100644
index 000000000..efa18d54e
--- /dev/null
+++ b/cli/tests/testdata/test/steps/failing_steps.ts
@@ -0,0 +1,27 @@
+Deno.test("nested failure", async (t) => {
+  const success = await t.step("step 1", async (t) => {
+    let success = await t.step("inner 1", () => {
+      throw new Error("Failed.");
+    });
+    if (success) throw new Error("Expected failure");
+
+    success = await t.step("inner 2", () => {});
+    if (!success) throw new Error("Expected success");
+  });
+
+  if (success) throw new Error("Expected failure");
+});
+
+Deno.test("multiple test step failures", async (t) => {
+  await t.step("step 1", () => {
+    throw new Error("Fail.");
+  });
+  await t.step("step 2", () => Promise.reject(new Error("Fail.")));
+});
+
+Deno.test("failing step in failing test", async (t) => {
+  await t.step("step 1", () => {
+    throw new Error("Fail.");
+  });
+  throw new Error("Fail test.");
+});
diff --git a/cli/tests/testdata/test/steps/ignored_steps.out b/cli/tests/testdata/test/steps/ignored_steps.out
new file mode 100644
index 000000000..c667a3d95
--- /dev/null
+++ b/cli/tests/testdata/test/steps/ignored_steps.out
@@ -0,0 +1,8 @@
+[WILDCARD]
+running 1 test from [WILDCARD]/ignored_steps.ts
+test ignored step ...
+  test step 1 ... ignored ([WILDCARD])
+  test step 2 ... ok ([WILDCARD])
+ok ([WILDCARD])
+
+test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out [WILDCARD]
diff --git a/cli/tests/testdata/test/steps/ignored_steps.ts b/cli/tests/testdata/test/steps/ignored_steps.ts
new file mode 100644
index 000000000..102b481fb
--- /dev/null
+++ b/cli/tests/testdata/test/steps/ignored_steps.ts
@@ -0,0 +1,16 @@
+Deno.test("ignored step", async (t) => {
+  let result = await t.step({
+    name: "step 1",
+    ignore: true,
+    fn: () => {
+      throw new Error("Fail.");
+    },
+  });
+  if (result !== false) throw new Error("Expected false.");
+  result = await t.step({
+    name: "step 2",
+    ignore: false,
+    fn: () => {},
+  });
+  if (result !== true) throw new Error("Expected true.");
+});
diff --git a/cli/tests/testdata/test/steps/invalid_usage.out b/cli/tests/testdata/test/steps/invalid_usage.out
new file mode 100644
index 000000000..b03ca57b6
--- /dev/null
+++ b/cli/tests/testdata/test/steps/invalid_usage.out
@@ -0,0 +1,111 @@
+[WILDCARD]
+running 7 tests from [WILDCARD]/invalid_usage.ts
+test capturing ...
+  test some step ... ok ([WILDCARD])
+FAILED ([WILDCARD])
+test top level missing await ...
+  test step ... pending ([WILDCARD])
+FAILED ([WILDCARD])
+test inner missing await ...
+  test step ...
+    test inner ... pending ([WILDCARD])
+      Error: Parent scope completed before test step finished execution. Ensure all steps are awaited (ex. `await t.step(...)`).
+          at postValidation [WILDCARD]
+          at testStepSanitizer [WILDCARD]
+  FAILED ([WILDCARD])
+    Error: There were still test steps running after the current scope finished execution. Ensure all steps are awaited (ex. `await t.step(...)`).
+        at postValidation [WILDCARD]
+        at testStepSanitizer [WILDCARD]
+        at async fn ([WILDCARD]/invalid_usage.ts:[WILDCARD])
+        at async Object.testStepSanitizer [WILDCARD]
+FAILED ([WILDCARD])
+test parallel steps with sanitizers ...
+  test step 1 ... pending ([WILDCARD])
+  test step 2 ... FAILED ([WILDCARD])
+    Error: Cannot start test step while another test step with sanitizers is running.
+     * parallel steps with sanitizers > step 1
+        at preValidation ([WILDCARD])
+        at testStepSanitizer ([WILDCARD])
+        at [WILDCARD]/invalid_usage.ts:[WILDCARD]
+        at [WILDCARD]
+FAILED ([WILDCARD])
+test parallel steps when first has sanitizer ...
+  test step 1 ... pending ([WILDCARD])
+  test step 2 ... FAILED ([WILDCARD])
+    Error: Cannot start test step while another test step with sanitizers is running.
+     * parallel steps when first has sanitizer > step 1
+        at preValidation ([WILDCARD])
+        at testStepSanitizer ([WILDCARD])
+        at [WILDCARD]/invalid_usage.ts:[WILDCARD]
+        at [WILDCARD]
+FAILED ([WILDCARD])
+test parallel steps when second has sanitizer ...
+  test step 1 ... ok ([WILDCARD])
+  test step 2 ... FAILED ([WILDCARD])
+    Error: Cannot start test step with sanitizers while another test step is running.
+     * parallel steps when second has sanitizer > step 1
+        at preValidation ([WILDCARD])
+        at testStepSanitizer ([WILDCARD])
+        at [WILDCARD]/invalid_usage.ts:[WILDCARD]
+        at [WILDCARD]
+FAILED ([WILDCARD])
+test parallel steps where only inner tests have sanitizers ...
+  test step 1 ...
+    test step inner ... ok ([WILDCARD])
+  ok ([WILDCARD])
+  test step 2 ...
+    test step inner ... FAILED ([WILDCARD])
+      Error: Cannot start test step with sanitizers while another test step is running.
+       * parallel steps where only inner tests have sanitizers > step 1
+          at preValidation ([WILDCARD])
+          at testStepSanitizer ([WILDCARD])
+          at [WILDCARD]/invalid_usage.ts:[WILDCARD]
+  FAILED ([WILDCARD])
+FAILED ([WILDCARD])
+
+failures:
+
+capturing
+Error: Cannot run test step after parent scope has finished execution. Ensure any `.step(...)` calls are executed before their parent scope completes execution.
+    at TestContext.step ([WILDCARD])
+    at [WILDCARD]/invalid_usage.ts:[WILDCARD]
+    at [WILDCARD]
+
+top level missing await
+Error: There were still test steps running after the current scope finished execution. Ensure all steps are awaited (ex. `await t.step(...)`).
+    at postValidation [WILDCARD]
+    at testStepSanitizer ([WILDCARD])
+    [WILDCARD]
+
+inner missing await
+Error: 1 test step failed.
+    at [WILDCARD]
+
+parallel steps with sanitizers
+Error: 1 test step failed.
+    at runTest ([WILDCARD])
+    at [WILDCARD]
+
+parallel steps when first has sanitizer
+Error: 1 test step failed.
+    at runTest ([WILDCARD])
+    at [WILDCARD]
+
+parallel steps when second has sanitizer
+Error: 1 test step failed.
+    at runTest ([WILDCARD])
+    at [WILDCARD]
+
+failures:
+
+	capturing
+	top level missing await
+	inner missing await
+	parallel steps with sanitizers
+	parallel steps when first has sanitizer
+	parallel steps when second has sanitizer
+	parallel steps where only inner tests have sanitizers
+
+test result: FAILED. 0 passed; 7 failed; 0 ignored; 0 measured; 0 filtered out ([WILDCARD])
+
+error: Test failed
diff --git a/cli/tests/testdata/test/steps/invalid_usage.ts b/cli/tests/testdata/test/steps/invalid_usage.ts
new file mode 100644
index 000000000..f670c842e
--- /dev/null
+++ b/cli/tests/testdata/test/steps/invalid_usage.ts
@@ -0,0 +1,122 @@
+import { deferred } from "../../../../../test_util/std/async/deferred.ts";
+
+Deno.test("capturing", async (t) => {
+  let capturedContext!: Deno.TestContext;
+  await t.step("some step", (t) => {
+    capturedContext = t;
+  });
+  // this should error because the scope of the tester has already completed
+  await capturedContext.step("next step", () => {});
+});
+
+Deno.test("top level missing await", (t) => {
+  t.step("step", () => {
+    return new Promise((resolve) => setTimeout(resolve, 10));
+  });
+});
+
+Deno.test({
+  name: "inner missing await",
+  fn: async (t) => {
+    await t.step("step", (t) => {
+      t.step("inner", () => {
+        return new Promise((resolve) => setTimeout(resolve, 10));
+      });
+    });
+    await new Promise((resolve) => setTimeout(resolve, 10));
+  },
+  sanitizeResources: false,
+  sanitizeOps: false,
+  sanitizeExit: false,
+});
+
+Deno.test("parallel steps with sanitizers", async (t) => {
+  // not allowed because steps with sanitizers cannot be run in parallel
+  const step1Entered = deferred();
+  const step2Finished = deferred();
+  const step1 = t.step("step 1", async () => {
+    step1Entered.resolve();
+    await step2Finished;
+  });
+  await step1Entered;
+  await t.step("step 2", () => {});
+  step2Finished.resolve();
+  await step1;
+});
+
+Deno.test("parallel steps when first has sanitizer", async (t) => {
+  const step1Entered = deferred();
+  const step2Finished = deferred();
+  const step1 = t.step({
+    name: "step 1",
+    fn: async () => {
+      step1Entered.resolve();
+      await step2Finished;
+    },
+  });
+  await step1Entered;
+  await t.step({
+    name: "step 2",
+    fn: () => {},
+    sanitizeOps: false,
+    sanitizeResources: false,
+    sanitizeExit: false,
+  });
+  step2Finished.resolve();
+  await step1;
+});
+
+Deno.test("parallel steps when second has sanitizer", async (t) => {
+  const step1Entered = deferred();
+  const step2Finished = deferred();
+  const step1 = t.step({
+    name: "step 1",
+    fn: async () => {
+      step1Entered.resolve();
+      await step2Finished;
+    },
+    sanitizeOps: false,
+    sanitizeResources: false,
+    sanitizeExit: false,
+  });
+  await step1Entered;
+  await t.step({
+    name: "step 2",
+    fn: async () => {
+      await new Promise((resolve) => setTimeout(resolve, 100));
+    },
+  });
+  step2Finished.resolve();
+  await step1;
+});
+
+Deno.test({
+  name: "parallel steps where only inner tests have sanitizers",
+  fn: async (t) => {
+    const step1Entered = deferred();
+    const step2Finished = deferred();
+    const step1 = t.step("step 1", async (t) => {
+      await t.step({
+        name: "step inner",
+        fn: async () => {
+          step1Entered.resolve();
+          await step2Finished;
+        },
+        sanitizeOps: true,
+      });
+    });
+    await step1Entered;
+    await t.step("step 2", async (t) => {
+      await t.step({
+        name: "step inner",
+        fn: () => {},
+        sanitizeOps: true,
+      });
+    });
+    step2Finished.resolve();
+    await step1;
+  },
+  sanitizeResources: false,
+  sanitizeOps: false,
+  sanitizeExit: false,
+});
diff --git a/cli/tests/testdata/test/steps/no_unstable_flag.out b/cli/tests/testdata/test/steps/no_unstable_flag.out
new file mode 100644
index 000000000..8fe6ba4f7
--- /dev/null
+++ b/cli/tests/testdata/test/steps/no_unstable_flag.out
@@ -0,0 +1,13 @@
+[WILDCARD]
+running 1 test from [WILDCARD]/no_unstable_flag.ts
+test description ... FAILED ([WILDCARD])
+
+failures:
+
+description
+Error: Test steps are unstable. The --unstable flag must be provided.
+    at [WILDCARD]
+
+failures:
+
+[WILDCARD]
diff --git a/cli/tests/testdata/test/steps/no_unstable_flag.ts b/cli/tests/testdata/test/steps/no_unstable_flag.ts
new file mode 100644
index 000000000..737efba11
--- /dev/null
+++ b/cli/tests/testdata/test/steps/no_unstable_flag.ts
@@ -0,0 +1,4 @@
+Deno.test("description", async (t) => {
+  // deno-lint-ignore no-explicit-any
+  await (t as any).step("step", () => {});
+});
diff --git a/cli/tests/testdata/test/steps/passing_steps.out b/cli/tests/testdata/test/steps/passing_steps.out
new file mode 100644
index 000000000..b92327d17
--- /dev/null
+++ b/cli/tests/testdata/test/steps/passing_steps.out
@@ -0,0 +1,38 @@
+[WILDCARD]
+running 5 tests from [WILDCARD]
+test description ...
+  test step 1 ...
+    test inner 1 ... ok ([WILDCARD]ms)
+    test inner 2 ... ok ([WILDCARD]ms)
+  ok ([WILDCARD]ms)
+ok ([WILDCARD]ms)
+test parallel steps without sanitizers ...
+  test step 1 ... ok ([WILDCARD])
+  test step 2 ... ok ([WILDCARD])
+ok ([WILDCARD])
+test parallel steps without sanitizers due to parent ...
+  test step 1 ... ok ([WILDCARD])
+  test step 2 ... ok ([WILDCARD])
+ok ([WILDCARD])
+test steps with disabled sanitizers, then enabled, then parallel disabled ...
+  test step 1 ...
+    test step 1 ...
+      test step 1 ...
+        test step 1 ... ok ([WILDCARD])
+        test step 1 ... ok ([WILDCARD])
+      ok ([WILDCARD])
+      test step 2 ... ok ([WILDCARD])
+    ok ([WILDCARD])
+  ok ([WILDCARD])
+ok ([WILDCARD])
+test steps buffered then streaming reporting ...
+  test step 1 ...
+    test step 1 - 1 ... ok ([WILDCARD])
+    test step 1 - 2 ...
+      test step 1 - 2 - 1 ... ok ([WILDCARD])
+    ok ([WILDCARD])
+  ok ([WILDCARD])
+  test step 2 ... ok ([WILDCARD])
+ok ([WILDCARD])
+
+test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out [WILDCARD]
diff --git a/cli/tests/testdata/test/steps/passing_steps.ts b/cli/tests/testdata/test/steps/passing_steps.ts
new file mode 100644
index 000000000..fbd52e2d3
--- /dev/null
+++ b/cli/tests/testdata/test/steps/passing_steps.ts
@@ -0,0 +1,120 @@
+import { deferred } from "../../../../../test_util/std/async/deferred.ts";
+
+Deno.test("description", async (t) => {
+  const success = await t.step("step 1", async (t) => {
+    await t.step("inner 1", () => {});
+    await t.step("inner 2", () => {});
+  });
+
+  if (!success) throw new Error("Expected the step to return true.");
+});
+
+Deno.test("parallel steps without sanitizers", async (t) => {
+  // allowed
+  await Promise.all([
+    t.step({
+      name: "step 1",
+      fn: async () => {
+        await new Promise((resolve) => setTimeout(resolve, 10));
+      },
+      sanitizeOps: false,
+      sanitizeResources: false,
+      sanitizeExit: false,
+    }),
+    t.step({
+      name: "step 2",
+      fn: async () => {
+        await new Promise((resolve) => setTimeout(resolve, 10));
+      },
+      sanitizeOps: false,
+      sanitizeResources: false,
+      sanitizeExit: false,
+    }),
+  ]);
+});
+
+Deno.test({
+  name: "parallel steps without sanitizers due to parent",
+  fn: async (t) => {
+    // allowed because parent disabled the sanitizers
+    await Promise.all([
+      t.step("step 1", async () => {
+        await new Promise((resolve) => setTimeout(resolve, 10));
+      }),
+      t.step("step 2", async () => {
+        await new Promise((resolve) => setTimeout(resolve, 10));
+      }),
+    ]);
+  },
+  sanitizeResources: false,
+  sanitizeOps: false,
+  sanitizeExit: false,
+});
+
+Deno.test({
+  name: "steps with disabled sanitizers, then enabled, then parallel disabled",
+  fn: async (t) => {
+    await t.step("step 1", async (t) => {
+      await t.step({
+        name: "step 1",
+        fn: async (t) => {
+          await Promise.all([
+            t.step({
+              name: "step 1",
+              fn: async (t) => {
+                await new Promise((resolve) => setTimeout(resolve, 10));
+                await Promise.all([
+                  t.step("step 1", () => {}),
+                  t.step("step 1", () => {}),
+                ]);
+              },
+              sanitizeExit: false,
+              sanitizeResources: false,
+              sanitizeOps: false,
+            }),
+            t.step({
+              name: "step 2",
+              fn: () => {},
+              sanitizeResources: false,
+              sanitizeOps: false,
+              sanitizeExit: false,
+            }),
+          ]);
+        },
+        sanitizeResources: true,
+        sanitizeOps: true,
+        sanitizeExit: true,
+      });
+    });
+  },
+  sanitizeResources: false,
+  sanitizeOps: false,
+  sanitizeExit: false,
+});
+
+Deno.test("steps buffered then streaming reporting", async (t) => {
+  // no sanitizers so this will be buffered
+  await t.step({
+    name: "step 1",
+    fn: async (t) => {
+      // also ensure the buffered tests display in order regardless of the second one finishing first
+      const step2Finished = deferred();
+      const step1 = t.step("step 1 - 1", async () => {
+        await step2Finished;
+      });
+      const step2 = t.step("step 1 - 2", async (t) => {
+        await t.step("step 1 - 2 - 1", () => {});
+      });
+      await step2;
+      step2Finished.resolve();
+      await step1;
+    },
+    sanitizeResources: false,
+    sanitizeOps: false,
+    sanitizeExit: false,
+  });
+
+  // now this will start streaming and we want to
+  // ensure it flushes the buffer of the last test
+  await t.step("step 2", async () => {});
+});
diff --git a/cli/tests/unit/test_util.ts b/cli/tests/unit/test_util.ts
index ee924fe8a..65d23af65 100644
--- a/cli/tests/unit/test_util.ts
+++ b/cli/tests/unit/test_util.ts
@@ -39,7 +39,7 @@ interface UnitTestOptions {
   permissions?: UnitTestPermissions;
 }
 
-type TestFunction = () => void | Promise<void>;
+type TestFunction = (tester: Deno.TestContext) => void | Promise<void>;
 
 export function unitTest(fn: TestFunction): void;
 export function unitTest(options: UnitTestOptions, fn: TestFunction): void;
diff --git a/cli/tests/unit/testing_test.ts b/cli/tests/unit/testing_test.ts
index 89b3cc31f..144246002 100644
--- a/cli/tests/unit/testing_test.ts
+++ b/cli/tests/unit/testing_test.ts
@@ -1,5 +1,5 @@
 // Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
-import { assertThrows, unitTest } from "./test_util.ts";
+import { assertRejects, assertThrows, unitTest } from "./test_util.ts";
 
 unitTest(function testFnOverloading() {
   // just verifying that you can use this test definition syntax
@@ -25,3 +25,41 @@ unitTest(function nameOfTestCaseCantBeEmpty() {
     "The test name can't be empty",
   );
 });
+
+unitTest(function invalidStepArguments(t) {
+  assertRejects(
+    async () => {
+      // deno-lint-ignore no-explicit-any
+      await (t as any).step("test");
+    },
+    TypeError,
+    "Expected function for second argument.",
+  );
+
+  assertRejects(
+    async () => {
+      // deno-lint-ignore no-explicit-any
+      await (t as any).step("test", "not a function");
+    },
+    TypeError,
+    "Expected function for second argument.",
+  );
+
+  assertRejects(
+    async () => {
+      // deno-lint-ignore no-explicit-any
+      await (t as any).step();
+    },
+    TypeError,
+    "Expected a test definition or name and function.",
+  );
+
+  assertRejects(
+    async () => {
+      // deno-lint-ignore no-explicit-any
+      await (t as any).step(() => {});
+    },
+    TypeError,
+    "Expected a test definition or name and function.",
+  );
+});