summaryrefslogtreecommitdiff
path: root/cli/tests/testdata/test
diff options
context:
space:
mode:
authorValentin Anger <syrupthinker@gryphno.de>2023-08-26 01:19:23 +0200
committerGitHub <noreply@github.com>2023-08-26 01:19:23 +0200
commita526cff0a9888a475d5b542efda443fe720a93d0 (patch)
tree9e6245c6ce5af9c25bd9cac11bd591d609edd504 /cli/tests/testdata/test
parent8bb4e10881730576bbb82e54ede1ebf5931194c3 (diff)
feat(cli/tools): add TAP test reporter (#14390) (#20073)
This PR adds a test reporter for the [Test Anything Protocol](https://testanything.org). It makes the following implementation decisions: - No TODO pragma, as there is no such marker in `Deno.test` - SKIP pragma for `ignore`d tests - Test steps are treated as TAP14 subtests - Support for this in consumers seems spotty - Some consumers will incorrectly interpret these markers, resulting in unexpected output - Considering the lack of support, and to avoid implementation complexity, subtests are at most one level deep (all test steps are in the same subtest) - To accommodate consumers that use comments to indicate test-suites (unspecced) - The test module path is output as a comment - This is disabled for `--parallel` testing - Failure diagnostics are output as JSON, which is also valid YAML - The structure is not specified, so the format roughly follows the spec example: ``` --- message: "Failed with error 'hostname peebles.example.com not found'" severity: fail found: hostname: 'peebles.example.com' address: ~ wanted: hostname: 'peebles.example.com' address: '85.193.201.85' at: file: test/dns-resolve.c line: 142 ... ```
Diffstat (limited to 'cli/tests/testdata/test')
-rw-r--r--cli/tests/testdata/test/steps/failing_steps.tap.out43
-rw-r--r--cli/tests/testdata/test/steps/ignored_steps.tap.out8
-rw-r--r--cli/tests/testdata/test/steps/passing_steps.tap.out42
3 files changed, 93 insertions, 0 deletions
diff --git a/cli/tests/testdata/test/steps/failing_steps.tap.out b/cli/tests/testdata/test/steps/failing_steps.tap.out
new file mode 100644
index 000000000..11b289f08
--- /dev/null
+++ b/cli/tests/testdata/test/steps/failing_steps.tap.out
@@ -0,0 +1,43 @@
+TAP version 14
+# ./test/steps/failing_steps.ts
+# Subtest: nested failure
+ not ok 1 - inner 1
+ ---
+ {"message":"Error: Failed.\n throw new Error(\"Failed.\");\n ^\n at [WILDCARD]/failing_steps.ts:[WILDCARD]\n[WILDCARD]","severity":"fail","at":{"file":"./test/steps/failing_steps.ts","line":[WILDCARD]}}
+ ...
+ ok 2 - inner 2
+ not ok 3 - step 1
+ ---
+ {"message":"1 test step failed.","severity":"fail","at":{"file":"./test/steps/failing_steps.ts","line":[WILDCARD]}}
+ ...
+ 1..3
+not ok 1 - nested failure
+ ---
+ {"message":"1 test step failed.","severity":"fail","at":{"file":"./test/steps/failing_steps.ts","line":[WILDCARD]}}
+ ...
+# Subtest: multiple test step failures
+ not ok 1 - step 1
+ ---
+ {"message":"Error: Fail.\n throw new Error(\"Fail.\");\n ^\n at [WILDCARD]/failing_steps.ts:[WILDCARD]\n[WILDCARD]","severity":"fail","at":{"file":"./test/steps/failing_steps.ts","line":[WILDCARD]}}
+ ...
+ not ok 2 - step 2
+ ---
+ {"message":"Error: Fail.\n await t.step(\"step 2\", () => Promise.reject(new Error(\"Fail.\")));\n ^\n at [WILDCARD]/failing_steps.ts:[WILDCARD]\n[WILDCARD]","severity":"fail","at":{"file":"./test/steps/failing_steps.ts","line":[WILDCARD]}}
+ ...
+ 1..2
+not ok 2 - multiple test step failures
+ ---
+ {"message":"2 test steps failed.","severity":"fail","at":{"file":"./test/steps/failing_steps.ts","line":[WILDCARD]}}
+ ...
+# Subtest: failing step in failing test
+ not ok 1 - step 1
+ ---
+ {"message":"Error: Fail.\n throw new Error(\"Fail.\");\n ^\n at [WILDCARD]/failing_steps.ts:[WILDCARD]\n[WILDCARD]","severity":"fail","at":{"file":"./test/steps/failing_steps.ts","line":[WILDCARD]}}
+ ...
+ 1..1
+not ok 3 - failing step in failing test
+ ---
+ {"message":"Error: Fail test.\n throw new Error(\"Fail test.\");\n ^\n at [WILDCARD]/failing_steps.ts:[WILDCARD]","severity":"fail","at":{"file":"./test/steps/failing_steps.ts","line":[WILDCARD]}}
+ ...
+1..3
+error: Test failed
diff --git a/cli/tests/testdata/test/steps/ignored_steps.tap.out b/cli/tests/testdata/test/steps/ignored_steps.tap.out
new file mode 100644
index 000000000..b2b2f5070
--- /dev/null
+++ b/cli/tests/testdata/test/steps/ignored_steps.tap.out
@@ -0,0 +1,8 @@
+TAP version 14
+# ./test/steps/ignored_steps.ts
+# Subtest: ignored step
+ ok 1 - step 1 # SKIP
+ ok 2 - step 2
+ 1..2
+ok 1 - ignored step
+1..1
diff --git a/cli/tests/testdata/test/steps/passing_steps.tap.out b/cli/tests/testdata/test/steps/passing_steps.tap.out
new file mode 100644
index 000000000..20a9fa312
--- /dev/null
+++ b/cli/tests/testdata/test/steps/passing_steps.tap.out
@@ -0,0 +1,42 @@
+TAP version 14
+# ./test/steps/passing_steps.ts
+# Subtest: description
+ ok 1 - inner 1
+ ok 2 - inner 2
+ ok 3 - step 1
+ 1..3
+ok 1 - description
+# Subtest: description function as first arg
+ ok 1 - inner1
+ ok 2 - inner1
+ ok 3 - step1
+ 1..3
+ok 2 - description function as first arg
+# Subtest: parallel steps without sanitizers
+ ok 1 - step 1
+ ok 2 - step 2
+ 1..2
+ok 3 - parallel steps without sanitizers
+# Subtest: parallel steps without sanitizers due to parent
+ ok 1 - step 1
+ ok 2 - step 2
+ 1..2
+ok 4 - parallel steps without sanitizers due to parent
+# Subtest: steps with disabled sanitizers, then enabled, then parallel disabled
+ ok 1 - step 2
+ ok 2 - step 1
+ ok 3 - step 1
+ ok 4 - step 1
+ ok 5 - step 1
+ ok 6 - step 1
+ 1..6
+ok 5 - steps with disabled sanitizers, then enabled, then parallel disabled
+# Subtest: steps buffered then streaming reporting
+ ok 1 - step 1 - 2 - 1
+ ok 2 - step 1 - 2
+ ok 3 - step 1 - 1
+ ok 4 - step 1
+ ok 5 - step 2
+ 1..5
+ok 6 - steps buffered then streaming reporting
+1..6