From 64e8c36805dab0362ed274a977bb9d0638b55b2a Mon Sep 17 00:00:00 2001 From: Yusuke Tanaka Date: Tue, 26 Mar 2024 00:08:46 +0900 Subject: fix(cli): output more detailed information for steps when using JUnit reporter (#22797) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This patch gets JUnit reporter to output more detailed information for test steps (subtests). ## Issue with previous implementation In the previous implementation, the test hierarchy was represented using several XML tags like the following: - `` corresponds to the entire test (one execution of `deno test` has exactly one `` tag) - `` corresponds to one file, such as `main_test.ts` - `` corresponds to one `Deno.test(...)` - `` corresponds to one `t.step(...)` This structure describes the test layers but one problem is that `` tag is used for any use cases so some tools that can ingest a JUnit XML file might not be able to interpret `` as subtests. ## How other tools address it Some of the testing frameworks in the ecosystem address this issue by fitting subtests into the `` layer. For instance, take a look at the following Go test file: ```go package main_test import "testing" func TestMain(t *testing.T) { t.Run("child 1", func(t *testing.T) { // OK }) t.Run("child 2", func(t *testing.T) { // Error t.Fatal("error") }) } ``` Running [gotestsum], we can get the output like this: ```xml === RUN TestMain/child_2 main_test.go:12: error --- FAIL: TestMain/child_2 (0.00s) === RUN TestMain --- FAIL: TestMain (0.00s) ``` This output shows that nested test cases are squashed into the `` layer by treating them as the same layer as their parent, `TestMain`. We can still distinguish nested ones by their `name` attributes that look like `TestMain/`. As described in #22795, [vitest] solves the issue in the same way as [gotestsum]. One downside of this would be that one test failure that happens in a nested test case will end up being counted multiple times, because not only the subtest but also its wrapping container(s) are considered to be failures. In fact, in the [gotestsum] output above, `TestMain/child_2` failed (which is totally expected) while its parent, `TestMain`, was also counted as failure. As https://github.com/denoland/deno/pull/20273#discussion_r1307558757 pointed out, there is a test runner that offers flexibility to prevent this, but I personally don't think the "duplicate failure count" issue is a big deal. ## How to fix the issue in this patch This patch fixes the issue with the same approach as [gotestsum] and [vitest]. More specifically, nested test cases are put into the `` level and their names are now represented as squashed test names concatenated by `>` (e.g. `parent 2 > child 1 > grandchild 1`). This change also allows us to put a detailed error message as `` tag within the `` tag, which should be handled nicely by third-party tools supporting JUnit XML. ## Extra fix Also, file paths embedded into XML outputs are changed from absolute path to relative path, which is helpful when running the test suites in several different environments like CI. Resolves #22795 [gotestsum]: https://github.com/gotestyourself/gotestsum [vitest]: https://vitest.dev/ --------- Co-authored-by: Bartek Iwańczuk --- tests/integration/test_tests.rs | 12 +++ .../test/junit_multiple_test_files.junit.out | 102 +++++++++++++++++++++ tests/testdata/test/nested_failures.junit.out | 47 ++++++++++ tests/testdata/test/nested_failures.ts | 23 +++++ tests/testdata/test/pass.junit.out | 36 ++++---- 5 files changed, 202 insertions(+), 18 deletions(-) create mode 100644 tests/testdata/test/junit_multiple_test_files.junit.out create mode 100644 tests/testdata/test/nested_failures.junit.out create mode 100644 tests/testdata/test/nested_failures.ts (limited to 'tests') diff --git a/tests/integration/test_tests.rs b/tests/integration/test_tests.rs index d5768b5ba..33abdf22c 100644 --- a/tests/integration/test_tests.rs +++ b/tests/integration/test_tests.rs @@ -283,6 +283,18 @@ itest!(junit { output: "test/pass.junit.out", }); +itest!(junit_nested { + args: "test --reporter junit test/nested_failures.ts", + output: "test/nested_failures.junit.out", + exit_code: 1, +}); + +itest!(junit_multiple_test_files { + args: "test --reporter junit test/pass.ts test/fail.ts", + output: "test/junit_multiple_test_files.junit.out", + exit_code: 1, +}); + #[test] fn junit_path() { let context = TestContextBuilder::new().use_temp_cwd().build(); diff --git a/tests/testdata/test/junit_multiple_test_files.junit.out b/tests/testdata/test/junit_multiple_test_files.junit.out new file mode 100644 index 000000000..bf6f3eaa4 --- /dev/null +++ b/tests/testdata/test/junit_multiple_test_files.junit.out @@ -0,0 +1,102 @@ +Check file:///[WILDCARD]/test/pass.ts +Check file:///[WILDCARD]/test/fail.ts + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Error + throw new Error(); + ^ + at file:///[WILDCARD]/test/fail.ts:2:9 + + + Error + throw new Error(); + ^ + at file:///[WILDCARD]/test/fail.ts:5:9 + + + Error + throw new Error(); + ^ + at file:///[WILDCARD]/test/fail.ts:8:9 + + + Error + throw new Error(); + ^ + at file:///[WILDCARD]/test/fail.ts:11:9 + + + Error + throw new Error(); + ^ + at file:///[WILDCARD]/test/fail.ts:14:9 + + + Error + throw new Error(); + ^ + at file:///[WILDCARD]/test/fail.ts:17:9 + + + Error + throw new Error(); + ^ + at file:///[WILDCARD]/test/fail.ts:20:9 + + + Error + throw new Error(); + ^ + at file:///[WILDCARD]/test/fail.ts:23:9 + + + Error + throw new Error(); + ^ + at file:///[WILDCARD]/test/fail.ts:26:9 + + + Error + throw new Error(); + ^ + at file:///[WILDCARD]/test/fail.ts:29:9 + + + +error: Test failed diff --git a/tests/testdata/test/nested_failures.junit.out b/tests/testdata/test/nested_failures.junit.out new file mode 100644 index 000000000..3e4d3c0d4 --- /dev/null +++ b/tests/testdata/test/nested_failures.junit.out @@ -0,0 +1,47 @@ +Check file:///[WILDCARD]/test/nested_failures.ts + + + + + 1 test step failed. + + + 2 test steps failed. + + + + + + + Error: Fail. + throw new Error("Fail."); + ^ + at file:///[WILDCARD]/test/nested_failures.ts:4:11 + [WILDCARD] + + + 1 test step failed. + + + + + Error: Fail. + throw new Error("Fail."); + ^ + at file:///[WILDCARD]/test/nested_failures.ts:12:13 + [WILDCARD] + + + Error: Fail. + throw new Error("Fail."); + ^ + at file:///[WILDCARD]/test/nested_failures.ts:16:11 + [WILDCARD] + + + + + + + +error: Test failed diff --git a/tests/testdata/test/nested_failures.ts b/tests/testdata/test/nested_failures.ts new file mode 100644 index 000000000..128e48aef --- /dev/null +++ b/tests/testdata/test/nested_failures.ts @@ -0,0 +1,23 @@ +Deno.test("parent 1", async (t) => { + await t.step("child 1", () => {}); + await t.step("child 2", () => { + throw new Error("Fail."); + }); +}); + +Deno.test("parent 2", async (t) => { + await t.step("child 1", async (t) => { + await t.step("grandchild 1", () => {}); + await t.step("grandchild 2", () => { + throw new Error("Fail."); + }); + }); + await t.step("child 2", () => { + throw new Error("Fail."); + }); +}); + +Deno.test("parent 3", async (t) => { + await t.step("child 1", () => {}); + await t.step("child 2", () => {}); +}); diff --git a/tests/testdata/test/pass.junit.out b/tests/testdata/test/pass.junit.out index b652dbf85..af9fd6f6b 100644 --- a/tests/testdata/test/pass.junit.out +++ b/tests/testdata/test/pass.junit.out @@ -1,38 +1,38 @@ -Check [WILDCARD]/testdata/test/pass.ts +Check file:///[WILDCARD]/test/pass.ts - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + -- cgit v1.2.3