chore(wpt): Mark a WPT test as failed if it exits before completion (#11371)

Currently, a WPT test is considered failed if its status code is
anything other than 0, regardless of whether the test suite completed
running or not, and any subtests that haven't finished running are not
considered to be failures.

But a test can exit with a zero status code before it has completed
running, if the event loop has run out of tasks because of a bug in one
of the ops, leading to false positives. This change fixes that.

2 files changed, 23 insertions, 19 deletions

diff --git a/tools/wpt.ts b/tools/wpt.ts
index 799e6b0c7..a3999a425 100755
--- a/tools/wpt.ts
+++ b/tools/wpt.ts
@@ -210,6 +210,14 @@ async function generateWptReport(
       : result.harnessStatus?.status === 0
       ? "OK"
       : "ERROR";
+    let message;
+    if (result.harnessStatus === null && result.status === 0) {
+      // If the only error is the event loop running out of tasks, using stderr
+      // as the message won't help.
+      message = "Event loop run out of tasks.";
+    } else {
+      message = result.harnessStatus?.message ?? (result.stderr.trim() || null);
+    }
     const reportResult = {
       test: test.url.pathname + test.url.search + test.url.hash,
       subtests: result.cases.map((case_) => {
@@ -231,9 +239,7 @@ async function generateWptReport(
         };
       }),
       status,
-      message: escapeLoneSurrogates(
-        result.harnessStatus?.message ?? (result.stderr.trim() || null),
-      ),
+      message: escapeLoneSurrogates(message),
       duration: result.duration,
       expected: status === "OK" ? undefined : "OK",
       "known_intermittent": [],
@@ -320,14 +326,14 @@ async function update() {
 
   const resultTests: Record<
     string,
-    { passed: string[]; failed: string[]; status: number }
+    { passed: string[]; failed: string[]; testSucceeded: boolean }
   > = {};
   for (const { test, result } of results) {
     if (!resultTests[test.path]) {
       resultTests[test.path] = {
         passed: [],
         failed: [],
-        status: result.status,
+        testSucceeded: result.status === 0 && result.harnessStatus !== null,
       };
     }
     for (const case_ of result.cases) {
@@ -342,11 +348,11 @@ async function update() {
   const currentExpectation = getExpectation();
 
   for (const path in resultTests) {
-    const { passed, failed, status } = resultTests[path];
+    const { passed, failed, testSucceeded } = resultTests[path];
     let finalExpectation: boolean | string[];
-    if (failed.length == 0 && status == 0) {
+    if (failed.length == 0 && testSucceeded) {
       finalExpectation = true;
-    } else if (failed.length > 0 && passed.length > 0 && status == 0) {
+    } else if (failed.length > 0 && passed.length > 0 && testSucceeded) {
       finalExpectation = failed;
     } else {
       finalExpectation = false;
@@ -413,7 +419,7 @@ function reportFinal(
       result,
       test.expectation,
     );
-    if (result.status !== 0) {
+    if (result.status !== 0 || result.harnessStatus === null) {
       if (test.expectation === false) {
         finalExpectedFailedAndFailedCount += 1;
       } else {
@@ -520,15 +526,18 @@ function analyzeTestResult(
 }
 
 function reportVariation(result: TestResult, expectation: boolean | string[]) {
-  if (result.status !== 0) {
+  if (result.status !== 0 || result.harnessStatus === null) {
     console.log(`test stderr:`);
     writeAllSync(Deno.stdout, new TextEncoder().encode(result.stderr));
 
     const expectFail = expectation === false;
+    const failReason = result.status !== 0
+      ? "runner failed during test"
+      : "the event loop run out of tasks during the test";
     console.log(
       `\nfile result: ${
         expectFail ? yellow("failed (expected)") : red("failed")
-      }. runner failed during test\n`,
+      }. ${failReason}\n`,
     );
     return;
   }
diff --git a/tools/wpt/expectation.json b/tools/wpt/expectation.json
index fbab59d68..4124efe7c 100644
--- a/tools/wpt/expectation.json
+++ b/tools/wpt/expectation.json
@@ -2252,12 +2252,7 @@
       "enqueue-with-detached-buffer.window.html": false
     },
     "readable-streams": {
-      "async-iterator.any.html": [
-        "Async-iterating a pull source manually",
-        "next() rejects if the stream errors",
-        "return() does not rejects if the stream has not errored yet",
-        "next() that succeeds; next() that reports an error; next()"
-      ],
+      "async-iterator.any.html": false,
       "bad-strategies.any.html": true,
       "bad-underlying-sources.any.html": true,
       "cancel.any.html": true,
@@ -2799,7 +2794,7 @@
     "reading-data-section": {
       "Determining-Encoding.any.html": true,
       "FileReader-event-handler-attributes.any.html": true,
-      "FileReader-multiple-reads.any.html": true,
+      "FileReader-multiple-reads.any.html": false,
       "filereader_abort.any.html": true,
       "filereader_error.any.html": true,
       "filereader_events.any.html": false,
@@ -2825,7 +2820,7 @@
         "type-long-settimeout.any.html": true
       },
       "microtask-queuing": {
-        "queue-microtask-exceptions.any.html": true,
+        "queue-microtask-exceptions.any.html": false,
         "queue-microtask.any.html": true
       }
     }