1 files changed, 79 insertions, 27 deletions

diff --git a/tools/wpt.ts b/tools/wpt.ts
index 12e9047a4..e7da43e64 100755
--- a/tools/wpt.ts
+++ b/tools/wpt.ts
@@ -137,7 +137,6 @@ async function setup() {
 }
 
 interface TestToRun {
-  sourcePath: string;
   path: string;
   url: URL;
   options: ManifestTestOptions;
@@ -146,7 +145,12 @@ interface TestToRun {
 
 async function run() {
   assert(Array.isArray(rest), "filter must be array");
-  const tests = discoverTestsToRun(rest.length == 0 ? undefined : rest);
+  const expectation = getExpectation();
+  const tests = discoverTestsToRun(
+    rest.length == 0 ? undefined : rest,
+    expectation,
+  );
+  assertAllExpectationsHaveTests(expectation, tests);
   console.log(`Going to run ${tests.length} test files.`);
 
   const results = await runWithTestUtil(false, async () => {
@@ -173,6 +177,43 @@ async function run() {
   Deno.exit(code);
 }
 
+// Check that all expectations in the expectations file have a test that will be
+// run.
+function assertAllExpectationsHaveTests(
+  expectation: Expectation,
+  testsToRun: TestToRun[],
+): void {
+  const tests = new Set(testsToRun.map((t) => t.path));
+  const missingTests: string[] = [];
+
+  function walk(parentExpectation: Expectation, parent: string) {
+    for (const key in parentExpectation) {
+      const path = `${parent}/${key}`;
+      const expectation = parentExpectation[key];
+      if (typeof expectation == "boolean" || Array.isArray(expectation)) {
+        if (!tests.has(path)) {
+          missingTests.push(path);
+        }
+      } else {
+        walk(expectation, path);
+      }
+    }
+  }
+
+  walk(expectation, "");
+
+  if (missingTests.length > 0) {
+    console.log(
+      red(
+        "Following tests are missing in manifest, but are present in expectations:",
+      ),
+    );
+    console.log("");
+    console.log(missingTests.join("\n"));
+    Deno.exit(1);
+  }
+}
+
 async function update() {
   assert(Array.isArray(rest), "filter must be array");
   const tests = discoverTestsToRun(rest.length == 0 ? undefined : rest, true);
@@ -204,8 +245,8 @@ async function update() {
     { passed: string[]; failed: string[]; status: number }
   > = {};
   for (const { test, result } of results) {
-    if (!resultTests[test.sourcePath]) {
-      resultTests[test.sourcePath] = {
+    if (!resultTests[test.path]) {
+      resultTests[test.path] = {
         passed: [],
         failed: [],
         status: result.status,
@@ -213,9 +254,9 @@ async function update() {
     }
     for (const case_ of result.cases) {
       if (case_.passed) {
-        resultTests[test.sourcePath].passed.push(case_.name);
+        resultTests[test.path].passed.push(case_.name);
       } else {
-        resultTests[test.sourcePath].failed.push(case_.name);
+        resultTests[test.path].failed.push(case_.name);
       }
     }
   }
@@ -485,27 +526,9 @@ function discoverTestsToRun(
     prefix: string,
   ) {
     for (const key in parentFolder) {
-      const sourcePath = `${prefix}/${key}`;
       const entry = parentFolder[key];
-      const expectation = Array.isArray(parentExpectation) ||
-          typeof parentExpectation == "boolean"
-        ? parentExpectation
-        : parentExpectation[key];
-
-      if (expectation === undefined) continue;
 
       if (Array.isArray(entry)) {
-        assert(
-          Array.isArray(expectation) || typeof expectation == "boolean",
-          "test entry must not have a folder expectation",
-        );
-        if (
-          filter &&
-          !filter.find((filter) => sourcePath.substring(1).startsWith(filter))
-        ) {
-          continue;
-        }
-
         for (
           const [path, options] of entry.slice(
             1,
@@ -514,16 +537,45 @@ function discoverTestsToRun(
           if (!path) continue;
           const url = new URL(path, "http://web-platform.test:8000");
           if (!url.pathname.endsWith(".any.html")) continue;
+          const finalPath = url.pathname + url.search;
+
+          const split = finalPath.split("/");
+          const finalKey = split[split.length - 1];
+
+          const expectation = Array.isArray(parentExpectation) ||
+              typeof parentExpectation == "boolean"
+            ? parentExpectation
+            : parentExpectation[finalKey];
+
+          if (expectation === undefined) continue;
+
+          assert(
+            Array.isArray(expectation) || typeof expectation == "boolean",
+            "test entry must not have a folder expectation",
+          );
+
+          if (
+            filter &&
+            !filter.find((filter) => finalPath.substring(1).startsWith(filter))
+          ) {
+            continue;
+          }
           testsToRun.push({
-            sourcePath,
-            path: url.pathname + url.search,
+            path: finalPath,
             url,
             options,
             expectation,
           });
         }
       } else {
-        walk(entry, expectation, sourcePath);
+        const expectation = Array.isArray(parentExpectation) ||
+            typeof parentExpectation == "boolean"
+          ? parentExpectation
+          : parentExpectation[key];
+
+        if (expectation === undefined) continue;
+
+        walk(entry, expectation, `${prefix}/${key}`);
       }
     }
   }