summaryrefslogtreecommitdiff
path: root/tools/wpt.ts
diff options
context:
space:
mode:
Diffstat (limited to 'tools/wpt.ts')
-rwxr-xr-xtools/wpt.ts533
1 files changed, 533 insertions, 0 deletions
diff --git a/tools/wpt.ts b/tools/wpt.ts
new file mode 100755
index 000000000..3aa5666f0
--- /dev/null
+++ b/tools/wpt.ts
@@ -0,0 +1,533 @@
+#!/usr/bin/env -S deno run --unstable --allow-write --allow-read --allow-net --allow-env --allow-run
+// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
+
+// This script is used to run WPT tests for Deno.
+
+import {
+ runSingleTest,
+ runWithTestUtil,
+ TestCaseResult,
+ TestResult,
+} from "./wpt/runner.ts";
+import {
+ assert,
+ autoConfig,
+ cargoBuild,
+ checkPy3Available,
+ Expectation,
+ getExpectation,
+ getExpectFailForCase,
+ getManifest,
+ json,
+ ManifestFolder,
+ ManifestTestOptions,
+ ManifestTestVariation,
+ quiet,
+ rest,
+ runPy,
+ updateManifest,
+} from "./wpt/utils.ts";
+import {
+ blue,
+ bold,
+ green,
+ red,
+ yellow,
+} from "https://deno.land/std@0.84.0/fmt/colors.ts";
+import { saveExpectation } from "./wpt/utils.ts";
+
+const command = Deno.args[0];
+
+switch (command) {
+ case "setup":
+ await checkPy3Available();
+ await updateManifest();
+ await setup();
+ break;
+
+ case "run":
+ await cargoBuild();
+ await run();
+ break;
+
+ case "update":
+ await cargoBuild();
+ await update();
+ break;
+
+ default:
+ console.log(`Possible commands:
+
+ setup
+ Validate that your environment is conigured correctly, or help you configure it.
+
+ run
+ Run all tests like specified in \`expectation.json\`.
+
+ update
+ Update the \`expectation.json\` to match the current reality.
+
+More details at https://deno.land/manual@master/contributing/web_platform_tests
+
+ `);
+ break;
+}
+
+async function setup() {
+ // TODO(lucacsonato): use this when 1.7.1 is released.
+ // const records = await Deno.resolveDns("web-platform.test", "A");
+ // const etcHostsConfigured = records[0] == "127.0.0.1";
+ const hostsFile = await Deno.readTextFile("/etc/hosts");
+ const etcHostsConfigured = hostsFile.includes("web-platform.test");
+
+ if (etcHostsConfigured) {
+ console.log("/etc/hosts is already configured.");
+ } else {
+ const autoConfigure = autoConfig ||
+ confirm(
+ "The WPT require certain entries to be present in your /etc/hosts file. Should these be configured automatically?",
+ );
+ if (autoConfigure) {
+ const proc = runPy(["wpt", "make-hosts-file"], { stdout: "piped" });
+ const status = await proc.status();
+ assert(status.success, "wpt make-hosts-file should not fail");
+ const entries = new TextDecoder().decode(await proc.output());
+ const hostsPath = Deno.build.os == "windows"
+ ? `${Deno.env.get("SystemRoot")}\\System32\\drivers\\etc\\hosts`
+ : "/etc/hosts";
+ const file = await Deno.open(hostsPath, { append: true }).catch((err) => {
+ if (err instanceof Deno.errors.PermissionDenied) {
+ throw new Error(
+ `Failed to open ${hostsPath} (permission error). Please run this command again with sudo, or configure the entries manually.`,
+ );
+ } else {
+ throw err;
+ }
+ });
+ await Deno.writeAll(
+ file,
+ new TextEncoder().encode(
+ "\n\n# Configured for Web Platform Tests (Deno)\n" + entries,
+ ),
+ );
+ console.log("Updated /etc/hosts");
+ } else {
+ console.log("Please configure the /etc/hosts entries manually.");
+ if (Deno.build.os == "windows") {
+ console.log("To do this run the following command in PowerShell:");
+ console.log("");
+ console.log(" cd test_util/wpt/");
+ console.log(
+ " python.exe wpt make-hosts-file | Out-File $env:SystemRoot\\System32\\drivers\\etc\\hosts -Encoding ascii -Append",
+ );
+ console.log("");
+ } else {
+ console.log("To do this run the following command in your shell:");
+ console.log("");
+ console.log(" cd test_util/wpt/");
+ console.log(
+ " python3 ./wpt make-hosts-file | sudo tee -a /etc/hosts",
+ );
+ console.log("");
+ }
+ }
+ }
+
+ console.log(green("Setup complete!"));
+}
+
+interface TestToRun {
+ sourcePath: string;
+ path: string;
+ url: URL;
+ options: ManifestTestOptions;
+ expectation: boolean | string[];
+}
+
+async function run() {
+ assert(Array.isArray(rest), "filter must be array");
+ const tests = discoverTestsToRun(rest.length == 0 ? undefined : rest);
+ console.log(`Going to run ${tests.length} test files.`);
+
+ const results = await runWithTestUtil(false, async () => {
+ const results = [];
+
+ for (const test of tests) {
+ console.log(`${blue("-".repeat(40))}\n${bold(test.path)}\n`);
+ const result = await runSingleTest(
+ test.url,
+ test.options,
+ json ? () => {} : createReportTestCase(test.expectation),
+ );
+ results.push({ test, result });
+ reportVariation(result, test.expectation);
+ }
+
+ return results;
+ });
+
+ if (json) {
+ await Deno.writeTextFile(json, JSON.stringify(results));
+ }
+ const code = reportFinal(results);
+ Deno.exit(code);
+}
+
+async function update() {
+ assert(Array.isArray(rest), "filter must be array");
+ const tests = discoverTestsToRun(rest.length == 0 ? undefined : rest, true);
+ console.log(`Going to run ${tests.length} test files.`);
+
+ const results = await runWithTestUtil(false, async () => {
+ const results = [];
+
+ for (const test of tests) {
+ console.log(`${blue("-".repeat(40))}\n${bold(test.path)}\n`);
+ const result = await runSingleTest(
+ test.url,
+ test.options,
+ json ? () => {} : createReportTestCase(test.expectation),
+ );
+ results.push({ test, result });
+ reportVariation(result, test.expectation);
+ }
+
+ return results;
+ });
+
+ if (json) {
+ await Deno.writeTextFile(json, JSON.stringify(results));
+ }
+
+ const resultTests: Record<
+ string,
+ { passed: string[]; failed: string[]; status: number }
+ > = {};
+ for (const { test, result } of results) {
+ if (!resultTests[test.sourcePath]) {
+ resultTests[test.sourcePath] = {
+ passed: [],
+ failed: [],
+ status: result.status,
+ };
+ }
+ for (const case_ of result.cases) {
+ if (case_.passed) {
+ resultTests[test.sourcePath].passed.push(case_.name);
+ } else {
+ resultTests[test.sourcePath].failed.push(case_.name);
+ }
+ }
+ }
+
+ const currentExpectation = getExpectation();
+
+ for (const path in resultTests) {
+ const { passed, failed, status } = resultTests[path];
+ let finalExpectation: boolean | string[];
+ if (failed.length == 0 && status == 0) {
+ finalExpectation = true;
+ } else if (failed.length > 0 && passed.length > 0 && status == 0) {
+ finalExpectation = failed;
+ } else {
+ finalExpectation = false;
+ }
+
+ insertExpectation(
+ path.slice(1).split("/"),
+ currentExpectation,
+ finalExpectation,
+ );
+ }
+
+ saveExpectation(currentExpectation);
+
+ reportFinal(results);
+
+ console.log(blue("Updated expectation.json to match reality."));
+
+ Deno.exit(0);
+}
+
+function insertExpectation(
+ segments: string[],
+ currentExpectation: Expectation,
+ finalExpectation: boolean | string[],
+) {
+ const segment = segments.shift();
+ assert(segment, "segments array must never be empty");
+ if (segments.length > 0) {
+ if (
+ !currentExpectation[segment] ||
+ Array.isArray(currentExpectation[segment]) ||
+ typeof currentExpectation[segment] === "boolean"
+ ) {
+ currentExpectation[segment] = {};
+ }
+ insertExpectation(
+ segments,
+ currentExpectation[segment] as Expectation,
+ finalExpectation,
+ );
+ } else {
+ currentExpectation[segment] = finalExpectation;
+ }
+}
+
+function reportFinal(
+ results: { test: TestToRun; result: TestResult }[],
+): number {
+ const finalTotalCount = results.length;
+ let finalFailedCount = 0;
+ const finalFailed: [string, TestCaseResult][] = [];
+ let finalExpectedFailedAndFailedCount = 0;
+ const finalExpectedFailedButPassedTests: [string, TestCaseResult][] = [];
+ const finalExpectedFailedButPassedFiles: string[] = [];
+ for (const { test, result } of results) {
+ const { failed, failedCount, expectedFailedButPassed } = analyzeTestResult(
+ result,
+ test.expectation,
+ );
+ if (result.status !== 0) {
+ if (test.expectation === false) {
+ finalExpectedFailedAndFailedCount += 1;
+ } else {
+ finalFailedCount += 1;
+ finalExpectedFailedButPassedFiles.push(test.path);
+ }
+ } else if (failedCount > 0) {
+ finalFailedCount += 1;
+ for (const case_ of failed) {
+ finalFailed.push([test.path, case_]);
+ }
+ for (const case_ of expectedFailedButPassed) {
+ finalExpectedFailedButPassedTests.push([test.path, case_]);
+ }
+ }
+ }
+ const finalPassedCount = finalTotalCount - finalFailedCount;
+
+ console.log(bold(blue("=".repeat(40))));
+
+ if (finalFailed.length > 0) {
+ console.log(`\nfailures:\n`);
+ }
+ for (const result of finalFailed) {
+ console.log(
+ ` ${JSON.stringify(`${result[0]} - ${result[1].name}`)}`,
+ );
+ }
+ if (finalExpectedFailedButPassedTests.length > 0) {
+ console.log(`\nexpected test failures that passed:\n`);
+ }
+ for (const result of finalExpectedFailedButPassedTests) {
+ console.log(
+ ` ${JSON.stringify(`${result[0]} - ${result[1].name}`)}`,
+ );
+ }
+ if (finalExpectedFailedButPassedFiles.length > 0) {
+ console.log(`\nexpected file failures that passed:\n`);
+ }
+ for (const result of finalExpectedFailedButPassedFiles) {
+ console.log(` ${JSON.stringify(result)}`);
+ }
+
+ console.log(
+ `\nfinal result: ${
+ finalFailedCount > 0 ? red("failed") : green("ok")
+ }. ${finalPassedCount} passed; ${finalFailedCount} failed; ${finalExpectedFailedAndFailedCount} expected failure; total ${finalTotalCount}\n`,
+ );
+
+ return finalFailedCount > 0 ? 1 : 0;
+}
+
+function analyzeTestResult(
+ result: TestResult,
+ expectation: boolean | string[],
+): {
+ failed: TestCaseResult[];
+ failedCount: number;
+ passedCount: number;
+ totalCount: number;
+ expectedFailedButPassed: TestCaseResult[];
+ expectedFailedButPassedCount: number;
+ expectedFailedAndFailedCount: number;
+} {
+ const failed = result.cases.filter(
+ (t) => !getExpectFailForCase(expectation, t.name) && !t.passed,
+ );
+ const expectedFailedButPassed = result.cases.filter(
+ (t) => getExpectFailForCase(expectation, t.name) && t.passed,
+ );
+ const expectedFailedButPassedCount = expectedFailedButPassed.length;
+ const failedCount = failed.length + expectedFailedButPassedCount;
+ const expectedFailedAndFailedCount = result.cases.filter(
+ (t) => getExpectFailForCase(expectation, t.name) && !t.passed,
+ ).length;
+ const totalCount = result.cases.length;
+ const passedCount = totalCount - failedCount - expectedFailedAndFailedCount;
+
+ return {
+ failed,
+ failedCount,
+ passedCount,
+ totalCount,
+ expectedFailedButPassed,
+ expectedFailedButPassedCount,
+ expectedFailedAndFailedCount,
+ };
+}
+
+function reportVariation(result: TestResult, expectation: boolean | string[]) {
+ if (result.status !== 0) {
+ console.log(`test stderr:`);
+ Deno.writeAllSync(Deno.stdout, new TextEncoder().encode(result.stderr));
+
+ const expectFail = expectation === false;
+ console.log(
+ `\nfile result: ${
+ expectFail ? yellow("failed (expected)") : red("failed")
+ }. runner failed during test\n`,
+ );
+ return;
+ }
+
+ const {
+ failed,
+ failedCount,
+ passedCount,
+ totalCount,
+ expectedFailedButPassed,
+ expectedFailedButPassedCount,
+ expectedFailedAndFailedCount,
+ } = analyzeTestResult(result, expectation);
+
+ if (failed.length > 0) {
+ console.log(`\nfailures:`);
+ }
+ for (const result of failed) {
+ console.log(`\n${result.name}\n${result.message}\n${result.stack}`);
+ }
+
+ if (failed.length > 0) {
+ console.log(`\nfailures:\n`);
+ }
+ for (const result of failed) {
+ console.log(` ${JSON.stringify(result.name)}`);
+ }
+ if (expectedFailedButPassedCount > 0) {
+ console.log(`\nexpected failures that passed:\n`);
+ }
+ for (const result of expectedFailedButPassed) {
+ console.log(` ${JSON.stringify(result.name)}`);
+ }
+ console.log(
+ `\nfile result: ${
+ failedCount > 0 ? red("failed") : green("ok")
+ }. ${passedCount} passed; ${failedCount} failed; ${expectedFailedAndFailedCount} expected failure; total ${totalCount}\n`,
+ );
+}
+
+function createReportTestCase(expectation: boolean | string[]) {
+ return function reportTestCase({ name, status }: TestCaseResult) {
+ const expectFail = getExpectFailForCase(expectation, name);
+ let simpleMessage = `test ${name} ... `;
+ switch (status) {
+ case 0:
+ if (expectFail) {
+ simpleMessage += red("ok (expected fail)");
+ } else {
+ simpleMessage += green("ok");
+ if (quiet) {
+ // don't print `ok` tests if --quiet is enabled
+ return;
+ }
+ }
+ break;
+ case 1:
+ if (expectFail) {
+ simpleMessage += yellow("failed (expected)");
+ } else {
+ simpleMessage += red("failed");
+ }
+ break;
+ case 2:
+ if (expectFail) {
+ simpleMessage += yellow("failed (expected)");
+ } else {
+ simpleMessage += red("failed (timeout)");
+ }
+ break;
+ case 3:
+ if (expectFail) {
+ simpleMessage += yellow("failed (expected)");
+ } else {
+ simpleMessage += red("failed (incomplete)");
+ }
+ break;
+ }
+
+ console.log(simpleMessage);
+ };
+}
+
+function discoverTestsToRun(
+ filter?: string[],
+ expectation: Expectation | string[] | boolean = getExpectation(),
+): TestToRun[] {
+ const manifestFolder = getManifest().items.testharness;
+
+ const testsToRun: TestToRun[] = [];
+
+ function walk(
+ parentFolder: ManifestFolder,
+ parentExpectation: Expectation | string[] | boolean,
+ prefix: string,
+ ) {
+ for (const key in parentFolder) {
+ const sourcePath = `${prefix}/${key}`;
+ const entry = parentFolder[key];
+ const expectation = Array.isArray(parentExpectation) ||
+ typeof parentExpectation == "boolean"
+ ? parentExpectation
+ : parentExpectation[key];
+
+ if (expectation === undefined) continue;
+
+ if (Array.isArray(entry)) {
+ assert(
+ Array.isArray(expectation) || typeof expectation == "boolean",
+ "test entry must not have a folder expectation",
+ );
+ if (
+ filter &&
+ !filter.find((filter) => sourcePath.substring(1).startsWith(filter))
+ ) {
+ continue;
+ }
+
+ for (
+ const [path, options] of entry.slice(
+ 1,
+ ) as ManifestTestVariation[]
+ ) {
+ if (!path) continue;
+ const url = new URL(path, "http://web-platform.test:8000");
+ if (!url.pathname.endsWith(".any.html")) continue;
+ testsToRun.push({
+ sourcePath,
+ path: url.pathname + url.search,
+ url,
+ options,
+ expectation,
+ });
+ }
+ } else {
+ walk(entry, expectation, sourcePath);
+ }
+ }
+ }
+ walk(manifestFolder, expectation, "");
+
+ return testsToRun;
+}
generated by cgit v1.2.3 (git 2.25.1) at 2025-07-24 19:36:45 +0000