diff options
Diffstat (limited to 'tools')
| -rwxr-xr-x | tools/wpt.ts | 60 | ||||
| -rw-r--r-- | tools/wpt/runner.ts | 18 | ||||
| -rw-r--r-- | tools/wpt/testharnessreport.js | 9 | ||||
| -rw-r--r-- | tools/wpt/utils.ts | 54 |
4 files changed, 139 insertions, 2 deletions
diff --git a/tools/wpt.ts b/tools/wpt.ts index e6f08dd34..8f8248880 100755 --- a/tools/wpt.ts +++ b/tools/wpt.ts @@ -15,6 +15,7 @@ import { cargoBuild, checkPy3Available, Expectation, + generateRunInfo, getExpectation, getExpectFailForCase, getManifest, @@ -26,6 +27,7 @@ import { rest, runPy, updateManifest, + wptreport, } from "./wpt/utils.ts"; import { blue, @@ -148,6 +150,7 @@ interface TestToRun { } async function run() { + const startTime = new Date().getTime(); assert(Array.isArray(rest), "filter must be array"); const expectation = getExpectation(); const tests = discoverTestsToRun( @@ -173,6 +176,7 @@ async function run() { return results; }); + const endTime = new Date().getTime(); if (json) { const minifiedResults = []; @@ -191,10 +195,66 @@ async function run() { } await Deno.writeTextFile(json, JSON.stringify(minifiedResults)); } + + if (wptreport) { + const report = await generateWptreport(results, startTime, endTime); + await Deno.writeTextFile(wptreport, JSON.stringify(report)); + } + const code = reportFinal(results); Deno.exit(code); } +async function generateWptreport( + results: { test: TestToRun; result: TestResult }[], + startTime: number, + endTime: number, +) { + const runInfo = await generateRunInfo(); + const reportResults = []; + for (const { test, result } of results) { + const status = result.status !== 0 + ? "CRASH" + : result.harnessStatus?.status === 0 + ? "OK" + : "ERROR"; + const reportResult = { + test: test.url.pathname + test.url.search + test.url.hash, + subtests: result.cases.map((case_) => { + let expected = undefined; + if (!case_.passed) { + if (typeof test.expectation === "boolean") { + expected = test.expectation ? "PASS" : "FAIL"; + } else { + expected = test.expectation.includes(case_.name) ? "FAIL" : "PASS"; + } + } + + return { + name: case_.name, + status: case_.passed ? "PASS" : "FAIL", + message: case_.message, + expected, + known_intermittent: [], + }; + }), + status, + message: result.harnessStatus?.message ?? + (result.stderr.trim() || null), + duration: result.duration, + expected: status === "OK" ? undefined : "OK", + "known_intermittent": [], + }; + reportResults.push(reportResult); + } + return { + "run_info": runInfo, + "time_start": startTime, + "time_end": endTime, + "results": reportResults, + }; +} + // Check that all expectations in the expectations file have a test that will be // run. function assertAllExpectationsHaveTests( diff --git a/tools/wpt/runner.ts b/tools/wpt/runner.ts index a0941b521..dcc88a123 100644 --- a/tools/wpt/runner.ts +++ b/tools/wpt/runner.ts @@ -47,10 +47,18 @@ export async function runWithTestUtil<T>( export interface TestResult { cases: TestCaseResult[]; + harnessStatus: TestHarnessStatus | null; + duration: number; status: number; stderr: string; } +export interface TestHarnessStatus { + status: number; + message: string | null; + stack: string | null; +} + export interface TestCaseResult { name: string; passed: boolean; @@ -71,6 +79,8 @@ export async function runSingleTest( }); await Deno.writeTextFile(tempFile, bundle); + const startTime = new Date().getTime(); + const proc = Deno.run({ cmd: [ join(ROOT_PATH, `./target/${release ? "release" : "debug"}/deno`), @@ -94,6 +104,8 @@ export async function runSingleTest( const cases = []; let stderr = ""; + let harnessStatus = null; + const lines = readLines(proc.stderr); for await (const line of lines) { if (line.startsWith("{")) { @@ -101,15 +113,21 @@ export async function runSingleTest( const result = { ...data, passed: data.status == 0 }; cases.push(result); reporter(result); + } else if (line.startsWith("#$#$#{")) { + harnessStatus = JSON.parse(line.slice(5)); } else { stderr += line + "\n"; console.error(stderr); } } + const duration = new Date().getTime() - startTime; + const { code } = await proc.status(); return { status: code, + harnessStatus, + duration, cases, stderr, }; diff --git a/tools/wpt/testharnessreport.js b/tools/wpt/testharnessreport.js index d3e783376..04251c852 100644 --- a/tools/wpt/testharnessreport.js +++ b/tools/wpt/testharnessreport.js @@ -10,6 +10,13 @@ window.add_result_callback(({ message, name, stack, status }) => { } }); -window.add_completion_callback((_tests, _harnessStatus) => { +window.add_completion_callback((_tests, harnessStatus) => { + const data = new TextEncoder().encode( + `#$#$#${JSON.stringify(harnessStatus)}\n`, + ); + let bytesWritten = 0; + while (bytesWritten < data.byteLength) { + bytesWritten += Deno.stderr.writeSync(data.subarray(bytesWritten)); + } Deno.exit(0); }); diff --git a/tools/wpt/utils.ts b/tools/wpt/utils.ts index cb454300b..3b1eb9965 100644 --- a/tools/wpt/utils.ts +++ b/tools/wpt/utils.ts @@ -6,6 +6,7 @@ import { join, ROOT_PATH } from "../util.js"; export const { json, + wptreport, quiet, release, rebuild, @@ -14,7 +15,7 @@ export const { } = parse(Deno.args, { "--": true, boolean: ["quiet", "release", "no-interactive"], - string: ["json"], + string: ["json", "wptreport"], }); /// PAGE ROOT @@ -145,3 +146,54 @@ export async function cargoBuild() { proc.close(); assert(status.success, "cargo build failed"); } + +/// WPTREPORT + +export async function generateRunInfo(): Promise<unknown> { + const oses = { + "windows": "win", + "darwin": "mac", + "linux": "linux", + }; + const proc = Deno.run({ + cmd: ["git", "rev-parse", "HEAD"], + cwd: join(ROOT_PATH, "test_util", "wpt"), + stdout: "piped", + }); + await proc.status(); + const revision = (new TextDecoder().decode(await proc.output())).trim(); + proc.close(); + const proc2 = Deno.run({ + cmd: [ + join(ROOT_PATH, `./target/${release ? "release" : "debug"}/deno`), + "eval", + "console.log(JSON.stringify(Deno.version))", + ], + cwd: join(ROOT_PATH, "test_util", "wpt"), + stdout: "piped", + }); + await proc2.status(); + const version = JSON.parse(new TextDecoder().decode(await proc2.output())); + proc2.close(); + const runInfo = { + "os": oses[Deno.build.os], + "processor": Deno.build.arch, + "version": "unknown", + "os_version": "unknown", + "bits": 64, + "has_sandbox": true, + "webrender": false, + "automation": false, + "linux_distro": "unknown", + "revision": revision, + "python_version": 3, + "product": "deno", + "debug": false, + "browser_version": version.deno, + "browser_channel": version.deno.includes("+") ? "canary" : "stable", + "verify": false, + "wasm": false, + "headless": true, + }; + return runInfo; +} |