diff options
Diffstat (limited to 'cli/js/40_testing.js')
| -rw-r--r-- | cli/js/40_testing.js | 335 |
1 files changed, 87 insertions, 248 deletions
diff --git a/cli/js/40_testing.js b/cli/js/40_testing.js index bcb5990a9..74fbb8da3 100644 --- a/cli/js/40_testing.js +++ b/cli/js/40_testing.js @@ -142,7 +142,8 @@ function assertOps(fn) { const pre = core.metrics(); const preTraces = new Map(core.opCallTraces); try { - await fn(desc); + const innerResult = await fn(desc); + if (innerResult) return innerResult; } finally { // Defer until next event loop turn - that way timeouts and intervals // cleared can actually be removed from resource table, otherwise @@ -150,9 +151,6 @@ function assertOps(fn) { await opSanitizerDelay(); await opSanitizerDelay(); } - - if (shouldSkipSanitizers(desc)) return; - const post = core.metrics(); const postTraces = new Map(core.opCallTraces); @@ -161,7 +159,7 @@ function assertOps(fn) { const dispatchedDiff = post.opsDispatchedAsync - pre.opsDispatchedAsync; const completedDiff = post.opsCompletedAsync - pre.opsCompletedAsync; - if (dispatchedDiff === completedDiff) return; + if (dispatchedDiff === completedDiff) return null; const details = []; for (const key in post.ops) { @@ -215,19 +213,7 @@ function assertOps(fn) { ); } } - - let msg = `Test case is leaking async ops. - - - ${ArrayPrototypeJoin(details, "\n - ")}`; - - if (!core.isOpCallTracingEnabled()) { - msg += - `\n\nTo get more details where ops were leaked, run again with --trace-ops flag.`; - } else { - msg += "\n"; - } - - throw assert(false, msg); + return { failed: { leakedOps: [details, core.isOpCallTracingEnabled()] } }; }; } @@ -372,12 +358,8 @@ function assertResources(fn) { /** @param desc {TestDescription | TestStepDescription} */ return async function resourceSanitizer(desc) { const pre = core.resources(); - await fn(desc); - - if (shouldSkipSanitizers(desc)) { - return; - } - + const innerResult = await fn(desc); + if (innerResult) return innerResult; const post = core.resources(); const allResources = new Set([ @@ -404,14 +386,10 @@ function assertResources(fn) { ArrayPrototypePush(details, detail); } } - - const message = `Test case is leaking ${details.length} resource${ - details.length === 1 ? "" : "s" - }: - - - ${details.join("\n - ")} -`; - assert(details.length === 0, message); + if (details.length == 0) { + return null; + } + return { failed: { leakedResources: details } }; }; } @@ -429,9 +407,8 @@ function assertExit(fn, isTest) { }); try { - await fn(...new SafeArrayIterator(params)); - } catch (err) { - throw err; + const innerResult = await fn(...new SafeArrayIterator(params)); + if (innerResult) return innerResult; } finally { setExitHandler(null); } @@ -441,79 +418,52 @@ function assertExit(fn, isTest) { function assertTestStepScopes(fn) { /** @param desc {TestDescription | TestStepDescription} */ return async function testStepSanitizer(desc) { - preValidation(); - // only report waiting after pre-validation - if (canStreamReporting(desc) && "parent" in desc) { - stepReportWait(desc); - } - await fn(MapPrototypeGet(testStates, desc.id).context); - testStepPostValidation(desc); - - function preValidation() { - const runningStepDescs = getRunningStepDescs(); - const runningStepDescsWithSanitizers = ArrayPrototypeFilter( - runningStepDescs, - (d) => usesSanitizer(d), - ); - - if (runningStepDescsWithSanitizers.length > 0) { - throw new Error( - "Cannot start test step while another test step with sanitizers is running.\n" + - runningStepDescsWithSanitizers - .map((d) => ` * ${getFullName(d)}`) - .join("\n"), - ); - } - - if (usesSanitizer(desc) && runningStepDescs.length > 0) { - throw new Error( - "Cannot start test step with sanitizers while another test step is running.\n" + - runningStepDescs.map((d) => ` * ${getFullName(d)}`).join("\n"), - ); - } - - function getRunningStepDescs() { - const results = []; - let childDesc = desc; - while (childDesc.parent != null) { - const state = MapPrototypeGet(testStates, childDesc.parent.id); - for (const siblingDesc of state.children) { - if (siblingDesc.id == childDesc.id) { - continue; - } - const siblingState = MapPrototypeGet(testStates, siblingDesc.id); - if (!siblingState.finalized) { - ArrayPrototypePush(results, siblingDesc); - } + function getRunningStepDescs() { + const results = []; + let childDesc = desc; + while (childDesc.parent != null) { + const state = MapPrototypeGet(testStates, childDesc.parent.id); + for (const siblingDesc of state.children) { + if (siblingDesc.id == childDesc.id) { + continue; + } + const siblingState = MapPrototypeGet(testStates, siblingDesc.id); + if (!siblingState.completed) { + ArrayPrototypePush(results, siblingDesc); } - childDesc = childDesc.parent; } - return results; + childDesc = childDesc.parent; } + return results; } - }; -} + const runningStepDescs = getRunningStepDescs(); + const runningStepDescsWithSanitizers = ArrayPrototypeFilter( + runningStepDescs, + (d) => usesSanitizer(d), + ); -function testStepPostValidation(desc) { - // check for any running steps - for (const childDesc of MapPrototypeGet(testStates, desc.id).children) { - if (MapPrototypeGet(testStates, childDesc.id).status == "pending") { - throw new Error( - "There were still test steps running after the current scope finished execution. Ensure all steps are awaited (ex. `await t.step(...)`).", - ); + if (runningStepDescsWithSanitizers.length > 0) { + return { + failed: { + overlapsWithSanitizers: runningStepDescsWithSanitizers.map( + getFullName, + ), + }, + }; } - } - // check if an ancestor already completed - let currentDesc = desc.parent; - while (currentDesc != null) { - if (MapPrototypeGet(testStates, currentDesc.id).finalized) { - throw new Error( - "Parent scope completed before test step finished execution. Ensure all steps are awaited (ex. `await t.step(...)`).", - ); + if (usesSanitizer(desc) && runningStepDescs.length > 0) { + return { + failed: { hasSanitizersAndOverlaps: runningStepDescs.map(getFullName) }, + }; } - currentDesc = currentDesc.parent; - } + await fn(MapPrototypeGet(testStates, desc.id).context); + for (const childDesc of MapPrototypeGet(testStates, desc.id).children) { + if (!MapPrototypeGet(testStates, childDesc.id).completed) { + return { failed: "incompleteSteps" }; + } + } + }; } function pledgePermissions(permissions) { @@ -573,18 +523,14 @@ function withPermissions(fn, permissions) { * @typedef {{ * context: TestContext, * children: TestStepDescription[], - * finalized: boolean, + * completed: boolean, * }} TestState * * @typedef {{ * context: TestContext, * children: TestStepDescription[], - * finalized: boolean, - * status: "pending" | "ok" | ""failed" | ignored", - * error: unknown, - * elapsed: number | null, - * reportedWait: boolean, - * reportedResult: boolean, + * completed: boolean, + * failed: boolean, * }} TestStepState * * @typedef {{ @@ -701,13 +647,6 @@ function test( // Delete this prop in case the user passed it. It's used to detect steps. delete testDesc.parent; - testDesc.fn = wrapTestFnWithSanitizers(testDesc.fn, testDesc); - if (testDesc.permissions) { - testDesc.fn = withPermissions( - testDesc.fn, - testDesc.permissions, - ); - } testDesc.origin = getTestOrigin(); const jsError = core.destructureError(new Error()); testDesc.location = { @@ -724,7 +663,7 @@ function test( MapPrototypeSet(testStates, testDesc.id, { context: createTestContext(testDesc), children: [], - finalized: false, + completed: false, }); } @@ -832,28 +771,20 @@ async function runTest(desc) { if (desc.ignore) { return "ignored"; } - + let testFn = wrapTestFnWithSanitizers(desc.fn, desc); + if (!("parent" in desc) && desc.permissions) { + testFn = withPermissions( + testFn, + desc.permissions, + ); + } try { - await desc.fn(desc); - const failCount = failedChildStepsCount(desc); - return failCount === 0 ? "ok" : { - "failed": core.destructureError( - new Error( - `${failCount} test step${failCount === 1 ? "" : "s"} failed.`, - ), - ), - }; + const result = await testFn(desc); + if (result) return result; + const failedSteps = failedChildStepsCount(desc); + return failedSteps === 0 ? "ok" : { failed: { failedSteps } }; } catch (error) { - return { - "failed": core.destructureError(error), - }; - } finally { - const state = MapPrototypeGet(testStates, desc.id); - state.finalized = true; - // ensure the children report their result - for (const childDesc of state.children) { - stepReportResult(childDesc); - } + return { failed: { jsError: core.destructureError(error) } }; } } @@ -1094,6 +1025,11 @@ async function runTests({ const earlier = DateNow(); const result = await runTest(desc); const elapsed = DateNow() - earlier; + const state = MapPrototypeGet(testStates, desc.id); + state.completed = true; + for (const childDesc of state.children) { + stepReportResult(childDesc, { failed: "incomplete" }, 0); + } ops.op_dispatch_test_event({ result: [desc.id, result, elapsed], }); @@ -1153,7 +1089,7 @@ async function runBenchmarks() { function getFullName(desc) { if ("parent" in desc) { - return `${desc.parent.name} > ${desc.name}`; + return `${getFullName(desc.parent)} ... ${desc.name}`; } return desc.name; } @@ -1162,74 +1098,23 @@ function usesSanitizer(desc) { return desc.sanitizeResources || desc.sanitizeOps || desc.sanitizeExit; } -function canStreamReporting(desc) { - let currentDesc = desc; - while (currentDesc != null) { - if (!usesSanitizer(currentDesc)) { - return false; - } - currentDesc = currentDesc.parent; - } - for (const childDesc of MapPrototypeGet(testStates, desc.id).children) { - const state = MapPrototypeGet(testStates, childDesc.id); - if (!usesSanitizer(childDesc) && !state.finalized) { - return false; - } - } - return true; -} - -function stepReportWait(desc) { - const state = MapPrototypeGet(testStates, desc.id); - if (state.reportedWait) { - return; - } - ops.op_dispatch_test_event({ stepWait: desc.id }); - state.reportedWait = true; -} - -function stepReportResult(desc) { +function stepReportResult(desc, result, elapsed) { const state = MapPrototypeGet(testStates, desc.id); - if (state.reportedResult) { - return; - } - stepReportWait(desc); for (const childDesc of state.children) { - stepReportResult(childDesc); - } - let result; - if (state.status == "pending" || state.status == "failed") { - result = { - [state.status]: state.error && core.destructureError(state.error), - }; - } else { - result = state.status; + stepReportResult(childDesc, { failed: "incomplete" }, 0); } ops.op_dispatch_test_event({ - stepResult: [desc.id, result, state.elapsed], + stepResult: [desc.id, result, elapsed], }); - state.reportedResult = true; } function failedChildStepsCount(desc) { return ArrayPrototypeFilter( MapPrototypeGet(testStates, desc.id).children, - (d) => MapPrototypeGet(testStates, d.id).status === "failed", + (d) => MapPrototypeGet(testStates, d.id).failed, ).length; } -/** If a test validation error already occurred then don't bother checking - * the sanitizers as that will create extra noise. - */ -function shouldSkipSanitizers(desc) { - try { - testStepPostValidation(desc); - return false; - } catch { - return true; - } -} - /** @param desc {TestDescription | TestStepDescription} */ function createTestContext(desc) { let parent; @@ -1266,7 +1151,7 @@ function createTestContext(desc) { * @param maybeFn {((t: TestContext) => void | Promise<void>) | undefined} */ async step(nameOrFnOrOptions, maybeFn) { - if (MapPrototypeGet(testStates, desc.id).finalized) { + if (MapPrototypeGet(testStates, desc.id).completed) { throw new Error( "Cannot run test step after parent scope has finished execution. " + "Ensure any `.step(...)` calls are executed before their parent scope completes execution.", @@ -1322,12 +1207,8 @@ function createTestContext(desc) { const state = { context: createTestContext(stepDesc), children: [], - finalized: false, - status: "pending", - error: null, - elapsed: null, - reportedWait: false, - reportedResult: false, + failed: false, + completed: false, }; MapPrototypeSet(testStates, stepDesc.id, state); ArrayPrototypePush( @@ -1335,56 +1216,14 @@ function createTestContext(desc) { stepDesc, ); - try { - if (stepDesc.ignore) { - state.status = "ignored"; - state.finalized = true; - if (canStreamReporting(stepDesc)) { - stepReportResult(stepDesc); - } - return false; - } - - const testFn = wrapTestFnWithSanitizers(stepDesc.fn, stepDesc); - const start = DateNow(); - - try { - await testFn(stepDesc); - - if (failedChildStepsCount(stepDesc) > 0) { - state.status = "failed"; - } else { - state.status = "ok"; - } - } catch (error) { - state.error = error; - state.status = "failed"; - } - - state.elapsed = DateNow() - start; - - if (MapPrototypeGet(testStates, stepDesc.parent.id).finalized) { - // always point this test out as one that was still running - // if the parent step finalized - state.status = "pending"; - } - - state.finalized = true; - - if (state.reportedWait && canStreamReporting(stepDesc)) { - stepReportResult(stepDesc); - } - - return state.status === "ok"; - } finally { - if (canStreamReporting(stepDesc.parent)) { - const parentState = MapPrototypeGet(testStates, stepDesc.parent.id); - // flush any buffered steps - for (const childDesc of parentState.children) { - stepReportResult(childDesc); - } - } - } + ops.op_dispatch_test_event({ stepWait: stepDesc.id }); + const earlier = DateNow(); + const result = await runTest(stepDesc); + const elapsed = DateNow() - earlier; + state.failed = !!result.failed; + state.completed = true; + stepReportResult(stepDesc, result, elapsed); + return result == "ok"; }, }; } |