diff options
| -rw-r--r-- | std/testing/README.md | 4 | ||||
| -rw-r--r-- | std/testing/bench.ts | 147 | ||||
| -rw-r--r-- | std/testing/bench_example.ts | 6 | ||||
| -rw-r--r-- | std/testing/bench_test.ts | 93 |
4 files changed, 198 insertions, 52 deletions
diff --git a/std/testing/README.md b/std/testing/README.md index 66d80d4bc..4a9249205 100644 --- a/std/testing/README.md +++ b/std/testing/README.md @@ -179,10 +179,6 @@ Runs all registered benchmarks serially. Filtering can be applied by setting `BenchmarkRunOptions.only` and/or `BenchmarkRunOptions.skip` to regular expressions matching benchmark names. -##### `runIfMain(meta: ImportMeta, opts?: BenchmarkRunOptions): Promise<void>` - -Runs specified benchmarks if the enclosing script is main. - ##### Other exports ```ts diff --git a/std/testing/bench.ts b/std/testing/bench.ts index cd7b89e8c..1ea07472e 100644 --- a/std/testing/bench.ts +++ b/std/testing/bench.ts @@ -1,6 +1,6 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -const { exit, noColor } = Deno; +const { noColor } = Deno; interface BenchmarkClock { start: number; @@ -30,6 +30,30 @@ export interface BenchmarkDefinition { export interface BenchmarkRunOptions { only?: RegExp; skip?: RegExp; + silent?: boolean; +} + +export interface BenchmarkResult { + name: string; + totalMs: number; + runsCount?: number; + runsAvgMs?: number; + runsMs?: number[]; +} + +export interface BenchmarkRunResult { + measured: number; + filtered: number; + results: BenchmarkResult[]; +} + +export class BenchmarkRunError extends Error { + benchmarkName?: string; + constructor(msg: string, benchmarkName?: string) { + super(msg); + this.name = "BenchmarkRunError"; + this.benchmarkName = benchmarkName; + } } function red(text: string): string { @@ -44,16 +68,22 @@ function verifyOr1Run(runs?: number): number { return runs && runs >= 1 && runs !== Infinity ? Math.floor(runs) : 1; } -function assertTiming(clock: BenchmarkClock): void { +function assertTiming(clock: BenchmarkClock, benchmarkName: string): void { // NaN indicates that a benchmark has not been timed properly if (!clock.stop) { - throw new Error("The benchmark timer's stop method must be called"); + throw new BenchmarkRunError( + `Running benchmarks FAILED during benchmark named [${benchmarkName}]. The benchmark timer's stop method must be called`, + benchmarkName + ); } else if (!clock.start) { - throw new Error("The benchmark timer's start method must be called"); + throw new BenchmarkRunError( + `Running benchmarks FAILED during benchmark named [${benchmarkName}]. The benchmark timer's start method must be called`, + benchmarkName + ); } else if (clock.start > clock.stop) { - throw new Error( - "The benchmark timer's start method must be called before its " + - "stop method" + throw new BenchmarkRunError( + `Running benchmarks FAILED during benchmark named [${benchmarkName}]. The benchmark timer's start method must be called before its stop method`, + benchmarkName ); } } @@ -93,7 +123,8 @@ export function bench( export async function runBenchmarks({ only = /[^\s]/, skip = /^\s*$/, -}: BenchmarkRunOptions = {}): Promise<void> { + silent, +}: BenchmarkRunOptions = {}): Promise<BenchmarkRunResult> { // Filtering candidates by the "only" and "skip" constraint const benchmarks: BenchmarkDefinition[] = candidates.filter( ({ name }): boolean => only.test(name) && !skip.test(name) @@ -101,19 +132,28 @@ export async function runBenchmarks({ // Init main counters and error flag const filtered = candidates.length - benchmarks.length; let measured = 0; - let failed = false; + let failError: Error | undefined = undefined; // Setting up a shared benchmark clock and timer const clock: BenchmarkClock = { start: NaN, stop: NaN }; const b = createBenchmarkTimer(clock); - // Iterating given benchmark definitions (await-in-loop) - console.log( - "running", - benchmarks.length, - `benchmark${benchmarks.length === 1 ? " ..." : "s ..."}` - ); + + if (!silent) { + // Iterating given benchmark definitions (await-in-loop) + console.log( + "running", + benchmarks.length, + `benchmark${benchmarks.length === 1 ? " ..." : "s ..."}` + ); + } + + // Initializing results array + const benchmarkResults: BenchmarkResult[] = []; for (const { name, runs = 0, func } of benchmarks) { - // See https://github.com/denoland/deno/pull/1452 about groupCollapsed - console.groupCollapsed(`benchmark ${name} ... `); + if (!silent) { + // See https://github.com/denoland/deno/pull/1452 about groupCollapsed + console.groupCollapsed(`benchmark ${name} ... `); + } + // Trying benchmark.func let result = ""; try { @@ -121,60 +161,83 @@ export async function runBenchmarks({ // b is a benchmark timer interfacing an unset (NaN) benchmark clock await func(b); // Making sure the benchmark was started/stopped properly - assertTiming(clock); + assertTiming(clock, name); result = `${clock.stop - clock.start}ms`; + // Adding one-time run to results + benchmarkResults.push({ name, totalMs: clock.stop - clock.start }); } else if (runs > 1) { // Averaging runs let pendingRuns = runs; let totalMs = 0; + // Initializing array holding individual runs ms + const runsMs = []; // Would be better 2 not run these serially while (true) { // b is a benchmark timer interfacing an unset (NaN) benchmark clock await func(b); // Making sure the benchmark was started/stopped properly - assertTiming(clock); + assertTiming(clock, name); // Summing up totalMs += clock.stop - clock.start; + // Adding partial result + runsMs.push(clock.stop - clock.start); // Resetting the benchmark clock clock.start = clock.stop = NaN; // Once all ran if (!--pendingRuns) { result = `${runs} runs avg: ${totalMs / runs}ms`; + // Adding result of multiple runs + benchmarkResults.push({ + name, + totalMs, + runsCount: runs, + runsAvgMs: totalMs / runs, + runsMs, + }); break; } } } } catch (err) { - failed = true; - console.groupEnd(); - console.error(red(err.stack)); + failError = err; + + if (!silent) { + console.groupEnd(); + console.error(red(err.stack)); + } + break; } - // Reporting - console.log(blue(result)); - console.groupEnd(); + + if (!silent) { + // Reporting + console.log(blue(result)); + console.groupEnd(); + } + measured++; // Resetting the benchmark clock clock.start = clock.stop = NaN; } - // Closing results - console.log( - `benchmark result: ${failed ? red("FAIL") : blue("DONE")}. ` + - `${measured} measured; ${filtered} filtered` - ); - // Making sure the program exit code is not zero in case of failure - if (failed) { - setTimeout((): void => exit(1), 0); + + if (!silent) { + // Closing results + console.log( + `benchmark result: ${!!failError ? red("FAIL") : blue("DONE")}. ` + + `${measured} measured; ${filtered} filtered` + ); } -} -/** Runs specified benchmarks if the enclosing script is main. */ -export function runIfMain( - meta: ImportMeta, - opts: BenchmarkRunOptions = {} -): Promise<void> { - if (meta.main) { - return runBenchmarks(opts); + // Making sure the program exit code is not zero in case of failure + if (!!failError) { + throw failError; } - return Promise.resolve(undefined); + + const benchmarkRunResult = { + measured, + filtered, + results: benchmarkResults, + }; + + return benchmarkRunResult; } diff --git a/std/testing/bench_example.ts b/std/testing/bench_example.ts index 401516cca..366521f85 100644 --- a/std/testing/bench_example.ts +++ b/std/testing/bench_example.ts @@ -1,5 +1,5 @@ // https://deno.land/std/testing/bench.ts -import { BenchmarkTimer, bench, runIfMain } from "./bench.ts"; +import { BenchmarkTimer, bench, runBenchmarks } from "./bench.ts"; // Basic bench(function forIncrementX1e9(b: BenchmarkTimer): void { @@ -26,4 +26,6 @@ bench(function throwing(b): void { }); // Bench control -runIfMain(import.meta, { skip: /throw/ }); +if (import.meta.main) { + runBenchmarks({ skip: /throw/ }); +} diff --git a/std/testing/bench_test.ts b/std/testing/bench_test.ts index 6dfc18b10..ef004807f 100644 --- a/std/testing/bench_test.ts +++ b/std/testing/bench_test.ts @@ -1,7 +1,11 @@ const { test } = Deno; -import { bench, runBenchmarks } from "./bench.ts"; - -import "./bench_example.ts"; +import { bench, runBenchmarks, BenchmarkRunError } from "./bench.ts"; +import { + assertEquals, + assert, + assertThrows, + assertThrowsAsync, +} from "./asserts.ts"; test({ name: "benching", @@ -57,6 +61,87 @@ test({ // Throws bc the timer's stop method is never called }); - await runBenchmarks({ skip: /throw/ }); + const benchResult = await runBenchmarks({ skip: /throw/ }); + + assertEquals(benchResult.measured, 5); + assertEquals(benchResult.filtered, 1); + assertEquals(benchResult.results.length, 5); + + const resultWithMultipleRunsFiltered = benchResult.results.filter( + (r) => r.name === "runs100ForIncrementX1e6" + ); + assertEquals(resultWithMultipleRunsFiltered.length, 1); + + const resultWithMultipleRuns = resultWithMultipleRunsFiltered[0]; + assert(!!resultWithMultipleRuns.runsCount); + assert(!!resultWithMultipleRuns.runsAvgMs); + assert(!!resultWithMultipleRuns.runsMs); + assertEquals(resultWithMultipleRuns.runsCount, 100); + assertEquals(resultWithMultipleRuns.runsMs!.length, 100); + }, +}); + +test({ + name: "benchWithoutName", + fn() { + assertThrows( + (): void => { + bench(() => {}); + }, + Error, + "The benchmark function must not be anonymous" + ); + }, +}); + +test({ + name: "benchWithoutStop", + fn: async function (): Promise<void> { + await assertThrowsAsync( + async (): Promise<void> => { + bench(function benchWithoutStop(b): void { + b.start(); + // Throws bc the timer's stop method is never called + }); + await runBenchmarks({ only: /benchWithoutStop/, silent: true }); + }, + BenchmarkRunError, + "The benchmark timer's stop method must be called" + ); + }, +}); + +test({ + name: "benchWithoutStart", + fn: async function (): Promise<void> { + await assertThrowsAsync( + async (): Promise<void> => { + bench(function benchWithoutStart(b): void { + b.stop(); + // Throws bc the timer's start method is never called + }); + await runBenchmarks({ only: /benchWithoutStart/, silent: true }); + }, + BenchmarkRunError, + "The benchmark timer's start method must be called" + ); + }, +}); + +test({ + name: "benchStopBeforeStart", + fn: async function (): Promise<void> { + await assertThrowsAsync( + async (): Promise<void> => { + bench(function benchStopBeforeStart(b): void { + b.stop(); + b.start(); + // Throws bc the timer's stop is called before start + }); + await runBenchmarks({ only: /benchStopBeforeStart/, silent: true }); + }, + BenchmarkRunError, + "The benchmark timer's start method must be called before its stop method" + ); }, }); |