feat(std/testing): benching progress callback (#5941)

2 files changed, 341 insertions, 36 deletions

diff --git a/std/testing/bench.ts b/std/testing/bench.ts
index 1ea07472e..1d27040f5 100644
--- a/std/testing/bench.ts
+++ b/std/testing/bench.ts
@@ -1,4 +1,5 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
+import { deepAssign } from "../_util/deep_assign.ts";
 
 const { noColor } = Deno;
 
@@ -23,30 +24,69 @@ export interface BenchmarkFunction {
 export interface BenchmarkDefinition {
   func: BenchmarkFunction;
   name: string;
+  /** Defines how many times the provided `func` should be benchmarked in succession */
   runs?: number;
 }
 
 /** Defines runBenchmark's run constraints by matching benchmark names. */
 export interface BenchmarkRunOptions {
+  /** Only benchmarks which name match this regexp will be run*/
   only?: RegExp;
+  /** Benchmarks which name match this regexp will be skipped */
   skip?: RegExp;
+  /** Setting it to true prevents default benchmarking progress logs to the commandline*/
   silent?: boolean;
 }
 
+/** Defines clearBenchmark's constraints by matching benchmark names. */
+export interface BenchmarkClearOptions {
+  /** Only benchmarks which name match this regexp will be removed */
+  only?: RegExp;
+  /** Benchmarks which name match this regexp will be kept */
+  skip?: RegExp;
+}
+
+/** Defines the result of a single benchmark */
 export interface BenchmarkResult {
+  /** The name of the benchmark */
   name: string;
+  /** The total time it took to run a given bechmark  */
   totalMs: number;
+  /** Times the benchmark was run in succession. Only defined if `runs` for this  bench is greater than 1. */
   runsCount?: number;
-  runsAvgMs?: number;
-  runsMs?: number[];
+  /** The average time of running the benchmark in milliseconds. Only defined if `runs` for this  bench is greater than 1. */
+  measuredRunsAvgMs?: number;
+  /** The individual measurements in millisecond it took to run the benchmark. Only defined if `runs` for this  bench is greater than 1. */
+  measuredRunsMs?: number[];
 }
 
+/** Defines the result of a `runBenchmarks` call */
 export interface BenchmarkRunResult {
-  measured: number;
+  /** How many benchmark were ignored by the provided `only` and `skip` */
   filtered: number;
+  /** The individual results for each benchmark that was run */
   results: BenchmarkResult[];
 }
 
+/** Defines the current progress during the run of `runBenchmarks` */
+export interface BenchmarkRunProgress extends BenchmarkRunResult {
+  /** List of the queued benchmarks to run with their name and their run count */
+  queued: Array<{ name: string; runsCount: number }>;
+  /** The currently running benchmark with its name, run count and the already finished measurements in milliseconds */
+  running?: { name: string; runsCount: number; measuredRunsMs: number[] };
+  /** Indicates in which state benchmarking currently is */
+  state: ProgressState;
+}
+
+/** Defines the states `BenchmarkRunProgress` can be in */
+export enum ProgressState {
+  BenchmarkingStart = "benchmarking_start",
+  BenchStart = "bench_start",
+  BenchPartialResult = "bench_partial_result",
+  BenchResult = "bench_result",
+  BenchmarkingEnd = "benchmarking_end",
+}
+
 export class BenchmarkRunError extends Error {
   benchmarkName?: string;
   constructor(msg: string, benchmarkName?: string) {
@@ -119,26 +159,56 @@ export function bench(
   }
 }
 
-/** Runs all registered and non-skipped benchmarks serially. */
-export async function runBenchmarks({
+/** Clears benchmark candidates which name matches `only` and doesn't match `skip`.
+ * Removes all candidates if options were not provided */
+export function clearBenchmarks({
   only = /[^\s]/,
-  skip = /^\s*$/,
-  silent,
-}: BenchmarkRunOptions = {}): Promise<BenchmarkRunResult> {
+  skip = /$^/,
+}: BenchmarkClearOptions = {}): void {
+  const keep = candidates.filter(
+    ({ name }): boolean => !only.test(name) || skip.test(name)
+  );
+  candidates.splice(0, candidates.length);
+  candidates.push(...keep);
+}
+
+/**
+ * Runs all registered and non-skipped benchmarks serially.
+ *
+ * @param [progressCb] provides the possibility to get updates of the current progress during the run of the benchmarking
+ * @returns results of the benchmarking
+ */
+export async function runBenchmarks(
+  { only = /[^\s]/, skip = /^\s*$/, silent }: BenchmarkRunOptions = {},
+  progressCb?: (progress: BenchmarkRunProgress) => void
+): Promise<BenchmarkRunResult> {
   // Filtering candidates by the "only" and "skip" constraint
   const benchmarks: BenchmarkDefinition[] = candidates.filter(
     ({ name }): boolean => only.test(name) && !skip.test(name)
   );
   // Init main counters and error flag
   const filtered = candidates.length - benchmarks.length;
-  let measured = 0;
   let failError: Error | undefined = undefined;
   // Setting up a shared benchmark clock and timer
   const clock: BenchmarkClock = { start: NaN, stop: NaN };
   const b = createBenchmarkTimer(clock);
 
+  // Init progress data
+  const progress: BenchmarkRunProgress = {
+    // bench.run is already ensured with verifyOr1Run on register
+    queued: benchmarks.map((bench) => ({
+      name: bench.name,
+      runsCount: bench.runs!,
+    })),
+    results: [],
+    filtered,
+    state: ProgressState.BenchmarkingStart,
+  };
+
+  // Publish initial progress data
+  publishProgress(progress, ProgressState.BenchmarkingStart, progressCb);
+
   if (!silent) {
-    // Iterating given benchmark definitions (await-in-loop)
     console.log(
       "running",
       benchmarks.length,
@@ -146,14 +216,25 @@ export async function runBenchmarks({
     );
   }
 
-  // Initializing results array
-  const benchmarkResults: BenchmarkResult[] = [];
+  // Iterating given benchmark definitions (await-in-loop)
   for (const { name, runs = 0, func } of benchmarks) {
     if (!silent) {
       // See https://github.com/denoland/deno/pull/1452 about groupCollapsed
       console.groupCollapsed(`benchmark ${name} ... `);
     }
 
+    // Remove benchmark from queued
+    const queueIndex = progress.queued.findIndex(
+      (queued) => queued.name === name && queued.runsCount === runs
+    );
+    if (queueIndex != -1) {
+      progress.queued.splice(queueIndex, 1);
+    }
+    // Init the progress of the running benchmark
+    progress.running = { name, runsCount: runs, measuredRunsMs: [] };
+    // Publish starting of a benchmark
+    publishProgress(progress, ProgressState.BenchStart, progressCb);
+
     // Trying benchmark.func
     let result = "";
     try {
@@ -162,38 +243,59 @@ export async function runBenchmarks({
         await func(b);
         // Making sure the benchmark was started/stopped properly
         assertTiming(clock, name);
-        result = `${clock.stop - clock.start}ms`;
+        // Calculate length of run
+        const measuredMs = clock.stop - clock.start;
+
+        result = `${measuredMs}ms`;
         // Adding one-time run to results
-        benchmarkResults.push({ name, totalMs: clock.stop - clock.start });
+        progress.results.push({ name, totalMs: measuredMs });
+        // Clear currently running
+        delete progress.running;
+        // Publish one-time run benchmark finish
+        publishProgress(progress, ProgressState.BenchResult, progressCb);
       } else if (runs > 1) {
         // Averaging runs
         let pendingRuns = runs;
         let totalMs = 0;
-        // Initializing array holding individual runs ms
-        const runsMs = [];
+
         // Would be better 2 not run these serially
         while (true) {
           // b is a benchmark timer interfacing an unset (NaN) benchmark clock
           await func(b);
           // Making sure the benchmark was started/stopped properly
           assertTiming(clock, name);
+
+          // Calculate length of run
+          const measuredMs = clock.stop - clock.start;
+
           // Summing up
-          totalMs += clock.stop - clock.start;
+          totalMs += measuredMs;
           // Adding partial result
-          runsMs.push(clock.stop - clock.start);
+          progress.running.measuredRunsMs.push(measuredMs);
+          // Publish partial benchmark results
+          publishProgress(
+            progress,
+            ProgressState.BenchPartialResult,
+            progressCb
+          );
+
           // Resetting the benchmark clock
           clock.start = clock.stop = NaN;
           // Once all ran
           if (!--pendingRuns) {
             result = `${runs} runs avg: ${totalMs / runs}ms`;
             // Adding result of multiple runs
-            benchmarkResults.push({
+            progress.results.push({
               name,
               totalMs,
               runsCount: runs,
-              runsAvgMs: totalMs / runs,
-              runsMs,
+              measuredRunsAvgMs: totalMs / runs,
+              measuredRunsMs: progress.running.measuredRunsMs,
             });
+            // Clear currently running
+            delete progress.running;
+            // Publish results of a multiple run benchmark
+            publishProgress(progress, ProgressState.BenchResult, progressCb);
             break;
           }
         }
@@ -215,29 +317,47 @@ export async function runBenchmarks({
       console.groupEnd();
     }
 
-    measured++;
     // Resetting the benchmark clock
     clock.start = clock.stop = NaN;
   }
 
+  // Indicate finished running
+  delete progress.queued;
+  // Publish final result in Cb too
+  publishProgress(progress, ProgressState.BenchmarkingEnd, progressCb);
+
   if (!silent) {
     // Closing results
     console.log(
       `benchmark result: ${!!failError ? red("FAIL") : blue("DONE")}. ` +
-        `${measured} measured; ${filtered} filtered`
+        `${progress.results.length} measured; ${filtered} filtered`
     );
   }
 
-  // Making sure the program exit code is not zero in case of failure
+  // Throw error if there was a failing benchmark
   if (!!failError) {
     throw failError;
   }
 
   const benchmarkRunResult = {
-    measured,
     filtered,
-    results: benchmarkResults,
+    results: progress.results,
   };
 
   return benchmarkRunResult;
 }
+
+function publishProgress(
+  progress: BenchmarkRunProgress,
+  state: ProgressState,
+  progressCb?: (progress: BenchmarkRunProgress) => void
+): void {
+  progressCb && progressCb(cloneProgressWithState(progress, state));
+}
+
+function cloneProgressWithState(
+  progress: BenchmarkRunProgress,
+  state: ProgressState
+): BenchmarkRunProgress {
+  return deepAssign({}, progress, { state }) as BenchmarkRunProgress;
+}
diff --git a/std/testing/bench_test.ts b/std/testing/bench_test.ts
index ef004807f..8e34b27a9 100644
--- a/std/testing/bench_test.ts
+++ b/std/testing/bench_test.ts
@@ -1,5 +1,12 @@
 const { test } = Deno;
-import { bench, runBenchmarks, BenchmarkRunError } from "./bench.ts";
+import {
+  bench,
+  runBenchmarks,
+  BenchmarkRunError,
+  clearBenchmarks,
+  BenchmarkRunProgress,
+  ProgressState,
+} from "./bench.ts";
 import {
   assertEquals,
   assert,
@@ -63,26 +70,27 @@ test({
 
     const benchResult = await runBenchmarks({ skip: /throw/ });
 
-    assertEquals(benchResult.measured, 5);
     assertEquals(benchResult.filtered, 1);
     assertEquals(benchResult.results.length, 5);
 
     const resultWithMultipleRunsFiltered = benchResult.results.filter(
-      (r) => r.name === "runs100ForIncrementX1e6"
+      ({ name }) => name === "runs100ForIncrementX1e6"
     );
     assertEquals(resultWithMultipleRunsFiltered.length, 1);
 
     const resultWithMultipleRuns = resultWithMultipleRunsFiltered[0];
     assert(!!resultWithMultipleRuns.runsCount);
-    assert(!!resultWithMultipleRuns.runsAvgMs);
-    assert(!!resultWithMultipleRuns.runsMs);
+    assert(!!resultWithMultipleRuns.measuredRunsAvgMs);
+    assert(!!resultWithMultipleRuns.measuredRunsMs);
     assertEquals(resultWithMultipleRuns.runsCount, 100);
-    assertEquals(resultWithMultipleRuns.runsMs!.length, 100);
+    assertEquals(resultWithMultipleRuns.measuredRunsMs!.length, 100);
+
+    clearBenchmarks();
   },
 });
 
 test({
-  name: "benchWithoutName",
+  name: "Bench without name should throw",
   fn() {
     assertThrows(
       (): void => {
@@ -95,7 +103,7 @@ test({
 });
 
 test({
-  name: "benchWithoutStop",
+  name: "Bench without stop should throw",
   fn: async function (): Promise<void> {
     await assertThrowsAsync(
       async (): Promise<void> => {
@@ -112,7 +120,7 @@ test({
 });
 
 test({
-  name: "benchWithoutStart",
+  name: "Bench without start should throw",
   fn: async function (): Promise<void> {
     await assertThrowsAsync(
       async (): Promise<void> => {
@@ -129,7 +137,7 @@ test({
 });
 
 test({
-  name: "benchStopBeforeStart",
+  name: "Bench with stop before start should throw",
   fn: async function (): Promise<void> {
     await assertThrowsAsync(
       async (): Promise<void> => {
@@ -145,3 +153,180 @@ test({
     );
   },
 });
+
+test({
+  name: "clearBenchmarks should clear all candidates",
+  fn: async function (): Promise<void> {
+    dummyBench("test");
+
+    clearBenchmarks();
+    const benchingResults = await runBenchmarks({ silent: true });
+
+    assertEquals(benchingResults.filtered, 0);
+    assertEquals(benchingResults.results.length, 0);
+  },
+});
+
+test({
+  name: "clearBenchmarks with only as option",
+  fn: async function (): Promise<void> {
+    // to reset candidates
+    clearBenchmarks();
+
+    dummyBench("test");
+    dummyBench("onlyclear");
+
+    clearBenchmarks({ only: /only/ });
+    const benchingResults = await runBenchmarks({ silent: true });
+
+    assertEquals(benchingResults.filtered, 0);
+    assertEquals(benchingResults.results.length, 1);
+    assertEquals(benchingResults.results[0].name, "test");
+  },
+});
+
+test({
+  name: "clearBenchmarks with skip as option",
+  fn: async function (): Promise<void> {
+    // to reset candidates
+    clearBenchmarks();
+
+    dummyBench("test");
+    dummyBench("skipclear");
+
+    clearBenchmarks({ skip: /skip/ });
+    const benchingResults = await runBenchmarks({ silent: true });
+
+    assertEquals(benchingResults.filtered, 0);
+    assertEquals(benchingResults.results.length, 1);
+    assertEquals(benchingResults.results[0].name, "skipclear");
+  },
+});
+
+test({
+  name: "clearBenchmarks with only and skip as option",
+  fn: async function (): Promise<void> {
+    // to reset candidates
+    clearBenchmarks();
+
+    dummyBench("test");
+    dummyBench("clearonly");
+    dummyBench("clearskip");
+    dummyBench("clearonly");
+
+    clearBenchmarks({ only: /clear/, skip: /skip/ });
+    const benchingResults = await runBenchmarks({ silent: true });
+
+    assertEquals(benchingResults.filtered, 0);
+    assertEquals(benchingResults.results.length, 2);
+    assert(!!benchingResults.results.find(({ name }) => name === "test"));
+    assert(!!benchingResults.results.find(({ name }) => name === "clearskip"));
+  },
+});
+
+test({
+  name: "progressCallback of runBenchmarks",
+  fn: async function (): Promise<void> {
+    clearBenchmarks();
+    dummyBench("skip");
+    dummyBench("single");
+    dummyBench("multiple", 2);
+
+    const progressCallbacks: BenchmarkRunProgress[] = [];
+
+    const benchingResults = await runBenchmarks(
+      { skip: /skip/, silent: true },
+      (progress) => {
+        // needs to be deep copied
+        progressCallbacks.push(progress);
+      }
+    );
+
+    let pc = 0;
+    // Assert initial progress before running
+    let progress = progressCallbacks[pc++];
+    assertEquals(progress.state, ProgressState.BenchmarkingStart);
+    assertEquals(progress.filtered, 1);
+    assertEquals(progress.queued.length, 2);
+    assertEquals(progress.running, undefined);
+    assertEquals(progress.results, []);
+
+    // Assert start of bench "single"
+    progress = progressCallbacks[pc++];
+    assertEquals(progress.state, ProgressState.BenchStart);
+    assertEquals(progress.filtered, 1);
+    assertEquals(progress.queued.length, 1);
+    assert(!!progress.queued.find(({ name }) => name == "multiple"));
+    assertEquals(progress.running, {
+      name: "single",
+      runsCount: 1,
+      measuredRunsMs: [],
+    });
+    assertEquals(progress.results, []);
+
+    // Assert result of bench "single"
+    progress = progressCallbacks[pc++];
+    assertEquals(progress.state, ProgressState.BenchResult);
+    assertEquals(progress.queued.length, 1);
+    assertEquals(progress.running, undefined);
+    assertEquals(progress.results.length, 1);
+    assert(!!progress.results.find(({ name }) => name == "single"));
+
+    // Assert start of bench "multiple"
+    progress = progressCallbacks[pc++];
+    assertEquals(progress.state, ProgressState.BenchStart);
+    assertEquals(progress.queued.length, 0);
+    assertEquals(progress.running, {
+      name: "multiple",
+      runsCount: 2,
+      measuredRunsMs: [],
+    });
+    assertEquals(progress.results.length, 1);
+
+    // Assert first result of bench "multiple"
+    progress = progressCallbacks[pc++];
+    assertEquals(progress.state, ProgressState.BenchPartialResult);
+    assertEquals(progress.queued.length, 0);
+    assertEquals(progress.running!.measuredRunsMs.length, 1);
+    assertEquals(progress.results.length, 1);
+
+    // Assert second result of bench "multiple"
+    progress = progressCallbacks[pc++];
+    assertEquals(progress.state, ProgressState.BenchPartialResult);
+    assertEquals(progress.queued.length, 0);
+    assertEquals(progress.running!.measuredRunsMs.length, 2);
+    assertEquals(progress.results.length, 1);
+
+    // Assert finish of bench "multiple"
+    progress = progressCallbacks[pc++];
+    assertEquals(progress.state, ProgressState.BenchResult);
+    assertEquals(progress.queued.length, 0);
+    assertEquals(progress.running, undefined);
+    assertEquals(progress.results.length, 2);
+    assert(!!progress.results.find(({ name }) => name == "single"));
+    const resultOfMultiple = progress.results.filter(
+      ({ name }) => name == "multiple"
+    );
+    assertEquals(resultOfMultiple.length, 1);
+    assert(!!resultOfMultiple[0].measuredRunsMs);
+    assert(!!resultOfMultiple[0].measuredRunsAvgMs);
+    assertEquals(resultOfMultiple[0].measuredRunsMs!.length, 2);
+
+    // The last progress should equal the final result from promise except the state property
+    progress = progressCallbacks[pc++];
+    assertEquals(progress.state, ProgressState.BenchmarkingEnd);
+    delete progress.state;
+    assertEquals(progress, benchingResults);
+  },
+});
+
+function dummyBench(name: string, runs = 1): void {
+  bench({
+    name,
+    runs,
+    func(b) {
+      b.start();
+      b.stop();
+    },
+  });
+}