1 files changed, 146 insertions, 26 deletions
diff --git a/std/testing/bench.ts b/std/testing/bench.ts
index 1ea07472e..1d27040f5 100644
--- a/std/testing/bench.ts
+++ b/std/testing/bench.ts
@@ -1,4 +1,5 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
+import { deepAssign } from "../_util/deep_assign.ts";
 
 const { noColor } = Deno;
 
@@ -23,30 +24,69 @@ export interface BenchmarkFunction {
 export interface BenchmarkDefinition {
   func: BenchmarkFunction;
   name: string;
+  /** Defines how many times the provided `func` should be benchmarked in succession */
   runs?: number;
 }
 
 /** Defines runBenchmark's run constraints by matching benchmark names. */
 export interface BenchmarkRunOptions {
+  /** Only benchmarks which name match this regexp will be run*/
   only?: RegExp;
+  /** Benchmarks which name match this regexp will be skipped */
   skip?: RegExp;
+  /** Setting it to true prevents default benchmarking progress logs to the commandline*/
   silent?: boolean;
 }
 
+/** Defines clearBenchmark's constraints by matching benchmark names. */
+export interface BenchmarkClearOptions {
+  /** Only benchmarks which name match this regexp will be removed */
+  only?: RegExp;
+  /** Benchmarks which name match this regexp will be kept */
+  skip?: RegExp;
+}
+
+/** Defines the result of a single benchmark */
 export interface BenchmarkResult {
+  /** The name of the benchmark */
   name: string;
+  /** The total time it took to run a given bechmark  */
   totalMs: number;
+  /** Times the benchmark was run in succession. Only defined if `runs` for this  bench is greater than 1. */
   runsCount?: number;
-  runsAvgMs?: number;
-  runsMs?: number[];
+  /** The average time of running the benchmark in milliseconds. Only defined if `runs` for this  bench is greater than 1. */
+  measuredRunsAvgMs?: number;
+  /** The individual measurements in millisecond it took to run the benchmark. Only defined if `runs` for this  bench is greater than 1. */
+  measuredRunsMs?: number[];
 }
 
+/** Defines the result of a `runBenchmarks` call */
 export interface BenchmarkRunResult {
-  measured: number;
+  /** How many benchmark were ignored by the provided `only` and `skip` */
   filtered: number;
+  /** The individual results for each benchmark that was run */
   results: BenchmarkResult[];
 }
 
+/** Defines the current progress during the run of `runBenchmarks` */
+export interface BenchmarkRunProgress extends BenchmarkRunResult {
+  /** List of the queued benchmarks to run with their name and their run count */
+  queued: Array<{ name: string; runsCount: number }>;
+  /** The currently running benchmark with its name, run count and the already finished measurements in milliseconds */
+  running?: { name: string; runsCount: number; measuredRunsMs: number[] };
+  /** Indicates in which state benchmarking currently is */
+  state: ProgressState;
+}
+
+/** Defines the states `BenchmarkRunProgress` can be in */
+export enum ProgressState {
+  BenchmarkingStart = "benchmarking_start",
+  BenchStart = "bench_start",
+  BenchPartialResult = "bench_partial_result",
+  BenchResult = "bench_result",
+  BenchmarkingEnd = "benchmarking_end",
+}
+
 export class BenchmarkRunError extends Error {
   benchmarkName?: string;
   constructor(msg: string, benchmarkName?: string) {
@@ -119,26 +159,56 @@ export function bench(
   }
 }
 
-/** Runs all registered and non-skipped benchmarks serially. */
-export async function runBenchmarks({
+/** Clears benchmark candidates which name matches `only` and doesn't match `skip`.
+ * Removes all candidates if options were not provided */
+export function clearBenchmarks({
   only = /[^\s]/,
-  skip = /^\s*$/,
-  silent,
-}: BenchmarkRunOptions = {}): Promise<BenchmarkRunResult> {
+  skip = /$^/,
+}: BenchmarkClearOptions = {}): void {
+  const keep = candidates.filter(
+    ({ name }): boolean => !only.test(name) || skip.test(name)
+  );
+  candidates.splice(0, candidates.length);
+  candidates.push(...keep);
+}
+
+/**
+ * Runs all registered and non-skipped benchmarks serially.
+ *
+ * @param [progressCb] provides the possibility to get updates of the current progress during the run of the benchmarking
+ * @returns results of the benchmarking
+ */
+export async function runBenchmarks(
+  { only = /[^\s]/, skip = /^\s*$/, silent }: BenchmarkRunOptions = {},
+  progressCb?: (progress: BenchmarkRunProgress) => void
+): Promise<BenchmarkRunResult> {
   // Filtering candidates by the "only" and "skip" constraint
   const benchmarks: BenchmarkDefinition[] = candidates.filter(
     ({ name }): boolean => only.test(name) && !skip.test(name)
   );
   // Init main counters and error flag
   const filtered = candidates.length - benchmarks.length;
-  let measured = 0;
   let failError: Error | undefined = undefined;
   // Setting up a shared benchmark clock and timer
   const clock: BenchmarkClock = { start: NaN, stop: NaN };
   const b = createBenchmarkTimer(clock);
 
+  // Init progress data
+  const progress: BenchmarkRunProgress = {
+    // bench.run is already ensured with verifyOr1Run on register
+    queued: benchmarks.map((bench) => ({
+      name: bench.name,
+      runsCount: bench.runs!,
+    })),
+    results: [],
+    filtered,
+    state: ProgressState.BenchmarkingStart,
+  };
+
+  // Publish initial progress data
+  publishProgress(progress, ProgressState.BenchmarkingStart, progressCb);
+
   if (!silent) {
-    // Iterating given benchmark definitions (await-in-loop)
     console.log(
       "running",
       benchmarks.length,
@@ -146,14 +216,25 @@ export async function runBenchmarks({
     );
   }
 
-  // Initializing results array
-  const benchmarkResults: BenchmarkResult[] = [];
+  // Iterating given benchmark definitions (await-in-loop)
   for (const { name, runs = 0, func } of benchmarks) {
     if (!silent) {
       // See https://github.com/denoland/deno/pull/1452 about groupCollapsed
       console.groupCollapsed(`benchmark ${name} ... `);
     }
 
+    // Remove benchmark from queued
+    const queueIndex = progress.queued.findIndex(
+      (queued) => queued.name === name && queued.runsCount === runs
+    );
+    if (queueIndex != -1) {
+      progress.queued.splice(queueIndex, 1);
+    }
+    // Init the progress of the running benchmark
+    progress.running = { name, runsCount: runs, measuredRunsMs: [] };
+    // Publish starting of a benchmark
+    publishProgress(progress, ProgressState.BenchStart, progressCb);
+
     // Trying benchmark.func
     let result = "";
     try {
@@ -162,38 +243,59 @@ export async function runBenchmarks({
         await func(b);
         // Making sure the benchmark was started/stopped properly
         assertTiming(clock, name);
-        result = `${clock.stop - clock.start}ms`;
+        // Calculate length of run
+        const measuredMs = clock.stop - clock.start;
+
+        result = `${measuredMs}ms`;
         // Adding one-time run to results
-        benchmarkResults.push({ name, totalMs: clock.stop - clock.start });
+        progress.results.push({ name, totalMs: measuredMs });
+        // Clear currently running
+        delete progress.running;
+        // Publish one-time run benchmark finish
+        publishProgress(progress, ProgressState.BenchResult, progressCb);
       } else if (runs > 1) {
         // Averaging runs
         let pendingRuns = runs;
         let totalMs = 0;
-        // Initializing array holding individual runs ms
-        const runsMs = [];
+
         // Would be better 2 not run these serially
         while (true) {
           // b is a benchmark timer interfacing an unset (NaN) benchmark clock
           await func(b);
           // Making sure the benchmark was started/stopped properly
           assertTiming(clock, name);
+
+          // Calculate length of run
+          const measuredMs = clock.stop - clock.start;
+
           // Summing up
-          totalMs += clock.stop - clock.start;
+          totalMs += measuredMs;
           // Adding partial result
-          runsMs.push(clock.stop - clock.start);
+          progress.running.measuredRunsMs.push(measuredMs);
+          // Publish partial benchmark results
+          publishProgress(
+            progress,
+            ProgressState.BenchPartialResult,
+            progressCb
+          );
+
           // Resetting the benchmark clock
           clock.start = clock.stop = NaN;
           // Once all ran
           if (!--pendingRuns) {
             result = `${runs} runs avg: ${totalMs / runs}ms`;
             // Adding result of multiple runs
-            benchmarkResults.push({
+            progress.results.push({
               name,
               totalMs,
               runsCount: runs,
-              runsAvgMs: totalMs / runs,
-              runsMs,
+              measuredRunsAvgMs: totalMs / runs,
+              measuredRunsMs: progress.running.measuredRunsMs,
             });
+            // Clear currently running
+            delete progress.running;
+            // Publish results of a multiple run benchmark
+            publishProgress(progress, ProgressState.BenchResult, progressCb);
             break;
           }
         }
@@ -215,29 +317,47 @@ export async function runBenchmarks({
       console.groupEnd();
     }
 
-    measured++;
     // Resetting the benchmark clock
     clock.start = clock.stop = NaN;
   }
 
+  // Indicate finished running
+  delete progress.queued;
+  // Publish final result in Cb too
+  publishProgress(progress, ProgressState.BenchmarkingEnd, progressCb);
+
   if (!silent) {
     // Closing results
     console.log(
       `benchmark result: ${!!failError ? red("FAIL") : blue("DONE")}. ` +
-        `${measured} measured; ${filtered} filtered`
+        `${progress.results.length} measured; ${filtered} filtered`
     );
   }
 
-  // Making sure the program exit code is not zero in case of failure
+  // Throw error if there was a failing benchmark
   if (!!failError) {
     throw failError;
   }
 
   const benchmarkRunResult = {
-    measured,
     filtered,
-    results: benchmarkResults,
+    results: progress.results,
   };
 
   return benchmarkRunResult;
 }
+
+function publishProgress(
+  progress: BenchmarkRunProgress,
+  state: ProgressState,
+  progressCb?: (progress: BenchmarkRunProgress) => void
+): void {
+  progressCb && progressCb(cloneProgressWithState(progress, state));
+}
+
+function cloneProgressWithState(
+  progress: BenchmarkRunProgress,
+  state: ProgressState
+): BenchmarkRunProgress {
+  return deepAssign({}, progress, { state }) as BenchmarkRunProgress;
+}