Return results in benchmark promise (#5842)

4 files changed, 198 insertions, 52 deletions

diff --git a/std/testing/README.md b/std/testing/README.md
index 66d80d4bc..4a9249205 100644
--- a/std/testing/README.md
+++ b/std/testing/README.md
@@ -179,10 +179,6 @@ Runs all registered benchmarks serially. Filtering can be applied by setting
 `BenchmarkRunOptions.only` and/or `BenchmarkRunOptions.skip` to regular
 expressions matching benchmark names.
 
-##### `runIfMain(meta: ImportMeta, opts?: BenchmarkRunOptions): Promise<void>`
-
-Runs specified benchmarks if the enclosing script is main.
-
 ##### Other exports
 
 ```ts
diff --git a/std/testing/bench.ts b/std/testing/bench.ts
index cd7b89e8c..1ea07472e 100644
--- a/std/testing/bench.ts
+++ b/std/testing/bench.ts
@@ -1,6 +1,6 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
 
-const { exit, noColor } = Deno;
+const { noColor } = Deno;
 
 interface BenchmarkClock {
   start: number;
@@ -30,6 +30,30 @@ export interface BenchmarkDefinition {
 export interface BenchmarkRunOptions {
   only?: RegExp;
   skip?: RegExp;
+  silent?: boolean;
+}
+
+export interface BenchmarkResult {
+  name: string;
+  totalMs: number;
+  runsCount?: number;
+  runsAvgMs?: number;
+  runsMs?: number[];
+}
+
+export interface BenchmarkRunResult {
+  measured: number;
+  filtered: number;
+  results: BenchmarkResult[];
+}
+
+export class BenchmarkRunError extends Error {
+  benchmarkName?: string;
+  constructor(msg: string, benchmarkName?: string) {
+    super(msg);
+    this.name = "BenchmarkRunError";
+    this.benchmarkName = benchmarkName;
+  }
 }
 
 function red(text: string): string {
@@ -44,16 +68,22 @@ function verifyOr1Run(runs?: number): number {
   return runs && runs >= 1 && runs !== Infinity ? Math.floor(runs) : 1;
 }
 
-function assertTiming(clock: BenchmarkClock): void {
+function assertTiming(clock: BenchmarkClock, benchmarkName: string): void {
   // NaN indicates that a benchmark has not been timed properly
   if (!clock.stop) {
-    throw new Error("The benchmark timer's stop method must be called");
+    throw new BenchmarkRunError(
+      `Running benchmarks FAILED during benchmark named [${benchmarkName}]. The benchmark timer's stop method must be called`,
+      benchmarkName
+    );
   } else if (!clock.start) {
-    throw new Error("The benchmark timer's start method must be called");
+    throw new BenchmarkRunError(
+      `Running benchmarks FAILED during benchmark named [${benchmarkName}]. The benchmark timer's start method must be called`,
+      benchmarkName
+    );
   } else if (clock.start > clock.stop) {
-    throw new Error(
-      "The benchmark timer's start method must be called before its " +
-        "stop method"
+    throw new BenchmarkRunError(
+      `Running benchmarks FAILED during benchmark named [${benchmarkName}]. The benchmark timer's start method must be called before its stop method`,
+      benchmarkName
     );
   }
 }
@@ -93,7 +123,8 @@ export function bench(
 export async function runBenchmarks({
   only = /[^\s]/,
   skip = /^\s*$/,
-}: BenchmarkRunOptions = {}): Promise<void> {
+  silent,
+}: BenchmarkRunOptions = {}): Promise<BenchmarkRunResult> {
   // Filtering candidates by the "only" and "skip" constraint
   const benchmarks: BenchmarkDefinition[] = candidates.filter(
     ({ name }): boolean => only.test(name) && !skip.test(name)
@@ -101,19 +132,28 @@ export async function runBenchmarks({
   // Init main counters and error flag
   const filtered = candidates.length - benchmarks.length;
   let measured = 0;
-  let failed = false;
+  let failError: Error | undefined = undefined;
   // Setting up a shared benchmark clock and timer
   const clock: BenchmarkClock = { start: NaN, stop: NaN };
   const b = createBenchmarkTimer(clock);
-  // Iterating given benchmark definitions (await-in-loop)
-  console.log(
-    "running",
-    benchmarks.length,
-    `benchmark${benchmarks.length === 1 ? " ..." : "s ..."}`
-  );
+
+  if (!silent) {
+    // Iterating given benchmark definitions (await-in-loop)
+    console.log(
+      "running",
+      benchmarks.length,
+      `benchmark${benchmarks.length === 1 ? " ..." : "s ..."}`
+    );
+  }
+
+  // Initializing results array
+  const benchmarkResults: BenchmarkResult[] = [];
   for (const { name, runs = 0, func } of benchmarks) {
-    // See https://github.com/denoland/deno/pull/1452 about groupCollapsed
-    console.groupCollapsed(`benchmark ${name} ... `);
+    if (!silent) {
+      // See https://github.com/denoland/deno/pull/1452 about groupCollapsed
+      console.groupCollapsed(`benchmark ${name} ... `);
+    }
+
     // Trying benchmark.func
     let result = "";
     try {
@@ -121,60 +161,83 @@ export async function runBenchmarks({
         // b is a benchmark timer interfacing an unset (NaN) benchmark clock
         await func(b);
         // Making sure the benchmark was started/stopped properly
-        assertTiming(clock);
+        assertTiming(clock, name);
         result = `${clock.stop - clock.start}ms`;
+        // Adding one-time run to results
+        benchmarkResults.push({ name, totalMs: clock.stop - clock.start });
       } else if (runs > 1) {
         // Averaging runs
         let pendingRuns = runs;
         let totalMs = 0;
+        // Initializing array holding individual runs ms
+        const runsMs = [];
         // Would be better 2 not run these serially
         while (true) {
           // b is a benchmark timer interfacing an unset (NaN) benchmark clock
           await func(b);
           // Making sure the benchmark was started/stopped properly
-          assertTiming(clock);
+          assertTiming(clock, name);
           // Summing up
           totalMs += clock.stop - clock.start;
+          // Adding partial result
+          runsMs.push(clock.stop - clock.start);
           // Resetting the benchmark clock
           clock.start = clock.stop = NaN;
           // Once all ran
           if (!--pendingRuns) {
             result = `${runs} runs avg: ${totalMs / runs}ms`;
+            // Adding result of multiple runs
+            benchmarkResults.push({
+              name,
+              totalMs,
+              runsCount: runs,
+              runsAvgMs: totalMs / runs,
+              runsMs,
+            });
             break;
           }
         }
       }
     } catch (err) {
-      failed = true;
-      console.groupEnd();
-      console.error(red(err.stack));
+      failError = err;
+
+      if (!silent) {
+        console.groupEnd();
+        console.error(red(err.stack));
+      }
+
       break;
     }
-    // Reporting
-    console.log(blue(result));
-    console.groupEnd();
+
+    if (!silent) {
+      // Reporting
+      console.log(blue(result));
+      console.groupEnd();
+    }
+
     measured++;
     // Resetting the benchmark clock
     clock.start = clock.stop = NaN;
   }
-  // Closing results
-  console.log(
-    `benchmark result: ${failed ? red("FAIL") : blue("DONE")}. ` +
-      `${measured} measured; ${filtered} filtered`
-  );
-  // Making sure the program exit code is not zero in case of failure
-  if (failed) {
-    setTimeout((): void => exit(1), 0);
+
+  if (!silent) {
+    // Closing results
+    console.log(
+      `benchmark result: ${!!failError ? red("FAIL") : blue("DONE")}. ` +
+        `${measured} measured; ${filtered} filtered`
+    );
   }
-}
 
-/** Runs specified benchmarks if the enclosing script is main. */
-export function runIfMain(
-  meta: ImportMeta,
-  opts: BenchmarkRunOptions = {}
-): Promise<void> {
-  if (meta.main) {
-    return runBenchmarks(opts);
+  // Making sure the program exit code is not zero in case of failure
+  if (!!failError) {
+    throw failError;
   }
-  return Promise.resolve(undefined);
+
+  const benchmarkRunResult = {
+    measured,
+    filtered,
+    results: benchmarkResults,
+  };
+
+  return benchmarkRunResult;
 }
diff --git a/std/testing/bench_example.ts b/std/testing/bench_example.ts
index 401516cca..366521f85 100644
--- a/std/testing/bench_example.ts
+++ b/std/testing/bench_example.ts
@@ -1,5 +1,5 @@
 // https://deno.land/std/testing/bench.ts
-import { BenchmarkTimer, bench, runIfMain } from "./bench.ts";
+import { BenchmarkTimer, bench, runBenchmarks } from "./bench.ts";
 
 // Basic
 bench(function forIncrementX1e9(b: BenchmarkTimer): void {
@@ -26,4 +26,6 @@ bench(function throwing(b): void {
 });
 
 // Bench control
-runIfMain(import.meta, { skip: /throw/ });
+if (import.meta.main) {
+  runBenchmarks({ skip: /throw/ });
+}
diff --git a/std/testing/bench_test.ts b/std/testing/bench_test.ts
index 6dfc18b10..ef004807f 100644
--- a/std/testing/bench_test.ts
+++ b/std/testing/bench_test.ts
@@ -1,7 +1,11 @@
 const { test } = Deno;
-import { bench, runBenchmarks } from "./bench.ts";
-
-import "./bench_example.ts";
+import { bench, runBenchmarks, BenchmarkRunError } from "./bench.ts";
+import {
+  assertEquals,
+  assert,
+  assertThrows,
+  assertThrowsAsync,
+} from "./asserts.ts";
 
 test({
   name: "benching",
@@ -57,6 +61,87 @@ test({
       // Throws bc the timer's stop method is never called
     });
 
-    await runBenchmarks({ skip: /throw/ });
+    const benchResult = await runBenchmarks({ skip: /throw/ });
+
+    assertEquals(benchResult.measured, 5);
+    assertEquals(benchResult.filtered, 1);
+    assertEquals(benchResult.results.length, 5);
+
+    const resultWithMultipleRunsFiltered = benchResult.results.filter(
+      (r) => r.name === "runs100ForIncrementX1e6"
+    );
+    assertEquals(resultWithMultipleRunsFiltered.length, 1);
+
+    const resultWithMultipleRuns = resultWithMultipleRunsFiltered[0];
+    assert(!!resultWithMultipleRuns.runsCount);
+    assert(!!resultWithMultipleRuns.runsAvgMs);
+    assert(!!resultWithMultipleRuns.runsMs);
+    assertEquals(resultWithMultipleRuns.runsCount, 100);
+    assertEquals(resultWithMultipleRuns.runsMs!.length, 100);
+  },
+});
+
+test({
+  name: "benchWithoutName",
+  fn() {
+    assertThrows(
+      (): void => {
+        bench(() => {});
+      },
+      Error,
+      "The benchmark function must not be anonymous"
+    );
+  },
+});
+
+test({
+  name: "benchWithoutStop",
+  fn: async function (): Promise<void> {
+    await assertThrowsAsync(
+      async (): Promise<void> => {
+        bench(function benchWithoutStop(b): void {
+          b.start();
+          // Throws bc the timer's stop method is never called
+        });
+        await runBenchmarks({ only: /benchWithoutStop/, silent: true });
+      },
+      BenchmarkRunError,
+      "The benchmark timer's stop method must be called"
+    );
+  },
+});
+
+test({
+  name: "benchWithoutStart",
+  fn: async function (): Promise<void> {
+    await assertThrowsAsync(
+      async (): Promise<void> => {
+        bench(function benchWithoutStart(b): void {
+          b.stop();
+          // Throws bc the timer's start method is never called
+        });
+        await runBenchmarks({ only: /benchWithoutStart/, silent: true });
+      },
+      BenchmarkRunError,
+      "The benchmark timer's start method must be called"
+    );
+  },
+});
+
+test({
+  name: "benchStopBeforeStart",
+  fn: async function (): Promise<void> {
+    await assertThrowsAsync(
+      async (): Promise<void> => {
+        bench(function benchStopBeforeStart(b): void {
+          b.stop();
+          b.start();
+          // Throws bc the timer's stop is called before start
+        });
+        await runBenchmarks({ only: /benchStopBeforeStart/, silent: true });
+      },
+      BenchmarkRunError,
+      "The benchmark timer's start method must be called before its stop method"
+    );
   },
 });