Move benching into testing. (denoland/deno_std#258)

Original: https://github.com/denoland/deno_std/commit/4de86f04de8c83f8af184cb67b56f4022c17864f

6 files changed, 354 insertions, 17 deletions

diff --git a/testing/README.md b/testing/README.md
index c134dd912..05cb8b92e 100644
--- a/testing/README.md
+++ b/testing/README.md
@@ -131,3 +131,77 @@ test(async function fails() {
   });
 });
 ```
+
+### Benching Usage
+
+Basic usage:
+
+```ts
+import { runBenchmarks, bench } from "https://deno.land/std/testing/bench.ts";
+
+bench(function forIncrementX1e9(b) {
+  b.start();
+  for (let i = 0; i < 1e9; i++);
+  b.stop();
+});
+
+runBenchmarks();
+```
+
+Averaging execution time over multiple runs:
+
+```ts
+bench({
+  name: "runs100ForIncrementX1e6",
+  runs: 100,
+  func(b) {
+    b.start();
+    for (let i = 0; i < 1e6; i++);
+    b.stop();
+  }
+});
+```
+
+#### Benching API
+
+##### `bench(benchmark: BenchmarkDefinition | BenchmarkFunction): void`
+
+Registers a benchmark that will be run once `runBenchmarks` is called.
+
+##### `runBenchmarks(opts?: BenchmarkRunOptions): Promise<void>`
+
+Runs all registered benchmarks serially. Filtering can be applied by setting
+`BenchmarkRunOptions.only` and/or `BenchmarkRunOptions.skip` to regular expressions matching benchmark names.
+
+##### `runIfMain(meta: ImportMeta, opts?: BenchmarkRunOptions): Promise<void>`
+
+Runs specified benchmarks if the enclosing script is main.
+
+##### Other exports
+
+```ts
+/** Provides methods for starting and stopping a benchmark clock. */
+export interface BenchmarkTimer {
+  start: () => void;
+  stop: () => void;
+}
+
+/** Defines a benchmark through a named function. */
+export interface BenchmarkFunction {
+  (b: BenchmarkTimer): void | Promise<void>;
+  name: string;
+}
+
+/** Defines a benchmark definition with configurable runs. */
+export interface BenchmarkDefinition {
+  func: BenchmarkFunction;
+  name: string;
+  runs?: number;
+}
+
+/** Defines runBenchmark's run constraints by matching benchmark names. */
+export interface BenchmarkRunOptions {
+  only?: RegExp;
+  skip?: RegExp;
+}
+```
diff --git a/testing/bench.ts b/testing/bench.ts
index bc2e569d2..0094f6292 100644
--- a/testing/bench.ts
+++ b/testing/bench.ts
@@ -1,16 +1,179 @@
-import { bench, runBenchmarks } from "./../benching/mod.ts";
-import { runTests } from "./mod.ts";
-
-bench(async function testingSerial(b) {
-  b.start();
-  await runTests();
-  b.stop();
-});
-
-bench(async function testingParallel(b) {
-  b.start();
-  await runTests({ parallel: true });
-  b.stop();
-});
-
-runBenchmarks({ only: /testing/ });
+// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
+
+const { exit, noColor } = Deno;
+
+interface BenchmarkClock {
+  start: number;
+  stop: number;
+}
+
+/** Provides methods for starting and stopping a benchmark clock. */
+export interface BenchmarkTimer {
+  start: () => void;
+  stop: () => void;
+}
+
+/** Defines a benchmark through a named function. */
+export interface BenchmarkFunction {
+  (b: BenchmarkTimer): void | Promise<void>;
+  name: string;
+}
+
+/** Defines a benchmark definition with configurable runs. */
+export interface BenchmarkDefinition {
+  func: BenchmarkFunction;
+  name: string;
+  runs?: number;
+}
+
+/** Defines runBenchmark's run constraints by matching benchmark names. */
+export interface BenchmarkRunOptions {
+  only?: RegExp;
+  skip?: RegExp;
+}
+
+function red(text: string): string {
+  return noColor ? text : `\x1b[31m${text}\x1b[0m`;
+}
+
+function blue(text: string): string {
+  return noColor ? text : `\x1b[34m${text}\x1b[0m`;
+}
+
+function verifyOr1Run(runs?: number): number {
+  return runs && runs >= 1 && runs !== Infinity ? Math.floor(runs) : 1;
+}
+
+function assertTiming(clock: BenchmarkClock): void {
+  // NaN indicates that a benchmark has not been timed properly
+  if (!clock.stop) {
+    throw new Error("The benchmark timer's stop method must be called");
+  } else if (!clock.start) {
+    throw new Error("The benchmark timer's start method must be called");
+  } else if (clock.start > clock.stop) {
+    throw new Error(
+      "The benchmark timer's start method must be called before its " +
+        "stop method"
+    );
+  }
+}
+
+function createBenchmarkTimer(clock: BenchmarkClock): BenchmarkTimer {
+  return {
+    start(): void {
+      clock.start = Date.now();
+    },
+    stop(): void {
+      clock.stop = Date.now();
+    }
+  };
+}
+
+const candidates: BenchmarkDefinition[] = [];
+
+/** Registers a benchmark as a candidate for the runBenchmarks executor. */
+export function bench(
+  benchmark: BenchmarkDefinition | BenchmarkFunction
+): void {
+  if (!benchmark.name) {
+    throw new Error("The benchmark function must not be anonymous");
+  }
+  if (typeof benchmark === "function") {
+    candidates.push({ name: benchmark.name, runs: 1, func: benchmark });
+  } else {
+    candidates.push({
+      name: benchmark.name,
+      runs: verifyOr1Run(benchmark.runs),
+      func: benchmark.func
+    });
+  }
+}
+
+/** Runs all registered and non-skipped benchmarks serially. */
+export async function runBenchmarks({
+  only = /[^\s]/,
+  skip = /^\s*$/
+}: BenchmarkRunOptions = {}): Promise<void> {
+  // Filtering candidates by the "only" and "skip" constraint
+  const benchmarks: BenchmarkDefinition[] = candidates.filter(
+    ({ name }) => only.test(name) && !skip.test(name)
+  );
+  // Init main counters and error flag
+  const filtered = candidates.length - benchmarks.length;
+  let measured = 0;
+  let failed = false;
+  // Setting up a shared benchmark clock and timer
+  const clock: BenchmarkClock = { start: NaN, stop: NaN };
+  const b = createBenchmarkTimer(clock);
+  // Iterating given benchmark definitions (await-in-loop)
+  console.log(
+    "running",
+    benchmarks.length,
+    `benchmark${benchmarks.length === 1 ? " ..." : "s ..."}`
+  );
+  for (const { name, runs = 0, func } of benchmarks) {
+    // See https://github.com/denoland/deno/pull/1452 about groupCollapsed
+    console.groupCollapsed(`benchmark ${name} ... `);
+    // Trying benchmark.func
+    let result = "";
+    try {
+      if (runs === 1) {
+        // b is a benchmark timer interfacing an unset (NaN) benchmark clock
+        await func(b);
+        // Making sure the benchmark was started/stopped properly
+        assertTiming(clock);
+        result = `${clock.stop - clock.start}ms`;
+      } else if (runs > 1) {
+        // Averaging runs
+        let pendingRuns = runs;
+        let totalMs = 0;
+        // Would be better 2 not run these serially
+        while (true) {
+          // b is a benchmark timer interfacing an unset (NaN) benchmark clock
+          await func(b);
+          // Making sure the benchmark was started/stopped properly
+          assertTiming(clock);
+          // Summing up
+          totalMs += clock.stop - clock.start;
+          // Resetting the benchmark clock
+          clock.start = clock.stop = NaN;
+          // Once all ran
+          if (!--pendingRuns) {
+            result = `${runs} runs avg: ${totalMs / runs}ms`;
+            break;
+          }
+        }
+      }
+    } catch (err) {
+      failed = true;
+      console.groupEnd();
+      console.error(red(err.stack));
+      break;
+    }
+    // Reporting
+    console.log(blue(result));
+    console.groupEnd();
+    measured++;
+    // Resetting the benchmark clock
+    clock.start = clock.stop = NaN;
+  }
+  // Closing results
+  console.log(
+    `benchmark result: ${failed ? red("FAIL") : blue("DONE")}. ` +
+      `${measured} measured; ${filtered} filtered`
+  );
+  // Making sure the program exit code is not zero in case of failure
+  if (failed) {
+    setTimeout(() => exit(1), 0);
+  }
+}
+
+/** Runs specified benchmarks if the enclosing script is main. */
+export async function runIfMain(
+  meta: ImportMeta,
+  opts?: BenchmarkRunOptions
+): Promise<void> {
+  if (meta.main) {
+    return runBenchmarks(opts);
+  }
+}
diff --git a/testing/bench_example.ts b/testing/bench_example.ts
new file mode 100644
index 000000000..86e25b9a6
--- /dev/null
+++ b/testing/bench_example.ts
@@ -0,0 +1,29 @@
+// https://deno.land/std/testing/bench.ts
+import { BenchmarkTimer, bench, runIfMain } from "./bench.ts";
+
+// Basic
+bench(function forIncrementX1e9(b: BenchmarkTimer) {
+  b.start();
+  for (let i = 0; i < 1e9; i++);
+  b.stop();
+});
+
+// Reporting average measured time for $runs runs of func
+bench({
+  name: "runs100ForIncrementX1e6",
+  runs: 100,
+  func(b) {
+    b.start();
+    for (let i = 0; i < 1e6; i++);
+    b.stop();
+  }
+});
+
+// Itsabug
+bench(function throwing(b) {
+  b.start();
+  // Throws bc the timer's stop method is never called
+});
+
+// Bench control
+runIfMain(import.meta, { skip: /throw/ });
diff --git a/testing/bench_test.ts b/testing/bench_test.ts
new file mode 100644
index 000000000..345f104a6
--- /dev/null
+++ b/testing/bench_test.ts
@@ -0,0 +1,52 @@
+import { test, runIfMain } from "./mod.ts";
+import { bench, runBenchmarks } from "./bench.ts";
+
+import "./bench_example.ts";
+
+test(async function benching() {
+  bench(function forIncrementX1e9(b) {
+    b.start();
+    for (let i = 0; i < 1e9; i++);
+    b.stop();
+  });
+
+  bench(function forDecrementX1e9(b) {
+    b.start();
+    for (let i = 1e9; i > 0; i--);
+    b.stop();
+  });
+
+  bench(async function forAwaitFetchDenolandX10(b) {
+    b.start();
+    for (let i = 0; i < 10; i++) {
+      await fetch("https://deno.land/");
+    }
+    b.stop();
+  });
+
+  bench(async function promiseAllFetchDenolandX10(b) {
+    const urls = new Array(10).fill("https://deno.land/");
+    b.start();
+    await Promise.all(urls.map((denoland: string) => fetch(denoland)));
+    b.stop();
+  });
+
+  bench({
+    name: "runs100ForIncrementX1e6",
+    runs: 100,
+    func(b) {
+      b.start();
+      for (let i = 0; i < 1e6; i++);
+      b.stop();
+    }
+  });
+
+  bench(function throwing(b) {
+    b.start();
+    // Throws bc the timer's stop method is never called
+  });
+
+  await runBenchmarks({ skip: /throw/ });
+});
+
+runIfMain(import.meta);
diff --git a/testing/test.ts b/testing/test.ts
index 367e28022..2359cba2e 100644
--- a/testing/test.ts
+++ b/testing/test.ts
@@ -11,6 +11,7 @@ import "./format_test.ts";
 import "./diff_test.ts";
 import "./pretty_test.ts";
 import "./asserts_test.ts";
+import "./bench_test.ts";
 
 test(function testingAssertEqualActualUncoercable() {
   let didThrow = false;
@@ -251,4 +252,4 @@ test(async function testingThrowsAsyncMsgNotIncludes() {
   assert(didThrow);
 });
 
-runIfMain(import.meta, { parallel: true });
+runIfMain(import.meta);
diff --git a/testing/testing_bench.ts b/testing/testing_bench.ts
new file mode 100644
index 000000000..0cc2f233b
--- /dev/null
+++ b/testing/testing_bench.ts
@@ -0,0 +1,18 @@
+import { bench, runIfMain } from "./bench.ts";
+import { runTests } from "./mod.ts";
+
+import "./asserts_test.ts";
+
+bench(async function testingSerial(b) {
+  b.start();
+  await runTests();
+  b.stop();
+});
+
+bench(async function testingParallel(b) {
+  b.start();
+  await runTests({ parallel: true });
+  b.stop();
+});
+
+runIfMain(import.meta);