feat(bench): update API, new console reporter (#14305)

This commit changes "deno bench" subcommand, by updating
the "Deno.bench" API as follows:
- remove "Deno.BenchDefinition.n"
- remove "Deno.BenchDefintion.warmup"
- add "Deno.BenchDefinition.group"
- add "Deno.BenchDefintion.baseline"

This is done because bench cases are no longer run fixed amount
of iterations, but instead they are run until there is difference between
subsequent runs that is statistically insiginificant.

Additionally, console reporter was rewritten completely, to looks
similar to "hyperfine" reporter.

1 files changed, 202 insertions, 69 deletions

diff --git a/runtime/js/40_testing.js b/runtime/js/40_testing.js
index c5adc1f59..7a5162e7f 100644
--- a/runtime/js/40_testing.js
+++ b/runtime/js/40_testing.js
@@ -9,16 +9,20 @@
   const { assert } = window.__bootstrap.infra;
   const {
     AggregateErrorPrototype,
+    ArrayFrom,
     ArrayPrototypeFilter,
     ArrayPrototypeJoin,
+    ArrayPrototypeMap,
     ArrayPrototypePush,
     ArrayPrototypeShift,
     ArrayPrototypeSome,
+    ArrayPrototypeSort,
     DateNow,
     Error,
     FunctionPrototype,
     Map,
     MapPrototypeHas,
+    MathCeil,
     ObjectKeys,
     ObjectPrototypeIsPrototypeOf,
     Promise,
@@ -434,6 +438,27 @@
     };
   }
 
+  function assertExitSync(fn, isTest) {
+    return function exitSanitizer(...params) {
+      setExitHandler((exitCode) => {
+        assert(
+          false,
+          `${
+            isTest ? "Test case" : "Bench"
+          } attempted to exit with exit code: ${exitCode}`,
+        );
+      });
+
+      try {
+        fn(...new SafeArrayIterator(params));
+      } catch (err) {
+        throw err;
+      } finally {
+        setExitHandler(null);
+      }
+    };
+  }
+
   function assertTestStepScopes(fn) {
     /** @param step {TestStep} */
     return async function testStepSanitizer(step) {
@@ -721,18 +746,14 @@
       benchDef = { ...defaults, ...nameOrFnOrOptions, fn, name };
     }
 
+    const AsyncFunction = (async () => {}).constructor;
+    benchDef.async = AsyncFunction === benchDef.fn.constructor;
+
     benchDef.fn = wrapBenchFnWithSanitizers(
-      reportBenchIteration(benchDef.fn),
+      benchDef.fn,
       benchDef,
     );
 
-    if (benchDef.permissions) {
-      benchDef.fn = withPermissions(
-        benchDef.fn,
-        benchDef.permissions,
-      );
-    }
-
     ArrayPrototypePush(benches, benchDef);
   }
 
@@ -823,37 +844,166 @@
     }
   }
 
-  async function runBench(bench) {
-    if (bench.ignore) {
-      return "ignored";
+  function compareMeasurements(a, b) {
+    if (a > b) return 1;
+    if (a < b) return -1;
+
+    return 0;
+  }
+
+  function benchStats(n, highPrecision, avg, min, max, all) {
+    return {
+      n,
+      min,
+      max,
+      p75: all[MathCeil(n * (75 / 100)) - 1],
+      p99: all[MathCeil(n * (99 / 100)) - 1],
+      p995: all[MathCeil(n * (99.5 / 100)) - 1],
+      p999: all[MathCeil(n * (99.9 / 100)) - 1],
+      avg: !highPrecision ? (avg / n) : MathCeil(avg / n),
+    };
+  }
+
+  async function benchMeasure(timeBudget, fn, step, sync) {
+    let n = 0;
+    let avg = 0;
+    let wavg = 0;
+    const all = [];
+    let min = Infinity;
+    let max = -Infinity;
+    const lowPrecisionThresholdInNs = 1e4;
+
+    // warmup step
+    let c = 0;
+    step.warmup = true;
+    let iterations = 20;
+    let budget = 10 * 1e6;
+
+    if (sync) {
+      while (budget > 0 || iterations-- > 0) {
+        const t1 = benchNow();
+
+        fn();
+        const iterationTime = benchNow() - t1;
+
+        c++;
+        wavg += iterationTime;
+        budget -= iterationTime;
+      }
+    } else {
+      while (budget > 0 || iterations-- > 0) {
+        const t1 = benchNow();
+
+        await fn();
+        const iterationTime = benchNow() - t1;
+
+        c++;
+        wavg += iterationTime;
+        budget -= iterationTime;
+      }
+    }
+
+    wavg /= c;
+
+    // measure step
+    step.warmup = false;
+
+    if (wavg > lowPrecisionThresholdInNs) {
+      let iterations = 10;
+      let budget = timeBudget * 1e6;
+
+      if (sync) {
+        while (budget > 0 || iterations-- > 0) {
+          const t1 = benchNow();
+
+          fn();
+          const iterationTime = benchNow() - t1;
+
+          n++;
+          avg += iterationTime;
+          budget -= iterationTime;
+          all.push(iterationTime);
+          if (iterationTime < min) min = iterationTime;
+          if (iterationTime > max) max = iterationTime;
+        }
+      } else {
+        while (budget > 0 || iterations-- > 0) {
+          const t1 = benchNow();
+
+          await fn();
+          const iterationTime = benchNow() - t1;
+
+          n++;
+          avg += iterationTime;
+          budget -= iterationTime;
+          all.push(iterationTime);
+          if (iterationTime < min) min = iterationTime;
+          if (iterationTime > max) max = iterationTime;
+        }
+      }
+    } else {
+      let iterations = 10;
+      let budget = timeBudget * 1e6;
+
+      if (sync) {
+        while (budget > 0 || iterations-- > 0) {
+          const t1 = benchNow();
+          for (let c = 0; c < lowPrecisionThresholdInNs; c++) fn();
+          const iterationTime = (benchNow() - t1) / lowPrecisionThresholdInNs;
+
+          n++;
+          avg += iterationTime;
+          all.push(iterationTime);
+          if (iterationTime < min) min = iterationTime;
+          if (iterationTime > max) max = iterationTime;
+          budget -= iterationTime * lowPrecisionThresholdInNs;
+        }
+      } else {
+        while (budget > 0 || iterations-- > 0) {
+          const t1 = benchNow();
+          for (let c = 0; c < lowPrecisionThresholdInNs; c++) await fn();
+          const iterationTime = (benchNow() - t1) / lowPrecisionThresholdInNs;
+
+          n++;
+          avg += iterationTime;
+          all.push(iterationTime);
+          if (iterationTime < min) min = iterationTime;
+          if (iterationTime > max) max = iterationTime;
+          budget -= iterationTime * lowPrecisionThresholdInNs;
+        }
+      }
     }
 
+    all.sort(compareMeasurements);
+    return benchStats(n, wavg > lowPrecisionThresholdInNs, avg, min, max, all);
+  }
+
+  async function runBench(bench) {
     const step = new BenchStep({
       name: bench.name,
       sanitizeExit: bench.sanitizeExit,
       warmup: false,
     });
 
-    try {
-      const warmupIterations = bench.warmupIterations;
-      step.warmup = true;
+    let token = null;
 
-      for (let i = 0; i < warmupIterations; i++) {
-        await bench.fn(step);
+    try {
+      if (bench.permissions) {
+        token = core.opSync(
+          "op_pledge_test_permissions",
+          serializePermissions(bench.permissions),
+        );
       }
 
-      const iterations = bench.n;
-      step.warmup = false;
+      const benchTimeInMs = 500;
+      const fn = bench.fn.bind(null, step);
+      const stats = await benchMeasure(benchTimeInMs, fn, step, !bench.async);
 
-      for (let i = 0; i < iterations; i++) {
-        await bench.fn(step);
-      }
-
-      return "ok";
+      return { ok: { stats, ...bench } };
     } catch (error) {
-      return {
-        "failed": formatError(error),
-      };
+      return { failed: { ...bench, error: formatError(error) } };
+    } finally {
+      if (token !== null) core.opSync("op_restore_test_permissions", token);
     }
   }
 
@@ -913,35 +1063,16 @@
     });
   }
 
-  function reportBenchResult(description, result, elapsed) {
+  function reportBenchResult(origin, result) {
     core.opSync("op_dispatch_bench_event", {
-      result: [description, result, elapsed],
+      result: [origin, result],
     });
   }
 
-  function reportBenchIteration(fn) {
-    return async function benchIteration(step) {
-      let now;
-      if (!step.warmup) {
-        now = benchNow();
-      }
-      await fn(step);
-      if (!step.warmup) {
-        reportIterationTime(benchNow() - now);
-      }
-    };
-  }
-
   function benchNow() {
     return core.opSync("op_bench_now");
   }
 
-  function reportIterationTime(time) {
-    core.opSync("op_dispatch_bench_event", {
-      iterationTime: time,
-    });
-  }
-
   async function runTests({
     filter = null,
     shuffle = null,
@@ -1013,32 +1144,34 @@
       createTestFilter(filter),
     );
 
+    let groups = new Set();
+    const benchmarks = ArrayPrototypeFilter(filtered, (bench) => !bench.ignore);
+
+    // make sure ungrouped benchmarks are placed above grouped
+    groups.add(undefined);
+
+    for (const bench of benchmarks) {
+      bench.group ||= undefined;
+      groups.add(bench.group);
+    }
+
+    groups = ArrayFrom(groups);
+    ArrayPrototypeSort(
+      benchmarks,
+      (a, b) => groups.indexOf(a.group) - groups.indexOf(b.group),
+    );
+
     reportBenchPlan({
       origin,
-      total: filtered.length,
-      filteredOut: benches.length - filtered.length,
+      total: benchmarks.length,
       usedOnly: only.length > 0,
+      names: ArrayPrototypeMap(benchmarks, (bench) => bench.name),
     });
 
-    for (const bench of filtered) {
-      // TODO(bartlomieju): probably needs some validation?
-      const iterations = bench.n ?? 1000;
-      const warmupIterations = bench.warmup ?? 1000;
-      const description = {
-        origin,
-        name: bench.name,
-        iterations,
-      };
-      bench.n = iterations;
-      bench.warmupIterations = warmupIterations;
-      const earlier = DateNow();
-
-      reportBenchWait(description);
-
-      const result = await runBench(bench);
-      const elapsed = DateNow() - earlier;
-
-      reportBenchResult(description, result, elapsed);
+    for (const bench of benchmarks) {
+      bench.baseline = !!bench.baseline;
+      reportBenchWait({ origin, ...bench });
+      reportBenchResult(origin, await runBench(bench));
     }
 
     globalThis.console = originalConsole;
@@ -1420,7 +1553,7 @@
    */
   function wrapBenchFnWithSanitizers(fn, opts) {
     if (opts.sanitizeExit) {
-      fn = assertExit(fn, false);
+      fn = opts.async ? assertExit(fn, false) : assertExitSync(fn, false);
     }
     return fn;
   }