From 5dbf5c82936a1975067101e25580790c8b7c50b7 Mon Sep 17 00:00:00 2001
From: Nayeem Rahman <nayeemrmn99@gmail.com>
Date: Sat, 26 Aug 2023 10:29:45 +0100
Subject: fix(bench): explicit timers don't force high precision measurements
 (#20272)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Disables `BenchContext::start()` and `BenchContext::end()` for low
precision benchmarks (less than 0.01s per iteration). Prints a warning
when they are used in such benchmarks, suggesting to remove them.

```ts
Deno.bench("noop", { group: "noops" }, () => {});

Deno.bench("noop with start/end", { group: "noops" }, (b) => {
  b.start();
  b.end();
});
```

Before:
```
cpu: 12th Gen Intel(R) Core(TM) i9-12900K
runtime: deno 1.36.2 (x86_64-unknown-linux-gnu)

file:///home/nayeem/projects/deno/temp3.ts
benchmark                    time (avg)        iter/s             (min … max)       p75       p99      p995
----------------------------------------------------------------------------- -----------------------------


noop                          2.63 ns/iter 380,674,131.4    (2.45 ns … 27.78 ns)   2.55 ns   4.03 ns   5.33 ns
noop with start and end     302.47 ns/iter   3,306,146.0     (200 ns … 151.2 µs)    300 ns    400 ns    400 ns

summary
  noop
   115.14x faster than noop with start and end
```

After:
```
cpu: 12th Gen Intel(R) Core(TM) i9-12900K
runtime: deno 1.36.1 (x86_64-unknown-linux-gnu)

file:///home/nayeem/projects/deno/temp3.ts
benchmark                    time (avg)        iter/s             (min … max)       p75       p99      p995
----------------------------------------------------------------------------- -----------------------------


noop                          3.01 ns/iter 332,565,561.7    (2.73 ns … 29.54 ns)   2.93 ns   5.29 ns   7.45 ns
noop with start and end       7.73 ns/iter 129,291,091.5    (6.61 ns … 46.76 ns)   7.87 ns  13.12 ns  15.32 ns
Warning start() and end() calls in "noop with start and end" are ignored because it averages less than 0.01s per iteration. Remove them for better results.

summary
  noop
   2.57x faster than noop with start and end
```
---
 cli/tests/integration/bench_tests.rs                             | 5 +++++
 .../testdata/bench/explicit_start_and_end_low_precision.out      | 9 +++++++++
 cli/tests/testdata/bench/explicit_start_and_end_low_precision.ts | 4 ++++
 3 files changed, 18 insertions(+)
 create mode 100644 cli/tests/testdata/bench/explicit_start_and_end_low_precision.out
 create mode 100644 cli/tests/testdata/bench/explicit_start_and_end_low_precision.ts

(limited to 'cli/tests')

diff --git a/cli/tests/integration/bench_tests.rs b/cli/tests/integration/bench_tests.rs
index 73d541aa6..214dfaa50 100644
--- a/cli/tests/integration/bench_tests.rs
+++ b/cli/tests/integration/bench_tests.rs
@@ -187,6 +187,11 @@ itest!(bench_explicit_start_end {
   exit_code: 1,
 });
 
+itest!(bench_explicit_start_end_low_precision {
+  args: "bench --quiet -A bench/explicit_start_and_end_low_precision.ts",
+  output: "bench/explicit_start_and_end_low_precision.out",
+});
+
 itest!(bench_with_config {
   args: "bench --config bench/collect/deno.jsonc bench/collect",
   exit_code: 0,
diff --git a/cli/tests/testdata/bench/explicit_start_and_end_low_precision.out b/cli/tests/testdata/bench/explicit_start_and_end_low_precision.out
new file mode 100644
index 000000000..e9ac5a864
--- /dev/null
+++ b/cli/tests/testdata/bench/explicit_start_and_end_low_precision.out
@@ -0,0 +1,9 @@
+cpu: [WILDCARD]
+runtime: deno [WILDCARD] ([WILDCARD])
+
+[WILDCARD]/explicit_start_and_end_low_precision.ts
+benchmark                    time (avg)        iter/s             (min … max)       p75       p99      p995
+----------------------------------------------------------------------------- -----------------------------
+noop with start and end  [WILDCARD] [WILDCARD]/iter[WILDCARD]([WILDCARD] … [WILDCARD]) [WILDCARD]
+Warning: start() and end() calls in "noop with start and end" are ignored because it averages less
+than 0.01s per iteration. Remove them for better results.
diff --git a/cli/tests/testdata/bench/explicit_start_and_end_low_precision.ts b/cli/tests/testdata/bench/explicit_start_and_end_low_precision.ts
new file mode 100644
index 000000000..23bdf19fe
--- /dev/null
+++ b/cli/tests/testdata/bench/explicit_start_and_end_low_precision.ts
@@ -0,0 +1,4 @@
+Deno.bench("noop with start and end", (b) => {
+  b.start();
+  b.end();
+});
-- 
cgit v1.2.3