feat(bench): update API, new console reporter (#14305)

This commit changes "deno bench" subcommand, by updating
the "Deno.bench" API as follows:
- remove "Deno.BenchDefinition.n"
- remove "Deno.BenchDefintion.warmup"
- add "Deno.BenchDefinition.group"
- add "Deno.BenchDefintion.baseline"

This is done because bench cases are no longer run fixed amount
of iterations, but instead they are run until there is difference between
subsequent runs that is statistically insiginificant.

Additionally, console reporter was rewritten completely, to looks
similar to "hyperfine" reporter.

1 files changed, 246 insertions, 186 deletions

diff --git a/cli/tools/bench.rs b/cli/tools/bench.rs
index 2d35c27c5..0947c2647 100644
--- a/cli/tools/bench.rs
+++ b/cli/tools/bench.rs
@@ -5,7 +5,6 @@ use crate::cache::CacherLoader;
 use crate::colors;
 use crate::compat;
 use crate::create_main_worker;
-use crate::display;
 use crate::emit;
 use crate::file_watcher;
 use crate::file_watcher::ResolutionResult;
@@ -35,15 +34,11 @@ use deno_graph::ModuleKind;
 use deno_runtime::permissions::Permissions;
 use deno_runtime::tokio_util::run_basic;
 use log::Level;
-use num_format::Locale;
-use num_format::ToFormattedString;
 use serde::Deserialize;
+use serde::Serialize;
 use std::collections::HashSet;
-use std::io::Write;
 use std::path::PathBuf;
 use std::sync::Arc;
-use std::time::Duration;
-use std::time::Instant;
 use tokio::sync::mpsc::unbounded_channel;
 use tokio::sync::mpsc::UnboundedSender;
 
@@ -53,14 +48,6 @@ struct BenchSpecifierOptions {
   filter: Option<String>,
 }
 
-#[derive(Debug, Clone, PartialEq, Deserialize, Eq, Hash)]
-#[serde(rename_all = "camelCase")]
-pub struct BenchDescription {
-  pub origin: String,
-  pub name: String,
-  pub iterations: u64,
-}
-
 #[derive(Debug, Clone, PartialEq, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub enum BenchOutput {
@@ -69,198 +56,294 @@ pub enum BenchOutput {
 
 #[derive(Debug, Clone, PartialEq, Deserialize)]
 #[serde(rename_all = "camelCase")]
-pub enum BenchResult {
-  Ok,
-  Ignored,
-  Failed(String),
-}
-
-#[derive(Debug, Clone, PartialEq, Deserialize)]
-#[serde(rename_all = "camelCase")]
 pub struct BenchPlan {
-  pub origin: String,
   pub total: usize,
-  pub filtered_out: usize,
+  pub origin: String,
   pub used_only: bool,
+  pub names: Vec<String>,
 }
 
 #[derive(Debug, Clone, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub enum BenchEvent {
   Plan(BenchPlan),
-  Wait(BenchDescription),
   Output(BenchOutput),
-  IterationTime(u64),
-  Result(BenchDescription, BenchResult, u64),
+  Wait(BenchMetadata),
+  Result(String, BenchResult),
 }
 
-#[derive(Debug, Clone)]
-pub struct BenchMeasures {
-  pub iterations: u64,
-  pub current_start: Instant,
-  pub measures: Vec<u128>,
+#[derive(Debug, Clone, PartialEq, Deserialize)]
+#[serde(rename_all = "camelCase")]
+pub enum BenchResult {
+  Ok(BenchMeasurement),
+  Failed(BenchFailure),
 }
 
-#[derive(Debug, Clone)]
-pub struct BenchSummary {
+#[derive(Debug, Clone, Serialize)]
+pub struct BenchReport {
   pub total: usize,
-  pub passed: usize,
   pub failed: usize,
-  pub ignored: usize,
-  pub filtered_out: usize,
-  pub measured: usize,
-  pub measures: Vec<BenchMeasures>,
-  pub current_bench: BenchMeasures,
-  pub failures: Vec<(BenchDescription, String)>,
+  pub failures: Vec<BenchFailure>,
+  pub measurements: Vec<BenchMeasurement>,
+}
+
+#[derive(Debug, Clone, PartialEq, Deserialize, Eq, Hash)]
+pub struct BenchMetadata {
+  pub name: String,
+  pub origin: String,
+  pub baseline: bool,
+  pub group: Option<String>,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct BenchMeasurement {
+  pub name: String,
+  pub baseline: bool,
+  pub stats: BenchStats,
+  pub group: Option<String>,
+}
+
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct BenchFailure {
+  pub name: String,
+  pub error: String,
+  pub baseline: bool,
+  pub group: Option<String>,
 }
 
-impl BenchSummary {
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct BenchStats {
+  pub n: u64,
+  pub min: f64,
+  pub max: f64,
+  pub avg: f64,
+  pub p75: f64,
+  pub p99: f64,
+  pub p995: f64,
+  pub p999: f64,
+}
+
+impl BenchReport {
   pub fn new() -> Self {
     Self {
       total: 0,
-      passed: 0,
       failed: 0,
-      ignored: 0,
-      filtered_out: 0,
-      measured: 0,
-      measures: Vec::new(),
-      current_bench: BenchMeasures {
-        iterations: 0,
-        current_start: Instant::now(),
-        measures: vec![],
-      },
       failures: Vec::new(),
+      measurements: Vec::new(),
     }
   }
+}
 
-  fn has_failed(&self) -> bool {
-    self.failed > 0 || !self.failures.is_empty()
-  }
-
-  fn has_pending(&self) -> bool {
-    self.total - self.passed - self.failed - self.ignored > 0
-  }
+fn create_reporter(show_output: bool) -> Box<dyn BenchReporter + Send> {
+  Box::new(ConsoleReporter::new(show_output))
 }
 
 pub trait BenchReporter {
+  fn report_group_summary(&mut self);
   fn report_plan(&mut self, plan: &BenchPlan);
-  fn report_wait(&mut self, description: &BenchDescription);
+  fn report_end(&mut self, report: &BenchReport);
+  fn report_wait(&mut self, wait: &BenchMetadata);
   fn report_output(&mut self, output: &BenchOutput);
-  fn report_result(
-    &mut self,
-    description: &BenchDescription,
-    result: &BenchResult,
-    elapsed: u64,
-    current_bench: &BenchMeasures,
-  );
-  fn report_summary(&mut self, summary: &BenchSummary, elapsed: &Duration);
+  fn report_result(&mut self, result: &BenchResult);
 }
 
-struct PrettyBenchReporter {
-  echo_output: bool,
+struct ConsoleReporter {
+  name: String,
+  show_output: bool,
+  has_ungrouped: bool,
+  group: Option<String>,
+  baseline: Option<BenchMeasurement>,
+  group_measurements: Vec<BenchMeasurement>,
+  options: Option<mitata::reporter::Options>,
 }
 
-impl PrettyBenchReporter {
-  fn new(echo_output: bool) -> Self {
-    Self { echo_output }
-  }
-
-  fn force_report_wait(&mut self, description: &BenchDescription) {
-    print!(
-      "bench {} ... {} iterations ",
-      description.name, description.iterations
-    );
-    // flush for faster feedback when line buffered
-    std::io::stdout().flush().unwrap();
+impl ConsoleReporter {
+  fn new(show_output: bool) -> Self {
+    Self {
+      show_output,
+      group: None,
+      options: None,
+      baseline: None,
+      name: String::new(),
+      has_ungrouped: false,
+      group_measurements: Vec::new(),
+    }
   }
 }
 
-impl BenchReporter for PrettyBenchReporter {
+impl BenchReporter for ConsoleReporter {
+  #[cold]
   fn report_plan(&mut self, plan: &BenchPlan) {
-    let inflection = if plan.total == 1 { "bench" } else { "benches" };
-    println!("running {} {} from {}", plan.total, inflection, plan.origin);
+    use std::sync::atomic::AtomicBool;
+    use std::sync::atomic::Ordering;
+    static FIRST_PLAN: AtomicBool = AtomicBool::new(true);
+
+    self.options = Some(mitata::reporter::Options::new(
+      &plan.names.iter().map(|x| x.as_str()).collect::<Vec<&str>>(),
+    ));
+
+    let options = self.options.as_mut().unwrap();
+
+    options.percentiles = true;
+    options.colors = colors::use_color();
+
+    if FIRST_PLAN
+      .compare_exchange(true, false, Ordering::SeqCst, Ordering::SeqCst)
+      .is_ok()
+    {
+      println!("{}", colors::gray(format!("cpu: {}", mitata::cpu::name())));
+      println!(
+        "{}\n",
+        colors::gray(format!(
+          "runtime: deno {} ({})",
+          crate::version::deno(),
+          env!("TARGET")
+        ))
+      );
+    } else {
+      println!();
+    }
+
+    println!(
+      "{}\n{}\n{}",
+      colors::gray(&plan.origin),
+      mitata::reporter::header(options),
+      mitata::reporter::br(options)
+    );
   }
 
-  fn report_wait(&mut self, description: &BenchDescription) {
-    self.force_report_wait(description);
+  fn report_wait(&mut self, wait: &BenchMetadata) {
+    self.name = wait.name.clone();
+
+    match &wait.group {
+      None => {
+        self.has_ungrouped = true;
+      }
+
+      Some(group) => {
+        if self.group.is_none()
+          && self.has_ungrouped
+          && self.group_measurements.is_empty()
+        {
+          println!();
+        }
+
+        if None == self.group || group != self.group.as_ref().unwrap() {
+          self.report_group_summary();
+        }
+
+        if (self.group.is_none() && self.has_ungrouped)
+          || (self.group.is_some() && self.group_measurements.is_empty())
+        {
+          println!();
+        }
+
+        self.group = Some(group.clone());
+      }
+    }
   }
 
   fn report_output(&mut self, output: &BenchOutput) {
-    if self.echo_output {
+    if self.show_output {
       match output {
-        BenchOutput::Console(line) => print!("{}", line),
+        BenchOutput::Console(line) => {
+          print!("{} {}", colors::gray(format!("{}:", self.name)), line)
+        }
       }
     }
   }
 
-  fn report_result(
-    &mut self,
-    _description: &BenchDescription,
-    result: &BenchResult,
-    elapsed: u64,
-    current_bench: &BenchMeasures,
-  ) {
-    let status = match result {
-      BenchResult::Ok => {
-        let ns_op = current_bench.measures.iter().sum::<u128>()
-          / current_bench.iterations as u128;
-        let min_op = current_bench.measures.iter().min().unwrap_or(&0);
-        let max_op = current_bench.measures.iter().max().unwrap_or(&0);
-        format!(
-          "{} ns/iter ({}..{} ns/iter) {}",
-          ns_op.to_formatted_string(&Locale::en),
-          min_op.to_formatted_string(&Locale::en),
-          max_op.to_formatted_string(&Locale::en),
-          colors::green("ok")
-        )
-      }
-      BenchResult::Ignored => colors::yellow("ignored").to_string(),
-      BenchResult::Failed(_) => colors::red("FAILED").to_string(),
-    };
+  fn report_result(&mut self, result: &BenchResult) {
+    let options = self.options.as_ref().unwrap();
 
-    println!(
-      "{} {}",
-      status,
-      colors::gray(format!("({})", display::human_elapsed(elapsed.into())))
-    );
-  }
+    match result {
+      BenchResult::Ok(bench) => {
+        let mut bench = bench.to_owned();
+
+        if bench.baseline && self.baseline.is_none() {
+          self.baseline = Some(bench.clone());
+        } else {
+          bench.baseline = false;
+        }
 
-  fn report_summary(&mut self, summary: &BenchSummary, elapsed: &Duration) {
-    if !summary.failures.is_empty() {
-      println!("\nfailures:\n");
-      for (description, error) in &summary.failures {
-        println!("{}", description.name);
-        println!("{}", error);
-        println!();
+        self.group_measurements.push(bench.clone());
+
+        println!(
+          "{}",
+          mitata::reporter::benchmark(
+            &bench.name,
+            &mitata::reporter::BenchmarkStats {
+              avg: bench.stats.avg,
+              min: bench.stats.min,
+              max: bench.stats.max,
+              p75: bench.stats.p75,
+              p99: bench.stats.p99,
+              p995: bench.stats.p995,
+            },
+            options
+          )
+        );
       }
 
-      println!("failures:\n");
-      for (description, _) in &summary.failures {
-        println!("\t{}", description.name);
+      BenchResult::Failed(failure) => {
+        println!(
+          "{}",
+          mitata::reporter::benchmark_error(
+            &failure.name,
+            &mitata::reporter::Error {
+              stack: None,
+              message: failure.error.clone(),
+            },
+            options
+          )
+        )
       }
-    }
+    };
+  }
 
-    let status = if summary.has_failed() || summary.has_pending() {
-      colors::red("FAILED").to_string()
-    } else {
-      colors::green("ok").to_string()
+  fn report_group_summary(&mut self) {
+    let options = match self.options.as_ref() {
+      None => return,
+      Some(options) => options,
     };
 
-    println!(
-      "\nbench result: {}. {} passed; {} failed; {} ignored; {} measured; {} filtered out {}\n",
-      status,
-      summary.passed,
-      summary.failed,
-      summary.ignored,
-      summary.measured,
-      summary.filtered_out,
-      colors::gray(format!("({})", display::human_elapsed(elapsed.as_millis()))),
-    );
+    if 2 <= self.group_measurements.len()
+      && (self.group.is_some()
+        || (self.group.is_none() && self.baseline.is_some()))
+    {
+      println!(
+        "\n{}",
+        mitata::reporter::summary(
+          &self
+            .group_measurements
+            .iter()
+            .map(|b| mitata::reporter::GroupBenchmark {
+              name: b.name.clone(),
+              baseline: b.baseline,
+              group: b.group.as_deref().unwrap_or("").to_owned(),
+
+              stats: mitata::reporter::BenchmarkStats {
+                avg: b.stats.avg,
+                min: b.stats.min,
+                max: b.stats.max,
+                p75: b.stats.p75,
+                p99: b.stats.p99,
+                p995: b.stats.p995,
+              },
+            })
+            .collect::<Vec<mitata::reporter::GroupBenchmark>>(),
+          options
+        )
+      );
+    }
+
+    self.baseline = None;
+    self.group_measurements.clear();
   }
-}
 
-fn create_reporter(echo_output: bool) -> Box<dyn BenchReporter + Send> {
-  Box::new(PrettyBenchReporter::new(echo_output))
+  fn report_end(&mut self, _: &BenchReport) {
+    self.report_group_summary();
+  }
 }
 
 /// Type check a collection of module and document specifiers.
@@ -367,20 +450,16 @@ async fn bench_specifiers(
     .buffer_unordered(1)
     .collect::<Vec<Result<Result<(), AnyError>, tokio::task::JoinError>>>();
 
-  let mut reporter = create_reporter(log_level != Some(Level::Error));
-
   let handler = {
     tokio::task::spawn(async move {
-      let earlier = Instant::now();
-      let mut summary = BenchSummary::new();
       let mut used_only = false;
+      let mut report = BenchReport::new();
+      let mut reporter = create_reporter(log_level != Some(Level::Error));
 
       while let Some(event) = receiver.recv().await {
         match event {
           BenchEvent::Plan(plan) => {
-            summary.total += plan.total;
-            summary.filtered_out += plan.filtered_out;
-
+            report.total += plan.total;
             if plan.used_only {
               used_only = true;
             }
@@ -388,51 +467,32 @@ async fn bench_specifiers(
             reporter.report_plan(&plan);
           }
 
-          BenchEvent::Wait(description) => {
-            reporter.report_wait(&description);
-            summary.current_bench = BenchMeasures {
-              iterations: description.iterations,
-              current_start: Instant::now(),
-              measures: Vec::with_capacity(
-                description.iterations.try_into().unwrap(),
-              ),
-            };
+          BenchEvent::Wait(metadata) => {
+            reporter.report_wait(&metadata);
           }
 
           BenchEvent::Output(output) => {
             reporter.report_output(&output);
           }
 
-          BenchEvent::IterationTime(iter_time) => {
-            summary.current_bench.measures.push(iter_time.into())
-          }
-
-          BenchEvent::Result(description, result, elapsed) => {
+          BenchEvent::Result(_origin, result) => {
             match &result {
-              BenchResult::Ok => {
-                summary.passed += 1;
-              }
-              BenchResult::Ignored => {
-                summary.ignored += 1;
+              BenchResult::Ok(bench) => {
+                report.measurements.push(bench.clone());
               }
-              BenchResult::Failed(error) => {
-                summary.failed += 1;
-                summary.failures.push((description.clone(), error.clone()));
+
+              BenchResult::Failed(failure) => {
+                report.failed += 1;
+                report.failures.push(failure.clone());
               }
-            }
+            };
 
-            reporter.report_result(
-              &description,
-              &result,
-              elapsed,
-              &summary.current_bench,
-            );
+            reporter.report_result(&result);
           }
         }
       }
 
-      let elapsed = Instant::now().duration_since(earlier);
-      reporter.report_summary(&summary, &elapsed);
+      reporter.report_end(&report);
 
       if used_only {
         return Err(generic_error(
@@ -440,7 +500,7 @@ async fn bench_specifiers(
         ));
       }
 
-      if summary.failed > 0 {
+      if report.failed > 0 {
         return Err(generic_error("Bench failed"));
       }