refactor(bench): Allocate IDs for benches (#14757)

4 files changed, 182 insertions, 96 deletions

diff --git a/cli/Cargo.toml b/cli/Cargo.toml
index fc09642e9..062f0cf70 100644
--- a/cli/Cargo.toml
+++ b/cli/Cargo.toml
@@ -72,6 +72,7 @@ eszip = "=0.20.0"
 fancy-regex = "=0.9.0"
 http = "=0.2.6"
 import_map = "=0.9.0"
+indexmap = "1.8.1"
 jsonc-parser = { version = "=0.19.0", features = ["serde"] }
 libc = "=0.2.126"
 log = { version = "=0.4.16", features = ["serde"] }
diff --git a/cli/ops/bench.rs b/cli/ops/bench.rs
index e028aa6b1..1c62452d4 100644
--- a/cli/ops/bench.rs
+++ b/cli/ops/bench.rs
@@ -1,4 +1,6 @@
+use crate::tools::bench::BenchDescription;
 use crate::tools::bench::BenchEvent;
+use crate::tools::test::TestFilter;
 use deno_core::error::generic_error;
 use deno_core::error::AnyError;
 use deno_core::op;
@@ -8,22 +10,32 @@ use deno_core::OpState;
 use deno_runtime::permissions::create_child_permissions;
 use deno_runtime::permissions::ChildPermissionsArg;
 use deno_runtime::permissions::Permissions;
+use serde::Deserialize;
+use serde::Serialize;
+use std::sync::atomic::AtomicUsize;
+use std::sync::atomic::Ordering;
 use std::time;
 use tokio::sync::mpsc::UnboundedSender;
 use uuid::Uuid;
 
-pub fn init(sender: UnboundedSender<BenchEvent>, unstable: bool) -> Extension {
+pub fn init(
+  sender: UnboundedSender<BenchEvent>,
+  filter: TestFilter,
+  unstable: bool,
+) -> Extension {
   Extension::builder()
     .ops(vec![
       op_pledge_test_permissions::decl(),
       op_restore_test_permissions::decl(),
       op_get_bench_origin::decl(),
+      op_register_bench::decl(),
       op_dispatch_bench_event::decl(),
       op_bench_now::decl(),
       op_bench_check_unstable::decl(),
     ])
     .state(move |state| {
       state.put(sender.clone());
+      state.put(filter.clone());
       state.put(Unstable(unstable));
       Ok(())
     })
@@ -97,6 +109,44 @@ fn op_get_bench_origin(state: &mut OpState) -> String {
   state.borrow::<ModuleSpecifier>().to_string()
 }
 
+#[derive(Debug, Deserialize)]
+#[serde(rename_all = "camelCase")]
+struct BenchInfo {
+  name: String,
+  origin: String,
+  baseline: bool,
+  group: Option<String>,
+}
+
+#[derive(Debug, Serialize)]
+#[serde(rename_all = "camelCase")]
+struct BenchRegisterResult {
+  id: usize,
+  filtered_out: bool,
+}
+
+static NEXT_ID: AtomicUsize = AtomicUsize::new(0);
+
+#[op]
+fn op_register_bench(
+  state: &mut OpState,
+  info: BenchInfo,
+) -> Result<BenchRegisterResult, AnyError> {
+  let id = NEXT_ID.fetch_add(1, Ordering::SeqCst);
+  let filter = state.borrow::<TestFilter>().clone();
+  let filtered_out = !filter.includes(&info.name);
+  let description = BenchDescription {
+    id,
+    name: info.name,
+    origin: info.origin,
+    baseline: info.baseline,
+    group: info.group,
+  };
+  let sender = state.borrow::<UnboundedSender<BenchEvent>>().clone();
+  sender.send(BenchEvent::Register(description)).ok();
+  Ok(BenchRegisterResult { id, filtered_out })
+}
+
 #[op]
 fn op_dispatch_bench_event(state: &mut OpState, event: BenchEvent) {
   let sender = state.borrow::<UnboundedSender<BenchEvent>>().clone();
diff --git a/cli/tools/bench.rs b/cli/tools/bench.rs
index afd85a8d0..d9e9c38c3 100644
--- a/cli/tools/bench.rs
+++ b/cli/tools/bench.rs
@@ -21,18 +21,21 @@ use crate::ops;
 use crate::proc_state::ProcState;
 use crate::resolver::ImportMapResolver;
 use crate::resolver::JsxResolver;
+use crate::tools::test::format_test_error;
+use crate::tools::test::TestFilter;
 
 use deno_core::error::generic_error;
 use deno_core::error::AnyError;
+use deno_core::error::JsError;
 use deno_core::futures::future;
 use deno_core::futures::stream;
 use deno_core::futures::FutureExt;
 use deno_core::futures::StreamExt;
-use deno_core::serde_json::json;
 use deno_core::ModuleSpecifier;
 use deno_graph::ModuleKind;
 use deno_runtime::permissions::Permissions;
 use deno_runtime::tokio_util::run_basic;
+use indexmap::IndexMap;
 use log::Level;
 use serde::Deserialize;
 use serde::Serialize;
@@ -50,12 +53,6 @@ struct BenchSpecifierOptions {
 
 #[derive(Debug, Clone, PartialEq, Deserialize)]
 #[serde(rename_all = "camelCase")]
-pub enum BenchOutput {
-  Console(String),
-}
-
-#[derive(Debug, Clone, PartialEq, Deserialize)]
-#[serde(rename_all = "camelCase")]
 pub struct BenchPlan {
   pub total: usize,
   pub origin: String,
@@ -67,28 +64,30 @@ pub struct BenchPlan {
 #[serde(rename_all = "camelCase")]
 pub enum BenchEvent {
   Plan(BenchPlan),
-  Output(BenchOutput),
-  Wait(BenchMetadata),
-  Result(String, BenchResult),
+  Output(String),
+  Register(BenchDescription),
+  Wait(usize),
+  Result(usize, BenchResult),
 }
 
 #[derive(Debug, Clone, PartialEq, Deserialize)]
 #[serde(rename_all = "camelCase")]
 pub enum BenchResult {
-  Ok(BenchMeasurement),
-  Failed(BenchFailure),
+  Ok(BenchStats),
+  Failed(Box<JsError>),
 }
 
-#[derive(Debug, Clone, Serialize)]
+#[derive(Debug, Clone)]
 pub struct BenchReport {
   pub total: usize,
   pub failed: usize,
-  pub failures: Vec<BenchFailure>,
-  pub measurements: Vec<BenchMeasurement>,
+  pub failures: Vec<(BenchDescription, Box<JsError>)>,
+  pub measurements: Vec<(BenchDescription, BenchStats)>,
 }
 
 #[derive(Debug, Clone, PartialEq, Deserialize, Eq, Hash)]
-pub struct BenchMetadata {
+pub struct BenchDescription {
+  pub id: usize,
   pub name: String,
   pub origin: String,
   pub baseline: bool,
@@ -96,22 +95,6 @@ pub struct BenchMetadata {
 }
 
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub struct BenchMeasurement {
-  pub name: String,
-  pub baseline: bool,
-  pub stats: BenchStats,
-  pub group: Option<String>,
-}
-
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
-pub struct BenchFailure {
-  pub name: String,
-  pub error: String,
-  pub baseline: bool,
-  pub group: Option<String>,
-}
-
-#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct BenchStats {
   pub n: u64,
   pub min: f64,
@@ -142,9 +125,10 @@ pub trait BenchReporter {
   fn report_group_summary(&mut self);
   fn report_plan(&mut self, plan: &BenchPlan);
   fn report_end(&mut self, report: &BenchReport);
-  fn report_wait(&mut self, wait: &BenchMetadata);
-  fn report_output(&mut self, output: &BenchOutput);
-  fn report_result(&mut self, result: &BenchResult);
+  fn report_register(&mut self, desc: &BenchDescription);
+  fn report_wait(&mut self, desc: &BenchDescription);
+  fn report_output(&mut self, output: &str);
+  fn report_result(&mut self, desc: &BenchDescription, result: &BenchResult);
 }
 
 struct ConsoleReporter {
@@ -152,8 +136,8 @@ struct ConsoleReporter {
   show_output: bool,
   has_ungrouped: bool,
   group: Option<String>,
-  baseline: Option<BenchMeasurement>,
-  group_measurements: Vec<BenchMeasurement>,
+  baseline: bool,
+  group_measurements: Vec<(BenchDescription, BenchStats)>,
   options: Option<mitata::reporter::Options>,
 }
 
@@ -163,7 +147,7 @@ impl ConsoleReporter {
       show_output,
       group: None,
       options: None,
-      baseline: None,
+      baseline: false,
       name: String::new(),
       has_ungrouped: false,
       group_measurements: Vec::new(),
@@ -181,7 +165,7 @@ impl BenchReporter for ConsoleReporter {
     self.report_group_summary();
 
     self.group = None;
-    self.baseline = None;
+    self.baseline = false;
     self.name = String::new();
     self.group_measurements.clear();
     self.options = Some(mitata::reporter::Options::new(
@@ -218,10 +202,12 @@ impl BenchReporter for ConsoleReporter {
     );
   }
 
-  fn report_wait(&mut self, wait: &BenchMetadata) {
-    self.name = wait.name.clone();
+  fn report_register(&mut self, _desc: &BenchDescription) {}
+
+  fn report_wait(&mut self, desc: &BenchDescription) {
+    self.name = desc.name.clone();
 
-    match &wait.group {
+    match &desc.group {
       None => {
         self.has_ungrouped = true;
       }
@@ -249,56 +235,52 @@ impl BenchReporter for ConsoleReporter {
     }
   }
 
-  fn report_output(&mut self, output: &BenchOutput) {
+  fn report_output(&mut self, output: &str) {
     if self.show_output {
-      match output {
-        BenchOutput::Console(line) => {
-          print!("{} {}", colors::gray(format!("{}:", self.name)), line)
-        }
-      }
+      print!("{} {}", colors::gray(format!("{}:", self.name)), output)
     }
   }
 
-  fn report_result(&mut self, result: &BenchResult) {
+  fn report_result(&mut self, desc: &BenchDescription, result: &BenchResult) {
     let options = self.options.as_ref().unwrap();
 
     match result {
-      BenchResult::Ok(bench) => {
-        let mut bench = bench.to_owned();
+      BenchResult::Ok(stats) => {
+        let mut desc = desc.clone();
 
-        if bench.baseline && self.baseline.is_none() {
-          self.baseline = Some(bench.clone());
+        if desc.baseline && !self.baseline {
+          self.baseline = true;
         } else {
-          bench.baseline = false;
+          desc.baseline = false;
         }
 
-        self.group_measurements.push(bench.clone());
-
         println!(
           "{}",
           mitata::reporter::benchmark(
-            &bench.name,
+            &desc.name,
             &mitata::reporter::BenchmarkStats {
-              avg: bench.stats.avg,
-              min: bench.stats.min,
-              max: bench.stats.max,
-              p75: bench.stats.p75,
-              p99: bench.stats.p99,
-              p995: bench.stats.p995,
+              avg: stats.avg,
+              min: stats.min,
+              max: stats.max,
+              p75: stats.p75,
+              p99: stats.p99,
+              p995: stats.p995,
             },
             options
           )
         );
+
+        self.group_measurements.push((desc, stats.clone()));
       }
 
-      BenchResult::Failed(failure) => {
+      BenchResult::Failed(js_error) => {
         println!(
           "{}",
           mitata::reporter::benchmark_error(
-            &failure.name,
+            &desc.name,
             &mitata::reporter::Error {
               stack: None,
-              message: failure.error.clone(),
+              message: format_test_error(js_error),
             },
             options
           )
@@ -314,8 +296,7 @@ impl BenchReporter for ConsoleReporter {
     };
 
     if 2 <= self.group_measurements.len()
-      && (self.group.is_some()
-        || (self.group.is_none() && self.baseline.is_some()))
+      && (self.group.is_some() || (self.group.is_none() && self.baseline))
     {
       println!(
         "\n{}",
@@ -323,18 +304,18 @@ impl BenchReporter for ConsoleReporter {
           &self
             .group_measurements
             .iter()
-            .map(|b| mitata::reporter::GroupBenchmark {
-              name: b.name.clone(),
-              baseline: b.baseline,
-              group: b.group.as_deref().unwrap_or("").to_owned(),
+            .map(|(d, s)| mitata::reporter::GroupBenchmark {
+              name: d.name.clone(),
+              baseline: d.baseline,
+              group: d.group.as_deref().unwrap_or("").to_owned(),
 
               stats: mitata::reporter::BenchmarkStats {
-                avg: b.stats.avg,
-                min: b.stats.min,
-                max: b.stats.max,
-                p75: b.stats.p75,
-                p99: b.stats.p99,
-                p995: b.stats.p995,
+                avg: s.avg,
+                min: s.min,
+                max: s.max,
+                p75: s.p75,
+                p99: s.p99,
+                p995: s.p995,
               },
             })
             .collect::<Vec<mitata::reporter::GroupBenchmark>>(),
@@ -343,7 +324,7 @@ impl BenchReporter for ConsoleReporter {
       );
     }
 
-    self.baseline = None;
+    self.baseline = false;
     self.group_measurements.clear();
   }
 
@@ -380,11 +361,12 @@ async fn bench_specifier(
   channel: UnboundedSender<BenchEvent>,
   options: BenchSpecifierOptions,
 ) -> Result<(), AnyError> {
+  let filter = TestFilter::from_flag(&options.filter);
   let mut worker = create_main_worker(
     &ps,
     specifier.clone(),
     permissions,
-    vec![ops::bench::init(channel.clone(), ps.flags.unstable)],
+    vec![ops::bench::init(channel.clone(), filter, ps.flags.unstable)],
     Default::default(),
   );
 
@@ -413,12 +395,7 @@ async fn bench_specifier(
 
   let bench_result = worker.js_runtime.execute_script(
     &located_script_name!(),
-    &format!(
-      r#"Deno[Deno.internal].runBenchmarks({})"#,
-      json!({
-        "filter": options.filter,
-      }),
-    ),
+    r#"Deno[Deno.internal].runBenchmarks()"#,
   )?;
 
   worker.js_runtime.resolve_value(bench_result).await?;
@@ -462,6 +439,7 @@ async fn bench_specifiers(
       let mut used_only = false;
       let mut report = BenchReport::new();
       let mut reporter = create_reporter(log_level != Some(Level::Error));
+      let mut benches = IndexMap::new();
 
       while let Some(event) = receiver.recv().await {
         match event {
@@ -474,27 +452,32 @@ async fn bench_specifiers(
             reporter.report_plan(&plan);
           }
 
-          BenchEvent::Wait(metadata) => {
-            reporter.report_wait(&metadata);
+          BenchEvent::Register(desc) => {
+            reporter.report_register(&desc);
+            benches.insert(desc.id, desc);
+          }
+
+          BenchEvent::Wait(id) => {
+            reporter.report_wait(benches.get(&id).unwrap());
           }
 
           BenchEvent::Output(output) => {
             reporter.report_output(&output);
           }
 
-          BenchEvent::Result(_origin, result) => {
-            match &result {
-              BenchResult::Ok(bench) => {
-                report.measurements.push(bench.clone());
+          BenchEvent::Result(id, result) => {
+            let desc = benches.get(&id).unwrap();
+            reporter.report_result(desc, &result);
+            match result {
+              BenchResult::Ok(stats) => {
+                report.measurements.push((desc.clone(), stats));
               }
 
               BenchResult::Failed(failure) => {
                 report.failed += 1;
-                report.failures.push(failure.clone());
+                report.failures.push((desc.clone(), failure));
               }
             };
-
-            reporter.report_result(&result);
           }
         }
       }
diff --git a/cli/tools/test.rs b/cli/tools/test.rs
index 4424ee28d..7dcc23dd1 100644
--- a/cli/tools/test.rs
+++ b/cli/tools/test.rs
@@ -77,6 +77,58 @@ pub enum TestMode {
   Both,
 }
 
+// TODO(nayeemrmn): This is only used for benches right now.
+#[derive(Clone, Debug, Default)]
+pub struct TestFilter {
+  pub substring: Option<String>,
+  pub regex: Option<Regex>,
+  pub include: Option<Vec<String>>,
+  pub exclude: Vec<String>,
+}
+
+impl TestFilter {
+  pub fn includes(&self, name: &String) -> bool {
+    if let Some(substring) = &self.substring {
+      if !name.contains(substring) {
+        return false;
+      }
+    }
+    if let Some(regex) = &self.regex {
+      if !regex.is_match(name) {
+        return false;
+      }
+    }
+    if let Some(include) = &self.include {
+      if !include.contains(name) {
+        return false;
+      }
+    }
+    if self.exclude.contains(name) {
+      return false;
+    }
+    true
+  }
+
+  pub fn from_flag(flag: &Option<String>) -> Self {
+    let mut substring = None;
+    let mut regex = None;
+    if let Some(flag) = flag {
+      if flag.starts_with('/') && flag.ends_with('/') {
+        let rs = flag.trim_start_matches('/').trim_end_matches('/');
+        regex =
+          Some(Regex::new(rs).unwrap_or_else(|_| Regex::new("$^").unwrap()));
+      } else {
+        substring = Some(flag.clone());
+      }
+    }
+    Self {
+      substring,
+      regex,
+      ..Default::default()
+    }
+  }
+}
+
 #[derive(Debug, Clone, PartialEq, Deserialize, Eq, Hash)]
 #[serde(rename_all = "camelCase")]
 pub struct TestLocation {