diff options
Diffstat (limited to 'cli/tools/bench.rs')
| -rw-r--r-- | cli/tools/bench.rs | 173 |
1 files changed, 78 insertions, 95 deletions
diff --git a/cli/tools/bench.rs b/cli/tools/bench.rs index afd85a8d0..d9e9c38c3 100644 --- a/cli/tools/bench.rs +++ b/cli/tools/bench.rs @@ -21,18 +21,21 @@ use crate::ops; use crate::proc_state::ProcState; use crate::resolver::ImportMapResolver; use crate::resolver::JsxResolver; +use crate::tools::test::format_test_error; +use crate::tools::test::TestFilter; use deno_core::error::generic_error; use deno_core::error::AnyError; +use deno_core::error::JsError; use deno_core::futures::future; use deno_core::futures::stream; use deno_core::futures::FutureExt; use deno_core::futures::StreamExt; -use deno_core::serde_json::json; use deno_core::ModuleSpecifier; use deno_graph::ModuleKind; use deno_runtime::permissions::Permissions; use deno_runtime::tokio_util::run_basic; +use indexmap::IndexMap; use log::Level; use serde::Deserialize; use serde::Serialize; @@ -50,12 +53,6 @@ struct BenchSpecifierOptions { #[derive(Debug, Clone, PartialEq, Deserialize)] #[serde(rename_all = "camelCase")] -pub enum BenchOutput { - Console(String), -} - -#[derive(Debug, Clone, PartialEq, Deserialize)] -#[serde(rename_all = "camelCase")] pub struct BenchPlan { pub total: usize, pub origin: String, @@ -67,28 +64,30 @@ pub struct BenchPlan { #[serde(rename_all = "camelCase")] pub enum BenchEvent { Plan(BenchPlan), - Output(BenchOutput), - Wait(BenchMetadata), - Result(String, BenchResult), + Output(String), + Register(BenchDescription), + Wait(usize), + Result(usize, BenchResult), } #[derive(Debug, Clone, PartialEq, Deserialize)] #[serde(rename_all = "camelCase")] pub enum BenchResult { - Ok(BenchMeasurement), - Failed(BenchFailure), + Ok(BenchStats), + Failed(Box<JsError>), } -#[derive(Debug, Clone, Serialize)] +#[derive(Debug, Clone)] pub struct BenchReport { pub total: usize, pub failed: usize, - pub failures: Vec<BenchFailure>, - pub measurements: Vec<BenchMeasurement>, + pub failures: Vec<(BenchDescription, Box<JsError>)>, + pub measurements: Vec<(BenchDescription, BenchStats)>, } #[derive(Debug, Clone, PartialEq, Deserialize, Eq, Hash)] -pub struct BenchMetadata { +pub struct BenchDescription { + pub id: usize, pub name: String, pub origin: String, pub baseline: bool, @@ -96,22 +95,6 @@ pub struct BenchMetadata { } #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct BenchMeasurement { - pub name: String, - pub baseline: bool, - pub stats: BenchStats, - pub group: Option<String>, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] -pub struct BenchFailure { - pub name: String, - pub error: String, - pub baseline: bool, - pub group: Option<String>, -} - -#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] pub struct BenchStats { pub n: u64, pub min: f64, @@ -142,9 +125,10 @@ pub trait BenchReporter { fn report_group_summary(&mut self); fn report_plan(&mut self, plan: &BenchPlan); fn report_end(&mut self, report: &BenchReport); - fn report_wait(&mut self, wait: &BenchMetadata); - fn report_output(&mut self, output: &BenchOutput); - fn report_result(&mut self, result: &BenchResult); + fn report_register(&mut self, desc: &BenchDescription); + fn report_wait(&mut self, desc: &BenchDescription); + fn report_output(&mut self, output: &str); + fn report_result(&mut self, desc: &BenchDescription, result: &BenchResult); } struct ConsoleReporter { @@ -152,8 +136,8 @@ struct ConsoleReporter { show_output: bool, has_ungrouped: bool, group: Option<String>, - baseline: Option<BenchMeasurement>, - group_measurements: Vec<BenchMeasurement>, + baseline: bool, + group_measurements: Vec<(BenchDescription, BenchStats)>, options: Option<mitata::reporter::Options>, } @@ -163,7 +147,7 @@ impl ConsoleReporter { show_output, group: None, options: None, - baseline: None, + baseline: false, name: String::new(), has_ungrouped: false, group_measurements: Vec::new(), @@ -181,7 +165,7 @@ impl BenchReporter for ConsoleReporter { self.report_group_summary(); self.group = None; - self.baseline = None; + self.baseline = false; self.name = String::new(); self.group_measurements.clear(); self.options = Some(mitata::reporter::Options::new( @@ -218,10 +202,12 @@ impl BenchReporter for ConsoleReporter { ); } - fn report_wait(&mut self, wait: &BenchMetadata) { - self.name = wait.name.clone(); + fn report_register(&mut self, _desc: &BenchDescription) {} + + fn report_wait(&mut self, desc: &BenchDescription) { + self.name = desc.name.clone(); - match &wait.group { + match &desc.group { None => { self.has_ungrouped = true; } @@ -249,56 +235,52 @@ impl BenchReporter for ConsoleReporter { } } - fn report_output(&mut self, output: &BenchOutput) { + fn report_output(&mut self, output: &str) { if self.show_output { - match output { - BenchOutput::Console(line) => { - print!("{} {}", colors::gray(format!("{}:", self.name)), line) - } - } + print!("{} {}", colors::gray(format!("{}:", self.name)), output) } } - fn report_result(&mut self, result: &BenchResult) { + fn report_result(&mut self, desc: &BenchDescription, result: &BenchResult) { let options = self.options.as_ref().unwrap(); match result { - BenchResult::Ok(bench) => { - let mut bench = bench.to_owned(); + BenchResult::Ok(stats) => { + let mut desc = desc.clone(); - if bench.baseline && self.baseline.is_none() { - self.baseline = Some(bench.clone()); + if desc.baseline && !self.baseline { + self.baseline = true; } else { - bench.baseline = false; + desc.baseline = false; } - self.group_measurements.push(bench.clone()); - println!( "{}", mitata::reporter::benchmark( - &bench.name, + &desc.name, &mitata::reporter::BenchmarkStats { - avg: bench.stats.avg, - min: bench.stats.min, - max: bench.stats.max, - p75: bench.stats.p75, - p99: bench.stats.p99, - p995: bench.stats.p995, + avg: stats.avg, + min: stats.min, + max: stats.max, + p75: stats.p75, + p99: stats.p99, + p995: stats.p995, }, options ) ); + + self.group_measurements.push((desc, stats.clone())); } - BenchResult::Failed(failure) => { + BenchResult::Failed(js_error) => { println!( "{}", mitata::reporter::benchmark_error( - &failure.name, + &desc.name, &mitata::reporter::Error { stack: None, - message: failure.error.clone(), + message: format_test_error(js_error), }, options ) @@ -314,8 +296,7 @@ impl BenchReporter for ConsoleReporter { }; if 2 <= self.group_measurements.len() - && (self.group.is_some() - || (self.group.is_none() && self.baseline.is_some())) + && (self.group.is_some() || (self.group.is_none() && self.baseline)) { println!( "\n{}", @@ -323,18 +304,18 @@ impl BenchReporter for ConsoleReporter { &self .group_measurements .iter() - .map(|b| mitata::reporter::GroupBenchmark { - name: b.name.clone(), - baseline: b.baseline, - group: b.group.as_deref().unwrap_or("").to_owned(), + .map(|(d, s)| mitata::reporter::GroupBenchmark { + name: d.name.clone(), + baseline: d.baseline, + group: d.group.as_deref().unwrap_or("").to_owned(), stats: mitata::reporter::BenchmarkStats { - avg: b.stats.avg, - min: b.stats.min, - max: b.stats.max, - p75: b.stats.p75, - p99: b.stats.p99, - p995: b.stats.p995, + avg: s.avg, + min: s.min, + max: s.max, + p75: s.p75, + p99: s.p99, + p995: s.p995, }, }) .collect::<Vec<mitata::reporter::GroupBenchmark>>(), @@ -343,7 +324,7 @@ impl BenchReporter for ConsoleReporter { ); } - self.baseline = None; + self.baseline = false; self.group_measurements.clear(); } @@ -380,11 +361,12 @@ async fn bench_specifier( channel: UnboundedSender<BenchEvent>, options: BenchSpecifierOptions, ) -> Result<(), AnyError> { + let filter = TestFilter::from_flag(&options.filter); let mut worker = create_main_worker( &ps, specifier.clone(), permissions, - vec![ops::bench::init(channel.clone(), ps.flags.unstable)], + vec![ops::bench::init(channel.clone(), filter, ps.flags.unstable)], Default::default(), ); @@ -413,12 +395,7 @@ async fn bench_specifier( let bench_result = worker.js_runtime.execute_script( &located_script_name!(), - &format!( - r#"Deno[Deno.internal].runBenchmarks({})"#, - json!({ - "filter": options.filter, - }), - ), + r#"Deno[Deno.internal].runBenchmarks()"#, )?; worker.js_runtime.resolve_value(bench_result).await?; @@ -462,6 +439,7 @@ async fn bench_specifiers( let mut used_only = false; let mut report = BenchReport::new(); let mut reporter = create_reporter(log_level != Some(Level::Error)); + let mut benches = IndexMap::new(); while let Some(event) = receiver.recv().await { match event { @@ -474,27 +452,32 @@ async fn bench_specifiers( reporter.report_plan(&plan); } - BenchEvent::Wait(metadata) => { - reporter.report_wait(&metadata); + BenchEvent::Register(desc) => { + reporter.report_register(&desc); + benches.insert(desc.id, desc); + } + + BenchEvent::Wait(id) => { + reporter.report_wait(benches.get(&id).unwrap()); } BenchEvent::Output(output) => { reporter.report_output(&output); } - BenchEvent::Result(_origin, result) => { - match &result { - BenchResult::Ok(bench) => { - report.measurements.push(bench.clone()); + BenchEvent::Result(id, result) => { + let desc = benches.get(&id).unwrap(); + reporter.report_result(desc, &result); + match result { + BenchResult::Ok(stats) => { + report.measurements.push((desc.clone(), stats)); } BenchResult::Failed(failure) => { report.failed += 1; - report.failures.push(failure.clone()); + report.failures.push((desc.clone(), failure)); } }; - - reporter.report_result(&result); } } } |