summaryrefslogtreecommitdiff
path: root/cli/bench/main.rs
diff options
context:
space:
mode:
Diffstat (limited to 'cli/bench/main.rs')
-rw-r--r--cli/bench/main.rs257
1 files changed, 149 insertions, 108 deletions
diff --git a/cli/bench/main.rs b/cli/bench/main.rs
index d6d5259f6..3e7cc61fc 100644
--- a/cli/bench/main.rs
+++ b/cli/bench/main.rs
@@ -3,7 +3,6 @@
use deno_core::error::AnyError;
use deno_core::serde_json;
use deno_core::serde_json::Value;
-use serde::Serialize;
use std::collections::HashMap;
use std::convert::From;
use std::env;
@@ -16,6 +15,7 @@ use std::time::SystemTime;
mod http;
mod lsp;
+mod metrics;
fn read_json(filename: &str) -> Result<Value> {
let f = fs::File::open(filename)?;
@@ -162,7 +162,7 @@ const RESULT_KEYS: &[&str] =
fn run_exec_time(
deno_exe: &Path,
target_dir: &Path,
-) -> Result<HashMap<String, HashMap<String, f64>>> {
+) -> Result<HashMap<String, HashMap<String, i64>>> {
let hyperfine_exe = test_util::prebuilt_tool_path("hyperfine");
let benchmark_file = target_dir.join("hyperfine_results.json");
@@ -203,7 +203,7 @@ fn run_exec_time(
true,
);
- let mut results = HashMap::<String, HashMap<String, f64>>::new();
+ let mut results = HashMap::<String, HashMap<String, i64>>::new();
let hyperfine_results = read_json(benchmark_file)?;
for ((name, _, _), data) in EXEC_TIME_BENCHMARKS.iter().zip(
hyperfine_results
@@ -220,7 +220,7 @@ fn run_exec_time(
data
.into_iter()
.filter(|(key, _)| RESULT_KEYS.contains(&key.as_str()))
- .map(|(key, val)| (key, val.as_f64().unwrap()))
+ .map(|(key, val)| (key, val.as_f64().unwrap() as i64))
.collect(),
);
}
@@ -228,7 +228,7 @@ fn run_exec_time(
Ok(results)
}
-fn rlib_size(target_dir: &std::path::Path, prefix: &str) -> u64 {
+fn rlib_size(target_dir: &std::path::Path, prefix: &str) -> i64 {
let mut size = 0;
let mut seen = std::collections::HashSet::new();
for entry in std::fs::read_dir(target_dir.join("deps")).unwrap() {
@@ -247,18 +247,18 @@ fn rlib_size(target_dir: &std::path::Path, prefix: &str) -> u64 {
}
}
assert!(size > 0);
- size
+ size as i64
}
const BINARY_TARGET_FILES: &[&str] =
&["CLI_SNAPSHOT.bin", "COMPILER_SNAPSHOT.bin"];
-fn get_binary_sizes(target_dir: &Path) -> Result<HashMap<String, u64>> {
- let mut sizes = HashMap::<String, u64>::new();
+fn get_binary_sizes(target_dir: &Path) -> Result<HashMap<String, i64>> {
+ let mut sizes = HashMap::<String, i64>::new();
let mut mtimes = HashMap::<String, SystemTime>::new();
sizes.insert(
"deno".to_string(),
- test_util::deno_exe_path().metadata()?.len(),
+ test_util::deno_exe_path().metadata()?.len() as i64,
);
// add up size for everything in target/release/deps/libswc*
@@ -294,7 +294,7 @@ fn get_binary_sizes(target_dir: &Path) -> Result<HashMap<String, u64>> {
}
mtimes.insert(filename.clone(), file_mtime);
- sizes.insert(filename, meta.len());
+ sizes.insert(filename, meta.len() as i64);
}
Ok(sizes)
@@ -304,8 +304,8 @@ const BUNDLES: &[(&str, &str)] = &[
("file_server", "./test_util/std/http/file_server.ts"),
("gist", "./test_util/std/examples/gist.ts"),
];
-fn bundle_benchmark(deno_exe: &Path) -> Result<HashMap<String, u64>> {
- let mut sizes = HashMap::<String, u64>::new();
+fn bundle_benchmark(deno_exe: &Path) -> Result<HashMap<String, i64>> {
+ let mut sizes = HashMap::<String, i64>::new();
for (name, url) in BUNDLES {
let path = format!("{}.bundle.js", name);
@@ -325,74 +325,15 @@ fn bundle_benchmark(deno_exe: &Path) -> Result<HashMap<String, u64>> {
let file = PathBuf::from(path);
assert!(file.is_file());
- sizes.insert(name.to_string(), file.metadata()?.len());
+ sizes.insert(name.to_string(), file.metadata()?.len() as i64);
let _ = fs::remove_file(file);
}
Ok(sizes)
}
-fn run_http(target_dir: &Path, new_data: &mut BenchResult) -> Result<()> {
- let stats = http::benchmark(target_dir)?;
-
- new_data.req_per_sec = stats
- .iter()
- .map(|(name, result)| (name.clone(), result.requests))
- .collect();
-
- new_data.max_latency = stats
- .iter()
- .map(|(name, result)| (name.clone(), result.latency))
- .collect();
-
- Ok(())
-}
-
-fn run_strace_benchmarks(
- deno_exe: &Path,
- new_data: &mut BenchResult,
-) -> Result<()> {
- use std::io::Read;
-
- let mut thread_count = HashMap::<String, u64>::new();
- let mut syscall_count = HashMap::<String, u64>::new();
-
- for (name, args, expected_exit_code) in EXEC_TIME_BENCHMARKS {
- let mut file = secure_tempfile::NamedTempFile::new()?;
-
- let exit_status = Command::new("strace")
- .args(&[
- "-c",
- "-f",
- "-o",
- file.path().to_str().unwrap(),
- deno_exe.to_str().unwrap(),
- ])
- .args(args.iter())
- .stdout(Stdio::null())
- .spawn()?
- .wait()?;
- let expected_exit_code = expected_exit_code.unwrap_or(0);
- assert_eq!(exit_status.code(), Some(expected_exit_code));
-
- let mut output = String::new();
- file.as_file_mut().read_to_string(&mut output)?;
-
- let strace_result = test_util::parse_strace_output(&output);
- let clone = strace_result.get("clone").map(|d| d.calls).unwrap_or(0) + 1;
- let total = strace_result.get("total").unwrap().calls;
- thread_count.insert(name.to_string(), clone);
- syscall_count.insert(name.to_string(), total);
- }
-
- new_data.thread_count = thread_count;
- new_data.syscall_count = syscall_count;
-
- Ok(())
-}
-
-fn run_max_mem_benchmark(deno_exe: &Path) -> Result<HashMap<String, u64>> {
- let mut results = HashMap::<String, u64>::new();
+fn run_max_mem_benchmark(deno_exe: &Path) -> Result<HashMap<String, i64>> {
+ let mut results = HashMap::<String, i64>::new();
for (name, args, return_code) in EXEC_TIME_BENCHMARKS {
let proc = Command::new("time")
@@ -408,7 +349,10 @@ fn run_max_mem_benchmark(deno_exe: &Path) -> Result<HashMap<String, u64>> {
}
let out = String::from_utf8(proc_result.stderr)?;
- results.insert(name.to_string(), test_util::parse_max_mem(&out).unwrap());
+ results.insert(
+ name.to_string(),
+ test_util::parse_max_mem(&out).unwrap() as i64,
+ );
}
Ok(results)
@@ -429,7 +373,8 @@ fn cargo_deps() -> usize {
count
}
-#[derive(Default, Serialize)]
+// TODO(@littledivy): Remove this, denoland/benchmark_data is deprecated.
+#[derive(Default, serde::Serialize)]
struct BenchResult {
created_at: String,
sha1: String,
@@ -437,27 +382,44 @@ struct BenchResult {
// TODO(ry) The "benchmark" benchmark should actually be called "exec_time".
// When this is changed, the historical data in gh-pages branch needs to be
// changed too.
- benchmark: HashMap<String, HashMap<String, f64>>,
- binary_size: HashMap<String, u64>,
- bundle_size: HashMap<String, u64>,
+ benchmark: HashMap<String, HashMap<String, i64>>,
+ binary_size: HashMap<String, i64>,
+ bundle_size: HashMap<String, i64>,
cargo_deps: usize,
- max_latency: HashMap<String, f64>,
- max_memory: HashMap<String, u64>,
- lsp_exec_time: HashMap<String, u64>,
- req_per_sec: HashMap<String, u64>,
- syscall_count: HashMap<String, u64>,
- thread_count: HashMap<String, u64>,
+ max_latency: HashMap<String, i64>,
+ max_memory: HashMap<String, i64>,
+ lsp_exec_time: HashMap<String, i64>,
+ req_per_sec: HashMap<String, i64>,
+ syscall_count: HashMap<String, i64>,
+ thread_count: HashMap<String, i64>,
}
-/*
- TODO(SyrupThinker)
- Switch to the #[bench] attribute once
- it is stabilized.
- Before that the #[test] tests won't be run because
- we replace the harness with our own runner here.
-*/
-fn main() -> Result<()> {
- if !env::args().any(|s| s == "--bench") {
+#[tokio::main]
+async fn main() -> Result<()> {
+ let mut args = env::args();
+
+ let mut benchmarks = vec![
+ "bundle",
+ "exec_time",
+ "binary_size",
+ "cargo_deps",
+ "lsp",
+ "http",
+ "strace",
+ "mem_usage",
+ ];
+
+ let mut found_bench = false;
+ let filter = args.nth(1);
+ if let Some(filter) = filter {
+ if filter != "--bench" {
+ benchmarks.retain(|s| s == &filter);
+ } else {
+ found_bench = true;
+ }
+ }
+
+ if !found_bench && !args.any(|s| s == "--bench") {
return Ok(());
}
@@ -465,7 +427,6 @@ fn main() -> Result<()> {
let target_dir = test_util::target_dir();
let deno_exe = test_util::deno_exe_path();
-
env::set_current_dir(&test_util::root_path())?;
let mut new_data = BenchResult {
@@ -481,27 +442,107 @@ fn main() -> Result<()> {
.0
.trim()
.to_string(),
- benchmark: run_exec_time(&deno_exe, &target_dir)?,
- binary_size: get_binary_sizes(&target_dir)?,
- bundle_size: bundle_benchmark(&deno_exe)?,
- cargo_deps: cargo_deps(),
- lsp_exec_time: lsp::benchmarks(&deno_exe)?,
..Default::default()
};
- if cfg!(not(target_os = "windows")) {
- run_http(&target_dir, &mut new_data)?;
+ let mut reporter = metrics::Reporter::new().await;
+
+ if benchmarks.contains(&"bundle") {
+ let bundle_size = bundle_benchmark(&deno_exe)?;
+ reporter.write("bundle_size", &bundle_size);
+ new_data.bundle_size = bundle_size;
+ }
+
+ if benchmarks.contains(&"exec_time") {
+ let exec_times = run_exec_time(&deno_exe, &target_dir)?;
+ for (name, data) in exec_times.iter() {
+ reporter.write_one("exec_time", name, *data.get("mean").unwrap());
+ }
+ new_data.benchmark = exec_times;
+ }
+
+ if benchmarks.contains(&"binary_size") {
+ let binary_sizes = get_binary_sizes(&target_dir)?;
+ reporter.write("binary_size", &binary_sizes);
+ new_data.binary_size = binary_sizes;
+ }
+
+ if benchmarks.contains(&"cargo_deps") {
+ let cargo_deps = cargo_deps();
+ reporter.write_one("cargo_deps", "cargo_deps", cargo_deps as i64);
+ new_data.cargo_deps = cargo_deps;
+ }
+
+ if benchmarks.contains(&"lsp") {
+ let lsp_exec_times = lsp::benchmarks(&deno_exe)?;
+ reporter.write("lsp_exec_time", &lsp_exec_times);
+ new_data.lsp_exec_time = lsp_exec_times;
}
- if cfg!(target_os = "linux") {
- run_strace_benchmarks(&deno_exe, &mut new_data)?;
- new_data.max_memory = run_max_mem_benchmark(&deno_exe)?;
+ if benchmarks.contains(&"http") && cfg!(not(target_os = "windows")) {
+ let stats = http::benchmark(&target_dir)?;
+ let req_per_sec = stats
+ .iter()
+ .map(|(name, result)| (name.clone(), result.requests as i64))
+ .collect();
+ reporter.write("req_per_sec", &req_per_sec);
+ new_data.req_per_sec = req_per_sec;
+ let max_latency = stats
+ .iter()
+ .map(|(name, result)| (name.clone(), result.latency as i64))
+ .collect();
+
+ reporter.write("max_latency", &max_latency);
+ new_data.max_latency = max_latency;
}
- println!("===== <BENCHMARK RESULTS>");
- serde_json::to_writer_pretty(std::io::stdout(), &new_data)?;
- println!("\n===== </BENCHMARK RESULTS>");
+ if cfg!(target_os = "linux") && benchmarks.contains(&"strace") {
+ use std::io::Read;
+
+ let mut thread_count = HashMap::<String, i64>::new();
+ let mut syscall_count = HashMap::<String, i64>::new();
+
+ for (name, args, expected_exit_code) in EXEC_TIME_BENCHMARKS {
+ let mut file = secure_tempfile::NamedTempFile::new()?;
+
+ let exit_status = Command::new("strace")
+ .args(&[
+ "-c",
+ "-f",
+ "-o",
+ file.path().to_str().unwrap(),
+ deno_exe.to_str().unwrap(),
+ ])
+ .args(args.iter())
+ .stdout(Stdio::null())
+ .spawn()?
+ .wait()?;
+ let expected_exit_code = expected_exit_code.unwrap_or(0);
+ assert_eq!(exit_status.code(), Some(expected_exit_code));
+
+ let mut output = String::new();
+ file.as_file_mut().read_to_string(&mut output)?;
+
+ let strace_result = test_util::parse_strace_output(&output);
+ let clone = strace_result.get("clone").map(|d| d.calls).unwrap_or(0) + 1;
+ let total = strace_result.get("total").unwrap().calls;
+ thread_count.insert(name.to_string(), clone as i64);
+ syscall_count.insert(name.to_string(), total as i64);
+ }
+
+ reporter.write("thread_count", &thread_count);
+ new_data.thread_count = thread_count;
+ reporter.write("syscall_count", &syscall_count);
+ new_data.syscall_count = syscall_count;
+ }
+
+ if benchmarks.contains(&"mem_usage") {
+ let max_memory = run_max_mem_benchmark(&deno_exe)?;
+ reporter.write("max_memory", &max_memory);
+ new_data.max_memory = max_memory;
+ }
+ reporter.submit().await;
if let Some(filename) = target_dir.join("bench.json").to_str() {
write_json(filename, &serde_json::to_value(&new_data)?)?;
} else {
generated by cgit v1.2.3 (git 2.25.1) at 2025-06-27 08:45:25 +0000