refactor(bench): continuous benchmarking improvements (#14821)

6 files changed, 269 insertions, 115 deletions

diff --git a/cli/Cargo.toml b/cli/Cargo.toml
index eba22c3e9..d9b760ee5 100644
--- a/cli/Cargo.toml
+++ b/cli/Cargo.toml
@@ -100,8 +100,12 @@ fwdansi = "=1.1.0"
 winapi = { version = "=0.3.9", features = ["knownfolders", "mswsock", "objbase", "shlobj", "tlhelp32", "winbase", "winerror", "winsock2"] }
 
 [dev-dependencies]
+csv = "1.1.6"
 deno_bench_util = { version = "0.52.0", path = "../bench_util" }
+dotenv = "0.15.0"
 flaky_test = "=0.1.0"
+google-storage1 = "3.1.0"
+once_cell = "=1.12.0"
 os_pipe = "=1.0.1"
 pretty_assertions = "=1.2.1"
 test_util = { path = "../test_util" }
diff --git a/cli/bench/README.md b/cli/bench/README.md
new file mode 100644
index 000000000..3117cff15
--- /dev/null
+++ b/cli/bench/README.md
@@ -0,0 +1,10 @@
+benchmark filtering:
+
+```
+cargo bench --bench deno_bench -- bundle
+```
+
+benchmark plots:
+
+new: https://denoland.grafana.net/d/vErC9VCnz/benchmarks?orgId=1 old:
+deno.land/benchmarks
diff --git a/cli/bench/http.rs b/cli/bench/http.rs
index 770642907..3646f472c 100644
--- a/cli/bench/http.rs
+++ b/cli/bench/http.rs
@@ -44,7 +44,7 @@ pub fn benchmark(
     if name.starts_with("node") {
       // node <path> <port>
       res.insert(
-        name,
+        file_stem.to_string(),
         run(
           &["node", path, &port.to_string()],
           port,
@@ -56,7 +56,7 @@ pub fn benchmark(
     } else {
       // deno run -A --unstable <path> <addr>
       res.insert(
-        name,
+        file_stem.to_string(),
         run(
           &[
             deno_exe,
diff --git a/cli/bench/lsp.rs b/cli/bench/lsp.rs
index 8397d23d7..7aa35d099 100644
--- a/cli/bench/lsp.rs
+++ b/cli/bench/lsp.rs
@@ -333,7 +333,7 @@ fn bench_startup_shutdown(deno_exe: &Path) -> Result<Duration, AnyError> {
 }
 
 /// Generate benchmarks for the LSP server.
-pub fn benchmarks(deno_exe: &Path) -> Result<HashMap<String, u64>, AnyError> {
+pub fn benchmarks(deno_exe: &Path) -> Result<HashMap<String, i64>, AnyError> {
   println!("-> Start benchmarking lsp");
   let mut exec_times = HashMap::new();
 
@@ -343,7 +343,7 @@ pub fn benchmarks(deno_exe: &Path) -> Result<HashMap<String, u64>, AnyError> {
     times.push(bench_startup_shutdown(deno_exe)?);
   }
   let mean =
-    (times.iter().sum::<Duration>() / times.len() as u32).as_millis() as u64;
+    (times.iter().sum::<Duration>() / times.len() as u32).as_millis() as i64;
   println!("      ({} runs, mean: {}ms)", times.len(), mean);
   exec_times.insert("startup_shutdown".to_string(), mean);
 
@@ -353,7 +353,7 @@ pub fn benchmarks(deno_exe: &Path) -> Result<HashMap<String, u64>, AnyError> {
     times.push(bench_big_file_edits(deno_exe)?);
   }
   let mean =
-    (times.iter().sum::<Duration>() / times.len() as u32).as_millis() as u64;
+    (times.iter().sum::<Duration>() / times.len() as u32).as_millis() as i64;
   println!("      ({} runs, mean: {}ms)", times.len(), mean);
   exec_times.insert("big_file_edits".to_string(), mean);
 
@@ -363,7 +363,7 @@ pub fn benchmarks(deno_exe: &Path) -> Result<HashMap<String, u64>, AnyError> {
     times.push(bench_find_replace(deno_exe)?);
   }
   let mean =
-    (times.iter().sum::<Duration>() / times.len() as u32).as_millis() as u64;
+    (times.iter().sum::<Duration>() / times.len() as u32).as_millis() as i64;
   println!("      ({} runs, mean: {}ms)", times.len(), mean);
   exec_times.insert("find_replace".to_string(), mean);
 
@@ -373,7 +373,7 @@ pub fn benchmarks(deno_exe: &Path) -> Result<HashMap<String, u64>, AnyError> {
     times.push(bench_code_lens(deno_exe)?);
   }
   let mean =
-    (times.iter().sum::<Duration>() / times.len() as u32).as_millis() as u64;
+    (times.iter().sum::<Duration>() / times.len() as u32).as_millis() as i64;
   println!("      ({} runs, mean: {}ms)", times.len(), mean);
   exec_times.insert("code_lens".to_string(), mean);
 
diff --git a/cli/bench/main.rs b/cli/bench/main.rs
index d6d5259f6..3e7cc61fc 100644
--- a/cli/bench/main.rs
+++ b/cli/bench/main.rs
@@ -3,7 +3,6 @@
 use deno_core::error::AnyError;
 use deno_core::serde_json;
 use deno_core::serde_json::Value;
-use serde::Serialize;
 use std::collections::HashMap;
 use std::convert::From;
 use std::env;
@@ -16,6 +15,7 @@ use std::time::SystemTime;
 
 mod http;
 mod lsp;
+mod metrics;
 
 fn read_json(filename: &str) -> Result<Value> {
   let f = fs::File::open(filename)?;
@@ -162,7 +162,7 @@ const RESULT_KEYS: &[&str] =
 fn run_exec_time(
   deno_exe: &Path,
   target_dir: &Path,
-) -> Result<HashMap<String, HashMap<String, f64>>> {
+) -> Result<HashMap<String, HashMap<String, i64>>> {
   let hyperfine_exe = test_util::prebuilt_tool_path("hyperfine");
 
   let benchmark_file = target_dir.join("hyperfine_results.json");
@@ -203,7 +203,7 @@ fn run_exec_time(
     true,
   );
 
-  let mut results = HashMap::<String, HashMap<String, f64>>::new();
+  let mut results = HashMap::<String, HashMap<String, i64>>::new();
   let hyperfine_results = read_json(benchmark_file)?;
   for ((name, _, _), data) in EXEC_TIME_BENCHMARKS.iter().zip(
     hyperfine_results
@@ -220,7 +220,7 @@ fn run_exec_time(
       data
         .into_iter()
         .filter(|(key, _)| RESULT_KEYS.contains(&key.as_str()))
-        .map(|(key, val)| (key, val.as_f64().unwrap()))
+        .map(|(key, val)| (key, val.as_f64().unwrap() as i64))
         .collect(),
     );
   }
@@ -228,7 +228,7 @@ fn run_exec_time(
   Ok(results)
 }
 
-fn rlib_size(target_dir: &std::path::Path, prefix: &str) -> u64 {
+fn rlib_size(target_dir: &std::path::Path, prefix: &str) -> i64 {
   let mut size = 0;
   let mut seen = std::collections::HashSet::new();
   for entry in std::fs::read_dir(target_dir.join("deps")).unwrap() {
@@ -247,18 +247,18 @@ fn rlib_size(target_dir: &std::path::Path, prefix: &str) -> u64 {
     }
   }
   assert!(size > 0);
-  size
+  size as i64
 }
 
 const BINARY_TARGET_FILES: &[&str] =
   &["CLI_SNAPSHOT.bin", "COMPILER_SNAPSHOT.bin"];
-fn get_binary_sizes(target_dir: &Path) -> Result<HashMap<String, u64>> {
-  let mut sizes = HashMap::<String, u64>::new();
+fn get_binary_sizes(target_dir: &Path) -> Result<HashMap<String, i64>> {
+  let mut sizes = HashMap::<String, i64>::new();
   let mut mtimes = HashMap::<String, SystemTime>::new();
 
   sizes.insert(
     "deno".to_string(),
-    test_util::deno_exe_path().metadata()?.len(),
+    test_util::deno_exe_path().metadata()?.len() as i64,
   );
 
   // add up size for everything in target/release/deps/libswc*
@@ -294,7 +294,7 @@ fn get_binary_sizes(target_dir: &Path) -> Result<HashMap<String, u64>> {
     }
 
     mtimes.insert(filename.clone(), file_mtime);
-    sizes.insert(filename, meta.len());
+    sizes.insert(filename, meta.len() as i64);
   }
 
   Ok(sizes)
@@ -304,8 +304,8 @@ const BUNDLES: &[(&str, &str)] = &[
   ("file_server", "./test_util/std/http/file_server.ts"),
   ("gist", "./test_util/std/examples/gist.ts"),
 ];
-fn bundle_benchmark(deno_exe: &Path) -> Result<HashMap<String, u64>> {
-  let mut sizes = HashMap::<String, u64>::new();
+fn bundle_benchmark(deno_exe: &Path) -> Result<HashMap<String, i64>> {
+  let mut sizes = HashMap::<String, i64>::new();
 
   for (name, url) in BUNDLES {
     let path = format!("{}.bundle.js", name);
@@ -325,74 +325,15 @@ fn bundle_benchmark(deno_exe: &Path) -> Result<HashMap<String, u64>> {
 
     let file = PathBuf::from(path);
     assert!(file.is_file());
-    sizes.insert(name.to_string(), file.metadata()?.len());
+    sizes.insert(name.to_string(), file.metadata()?.len() as i64);
     let _ = fs::remove_file(file);
   }
 
   Ok(sizes)
 }
 
-fn run_http(target_dir: &Path, new_data: &mut BenchResult) -> Result<()> {
-  let stats = http::benchmark(target_dir)?;
-
-  new_data.req_per_sec = stats
-    .iter()
-    .map(|(name, result)| (name.clone(), result.requests))
-    .collect();
-
-  new_data.max_latency = stats
-    .iter()
-    .map(|(name, result)| (name.clone(), result.latency))
-    .collect();
-
-  Ok(())
-}
-
-fn run_strace_benchmarks(
-  deno_exe: &Path,
-  new_data: &mut BenchResult,
-) -> Result<()> {
-  use std::io::Read;
-
-  let mut thread_count = HashMap::<String, u64>::new();
-  let mut syscall_count = HashMap::<String, u64>::new();
-
-  for (name, args, expected_exit_code) in EXEC_TIME_BENCHMARKS {
-    let mut file = secure_tempfile::NamedTempFile::new()?;
-
-    let exit_status = Command::new("strace")
-      .args(&[
-        "-c",
-        "-f",
-        "-o",
-        file.path().to_str().unwrap(),
-        deno_exe.to_str().unwrap(),
-      ])
-      .args(args.iter())
-      .stdout(Stdio::null())
-      .spawn()?
-      .wait()?;
-    let expected_exit_code = expected_exit_code.unwrap_or(0);
-    assert_eq!(exit_status.code(), Some(expected_exit_code));
-
-    let mut output = String::new();
-    file.as_file_mut().read_to_string(&mut output)?;
-
-    let strace_result = test_util::parse_strace_output(&output);
-    let clone = strace_result.get("clone").map(|d| d.calls).unwrap_or(0) + 1;
-    let total = strace_result.get("total").unwrap().calls;
-    thread_count.insert(name.to_string(), clone);
-    syscall_count.insert(name.to_string(), total);
-  }
-
-  new_data.thread_count = thread_count;
-  new_data.syscall_count = syscall_count;
-
-  Ok(())
-}
-
-fn run_max_mem_benchmark(deno_exe: &Path) -> Result<HashMap<String, u64>> {
-  let mut results = HashMap::<String, u64>::new();
+fn run_max_mem_benchmark(deno_exe: &Path) -> Result<HashMap<String, i64>> {
+  let mut results = HashMap::<String, i64>::new();
 
   for (name, args, return_code) in EXEC_TIME_BENCHMARKS {
     let proc = Command::new("time")
@@ -408,7 +349,10 @@ fn run_max_mem_benchmark(deno_exe: &Path) -> Result<HashMap<String, u64>> {
     }
     let out = String::from_utf8(proc_result.stderr)?;
 
-    results.insert(name.to_string(), test_util::parse_max_mem(&out).unwrap());
+    results.insert(
+      name.to_string(),
+      test_util::parse_max_mem(&out).unwrap() as i64,
+    );
   }
 
   Ok(results)
@@ -429,7 +373,8 @@ fn cargo_deps() -> usize {
   count
 }
 
-#[derive(Default, Serialize)]
+// TODO(@littledivy): Remove this, denoland/benchmark_data is deprecated.
+#[derive(Default, serde::Serialize)]
 struct BenchResult {
   created_at: String,
   sha1: String,
@@ -437,27 +382,44 @@ struct BenchResult {
   // TODO(ry) The "benchmark" benchmark should actually be called "exec_time".
   // When this is changed, the historical data in gh-pages branch needs to be
   // changed too.
-  benchmark: HashMap<String, HashMap<String, f64>>,
-  binary_size: HashMap<String, u64>,
-  bundle_size: HashMap<String, u64>,
+  benchmark: HashMap<String, HashMap<String, i64>>,
+  binary_size: HashMap<String, i64>,
+  bundle_size: HashMap<String, i64>,
   cargo_deps: usize,
-  max_latency: HashMap<String, f64>,
-  max_memory: HashMap<String, u64>,
-  lsp_exec_time: HashMap<String, u64>,
-  req_per_sec: HashMap<String, u64>,
-  syscall_count: HashMap<String, u64>,
-  thread_count: HashMap<String, u64>,
+  max_latency: HashMap<String, i64>,
+  max_memory: HashMap<String, i64>,
+  lsp_exec_time: HashMap<String, i64>,
+  req_per_sec: HashMap<String, i64>,
+  syscall_count: HashMap<String, i64>,
+  thread_count: HashMap<String, i64>,
 }
 
-/*
- TODO(SyrupThinker)
- Switch to the #[bench] attribute once
- it is stabilized.
- Before that the #[test] tests won't be run because
- we replace the harness with our own runner here.
-*/
-fn main() -> Result<()> {
-  if !env::args().any(|s| s == "--bench") {
+#[tokio::main]
+async fn main() -> Result<()> {
+  let mut args = env::args();
+
+  let mut benchmarks = vec![
+    "bundle",
+    "exec_time",
+    "binary_size",
+    "cargo_deps",
+    "lsp",
+    "http",
+    "strace",
+    "mem_usage",
+  ];
+
+  let mut found_bench = false;
+  let filter = args.nth(1);
+  if let Some(filter) = filter {
+    if filter != "--bench" {
+      benchmarks.retain(|s| s == &filter);
+    } else {
+      found_bench = true;
+    }
+  }
+
+  if !found_bench && !args.any(|s| s == "--bench") {
     return Ok(());
   }
 
@@ -465,7 +427,6 @@ fn main() -> Result<()> {
 
   let target_dir = test_util::target_dir();
   let deno_exe = test_util::deno_exe_path();
-
   env::set_current_dir(&test_util::root_path())?;
 
   let mut new_data = BenchResult {
@@ -481,27 +442,107 @@ fn main() -> Result<()> {
     .0
     .trim()
     .to_string(),
-    benchmark: run_exec_time(&deno_exe, &target_dir)?,
-    binary_size: get_binary_sizes(&target_dir)?,
-    bundle_size: bundle_benchmark(&deno_exe)?,
-    cargo_deps: cargo_deps(),
-    lsp_exec_time: lsp::benchmarks(&deno_exe)?,
     ..Default::default()
   };
 
-  if cfg!(not(target_os = "windows")) {
-    run_http(&target_dir, &mut new_data)?;
+  let mut reporter = metrics::Reporter::new().await;
+
+  if benchmarks.contains(&"bundle") {
+    let bundle_size = bundle_benchmark(&deno_exe)?;
+    reporter.write("bundle_size", &bundle_size);
+    new_data.bundle_size = bundle_size;
+  }
+
+  if benchmarks.contains(&"exec_time") {
+    let exec_times = run_exec_time(&deno_exe, &target_dir)?;
+    for (name, data) in exec_times.iter() {
+      reporter.write_one("exec_time", name, *data.get("mean").unwrap());
+    }
+    new_data.benchmark = exec_times;
+  }
+
+  if benchmarks.contains(&"binary_size") {
+    let binary_sizes = get_binary_sizes(&target_dir)?;
+    reporter.write("binary_size", &binary_sizes);
+    new_data.binary_size = binary_sizes;
+  }
+
+  if benchmarks.contains(&"cargo_deps") {
+    let cargo_deps = cargo_deps();
+    reporter.write_one("cargo_deps", "cargo_deps", cargo_deps as i64);
+    new_data.cargo_deps = cargo_deps;
+  }
+
+  if benchmarks.contains(&"lsp") {
+    let lsp_exec_times = lsp::benchmarks(&deno_exe)?;
+    reporter.write("lsp_exec_time", &lsp_exec_times);
+    new_data.lsp_exec_time = lsp_exec_times;
   }
 
-  if cfg!(target_os = "linux") {
-    run_strace_benchmarks(&deno_exe, &mut new_data)?;
-    new_data.max_memory = run_max_mem_benchmark(&deno_exe)?;
+  if benchmarks.contains(&"http") && cfg!(not(target_os = "windows")) {
+    let stats = http::benchmark(&target_dir)?;
+    let req_per_sec = stats
+      .iter()
+      .map(|(name, result)| (name.clone(), result.requests as i64))
+      .collect();
+    reporter.write("req_per_sec", &req_per_sec);
+    new_data.req_per_sec = req_per_sec;
+    let max_latency = stats
+      .iter()
+      .map(|(name, result)| (name.clone(), result.latency as i64))
+      .collect();
+
+    reporter.write("max_latency", &max_latency);
+    new_data.max_latency = max_latency;
   }
 
-  println!("===== <BENCHMARK RESULTS>");
-  serde_json::to_writer_pretty(std::io::stdout(), &new_data)?;
-  println!("\n===== </BENCHMARK RESULTS>");
+  if cfg!(target_os = "linux") && benchmarks.contains(&"strace") {
+    use std::io::Read;
+
+    let mut thread_count = HashMap::<String, i64>::new();
+    let mut syscall_count = HashMap::<String, i64>::new();
+
+    for (name, args, expected_exit_code) in EXEC_TIME_BENCHMARKS {
+      let mut file = secure_tempfile::NamedTempFile::new()?;
+
+      let exit_status = Command::new("strace")
+        .args(&[
+          "-c",
+          "-f",
+          "-o",
+          file.path().to_str().unwrap(),
+          deno_exe.to_str().unwrap(),
+        ])
+        .args(args.iter())
+        .stdout(Stdio::null())
+        .spawn()?
+        .wait()?;
+      let expected_exit_code = expected_exit_code.unwrap_or(0);
+      assert_eq!(exit_status.code(), Some(expected_exit_code));
+
+      let mut output = String::new();
+      file.as_file_mut().read_to_string(&mut output)?;
+
+      let strace_result = test_util::parse_strace_output(&output);
+      let clone = strace_result.get("clone").map(|d| d.calls).unwrap_or(0) + 1;
+      let total = strace_result.get("total").unwrap().calls;
+      thread_count.insert(name.to_string(), clone as i64);
+      syscall_count.insert(name.to_string(), total as i64);
+    }
+
+    reporter.write("thread_count", &thread_count);
+    new_data.thread_count = thread_count;
+    reporter.write("syscall_count", &syscall_count);
+    new_data.syscall_count = syscall_count;
+  }
+
+  if benchmarks.contains(&"mem_usage") {
+    let max_memory = run_max_mem_benchmark(&deno_exe)?;
+    reporter.write("max_memory", &max_memory);
+    new_data.max_memory = max_memory;
+  }
 
+  reporter.submit().await;
   if let Some(filename) = target_dir.join("bench.json").to_str() {
     write_json(filename, &serde_json::to_value(&new_data)?)?;
   } else {
diff --git a/cli/bench/metrics.rs b/cli/bench/metrics.rs
new file mode 100644
index 000000000..c49ca9020
--- /dev/null
+++ b/cli/bench/metrics.rs
@@ -0,0 +1,99 @@
+// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
+
+use google_storage1::api::Object;
+use google_storage1::hyper;
+use google_storage1::hyper_rustls;
+use google_storage1::oauth2;
+use google_storage1::Storage;
+use once_cell::sync::Lazy;
+use std::collections::HashMap;
+use std::io::Cursor;
+
+static GIT_HASH: Lazy<String> = Lazy::new(|| {
+  test_util::run_collect(&["git", "rev-parse", "HEAD"], None, None, None, true)
+    .0
+    .trim()
+    .to_string()
+});
+
+#[derive(serde::Serialize)]
+struct Metric {
+  name: String,
+  value: i64,
+  sha1: String,
+  #[serde(rename = "type")]
+  type_: String,
+  time: i64,
+}
+
+pub struct Reporter {
+  wtr: csv::Writer<Vec<u8>>,
+  gcloud_client: Option<Storage>,
+}
+
+impl Reporter {
+  pub async fn new() -> Self {
+    dotenv::dotenv().ok();
+    let gcloud_client =
+      match std::env::var("CI").map(|_| std::env::var("GOOGLE_SVC_KEY")) {
+        Ok(Ok(key_str)) => {
+          let secret = oauth2::parse_service_account_key(key_str)
+            .expect("Failed to load service account key");
+          let auth =
+            oauth2::authenticator::ServiceAccountAuthenticator::builder(secret)
+              .build()
+              .await
+              .unwrap();
+          let client = hyper::Client::builder().build(
+            hyper_rustls::HttpsConnectorBuilder::new()
+              .with_native_roots()
+              .https_or_http()
+              .enable_http1()
+              .enable_http2()
+              .build(),
+          );
+          Some(Storage::new(client, auth))
+        }
+        _ => None,
+      };
+    Self {
+      wtr: csv::Writer::from_writer(vec![]),
+      gcloud_client,
+    }
+  }
+
+  pub fn write_one(&mut self, type_: &str, name: &str, value: i64) {
+    self
+      .wtr
+      .serialize(Metric {
+        name: name.to_string(),
+        type_: type_.to_string(),
+        value,
+        sha1: GIT_HASH.clone(),
+        time: chrono::Utc::now().timestamp_millis(),
+      })
+      .unwrap();
+  }
+
+  pub fn write(&mut self, type_: &str, hashmap: &HashMap<String, i64>) {
+    for (name, value) in hashmap {
+      self.write_one(type_, name, *value);
+    }
+  }
+
+  pub async fn submit(mut self) {
+    self.wtr.flush().unwrap();
+    if let Some(client) = self.gcloud_client.take() {
+      let mut reader = Cursor::new(self.wtr.into_inner().unwrap());
+      let object: Object = Object::default();
+      client
+        .objects()
+        .insert(object, "deno_benchmark_data")
+        .name(&format!("{}.csv", *GIT_HASH))
+        .param("uploadType", "multipart")
+        .upload(&mut reader, "text/csv".parse().unwrap())
+        .await
+        .unwrap();
+    }
+  }
+}