feat(cli/tools): add TAP test reporter (#14390) (#20073)

This PR adds a test reporter for the [Test Anything
Protocol](https://testanything.org).

It makes the following implementation decisions:
- No TODO pragma, as there is no such marker in `Deno.test`
- SKIP pragma for `ignore`d tests
- Test steps are treated as TAP14 subtests
  - Support for this in consumers seems spotty
- Some consumers will incorrectly interpret these markers, resulting in
unexpected output
- Considering the lack of support, and to avoid implementation
complexity,
subtests are at most one level deep (all test steps are in the same
subtest)
- To accommodate consumers that use comments to indicate test-suites
(unspecced)
  - The test module path is output as a comment
  - This is disabled for `--parallel` testing
- Failure diagnostics are output as JSON, which is also valid YAML
- The structure is not specified, so the format roughly follows the spec
example:
  ```
  ---
  message: "Failed with error 'hostname peebles.example.com not found'"
  severity: fail
  found:
    hostname: 'peebles.example.com'
    address: ~
  wanted:
    hostname: 'peebles.example.com'
    address: '85.193.201.85'
  at:
    file: test/dns-resolve.c
    line: 142
  ...
  ```
  

3 files changed, 253 insertions, 0 deletions

diff --git a/cli/tools/test/mod.rs b/cli/tools/test/mod.rs
index 365b83268..9ad205e2e 100644
--- a/cli/tools/test/mod.rs
+++ b/cli/tools/test/mod.rs
@@ -89,6 +89,7 @@ use reporters::CompoundTestReporter;
 use reporters::DotTestReporter;
 use reporters::JunitTestReporter;
 use reporters::PrettyTestReporter;
+use reporters::TapTestReporter;
 use reporters::TestReporter;
 
 /// The test mode is used to determine how a specifier is to be tested.
@@ -404,6 +405,9 @@ fn get_test_reporter(options: &TestSpecifiersOptions) -> Box<dyn TestReporter> {
     TestReporterConfig::Junit => {
       Box::new(JunitTestReporter::new("-".to_string()))
     }
+    TestReporterConfig::Tap => Box::new(TapTestReporter::new(
+      options.concurrent_jobs > NonZeroUsize::new(1).unwrap(),
+    )),
   };
 
   if let Some(junit_path) = &options.junit_path {
diff --git a/cli/tools/test/reporters/mod.rs b/cli/tools/test/reporters/mod.rs
index a95c729dd..35d2776e4 100644
--- a/cli/tools/test/reporters/mod.rs
+++ b/cli/tools/test/reporters/mod.rs
@@ -7,11 +7,13 @@ mod compound;
 mod dot;
 mod junit;
 mod pretty;
+mod tap;
 
 pub use compound::CompoundTestReporter;
 pub use dot::DotTestReporter;
 pub use junit::JunitTestReporter;
 pub use pretty::PrettyTestReporter;
+pub use tap::TapTestReporter;
 
 pub trait TestReporter {
   fn report_register(&mut self, description: &TestDescription);
diff --git a/cli/tools/test/reporters/tap.rs b/cli/tools/test/reporters/tap.rs
new file mode 100644
index 000000000..c40c1eed7
--- /dev/null
+++ b/cli/tools/test/reporters/tap.rs
@@ -0,0 +1,247 @@
+// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
+
+use deno_core::serde_json::json;
+use deno_core::serde_json::{self};
+use serde::Serialize;
+
+use super::common;
+use super::fmt::to_relative_path_or_remote_url;
+use super::*;
+
+const VERSION_HEADER: &str = "TAP version 14";
+
+/// A test reporter for the Test Anything Protocol as defined at
+/// https://testanything.org/tap-version-14-specification.html
+pub struct TapTestReporter {
+  cwd: Url,
+  is_concurrent: bool,
+  header: bool,
+  planned: usize,
+  n: usize,
+  step_n: usize,
+  step_results: HashMap<usize, Vec<(TestStepDescription, TestStepResult)>>,
+}
+
+impl TapTestReporter {
+  pub fn new(is_concurrent: bool) -> TapTestReporter {
+    TapTestReporter {
+      cwd: Url::from_directory_path(std::env::current_dir().unwrap()).unwrap(),
+      is_concurrent,
+      header: false,
+      planned: 0,
+      n: 0,
+      step_n: 0,
+      step_results: HashMap::new(),
+    }
+  }
+
+  fn escape_description(description: &str) -> String {
+    description
+      .replace('\\', "\\\\")
+      .replace('\n', "\\n")
+      .replace('\r', "\\r")
+      .replace('#', "\\#")
+  }
+
+  fn print_diagnostic(
+    indent: usize,
+    failure: &TestFailure,
+    location: DiagnosticLocation,
+  ) {
+    // Unspecified behaviour:
+    // The diagnostic schema is not specified by the TAP spec,
+    // but there is an example, so we use it.
+
+    // YAML is a superset of JSON, so we can avoid a YAML dependency here.
+    // This makes the output less readable though.
+    let diagnostic = serde_json::to_string(&json!({
+      "message": failure.to_string(),
+      "severity": "fail".to_string(),
+      "at": location,
+    }))
+    .expect("failed to serialize TAP diagnostic");
+    println!("{:indent$}  ---", "", indent = indent);
+    println!("{:indent$}  {}", "", diagnostic, indent = indent);
+    println!("{:indent$}  ...", "", indent = indent);
+  }
+
+  fn print_line(
+    indent: usize,
+    status: &str,
+    step: usize,
+    description: &str,
+    directive: &str,
+  ) {
+    println!(
+      "{:indent$}{} {} - {}{}",
+      "",
+      status,
+      step,
+      Self::escape_description(description),
+      directive,
+      indent = indent
+    );
+  }
+
+  fn print_step_result(
+    &mut self,
+    desc: &TestStepDescription,
+    result: &TestStepResult,
+  ) {
+    if self.step_n == 0 {
+      println!("# Subtest: {}", desc.root_name)
+    }
+
+    let (status, directive) = match result {
+      TestStepResult::Ok => ("ok", ""),
+      TestStepResult::Ignored => ("ok", " # SKIP"),
+      TestStepResult::Failed(_failure) => ("not ok", ""),
+    };
+    self.step_n += 1;
+    Self::print_line(4, status, self.step_n, &desc.name, directive);
+
+    if let TestStepResult::Failed(failure) = result {
+      Self::print_diagnostic(
+        4,
+        failure,
+        DiagnosticLocation {
+          file: to_relative_path_or_remote_url(&self.cwd, &desc.origin),
+          line: desc.location.line_number,
+        },
+      );
+    }
+  }
+}
+
+impl TestReporter for TapTestReporter {
+  fn report_register(&mut self, _description: &TestDescription) {}
+
+  fn report_plan(&mut self, plan: &TestPlan) {
+    if !self.header {
+      println!("{}", VERSION_HEADER);
+      self.header = true;
+    }
+    self.planned += plan.total;
+
+    if !self.is_concurrent {
+      // Unspecified behavior: Consumers tend to interpret a comment as a test suite name.
+      // During concurrent execution these would not correspond to the actual test file, so skip them.
+      println!(
+        "# {}",
+        to_relative_path_or_remote_url(&self.cwd, &plan.origin)
+      )
+    }
+  }
+
+  fn report_wait(&mut self, _description: &TestDescription) {
+    // flush for faster feedback when line buffered
+    std::io::stdout().flush().unwrap();
+  }
+
+  fn report_output(&mut self, _output: &[u8]) {}
+
+  fn report_result(
+    &mut self,
+    description: &TestDescription,
+    result: &TestResult,
+    _elapsed: u64,
+  ) {
+    if self.is_concurrent {
+      let results = self.step_results.remove(&description.id);
+      for (desc, result) in results.iter().flat_map(|v| v.iter()) {
+        self.print_step_result(desc, result);
+      }
+    }
+
+    if self.step_n != 0 {
+      println!("    1..{}", self.step_n);
+      self.step_n = 0;
+    }
+
+    let (status, directive) = match result {
+      TestResult::Ok => ("ok", ""),
+      TestResult::Ignored => ("ok", " # SKIP"),
+      TestResult::Failed(_failure) => ("not ok", ""),
+      TestResult::Cancelled => ("not ok", ""),
+    };
+    self.n += 1;
+    Self::print_line(0, status, self.n, &description.name, directive);
+
+    if let TestResult::Failed(failure) = result {
+      Self::print_diagnostic(
+        0,
+        failure,
+        DiagnosticLocation {
+          file: to_relative_path_or_remote_url(&self.cwd, &description.origin),
+          line: description.location.line_number,
+        },
+      );
+    }
+  }
+
+  fn report_uncaught_error(&mut self, _origin: &str, _errorr: Box<JsError>) {}
+
+  fn report_step_register(&mut self, _description: &TestStepDescription) {}
+
+  fn report_step_wait(&mut self, _description: &TestStepDescription) {
+    // flush for faster feedback when line buffered
+    std::io::stdout().flush().unwrap();
+  }
+
+  fn report_step_result(
+    &mut self,
+    desc: &TestStepDescription,
+    result: &TestStepResult,
+    _elapsed: u64,
+    _tests: &IndexMap<usize, TestDescription>,
+    _test_steps: &IndexMap<usize, TestStepDescription>,
+  ) {
+    if self.is_concurrent {
+      // All subtests must be reported immediately before the parent test.
+      // So during concurrent execution we need to defer printing the results.
+      // TODO(SyrupThinker) This only outputs one level of subtests, it could support multiple.
+      self
+        .step_results
+        .entry(desc.root_id)
+        .or_default()
+        .push((desc.clone(), result.clone()));
+      return;
+    }
+
+    self.print_step_result(desc, result);
+  }
+
+  fn report_summary(
+    &mut self,
+    _elapsed: &Duration,
+    _tests: &IndexMap<usize, TestDescription>,
+    _test_steps: &IndexMap<usize, TestStepDescription>,
+  ) {
+    println!("1..{}", self.planned);
+  }
+
+  fn report_sigint(
+    &mut self,
+    tests_pending: &HashSet<usize>,
+    tests: &IndexMap<usize, TestDescription>,
+    test_steps: &IndexMap<usize, TestStepDescription>,
+  ) {
+    println!("Bail out! SIGINT received.");
+    common::report_sigint(&self.cwd, tests_pending, tests, test_steps);
+  }
+
+  fn flush_report(
+    &mut self,
+    _elapsed: &Duration,
+    _tests: &IndexMap<usize, TestDescription>,
+    _test_steps: &IndexMap<usize, TestStepDescription>,
+  ) -> anyhow::Result<()> {
+    Ok(())
+  }
+}
+
+#[derive(Serialize)]
+struct DiagnosticLocation {
+  file: String,
+  line: u32,
+}