chore(tests/specs): ability to have sub tests in file (#23667)

Allows writing named sub-tests. These are:

1. Filterable on the command line via `cargo test ...`
2. Run in parallel
3. Use a fresh temp and deno dir for each test (unlike steps)

1 files changed, 173 insertions, 57 deletions

diff --git a/tests/specs/mod.rs b/tests/specs/mod.rs
index fc61fefac..f367f5c77 100644
--- a/tests/specs/mod.rs
+++ b/tests/specs/mod.rs
@@ -1,6 +1,7 @@
 // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
 
 use std::cell::RefCell;
+use std::collections::BTreeMap;
 use std::collections::HashMap;
 use std::collections::HashSet;
 use std::panic::AssertUnwindSafe;
@@ -10,9 +11,14 @@ use std::sync::Arc;
 use deno_core::anyhow::Context;
 use deno_core::serde_json;
 use file_test_runner::collection::collect_tests_or_exit;
+use file_test_runner::collection::strategies::FileTestMapperStrategy;
 use file_test_runner::collection::strategies::TestPerDirectoryCollectionStrategy;
 use file_test_runner::collection::CollectOptions;
+use file_test_runner::collection::CollectTestsError;
+use file_test_runner::collection::CollectedCategoryOrTest;
 use file_test_runner::collection::CollectedTest;
+use file_test_runner::collection::CollectedTestCategory;
+use file_test_runner::TestResult;
 use serde::Deserialize;
 use test_util::tests_path;
 use test_util::PathRef;
@@ -27,6 +33,8 @@ enum VecOrString {
   String(String),
 }
 
+type JsonMap = serde_json::Map<String, serde_json::Value>;
+
 #[derive(Clone, Deserialize)]
 #[serde(deny_unknown_fields, rename_all = "camelCase")]
 struct MultiTestMetaData {
@@ -40,6 +48,68 @@ struct MultiTestMetaData {
   pub base: Option<String>,
   #[serde(default)]
   pub envs: HashMap<String, String>,
+  #[serde(default)]
+  pub tests: BTreeMap<String, JsonMap>,
+}
+
+impl MultiTestMetaData {
+  pub fn into_collected_tests(
+    mut self,
+    parent_test: &CollectedTest,
+  ) -> Vec<CollectedTest<serde_json::Value>> {
+    fn merge_json_value(
+      multi_test_meta_data: &MultiTestMetaData,
+      value: &mut JsonMap,
+    ) {
+      if let Some(base) = &multi_test_meta_data.base {
+        if !value.contains_key("base") {
+          value.insert("base".to_string(), base.clone().into());
+        }
+      }
+      if multi_test_meta_data.temp_dir && !value.contains_key("tempDir") {
+        value.insert("tempDir".to_string(), true.into());
+      }
+      if !multi_test_meta_data.envs.is_empty() {
+        if !value.contains_key("envs") {
+          value.insert("envs".to_string(), JsonMap::default().into());
+        }
+        let envs_obj = value.get_mut("envs").unwrap().as_object_mut().unwrap();
+        for (key, value) in &multi_test_meta_data.envs {
+          if !envs_obj.contains_key(key) {
+            envs_obj.insert(key.into(), value.clone().into());
+          }
+        }
+      }
+    }
+
+    let mut collected_tests = Vec::with_capacity(self.tests.len());
+    for (name, mut json_data) in std::mem::take(&mut self.tests) {
+      merge_json_value(&self, &mut json_data);
+      collected_tests.push(CollectedTest {
+        name: format!("{}::{}", parent_test.name, name),
+        path: parent_test.path.clone(),
+        data: serde_json::Value::Object(json_data),
+      });
+    }
+
+    collected_tests
+  }
+}
+
+#[derive(Clone, Deserialize)]
+#[serde(deny_unknown_fields, rename_all = "camelCase")]
+struct MultiStepMetaData {
+  /// Whether to copy all the non-assertion files in the current
+  /// test directory to a temporary directory before running the
+  /// steps.
+  #[serde(default)]
+  pub temp_dir: bool,
+  /// The base environment to use for the test.
+  #[serde(default)]
+  pub base: Option<String>,
+  #[serde(default)]
+  pub envs: HashMap<String, String>,
+  #[serde(default)]
   pub steps: Vec<StepMetaData>,
 }
 
@@ -55,8 +125,8 @@ struct SingleTestMetaData {
 }
 
 impl SingleTestMetaData {
-  pub fn into_multi(self) -> MultiTestMetaData {
-    MultiTestMetaData {
+  pub fn into_multi(self) -> MultiStepMetaData {
+    MultiStepMetaData {
       base: self.base,
       temp_dir: self.temp_dir,
       envs: Default::default(),
@@ -68,9 +138,6 @@ impl SingleTestMetaData {
 #[derive(Clone, Deserialize)]
 #[serde(deny_unknown_fields, rename_all = "camelCase")]
 struct StepMetaData {
-  /// Whether to clean the deno_dir before running the step.
-  #[serde(default)]
-  pub clean_deno_dir: bool,
   /// If the test should be retried multiple times on failure.
   #[serde(default)]
   pub flaky: bool,
@@ -87,13 +154,17 @@ struct StepMetaData {
 }
 
 pub fn main() {
-  let root_category = collect_tests_or_exit(CollectOptions {
-    base: tests_path().join("specs").to_path_buf(),
-    strategy: Box::new(TestPerDirectoryCollectionStrategy {
-      file_name: MANIFEST_FILE_NAME.to_string(),
-    }),
-    filter_override: None,
-  });
+  let root_category =
+    collect_tests_or_exit::<serde_json::Value>(CollectOptions {
+      base: tests_path().join("specs").to_path_buf(),
+      strategy: Box::new(FileTestMapperStrategy {
+        base_strategy: TestPerDirectoryCollectionStrategy {
+          file_name: MANIFEST_FILE_NAME.to_string(),
+        },
+        map: map_test_within_file,
+      }),
+      filter_override: None,
+    });
 
   if root_category.is_empty() {
     return; // all tests filtered out
@@ -103,49 +174,106 @@ pub fn main() {
   file_test_runner::run_tests(
     &root_category,
     file_test_runner::RunOptions { parallel: true },
-    Arc::new(|test| {
-      let diagnostic_logger = Rc::new(RefCell::new(Vec::<u8>::new()));
-      let result = file_test_runner::TestResult::from_maybe_panic(
-        AssertUnwindSafe(|| run_test(test, diagnostic_logger.clone())),
-      );
-      match result {
-        file_test_runner::TestResult::Passed
-        | file_test_runner::TestResult::Ignored => result,
-        file_test_runner::TestResult::Failed {
-          output: panic_output,
-        } => {
-          let mut output = diagnostic_logger.borrow().clone();
-          output.push(b'\n');
-          output.extend(panic_output);
-          file_test_runner::TestResult::Failed { output }
-        }
-        file_test_runner::TestResult::Steps(_) => unreachable!(),
-      }
-    }),
+    Arc::new(run_test),
   );
 }
 
-fn run_test(test: &CollectedTest, diagnostic_logger: Rc<RefCell<Vec<u8>>>) {
-  let metadata_path = PathRef::new(&test.path);
-  let metadata_value = metadata_path.read_jsonc_value();
+/// Maps a __test__.jsonc file to a category of tests if it contains a "test" object.
+fn map_test_within_file(
+  test: CollectedTest,
+) -> Result<CollectedCategoryOrTest<serde_json::Value>, CollectTestsError> {
+  let test_path = PathRef::new(&test.path);
+  let metadata_value = test_path.read_jsonc_value();
+  if metadata_value
+    .as_object()
+    .map(|o| o.contains_key("tests"))
+    .unwrap_or(false)
+  {
+    let data: MultiTestMetaData = serde_json::from_value(metadata_value)
+      .with_context(|| format!("Failed deserializing {}", test_path))
+      .map_err(CollectTestsError::Other)?;
+    Ok(CollectedCategoryOrTest::Category(CollectedTestCategory {
+      children: data
+        .into_collected_tests(&test)
+        .into_iter()
+        .map(CollectedCategoryOrTest::Test)
+        .collect(),
+      name: test.name,
+      path: test.path,
+    }))
+  } else {
+    Ok(CollectedCategoryOrTest::Test(CollectedTest {
+      name: test.name,
+      path: test.path,
+      data: metadata_value,
+    }))
+  }
+}
+
+fn run_test(test: &CollectedTest<serde_json::Value>) -> TestResult {
+  let cwd = PathRef::new(&test.path).parent();
+  let metadata_value = test.data.clone();
+  let diagnostic_logger = Rc::new(RefCell::new(Vec::<u8>::new()));
+  let result = TestResult::from_maybe_panic(AssertUnwindSafe(|| {
+    run_test_inner(metadata_value, &cwd, diagnostic_logger.clone())
+  }));
+  match result {
+    TestResult::Failed {
+      output: panic_output,
+    } => {
+      let mut output = diagnostic_logger.borrow().clone();
+      output.push(b'\n');
+      output.extend(panic_output);
+      TestResult::Failed { output }
+    }
+    TestResult::Passed | TestResult::Ignored | TestResult::SubTests(_) => {
+      result
+    }
+  }
+}
+
+fn run_test_inner(
+  metadata_value: serde_json::Value,
+  cwd: &PathRef,
+  diagnostic_logger: Rc<RefCell<Vec<u8>>>,
+) {
+  let metadata = deserialize_value(metadata_value);
+
+  let context = test_context_from_metadata(&metadata, cwd, diagnostic_logger);
+  for step in metadata.steps.iter().filter(|s| should_run_step(s)) {
+    let run_func = || run_step(step, &metadata, cwd, &context);
+    if step.flaky {
+      run_flaky(run_func);
+    } else {
+      run_func();
+    }
+  }
+}
+
+fn deserialize_value(metadata_value: serde_json::Value) -> MultiStepMetaData {
   // checking for "steps" leads to a more targeted error message
   // instead of when deserializing an untagged enum
-  let metadata = if metadata_value
+  if metadata_value
     .as_object()
-    .and_then(|o| o.get("steps"))
-    .is_some()
+    .map(|o| o.contains_key("steps"))
+    .unwrap_or(false)
   {
-    serde_json::from_value::<MultiTestMetaData>(metadata_value)
+    serde_json::from_value::<MultiStepMetaData>(metadata_value)
   } else {
     serde_json::from_value::<SingleTestMetaData>(metadata_value)
       .map(|s| s.into_multi())
   }
-  .with_context(|| format!("Failed to parse {}", metadata_path))
-  .unwrap();
+  .context("Failed to parse test spec")
+  .unwrap()
+}
 
+fn test_context_from_metadata(
+  metadata: &MultiStepMetaData,
+  cwd: &PathRef,
+  diagnostic_logger: Rc<RefCell<Vec<u8>>>,
+) -> test_util::TestContext {
   let mut builder = TestContextBuilder::new();
   builder = builder.logging_capture(diagnostic_logger);
-  let cwd = PathRef::new(test.path.parent().unwrap());
 
   if metadata.temp_dir {
     builder = builder.use_temp_cwd();
@@ -171,18 +299,10 @@ fn run_test(test: &CollectedTest, diagnostic_logger: Rc<RefCell<Vec<u8>>>) {
     // copy all the files in the cwd to a temp directory
     // excluding the metadata and assertion files
     let temp_dir = context.temp_dir().path();
-    let assertion_paths = resolve_test_and_assertion_files(&cwd, &metadata);
+    let assertion_paths = resolve_test_and_assertion_files(cwd, metadata);
     cwd.copy_to_recursive_with_exclusions(temp_dir, &assertion_paths);
   }
-
-  for step in metadata.steps.iter().filter(|s| should_run_step(s)) {
-    let run_func = || run_step(step, &metadata, &cwd, &context);
-    if step.flaky {
-      run_flaky(run_func);
-    } else {
-      run_func();
-    }
-  }
+  context
 }
 
 fn should_run_step(step: &StepMetaData) -> bool {
@@ -213,14 +333,10 @@ fn run_flaky(action: impl Fn()) {
 
 fn run_step(
   step: &StepMetaData,
-  metadata: &MultiTestMetaData,
+  metadata: &MultiStepMetaData,
   cwd: &PathRef,
   context: &test_util::TestContext,
 ) {
-  if step.clean_deno_dir {
-    context.deno_dir().path().remove_dir_all();
-  }
-
   let command = context
     .new_command()
     .envs(metadata.envs.iter().chain(step.envs.iter()));
@@ -248,7 +364,7 @@ fn run_step(
 
 fn resolve_test_and_assertion_files(
   dir: &PathRef,
-  metadata: &MultiTestMetaData,
+  metadata: &MultiStepMetaData,
 ) -> HashSet<PathRef> {
   let mut result = HashSet::with_capacity(metadata.steps.len() + 1);
   result.insert(dir.join(MANIFEST_FILE_NAME));