diff options
| -rw-r--r-- | cli/lsp/testing/collectors.rs | 54 | ||||
| -rw-r--r-- | cli/lsp/testing/definitions.rs | 70 | ||||
| -rw-r--r-- | cli/lsp/testing/execution.rs | 387 | ||||
| -rw-r--r-- | cli/lsp/testing/lsp_custom.rs | 10 | ||||
| -rw-r--r-- | cli/ops/testing.rs | 99 | ||||
| -rw-r--r-- | cli/tests/testdata/test/steps/failing_steps.out | 8 | ||||
| -rw-r--r-- | cli/tests/testdata/test/steps/ignored_steps.out | 2 | ||||
| -rw-r--r-- | cli/tests/testdata/test/steps/invalid_usage.out | 24 | ||||
| -rw-r--r-- | cli/tests/testdata/test/steps/passing_steps.out | 22 | ||||
| -rw-r--r-- | cli/tests/testdata/test/uncaught_errors.out | 3 | ||||
| -rw-r--r-- | cli/tools/test.rs | 204 | ||||
| -rw-r--r-- | runtime/js/40_testing.js | 753 |
12 files changed, 793 insertions, 843 deletions
diff --git a/cli/lsp/testing/collectors.rs b/cli/lsp/testing/collectors.rs index 537dd5806..d338ac088 100644 --- a/cli/lsp/testing/collectors.rs +++ b/cli/lsp/testing/collectors.rs @@ -15,7 +15,7 @@ fn arrow_to_steps( parent: &str, level: usize, arrow_expr: &ast::ArrowExpr, -) -> Option<Vec<TestDefinition>> { +) -> Vec<TestDefinition> { if let Some((maybe_test_context, maybe_step_var)) = parse_test_context_param(arrow_expr.params.get(0)) { @@ -26,14 +26,9 @@ fn arrow_to_steps( maybe_step_var, ); arrow_expr.body.visit_with(&mut collector); - let steps = collector.take(); - if !steps.is_empty() { - Some(steps) - } else { - None - } + collector.take() } else { - None + vec![] } } @@ -42,7 +37,7 @@ fn fn_to_steps( parent: &str, level: usize, function: &ast::Function, -) -> Option<Vec<TestDefinition>> { +) -> Vec<TestDefinition> { if let Some((maybe_test_context, maybe_step_var)) = parse_test_context_param(function.params.get(0).map(|p| &p.pat)) { @@ -53,14 +48,9 @@ fn fn_to_steps( maybe_step_var, ); function.body.visit_with(&mut collector); - let steps = collector.take(); - if !steps.is_empty() { - Some(steps) - } else { - None - } + collector.take() } else { - None + vec![] } } @@ -139,12 +129,12 @@ fn check_call_expr( parent: &str, node: &ast::CallExpr, level: usize, -) -> Option<(String, Option<Vec<TestDefinition>>)> { +) -> Option<(String, Vec<TestDefinition>)> { if let Some(expr) = node.args.get(0).map(|es| es.expr.as_ref()) { match expr { ast::Expr::Object(obj_lit) => { let mut maybe_name = None; - let mut steps = None; + let mut steps = vec![]; for prop in &obj_lit.props { if let ast::PropOrSpread::Prop(prop) = prop { match prop.as_ref() { @@ -203,7 +193,7 @@ fn check_call_expr( } ast::Expr::Lit(ast::Lit::Str(lit_str)) => { let name = lit_str.value.to_string(); - let mut steps = None; + let mut steps = vec![]; match node.args.get(1).map(|es| es.expr.as_ref()) { Some(ast::Expr::Fn(fn_expr)) => { steps = fn_to_steps(parent, level, &fn_expr.function); @@ -256,7 +246,7 @@ impl TestStepCollector { &mut self, name: N, range: SourceRange, - steps: Option<Vec<TestDefinition>>, + steps: Vec<TestDefinition>, ) { let step = TestDefinition::new_step( name.as_ref().to_string(), @@ -388,7 +378,7 @@ impl TestCollector { &mut self, name: N, range: SourceRange, - steps: Option<Vec<TestDefinition>>, + steps: Vec<TestDefinition>, ) { let definition = TestDefinition::new( &self.specifier, @@ -553,59 +543,59 @@ pub mod tests { level: 0, name: "test a".to_string(), range: new_range(12, 16), - steps: Some(vec![ + steps: vec![ TestDefinition { id: "4c7333a1e47721631224408c467f32751fe34b876cab5ec1f6ac71980ff15ad3".to_string(), level: 1, name: "a step".to_string(), range: new_range(83, 87), - steps: Some(vec![ + steps: vec![ TestDefinition { id: "abf356f59139b77574089615f896a6f501c010985d95b8a93abeb0069ccb2201".to_string(), level: 2, name: "sub step".to_string(), range: new_range(132, 136), - steps: None, + steps: vec![], } - ]) + ] } - ]), + ], }, TestDefinition { id: "86b4c821900e38fc89f24bceb0e45193608ab3f9d2a6019c7b6a5aceff5d7df2".to_string(), level: 0, name: "useFnName".to_string(), range: new_range(254, 258), - steps: Some(vec![ + steps: vec![ TestDefinition { id: "67a390d0084ae5fb88f3510c470a72a553581f1d0d5ba5fa89aee7a754f3953a".to_string(), level: 1, name: "step c".to_string(), range: new_range(313, 314), - steps: None, + steps: vec![], } - ]) + ] }, TestDefinition { id: "580eda89d7f5e619774c20e13b7d07a8e77c39cba101d60565144d48faa837cb".to_string(), level: 0, name: "test b".to_string(), range: new_range(358, 362), - steps: None, + steps: vec![], }, TestDefinition { id: "0b7c6bf3cd617018d33a1bf982a08fe088c5bb54fcd5eb9e802e7c137ec1af94".to_string(), level: 0, name: "test c".to_string(), range: new_range(420, 424), - steps: None, + steps: vec![], }, TestDefinition { id: "69d9fe87f64f5b66cb8b631d4fd2064e8224b8715a049be54276c42189ff8f9f".to_string(), level: 0, name: "test d".to_string(), range: new_range(480, 481), - steps: None, + steps: vec![], } ] ); diff --git a/cli/lsp/testing/definitions.rs b/cli/lsp/testing/definitions.rs index c810b6a25..14ac165fd 100644 --- a/cli/lsp/testing/definitions.rs +++ b/cli/lsp/testing/definitions.rs @@ -18,7 +18,7 @@ pub struct TestDefinition { pub level: usize, pub name: String, pub range: SourceRange, - pub steps: Option<Vec<TestDefinition>>, + pub steps: Vec<TestDefinition>, } impl TestDefinition { @@ -26,7 +26,7 @@ impl TestDefinition { specifier: &ModuleSpecifier, name: String, range: SourceRange, - steps: Option<Vec<TestDefinition>>, + steps: Vec<TestDefinition>, ) -> Self { let id = checksum::gen(&[specifier.as_str().as_bytes(), name.as_bytes()]); Self { @@ -43,7 +43,7 @@ impl TestDefinition { range: SourceRange, parent: String, level: usize, - steps: Option<Vec<TestDefinition>>, + steps: Vec<TestDefinition>, ) -> Self { let id = checksum::gen(&[ parent.as_bytes(), @@ -66,27 +66,18 @@ impl TestDefinition { lsp_custom::TestData { id: self.id.clone(), label: self.name.clone(), - steps: self.steps.as_ref().map(|steps| { - steps - .iter() - .map(|step| step.as_test_data(source_text_info)) - .collect() - }), + steps: self + .steps + .iter() + .map(|step| step.as_test_data(source_text_info)) + .collect(), range: Some(source_range_to_lsp_range(&self.range, source_text_info)), } } - fn find_step(&self, name: &str, level: usize) -> Option<&TestDefinition> { - if let Some(steps) = &self.steps { - for step in steps { - if step.name == name && step.level == level { - return Some(step); - } else if let Some(step) = step.find_step(name, level) { - return Some(step); - } - } - } - None + fn contains_id<S: AsRef<str>>(&self, id: S) -> bool { + let id = id.as_ref(); + self.id == id || self.steps.iter().any(|td| td.contains_id(id)) } } @@ -102,6 +93,16 @@ pub struct TestDefinitions { pub script_version: String, } +impl Default for TestDefinitions { + fn default() -> Self { + TestDefinitions { + script_version: "1".to_string(), + discovered: vec![], + injected: vec![], + } + } +} + impl TestDefinitions { /// Return the test definitions as a testing module notification. pub fn as_notification( @@ -137,6 +138,19 @@ impl TestDefinitions { }) } + /// Register a dynamically-detected test. Returns false if a test with the + /// same static id was already registered statically or dynamically. Otherwise + /// returns true. + pub fn inject(&mut self, data: lsp_custom::TestData) -> bool { + if self.discovered.iter().any(|td| td.contains_id(&data.id)) + || self.injected.iter().any(|td| td.id == data.id) + { + return false; + } + self.injected.push(data); + true + } + /// Return a test definition identified by the test ID. pub fn get_by_id<S: AsRef<str>>(&self, id: S) -> Option<&TestDefinition> { self @@ -144,20 +158,4 @@ impl TestDefinitions { .iter() .find(|td| td.id.as_str() == id.as_ref()) } - - /// Return a test definition by the test name. - pub fn get_by_name(&self, name: &str) -> Option<&TestDefinition> { - self.discovered.iter().find(|td| td.name.as_str() == name) - } - - pub fn get_step_by_name( - &self, - test_name: &str, - level: usize, - name: &str, - ) -> Option<&TestDefinition> { - self - .get_by_name(test_name) - .and_then(|td| td.find_step(name, level)) - } } diff --git a/cli/lsp/testing/execution.rs b/cli/lsp/testing/execution.rs index d5c0f6278..7c0552a0a 100644 --- a/cli/lsp/testing/execution.rs +++ b/cli/lsp/testing/execution.rs @@ -25,13 +25,13 @@ use deno_core::futures::future; use deno_core::futures::stream; use deno_core::futures::StreamExt; use deno_core::parking_lot::Mutex; -use deno_core::serde_json::json; -use deno_core::serde_json::Value; +use deno_core::parking_lot::RwLock; use deno_core::ModuleSpecifier; use deno_runtime::ops::io::Stdio; use deno_runtime::ops::io::StdioPipe; use deno_runtime::permissions::Permissions; use deno_runtime::tokio_util::run_local; +use indexmap::IndexMap; use std::collections::HashMap; use std::collections::HashSet; use std::sync::Arc; @@ -48,10 +48,10 @@ fn as_queue_and_filters( tests: &HashMap<ModuleSpecifier, TestDefinitions>, ) -> ( HashSet<ModuleSpecifier>, - HashMap<ModuleSpecifier, TestFilter>, + HashMap<ModuleSpecifier, LspTestFilter>, ) { let mut queue: HashSet<ModuleSpecifier> = HashSet::new(); - let mut filters: HashMap<ModuleSpecifier, TestFilter> = HashMap::new(); + let mut filters: HashMap<ModuleSpecifier, LspTestFilter> = HashMap::new(); if let Some(include) = ¶ms.include { for item in include { @@ -61,12 +61,12 @@ fn as_queue_and_filters( if let Some(test) = test_definitions.get_by_id(id) { let filter = filters.entry(item.text_document.uri.clone()).or_default(); - if let Some(include) = filter.maybe_include.as_mut() { + if let Some(include) = filter.include.as_mut() { include.insert(test.id.clone(), test.clone()); } else { let mut include = HashMap::new(); include.insert(test.id.clone(), test.clone()); - filter.maybe_include = Some(include); + filter.include = Some(include); } } } @@ -80,29 +80,20 @@ fn as_queue_and_filters( queue.extend(tests.keys().cloned()); } - if let Some(exclude) = ¶ms.exclude { - for item in exclude { - if let Some(test_definitions) = tests.get(&item.text_document.uri) { - if let Some(id) = &item.id { - // there is currently no way to filter out a specific test, so we have - // to ignore the exclusion - if item.step_id.is_none() { - if let Some(test) = test_definitions.get_by_id(id) { - let filter = - filters.entry(item.text_document.uri.clone()).or_default(); - if let Some(exclude) = filter.maybe_exclude.as_mut() { - exclude.insert(test.id.clone(), test.clone()); - } else { - let mut exclude = HashMap::new(); - exclude.insert(test.id.clone(), test.clone()); - filter.maybe_exclude = Some(exclude); - } - } + for item in ¶ms.exclude { + if let Some(test_definitions) = tests.get(&item.text_document.uri) { + if let Some(id) = &item.id { + // there is no way to exclude a test step + if item.step_id.is_none() { + if let Some(test) = test_definitions.get_by_id(id) { + let filter = + filters.entry(item.text_document.uri.clone()).or_default(); + filter.exclude.insert(test.id.clone(), test.clone()); } - } else { - // the entire test module is excluded - queue.remove(&item.text_document.uri); } + } else { + // the entire test module is excluded + queue.remove(&item.text_document.uri); } } } @@ -131,14 +122,14 @@ fn as_test_messages<S: AsRef<str>>( } #[derive(Debug, Clone, Default, PartialEq)] -struct TestFilter { - maybe_include: Option<HashMap<String, TestDefinition>>, - maybe_exclude: Option<HashMap<String, TestDefinition>>, +struct LspTestFilter { + include: Option<HashMap<String, TestDefinition>>, + exclude: HashMap<String, TestDefinition>, } -impl TestFilter { +impl LspTestFilter { fn as_ids(&self, test_definitions: &TestDefinitions) -> Vec<String> { - let ids: Vec<String> = if let Some(include) = &self.maybe_include { + let ids: Vec<String> = if let Some(include) = &self.include { include.keys().cloned().collect() } else { test_definitions @@ -147,33 +138,10 @@ impl TestFilter { .map(|td| td.id.clone()) .collect() }; - if let Some(exclude) = &self.maybe_exclude { - ids - .into_iter() - .filter(|id| !exclude.contains_key(id)) - .collect() - } else { - ids - } - } - - /// return the filter as a JSON value, suitable for sending as a filter to the - /// test runner. - fn as_test_options(&self) -> Value { - let maybe_include: Option<Vec<String>> = self - .maybe_include - .as_ref() - .map(|inc| inc.iter().map(|(_, td)| td.name.clone()).collect()); - let maybe_exclude: Option<Vec<String>> = self - .maybe_exclude - .as_ref() - .map(|ex| ex.iter().map(|(_, td)| td.name.clone()).collect()); - json!({ - "filter": { - "include": maybe_include, - "exclude": maybe_exclude, - } - }) + ids + .into_iter() + .filter(|id| !self.exclude.contains_key(id)) + .collect() } } @@ -184,14 +152,14 @@ async fn test_specifier( mode: test::TestMode, sender: &TestEventSender, token: CancellationToken, - options: Option<Value>, + filter: test::TestFilter, ) -> Result<(), AnyError> { if !token.is_cancelled() { let mut worker = create_main_worker( &ps, specifier.clone(), permissions, - vec![ops::testing::init(sender.clone())], + vec![ops::testing::init(sender.clone(), filter)], Stdio { stdin: StdioPipe::Inherit, stdout: StdioPipe::File(sender.stdout()), @@ -217,10 +185,9 @@ async fn test_specifier( worker.dispatch_load_event(&located_script_name!())?; - let options = options.unwrap_or_else(|| json!({})); let test_result = worker.js_runtime.execute_script( &located_script_name!(), - &format!(r#"Deno[Deno.internal].runTests({})"#, json!(options)), + r#"Deno[Deno.internal].runTests()"#, )?; worker.js_runtime.resolve_value(test_result).await?; @@ -241,7 +208,7 @@ async fn test_specifier( pub struct TestRun { id: u32, kind: lsp_custom::TestRunKind, - filters: HashMap<ModuleSpecifier, TestFilter>, + filters: HashMap<ModuleSpecifier, LspTestFilter>, queue: HashSet<ModuleSpecifier>, tests: Arc<Mutex<HashMap<ModuleSpecifier, TestDefinitions>>>, token: CancellationToken, @@ -343,13 +310,31 @@ impl TestRun { let mut queue = self.queue.iter().collect::<Vec<&ModuleSpecifier>>(); queue.sort(); + let tests: Arc<RwLock<IndexMap<usize, test::TestDescription>>> = + Arc::new(RwLock::new(IndexMap::new())); + let mut test_steps = IndexMap::new(); + + let tests_ = tests.clone(); let join_handles = queue.into_iter().map(move |specifier| { let specifier = specifier.clone(); let ps = ps.clone(); let permissions = permissions.clone(); let mut sender = sender.clone(); - let options = self.filters.get(&specifier).map(|f| f.as_test_options()); + let lsp_filter = self.filters.get(&specifier); + let filter = test::TestFilter { + substring: None, + regex: None, + include: lsp_filter.and_then(|f| { + f.include + .as_ref() + .map(|i| i.values().map(|t| t.name.clone()).collect()) + }), + exclude: lsp_filter + .map(|f| f.exclude.values().map(|t| t.name.clone()).collect()) + .unwrap_or_default(), + }; let token = self.token.clone(); + let tests = tests_.clone(); tokio::task::spawn_blocking(move || { let origin = specifier.to_string(); @@ -360,14 +345,23 @@ impl TestRun { test::TestMode::Executable, &sender, token, - options, + filter, )); if let Err(error) = file_result { if error.is::<JsError>() { sender.send(test::TestEvent::UncaughtError( - origin, + origin.clone(), Box::new(error.downcast::<JsError>().unwrap()), ))?; + for desc in tests.read().values() { + if desc.origin == origin { + sender.send(test::TestEvent::Result( + desc.id, + test::TestResult::Cancelled, + 0, + ))? + } + } } else { return Err(error); } @@ -396,6 +390,10 @@ impl TestRun { while let Some(event) = receiver.recv().await { match event { + test::TestEvent::Register(description) => { + reporter.report_register(&description); + tests.write().insert(description.id, description); + } test::TestEvent::Plan(plan) => { summary.total += plan.total; summary.filtered_out += plan.filtered_out; @@ -406,13 +404,14 @@ impl TestRun { reporter.report_plan(&plan); } - test::TestEvent::Wait(description) => { - reporter.report_wait(&description); + test::TestEvent::Wait(id) => { + reporter.report_wait(tests.read().get(&id).unwrap()); } test::TestEvent::Output(output) => { reporter.report_output(&output); } - test::TestEvent::Result(description, result, elapsed) => { + test::TestEvent::Result(id, result, elapsed) => { + let description = tests.read().get(&id).unwrap().clone(); match &result { test::TestResult::Ok => summary.passed += 1, test::TestResult::Ignored => summary.ignored += 1, @@ -420,6 +419,9 @@ impl TestRun { summary.failed += 1; summary.failures.push((description.clone(), error.clone())); } + test::TestResult::Cancelled => { + summary.failed += 1; + } } reporter.report_result(&description, &result, elapsed); @@ -429,10 +431,14 @@ impl TestRun { summary.failed += 1; summary.uncaught_errors.push((origin, error)); } - test::TestEvent::StepWait(description) => { - reporter.report_step_wait(&description); + test::TestEvent::StepRegister(description) => { + reporter.report_step_register(&description); + test_steps.insert(description.id, description); } - test::TestEvent::StepResult(description, result, duration) => { + test::TestEvent::StepWait(id) => { + reporter.report_step_wait(test_steps.get(&id).unwrap()); + } + test::TestEvent::StepResult(id, result, duration) => { match &result { test::TestStepResult::Ok => { summary.passed_steps += 1; @@ -447,7 +453,11 @@ impl TestRun { summary.pending_steps += 1; } } - reporter.report_step_result(&description, &result, duration); + reporter.report_step_result( + test_steps.get(&id).unwrap(), + &result, + duration, + ); } } @@ -562,10 +572,8 @@ impl From<&TestOrTestStepDescription> for lsp_custom::TestData { impl From<&test::TestDescription> for lsp_custom::TestData { fn from(desc: &test::TestDescription) -> Self { - let id = checksum::gen(&[desc.origin.as_bytes(), desc.name.as_bytes()]); - Self { - id, + id: desc.static_id(), label: desc.name.clone(), steps: Default::default(), range: None, @@ -576,14 +584,9 @@ impl From<&test::TestDescription> for lsp_custom::TestData { impl From<&test::TestDescription> for lsp_custom::TestIdentifier { fn from(desc: &test::TestDescription) -> Self { let uri = ModuleSpecifier::parse(&desc.origin).unwrap(); - let id = Some(checksum::gen(&[ - desc.origin.as_bytes(), - desc.name.as_bytes(), - ])); - Self { text_document: lsp::TextDocumentIdentifier { uri }, - id, + id: Some(desc.static_id()), step_id: None, } } @@ -591,14 +594,8 @@ impl From<&test::TestDescription> for lsp_custom::TestIdentifier { impl From<&test::TestStepDescription> for lsp_custom::TestData { fn from(desc: &test::TestStepDescription) -> Self { - let id = checksum::gen(&[ - desc.test.origin.as_bytes(), - &desc.level.to_be_bytes(), - desc.name.as_bytes(), - ]); - Self { - id, + id: desc.static_id(), label: desc.name.clone(), steps: Default::default(), range: None, @@ -608,21 +605,14 @@ impl From<&test::TestStepDescription> for lsp_custom::TestData { impl From<&test::TestStepDescription> for lsp_custom::TestIdentifier { fn from(desc: &test::TestStepDescription) -> Self { - let uri = ModuleSpecifier::parse(&desc.test.origin).unwrap(); - let id = Some(checksum::gen(&[ - desc.test.origin.as_bytes(), - desc.test.name.as_bytes(), - ])); - let step_id = Some(checksum::gen(&[ - desc.test.origin.as_bytes(), - &desc.level.to_be_bytes(), - desc.name.as_bytes(), - ])); - + let uri = ModuleSpecifier::parse(&desc.origin).unwrap(); Self { text_document: lsp::TextDocumentIdentifier { uri }, - id, - step_id, + id: Some(checksum::gen(&[ + desc.origin.as_bytes(), + desc.root_name.as_bytes(), + ])), + step_id: Some(desc.static_id()), } } } @@ -653,61 +643,28 @@ impl LspTestReporter { } } - fn add_step(&self, desc: &test::TestStepDescription) { - if let Ok(specifier) = ModuleSpecifier::parse(&desc.test.origin) { - let mut tests = self.tests.lock(); - let entry = - tests - .entry(specifier.clone()) - .or_insert_with(|| TestDefinitions { - discovered: Default::default(), - injected: Default::default(), - script_version: "1".to_string(), - }); - let mut prev: lsp_custom::TestData = desc.into(); - if let Some(stack) = self.stack.get(&desc.test.origin) { - for item in stack.iter().rev() { - let mut data: lsp_custom::TestData = item.into(); - data.steps = Some(vec![prev]); - prev = data; - } - entry.injected.push(prev.clone()); - let label = if let Some(root) = &self.maybe_root_uri { - specifier.as_str().replace(root.as_str(), "") - } else { - specifier - .path_segments() - .and_then(|s| s.last().map(|s| s.to_string())) - .unwrap_or_else(|| "<unknown>".to_string()) - }; - self - .client - .send_test_notification(TestingNotification::Module( - lsp_custom::TestModuleNotificationParams { - text_document: lsp::TextDocumentIdentifier { uri: specifier }, - kind: lsp_custom::TestModuleNotificationKind::Insert, - label, - tests: vec![prev], - }, - )); - } - } + fn progress(&self, message: lsp_custom::TestRunProgressMessage) { + self + .client + .send_test_notification(TestingNotification::Progress( + lsp_custom::TestRunProgressParams { + id: self.id, + message, + }, + )); } +} - /// Add a test which is being reported from the test runner but was not - /// statically identified - fn add_test(&self, desc: &test::TestDescription) { - if let Ok(specifier) = ModuleSpecifier::parse(&desc.origin) { - let mut tests = self.tests.lock(); - let entry = - tests - .entry(specifier.clone()) - .or_insert_with(|| TestDefinitions { - discovered: Default::default(), - injected: Default::default(), - script_version: "1".to_string(), - }); - entry.injected.push(desc.into()); +impl test::TestReporter for LspTestReporter { + fn report_plan(&mut self, _plan: &test::TestPlan) {} + + fn report_register(&mut self, desc: &test::TestDescription) { + let mut tests = self.tests.lock(); + let tds = tests + .entry(ModuleSpecifier::parse(&desc.location.file_name).unwrap()) + .or_default(); + if tds.inject(desc.into()) { + let specifier = ModuleSpecifier::parse(&desc.origin).unwrap(); let label = if let Some(root) = &self.maybe_root_uri { specifier.as_str().replace(root.as_str(), "") } else { @@ -729,49 +686,7 @@ impl LspTestReporter { } } - fn progress(&self, message: lsp_custom::TestRunProgressMessage) { - self - .client - .send_test_notification(TestingNotification::Progress( - lsp_custom::TestRunProgressParams { - id: self.id, - message, - }, - )); - } - - fn includes_step(&self, desc: &test::TestStepDescription) -> bool { - if let Ok(specifier) = ModuleSpecifier::parse(&desc.test.origin) { - let tests = self.tests.lock(); - if let Some(test_definitions) = tests.get(&specifier) { - return test_definitions - .get_step_by_name(&desc.test.name, desc.level, &desc.name) - .is_some(); - } - } - false - } - - fn includes_test(&self, desc: &test::TestDescription) -> bool { - if let Ok(specifier) = ModuleSpecifier::parse(&desc.origin) { - let tests = self.tests.lock(); - if let Some(test_definitions) = tests.get(&specifier) { - return test_definitions.get_by_name(&desc.name).is_some(); - } - } - false - } -} - -impl test::TestReporter for LspTestReporter { - fn report_plan(&mut self, _plan: &test::TestPlan) { - // there is nothing to do on report_plan - } - fn report_wait(&mut self, desc: &test::TestDescription) { - if !self.includes_test(desc) { - self.add_test(desc); - } self.current_origin = Some(desc.origin.clone()); let test: lsp_custom::TestIdentifier = desc.into(); let stack = self.stack.entry(desc.origin.clone()).or_default(); @@ -827,6 +742,13 @@ impl test::TestReporter for LspTestReporter { duration: Some(elapsed as u32), }) } + test::TestResult::Cancelled => { + self.progress(lsp_custom::TestRunProgressMessage::Failed { + test: desc.into(), + messages: vec![], + duration: Some(elapsed as u32), + }) + } } } @@ -861,13 +783,46 @@ impl test::TestReporter for LspTestReporter { } } - fn report_step_wait(&mut self, desc: &test::TestStepDescription) { - if !self.includes_step(desc) { - self.add_step(desc); + fn report_step_register(&mut self, desc: &test::TestStepDescription) { + let mut tests = self.tests.lock(); + let tds = tests + .entry(ModuleSpecifier::parse(&desc.location.file_name).unwrap()) + .or_default(); + if tds.inject(desc.into()) { + let specifier = ModuleSpecifier::parse(&desc.origin).unwrap(); + let mut prev: lsp_custom::TestData = desc.into(); + if let Some(stack) = self.stack.get(&desc.origin) { + for item in stack.iter().rev() { + let mut data: lsp_custom::TestData = item.into(); + data.steps = vec![prev]; + prev = data; + } + let label = if let Some(root) = &self.maybe_root_uri { + specifier.as_str().replace(root.as_str(), "") + } else { + specifier + .path_segments() + .and_then(|s| s.last().map(|s| s.to_string())) + .unwrap_or_else(|| "<unknown>".to_string()) + }; + self + .client + .send_test_notification(TestingNotification::Module( + lsp_custom::TestModuleNotificationParams { + text_document: lsp::TextDocumentIdentifier { uri: specifier }, + kind: lsp_custom::TestModuleNotificationKind::Insert, + label, + tests: vec![prev], + }, + )); + } } + } + + fn report_step_wait(&mut self, desc: &test::TestStepDescription) { let test: lsp_custom::TestIdentifier = desc.into(); - let stack = self.stack.entry(desc.test.origin.clone()).or_default(); - self.current_origin = Some(desc.test.origin.clone()); + let stack = self.stack.entry(desc.origin.clone()).or_default(); + self.current_origin = Some(desc.origin.clone()); assert!(!stack.is_empty()); stack.push(desc.into()); self.progress(lsp_custom::TestRunProgressMessage::Started { test }); @@ -879,7 +834,7 @@ impl test::TestReporter for LspTestReporter { result: &test::TestStepResult, elapsed: u64, ) { - let stack = self.stack.entry(desc.test.origin.clone()).or_default(); + let stack = self.stack.entry(desc.origin.clone()).or_default(); assert_eq!(stack.pop(), Some(desc.into())); match result { test::TestStepResult::Ok => { @@ -927,6 +882,7 @@ impl test::TestReporter for LspTestReporter { mod tests { use super::*; use crate::lsp::testing::collectors::tests::new_range; + use deno_core::serde_json::json; #[test] fn test_as_queue_and_filters() { @@ -941,7 +897,7 @@ mod tests { id: None, step_id: None, }]), - exclude: Some(vec![lsp_custom::TestIdentifier { + exclude: vec![lsp_custom::TestIdentifier { text_document: lsp::TextDocumentIdentifier { uri: specifier.clone(), }, @@ -950,7 +906,7 @@ mod tests { .to_string(), ), step_id: None, - }]), + }], }; let mut tests = HashMap::new(); let test_def_a = TestDefinition { @@ -959,7 +915,7 @@ mod tests { level: 0, name: "test a".to_string(), range: new_range(420, 424), - steps: None, + steps: vec![], }; let test_def_b = TestDefinition { id: "69d9fe87f64f5b66cb8b631d4fd2064e8224b8715a049be54276c42189ff8f9f" @@ -967,7 +923,7 @@ mod tests { level: 0, name: "test b".to_string(), range: new_range(480, 481), - steps: None, + steps: vec![], }; let test_definitions = TestDefinitions { discovered: vec![test_def_a, test_def_b.clone()], @@ -988,9 +944,9 @@ mod tests { let filter = maybe_filter.unwrap(); assert_eq!( filter, - &TestFilter { - maybe_include: None, - maybe_exclude: Some(exclude), + &LspTestFilter { + include: None, + exclude, } ); assert_eq!( @@ -1000,14 +956,5 @@ mod tests { .to_string() ] ); - assert_eq!( - filter.as_test_options(), - json!({ - "filter": { - "include": null, - "exclude": vec!["test b"], - } - }) - ); } } diff --git a/cli/lsp/testing/lsp_custom.rs b/cli/lsp/testing/lsp_custom.rs index 8182371ca..59df9884d 100644 --- a/cli/lsp/testing/lsp_custom.rs +++ b/cli/lsp/testing/lsp_custom.rs @@ -21,8 +21,9 @@ pub struct TestData { pub id: String, /// The human readable test to display for the test. pub label: String, - #[serde(skip_serializing_if = "Option::is_none")] - pub steps: Option<Vec<TestData>>, + #[serde(skip_serializing_if = "Vec::is_empty")] + #[serde(default)] + pub steps: Vec<TestData>, /// The range where the test is located. #[serde(skip_serializing_if = "Option::is_none")] pub range: Option<lsp::Range>, @@ -92,8 +93,9 @@ pub enum TestRunKind { pub struct TestRunRequestParams { pub id: u32, pub kind: TestRunKind, - #[serde(skip_serializing_if = "Option::is_none")] - pub exclude: Option<Vec<TestIdentifier>>, + #[serde(skip_serializing_if = "Vec::is_empty")] + #[serde(default)] + pub exclude: Vec<TestIdentifier>, #[serde(skip_serializing_if = "Option::is_none")] pub include: Option<Vec<TestIdentifier>>, } diff --git a/cli/ops/testing.rs b/cli/ops/testing.rs index 705353112..727ccdf66 100644 --- a/cli/ops/testing.rs +++ b/cli/ops/testing.rs @@ -1,7 +1,11 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. +use crate::tools::test::TestDescription; use crate::tools::test::TestEvent; use crate::tools::test::TestEventSender; +use crate::tools::test::TestFilter; +use crate::tools::test::TestLocation; +use crate::tools::test::TestStepDescription; use deno_core::error::generic_error; use deno_core::error::AnyError; @@ -12,18 +16,26 @@ use deno_core::OpState; use deno_runtime::permissions::create_child_permissions; use deno_runtime::permissions::ChildPermissionsArg; use deno_runtime::permissions::Permissions; +use serde::Deserialize; +use serde::Deserializer; +use serde::Serialize; +use std::sync::atomic::AtomicUsize; +use std::sync::atomic::Ordering; use uuid::Uuid; -pub fn init(sender: TestEventSender) -> Extension { +pub fn init(sender: TestEventSender, filter: TestFilter) -> Extension { Extension::builder() .ops(vec![ op_pledge_test_permissions::decl(), op_restore_test_permissions::decl(), op_get_test_origin::decl(), + op_register_test::decl(), + op_register_test_step::decl(), op_dispatch_test_event::decl(), ]) .state(move |state| { state.put(sender.clone()); + state.put(filter.clone()); Ok(()) }) .build() @@ -76,6 +88,91 @@ fn op_get_test_origin(state: &mut OpState) -> Result<String, AnyError> { Ok(state.borrow::<ModuleSpecifier>().to_string()) } +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct TestInfo { + name: String, + origin: String, + location: TestLocation, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +struct TestRegisterResult { + id: usize, + filtered_out: bool, +} + +static NEXT_ID: AtomicUsize = AtomicUsize::new(0); + +#[op] +fn op_register_test( + state: &mut OpState, + info: TestInfo, +) -> Result<TestRegisterResult, AnyError> { + let id = NEXT_ID.fetch_add(1, Ordering::SeqCst); + let filter = state.borrow::<TestFilter>().clone(); + let filtered_out = !filter.includes(&info.name); + let description = TestDescription { + id, + name: info.name, + origin: info.origin, + location: info.location, + }; + let mut sender = state.borrow::<TestEventSender>().clone(); + sender.send(TestEvent::Register(description)).ok(); + Ok(TestRegisterResult { id, filtered_out }) +} + +fn deserialize_parent<'de, D>(deserializer: D) -> Result<usize, D::Error> +where + D: Deserializer<'de>, +{ + #[derive(Deserialize)] + struct Parent { + id: usize, + } + Ok(Parent::deserialize(deserializer)?.id) +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct TestStepInfo { + name: String, + origin: String, + location: TestLocation, + level: usize, + #[serde(rename = "parent")] + #[serde(deserialize_with = "deserialize_parent")] + parent_id: usize, + root_id: usize, + root_name: String, +} + +#[op] +fn op_register_test_step( + state: &mut OpState, + info: TestStepInfo, +) -> Result<TestRegisterResult, AnyError> { + let id = NEXT_ID.fetch_add(1, Ordering::SeqCst); + let description = TestStepDescription { + id, + name: info.name, + origin: info.origin, + location: info.location, + level: info.level, + parent_id: info.parent_id, + root_id: info.root_id, + root_name: info.root_name, + }; + let mut sender = state.borrow::<TestEventSender>().clone(); + sender.send(TestEvent::StepRegister(description)).ok(); + Ok(TestRegisterResult { + id, + filtered_out: false, + }) +} + #[op] fn op_dispatch_test_event( state: &mut OpState, diff --git a/cli/tests/testdata/test/steps/failing_steps.out b/cli/tests/testdata/test/steps/failing_steps.out index 7c1d62488..d8c2bdf8d 100644 --- a/cli/tests/testdata/test/steps/failing_steps.out +++ b/cli/tests/testdata/test/steps/failing_steps.out @@ -9,8 +9,8 @@ nested failure ... at [WILDCARD]/failing_steps.ts:[WILDCARD] [WILDCARD] inner 2 ... ok ([WILDCARD]) - FAILED ([WILDCARD]) -FAILED ([WILDCARD]) + step 1 ... FAILED ([WILDCARD]) +nested failure ... FAILED ([WILDCARD]) multiple test step failures ... step 1 ... FAILED ([WILDCARD]) error: Error: Fail. @@ -23,7 +23,7 @@ multiple test step failures ... ^ at [WILDCARD]/failing_steps.ts:[WILDCARD] [WILDCARD] -FAILED ([WILDCARD]) +multiple test step failures ... FAILED ([WILDCARD]) failing step in failing test ... step 1 ... FAILED ([WILDCARD]) error: Error: Fail. @@ -31,7 +31,7 @@ failing step in failing test ... ^ at [WILDCARD]/failing_steps.ts:[WILDCARD] at [WILDCARD] -FAILED ([WILDCARD]) +failing step in failing test ... FAILED ([WILDCARD]) ERRORS diff --git a/cli/tests/testdata/test/steps/ignored_steps.out b/cli/tests/testdata/test/steps/ignored_steps.out index f80b6573f..2786e1e1a 100644 --- a/cli/tests/testdata/test/steps/ignored_steps.out +++ b/cli/tests/testdata/test/steps/ignored_steps.out @@ -3,6 +3,6 @@ running 1 test from ./test/steps/ignored_steps.ts ignored step ... step 1 ... ignored ([WILDCARD]) step 2 ... ok ([WILDCARD]) -ok ([WILDCARD]) +ignored step ... ok ([WILDCARD]) ok | 1 passed (1 step) | 0 failed | 0 ignored (1 step) [WILDCARD] diff --git a/cli/tests/testdata/test/steps/invalid_usage.out b/cli/tests/testdata/test/steps/invalid_usage.out index 395356e2d..dc97a5eed 100644 --- a/cli/tests/testdata/test/steps/invalid_usage.out +++ b/cli/tests/testdata/test/steps/invalid_usage.out @@ -2,23 +2,23 @@ running 7 tests from ./test/steps/invalid_usage.ts capturing ... some step ... ok ([WILDCARD]) -FAILED ([WILDCARD]) +capturing ... FAILED ([WILDCARD]) top level missing await ... step ... pending ([WILDCARD]) -FAILED ([WILDCARD]) +top level missing await ... FAILED ([WILDCARD]) inner missing await ... step ... inner ... pending ([WILDCARD]) error: Error: Parent scope completed before test step finished execution. Ensure all steps are awaited (ex. `await t.step(...)`). at [WILDCARD] at async TestContext.step [WILDCARD] - FAILED ([WILDCARD]) + step ... FAILED ([WILDCARD]) error: Error: There were still test steps running after the current scope finished execution. Ensure all steps are awaited (ex. `await t.step(...)`). await t.step("step", (t) => { ^ at [WILDCARD] at async fn ([WILDCARD]/invalid_usage.ts:[WILDCARD]) -FAILED ([WILDCARD]) +inner missing await ... FAILED ([WILDCARD]) parallel steps with sanitizers ... step 1 ... pending ([WILDCARD]) step 2 ... FAILED ([WILDCARD]) @@ -28,7 +28,7 @@ parallel steps with sanitizers ... ^ at [WILDCARD] at [WILDCARD]/invalid_usage.ts:[WILDCARD] -FAILED ([WILDCARD]) +parallel steps with sanitizers ... FAILED ([WILDCARD]) parallel steps when first has sanitizer ... step 1 ... pending ([WILDCARD]) step 2 ... FAILED ([WILDCARD]) @@ -38,7 +38,7 @@ parallel steps when first has sanitizer ... ^ at [WILDCARD] at [WILDCARD]/invalid_usage.ts:[WILDCARD] -FAILED ([WILDCARD]) +parallel steps when first has sanitizer ... FAILED ([WILDCARD]) parallel steps when second has sanitizer ... step 1 ... ok ([WILDCARD]) step 2 ... FAILED ([WILDCARD]) @@ -48,11 +48,11 @@ parallel steps when second has sanitizer ... ^ at [WILDCARD] at [WILDCARD]/invalid_usage.ts:[WILDCARD] -FAILED ([WILDCARD]) +parallel steps when second has sanitizer ... FAILED ([WILDCARD]) parallel steps where only inner tests have sanitizers ... step 1 ... step inner ... ok ([WILDCARD]) - ok ([WILDCARD]) + step 1 ... ok ([WILDCARD]) step 2 ... step inner ... FAILED ([WILDCARD]) error: Error: Cannot start test step with sanitizers while another test step is running. @@ -61,8 +61,8 @@ parallel steps where only inner tests have sanitizers ... ^ at [WILDCARD] at [WILDCARD]/invalid_usage.ts:[WILDCARD] - FAILED ([WILDCARD]) -FAILED ([WILDCARD]) + step 2 ... FAILED ([WILDCARD]) +parallel steps where only inner tests have sanitizers ... FAILED ([WILDCARD]) ERRORS @@ -75,8 +75,6 @@ error: Error: Cannot run test step after parent scope has finished execution. En top level missing await => ./test/steps/invalid_usage.ts:[WILDCARD] error: Error: There were still test steps running after the current scope finished execution. Ensure all steps are awaited (ex. `await t.step(...)`). - at postValidation [WILDCARD] - at testStepSanitizer ([WILDCARD]) [WILDCARD] inner missing await => ./test/steps/invalid_usage.ts:[WILDCARD] @@ -85,8 +83,6 @@ error: Error: 1 test step failed. parallel steps with sanitizers => ./test/steps/invalid_usage.ts:[WILDCARD] error: Error: There were still test steps running after the current scope finished execution. Ensure all steps are awaited (ex. `await t.step(...)`). - at postValidation [WILDCARD] - at testStepSanitizer ([WILDCARD]) [WILDCARD] parallel steps when first has sanitizer => ./test/steps/invalid_usage.ts:[WILDCARD] diff --git a/cli/tests/testdata/test/steps/passing_steps.out b/cli/tests/testdata/test/steps/passing_steps.out index bb1611910..5eacc571c 100644 --- a/cli/tests/testdata/test/steps/passing_steps.out +++ b/cli/tests/testdata/test/steps/passing_steps.out @@ -4,35 +4,35 @@ description ... step 1 ... inner 1 ... ok ([WILDCARD]ms) inner 2 ... ok ([WILDCARD]ms) - ok ([WILDCARD]ms) -ok ([WILDCARD]ms) + step 1 ... ok ([WILDCARD]ms) +description ... ok ([WILDCARD]ms) parallel steps without sanitizers ... step 1 ... ok ([WILDCARD]) step 2 ... ok ([WILDCARD]) -ok ([WILDCARD]) +parallel steps without sanitizers ... ok ([WILDCARD]) parallel steps without sanitizers due to parent ... step 1 ... ok ([WILDCARD]) step 2 ... ok ([WILDCARD]) -ok ([WILDCARD]) +parallel steps without sanitizers due to parent ... ok ([WILDCARD]) steps with disabled sanitizers, then enabled, then parallel disabled ... step 1 ... step 1 ... step 1 ... step 1 ... ok ([WILDCARD]) step 1 ... ok ([WILDCARD]) - ok ([WILDCARD]) + step 1 ... ok ([WILDCARD]) step 2 ... ok ([WILDCARD]) - ok ([WILDCARD]) - ok ([WILDCARD]) -ok ([WILDCARD]) + step 1 ... ok ([WILDCARD]) + step 1 ... ok ([WILDCARD]) +steps with disabled sanitizers, then enabled, then parallel disabled ... ok ([WILDCARD]) steps buffered then streaming reporting ... step 1 ... step 1 - 1 ... ok ([WILDCARD]) step 1 - 2 ... step 1 - 2 - 1 ... ok ([WILDCARD]) - ok ([WILDCARD]) - ok ([WILDCARD]) + step 1 - 2 ... ok ([WILDCARD]) + step 1 ... ok ([WILDCARD]) step 2 ... ok ([WILDCARD]) -ok ([WILDCARD]) +steps buffered then streaming reporting ... ok ([WILDCARD]) ok | 5 passed (18 steps) | 0 failed [WILDCARD] diff --git a/cli/tests/testdata/test/uncaught_errors.out b/cli/tests/testdata/test/uncaught_errors.out index 3c4dc2f9b..2eae72e21 100644 --- a/cli/tests/testdata/test/uncaught_errors.out +++ b/cli/tests/testdata/test/uncaught_errors.out @@ -3,6 +3,7 @@ foo 1 ... FAILED ([WILDCARD]) foo 2 ... ok ([WILDCARD]) foo 3 ... Uncaught error from ./test/uncaught_errors_1.ts FAILED +foo 3 ... cancelled (0ms) running 3 tests from ./test/uncaught_errors_2.ts bar 1 ... ok ([WILDCARD]) bar 2 ... FAILED ([WILDCARD]) @@ -53,6 +54,6 @@ bar 2 => ./test/uncaught_errors_2.ts:3:6 bar 3 => ./test/uncaught_errors_2.ts:6:6 ./test/uncaught_errors_3.ts (uncaught error) -FAILED | 2 passed | 5 failed ([WILDCARD]) +FAILED | 2 passed | 6 failed ([WILDCARD]) error: Test failed diff --git a/cli/tools/test.rs b/cli/tools/test.rs index d5317a761..f3d4d6f6c 100644 --- a/cli/tools/test.rs +++ b/cli/tools/test.rs @@ -3,6 +3,7 @@ use crate::args::Flags; use crate::args::TestFlags; use crate::args::TypeCheckMode; +use crate::checksum; use crate::colors; use crate::compat; use crate::create_main_worker; @@ -32,6 +33,7 @@ use deno_core::futures::stream; use deno_core::futures::FutureExt; use deno_core::futures::StreamExt; use deno_core::parking_lot::Mutex; +use deno_core::parking_lot::RwLock; use deno_core::serde_json::json; use deno_core::url::Url; use deno_core::ModuleSpecifier; @@ -40,6 +42,7 @@ use deno_runtime::ops::io::Stdio; use deno_runtime::ops::io::StdioPipe; use deno_runtime::permissions::Permissions; use deno_runtime::tokio_util::run_local; +use indexmap::IndexMap; use log::Level; use rand::rngs::SmallRng; use rand::seq::SliceRandom; @@ -47,7 +50,6 @@ use rand::SeedableRng; use regex::Regex; use serde::Deserialize; use std::collections::BTreeMap; -use std::collections::HashMap; use std::collections::HashSet; use std::fmt::Write as _; use std::io::Read; @@ -72,7 +74,6 @@ pub enum TestMode { Both, } -// TODO(nayeemrmn): This is only used for benches right now. #[derive(Clone, Debug, Default)] pub struct TestFilter { pub substring: Option<String>, @@ -135,11 +136,18 @@ pub struct TestLocation { #[derive(Debug, Clone, PartialEq, Deserialize, Eq, Hash)] #[serde(rename_all = "camelCase")] pub struct TestDescription { - pub origin: String, + pub id: usize, pub name: String, + pub origin: String, pub location: TestLocation, } +impl TestDescription { + pub fn static_id(&self) -> String { + checksum::gen(&[self.location.file_name.as_bytes(), self.name.as_bytes()]) + } +} + #[derive(Debug, Clone, PartialEq, Deserialize)] #[serde(rename_all = "camelCase")] pub enum TestOutput { @@ -153,14 +161,30 @@ pub enum TestResult { Ok, Ignored, Failed(Box<JsError>), + Cancelled, } #[derive(Debug, Clone, PartialEq, Deserialize)] #[serde(rename_all = "camelCase")] pub struct TestStepDescription { - pub test: TestDescription, - pub level: usize, + pub id: usize, pub name: String, + pub origin: String, + pub location: TestLocation, + pub level: usize, + pub parent_id: usize, + pub root_id: usize, + pub root_name: String, +} + +impl TestStepDescription { + pub fn static_id(&self) -> String { + checksum::gen(&[ + self.location.file_name.as_bytes(), + &self.level.to_be_bytes(), + self.name.as_bytes(), + ]) + } } #[derive(Debug, Clone, PartialEq, Deserialize)] @@ -194,13 +218,15 @@ pub struct TestPlan { #[derive(Debug, Clone, Deserialize)] #[serde(rename_all = "camelCase")] pub enum TestEvent { + Register(TestDescription), Plan(TestPlan), - Wait(TestDescription), + Wait(usize), Output(Vec<u8>), - Result(TestDescription, TestResult, u64), + Result(usize, TestResult, u64), UncaughtError(String, Box<JsError>), - StepWait(TestStepDescription), - StepResult(TestStepDescription, TestStepResult, u64), + StepRegister(TestStepDescription), + StepWait(usize), + StepResult(usize, TestStepResult, u64), } #[derive(Debug, Clone, Deserialize)] @@ -219,12 +245,12 @@ pub struct TestSummary { pub uncaught_errors: Vec<(String, Box<JsError>)>, } -#[derive(Debug, Clone, Deserialize)] +#[derive(Debug, Clone)] struct TestSpecifierOptions { compat_mode: bool, concurrent_jobs: NonZeroUsize, fail_fast: Option<NonZeroUsize>, - filter: Option<String>, + filter: TestFilter, shuffle: Option<u64>, trace_ops: bool, } @@ -250,13 +276,10 @@ impl TestSummary { fn has_failed(&self) -> bool { self.failed > 0 || !self.failures.is_empty() } - - fn has_pending(&self) -> bool { - self.total - self.passed - self.failed - self.ignored > 0 - } } pub trait TestReporter { + fn report_register(&mut self, plan: &TestDescription); fn report_plan(&mut self, plan: &TestPlan); fn report_wait(&mut self, description: &TestDescription); fn report_output(&mut self, output: &[u8]); @@ -267,6 +290,7 @@ pub trait TestReporter { elapsed: u64, ); fn report_uncaught_error(&mut self, origin: &str, error: &JsError); + fn report_step_register(&mut self, description: &TestStepDescription); fn report_step_wait(&mut self, description: &TestStepDescription); fn report_step_result( &mut self, @@ -285,9 +309,9 @@ enum DeferredStepOutput { struct PrettyTestReporter { concurrent: bool, echo_output: bool, - deferred_step_output: HashMap<TestDescription, Vec<DeferredStepOutput>>, + deferred_step_output: IndexMap<usize, Vec<DeferredStepOutput>>, in_new_line: bool, - last_wait_output_level: usize, + last_wait_id: Option<usize>, cwd: Url, did_have_user_output: bool, started_tests: bool, @@ -299,8 +323,8 @@ impl PrettyTestReporter { concurrent, echo_output, in_new_line: true, - deferred_step_output: HashMap::new(), - last_wait_output_level: 0, + deferred_step_output: IndexMap::new(), + last_wait_id: None, cwd: Url::from_directory_path(std::env::current_dir().unwrap()).unwrap(), did_have_user_output: false, started_tests: false, @@ -308,11 +332,14 @@ impl PrettyTestReporter { } fn force_report_wait(&mut self, description: &TestDescription) { + if !self.in_new_line { + println!(); + } print!("{} ...", description.name); self.in_new_line = false; // flush for faster feedback when line buffered std::io::stdout().flush().unwrap(); - self.last_wait_output_level = 0; + self.last_wait_id = Some(description.id); } fn to_relative_path_or_remote_url(&self, path_or_url: &str) -> String { @@ -329,15 +356,15 @@ impl PrettyTestReporter { } fn force_report_step_wait(&mut self, description: &TestStepDescription) { - let wrote_user_output = self.write_output_end(); - if !wrote_user_output && self.last_wait_output_level < description.level { + self.write_output_end(); + if !self.in_new_line { println!(); } print!("{}{} ...", " ".repeat(description.level), description.name); self.in_new_line = false; // flush for faster feedback when line buffered std::io::stdout().flush().unwrap(); - self.last_wait_output_level = description.level; + self.last_wait_id = Some(description.id); } fn force_report_step_result( @@ -353,19 +380,13 @@ impl PrettyTestReporter { TestStepResult::Failed(_) => colors::red("FAILED").to_string(), }; - let wrote_user_output = self.write_output_end(); - if !wrote_user_output && self.last_wait_output_level == description.level { - print!(" "); - } else { - print!("{}", " ".repeat(description.level)); - } - - if wrote_user_output { - print!("{} ... ", description.name); + self.write_output_end(); + if self.in_new_line || self.last_wait_id != Some(description.id) { + self.force_report_step_wait(description); } println!( - "{} {}", + " {} {}", status, colors::gray(format!("({})", display::human_elapsed(elapsed.into()))) ); @@ -380,19 +401,18 @@ impl PrettyTestReporter { self.in_new_line = true; } - fn write_output_end(&mut self) -> bool { + fn write_output_end(&mut self) { if self.did_have_user_output { println!("{}", colors::gray("----- output end -----")); self.in_new_line = true; self.did_have_user_output = false; - true - } else { - false } } } impl TestReporter for PrettyTestReporter { + fn report_register(&mut self, _description: &TestDescription) {} + fn report_plan(&mut self, plan: &TestPlan) { let inflection = if plan.total == 1 { "test" } else { "tests" }; println!( @@ -440,7 +460,8 @@ impl TestReporter for PrettyTestReporter { if self.concurrent { self.force_report_wait(description); - if let Some(step_outputs) = self.deferred_step_output.remove(description) + if let Some(step_outputs) = + self.deferred_step_output.remove(&description.id) { for step_output in step_outputs { match step_output { @@ -461,23 +482,20 @@ impl TestReporter for PrettyTestReporter { } } - let wrote_user_output = self.write_output_end(); - if !wrote_user_output && self.last_wait_output_level == 0 { - print!(" "); - } - - if wrote_user_output { - print!("{} ... ", description.name); + self.write_output_end(); + if self.in_new_line || self.last_wait_id != Some(description.id) { + self.force_report_wait(description); } let status = match result { TestResult::Ok => colors::green("ok").to_string(), TestResult::Ignored => colors::yellow("ignored").to_string(), TestResult::Failed(_) => colors::red("FAILED").to_string(), + TestResult::Cancelled => colors::gray("cancelled").to_string(), }; println!( - "{} {}", + " {} {}", status, colors::gray(format!("({})", display::human_elapsed(elapsed.into()))) ); @@ -494,15 +512,16 @@ impl TestReporter for PrettyTestReporter { colors::red("FAILED") ); self.in_new_line = true; - self.last_wait_output_level = 0; self.did_have_user_output = false; } + fn report_step_register(&mut self, _description: &TestStepDescription) {} + fn report_step_wait(&mut self, description: &TestStepDescription) { if self.concurrent { self .deferred_step_output - .entry(description.test.to_owned()) + .entry(description.root_id) .or_insert_with(Vec::new) .push(DeferredStepOutput::StepWait(description.clone())); } else { @@ -519,7 +538,7 @@ impl TestReporter for PrettyTestReporter { if self.concurrent { self .deferred_step_output - .entry(description.test.to_owned()) + .entry(description.root_id) .or_insert_with(Vec::new) .push(DeferredStepOutput::StepResult( description.clone(), @@ -597,7 +616,7 @@ impl TestReporter for PrettyTestReporter { } } - let status = if summary.has_failed() || summary.has_pending() { + let status = if summary.has_failed() { colors::red("FAILED").to_string() } else { colors::green("ok").to_string() @@ -737,7 +756,7 @@ async fn test_specifier( &ps, specifier.clone(), permissions, - vec![ops::testing::init(sender.clone())], + vec![ops::testing::init(sender.clone(), options.filter.clone())], Stdio { stdin: StdioPipe::Inherit, stdout: StdioPipe::File(sender.stdout()), @@ -807,10 +826,7 @@ async fn test_specifier( &located_script_name!(), &format!( r#"Deno[Deno.internal].runTests({})"#, - json!({ - "filter": options.filter, - "shuffle": options.shuffle, - }), + json!({ "shuffle": options.shuffle }), ), )?; @@ -1106,7 +1122,11 @@ async fn test_specifiers( let sender = TestEventSender::new(sender); let concurrent_jobs = options.concurrent_jobs; let fail_fast = options.fail_fast; + let tests: Arc<RwLock<IndexMap<usize, TestDescription>>> = + Arc::new(RwLock::new(IndexMap::new())); + let mut test_steps = IndexMap::new(); + let tests_ = tests.clone(); let join_handles = specifiers_with_mode.iter().map(move |(specifier, mode)| { let ps = ps.clone(); @@ -1115,6 +1135,7 @@ async fn test_specifiers( let mode = mode.clone(); let mut sender = sender.clone(); let options = options.clone(); + let tests = tests_.clone(); tokio::task::spawn_blocking(move || { let origin = specifier.to_string(); @@ -1129,9 +1150,18 @@ async fn test_specifiers( if let Err(error) = file_result { if error.is::<JsError>() { sender.send(TestEvent::UncaughtError( - origin, + origin.clone(), Box::new(error.downcast::<JsError>().unwrap()), ))?; + for desc in tests.read().values() { + if desc.origin == origin { + sender.send(TestEvent::Result( + desc.id, + TestResult::Cancelled, + 0, + ))? + } + } } else { return Err(error); } @@ -1150,11 +1180,17 @@ async fn test_specifiers( let handler = { tokio::task::spawn(async move { let earlier = Instant::now(); + let mut tests_with_result = HashSet::new(); let mut summary = TestSummary::new(); let mut used_only = false; while let Some(event) = receiver.recv().await { match event { + TestEvent::Register(description) => { + reporter.report_register(&description); + tests.write().insert(description.id, description); + } + TestEvent::Plan(plan) => { summary.total += plan.total; summary.filtered_out += plan.filtered_out; @@ -1166,29 +1202,34 @@ async fn test_specifiers( reporter.report_plan(&plan); } - TestEvent::Wait(description) => { - reporter.report_wait(&description); + TestEvent::Wait(id) => { + reporter.report_wait(tests.read().get(&id).unwrap()); } TestEvent::Output(output) => { reporter.report_output(&output); } - TestEvent::Result(description, result, elapsed) => { - match &result { - TestResult::Ok => { - summary.passed += 1; - } - TestResult::Ignored => { - summary.ignored += 1; - } - TestResult::Failed(error) => { - summary.failed += 1; - summary.failures.push((description.clone(), error.clone())); + TestEvent::Result(id, result, elapsed) => { + if tests_with_result.insert(id) { + let description = tests.read().get(&id).unwrap().clone(); + match &result { + TestResult::Ok => { + summary.passed += 1; + } + TestResult::Ignored => { + summary.ignored += 1; + } + TestResult::Failed(error) => { + summary.failed += 1; + summary.failures.push((description.clone(), error.clone())); + } + TestResult::Cancelled => { + summary.failed += 1; + } } + reporter.report_result(&description, &result, elapsed); } - - reporter.report_result(&description, &result, elapsed); } TestEvent::UncaughtError(origin, error) => { @@ -1197,11 +1238,16 @@ async fn test_specifiers( summary.uncaught_errors.push((origin, error)); } - TestEvent::StepWait(description) => { - reporter.report_step_wait(&description); + TestEvent::StepRegister(description) => { + reporter.report_step_register(&description); + test_steps.insert(description.id, description); + } + + TestEvent::StepWait(id) => { + reporter.report_step_wait(test_steps.get(&id).unwrap()); } - TestEvent::StepResult(description, result, duration) => { + TestEvent::StepResult(id, result, duration) => { match &result { TestStepResult::Ok => { summary.passed_steps += 1; @@ -1217,7 +1263,11 @@ async fn test_specifiers( } } - reporter.report_step_result(&description, &result, duration); + reporter.report_step_result( + test_steps.get(&id).unwrap(), + &result, + duration, + ); } } @@ -1366,7 +1416,7 @@ pub async fn run_tests( compat_mode: compat, concurrent_jobs: test_flags.concurrent_jobs, fail_fast: test_flags.fail_fast, - filter: test_flags.filter, + filter: TestFilter::from_flag(&test_flags.filter), shuffle: test_flags.shuffle, trace_ops: test_flags.trace_ops, }, @@ -1550,7 +1600,7 @@ pub async fn run_tests_with_watch( compat_mode: cli_options.compat(), concurrent_jobs: test_flags.concurrent_jobs, fail_fast: test_flags.fail_fast, - filter: filter.clone(), + filter: TestFilter::from_flag(&filter), shuffle: test_flags.shuffle, trace_ops: test_flags.trace_ops, }, diff --git a/runtime/js/40_testing.js b/runtime/js/40_testing.js index 48f2fefb8..fabdb8dc4 100644 --- a/runtime/js/40_testing.js +++ b/runtime/js/40_testing.js @@ -14,25 +14,20 @@ ArrayPrototypeMap, ArrayPrototypePush, ArrayPrototypeShift, - ArrayPrototypeSome, ArrayPrototypeSort, DateNow, Error, FunctionPrototype, Map, + MapPrototypeGet, MapPrototypeHas, + MapPrototypeSet, MathCeil, ObjectKeys, ObjectPrototypeIsPrototypeOf, Promise, - RegExp, - RegExpPrototypeTest, SafeArrayIterator, Set, - StringPrototypeEndsWith, - StringPrototypeIncludes, - StringPrototypeSlice, - StringPrototypeStartsWith, SymbolToStringTag, TypeError, } = window.__bootstrap.primordials; @@ -139,12 +134,12 @@ // ops. Note that "unref" ops are ignored since in nature that are // optional. function assertOps(fn) { - /** @param step {TestStep} */ - return async function asyncOpSanitizer(step) { + /** @param desc {TestDescription | TestStepDescription} */ + return async function asyncOpSanitizer(desc) { const pre = core.metrics(); const preTraces = new Map(core.opCallTraces); try { - await fn(step); + await fn(desc); } finally { // Defer until next event loop turn - that way timeouts and intervals // cleared can actually be removed from resource table, otherwise @@ -152,7 +147,7 @@ await opSanitizerDelay(); } - if (step.shouldSkipSanitizers) return; + if (shouldSkipSanitizers(desc)) return; const post = core.metrics(); const postTraces = new Map(core.opCallTraces); @@ -366,15 +361,13 @@ // Wrap test function in additional assertion that makes sure // the test case does not "leak" resources - ie. resource table after // the test has exactly the same contents as before the test. - function assertResources( - fn, - ) { - /** @param step {TestStep} */ - return async function resourceSanitizer(step) { + function assertResources(fn) { + /** @param desc {TestDescription | TestStepDescription} */ + return async function resourceSanitizer(desc) { const pre = core.resources(); - await fn(step); + await fn(desc); - if (step.shouldSkipSanitizers) { + if (shouldSkipSanitizers(desc)) { return; } @@ -396,12 +389,12 @@ const hint = resourceCloseHint(postResource); const detail = `${name} (rid ${resource}) was ${action1} during the test, but not ${action2} during the test. ${hint}`; - details.push(detail); + ArrayPrototypePush(details, detail); } else { const [name, action1, action2] = prettyResourceNames(preResource); const detail = `${name} (rid ${resource}) was ${action1} before the test started, but was ${action2} during the test. Do not close resources in a test that were not created during that test.`; - details.push(detail); + ArrayPrototypePush(details, detail); } } @@ -439,79 +432,81 @@ } function assertTestStepScopes(fn) { - /** @param step {TestStep} */ - return async function testStepSanitizer(step) { + /** @param desc {TestDescription | TestStepDescription} */ + return async function testStepSanitizer(desc) { preValidation(); // only report waiting after pre-validation - if (step.canStreamReporting()) { - step.reportWait(); + if (canStreamReporting(desc) && "parent" in desc) { + stepReportWait(desc); } - await fn(createTestContext(step)); - postValidation(); + await fn(MapPrototypeGet(testStates, desc.id).context); + testStepPostValidation(desc); function preValidation() { - const runningSteps = getPotentialConflictingRunningSteps(); - const runningStepsWithSanitizers = ArrayPrototypeFilter( - runningSteps, - (t) => t.usesSanitizer, + const runningStepDescs = getRunningStepDescs(); + const runningStepDescsWithSanitizers = ArrayPrototypeFilter( + runningStepDescs, + (d) => usesSanitizer(d), ); - if (runningStepsWithSanitizers.length > 0) { + if (runningStepDescsWithSanitizers.length > 0) { throw new Error( "Cannot start test step while another test step with sanitizers is running.\n" + - runningStepsWithSanitizers - .map((s) => ` * ${s.getFullName()}`) + runningStepDescsWithSanitizers + .map((d) => ` * ${getFullName(d)}`) .join("\n"), ); } - if (step.usesSanitizer && runningSteps.length > 0) { + if (usesSanitizer(desc) && runningStepDescs.length > 0) { throw new Error( "Cannot start test step with sanitizers while another test step is running.\n" + - runningSteps.map((s) => ` * ${s.getFullName()}`).join("\n"), + runningStepDescs.map((d) => ` * ${getFullName(d)}`).join("\n"), ); } - function getPotentialConflictingRunningSteps() { - /** @type {TestStep[]} */ + function getRunningStepDescs() { const results = []; - - let childStep = step; - for (const ancestor of step.ancestors()) { - for (const siblingStep of ancestor.children) { - if (siblingStep === childStep) { + let childDesc = desc; + while (childDesc.parent != null) { + const state = MapPrototypeGet(testStates, childDesc.parent.id); + for (const siblingDesc of state.children) { + if (siblingDesc.id == childDesc.id) { continue; } - if (!siblingStep.finalized) { - ArrayPrototypePush(results, siblingStep); + const siblingState = MapPrototypeGet(testStates, siblingDesc.id); + if (!siblingState.finalized) { + ArrayPrototypePush(results, siblingDesc); } } - childStep = ancestor; + childDesc = childDesc.parent; } return results; } } + }; + } - function postValidation() { - // check for any running steps - if (step.hasRunningChildren) { - throw new Error( - "There were still test steps running after the current scope finished execution. " + - "Ensure all steps are awaited (ex. `await t.step(...)`).", - ); - } + function testStepPostValidation(desc) { + // check for any running steps + for (const childDesc of MapPrototypeGet(testStates, desc.id).children) { + if (MapPrototypeGet(testStates, childDesc.id).status == "pending") { + throw new Error( + "There were still test steps running after the current scope finished execution. Ensure all steps are awaited (ex. `await t.step(...)`).", + ); + } + } - // check if an ancestor already completed - for (const ancestor of step.ancestors()) { - if (ancestor.finalized) { - throw new Error( - "Parent scope completed before test step finished execution. " + - "Ensure all steps are awaited (ex. `await t.step(...)`).", - ); - } - } + // check if an ancestor already completed + let currentDesc = desc.parent; + while (currentDesc != null) { + if (MapPrototypeGet(testStates, currentDesc.id).finalized) { + throw new Error( + "Parent scope completed before test step finished execution. Ensure all steps are awaited (ex. `await t.step(...)`).", + ); } - }; + currentDesc = currentDesc.parent; + } } function pledgePermissions(permissions) { @@ -541,6 +536,54 @@ * @typedef {{ * id: number, * name: string, + * fn: TestFunction + * origin: string, + * location: TestLocation, + * filteredOut: boolean, + * ignore: boolean, + * only: boolean. + * sanitizeOps: boolean, + * sanitizeResources: boolean, + * sanitizeExit: boolean, + * permissions: PermissionOptions, + * }} TestDescription + * + * @typedef {{ + * id: number, + * name: string, + * fn: TestFunction + * origin: string, + * location: TestLocation, + * ignore: boolean, + * level: number, + * parent: TestDescription | TestStepDescription, + * rootId: number, + * rootName: String, + * sanitizeOps: boolean, + * sanitizeResources: boolean, + * sanitizeExit: boolean, + * }} TestStepDescription + * + * @typedef {{ + * context: TestContext, + * children: TestStepDescription[], + * finalized: boolean, + * }} TestState + * + * @typedef {{ + * context: TestContext, + * children: TestStepDescription[], + * finalized: boolean, + * status: "pending" | "ok" | ""failed" | ignored", + * error: unknown, + * elapsed: number | null, + * reportedWait: boolean, + * reportedResult: boolean, + * }} TestStepState + * + * @typedef {{ + * id: number, + * name: string, * fn: BenchFunction * origin: string, * filteredOut: boolean, @@ -551,7 +594,10 @@ * }} BenchDescription */ - const tests = []; + /** @type {TestDescription[]} */ + const testDescs = []; + /** @type {Map<number, TestState | TestStepState>} */ + const testStates = new Map(); /** @type {BenchDescription[]} */ const benchDescs = []; let isTestOrBenchSubcommand = false; @@ -566,7 +612,7 @@ return; } - let testDef; + let testDesc; const defaults = { ignore: false, only: false, @@ -581,7 +627,7 @@ throw new TypeError("The test name can't be empty"); } if (typeof optionsOrFn === "function") { - testDef = { fn: optionsOrFn, name: nameOrFnOrOptions, ...defaults }; + testDesc = { fn: optionsOrFn, name: nameOrFnOrOptions, ...defaults }; } else { if (!maybeFn || typeof maybeFn !== "function") { throw new TypeError("Missing test function"); @@ -596,7 +642,7 @@ "Unexpected 'name' field in options, test name is already provided as the first argument.", ); } - testDef = { + testDesc = { ...defaults, ...optionsOrFn, fn: maybeFn, @@ -613,7 +659,7 @@ if (maybeFn != undefined) { throw new TypeError("Unexpected third argument to Deno.test()"); } - testDef = { + testDesc = { ...defaults, fn: nameOrFnOrOptions, name: nameOrFnOrOptions.name, @@ -643,29 +689,36 @@ if (!name) { throw new TypeError("The test name can't be empty"); } - testDef = { ...defaults, ...nameOrFnOrOptions, fn, name }; + testDesc = { ...defaults, ...nameOrFnOrOptions, fn, name }; } - testDef.fn = wrapTestFnWithSanitizers(testDef.fn, testDef); - - if (testDef.permissions) { - testDef.fn = withPermissions( - testDef.fn, - testDef.permissions, + // Delete this prop in case the user passed it. It's used to detect steps. + delete testDesc.parent; + testDesc.fn = wrapTestFnWithSanitizers(testDesc.fn, testDesc); + if (testDesc.permissions) { + testDesc.fn = withPermissions( + testDesc.fn, + testDesc.permissions, ); } - + testDesc.origin = getTestOrigin(); const jsError = Deno.core.destructureError(new Error()); - // Note: There might pop up a case where one of the filename, line number or - // column number from the caller isn't defined. We assume never for now. - // Make `TestDescription::location` optional if such a case is found. - testDef.location = { + testDesc.location = { fileName: jsError.frames[1].fileName, lineNumber: jsError.frames[1].lineNumber, columnNumber: jsError.frames[1].columnNumber, }; - ArrayPrototypePush(tests, testDef); + const { id, filteredOut } = core.opSync("op_register_test", testDesc); + testDesc.id = id; + testDesc.filteredOut = filteredOut; + + ArrayPrototypePush(testDescs, testDesc); + MapPrototypeSet(testStates, testDesc.id, { + context: createTestContext(testDesc), + children: [], + finalized: false, + }); } // Main bench function provided by Deno. @@ -769,58 +822,14 @@ ArrayPrototypePush(benchDescs, benchDesc); } - /** - * @param {string | { include?: string[], exclude?: string[] }} filter - * @returns {(def: { name: string }) => boolean} - */ - function createTestFilter(filter) { - if (!filter) { - return () => true; - } - - const regex = - typeof filter === "string" && StringPrototypeStartsWith(filter, "/") && - StringPrototypeEndsWith(filter, "/") - ? new RegExp(StringPrototypeSlice(filter, 1, filter.length - 1)) - : undefined; - - const filterIsObject = filter != null && typeof filter === "object"; - - return (def) => { - if (regex) { - return RegExpPrototypeTest(regex, def.name); - } - if (filterIsObject) { - if (filter.include && !filter.include.includes(def.name)) { - return false; - } else if (filter.exclude && filter.exclude.includes(def.name)) { - return false; - } else { - return true; - } - } - return StringPrototypeIncludes(def.name, filter); - }; - } - - async function runTest(test, description) { - if (test.ignore) { + async function runTest(desc) { + if (desc.ignore) { return "ignored"; } - const step = new TestStep({ - name: test.name, - parent: undefined, - parentContext: undefined, - rootTestDescription: description, - sanitizeOps: test.sanitizeOps, - sanitizeResources: test.sanitizeResources, - sanitizeExit: test.sanitizeExit, - }); - try { - await test.fn(step); - const failCount = step.failedChildStepsCount(); + await desc.fn(desc); + const failCount = failedChildStepsCount(desc); return failCount === 0 ? "ok" : { "failed": core.destructureError( new Error( @@ -833,10 +842,11 @@ "failed": core.destructureError(error), }; } finally { - step.finalized = true; + const state = MapPrototypeGet(testStates, desc.id); + state.finalized = true; // ensure the children report their result - for (const child of step.children) { - child.reportResult(); + for (const childDesc of state.children) { + stepReportResult(childDesc); } } } @@ -961,7 +971,7 @@ n++; avg += iterationTime; - all.push(iterationTime); + ArrayPrototypePush(all, iterationTime); if (iterationTime < min) min = iterationTime; if (iterationTime > max) max = iterationTime; budget -= iterationTime * lowPrecisionThresholdInNs; @@ -1018,36 +1028,6 @@ return origin; } - function reportTestPlan(plan) { - core.opSync("op_dispatch_test_event", { - plan, - }); - } - - function reportTestWait(test) { - core.opSync("op_dispatch_test_event", { - wait: test, - }); - } - - function reportTestResult(test, result, elapsed) { - core.opSync("op_dispatch_test_event", { - result: [test, result, elapsed], - }); - } - - function reportTestStepWait(testDescription) { - core.opSync("op_dispatch_test_event", { - stepWait: testDescription, - }); - } - - function reportTestStepResult(testDescription, result, elapsed) { - core.opSync("op_dispatch_test_event", { - stepResult: [testDescription, result, elapsed], - }); - } - function benchNow() { return core.opSync("op_bench_now"); } @@ -1060,24 +1040,24 @@ } async function runTests({ - filter = null, shuffle = null, } = {}) { core.setMacrotaskCallback(handleOpSanitizerDelayMacrotask); const origin = getTestOrigin(); - - const only = ArrayPrototypeFilter(tests, (test) => test.only); + const only = ArrayPrototypeFilter(testDescs, (test) => test.only); const filtered = ArrayPrototypeFilter( - only.length > 0 ? only : tests, - createTestFilter(filter), + only.length > 0 ? only : testDescs, + (desc) => !desc.filteredOut, ); - reportTestPlan({ - origin, - total: filtered.length, - filteredOut: tests.length - filtered.length, - usedOnly: only.length > 0, + core.opSync("op_dispatch_test_event", { + plan: { + origin, + total: filtered.length, + filteredOut: testDescs.length - filtered.length, + usedOnly: only.length > 0, + }, }); if (shuffle !== null) { @@ -1098,20 +1078,14 @@ } } - for (const test of filtered) { - const description = { - origin, - name: test.name, - location: test.location, - }; + for (const desc of filtered) { + core.opSync("op_dispatch_test_event", { wait: desc.id }); const earlier = DateNow(); - - reportTestWait(description); - - const result = await runTest(test, description); + const result = await runTest(desc); const elapsed = DateNow() - earlier; - - reportTestResult(description, result, elapsed); + core.opSync("op_dispatch_test_event", { + result: [desc.id, result, elapsed], + }); } } @@ -1166,320 +1140,225 @@ globalThis.console = originalConsole; } - /** - * @typedef {{ - * fn: (t: TestContext) => void | Promise<void>, - * name: string, - * ignore?: boolean, - * sanitizeOps?: boolean, - * sanitizeResources?: boolean, - * sanitizeExit?: boolean, - * }} TestStepDefinition - * - * @typedef {{ - * name: string, - * parent: TestStep | undefined, - * parentContext: TestContext | undefined, - * rootTestDescription: { origin: string; name: string }; - * sanitizeOps: boolean, - * sanitizeResources: boolean, - * sanitizeExit: boolean, - * }} TestStepParams - */ - - class TestStep { - /** @type {TestStepParams} */ - #params; - reportedWait = false; - #reportedResult = false; - finalized = false; - elapsed = 0; - /** @type "ok" | "ignored" | "pending" | "failed" */ - status = "pending"; - error = undefined; - /** @type {TestStep[]} */ - children = []; - - /** @param params {TestStepParams} */ - constructor(params) { - this.#params = params; - } - - get name() { - return this.#params.name; - } - - get parent() { - return this.#params.parent; - } - - get parentContext() { - return this.#params.parentContext; - } - - get rootTestDescription() { - return this.#params.rootTestDescription; - } - - get sanitizerOptions() { - return { - sanitizeResources: this.#params.sanitizeResources, - sanitizeOps: this.#params.sanitizeOps, - sanitizeExit: this.#params.sanitizeExit, - }; - } - - get usesSanitizer() { - return this.#params.sanitizeResources || - this.#params.sanitizeOps || - this.#params.sanitizeExit; - } - - /** If a test validation error already occurred then don't bother checking - * the sanitizers as that will create extra noise. - */ - get shouldSkipSanitizers() { - return this.hasRunningChildren || this.parent?.finalized; - } - - get hasRunningChildren() { - return ArrayPrototypeSome( - this.children, - /** @param step {TestStep} */ - (step) => step.status === "pending", - ); - } - - failedChildStepsCount() { - return ArrayPrototypeFilter( - this.children, - /** @param step {TestStep} */ - (step) => step.status === "failed", - ).length; + function getFullName(desc) { + if ("parent" in desc) { + return `${desc.parent.name} > ${desc.name}`; } + return desc.name; + } - canStreamReporting() { - // there should only ever be one sub step running when running with - // sanitizers, so we can use this to tell if we can stream reporting - return this.selfAndAllAncestorsUseSanitizer() && - this.children.every((c) => c.usesSanitizer || c.finalized); - } + function usesSanitizer(desc) { + return desc.sanitizeResources || desc.sanitizeOps || desc.sanitizeExit; + } - selfAndAllAncestorsUseSanitizer() { - if (!this.usesSanitizer) { + function canStreamReporting(desc) { + let currentDesc = desc; + while (currentDesc != null) { + if (!usesSanitizer(currentDesc)) { return false; } - - for (const ancestor of this.ancestors()) { - if (!ancestor.usesSanitizer) { - return false; - } - } - - return true; - } - - *ancestors() { - let ancestor = this.parent; - while (ancestor) { - yield ancestor; - ancestor = ancestor.parent; - } + currentDesc = currentDesc.parent; } - - getFullName() { - if (this.parent) { - return `${this.parent.getFullName()} > ${this.name}`; - } else { - return this.name; + for (const childDesc of MapPrototypeGet(testStates, desc.id).children) { + const state = MapPrototypeGet(testStates, childDesc.id); + if (!usesSanitizer(childDesc) && !state.finalized) { + return false; } } + return true; + } - reportWait() { - if (this.reportedWait || !this.parent) { - return; - } - - reportTestStepWait(this.#getTestStepDescription()); - - this.reportedWait = true; + function stepReportWait(desc) { + const state = MapPrototypeGet(testStates, desc.id); + if (state.reportedWait) { + return; } + core.opSync("op_dispatch_test_event", { stepWait: desc.id }); + state.reportedWait = true; + } - reportResult() { - if (this.#reportedResult || !this.parent) { - return; - } - - this.reportWait(); - - for (const child of this.children) { - child.reportResult(); - } - - reportTestStepResult( - this.#getTestStepDescription(), - this.#getStepResult(), - this.elapsed, - ); - - this.#reportedResult = true; + function stepReportResult(desc) { + const state = MapPrototypeGet(testStates, desc.id); + if (state.reportedResult) { + return; } - - #getStepResult() { - switch (this.status) { - case "ok": - return "ok"; - case "ignored": - return "ignored"; - case "pending": - return { - "pending": this.error && core.destructureError(this.error), - }; - case "failed": - return { - "failed": this.error && core.destructureError(this.error), - }; - default: - throw new Error(`Unhandled status: ${this.status}`); - } + stepReportWait(desc); + for (const childDesc of state.children) { + stepReportResult(childDesc); } - - #getTestStepDescription() { - return { - test: this.rootTestDescription, - name: this.name, - level: this.#getLevel(), + let result; + if (state.status == "pending" || state.status == "failed") { + result = { + [state.status]: state.error && core.destructureError(state.error), }; + } else { + result = state.status; } + core.opSync("op_dispatch_test_event", { + stepResult: [desc.id, result, state.elapsed], + }); + state.reportedResult = true; + } - #getLevel() { - let count = 0; - for (const _ of this.ancestors()) { - count++; - } - return count; + function failedChildStepsCount(desc) { + return ArrayPrototypeFilter( + MapPrototypeGet(testStates, desc.id).children, + (d) => MapPrototypeGet(testStates, d.id).status === "failed", + ).length; + } + + /** If a test validation error already occurred then don't bother checking + * the sanitizers as that will create extra noise. + */ + function shouldSkipSanitizers(desc) { + try { + testStepPostValidation(desc); + return false; + } catch { + return true; } } - /** @param parentStep {TestStep} */ - function createTestContext(parentStep) { + /** @param desc {TestDescription | TestStepDescription} */ + function createTestContext(desc) { + let parent; + let level; + let rootId; + let rootName; + if ("parent" in desc) { + parent = MapPrototypeGet(testStates, desc.parent.id).context; + level = desc.level; + rootId = desc.rootId; + rootName = desc.rootName; + } else { + parent = undefined; + level = 0; + rootId = desc.id; + rootName = desc.name; + } return { [SymbolToStringTag]: "TestContext", /** * The current test name. */ - name: parentStep.name, + name: desc.name, /** * Parent test context. */ - parent: parentStep.parentContext ?? undefined, + parent, /** * File Uri of the test code. */ - origin: parentStep.rootTestDescription.origin, + origin: desc.origin, /** * @param nameOrTestDefinition {string | TestStepDefinition} * @param fn {(t: TestContext) => void | Promise<void>} */ async step(nameOrTestDefinition, fn) { - if (parentStep.finalized) { + if (MapPrototypeGet(testStates, desc.id).finalized) { throw new Error( "Cannot run test step after parent scope has finished execution. " + "Ensure any `.step(...)` calls are executed before their parent scope completes execution.", ); } - const definition = getDefinition(); - const subStep = new TestStep({ - name: definition.name, - parent: parentStep, - parentContext: this, - rootTestDescription: parentStep.rootTestDescription, - sanitizeOps: getOrDefault( - definition.sanitizeOps, - parentStep.sanitizerOptions.sanitizeOps, - ), - sanitizeResources: getOrDefault( - definition.sanitizeResources, - parentStep.sanitizerOptions.sanitizeResources, - ), - sanitizeExit: getOrDefault( - definition.sanitizeExit, - parentStep.sanitizerOptions.sanitizeExit, - ), - }); - - ArrayPrototypePush(parentStep.children, subStep); + let stepDesc; + if (typeof nameOrTestDefinition === "string") { + if (!(ObjectPrototypeIsPrototypeOf(FunctionPrototype, fn))) { + throw new TypeError("Expected function for second argument."); + } + stepDesc = { + name: nameOrTestDefinition, + fn, + }; + } else if (typeof nameOrTestDefinition === "object") { + stepDesc = nameOrTestDefinition; + } else { + throw new TypeError( + "Expected a test definition or name and function.", + ); + } + stepDesc.ignore ??= false; + stepDesc.sanitizeOps ??= desc.sanitizeOps; + stepDesc.sanitizeResources ??= desc.sanitizeResources; + stepDesc.sanitizeExit ??= desc.sanitizeExit; + stepDesc.origin = getTestOrigin(); + const jsError = Deno.core.destructureError(new Error()); + stepDesc.location = { + fileName: jsError.frames[1].fileName, + lineNumber: jsError.frames[1].lineNumber, + columnNumber: jsError.frames[1].columnNumber, + }; + stepDesc.level = level + 1; + stepDesc.parent = desc; + stepDesc.rootId = rootId; + stepDesc.rootName = rootName; + const { id } = core.opSync("op_register_test_step", stepDesc); + stepDesc.id = id; + const state = { + context: createTestContext(stepDesc), + children: [], + finalized: false, + status: "pending", + error: null, + elapsed: null, + reportedWait: false, + reportedResult: false, + }; + MapPrototypeSet(testStates, stepDesc.id, state); + ArrayPrototypePush( + MapPrototypeGet(testStates, stepDesc.parent.id).children, + stepDesc, + ); try { - if (definition.ignore) { - subStep.status = "ignored"; - subStep.finalized = true; - if (subStep.canStreamReporting()) { - subStep.reportResult(); + if (stepDesc.ignore) { + state.status = "ignored"; + state.finalized = true; + if (canStreamReporting(stepDesc)) { + stepReportResult(stepDesc); } return false; } - const testFn = wrapTestFnWithSanitizers( - definition.fn, - subStep.sanitizerOptions, - ); + const testFn = wrapTestFnWithSanitizers(stepDesc.fn, stepDesc); const start = DateNow(); try { - await testFn(subStep); + await testFn(stepDesc); - if (subStep.failedChildStepsCount() > 0) { - subStep.status = "failed"; + if (failedChildStepsCount(stepDesc) > 0) { + state.status = "failed"; } else { - subStep.status = "ok"; + state.status = "ok"; } } catch (error) { - subStep.error = error; - subStep.status = "failed"; + state.error = error; + state.status = "failed"; } - subStep.elapsed = DateNow() - start; + state.elapsed = DateNow() - start; - if (subStep.parent?.finalized) { + if (MapPrototypeGet(testStates, stepDesc.parent.id).finalized) { // always point this test out as one that was still running // if the parent step finalized - subStep.status = "pending"; + state.status = "pending"; } - subStep.finalized = true; + state.finalized = true; - if (subStep.reportedWait && subStep.canStreamReporting()) { - subStep.reportResult(); + if (state.reportedWait && canStreamReporting(stepDesc)) { + stepReportResult(stepDesc); } - return subStep.status === "ok"; + return state.status === "ok"; } finally { - if (parentStep.canStreamReporting()) { + if (canStreamReporting(stepDesc.parent)) { + const parentState = MapPrototypeGet(testStates, stepDesc.parent.id); // flush any buffered steps - for (const parentChild of parentStep.children) { - parentChild.reportResult(); - } - } - } - - /** @returns {TestStepDefinition} */ - function getDefinition() { - if (typeof nameOrTestDefinition === "string") { - if (!(ObjectPrototypeIsPrototypeOf(FunctionPrototype, fn))) { - throw new TypeError("Expected function for second argument."); + for (const childDesc of parentState.children) { + stepReportResult(childDesc); } - return { - name: nameOrTestDefinition, - fn, - }; - } else if (typeof nameOrTestDefinition === "object") { - return nameOrTestDefinition; - } else { - throw new TypeError( - "Expected a test definition or name and function.", - ); } } }, @@ -1511,16 +1390,6 @@ return testFn; } - /** - * @template T - * @param value {T | undefined} - * @param defaultValue {T} - * @returns T - */ - function getOrDefault(value, defaultValue) { - return value == null ? defaultValue : value; - } - window.__bootstrap.internals = { ...window.__bootstrap.internals ?? {}, enableTestAndBench, |