diff options
36 files changed, 506 insertions, 14 deletions
diff --git a/cli/schemas/config-file.v1.json b/cli/schemas/config-file.v1.json index 56a8090f9..3ba803ef8 100644 --- a/cli/schemas/config-file.v1.json +++ b/cli/schemas/config-file.v1.json @@ -448,6 +448,13 @@ "type": "string", "required": true, "description": "The task to execute" + }, + "dependencies": { + "type": "array", + "items": { + "type": "string" + }, + "description": "Tasks that should be executed before this task" } } } diff --git a/cli/tools/task.rs b/cli/tools/task.rs index a13efbaf4..682dbf814 100644 --- a/cli/tools/task.rs +++ b/cli/tools/task.rs @@ -2,6 +2,7 @@ use std::collections::HashMap; use std::collections::HashSet; +use std::num::NonZeroUsize; use std::path::Path; use std::path::PathBuf; use std::rc::Rc; @@ -15,6 +16,10 @@ use deno_core::anyhow::anyhow; use deno_core::anyhow::bail; use deno_core::anyhow::Context; use deno_core::error::AnyError; +use deno_core::futures::future::LocalBoxFuture; +use deno_core::futures::stream::futures_unordered; +use deno_core::futures::FutureExt; +use deno_core::futures::StreamExt; use deno_core::url::Url; use deno_path_util::normalize_path; use deno_runtime::deno_node::NodeResolver; @@ -68,6 +73,13 @@ pub async fn execute_script( let node_resolver = factory.node_resolver().await?; let env_vars = task_runner::real_env_vars(); + let no_of_concurrent_tasks = if let Ok(value) = std::env::var("DENO_JOBS") { + value.parse::<NonZeroUsize>().ok() + } else { + std::thread::available_parallelism().ok() + } + .unwrap_or_else(|| NonZeroUsize::new(2).unwrap()); + let task_runner = TaskRunner { tasks_config, task_flags: &task_flags, @@ -75,7 +87,9 @@ pub async fn execute_script( node_resolver: node_resolver.as_ref(), env_vars, cli_options, + concurrency: no_of_concurrent_tasks.into(), }; + task_runner.run_task(task_name).await } @@ -93,30 +107,156 @@ struct TaskRunner<'a> { node_resolver: &'a NodeResolver, env_vars: HashMap<String, String>, cli_options: &'a CliOptions, + concurrency: usize, } impl<'a> TaskRunner<'a> { - async fn run_task( + pub async fn run_task( &self, - task_name: &String, + task_name: &str, ) -> Result<i32, deno_core::anyhow::Error> { - let Some((dir_url, task_or_script)) = self.tasks_config.task(task_name) - else { - if self.task_flags.is_run { - return Err(anyhow!("Task not found: {}", task_name)); + match sort_tasks_topo(task_name, &self.tasks_config) { + Ok(sorted) => self.run_tasks_in_parallel(sorted).await, + Err(err) => match err { + TaskError::NotFound(name) => { + if self.task_flags.is_run { + return Err(anyhow!("Task not found: {}", name)); + } + + log::error!("Task not found: {}", name); + if log::log_enabled!(log::Level::Error) { + self.print_available_tasks()?; + } + Ok(1) + } + TaskError::TaskDepCycle { path } => { + log::error!("Task cycle detected: {}", path.join(" -> ")); + Ok(1) + } + }, + } + } + + pub fn print_available_tasks(&self) -> Result<(), std::io::Error> { + print_available_tasks( + &mut std::io::stderr(), + &self.cli_options.start_dir, + &self.tasks_config, + ) + } + + async fn run_tasks_in_parallel( + &self, + task_names: Vec<String>, + ) -> Result<i32, deno_core::anyhow::Error> { + struct PendingTasksContext { + completed: HashSet<String>, + running: HashSet<String>, + task_names: Vec<String>, + } + + impl PendingTasksContext { + fn has_remaining_tasks(&self) -> bool { + self.completed.len() < self.task_names.len() } - log::error!("Task not found: {}", task_name); - if log::log_enabled!(log::Level::Error) { - print_available_tasks( - &mut std::io::stderr(), - &self.cli_options.start_dir, - &self.tasks_config, - )?; + fn mark_complete(&mut self, task_name: String) { + self.running.remove(&task_name); + self.completed.insert(task_name); } - return Ok(1); + + fn get_next_task<'a>( + &mut self, + runner: &'a TaskRunner<'a>, + ) -> Option<LocalBoxFuture<'a, Result<(i32, String), AnyError>>> { + for name in &self.task_names { + if self.completed.contains(name) || self.running.contains(name) { + continue; + } + + let should_run = if let Ok((_, def)) = runner.get_task(name) { + match def { + TaskOrScript::Task(_, def) => def + .dependencies + .iter() + .all(|dep| self.completed.contains(dep)), + TaskOrScript::Script(_, _) => true, + } + } else { + false + }; + + if !should_run { + continue; + } + + self.running.insert(name.clone()); + let name = name.clone(); + return Some( + async move { + runner + .run_task_no_dependencies(&name) + .await + .map(|exit_code| (exit_code, name)) + } + .boxed_local(), + ); + } + None + } + } + + let mut context = PendingTasksContext { + completed: HashSet::with_capacity(task_names.len()), + running: HashSet::with_capacity(self.concurrency), + task_names, + }; + + let mut queue = futures_unordered::FuturesUnordered::new(); + + while context.has_remaining_tasks() { + while queue.len() < self.concurrency { + if let Some(task) = context.get_next_task(self) { + queue.push(task); + } else { + break; + } + } + + // If queue is empty at this point, then there are no more tasks in the queue. + let Some(result) = queue.next().await else { + debug_assert_eq!(context.task_names.len(), 0); + break; + }; + + let (exit_code, name) = result?; + if exit_code > 0 { + return Ok(exit_code); + } + + context.mark_complete(name); + } + + Ok(0) + } + + fn get_task( + &self, + task_name: &str, + ) -> Result<(&Url, TaskOrScript), TaskError> { + let Some(result) = self.tasks_config.task(task_name) else { + return Err(TaskError::NotFound(task_name.to_string())); }; + Ok(result) + } + + async fn run_task_no_dependencies( + &self, + task_name: &String, + ) -> Result<i32, deno_core::anyhow::Error> { + let (dir_url, task_or_script) = self.get_task(task_name.as_str()).unwrap(); + match task_or_script { TaskOrScript::Task(_tasks, definition) => { self.run_deno_task(dir_url, task_name, definition).await @@ -234,6 +374,59 @@ impl<'a> TaskRunner<'a> { } } +#[derive(Debug)] +enum TaskError { + NotFound(String), + TaskDepCycle { path: Vec<String> }, +} + +fn sort_tasks_topo( + name: &str, + task_config: &WorkspaceTasksConfig, +) -> Result<Vec<String>, TaskError> { + fn sort_visit<'a>( + name: &'a str, + sorted: &mut Vec<String>, + mut path: Vec<&'a str>, + tasks_config: &'a WorkspaceTasksConfig, + ) -> Result<(), TaskError> { + // Already sorted + if sorted.iter().any(|sorted_name| sorted_name == name) { + return Ok(()); + } + + // Graph has a cycle + if path.contains(&name) { + path.push(name); + return Err(TaskError::TaskDepCycle { + path: path.iter().map(|s| s.to_string()).collect(), + }); + } + + let Some(def) = tasks_config.task(name) else { + return Err(TaskError::NotFound(name.to_string())); + }; + + if let TaskOrScript::Task(_, actual_def) = def.1 { + for dep in &actual_def.dependencies { + let mut path = path.clone(); + path.push(name); + sort_visit(dep, sorted, path, tasks_config)? + } + } + + sorted.push(name.to_string()); + + Ok(()) + } + + let mut sorted: Vec<String> = vec![]; + + sort_visit(name, &mut sorted, Vec::new(), task_config)?; + + Ok(sorted) +} + fn output_task(task_name: &str, script: &str) { log::info!( "{} {} {}", @@ -339,6 +532,14 @@ fn print_available_tasks( )?; } writeln!(writer, " {}", desc.task.command)?; + if !desc.task.dependencies.is_empty() { + writeln!( + writer, + " {} {}", + colors::gray("depends on:"), + colors::cyan(desc.task.dependencies.join(", ")) + )?; + } } Ok(()) diff --git a/tests/specs/task/dependencies/__test__.jsonc b/tests/specs/task/dependencies/__test__.jsonc new file mode 100644 index 000000000..38d085d79 --- /dev/null +++ b/tests/specs/task/dependencies/__test__.jsonc @@ -0,0 +1,61 @@ +{ + "tests": { + "basic1": { + "cwd": "basic1", + "tempDir": true, + "args": "task run", + "output": "./basic1.out" + }, + "basic2": { + "cwd": "basic2", + "tempDir": true, + "args": "task run", + "output": "./basic2.out" + }, + "cross_package": { + "cwd": "cross_package/package1", + "tempDir": true, + "args": "task run", + "output": "./cross_package.out", + "exitCode": 1 + }, + "diamond": { + "cwd": "diamond", + "tempDir": true, + "args": "task a", + "output": "./diamond.out" + }, + "diamond_list": { + "cwd": "diamond", + "tempDir": true, + "args": "task", + "output": "./diamond_list.out" + }, + "diamond_big": { + "cwd": "diamond_big", + "tempDir": true, + "args": "task a", + "output": "./diamond_big.out" + }, + "diamond_big_list": { + "cwd": "diamond_big", + "tempDir": true, + "args": "task", + "output": "./diamond_big_list.out" + }, + "cycle": { + "cwd": "cycle", + "tempDir": true, + "output": "./cycle.out", + "args": "task a", + "exitCode": 1 + }, + "cycle_2": { + "cwd": "cycle_2", + "tempDir": true, + "args": "task a", + "output": "./cycle_2.out", + "exitCode": 1 + } + } +} diff --git a/tests/specs/task/dependencies/basic1.out b/tests/specs/task/dependencies/basic1.out new file mode 100644 index 000000000..8c31d02b4 --- /dev/null +++ b/tests/specs/task/dependencies/basic1.out @@ -0,0 +1,12 @@ +Task build1 deno run ../build1.js +Task build2 deno run ../build2.js +[UNORDERED_START] +Starting build1 +build1 performing more work... +build1 finished +Starting build2 +build2 performing more work... +build2 finished +[UNORDERED_END] +Task run deno run ../run.js +run finished diff --git a/tests/specs/task/dependencies/basic1/deno.json b/tests/specs/task/dependencies/basic1/deno.json new file mode 100644 index 000000000..16bb9937e --- /dev/null +++ b/tests/specs/task/dependencies/basic1/deno.json @@ -0,0 +1,10 @@ +{ + "tasks": { + "build1": "deno run ../build1.js", + "build2": "deno run ../build2.js", + "run": { + "command": "deno run ../run.js", + "dependencies": ["build1", "build2"] + } + } +} diff --git a/tests/specs/task/dependencies/basic2.out b/tests/specs/task/dependencies/basic2.out new file mode 100644 index 000000000..24ccd0eb0 --- /dev/null +++ b/tests/specs/task/dependencies/basic2.out @@ -0,0 +1,10 @@ +Task build1 deno run ../build1.js +Starting build1 +build1 performing more work... +build1 finished +Task build2 deno run ../build2.js +Starting build2 +build2 performing more work... +build2 finished +Task run deno run ../run.js +run finished diff --git a/tests/specs/task/dependencies/basic2/deno.json b/tests/specs/task/dependencies/basic2/deno.json new file mode 100644 index 000000000..9a54926dd --- /dev/null +++ b/tests/specs/task/dependencies/basic2/deno.json @@ -0,0 +1,13 @@ +{ + "tasks": { + "build1": "deno run ../build1.js", + "build2": { + "command": "deno run ../build2.js", + "dependencies": ["build1"] + }, + "run": { + "command": "deno run ../run.js", + "dependencies": ["build2"] + } + } +} diff --git a/tests/specs/task/dependencies/build1.js b/tests/specs/task/dependencies/build1.js new file mode 100644 index 000000000..d14fb401a --- /dev/null +++ b/tests/specs/task/dependencies/build1.js @@ -0,0 +1,9 @@ +import { randomTimeout } from "./util.js"; + +console.log("Starting build1"); + +await randomTimeout(500, 750); +console.log("build1 performing more work..."); +await randomTimeout(500, 750); + +console.log("build1 finished"); diff --git a/tests/specs/task/dependencies/build2.js b/tests/specs/task/dependencies/build2.js new file mode 100644 index 000000000..3032a099a --- /dev/null +++ b/tests/specs/task/dependencies/build2.js @@ -0,0 +1,9 @@ +import { randomTimeout } from "./util.js"; + +console.log("Starting build2"); + +await randomTimeout(250, 750); +console.log("build2 performing more work..."); +await randomTimeout(250, 750); + +console.log("build2 finished"); diff --git a/tests/specs/task/dependencies/cross_package.out b/tests/specs/task/dependencies/cross_package.out new file mode 100644 index 000000000..a57f4de9f --- /dev/null +++ b/tests/specs/task/dependencies/cross_package.out @@ -0,0 +1,5 @@ +Task not found: ../package2:run +Available tasks: +- run + deno run.js + depends on: ../package2:run diff --git a/tests/specs/task/dependencies/cross_package/package1/deno.json b/tests/specs/task/dependencies/cross_package/package1/deno.json new file mode 100644 index 000000000..6684a1e2c --- /dev/null +++ b/tests/specs/task/dependencies/cross_package/package1/deno.json @@ -0,0 +1,8 @@ +{ + "tasks": { + "run": { + "command": "deno run.js", + "dependencies": ["../package2:run"] + } + } +} diff --git a/tests/specs/task/dependencies/cross_package/package2/deno.json b/tests/specs/task/dependencies/cross_package/package2/deno.json new file mode 100644 index 000000000..e45ec398f --- /dev/null +++ b/tests/specs/task/dependencies/cross_package/package2/deno.json @@ -0,0 +1,5 @@ +{ + "tasks": { + "run": "deno run.js" + } +} diff --git a/tests/specs/task/dependencies/cycle.out b/tests/specs/task/dependencies/cycle.out new file mode 100644 index 000000000..33352b0bb --- /dev/null +++ b/tests/specs/task/dependencies/cycle.out @@ -0,0 +1 @@ +Task cycle detected: a -> a diff --git a/tests/specs/task/dependencies/cycle/a.js b/tests/specs/task/dependencies/cycle/a.js new file mode 100644 index 000000000..688695558 --- /dev/null +++ b/tests/specs/task/dependencies/cycle/a.js @@ -0,0 +1 @@ +console.log("Running a"); diff --git a/tests/specs/task/dependencies/cycle/deno.jsonc b/tests/specs/task/dependencies/cycle/deno.jsonc new file mode 100644 index 000000000..31e67488c --- /dev/null +++ b/tests/specs/task/dependencies/cycle/deno.jsonc @@ -0,0 +1,8 @@ +{ + "tasks": { + "a": { + "command": "deno run a.js", + "dependencies": ["a"] + } + } +} diff --git a/tests/specs/task/dependencies/cycle_2.out b/tests/specs/task/dependencies/cycle_2.out new file mode 100644 index 000000000..89ef04a00 --- /dev/null +++ b/tests/specs/task/dependencies/cycle_2.out @@ -0,0 +1 @@ +Task cycle detected: a -> b -> a diff --git a/tests/specs/task/dependencies/cycle_2/a.js b/tests/specs/task/dependencies/cycle_2/a.js new file mode 100644 index 000000000..688695558 --- /dev/null +++ b/tests/specs/task/dependencies/cycle_2/a.js @@ -0,0 +1 @@ +console.log("Running a"); diff --git a/tests/specs/task/dependencies/cycle_2/b.js b/tests/specs/task/dependencies/cycle_2/b.js new file mode 100644 index 000000000..ed1addf1a --- /dev/null +++ b/tests/specs/task/dependencies/cycle_2/b.js @@ -0,0 +1 @@ +console.log("Running b"); diff --git a/tests/specs/task/dependencies/cycle_2/deno.jsonc b/tests/specs/task/dependencies/cycle_2/deno.jsonc new file mode 100644 index 000000000..5a5d38ec9 --- /dev/null +++ b/tests/specs/task/dependencies/cycle_2/deno.jsonc @@ -0,0 +1,12 @@ +{ + "tasks": { + "a": { + "command": "deno run a.js", + "dependencies": ["b"] + }, + "b": { + "command": "deno run b.js", + "dependencies": ["a"] + } + } +} diff --git a/tests/specs/task/dependencies/diamond.out b/tests/specs/task/dependencies/diamond.out new file mode 100644 index 000000000..75b06a35b --- /dev/null +++ b/tests/specs/task/dependencies/diamond.out @@ -0,0 +1,10 @@ +Task d deno run d.js +Running d +[UNORDERED_START] +Task b deno run b.js +Running b +Task c deno run c.js +Running c +[UNORDERED_END] +Task a deno run a.js +Running a diff --git a/tests/specs/task/dependencies/diamond/a.js b/tests/specs/task/dependencies/diamond/a.js new file mode 100644 index 000000000..688695558 --- /dev/null +++ b/tests/specs/task/dependencies/diamond/a.js @@ -0,0 +1 @@ +console.log("Running a"); diff --git a/tests/specs/task/dependencies/diamond/b.js b/tests/specs/task/dependencies/diamond/b.js new file mode 100644 index 000000000..ed1addf1a --- /dev/null +++ b/tests/specs/task/dependencies/diamond/b.js @@ -0,0 +1 @@ +console.log("Running b"); diff --git a/tests/specs/task/dependencies/diamond/c.js b/tests/specs/task/dependencies/diamond/c.js new file mode 100644 index 000000000..194d656be --- /dev/null +++ b/tests/specs/task/dependencies/diamond/c.js @@ -0,0 +1 @@ +console.log("Running c"); diff --git a/tests/specs/task/dependencies/diamond/d.js b/tests/specs/task/dependencies/diamond/d.js new file mode 100644 index 000000000..a9f231f83 --- /dev/null +++ b/tests/specs/task/dependencies/diamond/d.js @@ -0,0 +1 @@ +console.log("Running d"); diff --git a/tests/specs/task/dependencies/diamond/deno.jsonc b/tests/specs/task/dependencies/diamond/deno.jsonc new file mode 100644 index 000000000..07d0a9177 --- /dev/null +++ b/tests/specs/task/dependencies/diamond/deno.jsonc @@ -0,0 +1,22 @@ +{ + // a + // / \ + // b c + // \ / + // d + "tasks": { + "a": { + "command": "deno run a.js", + "dependencies": ["b", "c"] + }, + "b": { + "command": "deno run b.js", + "dependencies": ["d"] + }, + "c": { + "command": "deno run c.js", + "dependencies": ["d"] + }, + "d": "deno run d.js" + } +} diff --git a/tests/specs/task/dependencies/diamond_big.out b/tests/specs/task/dependencies/diamond_big.out new file mode 100644 index 000000000..f0b827b0d --- /dev/null +++ b/tests/specs/task/dependencies/diamond_big.out @@ -0,0 +1,13 @@ +Task e deno run e.js +Running e +[UNORDERED_START] +Task b deno run b.js +Running b +Task d deno run d.js +Running d +Task c deno run c.js +Running c +Finished b +[UNORDERED_END] +Task a deno run a.js +Running a diff --git a/tests/specs/task/dependencies/diamond_big/a.js b/tests/specs/task/dependencies/diamond_big/a.js new file mode 100644 index 000000000..688695558 --- /dev/null +++ b/tests/specs/task/dependencies/diamond_big/a.js @@ -0,0 +1 @@ +console.log("Running a"); diff --git a/tests/specs/task/dependencies/diamond_big/b.js b/tests/specs/task/dependencies/diamond_big/b.js new file mode 100644 index 000000000..4b00ef569 --- /dev/null +++ b/tests/specs/task/dependencies/diamond_big/b.js @@ -0,0 +1,4 @@ +console.log("Running b"); +setTimeout(() => { + console.log("Finished b"); +}, 10); diff --git a/tests/specs/task/dependencies/diamond_big/c.js b/tests/specs/task/dependencies/diamond_big/c.js new file mode 100644 index 000000000..194d656be --- /dev/null +++ b/tests/specs/task/dependencies/diamond_big/c.js @@ -0,0 +1 @@ +console.log("Running c"); diff --git a/tests/specs/task/dependencies/diamond_big/d.js b/tests/specs/task/dependencies/diamond_big/d.js new file mode 100644 index 000000000..a9f231f83 --- /dev/null +++ b/tests/specs/task/dependencies/diamond_big/d.js @@ -0,0 +1 @@ +console.log("Running d"); diff --git a/tests/specs/task/dependencies/diamond_big/deno.jsonc b/tests/specs/task/dependencies/diamond_big/deno.jsonc new file mode 100644 index 000000000..28ea7f695 --- /dev/null +++ b/tests/specs/task/dependencies/diamond_big/deno.jsonc @@ -0,0 +1,28 @@ +{ + // a + // / \ + // b c + // | | + // | d + // \ / + // e + "tasks": { + "a": { + "command": "deno run a.js", + "dependencies": ["b", "c"] + }, + "b": { + "command": "deno run b.js", + "dependencies": ["e"] + }, + "c": { + "command": "deno run c.js", + "dependencies": ["d"] + }, + "d": { + "command": "deno run d.js", + "dependencies": ["e"] + }, + "e": "deno run e.js" + } +} diff --git a/tests/specs/task/dependencies/diamond_big/e.js b/tests/specs/task/dependencies/diamond_big/e.js new file mode 100644 index 000000000..b36066c3d --- /dev/null +++ b/tests/specs/task/dependencies/diamond_big/e.js @@ -0,0 +1 @@ +console.log("Running e"); diff --git a/tests/specs/task/dependencies/diamond_big_list.out b/tests/specs/task/dependencies/diamond_big_list.out new file mode 100644 index 000000000..c95bcd272 --- /dev/null +++ b/tests/specs/task/dependencies/diamond_big_list.out @@ -0,0 +1,15 @@ +Available tasks: +- a + deno run a.js + depends on: b, c +- b + deno run b.js + depends on: e +- c + deno run c.js + depends on: d +- d + deno run d.js + depends on: e +- e + deno run e.js diff --git a/tests/specs/task/dependencies/diamond_list.out b/tests/specs/task/dependencies/diamond_list.out new file mode 100644 index 000000000..dfd725a40 --- /dev/null +++ b/tests/specs/task/dependencies/diamond_list.out @@ -0,0 +1,12 @@ +Available tasks: +- a + deno run a.js + depends on: b, c +- b + deno run b.js + depends on: d +- c + deno run c.js + depends on: d +- d + deno run d.js diff --git a/tests/specs/task/dependencies/run.js b/tests/specs/task/dependencies/run.js new file mode 100644 index 000000000..f457de6ab --- /dev/null +++ b/tests/specs/task/dependencies/run.js @@ -0,0 +1 @@ +console.log("run finished"); diff --git a/tests/specs/task/dependencies/util.js b/tests/specs/task/dependencies/util.js new file mode 100644 index 000000000..9579eb9c9 --- /dev/null +++ b/tests/specs/task/dependencies/util.js @@ -0,0 +1,4 @@ +export async function randomTimeout(min, max) { + const timeout = Math.floor(Math.random() * (max - min + 1) + min); + return new Promise((resolve) => setTimeout(resolve, timeout)); +} |