feat(test): add --shuffle flag to randomize test ordering (#11163)

11 files changed, 136 insertions, 8 deletions

diff --git a/Cargo.lock b/Cargo.lock
index d5adc0912..4ffb40c28 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -568,6 +568,7 @@ dependencies = [
  "os_pipe",
  "percent-encoding",
  "pin-project",
+ "rand 0.8.4",
  "regex",
  "ring",
  "rustyline",
@@ -645,7 +646,7 @@ version = "0.24.1"
 dependencies = [
  "deno_core",
  "deno_web",
- "rand 0.8.3",
+ "rand 0.8.4",
  "ring",
  "tokio",
  "uuid",
@@ -2529,9 +2530,9 @@ dependencies = [
 
 [[package]]
 name = "rand"
-version = "0.8.3"
+version = "0.8.4"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0ef9e7e66b4468674bfcb0c81af8b7fa0bb154fa9f28eb840da5c447baeb8d7e"
+checksum = "2e7573632e6454cf6b99d7aac4ccca54be06da05aca2ef7423d22d27d4d4bcd8"
 dependencies = [
  "libc",
  "rand_chacha 0.3.0",
@@ -3553,7 +3554,7 @@ checksum = "dac1c663cfc93810f88aed9b8941d48cabf856a1b111c29a40439018d870eb22"
 dependencies = [
  "cfg-if 1.0.0",
  "libc",
- "rand 0.8.3",
+ "rand 0.8.4",
  "redox_syscall",
  "remove_dir_all",
  "winapi 0.3.9",
@@ -3831,7 +3832,7 @@ dependencies = [
  "lazy_static",
  "log",
  "radix_trie",
- "rand 0.8.3",
+ "rand 0.8.4",
  "thiserror",
  "tokio",
  "trust-dns-proto",
@@ -3854,7 +3855,7 @@ dependencies = [
  "ipnet",
  "lazy_static",
  "log",
- "rand 0.8.3",
+ "rand 0.8.4",
  "serde",
  "smallvec",
  "thiserror",
@@ -3926,7 +3927,7 @@ dependencies = [
  "httparse",
  "input_buffer",
  "log",
- "rand 0.8.3",
+ "rand 0.8.4",
  "rustls",
  "sha-1",
  "thiserror",
diff --git a/cli/Cargo.toml b/cli/Cargo.toml
index 025369358..a4a8786a5 100644
--- a/cli/Cargo.toml
+++ b/cli/Cargo.toml
@@ -71,6 +71,7 @@ num_cpus = "1.13.0"
 percent-encoding = "2.1.0"
 pin-project = "1.0.6"
 regex = "1.4.3"
+rand = { version = "0.8.3", features = [ "small_rng" ] }
 ring = "0.16.20"
 rustyline = { version = "8.0.0", default-features = false }
 rustyline-derive = "0.4.0"
diff --git a/cli/flags.rs b/cli/flags.rs
index 6a51236c5..400798cbd 100644
--- a/cli/flags.rs
+++ b/cli/flags.rs
@@ -103,6 +103,7 @@ pub enum DenoSubcommand {
     allow_none: bool,
     include: Option<Vec<String>>,
     filter: Option<String>,
+    shuffle: Option<u64>,
     concurrent_jobs: usize,
   },
   Types,
@@ -1017,6 +1018,20 @@ fn test_subcommand<'a, 'b>() -> App<'a, 'b> {
         .help("Run tests with this string or pattern in the test name"),
     )
     .arg(
+      Arg::with_name("shuffle")
+        .long("shuffle")
+        .value_name("NUMBER")
+        .help("(UNSTABLE): Shuffle the order in which the tests are run")
+        .min_values(0)
+        .max_values(1)
+        .require_equals(true)
+        .takes_value(true)
+        .validator(|val: String| match val.parse::<u64>() {
+          Ok(_) => Ok(()),
+          Err(_) => Err("Shuffle seed should be a number".to_string()),
+        }),
+    )
+    .arg(
       Arg::with_name("coverage")
         .long("coverage")
         .require_equals(true)
@@ -1686,7 +1701,17 @@ fn test_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
   let quiet = matches.is_present("quiet");
   let filter = matches.value_of("filter").map(String::from);
 
-  flags.watch = matches.is_present("watch");
+  let shuffle = if matches.is_present("shuffle") {
+    let value = if let Some(value) = matches.value_of("shuffle") {
+      value.parse::<u64>().unwrap()
+    } else {
+      rand::random::<u64>()
+    };
+
+    Some(value)
+  } else {
+    None
+  };
 
   if matches.is_present("script_arg") {
     let script_arg: Vec<String> = matches
@@ -1730,6 +1755,7 @@ fn test_parse(flags: &mut Flags, matches: &clap::ArgMatches) {
     quiet,
     include,
     filter,
+    shuffle,
     allow_none,
     concurrent_jobs,
   };
@@ -3366,6 +3392,7 @@ mod tests {
           allow_none: true,
           quiet: false,
           include: Some(svec!["dir1/", "dir2/"]),
+          shuffle: None,
           concurrent_jobs: 1,
         },
         unstable: true,
diff --git a/cli/main.rs b/cli/main.rs
index 381ba7214..a61f94530 100644
--- a/cli/main.rs
+++ b/cli/main.rs
@@ -984,6 +984,7 @@ async fn test_command(
   quiet: bool,
   allow_none: bool,
   filter: Option<String>,
+  shuffle: Option<u64>,
   concurrent_jobs: usize,
 ) -> Result<(), AnyError> {
   if let Some(ref coverage_dir) = flags.coverage_dir {
@@ -1172,6 +1173,7 @@ async fn test_command(
           quiet,
           true,
           filter.clone(),
+          shuffle,
           concurrent_jobs,
         )
         .map(|res| res.map(|_| ()))
@@ -1207,6 +1209,7 @@ async fn test_command(
       quiet,
       allow_none,
       filter,
+      shuffle,
       concurrent_jobs,
     )
     .await?;
@@ -1314,6 +1317,7 @@ fn get_subcommand(
       include,
       allow_none,
       filter,
+      shuffle,
       concurrent_jobs,
     } => test_command(
       flags,
@@ -1324,6 +1328,7 @@ fn get_subcommand(
       quiet,
       allow_none,
       filter,
+      shuffle,
       concurrent_jobs,
     )
     .boxed_local(),
diff --git a/cli/tests/integration/test_tests.rs b/cli/tests/integration/test_tests.rs
index 3bc81c5d3..c029b154d 100644
--- a/cli/tests/integration/test_tests.rs
+++ b/cli/tests/integration/test_tests.rs
@@ -114,3 +114,15 @@ itest!(unhandled_rejection {
   exit_code: 1,
   output: "test/unhandled_rejection.out",
 });
+
+itest!(shuffle {
+  args: "test --shuffle test/shuffle",
+  exit_code: 0,
+  output_str: Some("[WILDCARD]"),
+});
+
+itest!(shuffle_with_seed {
+  args: "test --shuffle=42 test/shuffle",
+  exit_code: 0,
+  output: "test/shuffle.out",
+});
diff --git a/cli/tests/test/shuffle.out b/cli/tests/test/shuffle.out
new file mode 100644
index 000000000..04dd08ee2
--- /dev/null
+++ b/cli/tests/test/shuffle.out
@@ -0,0 +1,39 @@
+Check [WILDCARD]/test/shuffle/foo_test.ts
+Check [WILDCARD]/test/shuffle/baz_test.ts
+Check [WILDCARD]/test/shuffle/bar_test.ts
+running 10 tests from [WILDCARD]/test/shuffle/foo_test.ts
+test test 2 ... ok ([WILDCARD])
+test test 3 ... ok ([WILDCARD])
+test test 6 ... ok ([WILDCARD])
+test test 9 ... ok ([WILDCARD])
+test test 8 ... ok ([WILDCARD])
+test test 7 ... ok ([WILDCARD])
+test test 5 ... ok ([WILDCARD])
+test test 4 ... ok ([WILDCARD])
+test test 1 ... ok ([WILDCARD])
+test test 0 ... ok ([WILDCARD])
+running 10 tests from [WILDCARD]/test/shuffle/baz_test.ts
+test test 2 ... ok ([WILDCARD])
+test test 3 ... ok ([WILDCARD])
+test test 6 ... ok ([WILDCARD])
+test test 9 ... ok ([WILDCARD])
+test test 8 ... ok ([WILDCARD])
+test test 7 ... ok ([WILDCARD])
+test test 5 ... ok ([WILDCARD])
+test test 4 ... ok ([WILDCARD])
+test test 1 ... ok ([WILDCARD])
+test test 0 ... ok ([WILDCARD])
+running 10 tests from [WILDCARD]/test/shuffle/bar_test.ts
+test test 2 ... ok ([WILDCARD])
+test test 3 ... ok ([WILDCARD])
+test test 6 ... ok ([WILDCARD])
+test test 9 ... ok ([WILDCARD])
+test test 8 ... ok ([WILDCARD])
+test test 7 ... ok ([WILDCARD])
+test test 5 ... ok ([WILDCARD])
+test test 4 ... ok ([WILDCARD])
+test test 1 ... ok ([WILDCARD])
+test test 0 ... ok ([WILDCARD])
+
+test result: ok. 30 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out ([WILDCARD])
+
diff --git a/cli/tests/test/shuffle/bar_test.ts b/cli/tests/test/shuffle/bar_test.ts
new file mode 100644
index 000000000..ca118dc0d
--- /dev/null
+++ b/cli/tests/test/shuffle/bar_test.ts
@@ -0,0 +1,3 @@
+for (let i = 0; i < 10; i++) {
+  Deno.test(`test ${i}`, () => {});
+}
diff --git a/cli/tests/test/shuffle/baz_test.ts b/cli/tests/test/shuffle/baz_test.ts
new file mode 100644
index 000000000..ca118dc0d
--- /dev/null
+++ b/cli/tests/test/shuffle/baz_test.ts
@@ -0,0 +1,3 @@
+for (let i = 0; i < 10; i++) {
+  Deno.test(`test ${i}`, () => {});
+}
diff --git a/cli/tests/test/shuffle/foo_test.ts b/cli/tests/test/shuffle/foo_test.ts
new file mode 100644
index 000000000..ca118dc0d
--- /dev/null
+++ b/cli/tests/test/shuffle/foo_test.ts
@@ -0,0 +1,3 @@
+for (let i = 0; i < 10; i++) {
+  Deno.test(`test ${i}`, () => {});
+}
diff --git a/cli/tools/test_runner.rs b/cli/tools/test_runner.rs
index 96b101fa3..d1a0c6d65 100644
--- a/cli/tools/test_runner.rs
+++ b/cli/tools/test_runner.rs
@@ -21,6 +21,9 @@ use deno_core::serde_json::json;
 use deno_core::url::Url;
 use deno_core::ModuleSpecifier;
 use deno_runtime::permissions::Permissions;
+use rand::rngs::SmallRng;
+use rand::seq::SliceRandom;
+use rand::SeedableRng;
 use regex::Regex;
 use serde::Deserialize;
 use std::path::Path;
@@ -343,8 +346,19 @@ pub async fn run_tests(
   quiet: bool,
   allow_none: bool,
   filter: Option<String>,
+  shuffle: Option<u64>,
   concurrent_jobs: usize,
 ) -> Result<bool, AnyError> {
+  let test_modules = if let Some(seed) = shuffle {
+    let mut rng = SmallRng::seed_from_u64(seed);
+    let mut test_modules = test_modules.clone();
+    test_modules.sort();
+    test_modules.shuffle(&mut rng);
+    test_modules
+  } else {
+    test_modules
+  };
+
   if !doc_modules.is_empty() {
     let mut test_programs = Vec::new();
 
@@ -450,6 +464,7 @@ pub async fn run_tests(
   let test_options = json!({
       "disableLog": quiet,
       "filter": filter,
+      "shuffle": shuffle,
   });
 
   let test_module = deno_core::resolve_path("$deno$test.js")?;
diff --git a/runtime/js/40_testing.js b/runtime/js/40_testing.js
index 0106f895a..4ee863ee7 100644
--- a/runtime/js/40_testing.js
+++ b/runtime/js/40_testing.js
@@ -217,6 +217,7 @@ finishing test case.`;
   async function runTests({
     disableLog = false,
     filter = null,
+    shuffle = null,
   } = {}) {
     const originalConsole = globalThis.console;
     if (disableLog) {
@@ -234,6 +235,24 @@ finishing test case.`;
       only: only.length > 0,
     });
 
+    if (shuffle !== null) {
+      // http://en.wikipedia.org/wiki/Linear_congruential_generator
+      const nextInt = (function (state) {
+        const m = 0x80000000;
+        const a = 1103515245;
+        const c = 12345;
+
+        return function (max) {
+          return state = ((a * state + c) % m) % max;
+        };
+      }(shuffle));
+
+      for (let i = pending.length - 1; i > 0; i--) {
+        const j = nextInt(i);
+        [pending[i], pending[j]] = [pending[j], pending[i]];
+      }
+    }
+
     for (const test of pending) {
       const {
         name,