chore: refactor python tests to use unittest (#2414)

Move every test to a method on DenoTestCase.
test.py is a single TestSuite of every TestCase.

Add a Spawn context manager for http_server,
this is explicitly used where it's needed.
Each python test file can now be run independently
without needing to manually run http_server.

Add --help and consistent flags using argparse for
each python test, including --failfast.

Use ColorTextTestRunner so that '... ok' is green.

1 files changed, 104 insertions, 109 deletions

diff --git a/tools/test.py b/tools/test.py
index 4769b1697..3a44748a2 100755
--- a/tools/test.py
+++ b/tools/test.py
@@ -3,127 +3,122 @@
 # Runs the full test suite.
 # Usage: ./tools/test.py out/Debug
 import os
+import subprocess
 import sys
 import unittest
 
-from integration_tests import integration_tests
-from deno_dir_test import deno_dir_test
-from util import build_path, enable_ansi_colors, executable_suffix, run, rmtree
-from util import run_output, tests_path, green_ok
-from unit_tests import unit_tests
-from util_test import util_test
-from setup_test import setup_test
-from benchmark_test import benchmark_test
-from repl_test import repl_tests
-from fetch_test import fetch_test
-from fmt_test import fmt_test
-import subprocess
-import http_server
-
-
-def check_exists(filename):
-    if not os.path.exists(filename):
-        print "Required target doesn't exist:", filename
-        print "Run ./tools/build.py"
-        sys.exit(1)
-
-
-def test_no_color(deno_exe):
-    sys.stdout.write("no_color test...")
-    sys.stdout.flush()
-    t = os.path.join(tests_path, "no_color.js")
-    output = run_output([deno_exe, "run", t], merge_env={"NO_COLOR": "1"})
-    assert output.strip() == "noColor true"
-    t = os.path.join(tests_path, "no_color.js")
-    output = run_output([deno_exe, "run", t])
-    assert output.strip() == "noColor false"
-    print green_ok()
-
-
-def exec_path_test(deno_exe):
-    cmd = [deno_exe, "run", "tests/exec_path.ts"]
-    output = run_output(cmd)
-    assert deno_exe in output.strip()
+from benchmark_test import TestBenchmark
+from deno_dir_test import TestDenoDir
+from fetch_test import FetchTest
+from fmt_test import FmtTest
+from integration_tests import TestIntegrations
+from repl_test import TestRepl
+from setup_test import TestSetup
+from unit_tests import JsUnitTests
+from util_test import TestUtil
+
+from is_tty_test import TestIsTty
+# NOTE: These tests are skipped on Windows
+from permission_prompt_test import permission_prompt_tests
+from complex_permissions_test import complex_permissions_tests
+
+from http_server import spawn
+from util import (DenoTestCase, ColorTextTestRunner, enable_ansi_colors,
+                  executable_suffix, run, run_output, rmtree, tests_path,
+                  test_args)
+
+
+class TestTarget(DenoTestCase):
+    @staticmethod
+    def check_exists(filename):
+        if not os.path.exists(filename):
+            print "Required target doesn't exist:", filename
+            print "Run ./tools/build.py"
+            sys.exit(1)
+
+    def test_executable_exists(self):
+        self.check_exists(self.deno_exe)
+
+    def _test(self, executable):
+        "Test executable runs and exits with code 0."
+        bin_file = os.path.join(self.build_dir, executable + executable_suffix)
+        self.check_exists(bin_file)
+        run([bin_file])
+
+    def test_libdeno(self):
+        self._test("libdeno_test")
+
+    def test_cli(self):
+        self._test("cli_test")
+
+    def test_core(self):
+        self._test("deno_core_test")
+
+    def test_core_http_benchmark(self):
+        self._test("deno_core_http_bench_test")
+
+    def test_ts_library_builder(self):
+        run([
+            "node", "./node_modules/.bin/ts-node", "--project",
+            "tools/ts_library_builder/tsconfig.json",
+            "tools/ts_library_builder/test.ts"
+        ])
+
+    def test_no_color(self):
+        t = os.path.join(tests_path, "no_color.js")
+        output = run_output([self.deno_exe, "run", t],
+                            merge_env={"NO_COLOR": "1"})
+        assert output.strip() == "noColor true"
+        t = os.path.join(tests_path, "no_color.js")
+        output = run_output([self.deno_exe, "run", t])
+        assert output.strip() == "noColor false"
+
+    def test_exec_path(self):
+        cmd = [self.deno_exe, "run", "tests/exec_path.ts"]
+        output = run_output(cmd)
+        assert self.deno_exe in output.strip()
 
 
 def main(argv):
-    if len(argv) == 2:
-        build_dir = sys.argv[1]
-    elif len(argv) == 1:
-        build_dir = build_path()
-    else:
-        print "Usage: tools/test.py [build_dir]"
-        sys.exit(1)
-
-    deno_dir = os.path.join(build_dir, ".deno_test")
+    args = test_args(argv)
+
+    deno_dir = os.path.join(args.build_dir, ".deno_test")
     if os.path.isdir(deno_dir):
         rmtree(deno_dir)
     os.environ["DENO_DIR"] = deno_dir
 
     enable_ansi_colors()
 
-    http_server.spawn()
-
-    deno_exe = os.path.join(build_dir, "deno" + executable_suffix)
-    check_exists(deno_exe)
-
-    # Python/build tools testing
-    setup_test()
-    util_test()
-
-    run([
-        "node", "./node_modules/.bin/ts-node", "--project",
-        "tools/ts_library_builder/tsconfig.json",
-        "tools/ts_library_builder/test.ts"
-    ])
-
-    libdeno_test = os.path.join(build_dir, "libdeno_test" + executable_suffix)
-    check_exists(libdeno_test)
-    run([libdeno_test])
-
-    cli_test = os.path.join(build_dir, "cli_test" + executable_suffix)
-    check_exists(cli_test)
-    run([cli_test])
-
-    deno_core_test = os.path.join(build_dir,
-                                  "deno_core_test" + executable_suffix)
-    check_exists(deno_core_test)
-    run([deno_core_test])
-
-    deno_core_http_bench_test = os.path.join(
-        build_dir, "deno_core_http_bench_test" + executable_suffix)
-    check_exists(deno_core_http_bench_test)
-    run([deno_core_http_bench_test])
-
-    unit_tests(deno_exe)
-
-    fetch_test(deno_exe)
-    fmt_test(deno_exe)
-
-    integration_tests(deno_exe)
-
-    # TODO We currently skip testing the prompt and IsTTY in Windows completely.
-    # Windows does not support the pty module used for testing the permission
-    # prompt.
-    if os.name != 'nt':
-        from is_tty_test import is_tty_test
-        from permission_prompt_test import permission_prompt_test
-        from complex_permissions_test import complex_permissions_test
-        permission_prompt_test(deno_exe)
-        complex_permissions_test(deno_exe)
-        is_tty_test(deno_exe)
-
-    repl_tests(deno_exe)
-
-    rmtree(deno_dir)
-
-    deno_dir_test(deno_exe, deno_dir)
-
-    test_no_color(deno_exe)
-
-    benchmark_test(build_dir, deno_exe)
-    exec_path_test(deno_exe)
+    with spawn():
+        test_cases = [
+            TestSetup,
+            TestUtil,
+            TestTarget,
+            JsUnitTests,
+            FetchTest,
+            FmtTest,
+            TestIntegrations,
+            TestRepl,
+            TestDenoDir,
+            TestBenchmark,
+        ]
+        # These tests are skipped, but to make the test output less noisy
+        # we'll avoid triggering them.
+        if os.name != 'nt':
+            test_cases.append(TestIsTty)
+            test_cases += permission_prompt_tests()
+            test_cases += complex_permissions_tests()
+
+        suite = unittest.TestSuite([
+            unittest.TestLoader().loadTestsFromTestCase(tc)
+            for tc in test_cases
+        ])
+
+        result = ColorTextTestRunner(
+            verbosity=args.verbosity + 1, failfast=args.failfast).run(suite)
+        if not result.wasSuccessful():
+            sys.exit(1)
 
 
 if __name__ == '__main__':
-    sys.exit(main(sys.argv))
+    main(sys.argv[1:])