tools: refactor lint.py and format.py (#2950)

5 files changed, 136 insertions, 80 deletions

diff --git a/.eslintignore b/.eslintignore
index de3425c58..a76bd6824 100644
--- a/.eslintignore
+++ b/.eslintignore
@@ -1,3 +1,2 @@
 /js/deps/
-/js/gen/
 /tests/error_syntax.js
diff --git a/.gitignore b/.gitignore
index 5c16b784d..4118680a3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -16,4 +16,3 @@ node_modules
 /website/data.json
 /website/recent.json
 /website/*.bundle.js
-/js/gen
diff --git a/tools/format.py b/tools/format.py
index 5c9fcfcef..2628928f1 100755
--- a/tools/format.py
+++ b/tools/format.py
@@ -1,102 +1,96 @@
 #!/usr/bin/env python
 # Copyright 2018 the Deno authors. All rights reserved. MIT license.
-from glob import glob
 import os
 import sys
 import argparse
 from third_party import google_env, python_env
-from util import root_path, run, find_exts, platform, third_party_path
-
-parser = argparse.ArgumentParser()
-parser.add_argument("--js", help="only run prettier", action="store_true")
-parser.add_argument("--rs", help="only run rustfmt", action="store_true")
-parser.add_argument("--py", help="only run yapf", action="store_true")
-parser.add_argument("--gn", help="only run gn format", action="store_true")
-parser.add_argument("--cc", help="only run clang format", action="store_true")
-
-clang_format_path = os.path.join(third_party_path, "depot_tools",
-                                 "clang-format")
-prettier_path = os.path.join(third_party_path, "node_modules", "prettier",
-                             "bin-prettier.js")
-tools_path = os.path.join(root_path, "tools")
-rustfmt_config = os.path.join(root_path, ".rustfmt.toml")
+from util import git_ls_files, third_party_path, root_path, run
 
 
 def main():
     os.chdir(root_path)
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--cc", help="run clang-format", action="store_true")
+    parser.add_argument("--gn", help="run gn format", action="store_true")
+    parser.add_argument("--js", help="run prettier", action="store_true")
+    parser.add_argument("--py", help="run yapf", action="store_true")
+    parser.add_argument("--rs", help="run rustfmt", action="store_true")
     args = parser.parse_args()
+
     did_fmt = False
-    if args.rs:
-        rustfmt()
-        did_fmt = True
     if args.cc:
         clang_format()
         did_fmt = True
     if args.gn:
         gn_format()
         did_fmt = True
+    if args.js:
+        prettier()
+        did_fmt = True
     if args.py:
         yapf()
         did_fmt = True
-    if args.js:
-        prettier()
+    if args.rs:
+        rustfmt()
         did_fmt = True
+
     if not did_fmt:
-        rustfmt()
         clang_format()
         gn_format()
-        yapf()
         prettier()
-
-
-def qrun(cmd, env=None):
-    run(cmd, quiet=True, env=env)
+        yapf()
+        rustfmt()
 
 
 def clang_format():
     print "clang_format"
-    qrun([clang_format_path, "-i", "-style", "Google"] +
-         find_exts(["core"], [".cc", ".h"]))
-
-
-def rustfmt():
-    print "rustfmt"
-    qrun([
-        "rustfmt",
-        "--config-path",
-        rustfmt_config,
-    ] + find_exts(["cli", "core", "tools", "deno_typescript", "cli_snapshots"],
-                  [".rs"]))
+    exe = os.path.join(third_party_path, "depot_tools", "clang-format")
+    source_files = git_ls_files(root_path, ["*.cc", "*.h"])
+    run([exe, "-i", "-style", "Google", "--"] + source_files,
+        env=google_env(),
+        quiet=True)
 
 
 def gn_format():
     print "gn format"
-    for fn in ["BUILD.gn", ".gn"] + find_exts(
-        ["build_extra", "cli", "core", "deno_typescript", "cli_snapshots"],
-        [".gn", ".gni"]):
-        qrun(["third_party/depot_tools/gn", "format", fn], env=google_env())
+    exe = os.path.join(third_party_path, "depot_tools", "gn")
+    source_files = git_ls_files(root_path, ["*.gn", "*.gni"])
+    run([exe, "format", "--"] + source_files, env=google_env(), quiet=True)
+
+
+def prettier():
+    print "prettier"
+    script = os.path.join(third_party_path, "node_modules", "prettier",
+                          "bin-prettier.js")
+    source_files = git_ls_files(root_path, ["*.js", "*.json", "*.ts", "*.md"])
+    run(["node", script, "--write", "--loglevel=error", "--"] + source_files,
+        shell=False,
+        quiet=True)
 
 
 def yapf():
     print "yapf"
-    qrun(
-        [sys.executable, "third_party/python_packages/bin/yapf", "-i"] +
-        find_exts(["tools", "build_extra", "deno_typescript", "cli_snapshots"],
-                  [".py"],
-                  skip=["tools/clang"]),
-        env=python_env())
+    script = os.path.join(third_party_path, "python_packages", "bin", "yapf")
+    source_files = git_ls_files(root_path, ["*.py"])
+    run([sys.executable, script, "-i", "--"] + source_files,
+        env=python_env(),
+        shell=False,
+        quiet=True)
 
 
-def prettier():
-    print "prettier"
-    files = find_exts([
-        ".github", "js", "tests", "tools", "website", "core",
-        "deno_typescript", "cli_snapshots"
-    ], [".js", ".json", ".ts", ".md"],
-                      skip=["tools/clang", "js/deps", "js/gen"])
-    qrun(["node", prettier_path, "--write", "--loglevel=error"] +
-         ["rollup.config.js"] + files)
+def rustfmt():
+    print "rustfmt"
+    config_file = os.path.join(root_path, ".rustfmt.toml")
+    source_files = git_ls_files(root_path, ["*.rs"])
+    run([
+        "rustfmt",
+        "--config-path=" + config_file,
+        "--",
+    ] + source_files,
+        shell=False,
+        quiet=True)
 
 
-if __name__ == '__main__':
+if __name__ == "__main__":
     sys.exit(main())
diff --git a/tools/lint.py b/tools/lint.py
index 46a3c94ca..29ed86fa9 100755
--- a/tools/lint.py
+++ b/tools/lint.py
@@ -4,28 +4,66 @@
 
 import os
 import sys
-from util import enable_ansi_colors, find_exts, root_path, run, third_party_path
+from util import enable_ansi_colors, git_ls_files, root_path, run
+from util import third_party_path
 from third_party import python_env
 
-enable_ansi_colors()
 
-cpplint = os.path.join(third_party_path, "cpplint", "cpplint.py")
-eslint = os.path.join(third_party_path, "node_modules", "eslint", "bin",
-                      "eslint")
+def main():
+    enable_ansi_colors()
+    os.chdir(root_path)
+    cpplint()
+    eslint()
+    pylint()
 
-os.chdir(root_path)
-run([
-    sys.executable, cpplint, "--filter=-build/include_subdir",
-    "--repository=core/libdeno", "--extensions=cc,h", "--recursive", "core"
-])
 
-run([
-    "node", eslint, "--max-warnings=0", "./js/**/*.{ts,js}",
-    "./core/**/*.{ts,js}", "./tests/**/*.{ts,js}"
-])
+def cpplint():
+    print "cpplint"
+    script = os.path.join(third_party_path, "cpplint", "cpplint.py")
+    libdeno_dir = os.path.join(root_path, "core", "libdeno")
+    source_files = git_ls_files(libdeno_dir, ["*.cc", "*.h"])
+    run([
+        sys.executable,
+        script,
+        "--quiet",
+        "--filter=-build/include_subdir",
+        "--repository=" + libdeno_dir,
+        "--",
+    ] + source_files,
+        env=python_env(),
+        shell=False,
+        quiet=True)
 
-run([
-    sys.executable, "third_party/python_packages/pylint",
-    "--rcfile=third_party/depot_tools/pylintrc"
-] + find_exts(["tools", "build_extra"], [".py"], skip=["tools/clang"]),
-    env=python_env())
+
+def eslint():
+    print "eslint"
+    script = os.path.join(third_party_path, "node_modules", "eslint", "bin",
+                          "eslint")
+    # TODO: Files in 'deno_typescript', 'tools' and 'website' directories are
+    # currently not linted, but they should.
+    source_files = git_ls_files(
+        root_path,
+        ["*.js", "*.ts", ":!:deno_typescript/", ":!:tools/", ":!:website/"])
+    # Find all *directories* in the main repo that contain .ts/.js files.
+    source_dirs = set([os.path.dirname(f) for f in source_files])
+    # Within the source dirs, eslint does its own globbing, taking into account
+    # the exclusion rules listed in '.eslintignore'.
+    source_globs = ["%s/*.{js,ts}" % d for d in source_dirs]
+    run(["node", script, "--max-warnings=0", "--"] + source_globs,
+        shell=False,
+        quiet=True)
+
+
+def pylint():
+    print "pylint"
+    script = os.path.join(third_party_path, "python_packages", "pylint")
+    rcfile = os.path.join(third_party_path, "depot_tools", "pylintrc")
+    source_files = git_ls_files(root_path, ["*.py", ":!:gclient_config.py"])
+    run([sys.executable, script, "--rcfile=" + rcfile, "--"] + source_files,
+        env=python_env(),
+        shell=False,
+        quiet=True)
+
+
+if __name__ == "__main__":
+    sys.exit(main())
diff --git a/tools/util.py b/tools/util.py
index ce4c83573..7f5b95ea2 100644
--- a/tools/util.py
+++ b/tools/util.py
@@ -166,6 +166,32 @@ def touch(fname):
         open(fname, 'a').close()
 
 
+# Recursively list all files in (a subdirectory of) a git worktree.
+#   * Optionally, glob patterns may be specified to e.g. only list files with a
+#     certain extension.
+#   * Untracked files are included, unless they're listed in .gitignore.
+#   * Directory names themselves are not listed (but the files inside are).
+#   * Submodules and their contents are ignored entirely.
+#   * This function fails if the query matches no files.
+def git_ls_files(base_dir, patterns=None):
+    base_dir = os.path.abspath(base_dir)
+    args = [
+        "git", "-C", base_dir, "ls-files", "-z", "--exclude-standard",
+        "--cached", "--modified", "--others"
+    ]
+    if patterns:
+        args += ["--"] + patterns
+    output = subprocess.check_output(args)
+    files = [
+        os.path.normpath(os.path.join(base_dir, f)) for f in output.split("\0")
+        if f != ""
+    ]
+    if not files:
+        raise RuntimeError("git_ls_files: no files in '%s'" % base_dir +
+                           (" matching %s" % patterns if patterns else ""))
+    return files
+
+
 # Recursive search for files of certain extensions.
 #   * Recursive glob doesn't exist in python 2.7.
 #   * On windows, `os.walk()` unconditionally follows symlinks.