Clean up tools/

- Factor out tools/util.py
- Move js/*.py to tools.
- Rewrite tools/format.sh in python.
- Run lint first in travis.

7 files changed, 134 insertions, 94 deletions

diff --git a/tools/build_third_party.py b/tools/build_third_party.py
index e5c2885dc..87ed04783 100755
--- a/tools/build_third_party.py
+++ b/tools/build_third_party.py
@@ -10,58 +10,22 @@
 import os
 from os.path import join
 import subprocess
+from util import run, remove_and_symlink
 
 root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 third_party_path = join(root_path, "third_party")
 
-
-def main():
-    try:
-        os.makedirs(third_party_path)
-    except:
-        pass
-    os.chdir(third_party_path)
-    remove_and_symlink(join("..", "gclient_config.py"), ".gclient")
-    remove_and_symlink(join("..", "package.json"), "package.json")
-    remove_and_symlink(join("..", "yarn.lock"), "yarn.lock")
-    remove_and_symlink(join("v8", "third_party", "googletest"), "googletest")
-    remove_and_symlink(join("v8", "third_party", "jinja2"), "jinja2")
-    remove_and_symlink(join("v8", "third_party", "llvm-build"), "llvm-build")
-    remove_and_symlink(join("v8", "third_party", "markupsafe"), "markupsafe")
-    run(["gclient", "sync", "--no-history"])
-    run(["yarn"])
-
-
-def run(args):
-    print " ".join(args)
-    env = os.environ.copy()
-    subprocess.check_call(args, env=env)
-
-
-def remove_and_symlink(target, name):
-    try:
-        os.unlink(name)
-    except:
-        pass
-    os.symlink(target, name)
-
-
-def symlink(target, name, target_is_dir=False):
-    if os.name == "nt":
-        import ctypes
-        CreateSymbolicLinkW = ctypes.windll.kernel32.CreateSymbolicLinkW
-        CreateSymbolicLinkW.restype = ctypes.c_ubyte
-        CreateSymbolicLinkW.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p,
-                                        ctypes.c_uint32)
-
-        flags = 0x02  # SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE
-        if (target_is_dir):
-            flags |= 0x01  # SYMBOLIC_LINK_FLAG_DIRECTORY
-        if not CreateSymbolicLinkW(name, target, flags):
-            raise ctypes.WinError()
-    else:
-        os.symlink(target, name)
-
-
-if '__main__' == __name__:
-    main()
+try:
+    os.makedirs(third_party_path)
+except:
+    pass
+os.chdir(third_party_path)
+remove_and_symlink(join("..", "gclient_config.py"), ".gclient")
+remove_and_symlink(join("..", "package.json"), "package.json")
+remove_and_symlink(join("..", "yarn.lock"), "yarn.lock")
+remove_and_symlink(join("v8", "third_party", "googletest"), "googletest")
+remove_and_symlink(join("v8", "third_party", "jinja2"), "jinja2")
+remove_and_symlink(join("v8", "third_party", "llvm-build"), "llvm-build")
+remove_and_symlink(join("v8", "third_party", "markupsafe"), "markupsafe")
+run(["gclient", "sync", "--no-history"])
+run(["yarn"])
diff --git a/tools/flatbufferjs_hack.py b/tools/flatbufferjs_hack.py
new file mode 100755
index 000000000..163a1893c
--- /dev/null
+++ b/tools/flatbufferjs_hack.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+"""
+gn can only run python scripts.
+
+Generates flatbuffer TypeScript code.
+"""
+import subprocess
+import sys
+import os
+import shutil
+import util
+
+# TODO(ry) Ideally flatc output files should be written into target_gen_dir, but
+# its difficult to get this working in a way that parcel can resolve their
+# location. (Parcel does not support NODE_PATH.) Therefore this hack: write the
+# generated msg_generated.ts outputs into the js/ folder, and we check them into
+# the repo. Hopefully this hack can be removed at some point. If msg.fps is
+# changed, commit changes to the generated JS file.
+
+src = sys.argv[1]
+dst = sys.argv[2]
+stamp_file = sys.argv[3]
+
+shutil.copyfile(src, dst)
+
+util.touch(stamp_file)
diff --git a/tools/format.py b/tools/format.py
new file mode 100755
index 000000000..d1f555e5d
--- /dev/null
+++ b/tools/format.py
@@ -0,0 +1,25 @@
+#!/usr/bin/env python
+
+import os
+from glob import glob
+from util import run
+
+root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+
+os.chdir(root_path)
+# TODO(ry) Install clang-format in third_party.
+run(["clang-format", "-i", "-style", "Google"] + glob("src/*.cc") +
+    glob("src/*.h"))
+for fn in ["BUILD.gn", ".gn"] + glob("build_extra/**/*.gn*"):
+    run(["gn", "format", fn])
+# TODO(ry) Install yapf in third_party.
+run(["yapf", "-i"] + glob("tools/*.py"))
+# TODO(ry) Install prettier in third_party.
+run([
+    "prettier", "--write", "js/deno.d.ts", "js/main.ts", "js/mock_runtime.js",
+    "tsconfig.json"
+])
+# Do not format these.
+#  js/msg_generated.ts
+#  js/flatbuffers.js
+run(["rustfmt", "--write-mode", "overwrite"] + glob("src/*.rs"))
diff --git a/tools/format.sh b/tools/format.sh
deleted file mode 100755
index 44ca815ff..000000000
--- a/tools/format.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/bin/sh
-set -e
-cd `dirname "$0"`/..
-clang-format -i -style Google src/*.cc src/*.h
-
-gn format BUILD.gn
-gn format build_extra/deno.gni
-gn format build_extra/rust/rust.gni
-gn format build_extra/rust/BUILD.gn
-gn format .gn
-
-yapf -i js/*.py
-yapf -i tools/*.py
-
-prettier --write \
-  js/deno.d.ts \
-  js/main.ts \
-  js/mock_runtime.js \
-  tsconfig.json
-# Do not format these.
-#  js/msg_generated.ts
-#  js/flatbuffers.js
-
-rustfmt --write-mode overwrite src/*.rs
diff --git a/tools/lint.py b/tools/lint.py
index 1fa0d461d..452d44104 100755
--- a/tools/lint.py
+++ b/tools/lint.py
@@ -2,7 +2,7 @@
 # Does google-lint on c++ files and ts-lint on typescript files
 
 import os
-import subprocess
+from util import run
 
 root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
 third_party_path = os.path.join(root_path, "third_party")
@@ -10,21 +10,9 @@ cpplint = os.path.join(third_party_path, "cpplint", "cpplint.py")
 tslint = os.path.join(third_party_path, "node_modules", "tslint", "bin",
                       "tslint")
 
-
-def run(args):
-    print(" ".join(args))
-    env = os.environ.copy()
-    subprocess.check_call(args, env=env)
-
-
-def main():
-    os.chdir(root_path)
-    run([
-        "python", cpplint, "--filter=-build/include_subdir",
-        "--repository=src", "--extensions=cc,h", "--recursive", "src/."
-    ])
-    run(["node", tslint, "-p", ".", "--exclude", "js/msg_generated.ts"])
-
-
-if __name__ == "__main__":
-    main()
+os.chdir(root_path)
+run([
+    "python", cpplint, "--filter=-build/include_subdir", "--repository=src",
+    "--extensions=cc,h", "--recursive", "src/."
+])
+run(["node", tslint, "-p", ".", "--exclude", "js/msg_generated.ts"])
diff --git a/tools/run_node.py b/tools/run_node.py
new file mode 100755
index 000000000..afa861020
--- /dev/null
+++ b/tools/run_node.py
@@ -0,0 +1,19 @@
+#!/usr/bin/env python
+"""
+gn can only run python scripts. This launches a subprocess Node process.
+The working dir of this program is out/Debug/ (AKA root_build_dir)
+Before running node, we symlink js/node_modules to out/Debug/node_modules.
+"""
+import subprocess
+import sys
+import os
+import util
+
+root_path = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
+tools_path = os.path.join(root_path, "tools")
+third_party_path = os.path.join(root_path, "third_party")
+target_abs = os.path.join(third_party_path, "node_modules")
+target_rel = os.path.relpath(target_abs)
+
+util.remove_and_symlink(target_rel, "node_modules", True)
+util.run(["node"] + sys.argv[1:])
diff --git a/tools/util.py b/tools/util.py
new file mode 100644
index 000000000..1d44acd48
--- /dev/null
+++ b/tools/util.py
@@ -0,0 +1,42 @@
+# Copyright 2018 Ryan Dahl <ry@tinyclouds.org>
+# All rights reserved. MIT License.
+import os
+import subprocess
+
+
+def run(args):
+    print " ".join(args)
+    env = os.environ.copy()
+    subprocess.check_call(args, env=env)
+
+
+def remove_and_symlink(target, name, target_is_dir=False):
+    try:
+        os.unlink(name)
+    except:
+        pass
+    symlink(target, name, target_is_dir)
+
+
+def symlink(target, name, target_is_dir=False):
+    if os.name == "nt":
+        import ctypes
+        CreateSymbolicLinkW = ctypes.windll.kernel32.CreateSymbolicLinkW
+        CreateSymbolicLinkW.restype = ctypes.c_ubyte
+        CreateSymbolicLinkW.argtypes = (ctypes.c_wchar_p, ctypes.c_wchar_p,
+                                        ctypes.c_uint32)
+
+        flags = 0x02  # SYMBOLIC_LINK_FLAG_ALLOW_UNPRIVILEGED_CREATE
+        if (target_is_dir):
+            flags |= 0x01  # SYMBOLIC_LINK_FLAG_DIRECTORY
+        if not CreateSymbolicLinkW(name, target, flags):
+            raise ctypes.WinError()
+    else:
+        os.symlink(target, name)
+
+
+def touch(fname):
+    if os.path.exists(fname):
+        os.utime(fname, None)
+    else:
+        open(fname, 'a').close()