Build: make it possible to use ccache/sccache on windows

Also auto-detect the availability of sccache in setup.py.

5 files changed, 563 insertions, 5 deletions

diff --git a/BUILD.gn b/BUILD.gn
index c7251981d..5bb2d2ef2 100644
--- a/BUILD.gn
+++ b/BUILD.gn
@@ -5,6 +5,7 @@ import("//build_extra/flatbuffers/flatbuffer.gni")
 import("//build_extra/flatbuffers/rust/rust_flatbuffer.gni")
 import("//build_extra/deno.gni")
 import("//build_extra/rust/rust.gni")
+import("//build_extra/toolchain/validate.gni")
 
 group("default") {
   testonly = true
diff --git a/build_extra/toolchain/validate.gni b/build_extra/toolchain/validate.gni
new file mode 100644
index 000000000..79c013e8d
--- /dev/null
+++ b/build_extra/toolchain/validate.gni
@@ -0,0 +1,15 @@
+# Copyright 2018 the Deno authors. All rights reserved. MIT license.
+
+import("//build/toolchain/cc_wrapper.gni")
+
+# Verify that cc_wrapper is correctly set up on Windows.
+if (is_win && cc_wrapper != "" && custom_toolchain == "") {
+  suggested_toolchain = "//build_extra/toolchain/win:win_clang_$target_cpu"
+
+  # Use print instead of assert with message for readability.
+  print(
+      "The 'cc_wrapper' option isn't supported by the default Windows toolchain.")
+  print("To make it work, add the option:")
+  print("  custom_toolchain=\"$suggested_toolchain\"")
+  assert(custom_toolchain != "")
+}
diff --git a/build_extra/toolchain/win/BUILD.gn b/build_extra/toolchain/win/BUILD.gn
new file mode 100644
index 000000000..a90f3661d
--- /dev/null
+++ b/build_extra/toolchain/win/BUILD.gn
@@ -0,0 +1,509 @@
+# Automatically generated. See mods.gni.
+
+# Copyright (c) 2013 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+import("mods.gni")
+import("//build/config/clang/clang.gni")
+import("//build/config/compiler/compiler.gni")
+import("//build/config/sanitizers/sanitizers.gni")
+import("//build/config/win/visual_studio_version.gni")
+import("//build/toolchain/clang_static_analyzer.gni")
+import("//build/toolchain/goma.gni")
+import("//build/toolchain/toolchain.gni")
+
+# Should only be running on Windows.
+assert(is_win)
+
+# Setup the Visual Studio state.
+#
+# Its arguments are the VS path and the compiler wrapper tool. It will write
+# "environment.x86" and "environment.x64" to the build directory and return a
+# list to us.
+
+# This tool will is used as a wrapper for various commands below.
+tool_wrapper_path = rebase_path("tool_wrapper.py", root_build_dir)
+
+if (use_goma) {
+  if (host_os == "win") {
+    goma_prefix = "$goma_dir/gomacc.exe "
+  } else {
+    goma_prefix = "$goma_dir/gomacc "
+  }
+} else {
+  goma_prefix = ""
+}
+
+# Copy the VS runtime DLL for the default toolchain to the root build directory
+# so things will run.
+if (current_toolchain == default_toolchain) {
+  if (is_debug) {
+    configuration_name = "Debug"
+  } else {
+    configuration_name = "Release"
+  }
+  exec_script("$base_toolchain_dir/../../vs_toolchain.py",
+              [
+                "copy_dlls",
+                rebase_path(root_build_dir),
+                configuration_name,
+                target_cpu,
+              ])
+}
+
+# Parameters:
+#   environment: File name of environment file.
+#
+# You would also define a toolchain_args variable with at least these set:
+#   current_cpu: current_cpu to pass as a build arg
+#   current_os: current_os to pass as a build arg
+template("msvc_toolchain") {
+  toolchain(target_name) {
+    # When invoking this toolchain not as the default one, these args will be
+    # passed to the build. They are ignored when this is the default toolchain.
+    assert(defined(invoker.toolchain_args))
+    toolchain_args = {
+      if (defined(invoker.toolchain_args)) {
+        forward_variables_from(invoker.toolchain_args, "*")
+      }
+
+      # This value needs to be passed through unchanged.
+      host_toolchain = host_toolchain
+    }
+
+    # Make these apply to all tools below.
+    lib_switch = ""
+    lib_dir_switch = "/LIBPATH:"
+
+    # Object files go in this directory.
+    object_subdir = "{{target_out_dir}}/{{label_name}}"
+
+    env = invoker.environment
+
+    # When the invoker has explicitly overridden use_goma or cc_wrapper in the
+    # toolchain args, use those values, otherwise default to the global one.
+    # This works because the only reasonable override that toolchains might
+    # supply for these values are to force-disable them.
+    if (defined(toolchain_args.is_clang)) {
+      toolchain_uses_clang = toolchain_args.is_clang
+    } else {
+      toolchain_uses_clang = is_clang
+    }
+
+    cl = invoker.cl
+
+    if (toolchain_uses_clang && use_clang_static_analyzer) {
+      analyzer_prefix =
+          "$python_path " +
+          rebase_path("//build/toolchain/clang_static_analyzer_wrapper.py",
+                      root_build_dir) + " --mode=cl"
+      cl = "${analyzer_prefix} ${cl}"
+    }
+
+    if (use_lld) {
+      if (host_os == "win") {
+        lld_link = "lld-link.exe"
+      } else {
+        lld_link = "lld-link"
+      }
+      prefix = rebase_path("$clang_base_path/bin", root_build_dir)
+
+      # lld-link includes a replacement for lib.exe that can produce thin
+      # archives and understands bitcode (for lto builds).
+      lib = "$prefix/$lld_link /lib /llvmlibthin"
+      link = "$prefix/$lld_link"
+      if (host_os != "win") {
+        # See comment adding --rsp-quoting to $cl above for more information.
+        link = "$link --rsp-quoting=posix"
+      }
+    } else {
+      lib = "lib.exe"
+      link = "link.exe"
+    }
+
+    # If possible, pass system includes as flags to the compiler.  When that's
+    # not possible, load a full environment file (containing %INCLUDE% and
+    # %PATH%) -- e.g. 32-bit MSVS builds require %PATH% to be set and just
+    # passing in a list of include directories isn't enough.
+    if (defined(invoker.sys_include_flags)) {
+      env_wrapper = ""
+      sys_include_flags = "${invoker.sys_include_flags} "  # Note trailing space.
+    } else {
+      # clang-cl doesn't need this env hoop, so omit it there.
+      assert(!toolchain_uses_clang)
+      env_wrapper = "ninja -t msvc -e $env -- "  # Note trailing space.
+      sys_include_flags = ""
+    }
+
+    # ninja does not have -t msvc other than windows, and lld doesn't depend on
+    # mt.exe in PATH on non-Windows, so it's not needed there anyways.
+    if (defined(invoker.sys_lib_flags)) {
+      linker_wrapper = ""
+      sys_lib_flags = "${invoker.sys_lib_flags} "  # Note trailing space
+    } else if (use_lld) {
+      # Invoke ninja as wrapper instead of tool wrapper, because python
+      # invocation requires higher cpu usage compared to ninja invocation, and
+      # the python wrapper is only needed to work around link.exe problems.
+      # TODO(thakis): Remove wrapper once lld-link can merge manifests without
+      # relying on mt.exe being in %PATH% on Windows.
+      linker_wrapper = "ninja -t msvc -e $env -- "  # Note trailing space.
+      sys_lib_flags = ""
+    } else {
+      linker_wrapper =
+          "$python_path $tool_wrapper_path link-wrapper $env False "  # Note trailing space.
+      sys_lib_flags = ""
+    }
+
+    clflags = ""
+
+    # Pass /FC flag to the compiler if needed.
+    if (msvc_use_absolute_paths) {
+      clflags += "/FC "
+    }
+
+    tool("cc") {
+      precompiled_header_type = "msvc"
+      pdbname = "{{target_out_dir}}/{{label_name}}_c.pdb"
+
+      # Label names may have spaces in them so the pdbname must be quoted. The
+      # source and output don't need to be quoted because GN knows they're a
+      # full file name and will quote automatically when necessary.
+      depsformat = "msvc"
+      description = "CC {{output}}"
+      outputs = [
+        "$object_subdir/{{source_name_part}}.obj",
+      ]
+
+      command = "$env_wrapper$cl /nologo /showIncludes ${clflags} $sys_include_flags{{defines}} {{include_dirs}} {{cflags}} {{cflags_c}} /c {{source}} /Fo{{output}} /Fd\"$pdbname\""
+    }
+
+    tool("cxx") {
+      precompiled_header_type = "msvc"
+
+      # The PDB name needs to be different between C and C++ compiled files.
+      pdbname = "{{target_out_dir}}/{{label_name}}_cc.pdb"
+
+      # See comment in CC tool about quoting.
+      depsformat = "msvc"
+      description = "CXX {{output}}"
+      outputs = [
+        "$object_subdir/{{source_name_part}}.obj",
+      ]
+
+      command = "$env_wrapper$cl /nologo /showIncludes ${clflags} $sys_include_flags{{defines}} {{include_dirs}} {{cflags}} {{cflags_cc}} /c {{source}} /Fo{{output}} /Fd\"$pdbname\""
+    }
+
+    tool("rc") {
+      command = "$python_path $tool_wrapper_path rc-wrapper $env rc.exe {{defines}} {{include_dirs}} /fo{{output}} {{source}}"
+      depsformat = "msvc"
+      outputs = [
+        "$object_subdir/{{source_name_part}}.res",
+      ]
+      description = "RC {{output}}"
+    }
+
+    tool("asm") {
+      if (toolchain_args.current_cpu == "x64") {
+        ml = "ml64.exe"
+      } else {
+        ml = "ml.exe"
+      }
+      command = "$python_path $tool_wrapper_path asm-wrapper $env $ml {{defines}} {{include_dirs}} {{asmflags}} /c /Fo{{output}} {{source}}"
+      description = "ASM {{output}}"
+      outputs = [
+        "$object_subdir/{{source_name_part}}.obj",
+      ]
+    }
+
+    tool("alink") {
+      rspfile = "{{output}}.rsp"
+      command = "$linker_wrapper$lib /nologo ${sys_lib_flags}{{arflags}} /OUT:{{output}} @$rspfile"
+      description = "LIB {{output}}"
+      outputs = [
+        # Ignore {{output_extension}} and always use .lib, there's no reason to
+        # allow targets to override this extension on Windows.
+        "{{output_dir}}/{{target_output_name}}.lib",
+      ]
+      default_output_extension = ".lib"
+      default_output_dir = "{{target_out_dir}}"
+
+      # The use of inputs_newline is to work around a fixed per-line buffer
+      # size in the linker.
+      rspfile_content = "{{inputs_newline}}"
+    }
+
+    tool("solink") {
+      dllname = "{{output_dir}}/{{target_output_name}}{{output_extension}}"  # e.g. foo.dll
+      libname = "${dllname}.lib"  # e.g. foo.dll.lib
+      pdbname = "${dllname}.pdb"
+      rspfile = "${dllname}.rsp"
+      pool = "//build/toolchain:link_pool($default_toolchain)"
+
+      command = "$linker_wrapper$link /nologo ${sys_lib_flags}/IMPLIB:$libname /DLL /OUT:$dllname /PDB:$pdbname @$rspfile"
+
+      default_output_extension = ".dll"
+      default_output_dir = "{{root_out_dir}}"
+      description = "LINK(DLL) {{output}}"
+      outputs = [
+        dllname,
+        libname,
+      ]
+      link_output = libname
+      depend_output = libname
+      runtime_outputs = [ dllname ]
+      if (symbol_level != 0) {
+        outputs += [ pdbname ]
+        runtime_outputs += [ pdbname ]
+      }
+
+      # Since the above commands only updates the .lib file when it changes, ask
+      # Ninja to check if the timestamp actually changed to know if downstream
+      # dependencies should be recompiled.
+      restat = true
+
+      # The use of inputs_newline is to work around a fixed per-line buffer
+      # size in the linker.
+      rspfile_content = "{{libs}} {{solibs}} {{inputs_newline}} {{ldflags}}"
+    }
+
+    tool("solink_module") {
+      dllname = "{{output_dir}}/{{target_output_name}}{{output_extension}}"  # e.g. foo.dll
+      pdbname = "${dllname}.pdb"
+      rspfile = "${dllname}.rsp"
+      pool = "//build/toolchain:link_pool($default_toolchain)"
+
+      command = "$linker_wrapper$link /nologo ${sys_lib_flags}/DLL /OUT:$dllname /PDB:$pdbname @$rspfile"
+
+      default_output_extension = ".dll"
+      default_output_dir = "{{root_out_dir}}"
+      description = "LINK_MODULE(DLL) {{output}}"
+      outputs = [
+        dllname,
+      ]
+      if (symbol_level != 0) {
+        outputs += [ pdbname ]
+      }
+      runtime_outputs = outputs
+
+      # The use of inputs_newline is to work around a fixed per-line buffer
+      # size in the linker.
+      rspfile_content = "{{libs}} {{solibs}} {{inputs_newline}} {{ldflags}}"
+    }
+
+    tool("link") {
+      exename = "{{output_dir}}/{{target_output_name}}{{output_extension}}"
+      pdbname = "$exename.pdb"
+      rspfile = "$exename.rsp"
+      pool = "//build/toolchain:link_pool($default_toolchain)"
+
+      command = "$linker_wrapper$link /nologo ${sys_lib_flags}/OUT:$exename /PDB:$pdbname @$rspfile"
+
+      if (host_os == "win") {
+        shellprefix = "cmd /c"
+      } else {
+        shellprefix = ""
+      }
+      not_needed([ "shellprefix" ])
+
+      if (is_official_build) {
+        # On bots, the binary's PDB grow and eventually exceed 4G, causing the
+        # link to fail. As there's no utility to keeping the PDB around
+        # incrementally anyway in this config (because we're doing
+        # non-incremental LTCG builds), delete it before linking.
+        command = "$shellprefix $python_path $tool_wrapper_path delete-file $pdbname && $command"
+      }
+
+      default_output_extension = ".exe"
+      default_output_dir = "{{root_out_dir}}"
+      description = "LINK {{output}}"
+      outputs = [
+        exename,
+      ]
+      if (symbol_level != 0) {
+        outputs += [ pdbname ]
+      }
+      runtime_outputs = outputs
+
+      # The use of inputs_newline is to work around a fixed per-line buffer
+      # size in the linker.
+      rspfile_content = "{{inputs_newline}} {{libs}} {{solibs}} {{ldflags}}"
+    }
+
+    # These two are really entirely generic, but have to be repeated in
+    # each toolchain because GN doesn't allow a template to be used here.
+    # See //build/toolchain/toolchain.gni for details.
+    tool("stamp") {
+      command = stamp_command
+      description = stamp_description
+      pool = "//build/toolchain:action_pool($default_toolchain)"
+    }
+    tool("copy") {
+      command = copy_command
+      description = copy_description
+      pool = "//build/toolchain:action_pool($default_toolchain)"
+    }
+
+    tool("action") {
+      pool = "//build/toolchain:action_pool($default_toolchain)"
+    }
+  }
+}
+
+if (host_os == "win") {
+  clang_cl = "clang-cl.exe"
+} else {
+  clang_cl = "clang-cl"
+}
+
+if (target_cpu == "x86" || target_cpu == "x64") {
+  win_build_host_cpu = target_cpu
+} else {
+  win_build_host_cpu = host_cpu
+}
+
+# x86, arm and arm64 build cpu toolchains for Windows (not WinUWP). Only
+# define when the build cpu is one of these architectures since we don't
+# do any cross compiles when targeting x64-bit (the build does generate
+# some 64-bit stuff from x86/arm/arm64 target builds).
+if (win_build_host_cpu != "x64") {
+  build_cpu_toolchain_data =
+      exec_script("$base_toolchain_dir/setup_toolchain.py",
+                  [
+                    visual_studio_path,
+                    windows_sdk_path,
+                    visual_studio_runtime_dirs,
+                    host_os,
+                    win_build_host_cpu,
+                    "environment." + win_build_host_cpu,
+                  ],
+                  "scope")
+
+  msvc_toolchain(win_build_host_cpu) {
+    environment = "environment." + win_build_host_cpu
+    cl = "${goma_prefix}${cc_wrapper_prefix}\"${build_cpu_toolchain_data.vc_bin_dir}/cl.exe\""
+    if (host_os != "win") {
+      # For win cross build.
+      sys_lib_flags = "${build_cpu_toolchain_data.libpath_flags}"
+    }
+    toolchain_args = {
+      current_os = "win"
+      current_cpu = win_build_host_cpu
+      is_clang = false
+    }
+  }
+
+  msvc_toolchain("win_clang_" + win_build_host_cpu) {
+    environment = "environment." + win_build_host_cpu
+    prefix = rebase_path("$clang_base_path/bin", root_build_dir)
+    cl = "${goma_prefix}${cc_wrapper_prefix}$prefix/${clang_cl}"
+    sys_include_flags = "${build_cpu_toolchain_data.include_flags_imsvc}"
+    if (host_os != "win") {
+      # For win cross build.
+      sys_lib_flags = "${build_cpu_toolchain_data.libpath_flags}"
+    }
+
+    toolchain_args = {
+      current_os = "win"
+      current_cpu = win_build_host_cpu
+      is_clang = true
+    }
+  }
+}
+
+# 64-bit toolchains.
+x64_toolchain_data = exec_script("$base_toolchain_dir/setup_toolchain.py",
+                                 [
+                                   visual_studio_path,
+                                   windows_sdk_path,
+                                   visual_studio_runtime_dirs,
+                                   "win",
+                                   "x64",
+                                   "environment.x64",
+                                 ],
+                                 "scope")
+
+template("win_x64_toolchains") {
+  msvc_toolchain(target_name) {
+    environment = "environment.x64"
+    cl = "${goma_prefix}${cc_wrapper_prefix}\"${x64_toolchain_data.vc_bin_dir}/cl.exe\""
+    if (host_os != "win") {
+      # For win cross build
+      sys_lib_flags = "${x64_toolchain_data.libpath_flags}"
+    }
+
+    toolchain_args = {
+      if (defined(invoker.toolchain_args)) {
+        forward_variables_from(invoker.toolchain_args, "*")
+      }
+      is_clang = false
+      current_os = "win"
+      current_cpu = "x64"
+    }
+  }
+
+  msvc_toolchain("win_clang_" + target_name) {
+    environment = "environment.x64"
+    prefix = rebase_path("$clang_base_path/bin", root_build_dir)
+    cl = "${goma_prefix}${cc_wrapper_prefix}$prefix/${clang_cl}"
+    sys_include_flags = "${x64_toolchain_data.include_flags_imsvc}"
+    if (host_os != "win") {
+      # For win cross build
+      sys_lib_flags = "${x64_toolchain_data.libpath_flags}"
+    }
+
+    toolchain_args = {
+      if (defined(invoker.toolchain_args)) {
+        forward_variables_from(invoker.toolchain_args, "*")
+      }
+      is_clang = true
+      current_os = "win"
+      current_cpu = "x64"
+    }
+  }
+}
+
+win_x64_toolchains("x64") {
+  toolchain_args = {
+    # Use the defaults.
+  }
+}
+
+# The nacl_win64 toolchain is nearly identical to the plain x64 toolchain.
+# It's used solely for building nacl64.exe (//components/nacl/broker:nacl64).
+# The only reason it's a separate toolchain is so that it can force
+# is_component_build to false in the toolchain_args() block, because
+# building nacl64.exe in component style does not work.
+win_x64_toolchains("nacl_win64") {
+  toolchain_args = {
+    is_component_build = false
+  }
+}
+
+# WinUWP toolchains. Only define these when targeting them.
+
+if (target_os == "winuwp") {
+  assert(target_cpu == "x64" || target_cpu == "x86" || target_cpu == "arm" ||
+         target_cpu == "arm64")
+  store_cpu_toolchain_data =
+      exec_script("$base_toolchain_dir/setup_toolchain.py",
+                  [
+                    visual_studio_path,
+                    windows_sdk_path,
+                    visual_studio_runtime_dirs,
+                    target_os,
+                    target_cpu,
+                    "environment.store_" + target_cpu,
+                  ],
+                  "scope")
+
+  msvc_toolchain("uwp_" + target_cpu) {
+    environment = "environment.store_" + target_cpu
+    cl = "${goma_prefix}${cc_wrapper_prefix}\"${store_cpu_toolchain_data.vc_bin_dir}/cl.exe\""
+    toolchain_args = {
+      current_os = "winuwp"
+      current_cpu = target_cpu
+      is_clang = false
+    }
+  }
+}
diff --git a/build_extra/toolchain/win/mods.gni b/build_extra/toolchain/win/mods.gni
new file mode 100644
index 000000000..5355d7f9a
--- /dev/null
+++ b/build_extra/toolchain/win/mods.gni
@@ -0,0 +1,28 @@
+# Copyright 2018 the Deno authors. All rights reserved. MIT license.
+
+# The toolchain definitions in BUILD.gn are derived from Chromium's default
+# Windows toolchain. The only difference is that the 'cc_wrapper' build option
+# is honored, so we can use ccache or an equivalent tool.
+#
+# 'BUILD.gn' was generated as follows (using GNU sed):
+#
+#   (echo -e '# Automatically generated. See mods.gni.\n' |
+#    cat - ../../../build/toolchain/win/BUILD.gn          |
+#    sed '0,/import/s|import|import("mods.gni")\n&|'      |
+#    sed 's|exec_script("|&$base_toolchain_dir/|'         |
+#    sed 's|${goma_prefix}|&${cc_wrapper_prefix}|'        ) > BUILD.gn
+#   gn format BUILD.gn
+
+# Ensure the 'cc_wrapper' variable is in scope.
+import("//build/toolchain/cc_wrapper.gni")
+
+# Location of the original toolchain definition that this one is derived from.
+# Some python scripts that are run by BUILD.gni live here.
+base_toolchain_dir = "//build/toolchain/win"
+
+# If cc_wrapper if is set, wrap it in quotes and add a space to it.
+if (cc_wrapper == "") {
+  cc_wrapper_prefix = ""
+} else {
+  cc_wrapper_prefix = "\"$cc_wrapper\" "
+}
diff --git a/tools/setup.py b/tools/setup.py
index 883270ad1..cc6fef238 100755
--- a/tools/setup.py
+++ b/tools/setup.py
@@ -3,7 +3,7 @@ import third_party
 from util import run, build_path, build_mode
 import os
 import sys
-import distutils.spawn
+from distutils.spawn import find_executable
 
 third_party.fix_symlinks()
 third_party.download_gn()
@@ -24,10 +24,15 @@ def get_gn_args():
     if "DENO_BUILD_ARGS" in os.environ:
         out += os.environ["DENO_BUILD_ARGS"].split()
 
-    # Check if ccache is in the path, and if so we cc_wrapper.
-    ccache_path = distutils.spawn.find_executable("ccache")
-    if ccache_path:
-        out += [r'cc_wrapper="%s"' % ccache_path]
+    # Check if ccache or sccache are in the path, and if so we set cc_wrapper.
+    cc_wrapper = find_executable("ccache") or find_executable("sccache")
+    if cc_wrapper:
+        out += [r'cc_wrapper="%s"' % cc_wrapper]
+        # Windows needs a custom toolchain for cc_wrapper to work.
+        if os.name == "nt":
+            out += [
+                'custom_toolchain="//build_extra/toolchain/win:win_clang_x64"'
+            ]
 
     print "DENO_BUILD_ARGS:", out