Execute JS for the first time in Rust rewrite.

Implements code_fetch handler in Rust.

Add ability to embed string assets (for typescript declaration files)

Remove deno_cc and deno_cc_nosnapshot targets.

5 files changed, 104 insertions, 41 deletions

diff --git a/js/assets.ts b/js/assets.ts
new file mode 100644
index 000000000..850fc6c72
--- /dev/null
+++ b/js/assets.ts
@@ -0,0 +1,51 @@
+// Copyright 2018 Ryan Dahl <ry@tinyclouds.org>
+// All rights reserved. MIT License.
+
+// This file is formatted as it is because we are using the fact that Parcel
+// statically evaluates fs.readFileSync.
+import { readFileSync } from "fs";
+
+// tslint:disable:max-line-length
+// prettier-ignore
+export const assetSourceCode: { [key: string]: string } = {
+  "deno.d.ts": readFileSync(__dirname + "/deno.d.ts", "utf8"),
+  "lib.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.d.ts", "utf8"),
+  //"lib.dom.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.dom.d.ts", "utf8"),
+  "lib.dom.iterable.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.dom.iterable.d.ts", "utf8"),
+  "lib.es2015.collection.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2015.collection.d.ts", "utf8"),
+  "lib.es2015.core.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2015.core.d.ts", "utf8"),
+  //"lib.es2015.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2015.d.ts", "utf8"),
+  "lib.es2015.generator.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2015.generator.d.ts", "utf8"),
+  "lib.es2015.iterable.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2015.iterable.d.ts", "utf8"),
+  "lib.es2015.promise.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2015.promise.d.ts", "utf8"),
+  "lib.es2015.proxy.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2015.proxy.d.ts", "utf8"),
+  "lib.es2015.reflect.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2015.reflect.d.ts", "utf8"),
+  "lib.es2015.symbol.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2015.symbol.d.ts", "utf8"),
+  "lib.es2015.symbol.wellknown.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2015.symbol.wellknown.d.ts", "utf8"),
+  "lib.es2016.array.include.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2016.array.include.d.ts", "utf8"),
+  //"lib.es2016.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2016.d.ts", "utf8"),
+  //"lib.es2016.full.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2016.full.d.ts", "utf8"),
+  //"lib.es2017.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2017.d.ts", "utf8"),
+  //"lib.es2017.full.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2017.full.d.ts", "utf8"),
+  "lib.es2017.intl.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2017.intl.d.ts", "utf8"),
+  "lib.es2017.object.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2017.object.d.ts", "utf8"),
+  "lib.es2017.sharedmemory.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2017.sharedmemory.d.ts", "utf8"),
+  "lib.es2017.string.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2017.string.d.ts", "utf8"),
+  "lib.es2017.typedarrays.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2017.typedarrays.d.ts", "utf8"),
+  "lib.es2018.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2018.d.ts", "utf8"),
+  //"lib.es2018.full.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2018.full.d.ts", "utf8"),
+  "lib.es2018.promise.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2018.promise.d.ts", "utf8"),
+  "lib.es2018.regexp.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es2018.regexp.d.ts", "utf8"),
+  //"lib.es5.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es5.d.ts", "utf8"),
+  //"lib.es6.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.es6.d.ts", "utf8"),
+  "lib.esnext.array.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.esnext.array.d.ts", "utf8"),
+  "lib.esnext.asynciterable.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.esnext.asynciterable.d.ts", "utf8"),
+  "lib.esnext.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.esnext.d.ts", "utf8"),
+  //"lib.esnext.full.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.esnext.full.d.ts", "utf8"),
+  //"lib.scripthost.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.scripthost.d.ts", "utf8"),
+  //"lib.webworker.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/lib.webworker.d.ts", "utf8"),
+  //"protocol.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/protocol.d.ts", "utf8"),
+  //"tsserverlibrary.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/tsserverlibrary.d.ts", "utf8"),
+  "typescript.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/typescript.d.ts", "utf8"),
+  //"typescriptServices.d.ts": readFileSync(__dirname + "/../third_party/node_modules/typescript/lib/typescriptServices.d.ts", "utf8"),
+};
diff --git a/js/main.ts b/js/main.ts
index 379b69c97..c140304ef 100644
--- a/js/main.ts
+++ b/js/main.ts
@@ -1,12 +1,9 @@
 // tslint:disable-next-line:no-reference
 /// <reference path="deno.d.ts" />
-import * as ts from "typescript";
-
 import { flatbuffers } from "flatbuffers";
 import { deno as fbs } from "./msg_generated";
-import { assert } from "./util";
-
-// import * as runtime from "./runtime";
+import { assert, log } from "./util";
+import * as runtime from "./runtime";
 
 const globalEval = eval;
 const window = globalEval("this");
@@ -31,8 +28,6 @@ function startMsg(cmdId: number): Uint8Array {
 }
 
 window["denoMain"] = () => {
-  deno.print(`ts.version: ${ts.version}`);
-
   // First we send an empty "Start" message to let the privlaged side know we
   // are ready. The response should be a "StartRes" message containing the CLI
   // argv and other info.
@@ -55,17 +50,15 @@ window["denoMain"] = () => {
   assert(base.msg(startResMsg) != null);
 
   const cwd = startResMsg.cwd();
-  deno.print(`cwd: ${cwd}`);
+  log("cwd", cwd);
 
   const argv: string[] = [];
   for (let i = 0; i < startResMsg.argvLength(); i++) {
     argv.push(startResMsg.argv(i));
   }
-  deno.print(`argv ${argv}`);
+  log("argv", argv);
 
-  /* TODO(ry) Uncomment to test further message passing.
-  const inputFn = argv[0];
+  const inputFn = argv[1];
   const mod = runtime.resolveModule(inputFn, `${cwd}/`);
   mod.compileAndRun();
-  */
 };
diff --git a/js/os.ts b/js/os.ts
index 0c8ebede9..e4263009f 100644
--- a/js/os.ts
+++ b/js/os.ts
@@ -2,7 +2,8 @@
 // All rights reserved. MIT License.
 import { ModuleInfo } from "./types";
 import { deno as fbs } from "./msg_generated";
-import { assert, typedArrayToArrayBuffer } from "./util";
+import { assert } from "./util";
+import * as util from "./util";
 import { flatbuffers } from "flatbuffers";
 
 export function exit(exitCode = 0): void {
@@ -19,8 +20,7 @@ export function codeFetch(
   moduleSpecifier: string,
   containingFile: string
 ): ModuleInfo {
-  console.log("Hello from codeFetch");
-
+  util.log("os.ts codeFetch", moduleSpecifier, containingFile);
   // Send CodeFetch message
   const builder = new flatbuffers.Builder();
   const moduleSpecifier_ = builder.createString(moduleSpecifier);
@@ -33,23 +33,23 @@ export function codeFetch(
   fbs.Base.addMsg(builder, msg);
   fbs.Base.addMsgType(builder, fbs.Any.CodeFetch);
   builder.finish(fbs.Base.endBase(builder));
-  const payload = typedArrayToArrayBuffer(builder.asUint8Array());
-  const resBuf = deno.send("x", payload);
-
-  console.log("CodeFetch sent");
-
+  const resBuf = deno.send(builder.asUint8Array());
   // Process CodeFetchRes
   const bb = new flatbuffers.ByteBuffer(new Uint8Array(resBuf));
   const baseRes = fbs.Base.getRootAsBase(bb);
+  if (fbs.Any.NONE === baseRes.msgType()) {
+    throw Error(baseRes.error());
+  }
   assert(fbs.Any.CodeFetchRes === baseRes.msgType());
   const codeFetchRes = new fbs.CodeFetchRes();
   assert(baseRes.msg(codeFetchRes) != null);
-  return {
+  const r = {
     moduleName: codeFetchRes.moduleName(),
     filename: codeFetchRes.filename(),
     sourceCode: codeFetchRes.sourceCode(),
     outputCode: codeFetchRes.outputCode()
   };
+  return r;
 }
 
 export function codeCache(
@@ -57,28 +57,22 @@ export function codeCache(
   sourceCode: string,
   outputCode: string
 ): void {
+  util.log("os.ts codeCache", filename, sourceCode, outputCode);
   const builder = new flatbuffers.Builder();
-
   const filename_ = builder.createString(filename);
   const sourceCode_ = builder.createString(sourceCode);
   const outputCode_ = builder.createString(outputCode);
-
   fbs.CodeCache.startCodeCache(builder);
   fbs.CodeCache.addFilename(builder, filename_);
   fbs.CodeCache.addSourceCode(builder, sourceCode_);
   fbs.CodeCache.addOutputCode(builder, outputCode_);
   const msg = fbs.CodeCache.endCodeCache(builder);
-
   fbs.Base.startBase(builder);
   fbs.Base.addMsg(builder, msg);
+  fbs.Base.addMsgType(builder, fbs.Any.CodeCache);
   builder.finish(fbs.Base.endBase(builder));
-
-  // Maybe need to do another step?
-  // Base.finishBaseBuffer(builder, base);
-
-  const payload = typedArrayToArrayBuffer(builder.asUint8Array());
-  const resBuf = deno.send("x", payload);
-  assert(resBuf === null);
+  const resBuf = deno.send(builder.asUint8Array());
+  assert(resBuf == null);
 }
 
 export function readFileSync(filename: string): Uint8Array {
diff --git a/js/runtime.ts b/js/runtime.ts
index 9570dda1c..d234cdd85 100644
--- a/js/runtime.ts
+++ b/js/runtime.ts
@@ -11,6 +11,7 @@
 import * as ts from "typescript";
 import * as util from "./util";
 import { log } from "./util";
+import { assetSourceCode } from "./assets";
 import * as os from "./os";
 //import * as sourceMaps from "./v8_source_maps";
 import { window, globalEval } from "./globals";
@@ -22,9 +23,14 @@ const EOL = "\n";
 export type AmdFactory = (...args: any[]) => undefined | object;
 export type AmdDefine = (deps: string[], factory: AmdFactory) => void;
 
-/*
 // Uncaught exceptions are sent to window.onerror by the privlaged binding.
-window.onerror = (message, source, lineno, colno, error) => {
+window.onerror = (
+  message: string,
+  source: string,
+  lineno: number,
+  colno: number,
+  error: Error
+) => {
   // TODO Currently there is a bug in v8_source_maps.ts that causes a segfault
   // if it is used within window.onerror. To workaround we uninstall the
   // Error.prepareStackTrace handler. Users will get unmapped stack traces on
@@ -33,7 +39,6 @@ window.onerror = (message, source, lineno, colno, error) => {
   console.log(error.message, error.stack);
   os.exit(1);
 };
-*/
 
 /*
 export function setup(mainJs: string, mainMap: string): void {
@@ -147,11 +152,31 @@ export function resolveModule(
 ): null | FileModule {
   util.log("resolveModule", { moduleSpecifier, containingFile });
   util.assert(moduleSpecifier != null && moduleSpecifier.length > 0);
-  // We ask golang to sourceCodeFetch. It will load the sourceCode and if
-  // there is any outputCode cached, it will return that as well.
-  const fetchResponse = os.codeFetch(moduleSpecifier, containingFile);
-  const { filename, sourceCode, outputCode } = fetchResponse;
-  if (sourceCode.length === 0) {
+  let filename: string, sourceCode: string, outputCode: string;
+  if (
+    moduleSpecifier.startsWith("/$asset$/") ||
+    containingFile.startsWith("/$asset$/")
+  ) {
+    // Assets are compiled into the runtime javascript bundle.
+    const assetName = moduleSpecifier.split("/").pop();
+    sourceCode = assetSourceCode[assetName];
+    filename = "/$asset$/" + assetName;
+  } else {
+    // We query Rust with a CodeFetch message. It will load the sourceCode, and
+    // if there is any outputCode cached, will return that as well.
+    let fetchResponse;
+    try {
+      fetchResponse = os.codeFetch(moduleSpecifier, containingFile);
+    } catch (e) {
+      // TODO Only catch "no such file or directory" errors. Need error codes.
+      util.log("os.codeFetch error ignored", e.message);
+      return null;
+    }
+    filename = fetchResponse.filename;
+    sourceCode = fetchResponse.sourceCode;
+    outputCode = fetchResponse.outputCode;
+  }
+  if (sourceCode == null || sourceCode.length === 0) {
     return null;
   }
   util.log("resolveModule sourceCode length ", sourceCode.length);
@@ -293,7 +318,7 @@ class TypeScriptHost implements ts.LanguageServiceHost {
   }
 
   getDefaultLibFileName(options: ts.CompilerOptions): string {
-    const fn = ts.getDefaultLibFileName(options);
+    const fn = "lib.d.ts"; // ts.getDefaultLibFileName(options);
     util.log("getDefaultLibFileName", fn);
     const m = resolveModule(fn, "/$asset$/");
     return m.fileName;
diff --git a/js/util.ts b/js/util.ts
index 9f87ab63d..7216f9491 100644
--- a/js/util.ts
+++ b/js/util.ts
@@ -2,7 +2,7 @@
 // All rights reserved. MIT License.
 
 //import { debug } from "./main";
-const debug = true;
+const debug = false;
 
 import { TypedArray } from "./types";