Add web worker JS API (#1993)

* Refactored the way worker polling is scheduled and errors are handled.
* Share the worker future as a Shared

5 files changed, 168 insertions, 34 deletions

diff --git a/js/assets.ts b/js/assets.ts
index 436906783..b3a6e00db 100644
--- a/js/assets.ts
+++ b/js/assets.ts
@@ -44,12 +44,8 @@ import libEsnextDts from "/third_party/node_modules/typescript/lib/lib.esnext.d.
 import libEsnextIntlDts from "/third_party/node_modules/typescript/lib/lib.esnext.intl.d.ts!string";
 import libEsnextSymbolDts from "/third_party/node_modules/typescript/lib/lib.esnext.symbol.d.ts!string";
 
-// @internal
-export const assetSourceCode: { [key: string]: string } = {
-  // Generated library
-  "lib.deno_runtime.d.ts": libDts,
-
-  // Static libraries
+// Default static libraries for all compile jobs
+const defaultAssets: { [key: string]: string } = {
   "lib.es2015.collection.d.ts": libEs2015CollectionDts,
   "lib.es2015.core.d.ts": libEs2015CoreDts,
   "lib.es2015.d.ts": libEs2015Dts,
@@ -85,3 +81,11 @@ export const assetSourceCode: { [key: string]: string } = {
   "lib.esnext.intl.d.ts": libEsnextIntlDts,
   "lib.esnext.symbol.d.ts": libEsnextSymbolDts
 };
+
+// assests for normal compile jobs
+// @internal
+export const assetSourceCode: { [key: string]: string } = {
+  // Generated library
+  "lib.deno_runtime.d.ts": libDts,
+  ...defaultAssets
+};
diff --git a/js/compiler.ts b/js/compiler.ts
index 0f7070fd2..72ac391ea 100644
--- a/js/compiler.ts
+++ b/js/compiler.ts
@@ -46,6 +46,7 @@ type SourceMap = string;
 interface CompilerLookup {
   specifier: ModuleSpecifier;
   referrer: ContainingFile;
+  isWorker: boolean;
 }
 
 /** Abstraction of the APIs required from the `os` module so they can be
@@ -179,6 +180,8 @@ class Compiler implements ts.LanguageServiceHost, ts.FormatDiagnosticsHost {
   // testing
   private _ts: Ts = ts;
 
+  private readonly _assetsSourceCode: { [key: string]: string };
+
   /** The TypeScript language service often refers to the resolved fileName of
    * a module, this is a shortcut to avoid unnecessary module resolution logic
    * for modules that may have been initially resolved by a `moduleSpecifier`
@@ -239,9 +242,12 @@ class Compiler implements ts.LanguageServiceHost, ts.FormatDiagnosticsHost {
       // not null assertion
       moduleId = moduleSpecifier.split("/").pop()!;
       const assetName = moduleId.includes(".") ? moduleId : `${moduleId}.d.ts`;
-      assert(assetName in assetSourceCode, `No such asset "${assetName}"`);
+      assert(
+        assetName in this._assetsSourceCode,
+        `No such asset "${assetName}"`
+      );
       mediaType = msg.MediaType.TypeScript;
-      sourceCode = assetSourceCode[assetName];
+      sourceCode = this._assetsSourceCode[assetName];
       fileName = `${ASSETS}/${assetName}`;
     } else {
       // We query Rust with a CodeFetch message. It will load the sourceCode,
@@ -299,7 +305,8 @@ class Compiler implements ts.LanguageServiceHost, ts.FormatDiagnosticsHost {
     innerMap.set(moduleSpecifier, fileName);
   }
 
-  constructor() {
+  constructor(assetsSourceCode: { [key: string]: string }) {
+    this._assetsSourceCode = assetsSourceCode;
     this._service = this._ts.createLanguageService(this);
   }
 
@@ -498,7 +505,7 @@ class Compiler implements ts.LanguageServiceHost, ts.FormatDiagnosticsHost {
   }
 }
 
-const compiler = new Compiler();
+const compiler = new Compiler(assetSourceCode);
 
 // set global objects for compiler web worker
 window.clearTimeout = clearTimer;
@@ -514,17 +521,12 @@ window.TextEncoder = TextEncoder;
 // lazy instantiating the compiler web worker
 window.compilerMain = function compilerMain() {
   // workerMain should have already been called since a compiler is a worker.
-  const encoder = new TextEncoder();
-  const decoder = new TextDecoder();
-  window.onmessage = ({ data }: { data: Uint8Array }) => {
-    const json = decoder.decode(data);
-    const { specifier, referrer } = JSON.parse(json) as CompilerLookup;
+  window.onmessage = ({ data }: { data: CompilerLookup }) => {
+    const { specifier, referrer } = data;
 
     const result = compiler.compile(specifier, referrer);
 
-    const responseJson = JSON.stringify(result);
-    const response = encoder.encode(responseJson);
-    postMessage(response);
+    postMessage(result);
   };
 };
 
diff --git a/js/globals.ts b/js/globals.ts
index 5a0fb18ce..56956b4ad 100644
--- a/js/globals.ts
+++ b/js/globals.ts
@@ -102,7 +102,16 @@ export type TextDecoder = textEncoding.TextDecoder;
 
 window.performance = new performanceUtil.Performance();
 
+// This variable functioning correctly depends on `declareAsLet`
+// in //tools/ts_library_builder/main.ts
+window.onmessage = workers.onmessage;
+
 window.workerMain = workers.workerMain;
+window.workerClose = workers.workerClose;
+window.postMessage = workers.postMessage;
+
+window.Worker = workers.WorkerImpl;
+export type Worker = workers.Worker;
 
 // below are interfaces that are available in TypeScript but
 // have different signatures
diff --git a/js/main.ts b/js/main.ts
index e7f7e284e..c32f3ac9e 100644
--- a/js/main.ts
+++ b/js/main.ts
@@ -18,8 +18,8 @@ import * as deno from "./deno";
 // TODO(kitsonk) remove with `--types` below
 import libDts from "gen/cli/lib/lib.deno_runtime.d.ts!string";
 
-export default function denoMain(): void {
-  const startResMsg = os.start();
+export default function denoMain(name?: string): void {
+  const startResMsg = os.start(name);
 
   setVersions(startResMsg.denoVersion()!, startResMsg.v8Version()!);
 
diff --git a/js/workers.ts b/js/workers.ts
index bdfbed640..601ffa0b1 100644
--- a/js/workers.ts
+++ b/js/workers.ts
@@ -1,33 +1,110 @@
 // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
-import * as dispatch from "./dispatch";
+/* eslint-disable @typescript-eslint/no-explicit-any */
+import { sendAsync, sendSync } from "./dispatch";
 import * as msg from "gen/cli/msg_generated";
 import * as flatbuffers from "./flatbuffers";
 import { assert, log } from "./util";
+import { TextDecoder, TextEncoder } from "./text_encoding";
 import { window } from "./window";
 
-export async function postMessage(data: Uint8Array): Promise<void> {
+const encoder = new TextEncoder();
+const decoder = new TextDecoder();
+
+export function encodeMessage(data: any): Uint8Array {
+  const dataJson = JSON.stringify(data);
+  return encoder.encode(dataJson);
+}
+
+export function decodeMessage(dataIntArray: Uint8Array): any {
+  const dataJson = decoder.decode(dataIntArray);
+  return JSON.parse(dataJson);
+}
+
+function createWorker(specifier: string): number {
+  const builder = flatbuffers.createBuilder();
+  const specifier_ = builder.createString(specifier);
+  msg.CreateWorker.startCreateWorker(builder);
+  msg.CreateWorker.addSpecifier(builder, specifier_);
+  const inner = msg.CreateWorker.endCreateWorker(builder);
+  const baseRes = sendSync(builder, msg.Any.CreateWorker, inner);
+  assert(baseRes != null);
+  assert(
+    msg.Any.CreateWorkerRes === baseRes!.innerType(),
+    `base.innerType() unexpectedly is ${baseRes!.innerType()}`
+  );
+  const res = new msg.CreateWorkerRes();
+  assert(baseRes!.inner(res) != null);
+  return res.rid();
+}
+
+async function hostGetWorkerClosed(rid: number): Promise<void> {
+  const builder = flatbuffers.createBuilder();
+  msg.HostGetWorkerClosed.startHostGetWorkerClosed(builder);
+  msg.HostGetWorkerClosed.addRid(builder, rid);
+  const inner = msg.HostGetWorkerClosed.endHostGetWorkerClosed(builder);
+  await sendAsync(builder, msg.Any.HostGetWorkerClosed, inner);
+}
+
+function hostPostMessage(rid: number, data: any): void {
+  const dataIntArray = encodeMessage(data);
+  const builder = flatbuffers.createBuilder();
+  msg.HostPostMessage.startHostPostMessage(builder);
+  msg.HostPostMessage.addRid(builder, rid);
+  const inner = msg.HostPostMessage.endHostPostMessage(builder);
+  const baseRes = sendSync(
+    builder,
+    msg.Any.HostPostMessage,
+    inner,
+    dataIntArray
+  );
+  assert(baseRes != null);
+}
+
+async function hostGetMessage(rid: number): Promise<any> {
+  const builder = flatbuffers.createBuilder();
+  msg.HostGetMessage.startHostGetMessage(builder);
+  msg.HostGetMessage.addRid(builder, rid);
+  const inner = msg.HostGetMessage.endHostGetMessage(builder);
+  const baseRes = await sendAsync(builder, msg.Any.HostGetMessage, inner);
+  assert(baseRes != null);
+  assert(
+    msg.Any.HostGetMessageRes === baseRes!.innerType(),
+    `base.innerType() unexpectedly is ${baseRes!.innerType()}`
+  );
+  const res = new msg.HostGetMessageRes();
+  assert(baseRes!.inner(res) != null);
+
+  const dataArray = res.dataArray();
+  if (dataArray != null) {
+    return decodeMessage(dataArray);
+  } else {
+    return null;
+  }
+}
+
+// Stuff for workers
+export let onmessage: (e: { data: any }) => void = (): void => {};
+
+export function postMessage(data: any): void {
+  const dataIntArray = encodeMessage(data);
   const builder = flatbuffers.createBuilder();
   msg.WorkerPostMessage.startWorkerPostMessage(builder);
   const inner = msg.WorkerPostMessage.endWorkerPostMessage(builder);
-  const baseRes = await dispatch.sendAsync(
+  const baseRes = sendSync(
     builder,
     msg.Any.WorkerPostMessage,
     inner,
-    data
+    dataIntArray
   );
   assert(baseRes != null);
 }
 
-export async function getMessage(): Promise<null | Uint8Array> {
+export async function getMessage(): Promise<any> {
   log("getMessage");
   const builder = flatbuffers.createBuilder();
   msg.WorkerGetMessage.startWorkerGetMessage(builder);
   const inner = msg.WorkerGetMessage.endWorkerGetMessage(builder);
-  const baseRes = await dispatch.sendAsync(
-    builder,
-    msg.Any.WorkerGetMessage,
-    inner
-  );
+  const baseRes = await sendAsync(builder, msg.Any.WorkerGetMessage, inner);
   assert(baseRes != null);
   assert(
     msg.Any.WorkerGetMessageRes === baseRes!.innerType(),
@@ -37,14 +114,14 @@ export async function getMessage(): Promise<null | Uint8Array> {
   assert(baseRes!.inner(res) != null);
 
   const dataArray = res.dataArray();
-  if (dataArray == null) {
-    return null;
+  if (dataArray != null) {
+    return decodeMessage(dataArray);
   } else {
-    return new Uint8Array(dataArray!);
+    return null;
   }
 }
 
-let isClosing = false;
+export let isClosing = false;
 
 export function workerClose(): void {
   isClosing = true;
@@ -67,3 +144,45 @@ export async function workerMain(): Promise<void> {
     }
   }
 }
+
+export interface Worker {
+  onerror?: () => void;
+  onmessage?: (e: { data: any }) => void;
+  onmessageerror?: () => void;
+  postMessage(data: any): void;
+}
+
+export class WorkerImpl implements Worker {
+  private readonly rid: number;
+  private isClosing: boolean = false;
+  public onerror?: () => void;
+  public onmessage?: (data: any) => void;
+  public onmessageerror?: () => void;
+
+  constructor(specifier: string) {
+    this.rid = createWorker(specifier);
+    this.run();
+    hostGetWorkerClosed(this.rid).then(() => {
+      this.isClosing = true;
+    });
+  }
+
+  postMessage(data: any): void {
+    hostPostMessage(this.rid, data);
+  }
+
+  private async run(): Promise<void> {
+    while (!this.isClosing) {
+      const data = await hostGetMessage(this.rid);
+      if (data == null) {
+        log("worker got null message. quitting.");
+        break;
+      }
+      // TODO(afinch7) stop this from eating messages before onmessage has been assigned
+      if (this.onmessage) {
+        const event = { data };
+        this.onmessage(event);
+      }
+    }
+  }
+}