refactor(ts): remove op_cache (#5071)

This PR removes op_cache and refactors how Deno interacts with TS compiler.

Ultimate goal is to completely sandbox TS compiler worker; it should operate on
simple request -> response basis. With this commit TS compiler no longer
caches compiled sources as they are generated but rather collects all sources
and sends them back to Rust when compilation is done.

Additionally "Diagnostic" and its children got refactored to use "Deserialize" trait
instead of manually implementing JSON deserialization.

9 files changed, 157 insertions, 203 deletions

diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts
index d3cd67119..f1d339316 100644
--- a/cli/js/compiler.ts
+++ b/cli/js/compiler.ts
@@ -29,7 +29,10 @@ import {
   resolveModules,
 } from "./compiler/imports.ts";
 import {
-  createWriteFile,
+  EmmitedSource,
+  WriteFileCallback,
+  createCompileWriteFile,
+  createBundleWriteFile,
   CompilerRequestType,
   convertCompilerOptions,
   ignoredDiagnostics,
@@ -53,7 +56,6 @@ interface CompilerRequestCompile {
   config?: string;
   unstable: boolean;
   bundle: boolean;
-  outFile?: string;
   cwd: string;
 }
 
@@ -79,16 +81,20 @@ type CompilerRequest =
   | CompilerRequestRuntimeTranspile;
 
 interface CompileResult {
-  emitSkipped: boolean;
-  diagnostics?: Diagnostic;
+  emitMap?: Record<string, EmmitedSource>;
+  bundleOutput?: string;
+  diagnostics: Diagnostic;
 }
 
-type RuntimeCompileResult = [
-  undefined | DiagnosticItem[],
-  Record<string, string>
-];
+interface RuntimeCompileResult {
+  emitMap: Record<string, EmmitedSource>;
+  diagnostics: DiagnosticItem[];
+}
 
-type RuntimeBundleResult = [undefined | DiagnosticItem[], string];
+interface RuntimeBundleResult {
+  output: string;
+  diagnostics: DiagnosticItem[];
+}
 
 async function compile(
   request: CompilerRequestCompile
@@ -97,7 +103,6 @@ async function compile(
     bundle,
     config,
     configPath,
-    outFile,
     rootNames,
     target,
     unstable,
@@ -111,31 +116,32 @@ async function compile(
   // When a programme is emitted, TypeScript will call `writeFile` with
   // each file that needs to be emitted.  The Deno compiler host delegates
   // this, to make it easier to perform the right actions, which vary
-  // based a lot on the request.  For a `Compile` request, we need to
-  // cache all the files in the privileged side if we aren't bundling,
-  // and if we are bundling we need to enrich the bundle and either write
-  // out the bundle or log it to the console.
+  // based a lot on the request.
   const state: WriteFileState = {
     type: request.type,
+    emitMap: {},
     bundle,
     host: undefined,
-    outFile,
     rootNames,
   };
-  const writeFile = createWriteFile(state);
-
+  let writeFile: WriteFileCallback;
+  if (bundle) {
+    writeFile = createBundleWriteFile(state);
+  } else {
+    writeFile = createCompileWriteFile(state);
+  }
   const host = (state.host = new Host({
     bundle,
     target,
     writeFile,
     unstable,
   }));
-  let diagnostics: readonly ts.Diagnostic[] | undefined;
+  let diagnostics: readonly ts.Diagnostic[] = [];
 
   // if there is a configuration supplied, we need to parse that
   if (config && config.length && configPath) {
     const configResult = host.configure(cwd, configPath, config);
-    diagnostics = processConfigureResponse(configResult, configPath);
+    diagnostics = processConfigureResponse(configResult, configPath) || [];
   }
 
   // This will recursively analyse all the code for other imports,
@@ -147,10 +153,9 @@ async function compile(
     bundle || host.getCompilationSettings().checkJs
   );
 
-  let emitSkipped = true;
   // if there was a configuration and no diagnostics with it, we will continue
   // to generate the program and possibly emit it.
-  if (!diagnostics || (diagnostics && diagnostics.length === 0)) {
+  if (diagnostics.length === 0) {
     const options = host.getCompilationSettings();
     const program = ts.createProgram({
       rootNames,
@@ -168,23 +173,28 @@ async function compile(
       if (bundle) {
         // we only support a single root module when bundling
         assert(resolvedRootModules.length === 1);
-        // warning so it goes to stderr instead of stdout
-        console.warn(`Bundling "${resolvedRootModules[0]}"`);
         setRootExports(program, resolvedRootModules[0]);
       }
       const emitResult = program.emit();
-      emitSkipped = emitResult.emitSkipped;
+      assert(emitResult.emitSkipped === false, "Unexpected skip of the emit.");
       // emitResult.diagnostics is `readonly` in TS3.5+ and can't be assigned
       // without casting.
       diagnostics = emitResult.diagnostics;
     }
   }
 
+  let bundleOutput = undefined;
+
+  if (bundle) {
+    assert(state.bundleOutput);
+    bundleOutput = state.bundleOutput;
+  }
+
+  assert(state.emitMap);
   const result: CompileResult = {
-    emitSkipped,
-    diagnostics: diagnostics.length
-      ? fromTypeScriptDiagnostic(diagnostics)
-      : undefined,
+    emitMap: state.emitMap,
+    bundleOutput,
+    diagnostics: fromTypeScriptDiagnostic(diagnostics),
   };
 
   util.log("<<< compile end", {
@@ -259,9 +269,14 @@ async function runtimeCompile(
     rootNames,
     sources,
     emitMap: {},
-    emitBundle: undefined,
+    bundleOutput: undefined,
   };
-  const writeFile = createWriteFile(state);
+  let writeFile: WriteFileCallback;
+  if (bundle) {
+    writeFile = createBundleWriteFile(state);
+  } else {
+    writeFile = createCompileWriteFile(state);
+  }
 
   const host = (state.host = new Host({
     bundle,
@@ -314,12 +329,18 @@ async function runtimeCompile(
 
   const maybeDiagnostics = diagnostics.length
     ? fromTypeScriptDiagnostic(diagnostics).items
-    : undefined;
+    : [];
 
   if (bundle) {
-    return [maybeDiagnostics, state.emitBundle] as RuntimeBundleResult;
+    return {
+      diagnostics: maybeDiagnostics,
+      output: state.bundleOutput,
+    } as RuntimeBundleResult;
   } else {
-    return [maybeDiagnostics, state.emitMap] as RuntimeCompileResult;
+    return {
+      diagnostics: maybeDiagnostics,
+      emitMap: state.emitMap,
+    } as RuntimeCompileResult;
   }
 }
 
diff --git a/cli/js/compiler/api.ts b/cli/js/compiler/api.ts
index b7c57b528..a7d1e57a8 100644
--- a/cli/js/compiler/api.ts
+++ b/cli/js/compiler/api.ts
@@ -6,6 +6,8 @@
 import { DiagnosticItem } from "../diagnostics.ts";
 import * as util from "../util.ts";
 import * as runtimeCompilerOps from "../ops/runtime_compiler.ts";
+import { TranspileOnlyResult } from "../ops/runtime_compiler.ts";
+export { TranspileOnlyResult } from "../ops/runtime_compiler.ts";
 
 export interface CompilerOptions {
   allowJs?: boolean;
@@ -145,12 +147,8 @@ function checkRelative(specifier: string): string {
     : `./${specifier}`;
 }
 
-export interface TranspileOnlyResult {
-  source: string;
-  map?: string;
-}
-
-export async function transpileOnly(
+// TODO(bartlomieju): change return type to interface?
+export function transpileOnly(
   sources: Record<string, string>,
   options: CompilerOptions = {}
 ): Promise<Record<string, TranspileOnlyResult>> {
@@ -159,10 +157,10 @@ export async function transpileOnly(
     sources,
     options: JSON.stringify(options),
   };
-  const result = await runtimeCompilerOps.transpile(payload);
-  return JSON.parse(result);
+  return runtimeCompilerOps.transpile(payload);
 }
 
+// TODO(bartlomieju): change return type to interface?
 export async function compile(
   rootName: string,
   sources?: Record<string, string>,
@@ -180,9 +178,20 @@ export async function compile(
     options,
   });
   const result = await runtimeCompilerOps.compile(payload);
-  return JSON.parse(result);
+  util.assert(result.emitMap);
+  const maybeDiagnostics =
+    result.diagnostics.length === 0 ? undefined : result.diagnostics;
+
+  const emitMap: Record<string, string> = {};
+
+  for (const [key, emmitedSource] of Object.entries(result.emitMap)) {
+    emitMap[key] = emmitedSource.contents;
+  }
+
+  return [maybeDiagnostics, emitMap];
 }
 
+// TODO(bartlomieju): change return type to interface?
 export async function bundle(
   rootName: string,
   sources?: Record<string, string>,
@@ -200,5 +209,8 @@ export async function bundle(
     options,
   });
   const result = await runtimeCompilerOps.compile(payload);
-  return JSON.parse(result);
+  util.assert(result.output);
+  const maybeDiagnostics =
+    result.diagnostics.length === 0 ? undefined : result.diagnostics;
+  return [maybeDiagnostics, result.output];
 }
diff --git a/cli/js/compiler/host.ts b/cli/js/compiler/host.ts
index de2eacfa9..64b5e0245 100644
--- a/cli/js/compiler/host.ts
+++ b/cli/js/compiler/host.ts
@@ -255,16 +255,12 @@ export class Host implements ts.CompilerHost {
       assert(sourceFile != null);
       if (!sourceFile.tsSourceFile) {
         assert(sourceFile.sourceCode != null);
-        // even though we assert the extension for JSON modules to the compiler
-        // is TypeScript, TypeScript internally analyses the filename for its
-        // extension and tries to parse it as JSON instead of TS.  We have to
-        // change the filename to the TypeScript file.
+        const tsSourceFileName = fileName.startsWith(ASSETS)
+          ? sourceFile.filename
+          : fileName;
+
         sourceFile.tsSourceFile = ts.createSourceFile(
-          fileName.startsWith(ASSETS)
-            ? sourceFile.filename
-            : fileName.toLowerCase().endsWith(".json")
-            ? `${fileName}.ts`
-            : fileName,
+          tsSourceFileName,
           sourceFile.sourceCode,
           languageVersion
         );
@@ -294,15 +290,20 @@ export class Host implements ts.CompilerHost {
       containingFile,
     });
     return moduleNames.map((specifier) => {
-      const url = SourceFile.getUrl(specifier, containingFile);
-      const sourceFile = specifier.startsWith(ASSETS)
-        ? getAssetInternal(specifier)
-        : url
-        ? SourceFile.get(url)
-        : undefined;
+      const maybeUrl = SourceFile.getUrl(specifier, containingFile);
+
+      let sourceFile: SourceFile | undefined = undefined;
+
+      if (specifier.startsWith(ASSETS)) {
+        sourceFile = getAssetInternal(specifier);
+      } else if (typeof maybeUrl !== "undefined") {
+        sourceFile = SourceFile.get(maybeUrl);
+      }
+
       if (!sourceFile) {
         return undefined;
       }
+
       return {
         resolvedFileName: sourceFile.url,
         isExternalLibraryImport: specifier.startsWith(ASSETS),
diff --git a/cli/js/compiler/imports.ts b/cli/js/compiler/imports.ts
index de4402758..a811075be 100644
--- a/cli/js/compiler/imports.ts
+++ b/cli/js/compiler/imports.ts
@@ -122,11 +122,8 @@ export async function processImports(
       SourceFile.get(sourceFileJson.url) || new SourceFile(sourceFileJson);
     sourceFile.cache(specifiers[i][0], referrer);
     if (!sourceFile.processed) {
-      await processImports(
-        sourceFile.imports(processJsImports),
-        sourceFile.url,
-        processJsImports
-      );
+      const sourceFileImports = sourceFile.imports(processJsImports);
+      await processImports(sourceFileImports, sourceFile.url, processJsImports);
     }
   }
   return resolvedSources;
diff --git a/cli/js/compiler/sourcefile.ts b/cli/js/compiler/sourcefile.ts
index 3d547551f..d390c3f56 100644
--- a/cli/js/compiler/sourcefile.ts
+++ b/cli/js/compiler/sourcefile.ts
@@ -54,8 +54,6 @@ export class SourceFile {
   extension!: ts.Extension;
   filename!: string;
 
-  importedFiles?: Array<[string, string]>;
-
   mediaType!: MediaType;
   processed = false;
   sourceCode?: string;
@@ -93,14 +91,18 @@ export class SourceFile {
       return [];
     }
 
+    const readImportFiles = true;
+    const detectJsImports =
+      this.mediaType === MediaType.JavaScript ||
+      this.mediaType === MediaType.JSX;
+
     const preProcessedFileInfo = ts.preProcessFile(
       this.sourceCode,
-      true,
-      this.mediaType === MediaType.JavaScript ||
-        this.mediaType === MediaType.JSX
+      readImportFiles,
+      detectJsImports
     );
     this.processed = true;
-    const files = (this.importedFiles = [] as Array<[string, string]>);
+    const files: Array<[string, string]> = [];
 
     function process(references: Array<{ fileName: string }>): void {
       for (const { fileName } of references) {
@@ -160,8 +162,4 @@ export class SourceFile {
   static get(url: string): SourceFile | undefined {
     return moduleCache.get(url);
   }
-
-  static has(url: string): boolean {
-    return moduleCache.has(url);
-  }
 }
diff --git a/cli/js/compiler/util.ts b/cli/js/compiler/util.ts
index 35ce2e837..f3cbe5566 100644
--- a/cli/js/compiler/util.ts
+++ b/cli/js/compiler/util.ts
@@ -4,12 +4,16 @@ import { bold, cyan, yellow } from "../colors.ts";
 import { CompilerOptions } from "./api.ts";
 import { buildBundle } from "./bundler.ts";
 import { ConfigureResponse, Host } from "./host.ts";
-import { MediaType, SourceFile } from "./sourcefile.ts";
-import { atob, TextEncoder } from "../web/text_encoding.ts";
+import { atob } from "../web/text_encoding.ts";
 import * as compilerOps from "../ops/compiler.ts";
-import * as util from "../util.ts";
 import { assert } from "../util.ts";
-import { writeFileSync } from "../write_file.ts";
+
+export interface EmmitedSource {
+  // original filename
+  filename: string;
+  // compiled contents
+  contents: string;
+}
 
 export type WriteFileCallback = (
   fileName: string,
@@ -20,11 +24,10 @@ export type WriteFileCallback = (
 export interface WriteFileState {
   type: CompilerRequestType;
   bundle?: boolean;
+  bundleOutput?: string;
   host?: Host;
-  outFile?: string;
   rootNames: string[];
-  emitMap?: Record<string, string>;
-  emitBundle?: string;
+  emitMap?: Record<string, EmmitedSource>;
   sources?: Record<string, string>;
 }
 
@@ -38,87 +41,33 @@ export enum CompilerRequestType {
 
 export const OUT_DIR = "$deno$";
 
-function cache(
-  moduleId: string,
-  emittedFileName: string,
-  contents: string,
-  checkJs = false
-): void {
-  util.log("compiler::cache", { moduleId, emittedFileName, checkJs });
-  const sf = SourceFile.get(moduleId);
-
-  if (sf) {
-    // NOTE: JavaScript files are only cached to disk if `checkJs`
-    // option in on
-    if (sf.mediaType === MediaType.JavaScript && !checkJs) {
-      return;
-    }
-  }
-
-  if (emittedFileName.endsWith(".map")) {
-    // Source Map
-    compilerOps.cache(".map", moduleId, contents);
-  } else if (emittedFileName.endsWith(".js")) {
-    // Compiled JavaScript
-    compilerOps.cache(".js", moduleId, contents);
-  } else {
-    assert(false, `Trying to cache unhandled file type "${emittedFileName}"`);
-  }
-}
-
 export function getAsset(name: string): string {
   return compilerOps.getAsset(name);
 }
 
-export function createWriteFile(state: WriteFileState): WriteFileCallback {
-  const encoder = new TextEncoder();
-  if (state.type === CompilerRequestType.Compile) {
-    return function writeFile(
-      fileName: string,
-      data: string,
-      sourceFiles?: readonly ts.SourceFile[]
-    ): void {
-      assert(
-        sourceFiles != null,
-        `Unexpected emit of "${fileName}" which isn't part of a program.`
-      );
-      assert(state.host);
-      if (!state.bundle) {
-        assert(sourceFiles.length === 1);
-        cache(
-          sourceFiles[0].fileName,
-          fileName,
-          data,
-          state.host.getCompilationSettings().checkJs
-        );
-      } else {
-        // if the fileName is set to an internal value, just noop, this is
-        // used in the Rust unit tests.
-        if (state.outFile && state.outFile.startsWith(OUT_DIR)) {
-          return;
-        }
-        // we only support single root names for bundles
-        assert(
-          state.rootNames.length === 1,
-          `Only one root name supported.  Got "${JSON.stringify(
-            state.rootNames
-          )}"`
-        );
-        // this enriches the string with the loader and re-exports the
-        // exports of the root module
-        const content = buildBundle(state.rootNames[0], data, sourceFiles);
-        if (state.outFile) {
-          const encodedData = encoder.encode(content);
-          console.warn(`Emitting bundle to "${state.outFile}"`);
-          writeFileSync(state.outFile, encodedData);
-          console.warn(`${humanFileSize(encodedData.length)} emitted.`);
-        } else {
-          console.log(content);
-        }
-      }
-    };
-  }
+// TODO(bartlomieju): probably could be defined inline?
+export function createBundleWriteFile(
+  state: WriteFileState
+): WriteFileCallback {
+  return function writeFile(
+    _fileName: string,
+    data: string,
+    sourceFiles?: readonly ts.SourceFile[]
+  ): void {
+    assert(sourceFiles != null);
+    assert(state.host);
+    assert(state.emitMap);
+    assert(state.bundle);
+    // we only support single root names for bundles
+    assert(state.rootNames.length === 1);
+    state.bundleOutput = buildBundle(state.rootNames[0], data, sourceFiles);
+  };
+}
 
+// TODO(bartlomieju): probably could be defined inline?
+export function createCompileWriteFile(
+  state: WriteFileState
+): WriteFileCallback {
   return function writeFile(
     fileName: string,
     data: string,
@@ -127,24 +76,12 @@ export function createWriteFile(state: WriteFileState): WriteFileCallback {
     assert(sourceFiles != null);
     assert(state.host);
     assert(state.emitMap);
-    if (!state.bundle) {
-      assert(sourceFiles.length === 1);
-      state.emitMap[fileName] = data;
-      // we only want to cache the compiler output if we are resolving
-      // modules externally
-      if (!state.sources) {
-        cache(
-          sourceFiles[0].fileName,
-          fileName,
-          data,
-          state.host.getCompilationSettings().checkJs
-        );
-      }
-    } else {
-      // we only support single root names for bundles
-      assert(state.rootNames.length === 1);
-      state.emitBundle = buildBundle(state.rootNames[0], data, sourceFiles);
-    }
+    assert(!state.bundle);
+    assert(sourceFiles.length === 1);
+    state.emitMap[fileName] = {
+      filename: sourceFiles[0].fileName,
+      contents: data,
+    };
   };
 }
 
@@ -380,20 +317,6 @@ export function commonPath(paths: string[], sep = "/"): string {
   return prefix.endsWith(sep) ? prefix : `${prefix}${sep}`;
 }
 
-function humanFileSize(bytes: number): string {
-  const thresh = 1000;
-  if (Math.abs(bytes) < thresh) {
-    return bytes + " B";
-  }
-  const units = ["kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"];
-  let u = -1;
-  do {
-    bytes /= thresh;
-    ++u;
-  } while (Math.abs(bytes) >= thresh && u < units.length - 1);
-  return `${bytes.toFixed(1)} ${units[u]}`;
-}
-
 // @internal
 export function base64ToUint8Array(data: string): Uint8Array {
   const binString = atob(data);
diff --git a/cli/js/ops/compiler.ts b/cli/js/ops/compiler.ts
index 825cadc16..60f814741 100644
--- a/cli/js/ops/compiler.ts
+++ b/cli/js/ops/compiler.ts
@@ -38,15 +38,3 @@ export function getAsset(name: string): string {
   const sourceCodeBytes = core.dispatch(opId, encoder.encode(name));
   return decoder.decode(sourceCodeBytes!);
 }
-
-export function cache(
-  extension: string,
-  moduleId: string,
-  contents: string
-): void {
-  sendSync("op_cache", {
-    extension,
-    moduleId,
-    contents,
-  });
-}
diff --git a/cli/js/ops/runtime_compiler.ts b/cli/js/ops/runtime_compiler.ts
index b46670ace..5a89983ee 100644
--- a/cli/js/ops/runtime_compiler.ts
+++ b/cli/js/ops/runtime_compiler.ts
@@ -1,6 +1,7 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
 
 import { sendAsync } from "./dispatch_json.ts";
+import { DiagnosticItem } from "../diagnostics.ts";
 
 interface CompileRequest {
   rootName: string;
@@ -9,7 +10,13 @@ interface CompileRequest {
   bundle: boolean;
 }
 
-export function compile(request: CompileRequest): Promise<string> {
+interface CompileResponse {
+  diagnostics: DiagnosticItem[];
+  output?: string;
+  emitMap?: Record<string, Record<string, string>>;
+}
+
+export function compile(request: CompileRequest): Promise<CompileResponse> {
   return sendAsync("op_compile", request);
 }
 
@@ -18,6 +25,13 @@ interface TranspileRequest {
   options?: string;
 }
 
-export function transpile(request: TranspileRequest): Promise<string> {
+export interface TranspileOnlyResult {
+  source: string;
+  map?: string;
+}
+
+export function transpile(
+  request: TranspileRequest
+): Promise<Record<string, TranspileOnlyResult>> {
   return sendAsync("op_transpile", request);
 }
diff --git a/cli/js/tests/format_error_test.ts b/cli/js/tests/format_error_test.ts
index 0cb963ae6..ae7559c82 100644
--- a/cli/js/tests/format_error_test.ts
+++ b/cli/js/tests/format_error_test.ts
@@ -26,7 +26,7 @@ unitTest(function formatDiagnosticError() {
   try {
     Deno.formatDiagnostics(bad);
   } catch (e) {
-    assert(e instanceof TypeError);
+    assert(e instanceof Deno.errors.InvalidData);
     thrown = true;
   }
   assert(thrown);