refactor(ts): remove op_cache (#5071)

This PR removes op_cache and refactors how Deno interacts with TS compiler.

Ultimate goal is to completely sandbox TS compiler worker; it should operate on
simple request -> response basis. With this commit TS compiler no longer
caches compiled sources as they are generated but rather collects all sources
and sends them back to Rust when compilation is done.

Additionally "Diagnostic" and its children got refactored to use "Deserialize" trait
instead of manually implementing JSON deserialization.

5 files changed, 86 insertions, 155 deletions

diff --git a/cli/js/compiler/api.ts b/cli/js/compiler/api.ts
index b7c57b528..a7d1e57a8 100644
--- a/cli/js/compiler/api.ts
+++ b/cli/js/compiler/api.ts
@@ -6,6 +6,8 @@
 import { DiagnosticItem } from "../diagnostics.ts";
 import * as util from "../util.ts";
 import * as runtimeCompilerOps from "../ops/runtime_compiler.ts";
+import { TranspileOnlyResult } from "../ops/runtime_compiler.ts";
+export { TranspileOnlyResult } from "../ops/runtime_compiler.ts";
 
 export interface CompilerOptions {
   allowJs?: boolean;
@@ -145,12 +147,8 @@ function checkRelative(specifier: string): string {
     : `./${specifier}`;
 }
 
-export interface TranspileOnlyResult {
-  source: string;
-  map?: string;
-}
-
-export async function transpileOnly(
+// TODO(bartlomieju): change return type to interface?
+export function transpileOnly(
   sources: Record<string, string>,
   options: CompilerOptions = {}
 ): Promise<Record<string, TranspileOnlyResult>> {
@@ -159,10 +157,10 @@ export async function transpileOnly(
     sources,
     options: JSON.stringify(options),
   };
-  const result = await runtimeCompilerOps.transpile(payload);
-  return JSON.parse(result);
+  return runtimeCompilerOps.transpile(payload);
 }
 
+// TODO(bartlomieju): change return type to interface?
 export async function compile(
   rootName: string,
   sources?: Record<string, string>,
@@ -180,9 +178,20 @@ export async function compile(
     options,
   });
   const result = await runtimeCompilerOps.compile(payload);
-  return JSON.parse(result);
+  util.assert(result.emitMap);
+  const maybeDiagnostics =
+    result.diagnostics.length === 0 ? undefined : result.diagnostics;
+
+  const emitMap: Record<string, string> = {};
+
+  for (const [key, emmitedSource] of Object.entries(result.emitMap)) {
+    emitMap[key] = emmitedSource.contents;
+  }
+
+  return [maybeDiagnostics, emitMap];
 }
 
+// TODO(bartlomieju): change return type to interface?
 export async function bundle(
   rootName: string,
   sources?: Record<string, string>,
@@ -200,5 +209,8 @@ export async function bundle(
     options,
   });
   const result = await runtimeCompilerOps.compile(payload);
-  return JSON.parse(result);
+  util.assert(result.output);
+  const maybeDiagnostics =
+    result.diagnostics.length === 0 ? undefined : result.diagnostics;
+  return [maybeDiagnostics, result.output];
 }
diff --git a/cli/js/compiler/host.ts b/cli/js/compiler/host.ts
index de2eacfa9..64b5e0245 100644
--- a/cli/js/compiler/host.ts
+++ b/cli/js/compiler/host.ts
@@ -255,16 +255,12 @@ export class Host implements ts.CompilerHost {
       assert(sourceFile != null);
       if (!sourceFile.tsSourceFile) {
         assert(sourceFile.sourceCode != null);
-        // even though we assert the extension for JSON modules to the compiler
-        // is TypeScript, TypeScript internally analyses the filename for its
-        // extension and tries to parse it as JSON instead of TS.  We have to
-        // change the filename to the TypeScript file.
+        const tsSourceFileName = fileName.startsWith(ASSETS)
+          ? sourceFile.filename
+          : fileName;
+
         sourceFile.tsSourceFile = ts.createSourceFile(
-          fileName.startsWith(ASSETS)
-            ? sourceFile.filename
-            : fileName.toLowerCase().endsWith(".json")
-            ? `${fileName}.ts`
-            : fileName,
+          tsSourceFileName,
           sourceFile.sourceCode,
           languageVersion
         );
@@ -294,15 +290,20 @@ export class Host implements ts.CompilerHost {
       containingFile,
     });
     return moduleNames.map((specifier) => {
-      const url = SourceFile.getUrl(specifier, containingFile);
-      const sourceFile = specifier.startsWith(ASSETS)
-        ? getAssetInternal(specifier)
-        : url
-        ? SourceFile.get(url)
-        : undefined;
+      const maybeUrl = SourceFile.getUrl(specifier, containingFile);
+
+      let sourceFile: SourceFile | undefined = undefined;
+
+      if (specifier.startsWith(ASSETS)) {
+        sourceFile = getAssetInternal(specifier);
+      } else if (typeof maybeUrl !== "undefined") {
+        sourceFile = SourceFile.get(maybeUrl);
+      }
+
       if (!sourceFile) {
         return undefined;
       }
+
       return {
         resolvedFileName: sourceFile.url,
         isExternalLibraryImport: specifier.startsWith(ASSETS),
diff --git a/cli/js/compiler/imports.ts b/cli/js/compiler/imports.ts
index de4402758..a811075be 100644
--- a/cli/js/compiler/imports.ts
+++ b/cli/js/compiler/imports.ts
@@ -122,11 +122,8 @@ export async function processImports(
       SourceFile.get(sourceFileJson.url) || new SourceFile(sourceFileJson);
     sourceFile.cache(specifiers[i][0], referrer);
     if (!sourceFile.processed) {
-      await processImports(
-        sourceFile.imports(processJsImports),
-        sourceFile.url,
-        processJsImports
-      );
+      const sourceFileImports = sourceFile.imports(processJsImports);
+      await processImports(sourceFileImports, sourceFile.url, processJsImports);
     }
   }
   return resolvedSources;
diff --git a/cli/js/compiler/sourcefile.ts b/cli/js/compiler/sourcefile.ts
index 3d547551f..d390c3f56 100644
--- a/cli/js/compiler/sourcefile.ts
+++ b/cli/js/compiler/sourcefile.ts
@@ -54,8 +54,6 @@ export class SourceFile {
   extension!: ts.Extension;
   filename!: string;
 
-  importedFiles?: Array<[string, string]>;
-
   mediaType!: MediaType;
   processed = false;
   sourceCode?: string;
@@ -93,14 +91,18 @@ export class SourceFile {
       return [];
     }
 
+    const readImportFiles = true;
+    const detectJsImports =
+      this.mediaType === MediaType.JavaScript ||
+      this.mediaType === MediaType.JSX;
+
     const preProcessedFileInfo = ts.preProcessFile(
       this.sourceCode,
-      true,
-      this.mediaType === MediaType.JavaScript ||
-        this.mediaType === MediaType.JSX
+      readImportFiles,
+      detectJsImports
     );
     this.processed = true;
-    const files = (this.importedFiles = [] as Array<[string, string]>);
+    const files: Array<[string, string]> = [];
 
     function process(references: Array<{ fileName: string }>): void {
       for (const { fileName } of references) {
@@ -160,8 +162,4 @@ export class SourceFile {
   static get(url: string): SourceFile | undefined {
     return moduleCache.get(url);
   }
-
-  static has(url: string): boolean {
-    return moduleCache.has(url);
-  }
 }
diff --git a/cli/js/compiler/util.ts b/cli/js/compiler/util.ts
index 35ce2e837..f3cbe5566 100644
--- a/cli/js/compiler/util.ts
+++ b/cli/js/compiler/util.ts
@@ -4,12 +4,16 @@ import { bold, cyan, yellow } from "../colors.ts";
 import { CompilerOptions } from "./api.ts";
 import { buildBundle } from "./bundler.ts";
 import { ConfigureResponse, Host } from "./host.ts";
-import { MediaType, SourceFile } from "./sourcefile.ts";
-import { atob, TextEncoder } from "../web/text_encoding.ts";
+import { atob } from "../web/text_encoding.ts";
 import * as compilerOps from "../ops/compiler.ts";
-import * as util from "../util.ts";
 import { assert } from "../util.ts";
-import { writeFileSync } from "../write_file.ts";
+
+export interface EmmitedSource {
+  // original filename
+  filename: string;
+  // compiled contents
+  contents: string;
+}
 
 export type WriteFileCallback = (
   fileName: string,
@@ -20,11 +24,10 @@ export type WriteFileCallback = (
 export interface WriteFileState {
   type: CompilerRequestType;
   bundle?: boolean;
+  bundleOutput?: string;
   host?: Host;
-  outFile?: string;
   rootNames: string[];
-  emitMap?: Record<string, string>;
-  emitBundle?: string;
+  emitMap?: Record<string, EmmitedSource>;
   sources?: Record<string, string>;
 }
 
@@ -38,87 +41,33 @@ export enum CompilerRequestType {
 
 export const OUT_DIR = "$deno$";
 
-function cache(
-  moduleId: string,
-  emittedFileName: string,
-  contents: string,
-  checkJs = false
-): void {
-  util.log("compiler::cache", { moduleId, emittedFileName, checkJs });
-  const sf = SourceFile.get(moduleId);
-
-  if (sf) {
-    // NOTE: JavaScript files are only cached to disk if `checkJs`
-    // option in on
-    if (sf.mediaType === MediaType.JavaScript && !checkJs) {
-      return;
-    }
-  }
-
-  if (emittedFileName.endsWith(".map")) {
-    // Source Map
-    compilerOps.cache(".map", moduleId, contents);
-  } else if (emittedFileName.endsWith(".js")) {
-    // Compiled JavaScript
-    compilerOps.cache(".js", moduleId, contents);
-  } else {
-    assert(false, `Trying to cache unhandled file type "${emittedFileName}"`);
-  }
-}
-
 export function getAsset(name: string): string {
   return compilerOps.getAsset(name);
 }
 
-export function createWriteFile(state: WriteFileState): WriteFileCallback {
-  const encoder = new TextEncoder();
-  if (state.type === CompilerRequestType.Compile) {
-    return function writeFile(
-      fileName: string,
-      data: string,
-      sourceFiles?: readonly ts.SourceFile[]
-    ): void {
-      assert(
-        sourceFiles != null,
-        `Unexpected emit of "${fileName}" which isn't part of a program.`
-      );
-      assert(state.host);
-      if (!state.bundle) {
-        assert(sourceFiles.length === 1);
-        cache(
-          sourceFiles[0].fileName,
-          fileName,
-          data,
-          state.host.getCompilationSettings().checkJs
-        );
-      } else {
-        // if the fileName is set to an internal value, just noop, this is
-        // used in the Rust unit tests.
-        if (state.outFile && state.outFile.startsWith(OUT_DIR)) {
-          return;
-        }
-        // we only support single root names for bundles
-        assert(
-          state.rootNames.length === 1,
-          `Only one root name supported.  Got "${JSON.stringify(
-            state.rootNames
-          )}"`
-        );
-        // this enriches the string with the loader and re-exports the
-        // exports of the root module
-        const content = buildBundle(state.rootNames[0], data, sourceFiles);
-        if (state.outFile) {
-          const encodedData = encoder.encode(content);
-          console.warn(`Emitting bundle to "${state.outFile}"`);
-          writeFileSync(state.outFile, encodedData);
-          console.warn(`${humanFileSize(encodedData.length)} emitted.`);
-        } else {
-          console.log(content);
-        }
-      }
-    };
-  }
+// TODO(bartlomieju): probably could be defined inline?
+export function createBundleWriteFile(
+  state: WriteFileState
+): WriteFileCallback {
+  return function writeFile(
+    _fileName: string,
+    data: string,
+    sourceFiles?: readonly ts.SourceFile[]
+  ): void {
+    assert(sourceFiles != null);
+    assert(state.host);
+    assert(state.emitMap);
+    assert(state.bundle);
+    // we only support single root names for bundles
+    assert(state.rootNames.length === 1);
+    state.bundleOutput = buildBundle(state.rootNames[0], data, sourceFiles);
+  };
+}
 
+// TODO(bartlomieju): probably could be defined inline?
+export function createCompileWriteFile(
+  state: WriteFileState
+): WriteFileCallback {
   return function writeFile(
     fileName: string,
     data: string,
@@ -127,24 +76,12 @@ export function createWriteFile(state: WriteFileState): WriteFileCallback {
     assert(sourceFiles != null);
     assert(state.host);
     assert(state.emitMap);
-    if (!state.bundle) {
-      assert(sourceFiles.length === 1);
-      state.emitMap[fileName] = data;
-      // we only want to cache the compiler output if we are resolving
-      // modules externally
-      if (!state.sources) {
-        cache(
-          sourceFiles[0].fileName,
-          fileName,
-          data,
-          state.host.getCompilationSettings().checkJs
-        );
-      }
-    } else {
-      // we only support single root names for bundles
-      assert(state.rootNames.length === 1);
-      state.emitBundle = buildBundle(state.rootNames[0], data, sourceFiles);
-    }
+    assert(!state.bundle);
+    assert(sourceFiles.length === 1);
+    state.emitMap[fileName] = {
+      filename: sourceFiles[0].fileName,
+      contents: data,
+    };
   };
 }
 
@@ -380,20 +317,6 @@ export function commonPath(paths: string[], sep = "/"): string {
   return prefix.endsWith(sep) ? prefix : `${prefix}${sep}`;
 }
 
-function humanFileSize(bytes: number): string {
-  const thresh = 1000;
-  if (Math.abs(bytes) < thresh) {
-    return bytes + " B";
-  }
-  const units = ["kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"];
-  let u = -1;
-  do {
-    bytes /= thresh;
-    ++u;
-  } while (Math.abs(bytes) >= thresh && u < units.length - 1);
-  return `${bytes.toFixed(1)} ${units[u]}`;
-}
-
 // @internal
 export function base64ToUint8Array(data: string): Uint8Array {
   const binString = atob(data);