refactor: reorganize TS compiler (#5603)

5 files changed, 187 insertions, 199 deletions

diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs
index cbfb340dd..375142e07 100644
--- a/cli/file_fetcher.rs
+++ b/cli/file_fetcher.rs
@@ -535,7 +535,7 @@ impl SourceFileFetcher {
   }
 }
 
-fn map_file_extension(path: &Path) -> msg::MediaType {
+pub fn map_file_extension(path: &Path) -> msg::MediaType {
   match path.extension() {
     None => msg::MediaType::Unknown,
     Some(os_str) => match os_str.to_str() {
diff --git a/cli/js/compiler.ts b/cli/js/compiler.ts
index fa4a7526e..a1189dc20 100644
--- a/cli/js/compiler.ts
+++ b/cli/js/compiler.ts
@@ -18,20 +18,15 @@ import { Diagnostic, DiagnosticItem } from "./diagnostics.ts";
 import { fromTypeScriptDiagnostic } from "./diagnostics_util.ts";
 import { TranspileOnlyResult } from "./ops/runtime_compiler.ts";
 import { bootstrapWorkerRuntime } from "./runtime_worker.ts";
-import { assert } from "./util.ts";
-import * as util from "./util.ts";
-import { TextDecoder, TextEncoder } from "./web/text_encoding.ts";
+import { assert, log, notImplemented } from "./util.ts";
 import { core } from "./core.ts";
 
-const encoder = new TextEncoder();
-const decoder = new TextDecoder();
-
+// We really don't want to depend on JSON dispatch during snapshotting, so
+// this op exchanges strings with Rust as raw byte arrays.
 function getAsset(name: string): string {
   const opId = core.ops()["op_fetch_asset"];
-  // We really don't want to depend on JSON dispatch during snapshotting, so
-  // this op exchanges strings with Rust as raw byte arrays.
-  const sourceCodeBytes = core.dispatch(opId, encoder.encode(name));
-  return decoder.decode(sourceCodeBytes!);
+  const sourceCodeBytes = core.dispatch(opId, core.encode(name));
+  return core.decode(sourceCodeBytes!);
 }
 
 // Constants used by `normalizeString` and `resolvePath`
@@ -194,18 +189,6 @@ function getExtension(fileName: string, mediaType: MediaType): ts.Extension {
   }
 }
 
-/** Because we support providing types for JS files as well as X-TypeScript-Types
- * header we might be feeding TS compiler with different files than import specifiers
- * suggest. To accomplish that we keep track of two different specifiers:
- *  - original - the one in import statement (import "./foo.js")
- *  - mapped - if there's no type directive it's the same as original, otherwise
- *             it's unresolved specifier for type directive (/// @deno-types="./foo.d.ts")
- */
-interface SourceFileSpecifierMap {
-  original: string;
-  mapped: string;
-}
-
 /** A global cache of module source files that have been loaded.
  * This cache will be rewritten to be populated on compiler startup
  * with files provided from Rust in request message.
@@ -329,7 +312,7 @@ class Host implements ts.CompilerHost {
     path: string,
     configurationText: string
   ): ConfigureResponse {
-    util.log("compiler::host.configure", path);
+    log("compiler::host.configure", path);
     assert(configurationText);
     const { config, error } = ts.parseConfigFileTextToJson(
       path,
@@ -367,7 +350,7 @@ class Host implements ts.CompilerHost {
   /* TypeScript CompilerHost APIs */
 
   fileExists(_fileName: string): boolean {
-    return util.notImplemented();
+    return notImplemented();
   }
 
   getCanonicalFileName(fileName: string): string {
@@ -375,7 +358,7 @@ class Host implements ts.CompilerHost {
   }
 
   getCompilationSettings(): ts.CompilerOptions {
-    util.log("compiler::host.getCompilationSettings()");
+    log("compiler::host.getCompilationSettings()");
     return this.#options;
   }
 
@@ -384,7 +367,7 @@ class Host implements ts.CompilerHost {
   }
 
   getDefaultLibFileName(_options: ts.CompilerOptions): string {
-    util.log("compiler::host.getDefaultLibFileName()");
+    log("compiler::host.getDefaultLibFileName()");
     switch (this.#target) {
       case CompilerHostTarget.Main:
       case CompilerHostTarget.Runtime:
@@ -404,7 +387,7 @@ class Host implements ts.CompilerHost {
     onError?: (message: string) => void,
     shouldCreateNewSourceFile?: boolean
   ): ts.SourceFile | undefined {
-    util.log("compiler::host.getSourceFile", fileName);
+    log("compiler::host.getSourceFile", fileName);
     try {
       assert(!shouldCreateNewSourceFile);
       const sourceFile = fileName.startsWith(ASSETS)
@@ -436,21 +419,21 @@ class Host implements ts.CompilerHost {
   }
 
   readFile(_fileName: string): string | undefined {
-    return util.notImplemented();
+    return notImplemented();
   }
 
   resolveModuleNames(
     moduleNames: string[],
     containingFile: string
   ): Array<ts.ResolvedModuleFull | undefined> {
-    util.log("compiler::host.resolveModuleNames", {
+    log("compiler::host.resolveModuleNames", {
       moduleNames,
       containingFile,
     });
     return moduleNames.map((specifier) => {
       const maybeUrl = SourceFile.getResolvedUrl(specifier, containingFile);
 
-      util.log("compiler::host.resolveModuleNames maybeUrl", {
+      log("compiler::host.resolveModuleNames maybeUrl", {
         specifier,
         containingFile,
         maybeUrl,
@@ -488,7 +471,7 @@ class Host implements ts.CompilerHost {
     _onError?: (message: string) => void,
     sourceFiles?: readonly ts.SourceFile[]
   ): void {
-    util.log("compiler::host.writeFile", fileName);
+    log("compiler::host.writeFile", fileName);
     this.#writeFile(fileName, data, sourceFiles);
   }
 }
@@ -546,30 +529,6 @@ const _TS_SNAPSHOT_PROGRAM = ts.createProgram({
 // This function is called only during snapshotting process
 const SYSTEM_LOADER = getAsset("system_loader.js");
 
-function getMediaType(filename: string): MediaType {
-  const maybeExtension = /\.([a-zA-Z]+)$/.exec(filename);
-  if (!maybeExtension) {
-    util.log(`!!! Could not identify valid extension: "${filename}"`);
-    return MediaType.Unknown;
-  }
-  const [, extension] = maybeExtension;
-  switch (extension.toLowerCase()) {
-    case "js":
-      return MediaType.JavaScript;
-    case "jsx":
-      return MediaType.JSX;
-    case "ts":
-      return MediaType.TypeScript;
-    case "tsx":
-      return MediaType.TSX;
-    case "wasm":
-      return MediaType.Wasm;
-    default:
-      util.log(`!!! Unknown extension: "${extension}"`);
-      return MediaType.Unknown;
-  }
-}
-
 function buildLocalSourceFileCache(
   sourceFileMap: Record<string, SourceFileMapEntry>
 ): void {
@@ -578,7 +537,7 @@ function buildLocalSourceFileCache(
     SourceFile.addToCache({
       url: entry.url,
       filename: entry.url,
-      mediaType: getMediaType(entry.url),
+      mediaType: entry.mediaType,
       sourceCode: entry.sourceCode,
     });
 
@@ -673,7 +632,7 @@ function buildSourceFileCache(
   }
 }
 
-interface EmmitedSource {
+interface EmittedSource {
   // original filename
   filename: string;
   // compiled contents
@@ -692,7 +651,7 @@ interface WriteFileState {
   bundleOutput?: string;
   host?: Host;
   rootNames: string[];
-  emitMap?: Record<string, EmmitedSource>;
+  emitMap?: Record<string, EmittedSource>;
   sources?: Record<string, string>;
 }
 
@@ -1115,13 +1074,13 @@ type CompilerRequest =
   | CompilerRequestRuntimeTranspile;
 
 interface CompileResult {
-  emitMap?: Record<string, EmmitedSource>;
+  emitMap?: Record<string, EmittedSource>;
   bundleOutput?: string;
   diagnostics: Diagnostic;
 }
 
 interface RuntimeCompileResult {
-  emitMap: Record<string, EmmitedSource>;
+  emitMap: Record<string, EmittedSource>;
   diagnostics: DiagnosticItem[];
 }
 
@@ -1141,7 +1100,7 @@ function compile(request: CompilerRequestCompile): CompileResult {
     cwd,
     sourceFileMap,
   } = request;
-  util.log(">>> compile start", {
+  log(">>> compile start", {
     rootNames,
     type: CompilerRequestType[request.type],
   });
@@ -1221,7 +1180,7 @@ function compile(request: CompilerRequestCompile): CompileResult {
     diagnostics: fromTypeScriptDiagnostic(diagnostics),
   };
 
-  util.log("<<< compile end", {
+  log("<<< compile end", {
     rootNames,
     type: CompilerRequestType[request.type],
   });
@@ -1241,7 +1200,7 @@ function runtimeCompile(
     sourceFileMap,
   } = request;
 
-  util.log(">>> runtime compile start", {
+  log(">>> runtime compile start", {
     rootNames,
     bundle,
   });
@@ -1312,7 +1271,7 @@ function runtimeCompile(
   assert(emitResult.emitSkipped === false, "Unexpected skip of the emit.");
 
   assert(state.emitMap);
-  util.log("<<< runtime compile finish", {
+  log("<<< runtime compile finish", {
     rootNames,
     bundle,
     emitMap: Object.keys(state.emitMap),
@@ -1385,7 +1344,7 @@ async function tsCompilerOnMessage({
       break;
     }
     default:
-      util.log(
+      log(
         `!!! unhandled CompilerRequestType: ${
           (request as CompilerRequest).type
         } (${CompilerRequestType[(request as CompilerRequest).type]})`
diff --git a/cli/main.rs b/cli/main.rs
index 5985a1040..68ba5e776 100644
--- a/cli/main.rs
+++ b/cli/main.rs
@@ -72,10 +72,12 @@ use crate::file_fetcher::SourceFile;
 use crate::file_fetcher::SourceFileFetcher;
 use crate::fs as deno_fs;
 use crate::global_state::GlobalState;
+use crate::import_map::ImportMap;
 use crate::msg::MediaType;
 use crate::op_error::OpError;
 use crate::ops::io::get_stdio;
 use crate::permissions::Permissions;
+use crate::state::exit_unstable;
 use crate::state::State;
 use crate::tsc::TargetLib;
 use crate::worker::MainWorker;
@@ -396,12 +398,31 @@ async fn bundle_command(
     module_name = ModuleSpecifier::from(u)
   }
 
-  let global_state = GlobalState::new(flags)?;
   debug!(">>>>> bundle START");
-  let bundle_result = global_state
-    .ts_compiler
-    .bundle(global_state.clone(), module_name, out_file)
-    .await;
+  let compiler_config = tsc::CompilerConfig::load(flags.config_path.clone())?;
+
+  let maybe_import_map = match flags.import_map_path.as_ref() {
+    None => None,
+    Some(file_path) => {
+      if !flags.unstable {
+        exit_unstable("--importmap")
+      }
+      Some(ImportMap::load(file_path)?)
+    }
+  };
+
+  let global_state = GlobalState::new(flags)?;
+
+  let bundle_result = tsc::bundle(
+    &global_state,
+    compiler_config,
+    module_name,
+    maybe_import_map,
+    out_file,
+    global_state.flags.unstable,
+  )
+  .await;
+
   debug!(">>>>> bundle END");
   bundle_result
 }
diff --git a/cli/module_graph.rs b/cli/module_graph.rs
index 23a9fe26b..21e575cfd 100644
--- a/cli/module_graph.rs
+++ b/cli/module_graph.rs
@@ -1,5 +1,6 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
 
+use crate::file_fetcher::map_file_extension;
 use crate::file_fetcher::SourceFile;
 use crate::file_fetcher::SourceFileFetcher;
 use crate::import_map::ImportMap;
@@ -19,6 +20,7 @@ use serde::Serialize;
 use serde::Serializer;
 use std::collections::HashMap;
 use std::hash::BuildHasher;
+use std::path::PathBuf;
 use std::pin::Pin;
 
 fn serialize_module_specifier<S>(
@@ -258,9 +260,9 @@ impl ModuleGraphLoader {
       ModuleGraphFile {
         specifier: specifier.to_string(),
         url: specifier.to_string(),
+        media_type: map_file_extension(&PathBuf::from(specifier.clone()))
+          as i32,
         filename: specifier,
-        // ignored, it's set in TS worker
-        media_type: MediaType::JavaScript as i32,
         source_code,
         imports,
         referenced_files,
diff --git a/cli/tsc.rs b/cli/tsc.rs
index 41eb7e43c..0bfcf6266 100644
--- a/cli/tsc.rs
+++ b/cli/tsc.rs
@@ -165,6 +165,33 @@ lazy_static! {
     Regex::new(r#""checkJs"\s*?:\s*?true"#).unwrap();
 }
 
+/// Create a new worker with snapshot of TS compiler and setup compiler's
+/// runtime.
+fn create_compiler_worker(
+  global_state: GlobalState,
+  permissions: Permissions,
+) -> CompilerWorker {
+  // TODO(bartlomieju): these $deno$ specifiers should be unified for all subcommands
+  // like 'eval', 'repl'
+  let entry_point =
+    ModuleSpecifier::resolve_url_or_path("./__$deno$ts_compiler.ts").unwrap();
+  let worker_state =
+    State::new(global_state.clone(), Some(permissions), entry_point, true)
+      .expect("Unable to create worker state");
+
+  // TODO(bartlomieju): this metric is never used anywhere
+  // Count how many times we start the compiler worker.
+  global_state.compiler_starts.fetch_add(1, Ordering::SeqCst);
+
+  let mut worker = CompilerWorker::new(
+    "TS".to_string(),
+    startup_data::compiler_isolate_init(),
+    worker_state,
+  );
+  worker.execute("bootstrap.tsCompilerRuntime()").unwrap();
+  worker
+}
+
 #[derive(Clone)]
 pub enum TargetLib {
   Main,
@@ -312,7 +339,7 @@ struct EmittedSource {
 #[serde(rename_all = "camelCase")]
 struct BundleResponse {
   diagnostics: Diagnostic,
-  bundle_output: String,
+  bundle_output: Option<String>,
 }
 
 #[derive(Deserialize)]
@@ -356,126 +383,6 @@ impl TsCompiler {
     })))
   }
 
-  /// Create a new V8 worker with snapshot of TS compiler and setup compiler's
-  /// runtime.
-  fn setup_worker(
-    global_state: GlobalState,
-    permissions: Permissions,
-  ) -> CompilerWorker {
-    let entry_point =
-      ModuleSpecifier::resolve_url_or_path("./__$deno$ts_compiler.ts").unwrap();
-    let worker_state =
-      State::new(global_state.clone(), Some(permissions), entry_point, true)
-        .expect("Unable to create worker state");
-
-    // Count how many times we start the compiler worker.
-    global_state.compiler_starts.fetch_add(1, Ordering::SeqCst);
-
-    let mut worker = CompilerWorker::new(
-      "TS".to_string(),
-      startup_data::compiler_isolate_init(),
-      worker_state,
-    );
-    worker.execute("bootstrap.tsCompilerRuntime()").unwrap();
-    worker
-  }
-
-  pub async fn bundle(
-    &self,
-    global_state: GlobalState,
-    module_specifier: ModuleSpecifier,
-    out_file: Option<PathBuf>,
-  ) -> Result<(), ErrBox> {
-    debug!(
-      "Invoking the compiler to bundle. module_name: {}",
-      module_specifier.to_string()
-    );
-    eprintln!("Bundling {}", module_specifier.to_string());
-
-    let import_map: Option<ImportMap> =
-      match global_state.flags.import_map_path.as_ref() {
-        None => None,
-        Some(file_path) => {
-          if !global_state.flags.unstable {
-            exit_unstable("--importmap")
-          }
-          Some(ImportMap::load(file_path)?)
-        }
-      };
-    let permissions = Permissions::allow_all();
-    let mut module_graph_loader = ModuleGraphLoader::new(
-      global_state.file_fetcher.clone(),
-      import_map,
-      permissions.clone(),
-      false,
-      true,
-    );
-    module_graph_loader.add_to_graph(&module_specifier).await?;
-    let module_graph = module_graph_loader.get_graph();
-    let module_graph_json =
-      serde_json::to_value(module_graph).expect("Failed to serialize data");
-
-    let root_names = vec![module_specifier.to_string()];
-    let bundle = true;
-    let target = "main";
-    let unstable = global_state.flags.unstable;
-    let compiler_config = self.config.clone();
-    let cwd = std::env::current_dir().unwrap();
-    let j = match (compiler_config.path, compiler_config.content) {
-      (Some(config_path), Some(config_data)) => json!({
-        "type": msg::CompilerRequestType::Compile as i32,
-        "target": target,
-        "rootNames": root_names,
-        "bundle": bundle,
-        "unstable": unstable,
-        "configPath": config_path,
-        "config": str::from_utf8(&config_data).unwrap(),
-        "cwd": cwd,
-        "sourceFileMap": module_graph_json,
-      }),
-      _ => json!({
-        "type": msg::CompilerRequestType::Compile as i32,
-        "target": target,
-        "rootNames": root_names,
-        "bundle": bundle,
-        "unstable": unstable,
-        "cwd": cwd,
-        "sourceFileMap": module_graph_json,
-      }),
-    };
-
-    let req_msg = j.to_string().into_boxed_str().into_boxed_bytes();
-
-    let msg =
-      execute_in_same_thread(global_state.clone(), permissions, req_msg)
-        .await?;
-    let json_str = std::str::from_utf8(&msg).unwrap();
-    debug!("Message: {}", json_str);
-
-    let bundle_response: BundleResponse = serde_json::from_str(json_str)?;
-
-    if !bundle_response.diagnostics.items.is_empty() {
-      return Err(ErrBox::from(bundle_response.diagnostics));
-    }
-
-    let output_string = fmt::format_text(&bundle_response.bundle_output)?;
-
-    if let Some(out_file_) = out_file.as_ref() {
-      eprintln!("Emitting bundle to {:?}", out_file_);
-
-      let output_bytes = output_string.as_bytes();
-      let output_len = output_bytes.len();
-
-      deno_fs::write_file(out_file_, output_bytes, 0o666)?;
-      // TODO(bartlomieju): add "humanFileSize" method
-      eprintln!("{} bytes emitted.", output_len);
-    } else {
-      println!("{}", output_string);
-    }
-
-    Ok(())
-  }
-
   /// Mark given module URL as compiled to avoid multiple compilations of same
   /// module in single run.
   fn mark_compiled(&self, url: &Url) {
@@ -914,7 +821,7 @@ async fn execute_in_same_thread(
   permissions: Permissions,
   req: Buf,
 ) -> Result<Buf, ErrBox> {
-  let mut worker = TsCompiler::setup_worker(global_state.clone(), permissions);
+  let mut worker = create_compiler_worker(global_state.clone(), permissions);
   let handle = worker.thread_safe_handle();
   handle.post_message(req)?;
 
@@ -943,6 +850,100 @@ async fn execute_in_same_thread(
   }
 }
 
+pub async fn bundle(
+  global_state: &GlobalState,
+  compiler_config: CompilerConfig,
+  module_specifier: ModuleSpecifier,
+  maybe_import_map: Option<ImportMap>,
+  out_file: Option<PathBuf>,
+  unstable: bool,
+) -> Result<(), ErrBox> {
+  debug!(
+    "Invoking the compiler to bundle. module_name: {}",
+    module_specifier.to_string()
+  );
+  eprintln!("Bundling {}", module_specifier.to_string());
+
+  let permissions = Permissions::allow_all();
+  let mut module_graph_loader = ModuleGraphLoader::new(
+    global_state.file_fetcher.clone(),
+    maybe_import_map,
+    permissions.clone(),
+    false,
+    true,
+  );
+  module_graph_loader.add_to_graph(&module_specifier).await?;
+  let module_graph = module_graph_loader.get_graph();
+  let module_graph_json =
+    serde_json::to_value(module_graph).expect("Failed to serialize data");
+
+  let root_names = vec![module_specifier.to_string()];
+  let bundle = true;
+  let target = "main";
+  let cwd = std::env::current_dir().unwrap();
+
+  // TODO(bartlomieju): this is non-sense; CompilerConfig's `path` and `content` should
+  // be optional
+  let j = match (compiler_config.path, compiler_config.content) {
+    (Some(config_path), Some(config_data)) => json!({
+      "type": msg::CompilerRequestType::Compile as i32,
+      "target": target,
+      "rootNames": root_names,
+      "bundle": bundle,
+      "unstable": unstable,
+      "configPath": config_path,
+      "config": str::from_utf8(&config_data).unwrap(),
+      "cwd": cwd,
+      "sourceFileMap": module_graph_json,
+    }),
+    _ => json!({
+      "type": msg::CompilerRequestType::Compile as i32,
+      "target": target,
+      "rootNames": root_names,
+      "bundle": bundle,
+      "unstable": unstable,
+      "cwd": cwd,
+      "sourceFileMap": module_graph_json,
+    }),
+  };
+
+  let req_msg = j.to_string().into_boxed_str().into_boxed_bytes();
+
+  let msg =
+    execute_in_same_thread(global_state.clone(), permissions, req_msg).await?;
+  let json_str = std::str::from_utf8(&msg).unwrap();
+  debug!("Message: {}", json_str);
+
+  let bundle_response: BundleResponse = serde_json::from_str(json_str)?;
+
+  if !bundle_response.diagnostics.items.is_empty() {
+    return Err(ErrBox::from(bundle_response.diagnostics));
+  }
+
+  assert!(bundle_response.bundle_output.is_some());
+  let output = bundle_response.bundle_output.unwrap();
+
+  // TODO(bartlomieju): the rest of this function should be handled
+  // in `main.rs` - it has nothing to do with TypeScript...
+  let output_string = fmt::format_text(&output)?;
+
+  if let Some(out_file_) = out_file.as_ref() {
+    eprintln!("Emitting bundle to {:?}", out_file_);
+
+    let output_bytes = output_string.as_bytes();
+    let output_len = output_bytes.len();
+
+    deno_fs::write_file(out_file_, output_bytes, 0o666)?;
+    // TODO(bartlomieju): do we really need to show this info? (it doesn't respect --quiet flag)
+    // TODO(bartlomieju): add "humanFileSize" method
+    eprintln!("{} bytes emitted.", output_len);
+  } else {
+    println!("{}", output_string);
+  }
+
+  Ok(())
+}
+
 /// This function is used by `Deno.compile()` and `Deno.bundle()` APIs.
 pub async fn runtime_compile<S: BuildHasher>(
   global_state: GlobalState,
@@ -1138,10 +1139,15 @@ mod tests {
       String::from("$deno$/bundle.js"),
     ]);
 
-    let result = state
-      .ts_compiler
-      .bundle(state.clone(), module_name, None)
-      .await;
+    let result = bundle(
+      &state,
+      CompilerConfig::load(None).unwrap(),
+      module_name,
+      None,
+      None,
+      false,
+    )
+    .await;
     assert!(result.is_ok());
   }