refactor: cleanup compiler pipeline (#2686)

* remove fetch_source_file_and_maybe_compile_async and 
  replace it with State.fetch_compiled_module

* remove SourceFile.js_source()

* introduce CompiledModule which is basically the same as
  deno::SourceInfo and represents arbitrary file that has been 
  compiled to JS module

* introduce //cli/compilers module containing all compilers

* introduce JsCompiler which is a no-op compiler
  - output is the same as input, no compilation takes place
  - it is used for MediaType::JavaScript and MediaType::Unknown

* introduce JsonCompiler that wraps JSON in default export

* support JS-to-JS compilation using checkJs

1 files changed, 0 insertions, 699 deletions

diff --git a/cli/compiler.rs b/cli/compiler.rs
deleted file mode 100644
index f90337d02..000000000
--- a/cli/compiler.rs
+++ /dev/null
@@ -1,699 +0,0 @@
-// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
-use crate::diagnostics::Diagnostic;
-use crate::disk_cache::DiskCache;
-use crate::file_fetcher::SourceFile;
-use crate::file_fetcher::SourceFileFetcher;
-use crate::msg;
-use crate::resources;
-use crate::source_maps::SourceMapGetter;
-use crate::startup_data;
-use crate::state::*;
-use crate::version;
-use crate::worker::Worker;
-use deno::Buf;
-use deno::ErrBox;
-use deno::ModuleSpecifier;
-use futures::future::Either;
-use futures::Future;
-use futures::Stream;
-use ring;
-use std::collections::HashSet;
-use std::fmt::Write;
-use std::fs;
-use std::path::PathBuf;
-use std::str;
-use std::sync::atomic::Ordering;
-use std::sync::Mutex;
-use url::Url;
-
-/// Optional tuple which represents the state of the compiler
-/// configuration where the first is canonical name for the configuration file
-/// and a vector of the bytes of the contents of the configuration file.
-type CompilerConfig = Option<(PathBuf, Vec<u8>)>;
-
-/// Information associated with compiled file in cache.
-/// Includes source code path and state hash.
-/// version_hash is used to validate versions of the file
-/// and could be used to remove stale file in cache.
-pub struct CompiledFileMetadata {
-  pub source_path: PathBuf,
-  pub version_hash: String,
-}
-
-static SOURCE_PATH: &'static str = "source_path";
-static VERSION_HASH: &'static str = "version_hash";
-
-impl CompiledFileMetadata {
-  pub fn from_json_string(metadata_string: String) -> Option<Self> {
-    // TODO: use serde for deserialization
-    let maybe_metadata_json: serde_json::Result<serde_json::Value> =
-      serde_json::from_str(&metadata_string);
-
-    if let Ok(metadata_json) = maybe_metadata_json {
-      let source_path = metadata_json[SOURCE_PATH].as_str().map(PathBuf::from);
-      let version_hash = metadata_json[VERSION_HASH].as_str().map(String::from);
-
-      if source_path.is_none() || version_hash.is_none() {
-        return None;
-      }
-
-      return Some(CompiledFileMetadata {
-        source_path: source_path.unwrap(),
-        version_hash: version_hash.unwrap(),
-      });
-    }
-
-    None
-  }
-
-  pub fn to_json_string(self: &Self) -> Result<String, serde_json::Error> {
-    let mut value_map = serde_json::map::Map::new();
-
-    value_map.insert(SOURCE_PATH.to_owned(), json!(&self.source_path));
-    value_map.insert(VERSION_HASH.to_string(), json!(&self.version_hash));
-    serde_json::to_string(&value_map)
-  }
-}
-/// Creates the JSON message send to compiler.ts's onmessage.
-fn req(
-  root_names: Vec<String>,
-  compiler_config: CompilerConfig,
-  bundle: Option<String>,
-) -> Buf {
-  let j = if let Some((config_path, config_data)) = compiler_config {
-    json!({
-      "rootNames": root_names,
-      "bundle": bundle,
-      "configPath": config_path,
-      "config": str::from_utf8(&config_data).unwrap(),
-    })
-  } else {
-    json!({
-      "rootNames": root_names,
-      "bundle": bundle,
-    })
-  };
-  j.to_string().into_boxed_str().into_boxed_bytes()
-}
-
-fn gen_hash(v: Vec<&[u8]>) -> String {
-  let mut ctx = ring::digest::Context::new(&ring::digest::SHA1);
-  for src in v.iter() {
-    ctx.update(src);
-  }
-  let digest = ctx.finish();
-  let mut out = String::new();
-  // TODO There must be a better way to do this...
-  for byte in digest.as_ref() {
-    write!(&mut out, "{:02x}", byte).unwrap();
-  }
-  out
-}
-
-/// Emit a SHA1 hash based on source code, deno version and TS config.
-/// Used to check if a recompilation for source code is needed.
-pub fn source_code_version_hash(
-  source_code: &[u8],
-  version: &str,
-  config_hash: &[u8],
-) -> String {
-  gen_hash(vec![source_code, version.as_bytes(), config_hash])
-}
-
-fn load_config_file(
-  config_path: Option<String>,
-) -> (Option<PathBuf>, Option<Vec<u8>>) {
-  // take the passed flag and resolve the file name relative to the cwd
-  let config_file = match &config_path {
-    Some(config_file_name) => {
-      debug!("Compiler config file: {}", config_file_name);
-      let cwd = std::env::current_dir().unwrap();
-      Some(cwd.join(config_file_name))
-    }
-    _ => None,
-  };
-
-  // Convert the PathBuf to a canonicalized string.  This is needed by the
-  // compiler to properly deal with the configuration.
-  let config_path = match &config_file {
-    Some(config_file) => Some(config_file.canonicalize().unwrap().to_owned()),
-    _ => None,
-  };
-
-  // Load the contents of the configuration file
-  let config = match &config_file {
-    Some(config_file) => {
-      debug!("Attempt to load config: {}", config_file.to_str().unwrap());
-      match fs::read(&config_file) {
-        Ok(config_data) => Some(config_data.to_owned()),
-        _ => panic!(
-          "Error retrieving compiler config file at \"{}\"",
-          config_file.to_str().unwrap()
-        ),
-      }
-    }
-    _ => None,
-  };
-
-  (config_path, config)
-}
-
-pub struct TsCompiler {
-  pub file_fetcher: SourceFileFetcher,
-  pub config: CompilerConfig,
-  pub config_hash: Vec<u8>,
-  pub disk_cache: DiskCache,
-  /// Set of all URLs that have been compiled. This prevents double
-  /// compilation of module.
-  pub compiled: Mutex<HashSet<Url>>,
-  /// This setting is controlled by `--reload` flag. Unless the flag
-  /// is provided disk cache is used.
-  pub use_disk_cache: bool,
-}
-
-impl TsCompiler {
-  pub fn new(
-    file_fetcher: SourceFileFetcher,
-    disk_cache: DiskCache,
-    use_disk_cache: bool,
-    config_path: Option<String>,
-  ) -> Self {
-    let compiler_config = match load_config_file(config_path) {
-      (Some(config_path), Some(config)) => Some((config_path, config.to_vec())),
-      _ => None,
-    };
-
-    let config_bytes = match &compiler_config {
-      Some((_, config)) => config.clone(),
-      _ => b"".to_vec(),
-    };
-
-    Self {
-      file_fetcher,
-      disk_cache,
-      config: compiler_config,
-      config_hash: config_bytes,
-      compiled: Mutex::new(HashSet::new()),
-      use_disk_cache,
-    }
-  }
-
-  /// Create a new V8 worker with snapshot of TS compiler and setup compiler's runtime.
-  fn setup_worker(state: ThreadSafeState) -> Worker {
-    // Count how many times we start the compiler worker.
-    state.metrics.compiler_starts.fetch_add(1, Ordering::SeqCst);
-
-    let mut worker = Worker::new(
-      "TS".to_string(),
-      startup_data::compiler_isolate_init(),
-      // TODO(ry) Maybe we should use a separate state for the compiler.
-      // as was done previously.
-      state.clone(),
-    );
-    worker.execute("denoMain()").unwrap();
-    worker.execute("workerMain()").unwrap();
-    worker.execute("compilerMain()").unwrap();
-    worker
-  }
-
-  pub fn bundle_async(
-    self: &Self,
-    state: ThreadSafeState,
-    module_name: String,
-    out_file: String,
-  ) -> impl Future<Item = (), Error = ErrBox> {
-    debug!(
-      "Invoking the compiler to bundle. module_name: {}",
-      module_name
-    );
-
-    let root_names = vec![module_name.clone()];
-    let req_msg = req(root_names, self.config.clone(), Some(out_file));
-
-    let worker = TsCompiler::setup_worker(state.clone());
-    let resource = worker.state.resource.clone();
-    let compiler_rid = resource.rid;
-    let first_msg_fut =
-      resources::post_message_to_worker(compiler_rid, req_msg)
-        .then(move |_| worker)
-        .then(move |result| {
-          if let Err(err) = result {
-            // TODO(ry) Need to forward the error instead of exiting.
-            eprintln!("{}", err.to_string());
-            std::process::exit(1);
-          }
-          debug!("Sent message to worker");
-          let stream_future =
-            resources::get_message_stream_from_worker(compiler_rid)
-              .into_future();
-          stream_future.map(|(f, _rest)| f).map_err(|(f, _rest)| f)
-        });
-
-    first_msg_fut.map_err(|_| panic!("not handled")).and_then(
-      move |maybe_msg: Option<Buf>| {
-        debug!("Received message from worker");
-
-        if let Some(msg) = maybe_msg {
-          let json_str = std::str::from_utf8(&msg).unwrap();
-          debug!("Message: {}", json_str);
-          if let Some(diagnostics) = Diagnostic::from_emit_result(json_str) {
-            return Err(ErrBox::from(diagnostics));
-          }
-        }
-
-        Ok(())
-      },
-    )
-  }
-
-  /// Mark given module URL as compiled to avoid multiple compilations of same module
-  /// in single run.
-  fn mark_compiled(&self, url: &Url) {
-    let mut c = self.compiled.lock().unwrap();
-    c.insert(url.clone());
-  }
-
-  /// Check if given module URL has already been compiled and can be fetched directly from disk.
-  fn has_compiled(&self, url: &Url) -> bool {
-    let c = self.compiled.lock().unwrap();
-    c.contains(url)
-  }
-
-  /// Asynchronously compile module and all it's dependencies.
-  ///
-  /// This method compiled every module at most once.
-  ///
-  /// If `--reload` flag was provided then compiler will not on-disk cache and force recompilation.
-  ///
-  /// If compilation is required then new V8 worker is spawned with fresh TS compiler.
-  pub fn compile_async(
-    self: &Self,
-    state: ThreadSafeState,
-    source_file: &SourceFile,
-  ) -> impl Future<Item = SourceFile, Error = ErrBox> {
-    // TODO: maybe fetching of original SourceFile should be done here?
-
-    if source_file.media_type != msg::MediaType::TypeScript {
-      return Either::A(futures::future::ok(source_file.clone()));
-    }
-
-    if self.has_compiled(&source_file.url) {
-      match self.get_compiled_source_file(&source_file) {
-        Ok(compiled_module) => {
-          return Either::A(futures::future::ok(compiled_module));
-        }
-        Err(err) => {
-          return Either::A(futures::future::err(err));
-        }
-      }
-    }
-
-    if self.use_disk_cache {
-      // Try to load cached version:
-      // 1. check if there's 'meta' file
-      if let Some(metadata) = self.get_metadata(&source_file.url) {
-        // 2. compare version hashes
-        // TODO: it would probably be good idea to make it method implemented on SourceFile
-        let version_hash_to_validate = source_code_version_hash(
-          &source_file.source_code,
-          version::DENO,
-          &self.config_hash,
-        );
-
-        if metadata.version_hash == version_hash_to_validate {
-          debug!("load_cache metadata version hash match");
-          if let Ok(compiled_module) =
-            self.get_compiled_source_file(&source_file)
-          {
-            debug!(
-              "found cached compiled module: {:?}",
-              compiled_module.clone().filename
-            );
-            // TODO: store in in-process cache for subsequent access
-            return Either::A(futures::future::ok(compiled_module));
-          }
-        }
-      }
-    }
-
-    let source_file_ = source_file.clone();
-
-    debug!(">>>>> compile_sync START");
-    let module_url = source_file.url.clone();
-
-    debug!(
-      "Running rust part of compile_sync, module specifier: {}",
-      &source_file.url
-    );
-
-    let root_names = vec![module_url.to_string()];
-    let req_msg = req(root_names, self.config.clone(), None);
-
-    let worker = TsCompiler::setup_worker(state.clone());
-    let compiling_job = state.progress.add("Compile", &module_url.to_string());
-    let state_ = state.clone();
-
-    let resource = worker.state.resource.clone();
-    let compiler_rid = resource.rid;
-    let first_msg_fut =
-      resources::post_message_to_worker(compiler_rid, req_msg)
-        .then(move |_| worker)
-        .then(move |result| {
-          if let Err(err) = result {
-            // TODO(ry) Need to forward the error instead of exiting.
-            eprintln!("{}", err.to_string());
-            std::process::exit(1);
-          }
-          debug!("Sent message to worker");
-          let stream_future =
-            resources::get_message_stream_from_worker(compiler_rid)
-              .into_future();
-          stream_future.map(|(f, _rest)| f).map_err(|(f, _rest)| f)
-        });
-
-    let fut = first_msg_fut
-      .map_err(|_| panic!("not handled"))
-      .and_then(move |maybe_msg: Option<Buf>| {
-        debug!("Received message from worker");
-
-        if let Some(msg) = maybe_msg {
-          let json_str = std::str::from_utf8(&msg).unwrap();
-          debug!("Message: {}", json_str);
-          if let Some(diagnostics) = Diagnostic::from_emit_result(json_str) {
-            return Err(ErrBox::from(diagnostics));
-          }
-        }
-
-        Ok(())
-      }).and_then(move |_| {
-        // if we are this far it means compilation was successful and we can
-        // load compiled filed from disk
-        // TODO: can this be somehow called using `self.`?
-        state_
-          .ts_compiler
-          .get_compiled_source_file(&source_file_)
-          .map_err(|e| {
-            // TODO: this situation shouldn't happen
-            panic!("Expected to find compiled file: {}", e)
-          })
-      }).and_then(move |source_file_after_compile| {
-        // Explicit drop to keep reference alive until future completes.
-        drop(compiling_job);
-
-        Ok(source_file_after_compile)
-      }).then(move |r| {
-        debug!(">>>>> compile_sync END");
-        // TODO(ry) do this in worker's destructor.
-        // resource.close();
-        r
-      });
-
-    Either::B(fut)
-  }
-
-  /// Get associated `CompiledFileMetadata` for given module if it exists.
-  pub fn get_metadata(self: &Self, url: &Url) -> Option<CompiledFileMetadata> {
-    // Try to load cached version:
-    // 1. check if there's 'meta' file
-    let cache_key = self
-      .disk_cache
-      .get_cache_filename_with_extension(url, "meta");
-    if let Ok(metadata_bytes) = self.disk_cache.get(&cache_key) {
-      if let Ok(metadata) = std::str::from_utf8(&metadata_bytes) {
-        if let Some(read_metadata) =
-          CompiledFileMetadata::from_json_string(metadata.to_string())
-        {
-          return Some(read_metadata);
-        }
-      }
-    }
-
-    None
-  }
-
-  /// Return compiled JS file for given TS module.
-  // TODO: ideally we shouldn't construct SourceFile by hand, but it should be delegated to
-  // SourceFileFetcher
-  pub fn get_compiled_source_file(
-    self: &Self,
-    source_file: &SourceFile,
-  ) -> Result<SourceFile, ErrBox> {
-    let cache_key = self
-      .disk_cache
-      .get_cache_filename_with_extension(&source_file.url, "js");
-    let compiled_code = self.disk_cache.get(&cache_key)?;
-    let compiled_code_filename = self.disk_cache.location.join(cache_key);
-    debug!("compiled filename: {:?}", compiled_code_filename);
-
-    let compiled_module = SourceFile {
-      url: source_file.url.clone(),
-      filename: compiled_code_filename,
-      media_type: msg::MediaType::JavaScript,
-      source_code: compiled_code,
-    };
-
-    Ok(compiled_module)
-  }
-
-  /// Save compiled JS file for given TS module to on-disk cache.
-  ///
-  /// Along compiled file a special metadata file is saved as well containing
-  /// hash that can be validated to avoid unnecessary recompilation.
-  fn cache_compiled_file(
-    self: &Self,
-    module_specifier: &ModuleSpecifier,
-    contents: &str,
-  ) -> std::io::Result<()> {
-    let js_key = self
-      .disk_cache
-      .get_cache_filename_with_extension(module_specifier.as_url(), "js");
-    self
-      .disk_cache
-      .set(&js_key, contents.as_bytes())
-      .and_then(|_| {
-        self.mark_compiled(module_specifier.as_url());
-
-        let source_file = self
-          .file_fetcher
-          .fetch_source_file(&module_specifier)
-          .expect("Source file not found");
-
-        let version_hash = source_code_version_hash(
-          &source_file.source_code,
-          version::DENO,
-          &self.config_hash,
-        );
-
-        let compiled_file_metadata = CompiledFileMetadata {
-          source_path: source_file.filename.to_owned(),
-          version_hash,
-        };
-        let meta_key = self
-          .disk_cache
-          .get_cache_filename_with_extension(module_specifier.as_url(), "meta");
-        self.disk_cache.set(
-          &meta_key,
-          compiled_file_metadata.to_json_string()?.as_bytes(),
-        )
-      })
-  }
-
-  /// Return associated source map file for given TS module.
-  // TODO: ideally we shouldn't construct SourceFile by hand, but it should be delegated to
-  // SourceFileFetcher
-  pub fn get_source_map_file(
-    self: &Self,
-    module_specifier: &ModuleSpecifier,
-  ) -> Result<SourceFile, ErrBox> {
-    let cache_key = self
-      .disk_cache
-      .get_cache_filename_with_extension(module_specifier.as_url(), "js.map");
-    let source_code = self.disk_cache.get(&cache_key)?;
-    let source_map_filename = self.disk_cache.location.join(cache_key);
-    debug!("source map filename: {:?}", source_map_filename);
-
-    let source_map_file = SourceFile {
-      url: module_specifier.as_url().to_owned(),
-      filename: source_map_filename,
-      media_type: msg::MediaType::JavaScript,
-      source_code,
-    };
-
-    Ok(source_map_file)
-  }
-
-  /// Save source map file for given TS module to on-disk cache.
-  fn cache_source_map(
-    self: &Self,
-    module_specifier: &ModuleSpecifier,
-    contents: &str,
-  ) -> std::io::Result<()> {
-    let source_map_key = self
-      .disk_cache
-      .get_cache_filename_with_extension(module_specifier.as_url(), "js.map");
-    self.disk_cache.set(&source_map_key, contents.as_bytes())
-  }
-
-  /// This method is called by TS compiler via an "op".
-  pub fn cache_compiler_output(
-    self: &Self,
-    module_specifier: &ModuleSpecifier,
-    extension: &str,
-    contents: &str,
-  ) -> std::io::Result<()> {
-    match extension {
-      ".map" => self.cache_source_map(module_specifier, contents),
-      ".js" => self.cache_compiled_file(module_specifier, contents),
-      _ => unreachable!(),
-    }
-  }
-}
-
-impl SourceMapGetter for TsCompiler {
-  fn get_source_map(&self, script_name: &str) -> Option<Vec<u8>> {
-    self
-      .try_to_resolve_and_get_source_map(script_name)
-      .and_then(|out| Some(out.source_code))
-  }
-
-  fn get_source_line(&self, script_name: &str, line: usize) -> Option<String> {
-    self
-      .try_resolve_and_get_source_file(script_name)
-      .and_then(|out| {
-        str::from_utf8(&out.source_code).ok().and_then(|v| {
-          let lines: Vec<&str> = v.lines().collect();
-          assert!(lines.len() > line);
-          Some(lines[line].to_string())
-        })
-      })
-  }
-}
-
-// `SourceMapGetter` related methods
-impl TsCompiler {
-  fn try_to_resolve(self: &Self, script_name: &str) -> Option<ModuleSpecifier> {
-    // if `script_name` can't be resolved to ModuleSpecifier it's probably internal
-    // script (like `gen/cli/bundle/compiler.js`) so we won't be
-    // able to get source for it anyway
-    ModuleSpecifier::resolve_url(script_name).ok()
-  }
-
-  fn try_resolve_and_get_source_file(
-    &self,
-    script_name: &str,
-  ) -> Option<SourceFile> {
-    if let Some(module_specifier) = self.try_to_resolve(script_name) {
-      return match self.file_fetcher.fetch_source_file(&module_specifier) {
-        Ok(out) => Some(out),
-        Err(_) => None,
-      };
-    }
-
-    None
-  }
-
-  fn try_to_resolve_and_get_source_map(
-    &self,
-    script_name: &str,
-  ) -> Option<SourceFile> {
-    if let Some(module_specifier) = self.try_to_resolve(script_name) {
-      return match self.get_source_map_file(&module_specifier) {
-        Ok(out) => Some(out),
-        Err(_) => None,
-      };
-    }
-
-    None
-  }
-}
-
-#[cfg(test)]
-mod tests {
-  use super::*;
-  use crate::tokio_util;
-  use deno::ModuleSpecifier;
-  use std::path::PathBuf;
-
-  impl TsCompiler {
-    fn compile_sync(
-      self: &Self,
-      state: ThreadSafeState,
-      source_file: &SourceFile,
-    ) -> Result<SourceFile, ErrBox> {
-      tokio_util::block_on(self.compile_async(state, source_file))
-    }
-  }
-
-  #[test]
-  fn test_compile_sync() {
-    tokio_util::init(|| {
-      let specifier =
-        ModuleSpecifier::resolve_url_or_path("./tests/002_hello.ts").unwrap();
-
-      let mut out = SourceFile {
-        url: specifier.as_url().clone(),
-        filename: PathBuf::from("/tests/002_hello.ts"),
-        media_type: msg::MediaType::TypeScript,
-        source_code: include_bytes!("../tests/002_hello.ts").to_vec(),
-      };
-
-      let mock_state = ThreadSafeState::mock(vec![
-        String::from("./deno"),
-        String::from("hello.js"),
-      ]);
-      out = mock_state
-        .ts_compiler
-        .compile_sync(mock_state.clone(), &out)
-        .unwrap();
-      assert!(
-        out
-          .source_code
-          .starts_with("console.log(\"Hello World\");".as_bytes())
-      );
-    })
-  }
-
-  #[test]
-  fn test_bundle_async() {
-    let specifier = "./tests/002_hello.ts";
-    use deno::ModuleSpecifier;
-    let module_name = ModuleSpecifier::resolve_url_or_path(specifier)
-      .unwrap()
-      .to_string();
-
-    let state = ThreadSafeState::mock(vec![
-      String::from("./deno"),
-      String::from("./tests/002_hello.ts"),
-      String::from("$deno$/bundle.js"),
-    ]);
-    let out = state.ts_compiler.bundle_async(
-      state.clone(),
-      module_name,
-      String::from("$deno$/bundle.js"),
-    );
-    assert!(tokio_util::block_on(out).is_ok());
-  }
-
-  #[test]
-  fn test_source_code_version_hash() {
-    assert_eq!(
-      "08574f9cdeb94fd3fb9cdc7a20d086daeeb42bca",
-      source_code_version_hash(b"1+2", "0.4.0", b"{}")
-    );
-    // Different source_code should result in different hash.
-    assert_eq!(
-      "d8abe2ead44c3ff8650a2855bf1b18e559addd06",
-      source_code_version_hash(b"1", "0.4.0", b"{}")
-    );
-    // Different version should result in different hash.
-    assert_eq!(
-      "d6feffc5024d765d22c94977b4fe5975b59d6367",
-      source_code_version_hash(b"1", "0.1.0", b"{}")
-    );
-    // Different config should result in different hash.
-    assert_eq!(
-      "3b35db249b26a27decd68686f073a58266b2aec2",
-      source_code_version_hash(b"1", "0.4.0", b"{\"compilerOptions\": {}}")
-    );
-  }
-}