From 2e1ab8232156a23afd22834c1e707fb3403c0db6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Wed, 31 Jul 2019 19:16:03 +0200 Subject: refactor: cleanup compiler pipeline (#2686) * remove fetch_source_file_and_maybe_compile_async and replace it with State.fetch_compiled_module * remove SourceFile.js_source() * introduce CompiledModule which is basically the same as deno::SourceInfo and represents arbitrary file that has been compiled to JS module * introduce //cli/compilers module containing all compilers * introduce JsCompiler which is a no-op compiler - output is the same as input, no compilation takes place - it is used for MediaType::JavaScript and MediaType::Unknown * introduce JsonCompiler that wraps JSON in default export * support JS-to-JS compilation using checkJs --- cli/compiler.rs | 699 ------------------------------------- cli/compilers/js.rs | 25 ++ cli/compilers/json.rs | 26 ++ cli/compilers/mod.rs | 20 ++ cli/compilers/ts.rs | 740 ++++++++++++++++++++++++++++++++++++++++ cli/deno_dir.rs | 2 - cli/disk_cache.rs | 4 + cli/file_fetcher.rs | 21 -- cli/main.rs | 28 +- cli/state.rs | 115 ++++--- js/compiler.ts | 25 +- tests/038_checkjs.js | 6 + tests/038_checkjs.js.out | 15 + tests/038_checkjs.test | 5 + tests/038_checkjs.tsconfig.json | 5 + 15 files changed, 945 insertions(+), 791 deletions(-) delete mode 100644 cli/compiler.rs create mode 100644 cli/compilers/js.rs create mode 100644 cli/compilers/json.rs create mode 100644 cli/compilers/mod.rs create mode 100644 cli/compilers/ts.rs create mode 100644 tests/038_checkjs.js create mode 100644 tests/038_checkjs.js.out create mode 100644 tests/038_checkjs.test create mode 100644 tests/038_checkjs.tsconfig.json diff --git a/cli/compiler.rs b/cli/compiler.rs deleted file mode 100644 index f90337d02..000000000 --- a/cli/compiler.rs +++ /dev/null @@ -1,699 +0,0 @@ -// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. -use crate::diagnostics::Diagnostic; -use crate::disk_cache::DiskCache; -use crate::file_fetcher::SourceFile; -use crate::file_fetcher::SourceFileFetcher; -use crate::msg; -use crate::resources; -use crate::source_maps::SourceMapGetter; -use crate::startup_data; -use crate::state::*; -use crate::version; -use crate::worker::Worker; -use deno::Buf; -use deno::ErrBox; -use deno::ModuleSpecifier; -use futures::future::Either; -use futures::Future; -use futures::Stream; -use ring; -use std::collections::HashSet; -use std::fmt::Write; -use std::fs; -use std::path::PathBuf; -use std::str; -use std::sync::atomic::Ordering; -use std::sync::Mutex; -use url::Url; - -/// Optional tuple which represents the state of the compiler -/// configuration where the first is canonical name for the configuration file -/// and a vector of the bytes of the contents of the configuration file. -type CompilerConfig = Option<(PathBuf, Vec)>; - -/// Information associated with compiled file in cache. -/// Includes source code path and state hash. -/// version_hash is used to validate versions of the file -/// and could be used to remove stale file in cache. -pub struct CompiledFileMetadata { - pub source_path: PathBuf, - pub version_hash: String, -} - -static SOURCE_PATH: &'static str = "source_path"; -static VERSION_HASH: &'static str = "version_hash"; - -impl CompiledFileMetadata { - pub fn from_json_string(metadata_string: String) -> Option { - // TODO: use serde for deserialization - let maybe_metadata_json: serde_json::Result = - serde_json::from_str(&metadata_string); - - if let Ok(metadata_json) = maybe_metadata_json { - let source_path = metadata_json[SOURCE_PATH].as_str().map(PathBuf::from); - let version_hash = metadata_json[VERSION_HASH].as_str().map(String::from); - - if source_path.is_none() || version_hash.is_none() { - return None; - } - - return Some(CompiledFileMetadata { - source_path: source_path.unwrap(), - version_hash: version_hash.unwrap(), - }); - } - - None - } - - pub fn to_json_string(self: &Self) -> Result { - let mut value_map = serde_json::map::Map::new(); - - value_map.insert(SOURCE_PATH.to_owned(), json!(&self.source_path)); - value_map.insert(VERSION_HASH.to_string(), json!(&self.version_hash)); - serde_json::to_string(&value_map) - } -} -/// Creates the JSON message send to compiler.ts's onmessage. -fn req( - root_names: Vec, - compiler_config: CompilerConfig, - bundle: Option, -) -> Buf { - let j = if let Some((config_path, config_data)) = compiler_config { - json!({ - "rootNames": root_names, - "bundle": bundle, - "configPath": config_path, - "config": str::from_utf8(&config_data).unwrap(), - }) - } else { - json!({ - "rootNames": root_names, - "bundle": bundle, - }) - }; - j.to_string().into_boxed_str().into_boxed_bytes() -} - -fn gen_hash(v: Vec<&[u8]>) -> String { - let mut ctx = ring::digest::Context::new(&ring::digest::SHA1); - for src in v.iter() { - ctx.update(src); - } - let digest = ctx.finish(); - let mut out = String::new(); - // TODO There must be a better way to do this... - for byte in digest.as_ref() { - write!(&mut out, "{:02x}", byte).unwrap(); - } - out -} - -/// Emit a SHA1 hash based on source code, deno version and TS config. -/// Used to check if a recompilation for source code is needed. -pub fn source_code_version_hash( - source_code: &[u8], - version: &str, - config_hash: &[u8], -) -> String { - gen_hash(vec![source_code, version.as_bytes(), config_hash]) -} - -fn load_config_file( - config_path: Option, -) -> (Option, Option>) { - // take the passed flag and resolve the file name relative to the cwd - let config_file = match &config_path { - Some(config_file_name) => { - debug!("Compiler config file: {}", config_file_name); - let cwd = std::env::current_dir().unwrap(); - Some(cwd.join(config_file_name)) - } - _ => None, - }; - - // Convert the PathBuf to a canonicalized string. This is needed by the - // compiler to properly deal with the configuration. - let config_path = match &config_file { - Some(config_file) => Some(config_file.canonicalize().unwrap().to_owned()), - _ => None, - }; - - // Load the contents of the configuration file - let config = match &config_file { - Some(config_file) => { - debug!("Attempt to load config: {}", config_file.to_str().unwrap()); - match fs::read(&config_file) { - Ok(config_data) => Some(config_data.to_owned()), - _ => panic!( - "Error retrieving compiler config file at \"{}\"", - config_file.to_str().unwrap() - ), - } - } - _ => None, - }; - - (config_path, config) -} - -pub struct TsCompiler { - pub file_fetcher: SourceFileFetcher, - pub config: CompilerConfig, - pub config_hash: Vec, - pub disk_cache: DiskCache, - /// Set of all URLs that have been compiled. This prevents double - /// compilation of module. - pub compiled: Mutex>, - /// This setting is controlled by `--reload` flag. Unless the flag - /// is provided disk cache is used. - pub use_disk_cache: bool, -} - -impl TsCompiler { - pub fn new( - file_fetcher: SourceFileFetcher, - disk_cache: DiskCache, - use_disk_cache: bool, - config_path: Option, - ) -> Self { - let compiler_config = match load_config_file(config_path) { - (Some(config_path), Some(config)) => Some((config_path, config.to_vec())), - _ => None, - }; - - let config_bytes = match &compiler_config { - Some((_, config)) => config.clone(), - _ => b"".to_vec(), - }; - - Self { - file_fetcher, - disk_cache, - config: compiler_config, - config_hash: config_bytes, - compiled: Mutex::new(HashSet::new()), - use_disk_cache, - } - } - - /// Create a new V8 worker with snapshot of TS compiler and setup compiler's runtime. - fn setup_worker(state: ThreadSafeState) -> Worker { - // Count how many times we start the compiler worker. - state.metrics.compiler_starts.fetch_add(1, Ordering::SeqCst); - - let mut worker = Worker::new( - "TS".to_string(), - startup_data::compiler_isolate_init(), - // TODO(ry) Maybe we should use a separate state for the compiler. - // as was done previously. - state.clone(), - ); - worker.execute("denoMain()").unwrap(); - worker.execute("workerMain()").unwrap(); - worker.execute("compilerMain()").unwrap(); - worker - } - - pub fn bundle_async( - self: &Self, - state: ThreadSafeState, - module_name: String, - out_file: String, - ) -> impl Future { - debug!( - "Invoking the compiler to bundle. module_name: {}", - module_name - ); - - let root_names = vec![module_name.clone()]; - let req_msg = req(root_names, self.config.clone(), Some(out_file)); - - let worker = TsCompiler::setup_worker(state.clone()); - let resource = worker.state.resource.clone(); - let compiler_rid = resource.rid; - let first_msg_fut = - resources::post_message_to_worker(compiler_rid, req_msg) - .then(move |_| worker) - .then(move |result| { - if let Err(err) = result { - // TODO(ry) Need to forward the error instead of exiting. - eprintln!("{}", err.to_string()); - std::process::exit(1); - } - debug!("Sent message to worker"); - let stream_future = - resources::get_message_stream_from_worker(compiler_rid) - .into_future(); - stream_future.map(|(f, _rest)| f).map_err(|(f, _rest)| f) - }); - - first_msg_fut.map_err(|_| panic!("not handled")).and_then( - move |maybe_msg: Option| { - debug!("Received message from worker"); - - if let Some(msg) = maybe_msg { - let json_str = std::str::from_utf8(&msg).unwrap(); - debug!("Message: {}", json_str); - if let Some(diagnostics) = Diagnostic::from_emit_result(json_str) { - return Err(ErrBox::from(diagnostics)); - } - } - - Ok(()) - }, - ) - } - - /// Mark given module URL as compiled to avoid multiple compilations of same module - /// in single run. - fn mark_compiled(&self, url: &Url) { - let mut c = self.compiled.lock().unwrap(); - c.insert(url.clone()); - } - - /// Check if given module URL has already been compiled and can be fetched directly from disk. - fn has_compiled(&self, url: &Url) -> bool { - let c = self.compiled.lock().unwrap(); - c.contains(url) - } - - /// Asynchronously compile module and all it's dependencies. - /// - /// This method compiled every module at most once. - /// - /// If `--reload` flag was provided then compiler will not on-disk cache and force recompilation. - /// - /// If compilation is required then new V8 worker is spawned with fresh TS compiler. - pub fn compile_async( - self: &Self, - state: ThreadSafeState, - source_file: &SourceFile, - ) -> impl Future { - // TODO: maybe fetching of original SourceFile should be done here? - - if source_file.media_type != msg::MediaType::TypeScript { - return Either::A(futures::future::ok(source_file.clone())); - } - - if self.has_compiled(&source_file.url) { - match self.get_compiled_source_file(&source_file) { - Ok(compiled_module) => { - return Either::A(futures::future::ok(compiled_module)); - } - Err(err) => { - return Either::A(futures::future::err(err)); - } - } - } - - if self.use_disk_cache { - // Try to load cached version: - // 1. check if there's 'meta' file - if let Some(metadata) = self.get_metadata(&source_file.url) { - // 2. compare version hashes - // TODO: it would probably be good idea to make it method implemented on SourceFile - let version_hash_to_validate = source_code_version_hash( - &source_file.source_code, - version::DENO, - &self.config_hash, - ); - - if metadata.version_hash == version_hash_to_validate { - debug!("load_cache metadata version hash match"); - if let Ok(compiled_module) = - self.get_compiled_source_file(&source_file) - { - debug!( - "found cached compiled module: {:?}", - compiled_module.clone().filename - ); - // TODO: store in in-process cache for subsequent access - return Either::A(futures::future::ok(compiled_module)); - } - } - } - } - - let source_file_ = source_file.clone(); - - debug!(">>>>> compile_sync START"); - let module_url = source_file.url.clone(); - - debug!( - "Running rust part of compile_sync, module specifier: {}", - &source_file.url - ); - - let root_names = vec![module_url.to_string()]; - let req_msg = req(root_names, self.config.clone(), None); - - let worker = TsCompiler::setup_worker(state.clone()); - let compiling_job = state.progress.add("Compile", &module_url.to_string()); - let state_ = state.clone(); - - let resource = worker.state.resource.clone(); - let compiler_rid = resource.rid; - let first_msg_fut = - resources::post_message_to_worker(compiler_rid, req_msg) - .then(move |_| worker) - .then(move |result| { - if let Err(err) = result { - // TODO(ry) Need to forward the error instead of exiting. - eprintln!("{}", err.to_string()); - std::process::exit(1); - } - debug!("Sent message to worker"); - let stream_future = - resources::get_message_stream_from_worker(compiler_rid) - .into_future(); - stream_future.map(|(f, _rest)| f).map_err(|(f, _rest)| f) - }); - - let fut = first_msg_fut - .map_err(|_| panic!("not handled")) - .and_then(move |maybe_msg: Option| { - debug!("Received message from worker"); - - if let Some(msg) = maybe_msg { - let json_str = std::str::from_utf8(&msg).unwrap(); - debug!("Message: {}", json_str); - if let Some(diagnostics) = Diagnostic::from_emit_result(json_str) { - return Err(ErrBox::from(diagnostics)); - } - } - - Ok(()) - }).and_then(move |_| { - // if we are this far it means compilation was successful and we can - // load compiled filed from disk - // TODO: can this be somehow called using `self.`? - state_ - .ts_compiler - .get_compiled_source_file(&source_file_) - .map_err(|e| { - // TODO: this situation shouldn't happen - panic!("Expected to find compiled file: {}", e) - }) - }).and_then(move |source_file_after_compile| { - // Explicit drop to keep reference alive until future completes. - drop(compiling_job); - - Ok(source_file_after_compile) - }).then(move |r| { - debug!(">>>>> compile_sync END"); - // TODO(ry) do this in worker's destructor. - // resource.close(); - r - }); - - Either::B(fut) - } - - /// Get associated `CompiledFileMetadata` for given module if it exists. - pub fn get_metadata(self: &Self, url: &Url) -> Option { - // Try to load cached version: - // 1. check if there's 'meta' file - let cache_key = self - .disk_cache - .get_cache_filename_with_extension(url, "meta"); - if let Ok(metadata_bytes) = self.disk_cache.get(&cache_key) { - if let Ok(metadata) = std::str::from_utf8(&metadata_bytes) { - if let Some(read_metadata) = - CompiledFileMetadata::from_json_string(metadata.to_string()) - { - return Some(read_metadata); - } - } - } - - None - } - - /// Return compiled JS file for given TS module. - // TODO: ideally we shouldn't construct SourceFile by hand, but it should be delegated to - // SourceFileFetcher - pub fn get_compiled_source_file( - self: &Self, - source_file: &SourceFile, - ) -> Result { - let cache_key = self - .disk_cache - .get_cache_filename_with_extension(&source_file.url, "js"); - let compiled_code = self.disk_cache.get(&cache_key)?; - let compiled_code_filename = self.disk_cache.location.join(cache_key); - debug!("compiled filename: {:?}", compiled_code_filename); - - let compiled_module = SourceFile { - url: source_file.url.clone(), - filename: compiled_code_filename, - media_type: msg::MediaType::JavaScript, - source_code: compiled_code, - }; - - Ok(compiled_module) - } - - /// Save compiled JS file for given TS module to on-disk cache. - /// - /// Along compiled file a special metadata file is saved as well containing - /// hash that can be validated to avoid unnecessary recompilation. - fn cache_compiled_file( - self: &Self, - module_specifier: &ModuleSpecifier, - contents: &str, - ) -> std::io::Result<()> { - let js_key = self - .disk_cache - .get_cache_filename_with_extension(module_specifier.as_url(), "js"); - self - .disk_cache - .set(&js_key, contents.as_bytes()) - .and_then(|_| { - self.mark_compiled(module_specifier.as_url()); - - let source_file = self - .file_fetcher - .fetch_source_file(&module_specifier) - .expect("Source file not found"); - - let version_hash = source_code_version_hash( - &source_file.source_code, - version::DENO, - &self.config_hash, - ); - - let compiled_file_metadata = CompiledFileMetadata { - source_path: source_file.filename.to_owned(), - version_hash, - }; - let meta_key = self - .disk_cache - .get_cache_filename_with_extension(module_specifier.as_url(), "meta"); - self.disk_cache.set( - &meta_key, - compiled_file_metadata.to_json_string()?.as_bytes(), - ) - }) - } - - /// Return associated source map file for given TS module. - // TODO: ideally we shouldn't construct SourceFile by hand, but it should be delegated to - // SourceFileFetcher - pub fn get_source_map_file( - self: &Self, - module_specifier: &ModuleSpecifier, - ) -> Result { - let cache_key = self - .disk_cache - .get_cache_filename_with_extension(module_specifier.as_url(), "js.map"); - let source_code = self.disk_cache.get(&cache_key)?; - let source_map_filename = self.disk_cache.location.join(cache_key); - debug!("source map filename: {:?}", source_map_filename); - - let source_map_file = SourceFile { - url: module_specifier.as_url().to_owned(), - filename: source_map_filename, - media_type: msg::MediaType::JavaScript, - source_code, - }; - - Ok(source_map_file) - } - - /// Save source map file for given TS module to on-disk cache. - fn cache_source_map( - self: &Self, - module_specifier: &ModuleSpecifier, - contents: &str, - ) -> std::io::Result<()> { - let source_map_key = self - .disk_cache - .get_cache_filename_with_extension(module_specifier.as_url(), "js.map"); - self.disk_cache.set(&source_map_key, contents.as_bytes()) - } - - /// This method is called by TS compiler via an "op". - pub fn cache_compiler_output( - self: &Self, - module_specifier: &ModuleSpecifier, - extension: &str, - contents: &str, - ) -> std::io::Result<()> { - match extension { - ".map" => self.cache_source_map(module_specifier, contents), - ".js" => self.cache_compiled_file(module_specifier, contents), - _ => unreachable!(), - } - } -} - -impl SourceMapGetter for TsCompiler { - fn get_source_map(&self, script_name: &str) -> Option> { - self - .try_to_resolve_and_get_source_map(script_name) - .and_then(|out| Some(out.source_code)) - } - - fn get_source_line(&self, script_name: &str, line: usize) -> Option { - self - .try_resolve_and_get_source_file(script_name) - .and_then(|out| { - str::from_utf8(&out.source_code).ok().and_then(|v| { - let lines: Vec<&str> = v.lines().collect(); - assert!(lines.len() > line); - Some(lines[line].to_string()) - }) - }) - } -} - -// `SourceMapGetter` related methods -impl TsCompiler { - fn try_to_resolve(self: &Self, script_name: &str) -> Option { - // if `script_name` can't be resolved to ModuleSpecifier it's probably internal - // script (like `gen/cli/bundle/compiler.js`) so we won't be - // able to get source for it anyway - ModuleSpecifier::resolve_url(script_name).ok() - } - - fn try_resolve_and_get_source_file( - &self, - script_name: &str, - ) -> Option { - if let Some(module_specifier) = self.try_to_resolve(script_name) { - return match self.file_fetcher.fetch_source_file(&module_specifier) { - Ok(out) => Some(out), - Err(_) => None, - }; - } - - None - } - - fn try_to_resolve_and_get_source_map( - &self, - script_name: &str, - ) -> Option { - if let Some(module_specifier) = self.try_to_resolve(script_name) { - return match self.get_source_map_file(&module_specifier) { - Ok(out) => Some(out), - Err(_) => None, - }; - } - - None - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::tokio_util; - use deno::ModuleSpecifier; - use std::path::PathBuf; - - impl TsCompiler { - fn compile_sync( - self: &Self, - state: ThreadSafeState, - source_file: &SourceFile, - ) -> Result { - tokio_util::block_on(self.compile_async(state, source_file)) - } - } - - #[test] - fn test_compile_sync() { - tokio_util::init(|| { - let specifier = - ModuleSpecifier::resolve_url_or_path("./tests/002_hello.ts").unwrap(); - - let mut out = SourceFile { - url: specifier.as_url().clone(), - filename: PathBuf::from("/tests/002_hello.ts"), - media_type: msg::MediaType::TypeScript, - source_code: include_bytes!("../tests/002_hello.ts").to_vec(), - }; - - let mock_state = ThreadSafeState::mock(vec![ - String::from("./deno"), - String::from("hello.js"), - ]); - out = mock_state - .ts_compiler - .compile_sync(mock_state.clone(), &out) - .unwrap(); - assert!( - out - .source_code - .starts_with("console.log(\"Hello World\");".as_bytes()) - ); - }) - } - - #[test] - fn test_bundle_async() { - let specifier = "./tests/002_hello.ts"; - use deno::ModuleSpecifier; - let module_name = ModuleSpecifier::resolve_url_or_path(specifier) - .unwrap() - .to_string(); - - let state = ThreadSafeState::mock(vec![ - String::from("./deno"), - String::from("./tests/002_hello.ts"), - String::from("$deno$/bundle.js"), - ]); - let out = state.ts_compiler.bundle_async( - state.clone(), - module_name, - String::from("$deno$/bundle.js"), - ); - assert!(tokio_util::block_on(out).is_ok()); - } - - #[test] - fn test_source_code_version_hash() { - assert_eq!( - "08574f9cdeb94fd3fb9cdc7a20d086daeeb42bca", - source_code_version_hash(b"1+2", "0.4.0", b"{}") - ); - // Different source_code should result in different hash. - assert_eq!( - "d8abe2ead44c3ff8650a2855bf1b18e559addd06", - source_code_version_hash(b"1", "0.4.0", b"{}") - ); - // Different version should result in different hash. - assert_eq!( - "d6feffc5024d765d22c94977b4fe5975b59d6367", - source_code_version_hash(b"1", "0.1.0", b"{}") - ); - // Different config should result in different hash. - assert_eq!( - "3b35db249b26a27decd68686f073a58266b2aec2", - source_code_version_hash(b"1", "0.4.0", b"{\"compilerOptions\": {}}") - ); - } -} diff --git a/cli/compilers/js.rs b/cli/compilers/js.rs new file mode 100644 index 000000000..56c9b672e --- /dev/null +++ b/cli/compilers/js.rs @@ -0,0 +1,25 @@ +// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. +use crate::compilers::CompiledModule; +use crate::compilers::CompiledModuleFuture; +use crate::file_fetcher::SourceFile; +use crate::state::ThreadSafeState; +use std::str; + +pub struct JsCompiler {} + +impl JsCompiler { + pub fn compile_async( + self: &Self, + _state: ThreadSafeState, + source_file: &SourceFile, + ) -> Box { + let module = CompiledModule { + code: str::from_utf8(&source_file.source_code) + .unwrap() + .to_string(), + name: source_file.url.to_string(), + }; + + Box::new(futures::future::ok(module)) + } +} diff --git a/cli/compilers/json.rs b/cli/compilers/json.rs new file mode 100644 index 000000000..57e44d354 --- /dev/null +++ b/cli/compilers/json.rs @@ -0,0 +1,26 @@ +// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. +use crate::compilers::CompiledModule; +use crate::compilers::CompiledModuleFuture; +use crate::file_fetcher::SourceFile; +use crate::state::ThreadSafeState; +use std::str; + +pub struct JsonCompiler {} + +impl JsonCompiler { + pub fn compile_async( + self: &Self, + _state: ThreadSafeState, + source_file: &SourceFile, + ) -> Box { + let module = CompiledModule { + code: format!( + "export default {};", + str::from_utf8(&source_file.source_code).unwrap() + ), + name: source_file.url.to_string(), + }; + + Box::new(futures::future::ok(module)) + } +} diff --git a/cli/compilers/mod.rs b/cli/compilers/mod.rs new file mode 100644 index 000000000..fdc18d2bc --- /dev/null +++ b/cli/compilers/mod.rs @@ -0,0 +1,20 @@ +// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. +use deno::ErrBox; +use futures::Future; + +mod js; +mod json; +mod ts; + +pub use js::JsCompiler; +pub use json::JsonCompiler; +pub use ts::TsCompiler; + +#[derive(Debug, Clone)] +pub struct CompiledModule { + pub code: String, + pub name: String, +} + +pub type CompiledModuleFuture = + dyn Future + Send; diff --git a/cli/compilers/ts.rs b/cli/compilers/ts.rs new file mode 100644 index 000000000..bbfe33461 --- /dev/null +++ b/cli/compilers/ts.rs @@ -0,0 +1,740 @@ +// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. +use crate::compilers::CompiledModule; +use crate::compilers::CompiledModuleFuture; +use crate::deno_error::DenoError; +use crate::diagnostics::Diagnostic; +use crate::disk_cache::DiskCache; +use crate::file_fetcher::SourceFile; +use crate::file_fetcher::SourceFileFetcher; +use crate::msg; +use crate::msg::ErrorKind; +use crate::resources; +use crate::source_maps::SourceMapGetter; +use crate::startup_data; +use crate::state::*; +use crate::version; +use crate::worker::Worker; +use deno::Buf; +use deno::ErrBox; +use deno::ModuleSpecifier; +use futures::Future; +use futures::Stream; +use ring; +use std::collections::HashSet; +use std::fmt::Write; +use std::fs; +use std::path::PathBuf; +use std::str; +use std::sync::atomic::Ordering; +use std::sync::Mutex; +use url::Url; + +/// Struct which represents the state of the compiler +/// configuration where the first is canonical name for the configuration file, +/// second is a vector of the bytes of the contents of the configuration file, +/// third is bytes of the hash of contents. +#[derive(Clone)] +pub struct CompilerConfig { + pub path: Option, + pub content: Option>, + pub hash: Vec, +} + +impl CompilerConfig { + /// Take the passed flag and resolve the file name relative to the cwd. + pub fn load(config_path: Option) -> Result { + let config_file = match &config_path { + Some(config_file_name) => { + debug!("Compiler config file: {}", config_file_name); + let cwd = std::env::current_dir().unwrap(); + Some(cwd.join(config_file_name)) + } + _ => None, + }; + + // Convert the PathBuf to a canonicalized string. This is needed by the + // compiler to properly deal with the configuration. + let config_path = match &config_file { + Some(config_file) => Some(config_file.canonicalize().unwrap().to_owned()), + _ => None, + }; + + // Load the contents of the configuration file + let config = match &config_file { + Some(config_file) => { + debug!("Attempt to load config: {}", config_file.to_str().unwrap()); + let config = fs::read(&config_file)?; + Some(config) + } + _ => None, + }; + + let config_hash = match &config { + Some(bytes) => bytes.clone(), + _ => b"".to_vec(), + }; + + let ts_config = Self { + path: config_path, + content: config, + hash: config_hash, + }; + + Ok(ts_config) + } + + pub fn json(self: &Self) -> Result { + if self.content.is_none() { + return Ok(serde_json::Value::Null); + } + + let bytes = self.content.clone().unwrap(); + let json_string = std::str::from_utf8(&bytes)?; + match serde_json::from_str(&json_string) { + Ok(json_map) => Ok(json_map), + Err(_) => Err( + DenoError::new( + ErrorKind::InvalidInput, + "Compiler config is not a valid JSON".to_string(), + ).into(), + ), + } + } +} + +/// Information associated with compiled file in cache. +/// Includes source code path and state hash. +/// version_hash is used to validate versions of the file +/// and could be used to remove stale file in cache. +pub struct CompiledFileMetadata { + pub source_path: PathBuf, + pub version_hash: String, +} + +static SOURCE_PATH: &'static str = "source_path"; +static VERSION_HASH: &'static str = "version_hash"; + +impl CompiledFileMetadata { + pub fn from_json_string(metadata_string: String) -> Option { + // TODO: use serde for deserialization + let maybe_metadata_json: serde_json::Result = + serde_json::from_str(&metadata_string); + + if let Ok(metadata_json) = maybe_metadata_json { + let source_path = metadata_json[SOURCE_PATH].as_str().map(PathBuf::from); + let version_hash = metadata_json[VERSION_HASH].as_str().map(String::from); + + if source_path.is_none() || version_hash.is_none() { + return None; + } + + return Some(CompiledFileMetadata { + source_path: source_path.unwrap(), + version_hash: version_hash.unwrap(), + }); + } + + None + } + + pub fn to_json_string(self: &Self) -> Result { + let mut value_map = serde_json::map::Map::new(); + + value_map.insert(SOURCE_PATH.to_owned(), json!(&self.source_path)); + value_map.insert(VERSION_HASH.to_string(), json!(&self.version_hash)); + serde_json::to_string(&value_map) + } +} +/// Creates the JSON message send to compiler.ts's onmessage. +fn req( + root_names: Vec, + compiler_config: CompilerConfig, + bundle: Option, +) -> Buf { + let j = match (compiler_config.path, compiler_config.content) { + (Some(config_path), Some(config_data)) => json!({ + "rootNames": root_names, + "bundle": bundle, + "configPath": config_path, + "config": str::from_utf8(&config_data).unwrap(), + }), + _ => json!({ + "rootNames": root_names, + "bundle": bundle, + }), + }; + + j.to_string().into_boxed_str().into_boxed_bytes() +} + +fn gen_hash(v: Vec<&[u8]>) -> String { + let mut ctx = ring::digest::Context::new(&ring::digest::SHA1); + for src in v.iter() { + ctx.update(src); + } + let digest = ctx.finish(); + let mut out = String::new(); + // TODO There must be a better way to do this... + for byte in digest.as_ref() { + write!(&mut out, "{:02x}", byte).unwrap(); + } + out +} + +/// Emit a SHA1 hash based on source code, deno version and TS config. +/// Used to check if a recompilation for source code is needed. +pub fn source_code_version_hash( + source_code: &[u8], + version: &str, + config_hash: &[u8], +) -> String { + gen_hash(vec![source_code, version.as_bytes(), config_hash]) +} + +pub struct TsCompiler { + pub file_fetcher: SourceFileFetcher, + pub config: CompilerConfig, + pub disk_cache: DiskCache, + /// Set of all URLs that have been compiled. This prevents double + /// compilation of module. + pub compiled: Mutex>, + /// This setting is controlled by `--reload` flag. Unless the flag + /// is provided disk cache is used. + pub use_disk_cache: bool, + /// This setting is controlled by `compilerOptions.checkJs` + pub compile_js: bool, +} + +impl TsCompiler { + pub fn new( + file_fetcher: SourceFileFetcher, + disk_cache: DiskCache, + use_disk_cache: bool, + config_path: Option, + ) -> Result { + let config = CompilerConfig::load(config_path)?; + + // If `checkJs` is set to true in `compilerOptions` then we're gonna be compiling + // JavaScript files as well + let config_json = config.json()?; + let compile_js = match &config_json.get("compilerOptions") { + Some(serde_json::Value::Object(m)) => match m.get("checkJs") { + Some(serde_json::Value::Bool(bool_)) => *bool_, + _ => false, + }, + _ => false, + }; + + let compiler = Self { + file_fetcher, + disk_cache, + config, + compiled: Mutex::new(HashSet::new()), + use_disk_cache, + compile_js, + }; + + Ok(compiler) + } + + /// Create a new V8 worker with snapshot of TS compiler and setup compiler's runtime. + fn setup_worker(state: ThreadSafeState) -> Worker { + // Count how many times we start the compiler worker. + state.metrics.compiler_starts.fetch_add(1, Ordering::SeqCst); + + let mut worker = Worker::new( + "TS".to_string(), + startup_data::compiler_isolate_init(), + // TODO(ry) Maybe we should use a separate state for the compiler. + // as was done previously. + state.clone(), + ); + worker.execute("denoMain()").unwrap(); + worker.execute("workerMain()").unwrap(); + worker.execute("compilerMain()").unwrap(); + worker + } + + pub fn bundle_async( + self: &Self, + state: ThreadSafeState, + module_name: String, + out_file: String, + ) -> impl Future { + debug!( + "Invoking the compiler to bundle. module_name: {}", + module_name + ); + + let root_names = vec![module_name.clone()]; + let req_msg = req(root_names, self.config.clone(), Some(out_file)); + + let worker = TsCompiler::setup_worker(state.clone()); + let resource = worker.state.resource.clone(); + let compiler_rid = resource.rid; + let first_msg_fut = + resources::post_message_to_worker(compiler_rid, req_msg) + .then(move |_| worker) + .then(move |result| { + if let Err(err) = result { + // TODO(ry) Need to forward the error instead of exiting. + eprintln!("{}", err.to_string()); + std::process::exit(1); + } + debug!("Sent message to worker"); + let stream_future = + resources::get_message_stream_from_worker(compiler_rid) + .into_future(); + stream_future.map(|(f, _rest)| f).map_err(|(f, _rest)| f) + }); + + first_msg_fut.map_err(|_| panic!("not handled")).and_then( + move |maybe_msg: Option| { + debug!("Received message from worker"); + + if let Some(msg) = maybe_msg { + let json_str = std::str::from_utf8(&msg).unwrap(); + debug!("Message: {}", json_str); + if let Some(diagnostics) = Diagnostic::from_emit_result(json_str) { + return Err(ErrBox::from(diagnostics)); + } + } + + Ok(()) + }, + ) + } + + /// Mark given module URL as compiled to avoid multiple compilations of same module + /// in single run. + fn mark_compiled(&self, url: &Url) { + let mut c = self.compiled.lock().unwrap(); + c.insert(url.clone()); + } + + /// Check if given module URL has already been compiled and can be fetched directly from disk. + fn has_compiled(&self, url: &Url) -> bool { + let c = self.compiled.lock().unwrap(); + c.contains(url) + } + + /// Asynchronously compile module and all it's dependencies. + /// + /// This method compiled every module at most once. + /// + /// If `--reload` flag was provided then compiler will not on-disk cache and force recompilation. + /// + /// If compilation is required then new V8 worker is spawned with fresh TS compiler. + pub fn compile_async( + self: &Self, + state: ThreadSafeState, + source_file: &SourceFile, + ) -> Box { + if self.has_compiled(&source_file.url) { + return match self.get_compiled_module(&source_file.url) { + Ok(compiled) => Box::new(futures::future::ok(compiled)), + Err(err) => Box::new(futures::future::err(err)), + }; + } + + if self.use_disk_cache { + // Try to load cached version: + // 1. check if there's 'meta' file + if let Some(metadata) = self.get_metadata(&source_file.url) { + // 2. compare version hashes + // TODO: it would probably be good idea to make it method implemented on SourceFile + let version_hash_to_validate = source_code_version_hash( + &source_file.source_code, + version::DENO, + &self.config.hash, + ); + + if metadata.version_hash == version_hash_to_validate { + debug!("load_cache metadata version hash match"); + if let Ok(compiled_module) = + self.get_compiled_module(&source_file.url) + { + self.mark_compiled(&source_file.url); + return Box::new(futures::future::ok(compiled_module)); + } + } + } + } + + let source_file_ = source_file.clone(); + + debug!(">>>>> compile_sync START"); + let module_url = source_file.url.clone(); + + debug!( + "Running rust part of compile_sync, module specifier: {}", + &source_file.url + ); + + let root_names = vec![module_url.to_string()]; + let req_msg = req(root_names, self.config.clone(), None); + + let worker = TsCompiler::setup_worker(state.clone()); + let compiling_job = state.progress.add("Compile", &module_url.to_string()); + let state_ = state.clone(); + + let resource = worker.state.resource.clone(); + let compiler_rid = resource.rid; + let first_msg_fut = + resources::post_message_to_worker(compiler_rid, req_msg) + .then(move |_| worker) + .then(move |result| { + if let Err(err) = result { + // TODO(ry) Need to forward the error instead of exiting. + eprintln!("{}", err.to_string()); + std::process::exit(1); + } + debug!("Sent message to worker"); + let stream_future = + resources::get_message_stream_from_worker(compiler_rid) + .into_future(); + stream_future.map(|(f, _rest)| f).map_err(|(f, _rest)| f) + }); + + let fut = first_msg_fut + .map_err(|_| panic!("not handled")) + .and_then(move |maybe_msg: Option| { + debug!("Received message from worker"); + + if let Some(msg) = maybe_msg { + let json_str = std::str::from_utf8(&msg).unwrap(); + debug!("Message: {}", json_str); + if let Some(diagnostics) = Diagnostic::from_emit_result(json_str) { + return Err(ErrBox::from(diagnostics)); + } + } + + Ok(()) + }).and_then(move |_| { + // if we are this far it means compilation was successful and we can + // load compiled filed from disk + state_ + .ts_compiler + .get_compiled_module(&source_file_.url) + .map_err(|e| { + // TODO: this situation shouldn't happen + panic!("Expected to find compiled file: {}", e) + }) + }).and_then(move |compiled_module| { + // Explicit drop to keep reference alive until future completes. + drop(compiling_job); + + Ok(compiled_module) + }).then(move |r| { + debug!(">>>>> compile_sync END"); + // TODO(ry) do this in worker's destructor. + // resource.close(); + r + }); + + Box::new(fut) + } + + /// Get associated `CompiledFileMetadata` for given module if it exists. + pub fn get_metadata(self: &Self, url: &Url) -> Option { + // Try to load cached version: + // 1. check if there's 'meta' file + let cache_key = self + .disk_cache + .get_cache_filename_with_extension(url, "meta"); + if let Ok(metadata_bytes) = self.disk_cache.get(&cache_key) { + if let Ok(metadata) = std::str::from_utf8(&metadata_bytes) { + if let Some(read_metadata) = + CompiledFileMetadata::from_json_string(metadata.to_string()) + { + return Some(read_metadata); + } + } + } + + None + } + + pub fn get_compiled_module( + self: &Self, + module_url: &Url, + ) -> Result { + let compiled_source_file = self.get_compiled_source_file(module_url)?; + + let compiled_module = CompiledModule { + code: str::from_utf8(&compiled_source_file.source_code) + .unwrap() + .to_string(), + name: module_url.to_string(), + }; + + Ok(compiled_module) + } + + /// Return compiled JS file for given TS module. + // TODO: ideally we shouldn't construct SourceFile by hand, but it should be delegated to + // SourceFileFetcher + pub fn get_compiled_source_file( + self: &Self, + module_url: &Url, + ) -> Result { + let cache_key = self + .disk_cache + .get_cache_filename_with_extension(&module_url, "js"); + let compiled_code = self.disk_cache.get(&cache_key)?; + let compiled_code_filename = self.disk_cache.location.join(cache_key); + debug!("compiled filename: {:?}", compiled_code_filename); + + let compiled_module = SourceFile { + url: module_url.clone(), + filename: compiled_code_filename, + media_type: msg::MediaType::JavaScript, + source_code: compiled_code, + }; + + Ok(compiled_module) + } + + /// Save compiled JS file for given TS module to on-disk cache. + /// + /// Along compiled file a special metadata file is saved as well containing + /// hash that can be validated to avoid unnecessary recompilation. + fn cache_compiled_file( + self: &Self, + module_specifier: &ModuleSpecifier, + contents: &str, + ) -> std::io::Result<()> { + let js_key = self + .disk_cache + .get_cache_filename_with_extension(module_specifier.as_url(), "js"); + self + .disk_cache + .set(&js_key, contents.as_bytes()) + .and_then(|_| { + self.mark_compiled(module_specifier.as_url()); + + let source_file = self + .file_fetcher + .fetch_source_file(&module_specifier) + .expect("Source file not found"); + + let version_hash = source_code_version_hash( + &source_file.source_code, + version::DENO, + &self.config.hash, + ); + + let compiled_file_metadata = CompiledFileMetadata { + source_path: source_file.filename.to_owned(), + version_hash, + }; + let meta_key = self + .disk_cache + .get_cache_filename_with_extension(module_specifier.as_url(), "meta"); + self.disk_cache.set( + &meta_key, + compiled_file_metadata.to_json_string()?.as_bytes(), + ) + }) + } + + /// Return associated source map file for given TS module. + // TODO: ideally we shouldn't construct SourceFile by hand, but it should be delegated to + // SourceFileFetcher + pub fn get_source_map_file( + self: &Self, + module_specifier: &ModuleSpecifier, + ) -> Result { + let cache_key = self + .disk_cache + .get_cache_filename_with_extension(module_specifier.as_url(), "js.map"); + let source_code = self.disk_cache.get(&cache_key)?; + let source_map_filename = self.disk_cache.location.join(cache_key); + debug!("source map filename: {:?}", source_map_filename); + + let source_map_file = SourceFile { + url: module_specifier.as_url().to_owned(), + filename: source_map_filename, + media_type: msg::MediaType::JavaScript, + source_code, + }; + + Ok(source_map_file) + } + + /// Save source map file for given TS module to on-disk cache. + fn cache_source_map( + self: &Self, + module_specifier: &ModuleSpecifier, + contents: &str, + ) -> std::io::Result<()> { + let source_map_key = self + .disk_cache + .get_cache_filename_with_extension(module_specifier.as_url(), "js.map"); + self.disk_cache.set(&source_map_key, contents.as_bytes()) + } + + /// This method is called by TS compiler via an "op". + pub fn cache_compiler_output( + self: &Self, + module_specifier: &ModuleSpecifier, + extension: &str, + contents: &str, + ) -> std::io::Result<()> { + match extension { + ".map" => self.cache_source_map(module_specifier, contents), + ".js" => self.cache_compiled_file(module_specifier, contents), + _ => unreachable!(), + } + } +} + +impl SourceMapGetter for TsCompiler { + fn get_source_map(&self, script_name: &str) -> Option> { + self + .try_to_resolve_and_get_source_map(script_name) + .and_then(|out| Some(out.source_code)) + } + + fn get_source_line(&self, script_name: &str, line: usize) -> Option { + self + .try_resolve_and_get_source_file(script_name) + .and_then(|out| { + str::from_utf8(&out.source_code).ok().and_then(|v| { + let lines: Vec<&str> = v.lines().collect(); + assert!(lines.len() > line); + Some(lines[line].to_string()) + }) + }) + } +} + +// `SourceMapGetter` related methods +impl TsCompiler { + fn try_to_resolve(self: &Self, script_name: &str) -> Option { + // if `script_name` can't be resolved to ModuleSpecifier it's probably internal + // script (like `gen/cli/bundle/compiler.js`) so we won't be + // able to get source for it anyway + ModuleSpecifier::resolve_url(script_name).ok() + } + + fn try_resolve_and_get_source_file( + &self, + script_name: &str, + ) -> Option { + if let Some(module_specifier) = self.try_to_resolve(script_name) { + return match self.file_fetcher.fetch_source_file(&module_specifier) { + Ok(out) => Some(out), + Err(_) => None, + }; + } + + None + } + + fn try_to_resolve_and_get_source_map( + &self, + script_name: &str, + ) -> Option { + if let Some(module_specifier) = self.try_to_resolve(script_name) { + return match self.get_source_map_file(&module_specifier) { + Ok(out) => Some(out), + Err(_) => None, + }; + } + + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::tokio_util; + use deno::ModuleSpecifier; + use std::path::PathBuf; + + impl TsCompiler { + fn compile_sync( + self: &Self, + state: ThreadSafeState, + source_file: &SourceFile, + ) -> Result { + tokio_util::block_on(self.compile_async(state, source_file)) + } + } + + #[test] + fn test_compile_sync() { + tokio_util::init(|| { + let specifier = + ModuleSpecifier::resolve_url_or_path("./tests/002_hello.ts").unwrap(); + + let out = SourceFile { + url: specifier.as_url().clone(), + filename: PathBuf::from("/tests/002_hello.ts"), + media_type: msg::MediaType::TypeScript, + source_code: include_bytes!("../../tests/002_hello.ts").to_vec(), + }; + + let mock_state = ThreadSafeState::mock(vec![ + String::from("./deno"), + String::from("hello.js"), + ]); + let compiled = mock_state + .ts_compiler + .compile_sync(mock_state.clone(), &out) + .unwrap(); + assert!( + compiled + .code + .as_bytes() + .starts_with("console.log(\"Hello World\");".as_bytes()) + ); + }) + } + + #[test] + fn test_bundle_async() { + let specifier = "./tests/002_hello.ts"; + use deno::ModuleSpecifier; + let module_name = ModuleSpecifier::resolve_url_or_path(specifier) + .unwrap() + .to_string(); + + let state = ThreadSafeState::mock(vec![ + String::from("./deno"), + String::from("./tests/002_hello.ts"), + String::from("$deno$/bundle.js"), + ]); + let out = state.ts_compiler.bundle_async( + state.clone(), + module_name, + String::from("$deno$/bundle.js"), + ); + assert!(tokio_util::block_on(out).is_ok()); + } + + #[test] + fn test_source_code_version_hash() { + assert_eq!( + "08574f9cdeb94fd3fb9cdc7a20d086daeeb42bca", + source_code_version_hash(b"1+2", "0.4.0", b"{}") + ); + // Different source_code should result in different hash. + assert_eq!( + "d8abe2ead44c3ff8650a2855bf1b18e559addd06", + source_code_version_hash(b"1", "0.4.0", b"{}") + ); + // Different version should result in different hash. + assert_eq!( + "d6feffc5024d765d22c94977b4fe5975b59d6367", + source_code_version_hash(b"1", "0.1.0", b"{}") + ); + // Different config should result in different hash. + assert_eq!( + "3b35db249b26a27decd68686f073a58266b2aec2", + source_code_version_hash(b"1", "0.4.0", b"{\"compilerOptions\": {}}") + ); + } +} diff --git a/cli/deno_dir.rs b/cli/deno_dir.rs index c600b06fc..ac35922eb 100644 --- a/cli/deno_dir.rs +++ b/cli/deno_dir.rs @@ -17,8 +17,6 @@ pub struct DenoDir { } impl DenoDir { - // Must be called before using any function from this module. - // https://github.com/denoland/deno/blob/golang/deno_dir.go#L99-L111 pub fn new(custom_root: Option) -> std::io::Result { // Only setup once. let home_dir = dirs::home_dir().expect("Could not get home directory."); diff --git a/cli/disk_cache.rs b/cli/disk_cache.rs index 808cfe675..fdbe2cbd5 100644 --- a/cli/disk_cache.rs +++ b/cli/disk_cache.rs @@ -18,6 +18,10 @@ impl DiskCache { } } + // TODO(bartlomieju) this method is not working properly for Windows paths, + // Example: file:///C:/deno/js/unit_test_runner.ts + // would produce: C:deno\\js\\unit_test_runner.ts + // it should produce: file\deno\js\unit_test_runner.ts pub fn get_cache_filename(self: &Self, url: &Url) -> PathBuf { let mut out = PathBuf::new(); diff --git a/cli/file_fetcher.rs b/cli/file_fetcher.rs index 656ecfff0..79d6ede00 100644 --- a/cli/file_fetcher.rs +++ b/cli/file_fetcher.rs @@ -39,27 +39,6 @@ pub struct SourceFile { pub source_code: Vec, } -impl SourceFile { - // TODO(bartlomieju): this method should be implemented on new `CompiledSourceFile` - // trait and should be handled by "compiler pipeline" - pub fn js_source(&self) -> String { - if self.media_type == msg::MediaType::TypeScript { - panic!("TypeScript module has no JS source, did you forget to run it through compiler?"); - } - - // TODO: this should be done by compiler and JS module should be returned - if self.media_type == msg::MediaType::Json { - return format!( - "export default {};", - str::from_utf8(&self.source_code).unwrap() - ); - } - - // it's either JS or Unknown media type - str::from_utf8(&self.source_code).unwrap().to_string() - } -} - pub type SourceFileFuture = dyn Future + Send; diff --git a/cli/main.rs b/cli/main.rs index 452cdfa65..fb34a2c76 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -16,7 +16,7 @@ extern crate rand; extern crate url; mod ansi; -pub mod compiler; +pub mod compilers; pub mod deno_dir; pub mod deno_error; pub mod diagnostics; @@ -99,6 +99,7 @@ fn js_check(r: Result<(), ErrBox>) { } } +// TODO: we might want to rethink how this method works pub fn print_file_info( worker: Worker, module_specifier: &ModuleSpecifier, @@ -110,7 +111,7 @@ pub fn print_file_info( .file_fetcher .fetch_source_file_async(&module_specifier) .map_err(|err| println!("{}", err)) - .and_then(move |out| { + .and_then(|out| { println!( "{} {}", ansi::bold("local:".to_string()), @@ -125,18 +126,25 @@ pub fn print_file_info( state_ .clone() - .ts_compiler - .compile_async(state_.clone(), &out) + .fetch_compiled_module(&module_specifier_) .map_err(|e| { debug!("compiler error exiting!"); eprintln!("\n{}", e.to_string()); std::process::exit(1); }).and_then(move |compiled| { - if out.media_type == msg::MediaType::TypeScript { + if out.media_type == msg::MediaType::TypeScript + || (out.media_type == msg::MediaType::JavaScript + && state_.ts_compiler.compile_js) + { + let compiled_source_file = state_ + .ts_compiler + .get_compiled_source_file(&out.url) + .unwrap(); + println!( "{} {}", ansi::bold("compiled:".to_string()), - compiled.filename.to_str().unwrap(), + compiled_source_file.filename.to_str().unwrap(), ); } @@ -152,12 +160,8 @@ pub fn print_file_info( ); } - if let Some(deps) = worker - .state - .modules - .lock() - .unwrap() - .deps(&compiled.url.to_string()) + if let Some(deps) = + worker.state.modules.lock().unwrap().deps(&compiled.name) { println!("{}{}", ansi::bold("deps:\n".to_string()), deps.name); if let Some(ref depsdeps) = deps.deps { diff --git a/cli/state.rs b/cli/state.rs index cd7c1269d..047e2b7ed 100644 --- a/cli/state.rs +++ b/cli/state.rs @@ -1,11 +1,14 @@ // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license. -use crate::compiler::TsCompiler; +use crate::compilers::CompiledModule; +use crate::compilers::JsCompiler; +use crate::compilers::JsonCompiler; +use crate::compilers::TsCompiler; use crate::deno_dir; -use crate::file_fetcher::SourceFile; use crate::file_fetcher::SourceFileFetcher; use crate::flags; use crate::global_timer::GlobalTimer; use crate::import_map::ImportMap; +use crate::msg; use crate::ops; use crate::permissions::DenoPermissions; use crate::progress::Progress; @@ -26,6 +29,7 @@ use std; use std::collections::HashMap; use std::env; use std::ops::Deref; +use std::str; use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::sync::Mutex; @@ -77,6 +81,8 @@ pub struct State { pub seeded_rng: Option>, pub file_fetcher: SourceFileFetcher, + pub js_compiler: JsCompiler, + pub json_compiler: JsonCompiler, pub ts_compiler: TsCompiler, } @@ -103,28 +109,6 @@ impl ThreadSafeState { } } -pub fn fetch_source_file_and_maybe_compile_async( - state: &ThreadSafeState, - module_specifier: &ModuleSpecifier, -) -> impl Future { - let state_ = state.clone(); - - state_ - .file_fetcher - .fetch_source_file_async(&module_specifier) - .and_then(move |out| { - state_ - .clone() - .ts_compiler - .compile_async(state_.clone(), &out) - .map_err(|e| { - debug!("compiler error exiting!"); - eprintln!("\n{}", e.to_string()); - std::process::exit(1); - }) - }) -} - impl Loader for ThreadSafeState { fn resolve( &self, @@ -150,16 +134,14 @@ impl Loader for ThreadSafeState { module_specifier: &ModuleSpecifier, ) -> Box { self.metrics.resolve_count.fetch_add(1, Ordering::SeqCst); - Box::new( - fetch_source_file_and_maybe_compile_async(self, module_specifier).map( - |source_file| deno::SourceCodeInfo { - // Real module name, might be different from initial specifier - // due to redirections. - code: source_file.js_source(), - module_name: source_file.url.to_string(), - }, - ), - ) + Box::new(self.fetch_compiled_module(module_specifier).map( + |compiled_module| deno::SourceCodeInfo { + // Real module name, might be different from initial specifier + // due to redirections. + code: compiled_module.code, + module_name: compiled_module.name, + }, + )) } } @@ -192,36 +174,26 @@ impl ThreadSafeState { dir.gen_cache.clone(), !flags.reload, flags.config_path.clone(), - ); + )?; let main_module: Option = if argv_rest.len() <= 1 { None } else { let root_specifier = argv_rest[1].clone(); - match ModuleSpecifier::resolve_url_or_path(&root_specifier) { - Ok(specifier) => Some(specifier), - Err(e) => { - // TODO: handle unresolvable specifier - panic!("Unable to resolve root specifier: {:?}", e); - } - } + Some(ModuleSpecifier::resolve_url_or_path(&root_specifier)?) }; - let mut import_map = None; - if let Some(file_name) = &flags.import_map_path { - let base_url = match &main_module { - Some(module_specifier) => module_specifier.clone(), - None => unreachable!(), - }; - - match ImportMap::load(&base_url.to_string(), file_name) { - Ok(map) => import_map = Some(map), - Err(err) => { - println!("{:?}", err); - panic!("Error parsing import map"); - } + let import_map: Option = match &flags.import_map_path { + None => None, + Some(file_name) => { + let base_url = match &main_module { + Some(module_specifier) => module_specifier.clone(), + None => unreachable!(), + }; + let import_map = ImportMap::load(&base_url.to_string(), file_name)?; + Some(import_map) } - } + }; let mut seeded_rng = None; if let Some(seed) = flags.seed { @@ -249,11 +221,42 @@ impl ThreadSafeState { seeded_rng, file_fetcher, ts_compiler, + js_compiler: JsCompiler {}, + json_compiler: JsonCompiler {}, }; Ok(ThreadSafeState(Arc::new(state))) } + pub fn fetch_compiled_module( + self: &Self, + module_specifier: &ModuleSpecifier, + ) -> impl Future { + let state_ = self.clone(); + + self + .file_fetcher + .fetch_source_file_async(&module_specifier) + .and_then(move |out| match out.media_type { + msg::MediaType::Unknown => { + state_.js_compiler.compile_async(state_.clone(), &out) + } + msg::MediaType::Json => { + state_.json_compiler.compile_async(state_.clone(), &out) + } + msg::MediaType::TypeScript => { + state_.ts_compiler.compile_async(state_.clone(), &out) + } + msg::MediaType::JavaScript => { + if state_.ts_compiler.compile_js { + state_.ts_compiler.compile_async(state_.clone(), &out) + } else { + state_.js_compiler.compile_async(state_.clone(), &out) + } + } + }) + } + /// Read main module from argv pub fn main_module(&self) -> Option { match &self.main_module { diff --git a/js/compiler.ts b/js/compiler.ts index 34ac2f482..4203f753b 100644 --- a/js/compiler.ts +++ b/js/compiler.ts @@ -219,6 +219,8 @@ function getExtension( } class Host implements ts.CompilerHost { + extensionCache: Record = {}; + private readonly _options: ts.CompilerOptions = { allowJs: true, allowNonTsExtensions: true, @@ -370,10 +372,16 @@ class Host implements ts.CompilerHost { // This flags to the compiler to not go looking to transpile functional // code, anything that is in `/$asset$/` is just library code const isExternalLibraryImport = moduleName.startsWith(ASSETS); + const extension = getExtension( + resolvedFileName, + SourceFile.mediaType + ); + this.extensionCache[resolvedFileName] = extension; + const r = { resolvedFileName, isExternalLibraryImport, - extension: getExtension(resolvedFileName, SourceFile.mediaType) + extension }; return r; } else { @@ -401,6 +409,21 @@ class Host implements ts.CompilerHost { } else { assert(sourceFiles != null && sourceFiles.length == 1); const sourceFileName = sourceFiles![0].fileName; + const maybeExtension = this.extensionCache[sourceFileName]; + + if (maybeExtension) { + // NOTE: If it's a `.json` file we don't want to write it to disk. + // JSON files are loaded and used by TS compiler to check types, but we don't want + // to emit them to disk because output file is the same as input file. + if (maybeExtension === ts.Extension.Json) { + return; + } + + // NOTE: JavaScript files are only emitted to disk if `checkJs` option in on + if (maybeExtension === ts.Extension.Js && !this._options.checkJs) { + return; + } + } if (fileName.endsWith(".map")) { // Source Map diff --git a/tests/038_checkjs.js b/tests/038_checkjs.js new file mode 100644 index 000000000..628d3e376 --- /dev/null +++ b/tests/038_checkjs.js @@ -0,0 +1,6 @@ +// console.log intentionally misspelled to trigger a type error +consol.log("hello world!"); + +// the following error should be ignored and not output to the console +// eslint-disable-next-line +const foo = new Foo(); diff --git a/tests/038_checkjs.js.out b/tests/038_checkjs.js.out new file mode 100644 index 000000000..deaf77211 --- /dev/null +++ b/tests/038_checkjs.js.out @@ -0,0 +1,15 @@ +[WILDCARD] +error TS2552: Cannot find name 'consol'. Did you mean 'console'? + +[WILDCARD]tests/038_checkjs.js:2:1 + +2 consol.log("hello world!"); +[WILDCARD] +error TS2552: Cannot find name 'Foo'. Did you mean 'foo'? + +[WILDCARD]tests/038_checkjs.js:6:17 + +6 const foo = new Foo(); +[WILDCARD] +Found 2 errors. +[WILDCARD] \ No newline at end of file diff --git a/tests/038_checkjs.test b/tests/038_checkjs.test new file mode 100644 index 000000000..6385c9bb7 --- /dev/null +++ b/tests/038_checkjs.test @@ -0,0 +1,5 @@ +# checking if JS file is run through TS compiler +args: run --reload --config tests/038_checkjs.tsconfig.json tests/038_checkjs.js +check_stderr: true +exit_code: 1 +output: tests/038_checkjs.js.out diff --git a/tests/038_checkjs.tsconfig.json b/tests/038_checkjs.tsconfig.json new file mode 100644 index 000000000..08ac60b6c --- /dev/null +++ b/tests/038_checkjs.tsconfig.json @@ -0,0 +1,5 @@ +{ + "compilerOptions": { + "checkJs": true + } +} -- cgit v1.2.3