diff options
Diffstat (limited to 'cli/tsc.rs')
| -rw-r--r-- | cli/tsc.rs | 442 |
1 files changed, 255 insertions, 187 deletions
diff --git a/cli/tsc.rs b/cli/tsc.rs index 664721bc1..2eb8a35a0 100644 --- a/cli/tsc.rs +++ b/cli/tsc.rs @@ -5,10 +5,9 @@ use crate::diagnostics::DiagnosticItem; use crate::disk_cache::DiskCache; use crate::file_fetcher::SourceFile; use crate::file_fetcher::SourceFileFetcher; -use crate::fmt; -use crate::fs as deno_fs; use crate::global_state::GlobalState; use crate::import_map::ImportMap; +use crate::module_graph::ModuleGraphFile; use crate::module_graph::ModuleGraphLoader; use crate::msg; use crate::op_error::OpError; @@ -16,7 +15,6 @@ use crate::ops; use crate::permissions::Permissions; use crate::source_maps::SourceMapGetter; use crate::startup_data; -use crate::state::exit_unstable; use crate::state::State; use crate::version; use crate::web_worker::WebWorker; @@ -50,73 +48,69 @@ use std::sync::atomic::Ordering; use std::sync::Arc; use std::sync::Mutex; use std::task::Poll; -use std::time::Instant; use url::Url; -// TODO(bartlomieju): make static -pub fn get_available_libs() -> Vec<String> { - vec![ - "deno.ns".to_string(), - "deno.window".to_string(), - "deno.worker".to_string(), - "deno.shared_globals".to_string(), - "deno.unstable".to_string(), - "dom".to_string(), - "dom.iterable".to_string(), - "es5".to_string(), - "es6".to_string(), - "esnext".to_string(), - "es2020".to_string(), - "es2020.full".to_string(), - "es2019".to_string(), - "es2019.full".to_string(), - "es2018".to_string(), - "es2018.full".to_string(), - "es2017".to_string(), - "es2017.full".to_string(), - "es2016".to_string(), - "es2016.full".to_string(), - "es2015".to_string(), - "es2015.collection".to_string(), - "es2015.core".to_string(), - "es2015.generator".to_string(), - "es2015.iterable".to_string(), - "es2015.promise".to_string(), - "es2015.proxy".to_string(), - "es2015.reflect".to_string(), - "es2015.symbol".to_string(), - "es2015.symbol.wellknown".to_string(), - "es2016.array.include".to_string(), - "es2017.intl".to_string(), - "es2017.object".to_string(), - "es2017.sharedmemory".to_string(), - "es2017.string".to_string(), - "es2017.typedarrays".to_string(), - "es2018.asyncgenerator".to_string(), - "es2018.asynciterable".to_string(), - "es2018.intl".to_string(), - "es2018.promise".to_string(), - "es2018.regexp".to_string(), - "es2019.array".to_string(), - "es2019.object".to_string(), - "es2019.string".to_string(), - "es2019.symbol".to_string(), - "es2020.bigint".to_string(), - "es2020.promise".to_string(), - "es2020.string".to_string(), - "es2020.symbol.wellknown".to_string(), - "esnext.array".to_string(), - "esnext.asynciterable".to_string(), - "esnext.bigint".to_string(), - "esnext.intl".to_string(), - "esnext.promise".to_string(), - "esnext.string".to_string(), - "esnext.symbol".to_string(), - "scripthost".to_string(), - "webworker".to_string(), - "webworker.importscripts".to_string(), - ] -} +pub const AVAILABLE_LIBS: &[&str] = &[ + "deno.ns", + "deno.window", + "deno.worker", + "deno.shared_globals", + "deno.unstable", + "dom", + "dom.iterable", + "es5", + "es6", + "esnext", + "es2020", + "es2020.full", + "es2019", + "es2019.full", + "es2018", + "es2018.full", + "es2017", + "es2017.full", + "es2016", + "es2016.full", + "es2015", + "es2015.collection", + "es2015.core", + "es2015.generator", + "es2015.iterable", + "es2015.promise", + "es2015.proxy", + "es2015.reflect", + "es2015.symbol", + "es2015.symbol.wellknown", + "es2016.array.include", + "es2017.intl", + "es2017.object", + "es2017.sharedmemory", + "es2017.string", + "es2017.typedarrays", + "es2018.asyncgenerator", + "es2018.asynciterable", + "es2018.intl", + "es2018.promise", + "es2018.regexp", + "es2019.array", + "es2019.object", + "es2019.string", + "es2019.symbol", + "es2020.bigint", + "es2020.promise", + "es2020.string", + "es2020.symbol.wellknown", + "esnext.array", + "esnext.asynciterable", + "esnext.bigint", + "esnext.intl", + "esnext.promise", + "esnext.string", + "esnext.symbol", + "scripthost", + "webworker", + "webworker.importscripts", +]; #[derive(Debug, Clone)] pub struct CompiledModule { @@ -160,6 +154,7 @@ impl Future for CompilerWorker { } } +// TODO(bartlomieju): use JSONC parser from dprint instead of Regex lazy_static! { static ref CHECK_JS_RE: Regex = Regex::new(r#""checkJs"\s*?:\s*?true"#).unwrap(); @@ -175,9 +170,14 @@ fn create_compiler_worker( // like 'eval', 'repl' let entry_point = ModuleSpecifier::resolve_url_or_path("./__$deno$ts_compiler.ts").unwrap(); - let worker_state = - State::new(global_state.clone(), Some(permissions), entry_point, true) - .expect("Unable to create worker state"); + let worker_state = State::new( + global_state.clone(), + Some(permissions), + entry_point, + None, + true, + ) + .expect("Unable to create worker state"); // TODO(bartlomieju): this metric is never used anywhere // Count how many times we start the compiler worker. @@ -294,6 +294,28 @@ impl CompiledFileMetadata { } } +/// Information associated with compilation of a "module graph", +/// ie. entry point and all its dependencies. +/// It's used to perform cache invalidation if content of any +/// dependency changes. +#[derive(Deserialize, Serialize)] +pub struct GraphFileMetadata { + pub deps: Vec<String>, + pub version_hash: String, +} + +impl GraphFileMetadata { + pub fn from_json_string( + metadata_string: String, + ) -> Result<Self, serde_json::Error> { + serde_json::from_str::<Self>(&metadata_string) + } + + pub fn to_json_string(&self) -> Result<String, serde_json::Error> { + serde_json::to_string(self) + } +} + /// Emit a SHA256 hash based on source code, deno version and TS config. /// Used to check if a recompilation for source code is needed. pub fn source_code_version_hash( @@ -383,6 +405,7 @@ impl TsCompiler { }))) } + // TODO(bartlomieju): this method is no longer needed /// Mark given module URL as compiled to avoid multiple compilations of same /// module in single run. fn mark_compiled(&self, url: &Url) { @@ -390,11 +413,34 @@ impl TsCompiler { c.insert(url.clone()); } - /// Check if given module URL has already been compiled and can be fetched - /// directly from disk. - fn has_compiled(&self, url: &Url) -> bool { - let c = self.compiled.lock().unwrap(); - c.contains(url) + /// Check if there is compiled source in cache that is valid + /// and can be used again. + // TODO(bartlomieju): there should be check that cached file actually exists + fn has_compiled_source( + &self, + file_fetcher: &SourceFileFetcher, + url: &Url, + ) -> bool { + let specifier = ModuleSpecifier::from(url.clone()); + if let Some(source_file) = file_fetcher + .fetch_cached_source_file(&specifier, Permissions::allow_all()) + { + if let Some(metadata) = self.get_metadata(&url) { + // 2. compare version hashes + // TODO: it would probably be good idea to make it method implemented on SourceFile + let version_hash_to_validate = source_code_version_hash( + &source_file.source_code, + version::DENO, + &self.config.hash, + ); + + if metadata.version_hash == version_hash_to_validate { + return true; + } + } + } + + false } /// Asynchronously compile module and all it's dependencies. @@ -406,64 +452,43 @@ impl TsCompiler { /// /// If compilation is required then new V8 worker is spawned with fresh TS /// compiler. - pub async fn compile( + pub async fn compile_module_graph( &self, global_state: GlobalState, source_file: &SourceFile, target: TargetLib, permissions: Permissions, - is_dyn_import: bool, - ) -> Result<CompiledModule, ErrBox> { - if self.has_compiled(&source_file.url) { - return self.get_compiled_module(&source_file.url); - } + module_graph: HashMap<String, ModuleGraphFile>, + ) -> Result<(), ErrBox> { + let mut has_cached_version = false; if self.use_disk_cache { - // Try to load cached version: - // 1. check if there's 'meta' file - if let Some(metadata) = self.get_metadata(&source_file.url) { - // 2. compare version hashes - // TODO: it would probably be good idea to make it method implemented on SourceFile - let version_hash_to_validate = source_code_version_hash( - &source_file.source_code, - version::DENO, + if let Some(metadata) = self.get_graph_metadata(&source_file.url) { + has_cached_version = true; + + let version_hash = crate::checksum::gen(vec![ + version::DENO.as_bytes(), &self.config.hash, - ); + ]); - if metadata.version_hash == version_hash_to_validate { - debug!("load_cache metadata version hash match"); - if let Ok(compiled_module) = - self.get_compiled_module(&source_file.url) - { - self.mark_compiled(&source_file.url); - return Ok(compiled_module); - } + has_cached_version &= metadata.version_hash == version_hash; + has_cached_version &= self + .has_compiled_source(&global_state.file_fetcher, &source_file.url); + + for dep in metadata.deps { + let url = Url::parse(&dep).expect("Dep is not a valid url"); + has_cached_version &= + self.has_compiled_source(&global_state.file_fetcher, &url); } } } - let source_file_ = source_file.clone(); + + if has_cached_version { + return Ok(()); + } + let module_url = source_file.url.clone(); - let module_specifier = ModuleSpecifier::from(source_file.url.clone()); - let import_map: Option<ImportMap> = - match global_state.flags.import_map_path.as_ref() { - None => None, - Some(file_path) => { - if !global_state.flags.unstable { - exit_unstable("--importmap") - } - Some(ImportMap::load(file_path)?) - } - }; - let mut module_graph_loader = ModuleGraphLoader::new( - global_state.file_fetcher.clone(), - import_map, - permissions.clone(), - is_dyn_import, - true, - ); - module_graph_loader.add_to_graph(&module_specifier).await?; - let module_graph = module_graph_loader.get_graph(); let module_graph_json = serde_json::to_value(module_graph).expect("Failed to serialize data"); let target = match target { @@ -500,23 +525,17 @@ impl TsCompiler { let req_msg = j.to_string().into_boxed_str().into_boxed_bytes(); - let ts_compiler = self.clone(); - + // TODO(bartlomieju): lift this call up - TSC shouldn't print anything info!( "{} {}", colors::green("Compile".to_string()), module_url.to_string() ); - let start = Instant::now(); - let msg = execute_in_same_thread(global_state.clone(), permissions, req_msg) .await?; - let end = Instant::now(); - debug!("time spent in compiler thread {:#?}", end - start); - let json_str = std::str::from_utf8(&msg).unwrap(); let compile_response: CompileResponse = serde_json::from_str(json_str)?; @@ -525,8 +544,69 @@ impl TsCompiler { return Err(ErrBox::from(compile_response.diagnostics)); } + self.set_graph_metadata( + source_file.url.clone(), + &compile_response.emit_map, + )?; self.cache_emitted_files(compile_response.emit_map)?; - ts_compiler.get_compiled_module(&source_file_.url) + Ok(()) + } + + fn get_graph_metadata(&self, url: &Url) -> Option<GraphFileMetadata> { + // Try to load cached version: + // 1. check if there's 'meta' file + let cache_key = self + .disk_cache + .get_cache_filename_with_extension(url, "graph"); + if let Ok(metadata_bytes) = self.disk_cache.get(&cache_key) { + if let Ok(metadata) = std::str::from_utf8(&metadata_bytes) { + if let Ok(read_metadata) = + GraphFileMetadata::from_json_string(metadata.to_string()) + { + return Some(read_metadata); + } + } + } + + None + } + + fn set_graph_metadata( + &self, + url: Url, + emit_map: &HashMap<String, EmittedSource>, + ) -> std::io::Result<()> { + let version_hash = + crate::checksum::gen(vec![version::DENO.as_bytes(), &self.config.hash]); + let mut deps = vec![]; + + for (_emitted_name, source) in emit_map.iter() { + let specifier = ModuleSpecifier::resolve_url(&source.filename) + .expect("Should be a valid module specifier"); + + let source_file = self + .file_fetcher + .fetch_cached_source_file(&specifier, Permissions::allow_all()) + .expect("Source file not found"); + + // NOTE: JavaScript files are only cached to disk if `checkJs` + // option in on + if source_file.media_type == msg::MediaType::JavaScript + && !self.compile_js + { + continue; + } + + deps.push(specifier.to_string()); + } + + let graph_metadata = GraphFileMetadata { deps, version_hash }; + let meta_key = self + .disk_cache + .get_cache_filename_with_extension(&url, "graph"); + self + .disk_cache + .set(&meta_key, graph_metadata.to_json_string()?.as_bytes()) } /// Get associated `CompiledFileMetadata` for given module if it exists. @@ -557,10 +637,23 @@ impl TsCompiler { let specifier = ModuleSpecifier::resolve_url(&source.filename) .expect("Should be a valid module specifier"); + let source_file = self + .file_fetcher + .fetch_cached_source_file(&specifier, Permissions::allow_all()) + .expect("Source file not found"); + + // NOTE: JavaScript files are only cached to disk if `checkJs` + // option in on + if source_file.media_type == msg::MediaType::JavaScript + && !self.compile_js + { + continue; + } + if emitted_name.ends_with(".map") { self.cache_source_map(&specifier, &source.contents)?; } else if emitted_name.ends_with(".js") { - self.cache_compiled_file(&specifier, &source.contents)?; + self.cache_compiled_file(&specifier, source_file, &source.contents)?; } else { panic!("Trying to cache unknown file type {}", emitted_name); } @@ -618,20 +711,9 @@ impl TsCompiler { fn cache_compiled_file( &self, module_specifier: &ModuleSpecifier, + source_file: SourceFile, contents: &str, ) -> std::io::Result<()> { - let source_file = self - .file_fetcher - .fetch_cached_source_file(&module_specifier, Permissions::allow_all()) - .expect("Source file not found"); - - // NOTE: JavaScript files are only cached to disk if `checkJs` - // option in on - if source_file.media_type == msg::MediaType::JavaScript && !self.compile_js - { - return Ok(()); - } - // By default TSC output source map url that is relative; we need // to substitute it manually to correct file URL in DENO_DIR. let mut content_lines = contents @@ -664,10 +746,6 @@ impl TsCompiler { .get_cache_filename_with_extension(module_specifier.as_url(), "js"); self.disk_cache.set(&js_key, contents.as_bytes())?; self.mark_compiled(module_specifier.as_url()); - let source_file = self - .file_fetcher - .fetch_cached_source_file(&module_specifier, Permissions::allow_all()) - .expect("Source file not found"); let version_hash = source_code_version_hash( &source_file.source_code, @@ -720,18 +798,6 @@ impl TsCompiler { module_specifier: &ModuleSpecifier, contents: &str, ) -> std::io::Result<()> { - let source_file = self - .file_fetcher - .fetch_cached_source_file(&module_specifier, Permissions::allow_all()) - .expect("Source file not found"); - - // NOTE: JavaScript files are only cached to disk if `checkJs` - // option in on - if source_file.media_type == msg::MediaType::JavaScript && !self.compile_js - { - return Ok(()); - } - let js_key = self .disk_cache .get_cache_filename_with_extension(module_specifier.as_url(), "js"); @@ -854,14 +920,12 @@ pub async fn bundle( compiler_config: CompilerConfig, module_specifier: ModuleSpecifier, maybe_import_map: Option<ImportMap>, - out_file: Option<PathBuf>, unstable: bool, -) -> Result<(), ErrBox> { +) -> Result<String, ErrBox> { debug!( "Invoking the compiler to bundle. module_name: {}", module_specifier.to_string() ); - eprintln!("Bundling {}", module_specifier.to_string()); let permissions = Permissions::allow_all(); let mut module_graph_loader = ModuleGraphLoader::new( @@ -871,7 +935,9 @@ pub async fn bundle( false, true, ); - module_graph_loader.add_to_graph(&module_specifier).await?; + module_graph_loader + .add_to_graph(&module_specifier, None) + .await?; let module_graph = module_graph_loader.get_graph(); let module_graph_json = serde_json::to_value(module_graph).expect("Failed to serialize data"); @@ -921,26 +987,7 @@ pub async fn bundle( assert!(bundle_response.bundle_output.is_some()); let output = bundle_response.bundle_output.unwrap(); - - // TODO(bartlomieju): the rest of this function should be handled - // in `main.rs` - it has nothing to do with TypeScript... - let output_string = fmt::format_text(&output)?; - - if let Some(out_file_) = out_file.as_ref() { - eprintln!("Emitting bundle to {:?}", out_file_); - - let output_bytes = output_string.as_bytes(); - let output_len = output_bytes.len(); - - deno_fs::write_file(out_file_, output_bytes, 0o666)?; - // TODO(bartlomieju): do we really need to show this info? (it doesn't respect --quiet flag) - // TODO(bartlomieju): add "humanFileSize" method - eprintln!("{} bytes emitted.", output_len); - } else { - println!("{}", output_string); - } - - Ok(()) + Ok(output) } /// This function is used by `Deno.compile()` and `Deno.bundle()` APIs. @@ -968,7 +1015,9 @@ pub async fn runtime_compile<S: BuildHasher>( let module_specifier = ModuleSpecifier::resolve_import(root_name, "<unknown>")?; root_names.push(module_specifier.to_string()); - module_graph_loader.add_to_graph(&module_specifier).await?; + module_graph_loader + .add_to_graph(&module_specifier, None) + .await?; } // download all additional files from TSconfig and add them to root_names @@ -983,7 +1032,9 @@ pub async fn runtime_compile<S: BuildHasher>( .expect("type is not a string") .to_string(); let type_specifier = ModuleSpecifier::resolve_url_or_path(&type_str)?; - module_graph_loader.add_to_graph(&type_specifier).await?; + module_graph_loader + .add_to_graph(&type_specifier, None) + .await?; root_names.push(type_specifier.to_string()) } } @@ -1078,18 +1129,36 @@ mod tests { }; let mock_state = GlobalState::mock(vec![String::from("deno"), String::from("hello.ts")]); + + let mut module_graph_loader = ModuleGraphLoader::new( + mock_state.file_fetcher.clone(), + None, + Permissions::allow_all(), + false, + false, + ); + module_graph_loader + .add_to_graph(&specifier, None) + .await + .expect("Failed to create graph"); + let module_graph = module_graph_loader.get_graph(); + let result = mock_state .ts_compiler - .compile( + .compile_module_graph( mock_state.clone(), &out, TargetLib::Main, Permissions::allow_all(), - false, + module_graph, ) .await; assert!(result.is_ok()); - let source_code = result.unwrap().code; + let compiled_file = mock_state + .ts_compiler + .get_compiled_module(&out.url) + .unwrap(); + let source_code = compiled_file.code; assert!(source_code .as_bytes() .starts_with(b"\"use strict\";\nconsole.log(\"Hello World\");")); @@ -1143,7 +1212,6 @@ mod tests { CompilerConfig::load(None).unwrap(), module_name, None, - None, false, ) .await; |