From d672e1405dd7085a060625fc320d063f7f7970a2 Mon Sep 17 00:00:00 2001 From: Kitson Kelly Date: Tue, 3 Nov 2020 06:41:20 +1100 Subject: refactor(cli): cleanup compiler snapshot and tsc/module_graph (#8220) --- cli/Cargo.toml | 2 + cli/build.rs | 169 +++- cli/main.rs | 23 +- cli/module_graph.rs | 2209 +++++++++++++++++++++++++++++++++++++++++++ cli/module_graph2.rs | 2208 ------------------------------------------ cli/module_loader.rs | 2 +- cli/op_fetch_asset.rs | 115 --- cli/ops/runtime_compiler.rs | 8 +- cli/program_state.rs | 10 +- cli/tsc.rs | 750 +++++++++++++++ cli/tsc/99_main_compiler.js | 584 +++--------- cli/tsc2.rs | 726 -------------- 12 files changed, 3265 insertions(+), 3541 deletions(-) create mode 100644 cli/module_graph.rs delete mode 100644 cli/module_graph2.rs delete mode 100644 cli/op_fetch_asset.rs create mode 100644 cli/tsc.rs delete mode 100644 cli/tsc2.rs diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 731fb2efb..ec06e1f70 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -23,6 +23,8 @@ path = "./bench/main.rs" deno_core = { path = "../core", version = "0.66.0" } deno_web = { path = "../op_crates/web", version = "0.17.0" } deno_fetch = { path = "../op_crates/fetch", version = "0.9.0" } +regex = "1.3.9" +serde = { version = "1.0.116", features = ["derive"] } [target.'cfg(windows)'.build-dependencies] winres = "0.1.11" diff --git a/cli/build.rs b/cli/build.rs index 4819988e9..657041132 100644 --- a/cli/build.rs +++ b/cli/build.rs @@ -1,9 +1,13 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -mod op_fetch_asset; - +use deno_core::error::custom_error; +use deno_core::json_op_sync; +use deno_core::serde_json; +use deno_core::serde_json::json; use deno_core::JsRuntime; use deno_core::RuntimeOptions; +use regex::Regex; +use serde::Deserialize; use std::collections::HashMap; use std::env; use std::path::Path; @@ -46,46 +50,149 @@ fn create_runtime_snapshot(snapshot_path: &Path, files: Vec) { create_snapshot(js_runtime, snapshot_path, files); } +#[derive(Debug, Deserialize)] +struct LoadArgs { + /// The fully qualified specifier that should be loaded. + specifier: String, +} + fn create_compiler_snapshot( snapshot_path: &Path, files: Vec, cwd: &Path, ) { - let mut custom_libs: HashMap = HashMap::new(); - custom_libs - .insert("lib.deno.web.d.ts".to_string(), deno_web::get_declaration()); - custom_libs.insert( - "lib.deno.fetch.d.ts".to_string(), - deno_fetch::get_declaration(), - ); - custom_libs.insert( - "lib.deno.window.d.ts".to_string(), - cwd.join("dts/lib.deno.window.d.ts"), - ); - custom_libs.insert( - "lib.deno.worker.d.ts".to_string(), - cwd.join("dts/lib.deno.worker.d.ts"), - ); - custom_libs.insert( - "lib.deno.shared_globals.d.ts".to_string(), - cwd.join("dts/lib.deno.shared_globals.d.ts"), - ); - custom_libs.insert( - "lib.deno.ns.d.ts".to_string(), - cwd.join("dts/lib.deno.ns.d.ts"), - ); - custom_libs.insert( - "lib.deno.unstable.d.ts".to_string(), - cwd.join("dts/lib.deno.unstable.d.ts"), - ); + // libs that are being provided by op crates. + let mut op_crate_libs = HashMap::new(); + op_crate_libs.insert("deno.web", deno_web::get_declaration()); + op_crate_libs.insert("deno.fetch", deno_fetch::get_declaration()); + + // ensure we invalidate the build properly. + for (_, path) in op_crate_libs.iter() { + println!("cargo:rerun-if-changed={}", path.display()); + } + + // libs that should be loaded into the isolate before snapshotting. + let libs = vec![ + // Deno custom type libraries + "deno.window", + "deno.worker", + "deno.shared_globals", + "deno.ns", + "deno.unstable", + // Deno built-in type libraries + "es5", + "es2015.collection", + "es2015.core", + "es2015", + "es2015.generator", + "es2015.iterable", + "es2015.promise", + "es2015.proxy", + "es2015.reflect", + "es2015.symbol", + "es2015.symbol.wellknown", + "es2016.array.include", + "es2016", + "es2017", + "es2017.intl", + "es2017.object", + "es2017.sharedmemory", + "es2017.string", + "es2017.typedarrays", + "es2018.asyncgenerator", + "es2018.asynciterable", + "es2018", + "es2018.intl", + "es2018.promise", + "es2018.regexp", + "es2019.array", + "es2019", + "es2019.object", + "es2019.string", + "es2019.symbol", + "es2020.bigint", + "es2020", + "es2020.intl", + "es2020.promise", + "es2020.string", + "es2020.symbol.wellknown", + "esnext", + "esnext.intl", + "esnext.promise", + "esnext.string", + "esnext.weakref", + ]; + + // create a copy of the vector that includes any op crate libs to be passed + // to the JavaScript compiler to build into the snapshot + let mut build_libs = libs.clone(); + for (op_lib, _) in op_crate_libs.iter() { + build_libs.push(op_lib.to_owned()); + } + + let re_asset = Regex::new(r"asset:/{3}lib\.(\S+)\.d\.ts").expect("bad regex"); + let path_dts = cwd.join("dts"); + let build_specifier = "asset:///bootstrap.ts"; let mut js_runtime = JsRuntime::new(RuntimeOptions { will_snapshot: true, ..Default::default() }); js_runtime.register_op( - "op_fetch_asset", - op_fetch_asset::op_fetch_asset(custom_libs), + "op_build_info", + json_op_sync(move |_state, _args, _bufs| { + Ok(json!({ + "buildSpecifier": build_specifier, + "libs": build_libs, + })) + }), + ); + // using the same op that is used in `tsc.rs` for loading modules and reading + // files, but a slightly different implementation at build time. + js_runtime.register_op( + "op_load", + json_op_sync(move |_state, args, _bufs| { + let v: LoadArgs = serde_json::from_value(args)?; + // we need a basic file to send to tsc to warm it up. + if v.specifier == build_specifier { + Ok(json!({ + "data": r#"console.log("hello deno!");"#, + "hash": "1", + // this corresponds to `ts.ScriptKind.TypeScript` + "scriptKind": 3 + })) + // specifiers come across as `asset:///lib.{lib_name}.d.ts` and we need to + // parse out just the name so we can lookup the asset. + } else if let Some(caps) = re_asset.captures(&v.specifier) { + if let Some(lib) = caps.get(1).map(|m| m.as_str()) { + // if it comes from an op crate, we were supplied with the path to the + // file. + let path = if let Some(op_crate_lib) = op_crate_libs.get(lib) { + op_crate_lib.clone() + // otherwise we are will generate the path ourself + } else { + path_dts.join(format!("lib.{}.d.ts", lib)) + }; + let data = std::fs::read_to_string(path)?; + Ok(json!({ + "data": data, + "hash": "1", + // this corresponds to `ts.ScriptKind.TypeScript` + "scriptKind": 3 + })) + } else { + Err(custom_error( + "InvalidSpecifier", + format!("An invalid specifier was requested: {}", v.specifier), + )) + } + } else { + Err(custom_error( + "InvalidSpecifier", + format!("An invalid specifier was requested: {}", v.specifier), + )) + } + }), ); create_snapshot(js_runtime, snapshot_path, files); } diff --git a/cli/main.rs b/cli/main.rs index dc68546d5..7773f7d48 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -35,9 +35,8 @@ mod lint; mod lockfile; mod media_type; mod metrics; -mod module_graph2; +mod module_graph; mod module_loader; -mod op_fetch_asset; mod ops; mod permissions; mod program_state; @@ -49,7 +48,7 @@ mod specifier_handler; mod test_runner; mod text_encoding; mod tokio_util; -mod tsc2; +mod tsc; mod tsc_config; mod upgrade; mod version; @@ -177,7 +176,7 @@ async fn info_command( // so we allow access to all of them. Permissions::allow_all(), )?)); - let mut builder = module_graph2::GraphBuilder2::new( + let mut builder = module_graph::GraphBuilder::new( handler, program_state.maybe_import_map.clone(), program_state.lockfile.clone(), @@ -241,9 +240,9 @@ async fn cache_command( files: Vec, ) -> Result<(), AnyError> { let lib = if flags.unstable { - module_graph2::TypeLib::UnstableDenoWindow + module_graph::TypeLib::UnstableDenoWindow } else { - module_graph2::TypeLib::DenoWindow + module_graph::TypeLib::DenoWindow }; let program_state = ProgramState::new(flags)?; @@ -329,7 +328,7 @@ async fn bundle_command( // therefore we will allow the graph to access any module. Permissions::allow_all(), )?)); - let mut builder = module_graph2::GraphBuilder2::new( + let mut builder = module_graph::GraphBuilder::new( handler, program_state.maybe_import_map.clone(), program_state.lockfile.clone(), @@ -341,12 +340,12 @@ async fn bundle_command( if !flags.no_check { // TODO(@kitsonk) support bundling for workers let lib = if flags.unstable { - module_graph2::TypeLib::UnstableDenoWindow + module_graph::TypeLib::UnstableDenoWindow } else { - module_graph2::TypeLib::DenoWindow + module_graph::TypeLib::DenoWindow }; let graph = graph.clone(); - let result_info = graph.check(module_graph2::CheckOptions { + let result_info = graph.check(module_graph::CheckOptions { debug, emit: false, lib, @@ -364,7 +363,7 @@ async fn bundle_command( } let (output, stats, maybe_ignored_options) = - graph.bundle(module_graph2::BundleOptions { + graph.bundle(module_graph::BundleOptions { debug, maybe_config_path: flags.config_path, })?; @@ -563,7 +562,7 @@ async fn run_with_watch(flags: Flags, script: String) -> Result<(), AnyError> { &program_state, Permissions::allow_all(), )?)); - let mut builder = module_graph2::GraphBuilder2::new( + let mut builder = module_graph::GraphBuilder::new( handler, program_state.maybe_import_map.clone(), program_state.lockfile.clone(), diff --git a/cli/module_graph.rs b/cli/module_graph.rs new file mode 100644 index 000000000..d757aa0a1 --- /dev/null +++ b/cli/module_graph.rs @@ -0,0 +1,2209 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +use crate::ast; +use crate::ast::parse; +use crate::ast::transpile_module; +use crate::ast::BundleHook; +use crate::ast::Location; +use crate::ast::ParsedModule; +use crate::colors; +use crate::diagnostics::Diagnostics; +use crate::import_map::ImportMap; +use crate::info::ModuleGraphInfo; +use crate::info::ModuleInfo; +use crate::info::ModuleInfoMap; +use crate::info::ModuleInfoMapItem; +use crate::js; +use crate::lockfile::Lockfile; +use crate::media_type::MediaType; +use crate::specifier_handler::CachedModule; +use crate::specifier_handler::Dependency; +use crate::specifier_handler::DependencyMap; +use crate::specifier_handler::Emit; +use crate::specifier_handler::FetchFuture; +use crate::specifier_handler::SpecifierHandler; +use crate::tsc; +use crate::tsc_config::IgnoredCompilerOptions; +use crate::tsc_config::TsConfig; +use crate::version; +use crate::AnyError; + +use deno_core::error::Context; +use deno_core::futures::stream::FuturesUnordered; +use deno_core::futures::stream::StreamExt; +use deno_core::serde::Serialize; +use deno_core::serde::Serializer; +use deno_core::serde_json::json; +use deno_core::serde_json::Value; +use deno_core::ModuleResolutionError; +use deno_core::ModuleSpecifier; +use regex::Regex; +use serde::Deserialize; +use serde::Deserializer; +use std::cell::RefCell; +use std::collections::HashMap; +use std::collections::HashSet; +use std::error::Error; +use std::fmt; +use std::path::PathBuf; +use std::rc::Rc; +use std::result; +use std::sync::Arc; +use std::sync::Mutex; +use std::time::Instant; + +lazy_static! { + /// Matched the `@deno-types` pragma. + static ref DENO_TYPES_RE: Regex = + Regex::new(r#"(?i)^\s*@deno-types\s*=\s*(?:["']([^"']+)["']|(\S+))"#) + .unwrap(); + /// Matches a `/// ` comment reference. + static ref TRIPLE_SLASH_REFERENCE_RE: Regex = + Regex::new(r"(?i)^/\s*").unwrap(); + /// Matches a path reference, which adds a dependency to a module + static ref PATH_REFERENCE_RE: Regex = + Regex::new(r#"(?i)\spath\s*=\s*["']([^"']*)["']"#).unwrap(); + /// Matches a types reference, which for JavaScript files indicates the + /// location of types to use when type checking a program that includes it as + /// a dependency. + static ref TYPES_REFERENCE_RE: Regex = + Regex::new(r#"(?i)\stypes\s*=\s*["']([^"']*)["']"#).unwrap(); +} + +/// A group of errors that represent errors that can occur when interacting with +/// a module graph. +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum GraphError { + /// A module using the HTTPS protocol is trying to import a module with an + /// HTTP schema. + InvalidDowngrade(ModuleSpecifier, Location), + /// A remote module is trying to import a local module. + InvalidLocalImport(ModuleSpecifier, Location), + /// The source code is invalid, as it does not match the expected hash in the + /// lockfile. + InvalidSource(ModuleSpecifier, PathBuf), + /// An unexpected dependency was requested for a module. + MissingDependency(ModuleSpecifier, String), + /// An unexpected specifier was requested. + MissingSpecifier(ModuleSpecifier), + /// The current feature is not supported. + NotSupported(String), + /// A unsupported media type was attempted to be imported as a module. + UnsupportedImportType(ModuleSpecifier, MediaType), +} + +impl fmt::Display for GraphError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + match self { + GraphError::InvalidDowngrade(ref specifier, ref location) => write!(f, "Modules imported via https are not allowed to import http modules.\n Importing: {}\n at {}", specifier, location), + GraphError::InvalidLocalImport(ref specifier, ref location) => write!(f, "Remote modules are not allowed to import local modules. Consider using a dynamic import instead.\n Importing: {}\n at {}", specifier, location), + GraphError::InvalidSource(ref specifier, ref lockfile) => write!(f, "The source code is invalid, as it does not match the expected hash in the lock file.\n Specifier: {}\n Lock file: {}", specifier, lockfile.to_str().unwrap()), + GraphError::MissingDependency(ref referrer, specifier) => write!( + f, + "The graph is missing a dependency.\n Specifier: {} from {}", + specifier, referrer + ), + GraphError::MissingSpecifier(ref specifier) => write!( + f, + "The graph is missing a specifier.\n Specifier: {}", + specifier + ), + GraphError::NotSupported(ref msg) => write!(f, "{}", msg), + GraphError::UnsupportedImportType(ref specifier, ref media_type) => write!(f, "An unsupported media type was attempted to be imported as a module.\n Specifier: {}\n MediaType: {}", specifier, media_type), + } + } +} + +impl Error for GraphError {} + +/// A structure for handling bundle loading, which is implemented here, to +/// avoid a circular dependency with `ast`. +struct BundleLoader<'a> { + cm: Rc, + emit_options: &'a ast::EmitOptions, + globals: &'a swc_common::Globals, + graph: &'a Graph, +} + +impl<'a> BundleLoader<'a> { + pub fn new( + graph: &'a Graph, + emit_options: &'a ast::EmitOptions, + globals: &'a swc_common::Globals, + cm: Rc, + ) -> Self { + BundleLoader { + cm, + emit_options, + globals, + graph, + } + } +} + +impl swc_bundler::Load for BundleLoader<'_> { + fn load( + &self, + file: &swc_common::FileName, + ) -> Result<(Rc, swc_ecmascript::ast::Module), AnyError> + { + match file { + swc_common::FileName::Custom(filename) => { + let specifier = ModuleSpecifier::resolve_url_or_path(filename) + .context("Failed to convert swc FileName to ModuleSpecifier.")?; + if let Some(src) = self.graph.get_source(&specifier) { + let media_type = self + .graph + .get_media_type(&specifier) + .context("Looking up media type during bundling.")?; + transpile_module( + filename, + &src, + &media_type, + self.emit_options, + self.globals, + self.cm.clone(), + ) + } else { + Err( + GraphError::MissingDependency(specifier, "".to_string()) + .into(), + ) + } + } + _ => unreachable!("Received request for unsupported filename {:?}", file), + } + } +} + +/// An enum which represents the parsed out values of references in source code. +#[derive(Debug, Clone, Eq, PartialEq)] +enum TypeScriptReference { + Path(String), + Types(String), +} + +/// Determine if a comment contains a triple slash reference and optionally +/// return its kind and value. +fn parse_ts_reference(comment: &str) -> Option { + if !TRIPLE_SLASH_REFERENCE_RE.is_match(comment) { + None + } else if let Some(captures) = PATH_REFERENCE_RE.captures(comment) { + Some(TypeScriptReference::Path( + captures.get(1).unwrap().as_str().to_string(), + )) + } else if let Some(captures) = TYPES_REFERENCE_RE.captures(comment) { + Some(TypeScriptReference::Types( + captures.get(1).unwrap().as_str().to_string(), + )) + } else { + None + } +} + +/// Determine if a comment contains a `@deno-types` pragma and optionally return +/// its value. +fn parse_deno_types(comment: &str) -> Option { + if let Some(captures) = DENO_TYPES_RE.captures(comment) { + if let Some(m) = captures.get(1) { + Some(m.as_str().to_string()) + } else if let Some(m) = captures.get(2) { + Some(m.as_str().to_string()) + } else { + panic!("unreachable"); + } + } else { + None + } +} + +/// A hashing function that takes the source code, version and optionally a +/// user provided config and generates a string hash which can be stored to +/// determine if the cached emit is valid or not. +fn get_version(source: &str, version: &str, config: &[u8]) -> String { + crate::checksum::gen(&[source.as_bytes(), version.as_bytes(), config]) +} + +/// A logical representation of a module within a graph. +#[derive(Debug, Clone)] +struct Module { + dependencies: DependencyMap, + is_dirty: bool, + is_parsed: bool, + maybe_emit: Option, + maybe_emit_path: Option<(PathBuf, Option)>, + maybe_import_map: Option>>, + maybe_parsed_module: Option, + maybe_types: Option<(String, ModuleSpecifier)>, + maybe_version: Option, + media_type: MediaType, + specifier: ModuleSpecifier, + source: String, + source_path: PathBuf, +} + +impl Default for Module { + fn default() -> Self { + Module { + dependencies: HashMap::new(), + is_dirty: false, + is_parsed: false, + maybe_emit: None, + maybe_emit_path: None, + maybe_import_map: None, + maybe_parsed_module: None, + maybe_types: None, + maybe_version: None, + media_type: MediaType::Unknown, + specifier: ModuleSpecifier::resolve_url("file:///example.js").unwrap(), + source: "".to_string(), + source_path: PathBuf::new(), + } + } +} + +impl Module { + pub fn new( + cached_module: CachedModule, + is_root: bool, + maybe_import_map: Option>>, + ) -> Self { + // If this is a local root file, and its media type is unknown, set the + // media type to JavaScript. This allows easier ability to create "shell" + // scripts with Deno. + let media_type = if is_root + && !cached_module.is_remote + && cached_module.media_type == MediaType::Unknown + { + MediaType::JavaScript + } else { + cached_module.media_type + }; + let mut module = Module { + specifier: cached_module.specifier, + maybe_import_map, + media_type, + source: cached_module.source, + source_path: cached_module.source_path, + maybe_emit: cached_module.maybe_emit, + maybe_emit_path: cached_module.maybe_emit_path, + maybe_version: cached_module.maybe_version, + is_dirty: false, + ..Self::default() + }; + if module.maybe_import_map.is_none() { + if let Some(dependencies) = cached_module.maybe_dependencies { + module.dependencies = dependencies; + module.is_parsed = true; + } + } + module.maybe_types = if let Some(ref specifier) = cached_module.maybe_types + { + Some(( + specifier.clone(), + module + .resolve_import(&specifier, None) + .expect("could not resolve module"), + )) + } else { + None + }; + module + } + + /// Return `true` if the current hash of the module matches the stored + /// version. + pub fn is_emit_valid(&self, config: &[u8]) -> bool { + if let Some(version) = self.maybe_version.clone() { + version == get_version(&self.source, version::DENO, config) + } else { + false + } + } + + /// Parse a module, populating the structure with data retrieved from the + /// source of the module. + pub fn parse(&mut self) -> Result<(), AnyError> { + let parsed_module = + parse(self.specifier.as_str(), &self.source, &self.media_type)?; + + // parse out any triple slash references + for comment in parsed_module.get_leading_comments().iter() { + if let Some(ts_reference) = parse_ts_reference(&comment.text) { + let location = parsed_module.get_location(&comment.span); + match ts_reference { + TypeScriptReference::Path(import) => { + let specifier = + self.resolve_import(&import, Some(location.clone()))?; + let dep = self + .dependencies + .entry(import) + .or_insert_with(|| Dependency::new(location)); + dep.maybe_code = Some(specifier); + } + TypeScriptReference::Types(import) => { + let specifier = + self.resolve_import(&import, Some(location.clone()))?; + if self.media_type == MediaType::JavaScript + || self.media_type == MediaType::JSX + { + // TODO(kitsonk) we need to specifically update the cache when + // this value changes + self.maybe_types = Some((import.clone(), specifier)); + } else { + let dep = self + .dependencies + .entry(import) + .or_insert_with(|| Dependency::new(location)); + dep.maybe_type = Some(specifier); + } + } + } + } + } + + // Parse out all the syntactical dependencies for a module + let dependencies = parsed_module.analyze_dependencies(); + for desc in dependencies.iter().filter(|desc| { + desc.kind != swc_ecmascript::dep_graph::DependencyKind::Require + }) { + let location = Location { + filename: self.specifier.to_string(), + col: desc.col, + line: desc.line, + }; + + // In situations where there is a potential issue with resolving the + // import specifier, that ends up being a module resolution error for a + // code dependency, we should not throw in the `ModuleGraph` but instead + // wait until runtime and throw there, as with dynamic imports they need + // to be catchable, which means they need to be resolved at runtime. + let maybe_specifier = + match self.resolve_import(&desc.specifier, Some(location.clone())) { + Ok(specifier) => Some(specifier), + Err(any_error) => { + match any_error.downcast_ref::() { + Some(ModuleResolutionError::ImportPrefixMissing(_, _)) => None, + _ => { + return Err(any_error); + } + } + } + }; + + // Parse out any `@deno-types` pragmas and modify dependency + let maybe_type = if !desc.leading_comments.is_empty() { + let comment = desc.leading_comments.last().unwrap(); + if let Some(deno_types) = parse_deno_types(&comment.text).as_ref() { + Some(self.resolve_import(deno_types, Some(location.clone()))?) + } else { + None + } + } else { + None + }; + + let dep = self + .dependencies + .entry(desc.specifier.to_string()) + .or_insert_with(|| Dependency::new(location)); + dep.is_dynamic = desc.is_dynamic; + if let Some(specifier) = maybe_specifier { + if desc.kind == swc_ecmascript::dep_graph::DependencyKind::ExportType + || desc.kind == swc_ecmascript::dep_graph::DependencyKind::ImportType + { + dep.maybe_type = Some(specifier); + } else { + dep.maybe_code = Some(specifier); + } + } + // If the dependency wasn't a type only dependency already, and there is + // a `@deno-types` comment, then we will set the `maybe_type` dependency. + if maybe_type.is_some() && dep.maybe_type.is_none() { + dep.maybe_type = maybe_type; + } + } + + self.maybe_parsed_module = Some(parsed_module); + Ok(()) + } + + fn resolve_import( + &self, + specifier: &str, + maybe_location: Option, + ) -> Result { + let maybe_resolve = if let Some(import_map) = self.maybe_import_map.clone() + { + import_map + .borrow() + .resolve(specifier, self.specifier.as_str())? + } else { + None + }; + let specifier = if let Some(module_specifier) = maybe_resolve { + module_specifier + } else { + ModuleSpecifier::resolve_import(specifier, self.specifier.as_str())? + }; + + let referrer_scheme = self.specifier.as_url().scheme(); + let specifier_scheme = specifier.as_url().scheme(); + let location = maybe_location.unwrap_or(Location { + filename: self.specifier.to_string(), + line: 0, + col: 0, + }); + + // Disallow downgrades from HTTPS to HTTP + if referrer_scheme == "https" && specifier_scheme == "http" { + return Err( + GraphError::InvalidDowngrade(specifier.clone(), location).into(), + ); + } + + // Disallow a remote URL from trying to import a local URL + if (referrer_scheme == "https" || referrer_scheme == "http") + && !(specifier_scheme == "https" || specifier_scheme == "http") + { + return Err( + GraphError::InvalidLocalImport(specifier.clone(), location).into(), + ); + } + + Ok(specifier) + } + + /// Calculate the hashed version of the module and update the `maybe_version`. + pub fn set_version(&mut self, config: &[u8]) { + self.maybe_version = Some(get_version(&self.source, version::DENO, config)) + } + + pub fn size(&self) -> usize { + self.source.as_bytes().len() + } +} + +#[derive(Clone, Debug, Default, Eq, PartialEq)] +pub struct Stats(pub Vec<(String, u128)>); + +impl<'de> Deserialize<'de> for Stats { + fn deserialize(deserializer: D) -> result::Result + where + D: Deserializer<'de>, + { + let items: Vec<(String, u128)> = Deserialize::deserialize(deserializer)?; + Ok(Stats(items)) + } +} + +impl fmt::Display for Stats { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + writeln!(f, "Compilation statistics:")?; + for (key, value) in self.0.clone() { + writeln!(f, " {}: {}", key, value)?; + } + + Ok(()) + } +} + +/// A structure that provides information about a module graph result. +#[derive(Debug, Default)] +pub struct ResultInfo { + /// A structure which provides diagnostic information (usually from `tsc`) + /// about the code in the module graph. + pub diagnostics: Diagnostics, + /// Optionally ignored compiler options that represent any options that were + /// ignored if there was a user provided configuration. + pub maybe_ignored_options: Option, + /// A structure providing key metrics around the operation performed, in + /// milliseconds. + pub stats: Stats, +} + +/// Represents the "default" type library that should be used when type +/// checking the code in the module graph. Note that a user provided config +/// of `"lib"` would override this value. +#[derive(Debug, Clone, Eq, PartialEq)] +pub enum TypeLib { + DenoWindow, + DenoWorker, + UnstableDenoWindow, + UnstableDenoWorker, +} + +impl Default for TypeLib { + fn default() -> Self { + TypeLib::DenoWindow + } +} + +impl Serialize for TypeLib { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + let value = match self { + TypeLib::DenoWindow => vec!["deno.window".to_string()], + TypeLib::DenoWorker => vec!["deno.worker".to_string()], + TypeLib::UnstableDenoWindow => { + vec!["deno.window".to_string(), "deno.unstable".to_string()] + } + TypeLib::UnstableDenoWorker => { + vec!["deno.worker".to_string(), "deno.worker".to_string()] + } + }; + Serialize::serialize(&value, serializer) + } +} + +#[derive(Debug, Default)] +pub struct BundleOptions { + /// If `true` then debug logging will be output from the isolate. + pub debug: bool, + /// An optional string that points to a user supplied TypeScript configuration + /// file that augments the the default configuration passed to the TypeScript + /// compiler. + pub maybe_config_path: Option, +} + +#[derive(Debug, Default)] +pub struct CheckOptions { + /// If `true` then debug logging will be output from the isolate. + pub debug: bool, + /// Utilise the emit from `tsc` to update the emitted code for modules. + pub emit: bool, + /// The base type libraries that should be used when type checking. + pub lib: TypeLib, + /// An optional string that points to a user supplied TypeScript configuration + /// file that augments the the default configuration passed to the TypeScript + /// compiler. + pub maybe_config_path: Option, + /// Ignore any previously emits and ensure that all files are emitted from + /// source. + pub reload: bool, +} + +#[derive(Debug, Eq, PartialEq)] +pub enum BundleType { + /// Return the emitted contents of the program as a single "flattened" ES + /// module. + Esm, + // TODO(@kitsonk) once available in swc + // Iife, + /// Do not bundle the emit, instead returning each of the modules that are + /// part of the program as individual files. + None, +} + +impl Default for BundleType { + fn default() -> Self { + BundleType::None + } +} + +#[derive(Debug, Default)] +pub struct EmitOptions { + /// Indicate the form the result of the emit should take. + pub bundle_type: BundleType, + /// If `true` then debug logging will be output from the isolate. + pub debug: bool, + /// An optional map that contains user supplied TypeScript compiler + /// configuration options that are passed to the TypeScript compiler. + pub maybe_user_config: Option>, +} + +/// A structure which provides options when transpiling modules. +#[derive(Debug, Default)] +pub struct TranspileOptions { + /// If `true` then debug logging will be output from the isolate. + pub debug: bool, + /// An optional string that points to a user supplied TypeScript configuration + /// file that augments the the default configuration passed to the TypeScript + /// compiler. + pub maybe_config_path: Option, + /// Ignore any previously emits and ensure that all files are emitted from + /// source. + pub reload: bool, +} + +/// A dependency graph of modules, were the modules that have been inserted via +/// the builder will be loaded into the graph. Also provides an interface to +/// be able to manipulate and handle the graph. +#[derive(Debug, Clone)] +pub struct Graph { + /// A reference to the specifier handler that will retrieve and cache modules + /// for the graph. + handler: Rc>, + /// Optional TypeScript build info that will be passed to `tsc` if `tsc` is + /// invoked. + maybe_tsbuildinfo: Option, + /// The modules that are part of the graph. + modules: HashMap, + /// A map of redirects, where a module specifier is redirected to another + /// module specifier by the handler. All modules references should be + /// resolved internally via this, before attempting to access the module via + /// the handler, to make sure the correct modules is being dealt with. + redirects: HashMap, + /// The module specifiers that have been uniquely added to the graph, which + /// does not include any transient dependencies. + roots: Vec, + /// If all of the root modules are dynamically imported, then this is true. + /// This is used to ensure correct `--reload` behavior, where subsequent + /// calls to a module graph where the emit is already valid do not cause the + /// graph to re-emit. + roots_dynamic: bool, + // A reference to lock file that will be used to check module integrity. + maybe_lockfile: Option>>, +} + +impl Graph { + /// Create a new instance of a graph, ready to have modules loaded it. + /// + /// The argument `handler` is an instance of a structure that implements the + /// `SpecifierHandler` trait. + /// + pub fn new( + handler: Rc>, + maybe_lockfile: Option>>, + ) -> Self { + Graph { + handler, + maybe_tsbuildinfo: None, + modules: HashMap::new(), + redirects: HashMap::new(), + roots: Vec::new(), + roots_dynamic: true, + maybe_lockfile, + } + } + + /// Transform the module graph into a single JavaScript module which is + /// returned as a `String` in the result. + pub fn bundle( + &self, + options: BundleOptions, + ) -> Result<(String, Stats, Option), AnyError> { + if self.roots.is_empty() || self.roots.len() > 1 { + return Err(GraphError::NotSupported(format!("Bundling is only supported when there is a single root module in the graph. Found: {}", self.roots.len())).into()); + } + + let start = Instant::now(); + let root_specifier = self.roots[0].clone(); + let mut ts_config = TsConfig::new(json!({ + "checkJs": false, + "emitDecoratorMetadata": false, + "inlineSourceMap": true, + "jsx": "react", + "jsxFactory": "React.createElement", + "jsxFragmentFactory": "React.Fragment", + })); + let maybe_ignored_options = + ts_config.merge_tsconfig(options.maybe_config_path)?; + + let s = self.emit_bundle(&root_specifier, &ts_config.into())?; + let stats = Stats(vec![ + ("Files".to_string(), self.modules.len() as u128), + ("Total time".to_string(), start.elapsed().as_millis()), + ]); + + Ok((s, stats, maybe_ignored_options)) + } + + /// Type check the module graph, corresponding to the options provided. + pub fn check(self, options: CheckOptions) -> Result { + let mut config = TsConfig::new(json!({ + "allowJs": true, + // TODO(@kitsonk) is this really needed? + "esModuleInterop": true, + // Enabled by default to align to transpile/swc defaults + "experimentalDecorators": true, + "incremental": true, + "isolatedModules": true, + "lib": options.lib, + "module": "esnext", + "strict": true, + "target": "esnext", + "tsBuildInfoFile": "deno:///.tsbuildinfo", + })); + if options.emit { + config.merge(&json!({ + // TODO(@kitsonk) consider enabling this by default + // see: https://github.com/denoland/deno/issues/7732 + "emitDecoratorMetadata": false, + "jsx": "react", + "inlineSourceMap": true, + "outDir": "deno://", + "removeComments": true, + })); + } else { + config.merge(&json!({ + "noEmit": true, + })); + } + let maybe_ignored_options = + config.merge_tsconfig(options.maybe_config_path)?; + + // Short circuit if none of the modules require an emit, or all of the + // modules that require an emit have a valid emit. There is also an edge + // case where there are multiple imports of a dynamic module during a + // single invocation, if that is the case, even if there is a reload, we + // will simply look at if the emit is invalid, to avoid two checks for the + // same programme. + if !self.needs_emit(&config) + || (self.is_emit_valid(&config) + && (!options.reload || self.roots_dynamic)) + { + debug!("graph does not need to be checked or emitted."); + return Ok(ResultInfo { + maybe_ignored_options, + ..Default::default() + }); + } + + // TODO(@kitsonk) not totally happy with this here, but this is the first + // point where we know we are actually going to check the program. If we + // moved it out of here, we wouldn't know until after the check has already + // happened, which isn't informative to the users. + for specifier in &self.roots { + info!("{} {}", colors::green("Check"), specifier); + } + + let root_names = self.get_root_names(); + let maybe_tsbuildinfo = self.maybe_tsbuildinfo.clone(); + let hash_data = + vec![config.as_bytes(), version::DENO.as_bytes().to_owned()]; + let graph = Rc::new(RefCell::new(self)); + + let response = tsc::exec( + js::compiler_isolate_init(), + tsc::Request { + config: config.clone(), + debug: options.debug, + graph: graph.clone(), + hash_data, + maybe_tsbuildinfo, + root_names, + }, + )?; + + let mut graph = graph.borrow_mut(); + graph.maybe_tsbuildinfo = response.maybe_tsbuildinfo; + // Only process changes to the graph if there are no diagnostics and there + // were files emitted. + if response.diagnostics.is_empty() && !response.emitted_files.is_empty() { + let mut codes = HashMap::new(); + let mut maps = HashMap::new(); + let check_js = config.get_check_js(); + for emit in &response.emitted_files { + if let Some(specifiers) = &emit.maybe_specifiers { + assert!(specifiers.len() == 1, "Unexpected specifier length"); + // The specifier emitted might not be the redirected specifier, and + // therefore we need to ensure it is the correct one. + let specifier = graph.resolve_specifier(&specifiers[0]); + // Sometimes if tsc sees a CommonJS file it will _helpfully_ output it + // to ESM, which we don't really want unless someone has enabled the + // check_js option. + if !check_js + && graph.get_media_type(&specifier) == Some(MediaType::JavaScript) + { + debug!("skipping emit for {}", specifier); + continue; + } + match emit.media_type { + MediaType::JavaScript => { + codes.insert(specifier.clone(), emit.data.clone()); + } + MediaType::SourceMap => { + maps.insert(specifier.clone(), emit.data.clone()); + } + _ => unreachable!(), + } + } + } + let config = config.as_bytes(); + for (specifier, code) in codes.iter() { + if let Some(module) = graph.get_module_mut(specifier) { + module.maybe_emit = + Some(Emit::Cli((code.clone(), maps.get(specifier).cloned()))); + module.set_version(&config); + module.is_dirty = true; + } else { + return Err(GraphError::MissingSpecifier(specifier.clone()).into()); + } + } + } + graph.flush()?; + + Ok(ResultInfo { + diagnostics: response.diagnostics, + maybe_ignored_options, + stats: response.stats, + }) + } + + fn contains_module(&self, specifier: &ModuleSpecifier) -> bool { + let s = self.resolve_specifier(specifier); + self.modules.contains_key(s) + } + + /// Emit the module graph in a specific format. This is specifically designed + /// to be an "all-in-one" API for access by the runtime, allowing both + /// emitting single modules as well as bundles, using Deno module resolution + /// or supplied sources. + pub fn emit( + self, + options: EmitOptions, + ) -> Result<(HashMap, ResultInfo), AnyError> { + let mut config = TsConfig::new(json!({ + "allowJs": true, + // TODO(@kitsonk) consider enabling this by default + // see: https://github.com/denoland/deno/issues/7732 + "emitDecoratorMetadata": false, + "esModuleInterop": true, + "experimentalDecorators": true, + "isolatedModules": true, + "jsx": "react", + "lib": TypeLib::DenoWindow, + "module": "esnext", + "strict": true, + "target": "esnext", + })); + let opts = match options.bundle_type { + BundleType::Esm => json!({ + "checkJs": false, + "inlineSourceMap": false, + "noEmit": true, + "jsxFactory": "React.createElement", + "jsxFragmentFactory": "React.Fragment", + }), + BundleType::None => json!({ + "outDir": "deno://", + "removeComments": true, + "sourceMap": true, + }), + }; + config.merge(&opts); + let maybe_ignored_options = + if let Some(user_options) = &options.maybe_user_config { + config.merge_user_config(user_options)? + } else { + None + }; + + let root_names = self.get_root_names(); + let hash_data = + vec![config.as_bytes(), version::DENO.as_bytes().to_owned()]; + let graph = Rc::new(RefCell::new(self)); + + let response = tsc::exec( + js::compiler_isolate_init(), + tsc::Request { + config: config.clone(), + debug: options.debug, + graph: graph.clone(), + hash_data, + maybe_tsbuildinfo: None, + root_names, + }, + )?; + + let mut emitted_files = HashMap::new(); + match options.bundle_type { + BundleType::Esm => { + assert!( + response.emitted_files.is_empty(), + "No files should have been emitted from tsc." + ); + let graph = graph.borrow(); + assert_eq!( + graph.roots.len(), + 1, + "Only a single root module supported." + ); + let specifier = &graph.roots[0]; + let s = graph.emit_bundle(specifier, &config.into())?; + emitted_files.insert("deno:///bundle.js".to_string(), s); + } + BundleType::None => { + for emitted_file in &response.emitted_files { + assert!( + emitted_file.maybe_specifiers.is_some(), + "Orphaned file emitted." + ); + let specifiers = emitted_file.maybe_specifiers.clone().unwrap(); + assert_eq!( + specifiers.len(), + 1, + "An unexpected number of specifiers associated with emitted file." + ); + let specifier = specifiers[0].clone(); + let extension = match emitted_file.media_type { + MediaType::JavaScript => ".js", + MediaType::SourceMap => ".js.map", + _ => unreachable!(), + }; + let key = format!("{}{}", specifier, extension); + emitted_files.insert(key, emitted_file.data.clone()); + } + } + }; + + Ok(( + emitted_files, + ResultInfo { + diagnostics: response.diagnostics, + maybe_ignored_options, + stats: response.stats, + }, + )) + } + + /// Shared between `bundle()` and `emit()`. + fn emit_bundle( + &self, + specifier: &ModuleSpecifier, + emit_options: &ast::EmitOptions, + ) -> Result { + let cm = Rc::new(swc_common::SourceMap::new( + swc_common::FilePathMapping::empty(), + )); + let globals = swc_common::Globals::new(); + let loader = BundleLoader::new(self, emit_options, &globals, cm.clone()); + let hook = Box::new(BundleHook); + let bundler = swc_bundler::Bundler::new( + &globals, + cm.clone(), + loader, + self, + swc_bundler::Config::default(), + hook, + ); + let mut entries = HashMap::new(); + entries.insert( + "bundle".to_string(), + swc_common::FileName::Custom(specifier.to_string()), + ); + let output = bundler + .bundle(entries) + .context("Unable to output bundle during Graph::bundle().")?; + let mut buf = Vec::new(); + { + let mut emitter = swc_ecmascript::codegen::Emitter { + cfg: swc_ecmascript::codegen::Config { minify: false }, + cm: cm.clone(), + comments: None, + wr: Box::new(swc_ecmascript::codegen::text_writer::JsWriter::new( + cm, "\n", &mut buf, None, + )), + }; + + emitter + .emit_module(&output[0].module) + .context("Unable to emit bundle during Graph::bundle().")?; + } + + String::from_utf8(buf).context("Emitted bundle is an invalid utf-8 string.") + } + + /// Update the handler with any modules that are marked as _dirty_ and update + /// any build info if present. + fn flush(&mut self) -> Result<(), AnyError> { + let mut handler = self.handler.borrow_mut(); + for (_, module) in self.modules.iter_mut() { + if module.is_dirty { + if let Some(emit) = &module.maybe_emit { + handler.set_cache(&module.specifier, emit)?; + } + if let Some(version) = &module.maybe_version { + handler.set_version(&module.specifier, version.clone())?; + } + module.is_dirty = false; + } + } + for root_specifier in self.roots.iter() { + if let Some(tsbuildinfo) = &self.maybe_tsbuildinfo { + handler.set_tsbuildinfo(root_specifier, tsbuildinfo.to_owned())?; + } + } + + Ok(()) + } + + fn get_info( + &self, + specifier: &ModuleSpecifier, + seen: &mut HashSet, + totals: &mut HashMap, + ) -> ModuleInfo { + let not_seen = seen.insert(specifier.clone()); + let module = self.get_module(specifier).unwrap(); + let mut deps = Vec::new(); + let mut total_size = None; + + if not_seen { + let mut seen_deps = HashSet::new(); + // TODO(@kitsonk) https://github.com/denoland/deno/issues/7927 + for (_, dep) in module.dependencies.iter() { + // Check the runtime code dependency + if let Some(code_dep) = &dep.maybe_code { + if seen_deps.insert(code_dep.clone()) { + deps.push(self.get_info(code_dep, seen, totals)); + } + } + } + deps.sort(); + total_size = if let Some(total) = totals.get(specifier) { + Some(total.to_owned()) + } else { + let mut total = deps + .iter() + .map(|d| { + if let Some(total_size) = d.total_size { + total_size + } else { + 0 + } + }) + .sum(); + total += module.size(); + totals.insert(specifier.clone(), total); + Some(total) + }; + } + + ModuleInfo { + deps, + name: specifier.clone(), + size: module.size(), + total_size, + } + } + + fn get_info_map(&self) -> ModuleInfoMap { + let map = self + .modules + .iter() + .map(|(specifier, module)| { + let mut deps = HashSet::new(); + for (_, dep) in module.dependencies.iter() { + if let Some(code_dep) = &dep.maybe_code { + deps.insert(code_dep.clone()); + } + if let Some(type_dep) = &dep.maybe_type { + deps.insert(type_dep.clone()); + } + } + if let Some((_, types_dep)) = &module.maybe_types { + deps.insert(types_dep.clone()); + } + let item = ModuleInfoMapItem { + deps: deps.into_iter().collect(), + size: module.size(), + }; + (specifier.clone(), item) + }) + .collect(); + + ModuleInfoMap::new(map) + } + + pub fn get_media_type( + &self, + specifier: &ModuleSpecifier, + ) -> Option { + if let Some(module) = self.get_module(specifier) { + Some(module.media_type) + } else { + None + } + } + + fn get_module(&self, specifier: &ModuleSpecifier) -> Option<&Module> { + let s = self.resolve_specifier(specifier); + self.modules.get(s) + } + + fn get_module_mut( + &mut self, + specifier: &ModuleSpecifier, + ) -> Option<&mut Module> { + // this is duplicated code because `.resolve_specifier` requires an + // immutable borrow, but if `.resolve_specifier` is mut, then everything + // that calls it is is mut + let mut s = specifier; + while let Some(redirect) = self.redirects.get(s) { + s = redirect; + } + self.modules.get_mut(s) + } + + /// Consume graph and return list of all module specifiers contained in the + /// graph. + pub fn get_modules(&self) -> Vec { + self.modules.keys().map(|s| s.to_owned()).collect() + } + + /// Transform `self.roots` into something that works for `tsc`, because `tsc` + /// doesn't like root names without extensions that match its expectations, + /// nor does it have any concept of redirection, so we have to resolve all + /// that upfront before feeding it to `tsc`. + fn get_root_names(&self) -> Vec<(ModuleSpecifier, MediaType)> { + self + .roots + .iter() + .map(|ms| { + ( + // root modules can be redirects, so before we pass it to tsc we need + // to resolve the redirect + self.resolve_specifier(ms).clone(), + self.get_media_type(ms).unwrap(), + ) + }) + .collect() + } + + /// Get the source for a given module specifier. If the module is not part + /// of the graph, the result will be `None`. + pub fn get_source(&self, specifier: &ModuleSpecifier) -> Option { + if let Some(module) = self.get_module(specifier) { + Some(module.source.clone()) + } else { + None + } + } + + /// Return a structure which provides information about the module graph and + /// the relationship of the modules in the graph. This structure is used to + /// provide information for the `info` subcommand. + pub fn info(&self) -> Result { + if self.roots.is_empty() || self.roots.len() > 1 { + return Err(GraphError::NotSupported(format!("Info is only supported when there is a single root module in the graph. Found: {}", self.roots.len())).into()); + } + + let module = self.roots[0].clone(); + let m = self.get_module(&module).unwrap(); + + let mut seen = HashSet::new(); + let mut totals = HashMap::new(); + let info = self.get_info(&module, &mut seen, &mut totals); + + let files = self.get_info_map(); + let total_size = totals.get(&module).unwrap_or(&m.size()).to_owned(); + let (compiled, map) = + if let Some((emit_path, maybe_map_path)) = &m.maybe_emit_path { + (Some(emit_path.clone()), maybe_map_path.clone()) + } else { + (None, None) + }; + + Ok(ModuleGraphInfo { + compiled, + dep_count: self.modules.len() - 1, + file_type: m.media_type, + files, + info, + local: m.source_path.clone(), + map, + module, + total_size, + }) + } + + /// Determines if all of the modules in the graph that require an emit have + /// a valid emit. Returns `true` if all the modules have a valid emit, + /// otherwise false. + fn is_emit_valid(&self, config: &TsConfig) -> bool { + let check_js = config.get_check_js(); + let config = config.as_bytes(); + self.modules.iter().all(|(_, m)| { + let needs_emit = match m.media_type { + MediaType::TypeScript | MediaType::TSX | MediaType::JSX => true, + MediaType::JavaScript => check_js, + _ => false, + }; + if needs_emit { + m.is_emit_valid(&config) + } else { + true + } + }) + } + + /// Verify the subresource integrity of the graph based upon the optional + /// lockfile, updating the lockfile with any missing resources. This will + /// error if any of the resources do not match their lock status. + pub fn lock(&self) { + if let Some(lf) = self.maybe_lockfile.as_ref() { + let mut lockfile = lf.lock().unwrap(); + for (ms, module) in self.modules.iter() { + let specifier = module.specifier.to_string(); + let valid = lockfile.check_or_insert(&specifier, &module.source); + if !valid { + eprintln!( + "{}", + GraphError::InvalidSource(ms.clone(), lockfile.filename.clone()) + ); + std::process::exit(10); + } + } + } + } + + /// Determines if any of the modules in the graph are required to be emitted. + /// This is similar to `emit_valid()` except that the actual emit isn't + /// checked to determine if it is valid. + fn needs_emit(&self, config: &TsConfig) -> bool { + let check_js = config.get_check_js(); + self.modules.iter().any(|(_, m)| match m.media_type { + MediaType::TypeScript | MediaType::TSX | MediaType::JSX => true, + MediaType::JavaScript => check_js, + _ => false, + }) + } + + /// Given a string specifier and a referring module specifier, provide the + /// resulting module specifier and media type for the module that is part of + /// the graph. + /// + /// # Arguments + /// + /// * `specifier` - The string form of the module specifier that needs to be + /// resolved. + /// * `referrer` - The referring `ModuleSpecifier`. + /// * `prefer_types` - When resolving to a module specifier, determine if a + /// type dependency is preferred over a code dependency. This is set to + /// `true` when resolving module names for `tsc` as it needs the type + /// dependency over the code, while other consumers do not handle type only + /// dependencies. + pub fn resolve( + &self, + specifier: &str, + referrer: &ModuleSpecifier, + prefer_types: bool, + ) -> Result { + if !self.contains_module(referrer) { + return Err(GraphError::MissingSpecifier(referrer.to_owned()).into()); + } + let module = self.get_module(referrer).unwrap(); + if !module.dependencies.contains_key(specifier) { + return Err( + GraphError::MissingDependency( + referrer.to_owned(), + specifier.to_owned(), + ) + .into(), + ); + } + let dependency = module.dependencies.get(specifier).unwrap(); + // If there is a @deno-types pragma that impacts the dependency, then the + // maybe_type property will be set with that specifier, otherwise we use the + // specifier that point to the runtime code. + let resolved_specifier = if prefer_types && dependency.maybe_type.is_some() + { + dependency.maybe_type.clone().unwrap() + } else if let Some(code_specifier) = dependency.maybe_code.clone() { + code_specifier + } else { + return Err( + GraphError::MissingDependency( + referrer.to_owned(), + specifier.to_owned(), + ) + .into(), + ); + }; + if !self.contains_module(&resolved_specifier) { + return Err( + GraphError::MissingDependency( + referrer.to_owned(), + resolved_specifier.to_string(), + ) + .into(), + ); + } + let dep_module = self.get_module(&resolved_specifier).unwrap(); + // In the case that there is a X-TypeScript-Types or a triple-slash types, + // then the `maybe_types` specifier will be populated and we should use that + // instead. + let result = if prefer_types && dep_module.maybe_types.is_some() { + let (_, types) = dep_module.maybe_types.clone().unwrap(); + // It is possible that `types` points to a redirected specifier, so we + // need to ensure it resolves to the final specifier in the graph. + self.resolve_specifier(&types).clone() + } else { + dep_module.specifier.clone() + }; + + Ok(result) + } + + /// Takes a module specifier and returns the "final" specifier, accounting for + /// any redirects that may have occurred. + fn resolve_specifier<'a>( + &'a self, + specifier: &'a ModuleSpecifier, + ) -> &'a ModuleSpecifier { + let mut s = specifier; + let mut seen = HashSet::new(); + seen.insert(s.clone()); + while let Some(redirect) = self.redirects.get(s) { + if !seen.insert(redirect.clone()) { + eprintln!("An infinite loop of module redirections detected.\n Original specifier: {}", specifier); + break; + } + s = redirect; + if seen.len() > 5 { + eprintln!("An excessive number of module redirections detected.\n Original specifier: {}", specifier); + break; + } + } + s + } + + /// Transpile (only transform) the graph, updating any emitted modules + /// with the specifier handler. The result contains any performance stats + /// from the compiler and optionally any user provided configuration compiler + /// options that were ignored. + /// + /// # Arguments + /// + /// * `options` - A structure of options which impact how the code is + /// transpiled. + /// + pub fn transpile( + &mut self, + options: TranspileOptions, + ) -> Result<(Stats, Option), AnyError> { + let start = Instant::now(); + + let mut ts_config = TsConfig::new(json!({ + "checkJs": false, + "emitDecoratorMetadata": false, + "inlineSourceMap": true, + "jsx": "react", + "jsxFactory": "React.createElement", + "jsxFragmentFactory": "React.Fragment", + })); + + let maybe_ignored_options = + ts_config.merge_tsconfig(options.maybe_config_path)?; + + let emit_options: ast::EmitOptions = ts_config.clone().into(); + + let mut emit_count: u128 = 0; + let config = ts_config.as_bytes(); + for (_, module) in self.modules.iter_mut() { + // TODO(kitsonk) a lot of this logic should be refactored into `Module` as + // we start to support other methods on the graph. Especially managing + // the dirty state is something the module itself should "own". + + // if the module is a Dts file we should skip it + if module.media_type == MediaType::Dts { + continue; + } + // if we don't have check_js enabled, we won't touch non TypeScript + // modules + if !(emit_options.check_js + || module.media_type == MediaType::TSX + || module.media_type == MediaType::TypeScript) + { + continue; + } + // skip modules that already have a valid emit + if !options.reload && module.is_emit_valid(&config) { + continue; + } + if module.maybe_parsed_module.is_none() { + module.parse()?; + } + let parsed_module = module.maybe_parsed_module.clone().unwrap(); + let emit = parsed_module.transpile(&emit_options)?; + emit_count += 1; + module.maybe_emit = Some(Emit::Cli(emit)); + module.set_version(&config); + module.is_dirty = true; + } + self.flush()?; + + let stats = Stats(vec![ + ("Files".to_string(), self.modules.len() as u128), + ("Emitted".to_string(), emit_count), + ("Total time".to_string(), start.elapsed().as_millis()), + ]); + + Ok((stats, maybe_ignored_options)) + } +} + +impl swc_bundler::Resolve for Graph { + fn resolve( + &self, + referrer: &swc_common::FileName, + specifier: &str, + ) -> Result { + let referrer = if let swc_common::FileName::Custom(referrer) = referrer { + ModuleSpecifier::resolve_url_or_path(referrer) + .context("Cannot resolve swc FileName to a module specifier")? + } else { + unreachable!( + "An unexpected referrer was passed when bundling: {:?}", + referrer + ) + }; + let specifier = self.resolve(specifier, &referrer, false)?; + + Ok(swc_common::FileName::Custom(specifier.to_string())) + } +} + +/// A structure for building a dependency graph of modules. +pub struct GraphBuilder { + fetched: HashSet, + graph: Graph, + maybe_import_map: Option>>, + pending: FuturesUnordered, +} + +impl GraphBuilder { + pub fn new( + handler: Rc>, + maybe_import_map: Option, + maybe_lockfile: Option>>, + ) -> Self { + let internal_import_map = if let Some(import_map) = maybe_import_map { + Some(Rc::new(RefCell::new(import_map))) + } else { + None + }; + GraphBuilder { + graph: Graph::new(handler, maybe_lockfile), + fetched: HashSet::new(), + maybe_import_map: internal_import_map, + pending: FuturesUnordered::new(), + } + } + + /// Add a module into the graph based on a module specifier. The module + /// and any dependencies will be fetched from the handler. The module will + /// also be treated as a _root_ module in the graph. + pub async fn add( + &mut self, + specifier: &ModuleSpecifier, + is_dynamic: bool, + ) -> Result<(), AnyError> { + self.fetch(specifier, &None, is_dynamic)?; + + loop { + let cached_module = self.pending.next().await.unwrap()?; + let is_root = &cached_module.specifier == specifier; + self.visit(cached_module, is_root)?; + if self.pending.is_empty() { + break; + } + } + + if !self.graph.roots.contains(specifier) { + self.graph.roots.push(specifier.clone()); + self.graph.roots_dynamic = self.graph.roots_dynamic && is_dynamic; + if self.graph.maybe_tsbuildinfo.is_none() { + let handler = self.graph.handler.borrow(); + self.graph.maybe_tsbuildinfo = handler.get_tsbuildinfo(specifier)?; + } + } + + Ok(()) + } + + /// Request a module to be fetched from the handler and queue up its future + /// to be awaited to be resolved. + fn fetch( + &mut self, + specifier: &ModuleSpecifier, + maybe_referrer: &Option, + is_dynamic: bool, + ) -> Result<(), AnyError> { + if self.fetched.contains(&specifier) { + return Ok(()); + } + + self.fetched.insert(specifier.clone()); + let future = self.graph.handler.borrow_mut().fetch( + specifier.clone(), + maybe_referrer.clone(), + is_dynamic, + ); + self.pending.push(future); + + Ok(()) + } + + /// Visit a module that has been fetched, hydrating the module, analyzing its + /// dependencies if required, fetching those dependencies, and inserting the + /// module into the graph. + fn visit( + &mut self, + cached_module: CachedModule, + is_root: bool, + ) -> Result<(), AnyError> { + let specifier = cached_module.specifier.clone(); + let requested_specifier = cached_module.requested_specifier.clone(); + let mut module = + Module::new(cached_module, is_root, self.maybe_import_map.clone()); + match module.media_type { + MediaType::Json + | MediaType::SourceMap + | MediaType::TsBuildInfo + | MediaType::Unknown => { + return Err( + GraphError::UnsupportedImportType( + module.specifier, + module.media_type, + ) + .into(), + ); + } + _ => (), + } + if !module.is_parsed { + let has_types = module.maybe_types.is_some(); + module.parse()?; + if self.maybe_import_map.is_none() { + let mut handler = self.graph.handler.borrow_mut(); + handler.set_deps(&specifier, module.dependencies.clone())?; + if !has_types { + if let Some((types, _)) = module.maybe_types.clone() { + handler.set_types(&specifier, types)?; + } + } + } + } + for (_, dep) in module.dependencies.iter() { + let maybe_referrer = Some(dep.location.clone()); + if let Some(specifier) = dep.maybe_code.as_ref() { + self.fetch(specifier, &maybe_referrer, dep.is_dynamic)?; + } + if let Some(specifier) = dep.maybe_type.as_ref() { + self.fetch(specifier, &maybe_referrer, dep.is_dynamic)?; + } + } + if let Some((_, specifier)) = module.maybe_types.as_ref() { + self.fetch(specifier, &None, false)?; + } + if specifier != requested_specifier { + self + .graph + .redirects + .insert(requested_specifier, specifier.clone()); + } + self.graph.modules.insert(specifier, module); + + Ok(()) + } + + /// Move out the graph from the builder to be utilized further. An optional + /// lockfile can be provided, where if the sources in the graph do not match + /// the expected lockfile, an error will be logged and the process will exit. + pub fn get_graph(self) -> Graph { + self.graph.lock(); + self.graph + } +} + +#[cfg(test)] +pub mod tests { + use super::*; + + use crate::specifier_handler::MemoryHandler; + use deno_core::futures::future; + use std::env; + use std::fs; + use std::path::PathBuf; + use std::sync::Mutex; + + macro_rules! map ( + { $($key:expr => $value:expr),+ } => { + { + let mut m = ::std::collections::HashMap::new(); + $( + m.insert($key, $value); + )+ + m + } + }; + ); + + /// This is a testing mock for `SpecifierHandler` that uses a special file + /// system renaming to mock local and remote modules as well as provides + /// "spies" for the critical methods for testing purposes. + #[derive(Debug, Default)] + pub struct MockSpecifierHandler { + pub fixtures: PathBuf, + pub maybe_tsbuildinfo: Option, + pub tsbuildinfo_calls: Vec<(ModuleSpecifier, String)>, + pub cache_calls: Vec<(ModuleSpecifier, Emit)>, + pub deps_calls: Vec<(ModuleSpecifier, DependencyMap)>, + pub types_calls: Vec<(ModuleSpecifier, String)>, + pub version_calls: Vec<(ModuleSpecifier, String)>, + } + + impl MockSpecifierHandler { + fn get_cache( + &self, + specifier: ModuleSpecifier, + ) -> Result { + let specifier_text = specifier + .to_string() + .replace(":///", "_") + .replace("://", "_") + .replace("/", "-"); + let source_path = self.fixtures.join(specifier_text); + let media_type = MediaType::from(&source_path); + let source = fs::read_to_string(&source_path)?; + let is_remote = specifier.as_url().scheme() != "file"; + + Ok(CachedModule { + source, + requested_specifier: specifier.clone(), + source_path, + specifier, + media_type, + is_remote, + ..CachedModule::default() + }) + } + } + + impl SpecifierHandler for MockSpecifierHandler { + fn fetch( + &mut self, + specifier: ModuleSpecifier, + _maybe_referrer: Option, + _is_dynamic: bool, + ) -> FetchFuture { + Box::pin(future::ready(self.get_cache(specifier))) + } + fn get_tsbuildinfo( + &self, + _specifier: &ModuleSpecifier, + ) -> Result, AnyError> { + Ok(self.maybe_tsbuildinfo.clone()) + } + fn set_cache( + &mut self, + specifier: &ModuleSpecifier, + emit: &Emit, + ) -> Result<(), AnyError> { + self.cache_calls.push((specifier.clone(), emit.clone())); + Ok(()) + } + fn set_types( + &mut self, + specifier: &ModuleSpecifier, + types: String, + ) -> Result<(), AnyError> { + self.types_calls.push((specifier.clone(), types)); + Ok(()) + } + fn set_tsbuildinfo( + &mut self, + specifier: &ModuleSpecifier, + tsbuildinfo: String, + ) -> Result<(), AnyError> { + self.maybe_tsbuildinfo = Some(tsbuildinfo.clone()); + self + .tsbuildinfo_calls + .push((specifier.clone(), tsbuildinfo)); + Ok(()) + } + fn set_deps( + &mut self, + specifier: &ModuleSpecifier, + dependencies: DependencyMap, + ) -> Result<(), AnyError> { + self.deps_calls.push((specifier.clone(), dependencies)); + Ok(()) + } + fn set_version( + &mut self, + specifier: &ModuleSpecifier, + version: String, + ) -> Result<(), AnyError> { + self.version_calls.push((specifier.clone(), version)); + Ok(()) + } + } + + async fn setup( + specifier: ModuleSpecifier, + ) -> (Graph, Rc>) { + let c = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()); + let fixtures = c.join("tests/module_graph"); + let handler = Rc::new(RefCell::new(MockSpecifierHandler { + fixtures, + ..MockSpecifierHandler::default() + })); + let mut builder = GraphBuilder::new(handler.clone(), None, None); + builder + .add(&specifier, false) + .await + .expect("module not inserted"); + + (builder.get_graph(), handler) + } + + async fn setup_memory( + specifier: ModuleSpecifier, + sources: HashMap<&str, &str>, + ) -> Graph { + let sources: HashMap = sources + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + let handler = Rc::new(RefCell::new(MemoryHandler::new(sources))); + let mut builder = GraphBuilder::new(handler.clone(), None, None); + builder + .add(&specifier, false) + .await + .expect("module not inserted"); + + builder.get_graph() + } + + #[test] + fn test_get_version() { + let doc_a = "console.log(42);"; + let version_a = get_version(&doc_a, "1.2.3", b""); + let doc_b = "console.log(42);"; + let version_b = get_version(&doc_b, "1.2.3", b""); + assert_eq!(version_a, version_b); + + let version_c = get_version(&doc_a, "1.2.3", b"options"); + assert_ne!(version_a, version_c); + + let version_d = get_version(&doc_b, "1.2.3", b"options"); + assert_eq!(version_c, version_d); + + let version_e = get_version(&doc_a, "1.2.4", b""); + assert_ne!(version_a, version_e); + + let version_f = get_version(&doc_b, "1.2.4", b""); + assert_eq!(version_e, version_f); + } + + #[test] + fn test_module_emit_valid() { + let source = "console.log(42);".to_string(); + let maybe_version = Some(get_version(&source, version::DENO, b"")); + let module = Module { + source, + maybe_version, + ..Module::default() + }; + assert!(module.is_emit_valid(b"")); + + let source = "console.log(42);".to_string(); + let old_source = "console.log(43);"; + let maybe_version = Some(get_version(old_source, version::DENO, b"")); + let module = Module { + source, + maybe_version, + ..Module::default() + }; + assert!(!module.is_emit_valid(b"")); + + let source = "console.log(42);".to_string(); + let maybe_version = Some(get_version(&source, "0.0.0", b"")); + let module = Module { + source, + maybe_version, + ..Module::default() + }; + assert!(!module.is_emit_valid(b"")); + + let source = "console.log(42);".to_string(); + let module = Module { + source, + ..Module::default() + }; + assert!(!module.is_emit_valid(b"")); + } + + #[test] + fn test_module_set_version() { + let source = "console.log(42);".to_string(); + let expected = Some(get_version(&source, version::DENO, b"")); + let mut module = Module { + source, + ..Module::default() + }; + assert!(module.maybe_version.is_none()); + module.set_version(b""); + assert_eq!(module.maybe_version, expected); + } + + #[tokio::test] + async fn test_graph_bundle() { + let tests = vec![ + ("file:///tests/fixture01.ts", "fixture01.out"), + ("file:///tests/fixture02.ts", "fixture02.out"), + ("file:///tests/fixture03.ts", "fixture03.out"), + ("file:///tests/fixture04.ts", "fixture04.out"), + ("file:///tests/fixture05.ts", "fixture05.out"), + ("file:///tests/fixture06.ts", "fixture06.out"), + ("file:///tests/fixture07.ts", "fixture07.out"), + ("file:///tests/fixture08.ts", "fixture08.out"), + ("file:///tests/fixture09.ts", "fixture09.out"), + ("file:///tests/fixture10.ts", "fixture10.out"), + ("file:///tests/fixture11.ts", "fixture11.out"), + ("file:///tests/fixture12.ts", "fixture12.out"), + ("file:///tests/fixture13.ts", "fixture13.out"), + ("file:///tests/fixture14.ts", "fixture14.out"), + ]; + let c = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()); + let fixtures = c.join("tests/bundle"); + + for (specifier, expected_str) in tests { + let specifier = ModuleSpecifier::resolve_url_or_path(specifier).unwrap(); + let handler = Rc::new(RefCell::new(MockSpecifierHandler { + fixtures: fixtures.clone(), + ..MockSpecifierHandler::default() + })); + let mut builder = GraphBuilder::new(handler.clone(), None, None); + builder + .add(&specifier, false) + .await + .expect("module not inserted"); + let graph = builder.get_graph(); + let (actual, stats, maybe_ignored_options) = graph + .bundle(BundleOptions::default()) + .expect("could not bundle"); + assert_eq!(stats.0.len(), 2); + assert_eq!(maybe_ignored_options, None); + let expected_path = fixtures.join(expected_str); + let expected = fs::read_to_string(expected_path).unwrap(); + assert_eq!(actual, expected, "fixture: {}", specifier); + } + } + + #[tokio::test] + async fn test_graph_check_emit() { + let specifier = + ModuleSpecifier::resolve_url_or_path("file:///tests/main.ts") + .expect("could not resolve module"); + let (graph, handler) = setup(specifier).await; + let result_info = graph + .check(CheckOptions { + debug: false, + emit: true, + lib: TypeLib::DenoWindow, + maybe_config_path: None, + reload: false, + }) + .expect("should have checked"); + assert!(result_info.maybe_ignored_options.is_none()); + assert_eq!(result_info.stats.0.len(), 12); + println!("{}", result_info.diagnostics); + assert!(result_info.diagnostics.is_empty()); + let h = handler.borrow(); + assert_eq!(h.cache_calls.len(), 2); + assert_eq!(h.tsbuildinfo_calls.len(), 1); + } + + #[tokio::test] + async fn test_graph_check_no_emit() { + let specifier = + ModuleSpecifier::resolve_url_or_path("file:///tests/main.ts") + .expect("could not resolve module"); + let (graph, handler) = setup(specifier).await; + let result_info = graph + .check(CheckOptions { + debug: false, + emit: false, + lib: TypeLib::DenoWindow, + maybe_config_path: None, + reload: false, + }) + .expect("should have checked"); + assert!(result_info.maybe_ignored_options.is_none()); + assert_eq!(result_info.stats.0.len(), 12); + assert!(result_info.diagnostics.is_empty()); + let h = handler.borrow(); + assert_eq!(h.cache_calls.len(), 0); + assert_eq!(h.tsbuildinfo_calls.len(), 1); + } + + #[tokio::test] + async fn test_graph_check_user_config() { + let specifier = + ModuleSpecifier::resolve_url_or_path("file:///tests/checkwithconfig.ts") + .expect("could not resolve module"); + let (graph, handler) = setup(specifier.clone()).await; + let result_info = graph + .check(CheckOptions { + debug: false, + emit: true, + lib: TypeLib::DenoWindow, + maybe_config_path: Some( + "tests/module_graph/tsconfig_01.json".to_string(), + ), + reload: true, + }) + .expect("should have checked"); + assert!(result_info.maybe_ignored_options.is_none()); + assert!(result_info.diagnostics.is_empty()); + let h = handler.borrow(); + assert_eq!(h.version_calls.len(), 2); + let ver0 = h.version_calls[0].1.clone(); + let ver1 = h.version_calls[1].1.clone(); + + // let's do it all over again to ensure that the versions are determinstic + let (graph, handler) = setup(specifier).await; + let result_info = graph + .check(CheckOptions { + debug: false, + emit: true, + lib: TypeLib::DenoWindow, + maybe_config_path: Some( + "tests/module_graph/tsconfig_01.json".to_string(), + ), + reload: true, + }) + .expect("should have checked"); + assert!(result_info.maybe_ignored_options.is_none()); + assert!(result_info.diagnostics.is_empty()); + let h = handler.borrow(); + assert_eq!(h.version_calls.len(), 2); + assert!(h.version_calls[0].1 == ver0 || h.version_calls[0].1 == ver1); + assert!(h.version_calls[1].1 == ver0 || h.version_calls[1].1 == ver1); + } + + #[tokio::test] + async fn test_graph_emit() { + let specifier = + ModuleSpecifier::resolve_url_or_path("file:///a.ts").unwrap(); + let graph = setup_memory( + specifier, + map!( + "/a.ts" => r#" + import * as b from "./b.ts"; + + console.log(b); + "#, + "/b.ts" => r#" + export const b = "b"; + "# + ), + ) + .await; + let (emitted_files, result_info) = graph + .emit(EmitOptions { + bundle_type: BundleType::None, + debug: false, + maybe_user_config: None, + }) + .expect("should have emitted"); + assert!(result_info.diagnostics.is_empty()); + assert!(result_info.maybe_ignored_options.is_none()); + assert_eq!(emitted_files.len(), 4); + let out_a = emitted_files.get("file:///a.ts.js"); + assert!(out_a.is_some()); + let out_a = out_a.unwrap(); + assert!(out_a.starts_with("import * as b from")); + assert!(emitted_files.contains_key("file:///a.ts.js.map")); + let out_b = emitted_files.get("file:///b.ts.js"); + assert!(out_b.is_some()); + let out_b = out_b.unwrap(); + assert!(out_b.starts_with("export const b = \"b\";")); + assert!(emitted_files.contains_key("file:///b.ts.js.map")); + } + + #[tokio::test] + async fn test_graph_emit_bundle() { + let specifier = + ModuleSpecifier::resolve_url_or_path("file:///a.ts").unwrap(); + let graph = setup_memory( + specifier, + map!( + "/a.ts" => r#" + import * as b from "./b.ts"; + + console.log(b); + "#, + "/b.ts" => r#" + export const b = "b"; + "# + ), + ) + .await; + let (emitted_files, result_info) = graph + .emit(EmitOptions { + bundle_type: BundleType::Esm, + debug: false, + maybe_user_config: None, + }) + .expect("should have emitted"); + assert!(result_info.diagnostics.is_empty()); + assert!(result_info.maybe_ignored_options.is_none()); + assert_eq!(emitted_files.len(), 1); + let actual = emitted_files.get("deno:///bundle.js"); + assert!(actual.is_some()); + let actual = actual.unwrap(); + assert!(actual.contains("const b = \"b\";")); + assert!(actual.contains("console.log(b);")); + } + + #[tokio::test] + async fn test_graph_info() { + let specifier = + ModuleSpecifier::resolve_url_or_path("file:///tests/main.ts") + .expect("could not resolve module"); + let (graph, _) = setup(specifier).await; + let info = graph.info().expect("could not get info"); + assert!(info.compiled.is_none()); + assert_eq!(info.dep_count, 6); + assert_eq!(info.file_type, MediaType::TypeScript); + assert_eq!(info.files.0.len(), 7); + assert!(info.local.to_string_lossy().ends_with("file_tests-main.ts")); + assert!(info.map.is_none()); + assert_eq!( + info.module, + ModuleSpecifier::resolve_url_or_path("file:///tests/main.ts").unwrap() + ); + assert_eq!(info.total_size, 344); + } + + #[tokio::test] + async fn test_graph_import_json() { + let specifier = + ModuleSpecifier::resolve_url_or_path("file:///tests/importjson.ts") + .expect("could not resolve module"); + let c = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()); + let fixtures = c.join("tests/module_graph"); + let handler = Rc::new(RefCell::new(MockSpecifierHandler { + fixtures, + ..MockSpecifierHandler::default() + })); + let mut builder = GraphBuilder::new(handler.clone(), None, None); + builder + .add(&specifier, false) + .await + .expect_err("should have errored"); + } + + #[tokio::test] + async fn test_graph_transpile() { + // This is a complex scenario of transpiling, where we have TypeScript + // importing a JavaScript file (with type definitions) which imports + // TypeScript, JavaScript, and JavaScript with type definitions. + // For scenarios where we transpile, we only want the TypeScript files + // to be actually emitted. + // + // This also exercises "@deno-types" and type references. + let specifier = + ModuleSpecifier::resolve_url_or_path("file:///tests/main.ts") + .expect("could not resolve module"); + let (mut graph, handler) = setup(specifier).await; + let (stats, maybe_ignored_options) = + graph.transpile(TranspileOptions::default()).unwrap(); + assert_eq!(stats.0.len(), 3); + assert_eq!(maybe_ignored_options, None); + let h = handler.borrow(); + assert_eq!(h.cache_calls.len(), 2); + match &h.cache_calls[0].1 { + Emit::Cli((code, maybe_map)) => { + assert!( + code.contains("# sourceMappingURL=data:application/json;base64,") + ); + assert!(maybe_map.is_none()); + } + }; + match &h.cache_calls[1].1 { + Emit::Cli((code, maybe_map)) => { + assert!( + code.contains("# sourceMappingURL=data:application/json;base64,") + ); + assert!(maybe_map.is_none()); + } + }; + assert_eq!(h.deps_calls.len(), 7); + assert_eq!( + h.deps_calls[0].0, + ModuleSpecifier::resolve_url_or_path("file:///tests/main.ts").unwrap() + ); + assert_eq!(h.deps_calls[0].1.len(), 1); + assert_eq!( + h.deps_calls[1].0, + ModuleSpecifier::resolve_url_or_path("https://deno.land/x/lib/mod.js") + .unwrap() + ); + assert_eq!(h.deps_calls[1].1.len(), 3); + assert_eq!( + h.deps_calls[2].0, + ModuleSpecifier::resolve_url_or_path("https://deno.land/x/lib/mod.d.ts") + .unwrap() + ); + assert_eq!(h.deps_calls[2].1.len(), 3, "should have 3 dependencies"); + // sometimes the calls are not deterministic, and so checking the contents + // can cause some failures + assert_eq!(h.deps_calls[3].1.len(), 0, "should have no dependencies"); + assert_eq!(h.deps_calls[4].1.len(), 0, "should have no dependencies"); + assert_eq!(h.deps_calls[5].1.len(), 0, "should have no dependencies"); + assert_eq!(h.deps_calls[6].1.len(), 0, "should have no dependencies"); + } + + #[tokio::test] + async fn test_graph_transpile_user_config() { + let specifier = + ModuleSpecifier::resolve_url_or_path("https://deno.land/x/transpile.tsx") + .expect("could not resolve module"); + let (mut graph, handler) = setup(specifier).await; + let (_, maybe_ignored_options) = graph + .transpile(TranspileOptions { + debug: false, + maybe_config_path: Some("tests/module_graph/tsconfig.json".to_string()), + reload: false, + }) + .unwrap(); + assert_eq!( + maybe_ignored_options.unwrap().items, + vec!["target".to_string()], + "the 'target' options should have been ignored" + ); + let h = handler.borrow(); + assert_eq!(h.cache_calls.len(), 1, "only one file should be emitted"); + // FIXME(bartlomieju): had to add space in `
`, probably a quirk in swc_ecma_codegen + match &h.cache_calls[0].1 { + Emit::Cli((code, _)) => { + assert!( + code.contains("
Hello world!
"), + "jsx should have been preserved" + ); + } + } + } + + #[tokio::test] + async fn test_graph_with_lockfile() { + let c = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()); + let fixtures = c.join("tests/module_graph"); + let lockfile_path = fixtures.join("lockfile.json"); + let lockfile = + Lockfile::new(lockfile_path, false).expect("could not load lockfile"); + let maybe_lockfile = Some(Arc::new(Mutex::new(lockfile))); + let handler = Rc::new(RefCell::new(MockSpecifierHandler { + fixtures, + ..MockSpecifierHandler::default() + })); + let mut builder = GraphBuilder::new(handler.clone(), None, maybe_lockfile); + let specifier = + ModuleSpecifier::resolve_url_or_path("file:///tests/main.ts") + .expect("could not resolve module"); + builder + .add(&specifier, false) + .await + .expect("module not inserted"); + builder.get_graph(); + } +} diff --git a/cli/module_graph2.rs b/cli/module_graph2.rs deleted file mode 100644 index f795a7acb..000000000 --- a/cli/module_graph2.rs +++ /dev/null @@ -1,2208 +0,0 @@ -// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. - -use crate::ast; -use crate::ast::parse; -use crate::ast::transpile_module; -use crate::ast::BundleHook; -use crate::ast::Location; -use crate::ast::ParsedModule; -use crate::colors; -use crate::diagnostics::Diagnostics; -use crate::import_map::ImportMap; -use crate::info::ModuleGraphInfo; -use crate::info::ModuleInfo; -use crate::info::ModuleInfoMap; -use crate::info::ModuleInfoMapItem; -use crate::js; -use crate::lockfile::Lockfile; -use crate::media_type::MediaType; -use crate::specifier_handler::CachedModule; -use crate::specifier_handler::Dependency; -use crate::specifier_handler::DependencyMap; -use crate::specifier_handler::Emit; -use crate::specifier_handler::FetchFuture; -use crate::specifier_handler::SpecifierHandler; -use crate::tsc2; -use crate::tsc_config::IgnoredCompilerOptions; -use crate::tsc_config::TsConfig; -use crate::version; -use crate::AnyError; - -use deno_core::error::Context; -use deno_core::futures::stream::FuturesUnordered; -use deno_core::futures::stream::StreamExt; -use deno_core::serde::Serialize; -use deno_core::serde::Serializer; -use deno_core::serde_json::json; -use deno_core::serde_json::Value; -use deno_core::ModuleResolutionError; -use deno_core::ModuleSpecifier; -use regex::Regex; -use serde::Deserialize; -use serde::Deserializer; -use std::cell::RefCell; -use std::collections::HashMap; -use std::collections::HashSet; -use std::error::Error; -use std::fmt; -use std::path::PathBuf; -use std::rc::Rc; -use std::result; -use std::sync::Arc; -use std::sync::Mutex; -use std::time::Instant; - -lazy_static! { - /// Matched the `@deno-types` pragma. - static ref DENO_TYPES_RE: Regex = - Regex::new(r#"(?i)^\s*@deno-types\s*=\s*(?:["']([^"']+)["']|(\S+))"#) - .unwrap(); - /// Matches a `/// ` comment reference. - static ref TRIPLE_SLASH_REFERENCE_RE: Regex = - Regex::new(r"(?i)^/\s*").unwrap(); - /// Matches a path reference, which adds a dependency to a module - static ref PATH_REFERENCE_RE: Regex = - Regex::new(r#"(?i)\spath\s*=\s*["']([^"']*)["']"#).unwrap(); - /// Matches a types reference, which for JavaScript files indicates the - /// location of types to use when type checking a program that includes it as - /// a dependency. - static ref TYPES_REFERENCE_RE: Regex = - Regex::new(r#"(?i)\stypes\s*=\s*["']([^"']*)["']"#).unwrap(); -} - -/// A group of errors that represent errors that can occur when interacting with -/// a module graph. -#[derive(Debug, Clone, Eq, PartialEq)] -pub enum GraphError { - /// A module using the HTTPS protocol is trying to import a module with an - /// HTTP schema. - InvalidDowngrade(ModuleSpecifier, Location), - /// A remote module is trying to import a local module. - InvalidLocalImport(ModuleSpecifier, Location), - /// The source code is invalid, as it does not match the expected hash in the - /// lockfile. - InvalidSource(ModuleSpecifier, PathBuf), - /// An unexpected dependency was requested for a module. - MissingDependency(ModuleSpecifier, String), - /// An unexpected specifier was requested. - MissingSpecifier(ModuleSpecifier), - /// The current feature is not supported. - NotSupported(String), - /// A unsupported media type was attempted to be imported as a module. - UnsupportedImportType(ModuleSpecifier, MediaType), -} - -impl fmt::Display for GraphError { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - match self { - GraphError::InvalidDowngrade(ref specifier, ref location) => write!(f, "Modules imported via https are not allowed to import http modules.\n Importing: {}\n at {}", specifier, location), - GraphError::InvalidLocalImport(ref specifier, ref location) => write!(f, "Remote modules are not allowed to import local modules. Consider using a dynamic import instead.\n Importing: {}\n at {}", specifier, location), - GraphError::InvalidSource(ref specifier, ref lockfile) => write!(f, "The source code is invalid, as it does not match the expected hash in the lock file.\n Specifier: {}\n Lock file: {}", specifier, lockfile.to_str().unwrap()), - GraphError::MissingDependency(ref referrer, specifier) => write!( - f, - "The graph is missing a dependency.\n Specifier: {} from {}", - specifier, referrer - ), - GraphError::MissingSpecifier(ref specifier) => write!( - f, - "The graph is missing a specifier.\n Specifier: {}", - specifier - ), - GraphError::NotSupported(ref msg) => write!(f, "{}", msg), - GraphError::UnsupportedImportType(ref specifier, ref media_type) => write!(f, "An unsupported media type was attempted to be imported as a module.\n Specifier: {}\n MediaType: {}", specifier, media_type), - } - } -} - -impl Error for GraphError {} - -/// A structure for handling bundle loading, which is implemented here, to -/// avoid a circular dependency with `ast`. -struct BundleLoader<'a> { - cm: Rc, - emit_options: &'a ast::EmitOptions, - globals: &'a swc_common::Globals, - graph: &'a Graph2, -} - -impl<'a> BundleLoader<'a> { - pub fn new( - graph: &'a Graph2, - emit_options: &'a ast::EmitOptions, - globals: &'a swc_common::Globals, - cm: Rc, - ) -> Self { - BundleLoader { - cm, - emit_options, - globals, - graph, - } - } -} - -impl swc_bundler::Load for BundleLoader<'_> { - fn load( - &self, - file: &swc_common::FileName, - ) -> Result<(Rc, swc_ecmascript::ast::Module), AnyError> - { - match file { - swc_common::FileName::Custom(filename) => { - let specifier = ModuleSpecifier::resolve_url_or_path(filename) - .context("Failed to convert swc FileName to ModuleSpecifier.")?; - if let Some(src) = self.graph.get_source(&specifier) { - let media_type = self - .graph - .get_media_type(&specifier) - .context("Looking up media type during bundling.")?; - transpile_module( - filename, - &src, - &media_type, - self.emit_options, - self.globals, - self.cm.clone(), - ) - } else { - Err( - GraphError::MissingDependency(specifier, "".to_string()) - .into(), - ) - } - } - _ => unreachable!("Received request for unsupported filename {:?}", file), - } - } -} - -/// An enum which represents the parsed out values of references in source code. -#[derive(Debug, Clone, Eq, PartialEq)] -enum TypeScriptReference { - Path(String), - Types(String), -} - -/// Determine if a comment contains a triple slash reference and optionally -/// return its kind and value. -fn parse_ts_reference(comment: &str) -> Option { - if !TRIPLE_SLASH_REFERENCE_RE.is_match(comment) { - None - } else if let Some(captures) = PATH_REFERENCE_RE.captures(comment) { - Some(TypeScriptReference::Path( - captures.get(1).unwrap().as_str().to_string(), - )) - } else if let Some(captures) = TYPES_REFERENCE_RE.captures(comment) { - Some(TypeScriptReference::Types( - captures.get(1).unwrap().as_str().to_string(), - )) - } else { - None - } -} - -/// Determine if a comment contains a `@deno-types` pragma and optionally return -/// its value. -fn parse_deno_types(comment: &str) -> Option { - if let Some(captures) = DENO_TYPES_RE.captures(comment) { - if let Some(m) = captures.get(1) { - Some(m.as_str().to_string()) - } else if let Some(m) = captures.get(2) { - Some(m.as_str().to_string()) - } else { - panic!("unreachable"); - } - } else { - None - } -} - -/// A hashing function that takes the source code, version and optionally a -/// user provided config and generates a string hash which can be stored to -/// determine if the cached emit is valid or not. -fn get_version(source: &str, version: &str, config: &[u8]) -> String { - crate::checksum::gen(&[source.as_bytes(), version.as_bytes(), config]) -} - -/// A logical representation of a module within a graph. -#[derive(Debug, Clone)] -struct Module { - dependencies: DependencyMap, - is_dirty: bool, - is_parsed: bool, - maybe_emit: Option, - maybe_emit_path: Option<(PathBuf, Option)>, - maybe_import_map: Option>>, - maybe_parsed_module: Option, - maybe_types: Option<(String, ModuleSpecifier)>, - maybe_version: Option, - media_type: MediaType, - specifier: ModuleSpecifier, - source: String, - source_path: PathBuf, -} - -impl Default for Module { - fn default() -> Self { - Module { - dependencies: HashMap::new(), - is_dirty: false, - is_parsed: false, - maybe_emit: None, - maybe_emit_path: None, - maybe_import_map: None, - maybe_parsed_module: None, - maybe_types: None, - maybe_version: None, - media_type: MediaType::Unknown, - specifier: ModuleSpecifier::resolve_url("file:///example.js").unwrap(), - source: "".to_string(), - source_path: PathBuf::new(), - } - } -} - -impl Module { - pub fn new( - cached_module: CachedModule, - is_root: bool, - maybe_import_map: Option>>, - ) -> Self { - // If this is a local root file, and its media type is unknown, set the - // media type to JavaScript. This allows easier ability to create "shell" - // scripts with Deno. - let media_type = if is_root - && !cached_module.is_remote - && cached_module.media_type == MediaType::Unknown - { - MediaType::JavaScript - } else { - cached_module.media_type - }; - let mut module = Module { - specifier: cached_module.specifier, - maybe_import_map, - media_type, - source: cached_module.source, - source_path: cached_module.source_path, - maybe_emit: cached_module.maybe_emit, - maybe_emit_path: cached_module.maybe_emit_path, - maybe_version: cached_module.maybe_version, - is_dirty: false, - ..Self::default() - }; - if module.maybe_import_map.is_none() { - if let Some(dependencies) = cached_module.maybe_dependencies { - module.dependencies = dependencies; - module.is_parsed = true; - } - } - module.maybe_types = if let Some(ref specifier) = cached_module.maybe_types - { - Some(( - specifier.clone(), - module - .resolve_import(&specifier, None) - .expect("could not resolve module"), - )) - } else { - None - }; - module - } - - /// Return `true` if the current hash of the module matches the stored - /// version. - pub fn is_emit_valid(&self, config: &[u8]) -> bool { - if let Some(version) = self.maybe_version.clone() { - version == get_version(&self.source, version::DENO, config) - } else { - false - } - } - - /// Parse a module, populating the structure with data retrieved from the - /// source of the module. - pub fn parse(&mut self) -> Result<(), AnyError> { - let parsed_module = - parse(self.specifier.as_str(), &self.source, &self.media_type)?; - - // parse out any triple slash references - for comment in parsed_module.get_leading_comments().iter() { - if let Some(ts_reference) = parse_ts_reference(&comment.text) { - let location = parsed_module.get_location(&comment.span); - match ts_reference { - TypeScriptReference::Path(import) => { - let specifier = - self.resolve_import(&import, Some(location.clone()))?; - let dep = self - .dependencies - .entry(import) - .or_insert_with(|| Dependency::new(location)); - dep.maybe_code = Some(specifier); - } - TypeScriptReference::Types(import) => { - let specifier = - self.resolve_import(&import, Some(location.clone()))?; - if self.media_type == MediaType::JavaScript - || self.media_type == MediaType::JSX - { - // TODO(kitsonk) we need to specifically update the cache when - // this value changes - self.maybe_types = Some((import.clone(), specifier)); - } else { - let dep = self - .dependencies - .entry(import) - .or_insert_with(|| Dependency::new(location)); - dep.maybe_type = Some(specifier); - } - } - } - } - } - - // Parse out all the syntactical dependencies for a module - let dependencies = parsed_module.analyze_dependencies(); - for desc in dependencies.iter().filter(|desc| { - desc.kind != swc_ecmascript::dep_graph::DependencyKind::Require - }) { - let location = Location { - filename: self.specifier.to_string(), - col: desc.col, - line: desc.line, - }; - - // In situations where there is a potential issue with resolving the - // import specifier, that ends up being a module resolution error for a - // code dependency, we should not throw in the `ModuleGraph` but instead - // wait until runtime and throw there, as with dynamic imports they need - // to be catchable, which means they need to be resolved at runtime. - let maybe_specifier = - match self.resolve_import(&desc.specifier, Some(location.clone())) { - Ok(specifier) => Some(specifier), - Err(any_error) => { - match any_error.downcast_ref::() { - Some(ModuleResolutionError::ImportPrefixMissing(_, _)) => None, - _ => { - return Err(any_error); - } - } - } - }; - - // Parse out any `@deno-types` pragmas and modify dependency - let maybe_type = if !desc.leading_comments.is_empty() { - let comment = desc.leading_comments.last().unwrap(); - if let Some(deno_types) = parse_deno_types(&comment.text).as_ref() { - Some(self.resolve_import(deno_types, Some(location.clone()))?) - } else { - None - } - } else { - None - }; - - let dep = self - .dependencies - .entry(desc.specifier.to_string()) - .or_insert_with(|| Dependency::new(location)); - dep.is_dynamic = desc.is_dynamic; - if let Some(specifier) = maybe_specifier { - if desc.kind == swc_ecmascript::dep_graph::DependencyKind::ExportType - || desc.kind == swc_ecmascript::dep_graph::DependencyKind::ImportType - { - dep.maybe_type = Some(specifier); - } else { - dep.maybe_code = Some(specifier); - } - } - // If the dependency wasn't a type only dependency already, and there is - // a `@deno-types` comment, then we will set the `maybe_type` dependency. - if maybe_type.is_some() && dep.maybe_type.is_none() { - dep.maybe_type = maybe_type; - } - } - - self.maybe_parsed_module = Some(parsed_module); - Ok(()) - } - - fn resolve_import( - &self, - specifier: &str, - maybe_location: Option, - ) -> Result { - let maybe_resolve = if let Some(import_map) = self.maybe_import_map.clone() - { - import_map - .borrow() - .resolve(specifier, self.specifier.as_str())? - } else { - None - }; - let specifier = if let Some(module_specifier) = maybe_resolve { - module_specifier - } else { - ModuleSpecifier::resolve_import(specifier, self.specifier.as_str())? - }; - - let referrer_scheme = self.specifier.as_url().scheme(); - let specifier_scheme = specifier.as_url().scheme(); - let location = maybe_location.unwrap_or(Location { - filename: self.specifier.to_string(), - line: 0, - col: 0, - }); - - // Disallow downgrades from HTTPS to HTTP - if referrer_scheme == "https" && specifier_scheme == "http" { - return Err( - GraphError::InvalidDowngrade(specifier.clone(), location).into(), - ); - } - - // Disallow a remote URL from trying to import a local URL - if (referrer_scheme == "https" || referrer_scheme == "http") - && !(specifier_scheme == "https" || specifier_scheme == "http") - { - return Err( - GraphError::InvalidLocalImport(specifier.clone(), location).into(), - ); - } - - Ok(specifier) - } - - /// Calculate the hashed version of the module and update the `maybe_version`. - pub fn set_version(&mut self, config: &[u8]) { - self.maybe_version = Some(get_version(&self.source, version::DENO, config)) - } - - pub fn size(&self) -> usize { - self.source.as_bytes().len() - } -} - -#[derive(Clone, Debug, Default, Eq, PartialEq)] -pub struct Stats(pub Vec<(String, u128)>); - -impl<'de> Deserialize<'de> for Stats { - fn deserialize(deserializer: D) -> result::Result - where - D: Deserializer<'de>, - { - let items: Vec<(String, u128)> = Deserialize::deserialize(deserializer)?; - Ok(Stats(items)) - } -} - -impl fmt::Display for Stats { - fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { - writeln!(f, "Compilation statistics:")?; - for (key, value) in self.0.clone() { - writeln!(f, " {}: {}", key, value)?; - } - - Ok(()) - } -} - -/// A structure that provides information about a module graph result. -#[derive(Debug, Default)] -pub struct ResultInfo { - /// A structure which provides diagnostic information (usually from `tsc`) - /// about the code in the module graph. - pub diagnostics: Diagnostics, - /// Optionally ignored compiler options that represent any options that were - /// ignored if there was a user provided configuration. - pub maybe_ignored_options: Option, - /// A structure providing key metrics around the operation performed, in - /// milliseconds. - pub stats: Stats, -} - -/// Represents the "default" type library that should be used when type -/// checking the code in the module graph. Note that a user provided config -/// of `"lib"` would override this value. -#[derive(Debug, Clone, Eq, PartialEq)] -pub enum TypeLib { - DenoWindow, - DenoWorker, - UnstableDenoWindow, - UnstableDenoWorker, -} - -impl Default for TypeLib { - fn default() -> Self { - TypeLib::DenoWindow - } -} - -impl Serialize for TypeLib { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - let value = match self { - TypeLib::DenoWindow => vec!["deno.window".to_string()], - TypeLib::DenoWorker => vec!["deno.worker".to_string()], - TypeLib::UnstableDenoWindow => { - vec!["deno.window".to_string(), "deno.unstable".to_string()] - } - TypeLib::UnstableDenoWorker => { - vec!["deno.worker".to_string(), "deno.worker".to_string()] - } - }; - Serialize::serialize(&value, serializer) - } -} - -#[derive(Debug, Default)] -pub struct BundleOptions { - /// If `true` then debug logging will be output from the isolate. - pub debug: bool, - /// An optional string that points to a user supplied TypeScript configuration - /// file that augments the the default configuration passed to the TypeScript - /// compiler. - pub maybe_config_path: Option, -} - -#[derive(Debug, Default)] -pub struct CheckOptions { - /// If `true` then debug logging will be output from the isolate. - pub debug: bool, - /// Utilise the emit from `tsc` to update the emitted code for modules. - pub emit: bool, - /// The base type libraries that should be used when type checking. - pub lib: TypeLib, - /// An optional string that points to a user supplied TypeScript configuration - /// file that augments the the default configuration passed to the TypeScript - /// compiler. - pub maybe_config_path: Option, - /// Ignore any previously emits and ensure that all files are emitted from - /// source. - pub reload: bool, -} - -#[derive(Debug, Eq, PartialEq)] -pub enum BundleType { - /// Return the emitted contents of the program as a single "flattened" ES - /// module. - Esm, - // TODO(@kitsonk) once available in swc - // Iife, - /// Do not bundle the emit, instead returning each of the modules that are - /// part of the program as individual files. - None, -} - -impl Default for BundleType { - fn default() -> Self { - BundleType::None - } -} - -#[derive(Debug, Default)] -pub struct EmitOptions { - /// Indicate the form the result of the emit should take. - pub bundle_type: BundleType, - /// If `true` then debug logging will be output from the isolate. - pub debug: bool, - /// An optional map that contains user supplied TypeScript compiler - /// configuration options that are passed to the TypeScript compiler. - pub maybe_user_config: Option>, -} - -/// A structure which provides options when transpiling modules. -#[derive(Debug, Default)] -pub struct TranspileOptions { - /// If `true` then debug logging will be output from the isolate. - pub debug: bool, - /// An optional string that points to a user supplied TypeScript configuration - /// file that augments the the default configuration passed to the TypeScript - /// compiler. - pub maybe_config_path: Option, - /// Ignore any previously emits and ensure that all files are emitted from - /// source. - pub reload: bool, -} - -/// A dependency graph of modules, were the modules that have been inserted via -/// the builder will be loaded into the graph. Also provides an interface to -/// be able to manipulate and handle the graph. -#[derive(Debug, Clone)] -pub struct Graph2 { - /// A reference to the specifier handler that will retrieve and cache modules - /// for the graph. - handler: Rc>, - /// Optional TypeScript build info that will be passed to `tsc` if `tsc` is - /// invoked. - maybe_tsbuildinfo: Option, - /// The modules that are part of the graph. - modules: HashMap, - /// A map of redirects, where a module specifier is redirected to another - /// module specifier by the handler. All modules references should be - /// resolved internally via this, before attempting to access the module via - /// the handler, to make sure the correct modules is being dealt with. - redirects: HashMap, - /// The module specifiers that have been uniquely added to the graph, which - /// does not include any transient dependencies. - roots: Vec, - /// If all of the root modules are dynamically imported, then this is true. - /// This is used to ensure correct `--reload` behavior, where subsequent - /// calls to a module graph where the emit is already valid do not cause the - /// graph to re-emit. - roots_dynamic: bool, - // A reference to lock file that will be used to check module integrity. - maybe_lockfile: Option>>, -} - -impl Graph2 { - /// Create a new instance of a graph, ready to have modules loaded it. - /// - /// The argument `handler` is an instance of a structure that implements the - /// `SpecifierHandler` trait. - /// - pub fn new( - handler: Rc>, - maybe_lockfile: Option>>, - ) -> Self { - Graph2 { - handler, - maybe_tsbuildinfo: None, - modules: HashMap::new(), - redirects: HashMap::new(), - roots: Vec::new(), - roots_dynamic: true, - maybe_lockfile, - } - } - - /// Transform the module graph into a single JavaScript module which is - /// returned as a `String` in the result. - pub fn bundle( - &self, - options: BundleOptions, - ) -> Result<(String, Stats, Option), AnyError> { - if self.roots.is_empty() || self.roots.len() > 1 { - return Err(GraphError::NotSupported(format!("Bundling is only supported when there is a single root module in the graph. Found: {}", self.roots.len())).into()); - } - - let start = Instant::now(); - let root_specifier = self.roots[0].clone(); - let mut ts_config = TsConfig::new(json!({ - "checkJs": false, - "emitDecoratorMetadata": false, - "inlineSourceMap": true, - "jsx": "react", - "jsxFactory": "React.createElement", - "jsxFragmentFactory": "React.Fragment", - })); - let maybe_ignored_options = - ts_config.merge_tsconfig(options.maybe_config_path)?; - - let s = self.emit_bundle(&root_specifier, &ts_config.into())?; - let stats = Stats(vec![ - ("Files".to_string(), self.modules.len() as u128), - ("Total time".to_string(), start.elapsed().as_millis()), - ]); - - Ok((s, stats, maybe_ignored_options)) - } - - /// Type check the module graph, corresponding to the options provided. - pub fn check(self, options: CheckOptions) -> Result { - let mut config = TsConfig::new(json!({ - "allowJs": true, - // TODO(@kitsonk) is this really needed? - "esModuleInterop": true, - // Enabled by default to align to transpile/swc defaults - "experimentalDecorators": true, - "incremental": true, - "isolatedModules": true, - "lib": options.lib, - "module": "esnext", - "strict": true, - "target": "esnext", - "tsBuildInfoFile": "deno:///.tsbuildinfo", - })); - if options.emit { - config.merge(&json!({ - // TODO(@kitsonk) consider enabling this by default - // see: https://github.com/denoland/deno/issues/7732 - "emitDecoratorMetadata": false, - "jsx": "react", - "inlineSourceMap": true, - "outDir": "deno://", - "removeComments": true, - })); - } else { - config.merge(&json!({ - "noEmit": true, - })); - } - let maybe_ignored_options = - config.merge_tsconfig(options.maybe_config_path)?; - - // Short circuit if none of the modules require an emit, or all of the - // modules that require an emit have a valid emit. There is also an edge - // case where there are multiple imports of a dynamic module during a - // single invocation, if that is the case, even if there is a reload, we - // will simply look at if the emit is invalid, to avoid two checks for the - // same programme. - if !self.needs_emit(&config) - || (self.is_emit_valid(&config) - && (!options.reload || self.roots_dynamic)) - { - debug!("graph does not need to be checked or emitted."); - return Ok(ResultInfo { - maybe_ignored_options, - ..Default::default() - }); - } - - // TODO(@kitsonk) not totally happy with this here, but this is the first - // point where we know we are actually going to check the program. If we - // moved it out of here, we wouldn't know until after the check has already - // happened, which isn't informative to the users. - for specifier in &self.roots { - info!("{} {}", colors::green("Check"), specifier); - } - - let root_names = self.get_root_names(); - let maybe_tsbuildinfo = self.maybe_tsbuildinfo.clone(); - let hash_data = - vec![config.as_bytes(), version::DENO.as_bytes().to_owned()]; - let graph = Rc::new(RefCell::new(self)); - - let response = tsc2::exec( - js::compiler_isolate_init(), - tsc2::Request { - config: config.clone(), - debug: options.debug, - graph: graph.clone(), - hash_data, - maybe_tsbuildinfo, - root_names, - }, - )?; - - let mut graph = graph.borrow_mut(); - graph.maybe_tsbuildinfo = response.maybe_tsbuildinfo; - // Only process changes to the graph if there are no diagnostics and there - // were files emitted. - if response.diagnostics.is_empty() && !response.emitted_files.is_empty() { - let mut codes = HashMap::new(); - let mut maps = HashMap::new(); - let check_js = config.get_check_js(); - for emit in &response.emitted_files { - if let Some(specifiers) = &emit.maybe_specifiers { - assert!(specifiers.len() == 1, "Unexpected specifier length"); - // The specifier emitted might not be the redirected specifier, and - // therefore we need to ensure it is the correct one. - let specifier = graph.resolve_specifier(&specifiers[0]); - // Sometimes if tsc sees a CommonJS file it will _helpfully_ output it - // to ESM, which we don't really want unless someone has enabled the - // check_js option. - if !check_js - && graph.get_media_type(&specifier) == Some(MediaType::JavaScript) - { - debug!("skipping emit for {}", specifier); - continue; - } - match emit.media_type { - MediaType::JavaScript => { - codes.insert(specifier.clone(), emit.data.clone()); - } - MediaType::SourceMap => { - maps.insert(specifier.clone(), emit.data.clone()); - } - _ => unreachable!(), - } - } - } - let config = config.as_bytes(); - for (specifier, code) in codes.iter() { - if let Some(module) = graph.get_module_mut(specifier) { - module.maybe_emit = - Some(Emit::Cli((code.clone(), maps.get(specifier).cloned()))); - module.set_version(&config); - module.is_dirty = true; - } else { - return Err(GraphError::MissingSpecifier(specifier.clone()).into()); - } - } - } - graph.flush()?; - - Ok(ResultInfo { - diagnostics: response.diagnostics, - maybe_ignored_options, - stats: response.stats, - }) - } - - fn contains_module(&self, specifier: &ModuleSpecifier) -> bool { - let s = self.resolve_specifier(specifier); - self.modules.contains_key(s) - } - - /// Emit the module graph in a specific format. This is specifically designed - /// to be an "all-in-one" API for access by the runtime, allowing both - /// emitting single modules as well as bundles, using Deno module resolution - /// or supplied sources. - pub fn emit( - self, - options: EmitOptions, - ) -> Result<(HashMap, ResultInfo), AnyError> { - let mut config = TsConfig::new(json!({ - "allowJs": true, - // TODO(@kitsonk) consider enabling this by default - // see: https://github.com/denoland/deno/issues/7732 - "emitDecoratorMetadata": false, - "esModuleInterop": true, - "experimentalDecorators": true, - "isolatedModules": true, - "jsx": "react", - "lib": TypeLib::DenoWindow, - "module": "esnext", - "strict": true, - "target": "esnext", - })); - let opts = match options.bundle_type { - BundleType::Esm => json!({ - "checkJs": false, - "inlineSourceMap": false, - "noEmit": true, - "jsxFactory": "React.createElement", - "jsxFragmentFactory": "React.Fragment", - }), - BundleType::None => json!({ - "outDir": "deno://", - "removeComments": true, - "sourceMap": true, - }), - }; - config.merge(&opts); - let maybe_ignored_options = - if let Some(user_options) = &options.maybe_user_config { - config.merge_user_config(user_options)? - } else { - None - }; - - let root_names = self.get_root_names(); - let hash_data = - vec![config.as_bytes(), version::DENO.as_bytes().to_owned()]; - let graph = Rc::new(RefCell::new(self)); - - let response = tsc2::exec( - js::compiler_isolate_init(), - tsc2::Request { - config: config.clone(), - debug: options.debug, - graph: graph.clone(), - hash_data, - maybe_tsbuildinfo: None, - root_names, - }, - )?; - - let mut emitted_files = HashMap::new(); - match options.bundle_type { - BundleType::Esm => { - assert!( - response.emitted_files.is_empty(), - "No files should have been emitted from tsc." - ); - let graph = graph.borrow(); - assert_eq!( - graph.roots.len(), - 1, - "Only a single root module supported." - ); - let specifier = &graph.roots[0]; - let s = graph.emit_bundle(specifier, &config.into())?; - emitted_files.insert("deno:///bundle.js".to_string(), s); - } - BundleType::None => { - for emitted_file in &response.emitted_files { - assert!( - emitted_file.maybe_specifiers.is_some(), - "Orphaned file emitted." - ); - let specifiers = emitted_file.maybe_specifiers.clone().unwrap(); - assert_eq!( - specifiers.len(), - 1, - "An unexpected number of specifiers associated with emitted file." - ); - let specifier = specifiers[0].clone(); - let extension = match emitted_file.media_type { - MediaType::JavaScript => ".js", - MediaType::SourceMap => ".js.map", - _ => unreachable!(), - }; - let key = format!("{}{}", specifier, extension); - emitted_files.insert(key, emitted_file.data.clone()); - } - } - }; - - Ok(( - emitted_files, - ResultInfo { - diagnostics: response.diagnostics, - maybe_ignored_options, - stats: response.stats, - }, - )) - } - - /// Shared between `bundle()` and `emit()`. - fn emit_bundle( - &self, - specifier: &ModuleSpecifier, - emit_options: &ast::EmitOptions, - ) -> Result { - let cm = Rc::new(swc_common::SourceMap::new( - swc_common::FilePathMapping::empty(), - )); - let globals = swc_common::Globals::new(); - let loader = BundleLoader::new(self, emit_options, &globals, cm.clone()); - let hook = Box::new(BundleHook); - let bundler = swc_bundler::Bundler::new( - &globals, - cm.clone(), - loader, - self, - swc_bundler::Config::default(), - hook, - ); - let mut entries = HashMap::new(); - entries.insert( - "bundle".to_string(), - swc_common::FileName::Custom(specifier.to_string()), - ); - let output = bundler - .bundle(entries) - .context("Unable to output bundle during Graph2::bundle().")?; - let mut buf = Vec::new(); - { - let mut emitter = swc_ecmascript::codegen::Emitter { - cfg: swc_ecmascript::codegen::Config { minify: false }, - cm: cm.clone(), - comments: None, - wr: Box::new(swc_ecmascript::codegen::text_writer::JsWriter::new( - cm, "\n", &mut buf, None, - )), - }; - - emitter - .emit_module(&output[0].module) - .context("Unable to emit bundle during Graph2::bundle().")?; - } - - String::from_utf8(buf).context("Emitted bundle is an invalid utf-8 string.") - } - - /// Update the handler with any modules that are marked as _dirty_ and update - /// any build info if present. - fn flush(&mut self) -> Result<(), AnyError> { - let mut handler = self.handler.borrow_mut(); - for (_, module) in self.modules.iter_mut() { - if module.is_dirty { - if let Some(emit) = &module.maybe_emit { - handler.set_cache(&module.specifier, emit)?; - } - if let Some(version) = &module.maybe_version { - handler.set_version(&module.specifier, version.clone())?; - } - module.is_dirty = false; - } - } - for root_specifier in self.roots.iter() { - if let Some(tsbuildinfo) = &self.maybe_tsbuildinfo { - handler.set_tsbuildinfo(root_specifier, tsbuildinfo.to_owned())?; - } - } - - Ok(()) - } - - fn get_info( - &self, - specifier: &ModuleSpecifier, - seen: &mut HashSet, - totals: &mut HashMap, - ) -> ModuleInfo { - let not_seen = seen.insert(specifier.clone()); - let module = self.get_module(specifier).unwrap(); - let mut deps = Vec::new(); - let mut total_size = None; - - if not_seen { - let mut seen_deps = HashSet::new(); - // TODO(@kitsonk) https://github.com/denoland/deno/issues/7927 - for (_, dep) in module.dependencies.iter() { - // Check the runtime code dependency - if let Some(code_dep) = &dep.maybe_code { - if seen_deps.insert(code_dep.clone()) { - deps.push(self.get_info(code_dep, seen, totals)); - } - } - } - deps.sort(); - total_size = if let Some(total) = totals.get(specifier) { - Some(total.to_owned()) - } else { - let mut total = deps - .iter() - .map(|d| { - if let Some(total_size) = d.total_size { - total_size - } else { - 0 - } - }) - .sum(); - total += module.size(); - totals.insert(specifier.clone(), total); - Some(total) - }; - } - - ModuleInfo { - deps, - name: specifier.clone(), - size: module.size(), - total_size, - } - } - - fn get_info_map(&self) -> ModuleInfoMap { - let map = self - .modules - .iter() - .map(|(specifier, module)| { - let mut deps = HashSet::new(); - for (_, dep) in module.dependencies.iter() { - if let Some(code_dep) = &dep.maybe_code { - deps.insert(code_dep.clone()); - } - if let Some(type_dep) = &dep.maybe_type { - deps.insert(type_dep.clone()); - } - } - if let Some((_, types_dep)) = &module.maybe_types { - deps.insert(types_dep.clone()); - } - let item = ModuleInfoMapItem { - deps: deps.into_iter().collect(), - size: module.size(), - }; - (specifier.clone(), item) - }) - .collect(); - - ModuleInfoMap::new(map) - } - - pub fn get_media_type( - &self, - specifier: &ModuleSpecifier, - ) -> Option { - if let Some(module) = self.get_module(specifier) { - Some(module.media_type) - } else { - None - } - } - - fn get_module(&self, specifier: &ModuleSpecifier) -> Option<&Module> { - let s = self.resolve_specifier(specifier); - self.modules.get(s) - } - - fn get_module_mut( - &mut self, - specifier: &ModuleSpecifier, - ) -> Option<&mut Module> { - // this is duplicated code because `.resolve_specifier` requires an - // immutable borrow, but if `.resolve_specifier` is mut, then everything - // that calls it is is mut - let mut s = specifier; - while let Some(redirect) = self.redirects.get(s) { - s = redirect; - } - self.modules.get_mut(s) - } - - /// Consume graph and return list of all module specifiers contained in the - /// graph. - pub fn get_modules(&self) -> Vec { - self.modules.keys().map(|s| s.to_owned()).collect() - } - - /// Transform `self.roots` into something that works for `tsc`, because `tsc` - /// doesn't like root names without extensions that match its expectations, - /// nor does it have any concept of redirection, so we have to resolve all - /// that upfront before feeding it to `tsc`. - fn get_root_names(&self) -> Vec<(ModuleSpecifier, MediaType)> { - self - .roots - .iter() - .map(|ms| { - ( - // root modules can be redirects, so before we pass it to tsc we need - // to resolve the redirect - self.resolve_specifier(ms).clone(), - self.get_media_type(ms).unwrap(), - ) - }) - .collect() - } - - /// Get the source for a given module specifier. If the module is not part - /// of the graph, the result will be `None`. - pub fn get_source(&self, specifier: &ModuleSpecifier) -> Option { - if let Some(module) = self.get_module(specifier) { - Some(module.source.clone()) - } else { - None - } - } - - /// Return a structure which provides information about the module graph and - /// the relationship of the modules in the graph. This structure is used to - /// provide information for the `info` subcommand. - pub fn info(&self) -> Result { - if self.roots.is_empty() || self.roots.len() > 1 { - return Err(GraphError::NotSupported(format!("Info is only supported when there is a single root module in the graph. Found: {}", self.roots.len())).into()); - } - - let module = self.roots[0].clone(); - let m = self.get_module(&module).unwrap(); - - let mut seen = HashSet::new(); - let mut totals = HashMap::new(); - let info = self.get_info(&module, &mut seen, &mut totals); - - let files = self.get_info_map(); - let total_size = totals.get(&module).unwrap_or(&m.size()).to_owned(); - let (compiled, map) = - if let Some((emit_path, maybe_map_path)) = &m.maybe_emit_path { - (Some(emit_path.clone()), maybe_map_path.clone()) - } else { - (None, None) - }; - - Ok(ModuleGraphInfo { - compiled, - dep_count: self.modules.len() - 1, - file_type: m.media_type, - files, - info, - local: m.source_path.clone(), - map, - module, - total_size, - }) - } - - /// Determines if all of the modules in the graph that require an emit have - /// a valid emit. Returns `true` if all the modules have a valid emit, - /// otherwise false. - fn is_emit_valid(&self, config: &TsConfig) -> bool { - let check_js = config.get_check_js(); - let config = config.as_bytes(); - self.modules.iter().all(|(_, m)| { - let needs_emit = match m.media_type { - MediaType::TypeScript | MediaType::TSX | MediaType::JSX => true, - MediaType::JavaScript => check_js, - _ => false, - }; - if needs_emit { - m.is_emit_valid(&config) - } else { - true - } - }) - } - - /// Verify the subresource integrity of the graph based upon the optional - /// lockfile, updating the lockfile with any missing resources. This will - /// error if any of the resources do not match their lock status. - pub fn lock(&self) { - if let Some(lf) = self.maybe_lockfile.as_ref() { - let mut lockfile = lf.lock().unwrap(); - for (ms, module) in self.modules.iter() { - let specifier = module.specifier.to_string(); - let valid = lockfile.check_or_insert(&specifier, &module.source); - if !valid { - eprintln!( - "{}", - GraphError::InvalidSource(ms.clone(), lockfile.filename.clone()) - ); - std::process::exit(10); - } - } - } - } - - /// Determines if any of the modules in the graph are required to be emitted. - /// This is similar to `emit_valid()` except that the actual emit isn't - /// checked to determine if it is valid. - fn needs_emit(&self, config: &TsConfig) -> bool { - let check_js = config.get_check_js(); - self.modules.iter().any(|(_, m)| match m.media_type { - MediaType::TypeScript | MediaType::TSX | MediaType::JSX => true, - MediaType::JavaScript => check_js, - _ => false, - }) - } - - /// Given a string specifier and a referring module specifier, provide the - /// resulting module specifier and media type for the module that is part of - /// the graph. - /// - /// # Arguments - /// - /// * `specifier` - The string form of the module specifier that needs to be - /// resolved. - /// * `referrer` - The referring `ModuleSpecifier`. - /// * `prefer_types` - When resolving to a module specifier, determine if a - /// type dependency is preferred over a code dependency. This is set to - /// `true` when resolving module names for `tsc` as it needs the type - /// dependency over the code, while other consumers do not handle type only - /// dependencies. - pub fn resolve( - &self, - specifier: &str, - referrer: &ModuleSpecifier, - prefer_types: bool, - ) -> Result { - if !self.contains_module(referrer) { - return Err(GraphError::MissingSpecifier(referrer.to_owned()).into()); - } - let module = self.get_module(referrer).unwrap(); - if !module.dependencies.contains_key(specifier) { - return Err( - GraphError::MissingDependency( - referrer.to_owned(), - specifier.to_owned(), - ) - .into(), - ); - } - let dependency = module.dependencies.get(specifier).unwrap(); - // If there is a @deno-types pragma that impacts the dependency, then the - // maybe_type property will be set with that specifier, otherwise we use the - // specifier that point to the runtime code. - let resolved_specifier = if prefer_types && dependency.maybe_type.is_some() - { - dependency.maybe_type.clone().unwrap() - } else if let Some(code_specifier) = dependency.maybe_code.clone() { - code_specifier - } else { - return Err( - GraphError::MissingDependency( - referrer.to_owned(), - specifier.to_owned(), - ) - .into(), - ); - }; - if !self.contains_module(&resolved_specifier) { - return Err( - GraphError::MissingDependency( - referrer.to_owned(), - resolved_specifier.to_string(), - ) - .into(), - ); - } - let dep_module = self.get_module(&resolved_specifier).unwrap(); - // In the case that there is a X-TypeScript-Types or a triple-slash types, - // then the `maybe_types` specifier will be populated and we should use that - // instead. - let result = if prefer_types && dep_module.maybe_types.is_some() { - let (_, types) = dep_module.maybe_types.clone().unwrap(); - // It is possible that `types` points to a redirected specifier, so we - // need to ensure it resolves to the final specifier in the graph. - self.resolve_specifier(&types).clone() - } else { - dep_module.specifier.clone() - }; - - Ok(result) - } - - /// Takes a module specifier and returns the "final" specifier, accounting for - /// any redirects that may have occurred. - fn resolve_specifier<'a>( - &'a self, - specifier: &'a ModuleSpecifier, - ) -> &'a ModuleSpecifier { - let mut s = specifier; - let mut seen = HashSet::new(); - seen.insert(s.clone()); - while let Some(redirect) = self.redirects.get(s) { - if !seen.insert(redirect.clone()) { - eprintln!("An infinite loop of module redirections detected.\n Original specifier: {}", specifier); - break; - } - s = redirect; - if seen.len() > 5 { - eprintln!("An excessive number of module redirections detected.\n Original specifier: {}", specifier); - break; - } - } - s - } - - /// Transpile (only transform) the graph, updating any emitted modules - /// with the specifier handler. The result contains any performance stats - /// from the compiler and optionally any user provided configuration compiler - /// options that were ignored. - /// - /// # Arguments - /// - /// * `options` - A structure of options which impact how the code is - /// transpiled. - /// - pub fn transpile( - &mut self, - options: TranspileOptions, - ) -> Result<(Stats, Option), AnyError> { - let start = Instant::now(); - - let mut ts_config = TsConfig::new(json!({ - "checkJs": false, - "emitDecoratorMetadata": false, - "inlineSourceMap": true, - "jsx": "react", - "jsxFactory": "React.createElement", - "jsxFragmentFactory": "React.Fragment", - })); - - let maybe_ignored_options = - ts_config.merge_tsconfig(options.maybe_config_path)?; - - let emit_options: ast::EmitOptions = ts_config.clone().into(); - - let mut emit_count: u128 = 0; - let config = ts_config.as_bytes(); - for (_, module) in self.modules.iter_mut() { - // TODO(kitsonk) a lot of this logic should be refactored into `Module` as - // we start to support other methods on the graph. Especially managing - // the dirty state is something the module itself should "own". - - // if the module is a Dts file we should skip it - if module.media_type == MediaType::Dts { - continue; - } - // if we don't have check_js enabled, we won't touch non TypeScript - // modules - if !(emit_options.check_js - || module.media_type == MediaType::TSX - || module.media_type == MediaType::TypeScript) - { - continue; - } - // skip modules that already have a valid emit - if !options.reload && module.is_emit_valid(&config) { - continue; - } - if module.maybe_parsed_module.is_none() { - module.parse()?; - } - let parsed_module = module.maybe_parsed_module.clone().unwrap(); - let emit = parsed_module.transpile(&emit_options)?; - emit_count += 1; - module.maybe_emit = Some(Emit::Cli(emit)); - module.set_version(&config); - module.is_dirty = true; - } - self.flush()?; - - let stats = Stats(vec![ - ("Files".to_string(), self.modules.len() as u128), - ("Emitted".to_string(), emit_count), - ("Total time".to_string(), start.elapsed().as_millis()), - ]); - - Ok((stats, maybe_ignored_options)) - } -} - -impl swc_bundler::Resolve for Graph2 { - fn resolve( - &self, - referrer: &swc_common::FileName, - specifier: &str, - ) -> Result { - let referrer = if let swc_common::FileName::Custom(referrer) = referrer { - ModuleSpecifier::resolve_url_or_path(referrer) - .context("Cannot resolve swc FileName to a module specifier")? - } else { - unreachable!( - "An unexpected referrer was passed when bundling: {:?}", - referrer - ) - }; - let specifier = self.resolve(specifier, &referrer, false)?; - - Ok(swc_common::FileName::Custom(specifier.to_string())) - } -} - -/// A structure for building a dependency graph of modules. -pub struct GraphBuilder2 { - fetched: HashSet, - graph: Graph2, - maybe_import_map: Option>>, - pending: FuturesUnordered, -} - -impl GraphBuilder2 { - pub fn new( - handler: Rc>, - maybe_import_map: Option, - maybe_lockfile: Option>>, - ) -> Self { - let internal_import_map = if let Some(import_map) = maybe_import_map { - Some(Rc::new(RefCell::new(import_map))) - } else { - None - }; - GraphBuilder2 { - graph: Graph2::new(handler, maybe_lockfile), - fetched: HashSet::new(), - maybe_import_map: internal_import_map, - pending: FuturesUnordered::new(), - } - } - - /// Add a module into the graph based on a module specifier. The module - /// and any dependencies will be fetched from the handler. The module will - /// also be treated as a _root_ module in the graph. - pub async fn add( - &mut self, - specifier: &ModuleSpecifier, - is_dynamic: bool, - ) -> Result<(), AnyError> { - self.fetch(specifier, &None, is_dynamic)?; - - loop { - let cached_module = self.pending.next().await.unwrap()?; - let is_root = &cached_module.specifier == specifier; - self.visit(cached_module, is_root)?; - if self.pending.is_empty() { - break; - } - } - - if !self.graph.roots.contains(specifier) { - self.graph.roots.push(specifier.clone()); - self.graph.roots_dynamic = self.graph.roots_dynamic && is_dynamic; - if self.graph.maybe_tsbuildinfo.is_none() { - let handler = self.graph.handler.borrow(); - self.graph.maybe_tsbuildinfo = handler.get_tsbuildinfo(specifier)?; - } - } - - Ok(()) - } - - /// Request a module to be fetched from the handler and queue up its future - /// to be awaited to be resolved. - fn fetch( - &mut self, - specifier: &ModuleSpecifier, - maybe_referrer: &Option, - is_dynamic: bool, - ) -> Result<(), AnyError> { - if self.fetched.contains(&specifier) { - return Ok(()); - } - - self.fetched.insert(specifier.clone()); - let future = self.graph.handler.borrow_mut().fetch( - specifier.clone(), - maybe_referrer.clone(), - is_dynamic, - ); - self.pending.push(future); - - Ok(()) - } - - /// Visit a module that has been fetched, hydrating the module, analyzing its - /// dependencies if required, fetching those dependencies, and inserting the - /// module into the graph. - fn visit( - &mut self, - cached_module: CachedModule, - is_root: bool, - ) -> Result<(), AnyError> { - let specifier = cached_module.specifier.clone(); - let requested_specifier = cached_module.requested_specifier.clone(); - let mut module = - Module::new(cached_module, is_root, self.maybe_import_map.clone()); - match module.media_type { - MediaType::Json - | MediaType::SourceMap - | MediaType::TsBuildInfo - | MediaType::Unknown => { - return Err( - GraphError::UnsupportedImportType( - module.specifier, - module.media_type, - ) - .into(), - ); - } - _ => (), - } - if !module.is_parsed { - let has_types = module.maybe_types.is_some(); - module.parse()?; - if self.maybe_import_map.is_none() { - let mut handler = self.graph.handler.borrow_mut(); - handler.set_deps(&specifier, module.dependencies.clone())?; - if !has_types { - if let Some((types, _)) = module.maybe_types.clone() { - handler.set_types(&specifier, types)?; - } - } - } - } - for (_, dep) in module.dependencies.iter() { - let maybe_referrer = Some(dep.location.clone()); - if let Some(specifier) = dep.maybe_code.as_ref() { - self.fetch(specifier, &maybe_referrer, dep.is_dynamic)?; - } - if let Some(specifier) = dep.maybe_type.as_ref() { - self.fetch(specifier, &maybe_referrer, dep.is_dynamic)?; - } - } - if let Some((_, specifier)) = module.maybe_types.as_ref() { - self.fetch(specifier, &None, false)?; - } - if specifier != requested_specifier { - self - .graph - .redirects - .insert(requested_specifier, specifier.clone()); - } - self.graph.modules.insert(specifier, module); - - Ok(()) - } - - /// Move out the graph from the builder to be utilized further. An optional - /// lockfile can be provided, where if the sources in the graph do not match - /// the expected lockfile, an error will be logged and the process will exit. - pub fn get_graph(self) -> Graph2 { - self.graph.lock(); - self.graph - } -} - -#[cfg(test)] -pub mod tests { - use super::*; - - use crate::specifier_handler::MemoryHandler; - use deno_core::futures::future; - use std::env; - use std::fs; - use std::path::PathBuf; - use std::sync::Mutex; - - macro_rules! map ( - { $($key:expr => $value:expr),+ } => { - { - let mut m = ::std::collections::HashMap::new(); - $( - m.insert($key, $value); - )+ - m - } - }; - ); - - /// This is a testing mock for `SpecifierHandler` that uses a special file - /// system renaming to mock local and remote modules as well as provides - /// "spies" for the critical methods for testing purposes. - #[derive(Debug, Default)] - pub struct MockSpecifierHandler { - pub fixtures: PathBuf, - pub maybe_tsbuildinfo: Option, - pub tsbuildinfo_calls: Vec<(ModuleSpecifier, String)>, - pub cache_calls: Vec<(ModuleSpecifier, Emit)>, - pub deps_calls: Vec<(ModuleSpecifier, DependencyMap)>, - pub types_calls: Vec<(ModuleSpecifier, String)>, - pub version_calls: Vec<(ModuleSpecifier, String)>, - } - - impl MockSpecifierHandler { - fn get_cache( - &self, - specifier: ModuleSpecifier, - ) -> Result { - let specifier_text = specifier - .to_string() - .replace(":///", "_") - .replace("://", "_") - .replace("/", "-"); - let source_path = self.fixtures.join(specifier_text); - let media_type = MediaType::from(&source_path); - let source = fs::read_to_string(&source_path)?; - let is_remote = specifier.as_url().scheme() != "file"; - - Ok(CachedModule { - source, - requested_specifier: specifier.clone(), - source_path, - specifier, - media_type, - is_remote, - ..CachedModule::default() - }) - } - } - - impl SpecifierHandler for MockSpecifierHandler { - fn fetch( - &mut self, - specifier: ModuleSpecifier, - _maybe_referrer: Option, - _is_dynamic: bool, - ) -> FetchFuture { - Box::pin(future::ready(self.get_cache(specifier))) - } - fn get_tsbuildinfo( - &self, - _specifier: &ModuleSpecifier, - ) -> Result, AnyError> { - Ok(self.maybe_tsbuildinfo.clone()) - } - fn set_cache( - &mut self, - specifier: &ModuleSpecifier, - emit: &Emit, - ) -> Result<(), AnyError> { - self.cache_calls.push((specifier.clone(), emit.clone())); - Ok(()) - } - fn set_types( - &mut self, - specifier: &ModuleSpecifier, - types: String, - ) -> Result<(), AnyError> { - self.types_calls.push((specifier.clone(), types)); - Ok(()) - } - fn set_tsbuildinfo( - &mut self, - specifier: &ModuleSpecifier, - tsbuildinfo: String, - ) -> Result<(), AnyError> { - self.maybe_tsbuildinfo = Some(tsbuildinfo.clone()); - self - .tsbuildinfo_calls - .push((specifier.clone(), tsbuildinfo)); - Ok(()) - } - fn set_deps( - &mut self, - specifier: &ModuleSpecifier, - dependencies: DependencyMap, - ) -> Result<(), AnyError> { - self.deps_calls.push((specifier.clone(), dependencies)); - Ok(()) - } - fn set_version( - &mut self, - specifier: &ModuleSpecifier, - version: String, - ) -> Result<(), AnyError> { - self.version_calls.push((specifier.clone(), version)); - Ok(()) - } - } - - async fn setup( - specifier: ModuleSpecifier, - ) -> (Graph2, Rc>) { - let c = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()); - let fixtures = c.join("tests/module_graph"); - let handler = Rc::new(RefCell::new(MockSpecifierHandler { - fixtures, - ..MockSpecifierHandler::default() - })); - let mut builder = GraphBuilder2::new(handler.clone(), None, None); - builder - .add(&specifier, false) - .await - .expect("module not inserted"); - - (builder.get_graph(), handler) - } - - async fn setup_memory( - specifier: ModuleSpecifier, - sources: HashMap<&str, &str>, - ) -> Graph2 { - let sources: HashMap = sources - .iter() - .map(|(k, v)| (k.to_string(), v.to_string())) - .collect(); - let handler = Rc::new(RefCell::new(MemoryHandler::new(sources))); - let mut builder = GraphBuilder2::new(handler.clone(), None, None); - builder - .add(&specifier, false) - .await - .expect("module not inserted"); - - builder.get_graph() - } - - #[test] - fn test_get_version() { - let doc_a = "console.log(42);"; - let version_a = get_version(&doc_a, "1.2.3", b""); - let doc_b = "console.log(42);"; - let version_b = get_version(&doc_b, "1.2.3", b""); - assert_eq!(version_a, version_b); - - let version_c = get_version(&doc_a, "1.2.3", b"options"); - assert_ne!(version_a, version_c); - - let version_d = get_version(&doc_b, "1.2.3", b"options"); - assert_eq!(version_c, version_d); - - let version_e = get_version(&doc_a, "1.2.4", b""); - assert_ne!(version_a, version_e); - - let version_f = get_version(&doc_b, "1.2.4", b""); - assert_eq!(version_e, version_f); - } - - #[test] - fn test_module_emit_valid() { - let source = "console.log(42);".to_string(); - let maybe_version = Some(get_version(&source, version::DENO, b"")); - let module = Module { - source, - maybe_version, - ..Module::default() - }; - assert!(module.is_emit_valid(b"")); - - let source = "console.log(42);".to_string(); - let old_source = "console.log(43);"; - let maybe_version = Some(get_version(old_source, version::DENO, b"")); - let module = Module { - source, - maybe_version, - ..Module::default() - }; - assert!(!module.is_emit_valid(b"")); - - let source = "console.log(42);".to_string(); - let maybe_version = Some(get_version(&source, "0.0.0", b"")); - let module = Module { - source, - maybe_version, - ..Module::default() - }; - assert!(!module.is_emit_valid(b"")); - - let source = "console.log(42);".to_string(); - let module = Module { - source, - ..Module::default() - }; - assert!(!module.is_emit_valid(b"")); - } - - #[test] - fn test_module_set_version() { - let source = "console.log(42);".to_string(); - let expected = Some(get_version(&source, version::DENO, b"")); - let mut module = Module { - source, - ..Module::default() - }; - assert!(module.maybe_version.is_none()); - module.set_version(b""); - assert_eq!(module.maybe_version, expected); - } - - #[tokio::test] - async fn test_graph_bundle() { - let tests = vec![ - ("file:///tests/fixture01.ts", "fixture01.out"), - ("file:///tests/fixture02.ts", "fixture02.out"), - ("file:///tests/fixture03.ts", "fixture03.out"), - ("file:///tests/fixture04.ts", "fixture04.out"), - ("file:///tests/fixture05.ts", "fixture05.out"), - ("file:///tests/fixture06.ts", "fixture06.out"), - ("file:///tests/fixture07.ts", "fixture07.out"), - ("file:///tests/fixture08.ts", "fixture08.out"), - ("file:///tests/fixture09.ts", "fixture09.out"), - ("file:///tests/fixture10.ts", "fixture10.out"), - ("file:///tests/fixture11.ts", "fixture11.out"), - ("file:///tests/fixture12.ts", "fixture12.out"), - ("file:///tests/fixture13.ts", "fixture13.out"), - ("file:///tests/fixture14.ts", "fixture14.out"), - ]; - let c = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()); - let fixtures = c.join("tests/bundle"); - - for (specifier, expected_str) in tests { - let specifier = ModuleSpecifier::resolve_url_or_path(specifier).unwrap(); - let handler = Rc::new(RefCell::new(MockSpecifierHandler { - fixtures: fixtures.clone(), - ..MockSpecifierHandler::default() - })); - let mut builder = GraphBuilder2::new(handler.clone(), None, None); - builder - .add(&specifier, false) - .await - .expect("module not inserted"); - let graph = builder.get_graph(); - let (actual, stats, maybe_ignored_options) = graph - .bundle(BundleOptions::default()) - .expect("could not bundle"); - assert_eq!(stats.0.len(), 2); - assert_eq!(maybe_ignored_options, None); - let expected_path = fixtures.join(expected_str); - let expected = fs::read_to_string(expected_path).unwrap(); - assert_eq!(actual, expected, "fixture: {}", specifier); - } - } - - #[tokio::test] - async fn test_graph_check_emit() { - let specifier = - ModuleSpecifier::resolve_url_or_path("file:///tests/main.ts") - .expect("could not resolve module"); - let (graph, handler) = setup(specifier).await; - let result_info = graph - .check(CheckOptions { - debug: false, - emit: true, - lib: TypeLib::DenoWindow, - maybe_config_path: None, - reload: false, - }) - .expect("should have checked"); - assert!(result_info.maybe_ignored_options.is_none()); - assert_eq!(result_info.stats.0.len(), 12); - assert!(result_info.diagnostics.is_empty()); - let h = handler.borrow(); - assert_eq!(h.cache_calls.len(), 2); - assert_eq!(h.tsbuildinfo_calls.len(), 1); - } - - #[tokio::test] - async fn test_graph_check_no_emit() { - let specifier = - ModuleSpecifier::resolve_url_or_path("file:///tests/main.ts") - .expect("could not resolve module"); - let (graph, handler) = setup(specifier).await; - let result_info = graph - .check(CheckOptions { - debug: false, - emit: false, - lib: TypeLib::DenoWindow, - maybe_config_path: None, - reload: false, - }) - .expect("should have checked"); - assert!(result_info.maybe_ignored_options.is_none()); - assert_eq!(result_info.stats.0.len(), 12); - assert!(result_info.diagnostics.is_empty()); - let h = handler.borrow(); - assert_eq!(h.cache_calls.len(), 0); - assert_eq!(h.tsbuildinfo_calls.len(), 1); - } - - #[tokio::test] - async fn test_graph_check_user_config() { - let specifier = - ModuleSpecifier::resolve_url_or_path("file:///tests/checkwithconfig.ts") - .expect("could not resolve module"); - let (graph, handler) = setup(specifier.clone()).await; - let result_info = graph - .check(CheckOptions { - debug: false, - emit: true, - lib: TypeLib::DenoWindow, - maybe_config_path: Some( - "tests/module_graph/tsconfig_01.json".to_string(), - ), - reload: true, - }) - .expect("should have checked"); - assert!(result_info.maybe_ignored_options.is_none()); - assert!(result_info.diagnostics.is_empty()); - let h = handler.borrow(); - assert_eq!(h.version_calls.len(), 2); - let ver0 = h.version_calls[0].1.clone(); - let ver1 = h.version_calls[1].1.clone(); - - // let's do it all over again to ensure that the versions are determinstic - let (graph, handler) = setup(specifier).await; - let result_info = graph - .check(CheckOptions { - debug: false, - emit: true, - lib: TypeLib::DenoWindow, - maybe_config_path: Some( - "tests/module_graph/tsconfig_01.json".to_string(), - ), - reload: true, - }) - .expect("should have checked"); - assert!(result_info.maybe_ignored_options.is_none()); - assert!(result_info.diagnostics.is_empty()); - let h = handler.borrow(); - assert_eq!(h.version_calls.len(), 2); - assert!(h.version_calls[0].1 == ver0 || h.version_calls[0].1 == ver1); - assert!(h.version_calls[1].1 == ver0 || h.version_calls[1].1 == ver1); - } - - #[tokio::test] - async fn test_graph_emit() { - let specifier = - ModuleSpecifier::resolve_url_or_path("file:///a.ts").unwrap(); - let graph = setup_memory( - specifier, - map!( - "/a.ts" => r#" - import * as b from "./b.ts"; - - console.log(b); - "#, - "/b.ts" => r#" - export const b = "b"; - "# - ), - ) - .await; - let (emitted_files, result_info) = graph - .emit(EmitOptions { - bundle_type: BundleType::None, - debug: false, - maybe_user_config: None, - }) - .expect("should have emitted"); - assert!(result_info.diagnostics.is_empty()); - assert!(result_info.maybe_ignored_options.is_none()); - assert_eq!(emitted_files.len(), 4); - let out_a = emitted_files.get("file:///a.ts.js"); - assert!(out_a.is_some()); - let out_a = out_a.unwrap(); - assert!(out_a.starts_with("import * as b from")); - assert!(emitted_files.contains_key("file:///a.ts.js.map")); - let out_b = emitted_files.get("file:///b.ts.js"); - assert!(out_b.is_some()); - let out_b = out_b.unwrap(); - assert!(out_b.starts_with("export const b = \"b\";")); - assert!(emitted_files.contains_key("file:///b.ts.js.map")); - } - - #[tokio::test] - async fn test_graph_emit_bundle() { - let specifier = - ModuleSpecifier::resolve_url_or_path("file:///a.ts").unwrap(); - let graph = setup_memory( - specifier, - map!( - "/a.ts" => r#" - import * as b from "./b.ts"; - - console.log(b); - "#, - "/b.ts" => r#" - export const b = "b"; - "# - ), - ) - .await; - let (emitted_files, result_info) = graph - .emit(EmitOptions { - bundle_type: BundleType::Esm, - debug: false, - maybe_user_config: None, - }) - .expect("should have emitted"); - assert!(result_info.diagnostics.is_empty()); - assert!(result_info.maybe_ignored_options.is_none()); - assert_eq!(emitted_files.len(), 1); - let actual = emitted_files.get("deno:///bundle.js"); - assert!(actual.is_some()); - let actual = actual.unwrap(); - assert!(actual.contains("const b = \"b\";")); - assert!(actual.contains("console.log(b);")); - } - - #[tokio::test] - async fn test_graph_info() { - let specifier = - ModuleSpecifier::resolve_url_or_path("file:///tests/main.ts") - .expect("could not resolve module"); - let (graph, _) = setup(specifier).await; - let info = graph.info().expect("could not get info"); - assert!(info.compiled.is_none()); - assert_eq!(info.dep_count, 6); - assert_eq!(info.file_type, MediaType::TypeScript); - assert_eq!(info.files.0.len(), 7); - assert!(info.local.to_string_lossy().ends_with("file_tests-main.ts")); - assert!(info.map.is_none()); - assert_eq!( - info.module, - ModuleSpecifier::resolve_url_or_path("file:///tests/main.ts").unwrap() - ); - assert_eq!(info.total_size, 344); - } - - #[tokio::test] - async fn test_graph_import_json() { - let specifier = - ModuleSpecifier::resolve_url_or_path("file:///tests/importjson.ts") - .expect("could not resolve module"); - let c = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()); - let fixtures = c.join("tests/module_graph"); - let handler = Rc::new(RefCell::new(MockSpecifierHandler { - fixtures, - ..MockSpecifierHandler::default() - })); - let mut builder = GraphBuilder2::new(handler.clone(), None, None); - builder - .add(&specifier, false) - .await - .expect_err("should have errored"); - } - - #[tokio::test] - async fn test_graph_transpile() { - // This is a complex scenario of transpiling, where we have TypeScript - // importing a JavaScript file (with type definitions) which imports - // TypeScript, JavaScript, and JavaScript with type definitions. - // For scenarios where we transpile, we only want the TypeScript files - // to be actually emitted. - // - // This also exercises "@deno-types" and type references. - let specifier = - ModuleSpecifier::resolve_url_or_path("file:///tests/main.ts") - .expect("could not resolve module"); - let (mut graph, handler) = setup(specifier).await; - let (stats, maybe_ignored_options) = - graph.transpile(TranspileOptions::default()).unwrap(); - assert_eq!(stats.0.len(), 3); - assert_eq!(maybe_ignored_options, None); - let h = handler.borrow(); - assert_eq!(h.cache_calls.len(), 2); - match &h.cache_calls[0].1 { - Emit::Cli((code, maybe_map)) => { - assert!( - code.contains("# sourceMappingURL=data:application/json;base64,") - ); - assert!(maybe_map.is_none()); - } - }; - match &h.cache_calls[1].1 { - Emit::Cli((code, maybe_map)) => { - assert!( - code.contains("# sourceMappingURL=data:application/json;base64,") - ); - assert!(maybe_map.is_none()); - } - }; - assert_eq!(h.deps_calls.len(), 7); - assert_eq!( - h.deps_calls[0].0, - ModuleSpecifier::resolve_url_or_path("file:///tests/main.ts").unwrap() - ); - assert_eq!(h.deps_calls[0].1.len(), 1); - assert_eq!( - h.deps_calls[1].0, - ModuleSpecifier::resolve_url_or_path("https://deno.land/x/lib/mod.js") - .unwrap() - ); - assert_eq!(h.deps_calls[1].1.len(), 3); - assert_eq!( - h.deps_calls[2].0, - ModuleSpecifier::resolve_url_or_path("https://deno.land/x/lib/mod.d.ts") - .unwrap() - ); - assert_eq!(h.deps_calls[2].1.len(), 3, "should have 3 dependencies"); - // sometimes the calls are not deterministic, and so checking the contents - // can cause some failures - assert_eq!(h.deps_calls[3].1.len(), 0, "should have no dependencies"); - assert_eq!(h.deps_calls[4].1.len(), 0, "should have no dependencies"); - assert_eq!(h.deps_calls[5].1.len(), 0, "should have no dependencies"); - assert_eq!(h.deps_calls[6].1.len(), 0, "should have no dependencies"); - } - - #[tokio::test] - async fn test_graph_transpile_user_config() { - let specifier = - ModuleSpecifier::resolve_url_or_path("https://deno.land/x/transpile.tsx") - .expect("could not resolve module"); - let (mut graph, handler) = setup(specifier).await; - let (_, maybe_ignored_options) = graph - .transpile(TranspileOptions { - debug: false, - maybe_config_path: Some("tests/module_graph/tsconfig.json".to_string()), - reload: false, - }) - .unwrap(); - assert_eq!( - maybe_ignored_options.unwrap().items, - vec!["target".to_string()], - "the 'target' options should have been ignored" - ); - let h = handler.borrow(); - assert_eq!(h.cache_calls.len(), 1, "only one file should be emitted"); - // FIXME(bartlomieju): had to add space in `
`, probably a quirk in swc_ecma_codegen - match &h.cache_calls[0].1 { - Emit::Cli((code, _)) => { - assert!( - code.contains("
Hello world!
"), - "jsx should have been preserved" - ); - } - } - } - - #[tokio::test] - async fn test_graph_with_lockfile() { - let c = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()); - let fixtures = c.join("tests/module_graph"); - let lockfile_path = fixtures.join("lockfile.json"); - let lockfile = - Lockfile::new(lockfile_path, false).expect("could not load lockfile"); - let maybe_lockfile = Some(Arc::new(Mutex::new(lockfile))); - let handler = Rc::new(RefCell::new(MockSpecifierHandler { - fixtures, - ..MockSpecifierHandler::default() - })); - let mut builder = GraphBuilder2::new(handler.clone(), None, maybe_lockfile); - let specifier = - ModuleSpecifier::resolve_url_or_path("file:///tests/main.ts") - .expect("could not resolve module"); - builder - .add(&specifier, false) - .await - .expect("module not inserted"); - builder.get_graph(); - } -} diff --git a/cli/module_loader.rs b/cli/module_loader.rs index b19476fe2..1715f7f6b 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -1,7 +1,7 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. use crate::import_map::ImportMap; -use crate::module_graph2::TypeLib; +use crate::module_graph::TypeLib; use crate::permissions::Permissions; use crate::program_state::ProgramState; use deno_core::error::AnyError; diff --git a/cli/op_fetch_asset.rs b/cli/op_fetch_asset.rs deleted file mode 100644 index dcb54cde5..000000000 --- a/cli/op_fetch_asset.rs +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. -// Note: this module is used both in build.rs and main.rs. - -pub use deno_core::v8_set_flags; -use deno_core::BufVec; -use deno_core::Op; -use deno_core::OpState; -use std::cell::RefCell; -use std::collections::HashMap; -use std::path::PathBuf; -use std::rc::Rc; - -pub fn get_asset(name: &str) -> Option<&'static str> { - macro_rules! inc { - ($e:expr) => { - Some(include_str!(concat!("dts/", $e))) - }; - } - match name { - "bootstrap.ts" => Some("console.log(\"hello deno\");"), - "typescript.d.ts" => inc!("typescript.d.ts"), - "lib.dom.d.ts" => inc!("lib.dom.d.ts"), - "lib.dom.iterable.d.ts" => inc!("lib.dom.iterable.d.ts"), - "lib.es5.d.ts" => inc!("lib.es5.d.ts"), - "lib.es6.d.ts" => inc!("lib.es6.d.ts"), - "lib.esnext.d.ts" => inc!("lib.esnext.d.ts"), - "lib.es2020.d.ts" => inc!("lib.es2020.d.ts"), - "lib.es2020.full.d.ts" => inc!("lib.es2020.full.d.ts"), - "lib.es2019.d.ts" => inc!("lib.es2019.d.ts"), - "lib.es2019.full.d.ts" => inc!("lib.es2019.full.d.ts"), - "lib.es2018.d.ts" => inc!("lib.es2018.d.ts"), - "lib.es2018.full.d.ts" => inc!("lib.es2018.full.d.ts"), - "lib.es2017.d.ts" => inc!("lib.es2017.d.ts"), - "lib.es2017.full.d.ts" => inc!("lib.es2017.full.d.ts"), - "lib.es2016.d.ts" => inc!("lib.es2016.d.ts"), - "lib.es2016.full.d.ts" => inc!("lib.es2016.full.d.ts"), - "lib.es2015.d.ts" => inc!("lib.es2015.d.ts"), - "lib.es2015.collection.d.ts" => inc!("lib.es2015.collection.d.ts"), - "lib.es2015.core.d.ts" => inc!("lib.es2015.core.d.ts"), - "lib.es2015.generator.d.ts" => inc!("lib.es2015.generator.d.ts"), - "lib.es2015.iterable.d.ts" => inc!("lib.es2015.iterable.d.ts"), - "lib.es2015.promise.d.ts" => inc!("lib.es2015.promise.d.ts"), - "lib.es2015.proxy.d.ts" => inc!("lib.es2015.proxy.d.ts"), - "lib.es2015.reflect.d.ts" => inc!("lib.es2015.reflect.d.ts"), - "lib.es2015.symbol.d.ts" => inc!("lib.es2015.symbol.d.ts"), - "lib.es2015.symbol.wellknown.d.ts" => { - inc!("lib.es2015.symbol.wellknown.d.ts") - } - "lib.es2016.array.include.d.ts" => inc!("lib.es2016.array.include.d.ts"), - "lib.es2017.intl.d.ts" => inc!("lib.es2017.intl.d.ts"), - "lib.es2017.object.d.ts" => inc!("lib.es2017.object.d.ts"), - "lib.es2017.sharedmemory.d.ts" => inc!("lib.es2017.sharedmemory.d.ts"), - "lib.es2017.string.d.ts" => inc!("lib.es2017.string.d.ts"), - "lib.es2017.typedarrays.d.ts" => inc!("lib.es2017.typedarrays.d.ts"), - "lib.es2018.asyncgenerator.d.ts" => inc!("lib.es2018.asyncgenerator.d.ts"), - "lib.es2018.asynciterable.d.ts" => inc!("lib.es2018.asynciterable.d.ts"), - "lib.es2018.intl.d.ts" => inc!("lib.es2018.intl.d.ts"), - "lib.es2018.promise.d.ts" => inc!("lib.es2018.promise.d.ts"), - "lib.es2018.regexp.d.ts" => inc!("lib.es2018.regexp.d.ts"), - "lib.es2019.array.d.ts" => inc!("lib.es2019.array.d.ts"), - "lib.es2019.object.d.ts" => inc!("lib.es2019.object.d.ts"), - "lib.es2019.string.d.ts" => inc!("lib.es2019.string.d.ts"), - "lib.es2019.symbol.d.ts" => inc!("lib.es2019.symbol.d.ts"), - "lib.es2020.bigint.d.ts" => inc!("lib.es2020.bigint.d.ts"), - "lib.es2020.intl.d.ts" => inc!("lib.es2020.intl.d.ts"), - "lib.es2020.promise.d.ts" => inc!("lib.es2020.promise.d.ts"), - "lib.es2020.string.d.ts" => inc!("lib.es2020.string.d.ts"), - "lib.es2020.symbol.wellknown.d.ts" => { - inc!("lib.es2020.symbol.wellknown.d.ts") - } - "lib.esnext.intl.d.ts" => inc!("lib.esnext.intl.d.ts"), - "lib.esnext.promise.d.ts" => inc!("lib.esnext.promise.d.ts"), - "lib.esnext.string.d.ts" => inc!("lib.esnext.string.d.ts"), - "lib.esnext.weakref.d.ts" => inc!("lib.esnext.weakref.d.ts"), - "lib.scripthost.d.ts" => inc!("lib.scripthost.d.ts"), - "lib.webworker.d.ts" => inc!("lib.webworker.d.ts"), - "lib.webworker.importscripts.d.ts" => { - inc!("lib.webworker.importscripts.d.ts") - } - _ => None, - } -} - -/// Warning: Returns a non-JSON op dispatcher. Must be manually attached to -/// JsRuntime. -/// -/// TODO(@kitsonk) this is only used when building the snapshot, and needs to -/// be refactored somewhere else. It is no longer used by `main.rs` and -/// therefore requires the allow unused. -#[allow(unused)] -pub fn op_fetch_asset( - custom_assets: HashMap, -) -> impl Fn(Rc>, BufVec) -> Op { - for (_, path) in custom_assets.iter() { - println!("cargo:rerun-if-changed={}", path.display()); - } - move |_state: Rc>, bufs: BufVec| -> Op { - assert_eq!(bufs.len(), 1, "Invalid number of arguments"); - let name = std::str::from_utf8(&bufs[0]).unwrap(); - - let asset_code = if let Some(source_code) = get_asset(name) { - source_code.to_string() - } else if let Some(asset_path) = custom_assets.get(name) { - let source_code_vec = - std::fs::read(&asset_path).expect("Asset not found"); - let source_code = std::str::from_utf8(&source_code_vec).unwrap(); - source_code.to_string() - } else { - panic!("fetch_asset bad asset {}", name) - }; - - let vec = asset_code.into_bytes(); - deno_core::Op::Sync(vec.into_boxed_slice()) - } -} diff --git a/cli/ops/runtime_compiler.rs b/cli/ops/runtime_compiler.rs index 02d093375..f47f2fdb3 100644 --- a/cli/ops/runtime_compiler.rs +++ b/cli/ops/runtime_compiler.rs @@ -3,9 +3,9 @@ use crate::ast; use crate::colors; use crate::media_type::MediaType; -use crate::module_graph2::BundleType; -use crate::module_graph2::EmitOptions; -use crate::module_graph2::GraphBuilder2; +use crate::module_graph::BundleType; +use crate::module_graph::EmitOptions; +use crate::module_graph::GraphBuilder; use crate::permissions::Permissions; use crate::specifier_handler::FetchHandler; use crate::specifier_handler::MemoryHandler; @@ -65,7 +65,7 @@ async fn op_compile( runtime_permissions, )?)) }; - let mut builder = GraphBuilder2::new(handler, None, None); + let mut builder = GraphBuilder::new(handler, None, None); let specifier = ModuleSpecifier::resolve_url_or_path(&args.root_name) .context("The root specifier is invalid.")?; builder.add(&specifier, false).await?; diff --git a/cli/program_state.rs b/cli/program_state.rs index 027bbc792..cacb64ca5 100644 --- a/cli/program_state.rs +++ b/cli/program_state.rs @@ -8,10 +8,10 @@ use crate::import_map::ImportMap; use crate::inspector::InspectorServer; use crate::lockfile::Lockfile; use crate::media_type::MediaType; -use crate::module_graph2::CheckOptions; -use crate::module_graph2::GraphBuilder2; -use crate::module_graph2::TranspileOptions; -use crate::module_graph2::TypeLib; +use crate::module_graph::CheckOptions; +use crate::module_graph::GraphBuilder; +use crate::module_graph::TranspileOptions; +use crate::module_graph::TypeLib; use crate::permissions::Permissions; use crate::source_maps::SourceMapGetter; use crate::specifier_handler::FetchHandler; @@ -130,7 +130,7 @@ impl ProgramState { let handler = Rc::new(RefCell::new(FetchHandler::new(self, runtime_permissions)?)); let mut builder = - GraphBuilder2::new(handler, maybe_import_map, self.lockfile.clone()); + GraphBuilder::new(handler, maybe_import_map, self.lockfile.clone()); builder.add(&specifier, is_dynamic).await?; let mut graph = builder.get_graph(); let debug = self.flags.log_level == Some(log::Level::Debug); diff --git a/cli/tsc.rs b/cli/tsc.rs new file mode 100644 index 000000000..7f90dd7b2 --- /dev/null +++ b/cli/tsc.rs @@ -0,0 +1,750 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +use crate::diagnostics::Diagnostics; +use crate::media_type::MediaType; +use crate::module_graph::Graph; +use crate::module_graph::Stats; +use crate::tsc_config::TsConfig; + +use deno_core::error::anyhow; +use deno_core::error::bail; +use deno_core::error::AnyError; +use deno_core::error::Context; +use deno_core::json_op_sync; +use deno_core::serde_json; +use deno_core::serde_json::json; +use deno_core::serde_json::Value; +use deno_core::JsRuntime; +use deno_core::ModuleSpecifier; +use deno_core::OpFn; +use deno_core::RuntimeOptions; +use deno_core::Snapshot; +use serde::Deserialize; +use std::cell::RefCell; +use std::collections::HashMap; +use std::rc::Rc; + +/// Provide static assets that are not preloaded in the compiler snapshot. +fn get_asset(asset: &str) -> Option<&'static str> { + macro_rules! inc { + ($e:expr) => { + Some(include_str!(concat!("dts/", $e))) + }; + } + match asset { + "lib.dom.d.ts" => inc!("lib.dom.d.ts"), + "lib.dom.iterable.d.ts" => inc!("lib.dom.iterable.d.ts"), + "lib.es6.d.ts" => inc!("lib.es6.d.ts"), + "lib.es2016.full.d.ts" => inc!("lib.es2016.full.d.ts"), + "lib.es2017.full.d.ts" => inc!("lib.es2017.full.d.ts"), + "lib.es2018.full.d.ts" => inc!("lib.es2018.full.d.ts"), + "lib.es2019.full.d.ts" => inc!("lib.es2019.full.d.ts"), + "lib.es2020.full.d.ts" => inc!("lib.es2020.full.d.ts"), + "lib.esnext.full.d.ts" => inc!("lib.esnext.full.d.ts"), + "lib.scripthost.d.ts" => inc!("lib.scripthost.d.ts"), + "lib.webworker.d.ts" => inc!("lib.webworker.d.ts"), + "lib.webworker.importscripts.d.ts" => { + inc!("lib.webworker.importscripts.d.ts") + } + _ => None, + } +} + +fn get_maybe_hash( + maybe_source: &Option, + hash_data: &[Vec], +) -> Option { + if let Some(source) = maybe_source { + let mut data = vec![source.as_bytes().to_owned()]; + data.extend_from_slice(hash_data); + Some(crate::checksum::gen(&data)) + } else { + None + } +} + +#[derive(Debug, Clone, Default, Eq, PartialEq)] +pub struct EmittedFile { + pub data: String, + pub maybe_specifiers: Option>, + pub media_type: MediaType, +} + +/// A structure representing a request to be sent to the tsc runtime. +#[derive(Debug)] +pub struct Request { + /// The TypeScript compiler options which will be serialized and sent to + /// tsc. + pub config: TsConfig, + /// Indicates to the tsc runtime if debug logging should occur. + pub debug: bool, + pub graph: Rc>, + pub hash_data: Vec>, + pub maybe_tsbuildinfo: Option, + /// A vector of strings that represent the root/entry point modules for the + /// program. + pub root_names: Vec<(ModuleSpecifier, MediaType)>, +} + +#[derive(Debug, Clone, Eq, PartialEq)] +pub struct Response { + /// Any diagnostics that have been returned from the checker. + pub diagnostics: Diagnostics, + /// Any files that were emitted during the check. + pub emitted_files: Vec, + /// If there was any build info associated with the exec request. + pub maybe_tsbuildinfo: Option, + /// Statistics from the check. + pub stats: Stats, +} + +struct State { + hash_data: Vec>, + emitted_files: Vec, + graph: Rc>, + maybe_tsbuildinfo: Option, + maybe_response: Option, + root_map: HashMap, +} + +impl State { + pub fn new( + graph: Rc>, + hash_data: Vec>, + maybe_tsbuildinfo: Option, + root_map: HashMap, + ) -> Self { + State { + hash_data, + emitted_files: Vec::new(), + graph, + maybe_tsbuildinfo, + maybe_response: None, + root_map, + } + } +} + +fn op(op_fn: F) -> Box +where + F: Fn(&mut State, Value) -> Result + 'static, +{ + json_op_sync(move |s, args, _bufs| { + let state = s.borrow_mut::(); + op_fn(state, args) + }) +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct CreateHashArgs { + /// The string data to be used to generate the hash. This will be mixed with + /// other state data in Deno to derive the final hash. + data: String, +} + +fn create_hash(state: &mut State, args: Value) -> Result { + let v: CreateHashArgs = serde_json::from_value(args) + .context("Invalid request from JavaScript for \"op_create_hash\".")?; + let mut data = vec![v.data.as_bytes().to_owned()]; + data.extend_from_slice(&state.hash_data); + let hash = crate::checksum::gen(&data); + Ok(json!({ "hash": hash })) +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct EmitArgs { + /// The text data/contents of the file. + data: String, + /// The _internal_ filename for the file. This will be used to determine how + /// the file is cached and stored. + file_name: String, + /// A string representation of the specifier that was associated with a + /// module. This should be present on every module that represents a module + /// that was requested to be transformed. + maybe_specifiers: Option>, +} + +fn emit(state: &mut State, args: Value) -> Result { + let v: EmitArgs = serde_json::from_value(args) + .context("Invalid request from JavaScript for \"op_emit\".")?; + match v.file_name.as_ref() { + "deno:///.tsbuildinfo" => state.maybe_tsbuildinfo = Some(v.data), + _ => state.emitted_files.push(EmittedFile { + data: v.data, + maybe_specifiers: if let Some(specifiers) = &v.maybe_specifiers { + let specifiers = specifiers + .iter() + .map(|s| { + if let Some(remapped_specifier) = state.root_map.get(s) { + remapped_specifier.clone() + } else { + ModuleSpecifier::resolve_url_or_path(s).unwrap() + } + }) + .collect(); + Some(specifiers) + } else { + None + }, + media_type: MediaType::from(&v.file_name), + }), + } + + Ok(json!(true)) +} + +#[derive(Debug, Deserialize)] +struct LoadArgs { + /// The fully qualified specifier that should be loaded. + specifier: String, +} + +fn load(state: &mut State, args: Value) -> Result { + let v: LoadArgs = serde_json::from_value(args) + .context("Invalid request from JavaScript for \"op_load\".")?; + let specifier = ModuleSpecifier::resolve_url_or_path(&v.specifier) + .context("Error converting a string module specifier for \"op_load\".")?; + let mut hash: Option = None; + let mut media_type = MediaType::Unknown; + let data = if &v.specifier == "deno:///.tsbuildinfo" { + state.maybe_tsbuildinfo.clone() + // in certain situations we return a "blank" module to tsc and we need to + // handle the request for that module here. + } else if &v.specifier == "deno:///none.d.ts" { + hash = Some("1".to_string()); + media_type = MediaType::TypeScript; + Some("declare var a: any;\nexport = a;\n".to_string()) + } else if v.specifier.starts_with("asset:///") { + let name = v.specifier.replace("asset:///", ""); + let maybe_source = get_asset(&name).map(|s| s.to_string()); + hash = get_maybe_hash(&maybe_source, &state.hash_data); + media_type = MediaType::from(&v.specifier); + maybe_source + } else { + let graph = state.graph.borrow(); + let specifier = + if let Some(remapped_specifier) = state.root_map.get(&v.specifier) { + remapped_specifier.clone() + } else { + specifier + }; + let maybe_source = graph.get_source(&specifier); + media_type = if let Some(media_type) = graph.get_media_type(&specifier) { + media_type + } else { + MediaType::Unknown + }; + hash = get_maybe_hash(&maybe_source, &state.hash_data); + maybe_source + }; + + Ok( + json!({ "data": data, "hash": hash, "scriptKind": media_type.as_ts_script_kind() }), + ) +} + +#[derive(Debug, Deserialize)] +#[serde(rename_all = "camelCase")] +struct ResolveArgs { + /// The base specifier that the supplied specifier strings should be resolved + /// relative to. + base: String, + /// A list of specifiers that should be resolved. + specifiers: Vec, +} + +fn resolve(state: &mut State, args: Value) -> Result { + let v: ResolveArgs = serde_json::from_value(args) + .context("Invalid request from JavaScript for \"op_resolve\".")?; + let mut resolved: Vec<(String, String)> = Vec::new(); + let referrer = if let Some(remapped_base) = state.root_map.get(&v.base) { + remapped_base.clone() + } else { + ModuleSpecifier::resolve_url_or_path(&v.base).context( + "Error converting a string module specifier for \"op_resolve\".", + )? + }; + for specifier in &v.specifiers { + if specifier.starts_with("asset:///") { + resolved.push(( + specifier.clone(), + MediaType::from(specifier).as_ts_extension().to_string(), + )); + } else { + let graph = state.graph.borrow(); + match graph.resolve(specifier, &referrer, true) { + Ok(resolved_specifier) => { + let media_type = if let Some(media_type) = + graph.get_media_type(&resolved_specifier) + { + media_type + } else { + bail!( + "Unable to resolve media type for specifier: \"{}\"", + resolved_specifier + ) + }; + resolved.push(( + resolved_specifier.to_string(), + media_type.as_ts_extension(), + )); + } + // in certain situations, like certain dynamic imports, we won't have + // the source file in the graph, so we will return a fake module to + // make tsc happy. + Err(_) => { + resolved.push(("deno:///none.d.ts".to_string(), ".d.ts".to_string())); + } + } + } + } + + Ok(json!(resolved)) +} + +#[derive(Debug, Deserialize, Eq, PartialEq)] +struct RespondArgs { + pub diagnostics: Diagnostics, + pub stats: Stats, +} + +fn respond(state: &mut State, args: Value) -> Result { + let v: RespondArgs = serde_json::from_value(args) + .context("Error converting the result for \"op_respond\".")?; + state.maybe_response = Some(v); + Ok(json!(true)) +} + +/// Execute a request on the supplied snapshot, returning a response which +/// contains information, like any emitted files, diagnostics, statistics and +/// optionally an updated TypeScript build info. +pub fn exec( + snapshot: Snapshot, + request: Request, +) -> Result { + let mut runtime = JsRuntime::new(RuntimeOptions { + startup_snapshot: Some(snapshot), + ..Default::default() + }); + // tsc cannot handle root specifiers that don't have one of the "acceptable" + // extensions. Therefore, we have to check the root modules against their + // extensions and remap any that are unacceptable to tsc and add them to the + // op state so when requested, we can remap to the original specifier. + let mut root_map = HashMap::new(); + let root_names: Vec = request + .root_names + .iter() + .map(|(s, mt)| { + let ext_media_type = MediaType::from(&s.as_str().to_owned()); + if mt != &ext_media_type { + let new_specifier = format!("{}{}", s, mt.as_ts_extension()); + root_map.insert(new_specifier.clone(), s.clone()); + new_specifier + } else { + s.as_str().to_owned() + } + }) + .collect(); + + { + let op_state = runtime.op_state(); + let mut op_state = op_state.borrow_mut(); + op_state.put(State::new( + request.graph.clone(), + request.hash_data.clone(), + request.maybe_tsbuildinfo.clone(), + root_map, + )); + } + + runtime.register_op("op_create_hash", op(create_hash)); + runtime.register_op("op_emit", op(emit)); + runtime.register_op("op_load", op(load)); + runtime.register_op("op_resolve", op(resolve)); + runtime.register_op("op_respond", op(respond)); + + let startup_source = "globalThis.startup({ legacyFlag: false })"; + let request_value = json!({ + "config": request.config, + "debug": request.debug, + "rootNames": root_names, + }); + let request_str = request_value.to_string(); + let exec_source = format!("globalThis.exec({})", request_str); + + runtime + .execute("[native code]", startup_source) + .context("Could not properly start the compiler runtime.")?; + runtime.execute("[native_code]", &exec_source)?; + + let op_state = runtime.op_state(); + let mut op_state = op_state.borrow_mut(); + let state = op_state.take::(); + + if let Some(response) = state.maybe_response { + let diagnostics = response.diagnostics; + let emitted_files = state.emitted_files; + let maybe_tsbuildinfo = state.maybe_tsbuildinfo; + let stats = response.stats; + + Ok(Response { + diagnostics, + emitted_files, + maybe_tsbuildinfo, + stats, + }) + } else { + Err(anyhow!("The response for the exec request was not set.")) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::diagnostics::Diagnostic; + use crate::diagnostics::DiagnosticCategory; + use crate::js; + use crate::module_graph::tests::MockSpecifierHandler; + use crate::module_graph::GraphBuilder; + use crate::tsc_config::TsConfig; + use std::cell::RefCell; + use std::env; + use std::path::PathBuf; + + async fn setup( + maybe_specifier: Option, + maybe_hash_data: Option>>, + maybe_tsbuildinfo: Option, + ) -> State { + let specifier = maybe_specifier.unwrap_or_else(|| { + ModuleSpecifier::resolve_url_or_path("file:///main.ts").unwrap() + }); + let hash_data = maybe_hash_data.unwrap_or_else(|| vec![b"".to_vec()]); + let c = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()); + let fixtures = c.join("tests/tsc2"); + let handler = Rc::new(RefCell::new(MockSpecifierHandler { + fixtures, + ..MockSpecifierHandler::default() + })); + let mut builder = GraphBuilder::new(handler.clone(), None, None); + builder + .add(&specifier, false) + .await + .expect("module not inserted"); + let graph = Rc::new(RefCell::new(builder.get_graph())); + State::new(graph, hash_data, maybe_tsbuildinfo, HashMap::new()) + } + + async fn test_exec( + specifier: &ModuleSpecifier, + ) -> Result { + let hash_data = vec![b"something".to_vec()]; + let c = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()); + let fixtures = c.join("tests/tsc2"); + let handler = Rc::new(RefCell::new(MockSpecifierHandler { + fixtures, + ..Default::default() + })); + let mut builder = GraphBuilder::new(handler.clone(), None, None); + builder.add(&specifier, false).await?; + let graph = Rc::new(RefCell::new(builder.get_graph())); + let config = TsConfig::new(json!({ + "allowJs": true, + "checkJs": false, + "esModuleInterop": true, + "emitDecoratorMetadata": false, + "incremental": true, + "jsx": "react", + "jsxFactory": "React.createElement", + "jsxFragmentFactory": "React.Fragment", + "lib": ["deno.window"], + "module": "esnext", + "noEmit": true, + "outDir": "deno:///", + "strict": true, + "target": "esnext", + "tsBuildInfoFile": "deno:///.tsbuildinfo", + })); + let request = Request { + config, + debug: false, + graph, + hash_data, + maybe_tsbuildinfo: None, + root_names: vec![(specifier.clone(), MediaType::TypeScript)], + }; + exec(js::compiler_isolate_init(), request) + } + + #[tokio::test] + async fn test_create_hash() { + let mut state = setup(None, Some(vec![b"something".to_vec()]), None).await; + let actual = + create_hash(&mut state, json!({ "data": "some sort of content" })) + .expect("could not invoke op"); + assert_eq!( + actual, + json!({"hash": "ae92df8f104748768838916857a1623b6a3c593110131b0a00f81ad9dac16511"}) + ); + } + + #[tokio::test] + async fn test_emit() { + let mut state = setup(None, None, None).await; + let actual = emit( + &mut state, + json!({ + "data": "some file content", + "fileName": "cache:///some/file.js", + "maybeSpecifiers": ["file:///some/file.ts"] + }), + ) + .expect("should have invoked op"); + assert_eq!(actual, json!(true)); + assert_eq!(state.emitted_files.len(), 1); + assert!(state.maybe_tsbuildinfo.is_none()); + assert_eq!( + state.emitted_files[0], + EmittedFile { + data: "some file content".to_string(), + maybe_specifiers: Some(vec![ModuleSpecifier::resolve_url_or_path( + "file:///some/file.ts" + ) + .unwrap()]), + media_type: MediaType::JavaScript, + } + ); + } + + #[tokio::test] + async fn test_emit_tsbuildinfo() { + let mut state = setup(None, None, None).await; + let actual = emit( + &mut state, + json!({ + "data": "some file content", + "fileName": "deno:///.tsbuildinfo", + }), + ) + .expect("should have invoked op"); + assert_eq!(actual, json!(true)); + assert_eq!(state.emitted_files.len(), 0); + assert_eq!( + state.maybe_tsbuildinfo, + Some("some file content".to_string()) + ); + } + + #[tokio::test] + async fn test_load() { + let mut state = setup( + Some( + ModuleSpecifier::resolve_url_or_path("https://deno.land/x/mod.ts") + .unwrap(), + ), + None, + Some("some content".to_string()), + ) + .await; + let actual = load( + &mut state, + json!({ "specifier": "https://deno.land/x/mod.ts"}), + ) + .expect("should have invoked op"); + assert_eq!( + actual, + json!({ + "data": "console.log(\"hello deno\");\n", + "hash": "149c777056afcc973d5fcbe11421b6d5ddc57b81786765302030d7fc893bf729", + "scriptKind": 3, + }) + ); + } + + #[derive(Debug, Deserialize)] + #[serde(rename_all = "camelCase")] + struct LoadResponse { + data: String, + hash: Option, + script_kind: i64, + } + + #[tokio::test] + async fn test_load_asset() { + let mut state = setup( + Some( + ModuleSpecifier::resolve_url_or_path("https://deno.land/x/mod.ts") + .unwrap(), + ), + None, + Some("some content".to_string()), + ) + .await; + let value = + load(&mut state, json!({ "specifier": "asset:///lib.dom.d.ts" })) + .expect("should have invoked op"); + let actual: LoadResponse = + serde_json::from_value(value).expect("failed to deserialize"); + let expected = get_asset("lib.dom.d.ts").unwrap(); + assert_eq!(actual.data, expected); + assert!(actual.hash.is_some()); + assert_eq!(actual.script_kind, 3); + } + + #[tokio::test] + async fn test_load_tsbuildinfo() { + let mut state = setup( + Some( + ModuleSpecifier::resolve_url_or_path("https://deno.land/x/mod.ts") + .unwrap(), + ), + None, + Some("some content".to_string()), + ) + .await; + let actual = + load(&mut state, json!({ "specifier": "deno:///.tsbuildinfo"})) + .expect("should have invoked op"); + assert_eq!( + actual, + json!({ + "data": "some content", + "hash": null, + "scriptKind": 0, + }) + ); + } + + #[tokio::test] + async fn test_load_missing_specifier() { + let mut state = setup(None, None, None).await; + let actual = load( + &mut state, + json!({ "specifier": "https://deno.land/x/mod.ts"}), + ) + .expect("should have invoked op"); + assert_eq!( + actual, + json!({ + "data": null, + "hash": null, + "scriptKind": 0, + }) + ) + } + + #[tokio::test] + async fn test_resolve() { + let mut state = setup( + Some( + ModuleSpecifier::resolve_url_or_path("https://deno.land/x/a.ts") + .unwrap(), + ), + None, + None, + ) + .await; + let actual = resolve( + &mut state, + json!({ "base": "https://deno.land/x/a.ts", "specifiers": [ "./b.ts" ]}), + ) + .expect("should have invoked op"); + assert_eq!(actual, json!([["https://deno.land/x/b.ts", ".ts"]])); + } + + #[tokio::test] + async fn test_resolve_empty() { + let mut state = setup( + Some( + ModuleSpecifier::resolve_url_or_path("https://deno.land/x/a.ts") + .unwrap(), + ), + None, + None, + ) + .await; + let actual = resolve( + &mut state, + json!({ "base": "https://deno.land/x/a.ts", "specifiers": [ "./bad.ts" ]}), + ).expect("should have not errored"); + assert_eq!(actual, json!([["deno:///none.d.ts", ".d.ts"]])); + } + + #[tokio::test] + async fn test_respond() { + let mut state = setup(None, None, None).await; + let actual = respond( + &mut state, + json!({ + "diagnostics": [ + { + "messageText": "Unknown compiler option 'invalid'.", + "category": 1, + "code": 5023 + } + ], + "stats": [["a", 12]] + }), + ) + .expect("should have invoked op"); + assert_eq!(actual, json!(true)); + assert_eq!( + state.maybe_response, + Some(RespondArgs { + diagnostics: Diagnostics::new(vec![Diagnostic { + category: DiagnosticCategory::Error, + code: 5023, + start: None, + end: None, + message_text: Some( + "Unknown compiler option \'invalid\'.".to_string() + ), + message_chain: None, + source: None, + source_line: None, + file_name: None, + related_information: None, + }]), + stats: Stats(vec![("a".to_string(), 12)]) + }) + ); + } + + #[tokio::test] + async fn test_exec_basic() { + let specifier = + ModuleSpecifier::resolve_url_or_path("https://deno.land/x/a.ts").unwrap(); + let actual = test_exec(&specifier) + .await + .expect("exec should not have errored"); + assert!(actual.diagnostics.is_empty()); + assert!(actual.emitted_files.is_empty()); + assert!(actual.maybe_tsbuildinfo.is_some()); + assert_eq!(actual.stats.0.len(), 12); + } + + #[tokio::test] + async fn test_exec_reexport_dts() { + let specifier = + ModuleSpecifier::resolve_url_or_path("file:///reexports.ts").unwrap(); + let actual = test_exec(&specifier) + .await + .expect("exec should not have errored"); + assert!(actual.diagnostics.is_empty()); + assert!(actual.emitted_files.is_empty()); + assert!(actual.maybe_tsbuildinfo.is_some()); + assert_eq!(actual.stats.0.len(), 12); + } + + #[tokio::test] + async fn fix_lib_ref() { + let specifier = + ModuleSpecifier::resolve_url_or_path("file:///libref.ts").unwrap(); + let actual = test_exec(&specifier) + .await + .expect("exec should not have errored"); + assert!(actual.diagnostics.is_empty()); + } +} diff --git a/cli/tsc/99_main_compiler.js b/cli/tsc/99_main_compiler.js index e2a481d0f..33f34b806 100644 --- a/cli/tsc/99_main_compiler.js +++ b/cli/tsc/99_main_compiler.js @@ -1,19 +1,11 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. // This module is the entry point for "compiler" isolate, ie. the one -// that is created when Deno needs to compile TS/WASM to JS. -// -// It provides two functions that should be called by Rust: -// - `startup` -// This functions must be called when creating isolate -// to properly setup runtime. -// - `tsCompilerOnMessage` -// This function must be called when sending a request -// to the compiler. +// that is created when Deno needs to type check TypeScript, and in some +// instances convert TypeScript to JavaScript. // Removes the `__proto__` for security reasons. This intentionally makes // Deno non compliant with ECMA-262 Annex B.2.2.1 -// delete Object.prototype.__proto__; ((window) => { @@ -22,11 +14,6 @@ delete Object.prototype.__proto__; let logDebug = false; let logSource = "JS"; - /** Instructs the host to behave in a legacy fashion, with the legacy - * pipeline for handling code. Setting the value to `true` will cause the - * host to behave in the modern way. */ - let legacy = true; - function setLogDebug(debug, source) { logDebug = debug; if (source) { @@ -57,9 +44,7 @@ delete Object.prototype.__proto__; /** @type {Map} */ const sourceFileCache = new Map(); - /** - * @param {import("../dts/typescript").DiagnosticRelatedInformation} diagnostic - */ + /** @param {ts.DiagnosticRelatedInformation} diagnostic */ function fromRelatedInformation({ start, length, @@ -96,9 +81,7 @@ delete Object.prototype.__proto__; } } - /** - * @param {import("../dts/typescript").Diagnostic[]} diagnostics - */ + /** @param {ts.Diagnostic[]} diagnostics */ function fromTypeScriptDiagnostic(diagnostics) { return diagnostics.map(({ relatedInformation: ri, source, ...diag }) => { const value = fromRelatedInformation(diag); @@ -110,179 +93,63 @@ delete Object.prototype.__proto__; }); } - // We really don't want to depend on JSON dispatch during snapshotting, so - // this op exchanges strings with Rust as raw byte arrays. - function getAsset(name) { - const opId = core.ops()["op_fetch_asset"]; - const sourceCodeBytes = core.dispatch(opId, core.encode(name)); - return core.decode(sourceCodeBytes); - } - // Using incremental compile APIs requires that all // paths must be either relative or absolute. Since // analysis in Rust operates on fully resolved URLs, // it makes sense to use the same scheme here. const ASSETS = "asset:///"; - const OUT_DIR = "deno://"; const CACHE = "cache:///"; - // This constant is passed to compiler settings when - // doing incremental compiles. Contents of this - // file are passed back to Rust and saved to $DENO_DIR. - const TS_BUILD_INFO = "cache:///tsbuildinfo.json"; - const DEFAULT_COMPILE_OPTIONS = { - allowJs: false, - allowNonTsExtensions: true, - checkJs: false, + /** Diagnostics that are intentionally ignored when compiling TypeScript in + * Deno, as they provide misleading or incorrect information. */ + const IGNORED_DIAGNOSTICS = [ + // TS1208: All files must be modules when the '--isolatedModules' flag is + // provided. We can ignore because we guarantuee that all files are + // modules. + 1208, + // TS1375: 'await' expressions are only allowed at the top level of a file + // when that file is a module, but this file has no imports or exports. + // Consider adding an empty 'export {}' to make this file a module. + 1375, + // TS1103: 'for-await-of' statement is only allowed within an async function + // or async generator. + 1103, + // TS2306: File 'file:///Users/rld/src/deno/cli/tests/subdir/amd_like.js' is + // not a module. + 2306, + // TS2691: An import path cannot end with a '.ts' extension. Consider + // importing 'bad-module' instead. + 2691, + // TS5009: Cannot find the common subdirectory path for the input files. + 5009, + // TS5055: Cannot write file + // 'http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js' + // because it would overwrite input file. + 5055, + // TypeScript is overly opinionated that only CommonJS modules kinds can + // support JSON imports. Allegedly this was fixed in + // Microsoft/TypeScript#26825 but that doesn't seem to be working here, + // so we will ignore complaints about this compiler setting. + 5070, + // TS7016: Could not find a declaration file for module '...'. '...' + // implicitly has an 'any' type. This is due to `allowJs` being off by + // default but importing of a JavaScript module. + 7016, + ]; + + const SNAPSHOT_COMPILE_OPTIONS = { esModuleInterop: true, jsx: ts.JsxEmit.React, module: ts.ModuleKind.ESNext, - outDir: OUT_DIR, - sourceMap: true, + noEmit: true, strict: true, - removeComments: true, target: ts.ScriptTarget.ESNext, }; - const CompilerHostTarget = { - Main: "main", - Runtime: "runtime", - Worker: "worker", - }; - - // Warning! The values in this enum are duplicated in `cli/msg.rs` - // Update carefully! - const MediaType = { - 0: "JavaScript", - 1: "JSX", - 2: "TypeScript", - 3: "Dts", - 4: "TSX", - 5: "Json", - 6: "Wasm", - 7: "TsBuildInfo", - 8: "SourceMap", - 9: "Unknown", - JavaScript: 0, - JSX: 1, - TypeScript: 2, - Dts: 3, - TSX: 4, - Json: 5, - Wasm: 6, - TsBuildInfo: 7, - SourceMap: 8, - Unknown: 9, - }; - - function getExtension(fileName, mediaType) { - switch (mediaType) { - case MediaType.JavaScript: - return ts.Extension.Js; - case MediaType.JSX: - return ts.Extension.Jsx; - case MediaType.TypeScript: - return ts.Extension.Ts; - case MediaType.Dts: - return ts.Extension.Dts; - case MediaType.TSX: - return ts.Extension.Tsx; - case MediaType.Wasm: - // Custom marker for Wasm type. - return ts.Extension.Js; - case MediaType.Unknown: - default: - throw TypeError( - `Cannot resolve extension for "${fileName}" with mediaType "${ - MediaType[mediaType] - }".`, - ); - } - } - - /** A global cache of module source files that have been loaded. - * This cache will be rewritten to be populated on compiler startup - * with files provided from Rust in request message. - */ - const SOURCE_FILE_CACHE = new Map(); - /** A map of maps which cache resolved specifier for each import in a file. - * This cache is used so `resolveModuleNames` ops is called as few times - * as possible. - * - * First map's key is "referrer" URL ("file://a/b/c/mod.ts") - * Second map's key is "raw" import specifier ("./foo.ts") - * Second map's value is resolved import URL ("file:///a/b/c/foo.ts") - */ - const RESOLVED_SPECIFIER_CACHE = new Map(); - - class SourceFile { - constructor(json) { - this.processed = false; - Object.assign(this, json); - this.extension = getExtension(this.url, this.mediaType); - } - - static addToCache(json) { - if (SOURCE_FILE_CACHE.has(json.url)) { - throw new TypeError("SourceFile already exists"); - } - const sf = new SourceFile(json); - SOURCE_FILE_CACHE.set(sf.url, sf); - return sf; - } - - static getCached(url) { - return SOURCE_FILE_CACHE.get(url); - } - - static cacheResolvedUrl(resolvedUrl, rawModuleSpecifier, containingFile) { - containingFile = containingFile || ""; - let innerCache = RESOLVED_SPECIFIER_CACHE.get(containingFile); - if (!innerCache) { - innerCache = new Map(); - RESOLVED_SPECIFIER_CACHE.set(containingFile, innerCache); - } - innerCache.set(rawModuleSpecifier, resolvedUrl); - } - - static getResolvedUrl(moduleSpecifier, containingFile) { - const containingCache = RESOLVED_SPECIFIER_CACHE.get(containingFile); - if (containingCache) { - return containingCache.get(moduleSpecifier); - } - return undefined; - } - } - - function getAssetInternal(filename) { - const lastSegment = filename.split("/").pop(); - const url = ts.libMap.has(lastSegment) - ? ts.libMap.get(lastSegment) - : lastSegment; - const sourceFile = SourceFile.getCached(url); - if (sourceFile) { - return sourceFile; - } - const name = url.includes(".") ? url : `${url}.d.ts`; - const sourceCode = getAsset(name); - return SourceFile.addToCache({ - url, - filename: `${ASSETS}/${name}`, - mediaType: MediaType.TypeScript, - versionHash: "1", - sourceCode, - }); - } - - /** There was some private state in the legacy host, that is moved out to - * here which can then be refactored out later. */ - const legacyHostState = { - buildInfo: "", - target: CompilerHostTarget.Main, - writeFile: (_fileName, _data, _sourceFiles) => {}, - }; - - /** @type {import("../dts/typescript").CompilerHost} */ + /** An object literal of the incremental compiler host, which provides the + * specific "bindings" to the Deno environment that tsc needs to work. + * + * @type {ts.CompilerHost} */ const host = { fileExists(fileName) { debug(`host.fileExists("${fileName}")`); @@ -290,122 +157,61 @@ delete Object.prototype.__proto__; }, readFile(specifier) { debug(`host.readFile("${specifier}")`); - if (legacy) { - if (specifier == TS_BUILD_INFO) { - return legacyHostState.buildInfo; - } - return unreachable(); - } else { - return core.jsonOpSync("op_load", { specifier }).data; - } + return core.jsonOpSync("op_load", { specifier }).data; }, getSourceFile( specifier, languageVersion, - onError, - shouldCreateNewSourceFile, + _onError, + _shouldCreateNewSourceFile, ) { debug( `host.getSourceFile("${specifier}", ${ ts.ScriptTarget[languageVersion] })`, ); - if (legacy) { - try { - assert(!shouldCreateNewSourceFile); - const sourceFile = specifier.startsWith(ASSETS) - ? getAssetInternal(specifier) - : SourceFile.getCached(specifier); - assert(sourceFile != null); - if (!sourceFile.tsSourceFile) { - assert(sourceFile.sourceCode != null); - const tsSourceFileName = specifier.startsWith(ASSETS) - ? sourceFile.filename - : specifier; - - sourceFile.tsSourceFile = ts.createSourceFile( - tsSourceFileName, - sourceFile.sourceCode, - languageVersion, - ); - sourceFile.tsSourceFile.version = sourceFile.versionHash; - delete sourceFile.sourceCode; - - // This code is to support transition from the "legacy" compiler - // to the new one, by populating the new source file cache. - if ( - !sourceFileCache.has(specifier) && specifier.startsWith(ASSETS) - ) { - sourceFileCache.set(specifier, sourceFile.tsSourceFile); - } - } - return sourceFile.tsSourceFile; - } catch (e) { - if (onError) { - onError(String(e)); - } else { - throw e; - } - return undefined; - } - } else { - let sourceFile = sourceFileCache.get(specifier); - if (sourceFile) { - return sourceFile; - } - - /** @type {{ data: string; hash?: string; scriptKind: ts.ScriptKind }} */ - const { data, hash, scriptKind } = core.jsonOpSync( - "op_load", - { specifier }, - ); - assert( - data != null, - `"data" is unexpectedly null for "${specifier}".`, - ); - sourceFile = ts.createSourceFile( - specifier, - data, - languageVersion, - false, - scriptKind, - ); - sourceFile.moduleName = specifier; - sourceFile.version = hash; - sourceFileCache.set(specifier, sourceFile); + let sourceFile = sourceFileCache.get(specifier); + if (sourceFile) { return sourceFile; } + + /** @type {{ data: string; hash?: string; scriptKind: ts.ScriptKind }} */ + const { data, hash, scriptKind } = core.jsonOpSync( + "op_load", + { specifier }, + ); + assert( + data != null, + `"data" is unexpectedly null for "${specifier}".`, + ); + sourceFile = ts.createSourceFile( + specifier, + data, + languageVersion, + false, + scriptKind, + ); + sourceFile.moduleName = specifier; + sourceFile.version = hash; + sourceFileCache.set(specifier, sourceFile); + return sourceFile; }, getDefaultLibFileName() { - if (legacy) { - switch (legacyHostState.target) { - case CompilerHostTarget.Main: - case CompilerHostTarget.Runtime: - return `${ASSETS}/lib.deno.window.d.ts`; - case CompilerHostTarget.Worker: - return `${ASSETS}/lib.deno.worker.d.ts`; - } - } else { - return `${ASSETS}/lib.esnext.d.ts`; - } + return `${ASSETS}/lib.esnext.d.ts`; }, getDefaultLibLocation() { return ASSETS; }, writeFile(fileName, data, _writeByteOrderMark, _onError, sourceFiles) { debug(`host.writeFile("${fileName}")`); - if (legacy) { - legacyHostState.writeFile(fileName, data, sourceFiles); - } else { - let maybeSpecifiers; - if (sourceFiles) { - maybeSpecifiers = sourceFiles.map((sf) => sf.moduleName); - } - return core.jsonOpSync( - "op_emit", - { maybeSpecifiers, fileName, data }, - ); + let maybeSpecifiers; + if (sourceFiles) { + maybeSpecifiers = sourceFiles.map((sf) => sf.moduleName); } + return core.jsonOpSync( + "op_emit", + { maybeSpecifiers, fileName, data }, + ); }, getCurrentDirectory() { return CACHE; @@ -423,148 +229,24 @@ delete Object.prototype.__proto__; debug(`host.resolveModuleNames()`); debug(` base: ${base}`); debug(` specifiers: ${specifiers.join(", ")}`); - if (legacy) { - const resolved = specifiers.map((specifier) => { - const maybeUrl = SourceFile.getResolvedUrl(specifier, base); - - debug("compiler::host.resolveModuleNames maybeUrl", { - specifier, - maybeUrl, - }); - - let sourceFile = undefined; - - if (specifier.startsWith(ASSETS)) { - sourceFile = getAssetInternal(specifier); - } else if (typeof maybeUrl !== "undefined") { - sourceFile = SourceFile.getCached(maybeUrl); - } - - if (!sourceFile) { - return undefined; - } - - return { - resolvedFileName: sourceFile.url, - isExternalLibraryImport: specifier.startsWith(ASSETS), - extension: sourceFile.extension, - }; - }); - debug(resolved); - return resolved; - } else { - /** @type {Array<[string, import("../dts/typescript").Extension]>} */ - const resolved = core.jsonOpSync("op_resolve", { - specifiers, - base, - }); - let r = resolved.map(([resolvedFileName, extension]) => ({ - resolvedFileName, - extension, - isExternalLibraryImport: false, - })); - return r; - } + /** @type {Array<[string, ts.Extension]>} */ + const resolved = core.jsonOpSync("op_resolve", { + specifiers, + base, + }); + let r = resolved.map(([resolvedFileName, extension]) => ({ + resolvedFileName, + extension, + isExternalLibraryImport: false, + })); + return r; }, createHash(data) { return core.jsonOpSync("op_create_hash", { data }).hash; }, }; - // This is a hacky way of adding our libs to the libs available in TypeScript() - // as these are internal APIs of TypeScript which maintain valid libs - ts.libs.push("deno.ns", "deno.window", "deno.worker", "deno.shared_globals"); - ts.libMap.set("deno.ns", "lib.deno.ns.d.ts"); - ts.libMap.set("deno.web", "lib.deno.web.d.ts"); - ts.libMap.set("deno.fetch", "lib.deno.fetch.d.ts"); - ts.libMap.set("deno.window", "lib.deno.window.d.ts"); - ts.libMap.set("deno.worker", "lib.deno.worker.d.ts"); - ts.libMap.set("deno.shared_globals", "lib.deno.shared_globals.d.ts"); - ts.libMap.set("deno.unstable", "lib.deno.unstable.d.ts"); - - // TODO(@kitsonk) remove once added to TypeScript - ts.libs.push("esnext.weakref"); - ts.libMap.set("esnext.weakref", "lib.esnext.weakref.d.ts"); - - // this pre-populates the cache at snapshot time of our library files, so they - // are available in the future when needed. - host.getSourceFile( - `${ASSETS}lib.deno.ns.d.ts`, - ts.ScriptTarget.ESNext, - ); - host.getSourceFile( - `${ASSETS}lib.deno.web.d.ts`, - ts.ScriptTarget.ESNext, - ); - host.getSourceFile( - `${ASSETS}lib.deno.fetch.d.ts`, - ts.ScriptTarget.ESNext, - ); - host.getSourceFile( - `${ASSETS}lib.deno.window.d.ts`, - ts.ScriptTarget.ESNext, - ); - host.getSourceFile( - `${ASSETS}lib.deno.worker.d.ts`, - ts.ScriptTarget.ESNext, - ); - host.getSourceFile( - `${ASSETS}lib.deno.shared_globals.d.ts`, - ts.ScriptTarget.ESNext, - ); - host.getSourceFile( - `${ASSETS}lib.deno.unstable.d.ts`, - ts.ScriptTarget.ESNext, - ); - - // We never use this program; it's only created - // during snapshotting to hydrate and populate - // source file cache with lib declaration files. - const _TS_SNAPSHOT_PROGRAM = ts.createProgram({ - rootNames: [`${ASSETS}bootstrap.ts`], - options: DEFAULT_COMPILE_OPTIONS, - host, - }); - - const IGNORED_DIAGNOSTICS = [ - // TS2306: File 'file:///Users/rld/src/deno/cli/tests/subdir/amd_like.js' is - // not a module. - 2306, - // TS1375: 'await' expressions are only allowed at the top level of a file - // when that file is a module, but this file has no imports or exports. - // Consider adding an empty 'export {}' to make this file a module. - 1375, - // TS1103: 'for-await-of' statement is only allowed within an async function - // or async generator. - 1103, - // TS2691: An import path cannot end with a '.ts' extension. Consider - // importing 'bad-module' instead. - 2691, - // TS5009: Cannot find the common subdirectory path for the input files. - 5009, - // TS5055: Cannot write file - // 'http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js' - // because it would overwrite input file. - 5055, - // TypeScript is overly opinionated that only CommonJS modules kinds can - // support JSON imports. Allegedly this was fixed in - // Microsoft/TypeScript#26825 but that doesn't seem to be working here, - // so we will ignore complaints about this compiler setting. - 5070, - // TS7016: Could not find a declaration file for module '...'. '...' - // implicitly has an 'any' type. This is due to `allowJs` being off by - // default but importing of a JavaScript module. - 7016, - ]; - - const IGNORED_COMPILE_DIAGNOSTICS = [ - // TS1208: All files must be modules when the '--isolatedModules' flag is - // provided. We can ignore because we guarantuee that all files are - // modules. - 1208, - ]; - - /** @type {Array<{ key: string, value: number }>} */ + /** @type {Array<[string, number]>} */ const stats = []; let statsStart = 0; @@ -579,35 +261,31 @@ delete Object.prototype.__proto__; if ("getProgram" in program) { program = program.getProgram(); } - stats.push({ key: "Files", value: program.getSourceFiles().length }); - stats.push({ key: "Nodes", value: program.getNodeCount() }); - stats.push({ key: "Identifiers", value: program.getIdentifierCount() }); - stats.push({ key: "Symbols", value: program.getSymbolCount() }); - stats.push({ key: "Types", value: program.getTypeCount() }); - stats.push({ - key: "Instantiations", - value: program.getInstantiationCount(), - }); + stats.push(["Files", program.getSourceFiles().length]); + stats.push(["Nodes", program.getNodeCount()]); + stats.push(["Identifiers", program.getIdentifierCount()]); + stats.push(["Symbols", program.getSymbolCount()]); + stats.push(["Types", program.getTypeCount()]); + stats.push(["Instantiations", program.getInstantiationCount()]); } else if (fileCount != null) { - stats.push({ key: "Files", value: fileCount }); + stats.push(["Files", fileCount]); } const programTime = ts.performance.getDuration("Program"); const bindTime = ts.performance.getDuration("Bind"); const checkTime = ts.performance.getDuration("Check"); const emitTime = ts.performance.getDuration("Emit"); - stats.push({ key: "Parse time", value: programTime }); - stats.push({ key: "Bind time", value: bindTime }); - stats.push({ key: "Check time", value: checkTime }); - stats.push({ key: "Emit time", value: emitTime }); - stats.push({ - key: "Total TS time", - value: programTime + bindTime + checkTime + emitTime, - }); + stats.push(["Parse time", programTime]); + stats.push(["Bind time", bindTime]); + stats.push(["Check time", checkTime]); + stats.push(["Emit time", emitTime]); + stats.push( + ["Total TS time", programTime + bindTime + checkTime + emitTime], + ); } function performanceEnd() { const duration = new Date() - statsStart; - stats.push({ key: "Compile time", value: duration }); + stats.push(["Compile time", duration]); return stats; } @@ -645,17 +323,12 @@ delete Object.prototype.__proto__; ...program.getGlobalDiagnostics(), ...program.getSemanticDiagnostics(), ...emitDiagnostics, - ].filter(({ code }) => - !IGNORED_DIAGNOSTICS.includes(code) && - !IGNORED_COMPILE_DIAGNOSTICS.includes(code) - ); + ].filter(({ code }) => !IGNORED_DIAGNOSTICS.includes(code)); performanceProgram({ program }); - // TODO(@kitsonk) when legacy stats are removed, convert to just tuples - let stats = performanceEnd().map(({ key, value }) => [key, value]); core.jsonOpSync("op_respond", { diagnostics: fromTypeScriptDiagnostic(diagnostics), - stats, + stats: performanceEnd(), }); debug("<<< exec stop"); } @@ -665,7 +338,7 @@ delete Object.prototype.__proto__; /** Startup the runtime environment, setting various flags. * @param {{ debugFlag?: boolean; legacyFlag?: boolean; }} msg */ - function startup({ debugFlag = false, legacyFlag = true }) { + function startup({ debugFlag = false }) { if (hasStarted) { throw new Error("The compiler runtime already started."); } @@ -673,9 +346,42 @@ delete Object.prototype.__proto__; core.ops(); core.registerErrorClass("Error", Error); setLogDebug(!!debugFlag, "TS"); - legacy = legacyFlag; } + // Setup the compiler runtime during the build process. + core.ops(); + core.registerErrorClass("Error", Error); + + // A build time only op that provides some setup information that is used to + // ensure the snapshot is setup properly. + /** @type {{ buildSpecifier: string; libs: string[] }} */ + const { buildSpecifier, libs } = core.jsonOpSync("op_build_info", {}); + for (const lib of libs) { + let specifier = `lib.${lib}.d.ts`; + // we are using internal APIs here to "inject" our custom libraries into + // tsc, so things like `"lib": [ "deno.ns" ]` are supported. + if (!ts.libs.includes(lib)) { + ts.libs.push(lib); + ts.libMap.set(lib, `lib.${lib}.d.ts`); + } + // we are caching in memory common type libraries that will be re-used by + // tsc on when the snapshot is restored + assert( + host.getSourceFile(`${ASSETS}${specifier}`, ts.ScriptTarget.ESNext), + ); + } + // this helps ensure as much as possible is in memory that is re-usable + // before the snapshotting is done, which helps unsure fast "startup" for + // subsequent uses of tsc in Deno. + const TS_SNAPSHOT_PROGRAM = ts.createProgram({ + rootNames: [buildSpecifier], + options: SNAPSHOT_COMPILE_OPTIONS, + host, + }); + ts.getPreEmitDiagnostics(TS_SNAPSHOT_PROGRAM); + + // exposes the two functions that are called by `tsc::exec()` when type + // checking TypeScript. globalThis.startup = startup; globalThis.exec = exec; })(this); diff --git a/cli/tsc2.rs b/cli/tsc2.rs deleted file mode 100644 index 25767619e..000000000 --- a/cli/tsc2.rs +++ /dev/null @@ -1,726 +0,0 @@ -// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. - -use crate::diagnostics::Diagnostics; -use crate::media_type::MediaType; -use crate::module_graph2::Graph2; -use crate::module_graph2::Stats; -// TODO(@kitsonk) this needs to be refactored when we drop MG1 -use crate::op_fetch_asset::get_asset; -use crate::tsc_config::TsConfig; - -use deno_core::error::anyhow; -use deno_core::error::bail; -use deno_core::error::AnyError; -use deno_core::error::Context; -use deno_core::json_op_sync; -use deno_core::serde_json; -use deno_core::serde_json::json; -use deno_core::serde_json::Value; -use deno_core::JsRuntime; -use deno_core::ModuleSpecifier; -use deno_core::OpFn; -use deno_core::RuntimeOptions; -use deno_core::Snapshot; -use serde::Deserialize; -use std::cell::RefCell; -use std::collections::HashMap; -use std::rc::Rc; - -fn get_maybe_hash( - maybe_source: &Option, - hash_data: &[Vec], -) -> Option { - if let Some(source) = maybe_source { - let mut data = vec![source.as_bytes().to_owned()]; - data.extend_from_slice(hash_data); - Some(crate::checksum::gen(&data)) - } else { - None - } -} - -#[derive(Debug, Clone, Default, Eq, PartialEq)] -pub struct EmittedFile { - pub data: String, - pub maybe_specifiers: Option>, - pub media_type: MediaType, -} - -/// A structure representing a request to be sent to the tsc runtime. -#[derive(Debug)] -pub struct Request { - /// The TypeScript compiler options which will be serialized and sent to - /// tsc. - pub config: TsConfig, - /// Indicates to the tsc runtime if debug logging should occur. - pub debug: bool, - pub graph: Rc>, - pub hash_data: Vec>, - pub maybe_tsbuildinfo: Option, - /// A vector of strings that represent the root/entry point modules for the - /// program. - pub root_names: Vec<(ModuleSpecifier, MediaType)>, -} - -#[derive(Debug, Clone, Eq, PartialEq)] -pub struct Response { - /// Any diagnostics that have been returned from the checker. - pub diagnostics: Diagnostics, - /// Any files that were emitted during the check. - pub emitted_files: Vec, - /// If there was any build info associated with the exec request. - pub maybe_tsbuildinfo: Option, - /// Statistics from the check. - pub stats: Stats, -} - -struct State { - hash_data: Vec>, - emitted_files: Vec, - graph: Rc>, - maybe_tsbuildinfo: Option, - maybe_response: Option, - root_map: HashMap, -} - -impl State { - pub fn new( - graph: Rc>, - hash_data: Vec>, - maybe_tsbuildinfo: Option, - root_map: HashMap, - ) -> Self { - State { - hash_data, - emitted_files: Vec::new(), - graph, - maybe_tsbuildinfo, - maybe_response: None, - root_map, - } - } -} - -fn op(op_fn: F) -> Box -where - F: Fn(&mut State, Value) -> Result + 'static, -{ - json_op_sync(move |s, args, _bufs| { - let state = s.borrow_mut::(); - op_fn(state, args) - }) -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct CreateHashArgs { - /// The string data to be used to generate the hash. This will be mixed with - /// other state data in Deno to derive the final hash. - data: String, -} - -fn create_hash(state: &mut State, args: Value) -> Result { - let v: CreateHashArgs = serde_json::from_value(args) - .context("Invalid request from JavaScript for \"op_create_hash\".")?; - let mut data = vec![v.data.as_bytes().to_owned()]; - data.extend_from_slice(&state.hash_data); - let hash = crate::checksum::gen(&data); - Ok(json!({ "hash": hash })) -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct EmitArgs { - /// The text data/contents of the file. - data: String, - /// The _internal_ filename for the file. This will be used to determine how - /// the file is cached and stored. - file_name: String, - /// A string representation of the specifier that was associated with a - /// module. This should be present on every module that represents a module - /// that was requested to be transformed. - maybe_specifiers: Option>, -} - -fn emit(state: &mut State, args: Value) -> Result { - let v: EmitArgs = serde_json::from_value(args) - .context("Invalid request from JavaScript for \"op_emit\".")?; - match v.file_name.as_ref() { - "deno:///.tsbuildinfo" => state.maybe_tsbuildinfo = Some(v.data), - _ => state.emitted_files.push(EmittedFile { - data: v.data, - maybe_specifiers: if let Some(specifiers) = &v.maybe_specifiers { - let specifiers = specifiers - .iter() - .map(|s| { - if let Some(remapped_specifier) = state.root_map.get(s) { - remapped_specifier.clone() - } else { - ModuleSpecifier::resolve_url_or_path(s).unwrap() - } - }) - .collect(); - Some(specifiers) - } else { - None - }, - media_type: MediaType::from(&v.file_name), - }), - } - - Ok(json!(true)) -} - -#[derive(Debug, Deserialize)] -struct LoadArgs { - /// The fully qualified specifier that should be loaded. - specifier: String, -} - -fn load(state: &mut State, args: Value) -> Result { - let v: LoadArgs = serde_json::from_value(args) - .context("Invalid request from JavaScript for \"op_load\".")?; - let specifier = ModuleSpecifier::resolve_url_or_path(&v.specifier) - .context("Error converting a string module specifier for \"op_load\".")?; - let mut hash: Option = None; - let mut media_type = MediaType::Unknown; - let data = if &v.specifier == "deno:///.tsbuildinfo" { - state.maybe_tsbuildinfo.clone() - // in certain situations we return a "blank" module to tsc and we need to - // handle the request for that module here. - } else if &v.specifier == "deno:///none.d.ts" { - hash = Some("1".to_string()); - media_type = MediaType::TypeScript; - Some("declare var a: any;\nexport = a;\n".to_string()) - } else if v.specifier.starts_with("asset:///") { - let name = v.specifier.replace("asset:///", ""); - let maybe_source = get_asset(&name).map(|s| s.to_string()); - hash = get_maybe_hash(&maybe_source, &state.hash_data); - media_type = MediaType::from(&v.specifier); - maybe_source - } else { - let graph = state.graph.borrow(); - let specifier = - if let Some(remapped_specifier) = state.root_map.get(&v.specifier) { - remapped_specifier.clone() - } else { - specifier - }; - let maybe_source = graph.get_source(&specifier); - media_type = if let Some(media_type) = graph.get_media_type(&specifier) { - media_type - } else { - MediaType::Unknown - }; - hash = get_maybe_hash(&maybe_source, &state.hash_data); - maybe_source - }; - - Ok( - json!({ "data": data, "hash": hash, "scriptKind": media_type.as_ts_script_kind() }), - ) -} - -#[derive(Debug, Deserialize)] -#[serde(rename_all = "camelCase")] -struct ResolveArgs { - /// The base specifier that the supplied specifier strings should be resolved - /// relative to. - base: String, - /// A list of specifiers that should be resolved. - specifiers: Vec, -} - -fn resolve(state: &mut State, args: Value) -> Result { - let v: ResolveArgs = serde_json::from_value(args) - .context("Invalid request from JavaScript for \"op_resolve\".")?; - let mut resolved: Vec<(String, String)> = Vec::new(); - let referrer = if let Some(remapped_base) = state.root_map.get(&v.base) { - remapped_base.clone() - } else { - ModuleSpecifier::resolve_url_or_path(&v.base).context( - "Error converting a string module specifier for \"op_resolve\".", - )? - }; - for specifier in &v.specifiers { - if specifier.starts_with("asset:///") { - resolved.push(( - specifier.clone(), - MediaType::from(specifier).as_ts_extension().to_string(), - )); - } else { - let graph = state.graph.borrow(); - match graph.resolve(specifier, &referrer, true) { - Ok(resolved_specifier) => { - let media_type = if let Some(media_type) = - graph.get_media_type(&resolved_specifier) - { - media_type - } else { - bail!( - "Unable to resolve media type for specifier: \"{}\"", - resolved_specifier - ) - }; - resolved.push(( - resolved_specifier.to_string(), - media_type.as_ts_extension(), - )); - } - // in certain situations, like certain dynamic imports, we won't have - // the source file in the graph, so we will return a fake module to - // make tsc happy. - Err(_) => { - resolved.push(("deno:///none.d.ts".to_string(), ".d.ts".to_string())); - } - } - } - } - - Ok(json!(resolved)) -} - -#[derive(Debug, Deserialize, Eq, PartialEq)] -struct RespondArgs { - pub diagnostics: Diagnostics, - pub stats: Stats, -} - -fn respond(state: &mut State, args: Value) -> Result { - let v: RespondArgs = serde_json::from_value(args) - .context("Error converting the result for \"op_respond\".")?; - state.maybe_response = Some(v); - Ok(json!(true)) -} - -/// Execute a request on the supplied snapshot, returning a response which -/// contains information, like any emitted files, diagnostics, statistics and -/// optionally an updated TypeScript build info. -pub fn exec( - snapshot: Snapshot, - request: Request, -) -> Result { - let mut runtime = JsRuntime::new(RuntimeOptions { - startup_snapshot: Some(snapshot), - ..Default::default() - }); - // tsc cannot handle root specifiers that don't have one of the "acceptable" - // extensions. Therefore, we have to check the root modules against their - // extensions and remap any that are unacceptable to tsc and add them to the - // op state so when requested, we can remap to the original specifier. - let mut root_map = HashMap::new(); - let root_names: Vec = request - .root_names - .iter() - .map(|(s, mt)| { - let ext_media_type = MediaType::from(&s.as_str().to_owned()); - if mt != &ext_media_type { - let new_specifier = format!("{}{}", s, mt.as_ts_extension()); - root_map.insert(new_specifier.clone(), s.clone()); - new_specifier - } else { - s.as_str().to_owned() - } - }) - .collect(); - - { - let op_state = runtime.op_state(); - let mut op_state = op_state.borrow_mut(); - op_state.put(State::new( - request.graph.clone(), - request.hash_data.clone(), - request.maybe_tsbuildinfo.clone(), - root_map, - )); - } - - runtime.register_op("op_create_hash", op(create_hash)); - runtime.register_op("op_emit", op(emit)); - runtime.register_op("op_load", op(load)); - runtime.register_op("op_resolve", op(resolve)); - runtime.register_op("op_respond", op(respond)); - - let startup_source = "globalThis.startup({ legacyFlag: false })"; - let request_value = json!({ - "config": request.config, - "debug": request.debug, - "rootNames": root_names, - }); - let request_str = request_value.to_string(); - let exec_source = format!("globalThis.exec({})", request_str); - - runtime - .execute("[native code]", startup_source) - .context("Could not properly start the compiler runtime.")?; - runtime.execute("[native_code]", &exec_source)?; - - let op_state = runtime.op_state(); - let mut op_state = op_state.borrow_mut(); - let state = op_state.take::(); - - if let Some(response) = state.maybe_response { - let diagnostics = response.diagnostics; - let emitted_files = state.emitted_files; - let maybe_tsbuildinfo = state.maybe_tsbuildinfo; - let stats = response.stats; - - Ok(Response { - diagnostics, - emitted_files, - maybe_tsbuildinfo, - stats, - }) - } else { - Err(anyhow!("The response for the exec request was not set.")) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::diagnostics::Diagnostic; - use crate::diagnostics::DiagnosticCategory; - use crate::js; - use crate::module_graph2::tests::MockSpecifierHandler; - use crate::module_graph2::GraphBuilder2; - use crate::tsc_config::TsConfig; - use std::cell::RefCell; - use std::env; - use std::path::PathBuf; - - async fn setup( - maybe_specifier: Option, - maybe_hash_data: Option>>, - maybe_tsbuildinfo: Option, - ) -> State { - let specifier = maybe_specifier.unwrap_or_else(|| { - ModuleSpecifier::resolve_url_or_path("file:///main.ts").unwrap() - }); - let hash_data = maybe_hash_data.unwrap_or_else(|| vec![b"".to_vec()]); - let c = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()); - let fixtures = c.join("tests/tsc2"); - let handler = Rc::new(RefCell::new(MockSpecifierHandler { - fixtures, - ..MockSpecifierHandler::default() - })); - let mut builder = GraphBuilder2::new(handler.clone(), None, None); - builder - .add(&specifier, false) - .await - .expect("module not inserted"); - let graph = Rc::new(RefCell::new(builder.get_graph())); - State::new(graph, hash_data, maybe_tsbuildinfo, HashMap::new()) - } - - async fn test_exec( - specifier: &ModuleSpecifier, - ) -> Result { - let hash_data = vec![b"something".to_vec()]; - let c = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()); - let fixtures = c.join("tests/tsc2"); - let handler = Rc::new(RefCell::new(MockSpecifierHandler { - fixtures, - ..Default::default() - })); - let mut builder = GraphBuilder2::new(handler.clone(), None, None); - builder.add(&specifier, false).await?; - let graph = Rc::new(RefCell::new(builder.get_graph())); - let config = TsConfig::new(json!({ - "allowJs": true, - "checkJs": false, - "esModuleInterop": true, - "emitDecoratorMetadata": false, - "incremental": true, - "jsx": "react", - "jsxFactory": "React.createElement", - "jsxFragmentFactory": "React.Fragment", - "lib": ["deno.window"], - "module": "esnext", - "noEmit": true, - "outDir": "deno:///", - "strict": true, - "target": "esnext", - "tsBuildInfoFile": "deno:///.tsbuildinfo", - })); - let request = Request { - config, - debug: false, - graph, - hash_data, - maybe_tsbuildinfo: None, - root_names: vec![(specifier.clone(), MediaType::TypeScript)], - }; - exec(js::compiler_isolate_init(), request) - } - - #[tokio::test] - async fn test_create_hash() { - let mut state = setup(None, Some(vec![b"something".to_vec()]), None).await; - let actual = - create_hash(&mut state, json!({ "data": "some sort of content" })) - .expect("could not invoke op"); - assert_eq!( - actual, - json!({"hash": "ae92df8f104748768838916857a1623b6a3c593110131b0a00f81ad9dac16511"}) - ); - } - - #[tokio::test] - async fn test_emit() { - let mut state = setup(None, None, None).await; - let actual = emit( - &mut state, - json!({ - "data": "some file content", - "fileName": "cache:///some/file.js", - "maybeSpecifiers": ["file:///some/file.ts"] - }), - ) - .expect("should have invoked op"); - assert_eq!(actual, json!(true)); - assert_eq!(state.emitted_files.len(), 1); - assert!(state.maybe_tsbuildinfo.is_none()); - assert_eq!( - state.emitted_files[0], - EmittedFile { - data: "some file content".to_string(), - maybe_specifiers: Some(vec![ModuleSpecifier::resolve_url_or_path( - "file:///some/file.ts" - ) - .unwrap()]), - media_type: MediaType::JavaScript, - } - ); - } - - #[tokio::test] - async fn test_emit_tsbuildinfo() { - let mut state = setup(None, None, None).await; - let actual = emit( - &mut state, - json!({ - "data": "some file content", - "fileName": "deno:///.tsbuildinfo", - }), - ) - .expect("should have invoked op"); - assert_eq!(actual, json!(true)); - assert_eq!(state.emitted_files.len(), 0); - assert_eq!( - state.maybe_tsbuildinfo, - Some("some file content".to_string()) - ); - } - - #[tokio::test] - async fn test_load() { - let mut state = setup( - Some( - ModuleSpecifier::resolve_url_or_path("https://deno.land/x/mod.ts") - .unwrap(), - ), - None, - Some("some content".to_string()), - ) - .await; - let actual = load( - &mut state, - json!({ "specifier": "https://deno.land/x/mod.ts"}), - ) - .expect("should have invoked op"); - assert_eq!( - actual, - json!({ - "data": "console.log(\"hello deno\");\n", - "hash": "149c777056afcc973d5fcbe11421b6d5ddc57b81786765302030d7fc893bf729", - "scriptKind": 3, - }) - ); - } - - #[derive(Debug, Deserialize)] - #[serde(rename_all = "camelCase")] - struct LoadResponse { - data: String, - hash: Option, - script_kind: i64, - } - - #[tokio::test] - async fn test_load_asset() { - let mut state = setup( - Some( - ModuleSpecifier::resolve_url_or_path("https://deno.land/x/mod.ts") - .unwrap(), - ), - None, - Some("some content".to_string()), - ) - .await; - let value = - load(&mut state, json!({ "specifier": "asset:///lib.dom.d.ts" })) - .expect("should have invoked op"); - let actual: LoadResponse = - serde_json::from_value(value).expect("failed to deserialize"); - let expected = get_asset("lib.dom.d.ts").unwrap(); - assert_eq!(actual.data, expected); - assert!(actual.hash.is_some()); - assert_eq!(actual.script_kind, 3); - } - - #[tokio::test] - async fn test_load_tsbuildinfo() { - let mut state = setup( - Some( - ModuleSpecifier::resolve_url_or_path("https://deno.land/x/mod.ts") - .unwrap(), - ), - None, - Some("some content".to_string()), - ) - .await; - let actual = - load(&mut state, json!({ "specifier": "deno:///.tsbuildinfo"})) - .expect("should have invoked op"); - assert_eq!( - actual, - json!({ - "data": "some content", - "hash": null, - "scriptKind": 0, - }) - ); - } - - #[tokio::test] - async fn test_load_missing_specifier() { - let mut state = setup(None, None, None).await; - let actual = load( - &mut state, - json!({ "specifier": "https://deno.land/x/mod.ts"}), - ) - .expect("should have invoked op"); - assert_eq!( - actual, - json!({ - "data": null, - "hash": null, - "scriptKind": 0, - }) - ) - } - - #[tokio::test] - async fn test_resolve() { - let mut state = setup( - Some( - ModuleSpecifier::resolve_url_or_path("https://deno.land/x/a.ts") - .unwrap(), - ), - None, - None, - ) - .await; - let actual = resolve( - &mut state, - json!({ "base": "https://deno.land/x/a.ts", "specifiers": [ "./b.ts" ]}), - ) - .expect("should have invoked op"); - assert_eq!(actual, json!([["https://deno.land/x/b.ts", ".ts"]])); - } - - #[tokio::test] - async fn test_resolve_empty() { - let mut state = setup( - Some( - ModuleSpecifier::resolve_url_or_path("https://deno.land/x/a.ts") - .unwrap(), - ), - None, - None, - ) - .await; - let actual = resolve( - &mut state, - json!({ "base": "https://deno.land/x/a.ts", "specifiers": [ "./bad.ts" ]}), - ).expect("should have not errored"); - assert_eq!(actual, json!([["deno:///none.d.ts", ".d.ts"]])); - } - - #[tokio::test] - async fn test_respond() { - let mut state = setup(None, None, None).await; - let actual = respond( - &mut state, - json!({ - "diagnostics": [ - { - "messageText": "Unknown compiler option 'invalid'.", - "category": 1, - "code": 5023 - } - ], - "stats": [["a", 12]] - }), - ) - .expect("should have invoked op"); - assert_eq!(actual, json!(true)); - assert_eq!( - state.maybe_response, - Some(RespondArgs { - diagnostics: Diagnostics::new(vec![Diagnostic { - category: DiagnosticCategory::Error, - code: 5023, - start: None, - end: None, - message_text: Some( - "Unknown compiler option \'invalid\'.".to_string() - ), - message_chain: None, - source: None, - source_line: None, - file_name: None, - related_information: None, - }]), - stats: Stats(vec![("a".to_string(), 12)]) - }) - ); - } - - #[tokio::test] - async fn test_exec_basic() { - let specifier = - ModuleSpecifier::resolve_url_or_path("https://deno.land/x/a.ts").unwrap(); - let actual = test_exec(&specifier) - .await - .expect("exec should not have errored"); - assert!(actual.diagnostics.is_empty()); - assert!(actual.emitted_files.is_empty()); - assert!(actual.maybe_tsbuildinfo.is_some()); - assert_eq!(actual.stats.0.len(), 12); - } - - #[tokio::test] - async fn test_exec_reexport_dts() { - let specifier = - ModuleSpecifier::resolve_url_or_path("file:///reexports.ts").unwrap(); - let actual = test_exec(&specifier) - .await - .expect("exec should not have errored"); - assert!(actual.diagnostics.is_empty()); - assert!(actual.emitted_files.is_empty()); - assert!(actual.maybe_tsbuildinfo.is_some()); - assert_eq!(actual.stats.0.len(), 12); - } - - #[tokio::test] - async fn fix_lib_ref() { - let specifier = - ModuleSpecifier::resolve_url_or_path("file:///libref.ts").unwrap(); - let actual = test_exec(&specifier) - .await - .expect("exec should not have errored"); - assert!(actual.diagnostics.is_empty()); - } -} -- cgit v1.2.3