diff options
| -rw-r--r-- | cli/diagnostics.rs | 36 | ||||
| -rw-r--r-- | cli/dts/lib.deno.unstable.d.ts | 27 | ||||
| -rw-r--r-- | cli/main.rs | 34 | ||||
| -rw-r--r-- | cli/media_type.rs | 17 | ||||
| -rw-r--r-- | cli/module_graph.rs | 968 | ||||
| -rw-r--r-- | cli/module_graph2.rs | 500 | ||||
| -rw-r--r-- | cli/module_loader.rs | 20 | ||||
| -rw-r--r-- | cli/op_fetch_asset.rs | 7 | ||||
| -rw-r--r-- | cli/ops/runtime_compiler.rs | 76 | ||||
| -rw-r--r-- | cli/program_state.rs | 60 | ||||
| -rw-r--r-- | cli/rt/40_compiler_api.js | 17 | ||||
| -rw-r--r-- | cli/specifier_handler.rs | 217 | ||||
| -rw-r--r-- | cli/tests/compiler_api_test.ts | 135 | ||||
| -rw-r--r-- | cli/tests/compiler_js_error.ts | 1 | ||||
| -rw-r--r-- | cli/tests/compiler_js_error.ts.out | 4 | ||||
| -rw-r--r-- | cli/tests/integration_tests.rs | 6 | ||||
| -rw-r--r-- | cli/tests/lib_ref.ts | 6 | ||||
| -rw-r--r-- | cli/tests/lib_ref.ts.out | 2 | ||||
| -rw-r--r-- | cli/tests/lib_runtime_api.ts | 6 | ||||
| -rw-r--r-- | cli/tests/lib_runtime_api.ts.out | 2 | ||||
| -rw-r--r-- | cli/tsc.rs | 1005 | ||||
| -rw-r--r-- | cli/tsc/99_main_compiler.js | 418 | ||||
| -rw-r--r-- | cli/tsc_config.rs | 58 |
23 files changed, 852 insertions, 2770 deletions
diff --git a/cli/diagnostics.rs b/cli/diagnostics.rs index 1cd4ea234..ba21e5aa9 100644 --- a/cli/diagnostics.rs +++ b/cli/diagnostics.rs @@ -2,9 +2,11 @@ use crate::colors; +use deno_core::serde::Deserialize; +use deno_core::serde::Deserializer; +use deno_core::serde::Serialize; +use deno_core::serde::Serializer; use regex::Regex; -use serde::Deserialize; -use serde::Deserializer; use std::error::Error; use std::fmt; @@ -157,6 +159,21 @@ impl<'de> Deserialize<'de> for DiagnosticCategory { } } +impl Serialize for DiagnosticCategory { + fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error> + where + S: Serializer, + { + let value = match self { + DiagnosticCategory::Warning => 0 as i32, + DiagnosticCategory::Error => 1 as i32, + DiagnosticCategory::Suggestion => 2 as i32, + DiagnosticCategory::Message => 3 as i32, + }; + Serialize::serialize(&value, serializer) + } +} + impl From<i64> for DiagnosticCategory { fn from(value: i64) -> Self { match value { @@ -169,7 +186,7 @@ impl From<i64> for DiagnosticCategory { } } -#[derive(Debug, Deserialize, Clone, Eq, PartialEq)] +#[derive(Debug, Deserialize, Serialize, Clone, Eq, PartialEq)] #[serde(rename_all = "camelCase")] pub struct DiagnosticMessageChain { message_text: String, @@ -196,14 +213,14 @@ impl DiagnosticMessageChain { } } -#[derive(Debug, Deserialize, Clone, Eq, PartialEq)] +#[derive(Debug, Deserialize, Serialize, Clone, Eq, PartialEq)] #[serde(rename_all = "camelCase")] pub struct Position { pub line: u64, pub character: u64, } -#[derive(Debug, Deserialize, Clone, Eq, PartialEq)] +#[derive(Debug, Deserialize, Serialize, Clone, Eq, PartialEq)] #[serde(rename_all = "camelCase")] pub struct Diagnostic { pub category: DiagnosticCategory, @@ -367,6 +384,15 @@ impl<'de> Deserialize<'de> for Diagnostics { } } +impl Serialize for Diagnostics { + fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error> + where + S: Serializer, + { + Serialize::serialize(&self.0, serializer) + } +} + impl fmt::Display for Diagnostics { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { let mut i = 0; diff --git a/cli/dts/lib.deno.unstable.d.ts b/cli/dts/lib.deno.unstable.d.ts index 78a7e79c2..6eb39c21c 100644 --- a/cli/dts/lib.deno.unstable.d.ts +++ b/cli/dts/lib.deno.unstable.d.ts @@ -303,9 +303,6 @@ declare namespace Deno { /** Provide full support for iterables in `for..of`, spread and * destructuring when targeting ES5 or ES3. Defaults to `false`. */ downlevelIteration?: boolean; - /** Emit a UTF-8 Byte Order Mark (BOM) in the beginning of output files. - * Defaults to `false`. */ - emitBOM?: boolean; /** Only emit `.d.ts` declaration files. Defaults to `false`. */ emitDeclarationOnly?: boolean; /** Emit design-type metadata for decorated declarations in source. See issue @@ -316,8 +313,11 @@ declare namespace Deno { * ecosystem compatibility and enable `allowSyntheticDefaultImports` for type * system compatibility. Defaults to `true`. */ esModuleInterop?: boolean; - /** Enables experimental support for ES decorators. Defaults to `false`. */ + /** Enables experimental support for ES decorators. Defaults to `true`. */ experimentalDecorators?: boolean; + /** Import emit helpers (e.g. `__extends`, `__rest`, etc..) from + * [tslib](https://www.npmjs.com/package/tslib). */ + importHelpers?: boolean; /** Emit a single file with source maps instead of having a separate file. * Defaults to `false`. */ inlineSourceMap?: boolean; @@ -325,7 +325,7 @@ declare namespace Deno { * `inlineSourceMap` or `sourceMap` to be set. Defaults to `false`. */ inlineSources?: boolean; /** Perform additional checks to ensure that transpile only would be safe. - * Defaults to `false`. */ + * Defaults to `true`. */ isolatedModules?: boolean; /** Support JSX in `.tsx` files: `"react"`, `"preserve"`, `"react-native"`. * Defaults to `"react"`. */ @@ -333,12 +333,12 @@ declare namespace Deno { /** Specify the JSX factory function to use when targeting react JSX emit, * e.g. `React.createElement` or `h`. Defaults to `React.createElement`. */ jsxFactory?: string; + /** Specify the JSX fragment factory function to use when targeting react + * JSX emit, e.g. `Fragment`. Defaults to `React.Fragment`. */ + jsxFragmentFactory?: string; /** Resolve keyof to string valued property names only (no numbers or * symbols). Defaults to `false`. */ keyofStringsOnly?: string; - /** Emit class fields with ECMAScript-standard semantics. Defaults to `false`. - */ - useDefineForClassFields?: boolean; /** List of library files to be included in the compilation. If omitted, * then the Deno main runtime libs are used. */ lib?: string[]; @@ -389,10 +389,6 @@ declare namespace Deno { noUnusedLocals?: boolean; /** Report errors on unused parameters. Defaults to `false`. */ noUnusedParameters?: boolean; - /** Redirect output structure to the directory. This only impacts - * `Deno.compile` and only changes the emitted file names. Defaults to - * `undefined`. */ - outDir?: string; /** List of path mapping entries for module names to locations relative to the * `baseUrl`. Defaults to `undefined`. */ paths?: Record<string, string[]>; @@ -402,8 +398,6 @@ declare namespace Deno { /** Remove all comments except copy-right header comments beginning with * `/*!`. Defaults to `true`. */ removeComments?: boolean; - /** Include modules imported with `.json` extension. Defaults to `true`. */ - resolveJsonModule?: boolean; /** Specifies the root directory of input files. Only use to control the * output directory structure with `outDir`. Defaults to `undefined`. */ rootDir?: string; @@ -418,6 +412,8 @@ declare namespace Deno { * specified will be embedded in the sourceMap to direct the debugger where * the source files will be located. Defaults to `undefined`. */ sourceRoot?: string; + /** Skip type checking of all declaration files (`*.d.ts`). */ + skipLibCheck?: boolean; /** Enable all strict type checking options. Enabling `strict` enables * `noImplicitAny`, `noImplicitThis`, `alwaysStrict`, `strictBindCallApply`, * `strictNullChecks`, `strictFunctionTypes` and @@ -472,6 +468,9 @@ declare namespace Deno { * ``` */ types?: string[]; + /** Emit class fields with ECMAScript-standard semantics. Defaults to + * `false`. */ + useDefineForClassFields?: boolean; } /** **UNSTABLE**: new API, yet to be vetted. diff --git a/cli/main.rs b/cli/main.rs index 0d67286c4..dc68546d5 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -35,7 +35,6 @@ mod lint; mod lockfile; mod media_type; mod metrics; -mod module_graph; mod module_graph2; mod module_loader; mod op_fetch_asset; @@ -50,7 +49,6 @@ mod specifier_handler; mod test_runner; mod text_encoding; mod tokio_util; -mod tsc; mod tsc2; mod tsc_config; mod upgrade; @@ -242,6 +240,11 @@ async fn cache_command( flags: Flags, files: Vec<String>, ) -> Result<(), AnyError> { + let lib = if flags.unstable { + module_graph2::TypeLib::UnstableDenoWindow + } else { + module_graph2::TypeLib::DenoWindow + }; let program_state = ProgramState::new(flags)?; for file in files { @@ -249,7 +252,7 @@ async fn cache_command( program_state .prepare_module_load( specifier, - tsc::TargetLib::Main, + lib.clone(), Permissions::allow_all(), false, program_state.maybe_import_map.clone(), @@ -343,21 +346,20 @@ async fn bundle_command( module_graph2::TypeLib::DenoWindow }; let graph = graph.clone(); - let (stats, diagnostics, maybe_ignored_options) = - graph.check(module_graph2::CheckOptions { - debug, - emit: false, - lib, - maybe_config_path: flags.config_path.clone(), - reload: flags.reload, - })?; - - debug!("{}", stats); - if let Some(ignored_options) = maybe_ignored_options { + let result_info = graph.check(module_graph2::CheckOptions { + debug, + emit: false, + lib, + maybe_config_path: flags.config_path.clone(), + reload: flags.reload, + })?; + + debug!("{}", result_info.stats); + if let Some(ignored_options) = result_info.maybe_ignored_options { eprintln!("{}", ignored_options); } - if !diagnostics.is_empty() { - return Err(generic_error(diagnostics.to_string())); + if !result_info.diagnostics.is_empty() { + return Err(generic_error(result_info.diagnostics.to_string())); } } diff --git a/cli/media_type.rs b/cli/media_type.rs index 7d63439f6..cb26bcff5 100644 --- a/cli/media_type.rs +++ b/cli/media_type.rs @@ -1,5 +1,6 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +use deno_core::ModuleSpecifier; use serde::Serialize; use serde::Serializer; use std::fmt; @@ -60,6 +61,22 @@ impl<'a> From<&'a String> for MediaType { } } +impl<'a> From<&'a ModuleSpecifier> for MediaType { + fn from(specifier: &'a ModuleSpecifier) -> Self { + let url = specifier.as_url(); + let path = if url.scheme() == "file" { + if let Ok(path) = url.to_file_path() { + path + } else { + PathBuf::from(url.path()) + } + } else { + PathBuf::from(url.path()) + }; + MediaType::from_path(&path) + } +} + impl Default for MediaType { fn default() -> Self { MediaType::Unknown diff --git a/cli/module_graph.rs b/cli/module_graph.rs deleted file mode 100644 index f0645424e..000000000 --- a/cli/module_graph.rs +++ /dev/null @@ -1,968 +0,0 @@ -// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. - -use crate::ast::Location; -use crate::checksum; -use crate::file_fetcher::SourceFile; -use crate::file_fetcher::SourceFileFetcher; -use crate::import_map::ImportMap; -use crate::media_type::MediaType; -use crate::permissions::Permissions; -use crate::tsc::pre_process_file; -use crate::tsc::ImportDesc; -use crate::tsc::TsReferenceDesc; -use crate::tsc::TsReferenceKind; -use crate::tsc::AVAILABLE_LIBS; -use crate::version; -use deno_core::error::custom_error; -use deno_core::error::generic_error; -use deno_core::error::AnyError; -use deno_core::futures::stream::FuturesUnordered; -use deno_core::futures::stream::StreamExt; -use deno_core::futures::Future; -use deno_core::futures::FutureExt; -use deno_core::ModuleSpecifier; -use serde::Serialize; -use serde::Serializer; -use std::collections::HashMap; -use std::collections::HashSet; -use std::pin::Pin; - -// TODO(bartlomieju): it'd be great if this function returned -// more structured data and possibly format the same as TS diagnostics. -/// Decorate error with location of import that caused the error. -fn err_with_location( - e: AnyError, - maybe_location: Option<&Location>, -) -> AnyError { - if let Some(location) = maybe_location { - let location_str = format!( - "\nImported from \"{}:{}\"", - location.filename, location.line - ); - let err_str = e.to_string(); - generic_error(format!("{}{}", err_str, location_str)) - } else { - e - } -} - -/// Disallow http:// imports from modules loaded over https:// -fn validate_no_downgrade( - module_specifier: &ModuleSpecifier, - maybe_referrer: Option<&ModuleSpecifier>, - maybe_location: Option<&Location>, -) -> Result<(), AnyError> { - if let Some(referrer) = maybe_referrer.as_ref() { - if let "https" = referrer.as_url().scheme() { - if let "http" = module_specifier.as_url().scheme() { - let e = custom_error("PermissionDenied", - "Modules loaded over https:// are not allowed to import modules over http://" - ); - return Err(err_with_location(e, maybe_location)); - }; - }; - }; - - Ok(()) -} - -/// Verify that remote file doesn't try to statically import local file. -fn validate_no_file_from_remote( - module_specifier: &ModuleSpecifier, - maybe_referrer: Option<&ModuleSpecifier>, - maybe_location: Option<&Location>, -) -> Result<(), AnyError> { - if let Some(referrer) = maybe_referrer.as_ref() { - let referrer_url = referrer.as_url(); - match referrer_url.scheme() { - "http" | "https" => { - let specifier_url = module_specifier.as_url(); - match specifier_url.scheme() { - "http" | "https" => {} - _ => { - let e = custom_error( - "PermissionDenied", - "Remote modules are not allowed to statically import local \ - modules. Use dynamic import instead.", - ); - return Err(err_with_location(e, maybe_location)); - } - } - } - _ => {} - } - } - - Ok(()) -} - -// TODO(bartlomieju): handle imports/references in ambient contexts/TS modules -// https://github.com/denoland/deno/issues/6133 -fn resolve_imports_and_references( - referrer: ModuleSpecifier, - maybe_import_map: Option<&ImportMap>, - import_descs: Vec<ImportDesc>, - ref_descs: Vec<TsReferenceDesc>, -) -> Result<(Vec<ImportDescriptor>, Vec<ReferenceDescriptor>), AnyError> { - let mut imports = vec![]; - let mut references = vec![]; - - for import_desc in import_descs { - let maybe_resolved = if let Some(import_map) = maybe_import_map.as_ref() { - import_map.resolve(&import_desc.specifier, &referrer.to_string())? - } else { - None - }; - - let resolved_specifier = if let Some(resolved) = maybe_resolved { - resolved - } else { - ModuleSpecifier::resolve_import( - &import_desc.specifier, - &referrer.to_string(), - )? - }; - - let resolved_type_directive = - if let Some(types_specifier) = import_desc.deno_types.as_ref() { - Some(ModuleSpecifier::resolve_import( - &types_specifier, - &referrer.to_string(), - )?) - } else { - None - }; - - let import_descriptor = ImportDescriptor { - specifier: import_desc.specifier.to_string(), - resolved_specifier, - type_directive: import_desc.deno_types.clone(), - resolved_type_directive, - location: import_desc.location, - }; - - imports.push(import_descriptor); - } - - for ref_desc in ref_descs { - if AVAILABLE_LIBS.contains(&ref_desc.specifier.as_str()) { - continue; - } - - let resolved_specifier = ModuleSpecifier::resolve_import( - &ref_desc.specifier, - &referrer.to_string(), - )?; - - let reference_descriptor = ReferenceDescriptor { - specifier: ref_desc.specifier.to_string(), - resolved_specifier, - kind: ref_desc.kind, - location: ref_desc.location, - }; - - references.push(reference_descriptor); - } - - Ok((imports, references)) -} - -fn serialize_module_specifier<S>( - spec: &ModuleSpecifier, - s: S, -) -> Result<S::Ok, S::Error> -where - S: Serializer, -{ - s.serialize_str(&spec.to_string()) -} - -fn serialize_option_module_specifier<S>( - maybe_spec: &Option<ModuleSpecifier>, - s: S, -) -> Result<S::Ok, S::Error> -where - S: Serializer, -{ - if let Some(spec) = maybe_spec { - serialize_module_specifier(spec, s) - } else { - s.serialize_none() - } -} - -const SUPPORTED_MEDIA_TYPES: [MediaType; 4] = [ - MediaType::JavaScript, - MediaType::TypeScript, - MediaType::JSX, - MediaType::TSX, -]; - -pub type ModuleGraph = HashMap<String, ModuleGraphFile>; - -#[derive(Clone, Debug, Serialize, PartialEq)] -#[serde(rename_all = "camelCase")] -pub struct ImportDescriptor { - pub specifier: String, - #[serde(serialize_with = "serialize_module_specifier")] - pub resolved_specifier: ModuleSpecifier, - // These two fields are for support of @deno-types directive - // directly prepending import statement - pub type_directive: Option<String>, - #[serde(serialize_with = "serialize_option_module_specifier")] - pub resolved_type_directive: Option<ModuleSpecifier>, - #[serde(skip)] - pub location: Location, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct ReferenceDescriptor { - pub specifier: String, - #[serde(serialize_with = "serialize_module_specifier")] - pub resolved_specifier: ModuleSpecifier, - #[serde(skip)] - pub kind: TsReferenceKind, - #[serde(skip)] - pub location: Location, -} - -#[derive(Debug, Serialize)] -#[serde(rename_all = "camelCase")] -pub struct ModuleGraphFile { - pub specifier: String, - pub url: String, - pub redirect: Option<String>, - pub filename: String, - pub version_hash: String, - pub imports: Vec<ImportDescriptor>, - pub referenced_files: Vec<ReferenceDescriptor>, - pub lib_directives: Vec<ReferenceDescriptor>, - pub types_directives: Vec<ReferenceDescriptor>, - pub type_headers: Vec<ReferenceDescriptor>, - pub media_type: MediaType, - pub source_code: String, -} - -type SourceFileFuture = Pin< - Box<dyn Future<Output = Result<(ModuleSpecifier, SourceFile), AnyError>>>, ->; - -pub struct ModuleGraphLoader { - permissions: Permissions, - file_fetcher: SourceFileFetcher, - maybe_import_map: Option<ImportMap>, - pending_downloads: FuturesUnordered<SourceFileFuture>, - has_downloaded: HashSet<ModuleSpecifier>, - graph: ModuleGraph, - is_dyn_import: bool, - analyze_dynamic_imports: bool, -} - -impl ModuleGraphLoader { - pub fn new( - file_fetcher: SourceFileFetcher, - maybe_import_map: Option<ImportMap>, - permissions: Permissions, - is_dyn_import: bool, - analyze_dynamic_imports: bool, - ) -> Self { - Self { - file_fetcher, - permissions, - maybe_import_map, - pending_downloads: FuturesUnordered::new(), - has_downloaded: HashSet::new(), - graph: ModuleGraph::new(), - is_dyn_import, - analyze_dynamic_imports, - } - } - - /// This method is used to add specified module and all of its - /// dependencies to the graph. - /// - /// It resolves when all dependent modules have been fetched and analyzed. - /// - /// This method can be called multiple times. - pub async fn add_to_graph( - &mut self, - specifier: &ModuleSpecifier, - maybe_referrer: Option<ModuleSpecifier>, - ) -> Result<(), AnyError> { - self.download_module(specifier.clone(), maybe_referrer, None)?; - - loop { - let (specifier, source_file) = - self.pending_downloads.next().await.unwrap()?; - self.visit_module(&specifier, source_file)?; - if self.pending_downloads.is_empty() { - break; - } - } - - Ok(()) - } - - /// This method is used to create a graph from in-memory files stored in - /// a hash map. Useful for creating module graph for code received from - /// the runtime. - pub fn build_local_graph( - &mut self, - _root_name: &str, - source_map: &HashMap<String, String>, - ) -> Result<(), AnyError> { - for (spec, source_code) in source_map.iter() { - self.visit_memory_module(spec.to_string(), source_code.to_string())?; - } - - Ok(()) - } - - /// Consumes the loader and returns created graph. - pub fn get_graph(self) -> ModuleGraph { - self.graph - } - - fn visit_memory_module( - &mut self, - specifier: String, - source_code: String, - ) -> Result<(), AnyError> { - let mut referenced_files = vec![]; - let mut lib_directives = vec![]; - let mut types_directives = vec![]; - - // FIXME(bartlomieju): - // The resolveModules op only handles fully qualified URLs for referrer. - // However we will have cases where referrer is "/foo.ts". We add this dummy - // prefix "memory://" in order to use resolution logic. - let module_specifier = - if let Ok(spec) = ModuleSpecifier::resolve_url(&specifier) { - spec - } else { - ModuleSpecifier::resolve_url(&format!("memory://{}", specifier))? - }; - - let (raw_imports, raw_references) = pre_process_file( - &module_specifier.to_string(), - MediaType::from(&specifier), - &source_code, - self.analyze_dynamic_imports, - )?; - let (imports, references) = resolve_imports_and_references( - module_specifier.clone(), - self.maybe_import_map.as_ref(), - raw_imports, - raw_references, - )?; - - for ref_descriptor in references { - match ref_descriptor.kind { - TsReferenceKind::Lib => { - lib_directives.push(ref_descriptor); - } - TsReferenceKind::Types => { - types_directives.push(ref_descriptor); - } - TsReferenceKind::Path => { - referenced_files.push(ref_descriptor); - } - } - } - - self.graph.insert( - module_specifier.to_string(), - ModuleGraphFile { - specifier: specifier.to_string(), - url: specifier.to_string(), - redirect: None, - version_hash: "".to_string(), - media_type: MediaType::from(&specifier), - filename: specifier, - source_code, - imports, - referenced_files, - lib_directives, - types_directives, - type_headers: vec![], - }, - ); - Ok(()) - } - - // TODO(bartlomieju): decorate errors with import location in the source code - // https://github.com/denoland/deno/issues/5080 - fn download_module( - &mut self, - module_specifier: ModuleSpecifier, - maybe_referrer: Option<ModuleSpecifier>, - maybe_location: Option<Location>, - ) -> Result<(), AnyError> { - if self.has_downloaded.contains(&module_specifier) { - return Ok(()); - } - - validate_no_downgrade( - &module_specifier, - maybe_referrer.as_ref(), - maybe_location.as_ref(), - )?; - - if !self.is_dyn_import { - validate_no_file_from_remote( - &module_specifier, - maybe_referrer.as_ref(), - maybe_location.as_ref(), - )?; - } - - self.has_downloaded.insert(module_specifier.clone()); - let spec = module_specifier; - let file_fetcher = self.file_fetcher.clone(); - let perms = self.permissions.clone(); - - let load_future = async move { - let spec_ = spec.clone(); - let source_file = file_fetcher - .fetch_source_file(&spec_, maybe_referrer, perms) - .await - .map_err(|e| err_with_location(e, maybe_location.as_ref()))?; - - Ok((spec_.clone(), source_file)) - } - .boxed_local(); - - self.pending_downloads.push(load_future); - Ok(()) - } - - fn visit_module( - &mut self, - module_specifier: &ModuleSpecifier, - source_file: SourceFile, - ) -> Result<(), AnyError> { - let mut imports = vec![]; - let mut referenced_files = vec![]; - let mut lib_directives = vec![]; - let mut types_directives = vec![]; - let mut type_headers = vec![]; - - // IMPORTANT: source_file.url might be different than requested - // module_specifier because of HTTP redirects. In such - // situation we add an "empty" ModuleGraphFile with 'redirect' - // field set that will be later used in TS worker when building - // map of available source file. It will perform substitution - // for proper URL point to redirect target. - if module_specifier.as_url() != &source_file.url { - // TODO(bartlomieju): refactor, this is a band-aid - self.graph.insert( - module_specifier.to_string(), - ModuleGraphFile { - specifier: module_specifier.to_string(), - url: module_specifier.to_string(), - redirect: Some(source_file.url.to_string()), - filename: source_file.filename.to_str().unwrap().to_string(), - version_hash: checksum::gen(&[ - &source_file.source_code.as_bytes(), - &version::DENO.as_bytes(), - ]), - media_type: source_file.media_type, - source_code: "".to_string(), - imports: vec![], - referenced_files: vec![], - lib_directives: vec![], - types_directives: vec![], - type_headers: vec![], - }, - ); - } - - let module_specifier = ModuleSpecifier::from(source_file.url.clone()); - let version_hash = checksum::gen(&[ - &source_file.source_code.as_bytes(), - &version::DENO.as_bytes(), - ]); - let source_code = source_file.source_code.clone(); - - if SUPPORTED_MEDIA_TYPES.contains(&source_file.media_type) { - if let Some(types_specifier) = source_file.types_header { - let type_header = ReferenceDescriptor { - specifier: types_specifier.to_string(), - resolved_specifier: ModuleSpecifier::resolve_import( - &types_specifier, - &module_specifier.to_string(), - )?, - kind: TsReferenceKind::Types, - // TODO(bartlomieju): location is not needed in here and constructing - // location by hand is bad - location: Location { - filename: module_specifier.to_string(), - line: 0, - col: 0, - }, - }; - self.download_module( - type_header.resolved_specifier.clone(), - Some(module_specifier.clone()), - None, - )?; - type_headers.push(type_header); - } - - let (raw_imports, raw_refs) = pre_process_file( - &module_specifier.to_string(), - source_file.media_type, - &source_code, - self.analyze_dynamic_imports, - )?; - let (imports_, references) = resolve_imports_and_references( - module_specifier.clone(), - self.maybe_import_map.as_ref(), - raw_imports, - raw_refs, - )?; - - for import_descriptor in imports_ { - self.download_module( - import_descriptor.resolved_specifier.clone(), - Some(module_specifier.clone()), - Some(import_descriptor.location.clone()), - )?; - - if let Some(type_dir_url) = - import_descriptor.resolved_type_directive.as_ref() - { - self.download_module( - type_dir_url.clone(), - Some(module_specifier.clone()), - Some(import_descriptor.location.clone()), - )?; - } - - imports.push(import_descriptor); - } - - for ref_descriptor in references { - self.download_module( - ref_descriptor.resolved_specifier.clone(), - Some(module_specifier.clone()), - Some(ref_descriptor.location.clone()), - )?; - - match ref_descriptor.kind { - TsReferenceKind::Lib => { - lib_directives.push(ref_descriptor); - } - TsReferenceKind::Types => { - types_directives.push(ref_descriptor); - } - TsReferenceKind::Path => { - referenced_files.push(ref_descriptor); - } - } - } - } - - self.graph.insert( - module_specifier.to_string(), - ModuleGraphFile { - specifier: module_specifier.to_string(), - url: module_specifier.to_string(), - redirect: None, - version_hash, - filename: source_file.filename.to_str().unwrap().to_string(), - media_type: source_file.media_type, - source_code, - imports, - referenced_files, - lib_directives, - types_directives, - type_headers, - }, - ); - Ok(()) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::program_state::ProgramState; - use deno_core::serde_json; - use deno_core::serde_json::json; - - async fn build_graph( - module_specifier: &ModuleSpecifier, - ) -> Result<ModuleGraph, AnyError> { - let program_state = ProgramState::new(Default::default()).unwrap(); - let mut graph_loader = ModuleGraphLoader::new( - program_state.file_fetcher.clone(), - None, - Permissions::allow_all(), - false, - false, - ); - graph_loader.add_to_graph(&module_specifier, None).await?; - Ok(graph_loader.get_graph()) - } - - // TODO(bartlomieju): this test is flaky, because it's using 019_media_types - // file, reenable once Python server is replaced with Rust one. - #[ignore] - #[tokio::test] - async fn source_graph_fetch() { - let _http_server_guard = test_util::http_server(); - - let module_specifier = ModuleSpecifier::resolve_url_or_path( - "http://localhost:4545/cli/tests/019_media_types.ts", - ) - .unwrap(); - let graph = build_graph(&module_specifier) - .await - .expect("Failed to build graph"); - - let a = graph - .get("http://localhost:4545/cli/tests/019_media_types.ts") - .unwrap(); - - assert!(graph.contains_key( - "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js" - )); - assert!(graph.contains_key( - "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts" - )); - assert!(graph.contains_key("http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts")); - assert!(graph.contains_key( - "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts" - )); - assert!(graph.contains_key("http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js")); - assert!(graph.contains_key( - "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js" - )); - assert!(graph.contains_key( - "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js" - )); - assert!(graph.contains_key( - "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts" - )); - - assert_eq!( - serde_json::to_value(&a.imports).unwrap(), - json!([ - { - "specifier": "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts", - "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts", - "typeDirective": null, - "resolvedTypeDirective": null, - }, - { - "specifier": "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts", - "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts", - "typeDirective": null, - "resolvedTypeDirective": null, - }, - { - "specifier": "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts", - "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts", - "typeDirective": null, - "resolvedTypeDirective": null, - }, - { - "specifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts", - "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts", - "typeDirective": null, - "resolvedTypeDirective": null, - }, - { - "specifier": "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js", - "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js", - "typeDirective": null, - "resolvedTypeDirective": null, - }, - { - "specifier": "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js", - "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js", - "typeDirective": null, - "resolvedTypeDirective": null, - }, - { - "specifier": "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js", - "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js", - "typeDirective": null, - "resolvedTypeDirective": null, - }, - { - "specifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js", - "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js", - "typeDirective": null, - "resolvedTypeDirective": null, - }, - ]) - ); - } - - #[tokio::test] - async fn source_graph_type_references() { - let _http_server_guard = test_util::http_server(); - - let module_specifier = ModuleSpecifier::resolve_url_or_path( - "http://localhost:4545/cli/tests/type_definitions.ts", - ) - .unwrap(); - - let graph = build_graph(&module_specifier) - .await - .expect("Failed to build graph"); - - eprintln!("json {:#?}", serde_json::to_value(&graph).unwrap()); - - let a = graph - .get("http://localhost:4545/cli/tests/type_definitions.ts") - .unwrap(); - assert_eq!( - serde_json::to_value(&a.imports).unwrap(), - json!([ - { - "specifier": "./type_definitions/foo.js", - "resolvedSpecifier": "http://localhost:4545/cli/tests/type_definitions/foo.js", - "typeDirective": "./type_definitions/foo.d.ts", - "resolvedTypeDirective": "http://localhost:4545/cli/tests/type_definitions/foo.d.ts" - }, - { - "specifier": "./type_definitions/fizz.js", - "resolvedSpecifier": "http://localhost:4545/cli/tests/type_definitions/fizz.js", - "typeDirective": "./type_definitions/fizz.d.ts", - "resolvedTypeDirective": "http://localhost:4545/cli/tests/type_definitions/fizz.d.ts" - }, - { - "specifier": "./type_definitions/qat.ts", - "resolvedSpecifier": "http://localhost:4545/cli/tests/type_definitions/qat.ts", - "typeDirective": null, - "resolvedTypeDirective": null, - }, - ]) - ); - assert!(graph - .contains_key("http://localhost:4545/cli/tests/type_definitions/foo.js")); - assert!(graph.contains_key( - "http://localhost:4545/cli/tests/type_definitions/foo.d.ts" - )); - assert!(graph.contains_key( - "http://localhost:4545/cli/tests/type_definitions/fizz.js" - )); - assert!(graph.contains_key( - "http://localhost:4545/cli/tests/type_definitions/fizz.d.ts" - )); - assert!(graph - .contains_key("http://localhost:4545/cli/tests/type_definitions/qat.ts")); - } - - #[tokio::test] - async fn source_graph_type_references2() { - let _http_server_guard = test_util::http_server(); - - let module_specifier = ModuleSpecifier::resolve_url_or_path( - "http://localhost:4545/cli/tests/type_directives_02.ts", - ) - .unwrap(); - - let graph = build_graph(&module_specifier) - .await - .expect("Failed to build graph"); - - eprintln!("{:#?}", serde_json::to_value(&graph).unwrap()); - - let a = graph - .get("http://localhost:4545/cli/tests/type_directives_02.ts") - .unwrap(); - assert_eq!( - serde_json::to_value(&a.imports).unwrap(), - json!([ - { - "specifier": "./subdir/type_reference.js", - "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/type_reference.js", - "typeDirective": null, - "resolvedTypeDirective": null, - } - ]) - ); - - assert!(graph.contains_key( - "http://localhost:4545/cli/tests/subdir/type_reference.d.ts" - )); - - let b = graph - .get("http://localhost:4545/cli/tests/subdir/type_reference.js") - .unwrap(); - assert_eq!( - serde_json::to_value(&b.types_directives).unwrap(), - json!([ - { - "specifier": "./type_reference.d.ts", - "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/type_reference.d.ts", - } - ]) - ); - } - - #[tokio::test] - async fn source_graph_type_references3() { - let _http_server_guard = test_util::http_server(); - - let module_specifier = ModuleSpecifier::resolve_url_or_path( - "http://localhost:4545/cli/tests/type_directives_01.ts", - ) - .unwrap(); - - let graph = build_graph(&module_specifier) - .await - .expect("Failed to build graph"); - - let ts = graph - .get("http://localhost:4545/cli/tests/type_directives_01.ts") - .unwrap(); - assert_eq!( - serde_json::to_value(&ts.imports).unwrap(), - json!([ - { - "specifier": "http://127.0.0.1:4545/xTypeScriptTypes.js", - "resolvedSpecifier": "http://127.0.0.1:4545/xTypeScriptTypes.js", - "typeDirective": null, - "resolvedTypeDirective": null, - } - ]) - ); - - let headers = graph - .get("http://127.0.0.1:4545/xTypeScriptTypes.js") - .unwrap(); - assert_eq!( - serde_json::to_value(&headers.type_headers).unwrap(), - json!([ - { - "specifier": "./xTypeScriptTypes.d.ts", - "resolvedSpecifier": "http://127.0.0.1:4545/xTypeScriptTypes.d.ts" - } - ]) - ); - } - - #[tokio::test] - async fn source_graph_different_langs() { - let _http_server_guard = test_util::http_server(); - - // ModuleGraphLoader was mistakenly parsing this file as TSX - // https://github.com/denoland/deno/issues/5867 - - let module_specifier = ModuleSpecifier::resolve_url_or_path( - "http://localhost:4545/cli/tests/ts_with_generic.ts", - ) - .unwrap(); - - build_graph(&module_specifier) - .await - .expect("Failed to build graph"); - } -} - -// TODO(bartlomieju): use baseline tests from TSC to ensure -// compatibility -#[test] -fn test_pre_process_file() { - let source = r#" -// This comment is placed to make sure that directives are parsed -// even when they start on non-first line - -/// <reference lib="dom" /> -/// <reference types="./type_reference.d.ts" /> -/// <reference path="./type_reference/dep.ts" /> -// @deno-types="./type_definitions/foo.d.ts" -import { foo } from "./type_definitions/foo.js"; -// @deno-types="./type_definitions/fizz.d.ts" -import "./type_definitions/fizz.js"; - -/// <reference path="./type_reference/dep2.ts" /> - -import * as qat from "./type_definitions/qat.ts"; - -console.log(foo); -console.log(fizz); -console.log(qat.qat); -"#; - - let (imports, references) = - pre_process_file("some/file.ts", MediaType::TypeScript, source, true) - .expect("Failed to parse"); - - assert_eq!( - imports, - vec![ - ImportDesc { - specifier: "./type_definitions/foo.js".to_string(), - deno_types: Some("./type_definitions/foo.d.ts".to_string()), - location: Location { - filename: "some/file.ts".to_string(), - line: 9, - col: 0, - }, - }, - ImportDesc { - specifier: "./type_definitions/fizz.js".to_string(), - deno_types: Some("./type_definitions/fizz.d.ts".to_string()), - location: Location { - filename: "some/file.ts".to_string(), - line: 11, - col: 0, - }, - }, - ImportDesc { - specifier: "./type_definitions/qat.ts".to_string(), - deno_types: None, - location: Location { - filename: "some/file.ts".to_string(), - line: 15, - col: 0, - }, - }, - ] - ); - - // According to TS docs (https://www.typescriptlang.org/docs/handbook/triple-slash-directives.html) - // directives that are not at the top of the file are ignored, so only - // 3 references should be captured instead of 4. - let file_specifier = - ModuleSpecifier::resolve_url_or_path("some/file.ts").unwrap(); - assert_eq!( - references, - vec![ - TsReferenceDesc { - specifier: "dom".to_string(), - kind: TsReferenceKind::Lib, - location: Location { - filename: file_specifier.to_string(), - line: 5, - col: 0, - }, - }, - TsReferenceDesc { - specifier: "./type_reference.d.ts".to_string(), - kind: TsReferenceKind::Types, - location: Location { - filename: file_specifier.to_string(), - line: 6, - col: 0, - }, - }, - TsReferenceDesc { - specifier: "./type_reference/dep.ts".to_string(), - kind: TsReferenceKind::Path, - location: Location { - filename: file_specifier.to_string(), - line: 7, - col: 0, - }, - }, - ] - ); -} diff --git a/cli/module_graph2.rs b/cli/module_graph2.rs index 3fc900373..6ac27906d 100644 --- a/cli/module_graph2.rs +++ b/cli/module_graph2.rs @@ -1,9 +1,9 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. +use crate::ast; use crate::ast::parse; use crate::ast::transpile_module; use crate::ast::BundleHook; -use crate::ast::EmitOptions; use crate::ast::Location; use crate::ast::ParsedModule; use crate::colors; @@ -22,8 +22,7 @@ use crate::specifier_handler::DependencyMap; use crate::specifier_handler::Emit; use crate::specifier_handler::FetchFuture; use crate::specifier_handler::SpecifierHandler; -use crate::tsc2::exec; -use crate::tsc2::Request; +use crate::tsc2; use crate::tsc_config::IgnoredCompilerOptions; use crate::tsc_config::TsConfig; use crate::version; @@ -35,6 +34,7 @@ use deno_core::futures::stream::StreamExt; use deno_core::serde::Serialize; use deno_core::serde::Serializer; use deno_core::serde_json::json; +use deno_core::serde_json::Value; use deno_core::ModuleResolutionError; use deno_core::ModuleSpecifier; use regex::Regex; @@ -121,13 +121,13 @@ impl Error for GraphError {} struct BundleLoader<'a> { cm: Rc<swc_common::SourceMap>, graph: &'a Graph2, - emit_options: &'a EmitOptions, + emit_options: &'a ast::EmitOptions, } impl<'a> BundleLoader<'a> { pub fn new( graph: &'a Graph2, - emit_options: &'a EmitOptions, + emit_options: &'a ast::EmitOptions, cm: Rc<swc_common::SourceMap>, ) -> Self { BundleLoader { @@ -480,7 +480,7 @@ impl Module { } } -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, Default, Eq, PartialEq)] pub struct Stats(pub Vec<(String, u128)>); impl<'de> Deserialize<'de> for Stats { @@ -504,6 +504,23 @@ impl fmt::Display for Stats { } } +/// A structure that provides information about a module graph result. +#[derive(Debug, Default)] +pub struct ResultInfo { + /// A structure which provides diagnostic information (usually from `tsc`) + /// about the code in the module graph. + pub diagnostics: Diagnostics, + /// Optionally ignored compiler options that represent any options that were + /// ignored if there was a user provided configuration. + pub maybe_ignored_options: Option<IgnoredCompilerOptions>, + /// A structure providing key metrics around the operation performed, in + /// milliseconds. + pub stats: Stats, +} + +/// Represents the "default" type library that should be used when type +/// checking the code in the module graph. Note that a user provided config +/// of `"lib"` would override this value. #[derive(Debug, Clone, Eq, PartialEq)] pub enum TypeLib { DenoWindow, @@ -539,7 +556,11 @@ impl Serialize for TypeLib { #[derive(Debug, Default)] pub struct BundleOptions { + /// If `true` then debug logging will be output from the isolate. pub debug: bool, + /// An optional string that points to a user supplied TypeScript configuration + /// file that augments the the default configuration passed to the TypeScript + /// compiler. pub maybe_config_path: Option<String>, } @@ -560,6 +581,35 @@ pub struct CheckOptions { pub reload: bool, } +#[derive(Debug, Eq, PartialEq)] +pub enum BundleType { + /// Return the emitted contents of the program as a single "flattened" ES + /// module. + Esm, + // TODO(@kitsonk) once available in swc + // Iife, + /// Do not bundle the emit, instead returning each of the modules that are + /// part of the program as individual files. + None, +} + +impl Default for BundleType { + fn default() -> Self { + BundleType::None + } +} + +#[derive(Debug, Default)] +pub struct EmitOptions { + /// Indicate the form the result of the emit should take. + pub bundle_type: BundleType, + /// If `true` then debug logging will be output from the isolate. + pub debug: bool, + /// An optional map that contains user supplied TypeScript compiler + /// configuration options that are passed to the TypeScript compiler. + pub maybe_user_config: Option<HashMap<String, Value>>, +} + /// A structure which provides options when transpiling modules. #[derive(Debug, Default)] pub struct TranspileOptions { @@ -647,47 +697,8 @@ impl Graph2 { })); let maybe_ignored_options = ts_config.merge_tsconfig(options.maybe_config_path)?; - let emit_options: EmitOptions = ts_config.into(); - let cm = Rc::new(swc_common::SourceMap::new( - swc_common::FilePathMapping::empty(), - )); - let loader = BundleLoader::new(self, &emit_options, cm.clone()); - let hook = Box::new(BundleHook); - let globals = swc_common::Globals::new(); - let bundler = swc_bundler::Bundler::new( - &globals, - cm.clone(), - loader, - self, - swc_bundler::Config::default(), - hook, - ); - let mut entries = HashMap::new(); - entries.insert( - "bundle".to_string(), - swc_common::FileName::Custom(root_specifier.to_string()), - ); - let output = bundler - .bundle(entries) - .context("Unable to output bundle during Graph2::bundle().")?; - let mut buf = Vec::new(); - { - let mut emitter = swc_ecmascript::codegen::Emitter { - cfg: swc_ecmascript::codegen::Config { minify: false }, - cm: cm.clone(), - comments: None, - wr: Box::new(swc_ecmascript::codegen::text_writer::JsWriter::new( - cm, "\n", &mut buf, None, - )), - }; - - emitter - .emit_module(&output[0].module) - .context("Unable to emit bundle during Graph2::bundle().")?; - } - let s = String::from_utf8(buf) - .context("Emitted bundle is an invalid utf-8 string.")?; + let s = self.emit_bundle(&root_specifier, &ts_config.into())?; let stats = Stats(vec![ ("Files".to_string(), self.modules.len() as u128), ("Total time".to_string(), start.elapsed().as_millis()), @@ -697,11 +708,7 @@ impl Graph2 { } /// Type check the module graph, corresponding to the options provided. - pub fn check( - self, - options: CheckOptions, - ) -> Result<(Stats, Diagnostics, Option<IgnoredCompilerOptions>), AnyError> - { + pub fn check(self, options: CheckOptions) -> Result<ResultInfo, AnyError> { let mut config = TsConfig::new(json!({ "allowJs": true, // TODO(@kitsonk) is this really needed? @@ -745,11 +752,10 @@ impl Graph2 { && (!options.reload || self.roots_dynamic)) { debug!("graph does not need to be checked or emitted."); - return Ok(( - Stats(Vec::new()), - Diagnostics::default(), + return Ok(ResultInfo { maybe_ignored_options, - )); + ..Default::default() + }); } // TODO(@kitsonk) not totally happy with this here, but this is the first @@ -760,26 +766,15 @@ impl Graph2 { info!("{} {}", colors::green("Check"), specifier); } - let root_names: Vec<(ModuleSpecifier, MediaType)> = self - .roots - .iter() - .map(|ms| { - ( - // root modules can be redirects, so before we pass it to tsc we need - // to resolve the redirect - self.resolve_specifier(ms).clone(), - self.get_media_type(ms).unwrap(), - ) - }) - .collect(); + let root_names = self.get_root_names(); let maybe_tsbuildinfo = self.maybe_tsbuildinfo.clone(); let hash_data = vec![config.as_bytes(), version::DENO.as_bytes().to_owned()]; let graph = Rc::new(RefCell::new(self)); - let response = exec( + let response = tsc2::exec( js::compiler_isolate_init(), - Request { + tsc2::Request { config: config.clone(), debug: options.debug, graph: graph.clone(), @@ -837,7 +832,11 @@ impl Graph2 { } graph.flush()?; - Ok((response.stats, response.diagnostics, maybe_ignored_options)) + Ok(ResultInfo { + diagnostics: response.diagnostics, + maybe_ignored_options, + stats: response.stats, + }) } fn contains_module(&self, specifier: &ModuleSpecifier) -> bool { @@ -845,6 +844,165 @@ impl Graph2 { self.modules.contains_key(s) } + /// Emit the module graph in a specific format. This is specifically designed + /// to be an "all-in-one" API for access by the runtime, allowing both + /// emitting single modules as well as bundles, using Deno module resolution + /// or supplied sources. + pub fn emit( + self, + options: EmitOptions, + ) -> Result<(HashMap<String, String>, ResultInfo), AnyError> { + let mut config = TsConfig::new(json!({ + "allowJs": true, + // TODO(@kitsonk) consider enabling this by default + // see: https://github.com/denoland/deno/issues/7732 + "emitDecoratorMetadata": false, + "esModuleInterop": true, + "experimentalDecorators": true, + "isolatedModules": true, + "jsx": "react", + "lib": TypeLib::DenoWindow, + "module": "esnext", + "strict": true, + "target": "esnext", + })); + let opts = match options.bundle_type { + BundleType::Esm => json!({ + "checkJs": false, + "inlineSourceMap": false, + "noEmit": true, + "jsxFactory": "React.createElement", + "jsxFragmentFactory": "React.Fragment", + }), + BundleType::None => json!({ + "outDir": "deno://", + "removeComments": true, + "sourceMap": true, + }), + }; + config.merge(&opts); + let maybe_ignored_options = + if let Some(user_options) = &options.maybe_user_config { + config.merge_user_config(user_options)? + } else { + None + }; + + let root_names = self.get_root_names(); + let hash_data = + vec![config.as_bytes(), version::DENO.as_bytes().to_owned()]; + let graph = Rc::new(RefCell::new(self)); + + let response = tsc2::exec( + js::compiler_isolate_init(), + tsc2::Request { + config: config.clone(), + debug: options.debug, + graph: graph.clone(), + hash_data, + maybe_tsbuildinfo: None, + root_names, + }, + )?; + + let mut emitted_files = HashMap::new(); + match options.bundle_type { + BundleType::Esm => { + assert!( + response.emitted_files.is_empty(), + "No files should have been emitted from tsc." + ); + let graph = graph.borrow(); + assert_eq!( + graph.roots.len(), + 1, + "Only a single root module supported." + ); + let specifier = &graph.roots[0]; + let s = graph.emit_bundle(specifier, &config.into())?; + emitted_files.insert("deno:///bundle.js".to_string(), s); + } + BundleType::None => { + for emitted_file in &response.emitted_files { + assert!( + emitted_file.maybe_specifiers.is_some(), + "Orphaned file emitted." + ); + let specifiers = emitted_file.maybe_specifiers.clone().unwrap(); + assert_eq!( + specifiers.len(), + 1, + "An unexpected number of specifiers associated with emitted file." + ); + let specifier = specifiers[0].clone(); + let extension = match emitted_file.media_type { + MediaType::JavaScript => ".js", + MediaType::SourceMap => ".js.map", + _ => unreachable!(), + }; + let key = format!("{}{}", specifier, extension); + emitted_files.insert(key, emitted_file.data.clone()); + } + } + }; + + Ok(( + emitted_files, + ResultInfo { + diagnostics: response.diagnostics, + maybe_ignored_options, + stats: response.stats, + }, + )) + } + + /// Shared between `bundle()` and `emit()`. + fn emit_bundle( + &self, + specifier: &ModuleSpecifier, + emit_options: &ast::EmitOptions, + ) -> Result<String, AnyError> { + let cm = Rc::new(swc_common::SourceMap::new( + swc_common::FilePathMapping::empty(), + )); + let loader = BundleLoader::new(self, emit_options, cm.clone()); + let hook = Box::new(BundleHook); + let globals = swc_common::Globals::new(); + let bundler = swc_bundler::Bundler::new( + &globals, + cm.clone(), + loader, + self, + swc_bundler::Config::default(), + hook, + ); + let mut entries = HashMap::new(); + entries.insert( + "bundle".to_string(), + swc_common::FileName::Custom(specifier.to_string()), + ); + let output = bundler + .bundle(entries) + .context("Unable to output bundle during Graph2::bundle().")?; + let mut buf = Vec::new(); + { + let mut emitter = swc_ecmascript::codegen::Emitter { + cfg: swc_ecmascript::codegen::Config { minify: false }, + cm: cm.clone(), + comments: None, + wr: Box::new(swc_ecmascript::codegen::text_writer::JsWriter::new( + cm, "\n", &mut buf, None, + )), + }; + + emitter + .emit_module(&output[0].module) + .context("Unable to emit bundle during Graph2::bundle().")?; + } + + String::from_utf8(buf).context("Emitted bundle is an invalid utf-8 string.") + } + /// Update the handler with any modules that are marked as _dirty_ and update /// any build info if present. fn flush(&mut self) -> Result<(), AnyError> { @@ -963,22 +1121,6 @@ impl Graph2 { self.modules.get(s) } - /// Consume graph and return list of all module specifiers - /// contained in the graph. - pub fn get_modules(&self) -> Vec<ModuleSpecifier> { - self.modules.keys().map(|s| s.to_owned()).collect() - } - - /// Get the source for a given module specifier. If the module is not part - /// of the graph, the result will be `None`. - pub fn get_source(&self, specifier: &ModuleSpecifier) -> Option<String> { - if let Some(module) = self.get_module(specifier) { - Some(module.source.clone()) - } else { - None - } - } - fn get_module_mut( &mut self, specifier: &ModuleSpecifier, @@ -993,6 +1135,41 @@ impl Graph2 { self.modules.get_mut(s) } + /// Consume graph and return list of all module specifiers contained in the + /// graph. + pub fn get_modules(&self) -> Vec<ModuleSpecifier> { + self.modules.keys().map(|s| s.to_owned()).collect() + } + + /// Transform `self.roots` into something that works for `tsc`, because `tsc` + /// doesn't like root names without extensions that match its expectations, + /// nor does it have any concept of redirection, so we have to resolve all + /// that upfront before feeding it to `tsc`. + fn get_root_names(&self) -> Vec<(ModuleSpecifier, MediaType)> { + self + .roots + .iter() + .map(|ms| { + ( + // root modules can be redirects, so before we pass it to tsc we need + // to resolve the redirect + self.resolve_specifier(ms).clone(), + self.get_media_type(ms).unwrap(), + ) + }) + .collect() + } + + /// Get the source for a given module specifier. If the module is not part + /// of the graph, the result will be `None`. + pub fn get_source(&self, specifier: &ModuleSpecifier) -> Option<String> { + if let Some(module) = self.get_module(specifier) { + Some(module.source.clone()) + } else { + None + } + } + /// Return a structure which provides information about the module graph and /// the relationship of the modules in the graph. This structure is used to /// provide information for the `info` subcommand. @@ -1209,7 +1386,7 @@ impl Graph2 { let maybe_ignored_options = ts_config.merge_tsconfig(options.maybe_config_path)?; - let emit_options: EmitOptions = ts_config.clone().into(); + let emit_options: ast::EmitOptions = ts_config.clone().into(); let mut emit_count: u128 = 0; let config = ts_config.as_bytes(); @@ -1434,12 +1611,25 @@ impl GraphBuilder2 { pub mod tests { use super::*; + use crate::specifier_handler::MemoryHandler; use deno_core::futures::future; use std::env; use std::fs; use std::path::PathBuf; use std::sync::Mutex; + macro_rules! map ( + { $($key:expr => $value:expr),+ } => { + { + let mut m = ::std::collections::HashMap::new(); + $( + m.insert($key, $value); + )+ + m + } + }; + ); + /// This is a testing mock for `SpecifierHandler` that uses a special file /// system renaming to mock local and remote modules as well as provides /// "spies" for the critical methods for testing purposes. @@ -1465,20 +1655,7 @@ pub mod tests { .replace("://", "_") .replace("/", "-"); let source_path = self.fixtures.join(specifier_text); - let media_type = match source_path.extension().unwrap().to_str().unwrap() - { - "ts" => { - if source_path.to_string_lossy().ends_with(".d.ts") { - MediaType::Dts - } else { - MediaType::TypeScript - } - } - "tsx" => MediaType::TSX, - "js" => MediaType::JavaScript, - "jsx" => MediaType::JSX, - _ => MediaType::Unknown, - }; + let media_type = MediaType::from(&source_path); let source = fs::read_to_string(&source_path)?; let is_remote = specifier.as_url().scheme() != "file"; @@ -1572,6 +1749,24 @@ pub mod tests { (builder.get_graph(), handler) } + async fn setup_memory( + specifier: ModuleSpecifier, + sources: HashMap<&str, &str>, + ) -> Graph2 { + let sources: HashMap<String, String> = sources + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + let handler = Rc::new(RefCell::new(MemoryHandler::new(sources))); + let mut builder = GraphBuilder2::new(handler.clone(), None, None); + builder + .add(&specifier, false) + .await + .expect("module not inserted"); + + builder.get_graph() + } + #[test] fn test_get_version() { let doc_a = "console.log(42);"; @@ -1694,7 +1889,7 @@ pub mod tests { ModuleSpecifier::resolve_url_or_path("file:///tests/main.ts") .expect("could not resolve module"); let (graph, handler) = setup(specifier).await; - let (stats, diagnostics, maybe_ignored_options) = graph + let result_info = graph .check(CheckOptions { debug: false, emit: true, @@ -1703,9 +1898,9 @@ pub mod tests { reload: false, }) .expect("should have checked"); - assert!(maybe_ignored_options.is_none()); - assert_eq!(stats.0.len(), 12); - assert!(diagnostics.is_empty()); + assert!(result_info.maybe_ignored_options.is_none()); + assert_eq!(result_info.stats.0.len(), 12); + assert!(result_info.diagnostics.is_empty()); let h = handler.borrow(); assert_eq!(h.cache_calls.len(), 2); assert_eq!(h.tsbuildinfo_calls.len(), 1); @@ -1717,7 +1912,7 @@ pub mod tests { ModuleSpecifier::resolve_url_or_path("file:///tests/main.ts") .expect("could not resolve module"); let (graph, handler) = setup(specifier).await; - let (stats, diagnostics, maybe_ignored_options) = graph + let result_info = graph .check(CheckOptions { debug: false, emit: false, @@ -1726,9 +1921,9 @@ pub mod tests { reload: false, }) .expect("should have checked"); - assert!(maybe_ignored_options.is_none()); - assert_eq!(stats.0.len(), 12); - assert!(diagnostics.is_empty()); + assert!(result_info.maybe_ignored_options.is_none()); + assert_eq!(result_info.stats.0.len(), 12); + assert!(result_info.diagnostics.is_empty()); let h = handler.borrow(); assert_eq!(h.cache_calls.len(), 0); assert_eq!(h.tsbuildinfo_calls.len(), 1); @@ -1740,7 +1935,7 @@ pub mod tests { ModuleSpecifier::resolve_url_or_path("file:///tests/checkwithconfig.ts") .expect("could not resolve module"); let (graph, handler) = setup(specifier.clone()).await; - let (_, diagnostics, maybe_ignored_options) = graph + let result_info = graph .check(CheckOptions { debug: false, emit: true, @@ -1751,8 +1946,8 @@ pub mod tests { reload: true, }) .expect("should have checked"); - assert!(maybe_ignored_options.is_none()); - assert!(diagnostics.is_empty()); + assert!(result_info.maybe_ignored_options.is_none()); + assert!(result_info.diagnostics.is_empty()); let h = handler.borrow(); assert_eq!(h.version_calls.len(), 2); let ver0 = h.version_calls[0].1.clone(); @@ -1760,7 +1955,7 @@ pub mod tests { // let's do it all over again to ensure that the versions are determinstic let (graph, handler) = setup(specifier).await; - let (_, diagnostics, maybe_ignored_options) = graph + let result_info = graph .check(CheckOptions { debug: false, emit: true, @@ -1771,8 +1966,8 @@ pub mod tests { reload: true, }) .expect("should have checked"); - assert!(maybe_ignored_options.is_none()); - assert!(diagnostics.is_empty()); + assert!(result_info.maybe_ignored_options.is_none()); + assert!(result_info.diagnostics.is_empty()); let h = handler.borrow(); assert_eq!(h.version_calls.len(), 2); assert!(h.version_calls[0].1 == ver0 || h.version_calls[0].1 == ver1); @@ -1780,6 +1975,81 @@ pub mod tests { } #[tokio::test] + async fn test_graph_emit() { + let specifier = + ModuleSpecifier::resolve_url_or_path("file:///a.ts").unwrap(); + let graph = setup_memory( + specifier, + map!( + "/a.ts" => r#" + import * as b from "./b.ts"; + + console.log(b); + "#, + "/b.ts" => r#" + export const b = "b"; + "# + ), + ) + .await; + let (emitted_files, result_info) = graph + .emit(EmitOptions { + bundle_type: BundleType::None, + debug: false, + maybe_user_config: None, + }) + .expect("should have emitted"); + assert!(result_info.diagnostics.is_empty()); + assert!(result_info.maybe_ignored_options.is_none()); + assert_eq!(emitted_files.len(), 4); + let out_a = emitted_files.get("file:///a.ts.js"); + assert!(out_a.is_some()); + let out_a = out_a.unwrap(); + assert!(out_a.starts_with("import * as b from")); + assert!(emitted_files.contains_key("file:///a.ts.js.map")); + let out_b = emitted_files.get("file:///b.ts.js"); + assert!(out_b.is_some()); + let out_b = out_b.unwrap(); + assert!(out_b.starts_with("export const b = \"b\";")); + assert!(emitted_files.contains_key("file:///b.ts.js.map")); + } + + #[tokio::test] + async fn test_graph_emit_bundle() { + let specifier = + ModuleSpecifier::resolve_url_or_path("file:///a.ts").unwrap(); + let graph = setup_memory( + specifier, + map!( + "/a.ts" => r#" + import * as b from "./b.ts"; + + console.log(b); + "#, + "/b.ts" => r#" + export const b = "b"; + "# + ), + ) + .await; + let (emitted_files, result_info) = graph + .emit(EmitOptions { + bundle_type: BundleType::Esm, + debug: false, + maybe_user_config: None, + }) + .expect("should have emitted"); + assert!(result_info.diagnostics.is_empty()); + assert!(result_info.maybe_ignored_options.is_none()); + assert_eq!(emitted_files.len(), 1); + let actual = emitted_files.get("deno:///bundle.js"); + assert!(actual.is_some()); + let actual = actual.unwrap(); + assert!(actual.contains("const b = \"b\";")); + assert!(actual.contains("console.log(b);")); + } + + #[tokio::test] async fn test_graph_info() { let specifier = ModuleSpecifier::resolve_url_or_path("file:///tests/main.ts") diff --git a/cli/module_loader.rs b/cli/module_loader.rs index 39690465c..b19476fe2 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -1,9 +1,9 @@ // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. use crate::import_map::ImportMap; +use crate::module_graph2::TypeLib; use crate::permissions::Permissions; use crate::program_state::ProgramState; -use crate::tsc::TargetLib; use deno_core::error::AnyError; use deno_core::futures::future::FutureExt; use deno_core::futures::Future; @@ -21,7 +21,7 @@ pub struct CliModuleLoader { /// When flags contains a `.import_map_path` option, the content of the /// import map file will be resolved and set. pub import_map: Option<ImportMap>, - pub target_lib: TargetLib, + pub lib: TypeLib, pub is_main: bool, } @@ -29,7 +29,7 @@ impl CliModuleLoader { pub fn new(maybe_import_map: Option<ImportMap>) -> Rc<Self> { Rc::new(CliModuleLoader { import_map: maybe_import_map, - target_lib: TargetLib::Main, + lib: TypeLib::DenoWindow, is_main: true, }) } @@ -37,7 +37,7 @@ impl CliModuleLoader { pub fn new_for_worker() -> Rc<Self> { Rc::new(CliModuleLoader { import_map: None, - target_lib: TargetLib::Worker, + lib: TypeLib::DenoWorker, is_main: false, }) } @@ -117,13 +117,21 @@ impl ModuleLoader for CliModuleLoader { is_dynamic: bool, ) -> Pin<Box<dyn Future<Output = Result<(), AnyError>>>> { let specifier = specifier.clone(); - let target_lib = self.target_lib.clone(); let maybe_import_map = self.import_map.clone(); let state = op_state.borrow(); // The permissions that should be applied to any dynamically imported module let dynamic_permissions = state.borrow::<Permissions>().clone(); let program_state = state.borrow::<Arc<ProgramState>>().clone(); + let lib = if program_state.flags.unstable { + if self.lib == TypeLib::DenoWindow { + TypeLib::UnstableDenoWindow + } else { + TypeLib::UnstableDenoWorker + } + } else { + self.lib.clone() + }; drop(state); // TODO(bartlomieju): `prepare_module_load` should take `load_id` param @@ -131,7 +139,7 @@ impl ModuleLoader for CliModuleLoader { program_state .prepare_module_load( specifier, - target_lib, + lib, dynamic_permissions, is_dynamic, maybe_import_map, diff --git a/cli/op_fetch_asset.rs b/cli/op_fetch_asset.rs index 3ff8b782f..dcb54cde5 100644 --- a/cli/op_fetch_asset.rs +++ b/cli/op_fetch_asset.rs @@ -17,8 +17,6 @@ pub fn get_asset(name: &str) -> Option<&'static str> { }; } match name { - "system_loader.js" => Some(include_str!("system_loader.js")), - "system_loader_es5.js" => Some(include_str!("system_loader_es5.js")), "bootstrap.ts" => Some("console.log(\"hello deno\");"), "typescript.d.ts" => inc!("typescript.d.ts"), "lib.dom.d.ts" => inc!("lib.dom.d.ts"), @@ -85,6 +83,11 @@ pub fn get_asset(name: &str) -> Option<&'static str> { /// Warning: Returns a non-JSON op dispatcher. Must be manually attached to /// JsRuntime. +/// +/// TODO(@kitsonk) this is only used when building the snapshot, and needs to +/// be refactored somewhere else. It is no longer used by `main.rs` and +/// therefore requires the allow unused. +#[allow(unused)] pub fn op_fetch_asset( custom_assets: HashMap<String, PathBuf>, ) -> impl Fn(Rc<RefCell<OpState>>, BufVec) -> Op { diff --git a/cli/ops/runtime_compiler.rs b/cli/ops/runtime_compiler.rs index 5ceb90316..02d093375 100644 --- a/cli/ops/runtime_compiler.rs +++ b/cli/ops/runtime_compiler.rs @@ -3,17 +3,23 @@ use crate::ast; use crate::colors; use crate::media_type::MediaType; +use crate::module_graph2::BundleType; +use crate::module_graph2::EmitOptions; +use crate::module_graph2::GraphBuilder2; use crate::permissions::Permissions; -use crate::tsc::runtime_bundle; -use crate::tsc::runtime_compile; +use crate::specifier_handler::FetchHandler; +use crate::specifier_handler::MemoryHandler; +use crate::specifier_handler::SpecifierHandler; use crate::tsc_config; + use deno_core::error::AnyError; -use deno_core::futures::FutureExt; +use deno_core::error::Context; use deno_core::serde::Serialize; use deno_core::serde_json; use deno_core::serde_json::json; use deno_core::serde_json::Value; use deno_core::BufVec; +use deno_core::ModuleSpecifier; use deno_core::OpState; use serde::Deserialize; use std::cell::RefCell; @@ -39,35 +45,53 @@ async fn op_compile( args: Value, _data: BufVec, ) -> Result<Value, AnyError> { - super::check_unstable2(&state, "Deno.compile"); let args: CompileArgs = serde_json::from_value(args)?; - let cli_state = super::global_state2(&state); - let program_state = cli_state.clone(); - let permissions = { + if args.bundle { + super::check_unstable2(&state, "Deno.bundle"); + } else { + super::check_unstable2(&state, "Deno.compile"); + } + let program_state = super::global_state2(&state); + let runtime_permissions = { let state = state.borrow(); state.borrow::<Permissions>().clone() }; - let fut = if args.bundle { - runtime_bundle( - &program_state, - permissions, - &args.root_name, - &args.sources, - &args.options, - ) - .boxed_local() + let handler: Rc<RefCell<dyn SpecifierHandler>> = + if let Some(sources) = args.sources { + Rc::new(RefCell::new(MemoryHandler::new(sources))) + } else { + Rc::new(RefCell::new(FetchHandler::new( + &program_state, + runtime_permissions, + )?)) + }; + let mut builder = GraphBuilder2::new(handler, None, None); + let specifier = ModuleSpecifier::resolve_url_or_path(&args.root_name) + .context("The root specifier is invalid.")?; + builder.add(&specifier, false).await?; + let graph = builder.get_graph(); + let bundle_type = if args.bundle { + BundleType::Esm } else { - runtime_compile( - &program_state, - permissions, - &args.root_name, - &args.sources, - &args.options, - ) - .boxed_local() + BundleType::None }; - let result = fut.await?; - Ok(result) + let debug = program_state.flags.log_level == Some(log::Level::Debug); + let maybe_user_config: Option<HashMap<String, Value>> = + if let Some(options) = args.options { + Some(serde_json::from_str(&options)?) + } else { + None + }; + let (emitted_files, result_info) = graph.emit(EmitOptions { + bundle_type, + debug, + maybe_user_config, + })?; + + Ok(json!({ + "emittedFiles": emitted_files, + "diagnostics": result_info.diagnostics, + })) } #[derive(Deserialize, Debug)] diff --git a/cli/program_state.rs b/cli/program_state.rs index 9eeb72f23..027bbc792 100644 --- a/cli/program_state.rs +++ b/cli/program_state.rs @@ -15,9 +15,6 @@ use crate::module_graph2::TypeLib; use crate::permissions::Permissions; use crate::source_maps::SourceMapGetter; use crate::specifier_handler::FetchHandler; -use crate::tsc::CompiledModule; -use crate::tsc::TargetLib; -use crate::tsc::TsCompiler; use deno_core::error::generic_error; use deno_core::error::AnyError; @@ -37,6 +34,12 @@ pub fn exit_unstable(api_name: &str) { std::process::exit(70); } +// TODO(@kitsonk) probably can refactor this better with the graph. +pub struct CompiledModule { + pub code: String, + pub name: String, +} + /// This structure represents state of single "deno" program. /// /// It is shared by all created workers (thus V8 isolates). @@ -47,7 +50,6 @@ pub struct ProgramState { pub permissions: Permissions, pub dir: deno_dir::DenoDir, pub file_fetcher: SourceFileFetcher, - pub ts_compiler: TsCompiler, pub lockfile: Option<Arc<Mutex<Lockfile>>>, pub maybe_import_map: Option<ImportMap>, pub maybe_inspector_server: Option<Arc<InspectorServer>>, @@ -70,12 +72,6 @@ impl ProgramState { ca_file.as_deref(), )?; - let ts_compiler = TsCompiler::new( - file_fetcher.clone(), - flags.clone(), - dir.gen_cache.clone(), - )?; - let lockfile = if let Some(filename) = &flags.lock { let lockfile = Lockfile::new(filename.clone(), flags.lock_write)?; Some(Arc::new(Mutex::new(lockfile))) @@ -105,7 +101,6 @@ impl ProgramState { permissions: Permissions::from_flags(&flags), flags, file_fetcher, - ts_compiler, lockfile, maybe_import_map, maybe_inspector_server, @@ -120,7 +115,7 @@ impl ProgramState { pub async fn prepare_module_load( self: &Arc<Self>, specifier: ModuleSpecifier, - target_lib: TargetLib, + lib: TypeLib, runtime_permissions: Permissions, is_dynamic: bool, maybe_import_map: Option<ImportMap>, @@ -129,7 +124,7 @@ impl ProgramState { // Workers are subject to the current runtime permissions. We do the // permission check here early to avoid "wasting" time building a module // graph for a module that cannot be loaded. - if target_lib == TargetLib::Worker { + if lib == TypeLib::DenoWorker || lib == TypeLib::UnstableDenoWorker { runtime_permissions.check_specifier(&specifier)?; } let handler = @@ -153,37 +148,20 @@ impl ProgramState { eprintln!("{}", ignored_options); } } else { - let lib = match target_lib { - TargetLib::Main => { - if self.flags.unstable { - TypeLib::UnstableDenoWindow - } else { - TypeLib::DenoWindow - } - } - TargetLib::Worker => { - if self.flags.unstable { - TypeLib::UnstableDenoWorker - } else { - TypeLib::DenoWorker - } - } - }; - let (stats, diagnostics, maybe_ignored_options) = - graph.check(CheckOptions { - debug, - emit: true, - lib, - maybe_config_path, - reload: self.flags.reload, - })?; + let result_info = graph.check(CheckOptions { + debug, + emit: true, + lib, + maybe_config_path, + reload: self.flags.reload, + })?; - debug!("{}", stats); - if let Some(ignored_options) = maybe_ignored_options { + debug!("{}", result_info.stats); + if let Some(ignored_options) = result_info.maybe_ignored_options { eprintln!("{}", ignored_options); } - if !diagnostics.is_empty() { - return Err(generic_error(diagnostics.to_string())); + if !result_info.diagnostics.is_empty() { + return Err(generic_error(result_info.diagnostics.to_string())); } }; diff --git a/cli/rt/40_compiler_api.js b/cli/rt/40_compiler_api.js index ffe58559e..db12c857d 100644 --- a/cli/rt/40_compiler_api.js +++ b/cli/rt/40_compiler_api.js @@ -52,19 +52,14 @@ sources: !!sources, options, }); + /** @type {{ emittedFiles: Record<string, string>, diagnostics: any[] }} */ const result = await opCompile(payload); - util.assert(result.emitMap); + util.assert(result.emittedFiles); const maybeDiagnostics = result.diagnostics.length === 0 ? undefined : result.diagnostics; - const emitMap = {}; - - for (const [key, emittedSource] of Object.entries(result.emitMap)) { - emitMap[key] = emittedSource.contents; - } - - return [maybeDiagnostics, emitMap]; + return [maybeDiagnostics, result.emittedFiles]; } // TODO(bartlomieju): change return type to interface? @@ -84,12 +79,14 @@ sources: !!sources, options, }); + /** @type {{ emittedFiles: Record<string, string>, diagnostics: any[] }} */ const result = await opCompile(payload); - util.assert(result.output); + let output = result.emittedFiles["deno:///bundle.js"]; + util.assert(output); const maybeDiagnostics = result.diagnostics.length === 0 ? undefined : result.diagnostics; - return [maybeDiagnostics, result.output]; + return [maybeDiagnostics, output]; } window.__bootstrap.compilerApi = { diff --git a/cli/specifier_handler.rs b/cli/specifier_handler.rs index 988cad72b..aeb268544 100644 --- a/cli/specifier_handler.rs +++ b/cli/specifier_handler.rs @@ -8,7 +8,9 @@ use crate::media_type::MediaType; use crate::permissions::Permissions; use crate::program_state::ProgramState; +use deno_core::error::custom_error; use deno_core::error::AnyError; +use deno_core::futures::future; use deno_core::futures::Future; use deno_core::futures::FutureExt; use deno_core::serde_json; @@ -61,7 +63,6 @@ pub struct CachedModule { pub specifier: ModuleSpecifier, } -#[cfg(test)] impl Default for CachedModule { fn default() -> Self { let specifier = ModuleSpecifier::resolve_url("file:///example.js").unwrap(); @@ -422,12 +423,119 @@ impl SpecifierHandler for FetchHandler { } } +pub struct MemoryHandler { + sources: HashMap<String, String>, +} + +impl MemoryHandler { + pub fn new(sources: HashMap<String, String>) -> Self { + Self { sources } + } +} + +impl SpecifierHandler for MemoryHandler { + fn fetch( + &mut self, + specifier: ModuleSpecifier, + _maybe_referrer: Option<Location>, + _is_dynamic: bool, + ) -> FetchFuture { + let mut specifier_text = specifier.to_string(); + if !self.sources.contains_key(&specifier_text) { + specifier_text = specifier_text.replace("file:///", "/"); + if !self.sources.contains_key(&specifier_text) { + // Convert `C:/a/path/file.ts` to `/a/path/file.ts` + specifier_text = specifier_text[3..].to_string() + } + } + let result = if let Some(source) = self.sources.get(&specifier_text) { + let media_type = MediaType::from(&specifier); + let is_remote = specifier.as_url().scheme() != "file"; + + Ok(CachedModule { + source: source.to_string(), + requested_specifier: specifier.clone(), + specifier, + media_type, + is_remote, + ..Default::default() + }) + } else { + Err(custom_error( + "NotFound", + format!("Unable to find specifier in sources: {}", specifier), + )) + }; + + Box::pin(future::ready(result)) + } + + fn get_tsbuildinfo( + &self, + _specifier: &ModuleSpecifier, + ) -> Result<Option<String>, AnyError> { + Ok(None) + } + + fn set_cache( + &mut self, + _specifier: &ModuleSpecifier, + _emit: &Emit, + ) -> Result<(), AnyError> { + Ok(()) + } + + fn set_types( + &mut self, + _specifier: &ModuleSpecifier, + _types: String, + ) -> Result<(), AnyError> { + Ok(()) + } + + fn set_tsbuildinfo( + &mut self, + _specifier: &ModuleSpecifier, + _tsbuildinfo: String, + ) -> Result<(), AnyError> { + Ok(()) + } + + fn set_deps( + &mut self, + _specifier: &ModuleSpecifier, + _dependencies: DependencyMap, + ) -> Result<(), AnyError> { + Ok(()) + } + + fn set_version( + &mut self, + _specifier: &ModuleSpecifier, + _version: String, + ) -> Result<(), AnyError> { + Ok(()) + } +} + #[cfg(test)] pub mod tests { use super::*; use crate::http_cache::HttpCache; use tempfile::TempDir; + macro_rules! map ( + { $($key:expr => $value:expr),+ } => { + { + let mut m = ::std::collections::HashMap::new(); + $( + m.insert($key, $value); + )+ + m + } + }; + ); + fn setup() -> (TempDir, FetchHandler) { let temp_dir = TempDir::new().expect("could not setup"); let deno_dir = DenoDir::new(Some(temp_dir.path().to_path_buf())) @@ -522,4 +630,111 @@ pub mod tests { file_fetcher.fetch(specifier, None, false).await.unwrap(); assert_eq!(cached_module.is_remote, false); } + + #[tokio::test] + async fn test_memory_handler_fetch() { + let a_src = r#" + import * as b from "./b.ts"; + console.log(b); + "#; + let b_src = r#" + export const b = "b"; + "#; + let c_src = r#" + export const c = "c"; + "#; + let d_src = r#" + export const d: string; + "#; + let sources = map!( + "/a.ts" => a_src, + "/b.ts" => b_src, + "https://deno.land/x/c.js" => c_src, + "https://deno.land/x/d.d.ts" => d_src + ); + let sources: HashMap<String, String> = sources + .iter() + .map(|(k, v)| (k.to_string(), v.to_string())) + .collect(); + let mut handler = MemoryHandler::new(sources); + let specifier = + ModuleSpecifier::resolve_url_or_path("file:///a.ts").unwrap(); + let actual: CachedModule = handler + .fetch(specifier.clone(), None, false) + .await + .expect("could not fetch module"); + assert_eq!(actual.source, a_src.to_string()); + assert_eq!(actual.requested_specifier, specifier); + assert_eq!(actual.specifier, specifier); + assert_eq!(actual.media_type, MediaType::TypeScript); + assert_eq!(actual.is_remote, false); + + let specifier = + ModuleSpecifier::resolve_url_or_path("file:///b.ts").unwrap(); + let actual: CachedModule = handler + .fetch(specifier.clone(), None, false) + .await + .expect("could not fetch module"); + assert_eq!(actual.source, b_src.to_string()); + assert_eq!(actual.requested_specifier, specifier); + assert_eq!(actual.specifier, specifier); + assert_eq!(actual.media_type, MediaType::TypeScript); + assert_eq!(actual.is_remote, false); + + let specifier = + ModuleSpecifier::resolve_url_or_path("https://deno.land/x/c.js").unwrap(); + let actual: CachedModule = handler + .fetch(specifier.clone(), None, false) + .await + .expect("could not fetch module"); + assert_eq!(actual.source, c_src.to_string()); + assert_eq!(actual.requested_specifier, specifier); + assert_eq!(actual.specifier, specifier); + assert_eq!(actual.media_type, MediaType::JavaScript); + assert_eq!(actual.is_remote, true); + + let specifier = + ModuleSpecifier::resolve_url_or_path("https://deno.land/x/d.d.ts") + .unwrap(); + let actual: CachedModule = handler + .fetch(specifier.clone(), None, false) + .await + .expect("could not fetch module"); + assert_eq!(actual.source, d_src.to_string()); + assert_eq!(actual.requested_specifier, specifier); + assert_eq!(actual.specifier, specifier); + assert_eq!(actual.media_type, MediaType::Dts); + assert_eq!(actual.is_remote, true); + + let specifier = + ModuleSpecifier::resolve_url_or_path("https://deno.land/x/missing.ts") + .unwrap(); + handler + .fetch(specifier.clone(), None, false) + .await + .expect_err("should have errored"); + + let specifier = ModuleSpecifier::resolve_url_or_path("/a.ts").unwrap(); + let actual: CachedModule = handler + .fetch(specifier.clone(), None, false) + .await + .expect("could not fetch module"); + assert_eq!(actual.source, a_src.to_string()); + assert_eq!(actual.requested_specifier, specifier); + assert_eq!(actual.specifier, specifier); + assert_eq!(actual.media_type, MediaType::TypeScript); + assert_eq!(actual.is_remote, false); + + let specifier = + ModuleSpecifier::resolve_url_or_path("file:///C:/a.ts").unwrap(); + let actual: CachedModule = handler + .fetch(specifier.clone(), None, false) + .await + .expect("could not fetch module"); + assert_eq!(actual.source, a_src.to_string()); + assert_eq!(actual.requested_specifier, specifier); + assert_eq!(actual.specifier, specifier); + assert_eq!(actual.media_type, MediaType::TypeScript); + assert_eq!(actual.is_remote, false); + } } diff --git a/cli/tests/compiler_api_test.ts b/cli/tests/compiler_api_test.ts index a845e58c6..a236a3f3f 100644 --- a/cli/tests/compiler_api_test.ts +++ b/cli/tests/compiler_api_test.ts @@ -14,12 +14,11 @@ Deno.test({ }); assert(diagnostics == null); assert(actual); - assertEquals(Object.keys(actual), [ - "/bar.js.map", - "/bar.js", - "/foo.js.map", - "/foo.js", - ]); + const keys = Object.keys(actual).sort(); + assert(keys[0].endsWith("/bar.ts.js")); + assert(keys[1].endsWith("/bar.ts.js.map")); + assert(keys[2].endsWith("/foo.ts.js")); + assert(keys[3].endsWith("/foo.ts.js.map")); }, }); @@ -29,10 +28,10 @@ Deno.test({ const [diagnostics, actual] = await Deno.compile("./subdir/mod1.ts"); assert(diagnostics == null); assert(actual); - const keys = Object.keys(actual); + const keys = Object.keys(actual).sort(); assertEquals(keys.length, 6); - assert(keys[0].endsWith("print_hello.js.map")); - assert(keys[1].endsWith("print_hello.js")); + assert(keys[0].endsWith("cli/tests/subdir/mod1.ts.js")); + assert(keys[1].endsWith("cli/tests/subdir/mod1.ts.js.map")); }, }); @@ -51,8 +50,11 @@ Deno.test({ ); assert(diagnostics == null); assert(actual); - assertEquals(Object.keys(actual), ["/foo.js"]); - assert(actual["/foo.js"].startsWith("define(")); + const keys = Object.keys(actual); + assertEquals(keys.length, 1); + const key = keys[0]; + assert(key.endsWith("/foo.ts.js")); + assert(actual[key].startsWith("define(")); }, }); @@ -60,9 +62,9 @@ Deno.test({ name: "Deno.compile() - pass lib in compiler options", async fn() { const [diagnostics, actual] = await Deno.compile( - "/foo.ts", + "file:///foo.ts", { - "/foo.ts": `console.log(document.getElementById("foo")); + "file:///foo.ts": `console.log(document.getElementById("foo")); console.log(Deno.args);`, }, { @@ -71,45 +73,37 @@ Deno.test({ ); assert(diagnostics == null); assert(actual); - assertEquals(Object.keys(actual), ["/foo.js.map", "/foo.js"]); - }, -}); - -Deno.test({ - name: "Deno.compile() - pass outDir in compiler options", - async fn() { - const [diagnostics, actual] = await Deno.compile( - "src/foo.ts", - { - "src/foo.ts": "console.log('Hello world')", - }, - { - outDir: "./lib", - }, + assertEquals( + Object.keys(actual).sort(), + ["file:///foo.ts.js", "file:///foo.ts.js.map"], ); - assert(diagnostics == null); - assert(actual); - assertEquals(Object.keys(actual), ["lib/foo.js.map", "lib/foo.js"]); }, }); -Deno.test({ - name: "Deno.compile() - properly handles .d.ts files", - async fn() { - const [diagnostics, actual] = await Deno.compile( - "/foo.ts", - { - "/foo.ts": `console.log(Foo.bar);`, - }, - { - types: ["./subdir/foo_types.d.ts"], - }, - ); - assert(diagnostics == null); - assert(actual); - assertEquals(Object.keys(actual), ["/foo.js.map", "/foo.js"]); - }, -}); +// TODO(@kitsonk) figure the "right way" to restore support for types +// Deno.test({ +// name: "Deno.compile() - properly handles .d.ts files", +// async fn() { +// const [diagnostics, actual] = await Deno.compile( +// "/foo.ts", +// { +// "/foo.ts": `console.log(Foo.bar);`, +// "/foo_types.d.ts": `declare namespace Foo { +// const bar: string; +// }`, +// }, +// { +// types: ["/foo_types.d.ts"], +// }, +// ); +// assert(diagnostics == null); +// assert(actual); +// assertEquals( +// Object.keys(actual).sort(), +// ["file:///foo.ts.js", "file:///file.ts.js.map"], +// ); +// }, +// }); Deno.test({ name: "Deno.transpileOnly()", @@ -150,8 +144,7 @@ Deno.test({ "/bar.ts": `export const bar = "bar";\n`, }); assert(diagnostics == null); - assert(actual.includes(`__instantiate("foo", false)`)); - assert(actual.includes(`__exp["bar"]`)); + assert(actual.includes(`const bar = "bar"`)); }, }); @@ -160,26 +153,7 @@ Deno.test({ async fn() { const [diagnostics, actual] = await Deno.bundle("./subdir/mod1.ts"); assert(diagnostics == null); - assert(actual.includes(`__instantiate("mod1", false)`)); - assert(actual.includes(`__exp["printHello3"]`)); - }, -}); - -Deno.test({ - name: "Deno.bundle() - compiler config effects emit", - async fn() { - const [diagnostics, actual] = await Deno.bundle( - "/foo.ts", - { - "/foo.ts": `// random comment\nexport * from "./bar.ts";\n`, - "/bar.ts": `export const bar = "bar";\n`, - }, - { - removeComments: true, - }, - ); - assert(diagnostics == null); - assert(!actual.includes(`random`)); + assert(actual.length); }, }); @@ -191,22 +165,7 @@ Deno.test({ "/bar.js": `export const bar = "bar";\n`, }); assert(diagnostics == null); - assert(actual.includes(`System.register("bar",`)); - }, -}); - -Deno.test({ - name: "Deno.bundle - pre ES2017 uses ES5 loader", - async fn() { - const [diagnostics, actual] = await Deno.bundle( - "/foo.ts", - { - "/foo.ts": `console.log("hello world!")\n`, - }, - { target: "es2015" }, - ); - assert(diagnostics == null); - assert(actual.includes(`var __awaiter = `)); + assert(actual.includes(`const bar = "bar"`)); }, }); @@ -226,8 +185,8 @@ Deno.test({ name: "Deno.compile() - SWC diagnostics", async fn() { await assertThrowsAsync(async () => { - await Deno.compile("main.js", { - "main.js": ` + await Deno.compile("/main.js", { + "/main.js": ` export class Foo { constructor() { console.log("foo"); diff --git a/cli/tests/compiler_js_error.ts b/cli/tests/compiler_js_error.ts deleted file mode 100644 index 0b981ae3a..000000000 --- a/cli/tests/compiler_js_error.ts +++ /dev/null @@ -1 +0,0 @@ -Deno.compile("main.js", { "main.js": "console.log(foo);" }); diff --git a/cli/tests/compiler_js_error.ts.out b/cli/tests/compiler_js_error.ts.out deleted file mode 100644 index ef6ed7e80..000000000 --- a/cli/tests/compiler_js_error.ts.out +++ /dev/null @@ -1,4 +0,0 @@ -Check [WILDCARD]compiler_js_error.ts -error: Uncaught (in promise) Error: Error in TS compiler: -AssertionError: Unexpected skip of the emit. -[WILDCARD] diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs index 979a2ffad..64ef69ba1 100644 --- a/cli/tests/integration_tests.rs +++ b/cli/tests/integration_tests.rs @@ -3001,12 +3001,6 @@ itest!(deno_doc_import_map { output: "doc/use_import_map.out", }); -itest!(compiler_js_error { - args: "run --unstable compiler_js_error.ts", - output: "compiler_js_error.ts.out", - exit_code: 1, -}); - itest!(import_file_with_colon { args: "run --quiet --reload import_file_with_colon.ts", output: "import_file_with_colon.ts.out", diff --git a/cli/tests/lib_ref.ts b/cli/tests/lib_ref.ts index 0a4ce3fc7..7b7bc4eca 100644 --- a/cli/tests/lib_ref.ts +++ b/cli/tests/lib_ref.ts @@ -1,7 +1,7 @@ const [errors, program] = await Deno.compile( - "main.ts", + "/main.ts", { - "main.ts": + "/main.ts": `/// <reference lib="dom" />\n\ndocument.getElementById("foo");\nDeno.args;`, }, { @@ -11,4 +11,4 @@ const [errors, program] = await Deno.compile( ); console.log(errors); -console.log(Object.keys(program)); +console.log(Object.keys(program).sort()); diff --git a/cli/tests/lib_ref.ts.out b/cli/tests/lib_ref.ts.out index 9f8c62d8a..7aee0cc58 100644 --- a/cli/tests/lib_ref.ts.out +++ b/cli/tests/lib_ref.ts.out @@ -1,2 +1,2 @@ undefined -[ "main.js.map", "main.js" ] +[ "file:///[WILDCARD]main.ts.js", "file:///[WILDCARD]main.ts.js.map" ] diff --git a/cli/tests/lib_runtime_api.ts b/cli/tests/lib_runtime_api.ts index 788288f72..fc00825e9 100644 --- a/cli/tests/lib_runtime_api.ts +++ b/cli/tests/lib_runtime_api.ts @@ -1,7 +1,7 @@ const [errors, program] = await Deno.compile( - "main.ts", + "/main.ts", { - "main.ts": `document.getElementById("foo");`, + "/main.ts": `document.getElementById("foo");`, }, { lib: ["dom", "esnext"], @@ -9,4 +9,4 @@ const [errors, program] = await Deno.compile( ); console.log(errors); -console.log(Object.keys(program)); +console.log(Object.keys(program).sort()); diff --git a/cli/tests/lib_runtime_api.ts.out b/cli/tests/lib_runtime_api.ts.out index 9f8c62d8a..7aee0cc58 100644 --- a/cli/tests/lib_runtime_api.ts.out +++ b/cli/tests/lib_runtime_api.ts.out @@ -1,2 +1,2 @@ undefined -[ "main.js.map", "main.js" ] +[ "file:///[WILDCARD]main.ts.js", "file:///[WILDCARD]main.ts.js.map" ] diff --git a/cli/tsc.rs b/cli/tsc.rs deleted file mode 100644 index 303868d44..000000000 --- a/cli/tsc.rs +++ /dev/null @@ -1,1005 +0,0 @@ -// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. - -use crate::ast::parse; -use crate::ast::Location; -use crate::diagnostics::Diagnostics; -use crate::disk_cache::DiskCache; -use crate::file_fetcher::SourceFile; -use crate::file_fetcher::SourceFileFetcher; -use crate::flags::Flags; -use crate::fs::canonicalize_path; -use crate::js; -use crate::media_type::MediaType; -use crate::module_graph::ModuleGraph; -use crate::module_graph::ModuleGraphLoader; -use crate::permissions::Permissions; -use crate::program_state::ProgramState; -use crate::tsc_config; -use crate::version; -use deno_core::error::generic_error; -use deno_core::error::AnyError; -use deno_core::error::JsError; -use deno_core::json_op_sync; -use deno_core::serde_json; -use deno_core::serde_json::json; -use deno_core::serde_json::Value; -use deno_core::url::Url; -use deno_core::JsRuntime; -use deno_core::ModuleSpecifier; -use deno_core::RuntimeOptions; -use log::debug; -use regex::Regex; -use serde::Deserialize; -use serde::Serialize; -use serde::Serializer; -use sourcemap::SourceMap; -use std::collections::HashMap; -use std::collections::HashSet; -use std::fs; -use std::io; -use std::ops::Deref; -use std::path::PathBuf; -use std::str; -use std::sync::Arc; -use std::sync::Mutex; -use swc_common::comments::Comment; -use swc_common::comments::CommentKind; -use swc_ecmascript::dep_graph; - -pub const AVAILABLE_LIBS: &[&str] = &[ - "deno.ns", - "deno.window", - "deno.worker", - "deno.shared_globals", - "deno.unstable", - "dom", - "dom.iterable", - "es5", - "es6", - "esnext", - "es2020", - "es2020.full", - "es2019", - "es2019.full", - "es2018", - "es2018.full", - "es2017", - "es2017.full", - "es2016", - "es2016.full", - "es2015", - "es2015.collection", - "es2015.core", - "es2015.generator", - "es2015.iterable", - "es2015.promise", - "es2015.proxy", - "es2015.reflect", - "es2015.symbol", - "es2015.symbol.wellknown", - "es2016.array.include", - "es2017.intl", - "es2017.object", - "es2017.sharedmemory", - "es2017.string", - "es2017.typedarrays", - "es2018.asyncgenerator", - "es2018.asynciterable", - "es2018.intl", - "es2018.promise", - "es2018.regexp", - "es2019.array", - "es2019.object", - "es2019.string", - "es2019.symbol", - "es2020.bigint", - "es2020.promise", - "es2020.string", - "es2020.symbol.wellknown", - "esnext.array", - "esnext.asynciterable", - "esnext.bigint", - "esnext.intl", - "esnext.promise", - "esnext.string", - "esnext.symbol", - "esnext.weakref", - "scripthost", - "webworker", - "webworker.importscripts", -]; - -#[derive(Debug, Clone)] -pub struct CompiledModule { - pub code: String, - pub name: String, -} - -lazy_static! { - /// Matches the `@deno-types` pragma. - static ref DENO_TYPES_RE: Regex = - Regex::new(r#"(?i)^\s*@deno-types\s*=\s*(?:["']([^"']+)["']|(\S+))"#) - .unwrap(); - /// Matches a `/// <reference ... />` comment reference. - static ref TRIPLE_SLASH_REFERENCE_RE: Regex = - Regex::new(r"(?i)^/\s*<reference\s.*?/>").unwrap(); - /// Matches a path reference, which adds a dependency to a module - static ref PATH_REFERENCE_RE: Regex = - Regex::new(r#"(?i)\spath\s*=\s*["']([^"']*)["']"#).unwrap(); - /// Matches a types reference, which for JavaScript files indicates the - /// location of types to use when type checking a program that includes it as - /// a dependency. - static ref TYPES_REFERENCE_RE: Regex = - Regex::new(r#"(?i)\stypes\s*=\s*["']([^"']*)["']"#).unwrap(); - /// Matches a lib reference. - static ref LIB_REFERENCE_RE: Regex = - Regex::new(r#"(?i)\slib\s*=\s*["']([^"']*)["']"#).unwrap(); -} - -#[derive(Clone, Eq, PartialEq)] -pub enum TargetLib { - Main, - Worker, -} - -/// Struct which represents the state of the compiler -/// configuration where the first is canonical name for the configuration file, -/// second is a vector of the bytes of the contents of the configuration file, -/// third is bytes of the hash of contents. -#[derive(Clone)] -pub struct CompilerConfig { - pub path: Option<PathBuf>, - pub options: Value, - pub maybe_ignored_options: Option<tsc_config::IgnoredCompilerOptions>, - pub hash: String, - pub compile_js: bool, -} - -impl CompilerConfig { - /// Take the passed flag and resolve the file name relative to the cwd. - pub fn load(maybe_config_path: Option<String>) -> Result<Self, AnyError> { - if maybe_config_path.is_none() { - return Ok(Self { - path: Some(PathBuf::new()), - options: json!({}), - maybe_ignored_options: None, - hash: "".to_string(), - compile_js: false, - }); - } - - let raw_config_path = maybe_config_path.unwrap(); - debug!("Compiler config file: {}", raw_config_path); - let cwd = std::env::current_dir().unwrap(); - let config_file = cwd.join(raw_config_path); - - // Convert the PathBuf to a canonicalized string. This is needed by the - // compiler to properly deal with the configuration. - let config_path = canonicalize_path(&config_file).map_err(|_| { - io::Error::new( - io::ErrorKind::InvalidInput, - format!( - "Could not find the config file: {}", - config_file.to_string_lossy() - ), - ) - })?; - - // Load the contents of the configuration file - debug!("Attempt to load config: {}", config_path.to_str().unwrap()); - let config_bytes = fs::read(&config_file)?; - let config_hash = crate::checksum::gen(&[&config_bytes]); - let config_str = String::from_utf8(config_bytes)?; - - let (options, maybe_ignored_options) = if config_str.is_empty() { - (json!({}), None) - } else { - tsc_config::parse_config(&config_str, &config_path)? - }; - - // If `checkJs` is set to true in `compilerOptions` then we're gonna be compiling - // JavaScript files as well - let compile_js = options["checkJs"].as_bool().unwrap_or(false); - - Ok(Self { - path: Some(config_path), - options, - maybe_ignored_options, - hash: config_hash, - compile_js, - }) - } -} - -/// Information associated with compiled file in cache. -/// version_hash is used to validate versions of the file -/// and could be used to remove stale file in cache. -#[derive(Deserialize, Serialize)] -pub struct CompiledFileMetadata { - pub version_hash: String, -} - -impl CompiledFileMetadata { - pub fn to_json_string(&self) -> Result<String, serde_json::Error> { - serde_json::to_string(self) - } -} - -/// Emit a SHA256 hash based on source code, deno version and TS config. -/// Used to check if a recompilation for source code is needed. -fn source_code_version_hash( - source_code: &[u8], - version: &str, - config_hash: &[u8], -) -> String { - crate::checksum::gen(&[source_code, version.as_bytes(), config_hash]) -} - -pub struct TsCompilerInner { - pub file_fetcher: SourceFileFetcher, - pub flags: Flags, - pub config: CompilerConfig, - pub disk_cache: DiskCache, - /// Set of all URLs that have been compiled. This prevents double - /// compilation of module. - pub compiled: Mutex<HashSet<Url>>, - /// This setting is controlled by `--reload` flag. Unless the flag - /// is provided disk cache is used. - pub use_disk_cache: bool, - /// This setting is controlled by `compilerOptions.checkJs` - pub compile_js: bool, -} - -#[derive(Clone)] -pub struct TsCompiler(Arc<TsCompilerInner>); - -impl Deref for TsCompiler { - type Target = TsCompilerInner; - fn deref(&self) -> &Self::Target { - &self.0 - } -} - -#[derive(Deserialize, Debug)] -#[serde(rename_all = "camelCase")] -struct Stat { - key: String, - value: f64, -} - -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -struct EmittedSource { - filename: String, - contents: String, -} - -// TODO(bartlomieju): possible deduplicate once TS refactor is stabilized -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -#[allow(unused)] -struct RuntimeBundleResponse { - diagnostics: Diagnostics, - output: String, -} - -#[derive(Deserialize)] -#[serde(rename_all = "camelCase")] -struct RuntimeCompileResponse { - diagnostics: Diagnostics, - emit_map: HashMap<String, EmittedSource>, -} - -impl TsCompiler { - pub fn new( - file_fetcher: SourceFileFetcher, - flags: Flags, - disk_cache: DiskCache, - ) -> Result<Self, AnyError> { - let config = CompilerConfig::load(flags.config_path.clone())?; - let use_disk_cache = !flags.reload; - - Ok(TsCompiler(Arc::new(TsCompilerInner { - file_fetcher, - flags, - disk_cache, - compile_js: config.compile_js, - config, - compiled: Mutex::new(HashSet::new()), - use_disk_cache, - }))) - } - - /// Mark given module URL as compiled to avoid multiple compilations of same - /// module in single run. - fn mark_compiled(&self, url: &Url) { - let mut c = self.compiled.lock().unwrap(); - c.insert(url.clone()); - } - - fn cache_emitted_files( - &self, - emit_map: HashMap<String, EmittedSource>, - ) -> std::io::Result<()> { - for (emitted_name, source) in emit_map.iter() { - let specifier = ModuleSpecifier::resolve_url(&source.filename) - .expect("Should be a valid module specifier"); - - let source_file = self - .file_fetcher - .fetch_cached_source_file(&specifier, Permissions::allow_all()) - .expect("Source file not found"); - - // NOTE: JavaScript files are only cached to disk if `checkJs` - // option in on - if source_file.media_type == MediaType::JavaScript && !self.compile_js { - continue; - } - - if emitted_name.ends_with(".map") { - self.cache_source_map(&specifier, &source.contents)?; - } else if emitted_name.ends_with(".js") { - self.cache_compiled_file(&specifier, source_file, &source.contents)?; - } else { - panic!("Trying to cache unknown file type {}", emitted_name); - } - } - - Ok(()) - } - - /// Save compiled JS file for given TS module to on-disk cache. - /// - /// Along compiled file a special metadata file is saved as well containing - /// hash that can be validated to avoid unnecessary recompilation. - fn cache_compiled_file( - &self, - module_specifier: &ModuleSpecifier, - source_file: SourceFile, - contents: &str, - ) -> std::io::Result<()> { - let js_key = self - .disk_cache - .get_cache_filename_with_extension(module_specifier.as_url(), "js"); - self.disk_cache.set(&js_key, contents.as_bytes())?; - self.mark_compiled(module_specifier.as_url()); - - let version_hash = source_code_version_hash( - &source_file.source_code.as_bytes(), - version::DENO, - &self.config.hash.as_bytes(), - ); - - let compiled_file_metadata = CompiledFileMetadata { version_hash }; - let meta_key = self - .disk_cache - .get_cache_filename_with_extension(module_specifier.as_url(), "meta"); - self.disk_cache.set( - &meta_key, - compiled_file_metadata.to_json_string()?.as_bytes(), - ) - } - - /// Save source map file for given TS module to on-disk cache. - fn cache_source_map( - &self, - module_specifier: &ModuleSpecifier, - contents: &str, - ) -> std::io::Result<()> { - let js_key = self - .disk_cache - .get_cache_filename_with_extension(module_specifier.as_url(), "js"); - let js_path = self.disk_cache.location.join(js_key); - let js_file_url = - Url::from_file_path(js_path).expect("Bad file URL for file"); - - let source_map_key = self - .disk_cache - .get_cache_filename_with_extension(module_specifier.as_url(), "js.map"); - - let mut sm = SourceMap::from_slice(contents.as_bytes()) - .expect("Invalid source map content"); - sm.set_file(Some(&js_file_url.to_string())); - sm.set_source(0, &module_specifier.to_string()); - - let mut output: Vec<u8> = vec![]; - sm.to_writer(&mut output) - .expect("Failed to write source map"); - - self.disk_cache.set(&source_map_key, &output) - } -} - -#[derive(Debug, Deserialize)] -struct CreateHashArgs { - data: String, -} - -fn execute_in_tsc( - program_state: Arc<ProgramState>, - req: String, -) -> Result<String, AnyError> { - let mut js_runtime = JsRuntime::new(RuntimeOptions { - startup_snapshot: Some(js::compiler_isolate_init()), - ..Default::default() - }); - - let debug_flag = program_state - .flags - .log_level - .map_or(false, |l| l == log::Level::Debug); - let response = Arc::new(Mutex::new(None)); - - { - js_runtime.register_op( - "op_fetch_asset", - crate::op_fetch_asset::op_fetch_asset(HashMap::default()), - ); - let res = response.clone(); - js_runtime.register_op( - "op_compiler_respond", - json_op_sync(move |_state, args, _bufs| { - let mut response_slot = res.lock().unwrap(); - let replaced_value = response_slot.replace(args.to_string()); - assert!( - replaced_value.is_none(), - "op_compiler_respond found unexpected existing compiler output", - ); - Ok(json!({})) - }), - ); - js_runtime.register_op( - "op_create_hash", - json_op_sync(move |_s, args, _bufs| { - let v: CreateHashArgs = serde_json::from_value(args)?; - let hash = crate::checksum::gen(&[v.data.as_bytes()]); - Ok(json!({ "hash": hash })) - }), - ); - } - - let bootstrap_script = format!( - "globalThis.startup({{ debugFlag: {}, legacy: true }})", - debug_flag - ); - js_runtime.execute("<compiler>", &bootstrap_script)?; - - let script = format!("globalThis.tsCompilerOnMessage({{ data: {} }});", req); - js_runtime.execute("<compiler>", &script)?; - - let maybe_response = response.lock().unwrap().take(); - assert!( - maybe_response.is_some(), - "Unexpected missing response from TS compiler" - ); - - Ok(maybe_response.unwrap()) -} - -async fn create_runtime_module_graph( - program_state: &Arc<ProgramState>, - permissions: Permissions, - root_name: &str, - sources: &Option<HashMap<String, String>>, - type_files: Vec<String>, -) -> Result<(Vec<String>, ModuleGraph), AnyError> { - let mut root_names = vec![]; - let mut module_graph_loader = ModuleGraphLoader::new( - program_state.file_fetcher.clone(), - None, - permissions, - false, - false, - ); - - if let Some(s_map) = sources { - root_names.push(root_name.to_string()); - module_graph_loader.build_local_graph(root_name, s_map)?; - } else { - let module_specifier = - ModuleSpecifier::resolve_import(root_name, "<unknown>")?; - root_names.push(module_specifier.to_string()); - module_graph_loader - .add_to_graph(&module_specifier, None) - .await?; - } - - // download all additional files from TSconfig and add them to root_names - for type_file in type_files { - let type_specifier = ModuleSpecifier::resolve_url_or_path(&type_file)?; - module_graph_loader - .add_to_graph(&type_specifier, None) - .await?; - root_names.push(type_specifier.to_string()) - } - - Ok((root_names, module_graph_loader.get_graph())) -} - -fn extract_js_error(error: AnyError) -> AnyError { - match error.downcast::<JsError>() { - Ok(js_error) => { - let msg = format!("Error in TS compiler:\n{}", js_error); - generic_error(msg) - } - Err(error) => error, - } -} - -/// This function is used by `Deno.compile()` API. -pub async fn runtime_compile( - program_state: &Arc<ProgramState>, - permissions: Permissions, - root_name: &str, - sources: &Option<HashMap<String, String>>, - maybe_options: &Option<String>, -) -> Result<Value, AnyError> { - let mut user_options = if let Some(options) = maybe_options { - tsc_config::parse_raw_config(options)? - } else { - json!({}) - }; - - // Intentionally calling "take()" to replace value with `null` - otherwise TSC will try to load that file - // using `fileExists` API - let type_files = if let Some(types) = user_options["types"].take().as_array() - { - types - .iter() - .map(|type_value| type_value.as_str().unwrap_or("").to_string()) - .filter(|type_str| !type_str.is_empty()) - .collect() - } else { - vec![] - }; - - let unstable = program_state.flags.unstable; - - let mut lib = vec![]; - if let Some(user_libs) = user_options["lib"].take().as_array() { - let libs = user_libs - .iter() - .map(|type_value| type_value.as_str().unwrap_or("").to_string()) - .filter(|type_str| !type_str.is_empty()) - .collect::<Vec<String>>(); - lib.extend(libs); - } else { - lib.push("deno.window".to_string()); - } - - if unstable { - lib.push("deno.unstable".to_string()); - } - - let mut compiler_options = json!({ - "allowJs": false, - "allowNonTsExtensions": true, - "checkJs": false, - "esModuleInterop": true, - "isolatedModules": true, - "jsx": "react", - "module": "esnext", - "sourceMap": true, - "strict": true, - "removeComments": true, - "target": "esnext", - }); - - tsc_config::json_merge(&mut compiler_options, &user_options); - tsc_config::json_merge(&mut compiler_options, &json!({ "lib": lib })); - - let (root_names, module_graph) = create_runtime_module_graph( - &program_state, - permissions.clone(), - root_name, - sources, - type_files, - ) - .await?; - let module_graph_json = - serde_json::to_value(module_graph).expect("Failed to serialize data"); - - let req_msg = json!({ - "type": CompilerRequestType::RuntimeCompile, - "target": "runtime", - "rootNames": root_names, - "sourceFileMap": module_graph_json, - "compilerOptions": compiler_options, - }) - .to_string(); - - let compiler = program_state.ts_compiler.clone(); - - let json_str = - execute_in_tsc(program_state.clone(), req_msg).map_err(extract_js_error)?; - let response: RuntimeCompileResponse = serde_json::from_str(&json_str)?; - - if response.diagnostics.is_empty() && sources.is_none() { - compiler.cache_emitted_files(response.emit_map)?; - } - - // We're returning `Ok()` instead of `Err()` because it's not runtime - // error if there were diagnostics produced; we want to let user handle - // diagnostics in the runtime. - Ok(serde_json::from_str::<Value>(&json_str).unwrap()) -} - -/// This function is used by `Deno.bundle()` API. -pub async fn runtime_bundle( - program_state: &Arc<ProgramState>, - permissions: Permissions, - root_name: &str, - sources: &Option<HashMap<String, String>>, - maybe_options: &Option<String>, -) -> Result<Value, AnyError> { - let mut user_options = if let Some(options) = maybe_options { - tsc_config::parse_raw_config(options)? - } else { - json!({}) - }; - - // Intentionally calling "take()" to replace value with `null` - otherwise TSC will try to load that file - // using `fileExists` API - let type_files = if let Some(types) = user_options["types"].take().as_array() - { - types - .iter() - .map(|type_value| type_value.as_str().unwrap_or("").to_string()) - .filter(|type_str| !type_str.is_empty()) - .collect() - } else { - vec![] - }; - - let (root_names, module_graph) = create_runtime_module_graph( - &program_state, - permissions.clone(), - root_name, - sources, - type_files, - ) - .await?; - let module_graph_json = - serde_json::to_value(module_graph).expect("Failed to serialize data"); - - let unstable = program_state.flags.unstable; - - let mut lib = vec![]; - if let Some(user_libs) = user_options["lib"].take().as_array() { - let libs = user_libs - .iter() - .map(|type_value| type_value.as_str().unwrap_or("").to_string()) - .filter(|type_str| !type_str.is_empty()) - .collect::<Vec<String>>(); - lib.extend(libs); - } else { - lib.push("deno.window".to_string()); - } - - if unstable { - lib.push("deno.unstable".to_string()); - } - - let mut compiler_options = json!({ - "allowJs": false, - "allowNonTsExtensions": true, - "checkJs": false, - "esModuleInterop": true, - "jsx": "react", - "module": "esnext", - "outDir": null, - "sourceMap": true, - "strict": true, - "removeComments": true, - "target": "esnext", - }); - - let bundler_options = json!({ - "allowJs": true, - "inlineSourceMap": false, - "module": "system", - "outDir": null, - "outFile": "deno:///bundle.js", - // disabled until we have effective way to modify source maps - "sourceMap": false, - }); - - tsc_config::json_merge(&mut compiler_options, &user_options); - tsc_config::json_merge(&mut compiler_options, &json!({ "lib": lib })); - tsc_config::json_merge(&mut compiler_options, &bundler_options); - - let req_msg = json!({ - "type": CompilerRequestType::RuntimeBundle, - "target": "runtime", - "rootNames": root_names, - "sourceFileMap": module_graph_json, - "compilerOptions": compiler_options, - }) - .to_string(); - - let json_str = - execute_in_tsc(program_state.clone(), req_msg).map_err(extract_js_error)?; - let _response: RuntimeBundleResponse = serde_json::from_str(&json_str)?; - // We're returning `Ok()` instead of `Err()` because it's not runtime - // error if there were diagnostics produced; we want to let user handle - // diagnostics in the runtime. - Ok(serde_json::from_str::<Value>(&json_str).unwrap()) -} - -#[derive(Clone, Debug, PartialEq)] -pub struct ImportDesc { - pub specifier: String, - pub deno_types: Option<String>, - pub location: Location, -} - -#[derive(Clone, Debug, PartialEq)] -pub enum TsReferenceKind { - Lib, - Types, - Path, -} - -#[derive(Clone, Debug, PartialEq)] -pub struct TsReferenceDesc { - pub kind: TsReferenceKind, - pub specifier: String, - pub location: Location, -} - -// TODO(bartlomieju): handle imports in ambient contexts/TS modules -/// This function is a port of `ts.preProcessFile()` -/// -/// Additionally it captures `@deno-types` references directly -/// preceeding `import .. from` and `export .. from` statements. -pub fn pre_process_file( - file_name: &str, - media_type: MediaType, - source_code: &str, - analyze_dynamic_imports: bool, -) -> Result<(Vec<ImportDesc>, Vec<TsReferenceDesc>), AnyError> { - let specifier = ModuleSpecifier::resolve_url_or_path(file_name)?; - let module = parse(specifier.as_str(), source_code, &media_type)?; - - let dependency_descriptors = module.analyze_dependencies(); - - // for each import check if there's relevant @deno-types directive - let imports = dependency_descriptors - .iter() - .filter(|desc| desc.kind != dep_graph::DependencyKind::Require) - .filter(|desc| { - if analyze_dynamic_imports { - return true; - } - !desc.is_dynamic - }) - .map(|desc| { - let deno_types = get_deno_types(&desc.leading_comments); - ImportDesc { - specifier: desc.specifier.to_string(), - deno_types, - location: Location { - filename: file_name.to_string(), - col: desc.col, - line: desc.line, - }, - } - }) - .collect(); - - // analyze comment from beginning of the file and find TS directives - let comments = module.get_leading_comments(); - - let mut references = vec![]; - for comment in comments { - if comment.kind != CommentKind::Line { - continue; - } - - let text = comment.text.to_string(); - if let Some((kind, specifier)) = parse_ts_reference(text.trim()) { - let location = module.get_location(&comment.span); - references.push(TsReferenceDesc { - kind, - specifier, - location, - }); - } - } - Ok((imports, references)) -} - -fn get_deno_types(comments: &[Comment]) -> Option<String> { - if comments.is_empty() { - return None; - } - - // @deno-types must directly prepend import statement - hence - // checking last comment for span - let last = comments.last().unwrap(); - let comment = last.text.trim_start(); - parse_deno_types(&comment) -} - -fn parse_ts_reference(comment: &str) -> Option<(TsReferenceKind, String)> { - if !TRIPLE_SLASH_REFERENCE_RE.is_match(comment) { - return None; - } - - let (kind, specifier) = - if let Some(capture_groups) = PATH_REFERENCE_RE.captures(comment) { - (TsReferenceKind::Path, capture_groups.get(1).unwrap()) - } else if let Some(capture_groups) = TYPES_REFERENCE_RE.captures(comment) { - (TsReferenceKind::Types, capture_groups.get(1).unwrap()) - } else if let Some(capture_groups) = LIB_REFERENCE_RE.captures(comment) { - (TsReferenceKind::Lib, capture_groups.get(1).unwrap()) - } else { - return None; - }; - - Some((kind, specifier.as_str().to_string())) -} - -fn parse_deno_types(comment: &str) -> Option<String> { - if let Some(capture_groups) = DENO_TYPES_RE.captures(comment) { - if let Some(specifier) = capture_groups.get(1) { - return Some(specifier.as_str().to_string()); - } - if let Some(specifier) = capture_groups.get(2) { - return Some(specifier.as_str().to_string()); - } - } - - None -} - -// Warning! The values in this enum are duplicated in js/compiler.ts -// Update carefully! -#[repr(i32)] -#[derive(Clone, Copy, PartialEq, Debug)] -pub enum CompilerRequestType { - RuntimeCompile = 2, - RuntimeBundle = 3, -} - -impl Serialize for CompilerRequestType { - fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error> - where - S: Serializer, - { - let value: i32 = match self { - CompilerRequestType::RuntimeCompile => 2 as i32, - CompilerRequestType::RuntimeBundle => 3 as i32, - }; - Serialize::serialize(&value, serializer) - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::fs as deno_fs; - use tempfile::TempDir; - - #[test] - fn test_parse_deno_types() { - assert_eq!( - parse_deno_types("@deno-types=./a/b/c.d.ts"), - Some("./a/b/c.d.ts".to_string()) - ); - assert_eq!( - parse_deno_types("@deno-types=\"./a/b/c.d.ts\""), - Some("./a/b/c.d.ts".to_string()) - ); - assert_eq!( - parse_deno_types("@deno-types = https://dneo.land/x/some/package/a.d.ts"), - Some("https://dneo.land/x/some/package/a.d.ts".to_string()) - ); - assert_eq!( - parse_deno_types("@deno-types = ./a/b/c.d.ts"), - Some("./a/b/c.d.ts".to_string()) - ); - assert!(parse_deno_types("asdf").is_none()); - assert!(parse_deno_types("// deno-types = fooo").is_none()); - assert_eq!( - parse_deno_types("@deno-types=./a/b/c.d.ts some comment"), - Some("./a/b/c.d.ts".to_string()) - ); - assert_eq!( - parse_deno_types( - "@deno-types=./a/b/c.d.ts // some comment after slashes" - ), - Some("./a/b/c.d.ts".to_string()) - ); - assert_eq!( - parse_deno_types(r#"@deno-types="https://deno.land/x/foo/index.d.ts";"#), - Some("https://deno.land/x/foo/index.d.ts".to_string()) - ); - } - - #[test] - fn test_parse_ts_reference() { - assert_eq!( - parse_ts_reference(r#"/ <reference lib="deno.shared_globals" />"#), - Some((TsReferenceKind::Lib, "deno.shared_globals".to_string())) - ); - assert_eq!( - parse_ts_reference(r#"/ <reference path="./type/reference/dep.ts" />"#), - Some((TsReferenceKind::Path, "./type/reference/dep.ts".to_string())) - ); - assert_eq!( - parse_ts_reference(r#"/ <reference types="./type/reference.d.ts" />"#), - Some((TsReferenceKind::Types, "./type/reference.d.ts".to_string())) - ); - assert!(parse_ts_reference("asdf").is_none()); - assert!( - parse_ts_reference(r#"/ <reference unknown="unknown" />"#).is_none() - ); - assert!(parse_ts_reference(r#"/ <asset path="./styles.css" />"#).is_none()); - } - - #[test] - fn test_source_code_version_hash() { - assert_eq!( - "0185b42de0686b4c93c314daaa8dee159f768a9e9a336c2a5e3d5b8ca6c4208c", - source_code_version_hash(b"1+2", "0.4.0", b"{}") - ); - // Different source_code should result in different hash. - assert_eq!( - "e58631f1b6b6ce2b300b133ec2ad16a8a5ba6b7ecf812a8c06e59056638571ac", - source_code_version_hash(b"1", "0.4.0", b"{}") - ); - // Different version should result in different hash. - assert_eq!( - "307e6200347a88dbbada453102deb91c12939c65494e987d2d8978f6609b5633", - source_code_version_hash(b"1", "0.1.0", b"{}") - ); - // Different config should result in different hash. - assert_eq!( - "195eaf104a591d1d7f69fc169c60a41959c2b7a21373cd23a8f675f877ec385f", - source_code_version_hash(b"1", "0.4.0", b"{\"compilerOptions\": {}}") - ); - } - - #[test] - fn test_compile_js() { - let temp_dir = TempDir::new().expect("tempdir fail"); - let temp_dir_path = temp_dir.path(); - - let test_cases = vec![ - // valid JSON - (r#"{ "compilerOptions": { "checkJs": true } } "#, true), - // JSON with comment - ( - r#"{ - "compilerOptions": { - // force .js file compilation by Deno - "checkJs": true - } - }"#, - true, - ), - // without content - ("", false), - ]; - - let path = temp_dir_path.join("tsconfig.json"); - let path_str = path.to_str().unwrap().to_string(); - - for (json_str, expected) in test_cases { - deno_fs::write_file(&path, json_str.as_bytes(), 0o666).unwrap(); - let config = CompilerConfig::load(Some(path_str.clone())).unwrap(); - assert_eq!(config.compile_js, expected); - } - } - - #[test] - fn test_compiler_config_load() { - let temp_dir = TempDir::new().expect("tempdir fail"); - let temp_dir_path = temp_dir.path(); - let path = temp_dir_path.join("doesnotexist.json"); - let path_str = path.to_str().unwrap().to_string(); - let res = CompilerConfig::load(Some(path_str)); - assert!(res.is_err()); - } -} diff --git a/cli/tsc/99_main_compiler.js b/cli/tsc/99_main_compiler.js index b286f596d..e2a481d0f 100644 --- a/cli/tsc/99_main_compiler.js +++ b/cli/tsc/99_main_compiler.js @@ -118,9 +118,6 @@ delete Object.prototype.__proto__; return core.decode(sourceCodeBytes); } - // Constants used by `normalizeString` and `resolvePath` - const CHAR_DOT = 46; /* . */ - const CHAR_FORWARD_SLASH = 47; /* / */ // Using incremental compile APIs requires that all // paths must be either relative or absolute. Since // analysis in Rust operates on fully resolved URLs, @@ -218,18 +215,6 @@ delete Object.prototype.__proto__; */ const RESOLVED_SPECIFIER_CACHE = new Map(); - function parseCompilerOptions(compilerOptions) { - const { options, errors } = ts.convertCompilerOptionsFromJson( - compilerOptions, - "", - "tsconfig.json", - ); - return { - options, - diagnostics: errors.length ? errors : undefined, - }; - } - class SourceFile { constructor(json) { this.processed = false; @@ -541,95 +526,6 @@ delete Object.prototype.__proto__; host, }); - // This function is called only during snapshotting process - const SYSTEM_LOADER = getAsset("system_loader.js"); - const SYSTEM_LOADER_ES5 = getAsset("system_loader_es5.js"); - - function buildLocalSourceFileCache(sourceFileMap) { - for (const entry of Object.values(sourceFileMap)) { - assert(entry.sourceCode.length > 0); - SourceFile.addToCache({ - url: entry.url, - filename: entry.url, - mediaType: entry.mediaType, - sourceCode: entry.sourceCode, - versionHash: entry.versionHash, - }); - - for (const importDesc of entry.imports) { - let mappedUrl = importDesc.resolvedSpecifier; - const importedFile = sourceFileMap[importDesc.resolvedSpecifier]; - assert(importedFile); - const isJsOrJsx = importedFile.mediaType === MediaType.JavaScript || - importedFile.mediaType === MediaType.JSX; - // If JS or JSX perform substitution for types if available - if (isJsOrJsx) { - // @deno-types has highest precedence, followed by - // X-TypeScript-Types header - if (importDesc.resolvedTypeDirective) { - mappedUrl = importDesc.resolvedTypeDirective; - } else if (importedFile.typeHeaders.length > 0) { - const typeHeaders = importedFile.typeHeaders[0]; - mappedUrl = typeHeaders.resolvedSpecifier; - } else if (importedFile.typesDirectives.length > 0) { - const typeDirective = importedFile.typesDirectives[0]; - mappedUrl = typeDirective.resolvedSpecifier; - } - } - - mappedUrl = mappedUrl.replace("memory://", ""); - SourceFile.cacheResolvedUrl(mappedUrl, importDesc.specifier, entry.url); - } - for (const fileRef of entry.referencedFiles) { - SourceFile.cacheResolvedUrl( - fileRef.resolvedSpecifier.replace("memory://", ""), - fileRef.specifier, - entry.url, - ); - } - for (const fileRef of entry.libDirectives) { - SourceFile.cacheResolvedUrl( - fileRef.resolvedSpecifier.replace("memory://", ""), - fileRef.specifier, - entry.url, - ); - } - } - } - - // Warning! The values in this enum are duplicated in `cli/msg.rs` - // Update carefully! - const CompilerRequestType = { - RuntimeCompile: 2, - RuntimeBundle: 3, - }; - - function createBundleWriteFile(state) { - return function writeFile(_fileName, data, sourceFiles) { - assert(sourceFiles != null); - assert(state.options); - // we only support single root names for bundles - assert(state.rootNames.length === 1); - state.bundleOutput = buildBundle( - state.rootNames[0], - data, - sourceFiles, - state.options.target ?? ts.ScriptTarget.ESNext, - ); - }; - } - - function createRuntimeCompileWriteFile(state) { - return function writeFile(fileName, data, sourceFiles) { - assert(sourceFiles); - assert(sourceFiles.length === 1); - state.emitMap[fileName] = { - filename: sourceFiles[0].fileName, - contents: data, - }; - }; - } - const IGNORED_DIAGNOSTICS = [ // TS2306: File 'file:///Users/rld/src/deno/cli/tests/subdir/amd_like.js' is // not a module. @@ -674,7 +570,6 @@ delete Object.prototype.__proto__; function performanceStart() { stats.length = 0; - // TODO(kitsonk) replace with performance.mark() when landed statsStart = new Date(); ts.performance.enable(); } @@ -716,317 +611,6 @@ delete Object.prototype.__proto__; return stats; } - function normalizeString(path) { - let res = ""; - let lastSegmentLength = 0; - let lastSlash = -1; - let dots = 0; - let code; - for (let i = 0, len = path.length; i <= len; ++i) { - if (i < len) code = path.charCodeAt(i); - else if (code === CHAR_FORWARD_SLASH) break; - else code = CHAR_FORWARD_SLASH; - - if (code === CHAR_FORWARD_SLASH) { - if (lastSlash === i - 1 || dots === 1) { - // NOOP - } else if (lastSlash !== i - 1 && dots === 2) { - if ( - res.length < 2 || - lastSegmentLength !== 2 || - res.charCodeAt(res.length - 1) !== CHAR_DOT || - res.charCodeAt(res.length - 2) !== CHAR_DOT - ) { - if (res.length > 2) { - const lastSlashIndex = res.lastIndexOf("/"); - if (lastSlashIndex === -1) { - res = ""; - lastSegmentLength = 0; - } else { - res = res.slice(0, lastSlashIndex); - lastSegmentLength = res.length - 1 - res.lastIndexOf("/"); - } - lastSlash = i; - dots = 0; - continue; - } else if (res.length === 2 || res.length === 1) { - res = ""; - lastSegmentLength = 0; - lastSlash = i; - dots = 0; - continue; - } - } - } else { - if (res.length > 0) res += "/" + path.slice(lastSlash + 1, i); - else res = path.slice(lastSlash + 1, i); - lastSegmentLength = i - lastSlash - 1; - } - lastSlash = i; - dots = 0; - } else if (code === CHAR_DOT && dots !== -1) { - ++dots; - } else { - dots = -1; - } - } - return res; - } - - function commonPath(paths, sep = "/") { - const [first = "", ...remaining] = paths; - if (first === "" || remaining.length === 0) { - return first.substring(0, first.lastIndexOf(sep) + 1); - } - const parts = first.split(sep); - - let endOfPrefix = parts.length; - for (const path of remaining) { - const compare = path.split(sep); - for (let i = 0; i < endOfPrefix; i++) { - if (compare[i] !== parts[i]) { - endOfPrefix = i; - } - } - - if (endOfPrefix === 0) { - return ""; - } - } - const prefix = parts.slice(0, endOfPrefix).join(sep); - return prefix.endsWith(sep) ? prefix : `${prefix}${sep}`; - } - - let rootExports; - - function normalizeUrl(rootName) { - const match = /^(\S+:\/{2,3})(.+)$/.exec(rootName); - if (match) { - const [, protocol, path] = match; - return `${protocol}${normalizeString(path)}`; - } else { - return rootName; - } - } - - function buildBundle(rootName, data, sourceFiles, target) { - // when outputting to AMD and a single outfile, TypeScript makes up the module - // specifiers which are used to define the modules, and doesn't expose them - // publicly, so we have to try to replicate - const sources = sourceFiles.map((sf) => sf.fileName); - const sharedPath = commonPath(sources); - rootName = normalizeUrl(rootName) - .replace(sharedPath, "") - .replace(/\.\w+$/i, ""); - // If one of the modules requires support for top-level-await, TypeScript will - // emit the execute function as an async function. When this is the case we - // need to bubble up the TLA to the instantiation, otherwise we instantiate - // synchronously. - const hasTla = data.match(/execute:\sasync\sfunction\s/); - let instantiate; - if (rootExports && rootExports.length) { - instantiate = hasTla - ? `const __exp = await __instantiate("${rootName}", true);\n` - : `const __exp = __instantiate("${rootName}", false);\n`; - for (const rootExport of rootExports) { - if (rootExport === "default") { - instantiate += `export default __exp["${rootExport}"];\n`; - } else { - instantiate += - `export const ${rootExport} = __exp["${rootExport}"];\n`; - } - } - } else { - instantiate = hasTla - ? `await __instantiate("${rootName}", true);\n` - : `__instantiate("${rootName}", false);\n`; - } - const es5Bundle = target === ts.ScriptTarget.ES3 || - target === ts.ScriptTarget.ES5 || - target === ts.ScriptTarget.ES2015 || - target === ts.ScriptTarget.ES2016; - return `${ - es5Bundle ? SYSTEM_LOADER_ES5 : SYSTEM_LOADER - }\n${data}\n${instantiate}`; - } - - function setRootExports(program, rootModule) { - // get a reference to the type checker, this will let us find symbols from - // the AST. - const checker = program.getTypeChecker(); - // get a reference to the main source file for the bundle - const mainSourceFile = program.getSourceFile(rootModule); - assert(mainSourceFile); - // retrieve the internal TypeScript symbol for this AST node - const mainSymbol = checker.getSymbolAtLocation(mainSourceFile); - if (!mainSymbol) { - return; - } - rootExports = checker - .getExportsOfModule(mainSymbol) - // .getExportsOfModule includes type only symbols which are exported from - // the module, so we need to try to filter those out. While not critical - // someone looking at the bundle would think there is runtime code behind - // that when there isn't. There appears to be no clean way of figuring that - // out, so inspecting SymbolFlags that might be present that are type only - .filter( - (sym) => - sym.flags & ts.SymbolFlags.Class || - !( - sym.flags & ts.SymbolFlags.Interface || - sym.flags & ts.SymbolFlags.TypeLiteral || - sym.flags & ts.SymbolFlags.Signature || - sym.flags & ts.SymbolFlags.TypeParameter || - sym.flags & ts.SymbolFlags.TypeAlias || - sym.flags & ts.SymbolFlags.Type || - sym.flags & ts.SymbolFlags.Namespace || - sym.flags & ts.SymbolFlags.InterfaceExcludes || - sym.flags & ts.SymbolFlags.TypeParameterExcludes || - sym.flags & ts.SymbolFlags.TypeAliasExcludes - ), - ) - .map((sym) => sym.getName()); - } - - function runtimeCompile(request) { - const { compilerOptions, rootNames, target, sourceFileMap } = request; - - debug(">>> runtime compile start", { - rootNames, - }); - - // if there are options, convert them into TypeScript compiler options, - // and resolve any external file references - const result = parseCompilerOptions( - compilerOptions, - ); - const options = result.options; - // TODO(bartlomieju): this options is excluded by `ts.convertCompilerOptionsFromJson` - // however stuff breaks if it's not passed (type_directives_js_main.js, compiler_js_error.ts) - options.allowNonTsExtensions = true; - - buildLocalSourceFileCache(sourceFileMap); - - const state = { - rootNames, - emitMap: {}, - }; - legacyHostState.target = target; - legacyHostState.writeFile = createRuntimeCompileWriteFile(state); - const program = ts.createProgram({ - rootNames, - options, - host, - }); - - const diagnostics = ts - .getPreEmitDiagnostics(program) - .filter(({ code }) => - !IGNORED_DIAGNOSTICS.includes(code) && - !IGNORED_COMPILE_DIAGNOSTICS.includes(code) - ); - - const emitResult = program.emit(); - assert(emitResult.emitSkipped === false, "Unexpected skip of the emit."); - - debug("<<< runtime compile finish", { - rootNames, - emitMap: Object.keys(state.emitMap), - }); - - const maybeDiagnostics = diagnostics.length - ? fromTypeScriptDiagnostic(diagnostics) - : []; - - return { - diagnostics: maybeDiagnostics, - emitMap: state.emitMap, - }; - } - - function runtimeBundle(request) { - const { compilerOptions, rootNames, target, sourceFileMap } = request; - - debug(">>> runtime bundle start", { - rootNames, - }); - - // if there are options, convert them into TypeScript compiler options, - // and resolve any external file references - const result = parseCompilerOptions( - compilerOptions, - ); - const options = result.options; - // TODO(bartlomieju): this options is excluded by `ts.convertCompilerOptionsFromJson` - // however stuff breaks if it's not passed (type_directives_js_main.js, compiler_js_error.ts) - options.allowNonTsExtensions = true; - - buildLocalSourceFileCache(sourceFileMap); - - const state = { - rootNames, - bundleOutput: undefined, - }; - - legacyHostState.target = target; - legacyHostState.writeFile = createBundleWriteFile(state); - state.options = options; - - const program = ts.createProgram({ - rootNames, - options, - host, - }); - - setRootExports(program, rootNames[0]); - const diagnostics = ts - .getPreEmitDiagnostics(program) - .filter(({ code }) => !IGNORED_DIAGNOSTICS.includes(code)); - - const emitResult = program.emit(); - - assert(emitResult.emitSkipped === false, "Unexpected skip of the emit."); - - debug("<<< runtime bundle finish", { - rootNames, - }); - - const maybeDiagnostics = diagnostics.length - ? fromTypeScriptDiagnostic(diagnostics) - : []; - - return { - diagnostics: maybeDiagnostics, - output: state.bundleOutput, - }; - } - - function opCompilerRespond(msg) { - core.jsonOpSync("op_compiler_respond", msg); - } - - function tsCompilerOnMessage(msg) { - const request = msg.data; - switch (request.type) { - case CompilerRequestType.RuntimeCompile: { - const result = runtimeCompile(request); - opCompilerRespond(result); - break; - } - case CompilerRequestType.RuntimeBundle: { - const result = runtimeBundle(request); - opCompilerRespond(result); - break; - } - default: - throw new Error( - `!!! unhandled CompilerRequestType: ${request.type} (${ - CompilerRequestType[request.type] - })`, - ); - } - } - /** * @typedef {object} Request * @property {Record<string, any>} config @@ -1094,6 +678,4 @@ delete Object.prototype.__proto__; globalThis.startup = startup; globalThis.exec = exec; - // TODO(@kitsonk) remove when converted from legacy tsc - globalThis.tsCompilerOnMessage = tsCompilerOnMessage; })(this); diff --git a/cli/tsc_config.rs b/cli/tsc_config.rs index 52daf2e46..92332cca6 100644 --- a/cli/tsc_config.rs +++ b/cli/tsc_config.rs @@ -52,37 +52,49 @@ impl fmt::Display for IgnoredCompilerOptions { /// A static slice of all the compiler options that should be ignored that /// either have no effect on the compilation or would cause the emit to not work /// in Deno. -const IGNORED_COMPILER_OPTIONS: [&str; 10] = [ +const IGNORED_COMPILER_OPTIONS: &[&str] = &[ "allowSyntheticDefaultImports", + "allowUmdGlobalAccess", + "baseUrl", + "declaration", + "declarationMap", + "downlevelIteration", "esModuleInterop", + "emitDeclarationOnly", + "importHelpers", "inlineSourceMap", "inlineSources", // TODO(nayeemrmn): Add "isolatedModules" here for 1.6.0. "module", + "noEmitHelpers", "noLib", + "noResolve", + "outDir", + "paths", "preserveConstEnums", "reactNamespace", + "rootDir", + "rootDirs", + "skipLibCheck", "sourceMap", + "sourceRoot", "target", + "types", + "useDefineForClassFields", ]; -const IGNORED_RUNTIME_COMPILER_OPTIONS: [&str; 50] = [ - "allowUmdGlobalAccess", +const IGNORED_RUNTIME_COMPILER_OPTIONS: &[&str] = &[ "assumeChangesOnlyAffectDirectDependencies", - "baseUrl", "build", + "charset", "composite", - "declaration", - "declarationMap", "diagnostics", - "downlevelIteration", + "disableSizeLimit", "emitBOM", - "emitDeclarationOnly", "extendedDiagnostics", "forceConsistentCasingInFileNames", "generateCpuProfile", "help", - "importHelpers", "incremental", "init", "listEmittedFiles", @@ -92,29 +104,21 @@ const IGNORED_RUNTIME_COMPILER_OPTIONS: [&str; 50] = [ "moduleResolution", "newLine", "noEmit", - "noEmitHelpers", "noEmitOnError", - "noResolve", "out", "outDir", "outFile", - "paths", "preserveSymlinks", "preserveWatchOutput", "pretty", + "project", "resolveJsonModule", - "rootDir", - "rootDirs", "showConfig", "skipDefaultLibCheck", - "skipLibCheck", - "sourceRoot", "stripInternal", "traceResolution", "tsBuildInfoFile", - "types", "typeRoots", - "useDefineForClassFields", "version", "watch", ]; @@ -170,12 +174,6 @@ struct TSConfigJson { type_acquisition: Option<Value>, } -pub fn parse_raw_config(config_text: &str) -> Result<Value, AnyError> { - assert!(!config_text.is_empty()); - let jsonc = jsonc_parser::parse_to_value(config_text)?.unwrap(); - Ok(jsonc_to_serde(jsonc)) -} - fn parse_compiler_options( compiler_options: &HashMap<String, Value>, maybe_path: Option<PathBuf>, @@ -395,18 +393,6 @@ mod tests { } #[test] - fn test_parse_raw_config() { - let invalid_config_text = r#"{ - "compilerOptions": { - // comments are allowed - }"#; - let errbox = parse_raw_config(invalid_config_text).unwrap_err(); - assert!(errbox - .to_string() - .starts_with("Unterminated object on line 1")); - } - - #[test] fn test_tsconfig_as_bytes() { let mut tsconfig1 = TsConfig::new(json!({ "strict": true, |