diff options
Diffstat (limited to 'cli/module_graph.rs')
| -rw-r--r-- | cli/module_graph.rs | 509 |
1 files changed, 318 insertions, 191 deletions
diff --git a/cli/module_graph.rs b/cli/module_graph.rs index 9cded48a0..b5bde1a19 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -8,8 +8,10 @@ use crate::import_map::ImportMap; use crate::msg::MediaType; use crate::op_error::OpError; use crate::permissions::Permissions; -use crate::swc_util::analyze_dependencies_and_references; -use crate::swc_util::TsReferenceKind; +use crate::tsc::pre_process_file; +use crate::tsc::ImportDesc; +use crate::tsc::TsReferenceDesc; +use crate::tsc::TsReferenceKind; use crate::tsc::AVAILABLE_LIBS; use deno_core::ErrBox; use deno_core::ModuleSpecifier; @@ -21,20 +23,142 @@ use serde::Serialize; use serde::Serializer; use std::collections::HashMap; use std::collections::HashSet; -use std::hash::BuildHasher; use std::path::PathBuf; use std::pin::Pin; // TODO(bartlomieju): it'd be great if this function returned // more structured data and possibly format the same as TS diagnostics. /// Decorate error with location of import that caused the error. -fn err_with_location(e: ErrBox, location: &Location) -> ErrBox { - let location_str = format!( - "\nImported from \"{}:{}\"", - location.filename, location.line - ); - let err_str = e.to_string(); - OpError::other(format!("{}{}", err_str, location_str)).into() +fn err_with_location(e: ErrBox, maybe_location: Option<&Location>) -> ErrBox { + if let Some(location) = maybe_location { + let location_str = format!( + "\nImported from \"{}:{}\"", + location.filename, location.line + ); + let err_str = e.to_string(); + OpError::other(format!("{}{}", err_str, location_str)).into() + } else { + e + } +} + +/// Disallow http:// imports from modules loaded over https:// +fn validate_no_downgrade( + module_specifier: &ModuleSpecifier, + maybe_referrer: Option<&ModuleSpecifier>, + maybe_location: Option<&Location>, +) -> Result<(), ErrBox> { + if let Some(referrer) = maybe_referrer.as_ref() { + if let "https" = referrer.as_url().scheme() { + if let "http" = module_specifier.as_url().scheme() { + let e = OpError::permission_denied( + "Modules loaded over https:// are not allowed to import modules over http://".to_string() + ); + return Err(err_with_location(e.into(), maybe_location)); + }; + }; + }; + + Ok(()) +} + +/// Verify that remote file doesn't try to statically import local file. +fn validate_no_file_from_remote( + module_specifier: &ModuleSpecifier, + maybe_referrer: Option<&ModuleSpecifier>, + maybe_location: Option<&Location>, +) -> Result<(), ErrBox> { + if let Some(referrer) = maybe_referrer.as_ref() { + let referrer_url = referrer.as_url(); + match referrer_url.scheme() { + "http" | "https" => { + let specifier_url = module_specifier.as_url(); + match specifier_url.scheme() { + "http" | "https" => {} + _ => { + let e = OpError::permission_denied( + "Remote modules are not allowed to statically import local modules. Use dynamic import instead.".to_string() + ); + return Err(err_with_location(e.into(), maybe_location)); + } + } + } + _ => {} + } + } + + Ok(()) +} + +// TODO(bartlomieju): handle imports/references in ambient contexts/TS modules +// https://github.com/denoland/deno/issues/6133 +fn resolve_imports_and_references( + referrer: ModuleSpecifier, + maybe_import_map: Option<&ImportMap>, + import_descs: Vec<ImportDesc>, + ref_descs: Vec<TsReferenceDesc>, +) -> Result<(Vec<ImportDescriptor>, Vec<ReferenceDescriptor>), ErrBox> { + let mut imports = vec![]; + let mut references = vec![]; + + for import_desc in import_descs { + let maybe_resolved = if let Some(import_map) = maybe_import_map.as_ref() { + import_map.resolve(&import_desc.specifier, &referrer.to_string())? + } else { + None + }; + + let resolved_specifier = if let Some(resolved) = maybe_resolved { + resolved + } else { + ModuleSpecifier::resolve_import( + &import_desc.specifier, + &referrer.to_string(), + )? + }; + + let resolved_type_directive = + if let Some(types_specifier) = import_desc.deno_types.as_ref() { + Some(ModuleSpecifier::resolve_import( + &types_specifier, + &referrer.to_string(), + )?) + } else { + None + }; + + let import_descriptor = ImportDescriptor { + specifier: import_desc.specifier.to_string(), + resolved_specifier, + type_directive: import_desc.deno_types.clone(), + resolved_type_directive, + location: import_desc.location, + }; + + imports.push(import_descriptor); + } + + for ref_desc in ref_descs { + if AVAILABLE_LIBS.contains(&ref_desc.specifier.as_str()) { + continue; + } + + let resolved_specifier = ModuleSpecifier::resolve_import( + &ref_desc.specifier, + &referrer.to_string(), + )?; + + let reference_descriptor = ReferenceDescriptor { + specifier: ref_desc.specifier.to_string(), + resolved_specifier, + kind: ref_desc.kind, + location: ref_desc.location, + }; + + references.push(reference_descriptor); + } + + Ok((imports, references)) } fn serialize_module_specifier<S>( @@ -68,8 +192,7 @@ const SUPPORTED_MEDIA_TYPES: [MediaType; 4] = [ MediaType::TSX, ]; -#[derive(Debug, Serialize)] -pub struct ModuleGraph(HashMap<String, ModuleGraphFile>); +pub type ModuleGraph = HashMap<String, ModuleGraphFile>; #[derive(Debug, Serialize)] #[serde(rename_all = "camelCase")] @@ -82,6 +205,8 @@ pub struct ImportDescriptor { pub type_directive: Option<String>, #[serde(serialize_with = "serialize_option_module_specifier")] pub resolved_type_directive: Option<ModuleSpecifier>, + #[serde(skip)] + pub location: Location, } #[derive(Debug, Serialize)] @@ -90,6 +215,10 @@ pub struct ReferenceDescriptor { pub specifier: String, #[serde(serialize_with = "serialize_module_specifier")] pub resolved_specifier: ModuleSpecifier, + #[serde(skip)] + pub kind: TsReferenceKind, + #[serde(skip)] + pub location: Location, } #[derive(Debug, Serialize)] @@ -104,7 +233,7 @@ pub struct ModuleGraphFile { pub lib_directives: Vec<ReferenceDescriptor>, pub types_directives: Vec<ReferenceDescriptor>, pub type_headers: Vec<ReferenceDescriptor>, - pub media_type: i32, + pub media_type: MediaType, pub source_code: String, } @@ -117,7 +246,7 @@ pub struct ModuleGraphLoader { maybe_import_map: Option<ImportMap>, pending_downloads: FuturesUnordered<SourceFileFuture>, has_downloaded: HashSet<ModuleSpecifier>, - pub graph: ModuleGraph, + graph: ModuleGraph, is_dyn_import: bool, analyze_dynamic_imports: bool, } @@ -136,7 +265,7 @@ impl ModuleGraphLoader { maybe_import_map, pending_downloads: FuturesUnordered::new(), has_downloaded: HashSet::new(), - graph: ModuleGraph(HashMap::new()), + graph: ModuleGraph::new(), is_dyn_import, analyze_dynamic_imports, } @@ -153,7 +282,7 @@ impl ModuleGraphLoader { specifier: &ModuleSpecifier, maybe_referrer: Option<ModuleSpecifier>, ) -> Result<(), ErrBox> { - self.download_module(specifier.clone(), maybe_referrer)?; + self.download_module(specifier.clone(), maybe_referrer, None)?; loop { let (specifier, source_file) = @@ -170,10 +299,10 @@ impl ModuleGraphLoader { /// This method is used to create a graph from in-memory files stored in /// a hash map. Useful for creating module graph for code received from /// the runtime. - pub fn build_local_graph<S: BuildHasher>( + pub fn build_local_graph( &mut self, _root_name: &str, - source_map: &HashMap<String, String, S>, + source_map: &HashMap<String, String>, ) -> Result<(), ErrBox> { for (spec, source_code) in source_map.iter() { self.visit_memory_module(spec.to_string(), source_code.to_string())?; @@ -183,8 +312,8 @@ impl ModuleGraphLoader { } /// Consumes the loader and returns created graph. - pub fn get_graph(self) -> HashMap<String, ModuleGraphFile> { - self.graph.0 + pub fn get_graph(self) -> ModuleGraph { + self.graph } fn visit_memory_module( @@ -192,7 +321,6 @@ impl ModuleGraphLoader { specifier: String, source_code: String, ) -> Result<(), ErrBox> { - let mut imports = vec![]; let mut referenced_files = vec![]; let mut lib_directives = vec![]; let mut types_directives = vec![]; @@ -208,87 +336,40 @@ impl ModuleGraphLoader { ModuleSpecifier::resolve_url(&format!("memory://{}", specifier))? }; - let (import_descs, ref_descs) = analyze_dependencies_and_references( - &specifier, + let (raw_imports, raw_references) = pre_process_file( + &module_specifier.to_string(), map_file_extension(&PathBuf::from(&specifier)), &source_code, self.analyze_dynamic_imports, )?; + let (imports, references) = resolve_imports_and_references( + module_specifier.clone(), + self.maybe_import_map.as_ref(), + raw_imports, + raw_references, + )?; - for import_desc in import_descs { - let maybe_resolved = - if let Some(import_map) = self.maybe_import_map.as_ref() { - import_map - .resolve(&import_desc.specifier, &module_specifier.to_string())? - } else { - None - }; - - let resolved_specifier = if let Some(resolved) = maybe_resolved { - resolved - } else { - ModuleSpecifier::resolve_import( - &import_desc.specifier, - &module_specifier.to_string(), - )? - }; - - let resolved_type_directive = - if let Some(types_specifier) = import_desc.deno_types.as_ref() { - Some(ModuleSpecifier::resolve_import( - &types_specifier, - &module_specifier.to_string(), - )?) - } else { - None - }; - - let import_descriptor = ImportDescriptor { - specifier: import_desc.specifier.to_string(), - resolved_specifier, - type_directive: import_desc.deno_types, - resolved_type_directive, - }; - - imports.push(import_descriptor); - } - - for ref_desc in ref_descs { - if AVAILABLE_LIBS.contains(&ref_desc.specifier.as_str()) { - continue; - } - - let resolved_specifier = ModuleSpecifier::resolve_import( - &ref_desc.specifier, - &module_specifier.to_string(), - )?; - - let reference_descriptor = ReferenceDescriptor { - specifier: ref_desc.specifier.to_string(), - resolved_specifier, - }; - - match ref_desc.kind { + for ref_descriptor in references { + match ref_descriptor.kind { TsReferenceKind::Lib => { - lib_directives.push(reference_descriptor); + lib_directives.push(ref_descriptor); } TsReferenceKind::Types => { - types_directives.push(reference_descriptor); + types_directives.push(ref_descriptor); } TsReferenceKind::Path => { - referenced_files.push(reference_descriptor); + referenced_files.push(ref_descriptor); } } } - self.graph.0.insert( + self.graph.insert( module_specifier.to_string(), ModuleGraphFile { specifier: specifier.to_string(), url: specifier.to_string(), redirect: None, - media_type: map_file_extension(&PathBuf::from(specifier.clone())) - as i32, + media_type: map_file_extension(&PathBuf::from(specifier.clone())), filename: specifier, source_code, imports, @@ -307,43 +388,24 @@ impl ModuleGraphLoader { &mut self, module_specifier: ModuleSpecifier, maybe_referrer: Option<ModuleSpecifier>, + maybe_location: Option<Location>, ) -> Result<(), ErrBox> { if self.has_downloaded.contains(&module_specifier) { return Ok(()); } - // Disallow http:// imports from modules loaded over https:// - if let Some(referrer) = maybe_referrer.as_ref() { - if let "https" = referrer.as_url().scheme() { - if let "http" = module_specifier.as_url().scheme() { - let e = OpError::permission_denied( - "Modules loaded over https:// are not allowed to import modules over http://".to_string() - ); - return Err(e.into()); - }; - }; - }; + validate_no_downgrade( + &module_specifier, + maybe_referrer.as_ref(), + maybe_location.as_ref(), + )?; if !self.is_dyn_import { - // Verify that remote file doesn't try to statically import local file. - if let Some(referrer) = maybe_referrer.as_ref() { - let referrer_url = referrer.as_url(); - match referrer_url.scheme() { - "http" | "https" => { - let specifier_url = module_specifier.as_url(); - match specifier_url.scheme() { - "http" | "https" => {} - _ => { - let e = OpError::permission_denied( - "Remote modules are not allowed to statically import local modules. Use dynamic import instead.".to_string() - ); - return Err(e.into()); - } - } - } - _ => {} - } - } + validate_no_file_from_remote( + &module_specifier, + maybe_referrer.as_ref(), + maybe_location.as_ref(), + )?; } self.has_downloaded.insert(module_specifier.clone()); @@ -355,7 +417,9 @@ impl ModuleGraphLoader { let spec_ = spec.clone(); let source_file = file_fetcher .fetch_source_file(&spec_, maybe_referrer, perms) - .await?; + .await + .map_err(|e| err_with_location(e, maybe_location.as_ref()))?; + Ok((spec_.clone(), source_file)) } .boxed_local(); @@ -383,14 +447,14 @@ impl ModuleGraphLoader { // for proper URL point to redirect target. if module_specifier.as_url() != &source_file.url { // TODO(bartlomieju): refactor, this is a band-aid - self.graph.0.insert( + self.graph.insert( module_specifier.to_string(), ModuleGraphFile { specifier: module_specifier.to_string(), url: module_specifier.to_string(), redirect: Some(source_file.url.to_string()), filename: source_file.filename.to_str().unwrap().to_string(), - media_type: source_file.media_type as i32, + media_type: source_file.media_type, source_code: "".to_string(), imports: vec![], referenced_files: vec![], @@ -412,121 +476,85 @@ impl ModuleGraphLoader { &types_specifier, &module_specifier.to_string(), )?, + kind: TsReferenceKind::Types, + // TODO(bartlomieju): location is not needed in here and constructing + // location by hand is bad + location: Location { + filename: module_specifier.to_string(), + line: 0, + col: 0, + }, }; self.download_module( type_header.resolved_specifier.clone(), Some(module_specifier.clone()), + None, )?; type_headers.push(type_header); } - let (import_descs, ref_descs) = analyze_dependencies_and_references( + let (raw_imports, raw_refs) = pre_process_file( &module_specifier.to_string(), source_file.media_type, &source_code, self.analyze_dynamic_imports, )?; + let (imports_, references) = resolve_imports_and_references( + module_specifier.clone(), + self.maybe_import_map.as_ref(), + raw_imports, + raw_refs, + )?; - for import_desc in import_descs { - let maybe_resolved = - if let Some(import_map) = self.maybe_import_map.as_ref() { - import_map - .resolve(&import_desc.specifier, &module_specifier.to_string())? - } else { - None - }; - - let resolved_specifier = if let Some(resolved) = maybe_resolved { - resolved - } else { - ModuleSpecifier::resolve_import( - &import_desc.specifier, - &module_specifier.to_string(), - )? - }; - - let resolved_type_directive = - if let Some(types_specifier) = import_desc.deno_types.as_ref() { - Some(ModuleSpecifier::resolve_import( - &types_specifier, - &module_specifier.to_string(), - )?) - } else { - None - }; - - let import_descriptor = ImportDescriptor { - specifier: import_desc.specifier.to_string(), - resolved_specifier, - type_directive: import_desc.deno_types.clone(), - resolved_type_directive, - }; - - self - .download_module( - import_descriptor.resolved_specifier.clone(), - Some(module_specifier.clone()), - ) - .map_err(|e| err_with_location(e, &import_desc.location))?; + for import_descriptor in imports_ { + self.download_module( + import_descriptor.resolved_specifier.clone(), + Some(module_specifier.clone()), + Some(import_descriptor.location.clone()), + )?; if let Some(type_dir_url) = import_descriptor.resolved_type_directive.as_ref() { - self - .download_module( - type_dir_url.clone(), - Some(module_specifier.clone()), - ) - .map_err(|e| err_with_location(e, &import_desc.location))?; + self.download_module( + type_dir_url.clone(), + Some(module_specifier.clone()), + Some(import_descriptor.location.clone()), + )?; } imports.push(import_descriptor); } - for ref_desc in ref_descs { - if AVAILABLE_LIBS.contains(&ref_desc.specifier.as_str()) { - continue; - } - - let resolved_specifier = ModuleSpecifier::resolve_import( - &ref_desc.specifier, - &module_specifier.to_string(), + for ref_descriptor in references { + self.download_module( + ref_descriptor.resolved_specifier.clone(), + Some(module_specifier.clone()), + Some(ref_descriptor.location.clone()), )?; - let reference_descriptor = ReferenceDescriptor { - specifier: ref_desc.specifier.to_string(), - resolved_specifier, - }; - - self - .download_module( - reference_descriptor.resolved_specifier.clone(), - Some(module_specifier.clone()), - ) - .map_err(|e| err_with_location(e, &ref_desc.location))?; - - match ref_desc.kind { + match ref_descriptor.kind { TsReferenceKind::Lib => { - lib_directives.push(reference_descriptor); + lib_directives.push(ref_descriptor); } TsReferenceKind::Types => { - types_directives.push(reference_descriptor); + types_directives.push(ref_descriptor); } TsReferenceKind::Path => { - referenced_files.push(reference_descriptor); + referenced_files.push(ref_descriptor); } } } } - self.graph.0.insert( + self.graph.insert( module_specifier.to_string(), ModuleGraphFile { specifier: module_specifier.to_string(), url: module_specifier.to_string(), redirect: None, filename: source_file.filename.to_str().unwrap().to_string(), - media_type: source_file.media_type as i32, + media_type: source_file.media_type, source_code, imports, referenced_files, @@ -546,7 +574,7 @@ mod tests { async fn build_graph( module_specifier: &ModuleSpecifier, - ) -> Result<HashMap<String, ModuleGraphFile>, ErrBox> { + ) -> Result<ModuleGraph, ErrBox> { let global_state = GlobalState::new(Default::default()).unwrap(); let mut graph_loader = ModuleGraphLoader::new( global_state.file_fetcher.clone(), @@ -824,3 +852,102 @@ mod tests { drop(http_server_guard); } } + +// TODO(bartlomieju): use baseline tests from TSC to ensure +// compatibility +#[test] +fn test_pre_process_file() { + let source = r#" +// This comment is placed to make sure that directives are parsed +// even when they start on non-first line + +/// <reference lib="dom" /> +/// <reference types="./type_reference.d.ts" /> +/// <reference path="./type_reference/dep.ts" /> +// @deno-types="./type_definitions/foo.d.ts" +import { foo } from "./type_definitions/foo.js"; +// @deno-types="./type_definitions/fizz.d.ts" +import "./type_definitions/fizz.js"; + +/// <reference path="./type_reference/dep2.ts" /> + +import * as qat from "./type_definitions/qat.ts"; + +console.log(foo); +console.log(fizz); +console.log(qat.qat); +"#; + + let (imports, references) = + pre_process_file("some/file.ts", MediaType::TypeScript, source, true) + .expect("Failed to parse"); + + assert_eq!( + imports, + vec![ + ImportDesc { + specifier: "./type_definitions/foo.js".to_string(), + deno_types: Some("./type_definitions/foo.d.ts".to_string()), + location: Location { + filename: "some/file.ts".to_string(), + line: 9, + col: 0, + }, + }, + ImportDesc { + specifier: "./type_definitions/fizz.js".to_string(), + deno_types: Some("./type_definitions/fizz.d.ts".to_string()), + location: Location { + filename: "some/file.ts".to_string(), + line: 11, + col: 0, + }, + }, + ImportDesc { + specifier: "./type_definitions/qat.ts".to_string(), + deno_types: None, + location: Location { + filename: "some/file.ts".to_string(), + line: 15, + col: 0, + }, + }, + ] + ); + + // According to TS docs (https://www.typescriptlang.org/docs/handbook/triple-slash-directives.html) + // directives that are not at the top of the file are ignored, so only + // 3 references should be captured instead of 4. + assert_eq!( + references, + vec![ + TsReferenceDesc { + specifier: "dom".to_string(), + kind: TsReferenceKind::Lib, + location: Location { + filename: "some/file.ts".to_string(), + line: 5, + col: 0, + }, + }, + TsReferenceDesc { + specifier: "./type_reference.d.ts".to_string(), + kind: TsReferenceKind::Types, + location: Location { + filename: "some/file.ts".to_string(), + line: 6, + col: 0, + }, + }, + TsReferenceDesc { + specifier: "./type_reference/dep.ts".to_string(), + kind: TsReferenceKind::Path, + location: Location { + filename: "some/file.ts".to_string(), + line: 7, + col: 0, + }, + }, + ] + ); +} |