diff options
Diffstat (limited to 'cli/module_graph.rs')
| -rw-r--r-- | cli/module_graph.rs | 732 |
1 files changed, 732 insertions, 0 deletions
diff --git a/cli/module_graph.rs b/cli/module_graph.rs new file mode 100644 index 000000000..c21257d82 --- /dev/null +++ b/cli/module_graph.rs @@ -0,0 +1,732 @@ +// Copyright 2018-2020 the Deno authors. All rights reserved. MIT license. + +use crate::file_fetcher::SourceFile; +use crate::file_fetcher::SourceFileFetcher; +use crate::import_map::ImportMap; +use crate::msg::MediaType; +use crate::op_error::OpError; +use crate::permissions::Permissions; +use crate::swc_util::analyze_dependencies_and_references; +use crate::swc_util::TsReferenceKind; +use crate::tsc::get_available_libs; +use deno_core::ErrBox; +use deno_core::ModuleSpecifier; +use futures::stream::FuturesUnordered; +use futures::stream::StreamExt; +use futures::Future; +use futures::FutureExt; +use serde::Serialize; +use serde::Serializer; +use std::collections::HashMap; +use std::hash::BuildHasher; +use std::pin::Pin; + +fn serialize_module_specifier<S>( + spec: &ModuleSpecifier, + s: S, +) -> Result<S::Ok, S::Error> +where + S: Serializer, +{ + s.serialize_str(&spec.to_string()) +} + +fn serialize_option_module_specifier<S>( + maybe_spec: &Option<ModuleSpecifier>, + s: S, +) -> Result<S::Ok, S::Error> +where + S: Serializer, +{ + if let Some(spec) = maybe_spec { + serialize_module_specifier(spec, s) + } else { + s.serialize_none() + } +} + +#[derive(Debug, Serialize)] +pub struct ModuleGraph(HashMap<String, ModuleGraphFile>); + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ImportDescriptor { + specifier: String, + #[serde(serialize_with = "serialize_module_specifier")] + resolved_specifier: ModuleSpecifier, + // These two fields are for support of @deno-types directive + // directly prepending import statement + type_directive: Option<String>, + #[serde(serialize_with = "serialize_option_module_specifier")] + resolved_type_directive: Option<ModuleSpecifier>, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ReferenceDescriptor { + specifier: String, + #[serde(serialize_with = "serialize_module_specifier")] + resolved_specifier: ModuleSpecifier, +} + +#[derive(Debug, Serialize)] +#[serde(rename_all = "camelCase")] +pub struct ModuleGraphFile { + pub specifier: String, + pub url: String, + pub filename: String, + pub imports: Vec<ImportDescriptor>, + pub referenced_files: Vec<ReferenceDescriptor>, + pub lib_directives: Vec<ReferenceDescriptor>, + pub types_directives: Vec<ReferenceDescriptor>, + pub type_headers: Vec<ReferenceDescriptor>, + pub media_type: i32, + pub source_code: String, +} + +type SourceFileFuture = + Pin<Box<dyn Future<Output = Result<SourceFile, ErrBox>>>>; + +pub struct ModuleGraphLoader { + permissions: Permissions, + file_fetcher: SourceFileFetcher, + maybe_import_map: Option<ImportMap>, + pending_downloads: FuturesUnordered<SourceFileFuture>, + pub graph: ModuleGraph, + is_dyn_import: bool, + analyze_dynamic_imports: bool, +} + +impl ModuleGraphLoader { + pub fn new( + file_fetcher: SourceFileFetcher, + maybe_import_map: Option<ImportMap>, + permissions: Permissions, + is_dyn_import: bool, + analyze_dynamic_imports: bool, + ) -> Self { + Self { + file_fetcher, + permissions, + maybe_import_map, + pending_downloads: FuturesUnordered::new(), + graph: ModuleGraph(HashMap::new()), + is_dyn_import, + analyze_dynamic_imports, + } + } + + /// This method is used to add specified module and all of its + /// dependencies to the graph. + /// + /// It resolves when all dependent modules have been fetched and analyzed. + /// + /// This method can be called multiple times. + pub async fn add_to_graph( + &mut self, + specifier: &ModuleSpecifier, + ) -> Result<(), ErrBox> { + self.download_module(specifier.clone(), None)?; + + loop { + let source_file = self.pending_downloads.next().await.unwrap()?; + self.visit_module(&source_file.url.clone().into(), source_file)?; + if self.pending_downloads.is_empty() { + break; + } + } + + Ok(()) + } + + /// This method is used to create a graph from in-memory files stored in + /// a hash map. Useful for creating module graph for code received from + /// the runtime. + pub fn build_local_graph<S: BuildHasher>( + &mut self, + _root_name: &str, + source_map: &HashMap<String, String, S>, + ) -> Result<(), ErrBox> { + for (spec, source_code) in source_map.iter() { + self.visit_memory_module(spec.to_string(), source_code.to_string())?; + } + + Ok(()) + } + + /// Consumes the loader and returns created graph. + pub fn get_graph(self) -> HashMap<String, ModuleGraphFile> { + self.graph.0 + } + + fn visit_memory_module( + &mut self, + specifier: String, + source_code: String, + ) -> Result<(), ErrBox> { + let mut imports = vec![]; + let mut referenced_files = vec![]; + let mut lib_directives = vec![]; + let mut types_directives = vec![]; + + // FIXME(bartlomieju): + // The resolveModules op only handles fully qualified URLs for referrer. + // However we will have cases where referrer is "/foo.ts". We add this dummy + // prefix "memory://" in order to use resolution logic. + let module_specifier = + if let Ok(spec) = ModuleSpecifier::resolve_url(&specifier) { + spec + } else { + ModuleSpecifier::resolve_url(&format!("memory://{}", specifier))? + }; + + let (import_descs, ref_descs) = analyze_dependencies_and_references( + &source_code, + self.analyze_dynamic_imports, + )?; + + for import_desc in import_descs { + let maybe_resolved = + if let Some(import_map) = self.maybe_import_map.as_ref() { + import_map + .resolve(&import_desc.specifier, &module_specifier.to_string())? + } else { + None + }; + + let resolved_specifier = if let Some(resolved) = maybe_resolved { + resolved + } else { + ModuleSpecifier::resolve_import( + &import_desc.specifier, + &module_specifier.to_string(), + )? + }; + + let resolved_type_directive = + if let Some(types_specifier) = import_desc.deno_types.as_ref() { + Some(ModuleSpecifier::resolve_import( + &types_specifier, + &module_specifier.to_string(), + )?) + } else { + None + }; + + let import_descriptor = ImportDescriptor { + specifier: import_desc.specifier.to_string(), + resolved_specifier, + type_directive: import_desc.deno_types, + resolved_type_directive, + }; + + imports.push(import_descriptor); + } + + let available_libs = get_available_libs(); + + for ref_desc in ref_descs { + if available_libs.contains(&ref_desc.specifier) { + continue; + } + + let resolved_specifier = ModuleSpecifier::resolve_import( + &ref_desc.specifier, + &module_specifier.to_string(), + )?; + + let reference_descriptor = ReferenceDescriptor { + specifier: ref_desc.specifier.to_string(), + resolved_specifier, + }; + + match ref_desc.kind { + TsReferenceKind::Lib => { + lib_directives.push(reference_descriptor); + } + TsReferenceKind::Types => { + types_directives.push(reference_descriptor); + } + TsReferenceKind::Path => { + referenced_files.push(reference_descriptor); + } + } + } + + self.graph.0.insert( + module_specifier.to_string(), + ModuleGraphFile { + specifier: specifier.to_string(), + url: specifier.to_string(), + filename: specifier, + // ignored, it's set in TS worker + media_type: MediaType::JavaScript as i32, + source_code, + imports, + referenced_files, + lib_directives, + types_directives, + type_headers: vec![], + }, + ); + Ok(()) + } + + fn download_module( + &mut self, + module_specifier: ModuleSpecifier, + maybe_referrer: Option<ModuleSpecifier>, + ) -> Result<(), ErrBox> { + if self.graph.0.contains_key(&module_specifier.to_string()) { + return Ok(()); + } + + if !self.is_dyn_import { + // Verify that remote file doesn't try to statically import local file. + if let Some(referrer) = maybe_referrer.as_ref() { + let referrer_url = referrer.as_url(); + match referrer_url.scheme() { + "http" | "https" => { + let specifier_url = module_specifier.as_url(); + match specifier_url.scheme() { + "http" | "https" => {} + _ => { + let e = OpError::permission_denied("Remote module are not allowed to statically import local modules. Use dynamic import instead.".to_string()); + return Err(e.into()); + } + } + } + _ => {} + } + } + } + + let spec = module_specifier; + let file_fetcher = self.file_fetcher.clone(); + let perms = self.permissions.clone(); + + let load_future = async move { + let spec_ = spec.clone(); + let source_file = file_fetcher + .fetch_source_file(&spec_, maybe_referrer, perms) + .await?; + // FIXME(bartlomieju): + // because of redirects we may end up with wrong URL, + // substitute with original one + Ok(SourceFile { + url: spec_.as_url().to_owned(), + ..source_file + }) + } + .boxed_local(); + + self.pending_downloads.push(load_future); + Ok(()) + } + + fn visit_module( + &mut self, + module_specifier: &ModuleSpecifier, + source_file: SourceFile, + ) -> Result<(), ErrBox> { + let mut imports = vec![]; + let mut referenced_files = vec![]; + let mut lib_directives = vec![]; + let mut types_directives = vec![]; + let mut type_headers = vec![]; + + let source_code = String::from_utf8(source_file.source_code)?; + + if source_file.media_type == MediaType::JavaScript + || source_file.media_type == MediaType::TypeScript + { + if let Some(types_specifier) = source_file.types_header { + let type_header = ReferenceDescriptor { + specifier: types_specifier.to_string(), + resolved_specifier: ModuleSpecifier::resolve_import( + &types_specifier, + &module_specifier.to_string(), + )?, + }; + self.download_module( + type_header.resolved_specifier.clone(), + Some(module_specifier.clone()), + )?; + type_headers.push(type_header); + } + + let (import_descs, ref_descs) = analyze_dependencies_and_references( + &source_code, + self.analyze_dynamic_imports, + )?; + + for import_desc in import_descs { + let maybe_resolved = + if let Some(import_map) = self.maybe_import_map.as_ref() { + import_map + .resolve(&import_desc.specifier, &module_specifier.to_string())? + } else { + None + }; + + let resolved_specifier = if let Some(resolved) = maybe_resolved { + resolved + } else { + ModuleSpecifier::resolve_import( + &import_desc.specifier, + &module_specifier.to_string(), + )? + }; + + let resolved_type_directive = + if let Some(types_specifier) = import_desc.deno_types.as_ref() { + Some(ModuleSpecifier::resolve_import( + &types_specifier, + &module_specifier.to_string(), + )?) + } else { + None + }; + + let import_descriptor = ImportDescriptor { + specifier: import_desc.specifier.to_string(), + resolved_specifier, + type_directive: import_desc.deno_types, + resolved_type_directive, + }; + + self.download_module( + import_descriptor.resolved_specifier.clone(), + Some(module_specifier.clone()), + )?; + + if let Some(type_dir_url) = + import_descriptor.resolved_type_directive.as_ref() + { + self.download_module( + type_dir_url.clone(), + Some(module_specifier.clone()), + )?; + } + + imports.push(import_descriptor); + } + + let available_libs = get_available_libs(); + + for ref_desc in ref_descs { + if available_libs.contains(&ref_desc.specifier) { + continue; + } + + let resolved_specifier = ModuleSpecifier::resolve_import( + &ref_desc.specifier, + &module_specifier.to_string(), + )?; + + let reference_descriptor = ReferenceDescriptor { + specifier: ref_desc.specifier.to_string(), + resolved_specifier, + }; + + self.download_module( + reference_descriptor.resolved_specifier.clone(), + Some(module_specifier.clone()), + )?; + + match ref_desc.kind { + TsReferenceKind::Lib => { + lib_directives.push(reference_descriptor); + } + TsReferenceKind::Types => { + types_directives.push(reference_descriptor); + } + TsReferenceKind::Path => { + referenced_files.push(reference_descriptor); + } + } + } + } + + self.graph.0.insert( + module_specifier.to_string(), + ModuleGraphFile { + specifier: module_specifier.to_string(), + url: source_file.url.to_string(), + filename: source_file.filename.to_str().unwrap().to_string(), + media_type: source_file.media_type as i32, + source_code, + imports, + referenced_files, + lib_directives, + types_directives, + type_headers, + }, + ); + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::GlobalState; + + async fn build_graph( + module_specifier: &ModuleSpecifier, + ) -> Result<HashMap<String, ModuleGraphFile>, ErrBox> { + let global_state = GlobalState::new(Default::default()).unwrap(); + let mut graph_loader = ModuleGraphLoader::new( + global_state.file_fetcher.clone(), + None, + Permissions::allow_all(), + false, + false, + ); + graph_loader.add_to_graph(&module_specifier).await?; + Ok(graph_loader.get_graph()) + } + + #[tokio::test] + async fn source_graph_fetch() { + let http_server_guard = crate::test_util::http_server(); + + let module_specifier = ModuleSpecifier::resolve_url_or_path( + "http://localhost:4545/cli/tests/019_media_types.ts", + ) + .unwrap(); + let graph = build_graph(&module_specifier) + .await + .expect("Failed to build graph"); + + let a = graph + .get("http://localhost:4545/cli/tests/019_media_types.ts") + .unwrap(); + + assert!(graph.contains_key( + "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js" + )); + assert!(graph.contains_key( + "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts" + )); + assert!(graph.contains_key("http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts")); + assert!(graph.contains_key( + "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts" + )); + assert!(graph.contains_key("http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js")); + assert!(graph.contains_key( + "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js" + )); + assert!(graph.contains_key( + "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js" + )); + assert!(graph.contains_key( + "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts" + )); + + assert_eq!( + serde_json::to_value(&a.imports).unwrap(), + json!([ + { + "specifier": "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts", + "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_text_typescript.t1.ts", + "typeDirective": null, + "resolvedTypeDirective": null, + }, + { + "specifier": "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts", + "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_video_vdn.t2.ts", + "typeDirective": null, + "resolvedTypeDirective": null, + }, + { + "specifier": "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts", + "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_video_mp2t.t3.ts", + "typeDirective": null, + "resolvedTypeDirective": null, + }, + { + "specifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts", + "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_typescript.t4.ts", + "typeDirective": null, + "resolvedTypeDirective": null, + }, + { + "specifier": "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js", + "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_text_javascript.j1.js", + "typeDirective": null, + "resolvedTypeDirective": null, + }, + { + "specifier": "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js", + "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_application_ecmascript.j2.js", + "typeDirective": null, + "resolvedTypeDirective": null, + }, + { + "specifier": "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js", + "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_text_ecmascript.j3.js", + "typeDirective": null, + "resolvedTypeDirective": null, + }, + { + "specifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js", + "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/mt_application_x_javascript.j4.js", + "typeDirective": null, + "resolvedTypeDirective": null, + }, + ]) + ); + drop(http_server_guard); + } + + #[tokio::test] + async fn source_graph_type_references() { + let http_server_guard = crate::test_util::http_server(); + + let module_specifier = ModuleSpecifier::resolve_url_or_path( + "http://localhost:4545/cli/tests/type_definitions.ts", + ) + .unwrap(); + + let graph = build_graph(&module_specifier) + .await + .expect("Failed to build graph"); + + eprintln!("json {:#?}", serde_json::to_value(&graph).unwrap()); + + let a = graph + .get("http://localhost:4545/cli/tests/type_definitions.ts") + .unwrap(); + assert_eq!( + serde_json::to_value(&a.imports).unwrap(), + json!([ + { + "specifier": "./type_definitions/foo.js", + "resolvedSpecifier": "http://localhost:4545/cli/tests/type_definitions/foo.js", + "typeDirective": "./type_definitions/foo.d.ts", + "resolvedTypeDirective": "http://localhost:4545/cli/tests/type_definitions/foo.d.ts" + }, + { + "specifier": "./type_definitions/fizz.js", + "resolvedSpecifier": "http://localhost:4545/cli/tests/type_definitions/fizz.js", + "typeDirective": "./type_definitions/fizz.d.ts", + "resolvedTypeDirective": "http://localhost:4545/cli/tests/type_definitions/fizz.d.ts" + }, + { + "specifier": "./type_definitions/qat.ts", + "resolvedSpecifier": "http://localhost:4545/cli/tests/type_definitions/qat.ts", + "typeDirective": null, + "resolvedTypeDirective": null, + }, + ]) + ); + assert!(graph + .contains_key("http://localhost:4545/cli/tests/type_definitions/foo.js")); + assert!(graph.contains_key( + "http://localhost:4545/cli/tests/type_definitions/foo.d.ts" + )); + assert!(graph.contains_key( + "http://localhost:4545/cli/tests/type_definitions/fizz.js" + )); + assert!(graph.contains_key( + "http://localhost:4545/cli/tests/type_definitions/fizz.d.ts" + )); + assert!(graph + .contains_key("http://localhost:4545/cli/tests/type_definitions/qat.ts")); + + drop(http_server_guard); + } + + #[tokio::test] + async fn source_graph_type_references2() { + let http_server_guard = crate::test_util::http_server(); + + let module_specifier = ModuleSpecifier::resolve_url_or_path( + "http://localhost:4545/cli/tests/type_directives_02.ts", + ) + .unwrap(); + + let graph = build_graph(&module_specifier) + .await + .expect("Failed to build graph"); + + eprintln!("{:#?}", serde_json::to_value(&graph).unwrap()); + + let a = graph + .get("http://localhost:4545/cli/tests/type_directives_02.ts") + .unwrap(); + assert_eq!( + serde_json::to_value(&a.imports).unwrap(), + json!([ + { + "specifier": "./subdir/type_reference.js", + "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/type_reference.js", + "typeDirective": null, + "resolvedTypeDirective": null, + } + ]) + ); + + assert!(graph.contains_key( + "http://localhost:4545/cli/tests/subdir/type_reference.d.ts" + )); + + let b = graph + .get("http://localhost:4545/cli/tests/subdir/type_reference.js") + .unwrap(); + assert_eq!( + serde_json::to_value(&b.types_directives).unwrap(), + json!([ + { + "specifier": "./type_reference.d.ts", + "resolvedSpecifier": "http://localhost:4545/cli/tests/subdir/type_reference.d.ts", + } + ]) + ); + drop(http_server_guard); + } + + #[tokio::test] + async fn source_graph_type_references3() { + let http_server_guard = crate::test_util::http_server(); + + let module_specifier = ModuleSpecifier::resolve_url_or_path( + "http://localhost:4545/cli/tests/type_directives_01.ts", + ) + .unwrap(); + + let graph = build_graph(&module_specifier) + .await + .expect("Failed to build graph"); + + let ts = graph + .get("http://localhost:4545/cli/tests/type_directives_01.ts") + .unwrap(); + assert_eq!( + serde_json::to_value(&ts.imports).unwrap(), + json!([ + { + "specifier": "http://127.0.0.1:4545/xTypeScriptTypes.js", + "resolvedSpecifier": "http://127.0.0.1:4545/xTypeScriptTypes.js", + "typeDirective": null, + "resolvedTypeDirective": null, + } + ]) + ); + + let headers = graph + .get("http://127.0.0.1:4545/xTypeScriptTypes.js") + .unwrap(); + assert_eq!( + serde_json::to_value(&headers.type_headers).unwrap(), + json!([ + { + "specifier": "./xTypeScriptTypes.d.ts", + "resolvedSpecifier": "http://127.0.0.1:4545/xTypeScriptTypes.d.ts" + } + ]) + ); + drop(http_server_guard); + } +} |