diff options
author | Kitson Kelly <me@kitsonkelly.com> | 2020-12-15 16:52:55 +1100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2020-12-15 16:52:55 +1100 |
commit | 63a821b78b5a293c0eb5b3ecc18d67bde8331eda (patch) | |
tree | 4229af0ba8b9f56467f1355b0c53054b69039edf /cli/module_graph.rs | |
parent | b6d5ae1ecd125ce8a14cddc877cb66dbcfc9bb04 (diff) |
fix(cli): make dynamic import errors catchable (#8750)
Fixes #6259
Diffstat (limited to 'cli/module_graph.rs')
-rw-r--r-- | cli/module_graph.rs | 422 |
1 files changed, 284 insertions, 138 deletions
diff --git a/cli/module_graph.rs b/cli/module_graph.rs index 4144ee5ee..f5e08882e 100644 --- a/cli/module_graph.rs +++ b/cli/module_graph.rs @@ -28,6 +28,8 @@ use crate::tsc_config::TsConfig; use crate::version; use crate::AnyError; +use deno_core::error::anyhow; +use deno_core::error::custom_error; use deno_core::error::Context; use deno_core::futures::stream::FuturesUnordered; use deno_core::futures::stream::StreamExt; @@ -38,6 +40,7 @@ use deno_core::serde::Serializer; use deno_core::serde_json::json; use deno_core::serde_json::Value; use deno_core::ModuleResolutionError; +use deno_core::ModuleSource; use deno_core::ModuleSpecifier; use regex::Regex; use std::cell::RefCell; @@ -482,6 +485,10 @@ impl Module { Ok(specifier) } + pub fn set_emit(&mut self, code: String, maybe_map: Option<String>) { + self.maybe_emit = Some(Emit::Cli((code, maybe_map))); + } + /// Calculate the hashed version of the module and update the `maybe_version`. pub fn set_version(&mut self, config: &[u8]) { self.maybe_version = @@ -523,6 +530,9 @@ pub struct ResultInfo { /// A structure which provides diagnostic information (usually from `tsc`) /// about the code in the module graph. pub diagnostics: Diagnostics, + /// A map of specifiers to the result of their resolution in the module graph. + pub loadable_modules: + HashMap<ModuleSpecifier, Result<ModuleSource, AnyError>>, /// Optionally ignored compiler options that represent any options that were /// ignored if there was a user provided configuration. pub maybe_ignored_options: Option<IgnoredCompilerOptions>, @@ -637,6 +647,18 @@ pub struct TranspileOptions { pub reload: bool, } +#[derive(Debug, Clone)] +enum ModuleSlot { + /// The module fetch resulted in a non-recoverable error. + Err(Rc<AnyError>), + /// The the fetch resulted in a module. + Module(Box<Module>), + /// Used to denote a module that isn't part of the graph. + None, + /// The fetch of the module is pending. + Pending, +} + /// A dependency graph of modules, were the modules that have been inserted via /// the builder will be loaded into the graph. Also provides an interface to /// be able to manipulate and handle the graph. @@ -649,7 +671,7 @@ pub struct Graph { /// invoked. maybe_tsbuildinfo: Option<String>, /// The modules that are part of the graph. - modules: HashMap<ModuleSpecifier, Module>, + modules: HashMap<ModuleSpecifier, ModuleSlot>, /// A map of redirects, where a module specifier is redirected to another /// module specifier by the handler. All modules references should be /// resolved internally via this, before attempting to access the module via @@ -667,6 +689,44 @@ pub struct Graph { maybe_lockfile: Option<Arc<Mutex<Lockfile>>>, } +/// Convert a specifier and a module slot in a result to the module source which +/// is needed by Deno core for loading the module. +fn to_module_result( + (specifier, module_slot): (&ModuleSpecifier, &ModuleSlot), +) -> (ModuleSpecifier, Result<ModuleSource, AnyError>) { + match module_slot { + ModuleSlot::Err(err) => (specifier.clone(), Err(anyhow!(err.to_string()))), + ModuleSlot::Module(module) => ( + specifier.clone(), + if let Some(emit) = &module.maybe_emit { + match emit { + Emit::Cli((code, _)) => Ok(ModuleSource { + code: code.clone(), + module_url_found: module.specifier.to_string(), + module_url_specified: specifier.to_string(), + }), + } + } else { + match module.media_type { + MediaType::JavaScript | MediaType::Unknown => Ok(ModuleSource { + code: module.source.clone(), + module_url_found: module.specifier.to_string(), + module_url_specified: specifier.to_string(), + }), + _ => Err(custom_error( + "NotFound", + format!("Compiled module not found \"{}\"", specifier), + )), + } + }, + ), + _ => ( + specifier.clone(), + Err(anyhow!("Module \"{}\" unavailable.", specifier)), + ), + } +} + impl Graph { /// Create a new instance of a graph, ready to have modules loaded it. /// @@ -767,6 +827,7 @@ impl Graph { debug!("graph does not need to be checked or emitted."); return Ok(ResultInfo { maybe_ignored_options, + loadable_modules: self.get_loadable_modules(), ..Default::default() }); } @@ -834,9 +895,10 @@ impl Graph { } let config = config.as_bytes(); for (specifier, code) in codes.iter() { - if let Some(module) = graph.get_module_mut(specifier) { - module.maybe_emit = - Some(Emit::Cli((code.clone(), maps.get(specifier).cloned()))); + if let ModuleSlot::Module(module) = + graph.get_module_mut(specifier).unwrap() + { + module.set_emit(code.clone(), maps.get(specifier).cloned()); module.set_version(&config); module.is_dirty = true; } else { @@ -849,16 +911,12 @@ impl Graph { Ok(ResultInfo { diagnostics: response.diagnostics, + loadable_modules: graph.get_loadable_modules(), maybe_ignored_options, stats: response.stats, }) } - fn contains_module(&self, specifier: &ModuleSpecifier) -> bool { - let s = self.resolve_specifier(specifier); - self.modules.contains_key(s) - } - /// Emit the module graph in a specific format. This is specifically designed /// to be an "all-in-one" API for access by the runtime, allowing both /// emitting single modules as well as bundles, using Deno module resolution @@ -921,13 +979,13 @@ impl Graph { )?; let mut emitted_files = HashMap::new(); + let graph = graph.borrow(); match options.bundle_type { BundleType::Esm => { assert!( response.emitted_files.is_empty(), "No files should have been emitted from tsc." ); - let graph = graph.borrow(); assert_eq!( graph.roots.len(), 1, @@ -966,6 +1024,7 @@ impl Graph { emitted_files, ResultInfo { diagnostics: response.diagnostics, + loadable_modules: graph.get_loadable_modules(), maybe_ignored_options, stats: response.stats, }, @@ -1023,15 +1082,17 @@ impl Graph { /// any build info if present. fn flush(&mut self) -> Result<(), AnyError> { let mut handler = self.handler.borrow_mut(); - for (_, module) in self.modules.iter_mut() { - if module.is_dirty { - if let Some(emit) = &module.maybe_emit { - handler.set_cache(&module.specifier, emit)?; - } - if let Some(version) = &module.maybe_version { - handler.set_version(&module.specifier, version.clone())?; + for (_, module_slot) in self.modules.iter_mut() { + if let ModuleSlot::Module(module) = module_slot { + if module.is_dirty { + if let Some(emit) = &module.maybe_emit { + handler.set_cache(&module.specifier, emit)?; + } + if let Some(version) = &module.maybe_version { + handler.set_version(&module.specifier, version.clone())?; + } + module.is_dirty = false; } - module.is_dirty = false; } } for root_specifier in self.roots.iter() { @@ -1050,7 +1111,12 @@ impl Graph { totals: &mut HashMap<ModuleSpecifier, usize>, ) -> ModuleInfo { let not_seen = seen.insert(specifier.clone()); - let module = self.get_module(specifier).unwrap(); + let module = if let ModuleSlot::Module(module) = self.get_module(specifier) + { + module + } else { + unreachable!(); + }; let mut deps = Vec::new(); let mut total_size = None; @@ -1097,50 +1163,79 @@ impl Graph { let map = self .modules .iter() - .map(|(specifier, module)| { - let mut deps = BTreeSet::new(); - for (_, dep) in module.dependencies.iter() { - if let Some(code_dep) = &dep.maybe_code { - deps.insert(code_dep.clone()); + .filter_map(|(specifier, module_slot)| { + if let ModuleSlot::Module(module) = module_slot { + let mut deps = BTreeSet::new(); + for (_, dep) in module.dependencies.iter() { + if let Some(code_dep) = &dep.maybe_code { + deps.insert(code_dep.clone()); + } + if let Some(type_dep) = &dep.maybe_type { + deps.insert(type_dep.clone()); + } } - if let Some(type_dep) = &dep.maybe_type { - deps.insert(type_dep.clone()); + if let Some((_, types_dep)) = &module.maybe_types { + deps.insert(types_dep.clone()); } + let item = ModuleInfoMapItem { + deps: deps.into_iter().collect(), + size: module.size(), + }; + Some((specifier.clone(), item)) + } else { + None } - if let Some((_, types_dep)) = &module.maybe_types { - deps.insert(types_dep.clone()); - } - let item = ModuleInfoMapItem { - deps: deps.into_iter().collect(), - size: module.size(), - }; - (specifier.clone(), item) }) .collect(); ModuleInfoMap::new(map) } + /// Retrieve a map that contains a representation of each module in the graph + /// which can be used to provide code to a module loader without holding all + /// the state to be able to operate on the graph. + pub fn get_loadable_modules( + &self, + ) -> HashMap<ModuleSpecifier, Result<ModuleSource, AnyError>> { + let mut loadable_modules: HashMap< + ModuleSpecifier, + Result<ModuleSource, AnyError>, + > = self.modules.iter().map(to_module_result).collect(); + for (specifier, _) in self.redirects.iter() { + if let Some(module_slot) = + self.modules.get(self.resolve_specifier(specifier)) + { + let (_, result) = to_module_result((specifier, module_slot)); + loadable_modules.insert(specifier.clone(), result); + } + } + loadable_modules + } + pub fn get_media_type( &self, specifier: &ModuleSpecifier, ) -> Option<MediaType> { - if let Some(module) = self.get_module(specifier) { + if let ModuleSlot::Module(module) = self.get_module(specifier) { Some(module.media_type) } else { None } } - fn get_module(&self, specifier: &ModuleSpecifier) -> Option<&Module> { + fn get_module(&self, specifier: &ModuleSpecifier) -> &ModuleSlot { let s = self.resolve_specifier(specifier); - self.modules.get(s) + if let Some(module_slot) = self.modules.get(s) { + module_slot + } else { + &ModuleSlot::None + } } fn get_module_mut( &mut self, specifier: &ModuleSpecifier, - ) -> Option<&mut Module> { + ) -> Option<&mut ModuleSlot> { // this is duplicated code because `.resolve_specifier` requires an // immutable borrow, but if `.resolve_specifier` is mut, then everything // that calls it is is mut @@ -1174,12 +1269,14 @@ impl Graph { // files will not get emitted. To counter act that behavior, we will // include all modules that are emittable. let mut specifiers = HashSet::<&ModuleSpecifier>::new(); - for (_, module) in self.modules.iter() { - if module.media_type == MediaType::JSX - || module.media_type == MediaType::TypeScript - || module.media_type == MediaType::TSX - { - specifiers.insert(&module.specifier); + for (_, module_slot) in self.modules.iter() { + if let ModuleSlot::Module(module) = module_slot { + if module.media_type == MediaType::JSX + || module.media_type == MediaType::TypeScript + || module.media_type == MediaType::TSX + { + specifiers.insert(&module.specifier); + } } } // We should include all the original roots as well. @@ -1196,7 +1293,12 @@ impl Graph { // if the root module has a types specifier, we should be sending that // to tsc instead of the original specifier let specifier = self.resolve_specifier(ms); - let module = self.get_module(specifier).unwrap(); + let module = + if let ModuleSlot::Module(module) = self.get_module(specifier) { + module + } else { + panic!("missing module"); + }; let specifier = if let Some((_, types_specifier)) = &module.maybe_types { self.resolve_specifier(types_specifier) @@ -1216,7 +1318,7 @@ impl Graph { /// Get the source for a given module specifier. If the module is not part /// of the graph, the result will be `None`. pub fn get_source(&self, specifier: &ModuleSpecifier) -> Option<String> { - if let Some(module) = self.get_module(specifier) { + if let ModuleSlot::Module(module) = self.get_module(specifier) { Some(module.source.clone()) } else { None @@ -1232,7 +1334,11 @@ impl Graph { } let module = self.roots[0].clone(); - let m = self.get_module(&module).unwrap(); + let m = if let ModuleSlot::Module(module) = self.get_module(&module) { + module + } else { + return Err(GraphError::MissingSpecifier(module.clone()).into()); + }; let mut seen = HashSet::new(); let mut totals = HashMap::new(); @@ -1247,9 +1353,19 @@ impl Graph { (None, None) }; + let dep_count = self + .modules + .iter() + .filter_map(|(_, m)| match m { + ModuleSlot::Module(_) => Some(1), + _ => None, + }) + .count() + - 1; + Ok(ModuleGraphInfo { compiled, - dep_count: self.modules.len() - 1, + dep_count, file_type: m.media_type, files, info, @@ -1267,15 +1383,19 @@ impl Graph { let check_js = config.get_check_js(); let config = config.as_bytes(); self.modules.iter().all(|(_, m)| { - let needs_emit = match m.media_type { - MediaType::TypeScript | MediaType::TSX | MediaType::JSX => true, - MediaType::JavaScript => check_js, - _ => false, - }; - if needs_emit { - m.is_emit_valid(&config) + if let ModuleSlot::Module(m) = m { + let needs_emit = match m.media_type { + MediaType::TypeScript | MediaType::TSX | MediaType::JSX => true, + MediaType::JavaScript => check_js, + _ => false, + }; + if needs_emit { + m.is_emit_valid(&config) + } else { + true + } } else { - true + false } }) } @@ -1286,15 +1406,17 @@ impl Graph { pub fn lock(&self) { if let Some(lf) = self.maybe_lockfile.as_ref() { let mut lockfile = lf.lock().unwrap(); - for (ms, module) in self.modules.iter() { - let specifier = module.specifier.to_string(); - let valid = lockfile.check_or_insert(&specifier, &module.source); - if !valid { - eprintln!( - "{}", - GraphError::InvalidSource(ms.clone(), lockfile.filename.clone()) - ); - std::process::exit(10); + for (ms, module_slot) in self.modules.iter() { + if let ModuleSlot::Module(module) = module_slot { + let specifier = module.specifier.to_string(); + let valid = lockfile.check_or_insert(&specifier, &module.source); + if !valid { + eprintln!( + "{}", + GraphError::InvalidSource(ms.clone(), lockfile.filename.clone()) + ); + std::process::exit(10); + } } } } @@ -1305,9 +1427,12 @@ impl Graph { /// checked to determine if it is valid. fn needs_emit(&self, config: &TsConfig) -> bool { let check_js = config.get_check_js(); - self.modules.iter().any(|(_, m)| match m.media_type { - MediaType::TypeScript | MediaType::TSX | MediaType::JSX => true, - MediaType::JavaScript => check_js, + self.modules.iter().any(|(_, m)| match m { + ModuleSlot::Module(m) => match m.media_type { + MediaType::TypeScript | MediaType::TSX | MediaType::JSX => true, + MediaType::JavaScript => check_js, + _ => false, + }, _ => false, }) } @@ -1332,10 +1457,11 @@ impl Graph { referrer: &ModuleSpecifier, prefer_types: bool, ) -> Result<ModuleSpecifier, AnyError> { - if !self.contains_module(referrer) { - return Err(GraphError::MissingSpecifier(referrer.to_owned()).into()); - } - let module = self.get_module(referrer).unwrap(); + let module = if let ModuleSlot::Module(module) = self.get_module(referrer) { + module + } else { + return Err(GraphError::MissingSpecifier(referrer.clone()).into()); + }; if !module.dependencies.contains_key(specifier) { return Err( GraphError::MissingDependency( @@ -1363,7 +1489,11 @@ impl Graph { .into(), ); }; - if !self.contains_module(&resolved_specifier) { + let dep_module = if let ModuleSlot::Module(dep_module) = + self.get_module(&resolved_specifier) + { + dep_module + } else { return Err( GraphError::MissingDependency( referrer.to_owned(), @@ -1371,8 +1501,7 @@ impl Graph { ) .into(), ); - } - let dep_module = self.get_module(&resolved_specifier).unwrap(); + }; // In the case that there is a X-TypeScript-Types or a triple-slash types, // then the `maybe_types` specifier will be populated and we should use that // instead. @@ -1424,7 +1553,7 @@ impl Graph { pub fn transpile( &mut self, options: TranspileOptions, - ) -> Result<(Stats, Option<IgnoredCompilerOptions>), AnyError> { + ) -> Result<ResultInfo, AnyError> { let start = Instant::now(); let mut ts_config = TsConfig::new(json!({ @@ -1443,37 +1572,39 @@ impl Graph { let mut emit_count: u128 = 0; let config = ts_config.as_bytes(); - for (_, module) in self.modules.iter_mut() { - // TODO(kitsonk) a lot of this logic should be refactored into `Module` as - // we start to support other methods on the graph. Especially managing - // the dirty state is something the module itself should "own". - - // if the module is a Dts file we should skip it - if module.media_type == MediaType::Dts { - continue; - } - // if we don't have check_js enabled, we won't touch non TypeScript or JSX - // modules - if !(emit_options.check_js - || module.media_type == MediaType::JSX - || module.media_type == MediaType::TSX - || module.media_type == MediaType::TypeScript) - { - continue; - } - // skip modules that already have a valid emit - if !options.reload && module.is_emit_valid(&config) { - continue; - } - if module.maybe_parsed_module.is_none() { - module.parse()?; + for (_, module_slot) in self.modules.iter_mut() { + if let ModuleSlot::Module(module) = module_slot { + // TODO(kitsonk) a lot of this logic should be refactored into `Module` as + // we start to support other methods on the graph. Especially managing + // the dirty state is something the module itself should "own". + + // if the module is a Dts file we should skip it + if module.media_type == MediaType::Dts { + continue; + } + // if we don't have check_js enabled, we won't touch non TypeScript or JSX + // modules + if !(emit_options.check_js + || module.media_type == MediaType::JSX + || module.media_type == MediaType::TSX + || module.media_type == MediaType::TypeScript) + { + continue; + } + // skip modules that already have a valid emit + if !options.reload && module.is_emit_valid(&config) { + continue; + } + if module.maybe_parsed_module.is_none() { + module.parse()?; + } + let parsed_module = module.maybe_parsed_module.clone().unwrap(); + let emit = parsed_module.transpile(&emit_options)?; + emit_count += 1; + module.maybe_emit = Some(Emit::Cli(emit)); + module.set_version(&config); + module.is_dirty = true; } - let parsed_module = module.maybe_parsed_module.clone().unwrap(); - let emit = parsed_module.transpile(&emit_options)?; - emit_count += 1; - module.maybe_emit = Some(Emit::Cli(emit)); - module.set_version(&config); - module.is_dirty = true; } self.flush()?; @@ -1483,7 +1614,12 @@ impl Graph { ("Total time".to_string(), start.elapsed().as_millis()), ]); - Ok((stats, maybe_ignored_options)) + Ok(ResultInfo { + diagnostics: Default::default(), + loadable_modules: self.get_loadable_modules(), + maybe_ignored_options, + stats, + }) } } @@ -1510,7 +1646,6 @@ impl swc_bundler::Resolve for Graph { /// A structure for building a dependency graph of modules. pub struct GraphBuilder { - fetched: HashSet<ModuleSpecifier>, graph: Graph, maybe_import_map: Option<Rc<RefCell<ImportMap>>>, pending: FuturesUnordered<FetchFuture>, @@ -1529,7 +1664,6 @@ impl GraphBuilder { }; GraphBuilder { graph: Graph::new(handler, maybe_lockfile), - fetched: HashSet::new(), maybe_import_map: internal_import_map, pending: FuturesUnordered::new(), } @@ -1543,12 +1677,22 @@ impl GraphBuilder { specifier: &ModuleSpecifier, is_dynamic: bool, ) -> Result<(), AnyError> { - self.fetch(specifier, &None, is_dynamic)?; + self.fetch(specifier, &None, is_dynamic); loop { - let cached_module = self.pending.next().await.unwrap()?; - let is_root = &cached_module.specifier == specifier; - self.visit(cached_module, is_root)?; + match self.pending.next().await { + Some(Err((specifier, err))) => { + self + .graph + .modules + .insert(specifier, ModuleSlot::Err(Rc::new(err))); + } + Some(Ok(cached_module)) => { + let is_root = &cached_module.specifier == specifier; + self.visit(cached_module, is_root)?; + } + _ => {} + } if self.pending.is_empty() { break; } @@ -1573,20 +1717,19 @@ impl GraphBuilder { specifier: &ModuleSpecifier, maybe_referrer: &Option<Location>, is_dynamic: bool, - ) -> Result<(), AnyError> { - if self.fetched.contains(&specifier) { - return Ok(()); + ) { + if !self.graph.modules.contains_key(&specifier) { + self + .graph + .modules + .insert(specifier.clone(), ModuleSlot::Pending); + let future = self.graph.handler.borrow_mut().fetch( + specifier.clone(), + maybe_referrer.clone(), + is_dynamic, + ); + self.pending.push(future); } - - self.fetched.insert(specifier.clone()); - let future = self.graph.handler.borrow_mut().fetch( - specifier.clone(), - maybe_referrer.clone(), - is_dynamic, - ); - self.pending.push(future); - - Ok(()) } /// Visit a module that has been fetched, hydrating the module, analyzing its @@ -1632,14 +1775,14 @@ impl GraphBuilder { for (_, dep) in module.dependencies.iter() { let maybe_referrer = Some(dep.location.clone()); if let Some(specifier) = dep.maybe_code.as_ref() { - self.fetch(specifier, &maybe_referrer, dep.is_dynamic)?; + self.fetch(specifier, &maybe_referrer, dep.is_dynamic); } if let Some(specifier) = dep.maybe_type.as_ref() { - self.fetch(specifier, &maybe_referrer, dep.is_dynamic)?; + self.fetch(specifier, &maybe_referrer, dep.is_dynamic); } } if let Some((_, specifier)) = module.maybe_types.as_ref() { - self.fetch(specifier, &None, false)?; + self.fetch(specifier, &None, false); } if specifier != requested_specifier { self @@ -1647,7 +1790,10 @@ impl GraphBuilder { .redirects .insert(requested_specifier, specifier.clone()); } - self.graph.modules.insert(specifier, module); + self + .graph + .modules + .insert(specifier, ModuleSlot::Module(Box::new(module))); Ok(()) } @@ -1702,7 +1848,7 @@ pub mod tests { fn get_cache( &self, specifier: ModuleSpecifier, - ) -> Result<CachedModule, AnyError> { + ) -> Result<CachedModule, (ModuleSpecifier, AnyError)> { let specifier_text = specifier .to_string() .replace(":///", "_") @@ -1710,7 +1856,8 @@ pub mod tests { .replace("/", "-"); let source_path = self.fixtures.join(specifier_text); let media_type = MediaType::from(&source_path); - let source = fs::read_to_string(&source_path)?; + let source = fs::read_to_string(&source_path) + .map_err(|err| (specifier.clone(), err.into()))?; let is_remote = specifier.as_url().scheme() != "file"; Ok(CachedModule { @@ -2280,10 +2427,9 @@ pub mod tests { ModuleSpecifier::resolve_url_or_path("file:///tests/main.ts") .expect("could not resolve module"); let (mut graph, handler) = setup(specifier).await; - let (stats, maybe_ignored_options) = - graph.transpile(TranspileOptions::default()).unwrap(); - assert_eq!(stats.0.len(), 3); - assert_eq!(maybe_ignored_options, None); + let result_info = graph.transpile(TranspileOptions::default()).unwrap(); + assert_eq!(result_info.stats.0.len(), 3); + assert_eq!(result_info.maybe_ignored_options, None); let h = handler.borrow(); assert_eq!(h.cache_calls.len(), 2); match &h.cache_calls[0].1 { @@ -2334,7 +2480,7 @@ pub mod tests { ModuleSpecifier::resolve_url_or_path("https://deno.land/x/transpile.tsx") .expect("could not resolve module"); let (mut graph, handler) = setup(specifier).await; - let (_, maybe_ignored_options) = graph + let result_info = graph .transpile(TranspileOptions { debug: false, maybe_config_path: Some("tests/module_graph/tsconfig.json".to_string()), @@ -2342,7 +2488,7 @@ pub mod tests { }) .unwrap(); assert_eq!( - maybe_ignored_options.unwrap().items, + result_info.maybe_ignored_options.unwrap().items, vec!["target".to_string()], "the 'target' options should have been ignored" ); |