diff options
Diffstat (limited to 'cli/module_loader.rs')
| -rw-r--r-- | cli/module_loader.rs | 443 |
1 files changed, 407 insertions, 36 deletions
diff --git a/cli/module_loader.rs b/cli/module_loader.rs index 746743525..c2dfaad8f 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -1,18 +1,44 @@ // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. +use crate::args::CliOptions; +use crate::args::DenoSubcommand; +use crate::args::TsConfigType; use crate::args::TsTypeLib; +use crate::args::TypeCheckMode; +use crate::cache::Caches; +use crate::cache::DenoDir; +use crate::cache::ParsedSourceCache; +use crate::cache::TypeCheckCache; +use crate::emit::Emitter; +use crate::graph_util::graph_lock_or_exit; +use crate::graph_util::graph_valid_with_cli_options; +use crate::graph_util::ModuleGraphBuilder; +use crate::graph_util::ModuleGraphContainer; use crate::node; +use crate::node::NodeCodeTranslator; +use crate::node::NodeResolution; +use crate::npm::NpmPackageResolver; +use crate::npm::NpmResolution; +use crate::proc_state::CjsResolutionStore; +use crate::proc_state::FileWatcherReporter; use crate::proc_state::ProcState; +use crate::resolver::CliGraphResolver; +use crate::tools::check; +use crate::util::progress_bar::ProgressBar; use crate::util::text_encoding::code_without_source_map; use crate::util::text_encoding::source_map_from_code; use deno_ast::MediaType; use deno_core::anyhow::anyhow; use deno_core::anyhow::Context; +use deno_core::error::custom_error; +use deno_core::error::generic_error; use deno_core::error::AnyError; use deno_core::futures::future::FutureExt; use deno_core::futures::Future; +use deno_core::parking_lot::Mutex; use deno_core::resolve_url; +use deno_core::resolve_url_or_path; use deno_core::ModuleCode; use deno_core::ModuleLoader; use deno_core::ModuleSource; @@ -21,13 +47,210 @@ use deno_core::ModuleType; use deno_core::OpState; use deno_core::ResolutionKind; use deno_core::SourceMapGetter; +use deno_graph::source::Resolver; use deno_graph::EsmModule; use deno_graph::JsonModule; +use deno_graph::Module; +use deno_graph::Resolution; +use deno_lockfile::Lockfile; +use deno_runtime::deno_node::NodeResolutionMode; use deno_runtime::permissions::PermissionsContainer; +use deno_semver::npm::NpmPackageReqReference; +use std::borrow::Cow; use std::cell::RefCell; +use std::collections::HashSet; use std::pin::Pin; use std::rc::Rc; use std::str; +use std::sync::Arc; + +pub struct ModuleLoadPreparer { + options: Arc<CliOptions>, + caches: Arc<Caches>, + deno_dir: DenoDir, + graph_container: Arc<ModuleGraphContainer>, + lockfile: Option<Arc<Mutex<Lockfile>>>, + maybe_file_watcher_reporter: Option<FileWatcherReporter>, + module_graph_builder: Arc<ModuleGraphBuilder>, + npm_resolver: Arc<NpmPackageResolver>, + parsed_source_cache: Arc<ParsedSourceCache>, + progress_bar: ProgressBar, + resolver: Arc<CliGraphResolver>, +} + +impl ModuleLoadPreparer { + #[allow(clippy::too_many_arguments)] + pub fn new( + options: Arc<CliOptions>, + caches: Arc<Caches>, + deno_dir: DenoDir, + graph_container: Arc<ModuleGraphContainer>, + lockfile: Option<Arc<Mutex<Lockfile>>>, + maybe_file_watcher_reporter: Option<FileWatcherReporter>, + module_graph_builder: Arc<ModuleGraphBuilder>, + npm_resolver: Arc<NpmPackageResolver>, + parsed_source_cache: Arc<ParsedSourceCache>, + progress_bar: ProgressBar, + resolver: Arc<CliGraphResolver>, + ) -> Self { + Self { + options, + caches, + deno_dir, + graph_container, + lockfile, + maybe_file_watcher_reporter, + module_graph_builder, + npm_resolver, + parsed_source_cache, + progress_bar, + resolver, + } + } + + /// This method must be called for a module or a static importer of that + /// module before attempting to `load()` it from a `JsRuntime`. It will + /// populate the graph data in memory with the necessary source code, write + /// emits where necessary or report any module graph / type checking errors. + #[allow(clippy::too_many_arguments)] + pub async fn prepare_module_load( + &self, + roots: Vec<ModuleSpecifier>, + is_dynamic: bool, + lib: TsTypeLib, + root_permissions: PermissionsContainer, + dynamic_permissions: PermissionsContainer, + ) -> Result<(), AnyError> { + log::debug!("Preparing module load."); + let _pb_clear_guard = self.progress_bar.clear_guard(); + + let mut cache = self + .module_graph_builder + .create_fetch_cacher(root_permissions, dynamic_permissions); + let maybe_imports = self.options.to_maybe_imports()?; + let graph_resolver = self.resolver.as_graph_resolver(); + let graph_npm_resolver = self.resolver.as_graph_npm_resolver(); + let maybe_file_watcher_reporter: Option<&dyn deno_graph::source::Reporter> = + if let Some(reporter) = &self.maybe_file_watcher_reporter { + Some(reporter) + } else { + None + }; + + let analyzer = self.parsed_source_cache.as_analyzer(); + + log::debug!("Creating module graph."); + let mut graph_update_permit = + self.graph_container.acquire_update_permit().await; + let graph = graph_update_permit.graph_mut(); + + // Determine any modules that have already been emitted this session and + // should be skipped. + let reload_exclusions: HashSet<ModuleSpecifier> = + graph.specifiers().map(|(s, _)| s.clone()).collect(); + + self + .module_graph_builder + .build_graph_with_npm_resolution( + graph, + roots.clone(), + &mut cache, + deno_graph::BuildOptions { + is_dynamic, + imports: maybe_imports, + resolver: Some(graph_resolver), + npm_resolver: Some(graph_npm_resolver), + module_analyzer: Some(&*analyzer), + reporter: maybe_file_watcher_reporter, + }, + ) + .await?; + + // If there is a lockfile, validate the integrity of all the modules. + if let Some(lockfile) = &self.lockfile { + graph_lock_or_exit(graph, &mut lockfile.lock()); + } + + graph_valid_with_cli_options(graph, &roots, &self.options)?; + // save the graph and get a reference to the new graph + let graph = graph_update_permit.commit(); + + if graph.has_node_specifier + && self.options.type_check_mode() != TypeCheckMode::None + { + self + .npm_resolver + .inject_synthetic_types_node_package() + .await?; + } + + drop(_pb_clear_guard); + + // type check if necessary + if self.options.type_check_mode() != TypeCheckMode::None + && !self.graph_container.is_type_checked(&roots, lib) + { + // todo(dsherret): consolidate this with what's done in graph_util + log::debug!("Type checking."); + let maybe_config_specifier = self.options.maybe_config_file_specifier(); + let graph = Arc::new(graph.segment(&roots)); + let options = check::CheckOptions { + type_check_mode: self.options.type_check_mode(), + debug: self.options.log_level() == Some(log::Level::Debug), + maybe_config_specifier, + ts_config: self + .options + .resolve_ts_config_for_emit(TsConfigType::Check { lib })? + .ts_config, + log_checks: true, + reload: self.options.reload_flag() + && !roots.iter().all(|r| reload_exclusions.contains(r)), + }; + let check_cache = + TypeCheckCache::new(self.caches.type_checking_cache_db(&self.deno_dir)); + let check_result = + check::check(graph, &check_cache, self.npm_resolver.clone(), options)?; + self.graph_container.set_type_checked(&roots, lib); + if !check_result.diagnostics.is_empty() { + return Err(anyhow!(check_result.diagnostics)); + } + log::debug!("{}", check_result.stats); + } + + // any updates to the lockfile should be updated now + if let Some(ref lockfile) = self.lockfile { + let g = lockfile.lock(); + g.write()?; + } + + log::debug!("Prepared module load."); + + Ok(()) + } + + /// Helper around prepare_module_load that loads and type checks + /// the provided files. + pub async fn load_and_type_check_files( + &self, + files: &[String], + ) -> Result<(), AnyError> { + let lib = self.options.ts_type_lib_window(); + + let specifiers = files + .iter() + .map(|file| resolve_url_or_path(file, self.options.initial_cwd())) + .collect::<Result<Vec<_>, _>>()?; + self + .prepare_module_load( + specifiers, + false, + lib, + PermissionsContainer::allow_all(), + PermissionsContainer::allow_all(), + ) + .await + } +} struct ModuleCodeSource { pub code: ModuleCode, @@ -36,15 +259,24 @@ struct ModuleCodeSource { } pub struct CliModuleLoader { - pub lib: TsTypeLib, + lib: TsTypeLib, /// The initial set of permissions used to resolve the static imports in the /// worker. These are "allow all" for main worker, and parent thread /// permissions for Web Worker. - pub root_permissions: PermissionsContainer, + root_permissions: PermissionsContainer, /// Permissions used to resolve dynamic imports, these get passed as /// "root permissions" for Web Worker. dynamic_permissions: PermissionsContainer, - pub ps: ProcState, + cli_options: Arc<CliOptions>, + cjs_resolutions: Arc<CjsResolutionStore>, + emitter: Arc<Emitter>, + graph_container: Arc<ModuleGraphContainer>, + module_load_preparer: Arc<ModuleLoadPreparer>, + node_code_translator: Arc<NodeCodeTranslator>, + npm_resolution: Arc<NpmResolution>, + npm_resolver: Arc<NpmPackageResolver>, + parsed_source_cache: Arc<ParsedSourceCache>, + resolver: Arc<CliGraphResolver>, } impl CliModuleLoader { @@ -57,7 +289,16 @@ impl CliModuleLoader { lib: ps.options.ts_type_lib_window(), root_permissions, dynamic_permissions, - ps, + cli_options: ps.options.clone(), + cjs_resolutions: ps.cjs_resolutions.clone(), + emitter: ps.emitter.clone(), + graph_container: ps.graph_container.clone(), + module_load_preparer: ps.module_load_preparer.clone(), + node_code_translator: ps.node_code_translator.clone(), + npm_resolution: ps.npm_resolution.clone(), + npm_resolver: ps.npm_resolver.clone(), + parsed_source_cache: ps.parsed_source_cache.clone(), + resolver: ps.resolver.clone(), }) } @@ -70,7 +311,16 @@ impl CliModuleLoader { lib: ps.options.ts_type_lib_worker(), root_permissions, dynamic_permissions, - ps, + cli_options: ps.options.clone(), + cjs_resolutions: ps.cjs_resolutions.clone(), + emitter: ps.emitter.clone(), + graph_container: ps.graph_container.clone(), + module_load_preparer: ps.module_load_preparer.clone(), + node_code_translator: ps.node_code_translator.clone(), + npm_resolution: ps.npm_resolution.clone(), + npm_resolver: ps.npm_resolver.clone(), + parsed_source_cache: ps.parsed_source_cache.clone(), + resolver: ps.resolver.clone(), }) } @@ -83,7 +333,7 @@ impl CliModuleLoader { unreachable!(); // Node built-in modules should be handled internally. } - let graph = self.ps.graph(); + let graph = self.graph_container.graph(); match graph.get(specifier) { Some(deno_graph::Module::Json(JsonModule { source, @@ -116,11 +366,9 @@ impl CliModuleLoader { | MediaType::Jsx | MediaType::Tsx => { // get emit text - self.ps.emitter.emit_parsed_source( - specifier, - *media_type, - source, - )? + self + .emitter + .emit_parsed_source(specifier, *media_type, source)? } MediaType::TsBuildInfo | MediaType::Wasm | MediaType::SourceMap => { panic!("Unexpected media type {media_type} for {specifier}") @@ -128,7 +376,7 @@ impl CliModuleLoader { }; // at this point, we no longer need the parsed source in memory, so free it - self.ps.parsed_source_cache.free(specifier); + self.parsed_source_cache.free(specifier); Ok(ModuleCodeSource { code, @@ -152,7 +400,7 @@ impl CliModuleLoader { maybe_referrer: Option<&ModuleSpecifier>, is_dynamic: bool, ) -> Result<ModuleSource, AnyError> { - let code_source = if self.ps.npm_resolver.in_npm_package(specifier) { + let code_source = if self.npm_resolver.in_npm_package(specifier) { let file_path = specifier.to_file_path().unwrap(); let code = std::fs::read_to_string(&file_path).with_context(|| { let mut msg = "Unable to load ".to_string(); @@ -164,29 +412,24 @@ impl CliModuleLoader { msg })?; - let code = if self.ps.cjs_resolutions.lock().contains(specifier) { + let code = if self.cjs_resolutions.contains(specifier) { let mut permissions = if is_dynamic { self.dynamic_permissions.clone() } else { self.root_permissions.clone() }; // translate cjs to esm if it's cjs and inject node globals - node::translate_cjs_to_esm( - &self.ps.file_fetcher, + self.node_code_translator.translate_cjs_to_esm( specifier, code, MediaType::Cjs, - &self.ps.npm_resolver, - &self.ps.node_analysis_cache, &mut permissions, )? } else { // only inject node globals for esm - node::esm_code_with_node_globals( - &self.ps.node_analysis_cache, - specifier, - code, - )? + self + .node_code_translator + .esm_code_with_node_globals(specifier, code)? }; ModuleCodeSource { code: code.into(), @@ -196,7 +439,7 @@ impl CliModuleLoader { } else { self.load_prepared_module(specifier, maybe_referrer)? }; - let code = if self.ps.options.is_inspecting() { + let code = if self.cli_options.is_inspecting() { // we need the code with the source map in order for // it to work with --inspect or --inspect-brk code_source.code @@ -215,6 +458,23 @@ impl CliModuleLoader { &code_source.found_url, )) } + + fn handle_node_resolve_result( + &self, + result: Result<Option<node::NodeResolution>, AnyError>, + ) -> Result<ModuleSpecifier, AnyError> { + let response = match result? { + Some(response) => response, + None => return Err(generic_error("not found")), + }; + if let NodeResolution::CommonJs(specifier) = &response { + // remember that this was a common js resolution + self.cjs_resolutions.insert(specifier.clone()); + } else if let NodeResolution::BuiltIn(specifier) = &response { + return node::resolve_builtin_node_module(specifier); + } + Ok(response.into_url()) + } } impl ModuleLoader for CliModuleLoader { @@ -229,7 +489,117 @@ impl ModuleLoader for CliModuleLoader { } else { self.root_permissions.clone() }; - self.ps.resolve(specifier, referrer, &mut permissions) + + // TODO(bartlomieju): ideally we shouldn't need to call `current_dir()` on each + // call - maybe it should be caller's responsibility to pass it as an arg? + let cwd = std::env::current_dir().context("Unable to get CWD")?; + let referrer_result = deno_core::resolve_url_or_path(referrer, &cwd); + + if let Ok(referrer) = referrer_result.as_ref() { + if self.npm_resolver.in_npm_package(referrer) { + // we're in an npm package, so use node resolution + return self + .handle_node_resolve_result(node::node_resolve( + specifier, + referrer, + NodeResolutionMode::Execution, + &self.npm_resolver.as_require_npm_resolver(), + &mut permissions, + )) + .with_context(|| { + format!("Could not resolve '{specifier}' from '{referrer}'.") + }); + } + + let graph = self.graph_container.graph(); + let maybe_resolved = match graph.get(referrer) { + Some(Module::Esm(module)) => { + module.dependencies.get(specifier).map(|d| &d.maybe_code) + } + _ => None, + }; + + match maybe_resolved { + Some(Resolution::Ok(resolved)) => { + let specifier = &resolved.specifier; + + return match graph.get(specifier) { + Some(Module::Npm(module)) => self + .handle_node_resolve_result(node::node_resolve_npm_reference( + &module.nv_reference, + NodeResolutionMode::Execution, + &self.npm_resolver, + &mut permissions, + )) + .with_context(|| { + format!("Could not resolve '{}'.", module.nv_reference) + }), + Some(Module::Node(module)) => { + node::resolve_builtin_node_module(&module.module_name) + } + Some(Module::Esm(module)) => Ok(module.specifier.clone()), + Some(Module::Json(module)) => Ok(module.specifier.clone()), + Some(Module::External(module)) => { + Ok(node::resolve_specifier_into_node_modules(&module.specifier)) + } + None => Ok(specifier.clone()), + }; + } + Some(Resolution::Err(err)) => { + return Err(custom_error( + "TypeError", + format!("{}\n", err.to_string_with_range()), + )) + } + Some(Resolution::None) | None => {} + } + } + + // Built-in Node modules + if let Some(module_name) = specifier.strip_prefix("node:") { + return node::resolve_builtin_node_module(module_name); + } + + // FIXME(bartlomieju): this is a hacky way to provide compatibility with REPL + // and `Deno.core.evalContext` API. Ideally we should always have a referrer filled + // but sadly that's not the case due to missing APIs in V8. + let is_repl = + matches!(self.cli_options.sub_command(), DenoSubcommand::Repl(_)); + let referrer = if referrer.is_empty() && is_repl { + deno_core::resolve_path("./$deno$repl.ts", &cwd)? + } else { + referrer_result? + }; + + // FIXME(bartlomieju): this is another hack way to provide NPM specifier + // support in REPL. This should be fixed. + let resolution = self.resolver.resolve(specifier, &referrer); + + if is_repl { + let specifier = resolution + .as_ref() + .ok() + .map(Cow::Borrowed) + .or_else(|| ModuleSpecifier::parse(specifier).ok().map(Cow::Owned)); + if let Some(specifier) = specifier { + if let Ok(reference) = + NpmPackageReqReference::from_specifier(&specifier) + { + let reference = + self.npm_resolution.pkg_req_ref_to_nv_ref(reference)?; + return self + .handle_node_resolve_result(node::node_resolve_npm_reference( + &reference, + deno_runtime::deno_node::NodeResolutionMode::Execution, + &self.npm_resolver, + &mut permissions, + )) + .with_context(|| format!("Could not resolve '{reference}'.")); + } + } + } + + resolution } fn load( @@ -255,13 +625,13 @@ impl ModuleLoader for CliModuleLoader { _maybe_referrer: Option<String>, is_dynamic: bool, ) -> Pin<Box<dyn Future<Output = Result<(), AnyError>>>> { - if self.ps.npm_resolver.in_npm_package(specifier) { + if self.npm_resolver.in_npm_package(specifier) { // nothing to prepare return Box::pin(deno_core::futures::future::ready(Ok(()))); } let specifier = specifier.clone(); - let ps = self.ps.clone(); + let module_load_preparer = self.module_load_preparer.clone(); let dynamic_permissions = self.dynamic_permissions.clone(); let root_permissions = if is_dynamic { @@ -272,14 +642,15 @@ impl ModuleLoader for CliModuleLoader { let lib = self.lib; async move { - ps.prepare_module_load( - vec![specifier], - is_dynamic, - lib, - root_permissions, - dynamic_permissions, - ) - .await + module_load_preparer + .prepare_module_load( + vec![specifier], + is_dynamic, + lib, + root_permissions, + dynamic_permissions, + ) + .await } .boxed_local() } @@ -303,7 +674,7 @@ impl SourceMapGetter for CliModuleLoader { file_name: &str, line_number: usize, ) -> Option<String> { - let graph = self.ps.graph(); + let graph = self.graph_container.graph(); let code = match graph.get(&resolve_url(file_name).ok()?) { Some(deno_graph::Module::Esm(module)) => &module.source, Some(deno_graph::Module::Json(module)) => &module.source, |