diff options
author | Divy Srivastava <dj.srivastava23@gmail.com> | 2024-05-16 00:09:35 -0700 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-05-16 07:09:35 +0000 |
commit | 88983fb3eb5a085f7d358a7a98d5c738a21b5d27 (patch) | |
tree | d4d83c5bd668edc25d30616fd4a3decc1cea3fb9 /cli/module_loader.rs | |
parent | bba553bea5938932518dc6382e464968ce8374b4 (diff) |
fix(node): seperate worker module cache (#23634)
Construct a new module graph container for workers instead of sharing it
with the main worker.
Fixes #17248
Fixes #23461
---------
Co-authored-by: David Sherret <dsherret@gmail.com>
Diffstat (limited to 'cli/module_loader.rs')
-rw-r--r-- | cli/module_loader.rs | 437 |
1 files changed, 223 insertions, 214 deletions
diff --git a/cli/module_loader.rs b/cli/module_loader.rs index 7d8cb130b..9a8441ccd 100644 --- a/cli/module_loader.rs +++ b/cli/module_loader.rs @@ -1,5 +1,12 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +use std::borrow::Cow; +use std::cell::RefCell; +use std::pin::Pin; +use std::rc::Rc; +use std::str; +use std::sync::Arc; + use crate::args::jsr_url; use crate::args::CliOptions; use crate::args::DenoSubcommand; @@ -9,10 +16,12 @@ use crate::cache::ModuleInfoCache; use crate::cache::ParsedSourceCache; use crate::emit::Emitter; use crate::factory::CliFactory; +use crate::graph_container::MainModuleGraphContainer; +use crate::graph_container::ModuleGraphContainer; +use crate::graph_container::ModuleGraphUpdatePermit; use crate::graph_util::graph_lock_or_exit; use crate::graph_util::CreateGraphOptions; use crate::graph_util::ModuleGraphBuilder; -use crate::graph_util::ModuleGraphContainer; use crate::node; use crate::resolver::CliGraphResolver; use crate::resolver::CliNodeResolver; @@ -23,6 +32,7 @@ use crate::tools::check::TypeChecker; use crate::util::progress_bar::ProgressBar; use crate::util::text_encoding::code_without_source_map; use crate::util::text_encoding::source_map_from_code; +use crate::worker::ModuleLoaderAndSourceMapGetter; use crate::worker::ModuleLoaderFactory; use deno_ast::MediaType; @@ -36,7 +46,6 @@ use deno_core::futures::future::FutureExt; use deno_core::futures::Future; use deno_core::parking_lot::Mutex; use deno_core::resolve_url; -use deno_core::resolve_url_or_path; use deno_core::ModuleCodeString; use deno_core::ModuleLoader; use deno_core::ModuleSource; @@ -48,9 +57,11 @@ use deno_core::ResolutionKind; use deno_core::SourceMapGetter; use deno_graph::source::ResolutionMode; use deno_graph::source::Resolver; +use deno_graph::GraphKind; use deno_graph::JsModule; use deno_graph::JsonModule; use deno_graph::Module; +use deno_graph::ModuleGraph; use deno_graph::Resolution; use deno_lockfile::Lockfile; use deno_runtime::code_cache; @@ -58,12 +69,6 @@ use deno_runtime::deno_node::NodeResolutionMode; use deno_runtime::fs_util::code_timestamp; use deno_runtime::permissions::PermissionsContainer; use deno_semver::npm::NpmPackageReqReference; -use deno_terminal::colors; -use std::borrow::Cow; -use std::pin::Pin; -use std::rc::Rc; -use std::str; -use std::sync::Arc; pub async fn load_top_level_deps(factory: &CliFactory) -> Result<(), AnyError> { let npm_resolver = factory.npm_resolver().await?; @@ -83,12 +88,19 @@ pub async fn load_top_level_deps(factory: &CliFactory) -> Result<(), AnyError> { entry.value.cloned() } }) - .collect(); + .collect::<Vec<_>>(); + let mut graph_permit = factory + .main_module_graph_container() + .await? + .acquire_update_permit() + .await; + let graph = graph_permit.graph_mut(); factory .module_load_preparer() .await? .prepare_module_load( - roots, + graph, + &roots, false, factory.cli_options().ts_type_lib_window(), deno_runtime::permissions::PermissionsContainer::allow_all(), @@ -101,7 +113,6 @@ pub async fn load_top_level_deps(factory: &CliFactory) -> Result<(), AnyError> { pub struct ModuleLoadPreparer { options: Arc<CliOptions>, - graph_container: Arc<ModuleGraphContainer>, lockfile: Option<Arc<Mutex<Lockfile>>>, module_graph_builder: Arc<ModuleGraphBuilder>, progress_bar: ProgressBar, @@ -112,7 +123,6 @@ impl ModuleLoadPreparer { #[allow(clippy::too_many_arguments)] pub fn new( options: Arc<CliOptions>, - graph_container: Arc<ModuleGraphContainer>, lockfile: Option<Arc<Mutex<Lockfile>>>, module_graph_builder: Arc<ModuleGraphBuilder>, progress_bar: ProgressBar, @@ -120,7 +130,6 @@ impl ModuleLoadPreparer { ) -> Self { Self { options, - graph_container, lockfile, module_graph_builder, progress_bar, @@ -135,7 +144,8 @@ impl ModuleLoadPreparer { #[allow(clippy::too_many_arguments)] pub async fn prepare_module_load( &self, - roots: Vec<ModuleSpecifier>, + graph: &mut ModuleGraph, + roots: &[ModuleSpecifier], is_dynamic: bool, lib: TsTypeLib, permissions: PermissionsContainer, @@ -144,10 +154,7 @@ impl ModuleLoadPreparer { let _pb_clear_guard = self.progress_bar.clear_guard(); let mut cache = self.module_graph_builder.create_fetch_cacher(permissions); - log::debug!("Creating module graph."); - let mut graph_update_permit = - self.graph_container.acquire_update_permit().await; - let graph = graph_update_permit.graph_mut(); + log::debug!("Building module graph."); let has_type_checked = !graph.roots.is_empty(); self @@ -157,13 +164,13 @@ impl ModuleLoadPreparer { CreateGraphOptions { is_dynamic, graph_kind: graph.graph_kind(), - roots: roots.clone(), + roots: roots.to_vec(), loader: Some(&mut cache), }, ) .await?; - self.module_graph_builder.graph_roots_valid(graph, &roots)?; + self.module_graph_builder.graph_roots_valid(graph, roots)?; // If there is a lockfile... if let Some(lockfile) = &self.lockfile { @@ -174,9 +181,6 @@ impl ModuleLoadPreparer { lockfile.write().context("Failed writing lockfile.")?; } - // save the graph and get a reference to the new graph - let graph = graph_update_permit.commit(); - drop(_pb_clear_guard); // type check if necessary @@ -188,7 +192,7 @@ impl ModuleLoadPreparer { // created, we could avoid the clone of the graph here by providing // the actual graph on the first run and then getting the Arc<ModuleGraph> // back from the return value. - (*graph).clone(), + graph.clone(), check::CheckOptions { build_fast_check_graph: true, lib, @@ -204,154 +208,23 @@ impl ModuleLoadPreparer { Ok(()) } - - /// Helper around prepare_module_load that loads and type checks - /// the provided files. - pub async fn load_and_type_check_files( - &self, - files: &[String], - ) -> Result<(), AnyError> { - let lib = self.options.ts_type_lib_window(); - - let specifiers = self.collect_specifiers(files)?; - - if specifiers.is_empty() { - log::warn!("{} No matching files found.", colors::yellow("Warning")); - } - - self - .prepare_module_load( - specifiers, - false, - lib, - PermissionsContainer::allow_all(), - ) - .await - } - - fn collect_specifiers( - &self, - files: &[String], - ) -> Result<Vec<ModuleSpecifier>, AnyError> { - let excludes = self.options.resolve_config_excludes()?; - Ok( - files - .iter() - .filter_map(|file| { - let file_url = - resolve_url_or_path(file, self.options.initial_cwd()).ok()?; - if file_url.scheme() != "file" { - return Some(file_url); - } - // ignore local files that match any of files listed in `exclude` option - let file_path = file_url.to_file_path().ok()?; - if excludes.matches_path(&file_path) { - None - } else { - Some(file_url) - } - }) - .collect::<Vec<_>>(), - ) - } -} - -struct PreparedModuleLoader { - emitter: Arc<Emitter>, - graph_container: Arc<ModuleGraphContainer>, - parsed_source_cache: Arc<ParsedSourceCache>, -} - -impl PreparedModuleLoader { - pub fn load_prepared_module( - &self, - specifier: &ModuleSpecifier, - maybe_referrer: Option<&ModuleSpecifier>, - ) -> Result<ModuleCodeStringSource, AnyError> { - if specifier.scheme() == "node" { - unreachable!(); // Node built-in modules should be handled internally. - } - - let graph = self.graph_container.graph(); - match graph.get(specifier) { - Some(deno_graph::Module::Json(JsonModule { - source, - media_type, - specifier, - .. - })) => Ok(ModuleCodeStringSource { - code: source.clone().into(), - found_url: specifier.clone(), - media_type: *media_type, - }), - Some(deno_graph::Module::Js(JsModule { - source, - media_type, - specifier, - .. - })) => { - let code: ModuleCodeString = match media_type { - MediaType::JavaScript - | MediaType::Unknown - | MediaType::Cjs - | MediaType::Mjs - | MediaType::Json => source.clone().into(), - MediaType::Dts | MediaType::Dcts | MediaType::Dmts => { - Default::default() - } - MediaType::TypeScript - | MediaType::Mts - | MediaType::Cts - | MediaType::Jsx - | MediaType::Tsx => { - // get emit text - self - .emitter - .emit_parsed_source(specifier, *media_type, source)? - } - MediaType::TsBuildInfo | MediaType::Wasm | MediaType::SourceMap => { - panic!("Unexpected media type {media_type} for {specifier}") - } - }; - - // at this point, we no longer need the parsed source in memory, so free it - self.parsed_source_cache.free(specifier); - - Ok(ModuleCodeStringSource { - code, - found_url: specifier.clone(), - media_type: *media_type, - }) - } - Some( - deno_graph::Module::External(_) - | deno_graph::Module::Node(_) - | deno_graph::Module::Npm(_), - ) - | None => { - let mut msg = format!("Loading unprepared module: {specifier}"); - if let Some(referrer) = maybe_referrer { - msg = format!("{}, imported from: {}", msg, referrer.as_str()); - } - Err(anyhow!(msg)) - } - } - } } struct SharedCliModuleLoaderState { + graph_kind: GraphKind, lib_window: TsTypeLib, lib_worker: TsTypeLib, is_inspecting: bool, is_repl: bool, - graph_container: Arc<ModuleGraphContainer>, + code_cache: Option<Arc<CodeCache>>, + emitter: Arc<Emitter>, + main_module_graph_container: Arc<MainModuleGraphContainer>, + module_info_cache: Arc<ModuleInfoCache>, module_load_preparer: Arc<ModuleLoadPreparer>, - prepared_module_loader: PreparedModuleLoader, - resolver: Arc<CliGraphResolver>, node_resolver: Arc<CliNodeResolver>, npm_module_loader: NpmModuleLoader, - code_cache: Option<Arc<CodeCache>>, - module_info_cache: Arc<ModuleInfoCache>, + parsed_source_cache: Arc<ParsedSourceCache>, + resolver: Arc<CliGraphResolver>, } pub struct CliModuleLoaderFactory { @@ -362,18 +235,19 @@ impl CliModuleLoaderFactory { #[allow(clippy::too_many_arguments)] pub fn new( options: &CliOptions, + code_cache: Option<Arc<CodeCache>>, emitter: Arc<Emitter>, - graph_container: Arc<ModuleGraphContainer>, + main_module_graph_container: Arc<MainModuleGraphContainer>, + module_info_cache: Arc<ModuleInfoCache>, module_load_preparer: Arc<ModuleLoadPreparer>, - parsed_source_cache: Arc<ParsedSourceCache>, - resolver: Arc<CliGraphResolver>, node_resolver: Arc<CliNodeResolver>, npm_module_loader: NpmModuleLoader, - code_cache: Option<Arc<CodeCache>>, - module_info_cache: Arc<ModuleInfoCache>, + parsed_source_cache: Arc<ParsedSourceCache>, + resolver: Arc<CliGraphResolver>, ) -> Self { Self { shared: Arc::new(SharedCliModuleLoaderState { + graph_kind: options.graph_kind(), lib_window: options.ts_type_lib_window(), lib_worker: options.ts_type_lib_worker(), is_inspecting: options.is_inspecting(), @@ -381,34 +255,39 @@ impl CliModuleLoaderFactory { options.sub_command(), DenoSubcommand::Repl(_) | DenoSubcommand::Jupyter(_) ), - prepared_module_loader: PreparedModuleLoader { - emitter, - graph_container: graph_container.clone(), - parsed_source_cache, - }, - graph_container, + code_cache, + emitter, + main_module_graph_container, + module_info_cache, module_load_preparer, - resolver, node_resolver, npm_module_loader, - code_cache, - module_info_cache, + parsed_source_cache, + resolver, }), } } - fn create_with_lib( + fn create_with_lib<TGraphContainer: ModuleGraphContainer>( &self, + graph_container: TGraphContainer, lib: TsTypeLib, root_permissions: PermissionsContainer, dynamic_permissions: PermissionsContainer, - ) -> Rc<dyn ModuleLoader> { - Rc::new(CliModuleLoader { + ) -> ModuleLoaderAndSourceMapGetter { + let loader = Rc::new(CliModuleLoader { lib, root_permissions, dynamic_permissions, + graph_container, + emitter: self.shared.emitter.clone(), + parsed_source_cache: self.shared.parsed_source_cache.clone(), shared: self.shared.clone(), - }) + }); + ModuleLoaderAndSourceMapGetter { + module_loader: loader.clone(), + source_map_getter: Some(loader), + } } } @@ -417,8 +296,9 @@ impl ModuleLoaderFactory for CliModuleLoaderFactory { &self, root_permissions: PermissionsContainer, dynamic_permissions: PermissionsContainer, - ) -> Rc<dyn ModuleLoader> { + ) -> ModuleLoaderAndSourceMapGetter { self.create_with_lib( + (*self.shared.main_module_graph_container).clone(), self.shared.lib_window, root_permissions, dynamic_permissions, @@ -429,22 +309,20 @@ impl ModuleLoaderFactory for CliModuleLoaderFactory { &self, root_permissions: PermissionsContainer, dynamic_permissions: PermissionsContainer, - ) -> Rc<dyn ModuleLoader> { + ) -> ModuleLoaderAndSourceMapGetter { self.create_with_lib( + // create a fresh module graph for the worker + WorkerModuleGraphContainer::new(Arc::new(ModuleGraph::new( + self.shared.graph_kind, + ))), self.shared.lib_worker, root_permissions, dynamic_permissions, ) } - - fn create_source_map_getter(&self) -> Option<Rc<dyn SourceMapGetter>> { - Some(Rc::new(CliSourceMapGetter { - shared: self.shared.clone(), - })) - } } -struct CliModuleLoader { +struct CliModuleLoader<TGraphContainer: ModuleGraphContainer> { lib: TsTypeLib, /// The initial set of permissions used to resolve the static imports in the /// worker. These are "allow all" for main worker, and parent thread @@ -454,9 +332,12 @@ struct CliModuleLoader { /// "root permissions" for Web Worker. dynamic_permissions: PermissionsContainer, shared: Arc<SharedCliModuleLoaderState>, + emitter: Arc<Emitter>, + parsed_source_cache: Arc<ParsedSourceCache>, + graph_container: TGraphContainer, } -impl CliModuleLoader { +impl<TGraphContainer: ModuleGraphContainer> CliModuleLoader<TGraphContainer> { fn load_sync( &self, specifier: &ModuleSpecifier, @@ -476,10 +357,7 @@ impl CliModuleLoader { { result? } else { - self - .shared - .prepared_module_loader - .load_prepared_module(specifier, maybe_referrer)? + self.load_prepared_module(specifier, maybe_referrer)? }; let code = if self.shared.is_inspecting { // we need the code with the source map in order for @@ -581,7 +459,7 @@ impl CliModuleLoader { }; } - let graph = self.shared.graph_container.graph(); + let graph = self.graph_container.graph(); let maybe_resolved = match graph.get(referrer) { Some(Module::Js(module)) => { module.dependencies.get(specifier).map(|d| &d.maybe_code) @@ -695,9 +573,86 @@ impl CliModuleLoader { .map(|timestamp| timestamp.to_string())?; Ok(Some(timestamp)) } + + fn load_prepared_module( + &self, + specifier: &ModuleSpecifier, + maybe_referrer: Option<&ModuleSpecifier>, + ) -> Result<ModuleCodeStringSource, AnyError> { + if specifier.scheme() == "node" { + unreachable!(); // Node built-in modules should be handled internally. + } + + let graph = self.graph_container.graph(); + match graph.get(specifier) { + Some(deno_graph::Module::Json(JsonModule { + source, + media_type, + specifier, + .. + })) => Ok(ModuleCodeStringSource { + code: source.clone().into(), + found_url: specifier.clone(), + media_type: *media_type, + }), + Some(deno_graph::Module::Js(JsModule { + source, + media_type, + specifier, + .. + })) => { + let code: ModuleCodeString = match media_type { + MediaType::JavaScript + | MediaType::Unknown + | MediaType::Cjs + | MediaType::Mjs + | MediaType::Json => source.clone().into(), + MediaType::Dts | MediaType::Dcts | MediaType::Dmts => { + Default::default() + } + MediaType::TypeScript + | MediaType::Mts + | MediaType::Cts + | MediaType::Jsx + | MediaType::Tsx => { + // get emit text + self + .emitter + .emit_parsed_source(specifier, *media_type, source)? + } + MediaType::TsBuildInfo | MediaType::Wasm | MediaType::SourceMap => { + panic!("Unexpected media type {media_type} for {specifier}") + } + }; + + // at this point, we no longer need the parsed source in memory, so free it + self.parsed_source_cache.free(specifier); + + Ok(ModuleCodeStringSource { + code, + found_url: specifier.clone(), + media_type: *media_type, + }) + } + Some( + deno_graph::Module::External(_) + | deno_graph::Module::Node(_) + | deno_graph::Module::Npm(_), + ) + | None => { + let mut msg = format!("Loading unprepared module: {specifier}"); + if let Some(referrer) = maybe_referrer { + msg = format!("{}, imported from: {}", msg, referrer.as_str()); + } + Err(anyhow!(msg)) + } + } + } } -impl ModuleLoader for CliModuleLoader { +impl<TGraphContainer: ModuleGraphContainer> ModuleLoader + for CliModuleLoader<TGraphContainer> +{ fn resolve( &self, specifier: &str, @@ -747,13 +702,12 @@ impl ModuleLoader for CliModuleLoader { _maybe_referrer: Option<String>, is_dynamic: bool, ) -> Pin<Box<dyn Future<Output = Result<(), AnyError>>>> { - if let Some(result) = - self.shared.npm_module_loader.maybe_prepare_load(specifier) - { - return Box::pin(deno_core::futures::future::ready(result)); + if self.shared.node_resolver.in_npm_package(specifier) { + return Box::pin(deno_core::futures::future::ready(Ok(()))); } let specifier = specifier.clone(); + let graph_container = self.graph_container.clone(); let module_load_preparer = self.shared.module_load_preparer.clone(); let root_permissions = if is_dynamic { @@ -764,9 +718,19 @@ impl ModuleLoader for CliModuleLoader { let lib = self.lib; async move { + let mut update_permit = graph_container.acquire_update_permit().await; + let graph = update_permit.graph_mut(); module_load_preparer - .prepare_module_load(vec![specifier], is_dynamic, lib, root_permissions) - .await + .prepare_module_load( + graph, + &[specifier], + is_dynamic, + lib, + root_permissions, + ) + .await?; + update_permit.commit(); + Ok(()) } .boxed_local() } @@ -795,15 +759,13 @@ impl ModuleLoader for CliModuleLoader { ); } } - async {}.boxed_local() + std::future::ready(()).boxed_local() } } -struct CliSourceMapGetter { - shared: Arc<SharedCliModuleLoaderState>, -} - -impl SourceMapGetter for CliSourceMapGetter { +impl<TGraphContainer: ModuleGraphContainer> SourceMapGetter + for CliModuleLoader<TGraphContainer> +{ fn get_source_map(&self, file_name: &str) -> Option<Vec<u8>> { let specifier = resolve_url(file_name).ok()?; match specifier.scheme() { @@ -812,11 +774,7 @@ impl SourceMapGetter for CliSourceMapGetter { "wasm" | "file" | "http" | "https" | "data" | "blob" => (), _ => return None, } - let source = self - .shared - .prepared_module_loader - .load_prepared_module(&specifier, None) - .ok()?; + let source = self.load_prepared_module(&specifier, None).ok()?; source_map_from_code(&source.code) } @@ -825,7 +783,7 @@ impl SourceMapGetter for CliSourceMapGetter { file_name: &str, line_number: usize, ) -> Option<String> { - let graph = self.shared.graph_container.graph(); + let graph = self.graph_container.graph(); let code = match graph.get(&resolve_url(file_name).ok()?) { Some(deno_graph::Module::Js(module)) => &module.source, Some(deno_graph::Module::Json(module)) => &module.source, @@ -844,3 +802,54 @@ impl SourceMapGetter for CliSourceMapGetter { } } } + +/// Holds the `ModuleGraph` in workers. +#[derive(Clone)] +struct WorkerModuleGraphContainer { + // Allow only one request to update the graph data at a time, + // but allow other requests to read from it at any time even + // while another request is updating the data. + update_queue: Rc<deno_core::unsync::TaskQueue>, + inner: Rc<RefCell<Arc<ModuleGraph>>>, +} + +impl WorkerModuleGraphContainer { + pub fn new(module_graph: Arc<ModuleGraph>) -> Self { + Self { + update_queue: Default::default(), + inner: Rc::new(RefCell::new(module_graph)), + } + } +} + +impl ModuleGraphContainer for WorkerModuleGraphContainer { + async fn acquire_update_permit(&self) -> impl ModuleGraphUpdatePermit { + let permit = self.update_queue.acquire().await; + WorkerModuleGraphUpdatePermit { + permit, + inner: self.inner.clone(), + graph: (**self.inner.borrow()).clone(), + } + } + + fn graph(&self) -> Arc<ModuleGraph> { + self.inner.borrow().clone() + } +} + +struct WorkerModuleGraphUpdatePermit { + permit: deno_core::unsync::TaskQueuePermit, + inner: Rc<RefCell<Arc<ModuleGraph>>>, + graph: ModuleGraph, +} + +impl ModuleGraphUpdatePermit for WorkerModuleGraphUpdatePermit { + fn graph_mut(&mut self) -> &mut ModuleGraph { + &mut self.graph + } + + fn commit(self) { + *self.inner.borrow_mut() = Arc::new(self.graph); + drop(self.permit); // explicit drop for clarity + } +} |