refactor: break up `ProcState` (#18707)

1. Breaks up functionality within `ProcState` into several other structs
to break out the responsibilities (`ProcState` is only a data struct
now).
2. Moves towards being able to inject dependencies more easily and have
functionality only require what it needs.
3. Exposes `Arc<T>` around the "service structs" instead of it being
embedded within them. The idea behind embedding them was to reduce the
verbosity of needing to pass around `Arc<...>`, but I don't think it was
exactly working and as we move more of these structs to be more
injectable I don't think the extra verbosity will be a big deal.

1 files changed, 407 insertions, 36 deletions

diff --git a/cli/module_loader.rs b/cli/module_loader.rs
index 746743525..c2dfaad8f 100644
--- a/cli/module_loader.rs
+++ b/cli/module_loader.rs
@@ -1,18 +1,44 @@
 // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
 
+use crate::args::CliOptions;
+use crate::args::DenoSubcommand;
+use crate::args::TsConfigType;
 use crate::args::TsTypeLib;
+use crate::args::TypeCheckMode;
+use crate::cache::Caches;
+use crate::cache::DenoDir;
+use crate::cache::ParsedSourceCache;
+use crate::cache::TypeCheckCache;
+use crate::emit::Emitter;
+use crate::graph_util::graph_lock_or_exit;
+use crate::graph_util::graph_valid_with_cli_options;
+use crate::graph_util::ModuleGraphBuilder;
+use crate::graph_util::ModuleGraphContainer;
 use crate::node;
+use crate::node::NodeCodeTranslator;
+use crate::node::NodeResolution;
+use crate::npm::NpmPackageResolver;
+use crate::npm::NpmResolution;
+use crate::proc_state::CjsResolutionStore;
+use crate::proc_state::FileWatcherReporter;
 use crate::proc_state::ProcState;
+use crate::resolver::CliGraphResolver;
+use crate::tools::check;
+use crate::util::progress_bar::ProgressBar;
 use crate::util::text_encoding::code_without_source_map;
 use crate::util::text_encoding::source_map_from_code;
 
 use deno_ast::MediaType;
 use deno_core::anyhow::anyhow;
 use deno_core::anyhow::Context;
+use deno_core::error::custom_error;
+use deno_core::error::generic_error;
 use deno_core::error::AnyError;
 use deno_core::futures::future::FutureExt;
 use deno_core::futures::Future;
+use deno_core::parking_lot::Mutex;
 use deno_core::resolve_url;
+use deno_core::resolve_url_or_path;
 use deno_core::ModuleCode;
 use deno_core::ModuleLoader;
 use deno_core::ModuleSource;
@@ -21,13 +47,210 @@ use deno_core::ModuleType;
 use deno_core::OpState;
 use deno_core::ResolutionKind;
 use deno_core::SourceMapGetter;
+use deno_graph::source::Resolver;
 use deno_graph::EsmModule;
 use deno_graph::JsonModule;
+use deno_graph::Module;
+use deno_graph::Resolution;
+use deno_lockfile::Lockfile;
+use deno_runtime::deno_node::NodeResolutionMode;
 use deno_runtime::permissions::PermissionsContainer;
+use deno_semver::npm::NpmPackageReqReference;
+use std::borrow::Cow;
 use std::cell::RefCell;
+use std::collections::HashSet;
 use std::pin::Pin;
 use std::rc::Rc;
 use std::str;
+use std::sync::Arc;
+
+pub struct ModuleLoadPreparer {
+  options: Arc<CliOptions>,
+  caches: Arc<Caches>,
+  deno_dir: DenoDir,
+  graph_container: Arc<ModuleGraphContainer>,
+  lockfile: Option<Arc<Mutex<Lockfile>>>,
+  maybe_file_watcher_reporter: Option<FileWatcherReporter>,
+  module_graph_builder: Arc<ModuleGraphBuilder>,
+  npm_resolver: Arc<NpmPackageResolver>,
+  parsed_source_cache: Arc<ParsedSourceCache>,
+  progress_bar: ProgressBar,
+  resolver: Arc<CliGraphResolver>,
+}
+
+impl ModuleLoadPreparer {
+  #[allow(clippy::too_many_arguments)]
+  pub fn new(
+    options: Arc<CliOptions>,
+    caches: Arc<Caches>,
+    deno_dir: DenoDir,
+    graph_container: Arc<ModuleGraphContainer>,
+    lockfile: Option<Arc<Mutex<Lockfile>>>,
+    maybe_file_watcher_reporter: Option<FileWatcherReporter>,
+    module_graph_builder: Arc<ModuleGraphBuilder>,
+    npm_resolver: Arc<NpmPackageResolver>,
+    parsed_source_cache: Arc<ParsedSourceCache>,
+    progress_bar: ProgressBar,
+    resolver: Arc<CliGraphResolver>,
+  ) -> Self {
+    Self {
+      options,
+      caches,
+      deno_dir,
+      graph_container,
+      lockfile,
+      maybe_file_watcher_reporter,
+      module_graph_builder,
+      npm_resolver,
+      parsed_source_cache,
+      progress_bar,
+      resolver,
+    }
+  }
+
+  /// This method must be called for a module or a static importer of that
+  /// module before attempting to `load()` it from a `JsRuntime`. It will
+  /// populate the graph data in memory with the necessary source code, write
+  /// emits where necessary or report any module graph / type checking errors.
+  #[allow(clippy::too_many_arguments)]
+  pub async fn prepare_module_load(
+    &self,
+    roots: Vec<ModuleSpecifier>,
+    is_dynamic: bool,
+    lib: TsTypeLib,
+    root_permissions: PermissionsContainer,
+    dynamic_permissions: PermissionsContainer,
+  ) -> Result<(), AnyError> {
+    log::debug!("Preparing module load.");
+    let _pb_clear_guard = self.progress_bar.clear_guard();
+
+    let mut cache = self
+      .module_graph_builder
+      .create_fetch_cacher(root_permissions, dynamic_permissions);
+    let maybe_imports = self.options.to_maybe_imports()?;
+    let graph_resolver = self.resolver.as_graph_resolver();
+    let graph_npm_resolver = self.resolver.as_graph_npm_resolver();
+    let maybe_file_watcher_reporter: Option<&dyn deno_graph::source::Reporter> =
+      if let Some(reporter) = &self.maybe_file_watcher_reporter {
+        Some(reporter)
+      } else {
+        None
+      };
+
+    let analyzer = self.parsed_source_cache.as_analyzer();
+
+    log::debug!("Creating module graph.");
+    let mut graph_update_permit =
+      self.graph_container.acquire_update_permit().await;
+    let graph = graph_update_permit.graph_mut();
+
+    // Determine any modules that have already been emitted this session and
+    // should be skipped.
+    let reload_exclusions: HashSet<ModuleSpecifier> =
+      graph.specifiers().map(|(s, _)| s.clone()).collect();
+
+    self
+      .module_graph_builder
+      .build_graph_with_npm_resolution(
+        graph,
+        roots.clone(),
+        &mut cache,
+        deno_graph::BuildOptions {
+          is_dynamic,
+          imports: maybe_imports,
+          resolver: Some(graph_resolver),
+          npm_resolver: Some(graph_npm_resolver),
+          module_analyzer: Some(&*analyzer),
+          reporter: maybe_file_watcher_reporter,
+        },
+      )
+      .await?;
+
+    // If there is a lockfile, validate the integrity of all the modules.
+    if let Some(lockfile) = &self.lockfile {
+      graph_lock_or_exit(graph, &mut lockfile.lock());
+    }
+
+    graph_valid_with_cli_options(graph, &roots, &self.options)?;
+    // save the graph and get a reference to the new graph
+    let graph = graph_update_permit.commit();
+
+    if graph.has_node_specifier
+      && self.options.type_check_mode() != TypeCheckMode::None
+    {
+      self
+        .npm_resolver
+        .inject_synthetic_types_node_package()
+        .await?;
+    }
+
+    drop(_pb_clear_guard);
+
+    // type check if necessary
+    if self.options.type_check_mode() != TypeCheckMode::None
+      && !self.graph_container.is_type_checked(&roots, lib)
+    {
+      // todo(dsherret): consolidate this with what's done in graph_util
+      log::debug!("Type checking.");
+      let maybe_config_specifier = self.options.maybe_config_file_specifier();
+      let graph = Arc::new(graph.segment(&roots));
+      let options = check::CheckOptions {
+        type_check_mode: self.options.type_check_mode(),
+        debug: self.options.log_level() == Some(log::Level::Debug),
+        maybe_config_specifier,
+        ts_config: self
+          .options
+          .resolve_ts_config_for_emit(TsConfigType::Check { lib })?
+          .ts_config,
+        log_checks: true,
+        reload: self.options.reload_flag()
+          && !roots.iter().all(|r| reload_exclusions.contains(r)),
+      };
+      let check_cache =
+        TypeCheckCache::new(self.caches.type_checking_cache_db(&self.deno_dir));
+      let check_result =
+        check::check(graph, &check_cache, self.npm_resolver.clone(), options)?;
+      self.graph_container.set_type_checked(&roots, lib);
+      if !check_result.diagnostics.is_empty() {
+        return Err(anyhow!(check_result.diagnostics));
+      }
+      log::debug!("{}", check_result.stats);
+    }
+
+    // any updates to the lockfile should be updated now
+    if let Some(ref lockfile) = self.lockfile {
+      let g = lockfile.lock();
+      g.write()?;
+    }
+
+    log::debug!("Prepared module load.");
+
+    Ok(())
+  }
+
+  /// Helper around prepare_module_load that loads and type checks
+  /// the provided files.
+  pub async fn load_and_type_check_files(
+    &self,
+    files: &[String],
+  ) -> Result<(), AnyError> {
+    let lib = self.options.ts_type_lib_window();
+
+    let specifiers = files
+      .iter()
+      .map(|file| resolve_url_or_path(file, self.options.initial_cwd()))
+      .collect::<Result<Vec<_>, _>>()?;
+    self
+      .prepare_module_load(
+        specifiers,
+        false,
+        lib,
+        PermissionsContainer::allow_all(),
+        PermissionsContainer::allow_all(),
+      )
+      .await
+  }
+}
 
 struct ModuleCodeSource {
   pub code: ModuleCode,
@@ -36,15 +259,24 @@ struct ModuleCodeSource {
 }
 
 pub struct CliModuleLoader {
-  pub lib: TsTypeLib,
+  lib: TsTypeLib,
   /// The initial set of permissions used to resolve the static imports in the
   /// worker. These are "allow all" for main worker, and parent thread
   /// permissions for Web Worker.
-  pub root_permissions: PermissionsContainer,
+  root_permissions: PermissionsContainer,
   /// Permissions used to resolve dynamic imports, these get passed as
   /// "root permissions" for Web Worker.
   dynamic_permissions: PermissionsContainer,
-  pub ps: ProcState,
+  cli_options: Arc<CliOptions>,
+  cjs_resolutions: Arc<CjsResolutionStore>,
+  emitter: Arc<Emitter>,
+  graph_container: Arc<ModuleGraphContainer>,
+  module_load_preparer: Arc<ModuleLoadPreparer>,
+  node_code_translator: Arc<NodeCodeTranslator>,
+  npm_resolution: Arc<NpmResolution>,
+  npm_resolver: Arc<NpmPackageResolver>,
+  parsed_source_cache: Arc<ParsedSourceCache>,
+  resolver: Arc<CliGraphResolver>,
 }
 
 impl CliModuleLoader {
@@ -57,7 +289,16 @@ impl CliModuleLoader {
       lib: ps.options.ts_type_lib_window(),
       root_permissions,
       dynamic_permissions,
-      ps,
+      cli_options: ps.options.clone(),
+      cjs_resolutions: ps.cjs_resolutions.clone(),
+      emitter: ps.emitter.clone(),
+      graph_container: ps.graph_container.clone(),
+      module_load_preparer: ps.module_load_preparer.clone(),
+      node_code_translator: ps.node_code_translator.clone(),
+      npm_resolution: ps.npm_resolution.clone(),
+      npm_resolver: ps.npm_resolver.clone(),
+      parsed_source_cache: ps.parsed_source_cache.clone(),
+      resolver: ps.resolver.clone(),
     })
   }
 
@@ -70,7 +311,16 @@ impl CliModuleLoader {
       lib: ps.options.ts_type_lib_worker(),
       root_permissions,
       dynamic_permissions,
-      ps,
+      cli_options: ps.options.clone(),
+      cjs_resolutions: ps.cjs_resolutions.clone(),
+      emitter: ps.emitter.clone(),
+      graph_container: ps.graph_container.clone(),
+      module_load_preparer: ps.module_load_preparer.clone(),
+      node_code_translator: ps.node_code_translator.clone(),
+      npm_resolution: ps.npm_resolution.clone(),
+      npm_resolver: ps.npm_resolver.clone(),
+      parsed_source_cache: ps.parsed_source_cache.clone(),
+      resolver: ps.resolver.clone(),
     })
   }
 
@@ -83,7 +333,7 @@ impl CliModuleLoader {
       unreachable!(); // Node built-in modules should be handled internally.
     }
 
-    let graph = self.ps.graph();
+    let graph = self.graph_container.graph();
     match graph.get(specifier) {
       Some(deno_graph::Module::Json(JsonModule {
         source,
@@ -116,11 +366,9 @@ impl CliModuleLoader {
           | MediaType::Jsx
           | MediaType::Tsx => {
             // get emit text
-            self.ps.emitter.emit_parsed_source(
-              specifier,
-              *media_type,
-              source,
-            )?
+            self
+              .emitter
+              .emit_parsed_source(specifier, *media_type, source)?
           }
           MediaType::TsBuildInfo | MediaType::Wasm | MediaType::SourceMap => {
             panic!("Unexpected media type {media_type} for {specifier}")
@@ -128,7 +376,7 @@ impl CliModuleLoader {
         };
 
         // at this point, we no longer need the parsed source in memory, so free it
-        self.ps.parsed_source_cache.free(specifier);
+        self.parsed_source_cache.free(specifier);
 
         Ok(ModuleCodeSource {
           code,
@@ -152,7 +400,7 @@ impl CliModuleLoader {
     maybe_referrer: Option<&ModuleSpecifier>,
     is_dynamic: bool,
   ) -> Result<ModuleSource, AnyError> {
-    let code_source = if self.ps.npm_resolver.in_npm_package(specifier) {
+    let code_source = if self.npm_resolver.in_npm_package(specifier) {
       let file_path = specifier.to_file_path().unwrap();
       let code = std::fs::read_to_string(&file_path).with_context(|| {
         let mut msg = "Unable to load ".to_string();
@@ -164,29 +412,24 @@ impl CliModuleLoader {
         msg
       })?;
 
-      let code = if self.ps.cjs_resolutions.lock().contains(specifier) {
+      let code = if self.cjs_resolutions.contains(specifier) {
         let mut permissions = if is_dynamic {
           self.dynamic_permissions.clone()
         } else {
           self.root_permissions.clone()
         };
         // translate cjs to esm if it's cjs and inject node globals
-        node::translate_cjs_to_esm(
-          &self.ps.file_fetcher,
+        self.node_code_translator.translate_cjs_to_esm(
           specifier,
           code,
           MediaType::Cjs,
-          &self.ps.npm_resolver,
-          &self.ps.node_analysis_cache,
           &mut permissions,
         )?
       } else {
         // only inject node globals for esm
-        node::esm_code_with_node_globals(
-          &self.ps.node_analysis_cache,
-          specifier,
-          code,
-        )?
+        self
+          .node_code_translator
+          .esm_code_with_node_globals(specifier, code)?
       };
       ModuleCodeSource {
         code: code.into(),
@@ -196,7 +439,7 @@ impl CliModuleLoader {
     } else {
       self.load_prepared_module(specifier, maybe_referrer)?
     };
-    let code = if self.ps.options.is_inspecting() {
+    let code = if self.cli_options.is_inspecting() {
       // we need the code with the source map in order for
       // it to work with --inspect or --inspect-brk
       code_source.code
@@ -215,6 +458,23 @@ impl CliModuleLoader {
       &code_source.found_url,
     ))
   }
+
+  fn handle_node_resolve_result(
+    &self,
+    result: Result<Option<node::NodeResolution>, AnyError>,
+  ) -> Result<ModuleSpecifier, AnyError> {
+    let response = match result? {
+      Some(response) => response,
+      None => return Err(generic_error("not found")),
+    };
+    if let NodeResolution::CommonJs(specifier) = &response {
+      // remember that this was a common js resolution
+      self.cjs_resolutions.insert(specifier.clone());
+    } else if let NodeResolution::BuiltIn(specifier) = &response {
+      return node::resolve_builtin_node_module(specifier);
+    }
+    Ok(response.into_url())
+  }
 }
 
 impl ModuleLoader for CliModuleLoader {
@@ -229,7 +489,117 @@ impl ModuleLoader for CliModuleLoader {
     } else {
       self.root_permissions.clone()
     };
-    self.ps.resolve(specifier, referrer, &mut permissions)
+
+    // TODO(bartlomieju): ideally we shouldn't need to call `current_dir()` on each
+    // call - maybe it should be caller's responsibility to pass it as an arg?
+    let cwd = std::env::current_dir().context("Unable to get CWD")?;
+    let referrer_result = deno_core::resolve_url_or_path(referrer, &cwd);
+
+    if let Ok(referrer) = referrer_result.as_ref() {
+      if self.npm_resolver.in_npm_package(referrer) {
+        // we're in an npm package, so use node resolution
+        return self
+          .handle_node_resolve_result(node::node_resolve(
+            specifier,
+            referrer,
+            NodeResolutionMode::Execution,
+            &self.npm_resolver.as_require_npm_resolver(),
+            &mut permissions,
+          ))
+          .with_context(|| {
+            format!("Could not resolve '{specifier}' from '{referrer}'.")
+          });
+      }
+
+      let graph = self.graph_container.graph();
+      let maybe_resolved = match graph.get(referrer) {
+        Some(Module::Esm(module)) => {
+          module.dependencies.get(specifier).map(|d| &d.maybe_code)
+        }
+        _ => None,
+      };
+
+      match maybe_resolved {
+        Some(Resolution::Ok(resolved)) => {
+          let specifier = &resolved.specifier;
+
+          return match graph.get(specifier) {
+            Some(Module::Npm(module)) => self
+              .handle_node_resolve_result(node::node_resolve_npm_reference(
+                &module.nv_reference,
+                NodeResolutionMode::Execution,
+                &self.npm_resolver,
+                &mut permissions,
+              ))
+              .with_context(|| {
+                format!("Could not resolve '{}'.", module.nv_reference)
+              }),
+            Some(Module::Node(module)) => {
+              node::resolve_builtin_node_module(&module.module_name)
+            }
+            Some(Module::Esm(module)) => Ok(module.specifier.clone()),
+            Some(Module::Json(module)) => Ok(module.specifier.clone()),
+            Some(Module::External(module)) => {
+              Ok(node::resolve_specifier_into_node_modules(&module.specifier))
+            }
+            None => Ok(specifier.clone()),
+          };
+        }
+        Some(Resolution::Err(err)) => {
+          return Err(custom_error(
+            "TypeError",
+            format!("{}\n", err.to_string_with_range()),
+          ))
+        }
+        Some(Resolution::None) | None => {}
+      }
+    }
+
+    // Built-in Node modules
+    if let Some(module_name) = specifier.strip_prefix("node:") {
+      return node::resolve_builtin_node_module(module_name);
+    }
+
+    // FIXME(bartlomieju): this is a hacky way to provide compatibility with REPL
+    // and `Deno.core.evalContext` API. Ideally we should always have a referrer filled
+    // but sadly that's not the case due to missing APIs in V8.
+    let is_repl =
+      matches!(self.cli_options.sub_command(), DenoSubcommand::Repl(_));
+    let referrer = if referrer.is_empty() && is_repl {
+      deno_core::resolve_path("./$deno$repl.ts", &cwd)?
+    } else {
+      referrer_result?
+    };
+
+    // FIXME(bartlomieju): this is another hack way to provide NPM specifier
+    // support in REPL. This should be fixed.
+    let resolution = self.resolver.resolve(specifier, &referrer);
+
+    if is_repl {
+      let specifier = resolution
+        .as_ref()
+        .ok()
+        .map(Cow::Borrowed)
+        .or_else(|| ModuleSpecifier::parse(specifier).ok().map(Cow::Owned));
+      if let Some(specifier) = specifier {
+        if let Ok(reference) =
+          NpmPackageReqReference::from_specifier(&specifier)
+        {
+          let reference =
+            self.npm_resolution.pkg_req_ref_to_nv_ref(reference)?;
+          return self
+            .handle_node_resolve_result(node::node_resolve_npm_reference(
+              &reference,
+              deno_runtime::deno_node::NodeResolutionMode::Execution,
+              &self.npm_resolver,
+              &mut permissions,
+            ))
+            .with_context(|| format!("Could not resolve '{reference}'."));
+        }
+      }
+    }
+
+    resolution
   }
 
   fn load(
@@ -255,13 +625,13 @@ impl ModuleLoader for CliModuleLoader {
     _maybe_referrer: Option<String>,
     is_dynamic: bool,
   ) -> Pin<Box<dyn Future<Output = Result<(), AnyError>>>> {
-    if self.ps.npm_resolver.in_npm_package(specifier) {
+    if self.npm_resolver.in_npm_package(specifier) {
       // nothing to prepare
       return Box::pin(deno_core::futures::future::ready(Ok(())));
     }
 
     let specifier = specifier.clone();
-    let ps = self.ps.clone();
+    let module_load_preparer = self.module_load_preparer.clone();
 
     let dynamic_permissions = self.dynamic_permissions.clone();
     let root_permissions = if is_dynamic {
@@ -272,14 +642,15 @@ impl ModuleLoader for CliModuleLoader {
     let lib = self.lib;
 
     async move {
-      ps.prepare_module_load(
-        vec![specifier],
-        is_dynamic,
-        lib,
-        root_permissions,
-        dynamic_permissions,
-      )
-      .await
+      module_load_preparer
+        .prepare_module_load(
+          vec![specifier],
+          is_dynamic,
+          lib,
+          root_permissions,
+          dynamic_permissions,
+        )
+        .await
     }
     .boxed_local()
   }
@@ -303,7 +674,7 @@ impl SourceMapGetter for CliModuleLoader {
     file_name: &str,
     line_number: usize,
   ) -> Option<String> {
-    let graph = self.ps.graph();
+    let graph = self.graph_container.graph();
     let code = match graph.get(&resolve_url(file_name).ok()?) {
       Some(deno_graph::Module::Esm(module)) => &module.source,
       Some(deno_graph::Module::Json(module)) => &module.source,