refactor(cli): migrate run and cache to new infrastructure (#7996)

Co-authored-by: Ryan Dahl <ry@tinyclouds.org>

1 files changed, 159 insertions, 347 deletions

diff --git a/cli/program_state.rs b/cli/program_state.rs
index b921b6807..6e3a51663 100644
--- a/cli/program_state.rs
+++ b/cli/program_state.rs
@@ -8,16 +8,20 @@ use crate::import_map::ImportMap;
 use crate::inspector::InspectorServer;
 use crate::lockfile::Lockfile;
 use crate::media_type::MediaType;
-use crate::module_graph::ModuleGraphFile;
-use crate::module_graph::ModuleGraphLoader;
+use crate::module_graph2::CheckOptions;
 use crate::module_graph2::GraphBuilder2;
 use crate::module_graph2::TranspileOptions;
+use crate::module_graph2::TypeLib;
 use crate::permissions::Permissions;
+use crate::source_maps::SourceMapGetter;
 use crate::specifier_handler::FetchHandler;
 use crate::tsc::CompiledModule;
 use crate::tsc::TargetLib;
 use crate::tsc::TsCompiler;
+
+use deno_core::error::generic_error;
 use deno_core::error::AnyError;
+use deno_core::url::Url;
 use deno_core::ModuleSpecifier;
 use std::cell::RefCell;
 use std::env;
@@ -115,89 +119,66 @@ impl ProgramState {
   /// and traspilation.
   pub async fn prepare_module_load(
     self: &Arc<Self>,
-    module_specifier: ModuleSpecifier,
-    maybe_referrer: Option<ModuleSpecifier>,
+    specifier: ModuleSpecifier,
     target_lib: TargetLib,
-    permissions: Permissions,
-    is_dyn_import: bool,
+    dynamic_permissions: Permissions,
+    is_dynamic: bool,
     maybe_import_map: Option<ImportMap>,
   ) -> Result<(), AnyError> {
-    let module_specifier = module_specifier.clone();
+    let specifier = specifier.clone();
+    let handler =
+      Rc::new(RefCell::new(FetchHandler::new(self, dynamic_permissions)?));
+    let mut builder = GraphBuilder2::new(handler, maybe_import_map);
+    builder.add(&specifier, is_dynamic).await?;
+    let mut graph = builder.get_graph(&self.lockfile);
+    let debug = self.flags.log_level == Some(log::Level::Debug);
+    let maybe_config_path = self.flags.config_path.clone();
 
     if self.flags.no_check {
-      debug!("Transpiling root: {}", module_specifier);
-      // TODO(kitsonk) note that self.permissions != permissions, which is
-      // something that should be handled better in the future.
-      let handler =
-        Rc::new(RefCell::new(FetchHandler::new(self, permissions.clone())?));
-      let mut builder = GraphBuilder2::new(handler, maybe_import_map);
-      builder.insert(&module_specifier).await?;
-      let mut graph = builder.get_graph(&self.lockfile)?;
-
       let (stats, maybe_ignored_options) =
         graph.transpile(TranspileOptions {
-          debug: self.flags.log_level == Some(log::Level::Debug),
-          maybe_config_path: self.flags.config_path.clone(),
+          debug,
+          maybe_config_path,
+          reload: self.flags.reload,
         })?;
-
+      debug!("{}", stats);
       if let Some(ignored_options) = maybe_ignored_options {
         eprintln!("{}", ignored_options);
       }
-
-      debug!("{}", stats);
     } else {
-      let mut module_graph_loader = ModuleGraphLoader::new(
-        self.file_fetcher.clone(),
-        maybe_import_map,
-        permissions.clone(),
-        is_dyn_import,
-        false,
-      );
-      module_graph_loader
-        .add_to_graph(&module_specifier, maybe_referrer)
-        .await?;
-      let module_graph = module_graph_loader.get_graph();
-
-      let out = self
-        .file_fetcher
-        .fetch_cached_source_file(&module_specifier, permissions.clone())
-        .expect("Source file not found");
-
-      let module_graph_files = module_graph.values().collect::<Vec<_>>();
-      // Check integrity of every file in module graph
-      if let Some(ref lockfile) = self.lockfile {
-        let mut g = lockfile.lock().unwrap();
-
-        for graph_file in &module_graph_files {
-          let check_passed =
-            g.check_or_insert(&graph_file.url, &graph_file.source_code);
-
-          if !check_passed {
-            eprintln!(
-              "Subresource integrity check failed --lock={}\n{}",
-              g.filename.display(),
-              graph_file.url
-            );
-            std::process::exit(10);
+      let lib = match target_lib {
+        TargetLib::Main => {
+          if self.flags.unstable {
+            TypeLib::UnstableDenoWindow
+          } else {
+            TypeLib::DenoWindow
           }
         }
-      }
-
-      // Check if we need to compile files.
-      let should_compile = needs_compilation(
-        self.ts_compiler.compile_js,
-        out.media_type,
-        &module_graph_files,
-      );
-      let allow_js = should_allow_js(&module_graph_files);
+        TargetLib::Worker => {
+          if self.flags.unstable {
+            TypeLib::UnstableDenoWorker
+          } else {
+            TypeLib::DenoWorker
+          }
+        }
+      };
+      let (stats, diagnostics, maybe_ignored_options) =
+        graph.check(CheckOptions {
+          debug,
+          emit: true,
+          lib,
+          maybe_config_path,
+          reload: self.flags.reload,
+        })?;
 
-      if should_compile {
-        self
-          .ts_compiler
-          .compile(self, &out, target_lib, &module_graph, allow_js)
-          .await?;
+      debug!("{}", stats);
+      if let Some(ignored_options) = maybe_ignored_options {
+        eprintln!("{}", ignored_options);
       }
-    }
+      if !diagnostics.0.is_empty() {
+        return Err(generic_error(diagnostics.to_string()));
+      }
+    };
 
     if let Some(ref lockfile) = self.lockfile {
       let g = lockfile.lock().unwrap();
@@ -207,44 +188,39 @@ impl ProgramState {
     Ok(())
   }
 
-  // TODO(bartlomieju): this method doesn't need to be async anymore
-  /// This method is used after `prepare_module_load` finishes and JsRuntime
-  /// starts loading source and executing source code. This method shouldn't
-  /// perform any IO (besides $DENO_DIR) and only operate on sources collected
-  /// during `prepare_module_load`.
-  pub async fn fetch_compiled_module(
+  pub fn fetch_compiled_module(
     &self,
     module_specifier: ModuleSpecifier,
-    _maybe_referrer: Option<ModuleSpecifier>,
+    maybe_referrer: Option<ModuleSpecifier>,
   ) -> Result<CompiledModule, AnyError> {
     let out = self
       .file_fetcher
       .fetch_cached_source_file(&module_specifier, Permissions::allow_all())
       .expect("Cached source file doesn't exist");
 
-    // Check if we need to compile files
-    let was_compiled = match out.media_type {
-      MediaType::TypeScript | MediaType::TSX | MediaType::JSX => true,
-      MediaType::JavaScript => self.ts_compiler.compile_js,
-      _ => false,
-    };
-
-    let compiled_module = if was_compiled {
-      match self.ts_compiler.get_compiled_module(&out.url) {
-        Ok(module) => module,
-        Err(e) => {
-          let msg = format!(
-            "Failed to get compiled source code of \"{}\".\nReason: {}\n\
-            If the source file provides only type exports, prefer to use \"import type\" or \"export type\" syntax instead.",
-            out.url, e.to_string()
-          );
-          info!("{} {}", crate::colors::yellow("Warning"), msg);
-
-          CompiledModule {
-            code: "".to_string(),
-            name: out.url.to_string(),
-          }
-        }
+    let url = out.url.clone();
+    let compiled_module = if let Some((code, _)) = self.get_emit(&url) {
+      CompiledModule {
+        code: String::from_utf8(code).unwrap(),
+        name: out.url.to_string(),
+      }
+    // We expect a compiled source for any non-JavaScript files, except for
+    // local files that have an unknown media type and no referrer (root modules
+    // that do not have an extension.)
+    } else if out.media_type != MediaType::JavaScript
+      && !(out.media_type == MediaType::Unknown
+        && maybe_referrer.is_none()
+        && url.scheme() == "file")
+    {
+      let message = if let Some(referrer) = maybe_referrer {
+        format!("Compiled module not found \"{}\"\n  From: {}\n    If the source module contains only types, use `import type` and `export type` to import it instead.", module_specifier, referrer)
+      } else {
+        format!("Compiled module not found \"{}\"\n  If the source module contains only types, use `import type` and `export type` to import it instead.", module_specifier)
+      };
+      info!("{}: {}", crate::colors::yellow("warning"), message);
+      CompiledModule {
+        code: "".to_string(),
+        name: out.url.to_string(),
       }
     } else {
       CompiledModule {
@@ -256,6 +232,37 @@ impl ProgramState {
     Ok(compiled_module)
   }
 
+  // TODO(@kitsonk) this should be a straight forward API on file_fetcher or
+  // whatever future refactors do...
+  fn get_emit(&self, url: &Url) -> Option<(Vec<u8>, Option<Vec<u8>>)> {
+    match url.scheme() {
+      // we should only be looking for emits for schemes that denote external
+      // modules, which the disk_cache supports
+      "wasm" | "file" | "http" | "https" => (),
+      _ => {
+        return None;
+      }
+    }
+    let emit_path = self
+      .dir
+      .gen_cache
+      .get_cache_filename_with_extension(&url, "js");
+    let emit_map_path = self
+      .dir
+      .gen_cache
+      .get_cache_filename_with_extension(&url, "js.map");
+    if let Ok(code) = self.dir.gen_cache.get(&emit_path) {
+      let maybe_map = if let Ok(map) = self.dir.gen_cache.get(&emit_map_path) {
+        Some(map)
+      } else {
+        None
+      };
+      Some((code, maybe_map))
+    } else {
+      None
+    }
+  }
+
   /// Quits the process if the --unstable flag was not provided.
   ///
   /// This is intentionally a non-recoverable check so that people cannot probe
@@ -279,57 +286,62 @@ impl ProgramState {
   }
 }
 
-/// Determine if TS compiler should be run with `allowJs` setting on. This
-/// is the case when there's either:
-///  - a JavaScript file with non-JavaScript import
-///  - JSX import
-fn should_allow_js(module_graph_files: &[&ModuleGraphFile]) -> bool {
-  module_graph_files.iter().any(|module_file| {
-    if module_file.media_type == MediaType::JSX {
-      true
-    } else if module_file.media_type == MediaType::JavaScript {
-      module_file.imports.iter().any(|import_desc| {
-        let import_file = module_graph_files
-          .iter()
-          .find(|f| {
-            f.specifier == import_desc.resolved_specifier.to_string().as_str()
-          })
-          .expect("Failed to find imported file");
-        let media_type = import_file.media_type;
-        media_type == MediaType::TypeScript
-          || media_type == MediaType::TSX
-          || media_type == MediaType::JSX
-      })
+// TODO(@kitsonk) this is only temporary, but should be refactored to somewhere
+// else, like a refactored file_fetcher.
+impl SourceMapGetter for ProgramState {
+  fn get_source_map(&self, file_name: &str) -> Option<Vec<u8>> {
+    if let Ok(specifier) = ModuleSpecifier::resolve_url(file_name) {
+      if let Some((code, maybe_map)) = self.get_emit(&specifier.as_url()) {
+        if maybe_map.is_some() {
+          maybe_map
+        } else {
+          let code = String::from_utf8(code).unwrap();
+          let lines: Vec<&str> = code.split('\n').collect();
+          if let Some(last_line) = lines.last() {
+            if last_line
+              .starts_with("//# sourceMappingURL=data:application/json;base64,")
+            {
+              let input = last_line.trim_start_matches(
+                "//# sourceMappingURL=data:application/json;base64,",
+              );
+              let decoded_map = base64::decode(input)
+                .expect("Unable to decode source map from emitted file.");
+              Some(decoded_map)
+            } else {
+              None
+            }
+          } else {
+            None
+          }
+        }
+      } else {
+        None
+      }
     } else {
-      false
+      None
     }
-  })
-}
-
-// Compilation happens if either:
-// - `checkJs` is set to true in TS config
-// - entry point is a TS file
-// - any dependency in module graph is a TS file
-fn needs_compilation(
-  compile_js: bool,
-  media_type: MediaType,
-  module_graph_files: &[&ModuleGraphFile],
-) -> bool {
-  let mut needs_compilation = match media_type {
-    MediaType::TypeScript | MediaType::TSX | MediaType::JSX => true,
-    MediaType::JavaScript => compile_js,
-    _ => false,
-  };
-
-  needs_compilation |= module_graph_files.iter().any(|module_file| {
-    let media_type = module_file.media_type;
-
-    media_type == (MediaType::TypeScript)
-      || media_type == (MediaType::TSX)
-      || media_type == (MediaType::JSX)
-  });
+  }
 
-  needs_compilation
+  fn get_source_line(
+    &self,
+    file_name: &str,
+    line_number: usize,
+  ) -> Option<String> {
+    if let Ok(specifier) = ModuleSpecifier::resolve_url(file_name) {
+      self
+        .file_fetcher
+        .fetch_cached_source_file(&specifier, Permissions::allow_all())
+        .map(|out| {
+          // Do NOT use .lines(): it skips the terminating empty line.
+          // (due to internally using .split_terminator() instead of .split())
+          let lines: Vec<&str> = out.source_code.split('\n').collect();
+          assert!(lines.len() > line_number);
+          lines[line_number].to_string()
+        })
+    } else {
+      None
+    }
+  }
 }
 
 #[test]
@@ -337,203 +349,3 @@ fn thread_safe() {
   fn f<S: Send + Sync>(_: S) {}
   f(ProgramState::mock(vec![], None));
 }
-
-#[test]
-fn test_should_allow_js() {
-  use crate::ast::Location;
-  use crate::module_graph::ImportDescriptor;
-
-  assert!(should_allow_js(&[
-    &ModuleGraphFile {
-      specifier: "file:///some/file.ts".to_string(),
-      url: "file:///some/file.ts".to_string(),
-      redirect: None,
-      filename: "some/file.ts".to_string(),
-      imports: vec![],
-      version_hash: "1".to_string(),
-      referenced_files: vec![],
-      lib_directives: vec![],
-      types_directives: vec![],
-      type_headers: vec![],
-      media_type: MediaType::TypeScript,
-      source_code: "function foo() {}".to_string(),
-    },
-    &ModuleGraphFile {
-      specifier: "file:///some/file1.js".to_string(),
-      url: "file:///some/file1.js".to_string(),
-      redirect: None,
-      filename: "some/file1.js".to_string(),
-      version_hash: "1".to_string(),
-      imports: vec![ImportDescriptor {
-        specifier: "./file.ts".to_string(),
-        resolved_specifier: ModuleSpecifier::resolve_url(
-          "file:///some/file.ts",
-        )
-        .unwrap(),
-        type_directive: None,
-        resolved_type_directive: None,
-        location: Location {
-          filename: "file:///some/file1.js".to_string(),
-          line: 0,
-          col: 0,
-        },
-      }],
-      referenced_files: vec![],
-      lib_directives: vec![],
-      types_directives: vec![],
-      type_headers: vec![],
-      media_type: MediaType::JavaScript,
-      source_code: "function foo() {}".to_string(),
-    },
-  ],));
-
-  assert!(should_allow_js(&[
-    &ModuleGraphFile {
-      specifier: "file:///some/file.jsx".to_string(),
-      url: "file:///some/file.jsx".to_string(),
-      redirect: None,
-      filename: "some/file.jsx".to_string(),
-      imports: vec![],
-      version_hash: "1".to_string(),
-      referenced_files: vec![],
-      lib_directives: vec![],
-      types_directives: vec![],
-      type_headers: vec![],
-      media_type: MediaType::JSX,
-      source_code: "function foo() {}".to_string(),
-    },
-    &ModuleGraphFile {
-      specifier: "file:///some/file.ts".to_string(),
-      url: "file:///some/file.ts".to_string(),
-      redirect: None,
-      filename: "some/file.ts".to_string(),
-      version_hash: "1".to_string(),
-      imports: vec![ImportDescriptor {
-        specifier: "./file.jsx".to_string(),
-        resolved_specifier: ModuleSpecifier::resolve_url(
-          "file:///some/file.jsx",
-        )
-        .unwrap(),
-        type_directive: None,
-        resolved_type_directive: None,
-        location: Location {
-          filename: "file:///some/file1.ts".to_string(),
-          line: 0,
-          col: 0,
-        },
-      }],
-      referenced_files: vec![],
-      lib_directives: vec![],
-      types_directives: vec![],
-      type_headers: vec![],
-      media_type: MediaType::TypeScript,
-      source_code: "function foo() {}".to_string(),
-    },
-  ]));
-
-  assert!(!should_allow_js(&[
-    &ModuleGraphFile {
-      specifier: "file:///some/file.js".to_string(),
-      url: "file:///some/file.js".to_string(),
-      redirect: None,
-      filename: "some/file.js".to_string(),
-      imports: vec![],
-      referenced_files: vec![],
-      lib_directives: vec![],
-      types_directives: vec![],
-      version_hash: "1".to_string(),
-      type_headers: vec![],
-      media_type: MediaType::JavaScript,
-      source_code: "function foo() {}".to_string(),
-    },
-    &ModuleGraphFile {
-      specifier: "file:///some/file1.js".to_string(),
-      url: "file:///some/file1.js".to_string(),
-      redirect: None,
-      filename: "some/file1.js".to_string(),
-      imports: vec![ImportDescriptor {
-        specifier: "./file.js".to_string(),
-        resolved_specifier: ModuleSpecifier::resolve_url(
-          "file:///some/file.js",
-        )
-        .unwrap(),
-        type_directive: None,
-        resolved_type_directive: None,
-        location: Location {
-          filename: "file:///some/file.js".to_string(),
-          line: 0,
-          col: 0,
-        },
-      }],
-      referenced_files: vec![],
-      lib_directives: vec![],
-      types_directives: vec![],
-      version_hash: "1".to_string(),
-      type_headers: vec![],
-      media_type: MediaType::JavaScript,
-      source_code: "function foo() {}".to_string(),
-    },
-  ],));
-}
-
-#[test]
-fn test_needs_compilation() {
-  assert!(!needs_compilation(
-    false,
-    MediaType::JavaScript,
-    &[&ModuleGraphFile {
-      specifier: "some/file.js".to_string(),
-      url: "file:///some/file.js".to_string(),
-      redirect: None,
-      filename: "some/file.js".to_string(),
-      imports: vec![],
-      referenced_files: vec![],
-      lib_directives: vec![],
-      types_directives: vec![],
-      type_headers: vec![],
-      version_hash: "1".to_string(),
-      media_type: MediaType::JavaScript,
-      source_code: "function foo() {}".to_string(),
-    }],
-  ));
-
-  assert!(!needs_compilation(false, MediaType::JavaScript, &[]));
-  assert!(needs_compilation(true, MediaType::JavaScript, &[]));
-  assert!(needs_compilation(false, MediaType::TypeScript, &[]));
-  assert!(needs_compilation(false, MediaType::JSX, &[]));
-  assert!(needs_compilation(false, MediaType::TSX, &[]));
-  assert!(needs_compilation(
-    false,
-    MediaType::JavaScript,
-    &[
-      &ModuleGraphFile {
-        specifier: "file:///some/file.ts".to_string(),
-        url: "file:///some/file.ts".to_string(),
-        redirect: None,
-        filename: "some/file.ts".to_string(),
-        imports: vec![],
-        referenced_files: vec![],
-        lib_directives: vec![],
-        types_directives: vec![],
-        type_headers: vec![],
-        media_type: MediaType::TypeScript,
-        version_hash: "1".to_string(),
-        source_code: "function foo() {}".to_string(),
-      },
-      &ModuleGraphFile {
-        specifier: "file:///some/file1.js".to_string(),
-        url: "file:///some/file1.js".to_string(),
-        redirect: None,
-        filename: "some/file1.js".to_string(),
-        imports: vec![],
-        referenced_files: vec![],
-        lib_directives: vec![],
-        types_directives: vec![],
-        type_headers: vec![],
-        version_hash: "1".to_string(),
-        media_type: MediaType::JavaScript,
-        source_code: "function foo() {}".to_string(),
-      },
-    ],
-  ));
-}