refactor: TS compiler and module graph (#5817)

This PR addresses many problems with module graph loading
introduced in #5029, as well as many long standing issues.

"ModuleGraphLoader" has been wired to "ModuleLoader" implemented
on "State" - that means that dependency analysis and fetching is done
before spinning up TS compiler worker.

Basic dependency tracking for TS compilation has been implemented.

Errors caused by import statements are now annotated with import
location.

Co-authored-by: Ryan Dahl <ry@tinyclouds.org>

1 files changed, 47 insertions, 84 deletions

diff --git a/cli/state.rs b/cli/state.rs
index baac89216..00337c286 100644
--- a/cli/state.rs
+++ b/cli/state.rs
@@ -268,76 +268,23 @@ impl ModuleLoader for State {
     Ok(module_specifier)
   }
 
-  /// Given an absolute url, load its source code.
   fn load(
     &self,
     module_specifier: &ModuleSpecifier,
     maybe_referrer: Option<ModuleSpecifier>,
-    is_dyn_import: bool,
+    _is_dyn_import: bool,
   ) -> Pin<Box<deno_core::ModuleSourceFuture>> {
-    let module_specifier = module_specifier.clone();
-
-    // TODO(bartlomieju): this code is duplicated from module_graph.
-    // It should be removed when `prepare_load` will be used to load modules.
-    // Disallow http:// imports from modules loaded over https://
-    if let Some(referrer) = maybe_referrer.as_ref() {
-      if let "https" = referrer.as_url().scheme() {
-        if let "http" = module_specifier.as_url().scheme() {
-          let e = OpError::permission_denied(
-            "Modules loaded over https:// are not allowed to import modules over http://".to_string()
-          );
-          return async move { Err(e.into()) }.boxed_local();
-        }
-      }
-    }
-
-    if is_dyn_import {
-      if let Err(e) = self.check_dyn_import(&module_specifier) {
-        return async move { Err(e.into()) }.boxed_local();
-      }
-    } else {
-      // Verify that remote file doesn't try to statically import local file.
-      if let Some(referrer) = maybe_referrer.as_ref() {
-        let referrer_url = referrer.as_url();
-        match referrer_url.scheme() {
-          "http" | "https" => {
-            let specifier_url = module_specifier.as_url();
-            match specifier_url.scheme() {
-              "http" | "https" => {}
-              _ => {
-                let e = OpError::permission_denied(
-                  "Remote modules are not allowed to statically import local modules. Use dynamic import instead.".to_string()
-                );
-                return async move { Err(e.into()) }.boxed_local();
-              }
-            }
-          }
-          _ => {}
-        }
-      }
-    }
-
+    let module_specifier = module_specifier.to_owned();
     let mut state = self.borrow_mut();
     // TODO(bartlomieju): incrementing resolve_count here has no sense...
     state.metrics.resolve_count += 1;
     let module_url_specified = module_specifier.to_string();
     let global_state = state.global_state.clone();
-    let target_lib = state.target_lib.clone();
-    let permissions = if state.is_main {
-      Permissions::allow_all()
-    } else {
-      state.permissions.clone()
-    };
 
+    // TODO(bartlomieju): `fetch_compiled_module` should take `load_id` param
     let fut = async move {
       let compiled_module = global_state
-        .fetch_compiled_module(
-          module_specifier,
-          maybe_referrer,
-          target_lib,
-          permissions,
-          is_dyn_import,
-        )
+        .fetch_compiled_module(module_specifier, maybe_referrer)
         .await?;
       Ok(deno_core::ModuleSource {
         // Real module name, might be different from initial specifier
@@ -354,22 +301,47 @@ impl ModuleLoader for State {
   fn prepare_load(
     &self,
     _load_id: ModuleLoadId,
-    _module_specifier: &ModuleSpecifier,
-    _maybe_referrer: Option<String>,
-    _is_dyn_import: bool,
+    module_specifier: &ModuleSpecifier,
+    maybe_referrer: Option<String>,
+    is_dyn_import: bool,
   ) -> Pin<Box<dyn Future<Output = Result<(), ErrBox>>>> {
-    // TODO(bartlomieju):
-    // 1. recursively:
-    //    a) resolve specifier
-    //    b) check permission if dynamic import
-    //    c) fetch/download source code
-    //    d) parse the source code and extract all import/exports (dependencies)
-    //    e) add discovered deps and loop algorithm until no new dependencies
-    //        are discovered
-    // 2. run through appropriate compiler giving it access only to
-    //     discovered files
-
-    async { Ok(()) }.boxed_local()
+    let module_specifier = module_specifier.clone();
+    let state = self.borrow();
+    let target_lib = state.target_lib.clone();
+    let maybe_import_map = state.import_map.clone();
+    // Only "main" module is loaded without permission check,
+    // ie. module that is associated with "is_main" state
+    // and is not a dynamic import.
+    let permissions = if state.is_main && !is_dyn_import {
+      Permissions::allow_all()
+    } else {
+      state.permissions.clone()
+    };
+    let global_state = state.global_state.clone();
+    // TODO(bartlomieju): I'm not sure if it's correct to ignore
+    // bad referrer - this is the case for `Deno.core.evalContext()` where
+    // `ref_str` is `<unknown>`.
+    let maybe_referrer = if let Some(ref_str) = maybe_referrer {
+      ModuleSpecifier::resolve_url(&ref_str).ok()
+    } else {
+      None
+    };
+    drop(state);
+
+    // TODO(bartlomieju): `prepare_module_load` should take `load_id` param
+    async move {
+      global_state
+        .prepare_module_load(
+          module_specifier,
+          maybe_referrer,
+          target_lib,
+          permissions,
+          is_dyn_import,
+          maybe_import_map,
+        )
+        .await
+    }
+    .boxed_local()
   }
 }
 
@@ -379,19 +351,9 @@ impl State {
     global_state: GlobalState,
     shared_permissions: Option<Permissions>,
     main_module: ModuleSpecifier,
+    maybe_import_map: Option<ImportMap>,
     is_internal: bool,
   ) -> Result<Self, ErrBox> {
-    let import_map: Option<ImportMap> =
-      match global_state.flags.import_map_path.as_ref() {
-        None => None,
-        Some(file_path) => {
-          if !global_state.flags.unstable {
-            exit_unstable("--importmap")
-          }
-          Some(ImportMap::load(file_path)?)
-        }
-      };
-
     let seeded_rng = match global_state.flags.seed {
       Some(seed) => Some(StdRng::seed_from_u64(seed)),
       None => None,
@@ -407,7 +369,7 @@ impl State {
       global_state,
       main_module,
       permissions,
-      import_map,
+      import_map: maybe_import_map,
       metrics: Metrics::default(),
       global_timer: GlobalTimer::new(),
       workers: HashMap::new(),
@@ -529,6 +491,7 @@ impl State {
       GlobalState::mock(vec!["deno".to_string()]),
       None,
       module_specifier,
+      None,
       false,
     )
     .unwrap()