refactor(core): move ModuleMap to separate RefCell (#10656)

This commit moves bulk of the logic related to module loading
from "JsRuntime" to "ModuleMap".

Next steps are to rewrite the actual loading logic (represented by
"RecursiveModuleLoad") to be a part of "ModuleMap" as well --
that way we will be able to track multiple module loads from within
the map which should help me solve the problem of concurrent
loads (since all info about currently loading/loaded modules will
be contained in the ModuleMap, so we'll be able to know if actually
all required modules have been loaded).

1 files changed, 551 insertions, 31 deletions

diff --git a/core/modules.rs b/core/modules.rs
index b68d19def..a21286941 100644
--- a/core/modules.rs
+++ b/core/modules.rs
@@ -2,17 +2,22 @@
 
 use rusty_v8 as v8;
 
+use crate::bindings;
 use crate::error::generic_error;
 use crate::error::AnyError;
 use crate::module_specifier::ModuleSpecifier;
+use crate::runtime::exception_to_err_result;
 use crate::OpState;
 use futures::future::FutureExt;
 use futures::stream::FuturesUnordered;
 use futures::stream::Stream;
+use futures::stream::StreamFuture;
 use futures::stream::TryStreamExt;
+use log::debug;
 use std::cell::RefCell;
 use std::collections::HashMap;
 use std::collections::HashSet;
+use std::convert::TryFrom;
 use std::future::Future;
 use std::pin::Pin;
 use std::rc::Rc;
@@ -336,6 +341,28 @@ impl RecursiveModuleLoad {
     Ok(())
   }
 
+  pub fn is_currently_loading_main_module(&self) -> bool {
+    !self.is_dynamic_import() && self.state == LoadState::LoadingRoot
+  }
+
+  pub fn module_registered(&mut self, module_id: ModuleId) {
+    // If we just finished loading the root module, store the root module id.
+    if self.state == LoadState::LoadingRoot {
+      self.root_module_id = Some(module_id);
+      self.state = LoadState::LoadingImports;
+    }
+
+    if self.pending.is_empty() {
+      self.state = LoadState::Done;
+    }
+  }
+
+  /// Return root `ModuleId`; this function panics
+  /// if load is not finished yet.
+  pub fn expect_finished(&self) -> ModuleId {
+    self.root_module_id.expect("Root module id empty")
+  }
+
   pub fn add_import(
     &mut self,
     specifier: ModuleSpecifier,
@@ -383,6 +410,7 @@ impl Stream for RecursiveModuleLoad {
 
 pub struct ModuleInfo {
   pub id: ModuleId,
+  // Used in "bindings.rs" for "import.meta.main" property value.
   pub main: bool,
   pub name: String,
   pub import_specifiers: Vec<ModuleSpecifier>,
@@ -399,23 +427,41 @@ enum SymbolicModule {
 }
 
 /// A collection of JS modules.
-#[derive(Default)]
 pub struct ModuleMap {
+  // Handling of specifiers and v8 objects
   ids_by_handle: HashMap<v8::Global<v8::Module>, ModuleId>,
   handles_by_id: HashMap<ModuleId, v8::Global<v8::Module>>,
   info: HashMap<ModuleId, ModuleInfo>,
   by_name: HashMap<String, SymbolicModule>,
   next_module_id: ModuleId,
+
+  // Handling of futures for loading module sources
+  pub loader: Rc<dyn ModuleLoader>,
+  op_state: Rc<RefCell<OpState>>,
+  pub(crate) dynamic_import_map:
+    HashMap<ModuleLoadId, v8::Global<v8::PromiseResolver>>,
+  pub(crate) preparing_dynamic_imports:
+    FuturesUnordered<Pin<Box<PrepareLoadFuture>>>,
+  pub(crate) pending_dynamic_imports:
+    FuturesUnordered<StreamFuture<RecursiveModuleLoad>>,
 }
 
 impl ModuleMap {
-  pub fn new() -> ModuleMap {
+  pub fn new(
+    loader: Rc<dyn ModuleLoader>,
+    op_state: Rc<RefCell<OpState>>,
+  ) -> ModuleMap {
     Self {
-      handles_by_id: HashMap::new(),
       ids_by_handle: HashMap::new(),
+      handles_by_id: HashMap::new(),
       info: HashMap::new(),
       by_name: HashMap::new(),
       next_module_id: 1,
+      loader,
+      op_state,
+      dynamic_import_map: HashMap::new(),
+      preparing_dynamic_imports: FuturesUnordered::new(),
+      pending_dynamic_imports: FuturesUnordered::new(),
     }
   }
 
@@ -434,22 +480,52 @@ impl ModuleMap {
     }
   }
 
-  pub fn get_children(&self, id: ModuleId) -> Option<&Vec<ModuleSpecifier>> {
-    self.info.get(&id).map(|i| &i.import_specifiers)
-  }
-
-  pub fn is_registered(&self, specifier: &ModuleSpecifier) -> bool {
-    self.get_id(&specifier.to_string()).is_some()
-  }
-
-  pub fn register(
+  // Create and compile an ES module.
+  pub(crate) fn new_module(
     &mut self,
-    name: &str,
+    scope: &mut v8::HandleScope,
     main: bool,
-    handle: v8::Global<v8::Module>,
-    import_specifiers: Vec<ModuleSpecifier>,
-  ) -> ModuleId {
-    let name = String::from(name);
+    name: &str,
+    source: &str,
+  ) -> Result<ModuleId, AnyError> {
+    let name_str = v8::String::new(scope, name).unwrap();
+    let source_str = v8::String::new(scope, source).unwrap();
+
+    let origin = bindings::module_origin(scope, name_str);
+    let source = v8::script_compiler::Source::new(source_str, Some(&origin));
+
+    let tc_scope = &mut v8::TryCatch::new(scope);
+
+    let maybe_module = v8::script_compiler::compile_module(tc_scope, source);
+
+    if tc_scope.has_caught() {
+      assert!(maybe_module.is_none());
+      let e = tc_scope.exception().unwrap();
+      return exception_to_err_result(tc_scope, e, false);
+    }
+
+    let module = maybe_module.unwrap();
+
+    let mut import_specifiers: Vec<ModuleSpecifier> = vec![];
+    let module_requests = module.get_module_requests();
+    for i in 0..module_requests.length() {
+      let module_request = v8::Local::<v8::ModuleRequest>::try_from(
+        module_requests.get(tc_scope, i).unwrap(),
+      )
+      .unwrap();
+      let import_specifier = module_request
+        .get_specifier()
+        .to_rust_string_lossy(tc_scope);
+      let module_specifier = self.loader.resolve(
+        self.op_state.clone(),
+        &import_specifier,
+        name,
+        false,
+      )?;
+      import_specifiers.push(module_specifier);
+    }
+
+    let handle = v8::Global::<v8::Module>::new(tc_scope, module);
     let id = self.next_module_id;
     self.next_module_id += 1;
     self
@@ -462,11 +538,83 @@ impl ModuleMap {
       ModuleInfo {
         id,
         main,
-        name,
+        name: name.to_string(),
         import_specifiers,
       },
     );
-    id
+
+    Ok(id)
+  }
+
+  pub fn register_during_load(
+    &mut self,
+    scope: &mut v8::HandleScope,
+    module_source: ModuleSource,
+    load: &mut RecursiveModuleLoad,
+  ) -> Result<(), AnyError> {
+    let referrer_specifier =
+      crate::resolve_url(&module_source.module_url_found).unwrap();
+
+    // #A There are 3 cases to handle at this moment:
+    // 1. Source code resolved result have the same module name as requested
+    //    and is not yet registered
+    //     -> register
+    // 2. Source code resolved result have a different name as requested:
+    //   2a. The module with resolved module name has been registered
+    //     -> alias
+    //   2b. The module with resolved module name has not yet been registered
+    //     -> register & alias
+
+    // If necessary, register an alias.
+    if module_source.module_url_specified != module_source.module_url_found {
+      self.alias(
+        &module_source.module_url_specified,
+        &module_source.module_url_found,
+      );
+    }
+
+    let maybe_mod_id = self.get_id(&module_source.module_url_found);
+
+    let module_id = match maybe_mod_id {
+      Some(id) => {
+        // Module has already been registered.
+        debug!(
+          "Already-registered module fetched again: {}",
+          module_source.module_url_found
+        );
+        id
+      }
+      // Module not registered yet, do it now.
+      None => self.new_module(
+        scope,
+        load.is_currently_loading_main_module(),
+        &module_source.module_url_found,
+        &module_source.code,
+      )?,
+    };
+
+    // Now we must iterate over all imports of the module and load them.
+    let imports = self.get_children(module_id).unwrap().clone();
+
+    for module_specifier in imports {
+      let is_registered = self.is_registered(&module_specifier);
+      if !is_registered {
+        load
+          .add_import(module_specifier.to_owned(), referrer_specifier.clone());
+      }
+    }
+
+    load.module_registered(module_id);
+
+    Ok(())
+  }
+
+  pub fn get_children(&self, id: ModuleId) -> Option<&Vec<ModuleSpecifier>> {
+    self.info.get(&id).map(|i| &i.import_specifiers)
+  }
+
+  pub fn is_registered(&self, specifier: &ModuleSpecifier) -> bool {
+    self.get_id(specifier.as_str()).is_some()
   }
 
   pub fn alias(&mut self, name: &str, target: &str) {
@@ -499,17 +647,79 @@ impl ModuleMap {
   pub fn get_info_by_id(&self, id: &ModuleId) -> Option<&ModuleInfo> {
     self.info.get(id)
   }
+
+  pub fn load_main(
+    &self,
+    specifier: &str,
+    code: Option<String>,
+  ) -> RecursiveModuleLoad {
+    RecursiveModuleLoad::main(
+      self.op_state.clone(),
+      specifier,
+      code,
+      self.loader.clone(),
+    )
+  }
+
+  // Initiate loading of a module graph imported using `import()`.
+  pub fn load_dynamic_import(
+    &mut self,
+    specifier: &str,
+    referrer: &str,
+    resolver_handle: v8::Global<v8::PromiseResolver>,
+  ) {
+    let load = RecursiveModuleLoad::dynamic_import(
+      self.op_state.clone(),
+      specifier,
+      referrer,
+      self.loader.clone(),
+    );
+    self.dynamic_import_map.insert(load.id, resolver_handle);
+    let fut = load.prepare().boxed_local();
+    self.preparing_dynamic_imports.push(fut);
+  }
+
+  pub fn has_pending_dynamic_imports(&self) -> bool {
+    !(self.preparing_dynamic_imports.is_empty()
+      && self.pending_dynamic_imports.is_empty())
+  }
+
+  /// Called by `module_resolve_callback` during module instantiation.
+  pub fn resolve_callback<'s>(
+    &self,
+    scope: &mut v8::HandleScope<'s>,
+    specifier: &str,
+    referrer: &str,
+  ) -> Option<v8::Local<'s, v8::Module>> {
+    let resolved_specifier = self
+      .loader
+      .resolve(self.op_state.clone(), specifier, referrer, false)
+      .expect("Module should have been already resolved");
+
+    if let Some(id) = self.get_id(resolved_specifier.as_str()) {
+      if let Some(handle) = self.get_handle(id) {
+        return Some(v8::Local::new(scope, handle));
+      }
+    }
+
+    None
+  }
 }
 
 #[cfg(test)]
 mod tests {
   use super::*;
+  use crate::serialize_op_result;
   use crate::JsRuntime;
+  use crate::Op;
+  use crate::OpPayload;
   use crate::RuntimeOptions;
   use futures::future::FutureExt;
   use std::error::Error;
   use std::fmt;
   use std::future::Future;
+  use std::io;
+  use std::sync::atomic::{AtomicUsize, Ordering};
   use std::sync::Arc;
   use std::sync::Mutex;
 
@@ -704,9 +914,9 @@ mod tests {
       ]
     );
 
-    let state_rc = JsRuntime::state(runtime.v8_isolate());
-    let state = state_rc.borrow();
-    let modules = &state.module_map;
+    let module_map_rc = JsRuntime::module_map(runtime.v8_isolate());
+    let modules = module_map_rc.borrow();
+
     assert_eq!(modules.get_id("file:///a.js"), Some(a_id));
     let b_id = modules.get_id("file:///b.js").unwrap();
     let c_id = modules.get_id("file:///c.js").unwrap();
@@ -746,6 +956,319 @@ mod tests {
   "#;
 
   #[test]
+  fn test_mods() {
+    #[derive(Default)]
+    struct ModsLoader {
+      pub count: Arc<AtomicUsize>,
+    }
+
+    impl ModuleLoader for ModsLoader {
+      fn resolve(
+        &self,
+        _op_state: Rc<RefCell<OpState>>,
+        specifier: &str,
+        referrer: &str,
+        _is_main: bool,
+      ) -> Result<ModuleSpecifier, AnyError> {
+        self.count.fetch_add(1, Ordering::Relaxed);
+        assert_eq!(specifier, "./b.js");
+        assert_eq!(referrer, "file:///a.js");
+        let s = crate::resolve_import(specifier, referrer).unwrap();
+        Ok(s)
+      }
+
+      fn load(
+        &self,
+        _op_state: Rc<RefCell<OpState>>,
+        _module_specifier: &ModuleSpecifier,
+        _maybe_referrer: Option<ModuleSpecifier>,
+        _is_dyn_import: bool,
+      ) -> Pin<Box<ModuleSourceFuture>> {
+        unreachable!()
+      }
+    }
+
+    let loader = Rc::new(ModsLoader::default());
+
+    let resolve_count = loader.count.clone();
+    let dispatch_count = Arc::new(AtomicUsize::new(0));
+    let dispatch_count_ = dispatch_count.clone();
+
+    let dispatcher = move |state, payload: OpPayload| -> Op {
+      dispatch_count_.fetch_add(1, Ordering::Relaxed);
+      let (control, _): (u8, ()) = payload.deserialize().unwrap();
+      assert_eq!(control, 42);
+      let resp = (0, serialize_op_result(Ok(43), state));
+      Op::Async(Box::pin(futures::future::ready(resp)))
+    };
+
+    let mut runtime = JsRuntime::new(RuntimeOptions {
+      module_loader: Some(loader),
+      ..Default::default()
+    });
+    runtime.register_op("op_test", dispatcher);
+    runtime.sync_ops_cache();
+
+    runtime
+      .execute(
+        "setup.js",
+        r#"
+        function assert(cond) {
+          if (!cond) {
+            throw Error("assert");
+          }
+        }
+        "#,
+      )
+      .unwrap();
+
+    assert_eq!(dispatch_count.load(Ordering::Relaxed), 0);
+
+    let module_map_rc = JsRuntime::module_map(runtime.v8_isolate());
+
+    let (mod_a, mod_b) = {
+      let scope = &mut runtime.handle_scope();
+      let mut module_map = module_map_rc.borrow_mut();
+      let specifier_a = "file:///a.js".to_string();
+      let mod_a = module_map
+        .new_module(
+          scope,
+          true,
+          &specifier_a,
+          r#"
+          import { b } from './b.js'
+          if (b() != 'b') throw Error();
+          let control = 42;
+          Deno.core.opAsync("op_test", control);
+        "#,
+        )
+        .unwrap();
+
+      assert_eq!(dispatch_count.load(Ordering::Relaxed), 0);
+      let imports = module_map.get_children(mod_a);
+      assert_eq!(
+        imports,
+        Some(&vec![crate::resolve_url("file:///b.js").unwrap()])
+      );
+
+      let mod_b = module_map
+        .new_module(
+          scope,
+          false,
+          "file:///b.js",
+          "export function b() { return 'b' }",
+        )
+        .unwrap();
+      let imports = module_map.get_children(mod_b).unwrap();
+      assert_eq!(imports.len(), 0);
+      (mod_a, mod_b)
+    };
+
+    runtime.instantiate_module(mod_b).unwrap();
+    assert_eq!(dispatch_count.load(Ordering::Relaxed), 0);
+    assert_eq!(resolve_count.load(Ordering::SeqCst), 1);
+
+    runtime.instantiate_module(mod_a).unwrap();
+    assert_eq!(dispatch_count.load(Ordering::Relaxed), 0);
+
+    runtime.mod_evaluate(mod_a);
+    assert_eq!(dispatch_count.load(Ordering::Relaxed), 1);
+  }
+
+  #[test]
+  fn dyn_import_err() {
+    #[derive(Clone, Default)]
+    struct DynImportErrLoader {
+      pub count: Arc<AtomicUsize>,
+    }
+
+    impl ModuleLoader for DynImportErrLoader {
+      fn resolve(
+        &self,
+        _op_state: Rc<RefCell<OpState>>,
+        specifier: &str,
+        referrer: &str,
+        _is_main: bool,
+      ) -> Result<ModuleSpecifier, AnyError> {
+        self.count.fetch_add(1, Ordering::Relaxed);
+        assert_eq!(specifier, "/foo.js");
+        assert_eq!(referrer, "file:///dyn_import2.js");
+        let s = crate::resolve_import(specifier, referrer).unwrap();
+        Ok(s)
+      }
+
+      fn load(
+        &self,
+        _op_state: Rc<RefCell<OpState>>,
+        _module_specifier: &ModuleSpecifier,
+        _maybe_referrer: Option<ModuleSpecifier>,
+        _is_dyn_import: bool,
+      ) -> Pin<Box<ModuleSourceFuture>> {
+        async { Err(io::Error::from(io::ErrorKind::NotFound).into()) }.boxed()
+      }
+    }
+
+    // Test an erroneous dynamic import where the specified module isn't found.
+    run_in_task(|cx| {
+      let loader = Rc::new(DynImportErrLoader::default());
+      let count = loader.count.clone();
+      let mut runtime = JsRuntime::new(RuntimeOptions {
+        module_loader: Some(loader),
+        ..Default::default()
+      });
+
+      runtime
+        .execute(
+          "file:///dyn_import2.js",
+          r#"
+        (async () => {
+          await import("/foo.js");
+        })();
+        "#,
+        )
+        .unwrap();
+
+      assert_eq!(count.load(Ordering::Relaxed), 0);
+      // We should get an error here.
+      let result = runtime.poll_event_loop(cx);
+      if let Poll::Ready(Ok(_)) = result {
+        unreachable!();
+      }
+      assert_eq!(count.load(Ordering::Relaxed), 2);
+    })
+  }
+
+  #[derive(Clone, Default)]
+  struct DynImportOkLoader {
+    pub prepare_load_count: Arc<AtomicUsize>,
+    pub resolve_count: Arc<AtomicUsize>,
+    pub load_count: Arc<AtomicUsize>,
+  }
+
+  impl ModuleLoader for DynImportOkLoader {
+    fn resolve(
+      &self,
+      _op_state: Rc<RefCell<OpState>>,
+      specifier: &str,
+      referrer: &str,
+      _is_main: bool,
+    ) -> Result<ModuleSpecifier, AnyError> {
+      let c = self.resolve_count.fetch_add(1, Ordering::Relaxed);
+      assert!(c < 4);
+      assert_eq!(specifier, "./b.js");
+      assert_eq!(referrer, "file:///dyn_import3.js");
+      let s = crate::resolve_import(specifier, referrer).unwrap();
+      Ok(s)
+    }
+
+    fn load(
+      &self,
+      _op_state: Rc<RefCell<OpState>>,
+      specifier: &ModuleSpecifier,
+      _maybe_referrer: Option<ModuleSpecifier>,
+      _is_dyn_import: bool,
+    ) -> Pin<Box<ModuleSourceFuture>> {
+      self.load_count.fetch_add(1, Ordering::Relaxed);
+      let info = ModuleSource {
+        module_url_specified: specifier.to_string(),
+        module_url_found: specifier.to_string(),
+        code: "export function b() { return 'b' }".to_owned(),
+      };
+      async move { Ok(info) }.boxed()
+    }
+
+    fn prepare_load(
+      &self,
+      _op_state: Rc<RefCell<OpState>>,
+      _load_id: ModuleLoadId,
+      _module_specifier: &ModuleSpecifier,
+      _maybe_referrer: Option<String>,
+      _is_dyn_import: bool,
+    ) -> Pin<Box<dyn Future<Output = Result<(), AnyError>>>> {
+      self.prepare_load_count.fetch_add(1, Ordering::Relaxed);
+      async { Ok(()) }.boxed_local()
+    }
+  }
+
+  #[test]
+  fn dyn_import_ok() {
+    run_in_task(|cx| {
+      let loader = Rc::new(DynImportOkLoader::default());
+      let prepare_load_count = loader.prepare_load_count.clone();
+      let resolve_count = loader.resolve_count.clone();
+      let load_count = loader.load_count.clone();
+      let mut runtime = JsRuntime::new(RuntimeOptions {
+        module_loader: Some(loader),
+        ..Default::default()
+      });
+
+      // Dynamically import mod_b
+      runtime
+        .execute(
+          "file:///dyn_import3.js",
+          r#"
+          (async () => {
+            let mod = await import("./b.js");
+            if (mod.b() !== 'b') {
+              throw Error("bad1");
+            }
+            // And again!
+            mod = await import("./b.js");
+            if (mod.b() !== 'b') {
+              throw Error("bad2");
+            }
+          })();
+          "#,
+        )
+        .unwrap();
+
+      // First poll runs `prepare_load` hook.
+      assert!(matches!(runtime.poll_event_loop(cx), Poll::Pending));
+      assert_eq!(prepare_load_count.load(Ordering::Relaxed), 1);
+
+      // Second poll actually loads modules into the isolate.
+      assert!(matches!(runtime.poll_event_loop(cx), Poll::Ready(Ok(_))));
+      assert_eq!(resolve_count.load(Ordering::Relaxed), 4);
+      assert_eq!(load_count.load(Ordering::Relaxed), 2);
+      assert!(matches!(runtime.poll_event_loop(cx), Poll::Ready(Ok(_))));
+      assert_eq!(resolve_count.load(Ordering::Relaxed), 4);
+      assert_eq!(load_count.load(Ordering::Relaxed), 2);
+    })
+  }
+
+  #[test]
+  fn dyn_import_borrow_mut_error() {
+    // https://github.com/denoland/deno/issues/6054
+    run_in_task(|cx| {
+      let loader = Rc::new(DynImportOkLoader::default());
+      let prepare_load_count = loader.prepare_load_count.clone();
+      let mut runtime = JsRuntime::new(RuntimeOptions {
+        module_loader: Some(loader),
+        ..Default::default()
+      });
+      runtime.sync_ops_cache();
+      runtime
+        .execute(
+          "file:///dyn_import3.js",
+          r#"
+          (async () => {
+            let mod = await import("./b.js");
+            if (mod.b() !== 'b') {
+              throw Error("bad");
+            }
+          })();
+          "#,
+        )
+        .unwrap();
+      // First poll runs `prepare_load` hook.
+      let _ = runtime.poll_event_loop(cx);
+      assert_eq!(prepare_load_count.load(Ordering::Relaxed), 1);
+      // Second poll triggers error
+      let _ = runtime.poll_event_loop(cx);
+    })
+  }
+
+  #[test]
   fn test_circular_load() {
     let loader = MockLoader::new();
     let loads = loader.loads.clone();
@@ -772,9 +1295,8 @@ mod tests {
         ]
       );
 
-      let state_rc = JsRuntime::state(runtime.v8_isolate());
-      let state = state_rc.borrow();
-      let modules = &state.module_map;
+      let module_map_rc = JsRuntime::module_map(runtime.v8_isolate());
+      let modules = module_map_rc.borrow();
 
       assert_eq!(modules.get_id("file:///circular1.js"), Some(circular1_id));
       let circular2_id = modules.get_id("file:///circular2.js").unwrap();
@@ -845,9 +1367,8 @@ mod tests {
         ]
       );
 
-      let state_rc = JsRuntime::state(runtime.v8_isolate());
-      let state = state_rc.borrow();
-      let modules = &state.module_map;
+      let module_map_rc = JsRuntime::module_map(runtime.v8_isolate());
+      let modules = module_map_rc.borrow();
 
       assert_eq!(modules.get_id("file:///redirect1.js"), Some(redirect1_id));
 
@@ -992,9 +1513,8 @@ mod tests {
       vec!["file:///b.js", "file:///c.js", "file:///d.js"]
     );
 
-    let state_rc = JsRuntime::state(runtime.v8_isolate());
-    let state = state_rc.borrow();
-    let modules = &state.module_map;
+    let module_map_rc = JsRuntime::module_map(runtime.v8_isolate());
+    let modules = module_map_rc.borrow();
 
     assert_eq!(modules.get_id("file:///main_with_code.js"), Some(main_id));
     let b_id = modules.get_id("file:///b.js").unwrap();