refactor: Use ES modules for internal runtime code (#17648)

This PR refactors all internal js files (except core) to be written as
ES modules.
`__bootstrap`has been mostly replaced with static imports in form in
`internal:[path to file from repo root]`.
To specify if files are ESM, an `esm` method has been added to
`Extension`, similar to the `js` method.
A new ModuleLoader called `InternalModuleLoader` has been added to
enable the loading of internal specifiers, which is used in all
situations except when a snapshot is only loaded, and not a new one is
created from it.

---------

Co-authored-by: Bartek Iwańczuk <biwanczuk@gmail.com>

7 files changed, 217 insertions, 41 deletions

diff --git a/core/01_core.js b/core/01_core.js
index d3cd1d7be..6a6696bf8 100644
--- a/core/01_core.js
+++ b/core/01_core.js
@@ -427,6 +427,8 @@
   });
 
   ObjectAssign(globalThis.__bootstrap, { core });
+  const internals = {};
+  ObjectAssign(globalThis.__bootstrap, { internals });
   ObjectAssign(globalThis.Deno, { core });
 
   // Direct bindings on `globalThis`
diff --git a/core/bindings.rs b/core/bindings.rs
index d5f38d3c2..dce97000c 100644
--- a/core/bindings.rs
+++ b/core/bindings.rs
@@ -267,45 +267,47 @@ pub fn host_import_module_dynamically_callback<'s>(
     .unwrap()
     .to_rust_string_lossy(scope);
 
+  let is_internal_module = specifier_str.starts_with("internal:");
   let resolver = v8::PromiseResolver::new(scope).unwrap();
   let promise = resolver.get_promise(scope);
 
-  let assertions = parse_import_assertions(
-    scope,
-    import_assertions,
-    ImportAssertionsKind::DynamicImport,
-  );
+  if !is_internal_module {
+    let assertions = parse_import_assertions(
+      scope,
+      import_assertions,
+      ImportAssertionsKind::DynamicImport,
+    );
 
-  {
-    let tc_scope = &mut v8::TryCatch::new(scope);
-    validate_import_assertions(tc_scope, &assertions);
-    if tc_scope.has_caught() {
-      let e = tc_scope.exception().unwrap();
-      resolver.reject(tc_scope, e);
+    {
+      let tc_scope = &mut v8::TryCatch::new(scope);
+      validate_import_assertions(tc_scope, &assertions);
+      if tc_scope.has_caught() {
+        let e = tc_scope.exception().unwrap();
+        resolver.reject(tc_scope, e);
+      }
     }
-  }
-  let asserted_module_type =
-    get_asserted_module_type_from_assertions(&assertions);
+    let asserted_module_type =
+      get_asserted_module_type_from_assertions(&assertions);
 
-  let resolver_handle = v8::Global::new(scope, resolver);
-  {
-    let state_rc = JsRuntime::state(scope);
-    let module_map_rc = JsRuntime::module_map(scope);
+    let resolver_handle = v8::Global::new(scope, resolver);
+    {
+      let state_rc = JsRuntime::state(scope);
+      let module_map_rc = JsRuntime::module_map(scope);
 
-    debug!(
-      "dyn_import specifier {} referrer {} ",
-      specifier_str, referrer_name_str
-    );
-    ModuleMap::load_dynamic_import(
-      module_map_rc,
-      &specifier_str,
-      &referrer_name_str,
-      asserted_module_type,
-      resolver_handle,
-    );
-    state_rc.borrow_mut().notify_new_dynamic_import();
+      debug!(
+        "dyn_import specifier {} referrer {} ",
+        specifier_str, referrer_name_str
+      );
+      ModuleMap::load_dynamic_import(
+        module_map_rc,
+        &specifier_str,
+        &referrer_name_str,
+        asserted_module_type,
+        resolver_handle,
+      );
+      state_rc.borrow_mut().notify_new_dynamic_import();
+    }
   }
-
   // Map errors from module resolution (not JS errors from module execution) to
   // ones rethrown from this scope, so they include the call stack of the
   // dynamic import site. Error objects without any stack frames are assumed to
@@ -317,6 +319,14 @@ pub fn host_import_module_dynamically_callback<'s>(
 
   let promise = promise.catch(scope, map_err).unwrap();
 
+  if is_internal_module {
+    let message =
+      v8::String::new(scope, "Cannot load internal module from external code")
+        .unwrap();
+    let exception = v8::Exception::type_error(scope, message);
+    resolver.reject(scope, exception);
+  }
+
   Some(promise)
 }
 
diff --git a/core/extensions.rs b/core/extensions.rs
index 129e7b62a..b981e6da2 100644
--- a/core/extensions.rs
+++ b/core/extensions.rs
@@ -38,6 +38,7 @@ impl OpDecl {
 #[derive(Default)]
 pub struct Extension {
   js_files: Option<Vec<SourcePair>>,
+  esm_files: Option<Vec<SourcePair>>,
   ops: Option<Vec<OpDecl>>,
   opstate_fn: Option<Box<OpStateFn>>,
   middleware_fn: Option<Box<OpMiddlewareFn>>,
@@ -81,13 +82,20 @@ impl Extension {
 
   /// returns JS source code to be loaded into the isolate (either at snapshotting,
   /// or at startup).  as a vector of a tuple of the file name, and the source code.
-  pub fn init_js(&self) -> &[SourcePair] {
+  pub fn get_js_sources(&self) -> &[SourcePair] {
     match &self.js_files {
       Some(files) => files,
       None => &[],
     }
   }
 
+  pub fn get_esm_sources(&self) -> &[SourcePair] {
+    match &self.esm_files {
+      Some(files) => files,
+      None => &[],
+    }
+  }
+
   /// Called at JsRuntime startup to initialize ops in the isolate.
   pub fn init_ops(&mut self) -> Option<Vec<OpDecl>> {
     // TODO(@AaronO): maybe make op registration idempotent
@@ -145,6 +153,7 @@ impl Extension {
 #[derive(Default)]
 pub struct ExtensionBuilder {
   js: Vec<SourcePair>,
+  esm: Vec<SourcePair>,
   ops: Vec<OpDecl>,
   state: Option<Box<OpStateFn>>,
   middleware: Option<Box<OpMiddlewareFn>>,
@@ -164,6 +173,11 @@ impl ExtensionBuilder {
     self
   }
 
+  pub fn esm(&mut self, js_files: Vec<SourcePair>) -> &mut Self {
+    self.esm.extend(js_files);
+    self
+  }
+
   pub fn ops(&mut self, ops: Vec<OpDecl>) -> &mut Self {
     self.ops.extend(ops);
     self
@@ -195,10 +209,12 @@ impl ExtensionBuilder {
 
   pub fn build(&mut self) -> Extension {
     let js_files = Some(std::mem::take(&mut self.js));
+    let esm_files = Some(std::mem::take(&mut self.esm));
     let ops = Some(std::mem::take(&mut self.ops));
     let deps = Some(std::mem::take(&mut self.deps));
     Extension {
       js_files,
+      esm_files,
       ops,
       opstate_fn: self.state.take(),
       middleware_fn: self.middleware.take(),
diff --git a/core/lib.rs b/core/lib.rs
index 461b4fd20..868d6b749 100644
--- a/core/lib.rs
+++ b/core/lib.rs
@@ -73,6 +73,7 @@ pub use crate::module_specifier::ModuleResolutionError;
 pub use crate::module_specifier::ModuleSpecifier;
 pub use crate::module_specifier::DUMMY_SPECIFIER;
 pub use crate::modules::FsModuleLoader;
+pub use crate::modules::InternalModuleLoader;
 pub use crate::modules::ModuleId;
 pub use crate::modules::ModuleLoader;
 pub use crate::modules::ModuleSource;
diff --git a/core/modules.rs b/core/modules.rs
index 1b7169ea4..b57428070 100644
--- a/core/modules.rs
+++ b/core/modules.rs
@@ -292,6 +292,69 @@ impl ModuleLoader for NoopModuleLoader {
   }
 }
 
+pub struct InternalModuleLoader(Rc<dyn ModuleLoader>);
+
+impl InternalModuleLoader {
+  pub fn new(module_loader: Option<Rc<dyn ModuleLoader>>) -> Self {
+    InternalModuleLoader(
+      module_loader.unwrap_or_else(|| Rc::new(NoopModuleLoader)),
+    )
+  }
+}
+
+impl ModuleLoader for InternalModuleLoader {
+  fn resolve(
+    &self,
+    specifier: &str,
+    referrer: &str,
+    kind: ResolutionKind,
+  ) -> Result<ModuleSpecifier, Error> {
+    if let Ok(url_specifier) = ModuleSpecifier::parse(specifier) {
+      if url_specifier.scheme() == "internal" {
+        let referrer_specifier = ModuleSpecifier::parse(referrer).ok();
+        if referrer == "." || referrer_specifier.unwrap().scheme() == "internal"
+        {
+          return Ok(url_specifier);
+        } else {
+          return Err(generic_error(
+            "Cannot load internal module from external code",
+          ));
+        };
+      }
+    }
+
+    self.0.resolve(specifier, referrer, kind)
+  }
+
+  fn load(
+    &self,
+    module_specifier: &ModuleSpecifier,
+    maybe_referrer: Option<ModuleSpecifier>,
+    is_dyn_import: bool,
+  ) -> Pin<Box<ModuleSourceFuture>> {
+    self.0.load(module_specifier, maybe_referrer, is_dyn_import)
+  }
+
+  fn prepare_load(
+    &self,
+    op_state: Rc<RefCell<OpState>>,
+    module_specifier: &ModuleSpecifier,
+    maybe_referrer: Option<String>,
+    is_dyn_import: bool,
+  ) -> Pin<Box<dyn Future<Output = Result<(), Error>>>> {
+    if module_specifier.scheme() == "internal" {
+      return async { Ok(()) }.boxed_local();
+    }
+
+    self.0.prepare_load(
+      op_state,
+      module_specifier,
+      maybe_referrer,
+      is_dyn_import,
+    )
+  }
+}
+
 /// Basic file system module loader.
 ///
 /// Note that this loader will **block** event loop
@@ -2508,4 +2571,33 @@ if (import.meta.url != 'file:///main_with_code.js') throw Error();
       )
       .unwrap();
   }
+
+  #[test]
+  fn internal_module_loader() {
+    let loader = InternalModuleLoader::new(None);
+    assert!(loader
+      .resolve("internal:foo", "internal:bar", ResolutionKind::Import)
+      .is_ok());
+    assert_eq!(
+      loader
+        .resolve("internal:foo", "file://bar", ResolutionKind::Import)
+        .err()
+        .map(|e| e.to_string()),
+      Some("Cannot load internal module from external code".to_string())
+    );
+    assert_eq!(
+      loader
+        .resolve("file://foo", "file://bar", ResolutionKind::Import)
+        .err()
+        .map(|e| e.to_string()),
+      Some("Module loading is not supported".to_string())
+    );
+    assert_eq!(
+      loader
+        .resolve("file://foo", "internal:bar", ResolutionKind::Import)
+        .err()
+        .map(|e| e.to_string()),
+      Some("Module loading is not supported".to_string())
+    );
+  }
 }
diff --git a/core/runtime.rs b/core/runtime.rs
index 1418e5791..096d26ca3 100644
--- a/core/runtime.rs
+++ b/core/runtime.rs
@@ -13,17 +13,18 @@ use crate::modules::ModuleId;
 use crate::modules::ModuleLoadId;
 use crate::modules::ModuleLoader;
 use crate::modules::ModuleMap;
-use crate::modules::NoopModuleLoader;
 use crate::op_void_async;
 use crate::op_void_sync;
 use crate::ops::*;
 use crate::source_map::SourceMapCache;
 use crate::source_map::SourceMapGetter;
 use crate::Extension;
+use crate::NoopModuleLoader;
 use crate::OpMiddlewareFn;
 use crate::OpResult;
 use crate::OpState;
 use crate::PromiseId;
+use anyhow::Context as AnyhowContext;
 use anyhow::Error;
 use futures::channel::oneshot;
 use futures::future::poll_fn;
@@ -605,9 +606,16 @@ impl JsRuntime {
       None
     };
 
-    let loader = options
-      .module_loader
-      .unwrap_or_else(|| Rc::new(NoopModuleLoader));
+    let loader = if snapshot_options != SnapshotOptions::Load {
+      Rc::new(crate::modules::InternalModuleLoader::new(
+        options.module_loader,
+      ))
+    } else {
+      options
+        .module_loader
+        .unwrap_or_else(|| Rc::new(NoopModuleLoader))
+    };
+
     {
       let mut state = state_rc.borrow_mut();
       state.global_realm = Some(JsRealm(global_context.clone()));
@@ -805,10 +813,30 @@ impl JsRuntime {
     // Take extensions to avoid double-borrow
     let extensions = std::mem::take(&mut self.extensions_with_js);
     for ext in &extensions {
-      let js_files = ext.init_js();
-      for (filename, source) in js_files {
-        // TODO(@AaronO): use JsRuntime::execute_static() here to move src off heap
-        realm.execute_script(self.v8_isolate(), filename, source)?;
+      {
+        let js_files = ext.get_esm_sources();
+        for (filename, source) in js_files {
+          futures::executor::block_on(async {
+            let id = self
+              .load_side_module(
+                &ModuleSpecifier::parse(filename)?,
+                Some(source.to_string()),
+              )
+              .await?;
+            let receiver = self.mod_evaluate(id);
+            self.run_event_loop(false).await?;
+            receiver.await?
+          })
+          .with_context(|| format!("Couldn't execute '{filename}'"))?;
+        }
+      }
+
+      {
+        let js_files = ext.get_js_sources();
+        for (filename, source) in js_files {
+          // TODO(@AaronO): use JsRuntime::execute_static() here to move src off heap
+          realm.execute_script(self.v8_isolate(), filename, source)?;
+        }
       }
     }
     // Restore extensions
diff --git a/core/snapshot_util.rs b/core/snapshot_util.rs
index 8e397e262..8daaa9866 100644
--- a/core/snapshot_util.rs
+++ b/core/snapshot_util.rs
@@ -1,10 +1,12 @@
 // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license.
 
+use anyhow::Context;
 use std::path::Path;
 use std::path::PathBuf;
 
 use crate::Extension;
 use crate::JsRuntime;
+use crate::ModuleSpecifier;
 use crate::RuntimeOptions;
 use crate::Snapshot;
 
@@ -17,6 +19,7 @@ pub struct CreateSnapshotOptions {
   pub extensions: Vec<Extension>,
   pub extensions_with_js: Vec<Extension>,
   pub additional_files: Vec<PathBuf>,
+  pub additional_esm_files: Vec<PathBuf>,
   pub compression_cb: Option<Box<CompressionCb>>,
 }
 
@@ -44,6 +47,27 @@ pub fn create_snapshot(create_snapshot_options: CreateSnapshotOptions) {
       )
       .unwrap();
   }
+  for file in create_snapshot_options.additional_esm_files {
+    let display_path = file.strip_prefix(display_root).unwrap_or(&file);
+    let display_path_str = display_path.display().to_string();
+
+    let filename =
+      &("internal:".to_string() + &display_path_str.replace('\\', "/"));
+
+    futures::executor::block_on(async {
+      let id = js_runtime
+        .load_side_module(
+          &ModuleSpecifier::parse(filename)?,
+          Some(std::fs::read_to_string(&file)?),
+        )
+        .await?;
+      let receiver = js_runtime.mod_evaluate(id);
+      js_runtime.run_event_loop(false).await?;
+      receiver.await?
+    })
+    .with_context(|| format!("Couldn't execute '{}'", file.display()))
+    .unwrap();
+  }
 
   let snapshot = js_runtime.snapshot();
   let snapshot_slice: &[u8] = &snapshot;
@@ -79,9 +103,12 @@ pub fn create_snapshot(create_snapshot_options: CreateSnapshotOptions) {
   );
 }
 
+pub type FilterFn = Box<dyn Fn(&PathBuf) -> bool>;
+
 pub fn get_js_files(
   cargo_manifest_dir: &'static str,
   directory: &str,
+  filter: Option<FilterFn>,
 ) -> Vec<PathBuf> {
   let manifest_dir = Path::new(cargo_manifest_dir);
   let mut js_files = std::fs::read_dir(directory)
@@ -92,7 +119,7 @@ pub fn get_js_files(
     })
     .filter(|path| {
       path.extension().unwrap_or_default() == "js"
-        && !path.ends_with("99_main.js")
+        && filter.as_ref().map(|filter| filter(path)).unwrap_or(true)
     })
     .collect::<Vec<PathBuf>>();
   js_files.sort();