core: introduce extensions (#9800)

Extensions allow declarative extensions to "JsRuntime" (ops, state, JS or middleware).

This allows for:
- `op_crates` to be plug-and-play & self-contained, reducing complexity leaked to consumers
- op middleware (like metrics_op) to be opt-in and for new middleware (unstable, tracing,...)
- `MainWorker` and `WebWorker` to be composable, allowing users to extend workers with their ops whilst benefiting from the other infrastructure (inspector, etc...)

In short extensions improve deno's modularity, reducing complexity and leaky abstractions for embedders and the internal codebase.

3 files changed, 168 insertions, 0 deletions

diff --git a/core/extensions.rs b/core/extensions.rs
new file mode 100644
index 000000000..eb6f6f719
--- /dev/null
+++ b/core/extensions.rs
@@ -0,0 +1,105 @@
+use crate::error::AnyError;
+use crate::{OpFn, OpState};
+
+pub type SourcePair = (&'static str, &'static str);
+pub type OpPair = (&'static str, Box<OpFn>);
+pub type OpMiddlewareFn = dyn Fn(&'static str, Box<OpFn>) -> Box<OpFn>;
+pub type OpStateFn = dyn Fn(&mut OpState) -> Result<(), AnyError>;
+
+#[derive(Default)]
+pub struct Extension {
+  js_files: Option<Vec<SourcePair>>,
+  ops: Option<Vec<OpPair>>,
+  opstate_fn: Option<Box<OpStateFn>>,
+  middleware_fn: Option<Box<OpMiddlewareFn>>,
+  initialized: bool,
+}
+
+impl Extension {
+  pub fn new(
+    js_files: Option<Vec<SourcePair>>,
+    ops: Option<Vec<OpPair>>,
+    opstate_fn: Option<Box<OpStateFn>>,
+    middleware_fn: Option<Box<OpMiddlewareFn>>,
+  ) -> Self {
+    Self {
+      js_files,
+      ops,
+      opstate_fn,
+      middleware_fn,
+      initialized: false,
+    }
+  }
+
+  pub fn pure_js(js_files: Vec<SourcePair>) -> Self {
+    Self::new(Some(js_files), None, None, None)
+  }
+
+  pub fn with_ops(
+    js_files: Vec<SourcePair>,
+    ops: Vec<OpPair>,
+    opstate_fn: Option<Box<OpStateFn>>,
+  ) -> Self {
+    Self::new(Some(js_files), Some(ops), opstate_fn, None)
+  }
+}
+
+// Note: this used to be a trait, but we "downgraded" it to a single concrete type
+// for the initial iteration, it will likely become a trait in the future
+impl Extension {
+  /// returns JS source code to be loaded into the isolate (either at snapshotting,
+  /// or at startup).  as a vector of a tuple of the file name, and the source code.
+  pub(crate) fn init_js(&self) -> Vec<SourcePair> {
+    match &self.js_files {
+      Some(files) => files.clone(),
+      None => vec![],
+    }
+  }
+
+  /// Called at JsRuntime startup to initialize ops in the isolate.
+  pub(crate) fn init_ops(&mut self) -> Option<Vec<OpPair>> {
+    // TODO(@AaronO): maybe make op registration idempotent
+    if self.initialized {
+      panic!("init_ops called twice: not idempotent or correct");
+    }
+    self.initialized = true;
+
+    self.ops.take()
+  }
+
+  /// Allows setting up the initial op-state of an isolate at startup.
+  pub(crate) fn init_state(&self, state: &mut OpState) -> Result<(), AnyError> {
+    match &self.opstate_fn {
+      Some(ofn) => ofn(state),
+      None => Ok(()),
+    }
+  }
+
+  /// init_middleware lets us middleware op registrations, it's called before init_ops
+  pub(crate) fn init_middleware(&mut self) -> Option<Box<OpMiddlewareFn>> {
+    self.middleware_fn.take()
+  }
+}
+
+/// Helps embed JS files in an extension. Returns Vec<(&'static str, &'static str)>
+/// representing the filename and source code.
+///
+/// Example:
+/// ```ignore
+/// include_js_files!(
+///   prefix "deno:op_crates/hello",
+///   "01_hello.js",
+///   "02_goodbye.js",
+/// )
+/// ```
+#[macro_export]
+macro_rules! include_js_files {
+  (prefix $prefix:literal, $($file:literal,)+) => {
+    vec![
+      $((
+        concat!($prefix, "/", $file),
+        include_str!($file),
+      ),)+
+    ]
+  };
+}
diff --git a/core/lib.rs b/core/lib.rs
index 9b4ba230b..37055bcc8 100644
--- a/core/lib.rs
+++ b/core/lib.rs
@@ -3,6 +3,7 @@ mod async_cancel;
 mod async_cell;
 mod bindings;
 pub mod error;
+mod extensions;
 mod flags;
 mod gotham_state;
 mod module_specifier;
@@ -75,6 +76,9 @@ pub use crate::runtime::JsErrorCreateFn;
 pub use crate::runtime::JsRuntime;
 pub use crate::runtime::RuntimeOptions;
 pub use crate::runtime::Snapshot;
+// pub use crate::runtime_modules::include_js_files!;
+pub use crate::extensions::Extension;
+pub use crate::extensions::OpMiddlewareFn;
 pub use crate::zero_copy_buf::ZeroCopyBuf;
 
 pub fn v8_version() -> &'static str {
diff --git a/core/runtime.rs b/core/runtime.rs
index 1981df5f3..51b231eb0 100644
--- a/core/runtime.rs
+++ b/core/runtime.rs
@@ -20,6 +20,8 @@ use crate::modules::NoopModuleLoader;
 use crate::modules::PrepareLoadFuture;
 use crate::modules::RecursiveModuleLoad;
 use crate::ops::*;
+use crate::Extension;
+use crate::OpMiddlewareFn;
 use crate::OpPayload;
 use crate::OpResponse;
 use crate::OpState;
@@ -84,6 +86,7 @@ pub struct JsRuntime {
   snapshot_creator: Option<v8::SnapshotCreator>,
   has_snapshotted: bool,
   allocations: IsolateAllocations,
+  extensions: Vec<Extension>,
 }
 
 struct DynImportModEvaluate {
@@ -189,6 +192,10 @@ pub struct RuntimeOptions {
   /// executed tries to load modules.
   pub module_loader: Option<Rc<dyn ModuleLoader>>,
 
+  /// JsRuntime extensions, not to be confused with ES modules
+  /// these are sets of ops and other JS code to be initialized.
+  pub extensions: Vec<Extension>,
+
   /// V8 snapshot that should be loaded on startup.
   ///
   /// Currently can't be used with `will_snapshot`.
@@ -303,6 +310,7 @@ impl JsRuntime {
       snapshot_creator: maybe_snapshot_creator,
       has_snapshotted: false,
       allocations: IsolateAllocations::default(),
+      extensions: options.extensions,
     };
 
     if !has_startup_snapshot {
@@ -357,6 +365,57 @@ impl JsRuntime {
       .unwrap();
   }
 
+  /// Initializes JS of provided Extensions
+  // NOTE: this will probably change when streamlining snapshot flow
+  pub fn init_extension_js(&mut self) -> Result<(), AnyError> {
+    // Take extensions to avoid double-borrow
+    let mut extensions: Vec<Extension> = std::mem::take(&mut self.extensions);
+    for m in extensions.iter_mut() {
+      let js_files = m.init_js();
+      for (filename, source) in js_files {
+        // TODO(@AaronO): use JsRuntime::execute_static() here to move src off heap
+        self.execute(filename, source)?;
+      }
+    }
+    // Restore extensions
+    self.extensions = extensions;
+
+    Ok(())
+  }
+
+  /// Initializes ops of provided Extensions
+  // NOTE: this will probably change when streamlining snapshot flow
+  pub fn init_extension_ops(&mut self) -> Result<(), AnyError> {
+    let op_state = self.op_state();
+    // Take extensions to avoid double-borrow
+    let mut extensions: Vec<Extension> = std::mem::take(&mut self.extensions);
+
+    // Middleware
+    let middleware: Vec<Box<OpMiddlewareFn>> = extensions
+      .iter_mut()
+      .filter_map(|e| e.init_middleware())
+      .collect();
+    // macroware wraps an opfn in all the middleware
+    let macroware =
+      move |name, opfn| middleware.iter().fold(opfn, |opfn, m| m(name, opfn));
+
+    // Register ops
+    for e in extensions.iter_mut() {
+      e.init_state(&mut op_state.borrow_mut())?;
+      // Register each op after middlewaring it
+      let mut ops = e.init_ops().unwrap_or_default();
+      for (name, opfn) in ops {
+        self.register_op(name, macroware(name, opfn));
+      }
+    }
+    // Sync ops cache
+    self.sync_ops_cache();
+    // Restore extensions
+    self.extensions = extensions;
+
+    Ok(())
+  }
+
   /// Grabs a reference to core.js' handleAsyncMsgFromRust
   fn init_recv_cb(&mut self) {
     let scope = &mut self.handle_scope();