build: allow disabling snapshots for dev (#20048)

Closes #19399 (running without snapshots at all was suggested as an
alternative solution).

Adds a `__runtime_js_sources` pseudo-private feature to load extension
JS sources at runtime for faster development, instead of building and
loading snapshots or embedding sources in the binary. Will only work in
a development environment obviously.

Try running `cargo test --features __runtime_js_sources
integration::node_unit_tests::os_test`. Then break some behaviour in
`ext/node/polyfills/os.ts` e.g. make `function cpus() {}` return an
empty array, and run it again. Fix and then run again. No more build
time in between.

1 files changed, 60 insertions, 40 deletions

diff --git a/runtime/web_worker.rs b/runtime/web_worker.rs
index af1ae19e0..a3b93836c 100644
--- a/runtime/web_worker.rs
+++ b/runtime/web_worker.rs
@@ -4,6 +4,7 @@ use crate::inspector_server::InspectorServer;
 use crate::ops;
 use crate::permissions::PermissionsContainer;
 use crate::tokio_util::create_and_run_current_thread;
+use crate::worker::runtime;
 use crate::worker::FormatJsErrorFn;
 use crate::BootstrapOptions;
 use deno_broadcast_channel::InMemoryBroadcastChannel;
@@ -397,16 +398,17 @@ impl WebWorker {
 
     // NOTE(bartlomieju): ordering is important here, keep it in sync with
     // `runtime/build.rs`, `runtime/worker.rs` and `cli/build.rs`!
-    let mut extensions: Vec<Extension> = vec![
+
+    let mut extensions = vec![
       // Web APIs
-      deno_webidl::deno_webidl::init_ops(),
-      deno_console::deno_console::init_ops(),
-      deno_url::deno_url::init_ops(),
-      deno_web::deno_web::init_ops::<PermissionsContainer>(
+      deno_webidl::deno_webidl::init_ops_and_esm(),
+      deno_console::deno_console::init_ops_and_esm(),
+      deno_url::deno_url::init_ops_and_esm(),
+      deno_web::deno_web::init_ops_and_esm::<PermissionsContainer>(
         options.blob_store.clone(),
         Some(main_module.clone()),
       ),
-      deno_fetch::deno_fetch::init_ops::<PermissionsContainer>(
+      deno_fetch::deno_fetch::init_ops_and_esm::<PermissionsContainer>(
         deno_fetch::Options {
           user_agent: options.bootstrap.user_agent.clone(),
           root_cert_store_provider: options.root_cert_store_provider.clone(),
@@ -417,73 +419,89 @@ impl WebWorker {
           ..Default::default()
         },
       ),
-      deno_cache::deno_cache::init_ops::<SqliteBackedCache>(create_cache),
-      deno_websocket::deno_websocket::init_ops::<PermissionsContainer>(
+      deno_cache::deno_cache::init_ops_and_esm::<SqliteBackedCache>(
+        create_cache,
+      ),
+      deno_websocket::deno_websocket::init_ops_and_esm::<PermissionsContainer>(
         options.bootstrap.user_agent.clone(),
         options.root_cert_store_provider.clone(),
         options.unsafely_ignore_certificate_errors.clone(),
       ),
-      deno_webstorage::deno_webstorage::init_ops(None).disable(),
-      deno_crypto::deno_crypto::init_ops(options.seed),
-      deno_broadcast_channel::deno_broadcast_channel::init_ops(
+      deno_webstorage::deno_webstorage::init_ops_and_esm(None).disable(),
+      deno_crypto::deno_crypto::init_ops_and_esm(options.seed),
+      deno_broadcast_channel::deno_broadcast_channel::init_ops_and_esm(
         options.broadcast_channel.clone(),
         unstable,
       ),
-      deno_ffi::deno_ffi::init_ops::<PermissionsContainer>(unstable),
-      deno_net::deno_net::init_ops::<PermissionsContainer>(
+      deno_ffi::deno_ffi::init_ops_and_esm::<PermissionsContainer>(unstable),
+      deno_net::deno_net::init_ops_and_esm::<PermissionsContainer>(
         options.root_cert_store_provider.clone(),
         unstable,
         options.unsafely_ignore_certificate_errors.clone(),
       ),
-      deno_tls::deno_tls::init_ops(),
-      deno_kv::deno_kv::init_ops(
+      deno_tls::deno_tls::init_ops_and_esm(),
+      deno_kv::deno_kv::init_ops_and_esm(
         SqliteDbHandler::<PermissionsContainer>::new(None),
         unstable,
       ),
-      deno_napi::deno_napi::init_ops::<PermissionsContainer>(),
-      deno_http::deno_http::init_ops::<DefaultHttpPropertyExtractor>(),
-      deno_io::deno_io::init_ops(Some(options.stdio)),
-      deno_fs::deno_fs::init_ops::<PermissionsContainer>(
+      deno_napi::deno_napi::init_ops_and_esm::<PermissionsContainer>(),
+      deno_http::deno_http::init_ops_and_esm::<DefaultHttpPropertyExtractor>(),
+      deno_io::deno_io::init_ops_and_esm(Some(options.stdio)),
+      deno_fs::deno_fs::init_ops_and_esm::<PermissionsContainer>(
         unstable,
         options.fs.clone(),
       ),
-      deno_node::deno_node::init_ops::<PermissionsContainer>(
+      deno_node::deno_node::init_ops_and_esm::<PermissionsContainer>(
         options.npm_resolver,
         options.fs,
       ),
       // Runtime ops that are always initialized for WebWorkers
-      ops::web_worker::deno_web_worker::init_ops(),
-      ops::runtime::deno_runtime::init_ops(main_module.clone()),
-      ops::worker_host::deno_worker_host::init_ops(
+      ops::web_worker::deno_web_worker::init_ops_and_esm(),
+      ops::runtime::deno_runtime::init_ops_and_esm(main_module.clone()),
+      ops::worker_host::deno_worker_host::init_ops_and_esm(
         options.create_web_worker_cb.clone(),
         options.preload_module_cb.clone(),
         options.pre_execute_module_cb.clone(),
         options.format_js_error_fn.clone(),
       ),
-      ops::fs_events::deno_fs_events::init_ops(),
-      ops::os::deno_os_worker::init_ops(),
-      ops::permissions::deno_permissions::init_ops(),
-      ops::process::deno_process::init_ops(),
-      ops::signal::deno_signal::init_ops(),
-      ops::tty::deno_tty::init_ops(),
-      ops::http::deno_http_runtime::init_ops(),
-      deno_permissions_web_worker::init_ops(
+      ops::fs_events::deno_fs_events::init_ops_and_esm(),
+      ops::os::deno_os_worker::init_ops_and_esm(),
+      ops::permissions::deno_permissions::init_ops_and_esm(),
+      ops::process::deno_process::init_ops_and_esm(),
+      ops::signal::deno_signal::init_ops_and_esm(),
+      ops::tty::deno_tty::init_ops_and_esm(),
+      ops::http::deno_http_runtime::init_ops_and_esm(),
+      deno_permissions_web_worker::init_ops_and_esm(
         permissions,
         unstable,
         enable_testing_features,
       ),
+      runtime::init_ops_and_esm(),
     ];
 
-    // Append exts
+    for extension in &mut extensions {
+      #[cfg(not(feature = "__runtime_js_sources"))]
+      {
+        extension.js_files = std::borrow::Cow::Borrowed(&[]);
+        extension.esm_files = std::borrow::Cow::Borrowed(&[]);
+        extension.esm_entry_point = None;
+      }
+      #[cfg(feature = "__runtime_js_sources")]
+      {
+        use crate::worker::maybe_transpile_source;
+        for source in extension.esm_files.to_mut() {
+          maybe_transpile_source(source).unwrap();
+        }
+        for source in extension.js_files.to_mut() {
+          maybe_transpile_source(source).unwrap();
+        }
+      }
+    }
+
     extensions.extend(std::mem::take(&mut options.extensions));
 
-    #[cfg(not(feature = "dont_create_runtime_snapshot"))]
-    let startup_snapshot = options
-      .startup_snapshot
-      .unwrap_or_else(crate::js::deno_isolate_init);
-    #[cfg(feature = "dont_create_runtime_snapshot")]
-    let startup_snapshot = options.startup_snapshot
-      .expect("deno_runtime startup snapshot is not available with 'create_runtime_snapshot' Cargo feature.");
+    #[cfg(all(feature = "include_js_files_for_snapshotting", feature = "dont_create_runtime_snapshot", not(feature = "__runtime_js_sources")))]
+    options.startup_snapshot.as_ref().expect("Sources are not embedded, snapshotting was disabled and a user snapshot was not provided.");
 
     // Clear extension modules from the module map, except preserve `node:*`
     // modules as `node:` specifiers.
@@ -492,7 +510,9 @@ impl WebWorker {
 
     let mut js_runtime = JsRuntime::new(RuntimeOptions {
       module_loader: Some(options.module_loader.clone()),
-      startup_snapshot: Some(startup_snapshot),
+      startup_snapshot: options
+        .startup_snapshot
+        .or_else(crate::js::deno_isolate_init),
       source_map_getter: options.source_map_getter,
       get_error_class_fn: options.get_error_class_fn,
       shared_array_buffer_store: options.shared_array_buffer_store.clone(),