build: allow disabling snapshots for dev (#20048)

Closes #19399 (running without snapshots at all was suggested as an
alternative solution).

Adds a `__runtime_js_sources` pseudo-private feature to load extension
JS sources at runtime for faster development, instead of building and
loading snapshots or embedding sources in the binary. Will only work in
a development environment obviously.

Try running `cargo test --features __runtime_js_sources
integration::node_unit_tests::os_test`. Then break some behaviour in
`ext/node/polyfills/os.ts` e.g. make `function cpus() {}` return an
empty array, and run it again. Fix and then run again. No more build
time in between.

1 files changed, 129 insertions, 37 deletions

diff --git a/runtime/worker.rs b/runtime/worker.rs
index 5ce75cf71..060f5537b 100644
--- a/runtime/worker.rs
+++ b/runtime/worker.rs
@@ -18,6 +18,8 @@ use deno_core::futures::Future;
 use deno_core::v8;
 use deno_core::CompiledWasmModuleStore;
 use deno_core::Extension;
+#[cfg(feature = "__runtime_js_sources")]
+use deno_core::ExtensionFileSource;
 use deno_core::FsModuleLoader;
 use deno_core::GetErrorClassFn;
 use deno_core::JsRuntime;
@@ -59,6 +61,78 @@ impl ExitCode {
   }
 }
 
+// Duplicated in `build.rs`. Keep in sync!
+deno_core::extension!(
+  runtime,
+  esm_entry_point = "ext:runtime/90_deno_ns.js",
+  esm = [
+    dir "js",
+    "01_errors.js",
+    "01_version.ts",
+    "06_util.js",
+    "10_permissions.js",
+    "11_workers.js",
+    "13_buffer.js",
+    "30_os.js",
+    "40_fs_events.js",
+    "40_http.js",
+    "40_process.js",
+    "40_signals.js",
+    "40_tty.js",
+    "41_prompt.js",
+    "90_deno_ns.js",
+    "98_global_scope.js"
+  ],
+);
+
+// Duplicated in `build.rs`. Keep in sync!
+#[cfg(feature = "__runtime_js_sources")]
+pub fn maybe_transpile_source(
+  source: &mut ExtensionFileSource,
+) -> Result<(), AnyError> {
+  use deno_ast::MediaType;
+  use deno_ast::ParseParams;
+  use deno_ast::SourceTextInfo;
+  use deno_core::ExtensionFileSourceCode;
+  use std::path::Path;
+
+  // Always transpile `node:` built-in modules, since they might be TypeScript.
+  let media_type = if source.specifier.starts_with("node:") {
+    MediaType::TypeScript
+  } else {
+    MediaType::from_path(Path::new(&source.specifier))
+  };
+
+  match media_type {
+    MediaType::TypeScript => {}
+    MediaType::JavaScript => return Ok(()),
+    MediaType::Mjs => return Ok(()),
+    _ => panic!(
+      "Unsupported media type for snapshotting {media_type:?} for file {}",
+      source.specifier
+    ),
+  }
+  let code = source.load()?;
+
+  let parsed = deno_ast::parse_module(ParseParams {
+    specifier: source.specifier.to_string(),
+    text_info: SourceTextInfo::from_string(code.as_str().to_owned()),
+    media_type,
+    capture_tokens: false,
+    scope_analysis: false,
+    maybe_syntax: None,
+  })?;
+  let transpiled_source = parsed.transpile(&deno_ast::EmitOptions {
+    imports_not_used_as_values: deno_ast::ImportsNotUsedAsValues::Remove,
+    inline_source_map: false,
+    ..Default::default()
+  })?;
+
+  source.code =
+    ExtensionFileSourceCode::Computed(transpiled_source.text.into());
+  Ok(())
+}
+
 /// This worker is created and used by almost all
 /// subcommands in Deno executable.
 ///
@@ -226,14 +300,14 @@ impl MainWorker {
     // `runtime/build.rs`, `runtime/web_worker.rs` and `cli/build.rs`!
     let mut extensions = vec![
       // Web APIs
-      deno_webidl::deno_webidl::init_ops(),
-      deno_console::deno_console::init_ops(),
-      deno_url::deno_url::init_ops(),
-      deno_web::deno_web::init_ops::<PermissionsContainer>(
+      deno_webidl::deno_webidl::init_ops_and_esm(),
+      deno_console::deno_console::init_ops_and_esm(),
+      deno_url::deno_url::init_ops_and_esm(),
+      deno_web::deno_web::init_ops_and_esm::<PermissionsContainer>(
         options.blob_store.clone(),
         options.bootstrap.location.clone(),
       ),
-      deno_fetch::deno_fetch::init_ops::<PermissionsContainer>(
+      deno_fetch::deno_fetch::init_ops_and_esm::<PermissionsContainer>(
         deno_fetch::Options {
           user_agent: options.bootstrap.user_agent.clone(),
           root_cert_store_provider: options.root_cert_store_provider.clone(),
@@ -244,75 +318,91 @@ impl MainWorker {
           ..Default::default()
         },
       ),
-      deno_cache::deno_cache::init_ops::<SqliteBackedCache>(create_cache),
-      deno_websocket::deno_websocket::init_ops::<PermissionsContainer>(
+      deno_cache::deno_cache::init_ops_and_esm::<SqliteBackedCache>(
+        create_cache,
+      ),
+      deno_websocket::deno_websocket::init_ops_and_esm::<PermissionsContainer>(
         options.bootstrap.user_agent.clone(),
         options.root_cert_store_provider.clone(),
         options.unsafely_ignore_certificate_errors.clone(),
       ),
-      deno_webstorage::deno_webstorage::init_ops(
+      deno_webstorage::deno_webstorage::init_ops_and_esm(
         options.origin_storage_dir.clone(),
       ),
-      deno_crypto::deno_crypto::init_ops(options.seed),
-      deno_broadcast_channel::deno_broadcast_channel::init_ops(
+      deno_crypto::deno_crypto::init_ops_and_esm(options.seed),
+      deno_broadcast_channel::deno_broadcast_channel::init_ops_and_esm(
         options.broadcast_channel.clone(),
         unstable,
       ),
-      deno_ffi::deno_ffi::init_ops::<PermissionsContainer>(unstable),
-      deno_net::deno_net::init_ops::<PermissionsContainer>(
+      deno_ffi::deno_ffi::init_ops_and_esm::<PermissionsContainer>(unstable),
+      deno_net::deno_net::init_ops_and_esm::<PermissionsContainer>(
         options.root_cert_store_provider.clone(),
         unstable,
         options.unsafely_ignore_certificate_errors.clone(),
       ),
-      deno_tls::deno_tls::init_ops(),
-      deno_kv::deno_kv::init_ops(
+      deno_tls::deno_tls::init_ops_and_esm(),
+      deno_kv::deno_kv::init_ops_and_esm(
         SqliteDbHandler::<PermissionsContainer>::new(
           options.origin_storage_dir.clone(),
         ),
         unstable,
       ),
-      deno_napi::deno_napi::init_ops::<PermissionsContainer>(),
-      deno_http::deno_http::init_ops::<DefaultHttpPropertyExtractor>(),
-      deno_io::deno_io::init_ops(Some(options.stdio)),
-      deno_fs::deno_fs::init_ops::<PermissionsContainer>(
+      deno_napi::deno_napi::init_ops_and_esm::<PermissionsContainer>(),
+      deno_http::deno_http::init_ops_and_esm::<DefaultHttpPropertyExtractor>(),
+      deno_io::deno_io::init_ops_and_esm(Some(options.stdio)),
+      deno_fs::deno_fs::init_ops_and_esm::<PermissionsContainer>(
         unstable,
         options.fs.clone(),
       ),
-      deno_node::deno_node::init_ops::<PermissionsContainer>(
+      deno_node::deno_node::init_ops_and_esm::<PermissionsContainer>(
         options.npm_resolver,
         options.fs,
       ),
       // Ops from this crate
-      ops::runtime::deno_runtime::init_ops(main_module.clone()),
-      ops::worker_host::deno_worker_host::init_ops(
+      ops::runtime::deno_runtime::init_ops_and_esm(main_module.clone()),
+      ops::worker_host::deno_worker_host::init_ops_and_esm(
         options.create_web_worker_cb.clone(),
         options.web_worker_preload_module_cb.clone(),
         options.web_worker_pre_execute_module_cb.clone(),
         options.format_js_error_fn.clone(),
       ),
-      ops::fs_events::deno_fs_events::init_ops(),
-      ops::os::deno_os::init_ops(exit_code.clone()),
-      ops::permissions::deno_permissions::init_ops(),
-      ops::process::deno_process::init_ops(),
-      ops::signal::deno_signal::init_ops(),
-      ops::tty::deno_tty::init_ops(),
-      ops::http::deno_http_runtime::init_ops(),
-      deno_permissions_worker::init_ops(
+      ops::fs_events::deno_fs_events::init_ops_and_esm(),
+      ops::os::deno_os::init_ops_and_esm(exit_code.clone()),
+      ops::permissions::deno_permissions::init_ops_and_esm(),
+      ops::process::deno_process::init_ops_and_esm(),
+      ops::signal::deno_signal::init_ops_and_esm(),
+      ops::tty::deno_tty::init_ops_and_esm(),
+      ops::http::deno_http_runtime::init_ops_and_esm(),
+      deno_permissions_worker::init_ops_and_esm(
         permissions,
         unstable,
         enable_testing_features,
       ),
+      runtime::init_ops_and_esm(),
     ];
 
+    for extension in &mut extensions {
+      #[cfg(not(feature = "__runtime_js_sources"))]
+      {
+        extension.js_files = std::borrow::Cow::Borrowed(&[]);
+        extension.esm_files = std::borrow::Cow::Borrowed(&[]);
+        extension.esm_entry_point = None;
+      }
+      #[cfg(feature = "__runtime_js_sources")]
+      {
+        for source in extension.esm_files.to_mut() {
+          maybe_transpile_source(source).unwrap();
+        }
+        for source in extension.js_files.to_mut() {
+          maybe_transpile_source(source).unwrap();
+        }
+      }
+    }
+
     extensions.extend(std::mem::take(&mut options.extensions));
 
-    #[cfg(not(feature = "dont_create_runtime_snapshot"))]
-    let startup_snapshot = options
-      .startup_snapshot
-      .unwrap_or_else(crate::js::deno_isolate_init);
-    #[cfg(feature = "dont_create_runtime_snapshot")]
-    let startup_snapshot = options.startup_snapshot
-      .expect("deno_runtime startup snapshot is not available with 'create_runtime_snapshot' Cargo feature.");
+    #[cfg(all(feature = "include_js_files_for_snapshotting", feature = "dont_create_runtime_snapshot", not(feature = "__runtime_js_sources")))]
+    options.startup_snapshot.as_ref().expect("Sources are not embedded, snapshotting was disabled and a user snapshot was not provided.");
 
     // Clear extension modules from the module map, except preserve `node:*`
     // modules.
@@ -321,7 +411,9 @@ impl MainWorker {
 
     let mut js_runtime = JsRuntime::new(RuntimeOptions {
       module_loader: Some(options.module_loader.clone()),
-      startup_snapshot: Some(startup_snapshot),
+      startup_snapshot: options
+        .startup_snapshot
+        .or_else(crate::js::deno_isolate_init),
       create_params: options.create_params,
       source_map_getter: options.source_map_getter,
       get_error_class_fn: options.get_error_class_fn,