build: allow disabling snapshots for dev (#20048)

Closes #19399 (running without snapshots at all was suggested as an
alternative solution).

Adds a `__runtime_js_sources` pseudo-private feature to load extension
JS sources at runtime for faster development, instead of building and
loading snapshots or embedding sources in the binary. Will only work in
a development environment obviously.

Try running `cargo test --features __runtime_js_sources
integration::node_unit_tests::os_test`. Then break some behaviour in
`ext/node/polyfills/os.ts` e.g. make `function cpus() {}` return an
empty array, and run it again. Fix and then run again. No more build
time in between.

5 files changed, 61 insertions, 23 deletions

diff --git a/cli/Cargo.toml b/cli/Cargo.toml
index 5ee0bd22d..c1779c102 100644
--- a/cli/Cargo.toml
+++ b/cli/Cargo.toml
@@ -26,6 +26,11 @@ name = "lsp_bench_standalone"
 harness = false
 path = "./bench/lsp_bench_standalone.rs"
 
+[features]
+# A dev feature to disable creations and loading of snapshots in favor of
+# loading JS sources at runtime.
+__runtime_js_sources = ["deno_runtime/__runtime_js_sources"]
+
 [build-dependencies]
 deno_runtime = { workspace = true, features = ["snapshot_from_snapshot", "include_js_files_for_snapshotting"] }
 deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] }
diff --git a/cli/build.rs b/cli/build.rs
index ecf0d3cbe..206c80d1e 100644
--- a/cli/build.rs
+++ b/cli/build.rs
@@ -2,16 +2,10 @@
 
 use std::env;
 use std::path::PathBuf;
-use std::sync::Arc;
 
 use deno_core::snapshot_util::*;
-use deno_core::Extension;
 use deno_core::ExtensionFileSource;
 use deno_core::ExtensionFileSourceCode;
-use deno_runtime::deno_cache::SqliteBackedCache;
-use deno_runtime::deno_http::DefaultHttpPropertyExtractor;
-use deno_runtime::deno_kv::sqlite::SqliteDbHandler;
-use deno_runtime::permissions::PermissionsContainer;
 use deno_runtime::*;
 
 mod ts {
@@ -304,12 +298,10 @@ mod ts {
   }
 }
 
-// FIXME(bartlomieju): information about which extensions were
-// already snapshotted is not preserved in the snapshot. This should be
-// fixed, so we can reliably depend on that information.
-// deps = [runtime]
+// Duplicated in `ops/mod.rs`. Keep in sync!
 deno_core::extension!(
   cli,
+  deps = [runtime],
   esm_entry_point = "ext:cli/99_main.js",
   esm = [
     dir "js",
@@ -326,8 +318,16 @@ deno_core::extension!(
   }
 );
 
+#[cfg(not(feature = "__runtime_js_sources"))]
 #[must_use = "The files listed by create_cli_snapshot should be printed as 'cargo:rerun-if-changed' lines"]
 fn create_cli_snapshot(snapshot_path: PathBuf) -> CreateSnapshotOutput {
+  use deno_core::Extension;
+  use deno_runtime::deno_cache::SqliteBackedCache;
+  use deno_runtime::deno_http::DefaultHttpPropertyExtractor;
+  use deno_runtime::deno_kv::sqlite::SqliteDbHandler;
+  use deno_runtime::permissions::PermissionsContainer;
+  use std::sync::Arc;
+
   // NOTE(bartlomieju): ordering is important here, keep it in sync with
   // `runtime/worker.rs`, `runtime/web_worker.rs` and `runtime/build.rs`!
   let fs = Arc::new(deno_fs::RealFs);
@@ -367,13 +367,14 @@ fn create_cli_snapshot(snapshot_path: PathBuf) -> CreateSnapshotOutput {
     deno_io::deno_io::init_ops(Default::default()),
     deno_fs::deno_fs::init_ops::<PermissionsContainer>(false, fs.clone()),
     deno_node::deno_node::init_ops::<PermissionsContainer>(None, fs),
+    deno_runtime::runtime::init_ops(),
     cli::init_ops_and_esm(), // NOTE: This needs to be init_ops_and_esm!
   ];
 
   create_snapshot(CreateSnapshotOptions {
     cargo_manifest_dir: env!("CARGO_MANIFEST_DIR"),
     snapshot_path,
-    startup_snapshot: Some(deno_runtime::js::deno_isolate_init()),
+    startup_snapshot: deno_runtime::js::deno_isolate_init(),
     extensions,
     compression_cb: None,
     with_runtime_cb: None,
@@ -483,10 +484,13 @@ fn main() {
   let compiler_snapshot_path = o.join("COMPILER_SNAPSHOT.bin");
   ts::create_compiler_snapshot(compiler_snapshot_path, &c);
 
-  let cli_snapshot_path = o.join("CLI_SNAPSHOT.bin");
-  let output = create_cli_snapshot(cli_snapshot_path);
-  for path in output.files_loaded_during_snapshot {
-    println!("cargo:rerun-if-changed={}", path.display())
+  #[cfg(not(feature = "__runtime_js_sources"))]
+  {
+    let cli_snapshot_path = o.join("CLI_SNAPSHOT.bin");
+    let output = create_cli_snapshot(cli_snapshot_path);
+    for path in output.files_loaded_during_snapshot {
+      println!("cargo:rerun-if-changed={}", path.display())
+    }
   }
 
   #[cfg(target_os = "windows")]
diff --git a/cli/js.rs b/cli/js.rs
index 6a312a206..f87bf6d73 100644
--- a/cli/js.rs
+++ b/cli/js.rs
@@ -3,12 +3,20 @@
 use deno_core::Snapshot;
 use log::debug;
 
+#[cfg(not(feature = "__runtime_js_sources"))]
 static CLI_SNAPSHOT: &[u8] =
   include_bytes!(concat!(env!("OUT_DIR"), "/CLI_SNAPSHOT.bin"));
 
-pub fn deno_isolate_init() -> Snapshot {
+pub fn deno_isolate_init() -> Option<Snapshot> {
   debug!("Deno isolate init with snapshots.");
-  Snapshot::Static(CLI_SNAPSHOT)
+  #[cfg(not(feature = "__runtime_js_sources"))]
+  {
+    Some(Snapshot::Static(CLI_SNAPSHOT))
+  }
+  #[cfg(feature = "__runtime_js_sources")]
+  {
+    None
+  }
 }
 
 #[cfg(test)]
@@ -18,7 +26,7 @@ mod tests {
   #[test]
   fn runtime_snapshot() {
     let mut js_runtime = deno_core::JsRuntime::new(deno_core::RuntimeOptions {
-      startup_snapshot: Some(deno_isolate_init()),
+      startup_snapshot: deno_isolate_init(),
       ..Default::default()
     });
     js_runtime
diff --git a/cli/ops/mod.rs b/cli/ops/mod.rs
index 5066c44b9..7af5f14af 100644
--- a/cli/ops/mod.rs
+++ b/cli/ops/mod.rs
@@ -12,17 +12,38 @@ pub mod bench;
 pub mod testing;
 
 pub fn cli_exts(npm_resolver: Arc<CliNpmResolver>) -> Vec<Extension> {
-  vec![deno_cli::init_ops(npm_resolver)]
+  vec![
+    #[cfg(not(feature = "__runtime_js_sources"))]
+    cli::init_ops(npm_resolver),
+    #[cfg(feature = "__runtime_js_sources")]
+    cli::init_ops_and_esm(npm_resolver),
+  ]
 }
 
-deno_core::extension!(deno_cli,
+// ESM parts duplicated in `../build.rs`. Keep in sync!
+deno_core::extension!(cli,
+  deps = [runtime],
   ops = [op_npm_process_state],
+  esm_entry_point = "ext:cli/99_main.js",
+  esm = [
+    dir "js",
+    "40_testing.js",
+    "99_main.js"
+  ],
   options = {
     npm_resolver: Arc<CliNpmResolver>,
   },
   state = |state, options| {
     state.put(options.npm_resolver);
   },
+  customizer = |ext: &mut deno_core::Extension| {
+    ext.esm_files.to_mut().push(deno_core::ExtensionFileSource {
+      specifier: "ext:cli/runtime/js/99_main.js",
+      code: deno_core::ExtensionFileSourceCode::LoadedFromFsDuringSnapshot(
+        deno_runtime::js::PATH_FOR_99_MAIN_JS,
+      ),
+    });
+  },
 );
 
 #[op]
diff --git a/cli/worker.rs b/cli/worker.rs
index 235e9a225..a712dc9c6 100644
--- a/cli/worker.rs
+++ b/cli/worker.rs
@@ -450,7 +450,7 @@ impl CliMainWorkerFactory {
         inspect: shared.options.is_inspecting,
       },
       extensions,
-      startup_snapshot: Some(crate::js::deno_isolate_init()),
+      startup_snapshot: crate::js::deno_isolate_init(),
       create_params: None,
       unsafely_ignore_certificate_errors: shared
         .options
@@ -638,7 +638,7 @@ fn create_web_worker_callback(
         inspect: shared.options.is_inspecting,
       },
       extensions,
-      startup_snapshot: Some(crate::js::deno_isolate_init()),
+      startup_snapshot: crate::js::deno_isolate_init(),
       unsafely_ignore_certificate_errors: shared
         .options
         .unsafely_ignore_certificate_errors
@@ -688,7 +688,7 @@ mod tests {
     let permissions = PermissionsContainer::new(Permissions::default());
 
     let options = WorkerOptions {
-      startup_snapshot: Some(crate::js::deno_isolate_init()),
+      startup_snapshot: crate::js::deno_isolate_init(),
       ..Default::default()
     };