fix(runtime): Box the main future to avoid blowing up the stack (#19155)

This fixes `Unhandled exception at [...] Stack overflow` on Windows,
caused by the large size of the main future.

3 files changed, 130 insertions, 93 deletions

diff --git a/cli/main.rs b/cli/main.rs
index 023d5a208..031ab07e0 100644
--- a/cli/main.rs
+++ b/cli/main.rs
@@ -44,39 +44,72 @@ use args::CliOptions;
 use deno_core::anyhow::Context;
 use deno_core::error::AnyError;
 use deno_core::error::JsError;
+use deno_core::futures::FutureExt;
+use deno_core::task::JoinHandle;
 use deno_runtime::colors;
 use deno_runtime::fmt_errors::format_js_error;
 use deno_runtime::tokio_util::create_and_run_current_thread;
 use factory::CliFactory;
 use std::env;
 use std::env::current_exe;
+use std::future::Future;
 use std::path::PathBuf;
 
+/// Ensures that all subcommands return an i32 exit code and an [`AnyError`] error type.
+trait SubcommandOutput {
+  fn output(self) -> Result<i32, AnyError>;
+}
+
+impl SubcommandOutput for Result<i32, AnyError> {
+  fn output(self) -> Result<i32, AnyError> {
+    self
+  }
+}
+
+impl SubcommandOutput for Result<(), AnyError> {
+  fn output(self) -> Result<i32, AnyError> {
+    self.map(|_| 0)
+  }
+}
+
+impl SubcommandOutput for Result<(), std::io::Error> {
+  fn output(self) -> Result<i32, AnyError> {
+    self.map(|_| 0).map_err(|e| e.into())
+  }
+}
+
+/// Ensure that the subcommand runs in a task, rather than being directly executed. Since some of these
+/// futures are very large, this prevents the stack from getting blown out from passing them by value up
+/// the callchain (especially in debug mode when Rust doesn't have a chance to elide copies!).
+#[inline(always)]
+fn spawn_subcommand<F: Future<Output = T> + 'static, T: SubcommandOutput>(
+  f: F,
+) -> JoinHandle<Result<i32, AnyError>> {
+  deno_core::task::spawn(f.map(|r| r.output()))
+}
+
 async fn run_subcommand(flags: Flags) -> Result<i32, AnyError> {
-  match flags.subcommand.clone() {
-    DenoSubcommand::Bench(bench_flags) => {
+  let handle = match flags.subcommand.clone() {
+    DenoSubcommand::Bench(bench_flags) => spawn_subcommand(async {
       let cli_options = CliOptions::from_flags(flags)?;
       let bench_options = cli_options.resolve_bench_options(bench_flags)?;
       if cli_options.watch_paths().is_some() {
         tools::bench::run_benchmarks_with_watch(cli_options, bench_options)
-          .await?;
+          .await
       } else {
-        tools::bench::run_benchmarks(cli_options, bench_options).await?;
+        tools::bench::run_benchmarks(cli_options, bench_options).await
       }
-      Ok(0)
-    }
-    DenoSubcommand::Bundle(bundle_flags) => {
-      tools::bundle::bundle(flags, bundle_flags).await?;
-      Ok(0)
-    }
+    }),
+    DenoSubcommand::Bundle(bundle_flags) => spawn_subcommand(async {
+      tools::bundle::bundle(flags, bundle_flags).await
+    }),
     DenoSubcommand::Doc(doc_flags) => {
-      tools::doc::print_docs(flags, doc_flags).await?;
-      Ok(0)
+      spawn_subcommand(async { tools::doc::print_docs(flags, doc_flags).await })
     }
-    DenoSubcommand::Eval(eval_flags) => {
+    DenoSubcommand::Eval(eval_flags) => spawn_subcommand(async {
       tools::run::eval_command(flags, eval_flags).await
-    }
-    DenoSubcommand::Cache(cache_flags) => {
+    }),
+    DenoSubcommand::Cache(cache_flags) => spawn_subcommand(async move {
       let factory = CliFactory::from_flags(flags).await?;
       let module_load_preparer = factory.module_load_preparer().await?;
       let emitter = factory.emitter()?;
@@ -84,114 +117,102 @@ async fn run_subcommand(flags: Flags) -> Result<i32, AnyError> {
       module_load_preparer
         .load_and_type_check_files(&cache_flags.files)
         .await?;
-      emitter.cache_module_emits(&graph_container.graph())?;
-      Ok(0)
-    }
-    DenoSubcommand::Check(check_flags) => {
+      emitter.cache_module_emits(&graph_container.graph())
+    }),
+    DenoSubcommand::Check(check_flags) => spawn_subcommand(async move {
       let factory = CliFactory::from_flags(flags).await?;
       let module_load_preparer = factory.module_load_preparer().await?;
       module_load_preparer
         .load_and_type_check_files(&check_flags.files)
-        .await?;
-      Ok(0)
-    }
-    DenoSubcommand::Compile(compile_flags) => {
-      tools::compile::compile(flags, compile_flags).await?;
-      Ok(0)
-    }
-    DenoSubcommand::Coverage(coverage_flags) => {
-      tools::coverage::cover_files(flags, coverage_flags).await?;
-      Ok(0)
-    }
-    DenoSubcommand::Fmt(fmt_flags) => {
+        .await
+    }),
+    DenoSubcommand::Compile(compile_flags) => spawn_subcommand(async {
+      tools::compile::compile(flags, compile_flags).await
+    }),
+    DenoSubcommand::Coverage(coverage_flags) => spawn_subcommand(async {
+      tools::coverage::cover_files(flags, coverage_flags).await
+    }),
+    DenoSubcommand::Fmt(fmt_flags) => spawn_subcommand(async move {
       let cli_options = CliOptions::from_flags(flags.clone())?;
       let fmt_options = cli_options.resolve_fmt_options(fmt_flags)?;
-      tools::fmt::format(cli_options, fmt_options).await?;
-      Ok(0)
-    }
+      tools::fmt::format(cli_options, fmt_options).await
+    }),
     DenoSubcommand::Init(init_flags) => {
-      tools::init::init_project(init_flags).await?;
-      Ok(0)
+      spawn_subcommand(async { tools::init::init_project(init_flags).await })
     }
     DenoSubcommand::Info(info_flags) => {
-      tools::info::info(flags, info_flags).await?;
-      Ok(0)
-    }
-    DenoSubcommand::Install(install_flags) => {
-      tools::installer::install_command(flags, install_flags).await?;
-      Ok(0)
+      spawn_subcommand(async { tools::info::info(flags, info_flags).await })
     }
-    DenoSubcommand::Uninstall(uninstall_flags) => {
-      tools::installer::uninstall(uninstall_flags.name, uninstall_flags.root)?;
-      Ok(0)
-    }
-    DenoSubcommand::Lsp => {
-      lsp::start().await?;
-      Ok(0)
-    }
-    DenoSubcommand::Lint(lint_flags) => {
+    DenoSubcommand::Install(install_flags) => spawn_subcommand(async {
+      tools::installer::install_command(flags, install_flags).await
+    }),
+    DenoSubcommand::Uninstall(uninstall_flags) => spawn_subcommand(async {
+      tools::installer::uninstall(uninstall_flags.name, uninstall_flags.root)
+    }),
+    DenoSubcommand::Lsp => spawn_subcommand(async { lsp::start().await }),
+    DenoSubcommand::Lint(lint_flags) => spawn_subcommand(async {
       if lint_flags.rules {
         tools::lint::print_rules_list(lint_flags.json);
+        Ok(())
       } else {
         let cli_options = CliOptions::from_flags(flags)?;
         let lint_options = cli_options.resolve_lint_options(lint_flags)?;
-        tools::lint::lint(cli_options, lint_options).await?;
+        tools::lint::lint(cli_options, lint_options).await
       }
-      Ok(0)
-    }
+    }),
     DenoSubcommand::Repl(repl_flags) => {
-      tools::repl::run(flags, repl_flags).await
+      spawn_subcommand(async move { tools::repl::run(flags, repl_flags).await })
     }
-    DenoSubcommand::Run(run_flags) => {
+    DenoSubcommand::Run(run_flags) => spawn_subcommand(async move {
       if run_flags.is_stdin() {
         tools::run::run_from_stdin(flags).await
       } else {
         tools::run::run_script(flags).await
       }
-    }
-    DenoSubcommand::Task(task_flags) => {
+    }),
+    DenoSubcommand::Task(task_flags) => spawn_subcommand(async {
       tools::task::execute_script(flags, task_flags).await
-    }
+    }),
     DenoSubcommand::Test(test_flags) => {
-      if let Some(ref coverage_dir) = flags.coverage_dir {
-        std::fs::create_dir_all(coverage_dir)
-          .with_context(|| format!("Failed creating: {coverage_dir}"))?;
-        // this is set in order to ensure spawned processes use the same
-        // coverage directory
-        env::set_var(
-          "DENO_UNSTABLE_COVERAGE_DIR",
-          PathBuf::from(coverage_dir).canonicalize()?,
-        );
-      }
-      let cli_options = CliOptions::from_flags(flags)?;
-      let test_options = cli_options.resolve_test_options(test_flags)?;
-
-      if cli_options.watch_paths().is_some() {
-        tools::test::run_tests_with_watch(cli_options, test_options).await?;
-      } else {
-        tools::test::run_tests(cli_options, test_options).await?;
-      }
+      spawn_subcommand(async {
+        if let Some(ref coverage_dir) = flags.coverage_dir {
+          std::fs::create_dir_all(coverage_dir)
+            .with_context(|| format!("Failed creating: {coverage_dir}"))?;
+          // this is set in order to ensure spawned processes use the same
+          // coverage directory
+          env::set_var(
+            "DENO_UNSTABLE_COVERAGE_DIR",
+            PathBuf::from(coverage_dir).canonicalize()?,
+          );
+        }
+        let cli_options = CliOptions::from_flags(flags)?;
+        let test_options = cli_options.resolve_test_options(test_flags)?;
 
-      Ok(0)
+        if cli_options.watch_paths().is_some() {
+          tools::test::run_tests_with_watch(cli_options, test_options).await
+        } else {
+          tools::test::run_tests(cli_options, test_options).await
+        }
+      })
     }
     DenoSubcommand::Completions(completions_flags) => {
-      display::write_to_stdout_ignore_sigpipe(&completions_flags.buf)?;
-      Ok(0)
+      spawn_subcommand(async move {
+        display::write_to_stdout_ignore_sigpipe(&completions_flags.buf)
+      })
     }
-    DenoSubcommand::Types => {
+    DenoSubcommand::Types => spawn_subcommand(async move {
       let types = tsc::get_types_declaration_file_text(flags.unstable);
-      display::write_to_stdout_ignore_sigpipe(types.as_bytes())?;
-      Ok(0)
-    }
-    DenoSubcommand::Upgrade(upgrade_flags) => {
-      tools::upgrade::upgrade(flags, upgrade_flags).await?;
-      Ok(0)
-    }
-    DenoSubcommand::Vendor(vendor_flags) => {
-      tools::vendor::vendor(flags, vendor_flags).await?;
-      Ok(0)
-    }
-  }
+      display::write_to_stdout_ignore_sigpipe(types.as_bytes())
+    }),
+    DenoSubcommand::Upgrade(upgrade_flags) => spawn_subcommand(async {
+      tools::upgrade::upgrade(flags, upgrade_flags).await
+    }),
+    DenoSubcommand::Vendor(vendor_flags) => spawn_subcommand(async {
+      tools::vendor::vendor(flags, vendor_flags).await
+    }),
+  };
+
+  handle.await?
 }
 
 fn setup_panic_hook() {
diff --git a/core/task.rs b/core/task.rs
index 46a4c8c26..3e728a08f 100644
--- a/core/task.rs
+++ b/core/task.rs
@@ -44,6 +44,7 @@ impl<R> Future for JoinHandle<R> {
 /// Equivalent to [`tokio::task::spawn`], but does not require the future to be [`Send`]. Must only be
 /// used on a [`RuntimeFlavor::CurrentThread`] executor, though this is only checked when running with
 /// debug assertions.
+#[inline(always)]
 pub fn spawn<F: Future<Output = R> + 'static, R: 'static>(
   f: F,
 ) -> JoinHandle<R> {
@@ -60,6 +61,7 @@ pub fn spawn<F: Future<Output = R> + 'static, R: 'static>(
 
 /// Equivalent to [`tokio::task::spawn_blocking`]. Currently a thin wrapper around the tokio API, but this
 /// may change in the future.
+#[inline(always)]
 pub fn spawn_blocking<
   F: (FnOnce() -> R) + Send + 'static,
   R: Send + 'static,
@@ -89,6 +91,7 @@ impl<R> MaskResultAsSend<R> {
   }
 }
 
+#[repr(transparent)]
 pub struct MaskFutureAsSend<F> {
   future: F,
 }
@@ -102,6 +105,7 @@ impl<F> MaskFutureAsSend<F> {
   ///
   /// You must ensure that the future is actually used on the same
   /// thread, ie. always use current thread runtime flavor from Tokio.
+  #[inline(always)]
   pub unsafe fn new(future: F) -> Self {
     Self { future }
   }
diff --git a/runtime/tokio_util.rs b/runtime/tokio_util.rs
index ce6ef305f..d723006bd 100644
--- a/runtime/tokio_util.rs
+++ b/runtime/tokio_util.rs
@@ -16,14 +16,26 @@ pub fn create_basic_runtime() -> tokio::runtime::Runtime {
     .unwrap()
 }
 
+#[inline(always)]
 pub fn create_and_run_current_thread<F, R>(future: F) -> R
 where
   F: std::future::Future<Output = R> + 'static,
   R: Send + 'static,
 {
   let rt = create_basic_runtime();
+
+  // Since this is the main future, we want to box it in debug mode because it tends to be fairly
+  // large and the compiler won't optimize repeated copies. We also make this runtime factory
+  // function #[inline(always)] to avoid holding the unboxed, unused future on the stack.
+
+  #[cfg(debug_assertions)]
+  // SAFETY: this this is guaranteed to be running on a current-thread executor
+  let future = Box::pin(unsafe { MaskFutureAsSend::new(future) });
+
+  #[cfg(not(debug_assertions))]
   // SAFETY: this this is guaranteed to be running on a current-thread executor
   let future = unsafe { MaskFutureAsSend::new(future) };
+
   let join_handle = rt.spawn(future);
   rt.block_on(join_handle).unwrap().into_inner()
 }