diff options
| -rw-r--r-- | cli/Cargo.toml | 8 | ||||
| -rw-r--r-- | cli/build.rs | 4 | ||||
| -rw-r--r-- | core/Cargo.toml | 1 | ||||
| -rw-r--r-- | core/extensions.rs | 49 | ||||
| -rw-r--r-- | core/modules.rs | 8 | ||||
| -rw-r--r-- | core/runtime.rs | 6 | ||||
| -rw-r--r-- | core/snapshot_util.rs | 6 | ||||
| -rw-r--r-- | runtime/Cargo.toml | 11 | ||||
| -rw-r--r-- | runtime/build.rs | 33 | ||||
| -rw-r--r-- | runtime/js.rs | 18 | ||||
| -rw-r--r-- | runtime/web_worker.rs | 15 | ||||
| -rw-r--r-- | runtime/worker.rs | 15 |
12 files changed, 125 insertions, 49 deletions
diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 1a5ab72f0..f3bb67765 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -27,8 +27,8 @@ harness = false path = "./bench/lsp_bench_standalone.rs" [build-dependencies] -deno_runtime = { workspace = true, features = ["snapshot_from_snapshot"] } -deno_core.workspace = true +deno_runtime = { workspace = true, features = ["snapshot_from_snapshot", "include_js_files_for_snapshotting"] } +deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] } regex.workspace = true serde.workspace = true serde_json.workspace = true @@ -43,13 +43,13 @@ winres.workspace = true [dependencies] deno_ast = { workspace = true, features = ["bundler", "cjs", "codegen", "dep_graph", "module_specifier", "proposal", "react", "sourcemap", "transforms", "typescript", "view", "visit"] } -deno_core.workspace = true +deno_core = { workspace = true, features = ["include_js_files_for_snapshotting"] } deno_doc = "0.55.0" deno_emit = "0.15.0" deno_graph = "0.43.2" deno_lint = { version = "0.38.0", features = ["docs"] } deno_lockfile.workspace = true -deno_runtime.workspace = true +deno_runtime = { workspace = true, features = ["dont_create_runtime_snapshot", "include_js_files_for_snapshotting"] } deno_task_shell = "0.8.1" napi_sym.workspace = true diff --git a/cli/build.rs b/cli/build.rs index 807009a5d..01127fded 100644 --- a/cli/build.rs +++ b/cli/build.rs @@ -360,8 +360,8 @@ fn create_cli_snapshot(snapshot_path: PathBuf) { ); esm_files.push(ExtensionFileSource { specifier: "runtime/js/99_main.js".to_string(), - code: ExtensionFileSourceCode::IncludedInBinary( - deno_runtime::js::SOURCE_CODE_FOR_99_MAIN_JS, + code: ExtensionFileSourceCode::LoadedFromFsDuringSnapshot( + std::path::PathBuf::from(deno_runtime::js::PATH_FOR_99_MAIN_JS), ), }); let extensions_with_js = vec![Extension::builder("cli") diff --git a/core/Cargo.toml b/core/Cargo.toml index 2b4d40207..a246a7efe 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -16,6 +16,7 @@ path = "lib.rs" [features] default = ["v8_use_custom_libcxx"] v8_use_custom_libcxx = ["v8/use_custom_libcxx"] +include_js_files_for_snapshotting = [] [dependencies] anyhow.workspace = true diff --git a/core/extensions.rs b/core/extensions.rs index ab686d868..f84ab0a91 100644 --- a/core/extensions.rs +++ b/core/extensions.rs @@ -1,7 +1,9 @@ // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. use crate::OpState; +use anyhow::Context as _; use anyhow::Error; use std::cell::RefCell; +use std::path::PathBuf; use std::rc::Rc; use std::task::Context; use v8::fast_api::FastFunction; @@ -13,8 +15,24 @@ pub enum ExtensionFileSourceCode { /// will result in two copies of the source code being included - one in the /// snapshot, the other the static string in the `Extension`. IncludedInBinary(&'static str), - // TODO(bartlomieju): add more variants that allow to read file from the disk, - // and not include it in the binary. + + // Source code is loaded from a file on disk. It's meant to be used if the + // embedder is creating snapshots. Files will be loaded from the filesystem + // during the build time and they will only be present in the V8 snapshot. + LoadedFromFsDuringSnapshot(PathBuf), +} + +impl ExtensionFileSourceCode { + pub fn load(&self) -> Result<String, Error> { + match self { + ExtensionFileSourceCode::IncludedInBinary(code) => Ok(code.to_string()), + ExtensionFileSourceCode::LoadedFromFsDuringSnapshot(path) => { + let msg = format!("Failed to read \"{}\"", path.display()); + let code = std::fs::read_to_string(path).context(msg)?; + Ok(code) + } + } + } } #[derive(Clone, Debug)] @@ -299,6 +317,7 @@ impl ExtensionBuilder { /// - "internal:my_extension/js/01_hello.js" /// - "internal:my_extension/js/02_goodbye.js" /// ``` +#[cfg(not(feature = "include_js_files_for_snapshotting"))] #[macro_export] macro_rules! include_js_files { (dir $dir:literal, $($file:literal,)+) => { @@ -323,3 +342,29 @@ macro_rules! include_js_files { ] }; } + +#[cfg(feature = "include_js_files_for_snapshotting")] +#[macro_export] +macro_rules! include_js_files { + (dir $dir:literal, $($file:literal,)+) => { + vec![ + $($crate::ExtensionFileSource { + specifier: concat!($dir, "/", $file).to_string(), + code: $crate::ExtensionFileSourceCode::LoadedFromFsDuringSnapshot( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join($dir).join($file) + ), + },)+ + ] + }; + + ($($file:literal,)+) => { + vec![ + $($crate::ExtensionFileSource { + specifier: $file.to_string(), + code: $crate::ExtensionFileSourceCode::LoadedFromFsDuringSnapshot( + std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")).join($file) + ), + },)+ + ] + }; +} diff --git a/core/modules.rs b/core/modules.rs index 43dabf411..f1d4273e8 100644 --- a/core/modules.rs +++ b/core/modules.rs @@ -3,7 +3,6 @@ use crate::bindings; use crate::error::generic_error; use crate::extensions::ExtensionFileSource; -use crate::extensions::ExtensionFileSourceCode; use crate::module_specifier::ModuleSpecifier; use crate::resolve_import; use crate::resolve_url; @@ -403,10 +402,9 @@ impl ModuleLoader for InternalModuleLoader { let result = if let Some(load_callback) = &self.maybe_load_callback { load_callback(file_source) } else { - match file_source.code { - ExtensionFileSourceCode::IncludedInBinary(code) => { - Ok(code.to_string()) - } + match file_source.code.load() { + Ok(code) => Ok(code), + Err(err) => return futures::future::err(err).boxed_local(), } }; diff --git a/core/runtime.rs b/core/runtime.rs index ae2b0489b..d098c25b1 100644 --- a/core/runtime.rs +++ b/core/runtime.rs @@ -21,7 +21,6 @@ use crate::source_map::SourceMapCache; use crate::source_map::SourceMapGetter; use crate::Extension; use crate::ExtensionFileSource; -use crate::ExtensionFileSourceCode; use crate::NoopModuleLoader; use crate::OpMiddlewareFn; use crate::OpResult; @@ -869,14 +868,11 @@ impl JsRuntime { { let js_files = ext.get_js_sources(); for file_source in js_files { - let ExtensionFileSourceCode::IncludedInBinary(code) = - file_source.code; - // TODO(@AaronO): use JsRuntime::execute_static() here to move src off heap realm.execute_script( self.v8_isolate(), &file_source.specifier, - code, + &file_source.code.load()?, )?; } } diff --git a/core/snapshot_util.rs b/core/snapshot_util.rs index a4bf80227..5b0ba92a0 100644 --- a/core/snapshot_util.rs +++ b/core/snapshot_util.rs @@ -33,6 +33,12 @@ pub fn create_snapshot(create_snapshot_options: CreateSnapshotOptions) { snapshot_module_load_cb: create_snapshot_options.snapshot_module_load_cb, ..Default::default() }); + println!( + "JsRuntime for snapshot prepared, took {:#?} ({})", + Instant::now().saturating_duration_since(mark), + create_snapshot_options.snapshot_path.display() + ); + mark = Instant::now(); let snapshot = js_runtime.snapshot(); let snapshot_slice: &[u8] = &snapshot; diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 2d93f9dff..b2ef0637a 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -12,8 +12,17 @@ description = "Provides the deno runtime library" [features] # "fake" feature that allows to generate docs on docs.rs docsrs = [] -# feature that modifies the snapshot to allow extending it +# A feature that disables creation of startup snapshot during in the build script. +dont_create_runtime_snapshot = [] +# A feature that changes how startup snapshot is generated, that allows +# extending it in embedder crates. snapshot_from_snapshot = [] +# A feature that disables embedding of the JavaScript source files in the binary. +# With this feature enabled, the sources must be consumed during build time, +# by creating a startup snapshot. +include_js_files_for_snapshotting = [ + "deno_core/include_js_files_for_snapshotting", +] [lib] name = "deno_runtime" diff --git a/runtime/build.rs b/runtime/build.rs index a813d63cb..4678eab99 100644 --- a/runtime/build.rs +++ b/runtime/build.rs @@ -1,24 +1,25 @@ // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. -use deno_core::include_js_files; use std::env; use std::path::PathBuf; -// This is a shim that allows to generate documentation on docs.rs -mod not_docs { +#[cfg(all( + not(feature = "docsrs"), + not(feature = "dont_create_runtime_snapshot") +))] +mod startup_snapshot { use std::path::Path; use super::*; - use deno_cache::SqliteBackedCache; - use deno_core::snapshot_util::*; - use deno_core::Extension; - use deno_ast::MediaType; use deno_ast::ParseParams; use deno_ast::SourceTextInfo; + use deno_cache::SqliteBackedCache; use deno_core::error::AnyError; + use deno_core::include_js_files; + use deno_core::snapshot_util::*; + use deno_core::Extension; use deno_core::ExtensionFileSource; - use deno_core::ExtensionFileSourceCode; fn transpile_ts_for_snapshotting( file_source: &ExtensionFileSource, @@ -34,16 +35,15 @@ mod not_docs { file_source.specifier ), }; - - let ExtensionFileSourceCode::IncludedInBinary(code) = file_source.code; + let code = file_source.code.load()?; if !should_transpile { - return Ok(code.to_string()); + return Ok(code); } let parsed = deno_ast::parse_module(ParseParams { specifier: file_source.specifier.to_string(), - text_info: SourceTextInfo::from_string(code.to_string()), + text_info: SourceTextInfo::from_string(code), media_type, capture_tokens: false, scope_analysis: false, @@ -281,6 +281,7 @@ mod not_docs { #[cfg(not(feature = "snapshot_from_snapshot"))] { + use deno_core::ExtensionFileSourceCode; maybe_additional_extension = Some( Extension::builder("runtime_main") .dependencies(vec!["runtime"]) @@ -314,9 +315,13 @@ fn main() { // doesn't actually compile on docs.rs if env::var_os("DOCS_RS").is_some() { let snapshot_slice = &[]; + #[allow(clippy::needless_borrow)] std::fs::write(&runtime_snapshot_path, snapshot_slice).unwrap(); } - #[cfg(not(feature = "docsrs"))] - not_docs::build_snapshot(runtime_snapshot_path) + #[cfg(all( + not(feature = "docsrs"), + not(feature = "dont_create_runtime_snapshot") + ))] + startup_snapshot::build_snapshot(runtime_snapshot_path) } diff --git a/runtime/js.rs b/runtime/js.rs index 82e58a09c..57b1e7be4 100644 --- a/runtime/js.rs +++ b/runtime/js.rs @@ -1,16 +1,21 @@ // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. +#[cfg(not(feature = "dont_create_runtime_snapshot"))] use deno_core::Snapshot; +#[cfg(not(feature = "dont_create_runtime_snapshot"))] use log::debug; +#[cfg(not(feature = "dont_create_runtime_snapshot"))] use once_cell::sync::Lazy; +#[cfg(not(feature = "dont_create_runtime_snapshot"))] +static COMPRESSED_RUNTIME_SNAPSHOT: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/RUNTIME_SNAPSHOT.bin")); + +#[cfg(not(feature = "dont_create_runtime_snapshot"))] pub static RUNTIME_SNAPSHOT: Lazy<Box<[u8]>> = Lazy::new( #[allow(clippy::uninit_vec)] #[cold] #[inline(never)] || { - static COMPRESSED_RUNTIME_SNAPSHOT: &[u8] = - include_bytes!(concat!(env!("OUT_DIR"), "/RUNTIME_SNAPSHOT.bin")); - let size = u32::from_le_bytes(COMPRESSED_RUNTIME_SNAPSHOT[0..4].try_into().unwrap()) as usize; @@ -29,10 +34,15 @@ pub static RUNTIME_SNAPSHOT: Lazy<Box<[u8]>> = Lazy::new( }, ); +#[cfg(not(feature = "dont_create_runtime_snapshot"))] pub fn deno_isolate_init() -> Snapshot { debug!("Deno isolate init with snapshots."); Snapshot::Static(&RUNTIME_SNAPSHOT) } -#[cfg(feature = "snapshot_from_snapshot")] +#[cfg(not(feature = "include_js_files_for_snapshotting"))] pub static SOURCE_CODE_FOR_99_MAIN_JS: &str = include_str!("js/99_main.js"); + +#[cfg(feature = "include_js_files_for_snapshotting")] +pub static PATH_FOR_99_MAIN_JS: &str = + concat!(env!("CARGO_MANIFEST_DIR"), "/js/99_main.js"); diff --git a/runtime/web_worker.rs b/runtime/web_worker.rs index df75d79c1..bcf999b3b 100644 --- a/runtime/web_worker.rs +++ b/runtime/web_worker.rs @@ -1,7 +1,6 @@ // Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. use crate::colors; use crate::inspector_server::InspectorServer; -use crate::js; use crate::ops; use crate::ops::io::Stdio; use crate::permissions::PermissionsContainer; @@ -453,13 +452,17 @@ impl WebWorker { // Append exts extensions.extend(std::mem::take(&mut options.extensions)); + #[cfg(not(feature = "dont_create_runtime_snapshot"))] + let startup_snapshot = options + .startup_snapshot + .unwrap_or_else(crate::js::deno_isolate_init); + #[cfg(feature = "dont_create_runtime_snapshot")] + let startup_snapshot = options.startup_snapshot + .expect("deno_runtime startup snapshot is not available with 'create_runtime_snapshot' Cargo feature."); + let mut js_runtime = JsRuntime::new(RuntimeOptions { module_loader: Some(options.module_loader.clone()), - startup_snapshot: Some( - options - .startup_snapshot - .unwrap_or_else(js::deno_isolate_init), - ), + startup_snapshot: Some(startup_snapshot), source_map_getter: options.source_map_getter, get_error_class_fn: options.get_error_class_fn, shared_array_buffer_store: options.shared_array_buffer_store.clone(), diff --git a/runtime/worker.rs b/runtime/worker.rs index 141b5a650..d1998cd88 100644 --- a/runtime/worker.rs +++ b/runtime/worker.rs @@ -35,7 +35,6 @@ use deno_web::BlobStore; use log::debug; use crate::inspector_server::InspectorServer; -use crate::js; use crate::ops; use crate::ops::io::Stdio; use crate::permissions::PermissionsContainer; @@ -282,13 +281,17 @@ impl MainWorker { ]; extensions.extend(std::mem::take(&mut options.extensions)); + #[cfg(not(feature = "dont_create_runtime_snapshot"))] + let startup_snapshot = options + .startup_snapshot + .unwrap_or_else(crate::js::deno_isolate_init); + #[cfg(feature = "dont_create_runtime_snapshot")] + let startup_snapshot = options.startup_snapshot + .expect("deno_runtime startup snapshot is not available with 'create_runtime_snapshot' Cargo feature."); + let mut js_runtime = JsRuntime::new(RuntimeOptions { module_loader: Some(options.module_loader.clone()), - startup_snapshot: Some( - options - .startup_snapshot - .unwrap_or_else(js::deno_isolate_init), - ), + startup_snapshot: Some(startup_snapshot), source_map_getter: options.source_map_getter, get_error_class_fn: options.get_error_class_fn, shared_array_buffer_store: options.shared_array_buffer_store.clone(), |