diff options
| -rw-r--r-- | Cargo.lock | 24 | ||||
| -rw-r--r-- | Cargo.toml | 1 | ||||
| -rw-r--r-- | cli/Cargo.toml | 13 | ||||
| -rw-r--r-- | cli/build.rs | 385 | ||||
| -rw-r--r-- | cli/lsp/tsc.rs | 2 | ||||
| -rw-r--r-- | cli/main.rs | 2 | ||||
| -rw-r--r-- | cli/standalone.rs | 1 | ||||
| -rw-r--r-- | cli/tsc.rs | 28 | ||||
| -rw-r--r-- | runtime/Cargo.toml | 19 | ||||
| -rw-r--r-- | runtime/build.rs | 222 | ||||
| -rw-r--r-- | runtime/examples/hello_runtime.rs | 1 | ||||
| -rw-r--r-- | runtime/js.rs | 92 | ||||
| -rw-r--r-- | runtime/web_worker.rs | 5 | ||||
| -rw-r--r-- | runtime/worker.rs | 6 | ||||
| -rw-r--r-- | snapshots/Cargo.toml | 32 | ||||
| -rw-r--r-- | snapshots/build.rs | 10 | ||||
| -rw-r--r-- | snapshots/build_runtime.rs | 168 | ||||
| -rw-r--r-- | snapshots/build_tsc.rs | 328 | ||||
| -rw-r--r-- | snapshots/lib.rs | 77 |
19 files changed, 675 insertions, 741 deletions
diff --git a/Cargo.lock b/Cargo.lock index 3ba76f08f..a0d623938 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -782,14 +782,23 @@ dependencies = [ "data-url", "deno_ast", "deno_bench_util", + "deno_broadcast_channel", + "deno_console", "deno_core", + "deno_crypto", "deno_doc", "deno_emit", + "deno_fetch", "deno_graph", "deno_lint", + "deno_net", "deno_runtime", - "deno_snapshots", "deno_task_shell", + "deno_url", + "deno_web", + "deno_webgpu", + "deno_websocket", + "deno_webstorage", "dissimilar", "dotenv", "dprint-plugin-json", @@ -1155,19 +1164,6 @@ dependencies = [ ] [[package]] -name = "deno_snapshots" -version = "0.1.0" -dependencies = [ - "deno_core", - "deno_runtime", - "lzzzz", - "once_cell", - "regex", - "serde", - "zstd", -] - -[[package]] name = "deno_task_shell" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" diff --git a/Cargo.toml b/Cargo.toml index 071947c01..bb88cbaf3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,7 +9,6 @@ members = [ "ops", "runtime", "serde_v8", - "snapshots", "test_ffi", "test_util", "ext/broadcast_channel", diff --git a/cli/Cargo.toml b/cli/Cargo.toml index a82ec5644..094b2ce01 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -25,7 +25,17 @@ harness = false path = "./bench/lsp_bench_standalone.rs" [build-dependencies] -deno_runtime = { version = "0.67.0", path = "../runtime" } +deno_broadcast_channel = { version = "0.53.0", path = "../ext/broadcast_channel" } +deno_console = { version = "0.59.0", path = "../ext/console" } +deno_core = { version = "0.141.0", path = "../core" } +deno_crypto = { version = "0.73.0", path = "../ext/crypto" } +deno_fetch = { version = "0.82.0", path = "../ext/fetch" } +deno_net = { version = "0.51.0", path = "../ext/net" } +deno_url = { version = "0.59.0", path = "../ext/url" } +deno_web = { version = "0.90.0", path = "../ext/web" } +deno_webgpu = { version = "0.60.0", path = "../ext/webgpu" } +deno_websocket = { version = "0.64.0", path = "../ext/websocket" } +deno_webstorage = { version = "0.54.0", path = "../ext/webstorage" } regex = "=1.5.6" serde = { version = "=1.0.136", features = ["derive"] } zstd = '=0.11.1' @@ -42,7 +52,6 @@ deno_emit = "0.3.0" deno_graph = "0.28.0" deno_lint = { version = "0.31.0", features = ["docs"] } deno_runtime = { version = "0.67.0", path = "../runtime" } -deno_snapshots = { version = "0.1.0", path = "../snapshots" } deno_task_shell = "0.4.0" atty = "=0.2.14" diff --git a/cli/build.rs b/cli/build.rs index 095efa22f..1a4eaa425 100644 --- a/cli/build.rs +++ b/cli/build.rs @@ -1,19 +1,323 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. -use deno_runtime::deno_broadcast_channel; -use deno_runtime::deno_console; -use deno_runtime::deno_crypto; -use deno_runtime::deno_fetch; -use deno_runtime::deno_net; -use deno_runtime::deno_url; -use deno_runtime::deno_web; -use deno_runtime::deno_websocket; -use deno_runtime::deno_webstorage; - +use deno_core::error::custom_error; +use deno_core::error::AnyError; +use deno_core::op; +use deno_core::serde::Deserialize; +use deno_core::serde_json::json; +use deno_core::serde_json::Value; +use deno_core::Extension; +use deno_core::JsRuntime; +use deno_core::OpState; +use deno_core::RuntimeOptions; +use regex::Regex; +use std::collections::HashMap; use std::env; use std::path::Path; use std::path::PathBuf; +// TODO(bartlomieju): this module contains a lot of duplicated +// logic with `runtime/build.rs`, factor out to `deno_core`. +fn create_snapshot( + mut js_runtime: JsRuntime, + snapshot_path: &Path, + files: Vec<PathBuf>, +) { + // TODO(nayeemrmn): https://github.com/rust-lang/cargo/issues/3946 to get the + // workspace root. + let display_root = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap(); + for file in files { + println!("cargo:rerun-if-changed={}", file.display()); + let display_path = file.strip_prefix(display_root).unwrap(); + let display_path_str = display_path.display().to_string(); + js_runtime + .execute_script( + &("deno:".to_string() + &display_path_str.replace('\\', "/")), + &std::fs::read_to_string(&file).unwrap(), + ) + .unwrap(); + } + + let snapshot = js_runtime.snapshot(); + let snapshot_slice: &[u8] = &*snapshot; + println!("Snapshot size: {}", snapshot_slice.len()); + + let compressed_snapshot_with_size = { + let mut vec = vec![]; + + vec.extend_from_slice( + &u32::try_from(snapshot.len()) + .expect("snapshot larger than 4gb") + .to_le_bytes(), + ); + + vec.extend_from_slice( + &zstd::bulk::compress(snapshot_slice, 22) + .expect("snapshot compression failed"), + ); + + vec + }; + + println!( + "Snapshot compressed size: {}", + compressed_snapshot_with_size.len() + ); + + std::fs::write(&snapshot_path, compressed_snapshot_with_size).unwrap(); + println!("Snapshot written to: {} ", snapshot_path.display()); +} + +#[derive(Debug, Deserialize)] +struct LoadArgs { + /// The fully qualified specifier that should be loaded. + specifier: String, +} + +fn create_compiler_snapshot( + snapshot_path: &Path, + files: Vec<PathBuf>, + cwd: &Path, +) { + // libs that are being provided by op crates. + let mut op_crate_libs = HashMap::new(); + op_crate_libs.insert("deno.console", deno_console::get_declaration()); + op_crate_libs.insert("deno.url", deno_url::get_declaration()); + op_crate_libs.insert("deno.web", deno_web::get_declaration()); + op_crate_libs.insert("deno.fetch", deno_fetch::get_declaration()); + op_crate_libs.insert("deno.webgpu", deno_webgpu_get_declaration()); + op_crate_libs.insert("deno.websocket", deno_websocket::get_declaration()); + op_crate_libs.insert("deno.webstorage", deno_webstorage::get_declaration()); + op_crate_libs.insert("deno.crypto", deno_crypto::get_declaration()); + op_crate_libs.insert( + "deno.broadcast_channel", + deno_broadcast_channel::get_declaration(), + ); + op_crate_libs.insert("deno.net", deno_net::get_declaration()); + + // ensure we invalidate the build properly. + for (_, path) in op_crate_libs.iter() { + println!("cargo:rerun-if-changed={}", path.display()); + } + + // libs that should be loaded into the isolate before snapshotting. + let libs = vec![ + // Deno custom type libraries + "deno.window", + "deno.worker", + "deno.shared_globals", + "deno.ns", + "deno.unstable", + // Deno built-in type libraries + "es5", + "es2015.collection", + "es2015.core", + "es2015", + "es2015.generator", + "es2015.iterable", + "es2015.promise", + "es2015.proxy", + "es2015.reflect", + "es2015.symbol", + "es2015.symbol.wellknown", + "es2016.array.include", + "es2016", + "es2017", + "es2017.intl", + "es2017.object", + "es2017.sharedmemory", + "es2017.string", + "es2017.typedarrays", + "es2018.asyncgenerator", + "es2018.asynciterable", + "es2018", + "es2018.intl", + "es2018.promise", + "es2018.regexp", + "es2019.array", + "es2019", + "es2019.object", + "es2019.string", + "es2019.symbol", + "es2020.bigint", + "es2020", + "es2020.date", + "es2020.intl", + "es2020.number", + "es2020.promise", + "es2020.sharedmemory", + "es2020.string", + "es2020.symbol.wellknown", + "es2021", + "es2021.intl", + "es2021.promise", + "es2021.string", + "es2021.weakref", + "es2022", + "es2022.array", + "es2022.error", + "es2022.intl", + "es2022.object", + "es2022.string", + "esnext", + "esnext.array", + "esnext.intl", + ]; + + let path_dts = cwd.join("dts"); + // ensure we invalidate the build properly. + for name in libs.iter() { + println!( + "cargo:rerun-if-changed={}", + path_dts.join(format!("lib.{}.d.ts", name)).display() + ); + } + + // create a copy of the vector that includes any op crate libs to be passed + // to the JavaScript compiler to build into the snapshot + let mut build_libs = libs.clone(); + for (op_lib, _) in op_crate_libs.iter() { + build_libs.push(op_lib.to_owned()); + } + + #[op] + fn op_build_info(state: &mut OpState) -> Value { + let build_specifier = "asset:///bootstrap.ts"; + let build_libs = state.borrow::<Vec<&str>>(); + json!({ + "buildSpecifier": build_specifier, + "libs": build_libs, + }) + } + + #[op] + fn op_cwd() -> String { + "cache:///".into() + } + + #[op] + fn op_exists() -> bool { + false + } + + #[op] + fn op_script_version( + _state: &mut OpState, + _args: Value, + ) -> Result<Option<String>, AnyError> { + Ok(Some("1".to_string())) + } + + #[op] + // using the same op that is used in `tsc.rs` for loading modules and reading + // files, but a slightly different implementation at build time. + fn op_load(state: &mut OpState, args: LoadArgs) -> Result<Value, AnyError> { + let op_crate_libs = state.borrow::<HashMap<&str, PathBuf>>(); + let path_dts = state.borrow::<PathBuf>(); + let re_asset = + Regex::new(r"asset:/{3}lib\.(\S+)\.d\.ts").expect("bad regex"); + let build_specifier = "asset:///bootstrap.ts"; + + // we need a basic file to send to tsc to warm it up. + if args.specifier == build_specifier { + Ok(json!({ + "data": r#"console.log("hello deno!");"#, + "version": "1", + // this corresponds to `ts.ScriptKind.TypeScript` + "scriptKind": 3 + })) + // specifiers come across as `asset:///lib.{lib_name}.d.ts` and we need to + // parse out just the name so we can lookup the asset. + } else if let Some(caps) = re_asset.captures(&args.specifier) { + if let Some(lib) = caps.get(1).map(|m| m.as_str()) { + // if it comes from an op crate, we were supplied with the path to the + // file. + let path = if let Some(op_crate_lib) = op_crate_libs.get(lib) { + PathBuf::from(op_crate_lib).canonicalize().unwrap() + // otherwise we are will generate the path ourself + } else { + path_dts.join(format!("lib.{}.d.ts", lib)) + }; + let data = std::fs::read_to_string(path)?; + Ok(json!({ + "data": data, + "version": "1", + // this corresponds to `ts.ScriptKind.TypeScript` + "scriptKind": 3 + })) + } else { + Err(custom_error( + "InvalidSpecifier", + format!("An invalid specifier was requested: {}", args.specifier), + )) + } + } else { + Err(custom_error( + "InvalidSpecifier", + format!("An invalid specifier was requested: {}", args.specifier), + )) + } + } + let js_runtime = JsRuntime::new(RuntimeOptions { + will_snapshot: true, + extensions: vec![Extension::builder() + .ops(vec![ + op_build_info::decl(), + op_cwd::decl(), + op_exists::decl(), + op_load::decl(), + op_script_version::decl(), + ]) + .state(move |state| { + state.put(op_crate_libs.clone()); + state.put(build_libs.clone()); + state.put(path_dts.clone()); + + Ok(()) + }) + .build()], + ..Default::default() + }); + + create_snapshot(js_runtime, snapshot_path, files); +} + +fn ts_version() -> String { + std::fs::read_to_string("tsc/00_typescript.js") + .unwrap() + .lines() + .find(|l| l.contains("ts.version = ")) + .expect( + "Failed to find the pattern `ts.version = ` in typescript source code", + ) + .chars() + .skip_while(|c| !char::is_numeric(*c)) + .take_while(|c| *c != '"') + .collect::<String>() +} + +fn git_commit_hash() -> String { + if let Ok(output) = std::process::Command::new("git") + .arg("rev-list") + .arg("-1") + .arg("HEAD") + .output() + { + if output.status.success() { + std::str::from_utf8(&output.stdout[..40]) + .unwrap() + .to_string() + } else { + // When not in git repository + // (e.g. when the user install by `cargo install deno`) + "UNKNOWN".to_string() + } + } else { + // When there is no git command for some reason + "UNKNOWN".to_string() + } +} + fn main() { // Skip building from docs.rs. if env::var_os("DOCS_RS").is_some() { @@ -26,9 +330,8 @@ fn main() { if target != host { panic!("Cross compiling with snapshot is not supported."); } - - println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap()); - println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap()); + // To debug snapshot issues uncomment: + // op_fetch_asset::trace_serializer(); if let Ok(c) = env::var("DENO_CANARY") { println!("cargo:rustc-env=DENO_CANARY={}", c); @@ -82,6 +385,18 @@ fn main() { deno_net::get_declaration().display() ); + println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap()); + println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap()); + + let c = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap()); + let o = PathBuf::from(env::var_os("OUT_DIR").unwrap()); + + // Main snapshot + let compiler_snapshot_path = o.join("COMPILER_SNAPSHOT.bin"); + + let js_files = get_js_files("tsc"); + create_compiler_snapshot(&compiler_snapshot_path, js_files, &c); + #[cfg(target_os = "windows")] { let mut res = winres::WindowsResource::new(); @@ -99,38 +414,16 @@ fn deno_webgpu_get_declaration() -> PathBuf { manifest_dir.join("dts").join("lib.deno_webgpu.d.ts") } -fn git_commit_hash() -> String { - if let Ok(output) = std::process::Command::new("git") - .arg("rev-list") - .arg("-1") - .arg("HEAD") - .output() - { - if output.status.success() { - std::str::from_utf8(&output.stdout[..40]) - .unwrap() - .to_string() - } else { - // When not in git repository - // (e.g. when the user install by `cargo install deno`) - "UNKNOWN".to_string() - } - } else { - // When there is no git command for some reason - "UNKNOWN".to_string() - } -} - -fn ts_version() -> String { - std::fs::read_to_string("tsc/00_typescript.js") +fn get_js_files(d: &str) -> Vec<PathBuf> { + let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR")); + let mut js_files = std::fs::read_dir(d) .unwrap() - .lines() - .find(|l| l.contains("ts.version = ")) - .expect( - "Failed to find the pattern `ts.version = ` in typescript source code", - ) - .chars() - .skip_while(|c| !char::is_numeric(*c)) - .take_while(|c| *c != '"') - .collect::<String>() + .map(|dir_entry| { + let file = dir_entry.unwrap(); + manifest_dir.join(file.path()) + }) + .filter(|path| path.extension().unwrap_or_default() == "js") + .collect::<Vec<PathBuf>>(); + js_files.sort(); + js_files } diff --git a/cli/lsp/tsc.rs b/cli/lsp/tsc.rs index 9988a0ac9..c362d4ce3 100644 --- a/cli/lsp/tsc.rs +++ b/cli/lsp/tsc.rs @@ -2618,7 +2618,7 @@ fn op_script_version( fn js_runtime(performance: Arc<Performance>) -> JsRuntime { JsRuntime::new(RuntimeOptions { extensions: vec![init_extension(performance)], - startup_snapshot: Some(deno_snapshots::tsc_snapshot()), + startup_snapshot: Some(tsc::compiler_snapshot()), ..Default::default() }) } diff --git a/cli/main.rs b/cli/main.rs index 2747d763c..18eb58ff5 100644 --- a/cli/main.rs +++ b/cli/main.rs @@ -181,7 +181,6 @@ fn create_web_worker_callback( shared_array_buffer_store: Some(ps.shared_array_buffer_store.clone()), compiled_wasm_module_store: Some(ps.compiled_wasm_module_store.clone()), stdio: stdio.clone(), - startup_snapshot: Some(deno_snapshots::cli_snapshot()), }; WebWorker::bootstrap_from_options( @@ -263,7 +262,6 @@ pub fn create_main_worker( shared_array_buffer_store: Some(ps.shared_array_buffer_store.clone()), compiled_wasm_module_store: Some(ps.compiled_wasm_module_store.clone()), stdio, - startup_snapshot: Some(deno_snapshots::cli_snapshot()), }; MainWorker::bootstrap_from_options(main_module, permissions, options) diff --git a/cli/standalone.rs b/cli/standalone.rs index d130fbe2e..cd5390d2d 100644 --- a/cli/standalone.rs +++ b/cli/standalone.rs @@ -303,7 +303,6 @@ pub async fn run( shared_array_buffer_store: None, compiled_wasm_module_store: None, stdio: Default::default(), - startup_snapshot: Some(deno_snapshots::cli_snapshot()), }; let mut worker = MainWorker::bootstrap_from_options( main_module.clone(), diff --git a/cli/tsc.rs b/cli/tsc.rs index 95fdd305a..4065c6354 100644 --- a/cli/tsc.rs +++ b/cli/tsc.rs @@ -24,6 +24,7 @@ use deno_core::JsRuntime; use deno_core::ModuleSpecifier; use deno_core::OpState; use deno_core::RuntimeOptions; +use deno_core::Snapshot; use deno_graph::Resolved; use once_cell::sync::Lazy; use std::collections::HashMap; @@ -51,6 +52,27 @@ pub static SHARED_GLOBALS_LIB: &str = pub static WINDOW_LIB: &str = include_str!("dts/lib.deno.window.d.ts"); pub static UNSTABLE_NS_LIB: &str = include_str!("dts/lib.deno.unstable.d.ts"); +pub static COMPILER_SNAPSHOT: Lazy<Box<[u8]>> = Lazy::new( + #[cold] + #[inline(never)] + || { + static COMPRESSED_COMPILER_SNAPSHOT: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/COMPILER_SNAPSHOT.bin")); + + zstd::bulk::decompress( + &COMPRESSED_COMPILER_SNAPSHOT[4..], + u32::from_le_bytes(COMPRESSED_COMPILER_SNAPSHOT[0..4].try_into().unwrap()) + as usize, + ) + .unwrap() + .into_boxed_slice() + }, +); + +pub fn compiler_snapshot() -> Snapshot { + Snapshot::Static(&*COMPILER_SNAPSHOT) +} + macro_rules! inc { ($e:expr) => { include_str!(concat!("dts/", $e)) @@ -635,7 +657,7 @@ pub fn exec(request: Request) -> Result<Response, AnyError> { }) .collect(); let mut runtime = JsRuntime::new(RuntimeOptions { - startup_snapshot: Some(deno_snapshots::tsc_snapshot()), + startup_snapshot: Some(compiler_snapshot()), extensions: vec![Extension::builder() .ops(vec![ op_cwd::decl(), @@ -819,9 +841,9 @@ mod tests { } #[test] - fn test_tsc_snapshot() { + fn test_compiler_snapshot() { let mut js_runtime = deno_core::JsRuntime::new(deno_core::RuntimeOptions { - startup_snapshot: Some(deno_snapshots::tsc_snapshot()), + startup_snapshot: Some(compiler_snapshot()), ..Default::default() }); js_runtime diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml index 0f12281dd..fe50f9f1c 100644 --- a/runtime/Cargo.toml +++ b/runtime/Cargo.toml @@ -21,6 +21,25 @@ path = "lib.rs" name = "hello_runtime" path = "examples/hello_runtime.rs" +[build-dependencies] +deno_broadcast_channel = { version = "0.53.0", path = "../ext/broadcast_channel" } +deno_console = { version = "0.59.0", path = "../ext/console" } +deno_core = { version = "0.141.0", path = "../core" } +deno_crypto = { version = "0.73.0", path = "../ext/crypto" } +deno_fetch = { version = "0.82.0", path = "../ext/fetch" } +deno_ffi = { version = "0.46.0", path = "../ext/ffi" } +deno_http = { version = "0.53.0", path = "../ext/http" } +deno_net = { version = "0.51.0", path = "../ext/net" } +deno_tls = { version = "0.46.0", path = "../ext/tls" } +deno_url = { version = "0.59.0", path = "../ext/url" } +deno_web = { version = "0.90.0", path = "../ext/web" } +deno_webgpu = { version = "0.60.0", path = "../ext/webgpu" } +deno_webidl = { version = "0.59.0", path = "../ext/webidl" } +deno_websocket = { version = "0.64.0", path = "../ext/websocket" } +deno_webstorage = { version = "0.54.0", path = "../ext/webstorage" } + +lzzzz = '1.0' + [target.'cfg(windows)'.build-dependencies] winres = "0.1.11" winapi = "0.3.9" diff --git a/runtime/build.rs b/runtime/build.rs index 0cb113af1..eea7a3602 100644 --- a/runtime/build.rs +++ b/runtime/build.rs @@ -1,17 +1,219 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. +use std::env; +use std::path::Path; +use std::path::PathBuf; + +// This is a shim that allows to generate documentation on docs.rs +#[cfg(not(feature = "docsrs"))] +mod not_docs { + use super::*; + use deno_core::Extension; + use deno_core::JsRuntime; + use deno_core::RuntimeOptions; + + // TODO(bartlomieju): this module contains a lot of duplicated + // logic with `cli/build.rs`, factor out to `deno_core`. + fn create_snapshot( + mut js_runtime: JsRuntime, + snapshot_path: &Path, + files: Vec<PathBuf>, + ) { + // TODO(nayeemrmn): https://github.com/rust-lang/cargo/issues/3946 to get the + // workspace root. + let display_root = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap(); + for file in files { + println!("cargo:rerun-if-changed={}", file.display()); + let display_path = file.strip_prefix(display_root).unwrap(); + let display_path_str = display_path.display().to_string(); + js_runtime + .execute_script( + &("deno:".to_string() + &display_path_str.replace('\\', "/")), + &std::fs::read_to_string(&file).unwrap(), + ) + .unwrap(); + } + + let snapshot = js_runtime.snapshot(); + let snapshot_slice: &[u8] = &*snapshot; + println!("Snapshot size: {}", snapshot_slice.len()); + + let compressed_snapshot_with_size = { + let mut vec = vec![]; + + vec.extend_from_slice( + &u32::try_from(snapshot.len()) + .expect("snapshot larger than 4gb") + .to_le_bytes(), + ); + + lzzzz::lz4_hc::compress_to_vec( + snapshot_slice, + &mut vec, + lzzzz::lz4_hc::CLEVEL_MAX, + ) + .expect("snapshot compression failed"); + + vec + }; + + println!( + "Snapshot compressed size: {}", + compressed_snapshot_with_size.len() + ); + + std::fs::write(&snapshot_path, compressed_snapshot_with_size).unwrap(); + println!("Snapshot written to: {} ", snapshot_path.display()); + } + + struct Permissions; + + impl deno_fetch::FetchPermissions for Permissions { + fn check_net_url( + &mut self, + _url: &deno_core::url::Url, + ) -> Result<(), deno_core::error::AnyError> { + unreachable!("snapshotting!") + } + + fn check_read( + &mut self, + _p: &Path, + ) -> Result<(), deno_core::error::AnyError> { + unreachable!("snapshotting!") + } + } + + impl deno_websocket::WebSocketPermissions for Permissions { + fn check_net_url( + &mut self, + _url: &deno_core::url::Url, + ) -> Result<(), deno_core::error::AnyError> { + unreachable!("snapshotting!") + } + } + + impl deno_web::TimersPermission for Permissions { + fn allow_hrtime(&mut self) -> bool { + unreachable!("snapshotting!") + } + + fn check_unstable( + &self, + _state: &deno_core::OpState, + _api_name: &'static str, + ) { + unreachable!("snapshotting!") + } + } + + impl deno_ffi::FfiPermissions for Permissions { + fn check( + &mut self, + _path: Option<&Path>, + ) -> Result<(), deno_core::error::AnyError> { + unreachable!("snapshotting!") + } + } + + impl deno_net::NetPermissions for Permissions { + fn check_net<T: AsRef<str>>( + &mut self, + _host: &(T, Option<u16>), + ) -> Result<(), deno_core::error::AnyError> { + unreachable!("snapshotting!") + } + + fn check_read( + &mut self, + _p: &Path, + ) -> Result<(), deno_core::error::AnyError> { + unreachable!("snapshotting!") + } + + fn check_write( + &mut self, + _p: &Path, + ) -> Result<(), deno_core::error::AnyError> { + unreachable!("snapshotting!") + } + } + + fn create_runtime_snapshot(snapshot_path: &Path, files: Vec<PathBuf>) { + let extensions: Vec<Extension> = vec![ + deno_webidl::init(), + deno_console::init(), + deno_url::init(), + deno_tls::init(), + deno_web::init::<Permissions>( + deno_web::BlobStore::default(), + Default::default(), + ), + deno_fetch::init::<Permissions>(Default::default()), + deno_websocket::init::<Permissions>("".to_owned(), None, None), + deno_webstorage::init(None), + deno_crypto::init(None), + deno_webgpu::init(false), + deno_broadcast_channel::init( + deno_broadcast_channel::InMemoryBroadcastChannel::default(), + false, // No --unstable. + ), + deno_ffi::init::<Permissions>(false), + deno_net::init::<Permissions>( + None, false, // No --unstable. + None, + ), + deno_http::init(), + ]; + + let js_runtime = JsRuntime::new(RuntimeOptions { + will_snapshot: true, + extensions, + ..Default::default() + }); + create_snapshot(js_runtime, snapshot_path, files); + } + + fn get_js_files(d: &str) -> Vec<PathBuf> { + let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR")); + let mut js_files = std::fs::read_dir(d) + .unwrap() + .map(|dir_entry| { + let file = dir_entry.unwrap(); + manifest_dir.join(file.path()) + }) + .filter(|path| path.extension().unwrap_or_default() == "js") + .collect::<Vec<PathBuf>>(); + js_files.sort(); + js_files + } + + pub fn build_snapshot(runtime_snapshot_path: PathBuf) { + let js_files = get_js_files("js"); + create_runtime_snapshot(&runtime_snapshot_path, js_files); + } +} + fn main() { - // Skip building from docs.rs. - if std::env::var_os("DOCS_RS").is_some() { + // To debug snapshot issues uncomment: + // op_fetch_asset::trace_serializer(); + + println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap()); + println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap()); + let o = PathBuf::from(env::var_os("OUT_DIR").unwrap()); + + // Main snapshot + let runtime_snapshot_path = o.join("CLI_SNAPSHOT.bin"); + + // If we're building on docs.rs we just create + // and empty snapshot file and return, because `rusty_v8` + // doesn't actually compile on docs.rs + if env::var_os("DOCS_RS").is_some() { + let snapshot_slice = &[]; + std::fs::write(&runtime_snapshot_path, snapshot_slice).unwrap(); return; } - println!( - "cargo:rustc-env=TARGET={}", - std::env::var("TARGET").unwrap() - ); - println!( - "cargo:rustc-env=PROFILE={}", - std::env::var("PROFILE").unwrap() - ); + #[cfg(not(feature = "docsrs"))] + not_docs::build_snapshot(runtime_snapshot_path) } diff --git a/runtime/examples/hello_runtime.rs b/runtime/examples/hello_runtime.rs index 19f462d4f..07e42f0ff 100644 --- a/runtime/examples/hello_runtime.rs +++ b/runtime/examples/hello_runtime.rs @@ -58,7 +58,6 @@ async fn main() -> Result<(), AnyError> { shared_array_buffer_store: None, compiled_wasm_module_store: None, stdio: Default::default(), - startup_snapshot: None, }; let js_path = diff --git a/runtime/js.rs b/runtime/js.rs index a18dc3bd3..cdd479858 100644 --- a/runtime/js.rs +++ b/runtime/js.rs @@ -1,38 +1,58 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. -use deno_core::include_js_files; -use deno_core::Extension; - -pub fn init() -> Extension { - Extension::builder() - .js(include_js_files!( - prefix "deno:runtime", - // Generated with: - // bash -c "cd runtime && ls js/*.js | sort" - "js/01_build.js", - "js/01_errors.js", - "js/01_version.js", - "js/01_web_util.js", - "js/06_util.js", - "js/10_permissions.js", - "js/11_workers.js", - "js/12_io.js", - "js/13_buffer.js", - "js/30_fs.js", - "js/30_os.js", - "js/40_diagnostics.js", - "js/40_files.js", - "js/40_fs_events.js", - "js/40_http.js", - "js/40_process.js", - "js/40_read_file.js", - "js/40_signals.js", - "js/40_spawn.js", - "js/40_testing.js", - "js/40_tty.js", - "js/40_write_file.js", - "js/41_prompt.js", - "js/90_deno_ns.js", - "js/99_main.js", - )) - .build() +use deno_core::Snapshot; +use log::debug; +use once_cell::sync::Lazy; + +pub static CLI_SNAPSHOT: Lazy<Box<[u8]>> = Lazy::new( + #[allow(clippy::uninit_vec)] + #[cold] + #[inline(never)] + || { + static COMPRESSED_CLI_SNAPSHOT: &[u8] = + include_bytes!(concat!(env!("OUT_DIR"), "/CLI_SNAPSHOT.bin")); + + let size = + u32::from_le_bytes(COMPRESSED_CLI_SNAPSHOT[0..4].try_into().unwrap()) + as usize; + let mut vec = Vec::with_capacity(size); + + // SAFETY: vec is allocated with exact snapshot size (+ alignment) + // SAFETY: non zeroed bytes are overwritten with decompressed snapshot + unsafe { + vec.set_len(size); + } + + lzzzz::lz4::decompress(&COMPRESSED_CLI_SNAPSHOT[4..], &mut vec).unwrap(); + + vec.into_boxed_slice() + }, +); + +pub fn deno_isolate_init() -> Snapshot { + debug!("Deno isolate init with snapshots."); + Snapshot::Static(&*CLI_SNAPSHOT) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn cli_snapshot() { + let mut js_runtime = deno_core::JsRuntime::new(deno_core::RuntimeOptions { + startup_snapshot: Some(deno_isolate_init()), + ..Default::default() + }); + js_runtime + .execute_script( + "<anon>", + r#" + if (!(bootstrap.mainRuntime && bootstrap.workerRuntime)) { + throw Error("bad"); + } + console.log("we have console.log!!!"); + "#, + ) + .unwrap(); + } } diff --git a/runtime/web_worker.rs b/runtime/web_worker.rs index 19e344ee7..ba2c016cc 100644 --- a/runtime/web_worker.rs +++ b/runtime/web_worker.rs @@ -335,7 +335,6 @@ pub struct WebWorkerOptions { pub shared_array_buffer_store: Option<SharedArrayBufferStore>, pub compiled_wasm_module_store: Option<CompiledWasmModuleStore>, pub stdio: Stdio, - pub startup_snapshot: Option<deno_core::Snapshot>, } impl WebWorker { @@ -428,8 +427,6 @@ impl WebWorker { ops::tty::init(), deno_http::init(), ops::http::init(), - // Runtime JS - js::init(), // Permissions ext (worker specific state) perm_ext, ]; @@ -439,7 +436,7 @@ impl WebWorker { let mut js_runtime = JsRuntime::new(RuntimeOptions { module_loader: Some(options.module_loader.clone()), - startup_snapshot: options.startup_snapshot.take(), + startup_snapshot: Some(js::deno_isolate_init()), source_map_getter: options.source_map_getter, get_error_class_fn: options.get_error_class_fn, shared_array_buffer_store: options.shared_array_buffer_store.clone(), diff --git a/runtime/worker.rs b/runtime/worker.rs index f9274babe..5100f42da 100644 --- a/runtime/worker.rs +++ b/runtime/worker.rs @@ -81,7 +81,6 @@ pub struct WorkerOptions { pub shared_array_buffer_store: Option<SharedArrayBufferStore>, pub compiled_wasm_module_store: Option<CompiledWasmModuleStore>, pub stdio: Stdio, - pub startup_snapshot: Option<deno_core::Snapshot>, } impl MainWorker { @@ -169,8 +168,6 @@ impl MainWorker { ops::tty::init(), deno_http::init(), ops::http::init(), - // Runtime JS - js::init(), // Permissions ext (worker specific state) perm_ext, ]; @@ -178,7 +175,7 @@ impl MainWorker { let mut js_runtime = JsRuntime::new(RuntimeOptions { module_loader: Some(options.module_loader.clone()), - startup_snapshot: options.startup_snapshot.take(), + startup_snapshot: Some(js::deno_isolate_init()), source_map_getter: options.source_map_getter, get_error_class_fn: options.get_error_class_fn, shared_array_buffer_store: options.shared_array_buffer_store.clone(), @@ -432,7 +429,6 @@ mod tests { shared_array_buffer_store: None, compiled_wasm_module_store: None, stdio: Default::default(), - startup_snapshot: None, }; MainWorker::bootstrap_from_options(main_module, permissions, options) diff --git a/snapshots/Cargo.toml b/snapshots/Cargo.toml deleted file mode 100644 index 074d78db7..000000000 --- a/snapshots/Cargo.toml +++ /dev/null @@ -1,32 +0,0 @@ -# Copyright 2018-2021 the Deno authors. All rights reserved. MIT license. - -# IMPORTANT(bartlomieju): this crate is internal and shouldn't be published -# to crates.io - -[package] -name = "deno_snapshots" -version = "0.1.0" -authors = ["the Deno authors"] -edition = "2018" -license = "MIT" -repository = "https://github.com/denoland/deno" -description = "Provides snapshots of TSC & Deno (runtime+web+core)" - -[lib] -name = "deno_snapshots" -path = "lib.rs" - -[dependencies] -deno_core = { version = "0.141.0", path = "../core" } # For mock TSC #[op]s -deno_runtime = { version = "0.67.0", path = "../runtime" } -lzzzz = "1.0" -once_cell = "1.10.0" -zstd = "0.11.1" - -[build-dependencies] -deno_core = { version = "0.141.0", path = "../core" } # For mock TSC #[op]s -deno_runtime = { version = "0.67.0", path = "../runtime" } -lzzzz = "1.0" -regex = "1.5.6" -serde = { version = "1.0.125", features = ["derive"] } -zstd = "0.11.1" diff --git a/snapshots/build.rs b/snapshots/build.rs deleted file mode 100644 index df868591d..000000000 --- a/snapshots/build.rs +++ /dev/null @@ -1,10 +0,0 @@ -// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. - -mod build_runtime; -mod build_tsc; - -fn main() { - let out_dir = std::path::PathBuf::from(std::env::var_os("OUT_DIR").unwrap()); - build_runtime::create_runtime_snapshot(&out_dir.join("CLI_SNAPSHOT.bin")); - build_tsc::create_tsc_snapshot(&out_dir.join("COMPILER_SNAPSHOT.bin")); -} diff --git a/snapshots/build_runtime.rs b/snapshots/build_runtime.rs deleted file mode 100644 index ae1c67322..000000000 --- a/snapshots/build_runtime.rs +++ /dev/null @@ -1,168 +0,0 @@ -// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license. -use std::convert::TryFrom; -use std::path::Path; - -use deno_runtime::deno_broadcast_channel; -use deno_runtime::deno_console; -use deno_runtime::deno_core; -use deno_runtime::deno_crypto; -use deno_runtime::deno_fetch; -use deno_runtime::deno_ffi; -use deno_runtime::deno_http; -use deno_runtime::deno_net; -use deno_runtime::deno_tls; -use deno_runtime::deno_url; -use deno_runtime::deno_web; -use deno_runtime::deno_webgpu; -use deno_runtime::deno_webidl; -use deno_runtime::deno_websocket; -use deno_runtime::deno_webstorage; - -use deno_core::Extension; -use deno_core::JsRuntime; -use deno_core::RuntimeOptions; - -pub fn create_runtime_snapshot(snapshot_path: &Path) { - let extensions: Vec<Extension> = vec![ - deno_webidl::init(), - deno_console::init(), - deno_url::init(), - deno_tls::init(), - deno_web::init::<Permissions>( - deno_web::BlobStore::default(), - Default::default(), - ), - deno_fetch::init::<Permissions>(Default::default()), - deno_websocket::init::<Permissions>("".to_owned(), None, None), - deno_webstorage::init(None), - deno_crypto::init(None), - deno_webgpu::init(false), - deno_broadcast_channel::init( - deno_broadcast_channel::InMemoryBroadcastChannel::default(), - false, // No --unstable. - ), - deno_ffi::init::<Permissions>(false), - deno_net::init::<Permissions>( - None, false, // No --unstable. - None, - ), - deno_http::init(), - // Runtime JS - deno_runtime::js::init(), - ]; - - let js_runtime = JsRuntime::new(RuntimeOptions { - will_snapshot: true, - extensions, - ..Default::default() - }); - write_runtime_snapshot(js_runtime, snapshot_path); -} - -// TODO(bartlomieju): this module contains a lot of duplicated -// logic with `build_tsc.rs` -fn write_runtime_snapshot(mut js_runtime: JsRuntime, snapshot_path: &Path) { - let snapshot = js_runtime.snapshot(); - let snapshot_slice: &[u8] = &*snapshot; - println!("Snapshot size: {}", snapshot_slice.len()); - - let compressed_snapshot_with_size = { - let mut vec = vec![]; - - vec.extend_from_slice( - &u32::try_from(snapshot.len()) - .expect("snapshot larger than 4gb") - .to_le_bytes(), - ); - - lzzzz::lz4_hc::compress_to_vec( - snapshot_slice, - &mut vec, - lzzzz::lz4_hc::CLEVEL_MAX, - ) - .expect("snapshot compression failed"); - - vec - }; - - println!( - "Snapshot compressed size: {}", - compressed_snapshot_with_size.len() - ); - - std::fs::write(&snapshot_path, compressed_snapshot_with_size).unwrap(); - println!("Snapshot written to: {} ", snapshot_path.display()); -} - -struct Permissions; - -impl deno_fetch::FetchPermissions for Permissions { - fn check_net_url( - &mut self, - _url: &deno_core::url::Url, - ) -> Result<(), deno_core::error::AnyError> { - unreachable!("snapshotting!") - } - - fn check_read( - &mut self, - _p: &Path, - ) -> Result<(), deno_core::error::AnyError> { - unreachable!("snapshotting!") - } -} - -impl deno_websocket::WebSocketPermissions for Permissions { - fn check_net_url( - &mut self, - _url: &deno_core::url::Url, - ) -> Result<(), deno_core::error::AnyError> { - unreachable!("snapshotting!") - } -} - -impl deno_web::TimersPermission for Permissions { - fn allow_hrtime(&mut self) -> bool { - unreachable!("snapshotting!") - } - - fn check_unstable( - &self, - _state: &deno_core::OpState, - _api_name: &'static str, - ) { - unreachable!("snapshotting!") - } -} - -impl deno_ffi::FfiPermissions for Permissions { - fn check( - &mut self, - _path: Option<&Path>, - ) -> Result<(), deno_core::error::AnyError> { - unreachable!("snapshotting!") - } -} - -impl deno_net::NetPermissions for Permissions { - fn check_net<T: AsRef<str>>( - &mut self, - _host: &(T, Option<u16>), - ) -> Result<(), deno_core::error::AnyError> { - unreachable!("snapshotting!") - } - - fn check_read( - &mut self, - _p: &Path, - ) -> Result<(), deno_core::error::AnyError> { - unreachable!("snapshotting!") - } - - fn check_write( - &mut self, - _p: &Path, - ) -> Result<(), deno_core::error::AnyError> { - unreachable!("snapshotting!") - } -} diff --git a/snapshots/build_tsc.rs b/snapshots/build_tsc.rs deleted file mode 100644 index 5296ae23b..000000000 --- a/snapshots/build_tsc.rs +++ /dev/null @@ -1,328 +0,0 @@ -// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. - -use deno_runtime::deno_broadcast_channel; -use deno_runtime::deno_console; -use deno_runtime::deno_crypto; -use deno_runtime::deno_fetch; -use deno_runtime::deno_net; -use deno_runtime::deno_url; -use deno_runtime::deno_web; -use deno_runtime::deno_websocket; -use deno_runtime::deno_webstorage; - -use deno_runtime::deno_core::error::custom_error; -use deno_runtime::deno_core::error::AnyError; -use deno_runtime::deno_core::op; -use deno_runtime::deno_core::serde::Deserialize; -use deno_runtime::deno_core::serde_json::json; -use deno_runtime::deno_core::serde_json::Value; -use deno_runtime::deno_core::Extension; -use deno_runtime::deno_core::JsRuntime; -use deno_runtime::deno_core::OpState; -use deno_runtime::deno_core::RuntimeOptions; - -use regex::Regex; -use std::collections::HashMap; -use std::convert::TryFrom; -use std::env; -use std::path::Path; -use std::path::PathBuf; - -pub fn create_tsc_snapshot(snapshot_path: &Path) { - let mut js_runtime = JsRuntime::new(RuntimeOptions { - will_snapshot: true, - extensions: vec![tsc_snapshot_init()], - ..Default::default() - }); - load_js_files(&mut js_runtime); - write_snapshot(js_runtime, snapshot_path); -} - -// TODO(bartlomieju): this module contains a lot of duplicated -// logic with `build_runtime.rs` -fn write_snapshot(mut js_runtime: JsRuntime, snapshot_path: &Path) { - let snapshot = js_runtime.snapshot(); - let snapshot_slice: &[u8] = &*snapshot; - println!("Snapshot size: {}", snapshot_slice.len()); - - let compressed_snapshot_with_size = { - let mut vec = vec![]; - - vec.extend_from_slice( - &u32::try_from(snapshot.len()) - .expect("snapshot larger than 4gb") - .to_le_bytes(), - ); - - vec.extend_from_slice( - &zstd::bulk::compress(snapshot_slice, 22) - .expect("snapshot compression failed"), - ); - - vec - }; - - println!( - "Snapshot compressed size: {}", - compressed_snapshot_with_size.len() - ); - - std::fs::write(&snapshot_path, compressed_snapshot_with_size).unwrap(); - println!("Snapshot written to: {} ", snapshot_path.display()); -} - -#[derive(Debug, Deserialize)] -struct LoadArgs { - /// The fully qualified specifier that should be loaded. - specifier: String, -} - -fn tsc_snapshot_init() -> Extension { - // libs that are being provided by op crates. - let mut op_crate_libs = HashMap::new(); - op_crate_libs.insert("deno.console", deno_console::get_declaration()); - op_crate_libs.insert("deno.url", deno_url::get_declaration()); - op_crate_libs.insert("deno.web", deno_web::get_declaration()); - op_crate_libs.insert("deno.fetch", deno_fetch::get_declaration()); - op_crate_libs.insert("deno.webgpu", deno_webgpu_get_declaration()); - op_crate_libs.insert("deno.websocket", deno_websocket::get_declaration()); - op_crate_libs.insert("deno.webstorage", deno_webstorage::get_declaration()); - op_crate_libs.insert("deno.crypto", deno_crypto::get_declaration()); - op_crate_libs.insert( - "deno.broadcast_channel", - deno_broadcast_channel::get_declaration(), - ); - op_crate_libs.insert("deno.net", deno_net::get_declaration()); - - // ensure we invalidate the build properly. - for (_, path) in op_crate_libs.iter() { - println!("cargo:rerun-if-changed={}", path.display()); - } - - // libs that should be loaded into the isolate before snapshotting. - let libs = vec![ - // Deno custom type libraries - "deno.window", - "deno.worker", - "deno.shared_globals", - "deno.ns", - "deno.unstable", - // Deno built-in type libraries - "es5", - "es2015.collection", - "es2015.core", - "es2015", - "es2015.generator", - "es2015.iterable", - "es2015.promise", - "es2015.proxy", - "es2015.reflect", - "es2015.symbol", - "es2015.symbol.wellknown", - "es2016.array.include", - "es2016", - "es2017", - "es2017.intl", - "es2017.object", - "es2017.sharedmemory", - "es2017.string", - "es2017.typedarrays", - "es2018.asyncgenerator", - "es2018.asynciterable", - "es2018", - "es2018.intl", - "es2018.promise", - "es2018.regexp", - "es2019.array", - "es2019", - "es2019.object", - "es2019.string", - "es2019.symbol", - "es2020.bigint", - "es2020", - "es2020.date", - "es2020.intl", - "es2020.number", - "es2020.promise", - "es2020.sharedmemory", - "es2020.string", - "es2020.symbol.wellknown", - "es2021", - "es2021.intl", - "es2021.promise", - "es2021.string", - "es2021.weakref", - "es2022", - "es2022.array", - "es2022.error", - "es2022.intl", - "es2022.object", - "es2022.string", - "esnext", - "esnext.array", - "esnext.intl", - ]; - - let cli_dir = cli_dir(); - let path_dts = cli_dir.join("dts"); - // ensure we invalidate the build properly. - for name in libs.iter() { - println!( - "cargo:rerun-if-changed={}", - path_dts.join(format!("lib.{}.d.ts", name)).display() - ); - } - - // create a copy of the vector that includes any op crate libs to be passed - // to the JavaScript compiler to build into the snapshot - let mut build_libs = libs.clone(); - for (op_lib, _) in op_crate_libs.iter() { - build_libs.push(op_lib.to_owned()); - } - - #[op] - fn op_build_info(state: &mut OpState) -> Value { - let build_specifier = "asset:///bootstrap.ts"; - let build_libs = state.borrow::<Vec<&str>>(); - json!({ - "buildSpecifier": build_specifier, - "libs": build_libs, - }) - } - - #[op] - fn op_cwd() -> String { - "cache:///".into() - } - - #[op] - fn op_exists() -> bool { - false - } - - #[op] - fn op_script_version( - _state: &mut OpState, - _args: Value, - ) -> Result<Option<String>, AnyError> { - Ok(Some("1".to_string())) - } - - #[op] - // using the same op that is used in `tsc.rs` for loading modules and reading - // files, but a slightly different implementation at build time. - fn op_load(state: &mut OpState, args: LoadArgs) -> Result<Value, AnyError> { - let op_crate_libs = state.borrow::<HashMap<&str, PathBuf>>(); - let path_dts = state.borrow::<PathBuf>(); - let re_asset = - Regex::new(r"asset:/{3}lib\.(\S+)\.d\.ts").expect("bad regex"); - let build_specifier = "asset:///bootstrap.ts"; - - // we need a basic file to send to tsc to warm it up. - if args.specifier == build_specifier { - Ok(json!({ - "data": r#"console.log("hello deno!");"#, - "version": "1", - // this corresponds to `ts.ScriptKind.TypeScript` - "scriptKind": 3 - })) - // specifiers come across as `asset:///lib.{lib_name}.d.ts` and we need to - // parse out just the name so we can lookup the asset. - } else if let Some(caps) = re_asset.captures(&args.specifier) { - if let Some(lib) = caps.get(1).map(|m| m.as_str()) { - // if it comes from an op crate, we were supplied with the path to the - // file. - let path = if let Some(op_crate_lib) = op_crate_libs.get(lib) { - PathBuf::from(op_crate_lib).canonicalize().unwrap() - // otherwise we are will generate the path ourself - } else { - path_dts.join(format!("lib.{}.d.ts", lib)) - }; - let data = std::fs::read_to_string(path)?; - Ok(json!({ - "data": data, - "version": "1", - // this corresponds to `ts.ScriptKind.TypeScript` - "scriptKind": 3 - })) - } else { - Err(custom_error( - "InvalidSpecifier", - format!("An invalid specifier was requested: {}", args.specifier), - )) - } - } else { - Err(custom_error( - "InvalidSpecifier", - format!("An invalid specifier was requested: {}", args.specifier), - )) - } - } - - Extension::builder() - .ops(vec![ - op_build_info::decl(), - op_cwd::decl(), - op_exists::decl(), - op_load::decl(), - op_script_version::decl(), - ]) - .state(move |state| { - state.put(op_crate_libs.clone()); - state.put(build_libs.clone()); - state.put(path_dts.clone()); - - Ok(()) - }) - .build() -} - -fn deno_webgpu_get_declaration() -> PathBuf { - cli_dir().join("dts").join("lib.deno_webgpu.d.ts") -} - -fn load_js_files(js_runtime: &mut JsRuntime) { - let js_files = get_js_files(tsc_dir()); - let cwd = cli_dir(); - let display_root = cwd.parent().unwrap(); - for file in js_files { - println!("cargo:rerun-if-changed={}", file.display()); - let display_path = file.strip_prefix(display_root).unwrap(); - let display_path_str = display_path.display().to_string(); - js_runtime - .execute_script( - &("deno:".to_string() + &display_path_str.replace('\\', "/")), - &std::fs::read_to_string(&file).unwrap(), - ) - .unwrap(); - } -} - -fn root_dir() -> PathBuf { - // TODO(nayeemrmn): https://github.com/rust-lang/cargo/issues/3946 to get the workspace root. - Path::new(env!("CARGO_MANIFEST_DIR")) - .join("..") - .canonicalize() - .unwrap() -} - -fn cli_dir() -> PathBuf { - root_dir().join("cli") -} - -fn tsc_dir() -> PathBuf { - cli_dir().join("tsc") -} - -fn get_js_files(dir: PathBuf) -> Vec<PathBuf> { - let mut js_files = std::fs::read_dir(dir.clone()) - .unwrap() - .map(|dir_entry| { - let file = dir_entry.unwrap(); - dir.join(file.path()) - }) - .filter(|path| path.extension().unwrap_or_default() == "js") - .collect::<Vec<PathBuf>>(); - js_files.sort(); - js_files -} diff --git a/snapshots/lib.rs b/snapshots/lib.rs deleted file mode 100644 index 1d0b3ceb3..000000000 --- a/snapshots/lib.rs +++ /dev/null @@ -1,77 +0,0 @@ -use deno_core::Snapshot; -use once_cell::sync::Lazy; -use std::convert::TryInto; - -pub fn tsc_snapshot() -> Snapshot { - Snapshot::Static(&*COMPILER_SNAPSHOT) -} - -static COMPILER_SNAPSHOT: Lazy<Box<[u8]>> = Lazy::new( - #[cold] - #[inline(never)] - || { - static COMPRESSED_COMPILER_SNAPSHOT: &[u8] = - include_bytes!(concat!(env!("OUT_DIR"), "/COMPILER_SNAPSHOT.bin")); - - zstd::bulk::decompress( - &COMPRESSED_COMPILER_SNAPSHOT[4..], - u32::from_le_bytes(COMPRESSED_COMPILER_SNAPSHOT[0..4].try_into().unwrap()) - as usize, - ) - .unwrap() - .into_boxed_slice() - }, -); - -pub fn cli_snapshot() -> Snapshot { - Snapshot::Static(&*CLI_SNAPSHOT) -} - -static CLI_SNAPSHOT: Lazy<Box<[u8]>> = Lazy::new( - #[allow(clippy::uninit_vec)] - #[cold] - #[inline(never)] - || { - static COMPRESSED_CLI_SNAPSHOT: &[u8] = - include_bytes!(concat!(env!("OUT_DIR"), "/CLI_SNAPSHOT.bin")); - - let size = - u32::from_le_bytes(COMPRESSED_CLI_SNAPSHOT[0..4].try_into().unwrap()) - as usize; - let mut vec = Vec::with_capacity(size); - - // SAFETY: vec is allocated with exact snapshot size (+ alignment) - // SAFETY: non zeroed bytes are overwritten with decompressed snapshot - unsafe { - vec.set_len(size); - } - - lzzzz::lz4::decompress(&COMPRESSED_CLI_SNAPSHOT[4..], &mut vec).unwrap(); - - vec.into_boxed_slice() - }, -); - -#[cfg(test)] -mod tests { - use deno_runtime::deno_core; - - #[test] - fn cli_snapshot() { - let mut js_runtime = deno_core::JsRuntime::new(deno_core::RuntimeOptions { - startup_snapshot: Some(crate::cli_snapshot()), - ..Default::default() - }); - js_runtime - .execute_script( - "<anon>", - r#" - if (!(bootstrap.mainRuntime && bootstrap.workerRuntime)) { - throw Error("bad"); - } - console.log("we have console.log!!!"); - "#, - ) - .unwrap(); - } -} |