summaryrefslogtreecommitdiff
path: root/snapshots
diff options
context:
space:
mode:
Diffstat (limited to 'snapshots')
-rw-r--r--snapshots/Cargo.toml32
-rw-r--r--snapshots/build.rs10
-rw-r--r--snapshots/build_runtime.rs168
-rw-r--r--snapshots/build_tsc.rs328
-rw-r--r--snapshots/lib.rs77
5 files changed, 615 insertions, 0 deletions
diff --git a/snapshots/Cargo.toml b/snapshots/Cargo.toml
new file mode 100644
index 000000000..6266ed681
--- /dev/null
+++ b/snapshots/Cargo.toml
@@ -0,0 +1,32 @@
+# Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
+
+# IMPORTANT(bartlomieju): this crate is internal and shouldn't be published
+# to crates.io
+
+[package]
+name = "deno_snapshots"
+version = "0.0.0"
+authors = ["the Deno authors"]
+edition = "2018"
+license = "MIT"
+repository = "https://github.com/denoland/deno"
+description = "Provides snapshots of TSC & Deno (runtime+web+core)"
+
+[lib]
+name = "deno_snapshots"
+path = "lib.rs"
+
+[dependencies]
+deno_core = { version = "0.140.0", path = "../core" } # For mock TSC #[op]s
+deno_runtime = { version = "0.66.0", path = "../runtime" }
+lzzzz = "1.0"
+once_cell = "1.10.0"
+zstd = "0.11.1"
+
+[build-dependencies]
+deno_core = { version = "0.140.0", path = "../core" } # For mock TSC #[op]s
+deno_runtime = { version = "0.66.0", path = "../runtime" }
+lzzzz = "1.0"
+regex = "1.5.6"
+serde = { version = "1.0.125", features = ["derive"] }
+zstd = "0.11.1"
diff --git a/snapshots/build.rs b/snapshots/build.rs
new file mode 100644
index 000000000..df868591d
--- /dev/null
+++ b/snapshots/build.rs
@@ -0,0 +1,10 @@
+// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
+
+mod build_runtime;
+mod build_tsc;
+
+fn main() {
+ let out_dir = std::path::PathBuf::from(std::env::var_os("OUT_DIR").unwrap());
+ build_runtime::create_runtime_snapshot(&out_dir.join("CLI_SNAPSHOT.bin"));
+ build_tsc::create_tsc_snapshot(&out_dir.join("COMPILER_SNAPSHOT.bin"));
+}
diff --git a/snapshots/build_runtime.rs b/snapshots/build_runtime.rs
new file mode 100644
index 000000000..ae1c67322
--- /dev/null
+++ b/snapshots/build_runtime.rs
@@ -0,0 +1,168 @@
+// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
+use std::convert::TryFrom;
+use std::path::Path;
+
+use deno_runtime::deno_broadcast_channel;
+use deno_runtime::deno_console;
+use deno_runtime::deno_core;
+use deno_runtime::deno_crypto;
+use deno_runtime::deno_fetch;
+use deno_runtime::deno_ffi;
+use deno_runtime::deno_http;
+use deno_runtime::deno_net;
+use deno_runtime::deno_tls;
+use deno_runtime::deno_url;
+use deno_runtime::deno_web;
+use deno_runtime::deno_webgpu;
+use deno_runtime::deno_webidl;
+use deno_runtime::deno_websocket;
+use deno_runtime::deno_webstorage;
+
+use deno_core::Extension;
+use deno_core::JsRuntime;
+use deno_core::RuntimeOptions;
+
+pub fn create_runtime_snapshot(snapshot_path: &Path) {
+ let extensions: Vec<Extension> = vec![
+ deno_webidl::init(),
+ deno_console::init(),
+ deno_url::init(),
+ deno_tls::init(),
+ deno_web::init::<Permissions>(
+ deno_web::BlobStore::default(),
+ Default::default(),
+ ),
+ deno_fetch::init::<Permissions>(Default::default()),
+ deno_websocket::init::<Permissions>("".to_owned(), None, None),
+ deno_webstorage::init(None),
+ deno_crypto::init(None),
+ deno_webgpu::init(false),
+ deno_broadcast_channel::init(
+ deno_broadcast_channel::InMemoryBroadcastChannel::default(),
+ false, // No --unstable.
+ ),
+ deno_ffi::init::<Permissions>(false),
+ deno_net::init::<Permissions>(
+ None, false, // No --unstable.
+ None,
+ ),
+ deno_http::init(),
+ // Runtime JS
+ deno_runtime::js::init(),
+ ];
+
+ let js_runtime = JsRuntime::new(RuntimeOptions {
+ will_snapshot: true,
+ extensions,
+ ..Default::default()
+ });
+ write_runtime_snapshot(js_runtime, snapshot_path);
+}
+
+// TODO(bartlomieju): this module contains a lot of duplicated
+// logic with `build_tsc.rs`
+fn write_runtime_snapshot(mut js_runtime: JsRuntime, snapshot_path: &Path) {
+ let snapshot = js_runtime.snapshot();
+ let snapshot_slice: &[u8] = &*snapshot;
+ println!("Snapshot size: {}", snapshot_slice.len());
+
+ let compressed_snapshot_with_size = {
+ let mut vec = vec![];
+
+ vec.extend_from_slice(
+ &u32::try_from(snapshot.len())
+ .expect("snapshot larger than 4gb")
+ .to_le_bytes(),
+ );
+
+ lzzzz::lz4_hc::compress_to_vec(
+ snapshot_slice,
+ &mut vec,
+ lzzzz::lz4_hc::CLEVEL_MAX,
+ )
+ .expect("snapshot compression failed");
+
+ vec
+ };
+
+ println!(
+ "Snapshot compressed size: {}",
+ compressed_snapshot_with_size.len()
+ );
+
+ std::fs::write(&snapshot_path, compressed_snapshot_with_size).unwrap();
+ println!("Snapshot written to: {} ", snapshot_path.display());
+}
+
+struct Permissions;
+
+impl deno_fetch::FetchPermissions for Permissions {
+ fn check_net_url(
+ &mut self,
+ _url: &deno_core::url::Url,
+ ) -> Result<(), deno_core::error::AnyError> {
+ unreachable!("snapshotting!")
+ }
+
+ fn check_read(
+ &mut self,
+ _p: &Path,
+ ) -> Result<(), deno_core::error::AnyError> {
+ unreachable!("snapshotting!")
+ }
+}
+
+impl deno_websocket::WebSocketPermissions for Permissions {
+ fn check_net_url(
+ &mut self,
+ _url: &deno_core::url::Url,
+ ) -> Result<(), deno_core::error::AnyError> {
+ unreachable!("snapshotting!")
+ }
+}
+
+impl deno_web::TimersPermission for Permissions {
+ fn allow_hrtime(&mut self) -> bool {
+ unreachable!("snapshotting!")
+ }
+
+ fn check_unstable(
+ &self,
+ _state: &deno_core::OpState,
+ _api_name: &'static str,
+ ) {
+ unreachable!("snapshotting!")
+ }
+}
+
+impl deno_ffi::FfiPermissions for Permissions {
+ fn check(
+ &mut self,
+ _path: Option<&Path>,
+ ) -> Result<(), deno_core::error::AnyError> {
+ unreachable!("snapshotting!")
+ }
+}
+
+impl deno_net::NetPermissions for Permissions {
+ fn check_net<T: AsRef<str>>(
+ &mut self,
+ _host: &(T, Option<u16>),
+ ) -> Result<(), deno_core::error::AnyError> {
+ unreachable!("snapshotting!")
+ }
+
+ fn check_read(
+ &mut self,
+ _p: &Path,
+ ) -> Result<(), deno_core::error::AnyError> {
+ unreachable!("snapshotting!")
+ }
+
+ fn check_write(
+ &mut self,
+ _p: &Path,
+ ) -> Result<(), deno_core::error::AnyError> {
+ unreachable!("snapshotting!")
+ }
+}
diff --git a/snapshots/build_tsc.rs b/snapshots/build_tsc.rs
new file mode 100644
index 000000000..5296ae23b
--- /dev/null
+++ b/snapshots/build_tsc.rs
@@ -0,0 +1,328 @@
+// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
+
+use deno_runtime::deno_broadcast_channel;
+use deno_runtime::deno_console;
+use deno_runtime::deno_crypto;
+use deno_runtime::deno_fetch;
+use deno_runtime::deno_net;
+use deno_runtime::deno_url;
+use deno_runtime::deno_web;
+use deno_runtime::deno_websocket;
+use deno_runtime::deno_webstorage;
+
+use deno_runtime::deno_core::error::custom_error;
+use deno_runtime::deno_core::error::AnyError;
+use deno_runtime::deno_core::op;
+use deno_runtime::deno_core::serde::Deserialize;
+use deno_runtime::deno_core::serde_json::json;
+use deno_runtime::deno_core::serde_json::Value;
+use deno_runtime::deno_core::Extension;
+use deno_runtime::deno_core::JsRuntime;
+use deno_runtime::deno_core::OpState;
+use deno_runtime::deno_core::RuntimeOptions;
+
+use regex::Regex;
+use std::collections::HashMap;
+use std::convert::TryFrom;
+use std::env;
+use std::path::Path;
+use std::path::PathBuf;
+
+pub fn create_tsc_snapshot(snapshot_path: &Path) {
+ let mut js_runtime = JsRuntime::new(RuntimeOptions {
+ will_snapshot: true,
+ extensions: vec![tsc_snapshot_init()],
+ ..Default::default()
+ });
+ load_js_files(&mut js_runtime);
+ write_snapshot(js_runtime, snapshot_path);
+}
+
+// TODO(bartlomieju): this module contains a lot of duplicated
+// logic with `build_runtime.rs`
+fn write_snapshot(mut js_runtime: JsRuntime, snapshot_path: &Path) {
+ let snapshot = js_runtime.snapshot();
+ let snapshot_slice: &[u8] = &*snapshot;
+ println!("Snapshot size: {}", snapshot_slice.len());
+
+ let compressed_snapshot_with_size = {
+ let mut vec = vec![];
+
+ vec.extend_from_slice(
+ &u32::try_from(snapshot.len())
+ .expect("snapshot larger than 4gb")
+ .to_le_bytes(),
+ );
+
+ vec.extend_from_slice(
+ &zstd::bulk::compress(snapshot_slice, 22)
+ .expect("snapshot compression failed"),
+ );
+
+ vec
+ };
+
+ println!(
+ "Snapshot compressed size: {}",
+ compressed_snapshot_with_size.len()
+ );
+
+ std::fs::write(&snapshot_path, compressed_snapshot_with_size).unwrap();
+ println!("Snapshot written to: {} ", snapshot_path.display());
+}
+
+#[derive(Debug, Deserialize)]
+struct LoadArgs {
+ /// The fully qualified specifier that should be loaded.
+ specifier: String,
+}
+
+fn tsc_snapshot_init() -> Extension {
+ // libs that are being provided by op crates.
+ let mut op_crate_libs = HashMap::new();
+ op_crate_libs.insert("deno.console", deno_console::get_declaration());
+ op_crate_libs.insert("deno.url", deno_url::get_declaration());
+ op_crate_libs.insert("deno.web", deno_web::get_declaration());
+ op_crate_libs.insert("deno.fetch", deno_fetch::get_declaration());
+ op_crate_libs.insert("deno.webgpu", deno_webgpu_get_declaration());
+ op_crate_libs.insert("deno.websocket", deno_websocket::get_declaration());
+ op_crate_libs.insert("deno.webstorage", deno_webstorage::get_declaration());
+ op_crate_libs.insert("deno.crypto", deno_crypto::get_declaration());
+ op_crate_libs.insert(
+ "deno.broadcast_channel",
+ deno_broadcast_channel::get_declaration(),
+ );
+ op_crate_libs.insert("deno.net", deno_net::get_declaration());
+
+ // ensure we invalidate the build properly.
+ for (_, path) in op_crate_libs.iter() {
+ println!("cargo:rerun-if-changed={}", path.display());
+ }
+
+ // libs that should be loaded into the isolate before snapshotting.
+ let libs = vec![
+ // Deno custom type libraries
+ "deno.window",
+ "deno.worker",
+ "deno.shared_globals",
+ "deno.ns",
+ "deno.unstable",
+ // Deno built-in type libraries
+ "es5",
+ "es2015.collection",
+ "es2015.core",
+ "es2015",
+ "es2015.generator",
+ "es2015.iterable",
+ "es2015.promise",
+ "es2015.proxy",
+ "es2015.reflect",
+ "es2015.symbol",
+ "es2015.symbol.wellknown",
+ "es2016.array.include",
+ "es2016",
+ "es2017",
+ "es2017.intl",
+ "es2017.object",
+ "es2017.sharedmemory",
+ "es2017.string",
+ "es2017.typedarrays",
+ "es2018.asyncgenerator",
+ "es2018.asynciterable",
+ "es2018",
+ "es2018.intl",
+ "es2018.promise",
+ "es2018.regexp",
+ "es2019.array",
+ "es2019",
+ "es2019.object",
+ "es2019.string",
+ "es2019.symbol",
+ "es2020.bigint",
+ "es2020",
+ "es2020.date",
+ "es2020.intl",
+ "es2020.number",
+ "es2020.promise",
+ "es2020.sharedmemory",
+ "es2020.string",
+ "es2020.symbol.wellknown",
+ "es2021",
+ "es2021.intl",
+ "es2021.promise",
+ "es2021.string",
+ "es2021.weakref",
+ "es2022",
+ "es2022.array",
+ "es2022.error",
+ "es2022.intl",
+ "es2022.object",
+ "es2022.string",
+ "esnext",
+ "esnext.array",
+ "esnext.intl",
+ ];
+
+ let cli_dir = cli_dir();
+ let path_dts = cli_dir.join("dts");
+ // ensure we invalidate the build properly.
+ for name in libs.iter() {
+ println!(
+ "cargo:rerun-if-changed={}",
+ path_dts.join(format!("lib.{}.d.ts", name)).display()
+ );
+ }
+
+ // create a copy of the vector that includes any op crate libs to be passed
+ // to the JavaScript compiler to build into the snapshot
+ let mut build_libs = libs.clone();
+ for (op_lib, _) in op_crate_libs.iter() {
+ build_libs.push(op_lib.to_owned());
+ }
+
+ #[op]
+ fn op_build_info(state: &mut OpState) -> Value {
+ let build_specifier = "asset:///bootstrap.ts";
+ let build_libs = state.borrow::<Vec<&str>>();
+ json!({
+ "buildSpecifier": build_specifier,
+ "libs": build_libs,
+ })
+ }
+
+ #[op]
+ fn op_cwd() -> String {
+ "cache:///".into()
+ }
+
+ #[op]
+ fn op_exists() -> bool {
+ false
+ }
+
+ #[op]
+ fn op_script_version(
+ _state: &mut OpState,
+ _args: Value,
+ ) -> Result<Option<String>, AnyError> {
+ Ok(Some("1".to_string()))
+ }
+
+ #[op]
+ // using the same op that is used in `tsc.rs` for loading modules and reading
+ // files, but a slightly different implementation at build time.
+ fn op_load(state: &mut OpState, args: LoadArgs) -> Result<Value, AnyError> {
+ let op_crate_libs = state.borrow::<HashMap<&str, PathBuf>>();
+ let path_dts = state.borrow::<PathBuf>();
+ let re_asset =
+ Regex::new(r"asset:/{3}lib\.(\S+)\.d\.ts").expect("bad regex");
+ let build_specifier = "asset:///bootstrap.ts";
+
+ // we need a basic file to send to tsc to warm it up.
+ if args.specifier == build_specifier {
+ Ok(json!({
+ "data": r#"console.log("hello deno!");"#,
+ "version": "1",
+ // this corresponds to `ts.ScriptKind.TypeScript`
+ "scriptKind": 3
+ }))
+ // specifiers come across as `asset:///lib.{lib_name}.d.ts` and we need to
+ // parse out just the name so we can lookup the asset.
+ } else if let Some(caps) = re_asset.captures(&args.specifier) {
+ if let Some(lib) = caps.get(1).map(|m| m.as_str()) {
+ // if it comes from an op crate, we were supplied with the path to the
+ // file.
+ let path = if let Some(op_crate_lib) = op_crate_libs.get(lib) {
+ PathBuf::from(op_crate_lib).canonicalize().unwrap()
+ // otherwise we are will generate the path ourself
+ } else {
+ path_dts.join(format!("lib.{}.d.ts", lib))
+ };
+ let data = std::fs::read_to_string(path)?;
+ Ok(json!({
+ "data": data,
+ "version": "1",
+ // this corresponds to `ts.ScriptKind.TypeScript`
+ "scriptKind": 3
+ }))
+ } else {
+ Err(custom_error(
+ "InvalidSpecifier",
+ format!("An invalid specifier was requested: {}", args.specifier),
+ ))
+ }
+ } else {
+ Err(custom_error(
+ "InvalidSpecifier",
+ format!("An invalid specifier was requested: {}", args.specifier),
+ ))
+ }
+ }
+
+ Extension::builder()
+ .ops(vec![
+ op_build_info::decl(),
+ op_cwd::decl(),
+ op_exists::decl(),
+ op_load::decl(),
+ op_script_version::decl(),
+ ])
+ .state(move |state| {
+ state.put(op_crate_libs.clone());
+ state.put(build_libs.clone());
+ state.put(path_dts.clone());
+
+ Ok(())
+ })
+ .build()
+}
+
+fn deno_webgpu_get_declaration() -> PathBuf {
+ cli_dir().join("dts").join("lib.deno_webgpu.d.ts")
+}
+
+fn load_js_files(js_runtime: &mut JsRuntime) {
+ let js_files = get_js_files(tsc_dir());
+ let cwd = cli_dir();
+ let display_root = cwd.parent().unwrap();
+ for file in js_files {
+ println!("cargo:rerun-if-changed={}", file.display());
+ let display_path = file.strip_prefix(display_root).unwrap();
+ let display_path_str = display_path.display().to_string();
+ js_runtime
+ .execute_script(
+ &("deno:".to_string() + &display_path_str.replace('\\', "/")),
+ &std::fs::read_to_string(&file).unwrap(),
+ )
+ .unwrap();
+ }
+}
+
+fn root_dir() -> PathBuf {
+ // TODO(nayeemrmn): https://github.com/rust-lang/cargo/issues/3946 to get the workspace root.
+ Path::new(env!("CARGO_MANIFEST_DIR"))
+ .join("..")
+ .canonicalize()
+ .unwrap()
+}
+
+fn cli_dir() -> PathBuf {
+ root_dir().join("cli")
+}
+
+fn tsc_dir() -> PathBuf {
+ cli_dir().join("tsc")
+}
+
+fn get_js_files(dir: PathBuf) -> Vec<PathBuf> {
+ let mut js_files = std::fs::read_dir(dir.clone())
+ .unwrap()
+ .map(|dir_entry| {
+ let file = dir_entry.unwrap();
+ dir.join(file.path())
+ })
+ .filter(|path| path.extension().unwrap_or_default() == "js")
+ .collect::<Vec<PathBuf>>();
+ js_files.sort();
+ js_files
+}
diff --git a/snapshots/lib.rs b/snapshots/lib.rs
new file mode 100644
index 000000000..1d0b3ceb3
--- /dev/null
+++ b/snapshots/lib.rs
@@ -0,0 +1,77 @@
+use deno_core::Snapshot;
+use once_cell::sync::Lazy;
+use std::convert::TryInto;
+
+pub fn tsc_snapshot() -> Snapshot {
+ Snapshot::Static(&*COMPILER_SNAPSHOT)
+}
+
+static COMPILER_SNAPSHOT: Lazy<Box<[u8]>> = Lazy::new(
+ #[cold]
+ #[inline(never)]
+ || {
+ static COMPRESSED_COMPILER_SNAPSHOT: &[u8] =
+ include_bytes!(concat!(env!("OUT_DIR"), "/COMPILER_SNAPSHOT.bin"));
+
+ zstd::bulk::decompress(
+ &COMPRESSED_COMPILER_SNAPSHOT[4..],
+ u32::from_le_bytes(COMPRESSED_COMPILER_SNAPSHOT[0..4].try_into().unwrap())
+ as usize,
+ )
+ .unwrap()
+ .into_boxed_slice()
+ },
+);
+
+pub fn cli_snapshot() -> Snapshot {
+ Snapshot::Static(&*CLI_SNAPSHOT)
+}
+
+static CLI_SNAPSHOT: Lazy<Box<[u8]>> = Lazy::new(
+ #[allow(clippy::uninit_vec)]
+ #[cold]
+ #[inline(never)]
+ || {
+ static COMPRESSED_CLI_SNAPSHOT: &[u8] =
+ include_bytes!(concat!(env!("OUT_DIR"), "/CLI_SNAPSHOT.bin"));
+
+ let size =
+ u32::from_le_bytes(COMPRESSED_CLI_SNAPSHOT[0..4].try_into().unwrap())
+ as usize;
+ let mut vec = Vec::with_capacity(size);
+
+ // SAFETY: vec is allocated with exact snapshot size (+ alignment)
+ // SAFETY: non zeroed bytes are overwritten with decompressed snapshot
+ unsafe {
+ vec.set_len(size);
+ }
+
+ lzzzz::lz4::decompress(&COMPRESSED_CLI_SNAPSHOT[4..], &mut vec).unwrap();
+
+ vec.into_boxed_slice()
+ },
+);
+
+#[cfg(test)]
+mod tests {
+ use deno_runtime::deno_core;
+
+ #[test]
+ fn cli_snapshot() {
+ let mut js_runtime = deno_core::JsRuntime::new(deno_core::RuntimeOptions {
+ startup_snapshot: Some(crate::cli_snapshot()),
+ ..Default::default()
+ });
+ js_runtime
+ .execute_script(
+ "<anon>",
+ r#"
+ if (!(bootstrap.mainRuntime && bootstrap.workerRuntime)) {
+ throw Error("bad");
+ }
+ console.log("we have console.log!!!");
+ "#,
+ )
+ .unwrap();
+ }
+}
generated by cgit v1.2.3 (git 2.25.1) at 2025-12-10 09:36:06 +0000