summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorAaron O'Mullan <aaron.omullan@gmail.com>2022-06-24 10:04:45 -0300
committerGitHub <noreply@github.com>2022-06-24 15:04:45 +0200
commitfd5a12d7e25dc53238e2bbcffe970e646c1035f3 (patch)
tree251c3ec1a46067b02ef30fb48349962973016bf3
parentd39094913e91e5193f63459d9c5ca6ddc7779477 (diff)
refactor(snapshots): to their own crate (#14794)
Co-authored-by: Bartek IwaƄczuk <biwanczuk@gmail.com>
-rw-r--r--Cargo.lock24
-rw-r--r--Cargo.toml1
-rw-r--r--cli/Cargo.toml13
-rw-r--r--cli/build.rs385
-rw-r--r--cli/lsp/tsc.rs2
-rw-r--r--cli/main.rs2
-rw-r--r--cli/standalone.rs1
-rw-r--r--cli/tsc.rs28
-rw-r--r--runtime/Cargo.toml19
-rw-r--r--runtime/build.rs222
-rw-r--r--runtime/examples/hello_runtime.rs1
-rw-r--r--runtime/js.rs92
-rw-r--r--runtime/web_worker.rs5
-rw-r--r--runtime/worker.rs6
-rw-r--r--snapshots/Cargo.toml32
-rw-r--r--snapshots/build.rs10
-rw-r--r--snapshots/build_runtime.rs168
-rw-r--r--snapshots/build_tsc.rs328
-rw-r--r--snapshots/lib.rs77
19 files changed, 741 insertions, 675 deletions
diff --git a/Cargo.lock b/Cargo.lock
index 54ec13ac0..5165c654e 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -738,23 +738,14 @@ dependencies = [
"data-url",
"deno_ast",
"deno_bench_util",
- "deno_broadcast_channel",
- "deno_console",
"deno_core",
- "deno_crypto",
"deno_doc",
"deno_emit",
- "deno_fetch",
"deno_graph",
"deno_lint",
- "deno_net",
"deno_runtime",
+ "deno_snapshots",
"deno_task_shell",
- "deno_url",
- "deno_web",
- "deno_webgpu",
- "deno_websocket",
- "deno_webstorage",
"dissimilar",
"dprint-plugin-json",
"dprint-plugin-markdown",
@@ -1117,6 +1108,19 @@ dependencies = [
]
[[package]]
+name = "deno_snapshots"
+version = "0.0.0"
+dependencies = [
+ "deno_core",
+ "deno_runtime",
+ "lzzzz",
+ "once_cell",
+ "regex",
+ "serde",
+ "zstd",
+]
+
+[[package]]
name = "deno_task_shell"
version = "0.4.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
diff --git a/Cargo.toml b/Cargo.toml
index 95063652e..12a8b2cce 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -9,6 +9,7 @@ members = [
"ops",
"runtime",
"serde_v8",
+ "snapshots",
"test_ffi",
"test_util",
"ext/broadcast_channel",
diff --git a/cli/Cargo.toml b/cli/Cargo.toml
index 1ed342c6d..eba22c3e9 100644
--- a/cli/Cargo.toml
+++ b/cli/Cargo.toml
@@ -25,17 +25,7 @@ harness = false
path = "./bench/lsp_bench_standalone.rs"
[build-dependencies]
-deno_broadcast_channel = { version = "0.52.0", path = "../ext/broadcast_channel" }
-deno_console = { version = "0.58.0", path = "../ext/console" }
-deno_core = { version = "0.140.0", path = "../core" }
-deno_crypto = { version = "0.72.0", path = "../ext/crypto" }
-deno_fetch = { version = "0.81.0", path = "../ext/fetch" }
-deno_net = { version = "0.50.0", path = "../ext/net" }
-deno_url = { version = "0.58.0", path = "../ext/url" }
-deno_web = { version = "0.89.0", path = "../ext/web" }
-deno_webgpu = { version = "0.59.0", path = "../ext/webgpu" }
-deno_websocket = { version = "0.63.0", path = "../ext/websocket" }
-deno_webstorage = { version = "0.53.0", path = "../ext/webstorage" }
+deno_runtime = { version = "0.66.0", path = "../runtime" }
regex = "=1.5.6"
serde = { version = "=1.0.136", features = ["derive"] }
zstd = '=0.11.1'
@@ -52,6 +42,7 @@ deno_emit = "0.3.0"
deno_graph = "0.28.0"
deno_lint = { version = "0.31.0", features = ["docs"] }
deno_runtime = { version = "0.66.0", path = "../runtime" }
+deno_snapshots = { version = "0.0.0", path = "../snapshots" }
deno_task_shell = "0.4.0"
atty = "=0.2.14"
diff --git a/cli/build.rs b/cli/build.rs
index 1a4eaa425..095efa22f 100644
--- a/cli/build.rs
+++ b/cli/build.rs
@@ -1,323 +1,19 @@
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
-use deno_core::error::custom_error;
-use deno_core::error::AnyError;
-use deno_core::op;
-use deno_core::serde::Deserialize;
-use deno_core::serde_json::json;
-use deno_core::serde_json::Value;
-use deno_core::Extension;
-use deno_core::JsRuntime;
-use deno_core::OpState;
-use deno_core::RuntimeOptions;
-use regex::Regex;
-use std::collections::HashMap;
+use deno_runtime::deno_broadcast_channel;
+use deno_runtime::deno_console;
+use deno_runtime::deno_crypto;
+use deno_runtime::deno_fetch;
+use deno_runtime::deno_net;
+use deno_runtime::deno_url;
+use deno_runtime::deno_web;
+use deno_runtime::deno_websocket;
+use deno_runtime::deno_webstorage;
+
use std::env;
use std::path::Path;
use std::path::PathBuf;
-// TODO(bartlomieju): this module contains a lot of duplicated
-// logic with `runtime/build.rs`, factor out to `deno_core`.
-fn create_snapshot(
- mut js_runtime: JsRuntime,
- snapshot_path: &Path,
- files: Vec<PathBuf>,
-) {
- // TODO(nayeemrmn): https://github.com/rust-lang/cargo/issues/3946 to get the
- // workspace root.
- let display_root = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap();
- for file in files {
- println!("cargo:rerun-if-changed={}", file.display());
- let display_path = file.strip_prefix(display_root).unwrap();
- let display_path_str = display_path.display().to_string();
- js_runtime
- .execute_script(
- &("deno:".to_string() + &display_path_str.replace('\\', "/")),
- &std::fs::read_to_string(&file).unwrap(),
- )
- .unwrap();
- }
-
- let snapshot = js_runtime.snapshot();
- let snapshot_slice: &[u8] = &*snapshot;
- println!("Snapshot size: {}", snapshot_slice.len());
-
- let compressed_snapshot_with_size = {
- let mut vec = vec![];
-
- vec.extend_from_slice(
- &u32::try_from(snapshot.len())
- .expect("snapshot larger than 4gb")
- .to_le_bytes(),
- );
-
- vec.extend_from_slice(
- &zstd::bulk::compress(snapshot_slice, 22)
- .expect("snapshot compression failed"),
- );
-
- vec
- };
-
- println!(
- "Snapshot compressed size: {}",
- compressed_snapshot_with_size.len()
- );
-
- std::fs::write(&snapshot_path, compressed_snapshot_with_size).unwrap();
- println!("Snapshot written to: {} ", snapshot_path.display());
-}
-
-#[derive(Debug, Deserialize)]
-struct LoadArgs {
- /// The fully qualified specifier that should be loaded.
- specifier: String,
-}
-
-fn create_compiler_snapshot(
- snapshot_path: &Path,
- files: Vec<PathBuf>,
- cwd: &Path,
-) {
- // libs that are being provided by op crates.
- let mut op_crate_libs = HashMap::new();
- op_crate_libs.insert("deno.console", deno_console::get_declaration());
- op_crate_libs.insert("deno.url", deno_url::get_declaration());
- op_crate_libs.insert("deno.web", deno_web::get_declaration());
- op_crate_libs.insert("deno.fetch", deno_fetch::get_declaration());
- op_crate_libs.insert("deno.webgpu", deno_webgpu_get_declaration());
- op_crate_libs.insert("deno.websocket", deno_websocket::get_declaration());
- op_crate_libs.insert("deno.webstorage", deno_webstorage::get_declaration());
- op_crate_libs.insert("deno.crypto", deno_crypto::get_declaration());
- op_crate_libs.insert(
- "deno.broadcast_channel",
- deno_broadcast_channel::get_declaration(),
- );
- op_crate_libs.insert("deno.net", deno_net::get_declaration());
-
- // ensure we invalidate the build properly.
- for (_, path) in op_crate_libs.iter() {
- println!("cargo:rerun-if-changed={}", path.display());
- }
-
- // libs that should be loaded into the isolate before snapshotting.
- let libs = vec![
- // Deno custom type libraries
- "deno.window",
- "deno.worker",
- "deno.shared_globals",
- "deno.ns",
- "deno.unstable",
- // Deno built-in type libraries
- "es5",
- "es2015.collection",
- "es2015.core",
- "es2015",
- "es2015.generator",
- "es2015.iterable",
- "es2015.promise",
- "es2015.proxy",
- "es2015.reflect",
- "es2015.symbol",
- "es2015.symbol.wellknown",
- "es2016.array.include",
- "es2016",
- "es2017",
- "es2017.intl",
- "es2017.object",
- "es2017.sharedmemory",
- "es2017.string",
- "es2017.typedarrays",
- "es2018.asyncgenerator",
- "es2018.asynciterable",
- "es2018",
- "es2018.intl",
- "es2018.promise",
- "es2018.regexp",
- "es2019.array",
- "es2019",
- "es2019.object",
- "es2019.string",
- "es2019.symbol",
- "es2020.bigint",
- "es2020",
- "es2020.date",
- "es2020.intl",
- "es2020.number",
- "es2020.promise",
- "es2020.sharedmemory",
- "es2020.string",
- "es2020.symbol.wellknown",
- "es2021",
- "es2021.intl",
- "es2021.promise",
- "es2021.string",
- "es2021.weakref",
- "es2022",
- "es2022.array",
- "es2022.error",
- "es2022.intl",
- "es2022.object",
- "es2022.string",
- "esnext",
- "esnext.array",
- "esnext.intl",
- ];
-
- let path_dts = cwd.join("dts");
- // ensure we invalidate the build properly.
- for name in libs.iter() {
- println!(
- "cargo:rerun-if-changed={}",
- path_dts.join(format!("lib.{}.d.ts", name)).display()
- );
- }
-
- // create a copy of the vector that includes any op crate libs to be passed
- // to the JavaScript compiler to build into the snapshot
- let mut build_libs = libs.clone();
- for (op_lib, _) in op_crate_libs.iter() {
- build_libs.push(op_lib.to_owned());
- }
-
- #[op]
- fn op_build_info(state: &mut OpState) -> Value {
- let build_specifier = "asset:///bootstrap.ts";
- let build_libs = state.borrow::<Vec<&str>>();
- json!({
- "buildSpecifier": build_specifier,
- "libs": build_libs,
- })
- }
-
- #[op]
- fn op_cwd() -> String {
- "cache:///".into()
- }
-
- #[op]
- fn op_exists() -> bool {
- false
- }
-
- #[op]
- fn op_script_version(
- _state: &mut OpState,
- _args: Value,
- ) -> Result<Option<String>, AnyError> {
- Ok(Some("1".to_string()))
- }
-
- #[op]
- // using the same op that is used in `tsc.rs` for loading modules and reading
- // files, but a slightly different implementation at build time.
- fn op_load(state: &mut OpState, args: LoadArgs) -> Result<Value, AnyError> {
- let op_crate_libs = state.borrow::<HashMap<&str, PathBuf>>();
- let path_dts = state.borrow::<PathBuf>();
- let re_asset =
- Regex::new(r"asset:/{3}lib\.(\S+)\.d\.ts").expect("bad regex");
- let build_specifier = "asset:///bootstrap.ts";
-
- // we need a basic file to send to tsc to warm it up.
- if args.specifier == build_specifier {
- Ok(json!({
- "data": r#"console.log("hello deno!");"#,
- "version": "1",
- // this corresponds to `ts.ScriptKind.TypeScript`
- "scriptKind": 3
- }))
- // specifiers come across as `asset:///lib.{lib_name}.d.ts` and we need to
- // parse out just the name so we can lookup the asset.
- } else if let Some(caps) = re_asset.captures(&args.specifier) {
- if let Some(lib) = caps.get(1).map(|m| m.as_str()) {
- // if it comes from an op crate, we were supplied with the path to the
- // file.
- let path = if let Some(op_crate_lib) = op_crate_libs.get(lib) {
- PathBuf::from(op_crate_lib).canonicalize().unwrap()
- // otherwise we are will generate the path ourself
- } else {
- path_dts.join(format!("lib.{}.d.ts", lib))
- };
- let data = std::fs::read_to_string(path)?;
- Ok(json!({
- "data": data,
- "version": "1",
- // this corresponds to `ts.ScriptKind.TypeScript`
- "scriptKind": 3
- }))
- } else {
- Err(custom_error(
- "InvalidSpecifier",
- format!("An invalid specifier was requested: {}", args.specifier),
- ))
- }
- } else {
- Err(custom_error(
- "InvalidSpecifier",
- format!("An invalid specifier was requested: {}", args.specifier),
- ))
- }
- }
- let js_runtime = JsRuntime::new(RuntimeOptions {
- will_snapshot: true,
- extensions: vec![Extension::builder()
- .ops(vec![
- op_build_info::decl(),
- op_cwd::decl(),
- op_exists::decl(),
- op_load::decl(),
- op_script_version::decl(),
- ])
- .state(move |state| {
- state.put(op_crate_libs.clone());
- state.put(build_libs.clone());
- state.put(path_dts.clone());
-
- Ok(())
- })
- .build()],
- ..Default::default()
- });
-
- create_snapshot(js_runtime, snapshot_path, files);
-}
-
-fn ts_version() -> String {
- std::fs::read_to_string("tsc/00_typescript.js")
- .unwrap()
- .lines()
- .find(|l| l.contains("ts.version = "))
- .expect(
- "Failed to find the pattern `ts.version = ` in typescript source code",
- )
- .chars()
- .skip_while(|c| !char::is_numeric(*c))
- .take_while(|c| *c != '"')
- .collect::<String>()
-}
-
-fn git_commit_hash() -> String {
- if let Ok(output) = std::process::Command::new("git")
- .arg("rev-list")
- .arg("-1")
- .arg("HEAD")
- .output()
- {
- if output.status.success() {
- std::str::from_utf8(&output.stdout[..40])
- .unwrap()
- .to_string()
- } else {
- // When not in git repository
- // (e.g. when the user install by `cargo install deno`)
- "UNKNOWN".to_string()
- }
- } else {
- // When there is no git command for some reason
- "UNKNOWN".to_string()
- }
-}
-
fn main() {
// Skip building from docs.rs.
if env::var_os("DOCS_RS").is_some() {
@@ -330,8 +26,9 @@ fn main() {
if target != host {
panic!("Cross compiling with snapshot is not supported.");
}
- // To debug snapshot issues uncomment:
- // op_fetch_asset::trace_serializer();
+
+ println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap());
+ println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap());
if let Ok(c) = env::var("DENO_CANARY") {
println!("cargo:rustc-env=DENO_CANARY={}", c);
@@ -385,18 +82,6 @@ fn main() {
deno_net::get_declaration().display()
);
- println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap());
- println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap());
-
- let c = PathBuf::from(env::var_os("CARGO_MANIFEST_DIR").unwrap());
- let o = PathBuf::from(env::var_os("OUT_DIR").unwrap());
-
- // Main snapshot
- let compiler_snapshot_path = o.join("COMPILER_SNAPSHOT.bin");
-
- let js_files = get_js_files("tsc");
- create_compiler_snapshot(&compiler_snapshot_path, js_files, &c);
-
#[cfg(target_os = "windows")]
{
let mut res = winres::WindowsResource::new();
@@ -414,16 +99,38 @@ fn deno_webgpu_get_declaration() -> PathBuf {
manifest_dir.join("dts").join("lib.deno_webgpu.d.ts")
}
-fn get_js_files(d: &str) -> Vec<PathBuf> {
- let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR"));
- let mut js_files = std::fs::read_dir(d)
+fn git_commit_hash() -> String {
+ if let Ok(output) = std::process::Command::new("git")
+ .arg("rev-list")
+ .arg("-1")
+ .arg("HEAD")
+ .output()
+ {
+ if output.status.success() {
+ std::str::from_utf8(&output.stdout[..40])
+ .unwrap()
+ .to_string()
+ } else {
+ // When not in git repository
+ // (e.g. when the user install by `cargo install deno`)
+ "UNKNOWN".to_string()
+ }
+ } else {
+ // When there is no git command for some reason
+ "UNKNOWN".to_string()
+ }
+}
+
+fn ts_version() -> String {
+ std::fs::read_to_string("tsc/00_typescript.js")
.unwrap()
- .map(|dir_entry| {
- let file = dir_entry.unwrap();
- manifest_dir.join(file.path())
- })
- .filter(|path| path.extension().unwrap_or_default() == "js")
- .collect::<Vec<PathBuf>>();
- js_files.sort();
- js_files
+ .lines()
+ .find(|l| l.contains("ts.version = "))
+ .expect(
+ "Failed to find the pattern `ts.version = ` in typescript source code",
+ )
+ .chars()
+ .skip_while(|c| !char::is_numeric(*c))
+ .take_while(|c| *c != '"')
+ .collect::<String>()
}
diff --git a/cli/lsp/tsc.rs b/cli/lsp/tsc.rs
index 6ff4cba00..2a5bdc2d8 100644
--- a/cli/lsp/tsc.rs
+++ b/cli/lsp/tsc.rs
@@ -2618,7 +2618,7 @@ fn op_script_version(
fn js_runtime(performance: Arc<Performance>) -> JsRuntime {
JsRuntime::new(RuntimeOptions {
extensions: vec![init_extension(performance)],
- startup_snapshot: Some(tsc::compiler_snapshot()),
+ startup_snapshot: Some(deno_snapshots::tsc_snapshot()),
..Default::default()
})
}
diff --git a/cli/main.rs b/cli/main.rs
index 471afdffd..01dbba848 100644
--- a/cli/main.rs
+++ b/cli/main.rs
@@ -179,6 +179,7 @@ fn create_web_worker_callback(
shared_array_buffer_store: Some(ps.shared_array_buffer_store.clone()),
compiled_wasm_module_store: Some(ps.compiled_wasm_module_store.clone()),
stdio: stdio.clone(),
+ startup_snapshot: Some(deno_snapshots::cli_snapshot()),
};
WebWorker::bootstrap_from_options(
@@ -274,6 +275,7 @@ pub fn create_main_worker(
shared_array_buffer_store: Some(ps.shared_array_buffer_store.clone()),
compiled_wasm_module_store: Some(ps.compiled_wasm_module_store.clone()),
stdio,
+ startup_snapshot: Some(deno_snapshots::cli_snapshot()),
};
MainWorker::bootstrap_from_options(main_module, permissions, options)
diff --git a/cli/standalone.rs b/cli/standalone.rs
index f19329cbd..1e8429db0 100644
--- a/cli/standalone.rs
+++ b/cli/standalone.rs
@@ -307,6 +307,7 @@ pub async fn run(
shared_array_buffer_store: None,
compiled_wasm_module_store: None,
stdio: Default::default(),
+ startup_snapshot: Some(deno_snapshots::cli_snapshot()),
};
let mut worker = MainWorker::bootstrap_from_options(
main_module.clone(),
diff --git a/cli/tsc.rs b/cli/tsc.rs
index a61d7871d..b293ea3b2 100644
--- a/cli/tsc.rs
+++ b/cli/tsc.rs
@@ -24,7 +24,6 @@ use deno_core::JsRuntime;
use deno_core::ModuleSpecifier;
use deno_core::OpState;
use deno_core::RuntimeOptions;
-use deno_core::Snapshot;
use deno_graph::Resolved;
use once_cell::sync::Lazy;
use std::collections::HashMap;
@@ -52,27 +51,6 @@ pub static SHARED_GLOBALS_LIB: &str =
pub static WINDOW_LIB: &str = include_str!("dts/lib.deno.window.d.ts");
pub static UNSTABLE_NS_LIB: &str = include_str!("dts/lib.deno.unstable.d.ts");
-pub static COMPILER_SNAPSHOT: Lazy<Box<[u8]>> = Lazy::new(
- #[cold]
- #[inline(never)]
- || {
- static COMPRESSED_COMPILER_SNAPSHOT: &[u8] =
- include_bytes!(concat!(env!("OUT_DIR"), "/COMPILER_SNAPSHOT.bin"));
-
- zstd::bulk::decompress(
- &COMPRESSED_COMPILER_SNAPSHOT[4..],
- u32::from_le_bytes(COMPRESSED_COMPILER_SNAPSHOT[0..4].try_into().unwrap())
- as usize,
- )
- .unwrap()
- .into_boxed_slice()
- },
-);
-
-pub fn compiler_snapshot() -> Snapshot {
- Snapshot::Static(&*COMPILER_SNAPSHOT)
-}
-
macro_rules! inc {
($e:expr) => {
include_str!(concat!("dts/", $e))
@@ -657,7 +635,7 @@ pub fn exec(request: Request) -> Result<Response, AnyError> {
})
.collect();
let mut runtime = JsRuntime::new(RuntimeOptions {
- startup_snapshot: Some(compiler_snapshot()),
+ startup_snapshot: Some(deno_snapshots::tsc_snapshot()),
extensions: vec![Extension::builder()
.ops(vec![
op_cwd::decl(),
@@ -841,9 +819,9 @@ mod tests {
}
#[test]
- fn test_compiler_snapshot() {
+ fn test_tsc_snapshot() {
let mut js_runtime = deno_core::JsRuntime::new(deno_core::RuntimeOptions {
- startup_snapshot: Some(compiler_snapshot()),
+ startup_snapshot: Some(deno_snapshots::tsc_snapshot()),
..Default::default()
});
js_runtime
diff --git a/runtime/Cargo.toml b/runtime/Cargo.toml
index 7cb7b7820..652b2342d 100644
--- a/runtime/Cargo.toml
+++ b/runtime/Cargo.toml
@@ -21,25 +21,6 @@ path = "lib.rs"
name = "hello_runtime"
path = "examples/hello_runtime.rs"
-[build-dependencies]
-deno_broadcast_channel = { version = "0.52.0", path = "../ext/broadcast_channel" }
-deno_console = { version = "0.58.0", path = "../ext/console" }
-deno_core = { version = "0.140.0", path = "../core" }
-deno_crypto = { version = "0.72.0", path = "../ext/crypto" }
-deno_fetch = { version = "0.81.0", path = "../ext/fetch" }
-deno_ffi = { version = "0.45.0", path = "../ext/ffi" }
-deno_http = { version = "0.52.0", path = "../ext/http" }
-deno_net = { version = "0.50.0", path = "../ext/net" }
-deno_tls = { version = "0.45.0", path = "../ext/tls" }
-deno_url = { version = "0.58.0", path = "../ext/url" }
-deno_web = { version = "0.89.0", path = "../ext/web" }
-deno_webgpu = { version = "0.59.0", path = "../ext/webgpu" }
-deno_webidl = { version = "0.58.0", path = "../ext/webidl" }
-deno_websocket = { version = "0.63.0", path = "../ext/websocket" }
-deno_webstorage = { version = "0.53.0", path = "../ext/webstorage" }
-
-lzzzz = '1.0'
-
[target.'cfg(windows)'.build-dependencies]
winres = "0.1.11"
winapi = "0.3.9"
diff --git a/runtime/build.rs b/runtime/build.rs
index eea7a3602..0cb113af1 100644
--- a/runtime/build.rs
+++ b/runtime/build.rs
@@ -1,219 +1,17 @@
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
-use std::env;
-use std::path::Path;
-use std::path::PathBuf;
-
-// This is a shim that allows to generate documentation on docs.rs
-#[cfg(not(feature = "docsrs"))]
-mod not_docs {
- use super::*;
- use deno_core::Extension;
- use deno_core::JsRuntime;
- use deno_core::RuntimeOptions;
-
- // TODO(bartlomieju): this module contains a lot of duplicated
- // logic with `cli/build.rs`, factor out to `deno_core`.
- fn create_snapshot(
- mut js_runtime: JsRuntime,
- snapshot_path: &Path,
- files: Vec<PathBuf>,
- ) {
- // TODO(nayeemrmn): https://github.com/rust-lang/cargo/issues/3946 to get the
- // workspace root.
- let display_root = Path::new(env!("CARGO_MANIFEST_DIR")).parent().unwrap();
- for file in files {
- println!("cargo:rerun-if-changed={}", file.display());
- let display_path = file.strip_prefix(display_root).unwrap();
- let display_path_str = display_path.display().to_string();
- js_runtime
- .execute_script(
- &("deno:".to_string() + &display_path_str.replace('\\', "/")),
- &std::fs::read_to_string(&file).unwrap(),
- )
- .unwrap();
- }
-
- let snapshot = js_runtime.snapshot();
- let snapshot_slice: &[u8] = &*snapshot;
- println!("Snapshot size: {}", snapshot_slice.len());
-
- let compressed_snapshot_with_size = {
- let mut vec = vec![];
-
- vec.extend_from_slice(
- &u32::try_from(snapshot.len())
- .expect("snapshot larger than 4gb")
- .to_le_bytes(),
- );
-
- lzzzz::lz4_hc::compress_to_vec(
- snapshot_slice,
- &mut vec,
- lzzzz::lz4_hc::CLEVEL_MAX,
- )
- .expect("snapshot compression failed");
-
- vec
- };
-
- println!(
- "Snapshot compressed size: {}",
- compressed_snapshot_with_size.len()
- );
-
- std::fs::write(&snapshot_path, compressed_snapshot_with_size).unwrap();
- println!("Snapshot written to: {} ", snapshot_path.display());
- }
-
- struct Permissions;
-
- impl deno_fetch::FetchPermissions for Permissions {
- fn check_net_url(
- &mut self,
- _url: &deno_core::url::Url,
- ) -> Result<(), deno_core::error::AnyError> {
- unreachable!("snapshotting!")
- }
-
- fn check_read(
- &mut self,
- _p: &Path,
- ) -> Result<(), deno_core::error::AnyError> {
- unreachable!("snapshotting!")
- }
- }
-
- impl deno_websocket::WebSocketPermissions for Permissions {
- fn check_net_url(
- &mut self,
- _url: &deno_core::url::Url,
- ) -> Result<(), deno_core::error::AnyError> {
- unreachable!("snapshotting!")
- }
- }
-
- impl deno_web::TimersPermission for Permissions {
- fn allow_hrtime(&mut self) -> bool {
- unreachable!("snapshotting!")
- }
-
- fn check_unstable(
- &self,
- _state: &deno_core::OpState,
- _api_name: &'static str,
- ) {
- unreachable!("snapshotting!")
- }
- }
-
- impl deno_ffi::FfiPermissions for Permissions {
- fn check(
- &mut self,
- _path: Option<&Path>,
- ) -> Result<(), deno_core::error::AnyError> {
- unreachable!("snapshotting!")
- }
- }
-
- impl deno_net::NetPermissions for Permissions {
- fn check_net<T: AsRef<str>>(
- &mut self,
- _host: &(T, Option<u16>),
- ) -> Result<(), deno_core::error::AnyError> {
- unreachable!("snapshotting!")
- }
-
- fn check_read(
- &mut self,
- _p: &Path,
- ) -> Result<(), deno_core::error::AnyError> {
- unreachable!("snapshotting!")
- }
-
- fn check_write(
- &mut self,
- _p: &Path,
- ) -> Result<(), deno_core::error::AnyError> {
- unreachable!("snapshotting!")
- }
- }
-
- fn create_runtime_snapshot(snapshot_path: &Path, files: Vec<PathBuf>) {
- let extensions: Vec<Extension> = vec![
- deno_webidl::init(),
- deno_console::init(),
- deno_url::init(),
- deno_tls::init(),
- deno_web::init::<Permissions>(
- deno_web::BlobStore::default(),
- Default::default(),
- ),
- deno_fetch::init::<Permissions>(Default::default()),
- deno_websocket::init::<Permissions>("".to_owned(), None, None),
- deno_webstorage::init(None),
- deno_crypto::init(None),
- deno_webgpu::init(false),
- deno_broadcast_channel::init(
- deno_broadcast_channel::InMemoryBroadcastChannel::default(),
- false, // No --unstable.
- ),
- deno_ffi::init::<Permissions>(false),
- deno_net::init::<Permissions>(
- None, false, // No --unstable.
- None,
- ),
- deno_http::init(),
- ];
-
- let js_runtime = JsRuntime::new(RuntimeOptions {
- will_snapshot: true,
- extensions,
- ..Default::default()
- });
- create_snapshot(js_runtime, snapshot_path, files);
- }
-
- fn get_js_files(d: &str) -> Vec<PathBuf> {
- let manifest_dir = Path::new(env!("CARGO_MANIFEST_DIR"));
- let mut js_files = std::fs::read_dir(d)
- .unwrap()
- .map(|dir_entry| {
- let file = dir_entry.unwrap();
- manifest_dir.join(file.path())
- })
- .filter(|path| path.extension().unwrap_or_default() == "js")
- .collect::<Vec<PathBuf>>();
- js_files.sort();
- js_files
- }
-
- pub fn build_snapshot(runtime_snapshot_path: PathBuf) {
- let js_files = get_js_files("js");
- create_runtime_snapshot(&runtime_snapshot_path, js_files);
- }
-}
-
fn main() {
- // To debug snapshot issues uncomment:
- // op_fetch_asset::trace_serializer();
-
- println!("cargo:rustc-env=TARGET={}", env::var("TARGET").unwrap());
- println!("cargo:rustc-env=PROFILE={}", env::var("PROFILE").unwrap());
- let o = PathBuf::from(env::var_os("OUT_DIR").unwrap());
-
- // Main snapshot
- let runtime_snapshot_path = o.join("CLI_SNAPSHOT.bin");
-
- // If we're building on docs.rs we just create
- // and empty snapshot file and return, because `rusty_v8`
- // doesn't actually compile on docs.rs
- if env::var_os("DOCS_RS").is_some() {
- let snapshot_slice = &[];
- std::fs::write(&runtime_snapshot_path, snapshot_slice).unwrap();
+ // Skip building from docs.rs.
+ if std::env::var_os("DOCS_RS").is_some() {
return;
}
- #[cfg(not(feature = "docsrs"))]
- not_docs::build_snapshot(runtime_snapshot_path)
+ println!(
+ "cargo:rustc-env=TARGET={}",
+ std::env::var("TARGET").unwrap()
+ );
+ println!(
+ "cargo:rustc-env=PROFILE={}",
+ std::env::var("PROFILE").unwrap()
+ );
}
diff --git a/runtime/examples/hello_runtime.rs b/runtime/examples/hello_runtime.rs
index 07e42f0ff..19f462d4f 100644
--- a/runtime/examples/hello_runtime.rs
+++ b/runtime/examples/hello_runtime.rs
@@ -58,6 +58,7 @@ async fn main() -> Result<(), AnyError> {
shared_array_buffer_store: None,
compiled_wasm_module_store: None,
stdio: Default::default(),
+ startup_snapshot: None,
};
let js_path =
diff --git a/runtime/js.rs b/runtime/js.rs
index cdd479858..a18dc3bd3 100644
--- a/runtime/js.rs
+++ b/runtime/js.rs
@@ -1,58 +1,38 @@
// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
-use deno_core::Snapshot;
-use log::debug;
-use once_cell::sync::Lazy;
-
-pub static CLI_SNAPSHOT: Lazy<Box<[u8]>> = Lazy::new(
- #[allow(clippy::uninit_vec)]
- #[cold]
- #[inline(never)]
- || {
- static COMPRESSED_CLI_SNAPSHOT: &[u8] =
- include_bytes!(concat!(env!("OUT_DIR"), "/CLI_SNAPSHOT.bin"));
-
- let size =
- u32::from_le_bytes(COMPRESSED_CLI_SNAPSHOT[0..4].try_into().unwrap())
- as usize;
- let mut vec = Vec::with_capacity(size);
-
- // SAFETY: vec is allocated with exact snapshot size (+ alignment)
- // SAFETY: non zeroed bytes are overwritten with decompressed snapshot
- unsafe {
- vec.set_len(size);
- }
-
- lzzzz::lz4::decompress(&COMPRESSED_CLI_SNAPSHOT[4..], &mut vec).unwrap();
-
- vec.into_boxed_slice()
- },
-);
-
-pub fn deno_isolate_init() -> Snapshot {
- debug!("Deno isolate init with snapshots.");
- Snapshot::Static(&*CLI_SNAPSHOT)
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn cli_snapshot() {
- let mut js_runtime = deno_core::JsRuntime::new(deno_core::RuntimeOptions {
- startup_snapshot: Some(deno_isolate_init()),
- ..Default::default()
- });
- js_runtime
- .execute_script(
- "<anon>",
- r#"
- if (!(bootstrap.mainRuntime && bootstrap.workerRuntime)) {
- throw Error("bad");
- }
- console.log("we have console.log!!!");
- "#,
- )
- .unwrap();
- }
+use deno_core::include_js_files;
+use deno_core::Extension;
+
+pub fn init() -> Extension {
+ Extension::builder()
+ .js(include_js_files!(
+ prefix "deno:runtime",
+ // Generated with:
+ // bash -c "cd runtime && ls js/*.js | sort"
+ "js/01_build.js",
+ "js/01_errors.js",
+ "js/01_version.js",
+ "js/01_web_util.js",
+ "js/06_util.js",
+ "js/10_permissions.js",
+ "js/11_workers.js",
+ "js/12_io.js",
+ "js/13_buffer.js",
+ "js/30_fs.js",
+ "js/30_os.js",
+ "js/40_diagnostics.js",
+ "js/40_files.js",
+ "js/40_fs_events.js",
+ "js/40_http.js",
+ "js/40_process.js",
+ "js/40_read_file.js",
+ "js/40_signals.js",
+ "js/40_spawn.js",
+ "js/40_testing.js",
+ "js/40_tty.js",
+ "js/40_write_file.js",
+ "js/41_prompt.js",
+ "js/90_deno_ns.js",
+ "js/99_main.js",
+ ))
+ .build()
}
diff --git a/runtime/web_worker.rs b/runtime/web_worker.rs
index ba2c016cc..19e344ee7 100644
--- a/runtime/web_worker.rs
+++ b/runtime/web_worker.rs
@@ -335,6 +335,7 @@ pub struct WebWorkerOptions {
pub shared_array_buffer_store: Option<SharedArrayBufferStore>,
pub compiled_wasm_module_store: Option<CompiledWasmModuleStore>,
pub stdio: Stdio,
+ pub startup_snapshot: Option<deno_core::Snapshot>,
}
impl WebWorker {
@@ -427,6 +428,8 @@ impl WebWorker {
ops::tty::init(),
deno_http::init(),
ops::http::init(),
+ // Runtime JS
+ js::init(),
// Permissions ext (worker specific state)
perm_ext,
];
@@ -436,7 +439,7 @@ impl WebWorker {
let mut js_runtime = JsRuntime::new(RuntimeOptions {
module_loader: Some(options.module_loader.clone()),
- startup_snapshot: Some(js::deno_isolate_init()),
+ startup_snapshot: options.startup_snapshot.take(),
source_map_getter: options.source_map_getter,
get_error_class_fn: options.get_error_class_fn,
shared_array_buffer_store: options.shared_array_buffer_store.clone(),
diff --git a/runtime/worker.rs b/runtime/worker.rs
index e46b5c75f..acb50dc30 100644
--- a/runtime/worker.rs
+++ b/runtime/worker.rs
@@ -81,6 +81,7 @@ pub struct WorkerOptions {
pub shared_array_buffer_store: Option<SharedArrayBufferStore>,
pub compiled_wasm_module_store: Option<CompiledWasmModuleStore>,
pub stdio: Stdio,
+ pub startup_snapshot: Option<deno_core::Snapshot>,
}
impl MainWorker {
@@ -168,6 +169,8 @@ impl MainWorker {
ops::tty::init(),
deno_http::init(),
ops::http::init(),
+ // Runtime JS
+ js::init(),
// Permissions ext (worker specific state)
perm_ext,
];
@@ -175,7 +178,7 @@ impl MainWorker {
let mut js_runtime = JsRuntime::new(RuntimeOptions {
module_loader: Some(options.module_loader.clone()),
- startup_snapshot: Some(js::deno_isolate_init()),
+ startup_snapshot: options.startup_snapshot.take(),
source_map_getter: options.source_map_getter,
get_error_class_fn: options.get_error_class_fn,
shared_array_buffer_store: options.shared_array_buffer_store.clone(),
@@ -411,6 +414,7 @@ mod tests {
shared_array_buffer_store: None,
compiled_wasm_module_store: None,
stdio: Default::default(),
+ startup_snapshot: None,
};
MainWorker::bootstrap_from_options(main_module, permissions, options)
diff --git a/snapshots/Cargo.toml b/snapshots/Cargo.toml
new file mode 100644
index 000000000..6266ed681
--- /dev/null
+++ b/snapshots/Cargo.toml
@@ -0,0 +1,32 @@
+# Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
+
+# IMPORTANT(bartlomieju): this crate is internal and shouldn't be published
+# to crates.io
+
+[package]
+name = "deno_snapshots"
+version = "0.0.0"
+authors = ["the Deno authors"]
+edition = "2018"
+license = "MIT"
+repository = "https://github.com/denoland/deno"
+description = "Provides snapshots of TSC & Deno (runtime+web+core)"
+
+[lib]
+name = "deno_snapshots"
+path = "lib.rs"
+
+[dependencies]
+deno_core = { version = "0.140.0", path = "../core" } # For mock TSC #[op]s
+deno_runtime = { version = "0.66.0", path = "../runtime" }
+lzzzz = "1.0"
+once_cell = "1.10.0"
+zstd = "0.11.1"
+
+[build-dependencies]
+deno_core = { version = "0.140.0", path = "../core" } # For mock TSC #[op]s
+deno_runtime = { version = "0.66.0", path = "../runtime" }
+lzzzz = "1.0"
+regex = "1.5.6"
+serde = { version = "1.0.125", features = ["derive"] }
+zstd = "0.11.1"
diff --git a/snapshots/build.rs b/snapshots/build.rs
new file mode 100644
index 000000000..df868591d
--- /dev/null
+++ b/snapshots/build.rs
@@ -0,0 +1,10 @@
+// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
+
+mod build_runtime;
+mod build_tsc;
+
+fn main() {
+ let out_dir = std::path::PathBuf::from(std::env::var_os("OUT_DIR").unwrap());
+ build_runtime::create_runtime_snapshot(&out_dir.join("CLI_SNAPSHOT.bin"));
+ build_tsc::create_tsc_snapshot(&out_dir.join("COMPILER_SNAPSHOT.bin"));
+}
diff --git a/snapshots/build_runtime.rs b/snapshots/build_runtime.rs
new file mode 100644
index 000000000..ae1c67322
--- /dev/null
+++ b/snapshots/build_runtime.rs
@@ -0,0 +1,168 @@
+// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
+use std::convert::TryFrom;
+use std::path::Path;
+
+use deno_runtime::deno_broadcast_channel;
+use deno_runtime::deno_console;
+use deno_runtime::deno_core;
+use deno_runtime::deno_crypto;
+use deno_runtime::deno_fetch;
+use deno_runtime::deno_ffi;
+use deno_runtime::deno_http;
+use deno_runtime::deno_net;
+use deno_runtime::deno_tls;
+use deno_runtime::deno_url;
+use deno_runtime::deno_web;
+use deno_runtime::deno_webgpu;
+use deno_runtime::deno_webidl;
+use deno_runtime::deno_websocket;
+use deno_runtime::deno_webstorage;
+
+use deno_core::Extension;
+use deno_core::JsRuntime;
+use deno_core::RuntimeOptions;
+
+pub fn create_runtime_snapshot(snapshot_path: &Path) {
+ let extensions: Vec<Extension> = vec![
+ deno_webidl::init(),
+ deno_console::init(),
+ deno_url::init(),
+ deno_tls::init(),
+ deno_web::init::<Permissions>(
+ deno_web::BlobStore::default(),
+ Default::default(),
+ ),
+ deno_fetch::init::<Permissions>(Default::default()),
+ deno_websocket::init::<Permissions>("".to_owned(), None, None),
+ deno_webstorage::init(None),
+ deno_crypto::init(None),
+ deno_webgpu::init(false),
+ deno_broadcast_channel::init(
+ deno_broadcast_channel::InMemoryBroadcastChannel::default(),
+ false, // No --unstable.
+ ),
+ deno_ffi::init::<Permissions>(false),
+ deno_net::init::<Permissions>(
+ None, false, // No --unstable.
+ None,
+ ),
+ deno_http::init(),
+ // Runtime JS
+ deno_runtime::js::init(),
+ ];
+
+ let js_runtime = JsRuntime::new(RuntimeOptions {
+ will_snapshot: true,
+ extensions,
+ ..Default::default()
+ });
+ write_runtime_snapshot(js_runtime, snapshot_path);
+}
+
+// TODO(bartlomieju): this module contains a lot of duplicated
+// logic with `build_tsc.rs`
+fn write_runtime_snapshot(mut js_runtime: JsRuntime, snapshot_path: &Path) {
+ let snapshot = js_runtime.snapshot();
+ let snapshot_slice: &[u8] = &*snapshot;
+ println!("Snapshot size: {}", snapshot_slice.len());
+
+ let compressed_snapshot_with_size = {
+ let mut vec = vec![];
+
+ vec.extend_from_slice(
+ &u32::try_from(snapshot.len())
+ .expect("snapshot larger than 4gb")
+ .to_le_bytes(),
+ );
+
+ lzzzz::lz4_hc::compress_to_vec(
+ snapshot_slice,
+ &mut vec,
+ lzzzz::lz4_hc::CLEVEL_MAX,
+ )
+ .expect("snapshot compression failed");
+
+ vec
+ };
+
+ println!(
+ "Snapshot compressed size: {}",
+ compressed_snapshot_with_size.len()
+ );
+
+ std::fs::write(&snapshot_path, compressed_snapshot_with_size).unwrap();
+ println!("Snapshot written to: {} ", snapshot_path.display());
+}
+
+struct Permissions;
+
+impl deno_fetch::FetchPermissions for Permissions {
+ fn check_net_url(
+ &mut self,
+ _url: &deno_core::url::Url,
+ ) -> Result<(), deno_core::error::AnyError> {
+ unreachable!("snapshotting!")
+ }
+
+ fn check_read(
+ &mut self,
+ _p: &Path,
+ ) -> Result<(), deno_core::error::AnyError> {
+ unreachable!("snapshotting!")
+ }
+}
+
+impl deno_websocket::WebSocketPermissions for Permissions {
+ fn check_net_url(
+ &mut self,
+ _url: &deno_core::url::Url,
+ ) -> Result<(), deno_core::error::AnyError> {
+ unreachable!("snapshotting!")
+ }
+}
+
+impl deno_web::TimersPermission for Permissions {
+ fn allow_hrtime(&mut self) -> bool {
+ unreachable!("snapshotting!")
+ }
+
+ fn check_unstable(
+ &self,
+ _state: &deno_core::OpState,
+ _api_name: &'static str,
+ ) {
+ unreachable!("snapshotting!")
+ }
+}
+
+impl deno_ffi::FfiPermissions for Permissions {
+ fn check(
+ &mut self,
+ _path: Option<&Path>,
+ ) -> Result<(), deno_core::error::AnyError> {
+ unreachable!("snapshotting!")
+ }
+}
+
+impl deno_net::NetPermissions for Permissions {
+ fn check_net<T: AsRef<str>>(
+ &mut self,
+ _host: &(T, Option<u16>),
+ ) -> Result<(), deno_core::error::AnyError> {
+ unreachable!("snapshotting!")
+ }
+
+ fn check_read(
+ &mut self,
+ _p: &Path,
+ ) -> Result<(), deno_core::error::AnyError> {
+ unreachable!("snapshotting!")
+ }
+
+ fn check_write(
+ &mut self,
+ _p: &Path,
+ ) -> Result<(), deno_core::error::AnyError> {
+ unreachable!("snapshotting!")
+ }
+}
diff --git a/snapshots/build_tsc.rs b/snapshots/build_tsc.rs
new file mode 100644
index 000000000..5296ae23b
--- /dev/null
+++ b/snapshots/build_tsc.rs
@@ -0,0 +1,328 @@
+// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license.
+
+use deno_runtime::deno_broadcast_channel;
+use deno_runtime::deno_console;
+use deno_runtime::deno_crypto;
+use deno_runtime::deno_fetch;
+use deno_runtime::deno_net;
+use deno_runtime::deno_url;
+use deno_runtime::deno_web;
+use deno_runtime::deno_websocket;
+use deno_runtime::deno_webstorage;
+
+use deno_runtime::deno_core::error::custom_error;
+use deno_runtime::deno_core::error::AnyError;
+use deno_runtime::deno_core::op;
+use deno_runtime::deno_core::serde::Deserialize;
+use deno_runtime::deno_core::serde_json::json;
+use deno_runtime::deno_core::serde_json::Value;
+use deno_runtime::deno_core::Extension;
+use deno_runtime::deno_core::JsRuntime;
+use deno_runtime::deno_core::OpState;
+use deno_runtime::deno_core::RuntimeOptions;
+
+use regex::Regex;
+use std::collections::HashMap;
+use std::convert::TryFrom;
+use std::env;
+use std::path::Path;
+use std::path::PathBuf;
+
+pub fn create_tsc_snapshot(snapshot_path: &Path) {
+ let mut js_runtime = JsRuntime::new(RuntimeOptions {
+ will_snapshot: true,
+ extensions: vec![tsc_snapshot_init()],
+ ..Default::default()
+ });
+ load_js_files(&mut js_runtime);
+ write_snapshot(js_runtime, snapshot_path);
+}
+
+// TODO(bartlomieju): this module contains a lot of duplicated
+// logic with `build_runtime.rs`
+fn write_snapshot(mut js_runtime: JsRuntime, snapshot_path: &Path) {
+ let snapshot = js_runtime.snapshot();
+ let snapshot_slice: &[u8] = &*snapshot;
+ println!("Snapshot size: {}", snapshot_slice.len());
+
+ let compressed_snapshot_with_size = {
+ let mut vec = vec![];
+
+ vec.extend_from_slice(
+ &u32::try_from(snapshot.len())
+ .expect("snapshot larger than 4gb")
+ .to_le_bytes(),
+ );
+
+ vec.extend_from_slice(
+ &zstd::bulk::compress(snapshot_slice, 22)
+ .expect("snapshot compression failed"),
+ );
+
+ vec
+ };
+
+ println!(
+ "Snapshot compressed size: {}",
+ compressed_snapshot_with_size.len()
+ );
+
+ std::fs::write(&snapshot_path, compressed_snapshot_with_size).unwrap();
+ println!("Snapshot written to: {} ", snapshot_path.display());
+}
+
+#[derive(Debug, Deserialize)]
+struct LoadArgs {
+ /// The fully qualified specifier that should be loaded.
+ specifier: String,
+}
+
+fn tsc_snapshot_init() -> Extension {
+ // libs that are being provided by op crates.
+ let mut op_crate_libs = HashMap::new();
+ op_crate_libs.insert("deno.console", deno_console::get_declaration());
+ op_crate_libs.insert("deno.url", deno_url::get_declaration());
+ op_crate_libs.insert("deno.web", deno_web::get_declaration());
+ op_crate_libs.insert("deno.fetch", deno_fetch::get_declaration());
+ op_crate_libs.insert("deno.webgpu", deno_webgpu_get_declaration());
+ op_crate_libs.insert("deno.websocket", deno_websocket::get_declaration());
+ op_crate_libs.insert("deno.webstorage", deno_webstorage::get_declaration());
+ op_crate_libs.insert("deno.crypto", deno_crypto::get_declaration());
+ op_crate_libs.insert(
+ "deno.broadcast_channel",
+ deno_broadcast_channel::get_declaration(),
+ );
+ op_crate_libs.insert("deno.net", deno_net::get_declaration());
+
+ // ensure we invalidate the build properly.
+ for (_, path) in op_crate_libs.iter() {
+ println!("cargo:rerun-if-changed={}", path.display());
+ }
+
+ // libs that should be loaded into the isolate before snapshotting.
+ let libs = vec![
+ // Deno custom type libraries
+ "deno.window",
+ "deno.worker",
+ "deno.shared_globals",
+ "deno.ns",
+ "deno.unstable",
+ // Deno built-in type libraries
+ "es5",
+ "es2015.collection",
+ "es2015.core",
+ "es2015",
+ "es2015.generator",
+ "es2015.iterable",
+ "es2015.promise",
+ "es2015.proxy",
+ "es2015.reflect",
+ "es2015.symbol",
+ "es2015.symbol.wellknown",
+ "es2016.array.include",
+ "es2016",
+ "es2017",
+ "es2017.intl",
+ "es2017.object",
+ "es2017.sharedmemory",
+ "es2017.string",
+ "es2017.typedarrays",
+ "es2018.asyncgenerator",
+ "es2018.asynciterable",
+ "es2018",
+ "es2018.intl",
+ "es2018.promise",
+ "es2018.regexp",
+ "es2019.array",
+ "es2019",
+ "es2019.object",
+ "es2019.string",
+ "es2019.symbol",
+ "es2020.bigint",
+ "es2020",
+ "es2020.date",
+ "es2020.intl",
+ "es2020.number",
+ "es2020.promise",
+ "es2020.sharedmemory",
+ "es2020.string",
+ "es2020.symbol.wellknown",
+ "es2021",
+ "es2021.intl",
+ "es2021.promise",
+ "es2021.string",
+ "es2021.weakref",
+ "es2022",
+ "es2022.array",
+ "es2022.error",
+ "es2022.intl",
+ "es2022.object",
+ "es2022.string",
+ "esnext",
+ "esnext.array",
+ "esnext.intl",
+ ];
+
+ let cli_dir = cli_dir();
+ let path_dts = cli_dir.join("dts");
+ // ensure we invalidate the build properly.
+ for name in libs.iter() {
+ println!(
+ "cargo:rerun-if-changed={}",
+ path_dts.join(format!("lib.{}.d.ts", name)).display()
+ );
+ }
+
+ // create a copy of the vector that includes any op crate libs to be passed
+ // to the JavaScript compiler to build into the snapshot
+ let mut build_libs = libs.clone();
+ for (op_lib, _) in op_crate_libs.iter() {
+ build_libs.push(op_lib.to_owned());
+ }
+
+ #[op]
+ fn op_build_info(state: &mut OpState) -> Value {
+ let build_specifier = "asset:///bootstrap.ts";
+ let build_libs = state.borrow::<Vec<&str>>();
+ json!({
+ "buildSpecifier": build_specifier,
+ "libs": build_libs,
+ })
+ }
+
+ #[op]
+ fn op_cwd() -> String {
+ "cache:///".into()
+ }
+
+ #[op]
+ fn op_exists() -> bool {
+ false
+ }
+
+ #[op]
+ fn op_script_version(
+ _state: &mut OpState,
+ _args: Value,
+ ) -> Result<Option<String>, AnyError> {
+ Ok(Some("1".to_string()))
+ }
+
+ #[op]
+ // using the same op that is used in `tsc.rs` for loading modules and reading
+ // files, but a slightly different implementation at build time.
+ fn op_load(state: &mut OpState, args: LoadArgs) -> Result<Value, AnyError> {
+ let op_crate_libs = state.borrow::<HashMap<&str, PathBuf>>();
+ let path_dts = state.borrow::<PathBuf>();
+ let re_asset =
+ Regex::new(r"asset:/{3}lib\.(\S+)\.d\.ts").expect("bad regex");
+ let build_specifier = "asset:///bootstrap.ts";
+
+ // we need a basic file to send to tsc to warm it up.
+ if args.specifier == build_specifier {
+ Ok(json!({
+ "data": r#"console.log("hello deno!");"#,
+ "version": "1",
+ // this corresponds to `ts.ScriptKind.TypeScript`
+ "scriptKind": 3
+ }))
+ // specifiers come across as `asset:///lib.{lib_name}.d.ts` and we need to
+ // parse out just the name so we can lookup the asset.
+ } else if let Some(caps) = re_asset.captures(&args.specifier) {
+ if let Some(lib) = caps.get(1).map(|m| m.as_str()) {
+ // if it comes from an op crate, we were supplied with the path to the
+ // file.
+ let path = if let Some(op_crate_lib) = op_crate_libs.get(lib) {
+ PathBuf::from(op_crate_lib).canonicalize().unwrap()
+ // otherwise we are will generate the path ourself
+ } else {
+ path_dts.join(format!("lib.{}.d.ts", lib))
+ };
+ let data = std::fs::read_to_string(path)?;
+ Ok(json!({
+ "data": data,
+ "version": "1",
+ // this corresponds to `ts.ScriptKind.TypeScript`
+ "scriptKind": 3
+ }))
+ } else {
+ Err(custom_error(
+ "InvalidSpecifier",
+ format!("An invalid specifier was requested: {}", args.specifier),
+ ))
+ }
+ } else {
+ Err(custom_error(
+ "InvalidSpecifier",
+ format!("An invalid specifier was requested: {}", args.specifier),
+ ))
+ }
+ }
+
+ Extension::builder()
+ .ops(vec![
+ op_build_info::decl(),
+ op_cwd::decl(),
+ op_exists::decl(),
+ op_load::decl(),
+ op_script_version::decl(),
+ ])
+ .state(move |state| {
+ state.put(op_crate_libs.clone());
+ state.put(build_libs.clone());
+ state.put(path_dts.clone());
+
+ Ok(())
+ })
+ .build()
+}
+
+fn deno_webgpu_get_declaration() -> PathBuf {
+ cli_dir().join("dts").join("lib.deno_webgpu.d.ts")
+}
+
+fn load_js_files(js_runtime: &mut JsRuntime) {
+ let js_files = get_js_files(tsc_dir());
+ let cwd = cli_dir();
+ let display_root = cwd.parent().unwrap();
+ for file in js_files {
+ println!("cargo:rerun-if-changed={}", file.display());
+ let display_path = file.strip_prefix(display_root).unwrap();
+ let display_path_str = display_path.display().to_string();
+ js_runtime
+ .execute_script(
+ &("deno:".to_string() + &display_path_str.replace('\\', "/")),
+ &std::fs::read_to_string(&file).unwrap(),
+ )
+ .unwrap();
+ }
+}
+
+fn root_dir() -> PathBuf {
+ // TODO(nayeemrmn): https://github.com/rust-lang/cargo/issues/3946 to get the workspace root.
+ Path::new(env!("CARGO_MANIFEST_DIR"))
+ .join("..")
+ .canonicalize()
+ .unwrap()
+}
+
+fn cli_dir() -> PathBuf {
+ root_dir().join("cli")
+}
+
+fn tsc_dir() -> PathBuf {
+ cli_dir().join("tsc")
+}
+
+fn get_js_files(dir: PathBuf) -> Vec<PathBuf> {
+ let mut js_files = std::fs::read_dir(dir.clone())
+ .unwrap()
+ .map(|dir_entry| {
+ let file = dir_entry.unwrap();
+ dir.join(file.path())
+ })
+ .filter(|path| path.extension().unwrap_or_default() == "js")
+ .collect::<Vec<PathBuf>>();
+ js_files.sort();
+ js_files
+}
diff --git a/snapshots/lib.rs b/snapshots/lib.rs
new file mode 100644
index 000000000..1d0b3ceb3
--- /dev/null
+++ b/snapshots/lib.rs
@@ -0,0 +1,77 @@
+use deno_core::Snapshot;
+use once_cell::sync::Lazy;
+use std::convert::TryInto;
+
+pub fn tsc_snapshot() -> Snapshot {
+ Snapshot::Static(&*COMPILER_SNAPSHOT)
+}
+
+static COMPILER_SNAPSHOT: Lazy<Box<[u8]>> = Lazy::new(
+ #[cold]
+ #[inline(never)]
+ || {
+ static COMPRESSED_COMPILER_SNAPSHOT: &[u8] =
+ include_bytes!(concat!(env!("OUT_DIR"), "/COMPILER_SNAPSHOT.bin"));
+
+ zstd::bulk::decompress(
+ &COMPRESSED_COMPILER_SNAPSHOT[4..],
+ u32::from_le_bytes(COMPRESSED_COMPILER_SNAPSHOT[0..4].try_into().unwrap())
+ as usize,
+ )
+ .unwrap()
+ .into_boxed_slice()
+ },
+);
+
+pub fn cli_snapshot() -> Snapshot {
+ Snapshot::Static(&*CLI_SNAPSHOT)
+}
+
+static CLI_SNAPSHOT: Lazy<Box<[u8]>> = Lazy::new(
+ #[allow(clippy::uninit_vec)]
+ #[cold]
+ #[inline(never)]
+ || {
+ static COMPRESSED_CLI_SNAPSHOT: &[u8] =
+ include_bytes!(concat!(env!("OUT_DIR"), "/CLI_SNAPSHOT.bin"));
+
+ let size =
+ u32::from_le_bytes(COMPRESSED_CLI_SNAPSHOT[0..4].try_into().unwrap())
+ as usize;
+ let mut vec = Vec::with_capacity(size);
+
+ // SAFETY: vec is allocated with exact snapshot size (+ alignment)
+ // SAFETY: non zeroed bytes are overwritten with decompressed snapshot
+ unsafe {
+ vec.set_len(size);
+ }
+
+ lzzzz::lz4::decompress(&COMPRESSED_CLI_SNAPSHOT[4..], &mut vec).unwrap();
+
+ vec.into_boxed_slice()
+ },
+);
+
+#[cfg(test)]
+mod tests {
+ use deno_runtime::deno_core;
+
+ #[test]
+ fn cli_snapshot() {
+ let mut js_runtime = deno_core::JsRuntime::new(deno_core::RuntimeOptions {
+ startup_snapshot: Some(crate::cli_snapshot()),
+ ..Default::default()
+ });
+ js_runtime
+ .execute_script(
+ "<anon>",
+ r#"
+ if (!(bootstrap.mainRuntime && bootstrap.workerRuntime)) {
+ throw Error("bad");
+ }
+ console.log("we have console.log!!!");
+ "#,
+ )
+ .unwrap();
+ }
+}