diff options
Diffstat (limited to 'cli/standalone')
| -rw-r--r-- | cli/standalone/binary.rs | 307 | ||||
| -rw-r--r-- | cli/standalone/mod.rs | 325 |
2 files changed, 632 insertions, 0 deletions
diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs new file mode 100644 index 000000000..bca0aff2b --- /dev/null +++ b/cli/standalone/binary.rs @@ -0,0 +1,307 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +use std::io::Read; +use std::io::Seek; +use std::io::SeekFrom; +use std::io::Write; +use std::path::Path; +use std::sync::Arc; + +use deno_ast::ModuleSpecifier; +use deno_core::anyhow::Context; +use deno_core::error::AnyError; +use deno_core::futures::io::AllowStdIo; +use deno_core::futures::AsyncReadExt; +use deno_core::futures::AsyncSeekExt; +use deno_core::serde_json; +use deno_core::url::Url; +use deno_runtime::permissions::PermissionsOptions; +use log::Level; +use serde::Deserialize; +use serde::Serialize; + +use crate::args::CaData; +use crate::args::CliOptions; +use crate::args::CompileFlags; +use crate::cache::DenoDir; +use crate::file_fetcher::FileFetcher; +use crate::http_util::HttpClient; +use crate::util::progress_bar::ProgressBar; +use crate::util::progress_bar::ProgressBarStyle; + +const MAGIC_TRAILER: &[u8; 8] = b"d3n0l4nd"; + +#[derive(Deserialize, Serialize)] +pub struct Metadata { + pub argv: Vec<String>, + pub unstable: bool, + pub seed: Option<u64>, + pub permissions: PermissionsOptions, + pub location: Option<Url>, + pub v8_flags: Vec<String>, + pub log_level: Option<Level>, + pub ca_stores: Option<Vec<String>>, + pub ca_data: Option<Vec<u8>>, + pub unsafely_ignore_certificate_errors: Option<Vec<String>>, + pub maybe_import_map: Option<(Url, String)>, + pub entrypoint: ModuleSpecifier, +} + +pub fn write_binary_bytes( + writer: &mut impl Write, + original_bin: Vec<u8>, + metadata: &Metadata, + eszip: eszip::EszipV2, +) -> Result<(), AnyError> { + let metadata = serde_json::to_string(metadata)?.as_bytes().to_vec(); + let eszip_archive = eszip.into_bytes(); + + let eszip_pos = original_bin.len(); + let metadata_pos = eszip_pos + eszip_archive.len(); + let mut trailer = MAGIC_TRAILER.to_vec(); + trailer.write_all(&eszip_pos.to_be_bytes())?; + trailer.write_all(&metadata_pos.to_be_bytes())?; + + writer.write_all(&original_bin)?; + writer.write_all(&eszip_archive)?; + writer.write_all(&metadata)?; + writer.write_all(&trailer)?; + + Ok(()) +} + +pub fn is_standalone_binary(exe_path: &Path) -> bool { + let Ok(mut output_file) = std::fs::File::open(exe_path) else { + return false; + }; + if output_file.seek(SeekFrom::End(-24)).is_err() { + // This seek may fail because the file is too small to possibly be + // `deno compile` output. + return false; + } + let mut trailer = [0; 24]; + if output_file.read_exact(&mut trailer).is_err() { + return false; + }; + let (magic_trailer, _) = trailer.split_at(8); + magic_trailer == MAGIC_TRAILER +} + +/// This function will try to run this binary as a standalone binary +/// produced by `deno compile`. It determines if this is a standalone +/// binary by checking for the magic trailer string `d3n0l4nd` at EOF-24 (8 bytes * 3). +/// The magic trailer is followed by: +/// - a u64 pointer to the JS bundle embedded in the binary +/// - a u64 pointer to JSON metadata (serialized flags) embedded in the binary +/// These are dereferenced, and the bundle is executed under the configuration +/// specified by the metadata. If no magic trailer is present, this function +/// exits with `Ok(None)`. +pub async fn extract_standalone( + exe_path: &Path, + cli_args: Vec<String>, +) -> Result<Option<(Metadata, eszip::EszipV2)>, AnyError> { + let file = std::fs::File::open(exe_path)?; + + let mut bufreader = + deno_core::futures::io::BufReader::new(AllowStdIo::new(file)); + + let trailer_pos = bufreader.seek(SeekFrom::End(-24)).await?; + let mut trailer = [0; 24]; + bufreader.read_exact(&mut trailer).await?; + let (magic_trailer, rest) = trailer.split_at(8); + if magic_trailer != MAGIC_TRAILER { + return Ok(None); + } + + let (eszip_archive_pos, rest) = rest.split_at(8); + let metadata_pos = rest; + let eszip_archive_pos = u64_from_bytes(eszip_archive_pos)?; + let metadata_pos = u64_from_bytes(metadata_pos)?; + let metadata_len = trailer_pos - metadata_pos; + + bufreader.seek(SeekFrom::Start(eszip_archive_pos)).await?; + + let (eszip, loader) = eszip::EszipV2::parse(bufreader) + .await + .context("Failed to parse eszip header")?; + + let mut bufreader = loader.await.context("Failed to parse eszip archive")?; + + bufreader.seek(SeekFrom::Start(metadata_pos)).await?; + + let mut metadata = String::new(); + + bufreader + .take(metadata_len) + .read_to_string(&mut metadata) + .await + .context("Failed to read metadata from the current executable")?; + + let mut metadata: Metadata = serde_json::from_str(&metadata).unwrap(); + metadata.argv.append(&mut cli_args[1..].to_vec()); + + Ok(Some((metadata, eszip))) +} + +fn u64_from_bytes(arr: &[u8]) -> Result<u64, AnyError> { + let fixed_arr: &[u8; 8] = arr + .try_into() + .context("Failed to convert the buffer into a fixed-size array")?; + Ok(u64::from_be_bytes(*fixed_arr)) +} + +pub struct DenoCompileBinaryWriter { + file_fetcher: Arc<FileFetcher>, + client: HttpClient, + deno_dir: DenoDir, +} + +impl DenoCompileBinaryWriter { + pub fn new( + file_fetcher: Arc<FileFetcher>, + client: HttpClient, + deno_dir: DenoDir, + ) -> Self { + Self { + file_fetcher, + client, + deno_dir, + } + } + + pub async fn write_bin( + &self, + writer: &mut impl Write, + eszip: eszip::EszipV2, + module_specifier: &ModuleSpecifier, + compile_flags: &CompileFlags, + cli_options: &CliOptions, + ) -> Result<(), AnyError> { + // Select base binary based on target + let original_binary = + self.get_base_binary(compile_flags.target.clone()).await?; + + self + .write_standalone_binary( + writer, + original_binary, + eszip, + module_specifier, + cli_options, + compile_flags, + ) + .await + } + + async fn get_base_binary( + &self, + target: Option<String>, + ) -> Result<Vec<u8>, AnyError> { + if target.is_none() { + let path = std::env::current_exe()?; + return Ok(std::fs::read(path)?); + } + + let target = target.unwrap_or_else(|| env!("TARGET").to_string()); + let binary_name = format!("deno-{target}.zip"); + + let binary_path_suffix = if crate::version::is_canary() { + format!("canary/{}/{}", crate::version::GIT_COMMIT_HASH, binary_name) + } else { + format!("release/v{}/{}", env!("CARGO_PKG_VERSION"), binary_name) + }; + + let download_directory = self.deno_dir.dl_folder_path(); + let binary_path = download_directory.join(&binary_path_suffix); + + if !binary_path.exists() { + self + .download_base_binary(&download_directory, &binary_path_suffix) + .await?; + } + + let archive_data = std::fs::read(binary_path)?; + let temp_dir = tempfile::TempDir::new()?; + let base_binary_path = crate::tools::upgrade::unpack_into_dir( + archive_data, + target.contains("windows"), + &temp_dir, + )?; + let base_binary = std::fs::read(base_binary_path)?; + drop(temp_dir); // delete the temp dir + Ok(base_binary) + } + + async fn download_base_binary( + &self, + output_directory: &Path, + binary_path_suffix: &str, + ) -> Result<(), AnyError> { + let download_url = format!("https://dl.deno.land/{binary_path_suffix}"); + let maybe_bytes = { + let progress_bars = ProgressBar::new(ProgressBarStyle::DownloadBars); + let progress = progress_bars.update(&download_url); + + self + .client + .download_with_progress(download_url, &progress) + .await? + }; + let bytes = match maybe_bytes { + Some(bytes) => bytes, + None => { + log::info!("Download could not be found, aborting"); + std::process::exit(1) + } + }; + + std::fs::create_dir_all(output_directory)?; + let output_path = output_directory.join(binary_path_suffix); + std::fs::create_dir_all(output_path.parent().unwrap())?; + tokio::fs::write(output_path, bytes).await?; + Ok(()) + } + + /// This functions creates a standalone deno binary by appending a bundle + /// and magic trailer to the currently executing binary. + async fn write_standalone_binary( + &self, + writer: &mut impl Write, + original_bin: Vec<u8>, + eszip: eszip::EszipV2, + entrypoint: &ModuleSpecifier, + cli_options: &CliOptions, + compile_flags: &CompileFlags, + ) -> Result<(), AnyError> { + let ca_data = match cli_options.ca_data() { + Some(CaData::File(ca_file)) => Some( + std::fs::read(ca_file) + .with_context(|| format!("Reading: {ca_file}"))?, + ), + Some(CaData::Bytes(bytes)) => Some(bytes.clone()), + None => None, + }; + let maybe_import_map = cli_options + .resolve_import_map(&self.file_fetcher) + .await? + .map(|import_map| (import_map.base_url().clone(), import_map.to_json())); + let metadata = Metadata { + argv: compile_flags.args.clone(), + unstable: cli_options.unstable(), + seed: cli_options.seed(), + location: cli_options.location_flag().clone(), + permissions: cli_options.permissions_options(), + v8_flags: cli_options.v8_flags().clone(), + unsafely_ignore_certificate_errors: cli_options + .unsafely_ignore_certificate_errors() + .clone(), + log_level: cli_options.log_level(), + ca_stores: cli_options.ca_stores().clone(), + ca_data, + entrypoint: entrypoint.clone(), + maybe_import_map, + }; + + write_binary_bytes(writer, original_bin, &metadata, eszip) + } +} diff --git a/cli/standalone/mod.rs b/cli/standalone/mod.rs new file mode 100644 index 000000000..a2872e9b9 --- /dev/null +++ b/cli/standalone/mod.rs @@ -0,0 +1,325 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +use crate::args::CaData; +use crate::args::Flags; +use crate::colors; +use crate::file_fetcher::get_source_from_data_url; +use crate::ops; +use crate::proc_state::ProcState; +use crate::util::v8::construct_v8_flags; +use crate::version; +use crate::CliGraphResolver; +use deno_core::anyhow::Context; +use deno_core::error::type_error; +use deno_core::error::AnyError; +use deno_core::futures::task::LocalFutureObj; +use deno_core::futures::FutureExt; +use deno_core::located_script_name; +use deno_core::v8_set_flags; +use deno_core::ModuleLoader; +use deno_core::ModuleSpecifier; +use deno_core::ModuleType; +use deno_core::ResolutionKind; +use deno_graph::source::Resolver; +use deno_runtime::fmt_errors::format_js_error; +use deno_runtime::ops::worker_host::CreateWebWorkerCb; +use deno_runtime::ops::worker_host::WorkerEventCb; +use deno_runtime::permissions::Permissions; +use deno_runtime::permissions::PermissionsContainer; +use deno_runtime::web_worker::WebWorker; +use deno_runtime::web_worker::WebWorkerOptions; +use deno_runtime::worker::MainWorker; +use deno_runtime::worker::WorkerOptions; +use deno_runtime::BootstrapOptions; +use import_map::parse_from_json; +use log::Level; +use std::pin::Pin; +use std::rc::Rc; +use std::sync::Arc; + +mod binary; + +pub use binary::extract_standalone; +pub use binary::is_standalone_binary; +pub use binary::DenoCompileBinaryWriter; + +use self::binary::Metadata; + +#[derive(Clone)] +struct EmbeddedModuleLoader { + eszip: Arc<eszip::EszipV2>, + maybe_import_map_resolver: Option<Arc<CliGraphResolver>>, +} + +impl ModuleLoader for EmbeddedModuleLoader { + fn resolve( + &self, + specifier: &str, + referrer: &str, + _kind: ResolutionKind, + ) -> Result<ModuleSpecifier, AnyError> { + // Try to follow redirects when resolving. + let referrer = match self.eszip.get_module(referrer) { + Some(eszip::Module { ref specifier, .. }) => { + ModuleSpecifier::parse(specifier)? + } + None => { + let cwd = std::env::current_dir().context("Unable to get CWD")?; + deno_core::resolve_url_or_path(referrer, &cwd)? + } + }; + + self + .maybe_import_map_resolver + .as_ref() + .map(|r| r.resolve(specifier, &referrer)) + .unwrap_or_else(|| { + deno_core::resolve_import(specifier, referrer.as_str()) + .map_err(|err| err.into()) + }) + } + + fn load( + &self, + module_specifier: &ModuleSpecifier, + _maybe_referrer: Option<&ModuleSpecifier>, + _is_dynamic: bool, + ) -> Pin<Box<deno_core::ModuleSourceFuture>> { + let is_data_uri = get_source_from_data_url(module_specifier).ok(); + let module = self + .eszip + .get_module(module_specifier.as_str()) + .ok_or_else(|| type_error("Module not found")); + // TODO(mmastrac): This clone can probably be removed in the future if ModuleSpecifier is no longer a full-fledged URL + let module_specifier = module_specifier.clone(); + + async move { + if let Some((source, _)) = is_data_uri { + return Ok(deno_core::ModuleSource::new( + deno_core::ModuleType::JavaScript, + source.into(), + &module_specifier, + )); + } + + let module = module?; + let code = module.source().await.unwrap_or_default(); + let code = std::str::from_utf8(&code) + .map_err(|_| type_error("Module source is not utf-8"))? + .to_owned() + .into(); + + Ok(deno_core::ModuleSource::new( + match module.kind { + eszip::ModuleKind::JavaScript => ModuleType::JavaScript, + eszip::ModuleKind::Json => ModuleType::Json, + }, + code, + &module_specifier, + )) + } + .boxed_local() + } +} + +fn metadata_to_flags(metadata: &Metadata) -> Flags { + let permissions = metadata.permissions.clone(); + Flags { + argv: metadata.argv.clone(), + unstable: metadata.unstable, + seed: metadata.seed, + location: metadata.location.clone(), + allow_env: permissions.allow_env, + allow_hrtime: permissions.allow_hrtime, + allow_net: permissions.allow_net, + allow_ffi: permissions.allow_ffi, + allow_read: permissions.allow_read, + allow_run: permissions.allow_run, + allow_write: permissions.allow_write, + v8_flags: metadata.v8_flags.clone(), + log_level: metadata.log_level, + ca_stores: metadata.ca_stores.clone(), + ca_data: metadata.ca_data.clone().map(CaData::Bytes), + ..Default::default() + } +} + +fn web_worker_callback() -> Arc<WorkerEventCb> { + Arc::new(|worker| { + let fut = async move { Ok(worker) }; + LocalFutureObj::new(Box::new(fut)) + }) +} + +fn create_web_worker_callback( + ps: &ProcState, + module_loader: &Rc<EmbeddedModuleLoader>, +) -> Arc<CreateWebWorkerCb> { + let ps = ps.clone(); + let module_loader = module_loader.as_ref().clone(); + Arc::new(move |args| { + let module_loader = Rc::new(module_loader.clone()); + + let create_web_worker_cb = create_web_worker_callback(&ps, &module_loader); + let web_worker_cb = web_worker_callback(); + + let options = WebWorkerOptions { + bootstrap: BootstrapOptions { + args: ps.options.argv().clone(), + cpu_count: std::thread::available_parallelism() + .map(|p| p.get()) + .unwrap_or(1), + debug_flag: ps.options.log_level().map_or(false, |l| l == Level::Debug), + enable_testing_features: false, + locale: deno_core::v8::icu::get_language_tag(), + location: Some(args.main_module.clone()), + no_color: !colors::use_color(), + is_tty: colors::is_tty(), + runtime_version: version::deno().to_string(), + ts_version: version::TYPESCRIPT.to_string(), + unstable: ps.options.unstable(), + user_agent: version::get_user_agent().to_string(), + inspect: ps.options.is_inspecting(), + }, + extensions: ops::cli_exts(ps.npm_resolver.clone()), + startup_snapshot: Some(crate::js::deno_isolate_init()), + unsafely_ignore_certificate_errors: ps + .options + .unsafely_ignore_certificate_errors() + .clone(), + root_cert_store: Some(ps.root_cert_store.clone()), + seed: ps.options.seed(), + module_loader, + npm_resolver: None, // not currently supported + create_web_worker_cb, + preload_module_cb: web_worker_cb.clone(), + pre_execute_module_cb: web_worker_cb, + format_js_error_fn: Some(Arc::new(format_js_error)), + source_map_getter: None, + worker_type: args.worker_type, + maybe_inspector_server: None, + get_error_class_fn: Some(&get_error_class_name), + blob_store: ps.blob_store.clone(), + broadcast_channel: ps.broadcast_channel.clone(), + shared_array_buffer_store: Some(ps.shared_array_buffer_store.clone()), + compiled_wasm_module_store: Some(ps.compiled_wasm_module_store.clone()), + cache_storage_dir: None, + stdio: Default::default(), + }; + + WebWorker::bootstrap_from_options( + args.name, + args.permissions, + args.main_module, + args.worker_id, + options, + ) + }) +} + +pub async fn run( + eszip: eszip::EszipV2, + metadata: Metadata, +) -> Result<(), AnyError> { + let flags = metadata_to_flags(&metadata); + let main_module = &metadata.entrypoint; + let ps = ProcState::from_flags(flags).await?; + let permissions = PermissionsContainer::new(Permissions::from_options( + &metadata.permissions, + )?); + let module_loader = Rc::new(EmbeddedModuleLoader { + eszip: Arc::new(eszip), + maybe_import_map_resolver: metadata.maybe_import_map.map( + |(base, source)| { + Arc::new(CliGraphResolver::new( + None, + Some(Arc::new( + parse_from_json(&base, &source).unwrap().import_map, + )), + false, + ps.npm_api.clone(), + ps.npm_resolution.clone(), + ps.package_json_deps_installer.clone(), + )) + }, + ), + }); + let create_web_worker_cb = create_web_worker_callback(&ps, &module_loader); + let web_worker_cb = web_worker_callback(); + + v8_set_flags(construct_v8_flags(&metadata.v8_flags, vec![])); + + let options = WorkerOptions { + bootstrap: BootstrapOptions { + args: metadata.argv, + cpu_count: std::thread::available_parallelism() + .map(|p| p.get()) + .unwrap_or(1), + debug_flag: metadata + .log_level + .map(|l| l == Level::Debug) + .unwrap_or(false), + enable_testing_features: false, + locale: deno_core::v8::icu::get_language_tag(), + location: metadata.location, + no_color: !colors::use_color(), + is_tty: colors::is_tty(), + runtime_version: version::deno().to_string(), + ts_version: version::TYPESCRIPT.to_string(), + unstable: metadata.unstable, + user_agent: version::get_user_agent().to_string(), + inspect: ps.options.is_inspecting(), + }, + extensions: ops::cli_exts(ps.npm_resolver.clone()), + startup_snapshot: Some(crate::js::deno_isolate_init()), + unsafely_ignore_certificate_errors: metadata + .unsafely_ignore_certificate_errors, + root_cert_store: Some(ps.root_cert_store.clone()), + seed: metadata.seed, + source_map_getter: None, + format_js_error_fn: Some(Arc::new(format_js_error)), + create_web_worker_cb, + web_worker_preload_module_cb: web_worker_cb.clone(), + web_worker_pre_execute_module_cb: web_worker_cb, + maybe_inspector_server: None, + should_break_on_first_statement: false, + should_wait_for_inspector_session: false, + module_loader, + npm_resolver: None, // not currently supported + get_error_class_fn: Some(&get_error_class_name), + cache_storage_dir: None, + origin_storage_dir: None, + blob_store: ps.blob_store.clone(), + broadcast_channel: ps.broadcast_channel.clone(), + shared_array_buffer_store: Some(ps.shared_array_buffer_store.clone()), + compiled_wasm_module_store: Some(ps.compiled_wasm_module_store.clone()), + stdio: Default::default(), + }; + let mut worker = MainWorker::bootstrap_from_options( + main_module.clone(), + permissions, + options, + ); + worker.execute_main_module(main_module).await?; + worker.dispatch_load_event(located_script_name!())?; + + loop { + worker.run_event_loop(false).await?; + if !worker.dispatch_beforeunload_event(located_script_name!())? { + break; + } + } + + worker.dispatch_unload_event(located_script_name!())?; + std::process::exit(0); +} + +fn get_error_class_name(e: &AnyError) -> &'static str { + deno_runtime::errors::get_error_class_name(e).unwrap_or_else(|| { + panic!( + "Error '{}' contains boxed error of unsupported type:{}", + e, + e.chain().map(|e| format!("\n {e:?}")).collect::<String>() + ); + }) +} |