From 46b6037644c761369e689704f8e7b857959da155 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Tue, 19 Nov 2024 16:19:35 -0500 Subject: feat(compile): ability to embed local data files (#26934) ``` > deno compile --allow-read=. --include data-file.txt main.js ``` This only applies to files on the filesystem. For remote modules, that's going to have to wait for `import ... from "./data.txt" with { "type": "bytes" }` or whatever it will be. --- cli/standalone/binary.rs | 168 +++++++++++++++++++++++++++-------------------- 1 file changed, 95 insertions(+), 73 deletions(-) (limited to 'cli/standalone/binary.rs') diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index d18d77564..815399313 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -201,7 +201,8 @@ fn write_binary_bytes( compile_flags: &CompileFlags, ) -> Result<(), AnyError> { let data_section_bytes = - serialize_binary_data_section(metadata, npm_snapshot, remote_modules, vfs)?; + serialize_binary_data_section(metadata, npm_snapshot, remote_modules, vfs) + .context("Serializing binary data section.")?; let target = compile_flags.resolve_target(); if target.contains("linux") { @@ -364,6 +365,7 @@ pub fn extract_standalone( pub struct DenoCompileBinaryWriter<'a> { cjs_tracker: &'a CjsTracker, + cli_options: &'a CliOptions, deno_dir: &'a DenoDir, emitter: &'a Emitter, file_fetcher: &'a FileFetcher, @@ -377,6 +379,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { #[allow(clippy::too_many_arguments)] pub fn new( cjs_tracker: &'a CjsTracker, + cli_options: &'a CliOptions, deno_dir: &'a DenoDir, emitter: &'a Emitter, file_fetcher: &'a FileFetcher, @@ -387,6 +390,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { ) -> Self { Self { cjs_tracker, + cli_options, deno_dir, emitter, file_fetcher, @@ -403,8 +407,8 @@ impl<'a> DenoCompileBinaryWriter<'a> { graph: &ModuleGraph, root_dir_url: StandaloneRelativeFileBaseUrl<'_>, entrypoint: &ModuleSpecifier, + include_files: &[ModuleSpecifier], compile_flags: &CompileFlags, - cli_options: &CliOptions, ) -> Result<(), AnyError> { // Select base binary based on target let mut original_binary = self.get_base_binary(compile_flags).await?; @@ -417,7 +421,8 @@ impl<'a> DenoCompileBinaryWriter<'a> { target, ) } - set_windows_binary_to_gui(&mut original_binary)?; + set_windows_binary_to_gui(&mut original_binary) + .context("Setting windows binary to GUI.")?; } if compile_flags.icon.is_some() { let target = compile_flags.resolve_target(); @@ -435,7 +440,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { graph, root_dir_url, entrypoint, - cli_options, + include_files, compile_flags, ) .await @@ -478,10 +483,14 @@ impl<'a> DenoCompileBinaryWriter<'a> { if !binary_path.exists() { self .download_base_binary(&download_directory, &binary_path_suffix) - .await?; + .await + .context("Setting up base binary.")?; } - let archive_data = std::fs::read(binary_path)?; + let read_file = |path: &Path| -> Result, AnyError> { + std::fs::read(path).with_context(|| format!("Reading {}", path.display())) + }; + let archive_data = read_file(&binary_path)?; let temp_dir = tempfile::TempDir::new()?; let base_binary_path = archive::unpack_into_dir(archive::UnpackArgs { exe_name: "denort", @@ -490,7 +499,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { is_windows: target.contains("windows"), dest_path: temp_dir.path(), })?; - let base_binary = std::fs::read(base_binary_path)?; + let base_binary = read_file(&base_binary_path)?; drop(temp_dir); // delete the temp dir Ok(base_binary) } @@ -518,15 +527,19 @@ impl<'a> DenoCompileBinaryWriter<'a> { let bytes = match maybe_bytes { Some(bytes) => bytes, None => { - log::info!("Download could not be found, aborting"); - deno_runtime::exit(1); + bail!("Download could not be found, aborting"); } }; - std::fs::create_dir_all(output_directory)?; + let create_dir_all = |dir: &Path| { + std::fs::create_dir_all(dir) + .with_context(|| format!("Creating {}", dir.display())) + }; + create_dir_all(output_directory)?; let output_path = output_directory.join(binary_path_suffix); - std::fs::create_dir_all(output_path.parent().unwrap())?; - tokio::fs::write(output_path, bytes).await?; + create_dir_all(output_path.parent().unwrap())?; + std::fs::write(&output_path, bytes) + .with_context(|| format!("Writing {}", output_path.display()))?; Ok(()) } @@ -540,73 +553,79 @@ impl<'a> DenoCompileBinaryWriter<'a> { graph: &ModuleGraph, root_dir_url: StandaloneRelativeFileBaseUrl<'_>, entrypoint: &ModuleSpecifier, - cli_options: &CliOptions, + include_files: &[ModuleSpecifier], compile_flags: &CompileFlags, ) -> Result<(), AnyError> { - let ca_data = match cli_options.ca_data() { + let ca_data = match self.cli_options.ca_data() { Some(CaData::File(ca_file)) => Some( - std::fs::read(ca_file) - .with_context(|| format!("Reading: {ca_file}"))?, + std::fs::read(ca_file).with_context(|| format!("Reading {ca_file}"))?, ), Some(CaData::Bytes(bytes)) => Some(bytes.clone()), None => None, }; let root_path = root_dir_url.inner().to_file_path().unwrap(); - let (maybe_npm_vfs, node_modules, npm_snapshot) = match self - .npm_resolver - .as_inner() - { - InnerCliNpmResolverRef::Managed(managed) => { - let snapshot = - managed.serialized_valid_snapshot_for_system(&self.npm_system_info); - if !snapshot.as_serialized().packages.is_empty() { - let npm_vfs_builder = self.build_npm_vfs(&root_path, cli_options)?; + let (maybe_npm_vfs, node_modules, npm_snapshot) = + match self.npm_resolver.as_inner() { + InnerCliNpmResolverRef::Managed(managed) => { + let snapshot = + managed.serialized_valid_snapshot_for_system(&self.npm_system_info); + if !snapshot.as_serialized().packages.is_empty() { + let npm_vfs_builder = self + .build_npm_vfs(&root_path) + .context("Building npm vfs.")?; + ( + Some(npm_vfs_builder), + Some(NodeModules::Managed { + node_modules_dir: self + .npm_resolver + .root_node_modules_path() + .map(|path| { + root_dir_url + .specifier_key( + &ModuleSpecifier::from_directory_path(path).unwrap(), + ) + .into_owned() + }), + }), + Some(snapshot), + ) + } else { + (None, None, None) + } + } + InnerCliNpmResolverRef::Byonm(resolver) => { + let npm_vfs_builder = self.build_npm_vfs(&root_path)?; ( Some(npm_vfs_builder), - Some(NodeModules::Managed { - node_modules_dir: self.npm_resolver.root_node_modules_path().map( - |path| { + Some(NodeModules::Byonm { + root_node_modules_dir: resolver.root_node_modules_path().map( + |node_modules_dir| { root_dir_url .specifier_key( - &ModuleSpecifier::from_directory_path(path).unwrap(), + &ModuleSpecifier::from_directory_path(node_modules_dir) + .unwrap(), ) .into_owned() }, ), }), - Some(snapshot), + None, ) - } else { - (None, None, None) } - } - InnerCliNpmResolverRef::Byonm(resolver) => { - let npm_vfs_builder = self.build_npm_vfs(&root_path, cli_options)?; - ( - Some(npm_vfs_builder), - Some(NodeModules::Byonm { - root_node_modules_dir: resolver.root_node_modules_path().map( - |node_modules_dir| { - root_dir_url - .specifier_key( - &ModuleSpecifier::from_directory_path(node_modules_dir) - .unwrap(), - ) - .into_owned() - }, - ), - }), - None, - ) - } - }; + }; let mut vfs = if let Some(npm_vfs) = maybe_npm_vfs { npm_vfs } else { VfsBuilder::new(root_path.clone())? }; + for include_file in include_files { + let path = deno_path_util::url_to_file_path(include_file)?; + vfs + .add_file_at_path(&path) + .with_context(|| format!("Including {}", path.display()))?; + } let mut remote_modules_store = RemoteModulesStoreBuilder::default(); - let mut code_cache_key_hasher = if cli_options.code_cache_enabled() { + let mut code_cache_key_hasher = if self.cli_options.code_cache_enabled() { Some(FastInsecureHasher::new_deno_versioned()) } else { None @@ -672,7 +691,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { } remote_modules_store.add_redirects(&graph.redirects); - let env_vars_from_env_file = match cli_options.env_file_name() { + let env_vars_from_env_file = match self.cli_options.env_file_name() { Some(env_filenames) => { let mut aggregated_env_vars = IndexMap::new(); for env_filename in env_filenames.iter().rev() { @@ -688,16 +707,17 @@ impl<'a> DenoCompileBinaryWriter<'a> { let metadata = Metadata { argv: compile_flags.args.clone(), - seed: cli_options.seed(), + seed: self.cli_options.seed(), code_cache_key: code_cache_key_hasher.map(|h| h.finish()), - location: cli_options.location_flag().clone(), - permissions: cli_options.permission_flags().clone(), - v8_flags: cli_options.v8_flags().clone(), - unsafely_ignore_certificate_errors: cli_options + location: self.cli_options.location_flag().clone(), + permissions: self.cli_options.permission_flags().clone(), + v8_flags: self.cli_options.v8_flags().clone(), + unsafely_ignore_certificate_errors: self + .cli_options .unsafely_ignore_certificate_errors() .clone(), - log_level: cli_options.log_level(), - ca_stores: cli_options.ca_stores().clone(), + log_level: self.cli_options.log_level(), + ca_stores: self.cli_options.ca_stores().clone(), ca_data, env_vars_from_env_file, entrypoint_key: root_dir_url.specifier_key(entrypoint).into_owned(), @@ -740,11 +760,11 @@ impl<'a> DenoCompileBinaryWriter<'a> { node_modules, unstable_config: UnstableConfig { legacy_flag_enabled: false, - bare_node_builtins: cli_options.unstable_bare_node_builtins(), - sloppy_imports: cli_options.unstable_sloppy_imports(), - features: cli_options.unstable_features(), + bare_node_builtins: self.cli_options.unstable_bare_node_builtins(), + sloppy_imports: self.cli_options.unstable_sloppy_imports(), + features: self.cli_options.unstable_features(), }, - otel_config: cli_options.otel_config(), + otel_config: self.cli_options.otel_config(), }; write_binary_bytes( @@ -756,13 +776,10 @@ impl<'a> DenoCompileBinaryWriter<'a> { vfs, compile_flags, ) + .context("Writing binary bytes") } - fn build_npm_vfs( - &self, - root_path: &Path, - cli_options: &CliOptions, - ) -> Result { + fn build_npm_vfs(&self, root_path: &Path) -> Result { fn maybe_warn_different_system(system_info: &NpmSystemInfo) { if system_info != &NpmSystemInfo::default() { log::warn!("{} The node_modules directory may be incompatible with the target system.", crate::colors::yellow("Warning")); @@ -839,13 +856,18 @@ impl<'a> DenoCompileBinaryWriter<'a> { InnerCliNpmResolverRef::Byonm(_) => { maybe_warn_different_system(&self.npm_system_info); let mut builder = VfsBuilder::new(root_path.to_path_buf())?; - for pkg_json in cli_options.workspace().package_jsons() { + for pkg_json in self.cli_options.workspace().package_jsons() { builder.add_file_at_path(&pkg_json.path)?; } // traverse and add all the node_modules directories in the workspace let mut pending_dirs = VecDeque::new(); pending_dirs.push_back( - cli_options.workspace().root_dir().to_file_path().unwrap(), + self + .cli_options + .workspace() + .root_dir() + .to_file_path() + .unwrap(), ); while let Some(pending_dir) = pending_dirs.pop_front() { let mut entries = fs::read_dir(&pending_dir) -- cgit v1.2.3