From 1a0cb5b5312941521ab021cfe9eaed498f35900b Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 14 Oct 2024 20:48:39 -0400 Subject: feat(unstable): `--unstable-detect-cjs` for respecting explicit `"type": "commonjs"` (#26149) When using the `--unstable-detect-cjs` flag or adding `"unstable": ["detect-cjs"]` to a deno.json, it will make a JS file CJS if the closest package.json contains `"type": "commonjs"` and the file is not an ESM module (no TLA, no `import.meta`, no `import`/`export`). --- cli/standalone/binary.rs | 1 + 1 file changed, 1 insertion(+) (limited to 'cli/standalone/binary.rs') diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index 1290a238f..6e747bed4 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -622,6 +622,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { unstable_config: UnstableConfig { legacy_flag_enabled: false, bare_node_builtins: cli_options.unstable_bare_node_builtins(), + detect_cjs: cli_options.unstable_detect_cjs(), sloppy_imports: cli_options.unstable_sloppy_imports(), features: cli_options.unstable_features(), }, -- cgit v1.2.3 From 7c3c13cecf48219cdcb90dc0b5019686cdd88626 Mon Sep 17 00:00:00 2001 From: Nathan Whitaker <17734409+nathanwhit@users.noreply.github.com> Date: Tue, 15 Oct 2024 16:46:42 -0700 Subject: fix(install): retry downloads of registry info / tarballs (#26278) Fixes #26085. Adds a basic retry utility with some defaults, starts off with a 100ms wait, then 250ms, then 500ms I've applied the retry in the http client, reusing an existing function, so this also applies to retrying downloads of deno binaries in `upgrade` and `compile`. I can make a separate function that doesn't retry so this doesn't affect `upgrade` and `compile`, but it seemed desirable to have retries there too, so I left it in. --- cli/standalone/binary.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'cli/standalone/binary.rs') diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index 6e747bed4..52ee4eeb2 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -468,7 +468,11 @@ impl<'a> DenoCompileBinaryWriter<'a> { self .http_client_provider .get_or_create()? - .download_with_progress(download_url.parse()?, None, &progress) + .download_with_progress_and_retries( + download_url.parse()?, + None, + &progress, + ) .await? }; let bytes = match maybe_bytes { -- cgit v1.2.3 From eedf243b5ea98d22649bb0445444719a2fc12c59 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 24 Oct 2024 15:48:48 -0400 Subject: perf(compile): pass module source data from binary directly to v8 (#26494) This changes denort to pass a static reference of the moude source bytes found in the binary to v8 instead of copying it. --- cli/standalone/binary.rs | 465 ++++++++++++++++++++++++++++------------------- 1 file changed, 274 insertions(+), 191 deletions(-) (limited to 'cli/standalone/binary.rs') diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index 52ee4eeb2..0f8b0b49d 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -9,14 +9,18 @@ use std::ffi::OsString; use std::fs; use std::fs::File; use std::future::Future; +use std::io::ErrorKind; use std::io::Read; use std::io::Seek; use std::io::SeekFrom; use std::io::Write; +use std::ops::Range; use std::path::Path; use std::path::PathBuf; use std::process::Command; +use std::sync::Arc; +use deno_ast::MediaType; use deno_ast::ModuleSpecifier; use deno_config::workspace::PackageJsonDepResolution; use deno_config::workspace::ResolverWorkspaceJsrPackage; @@ -30,13 +34,22 @@ use deno_core::futures::AsyncReadExt; use deno_core::futures::AsyncSeekExt; use deno_core::serde_json; use deno_core::url::Url; +use deno_graph::source::RealFileSystem; +use deno_graph::ModuleGraph; +use deno_npm::resolution::SerializedNpmResolutionSnapshot; +use deno_npm::resolution::SerializedNpmResolutionSnapshotPackage; +use deno_npm::resolution::ValidSerializedNpmResolutionSnapshot; +use deno_npm::NpmPackageId; use deno_npm::NpmSystemInfo; +use deno_runtime::deno_fs; +use deno_runtime::deno_fs::FileSystem; +use deno_runtime::deno_fs::RealFs; +use deno_runtime::deno_io::fs::FsError; use deno_runtime::deno_node::PackageJson; use deno_semver::npm::NpmVersionReqParseError; use deno_semver::package::PackageReq; use deno_semver::Version; use deno_semver::VersionReqSpecifierParseError; -use eszip::EszipRelativeFileBaseUrl; use indexmap::IndexMap; use log::Level; use serde::Deserialize; @@ -49,6 +62,7 @@ use crate::args::NpmInstallDepsProvider; use crate::args::PermissionFlags; use crate::args::UnstableConfig; use crate::cache::DenoDir; +use crate::emit::Emitter; use crate::file_fetcher::FileFetcher; use crate::http_util::HttpClientProvider; use crate::npm::CliNpmResolver; @@ -60,12 +74,63 @@ use crate::util::fs::canonicalize_path_maybe_not_exists; use crate::util::progress_bar::ProgressBar; use crate::util::progress_bar::ProgressBarStyle; +use super::file_system::DenoCompileFileSystem; +use super::serialization::deserialize_binary_data_section; +use super::serialization::serialize_binary_data_section; +use super::serialization::DenoCompileModuleData; +use super::serialization::DeserializedDataSection; +use super::serialization::RemoteModulesStore; +use super::serialization::RemoteModulesStoreBuilder; use super::virtual_fs::FileBackedVfs; use super::virtual_fs::VfsBuilder; use super::virtual_fs::VfsRoot; use super::virtual_fs::VirtualDirectory; -const MAGIC_TRAILER: &[u8; 8] = b"d3n0l4nd"; +/// A URL that can be designated as the base for relative URLs. +/// +/// After creation, this URL may be used to get the key for a +/// module in the binary. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub struct StandaloneRelativeFileBaseUrl<'a>(&'a Url); + +impl<'a> From<&'a Url> for StandaloneRelativeFileBaseUrl<'a> { + fn from(url: &'a Url) -> Self { + Self(url) + } +} + +impl<'a> StandaloneRelativeFileBaseUrl<'a> { + pub fn new(url: &'a Url) -> Self { + debug_assert_eq!(url.scheme(), "file"); + Self(url) + } + + /// Gets the module map key of the provided specifier. + /// + /// * Descendant file specifiers will be made relative to the base. + /// * Non-descendant file specifiers will stay as-is (absolute). + /// * Non-file specifiers will stay as-is. + pub fn specifier_key<'b>(&self, target: &'b Url) -> Cow<'b, str> { + if target.scheme() != "file" { + return Cow::Borrowed(target.as_str()); + } + + match self.0.make_relative(target) { + Some(relative) => { + if relative.starts_with("../") { + Cow::Borrowed(target.as_str()) + } else { + Cow::Owned(relative) + } + } + None => Cow::Borrowed(target.as_str()), + } + } + + pub fn inner(&self) -> &Url { + self.0 + } +} #[derive(Deserialize, Serialize)] pub enum NodeModules { @@ -120,78 +185,23 @@ pub struct Metadata { pub unstable_config: UnstableConfig, } -pub fn load_npm_vfs(root_dir_path: PathBuf) -> Result { - let data = libsui::find_section("d3n0l4nd").unwrap(); - - // We do the first part sync so it can complete quickly - let trailer: [u8; TRAILER_SIZE] = data[0..TRAILER_SIZE].try_into().unwrap(); - let trailer = match Trailer::parse(&trailer)? { - None => panic!("Could not find trailer"), - Some(trailer) => trailer, - }; - let data = &data[TRAILER_SIZE..]; - - let vfs_data = - &data[trailer.npm_vfs_pos as usize..trailer.npm_files_pos as usize]; - let mut dir: VirtualDirectory = serde_json::from_slice(vfs_data)?; - - // align the name of the directory with the root dir - dir.name = root_dir_path - .file_name() - .unwrap() - .to_string_lossy() - .to_string(); - - let fs_root = VfsRoot { - dir, - root_path: root_dir_path, - start_file_offset: trailer.npm_files_pos, - }; - Ok(FileBackedVfs::new(data.to_vec(), fs_root)) -} - fn write_binary_bytes( mut file_writer: File, original_bin: Vec, metadata: &Metadata, - eszip: eszip::EszipV2, - npm_vfs: Option<&VirtualDirectory>, - npm_files: &Vec>, + npm_snapshot: Option, + remote_modules: &RemoteModulesStoreBuilder, + vfs: VfsBuilder, compile_flags: &CompileFlags, ) -> Result<(), AnyError> { - let metadata = serde_json::to_string(metadata)?.as_bytes().to_vec(); - let npm_vfs = serde_json::to_string(&npm_vfs)?.as_bytes().to_vec(); - let eszip_archive = eszip.into_bytes(); - - let mut writer = Vec::new(); - - // write the trailer, which includes the positions - // of the data blocks in the file - writer.write_all(&{ - let metadata_pos = eszip_archive.len() as u64; - let npm_vfs_pos = metadata_pos + (metadata.len() as u64); - let npm_files_pos = npm_vfs_pos + (npm_vfs.len() as u64); - Trailer { - eszip_pos: 0, - metadata_pos, - npm_vfs_pos, - npm_files_pos, - } - .as_bytes() - })?; - - writer.write_all(&eszip_archive)?; - writer.write_all(&metadata)?; - writer.write_all(&npm_vfs)?; - for file in npm_files { - writer.write_all(file)?; - } + let data_section_bytes = + serialize_binary_data_section(metadata, npm_snapshot, remote_modules, vfs)?; let target = compile_flags.resolve_target(); if target.contains("linux") { libsui::Elf::new(&original_bin).append( "d3n0l4nd", - &writer, + &data_section_bytes, &mut file_writer, )?; } else if target.contains("windows") { @@ -201,11 +211,11 @@ fn write_binary_bytes( pe = pe.set_icon(&icon)?; } - pe.write_resource("d3n0l4nd", writer)? + pe.write_resource("d3n0l4nd", data_section_bytes)? .build(&mut file_writer)?; } else if target.contains("darwin") { libsui::Macho::from(original_bin)? - .write_section("d3n0l4nd", writer)? + .write_section("d3n0l4nd", data_section_bytes)? .build_and_sign(&mut file_writer)?; } Ok(()) @@ -221,6 +231,63 @@ pub fn is_standalone_binary(exe_path: &Path) -> bool { || libsui::utils::is_macho(&data) } +pub struct StandaloneData { + pub fs: Arc, + pub metadata: Metadata, + pub modules: StandaloneModules, + pub npm_snapshot: Option, + pub root_path: PathBuf, + pub vfs: Arc, +} + +pub struct StandaloneModules { + remote_modules: RemoteModulesStore, + vfs: Arc, +} + +impl StandaloneModules { + pub fn resolve_specifier<'a>( + &'a self, + specifier: &'a ModuleSpecifier, + ) -> Result, AnyError> { + if specifier.scheme() == "file" { + Ok(Some(specifier)) + } else { + self.remote_modules.resolve_specifier(specifier) + } + } + + pub fn read<'a>( + &'a self, + specifier: &'a ModuleSpecifier, + ) -> Result>, AnyError> { + if specifier.scheme() == "file" { + let path = deno_path_util::url_to_file_path(specifier)?; + let bytes = match self.vfs.file_entry(&path) { + Ok(entry) => self.vfs.read_file_all(entry)?, + Err(err) if err.kind() == ErrorKind::NotFound => { + let bytes = match RealFs.read_file_sync(&path, None) { + Ok(bytes) => bytes, + Err(FsError::Io(err)) if err.kind() == ErrorKind::NotFound => { + return Ok(None) + } + Err(err) => return Err(err.into()), + }; + Cow::Owned(bytes) + } + Err(err) => return Err(err.into()), + }; + Ok(Some(DenoCompileModuleData { + media_type: MediaType::from_specifier(specifier), + specifier, + data: bytes, + })) + } else { + self.remote_modules.read(specifier) + } + } +} + /// This function will try to run this binary as a standalone binary /// produced by `deno compile`. It determines if this is a standalone /// binary by skipping over the trailer width at the end of the file, @@ -228,110 +295,66 @@ pub fn is_standalone_binary(exe_path: &Path) -> bool { /// the bundle is executed. If not, this function exits with `Ok(None)`. pub fn extract_standalone( cli_args: Cow>, -) -> Result< - Option>>, - AnyError, -> { +) -> Result, AnyError> { let Some(data) = libsui::find_section("d3n0l4nd") else { return Ok(None); }; - // We do the first part sync so it can complete quickly - let trailer = match Trailer::parse(&data[0..TRAILER_SIZE])? { + let DeserializedDataSection { + mut metadata, + npm_snapshot, + remote_modules, + mut vfs_dir, + vfs_files_data, + } = match deserialize_binary_data_section(data)? { + Some(data_section) => data_section, None => return Ok(None), - Some(trailer) => trailer, }; + let root_path = { + let maybe_current_exe = std::env::current_exe().ok(); + let current_exe_name = maybe_current_exe + .as_ref() + .and_then(|p| p.file_name()) + .map(|p| p.to_string_lossy()) + // should never happen + .unwrap_or_else(|| Cow::Borrowed("binary")); + std::env::temp_dir().join(format!("deno-compile-{}", current_exe_name)) + }; let cli_args = cli_args.into_owned(); - // If we have an eszip, read it out - Ok(Some(async move { - let bufreader = - deno_core::futures::io::BufReader::new(&data[TRAILER_SIZE..]); - - let (eszip, loader) = eszip::EszipV2::parse(bufreader) - .await - .context("Failed to parse eszip header")?; - - let bufreader = loader.await.context("Failed to parse eszip archive")?; - - let mut metadata = String::new(); - - bufreader - .take(trailer.metadata_len()) - .read_to_string(&mut metadata) - .await - .context("Failed to read metadata from the current executable")?; - - let mut metadata: Metadata = serde_json::from_str(&metadata).unwrap(); - metadata.argv.reserve(cli_args.len() - 1); - for arg in cli_args.into_iter().skip(1) { - metadata.argv.push(arg.into_string().unwrap()); - } - - Ok((metadata, eszip)) - })) -} - -const TRAILER_SIZE: usize = std::mem::size_of::() + 8; // 8 bytes for the magic trailer string - -struct Trailer { - eszip_pos: u64, - metadata_pos: u64, - npm_vfs_pos: u64, - npm_files_pos: u64, -} - -impl Trailer { - pub fn parse(trailer: &[u8]) -> Result, AnyError> { - let (magic_trailer, rest) = trailer.split_at(8); - if magic_trailer != MAGIC_TRAILER { - return Ok(None); - } - - let (eszip_archive_pos, rest) = rest.split_at(8); - let (metadata_pos, rest) = rest.split_at(8); - let (npm_vfs_pos, npm_files_pos) = rest.split_at(8); - let eszip_archive_pos = u64_from_bytes(eszip_archive_pos)?; - let metadata_pos = u64_from_bytes(metadata_pos)?; - let npm_vfs_pos = u64_from_bytes(npm_vfs_pos)?; - let npm_files_pos = u64_from_bytes(npm_files_pos)?; - Ok(Some(Trailer { - eszip_pos: eszip_archive_pos, - metadata_pos, - npm_vfs_pos, - npm_files_pos, - })) - } - - pub fn metadata_len(&self) -> u64 { - self.npm_vfs_pos - self.metadata_pos - } - - pub fn npm_vfs_len(&self) -> u64 { - self.npm_files_pos - self.npm_vfs_pos - } - - pub fn as_bytes(&self) -> Vec { - let mut trailer = MAGIC_TRAILER.to_vec(); - trailer.write_all(&self.eszip_pos.to_be_bytes()).unwrap(); - trailer.write_all(&self.metadata_pos.to_be_bytes()).unwrap(); - trailer.write_all(&self.npm_vfs_pos.to_be_bytes()).unwrap(); - trailer - .write_all(&self.npm_files_pos.to_be_bytes()) - .unwrap(); - trailer + metadata.argv.reserve(cli_args.len() - 1); + for arg in cli_args.into_iter().skip(1) { + metadata.argv.push(arg.into_string().unwrap()); } -} - -fn u64_from_bytes(arr: &[u8]) -> Result { - let fixed_arr: &[u8; 8] = arr - .try_into() - .context("Failed to convert the buffer into a fixed-size array")?; - Ok(u64::from_be_bytes(*fixed_arr)) + let vfs = { + // align the name of the directory with the root dir + vfs_dir.name = root_path.file_name().unwrap().to_string_lossy().to_string(); + + let fs_root = VfsRoot { + dir: vfs_dir, + root_path: root_path.clone(), + start_file_offset: 0, + }; + Arc::new(FileBackedVfs::new(Cow::Borrowed(vfs_files_data), fs_root)) + }; + let fs: Arc = + Arc::new(DenoCompileFileSystem::new(vfs.clone())); + Ok(Some(StandaloneData { + fs, + metadata, + modules: StandaloneModules { + remote_modules, + vfs: vfs.clone(), + }, + npm_snapshot, + root_path, + vfs, + })) } pub struct DenoCompileBinaryWriter<'a> { deno_dir: &'a DenoDir, + emitter: &'a Emitter, file_fetcher: &'a FileFetcher, http_client_provider: &'a HttpClientProvider, npm_resolver: &'a dyn CliNpmResolver, @@ -343,6 +366,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { #[allow(clippy::too_many_arguments)] pub fn new( deno_dir: &'a DenoDir, + emitter: &'a Emitter, file_fetcher: &'a FileFetcher, http_client_provider: &'a HttpClientProvider, npm_resolver: &'a dyn CliNpmResolver, @@ -351,6 +375,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { ) -> Self { Self { deno_dir, + emitter, file_fetcher, http_client_provider, npm_resolver, @@ -362,8 +387,8 @@ impl<'a> DenoCompileBinaryWriter<'a> { pub async fn write_bin( &self, writer: File, - eszip: eszip::EszipV2, - root_dir_url: EszipRelativeFileBaseUrl<'_>, + graph: &ModuleGraph, + root_dir_url: StandaloneRelativeFileBaseUrl<'_>, entrypoint: &ModuleSpecifier, compile_flags: &CompileFlags, cli_options: &CliOptions, @@ -390,15 +415,17 @@ impl<'a> DenoCompileBinaryWriter<'a> { ) } } - self.write_standalone_binary( - writer, - original_binary, - eszip, - root_dir_url, - entrypoint, - cli_options, - compile_flags, - ) + self + .write_standalone_binary( + writer, + original_binary, + graph, + root_dir_url, + entrypoint, + cli_options, + compile_flags, + ) + .await } async fn get_base_binary( @@ -493,12 +520,12 @@ impl<'a> DenoCompileBinaryWriter<'a> { /// This functions creates a standalone deno binary by appending a bundle /// and magic trailer to the currently executing binary. #[allow(clippy::too_many_arguments)] - fn write_standalone_binary( + async fn write_standalone_binary( &self, writer: File, original_bin: Vec, - mut eszip: eszip::EszipV2, - root_dir_url: EszipRelativeFileBaseUrl<'_>, + graph: &ModuleGraph, + root_dir_url: StandaloneRelativeFileBaseUrl<'_>, entrypoint: &ModuleSpecifier, cli_options: &CliOptions, compile_flags: &CompileFlags, @@ -512,19 +539,17 @@ impl<'a> DenoCompileBinaryWriter<'a> { None => None, }; let root_path = root_dir_url.inner().to_file_path().unwrap(); - let (npm_vfs, npm_files, node_modules) = match self.npm_resolver.as_inner() + let (maybe_npm_vfs, node_modules, npm_snapshot) = match self + .npm_resolver + .as_inner() { InnerCliNpmResolverRef::Managed(managed) => { let snapshot = managed.serialized_valid_snapshot_for_system(&self.npm_system_info); if !snapshot.as_serialized().packages.is_empty() { - let (root_dir, files) = self - .build_vfs(&root_path, cli_options)? - .into_dir_and_files(); - eszip.add_npm_snapshot(snapshot); + let npm_vfs_builder = self.build_npm_vfs(&root_path, cli_options)?; ( - Some(root_dir), - files, + Some(npm_vfs_builder), Some(NodeModules::Managed { node_modules_dir: self.npm_resolver.root_node_modules_path().map( |path| { @@ -536,18 +561,16 @@ impl<'a> DenoCompileBinaryWriter<'a> { }, ), }), + Some(snapshot), ) } else { - (None, Vec::new(), None) + (None, None, None) } } InnerCliNpmResolverRef::Byonm(resolver) => { - let (root_dir, files) = self - .build_vfs(&root_path, cli_options)? - .into_dir_and_files(); + let npm_vfs_builder = self.build_npm_vfs(&root_path, cli_options)?; ( - Some(root_dir), - files, + Some(npm_vfs_builder), Some(NodeModules::Byonm { root_node_modules_dir: resolver.root_node_modules_path().map( |node_modules_dir| { @@ -560,9 +583,67 @@ impl<'a> DenoCompileBinaryWriter<'a> { }, ), }), + None, ) } }; + let mut vfs = if let Some(npm_vfs) = maybe_npm_vfs { + npm_vfs + } else { + VfsBuilder::new(root_path.clone())? + }; + let mut remote_modules_store = RemoteModulesStoreBuilder::default(); + for module in graph.modules() { + if module.specifier().scheme() == "data" { + continue; // don't store data urls as an entry as they're in the code + } + let (maybe_source, media_type) = match module { + deno_graph::Module::Js(m) => { + // todo(https://github.com/denoland/deno_media_type/pull/12): use is_emittable() + let is_emittable = matches!( + m.media_type, + MediaType::TypeScript + | MediaType::Mts + | MediaType::Cts + | MediaType::Jsx + | MediaType::Tsx + ); + let source = if is_emittable { + let source = self + .emitter + .emit_parsed_source(&m.specifier, m.media_type, &m.source) + .await?; + source.to_vec() + } else { + m.source.as_bytes().to_vec() + }; + (Some(source), m.media_type) + } + deno_graph::Module::Json(m) => { + (Some(m.source.as_bytes().to_vec()), m.media_type) + } + deno_graph::Module::Npm(_) + | deno_graph::Module::Node(_) + | deno_graph::Module::External(_) => (None, MediaType::Unknown), + }; + if module.specifier().scheme() == "file" { + let file_path = deno_path_util::url_to_file_path(module.specifier())?; + vfs + .add_file_with_data( + &file_path, + match maybe_source { + Some(source) => source, + None => RealFs.read_file_sync(&file_path, None)?, + }, + ) + .with_context(|| { + format!("Failed adding '{}'", file_path.display()) + })?; + } else if let Some(source) = maybe_source { + remote_modules_store.add(module.specifier(), media_type, source); + } + } + remote_modules_store.add_redirects(&graph.redirects); let env_vars_from_env_file = match cli_options.env_file_name() { Some(env_filename) => { @@ -636,14 +717,14 @@ impl<'a> DenoCompileBinaryWriter<'a> { writer, original_bin, &metadata, - eszip, - npm_vfs.as_ref(), - &npm_files, + npm_snapshot.map(|s| s.into_serialized()), + &remote_modules_store, + vfs, compile_flags, ) } - fn build_vfs( + fn build_npm_vfs( &self, root_path: &Path, cli_options: &CliOptions, @@ -664,8 +745,8 @@ impl<'a> DenoCompileBinaryWriter<'a> { } else { // DO NOT include the user's registry url as it may contain credentials, // but also don't make this dependent on the registry url - let root_path = npm_resolver.global_cache_root_folder(); - let mut builder = VfsBuilder::new(root_path)?; + let global_cache_root_path = npm_resolver.global_cache_root_folder(); + let mut builder = VfsBuilder::new(global_cache_root_path)?; let mut packages = npm_resolver.all_system_packages(&self.npm_system_info); packages.sort_by(|a, b| a.id.cmp(&b.id)); // determinism @@ -675,12 +756,12 @@ impl<'a> DenoCompileBinaryWriter<'a> { builder.add_dir_recursive(&folder)?; } - // Flatten all the registries folders into a single "node_modules/localhost" folder + // Flatten all the registries folders into a single ".deno_compile_node_modules/localhost" folder // that will be used by denort when loading the npm cache. This avoids us exposing // the user's private registry information and means we don't have to bother // serializing all the different registry config into the binary. builder.with_root_dir(|root_dir| { - root_dir.name = "node_modules".to_string(); + root_dir.name = ".deno_compile_node_modules".to_string(); let mut new_entries = Vec::with_capacity(root_dir.entries.len()); let mut localhost_entries = IndexMap::new(); for entry in std::mem::take(&mut root_dir.entries) { @@ -715,6 +796,8 @@ impl<'a> DenoCompileBinaryWriter<'a> { root_dir.entries = new_entries; }); + builder.set_new_root_path(root_path.to_path_buf())?; + Ok(builder) } } -- cgit v1.2.3 From f0f476e58453d502ab5c118fc91d1475e6829967 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sat, 26 Oct 2024 13:41:09 -0400 Subject: perf: pass transpiled module to deno_core as known string (#26555) --- cli/standalone/binary.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cli/standalone/binary.rs') diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index 0f8b0b49d..f41f3003f 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -613,7 +613,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { .emitter .emit_parsed_source(&m.specifier, m.media_type, &m.source) .await?; - source.to_vec() + source.into_bytes() } else { m.source.as_bytes().to_vec() }; -- cgit v1.2.3 From 826e42a5b5880c974ae019a7a21aade6a718062c Mon Sep 17 00:00:00 2001 From: David Sherret Date: Fri, 1 Nov 2024 12:27:00 -0400 Subject: fix: improved support for cjs and cts modules (#26558) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * cts support * better cjs/cts type checking * deno compile cjs/cts support * More efficient detect cjs (going towards stabilization) * Determination of whether .js, .ts, .jsx, or .tsx is cjs or esm is only done after loading * Support `import x = require(...);` Co-authored-by: Bartek Iwańczuk --- cli/standalone/binary.rs | 38 +++++++++++++++++++++++++------------- 1 file changed, 25 insertions(+), 13 deletions(-) (limited to 'cli/standalone/binary.rs') diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index f41f3003f..9e2651226 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -21,6 +21,7 @@ use std::process::Command; use std::sync::Arc; use deno_ast::MediaType; +use deno_ast::ModuleKind; use deno_ast::ModuleSpecifier; use deno_config::workspace::PackageJsonDepResolution; use deno_config::workspace::ResolverWorkspaceJsrPackage; @@ -67,6 +68,7 @@ use crate::file_fetcher::FileFetcher; use crate::http_util::HttpClientProvider; use crate::npm::CliNpmResolver; use crate::npm::InnerCliNpmResolverRef; +use crate::resolver::CjsTracker; use crate::shared::ReleaseChannel; use crate::standalone::virtual_fs::VfsEntry; use crate::util::archive; @@ -257,6 +259,10 @@ impl StandaloneModules { } } + pub fn has_file(&self, path: &Path) -> bool { + self.vfs.file_entry(path).is_ok() + } + pub fn read<'a>( &'a self, specifier: &'a ModuleSpecifier, @@ -353,6 +359,7 @@ pub fn extract_standalone( } pub struct DenoCompileBinaryWriter<'a> { + cjs_tracker: &'a CjsTracker, deno_dir: &'a DenoDir, emitter: &'a Emitter, file_fetcher: &'a FileFetcher, @@ -365,6 +372,7 @@ pub struct DenoCompileBinaryWriter<'a> { impl<'a> DenoCompileBinaryWriter<'a> { #[allow(clippy::too_many_arguments)] pub fn new( + cjs_tracker: &'a CjsTracker, deno_dir: &'a DenoDir, emitter: &'a Emitter, file_fetcher: &'a FileFetcher, @@ -374,6 +382,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { npm_system_info: NpmSystemInfo, ) -> Self { Self { + cjs_tracker, deno_dir, emitter, file_fetcher, @@ -599,19 +608,21 @@ impl<'a> DenoCompileBinaryWriter<'a> { } let (maybe_source, media_type) = match module { deno_graph::Module::Js(m) => { - // todo(https://github.com/denoland/deno_media_type/pull/12): use is_emittable() - let is_emittable = matches!( - m.media_type, - MediaType::TypeScript - | MediaType::Mts - | MediaType::Cts - | MediaType::Jsx - | MediaType::Tsx - ); - let source = if is_emittable { + let source = if m.media_type.is_emittable() { + let is_cjs = self.cjs_tracker.is_cjs_with_known_is_script( + &m.specifier, + m.media_type, + m.is_script, + )?; + let module_kind = ModuleKind::from_is_cjs(is_cjs); let source = self .emitter - .emit_parsed_source(&m.specifier, m.media_type, &m.source) + .emit_parsed_source( + &m.specifier, + m.media_type, + module_kind, + &m.source, + ) .await?; source.into_bytes() } else { @@ -745,8 +756,9 @@ impl<'a> DenoCompileBinaryWriter<'a> { } else { // DO NOT include the user's registry url as it may contain credentials, // but also don't make this dependent on the registry url - let global_cache_root_path = npm_resolver.global_cache_root_folder(); - let mut builder = VfsBuilder::new(global_cache_root_path)?; + let global_cache_root_path = npm_resolver.global_cache_root_path(); + let mut builder = + VfsBuilder::new(global_cache_root_path.to_path_buf())?; let mut packages = npm_resolver.all_system_packages(&self.npm_system_info); packages.sort_by(|a, b| a.id.cmp(&b.id)); // determinism -- cgit v1.2.3 From 01de3317424cc870913dbe85ff3b80eadaf8cc87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Sat, 9 Nov 2024 15:19:46 +0000 Subject: perf(upgrade): cache downloaded binaries in DENO_DIR (#26108) Co-authored-by: Divy Srivastava --- cli/standalone/binary.rs | 74 ++++++++++-------------------------------------- 1 file changed, 15 insertions(+), 59 deletions(-) (limited to 'cli/standalone/binary.rs') diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index 9e2651226..a81e40ad9 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -63,6 +63,9 @@ use crate::args::NpmInstallDepsProvider; use crate::args::PermissionFlags; use crate::args::UnstableConfig; use crate::cache::DenoDir; +use crate::download_deno_binary::archive_name; +use crate::download_deno_binary::download_deno_binary; +use crate::download_deno_binary::BinaryKind; use crate::emit::Emitter; use crate::file_fetcher::FileFetcher; use crate::http_util::HttpClientProvider; @@ -452,36 +455,24 @@ impl<'a> DenoCompileBinaryWriter<'a> { } let target = compile_flags.resolve_target(); - let binary_name = format!("denort-{target}.zip"); - - let binary_path_suffix = - match crate::version::DENO_VERSION_INFO.release_channel { - ReleaseChannel::Canary => { - format!( - "canary/{}/{}", - crate::version::DENO_VERSION_INFO.git_hash, - binary_name - ) - } - _ => { - format!("release/v{}/{}", env!("CARGO_PKG_VERSION"), binary_name) - } - }; - let download_directory = self.deno_dir.dl_folder_path(); - let binary_path = download_directory.join(&binary_path_suffix); + let archive_name = archive_name(BinaryKind::Denort, &target); - if !binary_path.exists() { - self - .download_base_binary(&download_directory, &binary_path_suffix) - .await?; - } + let binary_path = download_deno_binary( + self.http_client_provider, + self.deno_dir, + BinaryKind::Denort, + &target, + crate::version::DENO_VERSION_INFO.version_or_git_hash(), + crate::version::DENO_VERSION_INFO.release_channel, + ) + .await?; let archive_data = std::fs::read(binary_path)?; let temp_dir = tempfile::TempDir::new()?; let base_binary_path = archive::unpack_into_dir(archive::UnpackArgs { - exe_name: "denort", - archive_name: &binary_name, + exe_name: BinaryKind::Denort.name(), + archive_name: &archive_name, archive_data: &archive_data, is_windows: target.contains("windows"), dest_path: temp_dir.path(), @@ -491,41 +482,6 @@ impl<'a> DenoCompileBinaryWriter<'a> { Ok(base_binary) } - async fn download_base_binary( - &self, - output_directory: &Path, - binary_path_suffix: &str, - ) -> Result<(), AnyError> { - let download_url = format!("https://dl.deno.land/{binary_path_suffix}"); - let maybe_bytes = { - let progress_bars = ProgressBar::new(ProgressBarStyle::DownloadBars); - let progress = progress_bars.update(&download_url); - - self - .http_client_provider - .get_or_create()? - .download_with_progress_and_retries( - download_url.parse()?, - None, - &progress, - ) - .await? - }; - let bytes = match maybe_bytes { - Some(bytes) => bytes, - None => { - log::info!("Download could not be found, aborting"); - std::process::exit(1) - } - }; - - std::fs::create_dir_all(output_directory)?; - let output_path = output_directory.join(binary_path_suffix); - std::fs::create_dir_all(output_path.parent().unwrap())?; - tokio::fs::write(output_path, bytes).await?; - Ok(()) - } - /// This functions creates a standalone deno binary by appending a bundle /// and magic trailer to the currently executing binary. #[allow(clippy::too_many_arguments)] -- cgit v1.2.3 From ce778a947e70616b435920fd6f1fc28c4b2a78d1 Mon Sep 17 00:00:00 2001 From: Divy Srivastava Date: Sun, 10 Nov 2024 09:00:44 +0530 Subject: Revert "perf(upgrade): cache downloaded binaries in DENO_DIR" (#26799) Reverts denoland/deno#26108 Tests are flaky on main https://github.com/denoland/deno/commit/01de3317424cc870913dbe85ff3b80eadaf8cc87 --- cli/standalone/binary.rs | 74 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 59 insertions(+), 15 deletions(-) (limited to 'cli/standalone/binary.rs') diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index a81e40ad9..9e2651226 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -63,9 +63,6 @@ use crate::args::NpmInstallDepsProvider; use crate::args::PermissionFlags; use crate::args::UnstableConfig; use crate::cache::DenoDir; -use crate::download_deno_binary::archive_name; -use crate::download_deno_binary::download_deno_binary; -use crate::download_deno_binary::BinaryKind; use crate::emit::Emitter; use crate::file_fetcher::FileFetcher; use crate::http_util::HttpClientProvider; @@ -455,24 +452,36 @@ impl<'a> DenoCompileBinaryWriter<'a> { } let target = compile_flags.resolve_target(); + let binary_name = format!("denort-{target}.zip"); + + let binary_path_suffix = + match crate::version::DENO_VERSION_INFO.release_channel { + ReleaseChannel::Canary => { + format!( + "canary/{}/{}", + crate::version::DENO_VERSION_INFO.git_hash, + binary_name + ) + } + _ => { + format!("release/v{}/{}", env!("CARGO_PKG_VERSION"), binary_name) + } + }; - let archive_name = archive_name(BinaryKind::Denort, &target); + let download_directory = self.deno_dir.dl_folder_path(); + let binary_path = download_directory.join(&binary_path_suffix); - let binary_path = download_deno_binary( - self.http_client_provider, - self.deno_dir, - BinaryKind::Denort, - &target, - crate::version::DENO_VERSION_INFO.version_or_git_hash(), - crate::version::DENO_VERSION_INFO.release_channel, - ) - .await?; + if !binary_path.exists() { + self + .download_base_binary(&download_directory, &binary_path_suffix) + .await?; + } let archive_data = std::fs::read(binary_path)?; let temp_dir = tempfile::TempDir::new()?; let base_binary_path = archive::unpack_into_dir(archive::UnpackArgs { - exe_name: BinaryKind::Denort.name(), - archive_name: &archive_name, + exe_name: "denort", + archive_name: &binary_name, archive_data: &archive_data, is_windows: target.contains("windows"), dest_path: temp_dir.path(), @@ -482,6 +491,41 @@ impl<'a> DenoCompileBinaryWriter<'a> { Ok(base_binary) } + async fn download_base_binary( + &self, + output_directory: &Path, + binary_path_suffix: &str, + ) -> Result<(), AnyError> { + let download_url = format!("https://dl.deno.land/{binary_path_suffix}"); + let maybe_bytes = { + let progress_bars = ProgressBar::new(ProgressBarStyle::DownloadBars); + let progress = progress_bars.update(&download_url); + + self + .http_client_provider + .get_or_create()? + .download_with_progress_and_retries( + download_url.parse()?, + None, + &progress, + ) + .await? + }; + let bytes = match maybe_bytes { + Some(bytes) => bytes, + None => { + log::info!("Download could not be found, aborting"); + std::process::exit(1) + } + }; + + std::fs::create_dir_all(output_directory)?; + let output_path = output_directory.join(binary_path_suffix); + std::fs::create_dir_all(output_path.parent().unwrap())?; + tokio::fs::write(output_path, bytes).await?; + Ok(()) + } + /// This functions creates a standalone deno binary by appending a bundle /// and magic trailer to the currently executing binary. #[allow(clippy::too_many_arguments)] -- cgit v1.2.3 From aa546189be730163ee5370029e4dfdb3b454ab96 Mon Sep 17 00:00:00 2001 From: snek Date: Wed, 13 Nov 2024 11:38:46 +0100 Subject: feat: OpenTelemetry Tracing API and Exporting (#26710) Initial import of OTEL code supporting tracing. Metrics soon to come. Implements APIs for https://jsr.io/@deno/otel so that code using OpenTelemetry.js just works tm. There is still a lot of work to do with configuration and adding built-in tracing to core APIs, which will come in followup PRs. --------- Co-authored-by: Luca Casonato --- cli/standalone/binary.rs | 3 +++ 1 file changed, 3 insertions(+) (limited to 'cli/standalone/binary.rs') diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index 9e2651226..960aad157 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -47,6 +47,7 @@ use deno_runtime::deno_fs::FileSystem; use deno_runtime::deno_fs::RealFs; use deno_runtime::deno_io::fs::FsError; use deno_runtime::deno_node::PackageJson; +use deno_runtime::ops::otel::OtelConfig; use deno_semver::npm::NpmVersionReqParseError; use deno_semver::package::PackageReq; use deno_semver::Version; @@ -185,6 +186,7 @@ pub struct Metadata { pub entrypoint_key: String, pub node_modules: Option, pub unstable_config: UnstableConfig, + pub otel_config: Option, // None means disabled. } fn write_binary_bytes( @@ -722,6 +724,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { sloppy_imports: cli_options.unstable_sloppy_imports(), features: cli_options.unstable_features(), }, + otel_config: cli_options.otel_config(), }; write_binary_bytes( -- cgit v1.2.3 From f091d1ad69b4e5217ae3272b641171781a372c4f Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 13 Nov 2024 10:10:09 -0500 Subject: feat(node): stabilize detecting if CJS via `"type": "commonjs"` in a package.json (#26439) This will respect `"type": "commonjs"` in a package.json to determine if `.js`/`.jsx`/`.ts`/.tsx` files are CJS or ESM. If the file is found to be ESM it will be loaded as ESM though. --- cli/standalone/binary.rs | 1 - 1 file changed, 1 deletion(-) (limited to 'cli/standalone/binary.rs') diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index 960aad157..b48e1c97c 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -720,7 +720,6 @@ impl<'a> DenoCompileBinaryWriter<'a> { unstable_config: UnstableConfig { legacy_flag_enabled: false, bare_node_builtins: cli_options.unstable_bare_node_builtins(), - detect_cjs: cli_options.unstable_detect_cjs(), sloppy_imports: cli_options.unstable_sloppy_imports(), features: cli_options.unstable_features(), }, -- cgit v1.2.3 From 4e899d48cffa95617266dd8f9aef54603a87ad82 Mon Sep 17 00:00:00 2001 From: snek Date: Thu, 14 Nov 2024 13:16:28 +0100 Subject: fix: otel resiliency (#26857) Improving the breadth of collected data, and ensuring that the collected data is more likely to be successfully reported. - Use `log` crate in more places - Hook up `log` crate to otel - Switch to process-wide otel processors - Handle places that use `process::exit` Also adds a more robust testing framework, with a deterministic tracing setting. Refs: https://github.com/denoland/deno/issues/26852 --- cli/standalone/binary.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'cli/standalone/binary.rs') diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index b48e1c97c..ebcbf3ee6 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -517,7 +517,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { Some(bytes) => bytes, None => { log::info!("Download could not be found, aborting"); - std::process::exit(1) + deno_runtime::exit(1); } }; -- cgit v1.2.3 From cff6e280c77afb0bc42a10348eeef5360db8f361 Mon Sep 17 00:00:00 2001 From: Bhuwan Pandit Date: Sun, 17 Nov 2024 22:49:35 +0000 Subject: feat(cli): support multiple env file argument (#26527) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #26425 ## Overview This PR adds support for specifying multiple environment files as arguments when using the Deno CLI. Subsequent files override pre-existing variables defined in previous files. If the same variable is defined in the environment and in the file, the value from the environment takes precedence. ## Example Usage ```bash deno run --allow-env --env-file --env-file=".env.one" --env-file=".env.two" script.ts ``` --------- Co-authored-by: Bartek Iwańczuk --- cli/standalone/binary.rs | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) (limited to 'cli/standalone/binary.rs') diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index ebcbf3ee6..3efd8ee14 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -659,9 +659,15 @@ impl<'a> DenoCompileBinaryWriter<'a> { remote_modules_store.add_redirects(&graph.redirects); let env_vars_from_env_file = match cli_options.env_file_name() { - Some(env_filename) => { - log::info!("{} Environment variables from the file \"{}\" were embedded in the generated executable file", crate::colors::yellow("Warning"), env_filename); - get_file_env_vars(env_filename.to_string())? + Some(env_filenames) => { + let mut aggregated_env_vars = IndexMap::new(); + for env_filename in env_filenames.iter().rev() { + log::info!("{} Environment variables from the file \"{}\" were embedded in the generated executable file", crate::colors::yellow("Warning"), env_filename); + + let env_vars = get_file_env_vars(env_filename.to_string())?; + aggregated_env_vars.extend(env_vars); + } + aggregated_env_vars } None => Default::default(), }; -- cgit v1.2.3 From dd4570ed85888d9659a2eec98437dbd6de4a5799 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 18 Nov 2024 15:09:28 -0500 Subject: perf(compile): code cache (#26528) Adds a lazily created code cache to `deno compile` by default. The code cache is created on first run to a single file in the temp directory and is only written once. After it's been written, the code cache becomes read only on subsequent runs. Only the modules loaded during startup are cached (dynamic imports are not code cached). The code cache can be disabled by compiling with `--no-code-cache`. --- cli/standalone/binary.rs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'cli/standalone/binary.rs') diff --git a/cli/standalone/binary.rs b/cli/standalone/binary.rs index 3efd8ee14..37753bafc 100644 --- a/cli/standalone/binary.rs +++ b/cli/standalone/binary.rs @@ -64,6 +64,7 @@ use crate::args::NpmInstallDepsProvider; use crate::args::PermissionFlags; use crate::args::UnstableConfig; use crate::cache::DenoDir; +use crate::cache::FastInsecureHasher; use crate::emit::Emitter; use crate::file_fetcher::FileFetcher; use crate::http_util::HttpClientProvider; @@ -174,6 +175,7 @@ pub struct SerializedWorkspaceResolver { pub struct Metadata { pub argv: Vec, pub seed: Option, + pub code_cache_key: Option, pub permissions: PermissionFlags, pub location: Option, pub v8_flags: Vec, @@ -604,10 +606,21 @@ impl<'a> DenoCompileBinaryWriter<'a> { VfsBuilder::new(root_path.clone())? }; let mut remote_modules_store = RemoteModulesStoreBuilder::default(); + let mut code_cache_key_hasher = if cli_options.code_cache_enabled() { + Some(FastInsecureHasher::new_deno_versioned()) + } else { + None + }; for module in graph.modules() { if module.specifier().scheme() == "data" { continue; // don't store data urls as an entry as they're in the code } + if let Some(hasher) = &mut code_cache_key_hasher { + if let Some(source) = module.source() { + hasher.write(module.specifier().as_str().as_bytes()); + hasher.write(source.as_bytes()); + } + } let (maybe_source, media_type) = match module { deno_graph::Module::Js(m) => { let source = if m.media_type.is_emittable() { @@ -675,6 +688,7 @@ impl<'a> DenoCompileBinaryWriter<'a> { let metadata = Metadata { argv: compile_flags.args.clone(), seed: cli_options.seed(), + code_cache_key: code_cache_key_hasher.map(|h| h.finish()), location: cli_options.location_flag().clone(), permissions: cli_options.permission_flags().clone(), v8_flags: cli_options.v8_flags().clone(), -- cgit v1.2.3