From 28aa489de9cd4f995ec2fc02e2c9d224e89f4c01 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 10 May 2023 20:06:59 -0400 Subject: feat(compile): unstable npm and node specifier support (#19005) This is the initial support for npm and node specifiers in `deno compile`. The npm packages are included in the binary and read from it via a virtual file system. This also supports the `--node-modules-dir` flag, dependencies specified in a package.json, and npm binary commands (ex. `deno compile --unstable npm:cowsay`) Closes #16632 --- cli/tools/compile.rs | 267 ++++++++++++++++++++++++++++++++++++++++++++++ cli/tools/mod.rs | 2 +- cli/tools/standalone.rs | 270 ----------------------------------------------- cli/tools/task.rs | 3 +- cli/tools/vendor/test.rs | 9 +- 5 files changed, 272 insertions(+), 279 deletions(-) create mode 100644 cli/tools/compile.rs delete mode 100644 cli/tools/standalone.rs (limited to 'cli/tools') diff --git a/cli/tools/compile.rs b/cli/tools/compile.rs new file mode 100644 index 000000000..f10a2d025 --- /dev/null +++ b/cli/tools/compile.rs @@ -0,0 +1,267 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +use crate::args::CompileFlags; +use crate::args::Flags; +use crate::factory::CliFactory; +use crate::graph_util::error_for_any_npm_specifier; +use crate::standalone::is_standalone_binary; +use crate::util::path::path_has_trailing_slash; +use deno_core::anyhow::bail; +use deno_core::anyhow::Context; +use deno_core::error::generic_error; +use deno_core::error::AnyError; +use deno_core::resolve_url_or_path; +use deno_runtime::colors; +use std::path::Path; +use std::path::PathBuf; +use std::sync::Arc; + +use super::installer::infer_name_from_url; + +pub async fn compile( + flags: Flags, + compile_flags: CompileFlags, +) -> Result<(), AnyError> { + let factory = CliFactory::from_flags(flags).await?; + let cli_options = factory.cli_options(); + let module_graph_builder = factory.module_graph_builder().await?; + let parsed_source_cache = factory.parsed_source_cache()?; + let binary_writer = factory.create_compile_binary_writer().await?; + let module_specifier = cli_options.resolve_main_module()?; + let module_roots = { + let mut vec = Vec::with_capacity(compile_flags.include.len() + 1); + vec.push(module_specifier.clone()); + for side_module in &compile_flags.include { + vec.push(resolve_url_or_path(side_module, cli_options.initial_cwd())?); + } + vec + }; + + let output_path = resolve_compile_executable_output_path( + &compile_flags, + cli_options.initial_cwd(), + ) + .await?; + + let graph = Arc::try_unwrap( + module_graph_builder + .create_graph_and_maybe_check(module_roots) + .await?, + ) + .unwrap(); + + if !cli_options.unstable() { + error_for_any_npm_specifier(&graph).context( + "Using npm specifiers with deno compile requires the --unstable flag.", + )?; + } + + let parser = parsed_source_cache.as_capturing_parser(); + let eszip = eszip::EszipV2::from_graph(graph, &parser, Default::default())?; + + log::info!( + "{} {} to {}", + colors::green("Compile"), + module_specifier.to_string(), + output_path.display(), + ); + validate_output_path(&output_path)?; + + let mut file = std::fs::File::create(&output_path)?; + binary_writer + .write_bin( + &mut file, + eszip, + &module_specifier, + &compile_flags, + cli_options, + ) + .await + .with_context(|| format!("Writing {}", output_path.display()))?; + drop(file); + + // set it as executable + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + let perms = std::fs::Permissions::from_mode(0o777); + std::fs::set_permissions(output_path, perms)?; + } + + Ok(()) +} + +/// This function writes out a final binary to specified path. If output path +/// is not already standalone binary it will return error instead. +fn validate_output_path(output_path: &Path) -> Result<(), AnyError> { + if output_path.exists() { + // If the output is a directory, throw error + if output_path.is_dir() { + bail!( + concat!( + "Could not compile to file '{}' because a directory exists with ", + "the same name. You can use the `--output ` flag to ", + "provide an alternative name." + ), + output_path.display() + ); + } + + // Make sure we don't overwrite any file not created by Deno compiler because + // this filename is chosen automatically in some cases. + if !is_standalone_binary(output_path) { + bail!( + concat!( + "Could not compile to file '{}' because the file already exists ", + "and cannot be overwritten. Please delete the existing file or ", + "use the `--output ` flag to ", + "provide an alternative name.", + ), + output_base.display(), + ); + } + std::fs::create_dir_all(output_base)?; + } + + Ok(()) +} + +async fn resolve_compile_executable_output_path( + compile_flags: &CompileFlags, + current_dir: &Path, +) -> Result { + let module_specifier = + resolve_url_or_path(&compile_flags.source_file, current_dir)?; + + let mut output = compile_flags.output.clone(); + + if let Some(out) = output.as_ref() { + if path_has_trailing_slash(out) { + if let Some(infer_file_name) = infer_name_from_url(&module_specifier) + .await + .map(PathBuf::from) + { + output = Some(out.join(infer_file_name)); + } + } else { + output = Some(out.to_path_buf()); + } + } + + if output.is_none() { + output = infer_name_from_url(&module_specifier) + .await + .map(PathBuf::from) + } + + output.ok_or_else(|| generic_error( + "An executable name was not provided. One could not be inferred from the URL. Aborting.", + )).map(|output| { + get_os_specific_filepath(output, &compile_flags.target) + }) +} + +fn get_os_specific_filepath( + output: PathBuf, + target: &Option, +) -> PathBuf { + let is_windows = match target { + Some(target) => target.contains("windows"), + None => cfg!(windows), + }; + if is_windows && output.extension().unwrap_or_default() != "exe" { + if let Some(ext) = output.extension() { + // keep version in my-exe-0.1.0 -> my-exe-0.1.0.exe + output.with_extension(format!("{}.exe", ext.to_string_lossy())) + } else { + output.with_extension("exe") + } + } else { + output + } +} + +#[cfg(test)] +mod test { + pub use super::*; + + #[tokio::test] + async fn resolve_compile_executable_output_path_target_linux() { + let path = resolve_compile_executable_output_path( + &CompileFlags { + source_file: "mod.ts".to_string(), + output: Some(PathBuf::from("./file")), + args: Vec::new(), + target: Some("x86_64-unknown-linux-gnu".to_string()), + include: vec![], + }, + &std::env::current_dir().unwrap(), + ) + .await + .unwrap(); + + // no extension, no matter what the operating system is + // because the target was specified as linux + // https://github.com/denoland/deno/issues/9667 + assert_eq!(path.file_name().unwrap(), "file"); + } + + #[tokio::test] + async fn resolve_compile_executable_output_path_target_windows() { + let path = resolve_compile_executable_output_path( + &CompileFlags { + source_file: "mod.ts".to_string(), + output: Some(PathBuf::from("./file")), + args: Vec::new(), + target: Some("x86_64-pc-windows-msvc".to_string()), + include: vec![], + }, + &std::env::current_dir().unwrap(), + ) + .await + .unwrap(); + assert_eq!(path.file_name().unwrap(), "file.exe"); + } + + #[test] + fn test_os_specific_file_path() { + fn run_test(path: &str, target: Option<&str>, expected: &str) { + assert_eq!( + get_os_specific_filepath( + PathBuf::from(path), + &target.map(|s| s.to_string()) + ), + PathBuf::from(expected) + ); + } + + if cfg!(windows) { + run_test("C:\\my-exe", None, "C:\\my-exe.exe"); + run_test("C:\\my-exe.exe", None, "C:\\my-exe.exe"); + run_test("C:\\my-exe-0.1.2", None, "C:\\my-exe-0.1.2.exe"); + } else { + run_test("my-exe", Some("linux"), "my-exe"); + run_test("my-exe-0.1.2", Some("linux"), "my-exe-0.1.2"); + } + + run_test("C:\\my-exe", Some("windows"), "C:\\my-exe.exe"); + run_test("C:\\my-exe.exe", Some("windows"), "C:\\my-exe.exe"); + run_test("C:\\my-exe.0.1.2", Some("windows"), "C:\\my-exe.0.1.2.exe"); + run_test("my-exe-0.1.2", Some("linux"), "my-exe-0.1.2"); + } +} diff --git a/cli/tools/mod.rs b/cli/tools/mod.rs index cf29435a7..c4a8306ab 100644 --- a/cli/tools/mod.rs +++ b/cli/tools/mod.rs @@ -3,6 +3,7 @@ pub mod bench; pub mod bundle; pub mod check; +pub mod compile; pub mod coverage; pub mod doc; pub mod fmt; @@ -12,7 +13,6 @@ pub mod installer; pub mod lint; pub mod repl; pub mod run; -pub mod standalone; pub mod task; pub mod test; pub mod upgrade; diff --git a/cli/tools/standalone.rs b/cli/tools/standalone.rs deleted file mode 100644 index d34e5da83..000000000 --- a/cli/tools/standalone.rs +++ /dev/null @@ -1,270 +0,0 @@ -// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. - -use crate::args::CompileFlags; -use crate::args::Flags; -use crate::factory::CliFactory; -use crate::graph_util::error_for_any_npm_specifier; -use crate::standalone::is_standalone_binary; -use crate::standalone::DenoCompileBinaryWriter; -use crate::util::path::path_has_trailing_slash; -use deno_core::anyhow::bail; -use deno_core::anyhow::Context; -use deno_core::error::generic_error; -use deno_core::error::AnyError; -use deno_core::resolve_url_or_path; -use deno_runtime::colors; -use std::path::Path; -use std::path::PathBuf; -use std::sync::Arc; - -use super::installer::infer_name_from_url; - -pub async fn compile( - flags: Flags, - compile_flags: CompileFlags, -) -> Result<(), AnyError> { - let factory = CliFactory::from_flags(flags).await?; - let cli_options = factory.cli_options(); - let file_fetcher = factory.file_fetcher()?; - let http_client = factory.http_client(); - let deno_dir = factory.deno_dir()?; - let module_graph_builder = factory.module_graph_builder().await?; - let parsed_source_cache = factory.parsed_source_cache()?; - - let binary_writer = - DenoCompileBinaryWriter::new(file_fetcher, http_client, deno_dir); - let module_specifier = cli_options.resolve_main_module()?; - let module_roots = { - let mut vec = Vec::with_capacity(compile_flags.include.len() + 1); - vec.push(module_specifier.clone()); - for side_module in &compile_flags.include { - vec.push(resolve_url_or_path(side_module, cli_options.initial_cwd())?); - } - vec - }; - - let output_path = resolve_compile_executable_output_path( - &compile_flags, - cli_options.initial_cwd(), - ) - .await?; - - let graph = Arc::try_unwrap( - module_graph_builder - .create_graph_and_maybe_check(module_roots) - .await?, - ) - .unwrap(); - - // at the moment, we don't support npm specifiers in deno_compile, so show an error - error_for_any_npm_specifier(&graph)?; - - let parser = parsed_source_cache.as_capturing_parser(); - let eszip = eszip::EszipV2::from_graph(graph, &parser, Default::default())?; - - log::info!( - "{} {} to {}", - colors::green("Compile"), - module_specifier.to_string(), - output_path.display(), - ); - validate_output_path(&output_path)?; - - let mut file = std::fs::File::create(&output_path)?; - binary_writer - .write_bin( - &mut file, - eszip, - &module_specifier, - &compile_flags, - cli_options, - ) - .await - .with_context(|| format!("Writing {}", output_path.display()))?; - drop(file); - - // set it as executable - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let perms = std::fs::Permissions::from_mode(0o777); - std::fs::set_permissions(output_path, perms)?; - } - - Ok(()) -} - -/// This function writes out a final binary to specified path. If output path -/// is not already standalone binary it will return error instead. -fn validate_output_path(output_path: &Path) -> Result<(), AnyError> { - if output_path.exists() { - // If the output is a directory, throw error - if output_path.is_dir() { - bail!( - concat!( - "Could not compile to file '{}' because a directory exists with ", - "the same name. You can use the `--output ` flag to ", - "provide an alternative name." - ), - output_path.display() - ); - } - - // Make sure we don't overwrite any file not created by Deno compiler because - // this filename is chosen automatically in some cases. - if !is_standalone_binary(output_path) { - bail!( - concat!( - "Could not compile to file '{}' because the file already exists ", - "and cannot be overwritten. Please delete the existing file or ", - "use the `--output ` flag to ", - "provide an alternative name.", - ), - output_base.display(), - ); - } - std::fs::create_dir_all(output_base)?; - } - - Ok(()) -} - -async fn resolve_compile_executable_output_path( - compile_flags: &CompileFlags, - current_dir: &Path, -) -> Result { - let module_specifier = - resolve_url_or_path(&compile_flags.source_file, current_dir)?; - - let mut output = compile_flags.output.clone(); - - if let Some(out) = output.as_ref() { - if path_has_trailing_slash(out) { - if let Some(infer_file_name) = infer_name_from_url(&module_specifier) - .await - .map(PathBuf::from) - { - output = Some(out.join(infer_file_name)); - } - } else { - output = Some(out.to_path_buf()); - } - } - - if output.is_none() { - output = infer_name_from_url(&module_specifier) - .await - .map(PathBuf::from) - } - - output.ok_or_else(|| generic_error( - "An executable name was not provided. One could not be inferred from the URL. Aborting.", - )).map(|output| { - get_os_specific_filepath(output, &compile_flags.target) - }) -} - -fn get_os_specific_filepath( - output: PathBuf, - target: &Option, -) -> PathBuf { - let is_windows = match target { - Some(target) => target.contains("windows"), - None => cfg!(windows), - }; - if is_windows && output.extension().unwrap_or_default() != "exe" { - if let Some(ext) = output.extension() { - // keep version in my-exe-0.1.0 -> my-exe-0.1.0.exe - output.with_extension(format!("{}.exe", ext.to_string_lossy())) - } else { - output.with_extension("exe") - } - } else { - output - } -} - -#[cfg(test)] -mod test { - pub use super::*; - - #[tokio::test] - async fn resolve_compile_executable_output_path_target_linux() { - let path = resolve_compile_executable_output_path( - &CompileFlags { - source_file: "mod.ts".to_string(), - output: Some(PathBuf::from("./file")), - args: Vec::new(), - target: Some("x86_64-unknown-linux-gnu".to_string()), - include: vec![], - }, - &std::env::current_dir().unwrap(), - ) - .await - .unwrap(); - - // no extension, no matter what the operating system is - // because the target was specified as linux - // https://github.com/denoland/deno/issues/9667 - assert_eq!(path.file_name().unwrap(), "file"); - } - - #[tokio::test] - async fn resolve_compile_executable_output_path_target_windows() { - let path = resolve_compile_executable_output_path( - &CompileFlags { - source_file: "mod.ts".to_string(), - output: Some(PathBuf::from("./file")), - args: Vec::new(), - target: Some("x86_64-pc-windows-msvc".to_string()), - include: vec![], - }, - &std::env::current_dir().unwrap(), - ) - .await - .unwrap(); - assert_eq!(path.file_name().unwrap(), "file.exe"); - } - - #[test] - fn test_os_specific_file_path() { - fn run_test(path: &str, target: Option<&str>, expected: &str) { - assert_eq!( - get_os_specific_filepath( - PathBuf::from(path), - &target.map(|s| s.to_string()) - ), - PathBuf::from(expected) - ); - } - - if cfg!(windows) { - run_test("C:\\my-exe", None, "C:\\my-exe.exe"); - run_test("C:\\my-exe.exe", None, "C:\\my-exe.exe"); - run_test("C:\\my-exe-0.1.2", None, "C:\\my-exe-0.1.2.exe"); - } else { - run_test("my-exe", Some("linux"), "my-exe"); - run_test("my-exe-0.1.2", Some("linux"), "my-exe-0.1.2"); - } - - run_test("C:\\my-exe", Some("windows"), "C:\\my-exe.exe"); - run_test("C:\\my-exe.exe", Some("windows"), "C:\\my-exe.exe"); - run_test("C:\\my-exe.0.1.2", Some("windows"), "C:\\my-exe.0.1.2.exe"); - run_test("my-exe-0.1.2", Some("linux"), "my-exe-0.1.2"); - } -} diff --git a/cli/tools/task.rs b/cli/tools/task.rs index 6380d3822..bf972e2db 100644 --- a/cli/tools/task.rs +++ b/cli/tools/task.rs @@ -64,12 +64,13 @@ pub async fn execute_script( .await; Ok(exit_code) } else if let Some(script) = package_json_scripts.get(task_name) { + let package_json_deps_provider = factory.package_json_deps_provider(); let package_json_deps_installer = factory.package_json_deps_installer().await?; let npm_resolver = factory.npm_resolver().await?; let node_resolver = factory.node_resolver().await?; - if let Some(package_deps) = package_json_deps_installer.package_deps() { + if let Some(package_deps) = package_json_deps_provider.deps() { for (key, value) in package_deps { if let Err(err) = value { log::info!( diff --git a/cli/tools/vendor/test.rs b/cli/tools/vendor/test.rs index 774ff0d58..e8a474ed3 100644 --- a/cli/tools/vendor/test.rs +++ b/cli/tools/vendor/test.rs @@ -22,7 +22,6 @@ use import_map::ImportMap; use crate::cache::ParsedSourceCache; use crate::npm::CliNpmRegistryApi; use crate::npm::NpmResolution; -use crate::npm::PackageJsonDepsInstaller; use crate::resolver::CliGraphResolver; use super::build::VendorEnvironment; @@ -270,18 +269,14 @@ async fn build_test_graph( None, None, )); - let deps_installer = Arc::new(PackageJsonDepsInstaller::new( - npm_registry_api.clone(), - npm_resolution.clone(), - None, - )); CliGraphResolver::new( None, Some(Arc::new(original_import_map)), false, npm_registry_api, npm_resolution, - deps_installer, + Default::default(), + Default::default(), ) }); let mut graph = ModuleGraph::default(); -- cgit v1.2.3