From 4d1a14ca7fa9496f36470a7771448a9b006b0204 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Bartek=20Iwa=C5=84czuk?= Date: Mon, 20 Feb 2023 19:14:06 +0100 Subject: feat: auto-discover package.json for npm dependencies (#17272) This commits adds auto-discovery of "package.json" file when running "deno run" and "deno task" subcommands. In case of "deno run" the "package.json" is being looked up starting from the directory of the script that is being run, stopping early if "deno.json(c)" file is found (ie. FS tree won't be traversed "up" from "deno.json"). When "package.json" is discovered the "--node-modules-dir" flag is implied, leading to creation of local "node_modules/" directory - we did that, because most tools relying on "package.json" will expect "node_modules/" directory to be present (eg. Vite). Additionally "dependencies" and "devDependencies" specified in the "package.json" are downloaded on startup. This is a stepping stone to supporting bare specifier imports, but the actual integration will be done in a follow up commit. --------- Co-authored-by: David Sherret --- cli/args/flags.rs | 59 ++++++++++++++-- cli/args/mod.rs | 178 +++++++++++++++++++++++++++++++++++++++++++++-- cli/args/package_json.rs | 167 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 393 insertions(+), 11 deletions(-) create mode 100644 cli/args/package_json.rs (limited to 'cli/args') diff --git a/cli/args/flags.rs b/cli/args/flags.rs index 5d1affb09..2c9f4c09d 100644 --- a/cli/args/flags.rs +++ b/cli/args/flags.rs @@ -327,7 +327,7 @@ pub struct Flags { pub cached_only: bool, pub type_check_mode: TypeCheckMode, pub config_flag: ConfigFlag, - pub node_modules_dir: bool, + pub node_modules_dir: Option, pub coverage_dir: Option, pub enable_testing_features: bool, pub ignore: Vec, @@ -503,6 +503,33 @@ impl Flags { } } + /// Extract path argument for `package.json` search paths. + /// If it returns Some(path), the `package.json` should be discovered + /// from the `path` dir. + /// If it returns None, the `package.json` file shouldn't be discovered at + /// all. + pub fn package_json_arg(&self) -> Option { + use DenoSubcommand::*; + + if let Run(RunFlags { script }) = &self.subcommand { + if let Ok(module_specifier) = deno_core::resolve_url_or_path(script) { + if module_specifier.scheme() == "file" { + let p = module_specifier + .to_file_path() + .unwrap() + .parent()? + .to_owned(); + return Some(p); + } else if module_specifier.scheme() == "npm" { + let p = std::env::current_dir().unwrap(); + return Some(p); + } + } + } + + None + } + pub fn has_permission(&self) -> bool { self.allow_all || self.allow_hrtime @@ -2309,7 +2336,12 @@ fn no_npm_arg<'a>() -> Arg<'a> { fn local_npm_arg<'a>() -> Arg<'a> { Arg::new("node-modules-dir") .long("node-modules-dir") - .help("Creates a local node_modules folder") + .min_values(0) + .max_values(1) + .takes_value(true) + .require_equals(true) + .possible_values(["true", "false"]) + .help("Creates a local node_modules folder. This option is implicitly true when a package.json is auto-discovered.") } fn unsafely_ignore_certificate_errors_arg<'a>() -> Arg<'a> { @@ -3247,9 +3279,7 @@ fn no_npm_arg_parse(flags: &mut Flags, matches: &clap::ArgMatches) { } fn local_npm_args_parse(flags: &mut Flags, matches: &ArgMatches) { - if matches.is_present("node-modules-dir") { - flags.node_modules_dir = true; - } + flags.node_modules_dir = optional_bool_parse(matches, "node-modules-dir"); } fn inspect_arg_validate(val: &str) -> Result<(), String> { @@ -5448,7 +5478,24 @@ mod tests { subcommand: DenoSubcommand::Run(RunFlags { script: "script.ts".to_string(), }), - node_modules_dir: true, + node_modules_dir: Some(true), + ..Flags::default() + } + ); + + let r = flags_from_vec(svec![ + "deno", + "run", + "--node-modules-dir=false", + "script.ts" + ]); + assert_eq!( + r.unwrap(), + Flags { + subcommand: DenoSubcommand::Run(RunFlags { + script: "script.ts".to_string(), + }), + node_modules_dir: Some(false), ..Flags::default() } ); diff --git a/cli/args/mod.rs b/cli/args/mod.rs index da36c7071..7cb2213e9 100644 --- a/cli/args/mod.rs +++ b/cli/args/mod.rs @@ -5,9 +5,13 @@ mod flags; mod flags_allow_net; mod import_map; mod lockfile; +pub mod package_json; pub use self::import_map::resolve_import_map_from_specifier; use ::import_map::ImportMap; + +use crate::npm::NpmResolutionSnapshot; +use crate::util::fs::canonicalize_path; pub use config_file::BenchConfig; pub use config_file::CompilerOptions; pub use config_file::ConfigFile; @@ -32,8 +36,11 @@ use deno_core::anyhow::Context; use deno_core::error::AnyError; use deno_core::normalize_path; use deno_core::parking_lot::Mutex; +use deno_core::serde_json; use deno_core::url::Url; +use deno_graph::npm::NpmPackageReq; use deno_runtime::colors; +use deno_runtime::deno_node::PackageJson; use deno_runtime::deno_tls::rustls; use deno_runtime::deno_tls::rustls::RootCertStore; use deno_runtime::deno_tls::rustls_native_certs::load_native_certs; @@ -41,17 +48,22 @@ use deno_runtime::deno_tls::rustls_pemfile; use deno_runtime::deno_tls::webpki_roots; use deno_runtime::inspector_server::InspectorServer; use deno_runtime::permissions::PermissionsOptions; +use once_cell::sync::Lazy; use std::collections::BTreeMap; +use std::collections::HashMap; +use std::collections::HashSet; use std::env; use std::io::BufReader; use std::io::Cursor; use std::net::SocketAddr; use std::num::NonZeroUsize; +use std::path::Path; use std::path::PathBuf; use std::sync::Arc; use crate::cache::DenoDir; use crate::file_fetcher::FileFetcher; +use crate::npm::NpmProcessState; use crate::util::fs::canonicalize_path_maybe_not_exists; use crate::version; @@ -375,6 +387,74 @@ fn resolve_lint_rules_options( } } +/// Discover `package.json` file. If `maybe_stop_at` is provided, we will stop +/// crawling up the directory tree at that path. +fn discover_package_json( + flags: &Flags, + maybe_stop_at: Option, +) -> Result, AnyError> { + pub fn discover_from( + start: &Path, + checked: &mut HashSet, + maybe_stop_at: Option, + ) -> Result, AnyError> { + const PACKAGE_JSON_NAME: &str = "package.json"; + + for ancestor in start.ancestors() { + if checked.insert(ancestor.to_path_buf()) { + let path = ancestor.join(PACKAGE_JSON_NAME); + + let source = match std::fs::read_to_string(&path) { + Ok(source) => source, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + if let Some(stop_at) = maybe_stop_at.as_ref() { + if ancestor == stop_at { + break; + } + } + continue; + } + Err(err) => bail!( + "Error loading package.json at {}. {:#}", + path.display(), + err + ), + }; + + let package_json = PackageJson::load_from_string(path.clone(), source)?; + log::debug!("package.json file found at '{}'", path.display()); + return Ok(Some(package_json)); + } + } + // No config file found. + log::debug!("No package.json file found"); + Ok(None) + } + + // TODO(bartlomieju): discover for all subcommands, but print warnings that + // `package.json` is ignored in bundle/compile/etc. + + if let Some(package_json_arg) = flags.package_json_arg() { + return discover_from( + &package_json_arg, + &mut HashSet::new(), + maybe_stop_at, + ); + } else if let crate::args::DenoSubcommand::Task(TaskFlags { + cwd: Some(path), + .. + }) = &flags.subcommand + { + // attempt to resolve the config file from the task subcommand's + // `--cwd` when specified + let task_cwd = canonicalize_path(&PathBuf::from(path))?; + return discover_from(&task_cwd, &mut HashSet::new(), None); + } + + log::debug!("No package.json file found"); + Ok(None) +} + /// Create and populate a root cert store based on the passed options and /// environment. pub fn get_root_cert_store( @@ -459,6 +539,21 @@ pub fn get_root_cert_store( Ok(root_cert_store) } +const RESOLUTION_STATE_ENV_VAR_NAME: &str = + "DENO_DONT_USE_INTERNAL_NODE_COMPAT_STATE"; + +static IS_NPM_MAIN: Lazy = + Lazy::new(|| std::env::var(RESOLUTION_STATE_ENV_VAR_NAME).is_ok()); + +static NPM_PROCESS_STATE: Lazy> = Lazy::new(|| { + let state = std::env::var(RESOLUTION_STATE_ENV_VAR_NAME).ok()?; + let state: NpmProcessState = serde_json::from_str(&state).ok()?; + // remove the environment variable so that sub processes + // that are spawned do not also use this. + std::env::remove_var(RESOLUTION_STATE_ENV_VAR_NAME); + Some(state) +}); + /// Overrides for the options below that when set will /// use these values over the values derived from the /// CLI flags or config file. @@ -474,6 +569,7 @@ pub struct CliOptions { // application need not concern itself with, so keep these private flags: Flags, maybe_config_file: Option, + maybe_package_json: Option, maybe_lockfile: Option>>, overrides: CliOptionOverrides, } @@ -483,6 +579,7 @@ impl CliOptions { flags: Flags, maybe_config_file: Option, maybe_lockfile: Option, + maybe_package_json: Option, ) -> Self { if let Some(insecure_allowlist) = flags.unsafely_ignore_certificate_errors.as_ref() @@ -503,6 +600,7 @@ impl CliOptions { Self { maybe_config_file, maybe_lockfile, + maybe_package_json, flags, overrides: Default::default(), } @@ -510,9 +608,30 @@ impl CliOptions { pub fn from_flags(flags: Flags) -> Result { let maybe_config_file = ConfigFile::discover(&flags)?; + + let mut maybe_package_json = None; + if let Some(config_file) = &maybe_config_file { + let specifier = config_file.specifier.clone(); + if specifier.scheme() == "file" { + let maybe_stop_at = specifier + .to_file_path() + .unwrap() + .parent() + .map(|p| p.to_path_buf()); + + maybe_package_json = discover_package_json(&flags, maybe_stop_at)?; + } + } else { + maybe_package_json = discover_package_json(&flags, None)?; + } let maybe_lock_file = lockfile::discover(&flags, maybe_config_file.as_ref())?; - Ok(Self::new(flags, maybe_config_file, maybe_lock_file)) + Ok(Self::new( + flags, + maybe_config_file, + maybe_lock_file, + maybe_package_json, + )) } pub fn maybe_config_file_specifier(&self) -> Option { @@ -576,7 +695,7 @@ impl CliOptions { }; resolve_import_map_from_specifier( &import_map_specifier, - self.get_maybe_config_file().as_ref(), + self.maybe_config_file().as_ref(), file_fetcher, ) .await @@ -586,21 +705,56 @@ impl CliOptions { .map(Some) } + fn get_npm_process_state(&self) -> Option<&NpmProcessState> { + if !self.is_npm_main() { + return None; + } + + (*NPM_PROCESS_STATE).as_ref() + } + + pub fn get_npm_resolution_snapshot(&self) -> Option { + if let Some(state) = self.get_npm_process_state() { + // TODO(bartlomieju): remove this clone + return Some(state.snapshot.clone()); + } + + None + } + + // If the main module should be treated as being in an npm package. + // This is triggered via a secret environment variable which is used + // for functionality like child_process.fork. Users should NOT depend + // on this functionality. + pub fn is_npm_main(&self) -> bool { + *IS_NPM_MAIN + } + /// Overrides the import map specifier to use. pub fn set_import_map_specifier(&mut self, path: Option) { self.overrides.import_map_specifier = Some(path); } pub fn node_modules_dir(&self) -> bool { - self.flags.node_modules_dir + if let Some(node_modules_dir) = self.flags.node_modules_dir { + return node_modules_dir; + } + + if let Some(npm_process_state) = self.get_npm_process_state() { + return npm_process_state.local_node_modules_path.is_some(); + } + + self.maybe_package_json.is_some() } /// Resolves the path to use for a local node_modules folder. pub fn resolve_local_node_modules_folder( &self, ) -> Result, AnyError> { - let path = if !self.flags.node_modules_dir { + let path = if !self.node_modules_dir() { return Ok(None); + } else if let Some(state) = self.get_npm_process_state() { + return Ok(state.local_node_modules_path.as_ref().map(PathBuf::from)); } else if let Some(config_path) = self .maybe_config_file .as_ref() @@ -699,10 +853,24 @@ impl CliOptions { } } - pub fn get_maybe_config_file(&self) -> &Option { + pub fn maybe_config_file(&self) -> &Option { &self.maybe_config_file } + pub fn maybe_package_json(&self) -> &Option { + &self.maybe_package_json + } + + pub fn maybe_package_json_deps( + &self, + ) -> Result>, AnyError> { + if let Some(package_json) = self.maybe_package_json() { + package_json::get_local_package_json_version_reqs(package_json).map(Some) + } else { + Ok(None) + } + } + pub fn resolve_fmt_options( &self, fmt_flags: FmtFlags, diff --git a/cli/args/package_json.rs b/cli/args/package_json.rs new file mode 100644 index 000000000..76d353c5e --- /dev/null +++ b/cli/args/package_json.rs @@ -0,0 +1,167 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. + +use std::collections::HashMap; + +use deno_core::anyhow::anyhow; +use deno_core::anyhow::bail; +use deno_core::error::AnyError; +use deno_graph::npm::NpmPackageReq; +use deno_graph::semver::VersionReq; +use deno_runtime::deno_node::PackageJson; + +/// Gets the name and raw version constraint taking into account npm +/// package aliases. +pub fn parse_dep_entry_name_and_raw_version<'a>( + key: &'a str, + value: &'a str, +) -> Result<(&'a str, &'a str), AnyError> { + if let Some(package_and_version) = value.strip_prefix("npm:") { + if let Some((name, version)) = package_and_version.rsplit_once('@') { + Ok((name, version)) + } else { + bail!("could not find @ symbol in npm url '{}'", value); + } + } else { + Ok((key, value)) + } +} + +/// Gets an application level package.json's npm package requirements. +/// +/// Note that this function is not general purpose. It is specifically for +/// parsing the application level package.json that the user has control +/// over. This is a design limitation to allow mapping these dependency +/// entries to npm specifiers which can then be used in the resolver. +pub fn get_local_package_json_version_reqs( + package_json: &PackageJson, +) -> Result, AnyError> { + fn insert_deps( + deps: Option<&HashMap>, + result: &mut HashMap, + ) -> Result<(), AnyError> { + if let Some(deps) = deps { + for (key, value) in deps { + let (name, version_req) = + parse_dep_entry_name_and_raw_version(key, value)?; + + let version_req = { + let result = VersionReq::parse_from_specifier(version_req); + match result { + Ok(version_req) => version_req, + Err(e) => { + let err = anyhow!("{:#}", e).context(concat!( + "Parsing version constraints in the application-level ", + "package.json is more strict at the moment" + )); + return Err(err); + } + } + }; + result.insert( + key.to_string(), + NpmPackageReq { + name: name.to_string(), + version_req: Some(version_req), + }, + ); + } + } + Ok(()) + } + + let deps = package_json.dependencies.as_ref(); + let dev_deps = package_json.dev_dependencies.as_ref(); + let mut result = HashMap::with_capacity( + deps.map(|d| d.len()).unwrap_or(0) + dev_deps.map(|d| d.len()).unwrap_or(0), + ); + + // insert the dev dependencies first so the dependencies will + // take priority and overwrite any collisions + insert_deps(dev_deps, &mut result)?; + insert_deps(deps, &mut result)?; + + Ok(result) +} + +#[cfg(test)] +mod test { + use pretty_assertions::assert_eq; + use std::path::PathBuf; + + use super::*; + + #[test] + fn test_parse_dep_entry_name_and_raw_version() { + let cases = [ + ("test", "^1.2", Ok(("test", "^1.2"))), + ("test", "1.x - 2.6", Ok(("test", "1.x - 2.6"))), + ("test", "npm:package@^1.2", Ok(("package", "^1.2"))), + ( + "test", + "npm:package", + Err("could not find @ symbol in npm url 'npm:package'"), + ), + ]; + for (key, value, expected_result) in cases { + let result = parse_dep_entry_name_and_raw_version(key, value); + match result { + Ok(result) => assert_eq!(result, expected_result.unwrap()), + Err(err) => assert_eq!(err.to_string(), expected_result.err().unwrap()), + } + } + } + + #[test] + fn test_get_local_package_json_version_reqs() { + let mut package_json = PackageJson::empty(PathBuf::from("/package.json")); + package_json.dependencies = Some(HashMap::from([ + ("test".to_string(), "^1.2".to_string()), + ("other".to_string(), "npm:package@~1.3".to_string()), + ])); + package_json.dev_dependencies = Some(HashMap::from([ + ("package_b".to_string(), "~2.2".to_string()), + // should be ignored + ("other".to_string(), "^3.2".to_string()), + ])); + let result = get_local_package_json_version_reqs(&package_json).unwrap(); + assert_eq!( + result, + HashMap::from([ + ( + "test".to_string(), + NpmPackageReq::from_str("test@^1.2").unwrap() + ), + ( + "other".to_string(), + NpmPackageReq::from_str("package@~1.3").unwrap() + ), + ( + "package_b".to_string(), + NpmPackageReq::from_str("package_b@~2.2").unwrap() + ) + ]) + ); + } + + #[test] + fn test_get_local_package_json_version_reqs_errors_non_npm_specifier() { + let mut package_json = PackageJson::empty(PathBuf::from("/package.json")); + package_json.dependencies = Some(HashMap::from([( + "test".to_string(), + "1.x - 1.3".to_string(), + )])); + let err = get_local_package_json_version_reqs(&package_json) + .err() + .unwrap(); + assert_eq!( + format!("{err:#}"), + concat!( + "Parsing version constraints in the application-level ", + "package.json is more strict at the moment: Invalid npm specifier ", + "version requirement. Unexpected character.\n", + " - 1.3\n", + " ~" + ) + ); + } +} -- cgit v1.2.3