feat: auto-discover package.json for npm dependencies (#17272)

This commits adds auto-discovery of "package.json" file when running
"deno run" and "deno task" subcommands. In case of "deno run" the
"package.json" is being looked up starting from the directory of the
script that is being run, stopping early if "deno.json(c)" file is found
(ie. FS tree won't be traversed "up" from "deno.json").

When "package.json" is discovered the "--node-modules-dir" flag is
implied, leading to creation of local "node_modules/" directory - we
did that, because most tools relying on "package.json" will expect
"node_modules/" directory to be present (eg. Vite). Additionally 
"dependencies" and "devDependencies" specified in the "package.json"
are downloaded on startup. 

This is a stepping stone to supporting bare specifier imports, but
the actual integration will be done in a follow up commit.

---------

Co-authored-by: David Sherret <dsherret@gmail.com>

4 files changed, 41 insertions, 158 deletions

diff --git a/cli/npm/mod.rs b/cli/npm/mod.rs
index 4be13707e..20db61081 100644
--- a/cli/npm/mod.rs
+++ b/cli/npm/mod.rs
@@ -16,3 +16,4 @@ pub use resolution::NpmPackageNodeId;
 pub use resolution::NpmResolutionPackage;
 pub use resolution::NpmResolutionSnapshot;
 pub use resolvers::NpmPackageResolver;
+pub use resolvers::NpmProcessState;
diff --git a/cli/npm/registry.rs b/cli/npm/registry.rs
index d267d2224..a758ae7be 100644
--- a/cli/npm/registry.rs
+++ b/cli/npm/registry.rs
@@ -24,6 +24,7 @@ use deno_graph::semver::VersionReq;
 use deno_runtime::colors;
 use serde::Serialize;
 
+use crate::args::package_json::parse_dep_entry_name_and_raw_version;
 use crate::args::CacheSetting;
 use crate::cache::CACHE_PERM;
 use crate::http_util::HttpClient;
@@ -113,30 +114,19 @@ impl NpmPackageVersionInfo {
     &self,
   ) -> Result<Vec<NpmDependencyEntry>, AnyError> {
     fn parse_dep_entry(
-      entry: (&String, &String),
+      (key, value): (&String, &String),
       kind: NpmDependencyEntryKind,
     ) -> Result<NpmDependencyEntry, AnyError> {
-      let bare_specifier = entry.0.clone();
       let (name, version_req) =
-        if let Some(package_and_version) = entry.1.strip_prefix("npm:") {
-          if let Some((name, version)) = package_and_version.rsplit_once('@') {
-            (name.to_string(), version.to_string())
-          } else {
-            bail!("could not find @ symbol in npm url '{}'", entry.1);
-          }
-        } else {
-          (entry.0.clone(), entry.1.clone())
-        };
+        parse_dep_entry_name_and_raw_version(key, value)?;
       let version_req =
-        VersionReq::parse_from_npm(&version_req).with_context(|| {
-          format!(
-            "error parsing version requirement for dependency: {bare_specifier}@{version_req}"
-          )
+        VersionReq::parse_from_npm(version_req).with_context(|| {
+          format!("error parsing version requirement for dependency: {key}@{version_req}")
         })?;
       Ok(NpmDependencyEntry {
         kind,
-        bare_specifier,
-        name,
+        bare_specifier: key.to_string(),
+        name: name.to_string(),
         version_req,
         peer_dep_version_req: None,
       })
@@ -339,7 +329,11 @@ impl RealNpmRegistryApiInner {
           .load_package_info_from_registry(name)
           .await
           .with_context(|| {
-          format!("Error getting response at {}", self.get_package_url(name))
+          format!(
+            "Error getting response at {} for package \"{}\"",
+            self.get_package_url(name),
+            name
+          )
         })?;
       }
       let maybe_package_info = maybe_package_info.map(Arc::new);
diff --git a/cli/npm/resolution/specifier.rs b/cli/npm/resolution/specifier.rs
index 36d93bca4..f8b3776a3 100644
--- a/cli/npm/resolution/specifier.rs
+++ b/cli/npm/resolution/specifier.rs
@@ -8,7 +8,6 @@ use std::collections::VecDeque;
 use deno_ast::ModuleSpecifier;
 use deno_graph::npm::NpmPackageReference;
 use deno_graph::npm::NpmPackageReq;
-use deno_graph::semver::VersionReq;
 use deno_graph::ModuleGraph;
 
 pub struct GraphNpmInfo {
@@ -182,7 +181,7 @@ pub fn resolve_graph_npm_info(graph: &ModuleGraph) -> GraphNpmInfo {
 
     let reqs = std::mem::take(&mut leaf.reqs);
     let mut reqs = reqs.into_iter().collect::<Vec<_>>();
-    reqs.sort_by(cmp_package_req);
+    reqs.sort();
     result.extend(reqs);
 
     let mut deps = std::mem::take(&mut leaf.dependencies)
@@ -380,46 +379,6 @@ fn cmp_folder_specifiers(a: &ModuleSpecifier, b: &ModuleSpecifier) -> Ordering {
   }
 }
 
-// Sort the package requirements alphabetically then the version
-// requirement in a way that will lead to the least number of
-// duplicate packages (so sort None last since it's `*`), but
-// mostly to create some determinism around how these are resolved.
-fn cmp_package_req(a: &NpmPackageReq, b: &NpmPackageReq) -> Ordering {
-  fn cmp_specifier_version_req(a: &VersionReq, b: &VersionReq) -> Ordering {
-    match a.tag() {
-      Some(a_tag) => match b.tag() {
-        Some(b_tag) => b_tag.cmp(a_tag), // sort descending
-        None => Ordering::Less,          // prefer a since tag
-      },
-      None => {
-        match b.tag() {
-          Some(_) => Ordering::Greater, // prefer b since tag
-          None => {
-            // At this point, just sort by text descending.
-            // We could maybe be a bit smarter here in the future.
-            b.to_string().cmp(&a.to_string())
-          }
-        }
-      }
-    }
-  }
-
-  match a.name.cmp(&b.name) {
-    Ordering::Equal => {
-      match &b.version_req {
-        Some(b_req) => {
-          match &a.version_req {
-            Some(a_req) => cmp_specifier_version_req(a_req, b_req),
-            None => Ordering::Greater, // prefer b, since a is *
-          }
-        }
-        None => Ordering::Less, // prefer a, since b is *
-      }
-    }
-    ordering => ordering,
-  }
-}
-
 #[cfg(test)]
 mod tests {
   use pretty_assertions::assert_eq;
@@ -485,31 +444,6 @@ mod tests {
   }
 
   #[test]
-  fn sorting_package_reqs() {
-    fn cmp_req(a: &str, b: &str) -> Ordering {
-      let a = NpmPackageReq::from_str(a).unwrap();
-      let b = NpmPackageReq::from_str(b).unwrap();
-      cmp_package_req(&a, &b)
-    }
-
-    // sort by name
-    assert_eq!(cmp_req("a", "b@1"), Ordering::Less);
-    assert_eq!(cmp_req("b@1", "a"), Ordering::Greater);
-    // prefer non-wildcard
-    assert_eq!(cmp_req("a", "a@1"), Ordering::Greater);
-    assert_eq!(cmp_req("a@1", "a"), Ordering::Less);
-    // prefer tag
-    assert_eq!(cmp_req("a@tag", "a"), Ordering::Less);
-    assert_eq!(cmp_req("a", "a@tag"), Ordering::Greater);
-    // sort tag descending
-    assert_eq!(cmp_req("a@latest-v1", "a@latest-v2"), Ordering::Greater);
-    assert_eq!(cmp_req("a@latest-v2", "a@latest-v1"), Ordering::Less);
-    // sort version req descending
-    assert_eq!(cmp_req("a@1", "a@2"), Ordering::Greater);
-    assert_eq!(cmp_req("a@2", "a@1"), Ordering::Less);
-  }
-
-  #[test]
   fn test_get_folder_path_specifier() {
     fn get(a: &str) -> String {
       get_folder_path_specifier(&ModuleSpecifier::parse(a).unwrap()).to_string()
diff --git a/cli/npm/resolvers/mod.rs b/cli/npm/resolvers/mod.rs
index 9dda160b0..a2638a15b 100644
--- a/cli/npm/resolvers/mod.rs
+++ b/cli/npm/resolvers/mod.rs
@@ -17,7 +17,6 @@ use deno_runtime::deno_node::NodeResolutionMode;
 use deno_runtime::deno_node::PathClean;
 use deno_runtime::deno_node::RequireNpmResolver;
 use global::GlobalNpmPackageResolver;
-use once_cell::sync::Lazy;
 use serde::Deserialize;
 use serde::Serialize;
 use std::collections::HashSet;
@@ -35,37 +34,11 @@ use super::NpmPackageNodeId;
 use super::NpmResolutionSnapshot;
 use super::RealNpmRegistryApi;
 
-const RESOLUTION_STATE_ENV_VAR_NAME: &str =
-  "DENO_DONT_USE_INTERNAL_NODE_COMPAT_STATE";
-
-static IS_NPM_MAIN: Lazy<bool> =
-  Lazy::new(|| std::env::var(RESOLUTION_STATE_ENV_VAR_NAME).is_ok());
-
 /// State provided to the process via an environment variable.
-#[derive(Debug, Serialize, Deserialize)]
-struct NpmProcessState {
-  snapshot: NpmResolutionSnapshot,
-  local_node_modules_path: Option<String>,
-}
-
-impl NpmProcessState {
-  pub fn was_set() -> bool {
-    *IS_NPM_MAIN
-  }
-
-  pub fn take() -> Option<NpmProcessState> {
-    // initialize the lazy before we remove the env var below
-    if !Self::was_set() {
-      return None;
-    }
-
-    let state = std::env::var(RESOLUTION_STATE_ENV_VAR_NAME).ok()?;
-    let state = serde_json::from_str(&state).ok()?;
-    // remove the environment variable so that sub processes
-    // that are spawned do not also use this.
-    std::env::remove_var(RESOLUTION_STATE_ENV_VAR_NAME);
-    Some(state)
-  }
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct NpmProcessState {
+  pub snapshot: NpmResolutionSnapshot,
+  pub local_node_modules_path: Option<String>,
 }
 
 #[derive(Clone)]
@@ -89,13 +62,8 @@ impl std::fmt::Debug for NpmPackageResolver {
 }
 
 impl NpmPackageResolver {
-  pub fn new(
-    cache: NpmCache,
-    api: RealNpmRegistryApi,
-    no_npm: bool,
-    local_node_modules_path: Option<PathBuf>,
-  ) -> Self {
-    Self::new_inner(cache, api, no_npm, local_node_modules_path, None, None)
+  pub fn new(cache: NpmCache, api: RealNpmRegistryApi) -> Self {
+    Self::new_inner(cache, api, false, None, None, None)
   }
 
   pub async fn new_with_maybe_lockfile(
@@ -103,32 +71,34 @@ impl NpmPackageResolver {
     api: RealNpmRegistryApi,
     no_npm: bool,
     local_node_modules_path: Option<PathBuf>,
+    initial_snapshot: Option<NpmResolutionSnapshot>,
     maybe_lockfile: Option<Arc<Mutex<Lockfile>>>,
   ) -> Result<Self, AnyError> {
-    let maybe_snapshot = if let Some(lockfile) = &maybe_lockfile {
-      if lockfile.lock().overwrite {
-        None
-      } else {
-        Some(
-          NpmResolutionSnapshot::from_lockfile(lockfile.clone(), &api)
-            .await
-            .with_context(|| {
-              format!(
-                "failed reading lockfile '{}'",
-                lockfile.lock().filename.display()
-              )
-            })?,
-        )
+    let mut initial_snapshot = initial_snapshot;
+
+    if initial_snapshot.is_none() {
+      if let Some(lockfile) = &maybe_lockfile {
+        if !lockfile.lock().overwrite {
+          initial_snapshot = Some(
+            NpmResolutionSnapshot::from_lockfile(lockfile.clone(), &api)
+              .await
+              .with_context(|| {
+                format!(
+                  "failed reading lockfile '{}'",
+                  lockfile.lock().filename.display()
+                )
+              })?,
+          )
+        }
       }
-    } else {
-      None
-    };
+    }
+
     Ok(Self::new_inner(
       cache,
       api,
       no_npm,
       local_node_modules_path,
-      maybe_snapshot,
+      initial_snapshot,
       maybe_lockfile,
     ))
   }
@@ -138,17 +108,9 @@ impl NpmPackageResolver {
     api: RealNpmRegistryApi,
     no_npm: bool,
     local_node_modules_path: Option<PathBuf>,
-    initial_snapshot: Option<NpmResolutionSnapshot>,
+    maybe_snapshot: Option<NpmResolutionSnapshot>,
     maybe_lockfile: Option<Arc<Mutex<Lockfile>>>,
   ) -> Self {
-    let process_npm_state = NpmProcessState::take();
-    let local_node_modules_path = local_node_modules_path.or_else(|| {
-      process_npm_state
-        .as_ref()
-        .and_then(|s| s.local_node_modules_path.as_ref().map(PathBuf::from))
-    });
-    let maybe_snapshot =
-      initial_snapshot.or_else(|| process_npm_state.map(|s| s.snapshot));
     let inner: Arc<dyn InnerNpmPackageResolver> = match &local_node_modules_path
     {
       Some(node_modules_folder) => Arc::new(LocalNpmPackageResolver::new(
@@ -289,14 +251,6 @@ impl NpmPackageResolver {
     self.inner.set_package_reqs(packages).await
   }
 
-  // If the main module should be treated as being in an npm package.
-  // This is triggered via a secret environment variable which is used
-  // for functionality like child_process.fork. Users should NOT depend
-  // on this functionality.
-  pub fn is_npm_main(&self) -> bool {
-    NpmProcessState::was_set()
-  }
-
   /// Gets the state of npm for the process.
   pub fn get_npm_process_state(&self) -> String {
     serde_json::to_string(&NpmProcessState {