fix(publish): lazily parse sources (#22301)

Closes #22290

4 files changed, 123 insertions, 53 deletions

diff --git a/cli/tools/doc.rs b/cli/tools/doc.rs
index acea65062..729ee05fc 100644
--- a/cli/tools/doc.rs
+++ b/cli/tools/doc.rs
@@ -4,6 +4,7 @@ use crate::args::DocFlags;
 use crate::args::DocHtmlFlag;
 use crate::args::DocSourceFileFlag;
 use crate::args::Flags;
+use crate::cache::LazyGraphSourceParser;
 use crate::colors;
 use crate::diagnostics::Diagnostic;
 use crate::diagnostics::DiagnosticLevel;
@@ -142,7 +143,10 @@ pub async fn doc(flags: Flags, doc_flags: DocFlags) -> Result<(), AnyError> {
 
       if doc_flags.lint {
         let diagnostics = doc_parser.take_diagnostics();
-        check_diagnostics(&**parsed_source_cache, &diagnostics)?;
+        check_diagnostics(
+          LazyGraphSourceParser::new(parsed_source_cache, &graph),
+          &diagnostics,
+        )?;
       }
 
       doc_nodes_by_url
@@ -413,7 +417,7 @@ impl Diagnostic for DocDiagnostic {
 }
 
 fn check_diagnostics(
-  parsed_source_cache: &dyn deno_graph::ParsedSourceStore,
+  source_parser: LazyGraphSourceParser,
   diagnostics: &[DocDiagnostic],
 ) -> Result<(), AnyError> {
   if diagnostics.is_empty() {
@@ -437,8 +441,8 @@ fn check_diagnostics(
     for (_, diagnostics_by_col) in diagnostics_by_lc {
       for (_, diagnostics) in diagnostics_by_col {
         for diagnostic in diagnostics {
-          let sources = SourceTextParsedSourceStore(parsed_source_cache);
-          eprintln!("{}", diagnostic.display(&sources));
+          let sources = SourceTextParsedSourceStore(source_parser);
+          log::error!("{}", diagnostic.display(&sources));
         }
       }
     }
diff --git a/cli/tools/registry/diagnostics.rs b/cli/tools/registry/diagnostics.rs
index 643a4fb2d..e7f947303 100644
--- a/cli/tools/registry/diagnostics.rs
+++ b/cli/tools/registry/diagnostics.rs
@@ -10,9 +10,9 @@ use deno_ast::swc::common::util::take::Take;
 use deno_core::anyhow::anyhow;
 use deno_core::error::AnyError;
 use deno_graph::FastCheckDiagnostic;
-use deno_graph::ParsedSourceStore;
 use lsp_types::Url;
 
+use crate::cache::LazyGraphSourceParser;
 use crate::diagnostics::Diagnostic;
 use crate::diagnostics::DiagnosticLevel;
 use crate::diagnostics::DiagnosticLocation;
@@ -33,7 +33,7 @@ pub struct PublishDiagnosticsCollector {
 impl PublishDiagnosticsCollector {
   pub fn print_and_error(
     &self,
-    sources: &dyn ParsedSourceStore,
+    sources: LazyGraphSourceParser,
   ) -> Result<(), AnyError> {
     let mut errors = 0;
     let mut has_zap_errors = false;
diff --git a/cli/tools/registry/mod.rs b/cli/tools/registry/mod.rs
index 5b3b70c4b..22f53dab4 100644
--- a/cli/tools/registry/mod.rs
+++ b/cli/tools/registry/mod.rs
@@ -27,6 +27,7 @@ use crate::args::deno_registry_url;
 use crate::args::CliOptions;
 use crate::args::Flags;
 use crate::args::PublishFlags;
+use crate::cache::LazyGraphSourceParser;
 use crate::cache::ParsedSourceCache;
 use crate::factory::CliFactory;
 use crate::graph_util::ModuleGraphBuilder;
@@ -90,6 +91,7 @@ fn get_deno_json_package_name(
 async fn prepare_publish(
   deno_json: &ConfigFile,
   source_cache: Arc<ParsedSourceCache>,
+  graph: Arc<deno_graph::ModuleGraph>,
   import_map: Arc<ImportMap>,
   diagnostics_collector: &PublishDiagnosticsCollector,
 ) -> Result<Rc<PreparedPublishPackage>, AnyError> {
@@ -140,7 +142,7 @@ async fn prepare_publish(
     let unfurler = ImportMapUnfurler::new(&import_map);
     tar::create_gzipped_tarball(
       &dir_path,
-      &*source_cache,
+      LazyGraphSourceParser::new(&source_cache, &graph),
       &diagnostics_collector,
       &unfurler,
       file_patterns,
@@ -639,19 +641,19 @@ async fn publish_package(
   Ok(())
 }
 
+struct PreparePackagesData {
+  publish_order_graph: PublishOrderGraph,
+  graph: Arc<deno_graph::ModuleGraph>,
+  package_by_name: HashMap<String, Rc<PreparedPublishPackage>>,
+}
+
 async fn prepare_packages_for_publishing(
   cli_factory: &CliFactory,
   no_zap: bool,
   diagnostics_collector: &PublishDiagnosticsCollector,
   deno_json: ConfigFile,
   import_map: Arc<ImportMap>,
-) -> Result<
-  (
-    PublishOrderGraph,
-    HashMap<String, Rc<PreparedPublishPackage>>,
-  ),
-  AnyError,
-> {
+) -> Result<PreparePackagesData, AnyError> {
   let maybe_workspace_config = deno_json.to_workspace_config()?;
   let module_graph_builder = cli_factory.module_graph_builder().await?.as_ref();
   let source_cache = cli_factory.parsed_source_cache();
@@ -660,7 +662,7 @@ async fn prepare_packages_for_publishing(
 
   let Some(workspace_config) = maybe_workspace_config else {
     let roots = resolve_config_file_roots_from_exports(&deno_json)?;
-    build_and_check_graph_for_publish(
+    let graph = build_and_check_graph_for_publish(
       module_graph_builder,
       type_checker,
       cli_options,
@@ -673,10 +675,10 @@ async fn prepare_packages_for_publishing(
       }],
     )
     .await?;
-    let mut prepared_package_by_name = HashMap::with_capacity(1);
     let package = prepare_publish(
       &deno_json,
       source_cache.clone(),
+      graph.clone(),
       import_map,
       diagnostics_collector,
     )
@@ -684,8 +686,12 @@ async fn prepare_packages_for_publishing(
     let package_name = format!("@{}/{}", package.scope, package.package);
     let publish_order_graph =
       PublishOrderGraph::new_single(package_name.clone());
-    prepared_package_by_name.insert(package_name, package);
-    return Ok((publish_order_graph, prepared_package_by_name));
+    let package_by_name = HashMap::from([(package_name, package)]);
+    return Ok(PreparePackagesData {
+      publish_order_graph,
+      graph,
+      package_by_name,
+    });
   };
 
   println!("Publishing a workspace...");
@@ -701,7 +707,7 @@ async fn prepare_packages_for_publishing(
   )
   .await?;
 
-  let mut prepared_package_by_name =
+  let mut package_by_name =
     HashMap::with_capacity(workspace_config.members.len());
   let publish_order_graph =
     publish_order::build_publish_order_graph(&graph, &roots)?;
@@ -712,11 +718,13 @@ async fn prepare_packages_for_publishing(
     .cloned()
     .map(|member| {
       let import_map = import_map.clone();
+      let graph = graph.clone();
       async move {
         let package = prepare_publish(
           &member.config_file,
           source_cache.clone(),
-          import_map.clone(),
+          graph,
+          import_map,
           diagnostics_collector,
         )
         .await
@@ -731,9 +739,13 @@ async fn prepare_packages_for_publishing(
   let results = deno_core::futures::future::join_all(results).await;
   for result in results {
     let (package_name, package) = result?;
-    prepared_package_by_name.insert(package_name, package);
+    package_by_name.insert(package_name, package);
   }
-  Ok((publish_order_graph, prepared_package_by_name))
+  Ok(PreparePackagesData {
+    publish_order_graph,
+    graph,
+    package_by_name,
+  })
 }
 
 async fn build_and_check_graph_for_publish(
@@ -828,20 +840,22 @@ pub async fn publish(
 
   let diagnostics_collector = PublishDiagnosticsCollector::default();
 
-  let (publish_order_graph, prepared_package_by_name) =
-    prepare_packages_for_publishing(
-      &cli_factory,
-      publish_flags.no_zap,
-      &diagnostics_collector,
-      config_file.clone(),
-      import_map,
-    )
-    .await?;
+  let prepared_data = prepare_packages_for_publishing(
+    &cli_factory,
+    publish_flags.no_zap,
+    &diagnostics_collector,
+    config_file.clone(),
+    import_map,
+  )
+  .await?;
 
-  diagnostics_collector
-    .print_and_error(&**cli_factory.parsed_source_cache())?;
+  let source_parser = LazyGraphSourceParser::new(
+    cli_factory.parsed_source_cache(),
+    &prepared_data.graph,
+  );
+  diagnostics_collector.print_and_error(source_parser)?;
 
-  if prepared_package_by_name.is_empty() {
+  if prepared_data.package_by_name.is_empty() {
     bail!("No packages to publish");
   }
 
@@ -855,8 +869,8 @@ pub async fn publish(
 
   perform_publish(
     cli_factory.http_client(),
-    publish_order_graph,
-    prepared_package_by_name,
+    prepared_data.publish_order_graph,
+    prepared_data.package_by_name,
     auth_method,
   )
   .await
diff --git a/cli/tools/registry/tar.rs b/cli/tools/registry/tar.rs
index 1dcfe2949..e63a76516 100644
--- a/cli/tools/registry/tar.rs
+++ b/cli/tools/registry/tar.rs
@@ -1,6 +1,7 @@
 // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
 
 use bytes::Bytes;
+use deno_ast::MediaType;
 use deno_config::glob::FilePatterns;
 use deno_core::anyhow::Context;
 use deno_core::error::AnyError;
@@ -13,6 +14,7 @@ use std::io::Write;
 use std::path::Path;
 use tar::Header;
 
+use crate::cache::LazyGraphSourceParser;
 use crate::tools::registry::paths::PackagePath;
 use crate::util::import_map::ImportMapUnfurler;
 
@@ -34,7 +36,7 @@ pub struct PublishableTarball {
 
 pub fn create_gzipped_tarball(
   dir: &Path,
-  source_cache: &dyn deno_graph::ParsedSourceStore,
+  source_parser: LazyGraphSourceParser,
   diagnostics_collector: &PublishDiagnosticsCollector,
   unfurler: &ImportMapUnfurler,
   file_patterns: Option<FilePatterns>,
@@ -122,25 +124,17 @@ pub fn create_gzipped_tarball(
         }
       }
 
-      let data = std::fs::read(path).with_context(|| {
-        format!("Unable to read file '{}'", entry.path().display())
-      })?;
+      let content = resolve_content_maybe_unfurling(
+        path,
+        &specifier,
+        unfurler,
+        source_parser,
+        diagnostics_collector,
+      )?;
       files.push(PublishableTarballFile {
         specifier: specifier.clone(),
-        size: data.len(),
+        size: content.len(),
       });
-      let content = match source_cache.get_parsed_source(&specifier) {
-        Some(parsed_source) => {
-          let mut reporter = |diagnostic| {
-            diagnostics_collector
-              .push(PublishDiagnostic::ImportMapUnfurl(diagnostic));
-          };
-          let content =
-            unfurler.unfurl(&specifier, &parsed_source, &mut reporter);
-          content.into_bytes()
-        }
-        None => data,
-      };
       tar
         .add_file(format!(".{}", path_str), &content)
         .with_context(|| {
@@ -172,6 +166,64 @@ pub fn create_gzipped_tarball(
   })
 }
 
+fn resolve_content_maybe_unfurling(
+  path: &Path,
+  specifier: &Url,
+  unfurler: &ImportMapUnfurler,
+  source_parser: LazyGraphSourceParser,
+  diagnostics_collector: &PublishDiagnosticsCollector,
+) -> Result<Vec<u8>, AnyError> {
+  let parsed_source = match source_parser.get_or_parse_source(specifier)? {
+    Some(parsed_source) => parsed_source,
+    None => {
+      let data = std::fs::read(path)
+        .with_context(|| format!("Unable to read file '{}'", path.display()))?;
+      let media_type = MediaType::from_specifier(specifier);
+
+      match media_type {
+        MediaType::JavaScript
+        | MediaType::Jsx
+        | MediaType::Mjs
+        | MediaType::Cjs
+        | MediaType::TypeScript
+        | MediaType::Mts
+        | MediaType::Cts
+        | MediaType::Dts
+        | MediaType::Dmts
+        | MediaType::Dcts
+        | MediaType::Tsx => {
+          // continue
+        }
+        MediaType::SourceMap
+        | MediaType::Unknown
+        | MediaType::Json
+        | MediaType::Wasm
+        | MediaType::TsBuildInfo => {
+          // not unfurlable data
+          return Ok(data);
+        }
+      }
+
+      let text = String::from_utf8(data)?;
+      deno_ast::parse_module(deno_ast::ParseParams {
+        specifier: specifier.to_string(),
+        text_info: deno_ast::SourceTextInfo::from_string(text),
+        media_type,
+        capture_tokens: false,
+        maybe_syntax: None,
+        scope_analysis: false,
+      })?
+    }
+  };
+
+  log::debug!("Unfurling {}", specifier);
+  let mut reporter = |diagnostic| {
+    diagnostics_collector.push(PublishDiagnostic::ImportMapUnfurl(diagnostic));
+  };
+  let content = unfurler.unfurl(specifier, &parsed_source, &mut reporter);
+  Ok(content.into_bytes())
+}
+
 struct TarGzArchive {
   builder: tar::Builder<Vec<u8>>,
 }