diff options
Diffstat (limited to 'cli/tools')
| -rw-r--r-- | cli/tools/check.rs | 108 | ||||
| -rw-r--r-- | cli/tools/registry/graph.rs | 154 | ||||
| -rw-r--r-- | cli/tools/registry/mod.rs | 113 | ||||
| -rw-r--r-- | cli/tools/registry/publish_order.rs | 89 |
4 files changed, 345 insertions, 119 deletions
diff --git a/cli/tools/check.rs b/cli/tools/check.rs index bde9a7612..7133f4d3f 100644 --- a/cli/tools/check.rs +++ b/cli/tools/check.rs @@ -1,6 +1,7 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. use std::collections::HashSet; +use std::collections::VecDeque; use std::sync::Arc; use deno_ast::MediaType; @@ -23,6 +24,7 @@ use crate::cache::FastInsecureHasher; use crate::cache::TypeCheckCache; use crate::npm::CliNpmResolver; use crate::tsc; +use crate::tsc::Diagnostics; use crate::version; /// Options for performing a check of a module graph. Note that the decision to @@ -68,6 +70,23 @@ impl TypeChecker { graph: Arc<ModuleGraph>, options: CheckOptions, ) -> Result<(), AnyError> { + let diagnostics = self.check_diagnostics(graph, options).await?; + if diagnostics.is_empty() { + Ok(()) + } else { + Err(diagnostics.into()) + } + } + + /// Type check the module graph returning its diagnostics. + /// + /// It is expected that it is determined if a check and/or emit is validated + /// before the function is called. + pub async fn check_diagnostics( + &self, + graph: Arc<ModuleGraph>, + options: CheckOptions, + ) -> Result<Diagnostics, AnyError> { // node built-in specifiers use the @types/node package to determine // types, so inject that now (the caller should do this after the lockfile // has been written) @@ -100,7 +119,7 @@ impl TypeChecker { type_check_mode, &ts_config, ) { - CheckHashResult::NoFiles => return Ok(()), + CheckHashResult::NoFiles => return Ok(Default::default()), CheckHashResult::Hash(hash) => Some(hash), } } @@ -111,7 +130,7 @@ impl TypeChecker { if !options.reload { if let Some(check_hash) = maybe_check_hash { if cache.has_check_hash(check_hash) { - return Ok(()); + return Ok(Default::default()); } } } @@ -152,7 +171,7 @@ impl TypeChecker { check_mode: type_check_mode, })?; - let diagnostics = if type_check_mode == TypeCheckMode::Local { + let mut diagnostics = if type_check_mode == TypeCheckMode::Local { response.diagnostics.filter(|d| { if let Some(file_name) = &d.file_name { if !file_name.starts_with("http") { @@ -175,6 +194,8 @@ impl TypeChecker { response.diagnostics }; + diagnostics.apply_fast_check_source_maps(&graph); + if let Some(tsbuildinfo) = response.maybe_tsbuildinfo { cache.set_tsbuildinfo(&graph.roots[0], &tsbuildinfo); } @@ -187,11 +208,7 @@ impl TypeChecker { log::debug!("{}", response.stats); - if diagnostics.is_empty() { - Ok(()) - } else { - Err(diagnostics.into()) - } + Ok(diagnostics) } } @@ -256,7 +273,12 @@ fn get_check_hash( } hasher.write_str(module.specifier.as_str()); - hasher.write_str(&module.source); + hasher.write_str( + module + .fast_check_module() + .map(|s| s.source.as_ref()) + .unwrap_or(&module.source), + ); } Module::Node(_) => { // the @types/node package will be in the resolved @@ -345,21 +367,18 @@ fn get_tsc_roots( )); } - let mut seen_roots = - HashSet::with_capacity(graph.imports.len() + graph.roots.len()); + let mut seen = + HashSet::with_capacity(graph.imports.len() + graph.specifiers_count()); + let mut pending = VecDeque::new(); // put in the global types first so that they're resolved before anything else for import in graph.imports.values() { for dep in import.dependencies.values() { let specifier = dep.get_type().or_else(|| dep.get_code()); if let Some(specifier) = &specifier { - if seen_roots.insert(*specifier) { - let maybe_entry = graph - .get(specifier) - .and_then(|m| maybe_get_check_entry(m, check_js)); - if let Some(entry) = maybe_entry { - result.push(entry); - } + let specifier = graph.resolve(specifier); + if seen.insert(specifier.clone()) { + pending.push_back(specifier); } } } @@ -367,23 +386,50 @@ fn get_tsc_roots( // then the roots for root in &graph.roots { - if let Some(module) = graph.get(root) { - if seen_roots.insert(root) { - if let Some(entry) = maybe_get_check_entry(module, check_js) { - result.push(entry); - } - } + let specifier = graph.resolve(root); + if seen.insert(specifier.clone()) { + pending.push_back(specifier); } } - // now the rest - result.extend(graph.modules().filter_map(|module| { - if seen_roots.contains(module.specifier()) { - None - } else { - maybe_get_check_entry(module, check_js) + // now walk the graph that only includes the fast check dependencies + while let Some(specifier) = pending.pop_front() { + let Some(module) = graph.get(&specifier) else { + continue; + }; + if let Some(entry) = maybe_get_check_entry(module, check_js) { + result.push(entry); + } + if let Some(module) = module.esm() { + let deps = module.dependencies_prefer_fast_check(); + for dep in deps.values() { + // walk both the code and type dependencies + if let Some(specifier) = dep.get_code() { + let specifier = graph.resolve(specifier); + if seen.insert(specifier.clone()) { + pending.push_back(specifier); + } + } + if let Some(specifier) = dep.get_type() { + let specifier = graph.resolve(specifier); + if seen.insert(specifier.clone()) { + pending.push_back(specifier); + } + } + } + + if let Some(dep) = module + .maybe_types_dependency + .as_ref() + .and_then(|d| d.dependency.ok()) + { + let specifier = graph.resolve(&dep.specifier); + if seen.insert(specifier.clone()) { + pending.push_back(specifier); + } + } } - })); + } result } diff --git a/cli/tools/registry/graph.rs b/cli/tools/registry/graph.rs new file mode 100644 index 000000000..04b37508c --- /dev/null +++ b/cli/tools/registry/graph.rs @@ -0,0 +1,154 @@ +// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. + +use std::collections::HashSet; +use std::collections::VecDeque; + +use deno_ast::ModuleSpecifier; +use deno_config::ConfigFile; +use deno_config::WorkspaceConfig; +use deno_core::anyhow::bail; +use deno_core::anyhow::Context; +use deno_core::error::AnyError; +use deno_graph::FastCheckDiagnostic; +use deno_graph::ModuleGraph; + +#[derive(Debug)] +pub struct MemberRoots { + pub name: String, + pub dir_url: ModuleSpecifier, + pub exports: Vec<ModuleSpecifier>, +} + +pub fn get_workspace_member_roots( + config: &WorkspaceConfig, +) -> Result<Vec<MemberRoots>, AnyError> { + let mut members = Vec::with_capacity(config.members.len()); + let mut seen_names = HashSet::with_capacity(config.members.len()); + for member in &config.members { + if !seen_names.insert(&member.package_name) { + bail!( + "Cannot have two workspace packages with the same name ('{}' at {})", + member.package_name, + member.path.display(), + ); + } + members.push(MemberRoots { + name: member.package_name.clone(), + dir_url: member.config_file.specifier.join("./").unwrap().clone(), + exports: resolve_config_file_roots_from_exports(&member.config_file)?, + }); + } + Ok(members) +} + +pub fn resolve_config_file_roots_from_exports( + config_file: &ConfigFile, +) -> Result<Vec<ModuleSpecifier>, AnyError> { + let exports_config = config_file + .to_exports_config() + .with_context(|| { + format!("Failed to parse exports at {}", config_file.specifier) + })? + .into_map(); + let mut exports = Vec::with_capacity(exports_config.len()); + for (_, value) in exports_config { + let entry_point = + config_file.specifier.join(&value).with_context(|| { + format!("Failed to join {} with {}", config_file.specifier, value) + })?; + exports.push(entry_point); + } + Ok(exports) +} + +pub fn surface_fast_check_type_graph_errors( + graph: &ModuleGraph, + packages: &[MemberRoots], +) -> Result<(), AnyError> { + let mut diagnostic_count = 0; + let mut seen_diagnostics = HashSet::new(); + let mut seen_modules = HashSet::with_capacity(graph.specifiers_count()); + for package in packages { + let mut pending = VecDeque::new(); + for export in &package.exports { + if seen_modules.insert(export.clone()) { + pending.push_back(export.clone()); + } + } + + 'analyze_package: while let Some(specifier) = pending.pop_front() { + let Ok(Some(module)) = graph.try_get_prefer_types(&specifier) else { + continue; + }; + let Some(esm_module) = module.esm() else { + continue; + }; + if let Some(diagnostic) = esm_module.fast_check_diagnostic() { + for diagnostic in diagnostic.flatten_multiple() { + if matches!( + diagnostic, + FastCheckDiagnostic::UnsupportedJavaScriptEntrypoint { .. } + ) { + // ignore JS packages for fast check + log::warn!( + concat!( + "{} Package '{}' is a JavaScript package without a corresponding ", + "declaration file. This may lead to a non-optimal experience for ", + "users of your package. For performance reasons, it's recommended ", + "to ship a corresponding TypeScript declaration file or to ", + "convert to TypeScript.", + ), + deno_runtime::colors::yellow("Warning"), + package.name, + ); + break 'analyze_package; // no need to keep analyzing this package + } else { + let message = diagnostic.message_with_range(); + if !seen_diagnostics.insert(message.clone()) { + continue; + } + + log::error!("\n{}", message); + diagnostic_count += 1; + } + } + } + + // analyze the next dependencies + for dep in esm_module.dependencies_prefer_fast_check().values() { + let Some(specifier) = graph.resolve_dependency_from_dep(dep, true) + else { + continue; + }; + + let dep_in_same_package = + specifier.as_str().starts_with(package.dir_url.as_str()); + if dep_in_same_package { + let is_new = seen_modules.insert(specifier.clone()); + if is_new { + pending.push_back(specifier.clone()); + } + } + } + } + } + + if diagnostic_count > 0 { + // for the time being, tell the user why we have these errors and the benefit they bring + log::error!( + concat!( + "\nFixing these fast check errors is required to make the code fast check compatible ", + "which enables type checking your package's TypeScript code with the same ", + "performance as if you had distributed declaration files. Do any of these ", + "errors seem too restrictive or incorrect? Please open an issue if so to ", + "help us improve: https://github.com/denoland/deno/issues\n", + ) + ); + bail!( + "Had {} fast check error{}.", + diagnostic_count, + if diagnostic_count == 1 { "" } else { "s" } + ) + } + Ok(()) +} diff --git a/cli/tools/registry/mod.rs b/cli/tools/registry/mod.rs index 36d5b4a7d..8293a87fe 100644 --- a/cli/tools/registry/mod.rs +++ b/cli/tools/registry/mod.rs @@ -24,16 +24,24 @@ use sha2::Digest; use crate::args::deno_registry_api_url; use crate::args::deno_registry_url; +use crate::args::CliOptions; use crate::args::Flags; use crate::args::PublishFlags; use crate::factory::CliFactory; +use crate::graph_util::ModuleGraphBuilder; use crate::http_util::HttpClient; +use crate::tools::check::CheckOptions; +use crate::tools::registry::graph::get_workspace_member_roots; +use crate::tools::registry::graph::resolve_config_file_roots_from_exports; +use crate::tools::registry::graph::surface_fast_check_type_graph_errors; +use crate::tools::registry::graph::MemberRoots; use crate::util::display::human_size; use crate::util::glob::PathOrPatternSet; use crate::util::import_map::ImportMapUnfurler; mod api; mod auth; +mod graph; mod publish_order; mod tar; @@ -41,6 +49,8 @@ use auth::get_auth_method; use auth::AuthMethod; use publish_order::PublishOrderGraph; +use super::check::TypeChecker; + use self::tar::PublishableTarball; fn ring_bell() { @@ -64,6 +74,15 @@ impl PreparedPublishPackage { static SUGGESTED_ENTRYPOINTS: [&str; 4] = ["mod.ts", "mod.js", "index.ts", "index.js"]; +fn get_deno_json_package_name( + deno_json: &ConfigFile, +) -> Result<String, AnyError> { + match deno_json.json.name.clone() { + Some(name) => Ok(name), + None => bail!("{} is missing 'name' field", deno_json.specifier), + } +} + async fn prepare_publish( deno_json: &ConfigFile, import_map: Arc<ImportMap>, @@ -73,9 +92,7 @@ async fn prepare_publish( let Some(version) = deno_json.json.version.clone() else { bail!("{} is missing 'version' field", deno_json.specifier); }; - let Some(name) = deno_json.json.name.clone() else { - bail!("{} is missing 'name' field", deno_json.specifier); - }; + let name = get_deno_json_package_name(deno_json)?; if deno_json.json.exports.is_none() { let mut suggested_entrypoint = None; @@ -122,6 +139,8 @@ async fn prepare_publish( }) .await??; + log::debug!("Tarball size ({}): {}", name, tarball.bytes.len()); + Ok(Rc::new(PreparedPublishPackage { scope: scope.to_string(), package: package_name.to_string(), @@ -665,11 +684,26 @@ async fn prepare_packages_for_publishing( AnyError, > { let maybe_workspace_config = deno_json.to_workspace_config()?; + let module_graph_builder = cli_factory.module_graph_builder().await?.as_ref(); + let type_checker = cli_factory.type_checker().await?; + let cli_options = cli_factory.cli_options(); let Some(workspace_config) = maybe_workspace_config else { + let roots = resolve_config_file_roots_from_exports(&deno_json)?; + build_and_check_graph_for_publish( + module_graph_builder, + type_checker, + cli_options, + &[MemberRoots { + name: get_deno_json_package_name(&deno_json)?, + dir_url: deno_json.specifier.join("./").unwrap().clone(), + exports: roots, + }], + ) + .await?; let mut prepared_package_by_name = HashMap::with_capacity(1); let package = prepare_publish(&deno_json, import_map).await?; - let package_name = package.package.clone(); + let package_name = format!("@{}/{}", package.scope, package.package); let publish_order_graph = PublishOrderGraph::new_single(package_name.clone()); prepared_package_by_name.insert(package_name, package); @@ -677,14 +711,21 @@ async fn prepare_packages_for_publishing( }; println!("Publishing a workspace..."); - let mut prepared_package_by_name = - HashMap::with_capacity(workspace_config.members.len()); - let publish_order_graph = publish_order::build_publish_graph( - &workspace_config, - cli_factory.module_graph_builder().await?.as_ref(), + // create the module graph + let roots = get_workspace_member_roots(&workspace_config)?; + let graph = build_and_check_graph_for_publish( + module_graph_builder, + type_checker, + cli_options, + &roots, ) .await?; + let mut prepared_package_by_name = + HashMap::with_capacity(workspace_config.members.len()); + let publish_order_graph = + publish_order::build_publish_order_graph(&graph, &roots)?; + let results = workspace_config .members @@ -712,6 +753,55 @@ async fn prepare_packages_for_publishing( Ok((publish_order_graph, prepared_package_by_name)) } +async fn build_and_check_graph_for_publish( + module_graph_builder: &ModuleGraphBuilder, + type_checker: &TypeChecker, + cli_options: &CliOptions, + packages: &[MemberRoots], +) -> Result<Arc<deno_graph::ModuleGraph>, deno_core::anyhow::Error> { + let graph = Arc::new( + module_graph_builder + .create_graph_with_options(crate::graph_util::CreateGraphOptions { + // All because we're going to use this same graph to determine the publish order later + graph_kind: deno_graph::GraphKind::All, + roots: packages + .iter() + .flat_map(|r| r.exports.iter()) + .cloned() + .collect(), + workspace_fast_check: true, + loader: None, + }) + .await?, + ); + graph.valid()?; + log::info!("Checking fast check type graph for errors..."); + surface_fast_check_type_graph_errors(&graph, packages)?; + log::info!("Ensuring type checks..."); + let diagnostics = type_checker + .check_diagnostics( + graph.clone(), + CheckOptions { + lib: cli_options.ts_type_lib_window(), + log_ignored_options: false, + reload: cli_options.reload_flag(), + }, + ) + .await?; + if !diagnostics.is_empty() { + bail!( + concat!( + "{:#}\n\n", + "You may have discovered a bug in Deno's fast check implementation. ", + "Fast check is still early days and we would appreciate if you log a ", + "bug if you believe this is one: https://github.com/denoland/deno/issues/" + ), + diagnostics + ); + } + Ok(graph) +} + pub async fn publish( flags: Flags, publish_flags: PublishFlags, @@ -728,10 +818,7 @@ pub async fn publish( Arc::new(ImportMap::new(Url::parse("file:///dev/null").unwrap())) }); - let initial_cwd = - std::env::current_dir().with_context(|| "Failed getting cwd.")?; - - let directory_path = initial_cwd.join(publish_flags.directory); + let directory_path = cli_factory.cli_options().initial_cwd(); // TODO: doesn't handle jsonc let deno_json_path = directory_path.join("deno.json"); let deno_json = ConfigFile::read(&deno_json_path).with_context(|| { diff --git a/cli/tools/registry/publish_order.rs b/cli/tools/registry/publish_order.rs index 965da8341..4071c42ca 100644 --- a/cli/tools/registry/publish_order.rs +++ b/cli/tools/registry/publish_order.rs @@ -4,13 +4,11 @@ use std::collections::HashMap; use std::collections::HashSet; use std::collections::VecDeque; -use deno_ast::ModuleSpecifier; -use deno_config::WorkspaceConfig; use deno_core::anyhow::bail; -use deno_core::anyhow::Context; use deno_core::error::AnyError; +use deno_graph::ModuleGraph; -use crate::graph_util::ModuleGraphBuilder; +use super::graph::MemberRoots; pub struct PublishOrderGraph { packages: HashMap<String, HashSet<String>>, @@ -122,80 +120,21 @@ impl PublishOrderGraph { } } -pub async fn build_publish_graph( - workspace_config: &WorkspaceConfig, - module_graph_builder: &ModuleGraphBuilder, +pub fn build_publish_order_graph( + graph: &ModuleGraph, + roots: &[MemberRoots], ) -> Result<PublishOrderGraph, AnyError> { - let roots = get_workspace_roots(workspace_config)?; - let graph = module_graph_builder - .create_graph( - deno_graph::GraphKind::All, - roots.iter().flat_map(|r| r.exports.clone()).collect(), - ) - .await?; - graph.valid()?; - let packages = build_pkg_deps(graph, roots); - Ok(build_graph(packages)) -} - -#[derive(Debug)] -struct MemberRoots { - name: String, - dir_url: ModuleSpecifier, - exports: Vec<ModuleSpecifier>, -} - -fn get_workspace_roots( - config: &WorkspaceConfig, -) -> Result<Vec<MemberRoots>, AnyError> { - let mut members = Vec::with_capacity(config.members.len()); - let mut seen_names = HashSet::with_capacity(config.members.len()); - for member in &config.members { - let exports_config = member - .config_file - .to_exports_config() - .with_context(|| { - format!( - "Failed to parse exports at {}", - member.config_file.specifier - ) - })? - .into_map(); - if !seen_names.insert(&member.package_name) { - bail!( - "Cannot have two workspace packages with the same name ('{}' at {})", - member.package_name, - member.path.display(), - ); - } - let mut member_root = MemberRoots { - name: member.package_name.clone(), - dir_url: member.config_file.specifier.join("./").unwrap().clone(), - exports: Vec::with_capacity(exports_config.len()), - }; - for (_, value) in exports_config { - let entry_point = - member.config_file.specifier.join(&value).with_context(|| { - format!( - "Failed to join {} with {}", - member.config_file.specifier, value - ) - })?; - member_root.exports.push(entry_point); - } - members.push(member_root); - } - Ok(members) + Ok(build_publish_order_graph_from_pkgs_deps(packages)) } fn build_pkg_deps( - graph: deno_graph::ModuleGraph, - roots: Vec<MemberRoots>, + graph: &deno_graph::ModuleGraph, + roots: &[MemberRoots], ) -> HashMap<String, HashSet<String>> { let mut members = HashMap::with_capacity(roots.len()); let mut seen_modules = HashSet::with_capacity(graph.modules().count()); - for root in &roots { + for root in roots { let mut deps = HashSet::new(); let mut pending = VecDeque::new(); pending.extend(root.exports.clone()); @@ -243,7 +182,7 @@ fn build_pkg_deps( members } -fn build_graph( +fn build_publish_order_graph_from_pkgs_deps( packages: HashMap<String, HashSet<String>>, ) -> PublishOrderGraph { let mut in_degree = HashMap::new(); @@ -273,7 +212,7 @@ mod test { #[test] fn test_graph_no_deps() { - let mut graph = build_graph(HashMap::from([ + let mut graph = build_publish_order_graph_from_pkgs_deps(HashMap::from([ ("a".to_string(), HashSet::new()), ("b".to_string(), HashSet::new()), ("c".to_string(), HashSet::new()), @@ -293,7 +232,7 @@ mod test { #[test] fn test_graph_single_dep() { - let mut graph = build_graph(HashMap::from([ + let mut graph = build_publish_order_graph_from_pkgs_deps(HashMap::from([ ("a".to_string(), HashSet::from(["b".to_string()])), ("b".to_string(), HashSet::from(["c".to_string()])), ("c".to_string(), HashSet::new()), @@ -310,7 +249,7 @@ mod test { #[test] fn test_graph_multiple_dep() { - let mut graph = build_graph(HashMap::from([ + let mut graph = build_publish_order_graph_from_pkgs_deps(HashMap::from([ ( "a".to_string(), HashSet::from(["b".to_string(), "c".to_string()]), @@ -342,7 +281,7 @@ mod test { #[test] fn test_graph_circular_dep() { - let mut graph = build_graph(HashMap::from([ + let mut graph = build_publish_order_graph_from_pkgs_deps(HashMap::from([ ("a".to_string(), HashSet::from(["b".to_string()])), ("b".to_string(), HashSet::from(["c".to_string()])), ("c".to_string(), HashSet::from(["a".to_string()])), |