feat(cli/doc): use type definitions "deno doc" if available (#8459)

This commit adds support for type definitions in "deno doc";
with this change "deno doc" is able to leverage the same directives
as TS compiler.

Co-authored-by: Bartek Iwańczuk <biwanczuk@gmail.com>

10 files changed, 232 insertions, 132 deletions

diff --git a/cli/main.rs b/cli/main.rs
index e957b9342..7071d775a 100644
--- a/cli/main.rs
+++ b/cli/main.rs
@@ -44,12 +44,10 @@ mod tsc_config;
 mod version;
 
 use crate::file_fetcher::File;
-use crate::file_fetcher::FileFetcher;
 use crate::file_watcher::ModuleResolutionResult;
 use crate::flags::DenoSubcommand;
 use crate::flags::Flags;
 use crate::fmt_errors::PrettyJsError;
-use crate::import_map::ImportMap;
 use crate::media_type::MediaType;
 use crate::module_loader::CliModuleLoader;
 use crate::program_state::exit_unstable;
@@ -66,8 +64,6 @@ use deno_core::serde_json;
 use deno_core::serde_json::json;
 use deno_core::v8_set_flags;
 use deno_core::ModuleSpecifier;
-use deno_doc as doc;
-use deno_doc::parser::DocFileLoader;
 use deno_runtime::ops::worker_host::CreateWebWorkerCb;
 use deno_runtime::permissions::Permissions;
 use deno_runtime::web_worker::WebWorker;
@@ -228,7 +224,9 @@ pub fn create_main_worker(
   worker
 }
 
-fn write_to_stdout_ignore_sigpipe(bytes: &[u8]) -> Result<(), std::io::Error> {
+pub fn write_to_stdout_ignore_sigpipe(
+  bytes: &[u8],
+) -> Result<(), std::io::Error> {
   use std::io::ErrorKind;
 
   match std::io::stdout().write_all(bytes) {
@@ -240,7 +238,7 @@ fn write_to_stdout_ignore_sigpipe(bytes: &[u8]) -> Result<(), std::io::Error> {
   }
 }
 
-fn write_json_to_stdout<T>(value: &T) -> Result<(), AnyError>
+pub fn write_json_to_stdout<T>(value: &T) -> Result<(), AnyError>
 where
   T: ?Sized + serde::ser::Serialize,
 {
@@ -278,7 +276,7 @@ fn print_cache_info(
   }
 }
 
-fn get_types(unstable: bool) -> String {
+pub fn get_types(unstable: bool) -> String {
   let mut types = format!(
     "{}\n{}\n{}\n{}\n{}\n{}",
     crate::tsc::DENO_NS_LIB,
@@ -684,59 +682,6 @@ async fn bundle_command(
   Ok(())
 }
 
-struct DocLoader {
-  fetcher: FileFetcher,
-  maybe_import_map: Option<ImportMap>,
-}
-
-impl DocFileLoader for DocLoader {
-  fn resolve(
-    &self,
-    specifier: &str,
-    referrer: &str,
-  ) -> Result<String, doc::DocError> {
-    let maybe_resolved =
-      if let Some(import_map) = self.maybe_import_map.as_ref() {
-        import_map
-          .resolve(specifier, referrer)
-          .map_err(|e| doc::DocError::Resolve(e.to_string()))?
-      } else {
-        None
-      };
-
-    let resolved_specifier = if let Some(resolved) = maybe_resolved {
-      resolved
-    } else {
-      deno_core::resolve_import(specifier, referrer)
-        .map_err(|e| doc::DocError::Resolve(e.to_string()))?
-    };
-
-    Ok(resolved_specifier.to_string())
-  }
-
-  fn load_source_code(
-    &self,
-    specifier: &str,
-  ) -> Pin<Box<dyn Future<Output = Result<String, doc::DocError>>>> {
-    let fetcher = self.fetcher.clone();
-    let specifier =
-      resolve_url_or_path(specifier).expect("Expected valid specifier");
-    async move {
-      let source_file = fetcher
-        .fetch(&specifier, &Permissions::allow_all())
-        .await
-        .map_err(|e| {
-          doc::DocError::Io(std::io::Error::new(
-            std::io::ErrorKind::Other,
-            e.to_string(),
-          ))
-        })?;
-      Ok(source_file.source)
-    }
-    .boxed_local()
-  }
-}
-
 async fn doc_command(
   flags: Flags,
   source_file: Option<String>,
@@ -744,64 +689,7 @@ async fn doc_command(
   maybe_filter: Option<String>,
   private: bool,
 ) -> Result<(), AnyError> {
-  let program_state = ProgramState::build(flags.clone()).await?;
-  let source_file = source_file.unwrap_or_else(|| "--builtin".to_string());
-
-  let loader = Box::new(DocLoader {
-    fetcher: program_state.file_fetcher.clone(),
-    maybe_import_map: program_state.maybe_import_map.clone(),
-  });
-  let doc_parser = doc::DocParser::new(loader, private);
-
-  let parse_result = if source_file == "--builtin" {
-    let syntax = ast::get_syntax(&MediaType::Dts);
-    doc_parser.parse_source(
-      "lib.deno.d.ts",
-      syntax,
-      get_types(flags.unstable).as_str(),
-    )
-  } else {
-    let path = PathBuf::from(&source_file);
-    let media_type = MediaType::from(&path);
-    let syntax = ast::get_syntax(&media_type);
-    let module_specifier = resolve_url_or_path(&source_file).unwrap();
-    doc_parser
-      .parse_with_reexports(&module_specifier.to_string(), syntax)
-      .await
-  };
-
-  let mut doc_nodes = match parse_result {
-    Ok(nodes) => nodes,
-    Err(e) => {
-      eprintln!("{}", e);
-      std::process::exit(1);
-    }
-  };
-
-  if json {
-    write_json_to_stdout(&doc_nodes)
-  } else {
-    doc_nodes.retain(|doc_node| doc_node.kind != doc::DocNodeKind::Import);
-    let details = if let Some(filter) = maybe_filter {
-      let nodes =
-        doc::find_nodes_by_name_recursively(doc_nodes, filter.clone());
-      if nodes.is_empty() {
-        eprintln!("Node {} was not found!", filter);
-        std::process::exit(1);
-      }
-      format!(
-        "{}",
-        doc::DocPrinter::new(&nodes, colors::use_color(), private)
-      )
-    } else {
-      format!(
-        "{}",
-        doc::DocPrinter::new(&doc_nodes, colors::use_color(), private)
-      )
-    };
-
-    write_to_stdout_ignore_sigpipe(details.as_bytes()).map_err(AnyError::from)
-  }
+  tools::doc::print_docs(flags, source_file, json, maybe_filter, private).await
 }
 
 async fn format_command(
diff --git a/cli/tests/doc/types_header.out b/cli/tests/doc/types_header.out
new file mode 100644
index 000000000..ccff1a373
--- /dev/null
+++ b/cli/tests/doc/types_header.out
@@ -0,0 +1,6 @@
+Download http://127.0.0.1:4545/xTypeScriptTypes.js
+Download http://127.0.0.1:4545/xTypeScriptTypes.d.ts
+Defined in http://127.0.0.1:4545/xTypeScriptTypes.d.ts:1:0 
+
+const foo: "foo"
+
diff --git a/cli/tests/doc/types_header.ts b/cli/tests/doc/types_header.ts
new file mode 100644
index 000000000..b64c8d000
--- /dev/null
+++ b/cli/tests/doc/types_header.ts
@@ -0,0 +1 @@
+export * from "http://127.0.0.1:4545/xTypeScriptTypes.js";
diff --git a/cli/tests/doc/types_hint.out b/cli/tests/doc/types_hint.out
new file mode 100644
index 000000000..7eb05faed
--- /dev/null
+++ b/cli/tests/doc/types_hint.out
@@ -0,0 +1,5 @@
+Defined in [WILDCARD]/type_definitions/foo.d.ts:2:0 
+
+const foo: string
+  An exported value.
+
diff --git a/cli/tests/doc/types_hint.ts b/cli/tests/doc/types_hint.ts
new file mode 100644
index 000000000..bacea46db
--- /dev/null
+++ b/cli/tests/doc/types_hint.ts
@@ -0,0 +1,2 @@
+// @deno-types="../type_definitions/foo.d.ts"
+export * from "../type_definitions/foo.js";
diff --git a/cli/tests/doc/types_ref.js b/cli/tests/doc/types_ref.js
new file mode 100644
index 000000000..03d8b5570
--- /dev/null
+++ b/cli/tests/doc/types_ref.js
@@ -0,0 +1,2 @@
+/// <reference types="../type_definitions/foo.d.ts" />
+export const foo = "foo";
diff --git a/cli/tests/doc/types_ref.out b/cli/tests/doc/types_ref.out
new file mode 100644
index 000000000..7eb05faed
--- /dev/null
+++ b/cli/tests/doc/types_ref.out
@@ -0,0 +1,5 @@
+Defined in [WILDCARD]/type_definitions/foo.d.ts:2:0 
+
+const foo: string
+  An exported value.
+
diff --git a/cli/tests/integration_tests.rs b/cli/tests/integration_tests.rs
index 497028cd1..8b8d6354b 100644
--- a/cli/tests/integration_tests.rs
+++ b/cli/tests/integration_tests.rs
@@ -3669,20 +3669,10 @@ console.log("finish");
     exit_code: 0,
   });
 
-  itest!(deno_doc_builtin {
-    args: "doc",
-    output: "deno_doc_builtin.out",
-  });
-
-  itest!(deno_doc {
-    args: "doc deno_doc.ts",
-    output: "deno_doc.out",
-  });
-
-  itest!(deno_doc_import_map {
-    args:
-      "doc --unstable --import-map=doc/import_map.json doc/use_import_map.js",
-    output: "doc/use_import_map.out",
+  itest!(deno_doc_types_header_direct {
+    args: "doc --reload http://127.0.0.1:4545/xTypeScriptTypes.js",
+    output: "doc/types_header.out",
+    http_server: true,
   });
 
   itest!(import_data_url_error_stack {
@@ -3945,6 +3935,42 @@ console.log("finish");
     assert_eq!(output.stderr, b"");
   }
 
+  mod doc {
+    use super::*;
+
+    itest!(deno_doc_builtin {
+      args: "doc",
+      output: "deno_doc_builtin.out",
+    });
+
+    itest!(deno_doc {
+      args: "doc deno_doc.ts",
+      output: "deno_doc.out",
+    });
+
+    itest!(deno_doc_import_map {
+      args:
+        "doc --unstable --import-map=doc/import_map.json doc/use_import_map.js",
+      output: "doc/use_import_map.out",
+    });
+
+    itest!(deno_doc_types_hint {
+      args: "doc doc/types_hint.ts",
+      output: "doc/types_hint.out",
+    });
+
+    itest!(deno_doc_types_ref {
+      args: "doc doc/types_ref.js",
+      output: "doc/types_ref.out",
+    });
+
+    itest!(deno_doc_types_header {
+      args: "doc --reload doc/types_header.ts",
+      output: "doc/types_header.out",
+      http_server: true,
+    });
+  }
+
   mod coverage {
     use super::*;
 
diff --git a/cli/tools/doc.rs b/cli/tools/doc.rs
new file mode 100644
index 000000000..9725076d6
--- /dev/null
+++ b/cli/tools/doc.rs
@@ -0,0 +1,164 @@
+// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
+
+use crate::ast;
+use crate::colors;
+use crate::file_fetcher::File;
+use crate::flags::Flags;
+use crate::get_types;
+use crate::media_type::MediaType;
+use crate::module_graph;
+use crate::program_state::ProgramState;
+use crate::specifier_handler::FetchHandler;
+use crate::write_json_to_stdout;
+use crate::write_to_stdout_ignore_sigpipe;
+use deno_core::error::AnyError;
+use deno_core::futures::future::FutureExt;
+use deno_core::futures::Future;
+use deno_core::resolve_url_or_path;
+use deno_doc as doc;
+use deno_doc::parser::DocFileLoader;
+use deno_runtime::permissions::Permissions;
+use std::path::PathBuf;
+use std::pin::Pin;
+use std::sync::Arc;
+use std::sync::Mutex;
+
+/// When parsing lib.deno.d.ts, only `DocParser::parse_source` is used,
+/// which never even references the loader, so this is just a stub for that scenario.
+///
+/// TODO(Liamolucko): Refactor `deno_doc` so this isn't necessary.
+struct StubDocLoader;
+
+impl DocFileLoader for StubDocLoader {
+  fn resolve(
+    &self,
+    _specifier: &str,
+    _referrer: &str,
+  ) -> Result<String, doc::DocError> {
+    unreachable!()
+  }
+
+  fn load_source_code(
+    &self,
+    _specifier: &str,
+  ) -> Pin<Box<dyn Future<Output = Result<String, doc::DocError>>>> {
+    unreachable!()
+  }
+}
+
+impl DocFileLoader for module_graph::Graph {
+  fn resolve(
+    &self,
+    specifier: &str,
+    referrer: &str,
+  ) -> Result<String, doc::DocError> {
+    let referrer =
+      resolve_url_or_path(referrer).expect("Expected valid specifier");
+    match self.resolve(specifier, &referrer, true) {
+      Ok(specifier) => Ok(specifier.to_string()),
+      Err(e) => Err(doc::DocError::Resolve(e.to_string())),
+    }
+  }
+
+  fn load_source_code(
+    &self,
+    specifier: &str,
+  ) -> Pin<Box<dyn Future<Output = Result<String, doc::DocError>>>> {
+    let specifier =
+      resolve_url_or_path(specifier).expect("Expected valid specifier");
+    let source = self.get_source(&specifier).expect("Unknown dependency");
+    async move { Ok(source) }.boxed_local()
+  }
+}
+
+pub async fn print_docs(
+  flags: Flags,
+  source_file: Option<String>,
+  json: bool,
+  maybe_filter: Option<String>,
+  private: bool,
+) -> Result<(), AnyError> {
+  let program_state = ProgramState::build(flags.clone()).await?;
+  let source_file = source_file.unwrap_or_else(|| "--builtin".to_string());
+
+  let parse_result = if source_file == "--builtin" {
+    let loader = Box::new(StubDocLoader);
+    let doc_parser = doc::DocParser::new(loader, private);
+
+    let syntax = ast::get_syntax(&MediaType::Dts);
+    doc_parser.parse_source(
+      "lib.deno.d.ts",
+      syntax,
+      get_types(flags.unstable).as_str(),
+    )
+  } else {
+    let module_specifier = resolve_url_or_path(&source_file).unwrap();
+
+    // If the root module has external types, the module graph won't redirect it,
+    // so instead create a dummy file which exports everything from the actual file being documented.
+    let root_specifier = resolve_url_or_path("./$deno$doc.ts").unwrap();
+    let root = File {
+      local: PathBuf::from("./$deno$doc.ts"),
+      maybe_types: None,
+      media_type: MediaType::TypeScript,
+      source: format!("export * from \"{}\";", module_specifier),
+      specifier: root_specifier.clone(),
+    };
+
+    // Save our fake file into file fetcher cache.
+    program_state.file_fetcher.insert_cached(root);
+
+    let handler = Arc::new(Mutex::new(FetchHandler::new(
+      &program_state,
+      Permissions::allow_all(),
+    )?));
+    let mut builder = module_graph::GraphBuilder::new(
+      handler,
+      program_state.maybe_import_map.clone(),
+      program_state.lockfile.clone(),
+    );
+    builder.add(&root_specifier, false).await?;
+    let graph = builder.get_graph();
+
+    let doc_parser = doc::DocParser::new(Box::new(graph), private);
+    doc_parser
+      .parse_with_reexports(
+        root_specifier.as_str(),
+        ast::get_syntax(&MediaType::TypeScript),
+      )
+      .await
+  };
+
+  let mut doc_nodes = match parse_result {
+    Ok(nodes) => nodes,
+    Err(e) => {
+      eprintln!("{}", e);
+      std::process::exit(1);
+    }
+  };
+
+  if json {
+    write_json_to_stdout(&doc_nodes)
+  } else {
+    doc_nodes.retain(|doc_node| doc_node.kind != doc::DocNodeKind::Import);
+    let details = if let Some(filter) = maybe_filter {
+      let nodes =
+        doc::find_nodes_by_name_recursively(doc_nodes, filter.clone());
+      if nodes.is_empty() {
+        eprintln!("Node {} was not found!", filter);
+        std::process::exit(1);
+      }
+      format!(
+        "{}",
+        doc::DocPrinter::new(&nodes, colors::use_color(), private)
+      )
+    } else {
+      format!(
+        "{}",
+        doc::DocPrinter::new(&doc_nodes, colors::use_color(), private)
+      )
+    };
+
+    write_to_stdout_ignore_sigpipe(details.as_bytes()).map_err(AnyError::from)
+  }
+}
diff --git a/cli/tools/mod.rs b/cli/tools/mod.rs
index 089232b99..cd00f6a86 100644
--- a/cli/tools/mod.rs
+++ b/cli/tools/mod.rs
@@ -1,6 +1,7 @@
 // Copyright 2018-2021 the Deno authors. All rights reserved. MIT license.
 
 pub mod coverage;
+pub mod doc;
 pub mod fmt;
 pub mod installer;
 pub mod lint;