refactor: rewrite TS dependency analysis in Rust (#5029)

This commit completely overhauls how module analysis is 
performed in TS compiler by moving the logic to Rust.

In the current setup module analysis is performed using 
"ts.preProcessFile" API in a special TS compiler worker 
running on a separate thread.

"ts.preProcessFile" allowed us to build a lot of functionality
in CLI including X-TypeScript-Types header support 
and @deno-types directive support. Unfortunately at the 
same time complexity of the ops required to perform 
supporting tasks exploded and caused some hidden 
permission escapes.

This PR introduces "ModuleGraphLoader" which can parse
source and load recursively all dependent source files; as 
well as declaration files. All dependencies used in TS 
compiler and now fetched and collected upfront in Rust 
before spinning up TS compiler.

To achieve feature parity with existing APIs this commit 
includes a lot of changes:

* add "ModuleGraphLoader"
  - can fetch local and remote sources
  - parses source code using SWC and extracts imports, exports, file references, special 
     headers
  - this struct inherited all of the hidden complexity and cruft from TS version and requires 
     several follow up PRs
* rewrite cli/tsc.rs to perform module analysis upfront and send all required source code to 
  TS worker in one message
* remove op_resolve_modules and op_fetch_source_files from cli/ops/compiler.rs
* run TS worker on the same thread

1 files changed, 292 insertions, 56 deletions

diff --git a/cli/swc_util.rs b/cli/swc_util.rs
index f579cbd36..e07486cd7 100644
--- a/cli/swc_util.rs
+++ b/cli/swc_util.rs
@@ -1,5 +1,6 @@
 // Copyright 2018-2020 the Deno authors. All rights reserved. MIT license.
 use crate::swc_common;
+use crate::swc_common::comments::CommentKind;
 use crate::swc_common::comments::Comments;
 use crate::swc_common::errors::Diagnostic;
 use crate::swc_common::errors::DiagnosticBuilder;
@@ -158,10 +159,16 @@ impl AstParser {
     &self,
     span: Span,
   ) -> Vec<swc_common::comments::Comment> {
-    self
-      .comments
-      .take_leading_comments(span.lo())
-      .unwrap_or_else(Vec::new)
+    let maybe_comments = self.comments.take_leading_comments(span.lo());
+
+    if let Some(comments) = maybe_comments {
+      // clone the comments and put them back in map
+      let to_return = comments.clone();
+      self.comments.add_leading(span.lo(), comments);
+      to_return
+    } else {
+      vec![]
+    }
   }
 }
 
@@ -240,80 +247,309 @@ impl Visit for DependencyVisitor {
   }
 }
 
-/// Given file name and source code return vector
-/// of unresolved import specifiers.
-///
-/// Returned vector may contain duplicate entries.
-///
-/// Second argument allows to configure if dynamic
-/// imports should be analyzed.
-///
-/// NOTE: Only statically analyzable dynamic imports
-/// are considered; ie. the ones that have plain string specifier:
-///
-///    await import("./fizz.ts")
-///
-/// These imports will be ignored:
-///
-///    await import(`./${dir}/fizz.ts`)
-///    await import("./" + "fizz.ts")
-#[allow(unused)]
-pub fn analyze_dependencies(
+#[derive(Clone, Debug, PartialEq)]
+enum DependencyKind {
+  Import,
+  DynamicImport,
+  Export,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+struct DependencyDescriptor {
+  span: Span,
+  specifier: String,
+  kind: DependencyKind,
+}
+
+struct NewDependencyVisitor {
+  dependencies: Vec<DependencyDescriptor>,
+}
+
+impl Visit for NewDependencyVisitor {
+  fn visit_import_decl(
+    &mut self,
+    import_decl: &swc_ecma_ast::ImportDecl,
+    _parent: &dyn Node,
+  ) {
+    let src_str = import_decl.src.value.to_string();
+    self.dependencies.push(DependencyDescriptor {
+      specifier: src_str,
+      kind: DependencyKind::Import,
+      span: import_decl.span,
+    });
+  }
+
+  fn visit_named_export(
+    &mut self,
+    named_export: &swc_ecma_ast::NamedExport,
+    _parent: &dyn Node,
+  ) {
+    if let Some(src) = &named_export.src {
+      let src_str = src.value.to_string();
+      self.dependencies.push(DependencyDescriptor {
+        specifier: src_str,
+        kind: DependencyKind::Export,
+        span: named_export.span,
+      });
+    }
+  }
+
+  fn visit_export_all(
+    &mut self,
+    export_all: &swc_ecma_ast::ExportAll,
+    _parent: &dyn Node,
+  ) {
+    let src_str = export_all.src.value.to_string();
+    self.dependencies.push(DependencyDescriptor {
+      specifier: src_str,
+      kind: DependencyKind::Export,
+      span: export_all.span,
+    });
+  }
+
+  fn visit_call_expr(
+    &mut self,
+    call_expr: &swc_ecma_ast::CallExpr,
+    parent: &dyn Node,
+  ) {
+    use swc_ecma_ast::Expr::*;
+    use swc_ecma_ast::ExprOrSuper::*;
+
+    swc_ecma_visit::visit_call_expr(self, call_expr, parent);
+    let boxed_expr = match call_expr.callee.clone() {
+      Super(_) => return,
+      Expr(boxed) => boxed,
+    };
+
+    match &*boxed_expr {
+      Ident(ident) => {
+        if &ident.sym.to_string() != "import" {
+          return;
+        }
+      }
+      _ => return,
+    };
+
+    if let Some(arg) = call_expr.args.get(0) {
+      match &*arg.expr {
+        Lit(lit) => {
+          if let swc_ecma_ast::Lit::Str(str_) = lit {
+            let src_str = str_.value.to_string();
+            self.dependencies.push(DependencyDescriptor {
+              specifier: src_str,
+              kind: DependencyKind::DynamicImport,
+              span: call_expr.span,
+            });
+          }
+        }
+        _ => return,
+      }
+    }
+  }
+}
+
+fn get_deno_types(parser: &AstParser, span: Span) -> Option<String> {
+  let comments = parser.get_span_comments(span);
+
+  if comments.is_empty() {
+    return None;
+  }
+
+  // @deno-types must directly prepend import statement - hence
+  // checking last comment for span
+  let last = comments.last().unwrap();
+  let comment = last.text.trim_start();
+
+  if comment.starts_with("@deno-types") {
+    let split: Vec<String> =
+      comment.split('=').map(|s| s.to_string()).collect();
+    assert_eq!(split.len(), 2);
+    let specifier_in_quotes = split.get(1).unwrap().to_string();
+    let specifier = specifier_in_quotes
+      .trim_start_matches('\"')
+      .trim_start_matches('\'')
+      .trim_end_matches('\"')
+      .trim_end_matches('\'')
+      .to_string();
+    return Some(specifier);
+  }
+
+  None
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct ImportDescriptor {
+  pub specifier: String,
+  pub deno_types: Option<String>,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub enum TsReferenceKind {
+  Lib,
+  Types,
+  Path,
+}
+
+#[derive(Clone, Debug, PartialEq)]
+pub struct TsReferenceDescriptor {
+  pub kind: TsReferenceKind,
+  pub specifier: String,
+}
+
+pub fn analyze_dependencies_and_references(
   source_code: &str,
   analyze_dynamic_imports: bool,
-) -> Result<Vec<String>, SwcDiagnosticBuffer> {
+) -> Result<
+  (Vec<ImportDescriptor>, Vec<TsReferenceDescriptor>),
+  SwcDiagnosticBuffer,
+> {
   let parser = AstParser::new();
   parser.parse_module("root.ts", source_code, |parse_result| {
     let module = parse_result?;
-    let mut collector = DependencyVisitor {
+    let mut collector = NewDependencyVisitor {
       dependencies: vec![],
-      analyze_dynamic_imports,
     };
+    let module_span = module.span;
     collector.visit_module(&module, &module);
-    Ok(collector.dependencies)
+
+    let dependency_descriptors = collector.dependencies;
+
+    // for each import check if there's relevant @deno-types directive
+    let imports = dependency_descriptors
+      .iter()
+      .filter(|desc| {
+        if analyze_dynamic_imports {
+          return true;
+        }
+
+        desc.kind != DependencyKind::DynamicImport
+      })
+      .map(|desc| {
+        if desc.kind == DependencyKind::Import {
+          let deno_types = get_deno_types(&parser, desc.span);
+          ImportDescriptor {
+            specifier: desc.specifier.to_string(),
+            deno_types,
+          }
+        } else {
+          ImportDescriptor {
+            specifier: desc.specifier.to_string(),
+            deno_types: None,
+          }
+        }
+      })
+      .collect();
+
+    // analyze comment from beginning of the file and find TS directives
+    let comments = parser
+      .comments
+      .take_leading_comments(module_span.lo())
+      .unwrap_or_else(|| vec![]);
+
+    let mut references = vec![];
+    for comment in comments {
+      if comment.kind != CommentKind::Line {
+        continue;
+      }
+
+      // TODO(bartlomieju): you can do better than that...
+      let text = comment.text.to_string();
+      let (kind, specifier_in_quotes) =
+        if text.starts_with("/ <reference path=") {
+          (
+            TsReferenceKind::Path,
+            text.trim_start_matches("/ <reference path="),
+          )
+        } else if text.starts_with("/ <reference lib=") {
+          (
+            TsReferenceKind::Lib,
+            text.trim_start_matches("/ <reference lib="),
+          )
+        } else if text.starts_with("/ <reference types=") {
+          (
+            TsReferenceKind::Types,
+            text.trim_start_matches("/ <reference types="),
+          )
+        } else {
+          continue;
+        };
+      let specifier = specifier_in_quotes
+        .trim_end_matches("/>")
+        .trim_end()
+        .trim_start_matches('\"')
+        .trim_start_matches('\'')
+        .trim_end_matches('\"')
+        .trim_end_matches('\'')
+        .to_string();
+
+      references.push(TsReferenceDescriptor { kind, specifier });
+    }
+    Ok((imports, references))
   })
 }
 
 #[test]
-fn test_analyze_dependencies() {
+fn test_analyze_dependencies_and_directives() {
   let source = r#"
-import { foo } from "./foo.ts";
-export { bar } from "./foo.ts";
-export * from "./bar.ts";
+// This comment is placed to make sure that directives are parsed
+// even when they start on non-first line
+  
+/// <reference lib="dom" />
+/// <reference types="./type_reference.d.ts" />
+/// <reference path="./type_reference/dep.ts" />
+// @deno-types="./type_definitions/foo.d.ts"
+import { foo } from "./type_definitions/foo.js";
+// @deno-types="./type_definitions/fizz.d.ts"
+import "./type_definitions/fizz.js";
+
+/// <reference path="./type_reference/dep2.ts" />
+
+import * as qat from "./type_definitions/qat.ts";
+
+console.log(foo);
+console.log(fizz);
+console.log(qat.qat);  
 "#;
 
-  let dependencies =
-    analyze_dependencies(source, false).expect("Failed to parse");
+  let (imports, references) =
+    analyze_dependencies_and_references(source, true).expect("Failed to parse");
+
   assert_eq!(
-    dependencies,
+    imports,
     vec![
-      "./foo.ts".to_string(),
-      "./foo.ts".to_string(),
-      "./bar.ts".to_string(),
+      ImportDescriptor {
+        specifier: "./type_definitions/foo.js".to_string(),
+        deno_types: Some("./type_definitions/foo.d.ts".to_string())
+      },
+      ImportDescriptor {
+        specifier: "./type_definitions/fizz.js".to_string(),
+        deno_types: Some("./type_definitions/fizz.d.ts".to_string())
+      },
+      ImportDescriptor {
+        specifier: "./type_definitions/qat.ts".to_string(),
+        deno_types: None
+      },
     ]
   );
-}
-
-#[test]
-fn test_analyze_dependencies_dyn_imports() {
-  let source = r#"
-import { foo } from "./foo.ts";
-export { bar } from "./foo.ts";
-export * from "./bar.ts";
-
-const a = await import("./fizz.ts");
-const a = await import("./" + "buzz.ts");
-"#;
 
-  let dependencies =
-    analyze_dependencies(source, true).expect("Failed to parse");
+  // According to TS docs (https://www.typescriptlang.org/docs/handbook/triple-slash-directives.html)
+  // directives that are not at the top of the file are ignored, so only
+  // 3 references should be captured instead of 4.
   assert_eq!(
-    dependencies,
+    references,
     vec![
-      "./foo.ts".to_string(),
-      "./foo.ts".to_string(),
-      "./bar.ts".to_string(),
-      "./fizz.ts".to_string(),
+      TsReferenceDescriptor {
+        specifier: "dom".to_string(),
+        kind: TsReferenceKind::Lib,
+      },
+      TsReferenceDescriptor {
+        specifier: "./type_reference.d.ts".to_string(),
+        kind: TsReferenceKind::Types,
+      },
+      TsReferenceDescriptor {
+        specifier: "./type_reference/dep.ts".to_string(),
+        kind: TsReferenceKind::Path,
+      },
     ]
   );
 }