refactor(compiler): split code paths for compile and bundle (#6304)

* refactor "compile" and "runtimeCompile" in "compiler.ts" and factor out
separate methods for "compile" and "bundle" operations

* remove noisy debug output from "compiler.ts"
 
* provide "Serialize" implementations for enums in "msg.rs"

* rename "analyze_dependencies_and_references" to "pre_process_file" and
move it to "tsc.rs"

* refactor ModuleGraph to use more concrete types and properly annotate
locations where errors occur

* remove dead code from "file_fetcher.rs" - "SourceFile.types_url" is no
longer needed, as type reference parsing is done in "ModuleGraph"

* remove unneeded field "source_path" from ".meta" files stored for
compiled source file (towards #6080)

1 files changed, 318 insertions, 191 deletions

diff --git a/cli/module_graph.rs b/cli/module_graph.rs
index 9cded48a0..b5bde1a19 100644
--- a/cli/module_graph.rs
+++ b/cli/module_graph.rs
@@ -8,8 +8,10 @@ use crate::import_map::ImportMap;
 use crate::msg::MediaType;
 use crate::op_error::OpError;
 use crate::permissions::Permissions;
-use crate::swc_util::analyze_dependencies_and_references;
-use crate::swc_util::TsReferenceKind;
+use crate::tsc::pre_process_file;
+use crate::tsc::ImportDesc;
+use crate::tsc::TsReferenceDesc;
+use crate::tsc::TsReferenceKind;
 use crate::tsc::AVAILABLE_LIBS;
 use deno_core::ErrBox;
 use deno_core::ModuleSpecifier;
@@ -21,20 +23,142 @@ use serde::Serialize;
 use serde::Serializer;
 use std::collections::HashMap;
 use std::collections::HashSet;
-use std::hash::BuildHasher;
 use std::path::PathBuf;
 use std::pin::Pin;
 
 // TODO(bartlomieju): it'd be great if this function returned
 // more structured data and possibly format the same as TS diagnostics.
 /// Decorate error with location of import that caused the error.
-fn err_with_location(e: ErrBox, location: &Location) -> ErrBox {
-  let location_str = format!(
-    "\nImported from \"{}:{}\"",
-    location.filename, location.line
-  );
-  let err_str = e.to_string();
-  OpError::other(format!("{}{}", err_str, location_str)).into()
+fn err_with_location(e: ErrBox, maybe_location: Option<&Location>) -> ErrBox {
+  if let Some(location) = maybe_location {
+    let location_str = format!(
+      "\nImported from \"{}:{}\"",
+      location.filename, location.line
+    );
+    let err_str = e.to_string();
+    OpError::other(format!("{}{}", err_str, location_str)).into()
+  } else {
+    e
+  }
+}
+
+/// Disallow http:// imports from modules loaded over https://
+fn validate_no_downgrade(
+  module_specifier: &ModuleSpecifier,
+  maybe_referrer: Option<&ModuleSpecifier>,
+  maybe_location: Option<&Location>,
+) -> Result<(), ErrBox> {
+  if let Some(referrer) = maybe_referrer.as_ref() {
+    if let "https" = referrer.as_url().scheme() {
+      if let "http" = module_specifier.as_url().scheme() {
+        let e = OpError::permission_denied(
+          "Modules loaded over https:// are not allowed to import modules over http://".to_string()
+        );
+        return Err(err_with_location(e.into(), maybe_location));
+      };
+    };
+  };
+
+  Ok(())
+}
+
+/// Verify that remote file doesn't try to statically import local file.
+fn validate_no_file_from_remote(
+  module_specifier: &ModuleSpecifier,
+  maybe_referrer: Option<&ModuleSpecifier>,
+  maybe_location: Option<&Location>,
+) -> Result<(), ErrBox> {
+  if let Some(referrer) = maybe_referrer.as_ref() {
+    let referrer_url = referrer.as_url();
+    match referrer_url.scheme() {
+      "http" | "https" => {
+        let specifier_url = module_specifier.as_url();
+        match specifier_url.scheme() {
+          "http" | "https" => {}
+          _ => {
+            let e = OpError::permission_denied(
+              "Remote modules are not allowed to statically import local modules. Use dynamic import instead.".to_string()
+            );
+            return Err(err_with_location(e.into(), maybe_location));
+          }
+        }
+      }
+      _ => {}
+    }
+  }
+
+  Ok(())
+}
+
+// TODO(bartlomieju): handle imports/references in ambient contexts/TS modules
+// https://github.com/denoland/deno/issues/6133
+fn resolve_imports_and_references(
+  referrer: ModuleSpecifier,
+  maybe_import_map: Option<&ImportMap>,
+  import_descs: Vec<ImportDesc>,
+  ref_descs: Vec<TsReferenceDesc>,
+) -> Result<(Vec<ImportDescriptor>, Vec<ReferenceDescriptor>), ErrBox> {
+  let mut imports = vec![];
+  let mut references = vec![];
+
+  for import_desc in import_descs {
+    let maybe_resolved = if let Some(import_map) = maybe_import_map.as_ref() {
+      import_map.resolve(&import_desc.specifier, &referrer.to_string())?
+    } else {
+      None
+    };
+
+    let resolved_specifier = if let Some(resolved) = maybe_resolved {
+      resolved
+    } else {
+      ModuleSpecifier::resolve_import(
+        &import_desc.specifier,
+        &referrer.to_string(),
+      )?
+    };
+
+    let resolved_type_directive =
+      if let Some(types_specifier) = import_desc.deno_types.as_ref() {
+        Some(ModuleSpecifier::resolve_import(
+          &types_specifier,
+          &referrer.to_string(),
+        )?)
+      } else {
+        None
+      };
+
+    let import_descriptor = ImportDescriptor {
+      specifier: import_desc.specifier.to_string(),
+      resolved_specifier,
+      type_directive: import_desc.deno_types.clone(),
+      resolved_type_directive,
+      location: import_desc.location,
+    };
+
+    imports.push(import_descriptor);
+  }
+
+  for ref_desc in ref_descs {
+    if AVAILABLE_LIBS.contains(&ref_desc.specifier.as_str()) {
+      continue;
+    }
+
+    let resolved_specifier = ModuleSpecifier::resolve_import(
+      &ref_desc.specifier,
+      &referrer.to_string(),
+    )?;
+
+    let reference_descriptor = ReferenceDescriptor {
+      specifier: ref_desc.specifier.to_string(),
+      resolved_specifier,
+      kind: ref_desc.kind,
+      location: ref_desc.location,
+    };
+
+    references.push(reference_descriptor);
+  }
+
+  Ok((imports, references))
 }
 
 fn serialize_module_specifier<S>(
@@ -68,8 +192,7 @@ const SUPPORTED_MEDIA_TYPES: [MediaType; 4] = [
   MediaType::TSX,
 ];
 
-#[derive(Debug, Serialize)]
-pub struct ModuleGraph(HashMap<String, ModuleGraphFile>);
+pub type ModuleGraph = HashMap<String, ModuleGraphFile>;
 
 #[derive(Debug, Serialize)]
 #[serde(rename_all = "camelCase")]
@@ -82,6 +205,8 @@ pub struct ImportDescriptor {
   pub type_directive: Option<String>,
   #[serde(serialize_with = "serialize_option_module_specifier")]
   pub resolved_type_directive: Option<ModuleSpecifier>,
+  #[serde(skip)]
+  pub location: Location,
 }
 
 #[derive(Debug, Serialize)]
@@ -90,6 +215,10 @@ pub struct ReferenceDescriptor {
   pub specifier: String,
   #[serde(serialize_with = "serialize_module_specifier")]
   pub resolved_specifier: ModuleSpecifier,
+  #[serde(skip)]
+  pub kind: TsReferenceKind,
+  #[serde(skip)]
+  pub location: Location,
 }
 
 #[derive(Debug, Serialize)]
@@ -104,7 +233,7 @@ pub struct ModuleGraphFile {
   pub lib_directives: Vec<ReferenceDescriptor>,
   pub types_directives: Vec<ReferenceDescriptor>,
   pub type_headers: Vec<ReferenceDescriptor>,
-  pub media_type: i32,
+  pub media_type: MediaType,
   pub source_code: String,
 }
 
@@ -117,7 +246,7 @@ pub struct ModuleGraphLoader {
   maybe_import_map: Option<ImportMap>,
   pending_downloads: FuturesUnordered<SourceFileFuture>,
   has_downloaded: HashSet<ModuleSpecifier>,
-  pub graph: ModuleGraph,
+  graph: ModuleGraph,
   is_dyn_import: bool,
   analyze_dynamic_imports: bool,
 }
@@ -136,7 +265,7 @@ impl ModuleGraphLoader {
       maybe_import_map,
       pending_downloads: FuturesUnordered::new(),
       has_downloaded: HashSet::new(),
-      graph: ModuleGraph(HashMap::new()),
+      graph: ModuleGraph::new(),
       is_dyn_import,
       analyze_dynamic_imports,
     }
@@ -153,7 +282,7 @@ impl ModuleGraphLoader {
     specifier: &ModuleSpecifier,
     maybe_referrer: Option<ModuleSpecifier>,
   ) -> Result<(), ErrBox> {
-    self.download_module(specifier.clone(), maybe_referrer)?;
+    self.download_module(specifier.clone(), maybe_referrer, None)?;
 
     loop {
       let (specifier, source_file) =
@@ -170,10 +299,10 @@ impl ModuleGraphLoader {
   /// This method is used to create a graph from in-memory files stored in
   /// a hash map. Useful for creating module graph for code received from
   /// the runtime.
-  pub fn build_local_graph<S: BuildHasher>(
+  pub fn build_local_graph(
     &mut self,
     _root_name: &str,
-    source_map: &HashMap<String, String, S>,
+    source_map: &HashMap<String, String>,
   ) -> Result<(), ErrBox> {
     for (spec, source_code) in source_map.iter() {
       self.visit_memory_module(spec.to_string(), source_code.to_string())?;
@@ -183,8 +312,8 @@ impl ModuleGraphLoader {
   }
 
   /// Consumes the loader and returns created graph.
-  pub fn get_graph(self) -> HashMap<String, ModuleGraphFile> {
-    self.graph.0
+  pub fn get_graph(self) -> ModuleGraph {
+    self.graph
   }
 
   fn visit_memory_module(
@@ -192,7 +321,6 @@ impl ModuleGraphLoader {
     specifier: String,
     source_code: String,
   ) -> Result<(), ErrBox> {
-    let mut imports = vec![];
     let mut referenced_files = vec![];
     let mut lib_directives = vec![];
     let mut types_directives = vec![];
@@ -208,87 +336,40 @@ impl ModuleGraphLoader {
         ModuleSpecifier::resolve_url(&format!("memory://{}", specifier))?
       };
 
-    let (import_descs, ref_descs) = analyze_dependencies_and_references(
-      &specifier,
+    let (raw_imports, raw_references) = pre_process_file(
+      &module_specifier.to_string(),
       map_file_extension(&PathBuf::from(&specifier)),
       &source_code,
       self.analyze_dynamic_imports,
     )?;
+    let (imports, references) = resolve_imports_and_references(
+      module_specifier.clone(),
+      self.maybe_import_map.as_ref(),
+      raw_imports,
+      raw_references,
+    )?;
 
-    for import_desc in import_descs {
-      let maybe_resolved =
-        if let Some(import_map) = self.maybe_import_map.as_ref() {
-          import_map
-            .resolve(&import_desc.specifier, &module_specifier.to_string())?
-        } else {
-          None
-        };
-
-      let resolved_specifier = if let Some(resolved) = maybe_resolved {
-        resolved
-      } else {
-        ModuleSpecifier::resolve_import(
-          &import_desc.specifier,
-          &module_specifier.to_string(),
-        )?
-      };
-
-      let resolved_type_directive =
-        if let Some(types_specifier) = import_desc.deno_types.as_ref() {
-          Some(ModuleSpecifier::resolve_import(
-            &types_specifier,
-            &module_specifier.to_string(),
-          )?)
-        } else {
-          None
-        };
-
-      let import_descriptor = ImportDescriptor {
-        specifier: import_desc.specifier.to_string(),
-        resolved_specifier,
-        type_directive: import_desc.deno_types,
-        resolved_type_directive,
-      };
-
-      imports.push(import_descriptor);
-    }
-
-    for ref_desc in ref_descs {
-      if AVAILABLE_LIBS.contains(&ref_desc.specifier.as_str()) {
-        continue;
-      }
-
-      let resolved_specifier = ModuleSpecifier::resolve_import(
-        &ref_desc.specifier,
-        &module_specifier.to_string(),
-      )?;
-
-      let reference_descriptor = ReferenceDescriptor {
-        specifier: ref_desc.specifier.to_string(),
-        resolved_specifier,
-      };
-
-      match ref_desc.kind {
+    for ref_descriptor in references {
+      match ref_descriptor.kind {
         TsReferenceKind::Lib => {
-          lib_directives.push(reference_descriptor);
+          lib_directives.push(ref_descriptor);
         }
         TsReferenceKind::Types => {
-          types_directives.push(reference_descriptor);
+          types_directives.push(ref_descriptor);
         }
         TsReferenceKind::Path => {
-          referenced_files.push(reference_descriptor);
+          referenced_files.push(ref_descriptor);
         }
       }
     }
 
-    self.graph.0.insert(
+    self.graph.insert(
       module_specifier.to_string(),
       ModuleGraphFile {
         specifier: specifier.to_string(),
         url: specifier.to_string(),
         redirect: None,
-        media_type: map_file_extension(&PathBuf::from(specifier.clone()))
-          as i32,
+        media_type: map_file_extension(&PathBuf::from(specifier.clone())),
         filename: specifier,
         source_code,
         imports,
@@ -307,43 +388,24 @@ impl ModuleGraphLoader {
     &mut self,
     module_specifier: ModuleSpecifier,
     maybe_referrer: Option<ModuleSpecifier>,
+    maybe_location: Option<Location>,
   ) -> Result<(), ErrBox> {
     if self.has_downloaded.contains(&module_specifier) {
       return Ok(());
     }
 
-    // Disallow http:// imports from modules loaded over https://
-    if let Some(referrer) = maybe_referrer.as_ref() {
-      if let "https" = referrer.as_url().scheme() {
-        if let "http" = module_specifier.as_url().scheme() {
-          let e = OpError::permission_denied(
-            "Modules loaded over https:// are not allowed to import modules over http://".to_string()
-          );
-          return Err(e.into());
-        };
-      };
-    };
+    validate_no_downgrade(
+      &module_specifier,
+      maybe_referrer.as_ref(),
+      maybe_location.as_ref(),
+    )?;
 
     if !self.is_dyn_import {
-      // Verify that remote file doesn't try to statically import local file.
-      if let Some(referrer) = maybe_referrer.as_ref() {
-        let referrer_url = referrer.as_url();
-        match referrer_url.scheme() {
-          "http" | "https" => {
-            let specifier_url = module_specifier.as_url();
-            match specifier_url.scheme() {
-              "http" | "https" => {}
-              _ => {
-                let e = OpError::permission_denied(
-                  "Remote modules are not allowed to statically import local modules. Use dynamic import instead.".to_string()
-                );
-                return Err(e.into());
-              }
-            }
-          }
-          _ => {}
-        }
-      }
+      validate_no_file_from_remote(
+        &module_specifier,
+        maybe_referrer.as_ref(),
+        maybe_location.as_ref(),
+      )?;
     }
 
     self.has_downloaded.insert(module_specifier.clone());
@@ -355,7 +417,9 @@ impl ModuleGraphLoader {
       let spec_ = spec.clone();
       let source_file = file_fetcher
         .fetch_source_file(&spec_, maybe_referrer, perms)
-        .await?;
+        .await
+        .map_err(|e| err_with_location(e, maybe_location.as_ref()))?;
+
       Ok((spec_.clone(), source_file))
     }
     .boxed_local();
@@ -383,14 +447,14 @@ impl ModuleGraphLoader {
     // for proper URL point to redirect target.
     if module_specifier.as_url() != &source_file.url {
       // TODO(bartlomieju): refactor, this is a band-aid
-      self.graph.0.insert(
+      self.graph.insert(
         module_specifier.to_string(),
         ModuleGraphFile {
           specifier: module_specifier.to_string(),
           url: module_specifier.to_string(),
           redirect: Some(source_file.url.to_string()),
           filename: source_file.filename.to_str().unwrap().to_string(),
-          media_type: source_file.media_type as i32,
+          media_type: source_file.media_type,
           source_code: "".to_string(),
           imports: vec![],
           referenced_files: vec![],
@@ -412,121 +476,85 @@ impl ModuleGraphLoader {
             &types_specifier,
             &module_specifier.to_string(),
           )?,
+          kind: TsReferenceKind::Types,
+          // TODO(bartlomieju): location is not needed in here and constructing
+          // location by hand is bad
+          location: Location {
+            filename: module_specifier.to_string(),
+            line: 0,
+            col: 0,
+          },
         };
         self.download_module(
           type_header.resolved_specifier.clone(),
           Some(module_specifier.clone()),
+          None,
         )?;
         type_headers.push(type_header);
       }
 
-      let (import_descs, ref_descs) = analyze_dependencies_and_references(
+      let (raw_imports, raw_refs) = pre_process_file(
         &module_specifier.to_string(),
         source_file.media_type,
         &source_code,
         self.analyze_dynamic_imports,
       )?;
+      let (imports_, references) = resolve_imports_and_references(
+        module_specifier.clone(),
+        self.maybe_import_map.as_ref(),
+        raw_imports,
+        raw_refs,
+      )?;
 
-      for import_desc in import_descs {
-        let maybe_resolved =
-          if let Some(import_map) = self.maybe_import_map.as_ref() {
-            import_map
-              .resolve(&import_desc.specifier, &module_specifier.to_string())?
-          } else {
-            None
-          };
-
-        let resolved_specifier = if let Some(resolved) = maybe_resolved {
-          resolved
-        } else {
-          ModuleSpecifier::resolve_import(
-            &import_desc.specifier,
-            &module_specifier.to_string(),
-          )?
-        };
-
-        let resolved_type_directive =
-          if let Some(types_specifier) = import_desc.deno_types.as_ref() {
-            Some(ModuleSpecifier::resolve_import(
-              &types_specifier,
-              &module_specifier.to_string(),
-            )?)
-          } else {
-            None
-          };
-
-        let import_descriptor = ImportDescriptor {
-          specifier: import_desc.specifier.to_string(),
-          resolved_specifier,
-          type_directive: import_desc.deno_types.clone(),
-          resolved_type_directive,
-        };
-
-        self
-          .download_module(
-            import_descriptor.resolved_specifier.clone(),
-            Some(module_specifier.clone()),
-          )
-          .map_err(|e| err_with_location(e, &import_desc.location))?;
+      for import_descriptor in imports_ {
+        self.download_module(
+          import_descriptor.resolved_specifier.clone(),
+          Some(module_specifier.clone()),
+          Some(import_descriptor.location.clone()),
+        )?;
 
         if let Some(type_dir_url) =
           import_descriptor.resolved_type_directive.as_ref()
         {
-          self
-            .download_module(
-              type_dir_url.clone(),
-              Some(module_specifier.clone()),
-            )
-            .map_err(|e| err_with_location(e, &import_desc.location))?;
+          self.download_module(
+            type_dir_url.clone(),
+            Some(module_specifier.clone()),
+            Some(import_descriptor.location.clone()),
+          )?;
         }
 
         imports.push(import_descriptor);
       }
 
-      for ref_desc in ref_descs {
-        if AVAILABLE_LIBS.contains(&ref_desc.specifier.as_str()) {
-          continue;
-        }
-
-        let resolved_specifier = ModuleSpecifier::resolve_import(
-          &ref_desc.specifier,
-          &module_specifier.to_string(),
+      for ref_descriptor in references {
+        self.download_module(
+          ref_descriptor.resolved_specifier.clone(),
+          Some(module_specifier.clone()),
+          Some(ref_descriptor.location.clone()),
         )?;
 
-        let reference_descriptor = ReferenceDescriptor {
-          specifier: ref_desc.specifier.to_string(),
-          resolved_specifier,
-        };
-
-        self
-          .download_module(
-            reference_descriptor.resolved_specifier.clone(),
-            Some(module_specifier.clone()),
-          )
-          .map_err(|e| err_with_location(e, &ref_desc.location))?;
-
-        match ref_desc.kind {
+        match ref_descriptor.kind {
           TsReferenceKind::Lib => {
-            lib_directives.push(reference_descriptor);
+            lib_directives.push(ref_descriptor);
           }
           TsReferenceKind::Types => {
-            types_directives.push(reference_descriptor);
+            types_directives.push(ref_descriptor);
           }
           TsReferenceKind::Path => {
-            referenced_files.push(reference_descriptor);
+            referenced_files.push(ref_descriptor);
           }
         }
       }
     }
 
-    self.graph.0.insert(
+    self.graph.insert(
       module_specifier.to_string(),
       ModuleGraphFile {
         specifier: module_specifier.to_string(),
         url: module_specifier.to_string(),
         redirect: None,
         filename: source_file.filename.to_str().unwrap().to_string(),
-        media_type: source_file.media_type as i32,
+        media_type: source_file.media_type,
         source_code,
         imports,
         referenced_files,
@@ -546,7 +574,7 @@ mod tests {
 
   async fn build_graph(
     module_specifier: &ModuleSpecifier,
-  ) -> Result<HashMap<String, ModuleGraphFile>, ErrBox> {
+  ) -> Result<ModuleGraph, ErrBox> {
     let global_state = GlobalState::new(Default::default()).unwrap();
     let mut graph_loader = ModuleGraphLoader::new(
       global_state.file_fetcher.clone(),
@@ -824,3 +852,102 @@ mod tests {
     drop(http_server_guard);
   }
 }
+
+// TODO(bartlomieju): use baseline tests from TSC to ensure
+// compatibility
+#[test]
+fn test_pre_process_file() {
+  let source = r#"
+// This comment is placed to make sure that directives are parsed
+// even when they start on non-first line
+  
+/// <reference lib="dom" />
+/// <reference types="./type_reference.d.ts" />
+/// <reference path="./type_reference/dep.ts" />
+// @deno-types="./type_definitions/foo.d.ts"
+import { foo } from "./type_definitions/foo.js";
+// @deno-types="./type_definitions/fizz.d.ts"
+import "./type_definitions/fizz.js";
+
+/// <reference path="./type_reference/dep2.ts" />
+
+import * as qat from "./type_definitions/qat.ts";
+
+console.log(foo);
+console.log(fizz);
+console.log(qat.qat);  
+"#;
+
+  let (imports, references) =
+    pre_process_file("some/file.ts", MediaType::TypeScript, source, true)
+      .expect("Failed to parse");
+
+  assert_eq!(
+    imports,
+    vec![
+      ImportDesc {
+        specifier: "./type_definitions/foo.js".to_string(),
+        deno_types: Some("./type_definitions/foo.d.ts".to_string()),
+        location: Location {
+          filename: "some/file.ts".to_string(),
+          line: 9,
+          col: 0,
+        },
+      },
+      ImportDesc {
+        specifier: "./type_definitions/fizz.js".to_string(),
+        deno_types: Some("./type_definitions/fizz.d.ts".to_string()),
+        location: Location {
+          filename: "some/file.ts".to_string(),
+          line: 11,
+          col: 0,
+        },
+      },
+      ImportDesc {
+        specifier: "./type_definitions/qat.ts".to_string(),
+        deno_types: None,
+        location: Location {
+          filename: "some/file.ts".to_string(),
+          line: 15,
+          col: 0,
+        },
+      },
+    ]
+  );
+
+  // According to TS docs (https://www.typescriptlang.org/docs/handbook/triple-slash-directives.html)
+  // directives that are not at the top of the file are ignored, so only
+  // 3 references should be captured instead of 4.
+  assert_eq!(
+    references,
+    vec![
+      TsReferenceDesc {
+        specifier: "dom".to_string(),
+        kind: TsReferenceKind::Lib,
+        location: Location {
+          filename: "some/file.ts".to_string(),
+          line: 5,
+          col: 0,
+        },
+      },
+      TsReferenceDesc {
+        specifier: "./type_reference.d.ts".to_string(),
+        kind: TsReferenceKind::Types,
+        location: Location {
+          filename: "some/file.ts".to_string(),
+          line: 6,
+          col: 0,
+        },
+      },
+      TsReferenceDesc {
+        specifier: "./type_reference/dep.ts".to_string(),
+        kind: TsReferenceKind::Path,
+        location: Location {
+          filename: "some/file.ts".to_string(),
+          line: 7,
+          col: 0,
+        },
+      },
+    ]
+  );
+}