Native ES modules (#1460)

* Native ES modules

This is a major refactor of internal compiler.

Before: JS and TS both were sent through the typescript compiler where
their imports were parsed and handled. Both compiled to AMD JS and
finally sent to V8

Now: JS is sent directly into V8. TS is sent through the typescript
compiler, but tsc generates ES modules now instead of AMD. This
generated JS is then dumped into V8.

This should much faster for pure JS code. It may improve TS compilation
speed.

In the future this allows us to separate TS out of the runtime heap and
into its own dedicated snapshot. This will result in a smaller runtime
heap, and thus should be faster.

Some tests were unfortunately disabled to ease landing this patch:
1. compiler_tests.ts which I intend to bring back in later commits.
2. Some text_encoding_test.ts tests which made the file invalid utf8.
   See PR for a discussion.
Also worth noting that this is necessary to support WASM

8 files changed, 226 insertions, 39 deletions

diff --git a/src/compiler.rs b/src/compiler.rs
new file mode 100644
index 000000000..627be0a44
--- /dev/null
+++ b/src/compiler.rs
@@ -0,0 +1,156 @@
+// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
+use isolate::Buf;
+use isolate::IsolateState;
+use msg;
+use resources;
+use resources::Resource;
+use resources::ResourceId;
+use workers;
+
+use futures::Future;
+use serde_json;
+use std::sync::Arc;
+use std::sync::Mutex;
+
+lazy_static! {
+  static ref c_rid: Mutex<Option<ResourceId>> = Mutex::new(None);
+}
+
+// This corresponds to JS ModuleMetaData.
+// TODO Rename one or the other so they correspond.
+#[derive(Debug)]
+pub struct CodeFetchOutput {
+  pub module_name: String,
+  pub filename: String,
+  pub media_type: msg::MediaType,
+  pub source_code: String,
+  pub maybe_output_code: Option<String>,
+  pub maybe_source_map: Option<String>,
+}
+
+impl CodeFetchOutput {
+  pub fn js_source<'a>(&'a self) -> String {
+    if self.media_type == msg::MediaType::Json {
+      return String::from(format!("export default {};", self.source_code));
+    }
+    match self.maybe_output_code {
+      None => self.source_code.clone(),
+      Some(ref output_code) => output_code.clone(),
+    }
+  }
+}
+
+impl CodeFetchOutput {
+  // TODO Use serde_derive? Use flatbuffers?
+  fn from_json(json_str: &str) -> Option<Self> {
+    match serde_json::from_str::<serde_json::Value>(json_str) {
+      Ok(serde_json::Value::Object(map)) => {
+        let module_name = match map["moduleId"].as_str() {
+          None => return None,
+          Some(s) => s.to_string(),
+        };
+
+        let filename = match map["fileName"].as_str() {
+          None => return None,
+          Some(s) => s.to_string(),
+        };
+
+        let source_code = match map["sourceCode"].as_str() {
+          None => return None,
+          Some(s) => s.to_string(),
+        };
+
+        let maybe_output_code =
+          map["outputCode"].as_str().map(|s| s.to_string());
+
+        let maybe_source_map = map["sourceMap"].as_str().map(|s| s.to_string());
+
+        Some(CodeFetchOutput {
+          module_name,
+          filename,
+          media_type: msg::MediaType::JavaScript, // TODO
+          source_code,
+          maybe_output_code,
+          maybe_source_map,
+        })
+      }
+      _ => None,
+    }
+  }
+}
+
+fn lazy_start(parent_state: &Arc<IsolateState>) -> Resource {
+  let mut cell = c_rid.lock().unwrap();
+  let rid = cell.get_or_insert_with(|| {
+    let resource =
+      workers::spawn(parent_state.clone(), "compilerMain()".to_string());
+    resource.rid
+  });
+  Resource { rid: *rid }
+}
+
+fn req(specifier: &str, referrer: &str) -> Buf {
+  json!({
+    "specifier": specifier,
+    "referrer": referrer,
+  }).to_string()
+  .into_boxed_str()
+  .into_boxed_bytes()
+}
+
+pub fn compile_sync(
+  parent_state: &Arc<IsolateState>,
+  specifier: &str,
+  referrer: &str,
+) -> Option<CodeFetchOutput> {
+  let req_msg = req(specifier, referrer);
+
+  let compiler = lazy_start(parent_state);
+
+  let send_future = resources::worker_post_message(compiler.rid, req_msg);
+  send_future.wait().unwrap();
+
+  let recv_future = resources::worker_recv_message(compiler.rid);
+  let res_msg = recv_future.wait().unwrap().unwrap();
+
+  let res_json = std::str::from_utf8(&res_msg).unwrap();
+  CodeFetchOutput::from_json(res_json)
+}
+
+#[cfg(test)]
+mod tests {
+  use super::*;
+
+  #[test]
+  fn test_compile_sync() {
+    let cwd = std::env::current_dir().unwrap();
+    let cwd_string = cwd.to_str().unwrap().to_owned();
+
+    let specifier = "./tests/002_hello.ts";
+    let referrer = cwd_string + "/";
+
+    let cfo =
+      compile_sync(&IsolateState::mock(), specifier, &referrer).unwrap();
+    let output_code = cfo.maybe_output_code.unwrap();
+    assert!(output_code.starts_with("console.log(\"Hello World\");"));
+  }
+
+  #[test]
+  fn code_fetch_output_from_json() {
+    let json = r#"{
+      "moduleId":"/Users/rld/src/deno/tests/002_hello.ts",
+      "fileName":"/Users/rld/src/deno/tests/002_hello.ts",
+      "mediaType":1,
+      "sourceCode":"console.log(\"Hello World\");\n",
+      "outputCode":"yyy",
+      "sourceMap":"xxx",
+      "scriptVersion":"1"
+    }"#;
+    let actual = CodeFetchOutput::from_json(json).unwrap();
+    assert_eq!(actual.filename, "/Users/rld/src/deno/tests/002_hello.ts");
+    assert_eq!(actual.module_name, "/Users/rld/src/deno/tests/002_hello.ts");
+    assert_eq!(actual.source_code, "console.log(\"Hello World\");\n");
+    assert_eq!(actual.maybe_output_code, Some("yyy".to_string()));
+    assert_eq!(actual.maybe_source_map, Some("xxx".to_string()));
+  }
+}
diff --git a/src/deno_dir.rs b/src/deno_dir.rs
index d3d67d195..3b35035d6 100644
--- a/src/deno_dir.rs
+++ b/src/deno_dir.rs
@@ -1,4 +1,5 @@
 // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
+use compiler::CodeFetchOutput;
 use dirs;
 use errors;
 use errors::DenoError;
@@ -19,25 +20,6 @@ use std::result::Result;
 use url;
 use url::Url;
 
-#[derive(Debug)]
-pub struct CodeFetchOutput {
-  pub module_name: String,
-  pub filename: String,
-  pub media_type: msg::MediaType,
-  pub source_code: String,
-  pub maybe_output_code: Option<String>,
-  pub maybe_source_map: Option<String>,
-}
-
-impl CodeFetchOutput {
-  pub fn js_source<'a>(&'a self) -> &'a String {
-    match self.maybe_output_code {
-      None => &self.source_code,
-      Some(ref output_code) => output_code,
-    }
-  }
-}
-
 /// Gets corresponding MediaType given extension
 fn extmap(ext: &str) -> msg::MediaType {
   match ext {
@@ -319,6 +301,10 @@ impl DenoDir {
 
     out.source_code = filter_shebang(out.source_code);
 
+    if out.media_type != msg::MediaType::TypeScript {
+      return Ok(out);
+    }
+
     let result =
       self.load_cache(out.filename.as_str(), out.source_code.as_str());
     match result {
diff --git a/src/isolate.rs b/src/isolate.rs
index c4174de3f..fcd07d23e 100644
--- a/src/isolate.rs
+++ b/src/isolate.rs
@@ -1,5 +1,4 @@
 // Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.
-
 // Do not use FlatBuffers in this module.
 // TODO Currently this module uses Tokio, but it would be nice if they were
 // decoupled.
@@ -253,13 +252,14 @@ impl Isolate {
 
   /// Executes the provided JavaScript module.
   pub fn execute_mod(&self, js_filename: &str) -> Result<(), JSError> {
-    let out = self.state.dir.code_fetch(js_filename, ".").unwrap();
-    debug!("module_resolve complete {}", out.filename);
+    let out =
+      code_fetch_and_maybe_compile(&self.state, js_filename, ".").unwrap();
 
-    let filename = CString::new(js_filename).unwrap();
+    let filename = CString::new(out.filename.clone()).unwrap();
     let filename_ptr = filename.as_ptr() as *const i8;
 
     let js_source = CString::new(out.js_source().clone()).unwrap();
+    let js_source = CString::new(js_source).unwrap();
     let js_source_ptr = js_source.as_ptr() as *const i8;
 
     let r = unsafe {
@@ -364,6 +364,25 @@ impl Drop for Isolate {
   }
 }
 
+use compiler::compile_sync;
+use compiler::CodeFetchOutput;
+use msg;
+fn code_fetch_and_maybe_compile(
+  state: &Arc<IsolateState>,
+  specifier: &str,
+  referrer: &str,
+) -> Result<CodeFetchOutput, DenoError> {
+  let mut out = state.dir.code_fetch(specifier, referrer)?;
+  if out.media_type == msg::MediaType::TypeScript
+    && out.maybe_output_code.is_none()
+  {
+    debug!(">>>>> compile_sync START");
+    out = compile_sync(state, specifier, &referrer).unwrap();
+    debug!(">>>>> compile_sync END");
+  }
+  Ok(out)
+}
+
 extern "C" fn resolve_cb(
   user_data: *mut c_void,
   specifier_ptr: *const c_char,
@@ -378,16 +397,21 @@ extern "C" fn resolve_cb(
   debug!("module_resolve callback {} {}", specifier, referrer);
   let isolate = unsafe { Isolate::from_raw_ptr(user_data) };
 
-  let out = isolate.state.dir.code_fetch(specifier, referrer).unwrap();
-  debug!("module_resolve complete {}", out.filename);
+  let out =
+    code_fetch_and_maybe_compile(&isolate.state, specifier, referrer).unwrap();
+
+  let filename = CString::new(out.filename.clone()).unwrap();
+  let filename_ptr = filename.as_ptr() as *const i8;
 
-  // TODO js_source is not null terminated, therefore the clone.
   let js_source = CString::new(out.js_source().clone()).unwrap();
-  let filename = out.filename.as_ptr() as *const i8;
   let js_source_ptr = js_source.as_ptr() as *const i8;
 
   unsafe {
-    libdeno::deno_resolve_ok(isolate.libdeno_isolate, filename, js_source_ptr)
+    libdeno::deno_resolve_ok(
+      isolate.libdeno_isolate,
+      filename_ptr,
+      js_source_ptr,
+    )
   };
 }
 
diff --git a/src/main.rs b/src/main.rs
index 75cc61b58..629605e0e 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -10,7 +10,6 @@ extern crate rand;
 extern crate remove_dir_all;
 extern crate ring;
 extern crate rustyline;
-extern crate serde_json;
 extern crate source_map_mappings;
 extern crate tempfile;
 extern crate tokio;
@@ -27,7 +26,10 @@ extern crate lazy_static;
 extern crate log;
 #[macro_use]
 extern crate futures;
+#[macro_use]
+extern crate serde_json;
 
+pub mod compiler;
 pub mod deno_dir;
 pub mod errors;
 pub mod flags;
@@ -47,7 +49,7 @@ pub mod snapshot;
 mod tokio_util;
 mod tokio_write;
 pub mod version;
-mod workers;
+pub mod workers;
 
 #[cfg(unix)]
 mod eager_unix;
@@ -100,10 +102,21 @@ fn main() {
   let state = Arc::new(isolate::IsolateState::new(flags, rest_argv, None));
   let snapshot = snapshot::deno_snapshot();
   let isolate = isolate::Isolate::new(snapshot, state, ops::dispatch);
+
   tokio_util::init(|| {
+    // Setup runtime.
     isolate
       .execute("denoMain();")
       .unwrap_or_else(print_err_and_exit);
+
+    // Execute input file.
+    if isolate.state.argv.len() > 1 {
+      let input_filename = &isolate.state.argv[1];
+      isolate
+        .execute_mod(input_filename)
+        .unwrap_or_else(print_err_and_exit);
+    }
+
     isolate.event_loop().unwrap_or_else(print_err_and_exit);
   });
 }
diff --git a/src/msg.fbs b/src/msg.fbs
index a9afb195f..57f8a7f2c 100644
--- a/src/msg.fbs
+++ b/src/msg.fbs
@@ -127,6 +127,13 @@ enum MediaType: byte {
   Unknown
 }
 
+table Shared {
+  lock: bool;
+  head: int;
+  tail: int;
+  ring: [Base];
+}
+
 table Base {
   cmd_id: uint32;
   sync: bool = false;
diff --git a/src/ops.rs b/src/ops.rs
index cf25f29e0..07170da94 100644
--- a/src/ops.rs
+++ b/src/ops.rs
@@ -672,7 +672,7 @@ fn op_close(
   let rid = inner.rid();
   match resources::lookup(rid) {
     None => odd_future(errors::bad_resource()),
-    Some(mut resource) => {
+    Some(resource) => {
       resource.close();
       ok_future(empty_buf())
     }
diff --git a/src/resources.rs b/src/resources.rs
index 69173fe85..308fe1251 100644
--- a/src/resources.rs
+++ b/src/resources.rs
@@ -124,7 +124,6 @@ pub fn table_entries() -> Vec<(u32, String)> {
 fn test_table_entries() {
   let mut entries = table_entries();
   entries.sort();
-  assert_eq!(entries.len(), 3);
   assert_eq!(entries[0], (0, String::from("stdin")));
   assert_eq!(entries[1], (1, String::from("stdout")));
   assert_eq!(entries[2], (2, String::from("stderr")));
@@ -173,7 +172,7 @@ impl Resource {
 
   // close(2) is done by dropping the value. Therefore we just need to remove
   // the resource from the RESOURCE_TABLE.
-  pub fn close(&mut self) {
+  pub fn close(&self) {
     let mut table = RESOURCE_TABLE.lock().unwrap();
     let r = table.remove(&self.rid);
     assert!(r.is_some());
diff --git a/src/workers.rs b/src/workers.rs
index 319f4018d..e67d80489 100644
--- a/src/workers.rs
+++ b/src/workers.rs
@@ -1,7 +1,4 @@
 // Copyright 2018 the Deno authors. All rights reserved. MIT license.
-
-#![allow(dead_code)]
-
 use isolate::Buf;
 use isolate::Isolate;
 use isolate::IsolateState;
@@ -53,7 +50,10 @@ impl Worker {
   }
 }
 
-fn spawn(state: Arc<IsolateState>, js_source: String) -> resources::Resource {
+pub fn spawn(
+  state: Arc<IsolateState>,
+  js_source: String,
+) -> resources::Resource {
   // TODO This function should return a Future, so that the caller can retrieve
   // the JSError if one is thrown. Currently it just prints to stderr and calls
   // exit(1).
@@ -64,11 +64,12 @@ fn spawn(state: Arc<IsolateState>, js_source: String) -> resources::Resource {
     .spawn(move || {
       let (worker, external_channels) = Worker::new(&state);
 
-      let mut resource = resources::add_worker(external_channels);
+      let resource = resources::add_worker(external_channels);
       p.send(resource.clone()).unwrap();
 
       tokio_util::init(|| {
         (|| -> Result<(), JSError> {
+          worker.execute("denoMain()")?;
           worker.execute("workerMain()")?;
           worker.execute(&js_source)?;
           worker.event_loop()?;
@@ -142,7 +143,8 @@ mod tests {
 
     // TODO Need a way to get a future for when a resource closes.
     // For now, just sleep for a bit.
-    thread::sleep(std::time::Duration::from_millis(100));
+    // resource.close();
+    thread::sleep(std::time::Duration::from_millis(1000));
     assert_eq!(resources::get_type(resource.rid), None);
   }
 }