diff options
Diffstat (limited to 'core')
| -rw-r--r-- | core/Cargo.toml | 5 | ||||
| -rw-r--r-- | core/benches/op_baseline.rs | 88 | ||||
| -rw-r--r-- | core/bindings.rs | 54 | ||||
| -rw-r--r-- | core/core.js | 8 | ||||
| -rw-r--r-- | core/examples/http_bench_json_ops.rs | 2 | ||||
| -rw-r--r-- | core/extensions.rs | 22 | ||||
| -rw-r--r-- | core/inspector.rs | 755 | ||||
| -rw-r--r-- | core/lib.rs | 6 | ||||
| -rw-r--r-- | core/modules.rs | 596 | ||||
| -rw-r--r-- | core/ops_json.rs | 2 | ||||
| -rw-r--r-- | core/runtime.rs | 753 |
11 files changed, 1539 insertions, 752 deletions
diff --git a/core/Cargo.toml b/core/Cargo.toml index e22016e67..7c11bfbc1 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -34,8 +34,3 @@ path = "examples/http_bench_json_ops.rs" # These dependencies are only used for the 'http_bench_*_ops' examples. [dev-dependencies] tokio = { version = "1.6.1", features = ["full"] } -bencher = "0.1" - -[[bench]] -name = "op_baseline" -harness = false diff --git a/core/benches/op_baseline.rs b/core/benches/op_baseline.rs deleted file mode 100644 index ecac4ca26..000000000 --- a/core/benches/op_baseline.rs +++ /dev/null @@ -1,88 +0,0 @@ -use bencher::{benchmark_group, benchmark_main, Bencher}; - -use deno_core::error::AnyError; -use deno_core::op_async; -use deno_core::op_sync; -use deno_core::serialize_op_result; -use deno_core::v8; -use deno_core::JsRuntime; -use deno_core::Op; -use deno_core::OpState; - -use std::cell::RefCell; -use std::rc::Rc; - -fn create_js_runtime() -> JsRuntime { - let mut runtime = JsRuntime::new(Default::default()); - runtime.register_op("pi_json", op_sync(|_, _: (), _: ()| Ok(314159))); - runtime.register_op("pi_async", op_async(op_pi_async)); - runtime.register_op("nop", |state, _| { - Op::Sync(serialize_op_result(Ok(9), state)) - }); - runtime.sync_ops_cache(); - - runtime -} - -// this is a function since async closures aren't stable -async fn op_pi_async( - _: Rc<RefCell<OpState>>, - _: (), - _: (), -) -> Result<i64, AnyError> { - Ok(314159) -} - -pub fn bench_runtime_js(b: &mut Bencher, src: &str) { - let mut runtime = create_js_runtime(); - let scope = &mut runtime.handle_scope(); - let code = v8::String::new(scope, src).unwrap(); - let script = v8::Script::compile(scope, code, None).unwrap(); - b.iter(|| { - script.run(scope).unwrap(); - }); -} - -pub fn bench_runtime_js_async(b: &mut Bencher, src: &str) { - let mut runtime = create_js_runtime(); - let tokio_runtime = tokio::runtime::Builder::new_current_thread() - .enable_all() - .build() - .unwrap(); - - b.iter(|| { - runtime.execute("inner_loop", src).unwrap(); - let future = runtime.run_event_loop(); - tokio_runtime.block_on(future).unwrap(); - }); -} - -fn bench_op_pi_json(b: &mut Bencher) { - bench_runtime_js( - b, - r#"for(let i=0; i < 1e3; i++) { - Deno.core.opSync("pi_json", null); - }"#, - ); -} - -fn bench_op_nop(b: &mut Bencher) { - bench_runtime_js( - b, - r#"for(let i=0; i < 1e3; i++) { - Deno.core.opSync("nop", null, null, null); - }"#, - ); -} - -fn bench_op_async(b: &mut Bencher) { - bench_runtime_js_async( - b, - r#"for(let i=0; i < 1e3; i++) { - Deno.core.opAsync("pi_async", null); - }"#, - ); -} - -benchmark_group!(benches, bench_op_pi_json, bench_op_nop, bench_op_async); -benchmark_main!(benches); diff --git a/core/bindings.rs b/core/bindings.rs index f6c94b335..5fb57aac3 100644 --- a/core/bindings.rs +++ b/core/bindings.rs @@ -8,6 +8,7 @@ use crate::OpPayload; use crate::OpTable; use crate::PromiseId; use crate::ZeroCopyBuf; +use log::debug; use rusty_v8 as v8; use serde::Serialize; use serde_v8::to_v8; @@ -179,8 +180,18 @@ pub extern "C" fn host_import_module_dynamically_callback( let resolver_handle = v8::Global::new(scope, resolver); { let state_rc = JsRuntime::state(scope); - let mut state = state_rc.borrow_mut(); - state.dyn_import_cb(resolver_handle, &specifier_str, &referrer_name_str); + let module_map_rc = JsRuntime::module_map(scope); + + debug!( + "dyn_import specifier {} referrer {} ", + specifier_str, referrer_name_str + ); + module_map_rc.borrow_mut().load_dynamic_import( + &specifier_str, + &referrer_name_str, + resolver_handle, + ); + state_rc.borrow_mut().notify_new_dynamic_import(); } // Map errors from module resolution (not JS errors from module execution) to @@ -218,12 +229,11 @@ pub extern "C" fn host_initialize_import_meta_object_callback( meta: v8::Local<v8::Object>, ) { let scope = &mut unsafe { v8::CallbackScope::new(context) }; - let state_rc = JsRuntime::state(scope); - let state = state_rc.borrow(); + let module_map_rc = JsRuntime::module_map(scope); + let module_map = module_map_rc.borrow(); let module_global = v8::Global::new(scope, module); - let info = state - .module_map + let info = module_map .get_info(&module_global) .expect("Module not found"); @@ -573,7 +583,11 @@ fn queue_microtask( }; } -// Called by V8 during `Isolate::mod_instantiate`. +/// Called by V8 during `JsRuntime::instantiate_module`. +/// +/// This function borrows `ModuleMap` from the isolate slot, +/// so it is crucial to ensure there are no existing borrows +/// of `ModuleMap` when `JsRuntime::instantiate_module` is called. pub fn module_resolve_callback<'s>( context: v8::Local<'s, v8::Context>, specifier: v8::Local<'s, v8::String>, @@ -582,32 +596,22 @@ pub fn module_resolve_callback<'s>( ) -> Option<v8::Local<'s, v8::Module>> { let scope = &mut unsafe { v8::CallbackScope::new(context) }; - let state_rc = JsRuntime::state(scope); - let state = state_rc.borrow(); + let module_map_rc = JsRuntime::module_map(scope); + let module_map = module_map_rc.borrow(); let referrer_global = v8::Global::new(scope, referrer); - let referrer_info = state - .module_map + + let referrer_info = module_map .get_info(&referrer_global) .expect("ModuleInfo not found"); let referrer_name = referrer_info.name.to_string(); let specifier_str = specifier.to_rust_string_lossy(scope); - let resolved_specifier = state - .loader - .resolve( - state.op_state.clone(), - &specifier_str, - &referrer_name, - false, - ) - .expect("Module should have been already resolved"); - - if let Some(id) = state.module_map.get_id(resolved_specifier.as_str()) { - if let Some(handle) = state.module_map.get_handle(id) { - return Some(v8::Local::new(scope, handle)); - } + let maybe_module = + module_map.resolve_callback(scope, &specifier_str, &referrer_name); + if let Some(module) = maybe_module { + return Some(module); } let msg = format!( diff --git a/core/core.js b/core/core.js index 32ff7758e..8e06a3e45 100644 --- a/core/core.js +++ b/core/core.js @@ -108,16 +108,16 @@ return res; } - function opAsync(opName, args = null, zeroCopy = null) { + function opAsync(opName, arg1 = null, arg2 = null) { const promiseId = nextPromiseId++; - const maybeError = dispatch(opName, promiseId, args, zeroCopy); + const maybeError = dispatch(opName, promiseId, arg1, arg2); // Handle sync error (e.g: error parsing args) if (maybeError) return unwrapOpResult(maybeError); return setPromise(promiseId).then(unwrapOpResult); } - function opSync(opName, args = null, zeroCopy = null) { - return unwrapOpResult(dispatch(opName, null, args, zeroCopy)); + function opSync(opName, arg1 = null, arg2 = null) { + return unwrapOpResult(dispatch(opName, null, arg1, arg2)); } function resources() { diff --git a/core/examples/http_bench_json_ops.rs b/core/examples/http_bench_json_ops.rs index 88556cea7..a29ebfcf2 100644 --- a/core/examples/http_bench_json_ops.rs +++ b/core/examples/http_bench_json_ops.rs @@ -223,7 +223,7 @@ fn main() { include_str!("http_bench_json_ops.js"), ) .unwrap(); - js_runtime.run_event_loop().await + js_runtime.run_event_loop(false).await }; runtime.block_on(future).unwrap(); } diff --git a/core/extensions.rs b/core/extensions.rs index e38d11fc9..54d648071 100644 --- a/core/extensions.rs +++ b/core/extensions.rs @@ -1,7 +1,8 @@ use crate::error::AnyError; use crate::{OpFn, OpState}; -pub type SourcePair = (&'static str, &'static str); +pub type SourcePair = (&'static str, Box<SourceLoadFn>); +pub type SourceLoadFn = dyn Fn() -> Result<String, AnyError>; pub type OpPair = (&'static str, Box<OpFn>); pub type OpMiddlewareFn = dyn Fn(&'static str, Box<OpFn>) -> Box<OpFn>; pub type OpStateFn = dyn Fn(&mut OpState) -> Result<(), AnyError>; @@ -24,10 +25,10 @@ impl Extension { /// returns JS source code to be loaded into the isolate (either at snapshotting, /// or at startup). as a vector of a tuple of the file name, and the source code. - pub fn init_js(&self) -> Vec<SourcePair> { + pub fn init_js(&self) -> &[SourcePair] { match &self.js_files { - Some(files) => files.clone(), - None => vec![], + Some(files) => files, + None => &[], } } @@ -104,8 +105,9 @@ impl ExtensionBuilder { } } } -/// Helps embed JS files in an extension. Returns Vec<(&'static str, &'static str)> -/// representing the filename and source code. +/// Helps embed JS files in an extension. Returns Vec<(&'static str, Box<SourceLoadFn>)> +/// representing the filename and source code. This is only meant for extensions +/// that will be snapshotted, as code will be loaded at runtime. /// /// Example: /// ```ignore @@ -121,7 +123,13 @@ macro_rules! include_js_files { vec![ $(( concat!($prefix, "/", $file), - include_str!($file), + Box::new(|| { + let c = std::path::PathBuf::from(env!("CARGO_MANIFEST_DIR")); + let path = c.join($file); + println!("cargo:rerun-if-changed={}", path.display()); + let src = std::fs::read_to_string(path)?; + Ok(src) + }), ),)+ ] }; diff --git a/core/inspector.rs b/core/inspector.rs new file mode 100644 index 000000000..f6e348d88 --- /dev/null +++ b/core/inspector.rs @@ -0,0 +1,755 @@ +// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license. + +//! The documentation for the inspector API is sparse, but these are helpful: +//! https://chromedevtools.github.io/devtools-protocol/ +//! https://hyperandroid.com/2020/02/12/v8-inspector-from-an-embedder-standpoint/ + +use crate::error::generic_error; +use crate::error::AnyError; +use crate::futures::channel::mpsc; +use crate::futures::channel::mpsc::UnboundedReceiver; +use crate::futures::channel::mpsc::UnboundedSender; +use crate::futures::channel::oneshot; +use crate::futures::future::select; +use crate::futures::future::Either; +use crate::futures::future::Future; +use crate::futures::prelude::*; +use crate::futures::stream::FuturesUnordered; +use crate::futures::stream::StreamExt; +use crate::futures::task; +use crate::futures::task::Context; +use crate::futures::task::Poll; +use crate::serde_json; +use crate::serde_json::json; +use crate::serde_json::Value; +use crate::v8; +use std::cell::BorrowMutError; +use std::cell::RefCell; +use std::collections::HashMap; +use std::ffi::c_void; +use std::mem::replace; +use std::mem::take; +use std::mem::MaybeUninit; +use std::pin::Pin; +use std::ptr; +use std::ptr::NonNull; +use std::rc::Rc; +use std::sync::Arc; +use std::sync::Mutex; +use std::thread; + +/// If first argument is `None` then it's a notification, otherwise +/// it's a message. +pub type SessionProxySender = UnboundedSender<(Option<i32>, String)>; +// TODO(bartlomieju): does it even need to send a Result? +// It seems `Vec<u8>` would be enough +pub type SessionProxyReceiver = UnboundedReceiver<Result<Vec<u8>, AnyError>>; + +/// Encapsulates an UnboundedSender/UnboundedReceiver pair that together form +/// a duplex channel for sending/receiving messages in V8 session. +pub struct InspectorSessionProxy { + pub tx: SessionProxySender, + pub rx: SessionProxyReceiver, +} + +impl InspectorSessionProxy { + pub fn split(self) -> (SessionProxySender, SessionProxyReceiver) { + (self.tx, self.rx) + } +} + +#[derive(Clone, Copy)] +enum PollState { + Idle, + Woken, + Polling, + Parked, + Dropped, +} + +/// This structure is used responsible for providing inspector interface +/// to the `JsRuntime`. +/// +/// It stores an instance of `v8::inspector::V8Inspector` and additionally +/// implements `v8::inspector::V8InspectorClientImpl`. +/// +/// After creating this structure it's possible to connect multiple sessions +/// to the inspector, in case of Deno it's either: a "websocket session" that +/// provides integration with Chrome Devtools, or an "in-memory session" that +/// is used for REPL or converage collection. +pub struct JsRuntimeInspector { + v8_inspector_client: v8::inspector::V8InspectorClientBase, + v8_inspector: Rc<RefCell<v8::UniquePtr<v8::inspector::V8Inspector>>>, + new_session_tx: UnboundedSender<InspectorSessionProxy>, + sessions: RefCell<SessionContainer>, + flags: RefCell<InspectorFlags>, + waker: Arc<InspectorWaker>, + deregister_tx: Option<oneshot::Sender<()>>, +} + +impl Drop for JsRuntimeInspector { + fn drop(&mut self) { + // Since the waker is cloneable, it might outlive the inspector itself. + // Set the poll state to 'dropped' so it doesn't attempt to request an + // interrupt from the isolate. + self.waker.update(|w| w.poll_state = PollState::Dropped); + // TODO(bartlomieju): this comment is out of date + // V8 automatically deletes all sessions when an `V8Inspector` instance is + // deleted, however InspectorSession also has a drop handler that cleans + // up after itself. To avoid a double free, make sure the inspector is + // dropped last. + take(&mut *self.sessions.borrow_mut()); + + // Notify counterparty that this instance is being destroyed. Ignoring + // result because counterparty waiting for the signal might have already + // dropped the other end of channel. + if let Some(deregister_tx) = self.deregister_tx.take() { + let _ = deregister_tx.send(()); + } + } +} + +impl v8::inspector::V8InspectorClientImpl for JsRuntimeInspector { + fn base(&self) -> &v8::inspector::V8InspectorClientBase { + &self.v8_inspector_client + } + + fn base_mut(&mut self) -> &mut v8::inspector::V8InspectorClientBase { + &mut self.v8_inspector_client + } + + fn run_message_loop_on_pause(&mut self, context_group_id: i32) { + assert_eq!(context_group_id, JsRuntimeInspector::CONTEXT_GROUP_ID); + self.flags.borrow_mut().on_pause = true; + let _ = self.poll_sessions(None); + } + + fn quit_message_loop_on_pause(&mut self) { + self.flags.borrow_mut().on_pause = false; + } + + fn run_if_waiting_for_debugger(&mut self, context_group_id: i32) { + assert_eq!(context_group_id, JsRuntimeInspector::CONTEXT_GROUP_ID); + self.flags.borrow_mut().session_handshake_done = true; + } +} + +/// `JsRuntimeInspector` implements a Future so that it can poll for new incoming +/// connections and messages from the WebSocket server. The Worker that owns +/// this `JsRuntimeInspector` will call this function from `Worker::poll()`. +impl Future for JsRuntimeInspector { + type Output = (); + fn poll(self: Pin<&mut Self>, cx: &mut Context) -> Poll<()> { + self.poll_sessions(Some(cx)).unwrap() + } +} + +impl JsRuntimeInspector { + /// Currently Deno supports only a single context in `JsRuntime` + /// and thus it's id is provided as an associated contant. + const CONTEXT_GROUP_ID: i32 = 1; + + pub fn new( + isolate: &mut v8::OwnedIsolate, + context: v8::Global<v8::Context>, + ) -> Box<Self> { + let scope = &mut v8::HandleScope::new(isolate); + + let (new_session_tx, new_session_rx) = + mpsc::unbounded::<InspectorSessionProxy>(); + + let v8_inspector_client = + v8::inspector::V8InspectorClientBase::new::<Self>(); + + let flags = InspectorFlags::new(); + let waker = InspectorWaker::new(scope.thread_safe_handle()); + + // Create JsRuntimeInspector instance. + let mut self_ = Box::new(Self { + v8_inspector_client, + v8_inspector: Default::default(), + sessions: Default::default(), + new_session_tx, + flags, + waker, + deregister_tx: None, + }); + self_.v8_inspector = Rc::new(RefCell::new( + v8::inspector::V8Inspector::create(scope, &mut *self_).into(), + )); + self_.sessions = + SessionContainer::new(self_.v8_inspector.clone(), new_session_rx); + + // Tell the inspector about the global context. + let context = v8::Local::new(scope, context); + let context_name = v8::inspector::StringView::from(&b"global context"[..]); + self_ + .v8_inspector + .borrow_mut() + .as_mut() + .unwrap() + .context_created(context, Self::CONTEXT_GROUP_ID, context_name); + + // Poll the session handler so we will get notified whenever there is + // new_incoming debugger activity. + let _ = self_.poll_sessions(None).unwrap(); + + self_ + } + + pub fn has_active_sessions(&self) -> bool { + let sessions = self.sessions.borrow(); + !sessions.established.is_empty() || sessions.handshake.is_some() + } + + fn poll_sessions( + &self, + mut invoker_cx: Option<&mut Context>, + ) -> Result<Poll<()>, BorrowMutError> { + // The futures this function uses do not have re-entrant poll() functions. + // However it is can happpen that poll_sessions() gets re-entered, e.g. + // when an interrupt request is honored while the inspector future is polled + // by the task executor. We let the caller know by returning some error. + let mut sessions = self.sessions.try_borrow_mut()?; + + self.waker.update(|w| { + match w.poll_state { + PollState::Idle | PollState::Woken => w.poll_state = PollState::Polling, + _ => unreachable!(), + }; + }); + + // Create a new Context object that will make downstream futures + // use the InspectorWaker when they are ready to be polled again. + let waker_ref = task::waker_ref(&self.waker); + let cx = &mut Context::from_waker(&waker_ref); + + loop { + loop { + // Do one "handshake" with a newly connected session at a time. + if let Some(session) = &mut sessions.handshake { + let poll_result = session.poll_unpin(cx); + let handshake_done = + replace(&mut self.flags.borrow_mut().session_handshake_done, false); + match poll_result { + Poll::Pending if handshake_done => { + let session = sessions.handshake.take().unwrap(); + sessions.established.push(session); + take(&mut self.flags.borrow_mut().waiting_for_session); + } + Poll::Ready(_) => sessions.handshake = None, + Poll::Pending => break, + }; + } + + // Accept new connections. + match sessions.new_incoming.poll_next_unpin(cx) { + Poll::Ready(Some(session)) => { + let prev = sessions.handshake.replace(session); + assert!(prev.is_none()); + continue; + } + Poll::Ready(None) => {} + Poll::Pending => {} + } + + // Poll established sessions. + match sessions.established.poll_next_unpin(cx) { + Poll::Ready(Some(_)) => continue, + Poll::Ready(None) => break, + Poll::Pending => break, + }; + } + + let should_block = sessions.handshake.is_some() + || self.flags.borrow().on_pause + || self.flags.borrow().waiting_for_session; + + let new_state = self.waker.update(|w| { + match w.poll_state { + PollState::Woken => { + // The inspector was woken while the session handler was being + // polled, so we poll it another time. + w.poll_state = PollState::Polling; + } + PollState::Polling if !should_block => { + // The session handler doesn't need to be polled any longer, and + // there's no reason to block (execution is not paused), so this + // function is about to return. + w.poll_state = PollState::Idle; + // Register the task waker that can be used to wake the parent + // task that will poll the inspector future. + if let Some(cx) = invoker_cx.take() { + w.task_waker.replace(cx.waker().clone()); + } + // Register the address of the inspector, which allows the waker + // to request an interrupt from the isolate. + w.inspector_ptr = NonNull::new(self as *const _ as *mut Self); + } + PollState::Polling if should_block => { + // Isolate execution has been paused but there are no more + // events to process, so this thread will be parked. Therefore, + // store the current thread handle in the waker so it knows + // which thread to unpark when new events arrive. + w.poll_state = PollState::Parked; + w.parked_thread.replace(thread::current()); + } + _ => unreachable!(), + }; + w.poll_state + }); + match new_state { + PollState::Idle => break Ok(Poll::Pending), // Yield to task. + PollState::Polling => {} // Poll the session handler again. + PollState::Parked => thread::park(), // Park the thread. + _ => unreachable!(), + }; + } + } + + /// This function blocks the thread until at least one inspector client has + /// established a websocket connection and successfully completed the + /// handshake. After that, it instructs V8 to pause at the next statement. + pub fn wait_for_session_and_break_on_next_statement(&mut self) { + loop { + match self.sessions.get_mut().established.iter_mut().next() { + Some(session) => break session.break_on_next_statement(), + None => { + self.flags.get_mut().waiting_for_session = true; + let _ = self.poll_sessions(None).unwrap(); + } + }; + } + } + + /// Obtain a sender for proxy channels. + /// + /// After a proxy is sent inspector will wait for a "handshake". + /// Frontend must send "Runtime.runIfWaitingForDebugger" message to + /// complete the handshake. + pub fn get_session_sender(&self) -> UnboundedSender<InspectorSessionProxy> { + self.new_session_tx.clone() + } + + /// Create a channel that notifies the frontend when inspector is dropped. + /// + /// NOTE: Only a single handler is currently available. + pub fn add_deregister_handler(&mut self) -> oneshot::Receiver<()> { + let (tx, rx) = oneshot::channel::<()>(); + let prev = self.deregister_tx.replace(tx); + assert!( + prev.is_none(), + "Only a single deregister handler is allowed" + ); + rx + } + + /// Create a local inspector session that can be used on + /// the same thread as the isolate. + pub fn create_local_session(&self) -> LocalInspectorSession { + // The 'outbound' channel carries messages sent to the session. + let (outbound_tx, outbound_rx) = mpsc::unbounded(); + + // The 'inbound' channel carries messages received from the session. + let (inbound_tx, inbound_rx) = mpsc::unbounded(); + + let proxy = InspectorSessionProxy { + tx: outbound_tx, + rx: inbound_rx, + }; + + // InspectorSessions for a local session is added directly to the "established" + // sessions, so it doesn't need to go through the session sender and handshake + // phase. + let inspector_session = + InspectorSession::new(self.v8_inspector.clone(), proxy); + self + .sessions + .borrow_mut() + .established + .push(inspector_session); + take(&mut self.flags.borrow_mut().waiting_for_session); + + LocalInspectorSession::new(inbound_tx, outbound_rx) + } +} + +#[derive(Default)] +struct InspectorFlags { + waiting_for_session: bool, + session_handshake_done: bool, + on_pause: bool, +} + +impl InspectorFlags { + fn new() -> RefCell<Self> { + let self_ = Self::default(); + RefCell::new(self_) + } +} + +/// A helper structure that helps coordinate sessions during different +/// parts of their lifecycle. +struct SessionContainer { + new_incoming: Pin<Box<dyn Stream<Item = Box<InspectorSession>> + 'static>>, + handshake: Option<Box<InspectorSession>>, + established: FuturesUnordered<Box<InspectorSession>>, +} + +impl SessionContainer { + fn new( + v8_inspector: Rc<RefCell<v8::UniquePtr<v8::inspector::V8Inspector>>>, + new_session_rx: UnboundedReceiver<InspectorSessionProxy>, + ) -> RefCell<Self> { + let new_incoming = new_session_rx + .map(move |session_proxy| { + InspectorSession::new(v8_inspector.clone(), session_proxy) + }) + .boxed_local(); + let self_ = Self { + new_incoming, + ..Default::default() + }; + RefCell::new(self_) + } +} + +impl Default for SessionContainer { + fn default() -> Self { + Self { + new_incoming: stream::empty().boxed_local(), + handshake: None, + established: FuturesUnordered::new(), + } + } +} + +struct InspectorWakerInner { + poll_state: PollState, + task_waker: Option<task::Waker>, + parked_thread: Option<thread::Thread>, + inspector_ptr: Option<NonNull<JsRuntimeInspector>>, + isolate_handle: v8::IsolateHandle, +} + +unsafe impl Send for InspectorWakerInner {} + +struct InspectorWaker(Mutex<InspectorWakerInner>); + +impl InspectorWaker { + fn new(isolate_handle: v8::IsolateHandle) -> Arc<Self> { + let inner = InspectorWakerInner { + poll_state: PollState::Idle, + task_waker: None, + parked_thread: None, + inspector_ptr: None, + isolate_handle, + }; + Arc::new(Self(Mutex::new(inner))) + } + + fn update<F, R>(&self, update_fn: F) -> R + where + F: FnOnce(&mut InspectorWakerInner) -> R, + { + let mut g = self.0.lock().unwrap(); + update_fn(&mut g) + } +} + +impl task::ArcWake for InspectorWaker { + fn wake_by_ref(arc_self: &Arc<Self>) { + arc_self.update(|w| { + match w.poll_state { + PollState::Idle => { + // Wake the task, if any, that has polled the Inspector future last. + if let Some(waker) = w.task_waker.take() { + waker.wake() + } + // Request an interrupt from the isolate if it's running and there's + // not unhandled interrupt request in flight. + if let Some(arg) = w + .inspector_ptr + .take() + .map(|ptr| ptr.as_ptr() as *mut c_void) + { + w.isolate_handle.request_interrupt(handle_interrupt, arg); + } + extern "C" fn handle_interrupt( + _isolate: &mut v8::Isolate, + arg: *mut c_void, + ) { + let inspector = unsafe { &*(arg as *mut JsRuntimeInspector) }; + let _ = inspector.poll_sessions(None); + } + } + PollState::Parked => { + // Unpark the isolate thread. + let parked_thread = w.parked_thread.take().unwrap(); + assert_ne!(parked_thread.id(), thread::current().id()); + parked_thread.unpark(); + } + _ => {} + }; + w.poll_state = PollState::Woken; + }); + } +} + +/// An inspector session that proxies messages to concrete "transport layer", +/// eg. Websocket or another set of channels. +struct InspectorSession { + v8_channel: v8::inspector::ChannelBase, + v8_session: Rc<RefCell<v8::UniqueRef<v8::inspector::V8InspectorSession>>>, + proxy_tx: SessionProxySender, + proxy_rx_handler: Pin<Box<dyn Future<Output = ()> + 'static>>, +} + +impl InspectorSession { + const CONTEXT_GROUP_ID: i32 = 1; + + pub fn new( + v8_inspector_rc: Rc<RefCell<v8::UniquePtr<v8::inspector::V8Inspector>>>, + session_proxy: InspectorSessionProxy, + ) -> Box<Self> { + new_box_with(move |self_ptr| { + let v8_channel = v8::inspector::ChannelBase::new::<Self>(); + let mut v8_inspector = v8_inspector_rc.borrow_mut(); + let v8_inspector_ptr = v8_inspector.as_mut().unwrap(); + let v8_session = Rc::new(RefCell::new(v8_inspector_ptr.connect( + Self::CONTEXT_GROUP_ID, + // Todo(piscisaureus): V8Inspector::connect() should require that + // the 'v8_channel' argument cannot move. + unsafe { &mut *self_ptr }, + v8::inspector::StringView::empty(), + ))); + + let (proxy_tx, proxy_rx) = session_proxy.split(); + let proxy_rx_handler = + Self::receive_from_proxy(v8_session.clone(), proxy_rx); + + Self { + v8_channel, + v8_session, + proxy_tx, + proxy_rx_handler, + } + }) + } + + // Dispatch message to V8 session + #[allow(unused)] + fn dispatch_message(&mut self, msg: Vec<u8>) { + let msg = v8::inspector::StringView::from(msg.as_slice()); + let mut v8_session = self.v8_session.borrow_mut(); + let v8_session_ptr = v8_session.as_mut(); + v8_session_ptr.dispatch_protocol_message(msg); + } + + // TODO(bartlomieju): this function should be reworked into `impl Future` + // or `impl Stream` + /// Returns a future that receives messages from the proxy and dispatches + /// them to the V8 session. + fn receive_from_proxy( + v8_session_rc: Rc< + RefCell<v8::UniqueRef<v8::inspector::V8InspectorSession>>, + >, + proxy_rx: SessionProxyReceiver, + ) -> Pin<Box<dyn Future<Output = ()> + 'static>> { + async move { + let result = proxy_rx + .map_ok(move |msg| { + let msg = v8::inspector::StringView::from(msg.as_slice()); + let mut v8_session = v8_session_rc.borrow_mut(); + let v8_session_ptr = v8_session.as_mut(); + v8_session_ptr.dispatch_protocol_message(msg); + }) + .try_collect::<()>() + .await; + + // TODO(bartlomieju): ideally these prints should be moved + // to `server.rs` as they are unwanted in context of REPL/coverage collection + // but right now they do not pose a huge problem. Investigate how to + // move them to `server.rs`. + match result { + Ok(_) => eprintln!("Debugger session ended."), + Err(err) => eprintln!("Debugger session ended: {}.", err), + }; + } + .boxed_local() + } + + fn send_message( + &self, + maybe_call_id: Option<i32>, + msg: v8::UniquePtr<v8::inspector::StringBuffer>, + ) { + let msg = msg.unwrap().string().to_string(); + let _ = self.proxy_tx.unbounded_send((maybe_call_id, msg)); + } + + pub fn break_on_next_statement(&mut self) { + let reason = v8::inspector::StringView::from(&b"debugCommand"[..]); + let detail = v8::inspector::StringView::empty(); + self + .v8_session + .borrow_mut() + .as_mut() + .schedule_pause_on_next_statement(reason, detail); + } +} + +impl v8::inspector::ChannelImpl for InspectorSession { + fn base(&self) -> &v8::inspector::ChannelBase { + &self.v8_channel + } + + fn base_mut(&mut self) -> &mut v8::inspector::ChannelBase { + &mut self.v8_channel + } + + fn send_response( + &mut self, + call_id: i32, + message: v8::UniquePtr<v8::inspector::StringBuffer>, + ) { + self.send_message(Some(call_id), message); + } + + fn send_notification( + &mut self, + message: v8::UniquePtr<v8::inspector::StringBuffer>, + ) { + self.send_message(None, message); + } + + fn flush_protocol_notifications(&mut self) {} +} + +impl Future for InspectorSession { + type Output = (); + fn poll(mut self: Pin<&mut Self>, cx: &mut Context) -> Poll<Self::Output> { + self.proxy_rx_handler.poll_unpin(cx) + } +} + +/// A local inspector session that can be used to send and receive protocol messages directly on +/// the same thread as an isolate. +pub struct LocalInspectorSession { + v8_session_tx: UnboundedSender<Result<Vec<u8>, AnyError>>, + v8_session_rx: UnboundedReceiver<(Option<i32>, String)>, + response_tx_map: HashMap<i32, oneshot::Sender<serde_json::Value>>, + next_message_id: i32, + notification_queue: Vec<Value>, +} + +impl LocalInspectorSession { + pub fn new( + v8_session_tx: UnboundedSender<Result<Vec<u8>, AnyError>>, + v8_session_rx: UnboundedReceiver<(Option<i32>, String)>, + ) -> Self { + let response_tx_map = HashMap::new(); + let next_message_id = 0; + + let notification_queue = Vec::new(); + + Self { + v8_session_tx, + v8_session_rx, + response_tx_map, + next_message_id, + notification_queue, + } + } + + pub fn notifications(&mut self) -> Vec<Value> { + self.notification_queue.split_off(0) + } + + pub async fn post_message( + &mut self, + method: &str, + params: Option<serde_json::Value>, + ) -> Result<serde_json::Value, AnyError> { + let id = self.next_message_id; + self.next_message_id += 1; + + let (response_tx, mut response_rx) = + oneshot::channel::<serde_json::Value>(); + self.response_tx_map.insert(id, response_tx); + + let message = json!({ + "id": id, + "method": method, + "params": params, + }); + + let raw_message = serde_json::to_string(&message).unwrap(); + self + .v8_session_tx + .unbounded_send(Ok(raw_message.as_bytes().to_vec())) + .unwrap(); + + loop { + let receive_fut = self.receive_from_v8_session().boxed_local(); + match select(receive_fut, &mut response_rx).await { + Either::Left(_) => continue, + Either::Right((result, _)) => { + let response = result?; + if let Some(error) = response.get("error") { + return Err(generic_error(error.to_string())); + } + + let result = response.get("result").unwrap().clone(); + return Ok(result); + } + } + } + } + + async fn receive_from_v8_session(&mut self) { + let (maybe_call_id, message) = self.v8_session_rx.next().await.unwrap(); + // If there's no call_id then it's a notification + if let Some(call_id) = maybe_call_id { + let message: serde_json::Value = match serde_json::from_str(&message) { + Ok(v) => v, + Err(error) => match error.classify() { + serde_json::error::Category::Syntax => json!({ + "id": call_id, + "result": { + "result": { + "type": "error", + "description": "Unterminated string literal", + "value": "Unterminated string literal", + }, + "exceptionDetails": { + "exceptionId": 0, + "text": "Unterminated string literal", + "lineNumber": 0, + "columnNumber": 0 + }, + }, + }), + _ => panic!("Could not parse inspector message"), + }, + }; + + self + .response_tx_map + .remove(&call_id) + .unwrap() + .send(message) + .unwrap(); + } else { + let message = serde_json::from_str(&message).unwrap(); + self.notification_queue.push(message); + } + } +} + +fn new_box_with<T>(new_fn: impl FnOnce(*mut T) -> T) -> Box<T> { + let b = Box::new(MaybeUninit::<T>::uninit()); + let p = Box::into_raw(b) as *mut T; + unsafe { ptr::write(p, new_fn(p)) }; + unsafe { Box::from_raw(p) } +} diff --git a/core/lib.rs b/core/lib.rs index 1f75b0ee3..7d49c1420 100644 --- a/core/lib.rs +++ b/core/lib.rs @@ -6,6 +6,7 @@ pub mod error; mod extensions; mod flags; mod gotham_state; +mod inspector; mod module_specifier; mod modules; mod normalize_path; @@ -37,6 +38,9 @@ pub use crate::async_cell::AsyncRefFuture; pub use crate::async_cell::RcLike; pub use crate::async_cell::RcRef; pub use crate::flags::v8_set_flags; +pub use crate::inspector::InspectorSessionProxy; +pub use crate::inspector::JsRuntimeInspector; +pub use crate::inspector::LocalInspectorSession; pub use crate::module_specifier::resolve_import; pub use crate::module_specifier::resolve_path; pub use crate::module_specifier::resolve_url; @@ -51,6 +55,8 @@ pub use crate::modules::ModuleLoader; pub use crate::modules::ModuleSource; pub use crate::modules::ModuleSourceFuture; pub use crate::modules::NoopModuleLoader; +// TODO(bartlomieju): this struct should be implementation +// detail nad not be public pub use crate::modules::RecursiveModuleLoad; pub use crate::normalize_path::normalize_path; pub use crate::ops::serialize_op_result; diff --git a/core/modules.rs b/core/modules.rs index b68d19def..642dcc26a 100644 --- a/core/modules.rs +++ b/core/modules.rs @@ -2,17 +2,22 @@ use rusty_v8 as v8; +use crate::bindings; use crate::error::generic_error; use crate::error::AnyError; use crate::module_specifier::ModuleSpecifier; +use crate::runtime::exception_to_err_result; use crate::OpState; use futures::future::FutureExt; use futures::stream::FuturesUnordered; use futures::stream::Stream; +use futures::stream::StreamFuture; use futures::stream::TryStreamExt; +use log::debug; use std::cell::RefCell; use std::collections::HashMap; use std::collections::HashSet; +use std::convert::TryFrom; use std::future::Future; use std::pin::Pin; use std::rc::Rc; @@ -336,6 +341,28 @@ impl RecursiveModuleLoad { Ok(()) } + pub fn is_currently_loading_main_module(&self) -> bool { + !self.is_dynamic_import() && self.state == LoadState::LoadingRoot + } + + pub fn module_registered(&mut self, module_id: ModuleId) { + // If we just finished loading the root module, store the root module id. + if self.state == LoadState::LoadingRoot { + self.root_module_id = Some(module_id); + self.state = LoadState::LoadingImports; + } + + if self.pending.is_empty() { + self.state = LoadState::Done; + } + } + + /// Return root `ModuleId`; this function panics + /// if load is not finished yet. + pub fn expect_finished(&self) -> ModuleId { + self.root_module_id.expect("Root module id empty") + } + pub fn add_import( &mut self, specifier: ModuleSpecifier, @@ -383,6 +410,7 @@ impl Stream for RecursiveModuleLoad { pub struct ModuleInfo { pub id: ModuleId, + // Used in "bindings.rs" for "import.meta.main" property value. pub main: bool, pub name: String, pub import_specifiers: Vec<ModuleSpecifier>, @@ -399,23 +427,41 @@ enum SymbolicModule { } /// A collection of JS modules. -#[derive(Default)] pub struct ModuleMap { + // Handling of specifiers and v8 objects ids_by_handle: HashMap<v8::Global<v8::Module>, ModuleId>, handles_by_id: HashMap<ModuleId, v8::Global<v8::Module>>, info: HashMap<ModuleId, ModuleInfo>, by_name: HashMap<String, SymbolicModule>, next_module_id: ModuleId, + + // Handling of futures for loading module sources + pub loader: Rc<dyn ModuleLoader>, + op_state: Rc<RefCell<OpState>>, + pub(crate) dynamic_import_map: + HashMap<ModuleLoadId, v8::Global<v8::PromiseResolver>>, + pub(crate) preparing_dynamic_imports: + FuturesUnordered<Pin<Box<PrepareLoadFuture>>>, + pub(crate) pending_dynamic_imports: + FuturesUnordered<StreamFuture<RecursiveModuleLoad>>, } impl ModuleMap { - pub fn new() -> ModuleMap { + pub fn new( + loader: Rc<dyn ModuleLoader>, + op_state: Rc<RefCell<OpState>>, + ) -> ModuleMap { Self { - handles_by_id: HashMap::new(), ids_by_handle: HashMap::new(), + handles_by_id: HashMap::new(), info: HashMap::new(), by_name: HashMap::new(), next_module_id: 1, + loader, + op_state, + dynamic_import_map: HashMap::new(), + preparing_dynamic_imports: FuturesUnordered::new(), + pending_dynamic_imports: FuturesUnordered::new(), } } @@ -434,22 +480,52 @@ impl ModuleMap { } } - pub fn get_children(&self, id: ModuleId) -> Option<&Vec<ModuleSpecifier>> { - self.info.get(&id).map(|i| &i.import_specifiers) - } - - pub fn is_registered(&self, specifier: &ModuleSpecifier) -> bool { - self.get_id(&specifier.to_string()).is_some() - } - - pub fn register( + // Create and compile an ES module. + pub(crate) fn new_module( &mut self, - name: &str, + scope: &mut v8::HandleScope, main: bool, - handle: v8::Global<v8::Module>, - import_specifiers: Vec<ModuleSpecifier>, - ) -> ModuleId { - let name = String::from(name); + name: &str, + source: &str, + ) -> Result<ModuleId, AnyError> { + let name_str = v8::String::new(scope, name).unwrap(); + let source_str = v8::String::new(scope, source).unwrap(); + + let origin = bindings::module_origin(scope, name_str); + let source = v8::script_compiler::Source::new(source_str, Some(&origin)); + + let tc_scope = &mut v8::TryCatch::new(scope); + + let maybe_module = v8::script_compiler::compile_module(tc_scope, source); + + if tc_scope.has_caught() { + assert!(maybe_module.is_none()); + let e = tc_scope.exception().unwrap(); + return exception_to_err_result(tc_scope, e, false); + } + + let module = maybe_module.unwrap(); + + let mut import_specifiers: Vec<ModuleSpecifier> = vec![]; + let module_requests = module.get_module_requests(); + for i in 0..module_requests.length() { + let module_request = v8::Local::<v8::ModuleRequest>::try_from( + module_requests.get(tc_scope, i).unwrap(), + ) + .unwrap(); + let import_specifier = module_request + .get_specifier() + .to_rust_string_lossy(tc_scope); + let module_specifier = self.loader.resolve( + self.op_state.clone(), + &import_specifier, + name, + false, + )?; + import_specifiers.push(module_specifier); + } + + let handle = v8::Global::<v8::Module>::new(tc_scope, module); let id = self.next_module_id; self.next_module_id += 1; self @@ -462,11 +538,83 @@ impl ModuleMap { ModuleInfo { id, main, - name, + name: name.to_string(), import_specifiers, }, ); - id + + Ok(id) + } + + pub fn register_during_load( + &mut self, + scope: &mut v8::HandleScope, + module_source: ModuleSource, + load: &mut RecursiveModuleLoad, + ) -> Result<(), AnyError> { + let referrer_specifier = + crate::resolve_url(&module_source.module_url_found).unwrap(); + + // #A There are 3 cases to handle at this moment: + // 1. Source code resolved result have the same module name as requested + // and is not yet registered + // -> register + // 2. Source code resolved result have a different name as requested: + // 2a. The module with resolved module name has been registered + // -> alias + // 2b. The module with resolved module name has not yet been registered + // -> register & alias + + // If necessary, register an alias. + if module_source.module_url_specified != module_source.module_url_found { + self.alias( + &module_source.module_url_specified, + &module_source.module_url_found, + ); + } + + let maybe_mod_id = self.get_id(&module_source.module_url_found); + + let module_id = match maybe_mod_id { + Some(id) => { + // Module has already been registered. + debug!( + "Already-registered module fetched again: {}", + module_source.module_url_found + ); + id + } + // Module not registered yet, do it now. + None => self.new_module( + scope, + load.is_currently_loading_main_module(), + &module_source.module_url_found, + &module_source.code, + )?, + }; + + // Now we must iterate over all imports of the module and load them. + let imports = self.get_children(module_id).unwrap().clone(); + + for module_specifier in imports { + let is_registered = self.is_registered(&module_specifier); + if !is_registered { + load + .add_import(module_specifier.to_owned(), referrer_specifier.clone()); + } + } + + load.module_registered(module_id); + + Ok(()) + } + + pub fn get_children(&self, id: ModuleId) -> Option<&Vec<ModuleSpecifier>> { + self.info.get(&id).map(|i| &i.import_specifiers) + } + + pub fn is_registered(&self, specifier: &ModuleSpecifier) -> bool { + self.get_id(specifier.as_str()).is_some() } pub fn alias(&mut self, name: &str, target: &str) { @@ -499,17 +647,79 @@ impl ModuleMap { pub fn get_info_by_id(&self, id: &ModuleId) -> Option<&ModuleInfo> { self.info.get(id) } + + pub fn load_main( + &self, + specifier: &str, + code: Option<String>, + ) -> RecursiveModuleLoad { + RecursiveModuleLoad::main( + self.op_state.clone(), + specifier, + code, + self.loader.clone(), + ) + } + + // Initiate loading of a module graph imported using `import()`. + pub fn load_dynamic_import( + &mut self, + specifier: &str, + referrer: &str, + resolver_handle: v8::Global<v8::PromiseResolver>, + ) { + let load = RecursiveModuleLoad::dynamic_import( + self.op_state.clone(), + specifier, + referrer, + self.loader.clone(), + ); + self.dynamic_import_map.insert(load.id, resolver_handle); + let fut = load.prepare().boxed_local(); + self.preparing_dynamic_imports.push(fut); + } + + pub fn has_pending_dynamic_imports(&self) -> bool { + !(self.preparing_dynamic_imports.is_empty() + && self.pending_dynamic_imports.is_empty()) + } + + /// Called by `module_resolve_callback` during module instantiation. + pub fn resolve_callback<'s>( + &self, + scope: &mut v8::HandleScope<'s>, + specifier: &str, + referrer: &str, + ) -> Option<v8::Local<'s, v8::Module>> { + let resolved_specifier = self + .loader + .resolve(self.op_state.clone(), specifier, referrer, false) + .expect("Module should have been already resolved"); + + if let Some(id) = self.get_id(resolved_specifier.as_str()) { + if let Some(handle) = self.get_handle(id) { + return Some(v8::Local::new(scope, handle)); + } + } + + None + } } #[cfg(test)] mod tests { use super::*; + use crate::serialize_op_result; use crate::JsRuntime; + use crate::Op; + use crate::OpPayload; use crate::RuntimeOptions; use futures::future::FutureExt; use std::error::Error; use std::fmt; use std::future::Future; + use std::io; + use std::sync::atomic::{AtomicUsize, Ordering}; use std::sync::Arc; use std::sync::Mutex; @@ -692,7 +902,7 @@ mod tests { let a_id = futures::executor::block_on(a_id_fut).expect("Failed to load"); runtime.mod_evaluate(a_id); - futures::executor::block_on(runtime.run_event_loop()).unwrap(); + futures::executor::block_on(runtime.run_event_loop(false)).unwrap(); let l = loads.lock().unwrap(); assert_eq!( l.to_vec(), @@ -704,9 +914,9 @@ mod tests { ] ); - let state_rc = JsRuntime::state(runtime.v8_isolate()); - let state = state_rc.borrow(); - let modules = &state.module_map; + let module_map_rc = JsRuntime::module_map(runtime.v8_isolate()); + let modules = module_map_rc.borrow(); + assert_eq!(modules.get_id("file:///a.js"), Some(a_id)); let b_id = modules.get_id("file:///b.js").unwrap(); let c_id = modules.get_id("file:///c.js").unwrap(); @@ -746,6 +956,325 @@ mod tests { "#; #[test] + fn test_mods() { + #[derive(Default)] + struct ModsLoader { + pub count: Arc<AtomicUsize>, + } + + impl ModuleLoader for ModsLoader { + fn resolve( + &self, + _op_state: Rc<RefCell<OpState>>, + specifier: &str, + referrer: &str, + _is_main: bool, + ) -> Result<ModuleSpecifier, AnyError> { + self.count.fetch_add(1, Ordering::Relaxed); + assert_eq!(specifier, "./b.js"); + assert_eq!(referrer, "file:///a.js"); + let s = crate::resolve_import(specifier, referrer).unwrap(); + Ok(s) + } + + fn load( + &self, + _op_state: Rc<RefCell<OpState>>, + _module_specifier: &ModuleSpecifier, + _maybe_referrer: Option<ModuleSpecifier>, + _is_dyn_import: bool, + ) -> Pin<Box<ModuleSourceFuture>> { + unreachable!() + } + } + + let loader = Rc::new(ModsLoader::default()); + + let resolve_count = loader.count.clone(); + let dispatch_count = Arc::new(AtomicUsize::new(0)); + let dispatch_count_ = dispatch_count.clone(); + + let dispatcher = move |state, payload: OpPayload| -> Op { + dispatch_count_.fetch_add(1, Ordering::Relaxed); + let (control, _): (u8, ()) = payload.deserialize().unwrap(); + assert_eq!(control, 42); + let resp = (0, serialize_op_result(Ok(43), state)); + Op::Async(Box::pin(futures::future::ready(resp))) + }; + + let mut runtime = JsRuntime::new(RuntimeOptions { + module_loader: Some(loader), + ..Default::default() + }); + runtime.register_op("op_test", dispatcher); + runtime.sync_ops_cache(); + + runtime + .execute( + "setup.js", + r#" + function assert(cond) { + if (!cond) { + throw Error("assert"); + } + } + "#, + ) + .unwrap(); + + assert_eq!(dispatch_count.load(Ordering::Relaxed), 0); + + let module_map_rc = JsRuntime::module_map(runtime.v8_isolate()); + + let (mod_a, mod_b) = { + let scope = &mut runtime.handle_scope(); + let mut module_map = module_map_rc.borrow_mut(); + let specifier_a = "file:///a.js".to_string(); + let mod_a = module_map + .new_module( + scope, + true, + &specifier_a, + r#" + import { b } from './b.js' + if (b() != 'b') throw Error(); + let control = 42; + Deno.core.opAsync("op_test", control); + "#, + ) + .unwrap(); + + assert_eq!(dispatch_count.load(Ordering::Relaxed), 0); + let imports = module_map.get_children(mod_a); + assert_eq!( + imports, + Some(&vec![crate::resolve_url("file:///b.js").unwrap()]) + ); + + let mod_b = module_map + .new_module( + scope, + false, + "file:///b.js", + "export function b() { return 'b' }", + ) + .unwrap(); + let imports = module_map.get_children(mod_b).unwrap(); + assert_eq!(imports.len(), 0); + (mod_a, mod_b) + }; + + runtime.instantiate_module(mod_b).unwrap(); + assert_eq!(dispatch_count.load(Ordering::Relaxed), 0); + assert_eq!(resolve_count.load(Ordering::SeqCst), 1); + + runtime.instantiate_module(mod_a).unwrap(); + assert_eq!(dispatch_count.load(Ordering::Relaxed), 0); + + runtime.mod_evaluate(mod_a); + assert_eq!(dispatch_count.load(Ordering::Relaxed), 1); + } + + #[test] + fn dyn_import_err() { + #[derive(Clone, Default)] + struct DynImportErrLoader { + pub count: Arc<AtomicUsize>, + } + + impl ModuleLoader for DynImportErrLoader { + fn resolve( + &self, + _op_state: Rc<RefCell<OpState>>, + specifier: &str, + referrer: &str, + _is_main: bool, + ) -> Result<ModuleSpecifier, AnyError> { + self.count.fetch_add(1, Ordering::Relaxed); + assert_eq!(specifier, "/foo.js"); + assert_eq!(referrer, "file:///dyn_import2.js"); + let s = crate::resolve_import(specifier, referrer).unwrap(); + Ok(s) + } + + fn load( + &self, + _op_state: Rc<RefCell<OpState>>, + _module_specifier: &ModuleSpecifier, + _maybe_referrer: Option<ModuleSpecifier>, + _is_dyn_import: bool, + ) -> Pin<Box<ModuleSourceFuture>> { + async { Err(io::Error::from(io::ErrorKind::NotFound).into()) }.boxed() + } + } + + // Test an erroneous dynamic import where the specified module isn't found. + run_in_task(|cx| { + let loader = Rc::new(DynImportErrLoader::default()); + let count = loader.count.clone(); + let mut runtime = JsRuntime::new(RuntimeOptions { + module_loader: Some(loader), + ..Default::default() + }); + + runtime + .execute( + "file:///dyn_import2.js", + r#" + (async () => { + await import("/foo.js"); + })(); + "#, + ) + .unwrap(); + + assert_eq!(count.load(Ordering::Relaxed), 0); + // We should get an error here. + let result = runtime.poll_event_loop(cx, false); + if let Poll::Ready(Ok(_)) = result { + unreachable!(); + } + assert_eq!(count.load(Ordering::Relaxed), 2); + }) + } + + #[derive(Clone, Default)] + struct DynImportOkLoader { + pub prepare_load_count: Arc<AtomicUsize>, + pub resolve_count: Arc<AtomicUsize>, + pub load_count: Arc<AtomicUsize>, + } + + impl ModuleLoader for DynImportOkLoader { + fn resolve( + &self, + _op_state: Rc<RefCell<OpState>>, + specifier: &str, + referrer: &str, + _is_main: bool, + ) -> Result<ModuleSpecifier, AnyError> { + let c = self.resolve_count.fetch_add(1, Ordering::Relaxed); + assert!(c < 4); + assert_eq!(specifier, "./b.js"); + assert_eq!(referrer, "file:///dyn_import3.js"); + let s = crate::resolve_import(specifier, referrer).unwrap(); + Ok(s) + } + + fn load( + &self, + _op_state: Rc<RefCell<OpState>>, + specifier: &ModuleSpecifier, + _maybe_referrer: Option<ModuleSpecifier>, + _is_dyn_import: bool, + ) -> Pin<Box<ModuleSourceFuture>> { + self.load_count.fetch_add(1, Ordering::Relaxed); + let info = ModuleSource { + module_url_specified: specifier.to_string(), + module_url_found: specifier.to_string(), + code: "export function b() { return 'b' }".to_owned(), + }; + async move { Ok(info) }.boxed() + } + + fn prepare_load( + &self, + _op_state: Rc<RefCell<OpState>>, + _load_id: ModuleLoadId, + _module_specifier: &ModuleSpecifier, + _maybe_referrer: Option<String>, + _is_dyn_import: bool, + ) -> Pin<Box<dyn Future<Output = Result<(), AnyError>>>> { + self.prepare_load_count.fetch_add(1, Ordering::Relaxed); + async { Ok(()) }.boxed_local() + } + } + + #[test] + fn dyn_import_ok() { + run_in_task(|cx| { + let loader = Rc::new(DynImportOkLoader::default()); + let prepare_load_count = loader.prepare_load_count.clone(); + let resolve_count = loader.resolve_count.clone(); + let load_count = loader.load_count.clone(); + let mut runtime = JsRuntime::new(RuntimeOptions { + module_loader: Some(loader), + ..Default::default() + }); + + // Dynamically import mod_b + runtime + .execute( + "file:///dyn_import3.js", + r#" + (async () => { + let mod = await import("./b.js"); + if (mod.b() !== 'b') { + throw Error("bad1"); + } + // And again! + mod = await import("./b.js"); + if (mod.b() !== 'b') { + throw Error("bad2"); + } + })(); + "#, + ) + .unwrap(); + + // First poll runs `prepare_load` hook. + assert!(matches!(runtime.poll_event_loop(cx, false), Poll::Pending)); + assert_eq!(prepare_load_count.load(Ordering::Relaxed), 1); + + // Second poll actually loads modules into the isolate. + assert!(matches!( + runtime.poll_event_loop(cx, false), + Poll::Ready(Ok(_)) + )); + assert_eq!(resolve_count.load(Ordering::Relaxed), 4); + assert_eq!(load_count.load(Ordering::Relaxed), 2); + assert!(matches!( + runtime.poll_event_loop(cx, false), + Poll::Ready(Ok(_)) + )); + assert_eq!(resolve_count.load(Ordering::Relaxed), 4); + assert_eq!(load_count.load(Ordering::Relaxed), 2); + }) + } + + #[test] + fn dyn_import_borrow_mut_error() { + // https://github.com/denoland/deno/issues/6054 + run_in_task(|cx| { + let loader = Rc::new(DynImportOkLoader::default()); + let prepare_load_count = loader.prepare_load_count.clone(); + let mut runtime = JsRuntime::new(RuntimeOptions { + module_loader: Some(loader), + ..Default::default() + }); + runtime.sync_ops_cache(); + runtime + .execute( + "file:///dyn_import3.js", + r#" + (async () => { + let mod = await import("./b.js"); + if (mod.b() !== 'b') { + throw Error("bad"); + } + })(); + "#, + ) + .unwrap(); + // First poll runs `prepare_load` hook. + let _ = runtime.poll_event_loop(cx, false); + assert_eq!(prepare_load_count.load(Ordering::Relaxed), 1); + // Second poll triggers error + let _ = runtime.poll_event_loop(cx, false); + }) + } + + #[test] fn test_circular_load() { let loader = MockLoader::new(); let loads = loader.loads.clone(); @@ -760,7 +1289,7 @@ mod tests { assert!(result.is_ok()); let circular1_id = result.unwrap(); runtime.mod_evaluate(circular1_id); - runtime.run_event_loop().await.unwrap(); + runtime.run_event_loop(false).await.unwrap(); let l = loads.lock().unwrap(); assert_eq!( @@ -772,9 +1301,8 @@ mod tests { ] ); - let state_rc = JsRuntime::state(runtime.v8_isolate()); - let state = state_rc.borrow(); - let modules = &state.module_map; + let module_map_rc = JsRuntime::module_map(runtime.v8_isolate()); + let modules = module_map_rc.borrow(); assert_eq!(modules.get_id("file:///circular1.js"), Some(circular1_id)); let circular2_id = modules.get_id("file:///circular2.js").unwrap(); @@ -834,7 +1362,7 @@ mod tests { assert!(result.is_ok()); let redirect1_id = result.unwrap(); runtime.mod_evaluate(redirect1_id); - runtime.run_event_loop().await.unwrap(); + runtime.run_event_loop(false).await.unwrap(); let l = loads.lock().unwrap(); assert_eq!( l.to_vec(), @@ -845,9 +1373,8 @@ mod tests { ] ); - let state_rc = JsRuntime::state(runtime.v8_isolate()); - let state = state_rc.borrow(); - let modules = &state.module_map; + let module_map_rc = JsRuntime::module_map(runtime.v8_isolate()); + let modules = module_map_rc.borrow(); assert_eq!(modules.get_id("file:///redirect1.js"), Some(redirect1_id)); @@ -984,7 +1511,7 @@ mod tests { futures::executor::block_on(main_id_fut).expect("Failed to load"); runtime.mod_evaluate(main_id); - futures::executor::block_on(runtime.run_event_loop()).unwrap(); + futures::executor::block_on(runtime.run_event_loop(false)).unwrap(); let l = loads.lock().unwrap(); assert_eq!( @@ -992,9 +1519,8 @@ mod tests { vec!["file:///b.js", "file:///c.js", "file:///d.js"] ); - let state_rc = JsRuntime::state(runtime.v8_isolate()); - let state = state_rc.borrow(); - let modules = &state.module_map; + let module_map_rc = JsRuntime::module_map(runtime.v8_isolate()); + let modules = module_map_rc.borrow(); assert_eq!(modules.get_id("file:///main_with_code.js"), Some(main_id)); let b_id = modules.get_id("file:///b.js").unwrap(); diff --git a/core/ops_json.rs b/core/ops_json.rs index bc04d9dd4..12fd9fdbc 100644 --- a/core/ops_json.rs +++ b/core/ops_json.rs @@ -134,7 +134,7 @@ mod tests { "#, ) .unwrap(); - let e = runtime.run_event_loop().await.unwrap_err().to_string(); + let e = runtime.run_event_loop(false).await.unwrap_err().to_string(); println!("{}", e); assert!(e.contains("Error: foo")); assert!(e.contains("at async f1 (<init>:")); diff --git a/core/runtime.rs b/core/runtime.rs index 96281ef9f..71aad8e0b 100644 --- a/core/runtime.rs +++ b/core/runtime.rs @@ -8,17 +8,13 @@ use crate::error::generic_error; use crate::error::AnyError; use crate::error::ErrWithV8Handle; use crate::error::JsError; -use crate::futures::FutureExt; +use crate::inspector::JsRuntimeInspector; use crate::module_specifier::ModuleSpecifier; -use crate::modules::LoadState; use crate::modules::ModuleId; use crate::modules::ModuleLoadId; use crate::modules::ModuleLoader; use crate::modules::ModuleMap; -use crate::modules::ModuleSource; use crate::modules::NoopModuleLoader; -use crate::modules::PrepareLoadFuture; -use crate::modules::RecursiveModuleLoad; use crate::ops::*; use crate::Extension; use crate::OpMiddlewareFn; @@ -28,12 +24,11 @@ use crate::OpState; use crate::PromiseId; use futures::channel::mpsc; use futures::future::poll_fn; +use futures::future::FutureExt; use futures::stream::FuturesUnordered; use futures::stream::StreamExt; -use futures::stream::StreamFuture; use futures::task::AtomicWaker; use futures::Future; -use log::debug; use std::any::Any; use std::cell::RefCell; use std::collections::HashMap; @@ -82,6 +77,7 @@ pub struct JsRuntime { // This is an Option<OwnedIsolate> instead of just OwnedIsolate to workaround // an safety issue with SnapshotCreator. See JsRuntime::drop. v8_isolate: Option<v8::OwnedIsolate>, + inspector: Option<Box<JsRuntimeInspector>>, snapshot_creator: Option<v8::SnapshotCreator>, has_snapshotted: bool, allocations: IsolateAllocations, @@ -115,17 +111,15 @@ pub(crate) struct JsRuntimeState { pub(crate) pending_unref_ops: FuturesUnordered<PendingOpFuture>, pub(crate) have_unpolled_ops: bool, pub(crate) op_state: Rc<RefCell<OpState>>, - pub loader: Rc<dyn ModuleLoader>, - pub module_map: ModuleMap, - pub(crate) dyn_import_map: - HashMap<ModuleLoadId, v8::Global<v8::PromiseResolver>>, - preparing_dyn_imports: FuturesUnordered<Pin<Box<PrepareLoadFuture>>>, - pending_dyn_imports: FuturesUnordered<StreamFuture<RecursiveModuleLoad>>, waker: AtomicWaker, } impl Drop for JsRuntime { fn drop(&mut self) { + // The Isolate object must outlive the Inspector object, but this is + // currently not enforced by the type system. + self.inspector.take(); + if let Some(creator) = self.snapshot_creator.take() { // TODO(ry): in rusty_v8, `SnapShotCreator::get_owned_isolate()` returns // a `struct OwnedIsolate` which is not actually owned, hence the need @@ -211,6 +205,9 @@ pub struct RuntimeOptions { /// V8 platform instance to use. Used when Deno initializes V8 /// (which it only does once), otherwise it's silenty dropped. pub v8_platform: Option<v8::UniquePtr<v8::Platform>>, + + /// Create a V8 inspector and attach to the runtime. + pub attach_inspector: bool, } impl JsRuntime { @@ -271,6 +268,14 @@ impl JsRuntime { (isolate, None) }; + let maybe_inspector = if options.attach_inspector { + let inspector = + JsRuntimeInspector::new(&mut isolate, global_context.clone()); + Some(inspector) + } else { + None + }; + let loader = options .module_loader .unwrap_or_else(|| Rc::new(NoopModuleLoader)); @@ -284,6 +289,8 @@ impl JsRuntime { op_state.get_error_class_fn = get_error_class_fn; } + let op_state = Rc::new(RefCell::new(op_state)); + isolate.set_slot(Rc::new(RefCell::new(JsRuntimeState { global_context: Some(global_context), pending_promise_exceptions: HashMap::new(), @@ -294,16 +301,14 @@ impl JsRuntime { js_error_create_fn, pending_ops: FuturesUnordered::new(), pending_unref_ops: FuturesUnordered::new(), - op_state: Rc::new(RefCell::new(op_state)), + op_state: op_state.clone(), have_unpolled_ops: false, - module_map: ModuleMap::new(), - loader, - dyn_import_map: HashMap::new(), - preparing_dyn_imports: FuturesUnordered::new(), - pending_dyn_imports: FuturesUnordered::new(), waker: AtomicWaker::new(), }))); + let module_map = ModuleMap::new(loader, op_state); + isolate.set_slot(Rc::new(RefCell::new(module_map))); + // Add builtins extension options .extensions @@ -311,6 +316,7 @@ impl JsRuntime { let mut js_runtime = Self { v8_isolate: Some(isolate), + inspector: maybe_inspector, snapshot_creator: maybe_snapshot_creator, has_snapshotted: false, allocations: IsolateAllocations::default(), @@ -341,6 +347,10 @@ impl JsRuntime { self.v8_isolate.as_mut().unwrap() } + pub fn inspector(&mut self) -> Option<&mut Box<JsRuntimeInspector>> { + self.inspector.as_mut() + } + pub fn handle_scope(&mut self) -> v8::HandleScope { let context = self.global_context(); v8::HandleScope::with_context(self.v8_isolate(), context) @@ -363,6 +373,11 @@ impl JsRuntime { s.clone() } + pub(crate) fn module_map(isolate: &v8::Isolate) -> Rc<RefCell<ModuleMap>> { + let module_map = isolate.get_slot::<Rc<RefCell<ModuleMap>>>().unwrap(); + module_map.clone() + } + /// Initializes JS of provided Extensions fn init_extension_js(&mut self) -> Result<(), AnyError> { // Take extensions to avoid double-borrow @@ -370,8 +385,9 @@ impl JsRuntime { for m in extensions.iter_mut() { let js_files = m.init_js(); for (filename, source) in js_files { + let source = source()?; // TODO(@AaronO): use JsRuntime::execute_static() here to move src off heap - self.execute(filename, source)?; + self.execute(filename, &source)?; } } // Restore extensions @@ -495,8 +511,14 @@ impl JsRuntime { // TODO(piscisaureus): The rusty_v8 type system should enforce this. state.borrow_mut().global_context.take(); - // Drop v8::Global handles before snapshotting - std::mem::take(&mut state.borrow_mut().module_map); + // Overwrite existing ModuleMap to drop v8::Global handles + self + .v8_isolate() + .set_slot(Rc::new(RefCell::new(ModuleMap::new( + Rc::new(NoopModuleLoader), + state.borrow().op_state.clone(), + )))); + // Drop other v8::Global handles before snapshotting std::mem::take(&mut state.borrow_mut().js_recv_cb); let snapshot_creator = self.snapshot_creator.as_mut().unwrap(); @@ -570,16 +592,28 @@ impl JsRuntime { /// This future resolves when: /// - there are no more pending dynamic imports /// - there are no more pending ops - pub async fn run_event_loop(&mut self) -> Result<(), AnyError> { - poll_fn(|cx| self.poll_event_loop(cx)).await + /// - there are no more active inspector sessions (only if `wait_for_inspector` is set to true) + pub async fn run_event_loop( + &mut self, + wait_for_inspector: bool, + ) -> Result<(), AnyError> { + poll_fn(|cx| self.poll_event_loop(cx, wait_for_inspector)).await } /// Runs a single tick of event loop + /// + /// If `wait_for_inspector` is set to true event loop + /// will return `Poll::Pending` if there are active inspector sessions. pub fn poll_event_loop( &mut self, cx: &mut Context, + wait_for_inspector: bool, ) -> Poll<Result<(), AnyError>> { + // We always poll the inspector if it exists. + let _ = self.inspector().map(|i| i.poll_unpin(cx)); + let state_rc = Self::state(self.v8_isolate()); + let module_map_rc = Self::module_map(self.v8_isolate()); { let state = state_rc.borrow(); state.waker.register(cx.waker()); @@ -610,21 +644,29 @@ impl JsRuntime { self.evaluate_pending_module(); let state = state_rc.borrow(); + let module_map = module_map_rc.borrow(); + let has_pending_ops = !state.pending_ops.is_empty(); - let has_pending_dyn_imports = !{ - state.preparing_dyn_imports.is_empty() - && state.pending_dyn_imports.is_empty() - }; + let has_pending_dyn_imports = module_map.has_pending_dynamic_imports(); let has_pending_dyn_module_evaluation = !state.pending_dyn_mod_evaluate.is_empty(); let has_pending_module_evaluation = state.pending_mod_evaluate.is_some(); + let inspector_has_active_sessions = self + .inspector + .as_ref() + .map(|i| i.has_active_sessions()) + .unwrap_or(false); if !has_pending_ops && !has_pending_dyn_imports && !has_pending_dyn_module_evaluation && !has_pending_module_evaluation { + if wait_for_inspector && inspector_has_active_sessions { + return Poll::Pending; + } + return Poll::Ready(Ok(())); } @@ -653,8 +695,7 @@ impl JsRuntime { let mut msg = "Dynamically imported module evaluation is still pending but there are no pending ops. This situation is often caused by unresolved promise. Pending dynamic modules:\n".to_string(); for pending_evaluate in &state.pending_dyn_mod_evaluate { - let module_info = state - .module_map + let module_info = module_map .get_info_by_id(&pending_evaluate.module_id) .unwrap(); msg.push_str(&format!("- {}", module_info.name.as_str())); @@ -680,25 +721,11 @@ where } impl JsRuntimeState { - // Called by V8 during `Isolate::mod_instantiate`. - pub fn dyn_import_cb( - &mut self, - resolver_handle: v8::Global<v8::PromiseResolver>, - specifier: &str, - referrer: &str, - ) { - debug!("dyn_import specifier {} referrer {} ", specifier, referrer); - - let load = RecursiveModuleLoad::dynamic_import( - self.op_state.clone(), - specifier, - referrer, - self.loader.clone(), - ); - self.dyn_import_map.insert(load.id, resolver_handle); + /// Called by `bindings::host_import_module_dynamically_callback` + /// after initiating new dynamic import load. + pub fn notify_new_dynamic_import(&mut self) { + // Notify event loop to poll again soon. self.waker.wake(); - let fut = load.prepare().boxed_local(); - self.preparing_dyn_imports.push(fut); } } @@ -744,116 +771,53 @@ pub(crate) fn exception_to_err_result<'s, T>( // Related to module loading impl JsRuntime { - /// Low-level module creation. - /// - /// Called during module loading or dynamic import loading. - fn mod_new( + pub(crate) fn instantiate_module( &mut self, - main: bool, - name: &str, - source: &str, - ) -> Result<ModuleId, AnyError> { - let state_rc = Self::state(self.v8_isolate()); - let scope = &mut self.handle_scope(); - - let name_str = v8::String::new(scope, name).unwrap(); - let source_str = v8::String::new(scope, source).unwrap(); - - let origin = bindings::module_origin(scope, name_str); - let source = v8::script_compiler::Source::new(source_str, Some(&origin)); - - let tc_scope = &mut v8::TryCatch::new(scope); - - let maybe_module = v8::script_compiler::compile_module(tc_scope, source); - - if tc_scope.has_caught() { - assert!(maybe_module.is_none()); - let e = tc_scope.exception().unwrap(); - return exception_to_err_result(tc_scope, e, false); - } - - let module = maybe_module.unwrap(); - - let mut import_specifiers: Vec<ModuleSpecifier> = vec![]; - let module_requests = module.get_module_requests(); - for i in 0..module_requests.length() { - let module_request = v8::Local::<v8::ModuleRequest>::try_from( - module_requests.get(tc_scope, i).unwrap(), - ) - .unwrap(); - let import_specifier = module_request - .get_specifier() - .to_rust_string_lossy(tc_scope); - let state = state_rc.borrow(); - let module_specifier = state.loader.resolve( - state.op_state.clone(), - &import_specifier, - name, - false, - )?; - import_specifiers.push(module_specifier); - } - - let id = state_rc.borrow_mut().module_map.register( - name, - main, - v8::Global::<v8::Module>::new(tc_scope, module), - import_specifiers, - ); - - Ok(id) - } - - /// Instantiates a ES module - /// - /// `AnyError` can be downcast to a type that exposes additional information - /// about the V8 exception. By default this type is `JsError`, however it may - /// be a different type if `RuntimeOptions::js_error_create_fn` has been set. - fn mod_instantiate(&mut self, id: ModuleId) -> Result<(), AnyError> { - let state_rc = Self::state(self.v8_isolate()); + id: ModuleId, + ) -> Result<(), AnyError> { + let module_map_rc = Self::module_map(self.v8_isolate()); let scope = &mut self.handle_scope(); let tc_scope = &mut v8::TryCatch::new(scope); - let module = state_rc + let module = module_map_rc .borrow() - .module_map .get_handle(id) .map(|handle| v8::Local::new(tc_scope, handle)) .expect("ModuleInfo not found"); if module.get_status() == v8::ModuleStatus::Errored { let exception = module.get_exception(); - exception_to_err_result(tc_scope, exception, false) - .map_err(|err| attach_handle_to_error(tc_scope, err, exception)) - } else { - let instantiate_result = - module.instantiate_module(tc_scope, bindings::module_resolve_callback); - match instantiate_result { - Some(_) => Ok(()), - None => { - let exception = tc_scope.exception().unwrap(); - exception_to_err_result(tc_scope, exception, false) - .map_err(|err| attach_handle_to_error(tc_scope, err, exception)) - } - } + let err = exception_to_err_result(tc_scope, exception, false) + .map_err(|err| attach_handle_to_error(tc_scope, err, exception)); + return err; + } + + // IMPORTANT: No borrows to `ModuleMap` can be held at this point because + // `module_resolve_callback` will be calling into `ModuleMap` from within + // the isolate. + let instantiate_result = + module.instantiate_module(tc_scope, bindings::module_resolve_callback); + + if instantiate_result.is_none() { + let exception = tc_scope.exception().unwrap(); + let err = exception_to_err_result(tc_scope, exception, false) + .map_err(|err| attach_handle_to_error(tc_scope, err, exception)); + return err; } + + Ok(()) } - /// Evaluates an already instantiated ES module. - /// - /// `AnyError` can be downcast to a type that exposes additional information - /// about the V8 exception. By default this type is `JsError`, however it may - /// be a different type if `RuntimeOptions::js_error_create_fn` has been set. - pub fn dyn_mod_evaluate( + fn dynamic_import_module_evaluate( &mut self, load_id: ModuleLoadId, id: ModuleId, ) -> Result<(), AnyError> { let state_rc = Self::state(self.v8_isolate()); + let module_map_rc = Self::module_map(self.v8_isolate()); - let module_handle = state_rc + let module_handle = module_map_rc .borrow() - .module_map .get_handle(id) .expect("ModuleInfo not found"); @@ -937,11 +901,11 @@ impl JsRuntime { id: ModuleId, ) -> mpsc::Receiver<Result<(), AnyError>> { let state_rc = Self::state(self.v8_isolate()); + let module_map_rc = Self::module_map(self.v8_isolate()); let scope = &mut self.handle_scope(); - let module = state_rc + let module = module_map_rc .borrow() - .module_map .get_handle(id) .map(|handle| v8::Local::new(scope, handle)) .expect("ModuleInfo not found"); @@ -999,15 +963,15 @@ impl JsRuntime { receiver } - fn dyn_import_error(&mut self, id: ModuleLoadId, err: AnyError) { - let state_rc = Self::state(self.v8_isolate()); + fn dynamic_import_reject(&mut self, id: ModuleLoadId, err: AnyError) { + let module_map_rc = Self::module_map(self.v8_isolate()); let scope = &mut self.handle_scope(); - let resolver_handle = state_rc + let resolver_handle = module_map_rc .borrow_mut() - .dyn_import_map + .dynamic_import_map .remove(&id) - .expect("Invalid dyn import id"); + .expect("Invalid dynamic import id"); let resolver = resolver_handle.get(scope); let exception = err @@ -1019,25 +983,28 @@ impl JsRuntime { v8::Exception::type_error(scope, message) }); + // IMPORTANT: No borrows to `ModuleMap` can be held at this point because + // rejecting the promise might initiate another `import()` which will + // in turn call `bindings::host_import_module_dynamically_callback` which + // will reach into `ModuleMap` from within the isolate. resolver.reject(scope, exception).unwrap(); scope.perform_microtask_checkpoint(); } - fn dyn_import_done(&mut self, id: ModuleLoadId, mod_id: ModuleId) { - let state_rc = Self::state(self.v8_isolate()); + fn dynamic_import_resolve(&mut self, id: ModuleLoadId, mod_id: ModuleId) { + let module_map_rc = Self::module_map(self.v8_isolate()); let scope = &mut self.handle_scope(); - let resolver_handle = state_rc + let resolver_handle = module_map_rc .borrow_mut() - .dyn_import_map + .dynamic_import_map .remove(&id) - .expect("Invalid dyn import id"); + .expect("Invalid dynamic import id"); let resolver = resolver_handle.get(scope); let module = { - let state = state_rc.borrow(); - state - .module_map + module_map_rc + .borrow() .get_handle(mod_id) .map(|handle| v8::Local::new(scope, handle)) .expect("Dyn import module info not found") @@ -1045,6 +1012,10 @@ impl JsRuntime { // Resolution success assert_eq!(module.get_status(), v8::ModuleStatus::Evaluated); + // IMPORTANT: No borrows to `ModuleMap` can be held at this point because + // resolving the promise might initiate another `import()` which will + // in turn call `bindings::host_import_module_dynamically_callback` which + // will reach into `ModuleMap` from within the isolate. let module_namespace = module.get_module_namespace(); resolver.resolve(scope, module_namespace).unwrap(); scope.perform_microtask_checkpoint(); @@ -1054,16 +1025,16 @@ impl JsRuntime { &mut self, cx: &mut Context, ) -> Poll<Result<(), AnyError>> { - let state_rc = Self::state(self.v8_isolate()); + let module_map_rc = Self::module_map(self.v8_isolate()); - if state_rc.borrow().preparing_dyn_imports.is_empty() { + if module_map_rc.borrow().preparing_dynamic_imports.is_empty() { return Poll::Ready(Ok(())); } loop { - let poll_result = state_rc + let poll_result = module_map_rc .borrow_mut() - .preparing_dyn_imports + .preparing_dynamic_imports .poll_next_unpin(cx); if let Poll::Ready(Some(prepare_poll)) = poll_result { @@ -1072,13 +1043,13 @@ impl JsRuntime { match prepare_result { Ok(load) => { - state_rc + module_map_rc .borrow_mut() - .pending_dyn_imports + .pending_dynamic_imports .push(load.into_future()); } Err(err) => { - self.dyn_import_error(dyn_import_id, err); + self.dynamic_import_reject(dyn_import_id, err); } } // Continue polling for more prepared dynamic imports. @@ -1094,16 +1065,16 @@ impl JsRuntime { &mut self, cx: &mut Context, ) -> Poll<Result<(), AnyError>> { - let state_rc = Self::state(self.v8_isolate()); + let module_map_rc = Self::module_map(self.v8_isolate()); - if state_rc.borrow().pending_dyn_imports.is_empty() { + if module_map_rc.borrow().pending_dynamic_imports.is_empty() { return Poll::Ready(Ok(())); } loop { - let poll_result = state_rc + let poll_result = module_map_rc .borrow_mut() - .pending_dyn_imports + .pending_dynamic_imports .poll_next_unpin(cx); if let Poll::Ready(Some(load_stream_poll)) = poll_result { @@ -1117,33 +1088,40 @@ impl JsRuntime { // A module (not necessarily the one dynamically imported) has been // fetched. Create and register it, and if successful, poll for the // next recursive-load event related to this dynamic import. - match self.register_during_load(info, &mut load) { + let register_result = + module_map_rc.borrow_mut().register_during_load( + &mut self.handle_scope(), + info, + &mut load, + ); + + match register_result { Ok(()) => { // Keep importing until it's fully drained - state_rc + module_map_rc .borrow_mut() - .pending_dyn_imports + .pending_dynamic_imports .push(load.into_future()); } - Err(err) => self.dyn_import_error(dyn_import_id, err), + Err(err) => self.dynamic_import_reject(dyn_import_id, err), } } Err(err) => { // A non-javascript error occurred; this could be due to a an invalid // module specifier, or a problem with the source map, or a failure // to fetch the module source code. - self.dyn_import_error(dyn_import_id, err) + self.dynamic_import_reject(dyn_import_id, err) } } } else { // The top-level module from a dynamic import has been instantiated. // Load is done. - let module_id = load.root_module_id.unwrap(); - let result = self.mod_instantiate(module_id); + let module_id = load.expect_finished(); + let result = self.instantiate_module(module_id); if let Err(err) = result { - self.dyn_import_error(dyn_import_id, err); + self.dynamic_import_reject(dyn_import_id, err); } - self.dyn_mod_evaluate(dyn_import_id, module_id)?; + self.dynamic_import_module_evaluate(dyn_import_id, module_id)?; } // Continue polling for more ready dynamic imports. @@ -1252,10 +1230,10 @@ impl JsRuntime { if let Some(result) = maybe_result { match result { Ok((dyn_import_id, module_id)) => { - self.dyn_import_done(dyn_import_id, module_id); + self.dynamic_import_resolve(dyn_import_id, module_id); } Err((dyn_import_id, err1)) => { - self.dyn_import_error(dyn_import_id, err1); + self.dynamic_import_reject(dyn_import_id, err1); } } } else { @@ -1264,90 +1242,6 @@ impl JsRuntime { } } - fn register_during_load( - &mut self, - info: ModuleSource, - load: &mut RecursiveModuleLoad, - ) -> Result<(), AnyError> { - let ModuleSource { - code, - module_url_specified, - module_url_found, - } = info; - - let is_main = - load.state == LoadState::LoadingRoot && !load.is_dynamic_import(); - let referrer_specifier = crate::resolve_url(&module_url_found).unwrap(); - - let state_rc = Self::state(self.v8_isolate()); - // #A There are 3 cases to handle at this moment: - // 1. Source code resolved result have the same module name as requested - // and is not yet registered - // -> register - // 2. Source code resolved result have a different name as requested: - // 2a. The module with resolved module name has been registered - // -> alias - // 2b. The module with resolved module name has not yet been registered - // -> register & alias - - // If necessary, register an alias. - if module_url_specified != module_url_found { - let mut state = state_rc.borrow_mut(); - state - .module_map - .alias(&module_url_specified, &module_url_found); - } - - let maybe_mod_id = { - let state = state_rc.borrow(); - state.module_map.get_id(&module_url_found) - }; - - let module_id = match maybe_mod_id { - Some(id) => { - // Module has already been registered. - debug!( - "Already-registered module fetched again: {}", - module_url_found - ); - id - } - // Module not registered yet, do it now. - None => self.mod_new(is_main, &module_url_found, &code)?, - }; - - // Now we must iterate over all imports of the module and load them. - let imports = { - let state_rc = Self::state(self.v8_isolate()); - let state = state_rc.borrow(); - state.module_map.get_children(module_id).unwrap().clone() - }; - - for module_specifier in imports { - let is_registered = { - let state_rc = Self::state(self.v8_isolate()); - let state = state_rc.borrow(); - state.module_map.is_registered(&module_specifier) - }; - if !is_registered { - load - .add_import(module_specifier.to_owned(), referrer_specifier.clone()); - } - } - - // If we just finished loading the root module, store the root module id. - if load.state == LoadState::LoadingRoot { - load.root_module_id = Some(module_id); - load.state = LoadState::LoadingImports; - } - - if load.pending.is_empty() { - load.state = LoadState::Done; - } - - Ok(()) - } - /// Asynchronously load specified module and all of its dependencies /// /// User must call `JsRuntime::mod_evaluate` with returned `ModuleId` @@ -1357,29 +1251,24 @@ impl JsRuntime { specifier: &ModuleSpecifier, code: Option<String>, ) -> Result<ModuleId, AnyError> { - let loader = { - let state_rc = Self::state(self.v8_isolate()); - let state = state_rc.borrow(); - state.loader.clone() - }; + let module_map_rc = Self::module_map(self.v8_isolate()); + + let load = module_map_rc.borrow().load_main(specifier.as_str(), code); - let load = RecursiveModuleLoad::main( - self.op_state(), - &specifier.to_string(), - code, - loader, - ); let (_load_id, prepare_result) = load.prepare().await; let mut load = prepare_result?; while let Some(info_result) = load.next().await { let info = info_result?; - self.register_during_load(info, &mut load)?; + let scope = &mut self.handle_scope(); + module_map_rc + .borrow_mut() + .register_during_load(scope, info, &mut load)?; } - let root_id = load.root_module_id.expect("Root module id empty"); - self.mod_instantiate(root_id).map(|_| root_id) + let root_id = load.expect_finished(); + self.instantiate_module(root_id).map(|_| root_id) } fn poll_pending_ops( @@ -1525,8 +1414,6 @@ pub mod tests { use crate::op_sync; use crate::ZeroCopyBuf; use futures::future::lazy; - use futures::FutureExt; - use std::io; use std::ops::FnOnce; use std::rc::Rc; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -1719,7 +1606,7 @@ pub mod tests { "#, ) .unwrap(); - if let Poll::Ready(Err(_)) = runtime.poll_event_loop(&mut cx) { + if let Poll::Ready(Err(_)) = runtime.poll_event_loop(&mut cx, false) { unreachable!(); } }); @@ -1745,7 +1632,7 @@ pub mod tests { include_str!("encode_decode_test.js"), ) .unwrap(); - if let Poll::Ready(Err(_)) = runtime.poll_event_loop(&mut cx) { + if let Poll::Ready(Err(_)) = runtime.poll_event_loop(&mut cx, false) { unreachable!(); } }); @@ -1761,7 +1648,7 @@ pub mod tests { include_str!("serialize_deserialize_test.js"), ) .unwrap(); - if let Poll::Ready(Err(_)) = runtime.poll_event_loop(&mut cx) { + if let Poll::Ready(Err(_)) = runtime.poll_event_loop(&mut cx, false) { unreachable!(); } }); @@ -1794,7 +1681,7 @@ pub mod tests { include_str!("error_builder_test.js"), ) .unwrap(); - if let Poll::Ready(Err(_)) = runtime.poll_event_loop(&mut cx) { + if let Poll::Ready(Err(_)) = runtime.poll_event_loop(&mut cx, false) { unreachable!(); } }); @@ -1929,312 +1816,6 @@ pub mod tests { } #[test] - fn test_mods() { - #[derive(Default)] - struct ModsLoader { - pub count: Arc<AtomicUsize>, - } - - impl ModuleLoader for ModsLoader { - fn resolve( - &self, - _op_state: Rc<RefCell<OpState>>, - specifier: &str, - referrer: &str, - _is_main: bool, - ) -> Result<ModuleSpecifier, AnyError> { - self.count.fetch_add(1, Ordering::Relaxed); - assert_eq!(specifier, "./b.js"); - assert_eq!(referrer, "file:///a.js"); - let s = crate::resolve_import(specifier, referrer).unwrap(); - Ok(s) - } - - fn load( - &self, - _op_state: Rc<RefCell<OpState>>, - _module_specifier: &ModuleSpecifier, - _maybe_referrer: Option<ModuleSpecifier>, - _is_dyn_import: bool, - ) -> Pin<Box<ModuleSourceFuture>> { - unreachable!() - } - } - - let loader = Rc::new(ModsLoader::default()); - - let resolve_count = loader.count.clone(); - let dispatch_count = Arc::new(AtomicUsize::new(0)); - let dispatch_count_ = dispatch_count.clone(); - - let dispatcher = move |state, payload: OpPayload| -> Op { - dispatch_count_.fetch_add(1, Ordering::Relaxed); - let (control, _): (u8, ()) = payload.deserialize().unwrap(); - assert_eq!(control, 42); - let resp = (0, serialize_op_result(Ok(43), state)); - Op::Async(Box::pin(futures::future::ready(resp))) - }; - - let mut runtime = JsRuntime::new(RuntimeOptions { - module_loader: Some(loader), - ..Default::default() - }); - runtime.register_op("op_test", dispatcher); - runtime.sync_ops_cache(); - - runtime - .execute( - "setup.js", - r#" - function assert(cond) { - if (!cond) { - throw Error("assert"); - } - } - "#, - ) - .unwrap(); - - assert_eq!(dispatch_count.load(Ordering::Relaxed), 0); - - let specifier_a = "file:///a.js".to_string(); - let mod_a = runtime - .mod_new( - true, - &specifier_a, - r#" - import { b } from './b.js' - if (b() != 'b') throw Error(); - let control = 42; - Deno.core.opAsync("op_test", control); - "#, - ) - .unwrap(); - assert_eq!(dispatch_count.load(Ordering::Relaxed), 0); - - let state_rc = JsRuntime::state(runtime.v8_isolate()); - { - let state = state_rc.borrow(); - let imports = state.module_map.get_children(mod_a); - assert_eq!( - imports, - Some(&vec![crate::resolve_url("file:///b.js").unwrap()]) - ); - } - let mod_b = runtime - .mod_new(false, "file:///b.js", "export function b() { return 'b' }") - .unwrap(); - { - let state = state_rc.borrow(); - let imports = state.module_map.get_children(mod_b).unwrap(); - assert_eq!(imports.len(), 0); - } - - runtime.mod_instantiate(mod_b).unwrap(); - assert_eq!(dispatch_count.load(Ordering::Relaxed), 0); - assert_eq!(resolve_count.load(Ordering::SeqCst), 1); - - runtime.mod_instantiate(mod_a).unwrap(); - assert_eq!(dispatch_count.load(Ordering::Relaxed), 0); - - runtime.mod_evaluate(mod_a); - assert_eq!(dispatch_count.load(Ordering::Relaxed), 1); - } - - #[test] - fn dyn_import_err() { - #[derive(Clone, Default)] - struct DynImportErrLoader { - pub count: Arc<AtomicUsize>, - } - - impl ModuleLoader for DynImportErrLoader { - fn resolve( - &self, - _op_state: Rc<RefCell<OpState>>, - specifier: &str, - referrer: &str, - _is_main: bool, - ) -> Result<ModuleSpecifier, AnyError> { - self.count.fetch_add(1, Ordering::Relaxed); - assert_eq!(specifier, "/foo.js"); - assert_eq!(referrer, "file:///dyn_import2.js"); - let s = crate::resolve_import(specifier, referrer).unwrap(); - Ok(s) - } - - fn load( - &self, - _op_state: Rc<RefCell<OpState>>, - _module_specifier: &ModuleSpecifier, - _maybe_referrer: Option<ModuleSpecifier>, - _is_dyn_import: bool, - ) -> Pin<Box<ModuleSourceFuture>> { - async { Err(io::Error::from(io::ErrorKind::NotFound).into()) }.boxed() - } - } - - // Test an erroneous dynamic import where the specified module isn't found. - run_in_task(|cx| { - let loader = Rc::new(DynImportErrLoader::default()); - let count = loader.count.clone(); - let mut runtime = JsRuntime::new(RuntimeOptions { - module_loader: Some(loader), - ..Default::default() - }); - - runtime - .execute( - "file:///dyn_import2.js", - r#" - (async () => { - await import("/foo.js"); - })(); - "#, - ) - .unwrap(); - - assert_eq!(count.load(Ordering::Relaxed), 0); - // We should get an error here. - let result = runtime.poll_event_loop(cx); - if let Poll::Ready(Ok(_)) = result { - unreachable!(); - } - assert_eq!(count.load(Ordering::Relaxed), 2); - }) - } - - #[derive(Clone, Default)] - struct DynImportOkLoader { - pub prepare_load_count: Arc<AtomicUsize>, - pub resolve_count: Arc<AtomicUsize>, - pub load_count: Arc<AtomicUsize>, - } - - impl ModuleLoader for DynImportOkLoader { - fn resolve( - &self, - _op_state: Rc<RefCell<OpState>>, - specifier: &str, - referrer: &str, - _is_main: bool, - ) -> Result<ModuleSpecifier, AnyError> { - let c = self.resolve_count.fetch_add(1, Ordering::Relaxed); - assert!(c < 4); - assert_eq!(specifier, "./b.js"); - assert_eq!(referrer, "file:///dyn_import3.js"); - let s = crate::resolve_import(specifier, referrer).unwrap(); - Ok(s) - } - - fn load( - &self, - _op_state: Rc<RefCell<OpState>>, - specifier: &ModuleSpecifier, - _maybe_referrer: Option<ModuleSpecifier>, - _is_dyn_import: bool, - ) -> Pin<Box<ModuleSourceFuture>> { - self.load_count.fetch_add(1, Ordering::Relaxed); - let info = ModuleSource { - module_url_specified: specifier.to_string(), - module_url_found: specifier.to_string(), - code: "export function b() { return 'b' }".to_owned(), - }; - async move { Ok(info) }.boxed() - } - - fn prepare_load( - &self, - _op_state: Rc<RefCell<OpState>>, - _load_id: ModuleLoadId, - _module_specifier: &ModuleSpecifier, - _maybe_referrer: Option<String>, - _is_dyn_import: bool, - ) -> Pin<Box<dyn Future<Output = Result<(), AnyError>>>> { - self.prepare_load_count.fetch_add(1, Ordering::Relaxed); - async { Ok(()) }.boxed_local() - } - } - - #[test] - fn dyn_import_ok() { - run_in_task(|cx| { - let loader = Rc::new(DynImportOkLoader::default()); - let prepare_load_count = loader.prepare_load_count.clone(); - let resolve_count = loader.resolve_count.clone(); - let load_count = loader.load_count.clone(); - let mut runtime = JsRuntime::new(RuntimeOptions { - module_loader: Some(loader), - ..Default::default() - }); - - // Dynamically import mod_b - runtime - .execute( - "file:///dyn_import3.js", - r#" - (async () => { - let mod = await import("./b.js"); - if (mod.b() !== 'b') { - throw Error("bad1"); - } - // And again! - mod = await import("./b.js"); - if (mod.b() !== 'b') { - throw Error("bad2"); - } - })(); - "#, - ) - .unwrap(); - - // First poll runs `prepare_load` hook. - assert!(matches!(runtime.poll_event_loop(cx), Poll::Pending)); - assert_eq!(prepare_load_count.load(Ordering::Relaxed), 1); - - // Second poll actually loads modules into the isolate. - assert!(matches!(runtime.poll_event_loop(cx), Poll::Ready(Ok(_)))); - assert_eq!(resolve_count.load(Ordering::Relaxed), 4); - assert_eq!(load_count.load(Ordering::Relaxed), 2); - assert!(matches!(runtime.poll_event_loop(cx), Poll::Ready(Ok(_)))); - assert_eq!(resolve_count.load(Ordering::Relaxed), 4); - assert_eq!(load_count.load(Ordering::Relaxed), 2); - }) - } - - #[test] - fn dyn_import_borrow_mut_error() { - // https://github.com/denoland/deno/issues/6054 - run_in_task(|cx| { - let loader = Rc::new(DynImportOkLoader::default()); - let prepare_load_count = loader.prepare_load_count.clone(); - let mut runtime = JsRuntime::new(RuntimeOptions { - module_loader: Some(loader), - ..Default::default() - }); - runtime.sync_ops_cache(); - runtime - .execute( - "file:///dyn_import3.js", - r#" - (async () => { - let mod = await import("./b.js"); - if (mod.b() !== 'b') { - throw Error("bad"); - } - })(); - "#, - ) - .unwrap(); - // First poll runs `prepare_load` hook. - let _ = runtime.poll_event_loop(cx); - assert_eq!(prepare_load_count.load(Ordering::Relaxed), 1); - // Second poll triggers error - let _ = runtime.poll_event_loop(cx); - }) - } - - #[test] fn es_snapshot() { #[derive(Default)] struct ModsLoader; @@ -2280,7 +1861,7 @@ pub mod tests { .unwrap(); runtime.mod_evaluate(module_id); - futures::executor::block_on(runtime.run_event_loop()).unwrap(); + futures::executor::block_on(runtime.run_event_loop(false)).unwrap(); let _snapshot = runtime.snapshot(); } @@ -2359,7 +1940,7 @@ main(); at async error_async_stack.js:4:5 at async error_async_stack.js:10:5"#; - match runtime.poll_event_loop(cx) { + match runtime.poll_event_loop(cx, false) { Poll::Ready(Err(e)) => { assert_eq!(e.to_string(), expected_error); } |