diff options
Diffstat (limited to 'ext/fetch/lib.rs')
-rw-r--r-- | ext/fetch/lib.rs | 276 |
1 files changed, 185 insertions, 91 deletions
diff --git a/ext/fetch/lib.rs b/ext/fetch/lib.rs index 88f303852..c8e93b9fe 100644 --- a/ext/fetch/lib.rs +++ b/ext/fetch/lib.rs @@ -1,5 +1,6 @@ // Copyright 2018-2024 the Deno authors. All rights reserved. MIT license. +pub mod dns; mod fs_fetch_handler; mod proxy; #[cfg(test)] @@ -17,10 +18,6 @@ use std::sync::Arc; use std::task::Context; use std::task::Poll; -use deno_core::anyhow::anyhow; -use deno_core::anyhow::Error; -use deno_core::error::type_error; -use deno_core::error::AnyError; use deno_core::futures::stream::Peekable; use deno_core::futures::Future; use deno_core::futures::FutureExt; @@ -28,6 +25,7 @@ use deno_core::futures::Stream; use deno_core::futures::StreamExt; use deno_core::futures::TryFutureExt; use deno_core::op2; +use deno_core::url; use deno_core::url::Url; use deno_core::AsyncRefCell; use deno_core::AsyncResult; @@ -42,6 +40,7 @@ use deno_core::OpState; use deno_core::RcRef; use deno_core::Resource; use deno_core::ResourceId; +use deno_permissions::PermissionCheckError; use deno_tls::rustls::RootCertStore; use deno_tls::Proxy; use deno_tls::RootCertStoreProvider; @@ -68,6 +67,7 @@ use http_body_util::BodyExt; use hyper::body::Frame; use hyper_util::client::legacy::connect::HttpConnector; use hyper_util::client::legacy::connect::HttpInfo; +use hyper_util::client::legacy::Builder as HyperClientBuilder; use hyper_util::rt::TokioExecutor; use hyper_util::rt::TokioTimer; use serde::Deserialize; @@ -86,16 +86,30 @@ pub struct Options { pub user_agent: String, pub root_cert_store_provider: Option<Arc<dyn RootCertStoreProvider>>, pub proxy: Option<Proxy>, + /// A callback to customize HTTP client configuration. + /// + /// The settings applied with this hook may be overridden by the options + /// provided through `Deno.createHttpClient()` API. For instance, if the hook + /// calls [`hyper_util::client::legacy::Builder::pool_max_idle_per_host`] with + /// a value of 99, and a user calls `Deno.createHttpClient({ poolMaxIdlePerHost: 42 })`, + /// the value that will take effect is 42. + /// + /// For more info on what can be configured, see [`hyper_util::client::legacy::Builder`]. + pub client_builder_hook: Option<fn(HyperClientBuilder) -> HyperClientBuilder>, #[allow(clippy::type_complexity)] - pub request_builder_hook: - Option<fn(&mut http::Request<ReqBody>) -> Result<(), AnyError>>, + pub request_builder_hook: Option< + fn(&mut http::Request<ReqBody>) -> Result<(), deno_core::error::AnyError>, + >, pub unsafely_ignore_certificate_errors: Option<Vec<String>>, pub client_cert_chain_and_key: TlsKeys, pub file_fetch_handler: Rc<dyn FetchHandler>, + pub resolver: dns::Resolver, } impl Options { - pub fn root_cert_store(&self) -> Result<Option<RootCertStore>, AnyError> { + pub fn root_cert_store( + &self, + ) -> Result<Option<RootCertStore>, deno_core::error::AnyError> { Ok(match &self.root_cert_store_provider { Some(provider) => Some(provider.get_or_try_init()?.clone()), None => None, @@ -109,10 +123,12 @@ impl Default for Options { user_agent: "".to_string(), root_cert_store_provider: None, proxy: None, + client_builder_hook: None, request_builder_hook: None, unsafely_ignore_certificate_errors: None, client_cert_chain_and_key: TlsKeys::Null, file_fetch_handler: Rc::new(DefaultFileFetchHandler), + resolver: dns::Resolver::default(), } } } @@ -144,6 +160,51 @@ deno_core::extension!(deno_fetch, }, ); +#[derive(Debug, thiserror::Error)] +pub enum FetchError { + #[error(transparent)] + Resource(deno_core::error::AnyError), + #[error(transparent)] + Permission(#[from] PermissionCheckError), + #[error("NetworkError when attempting to fetch resource")] + NetworkError, + #[error("Fetching files only supports the GET method: received {0}")] + FsNotGet(Method), + #[error("Invalid URL {0}")] + InvalidUrl(Url), + #[error(transparent)] + InvalidHeaderName(#[from] http::header::InvalidHeaderName), + #[error(transparent)] + InvalidHeaderValue(#[from] http::header::InvalidHeaderValue), + #[error("{0:?}")] + DataUrl(data_url::DataUrlError), + #[error("{0:?}")] + Base64(data_url::forgiving_base64::InvalidBase64), + #[error("Blob for the given URL not found.")] + BlobNotFound, + #[error("Url scheme '{0}' not supported")] + SchemeNotSupported(String), + #[error("Request was cancelled")] + RequestCanceled, + #[error(transparent)] + Http(#[from] http::Error), + #[error(transparent)] + ClientCreate(#[from] HttpClientCreateError), + #[error(transparent)] + Url(#[from] url::ParseError), + #[error(transparent)] + Method(#[from] http::method::InvalidMethod), + #[error(transparent)] + ClientSend(#[from] ClientSendError), + #[error(transparent)] + RequestBuilderHook(deno_core::error::AnyError), + #[error(transparent)] + Io(#[from] std::io::Error), + // Only used for node upgrade + #[error(transparent)] + Hyper(#[from] hyper::Error), +} + pub type CancelableResponseFuture = Pin<Box<dyn Future<Output = CancelableResponseResult>>>; @@ -170,11 +231,7 @@ impl FetchHandler for DefaultFileFetchHandler { _state: &mut OpState, _url: &Url, ) -> (CancelableResponseFuture, Option<Rc<CancelHandle>>) { - let fut = async move { - Ok(Err(type_error( - "NetworkError when attempting to fetch resource", - ))) - }; + let fut = async move { Ok(Err(FetchError::NetworkError)) }; (Box::pin(fut), None) } } @@ -191,7 +248,7 @@ pub struct FetchReturn { pub fn get_or_create_client_from_state( state: &mut OpState, -) -> Result<Client, AnyError> { +) -> Result<Client, HttpClientCreateError> { if let Some(client) = state.try_borrow::<Client>() { Ok(client.clone()) } else { @@ -204,13 +261,16 @@ pub fn get_or_create_client_from_state( pub fn create_client_from_options( options: &Options, -) -> Result<Client, AnyError> { +) -> Result<Client, HttpClientCreateError> { create_http_client( &options.user_agent, CreateHttpClientOptions { - root_cert_store: options.root_cert_store()?, + root_cert_store: options + .root_cert_store() + .map_err(HttpClientCreateError::RootCertStore)?, ca_certs: vec![], proxy: options.proxy.clone(), + dns_resolver: options.resolver.clone(), unsafely_ignore_certificate_errors: options .unsafely_ignore_certificate_errors .clone(), @@ -223,6 +283,7 @@ pub fn create_client_from_options( pool_idle_timeout: None, http1: true, http2: true, + client_builder_hook: options.client_builder_hook, }, ) } @@ -230,7 +291,9 @@ pub fn create_client_from_options( #[allow(clippy::type_complexity)] pub struct ResourceToBodyAdapter( Rc<dyn Resource>, - Option<Pin<Box<dyn Future<Output = Result<BufView, Error>>>>>, + Option< + Pin<Box<dyn Future<Output = Result<BufView, deno_core::error::AnyError>>>>, + >, ); impl ResourceToBodyAdapter { @@ -246,7 +309,7 @@ unsafe impl Send for ResourceToBodyAdapter {} unsafe impl Sync for ResourceToBodyAdapter {} impl Stream for ResourceToBodyAdapter { - type Item = Result<Bytes, Error>; + type Item = Result<Bytes, deno_core::error::AnyError>; fn poll_next( self: Pin<&mut Self>, @@ -276,7 +339,7 @@ impl Stream for ResourceToBodyAdapter { impl hyper::body::Body for ResourceToBodyAdapter { type Data = Bytes; - type Error = Error; + type Error = deno_core::error::AnyError; fn poll_frame( self: Pin<&mut Self>, @@ -301,13 +364,13 @@ pub trait FetchPermissions { &mut self, url: &Url, api_name: &str, - ) -> Result<(), AnyError>; + ) -> Result<(), PermissionCheckError>; #[must_use = "the resolved return value to mitigate time-of-check to time-of-use issues"] fn check_read<'a>( &mut self, p: &'a Path, api_name: &str, - ) -> Result<Cow<'a, Path>, AnyError>; + ) -> Result<Cow<'a, Path>, PermissionCheckError>; } impl FetchPermissions for deno_permissions::PermissionsContainer { @@ -316,7 +379,7 @@ impl FetchPermissions for deno_permissions::PermissionsContainer { &mut self, url: &Url, api_name: &str, - ) -> Result<(), AnyError> { + ) -> Result<(), PermissionCheckError> { deno_permissions::PermissionsContainer::check_net_url(self, url, api_name) } @@ -325,7 +388,7 @@ impl FetchPermissions for deno_permissions::PermissionsContainer { &mut self, path: &'a Path, api_name: &str, - ) -> Result<Cow<'a, Path>, AnyError> { + ) -> Result<Cow<'a, Path>, PermissionCheckError> { deno_permissions::PermissionsContainer::check_read_path( self, path, @@ -346,12 +409,15 @@ pub fn op_fetch<FP>( has_body: bool, #[buffer] data: Option<JsBuffer>, #[smi] resource: Option<ResourceId>, -) -> Result<FetchReturn, AnyError> +) -> Result<FetchReturn, FetchError> where FP: FetchPermissions + 'static, { let (client, allow_host) = if let Some(rid) = client_rid { - let r = state.resource_table.get::<HttpClientResource>(rid)?; + let r = state + .resource_table + .get::<HttpClientResource>(rid) + .map_err(FetchError::Resource)?; (r.client.clone(), r.allow_host) } else { (get_or_create_client_from_state(state)?, false) @@ -364,9 +430,7 @@ where let scheme = url.scheme(); let (request_rid, cancel_handle_rid) = match scheme { "file" => { - let path = url.to_file_path().map_err(|_| { - type_error("NetworkError when attempting to fetch resource") - })?; + let path = url.to_file_path().map_err(|_| FetchError::NetworkError)?; let permissions = state.borrow_mut::<FP>(); let path = permissions.check_read(&path, "fetch()")?; let url = match path { @@ -375,9 +439,7 @@ where }; if method != Method::GET { - return Err(type_error(format!( - "Fetching files only supports the GET method: received {method}" - ))); + return Err(FetchError::FsNotGet(method)); } let Options { @@ -402,7 +464,7 @@ where let uri = url .as_str() .parse::<Uri>() - .map_err(|_| type_error(format!("Invalid URL {url}")))?; + .map_err(|_| FetchError::InvalidUrl(url.clone()))?; let mut con_len = None; let body = if has_body { @@ -416,7 +478,10 @@ where .boxed() } (_, Some(resource)) => { - let resource = state.resource_table.take_any(resource)?; + let resource = state + .resource_table + .take_any(resource) + .map_err(FetchError::Resource)?; match resource.size_hint() { (body_size, Some(n)) if body_size == n && body_size > 0 => { con_len = Some(body_size); @@ -453,10 +518,8 @@ where } for (key, value) in headers { - let name = HeaderName::from_bytes(&key) - .map_err(|err| type_error(err.to_string()))?; - let v = HeaderValue::from_bytes(&value) - .map_err(|err| type_error(err.to_string()))?; + let name = HeaderName::from_bytes(&key)?; + let v = HeaderValue::from_bytes(&value)?; if (name != HOST || allow_host) && name != CONTENT_LENGTH { request.headers_mut().append(name, v); @@ -474,20 +537,18 @@ where let options = state.borrow::<Options>(); if let Some(request_builder_hook) = options.request_builder_hook { request_builder_hook(&mut request) - .map_err(|err| type_error(err.to_string()))?; + .map_err(FetchError::RequestBuilderHook)?; } let cancel_handle = CancelHandle::new_rc(); let cancel_handle_ = cancel_handle.clone(); - let fut = { - async move { - client - .send(request) - .map_err(Into::into) - .or_cancel(cancel_handle_) - .await - } + let fut = async move { + client + .send(request) + .map_err(Into::into) + .or_cancel(cancel_handle_) + .await }; let request_rid = state.resource_table.add(FetchRequestResource { @@ -501,12 +562,10 @@ where (request_rid, Some(cancel_handle_rid)) } "data" => { - let data_url = DataUrl::process(url.as_str()) - .map_err(|e| type_error(format!("{e:?}")))?; + let data_url = + DataUrl::process(url.as_str()).map_err(FetchError::DataUrl)?; - let (body, _) = data_url - .decode_to_vec() - .map_err(|e| type_error(format!("{e:?}")))?; + let (body, _) = data_url.decode_to_vec().map_err(FetchError::Base64)?; let body = http_body_util::Full::new(body.into()) .map_err(|never| match never {}) .boxed(); @@ -528,11 +587,9 @@ where "blob" => { // Blob URL resolution happens in the JS side of fetch. If we got here is // because the URL isn't an object URL. - return Err(type_error("Blob for the given URL not found.")); - } - _ => { - return Err(type_error(format!("Url scheme '{scheme}' not supported"))) + return Err(FetchError::BlobNotFound); } + _ => return Err(FetchError::SchemeNotSupported(scheme.to_string())), }; Ok(FetchReturn { @@ -564,11 +621,12 @@ pub struct FetchResponse { pub async fn op_fetch_send( state: Rc<RefCell<OpState>>, #[smi] rid: ResourceId, -) -> Result<FetchResponse, AnyError> { +) -> Result<FetchResponse, FetchError> { let request = state .borrow_mut() .resource_table - .take::<FetchRequestResource>(rid)?; + .take::<FetchRequestResource>(rid) + .map_err(FetchError::Resource)?; let request = Rc::try_unwrap(request) .ok() @@ -581,22 +639,23 @@ pub async fn op_fetch_send( // If any error in the chain is a hyper body error, return that as a special result we can use to // reconstruct an error chain (eg: `new TypeError(..., { cause: new Error(...) })`). // TODO(mmastrac): it would be a lot easier if we just passed a v8::Global through here instead - let mut err_ref: &dyn std::error::Error = err.as_ref(); - while let Some(err_src) = std::error::Error::source(err_ref) { - if let Some(err_src) = err_src.downcast_ref::<hyper::Error>() { - if let Some(err_src) = std::error::Error::source(err_src) { - return Ok(FetchResponse { - error: Some((err.to_string(), err_src.to_string())), - ..Default::default() - }); + + if let FetchError::ClientSend(err_src) = &err { + if let Some(client_err) = std::error::Error::source(&err_src.source) { + if let Some(err_src) = client_err.downcast_ref::<hyper::Error>() { + if let Some(err_src) = std::error::Error::source(err_src) { + return Ok(FetchResponse { + error: Some((err.to_string(), err_src.to_string())), + ..Default::default() + }); + } } } - err_ref = err_src; } - return Err(type_error(err.to_string())); + return Err(err); } - Err(_) => return Err(type_error("Request was cancelled")), + Err(_) => return Err(FetchError::RequestCanceled), }; let status = res.status(); @@ -636,7 +695,7 @@ pub async fn op_fetch_send( } type CancelableResponseResult = - Result<Result<http::Response<ResBody>, AnyError>, Canceled>; + Result<Result<http::Response<ResBody>, FetchError>, Canceled>; pub struct FetchRequestResource { pub future: Pin<Box<dyn Future<Output = CancelableResponseResult>>>, @@ -691,7 +750,7 @@ impl FetchResponseResource { } } - pub async fn upgrade(self) -> Result<hyper::upgrade::Upgraded, AnyError> { + pub async fn upgrade(self) -> Result<hyper::upgrade::Upgraded, hyper::Error> { let reader = self.response_reader.into_inner(); match reader { FetchResponseReader::Start(resp) => Ok(hyper::upgrade::on(resp).await?), @@ -746,7 +805,9 @@ impl Resource for FetchResponseResource { // safely call `await` on it without creating a race condition. Some(_) => match reader.as_mut().next().await.unwrap() { Ok(chunk) => assert!(chunk.is_empty()), - Err(err) => break Err(type_error(err.to_string())), + Err(err) => { + break Err(deno_core::error::type_error(err.to_string())) + } }, None => break Ok(BufView::empty()), } @@ -791,6 +852,8 @@ pub struct CreateHttpClientArgs { proxy: Option<Proxy>, pool_max_idle_per_host: Option<usize>, pool_idle_timeout: Option<serde_json::Value>, + #[serde(default)] + use_hickory_resolver: bool, #[serde(default = "default_true")] http1: bool, #[serde(default = "default_true")] @@ -809,7 +872,7 @@ pub fn op_fetch_custom_client<FP>( state: &mut OpState, #[serde] args: CreateHttpClientArgs, #[cppgc] tls_keys: &TlsKeysHolder, -) -> Result<ResourceId, AnyError> +) -> Result<ResourceId, FetchError> where FP: FetchPermissions + 'static, { @@ -829,9 +892,18 @@ where let client = create_http_client( &options.user_agent, CreateHttpClientOptions { - root_cert_store: options.root_cert_store()?, + root_cert_store: options + .root_cert_store() + .map_err(HttpClientCreateError::RootCertStore)?, ca_certs, proxy: args.proxy, + dns_resolver: if args.use_hickory_resolver { + dns::Resolver::hickory() + .map_err(deno_core::error::AnyError::new) + .map_err(FetchError::Resource)? + } else { + dns::Resolver::default() + }, unsafely_ignore_certificate_errors: options .unsafely_ignore_certificate_errors .clone(), @@ -849,6 +921,7 @@ where ), http1: args.http1, http2: args.http2, + client_builder_hook: options.client_builder_hook, }, )?; @@ -863,12 +936,14 @@ pub struct CreateHttpClientOptions { pub root_cert_store: Option<RootCertStore>, pub ca_certs: Vec<Vec<u8>>, pub proxy: Option<Proxy>, + pub dns_resolver: dns::Resolver, pub unsafely_ignore_certificate_errors: Option<Vec<String>>, pub client_cert_chain_and_key: Option<TlsKey>, pub pool_max_idle_per_host: Option<usize>, pub pool_idle_timeout: Option<Option<u64>>, pub http1: bool, pub http2: bool, + pub client_builder_hook: Option<fn(HyperClientBuilder) -> HyperClientBuilder>, } impl Default for CreateHttpClientOptions { @@ -877,29 +952,46 @@ impl Default for CreateHttpClientOptions { root_cert_store: None, ca_certs: vec![], proxy: None, + dns_resolver: dns::Resolver::default(), unsafely_ignore_certificate_errors: None, client_cert_chain_and_key: None, pool_max_idle_per_host: None, pool_idle_timeout: None, http1: true, http2: true, + client_builder_hook: None, } } } +#[derive(Debug, thiserror::Error)] +pub enum HttpClientCreateError { + #[error(transparent)] + Tls(deno_tls::TlsError), + #[error("Illegal characters in User-Agent: received {0}")] + InvalidUserAgent(String), + #[error("invalid proxy url")] + InvalidProxyUrl, + #[error("Cannot create Http Client: either `http1` or `http2` needs to be set to true")] + HttpVersionSelectionInvalid, + #[error(transparent)] + RootCertStore(deno_core::error::AnyError), +} + /// Create new instance of async Client. This client supports /// proxies and doesn't follow redirects. pub fn create_http_client( user_agent: &str, options: CreateHttpClientOptions, -) -> Result<Client, AnyError> { +) -> Result<Client, HttpClientCreateError> { let mut tls_config = deno_tls::create_client_config( options.root_cert_store, options.ca_certs, options.unsafely_ignore_certificate_errors, options.client_cert_chain_and_key.into(), deno_tls::SocketUse::Http, - )?; + ) + .map_err(HttpClientCreateError::Tls)?; // Proxy TLS should not send ALPN tls_config.alpn_protocols.clear(); @@ -915,24 +1007,26 @@ pub fn create_http_client( tls_config.alpn_protocols = alpn_protocols; let tls_config = Arc::from(tls_config); - let mut http_connector = HttpConnector::new(); + let mut http_connector = + HttpConnector::new_with_resolver(options.dns_resolver.clone()); http_connector.enforce_http(false); let user_agent = user_agent.parse::<HeaderValue>().map_err(|_| { - type_error(format!( - "Illegal characters in User-Agent: received {user_agent}" - )) + HttpClientCreateError::InvalidUserAgent(user_agent.to_string()) })?; - let mut builder = - hyper_util::client::legacy::Builder::new(TokioExecutor::new()); + let mut builder = HyperClientBuilder::new(TokioExecutor::new()); builder.timer(TokioTimer::new()); builder.pool_timer(TokioTimer::new()); + if let Some(client_builder_hook) = options.client_builder_hook { + builder = client_builder_hook(builder); + } + let mut proxies = proxy::from_env(); if let Some(proxy) = options.proxy { let mut intercept = proxy::Intercept::all(&proxy.url) - .ok_or_else(|| type_error("invalid proxy url"))?; + .ok_or_else(|| HttpClientCreateError::InvalidProxyUrl)?; if let Some(basic_auth) = &proxy.basic_auth { intercept.set_auth(&basic_auth.username, &basic_auth.password); } @@ -964,7 +1058,7 @@ pub fn create_http_client( } (true, true) => {} (false, false) => { - return Err(type_error("Cannot create Http Client: either `http1` or `http2` needs to be set to true")) + return Err(HttpClientCreateError::HttpVersionSelectionInvalid) } } @@ -980,10 +1074,8 @@ pub fn create_http_client( #[op2] #[serde] -pub fn op_utf8_to_byte_string( - #[string] input: String, -) -> Result<ByteString, AnyError> { - Ok(input.into()) +pub fn op_utf8_to_byte_string(#[string] input: String) -> ByteString { + input.into() } #[derive(Clone, Debug)] @@ -994,7 +1086,7 @@ pub struct Client { user_agent: HeaderValue, } -type Connector = proxy::ProxyConnector<HttpConnector>; +type Connector = proxy::ProxyConnector<HttpConnector<dns::Resolver>>; // clippy is wrong here #[allow(clippy::declare_interior_mutable_const)] @@ -1003,7 +1095,7 @@ const STAR_STAR: HeaderValue = HeaderValue::from_static("*/*"); #[derive(Debug)] pub struct ClientSendError { uri: Uri, - source: hyper_util::client::legacy::Error, + pub source: hyper_util::client::legacy::Error, } impl ClientSendError { @@ -1075,12 +1167,14 @@ impl Client { .oneshot(req) .await .map_err(|e| ClientSendError { uri, source: e })?; - Ok(resp.map(|b| b.map_err(|e| anyhow!(e)).boxed())) + Ok(resp.map(|b| b.map_err(|e| deno_core::anyhow::anyhow!(e)).boxed())) } } -pub type ReqBody = http_body_util::combinators::BoxBody<Bytes, Error>; -pub type ResBody = http_body_util::combinators::BoxBody<Bytes, Error>; +pub type ReqBody = + http_body_util::combinators::BoxBody<Bytes, deno_core::error::AnyError>; +pub type ResBody = + http_body_util::combinators::BoxBody<Bytes, deno_core::error::AnyError>; /// Copied from https://github.com/seanmonstar/reqwest/blob/b9d62a0323d96f11672a61a17bf8849baec00275/src/async_impl/request.rs#L572 /// Check the request URL for a "username:password" type authority, and if |