summaryrefslogtreecommitdiff
path: root/ext/fetch
diff options
context:
space:
mode:
Diffstat (limited to 'ext/fetch')
-rw-r--r--ext/fetch/22_body.js14
-rw-r--r--ext/fetch/23_request.js16
-rw-r--r--ext/fetch/23_response.js105
-rw-r--r--ext/fetch/Cargo.toml4
-rw-r--r--ext/fetch/dns.rs116
-rw-r--r--ext/fetch/fs_fetch_handler.rs5
-rw-r--r--ext/fetch/lib.deno_fetch.d.ts2
-rw-r--r--ext/fetch/lib.rs276
-rw-r--r--ext/fetch/tests.rs80
9 files changed, 465 insertions, 153 deletions
diff --git a/ext/fetch/22_body.js b/ext/fetch/22_body.js
index 61a06b4af..c7e977c0b 100644
--- a/ext/fetch/22_body.js
+++ b/ext/fetch/22_body.js
@@ -15,6 +15,7 @@ import { core, primordials } from "ext:core/mod.js";
const {
isAnyArrayBuffer,
isArrayBuffer,
+ isStringObject,
} = core;
const {
ArrayBufferIsView,
@@ -466,6 +467,8 @@ function extractBody(object) {
if (object.locked || isReadableStreamDisturbed(object)) {
throw new TypeError("ReadableStream is locked or disturbed");
}
+ } else if (object[webidl.AsyncIterable] === webidl.AsyncIterable) {
+ stream = ReadableStream.from(object.open());
}
if (typeof source === "string") {
// WARNING: this deviates from spec (expects length to be set)
@@ -483,6 +486,9 @@ function extractBody(object) {
return { body, contentType };
}
+webidl.converters["async iterable<Uint8Array>"] = webidl
+ .createAsyncIterableConverter(webidl.converters.Uint8Array);
+
webidl.converters["BodyInit_DOMString"] = (V, prefix, context, opts) => {
// Union for (ReadableStream or Blob or ArrayBufferView or ArrayBuffer or FormData or URLSearchParams or USVString)
if (ObjectPrototypeIsPrototypeOf(ReadableStreamPrototype, V)) {
@@ -501,6 +507,14 @@ webidl.converters["BodyInit_DOMString"] = (V, prefix, context, opts) => {
if (ArrayBufferIsView(V)) {
return webidl.converters["ArrayBufferView"](V, prefix, context, opts);
}
+ if (webidl.isAsyncIterable(V) && !isStringObject(V)) {
+ return webidl.converters["async iterable<Uint8Array>"](
+ V,
+ prefix,
+ context,
+ opts,
+ );
+ }
}
// BodyInit conversion is passed to extractBody(), which calls core.encode().
// core.encode() will UTF-8 encode strings with replacement, being equivalent to the USV normalization.
diff --git a/ext/fetch/23_request.js b/ext/fetch/23_request.js
index 6211e927d..61cac22d2 100644
--- a/ext/fetch/23_request.js
+++ b/ext/fetch/23_request.js
@@ -269,19 +269,25 @@ class Request {
/** @type {AbortSignal} */
get [_signal]() {
const signal = this[_signalCache];
- // This signal not been created yet, and the request is still in progress
- if (signal === undefined) {
+ // This signal has not been created yet, but the request has already completed
+ if (signal === false) {
const signal = newSignal();
this[_signalCache] = signal;
+ signal[signalAbort](signalAbortError);
return signal;
}
- // This signal has not been created yet, but the request has already completed
- if (signal === false) {
+
+ // This signal not been created yet, and the request is still in progress
+ if (signal === undefined) {
const signal = newSignal();
this[_signalCache] = signal;
- signal[signalAbort](signalAbortError);
+ this[_request].onCancel?.(() => {
+ signal[signalAbort](signalAbortError);
+ });
+
return signal;
}
+
return signal;
}
get [_mimeType]() {
diff --git a/ext/fetch/23_response.js b/ext/fetch/23_response.js
index ff4ad5fac..278dcb7de 100644
--- a/ext/fetch/23_response.js
+++ b/ext/fetch/23_response.js
@@ -61,6 +61,15 @@ const _mimeType = Symbol("mime type");
const _body = Symbol("body");
const _brand = webidl.brand;
+// it's slightly faster to cache these
+const webidlConvertersBodyInitDomString =
+ webidl.converters["BodyInit_DOMString?"];
+const webidlConvertersUSVString = webidl.converters["USVString"];
+const webidlConvertersUnsignedShort = webidl.converters["unsigned short"];
+const webidlConvertersAny = webidl.converters["any"];
+const webidlConvertersByteString = webidl.converters["ByteString"];
+const webidlConvertersHeadersInit = webidl.converters["HeadersInit"];
+
/**
* @typedef InnerResponse
* @property {"basic" | "cors" | "default" | "error" | "opaque" | "opaqueredirect"} type
@@ -259,8 +268,8 @@ class Response {
*/
static redirect(url, status = 302) {
const prefix = "Failed to execute 'Response.redirect'";
- url = webidl.converters["USVString"](url, prefix, "Argument 1");
- status = webidl.converters["unsigned short"](status, prefix, "Argument 2");
+ url = webidlConvertersUSVString(url, prefix, "Argument 1");
+ status = webidlConvertersUnsignedShort(status, prefix, "Argument 2");
const baseURL = getLocationHref();
const parsedURL = new URL(url, baseURL);
@@ -286,8 +295,8 @@ class Response {
*/
static json(data = undefined, init = { __proto__: null }) {
const prefix = "Failed to execute 'Response.json'";
- data = webidl.converters.any(data);
- init = webidl.converters["ResponseInit_fast"](init, prefix, "Argument 2");
+ data = webidlConvertersAny(data);
+ init = webidlConvertersResponseInitFast(init, prefix, "Argument 2");
const str = serializeJSValueToJSONString(data);
const res = extractBody(str);
@@ -313,8 +322,8 @@ class Response {
}
const prefix = "Failed to construct 'Response'";
- body = webidl.converters["BodyInit_DOMString?"](body, prefix, "Argument 1");
- init = webidl.converters["ResponseInit_fast"](init, prefix, "Argument 2");
+ body = webidlConvertersBodyInitDomString(body, prefix, "Argument 1");
+ init = webidlConvertersResponseInitFast(init, prefix, "Argument 2");
this[_response] = newInnerResponse();
this[_headers] = headersFromHeaderList(
@@ -443,47 +452,49 @@ webidl.converters["Response"] = webidl.createInterfaceConverter(
"Response",
ResponsePrototype,
);
-webidl.converters["ResponseInit"] = webidl.createDictionaryConverter(
- "ResponseInit",
- [{
- key: "status",
- defaultValue: 200,
- converter: webidl.converters["unsigned short"],
- }, {
- key: "statusText",
- defaultValue: "",
- converter: webidl.converters["ByteString"],
- }, {
- key: "headers",
- converter: webidl.converters["HeadersInit"],
- }],
-);
-webidl.converters["ResponseInit_fast"] = function (
- init,
- prefix,
- context,
- opts,
-) {
- if (init === undefined || init === null) {
- return { status: 200, statusText: "", headers: undefined };
- }
- // Fast path, if not a proxy
- if (typeof init === "object" && !core.isProxy(init)) {
- // Not a proxy fast path
- const status = init.status !== undefined
- ? webidl.converters["unsigned short"](init.status)
- : 200;
- const statusText = init.statusText !== undefined
- ? webidl.converters["ByteString"](init.statusText)
- : "";
- const headers = init.headers !== undefined
- ? webidl.converters["HeadersInit"](init.headers)
- : undefined;
- return { status, statusText, headers };
- }
- // Slow default path
- return webidl.converters["ResponseInit"](init, prefix, context, opts);
-};
+const webidlConvertersResponseInit = webidl.converters["ResponseInit"] = webidl
+ .createDictionaryConverter(
+ "ResponseInit",
+ [{
+ key: "status",
+ defaultValue: 200,
+ converter: webidlConvertersUnsignedShort,
+ }, {
+ key: "statusText",
+ defaultValue: "",
+ converter: webidlConvertersByteString,
+ }, {
+ key: "headers",
+ converter: webidlConvertersHeadersInit,
+ }],
+ );
+const webidlConvertersResponseInitFast = webidl
+ .converters["ResponseInit_fast"] = function (
+ init,
+ prefix,
+ context,
+ opts,
+ ) {
+ if (init === undefined || init === null) {
+ return { status: 200, statusText: "", headers: undefined };
+ }
+ // Fast path, if not a proxy
+ if (typeof init === "object" && !core.isProxy(init)) {
+ // Not a proxy fast path
+ const status = init.status !== undefined
+ ? webidlConvertersUnsignedShort(init.status)
+ : 200;
+ const statusText = init.statusText !== undefined
+ ? webidlConvertersByteString(init.statusText)
+ : "";
+ const headers = init.headers !== undefined
+ ? webidlConvertersHeadersInit(init.headers)
+ : undefined;
+ return { status, statusText, headers };
+ }
+ // Slow default path
+ return webidlConvertersResponseInit(init, prefix, context, opts);
+ };
/**
* @param {Response} response
diff --git a/ext/fetch/Cargo.toml b/ext/fetch/Cargo.toml
index cc9e4f03d..00c85f2aa 100644
--- a/ext/fetch/Cargo.toml
+++ b/ext/fetch/Cargo.toml
@@ -2,7 +2,7 @@
[package]
name = "deno_fetch"
-version = "0.195.0"
+version = "0.201.0"
authors.workspace = true
edition.workspace = true
license.workspace = true
@@ -22,6 +22,7 @@ deno_permissions.workspace = true
deno_tls.workspace = true
dyn-clone = "1"
error_reporter = "1"
+hickory-resolver.workspace = true
http.workspace = true
http-body-util.workspace = true
hyper.workspace = true
@@ -32,6 +33,7 @@ percent-encoding.workspace = true
rustls-webpki.workspace = true
serde.workspace = true
serde_json.workspace = true
+thiserror.workspace = true
tokio.workspace = true
tokio-rustls.workspace = true
tokio-socks.workspace = true
diff --git a/ext/fetch/dns.rs b/ext/fetch/dns.rs
new file mode 100644
index 000000000..9e21a4c34
--- /dev/null
+++ b/ext/fetch/dns.rs
@@ -0,0 +1,116 @@
+// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
+use std::future::Future;
+use std::io;
+use std::net::SocketAddr;
+use std::pin::Pin;
+use std::task::Poll;
+use std::task::{self};
+use std::vec;
+
+use hickory_resolver::error::ResolveError;
+use hickory_resolver::name_server::GenericConnector;
+use hickory_resolver::name_server::TokioRuntimeProvider;
+use hickory_resolver::AsyncResolver;
+use hyper_util::client::legacy::connect::dns::GaiResolver;
+use hyper_util::client::legacy::connect::dns::Name;
+use tokio::task::JoinHandle;
+use tower::Service;
+
+#[derive(Clone, Debug)]
+pub enum Resolver {
+ /// A resolver using blocking `getaddrinfo` calls in a threadpool.
+ Gai(GaiResolver),
+ /// hickory-resolver's userspace resolver.
+ Hickory(AsyncResolver<GenericConnector<TokioRuntimeProvider>>),
+}
+
+impl Default for Resolver {
+ fn default() -> Self {
+ Self::gai()
+ }
+}
+
+impl Resolver {
+ pub fn gai() -> Self {
+ Self::Gai(GaiResolver::new())
+ }
+
+ /// Create a [`AsyncResolver`] from system conf.
+ pub fn hickory() -> Result<Self, ResolveError> {
+ Ok(Self::Hickory(
+ hickory_resolver::AsyncResolver::tokio_from_system_conf()?,
+ ))
+ }
+
+ pub fn hickory_from_async_resolver(
+ resolver: AsyncResolver<GenericConnector<TokioRuntimeProvider>>,
+ ) -> Self {
+ Self::Hickory(resolver)
+ }
+}
+
+type SocketAddrs = vec::IntoIter<SocketAddr>;
+
+pub struct ResolveFut {
+ inner: JoinHandle<Result<SocketAddrs, io::Error>>,
+}
+
+impl Future for ResolveFut {
+ type Output = Result<SocketAddrs, io::Error>;
+
+ fn poll(
+ mut self: Pin<&mut Self>,
+ cx: &mut task::Context<'_>,
+ ) -> Poll<Self::Output> {
+ Pin::new(&mut self.inner).poll(cx).map(|res| match res {
+ Ok(Ok(addrs)) => Ok(addrs),
+ Ok(Err(e)) => Err(e),
+ Err(join_err) => {
+ if join_err.is_cancelled() {
+ Err(io::Error::new(io::ErrorKind::Interrupted, join_err))
+ } else {
+ Err(io::Error::new(io::ErrorKind::Other, join_err))
+ }
+ }
+ })
+ }
+}
+
+impl Service<Name> for Resolver {
+ type Response = SocketAddrs;
+ type Error = io::Error;
+ type Future = ResolveFut;
+
+ fn poll_ready(
+ &mut self,
+ _cx: &mut task::Context<'_>,
+ ) -> Poll<Result<(), io::Error>> {
+ Poll::Ready(Ok(()))
+ }
+
+ fn call(&mut self, name: Name) -> Self::Future {
+ let task = match self {
+ Resolver::Gai(gai_resolver) => {
+ let mut resolver = gai_resolver.clone();
+ tokio::spawn(async move {
+ let result = resolver.call(name).await?;
+ let x: Vec<_> = result.into_iter().collect();
+ let iter: SocketAddrs = x.into_iter();
+ Ok(iter)
+ })
+ }
+ Resolver::Hickory(async_resolver) => {
+ let resolver = async_resolver.clone();
+ tokio::spawn(async move {
+ let result = resolver.lookup_ip(name.as_str()).await?;
+
+ let x: Vec<_> =
+ result.into_iter().map(|x| SocketAddr::new(x, 0)).collect();
+ let iter: SocketAddrs = x.into_iter();
+ Ok(iter)
+ })
+ }
+ };
+ ResolveFut { inner: task }
+ }
+}
diff --git a/ext/fetch/fs_fetch_handler.rs b/ext/fetch/fs_fetch_handler.rs
index 4c2b81f35..c236dd9c6 100644
--- a/ext/fetch/fs_fetch_handler.rs
+++ b/ext/fetch/fs_fetch_handler.rs
@@ -4,7 +4,6 @@ use crate::CancelHandle;
use crate::CancelableResponseFuture;
use crate::FetchHandler;
-use deno_core::error::type_error;
use deno_core::futures::FutureExt;
use deno_core::futures::TryFutureExt;
use deno_core::futures::TryStreamExt;
@@ -42,9 +41,7 @@ impl FetchHandler for FsFetchHandler {
.map_err(|_| ())?;
Ok::<_, ()>(response)
}
- .map_err(move |_| {
- type_error("NetworkError when attempting to fetch resource")
- })
+ .map_err(move |_| super::FetchError::NetworkError)
.or_cancel(&cancel_handle)
.boxed_local();
diff --git a/ext/fetch/lib.deno_fetch.d.ts b/ext/fetch/lib.deno_fetch.d.ts
index d219a3859..8614dec89 100644
--- a/ext/fetch/lib.deno_fetch.d.ts
+++ b/ext/fetch/lib.deno_fetch.d.ts
@@ -163,6 +163,8 @@ type BodyInit =
| FormData
| URLSearchParams
| ReadableStream<Uint8Array>
+ | Iterable<Uint8Array>
+ | AsyncIterable<Uint8Array>
| string;
/** @category Fetch */
type RequestDestination =
diff --git a/ext/fetch/lib.rs b/ext/fetch/lib.rs
index 88f303852..c8e93b9fe 100644
--- a/ext/fetch/lib.rs
+++ b/ext/fetch/lib.rs
@@ -1,5 +1,6 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
+pub mod dns;
mod fs_fetch_handler;
mod proxy;
#[cfg(test)]
@@ -17,10 +18,6 @@ use std::sync::Arc;
use std::task::Context;
use std::task::Poll;
-use deno_core::anyhow::anyhow;
-use deno_core::anyhow::Error;
-use deno_core::error::type_error;
-use deno_core::error::AnyError;
use deno_core::futures::stream::Peekable;
use deno_core::futures::Future;
use deno_core::futures::FutureExt;
@@ -28,6 +25,7 @@ use deno_core::futures::Stream;
use deno_core::futures::StreamExt;
use deno_core::futures::TryFutureExt;
use deno_core::op2;
+use deno_core::url;
use deno_core::url::Url;
use deno_core::AsyncRefCell;
use deno_core::AsyncResult;
@@ -42,6 +40,7 @@ use deno_core::OpState;
use deno_core::RcRef;
use deno_core::Resource;
use deno_core::ResourceId;
+use deno_permissions::PermissionCheckError;
use deno_tls::rustls::RootCertStore;
use deno_tls::Proxy;
use deno_tls::RootCertStoreProvider;
@@ -68,6 +67,7 @@ use http_body_util::BodyExt;
use hyper::body::Frame;
use hyper_util::client::legacy::connect::HttpConnector;
use hyper_util::client::legacy::connect::HttpInfo;
+use hyper_util::client::legacy::Builder as HyperClientBuilder;
use hyper_util::rt::TokioExecutor;
use hyper_util::rt::TokioTimer;
use serde::Deserialize;
@@ -86,16 +86,30 @@ pub struct Options {
pub user_agent: String,
pub root_cert_store_provider: Option<Arc<dyn RootCertStoreProvider>>,
pub proxy: Option<Proxy>,
+ /// A callback to customize HTTP client configuration.
+ ///
+ /// The settings applied with this hook may be overridden by the options
+ /// provided through `Deno.createHttpClient()` API. For instance, if the hook
+ /// calls [`hyper_util::client::legacy::Builder::pool_max_idle_per_host`] with
+ /// a value of 99, and a user calls `Deno.createHttpClient({ poolMaxIdlePerHost: 42 })`,
+ /// the value that will take effect is 42.
+ ///
+ /// For more info on what can be configured, see [`hyper_util::client::legacy::Builder`].
+ pub client_builder_hook: Option<fn(HyperClientBuilder) -> HyperClientBuilder>,
#[allow(clippy::type_complexity)]
- pub request_builder_hook:
- Option<fn(&mut http::Request<ReqBody>) -> Result<(), AnyError>>,
+ pub request_builder_hook: Option<
+ fn(&mut http::Request<ReqBody>) -> Result<(), deno_core::error::AnyError>,
+ >,
pub unsafely_ignore_certificate_errors: Option<Vec<String>>,
pub client_cert_chain_and_key: TlsKeys,
pub file_fetch_handler: Rc<dyn FetchHandler>,
+ pub resolver: dns::Resolver,
}
impl Options {
- pub fn root_cert_store(&self) -> Result<Option<RootCertStore>, AnyError> {
+ pub fn root_cert_store(
+ &self,
+ ) -> Result<Option<RootCertStore>, deno_core::error::AnyError> {
Ok(match &self.root_cert_store_provider {
Some(provider) => Some(provider.get_or_try_init()?.clone()),
None => None,
@@ -109,10 +123,12 @@ impl Default for Options {
user_agent: "".to_string(),
root_cert_store_provider: None,
proxy: None,
+ client_builder_hook: None,
request_builder_hook: None,
unsafely_ignore_certificate_errors: None,
client_cert_chain_and_key: TlsKeys::Null,
file_fetch_handler: Rc::new(DefaultFileFetchHandler),
+ resolver: dns::Resolver::default(),
}
}
}
@@ -144,6 +160,51 @@ deno_core::extension!(deno_fetch,
},
);
+#[derive(Debug, thiserror::Error)]
+pub enum FetchError {
+ #[error(transparent)]
+ Resource(deno_core::error::AnyError),
+ #[error(transparent)]
+ Permission(#[from] PermissionCheckError),
+ #[error("NetworkError when attempting to fetch resource")]
+ NetworkError,
+ #[error("Fetching files only supports the GET method: received {0}")]
+ FsNotGet(Method),
+ #[error("Invalid URL {0}")]
+ InvalidUrl(Url),
+ #[error(transparent)]
+ InvalidHeaderName(#[from] http::header::InvalidHeaderName),
+ #[error(transparent)]
+ InvalidHeaderValue(#[from] http::header::InvalidHeaderValue),
+ #[error("{0:?}")]
+ DataUrl(data_url::DataUrlError),
+ #[error("{0:?}")]
+ Base64(data_url::forgiving_base64::InvalidBase64),
+ #[error("Blob for the given URL not found.")]
+ BlobNotFound,
+ #[error("Url scheme '{0}' not supported")]
+ SchemeNotSupported(String),
+ #[error("Request was cancelled")]
+ RequestCanceled,
+ #[error(transparent)]
+ Http(#[from] http::Error),
+ #[error(transparent)]
+ ClientCreate(#[from] HttpClientCreateError),
+ #[error(transparent)]
+ Url(#[from] url::ParseError),
+ #[error(transparent)]
+ Method(#[from] http::method::InvalidMethod),
+ #[error(transparent)]
+ ClientSend(#[from] ClientSendError),
+ #[error(transparent)]
+ RequestBuilderHook(deno_core::error::AnyError),
+ #[error(transparent)]
+ Io(#[from] std::io::Error),
+ // Only used for node upgrade
+ #[error(transparent)]
+ Hyper(#[from] hyper::Error),
+}
+
pub type CancelableResponseFuture =
Pin<Box<dyn Future<Output = CancelableResponseResult>>>;
@@ -170,11 +231,7 @@ impl FetchHandler for DefaultFileFetchHandler {
_state: &mut OpState,
_url: &Url,
) -> (CancelableResponseFuture, Option<Rc<CancelHandle>>) {
- let fut = async move {
- Ok(Err(type_error(
- "NetworkError when attempting to fetch resource",
- )))
- };
+ let fut = async move { Ok(Err(FetchError::NetworkError)) };
(Box::pin(fut), None)
}
}
@@ -191,7 +248,7 @@ pub struct FetchReturn {
pub fn get_or_create_client_from_state(
state: &mut OpState,
-) -> Result<Client, AnyError> {
+) -> Result<Client, HttpClientCreateError> {
if let Some(client) = state.try_borrow::<Client>() {
Ok(client.clone())
} else {
@@ -204,13 +261,16 @@ pub fn get_or_create_client_from_state(
pub fn create_client_from_options(
options: &Options,
-) -> Result<Client, AnyError> {
+) -> Result<Client, HttpClientCreateError> {
create_http_client(
&options.user_agent,
CreateHttpClientOptions {
- root_cert_store: options.root_cert_store()?,
+ root_cert_store: options
+ .root_cert_store()
+ .map_err(HttpClientCreateError::RootCertStore)?,
ca_certs: vec![],
proxy: options.proxy.clone(),
+ dns_resolver: options.resolver.clone(),
unsafely_ignore_certificate_errors: options
.unsafely_ignore_certificate_errors
.clone(),
@@ -223,6 +283,7 @@ pub fn create_client_from_options(
pool_idle_timeout: None,
http1: true,
http2: true,
+ client_builder_hook: options.client_builder_hook,
},
)
}
@@ -230,7 +291,9 @@ pub fn create_client_from_options(
#[allow(clippy::type_complexity)]
pub struct ResourceToBodyAdapter(
Rc<dyn Resource>,
- Option<Pin<Box<dyn Future<Output = Result<BufView, Error>>>>>,
+ Option<
+ Pin<Box<dyn Future<Output = Result<BufView, deno_core::error::AnyError>>>>,
+ >,
);
impl ResourceToBodyAdapter {
@@ -246,7 +309,7 @@ unsafe impl Send for ResourceToBodyAdapter {}
unsafe impl Sync for ResourceToBodyAdapter {}
impl Stream for ResourceToBodyAdapter {
- type Item = Result<Bytes, Error>;
+ type Item = Result<Bytes, deno_core::error::AnyError>;
fn poll_next(
self: Pin<&mut Self>,
@@ -276,7 +339,7 @@ impl Stream for ResourceToBodyAdapter {
impl hyper::body::Body for ResourceToBodyAdapter {
type Data = Bytes;
- type Error = Error;
+ type Error = deno_core::error::AnyError;
fn poll_frame(
self: Pin<&mut Self>,
@@ -301,13 +364,13 @@ pub trait FetchPermissions {
&mut self,
url: &Url,
api_name: &str,
- ) -> Result<(), AnyError>;
+ ) -> Result<(), PermissionCheckError>;
#[must_use = "the resolved return value to mitigate time-of-check to time-of-use issues"]
fn check_read<'a>(
&mut self,
p: &'a Path,
api_name: &str,
- ) -> Result<Cow<'a, Path>, AnyError>;
+ ) -> Result<Cow<'a, Path>, PermissionCheckError>;
}
impl FetchPermissions for deno_permissions::PermissionsContainer {
@@ -316,7 +379,7 @@ impl FetchPermissions for deno_permissions::PermissionsContainer {
&mut self,
url: &Url,
api_name: &str,
- ) -> Result<(), AnyError> {
+ ) -> Result<(), PermissionCheckError> {
deno_permissions::PermissionsContainer::check_net_url(self, url, api_name)
}
@@ -325,7 +388,7 @@ impl FetchPermissions for deno_permissions::PermissionsContainer {
&mut self,
path: &'a Path,
api_name: &str,
- ) -> Result<Cow<'a, Path>, AnyError> {
+ ) -> Result<Cow<'a, Path>, PermissionCheckError> {
deno_permissions::PermissionsContainer::check_read_path(
self,
path,
@@ -346,12 +409,15 @@ pub fn op_fetch<FP>(
has_body: bool,
#[buffer] data: Option<JsBuffer>,
#[smi] resource: Option<ResourceId>,
-) -> Result<FetchReturn, AnyError>
+) -> Result<FetchReturn, FetchError>
where
FP: FetchPermissions + 'static,
{
let (client, allow_host) = if let Some(rid) = client_rid {
- let r = state.resource_table.get::<HttpClientResource>(rid)?;
+ let r = state
+ .resource_table
+ .get::<HttpClientResource>(rid)
+ .map_err(FetchError::Resource)?;
(r.client.clone(), r.allow_host)
} else {
(get_or_create_client_from_state(state)?, false)
@@ -364,9 +430,7 @@ where
let scheme = url.scheme();
let (request_rid, cancel_handle_rid) = match scheme {
"file" => {
- let path = url.to_file_path().map_err(|_| {
- type_error("NetworkError when attempting to fetch resource")
- })?;
+ let path = url.to_file_path().map_err(|_| FetchError::NetworkError)?;
let permissions = state.borrow_mut::<FP>();
let path = permissions.check_read(&path, "fetch()")?;
let url = match path {
@@ -375,9 +439,7 @@ where
};
if method != Method::GET {
- return Err(type_error(format!(
- "Fetching files only supports the GET method: received {method}"
- )));
+ return Err(FetchError::FsNotGet(method));
}
let Options {
@@ -402,7 +464,7 @@ where
let uri = url
.as_str()
.parse::<Uri>()
- .map_err(|_| type_error(format!("Invalid URL {url}")))?;
+ .map_err(|_| FetchError::InvalidUrl(url.clone()))?;
let mut con_len = None;
let body = if has_body {
@@ -416,7 +478,10 @@ where
.boxed()
}
(_, Some(resource)) => {
- let resource = state.resource_table.take_any(resource)?;
+ let resource = state
+ .resource_table
+ .take_any(resource)
+ .map_err(FetchError::Resource)?;
match resource.size_hint() {
(body_size, Some(n)) if body_size == n && body_size > 0 => {
con_len = Some(body_size);
@@ -453,10 +518,8 @@ where
}
for (key, value) in headers {
- let name = HeaderName::from_bytes(&key)
- .map_err(|err| type_error(err.to_string()))?;
- let v = HeaderValue::from_bytes(&value)
- .map_err(|err| type_error(err.to_string()))?;
+ let name = HeaderName::from_bytes(&key)?;
+ let v = HeaderValue::from_bytes(&value)?;
if (name != HOST || allow_host) && name != CONTENT_LENGTH {
request.headers_mut().append(name, v);
@@ -474,20 +537,18 @@ where
let options = state.borrow::<Options>();
if let Some(request_builder_hook) = options.request_builder_hook {
request_builder_hook(&mut request)
- .map_err(|err| type_error(err.to_string()))?;
+ .map_err(FetchError::RequestBuilderHook)?;
}
let cancel_handle = CancelHandle::new_rc();
let cancel_handle_ = cancel_handle.clone();
- let fut = {
- async move {
- client
- .send(request)
- .map_err(Into::into)
- .or_cancel(cancel_handle_)
- .await
- }
+ let fut = async move {
+ client
+ .send(request)
+ .map_err(Into::into)
+ .or_cancel(cancel_handle_)
+ .await
};
let request_rid = state.resource_table.add(FetchRequestResource {
@@ -501,12 +562,10 @@ where
(request_rid, Some(cancel_handle_rid))
}
"data" => {
- let data_url = DataUrl::process(url.as_str())
- .map_err(|e| type_error(format!("{e:?}")))?;
+ let data_url =
+ DataUrl::process(url.as_str()).map_err(FetchError::DataUrl)?;
- let (body, _) = data_url
- .decode_to_vec()
- .map_err(|e| type_error(format!("{e:?}")))?;
+ let (body, _) = data_url.decode_to_vec().map_err(FetchError::Base64)?;
let body = http_body_util::Full::new(body.into())
.map_err(|never| match never {})
.boxed();
@@ -528,11 +587,9 @@ where
"blob" => {
// Blob URL resolution happens in the JS side of fetch. If we got here is
// because the URL isn't an object URL.
- return Err(type_error("Blob for the given URL not found."));
- }
- _ => {
- return Err(type_error(format!("Url scheme '{scheme}' not supported")))
+ return Err(FetchError::BlobNotFound);
}
+ _ => return Err(FetchError::SchemeNotSupported(scheme.to_string())),
};
Ok(FetchReturn {
@@ -564,11 +621,12 @@ pub struct FetchResponse {
pub async fn op_fetch_send(
state: Rc<RefCell<OpState>>,
#[smi] rid: ResourceId,
-) -> Result<FetchResponse, AnyError> {
+) -> Result<FetchResponse, FetchError> {
let request = state
.borrow_mut()
.resource_table
- .take::<FetchRequestResource>(rid)?;
+ .take::<FetchRequestResource>(rid)
+ .map_err(FetchError::Resource)?;
let request = Rc::try_unwrap(request)
.ok()
@@ -581,22 +639,23 @@ pub async fn op_fetch_send(
// If any error in the chain is a hyper body error, return that as a special result we can use to
// reconstruct an error chain (eg: `new TypeError(..., { cause: new Error(...) })`).
// TODO(mmastrac): it would be a lot easier if we just passed a v8::Global through here instead
- let mut err_ref: &dyn std::error::Error = err.as_ref();
- while let Some(err_src) = std::error::Error::source(err_ref) {
- if let Some(err_src) = err_src.downcast_ref::<hyper::Error>() {
- if let Some(err_src) = std::error::Error::source(err_src) {
- return Ok(FetchResponse {
- error: Some((err.to_string(), err_src.to_string())),
- ..Default::default()
- });
+
+ if let FetchError::ClientSend(err_src) = &err {
+ if let Some(client_err) = std::error::Error::source(&err_src.source) {
+ if let Some(err_src) = client_err.downcast_ref::<hyper::Error>() {
+ if let Some(err_src) = std::error::Error::source(err_src) {
+ return Ok(FetchResponse {
+ error: Some((err.to_string(), err_src.to_string())),
+ ..Default::default()
+ });
+ }
}
}
- err_ref = err_src;
}
- return Err(type_error(err.to_string()));
+ return Err(err);
}
- Err(_) => return Err(type_error("Request was cancelled")),
+ Err(_) => return Err(FetchError::RequestCanceled),
};
let status = res.status();
@@ -636,7 +695,7 @@ pub async fn op_fetch_send(
}
type CancelableResponseResult =
- Result<Result<http::Response<ResBody>, AnyError>, Canceled>;
+ Result<Result<http::Response<ResBody>, FetchError>, Canceled>;
pub struct FetchRequestResource {
pub future: Pin<Box<dyn Future<Output = CancelableResponseResult>>>,
@@ -691,7 +750,7 @@ impl FetchResponseResource {
}
}
- pub async fn upgrade(self) -> Result<hyper::upgrade::Upgraded, AnyError> {
+ pub async fn upgrade(self) -> Result<hyper::upgrade::Upgraded, hyper::Error> {
let reader = self.response_reader.into_inner();
match reader {
FetchResponseReader::Start(resp) => Ok(hyper::upgrade::on(resp).await?),
@@ -746,7 +805,9 @@ impl Resource for FetchResponseResource {
// safely call `await` on it without creating a race condition.
Some(_) => match reader.as_mut().next().await.unwrap() {
Ok(chunk) => assert!(chunk.is_empty()),
- Err(err) => break Err(type_error(err.to_string())),
+ Err(err) => {
+ break Err(deno_core::error::type_error(err.to_string()))
+ }
},
None => break Ok(BufView::empty()),
}
@@ -791,6 +852,8 @@ pub struct CreateHttpClientArgs {
proxy: Option<Proxy>,
pool_max_idle_per_host: Option<usize>,
pool_idle_timeout: Option<serde_json::Value>,
+ #[serde(default)]
+ use_hickory_resolver: bool,
#[serde(default = "default_true")]
http1: bool,
#[serde(default = "default_true")]
@@ -809,7 +872,7 @@ pub fn op_fetch_custom_client<FP>(
state: &mut OpState,
#[serde] args: CreateHttpClientArgs,
#[cppgc] tls_keys: &TlsKeysHolder,
-) -> Result<ResourceId, AnyError>
+) -> Result<ResourceId, FetchError>
where
FP: FetchPermissions + 'static,
{
@@ -829,9 +892,18 @@ where
let client = create_http_client(
&options.user_agent,
CreateHttpClientOptions {
- root_cert_store: options.root_cert_store()?,
+ root_cert_store: options
+ .root_cert_store()
+ .map_err(HttpClientCreateError::RootCertStore)?,
ca_certs,
proxy: args.proxy,
+ dns_resolver: if args.use_hickory_resolver {
+ dns::Resolver::hickory()
+ .map_err(deno_core::error::AnyError::new)
+ .map_err(FetchError::Resource)?
+ } else {
+ dns::Resolver::default()
+ },
unsafely_ignore_certificate_errors: options
.unsafely_ignore_certificate_errors
.clone(),
@@ -849,6 +921,7 @@ where
),
http1: args.http1,
http2: args.http2,
+ client_builder_hook: options.client_builder_hook,
},
)?;
@@ -863,12 +936,14 @@ pub struct CreateHttpClientOptions {
pub root_cert_store: Option<RootCertStore>,
pub ca_certs: Vec<Vec<u8>>,
pub proxy: Option<Proxy>,
+ pub dns_resolver: dns::Resolver,
pub unsafely_ignore_certificate_errors: Option<Vec<String>>,
pub client_cert_chain_and_key: Option<TlsKey>,
pub pool_max_idle_per_host: Option<usize>,
pub pool_idle_timeout: Option<Option<u64>>,
pub http1: bool,
pub http2: bool,
+ pub client_builder_hook: Option<fn(HyperClientBuilder) -> HyperClientBuilder>,
}
impl Default for CreateHttpClientOptions {
@@ -877,29 +952,46 @@ impl Default for CreateHttpClientOptions {
root_cert_store: None,
ca_certs: vec![],
proxy: None,
+ dns_resolver: dns::Resolver::default(),
unsafely_ignore_certificate_errors: None,
client_cert_chain_and_key: None,
pool_max_idle_per_host: None,
pool_idle_timeout: None,
http1: true,
http2: true,
+ client_builder_hook: None,
}
}
}
+#[derive(Debug, thiserror::Error)]
+pub enum HttpClientCreateError {
+ #[error(transparent)]
+ Tls(deno_tls::TlsError),
+ #[error("Illegal characters in User-Agent: received {0}")]
+ InvalidUserAgent(String),
+ #[error("invalid proxy url")]
+ InvalidProxyUrl,
+ #[error("Cannot create Http Client: either `http1` or `http2` needs to be set to true")]
+ HttpVersionSelectionInvalid,
+ #[error(transparent)]
+ RootCertStore(deno_core::error::AnyError),
+}
+
/// Create new instance of async Client. This client supports
/// proxies and doesn't follow redirects.
pub fn create_http_client(
user_agent: &str,
options: CreateHttpClientOptions,
-) -> Result<Client, AnyError> {
+) -> Result<Client, HttpClientCreateError> {
let mut tls_config = deno_tls::create_client_config(
options.root_cert_store,
options.ca_certs,
options.unsafely_ignore_certificate_errors,
options.client_cert_chain_and_key.into(),
deno_tls::SocketUse::Http,
- )?;
+ )
+ .map_err(HttpClientCreateError::Tls)?;
// Proxy TLS should not send ALPN
tls_config.alpn_protocols.clear();
@@ -915,24 +1007,26 @@ pub fn create_http_client(
tls_config.alpn_protocols = alpn_protocols;
let tls_config = Arc::from(tls_config);
- let mut http_connector = HttpConnector::new();
+ let mut http_connector =
+ HttpConnector::new_with_resolver(options.dns_resolver.clone());
http_connector.enforce_http(false);
let user_agent = user_agent.parse::<HeaderValue>().map_err(|_| {
- type_error(format!(
- "Illegal characters in User-Agent: received {user_agent}"
- ))
+ HttpClientCreateError::InvalidUserAgent(user_agent.to_string())
})?;
- let mut builder =
- hyper_util::client::legacy::Builder::new(TokioExecutor::new());
+ let mut builder = HyperClientBuilder::new(TokioExecutor::new());
builder.timer(TokioTimer::new());
builder.pool_timer(TokioTimer::new());
+ if let Some(client_builder_hook) = options.client_builder_hook {
+ builder = client_builder_hook(builder);
+ }
+
let mut proxies = proxy::from_env();
if let Some(proxy) = options.proxy {
let mut intercept = proxy::Intercept::all(&proxy.url)
- .ok_or_else(|| type_error("invalid proxy url"))?;
+ .ok_or_else(|| HttpClientCreateError::InvalidProxyUrl)?;
if let Some(basic_auth) = &proxy.basic_auth {
intercept.set_auth(&basic_auth.username, &basic_auth.password);
}
@@ -964,7 +1058,7 @@ pub fn create_http_client(
}
(true, true) => {}
(false, false) => {
- return Err(type_error("Cannot create Http Client: either `http1` or `http2` needs to be set to true"))
+ return Err(HttpClientCreateError::HttpVersionSelectionInvalid)
}
}
@@ -980,10 +1074,8 @@ pub fn create_http_client(
#[op2]
#[serde]
-pub fn op_utf8_to_byte_string(
- #[string] input: String,
-) -> Result<ByteString, AnyError> {
- Ok(input.into())
+pub fn op_utf8_to_byte_string(#[string] input: String) -> ByteString {
+ input.into()
}
#[derive(Clone, Debug)]
@@ -994,7 +1086,7 @@ pub struct Client {
user_agent: HeaderValue,
}
-type Connector = proxy::ProxyConnector<HttpConnector>;
+type Connector = proxy::ProxyConnector<HttpConnector<dns::Resolver>>;
// clippy is wrong here
#[allow(clippy::declare_interior_mutable_const)]
@@ -1003,7 +1095,7 @@ const STAR_STAR: HeaderValue = HeaderValue::from_static("*/*");
#[derive(Debug)]
pub struct ClientSendError {
uri: Uri,
- source: hyper_util::client::legacy::Error,
+ pub source: hyper_util::client::legacy::Error,
}
impl ClientSendError {
@@ -1075,12 +1167,14 @@ impl Client {
.oneshot(req)
.await
.map_err(|e| ClientSendError { uri, source: e })?;
- Ok(resp.map(|b| b.map_err(|e| anyhow!(e)).boxed()))
+ Ok(resp.map(|b| b.map_err(|e| deno_core::anyhow::anyhow!(e)).boxed()))
}
}
-pub type ReqBody = http_body_util::combinators::BoxBody<Bytes, Error>;
-pub type ResBody = http_body_util::combinators::BoxBody<Bytes, Error>;
+pub type ReqBody =
+ http_body_util::combinators::BoxBody<Bytes, deno_core::error::AnyError>;
+pub type ResBody =
+ http_body_util::combinators::BoxBody<Bytes, deno_core::error::AnyError>;
/// Copied from https://github.com/seanmonstar/reqwest/blob/b9d62a0323d96f11672a61a17bf8849baec00275/src/async_impl/request.rs#L572
/// Check the request URL for a "username:password" type authority, and if
diff --git a/ext/fetch/tests.rs b/ext/fetch/tests.rs
index dad1b34a9..e053c6b1c 100644
--- a/ext/fetch/tests.rs
+++ b/ext/fetch/tests.rs
@@ -1,6 +1,8 @@
// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
use std::net::SocketAddr;
+use std::sync::atomic::AtomicUsize;
+use std::sync::atomic::Ordering::SeqCst;
use std::sync::Arc;
use bytes::Bytes;
@@ -10,6 +12,8 @@ use http_body_util::BodyExt;
use tokio::io::AsyncReadExt;
use tokio::io::AsyncWriteExt;
+use crate::dns;
+
use super::create_http_client;
use super::CreateHttpClientOptions;
@@ -17,6 +21,53 @@ static EXAMPLE_CRT: &[u8] = include_bytes!("../tls/testdata/example1_cert.der");
static EXAMPLE_KEY: &[u8] =
include_bytes!("../tls/testdata/example1_prikey.der");
+#[test]
+fn test_userspace_resolver() {
+ let thread_counter = Arc::new(AtomicUsize::new(0));
+
+ let thread_counter_ref = thread_counter.clone();
+ let rt = tokio::runtime::Builder::new_current_thread()
+ .enable_all()
+ .on_thread_start(move || {
+ thread_counter_ref.fetch_add(1, SeqCst);
+ })
+ .build()
+ .unwrap();
+
+ rt.block_on(async move {
+ assert_eq!(thread_counter.load(SeqCst), 0);
+ let src_addr = create_https_server(true).await;
+ assert_eq!(src_addr.ip().to_string(), "127.0.0.1");
+ // use `localhost` to ensure dns step happens.
+ let addr = format!("localhost:{}", src_addr.port());
+
+ let hickory = hickory_resolver::AsyncResolver::tokio(
+ Default::default(),
+ Default::default(),
+ );
+
+ assert_eq!(thread_counter.load(SeqCst), 0);
+ rust_test_client_with_resolver(
+ None,
+ addr.clone(),
+ "https",
+ http::Version::HTTP_2,
+ dns::Resolver::hickory_from_async_resolver(hickory),
+ )
+ .await;
+ assert_eq!(thread_counter.load(SeqCst), 0, "userspace resolver shouldn't spawn new threads.");
+ rust_test_client_with_resolver(
+ None,
+ addr.clone(),
+ "https",
+ http::Version::HTTP_2,
+ dns::Resolver::gai(),
+ )
+ .await;
+ assert_eq!(thread_counter.load(SeqCst), 1, "getaddrinfo is called inside spawn_blocking, so tokio spawn a new worker thread for it.");
+ });
+}
+
#[tokio::test]
async fn test_https_proxy_http11() {
let src_addr = create_https_server(false).await;
@@ -52,27 +103,30 @@ async fn test_socks_proxy_h2() {
run_test_client(prx_addr, src_addr, "socks5", http::Version::HTTP_2).await;
}
-async fn run_test_client(
- prx_addr: SocketAddr,
- src_addr: SocketAddr,
+async fn rust_test_client_with_resolver(
+ prx_addr: Option<SocketAddr>,
+ src_addr: String,
proto: &str,
ver: http::Version,
+ resolver: dns::Resolver,
) {
let client = create_http_client(
"fetch/test",
CreateHttpClientOptions {
root_cert_store: None,
ca_certs: vec![],
- proxy: Some(deno_tls::Proxy {
- url: format!("{}://{}", proto, prx_addr),
+ proxy: prx_addr.map(|p| deno_tls::Proxy {
+ url: format!("{}://{}", proto, p),
basic_auth: None,
}),
unsafely_ignore_certificate_errors: Some(vec![]),
client_cert_chain_and_key: None,
pool_max_idle_per_host: None,
pool_idle_timeout: None,
+ dns_resolver: resolver,
http1: true,
http2: true,
+ client_builder_hook: None,
},
)
.unwrap();
@@ -92,6 +146,22 @@ async fn run_test_client(
assert_eq!(hello, "hello from server");
}
+async fn run_test_client(
+ prx_addr: SocketAddr,
+ src_addr: SocketAddr,
+ proto: &str,
+ ver: http::Version,
+) {
+ rust_test_client_with_resolver(
+ Some(prx_addr),
+ src_addr.to_string(),
+ proto,
+ ver,
+ Default::default(),
+ )
+ .await
+}
+
async fn create_https_server(allow_h2: bool) -> SocketAddr {
let mut tls_config = deno_tls::rustls::server::ServerConfig::builder()
.with_no_client_auth()
generated by cgit v1.2.3 (git 2.25.1) at 2025-06-26 20:20:41 +0000