summaryrefslogtreecommitdiff
path: root/cli/http_util.rs
diff options
context:
space:
mode:
authorRyan Dahl <ry@tinyclouds.org>2024-07-13 17:08:23 -0400
committerGitHub <noreply@github.com>2024-07-13 21:08:23 +0000
commite0cfc9da39e1d05e6a95c89c41cff8ae34fcbd66 (patch)
tree97e291e29e8e0e49796f3929e9bf5f42d0e5f76c /cli/http_util.rs
parentf6fd6619e708a515831f707438368d81b0c9aa56 (diff)
Revert "refactor(fetch): reimplement fetch with hyper instead of reqwest (#24237)" (#24574)
This reverts commit f6fd6619e708a515831f707438368d81b0c9aa56. I'm seeing a difference between canary and 1.45.2. In `deno-docs/reference_gen` I can't download dax when running `deno task types` ``` ~/src/deno-docs/reference_gen# deno upgrade --canary Looking up latest canary version Found latest version f6fd6619e708a515831f707438368d81b0c9aa56 Downloading https://dl.deno.land/canary/f6fd6619e708a515831f707438368d81b0c9aa56/deno-aarch64-apple-darwin.zip Deno is upgrading to version f6fd6619e708a515831f707438368d81b0c9aa56 Archive: /var/folders/9v/kys6gqns6kl8nksyn4l1f9v40000gn/T/.tmpb5lDnq/deno.zip inflating: deno Upgraded successfully ~/src/deno-docs/reference_gen# deno -v deno 1.45.2+f6fd661 ~/src/deno-docs/reference_gen# rm -rf /Users/ry/Library/Caches/deno ~/src/deno-docs/reference_gen# deno task types Task types deno task types:deno && deno task types:node Task types:deno deno run --allow-read --allow-write --allow-run --allow-env --allow-sys deno-docs.ts error: JSR package manifest for '@david/dax' failed to load. expected value at line 1 column 1 at file:///Users/ry/src/deno-docs/reference_gen/deno-docs.ts:2:15 ~/src/deno-docs/reference_gen# deno upgrade --version 1.45.2 Downloading https://github.com/denoland/deno/releases/download/v1.45.2/deno-aarch64-apple-darwin.zip Deno is upgrading to version 1.45.2 Archive: /var/folders/9v/kys6gqns6kl8nksyn4l1f9v40000gn/T/.tmp3R7uhF/deno.zip inflating: deno Upgraded successfully ~/src/deno-docs/reference_gen# rm -rf /Users/ry/Library/Caches/deno ~/src/deno-docs/reference_gen# deno task types Task types deno task types:deno && deno task types:node Task types:deno deno run --allow-read --allow-write --allow-run --allow-env --allow-sys deno-docs.ts Task types:node deno run --allow-read --allow-write=. --allow-env --allow-sys node-docs.ts ```
Diffstat (limited to 'cli/http_util.rs')
-rw-r--r--cli/http_util.rs240
1 files changed, 65 insertions, 175 deletions
diff --git a/cli/http_util.rs b/cli/http_util.rs
index a8646c188..18c0687bd 100644
--- a/cli/http_util.rs
+++ b/cli/http_util.rs
@@ -12,22 +12,18 @@ use deno_core::error::generic_error;
use deno_core::error::AnyError;
use deno_core::futures::StreamExt;
use deno_core::parking_lot::Mutex;
-use deno_core::serde;
-use deno_core::serde_json;
use deno_core::url::Url;
-use deno_runtime::deno_fetch;
use deno_runtime::deno_fetch::create_http_client;
+use deno_runtime::deno_fetch::reqwest;
+use deno_runtime::deno_fetch::reqwest::header::HeaderName;
+use deno_runtime::deno_fetch::reqwest::header::HeaderValue;
+use deno_runtime::deno_fetch::reqwest::header::ACCEPT;
+use deno_runtime::deno_fetch::reqwest::header::AUTHORIZATION;
+use deno_runtime::deno_fetch::reqwest::header::IF_NONE_MATCH;
+use deno_runtime::deno_fetch::reqwest::header::LOCATION;
+use deno_runtime::deno_fetch::reqwest::StatusCode;
use deno_runtime::deno_fetch::CreateHttpClientOptions;
use deno_runtime::deno_tls::RootCertStoreProvider;
-use http::header::HeaderName;
-use http::header::HeaderValue;
-use http::header::ACCEPT;
-use http::header::AUTHORIZATION;
-use http::header::IF_NONE_MATCH;
-use http::header::LOCATION;
-use http::StatusCode;
-use http_body_util::BodyExt;
-
use std::collections::HashMap;
use std::sync::Arc;
use std::thread::ThreadId;
@@ -212,7 +208,8 @@ pub struct HttpClientProvider {
// it's not safe to share a reqwest::Client across tokio runtimes,
// so we store these Clients keyed by thread id
// https://github.com/seanmonstar/reqwest/issues/1148#issuecomment-910868788
- clients_by_thread_id: Mutex<HashMap<ThreadId, deno_fetch::Client>>,
+ #[allow(clippy::disallowed_types)] // reqwest::Client allowed here
+ clients_by_thread_id: Mutex<HashMap<ThreadId, reqwest::Client>>,
}
impl std::fmt::Debug for HttpClientProvider {
@@ -273,15 +270,9 @@ pub struct BadResponseError {
#[derive(Debug, Error)]
pub enum DownloadError {
#[error(transparent)]
- Fetch(AnyError),
- #[error(transparent)]
- UrlParse(#[from] deno_core::url::ParseError),
- #[error(transparent)]
- HttpParse(#[from] http::Error),
- #[error(transparent)]
- Json(#[from] serde_json::Error),
+ Reqwest(#[from] reqwest::Error),
#[error(transparent)]
- ToStr(#[from] http::header::ToStrError),
+ ToStr(#[from] reqwest::header::ToStrError),
#[error("Redirection from '{}' did not provide location header", .request_url)]
NoRedirectHeader { request_url: Url },
#[error("Too many redirects.")]
@@ -292,7 +283,8 @@ pub enum DownloadError {
#[derive(Debug)]
pub struct HttpClient {
- client: deno_fetch::Client,
+ #[allow(clippy::disallowed_types)] // reqwest::Client allowed here
+ client: reqwest::Client,
// don't allow sending this across threads because then
// it might be shared accidentally across tokio runtimes
// which will cause issues
@@ -303,56 +295,22 @@ pub struct HttpClient {
impl HttpClient {
// DO NOT make this public. You should always be creating one of these from
// the HttpClientProvider
- fn new(client: deno_fetch::Client) -> Self {
+ #[allow(clippy::disallowed_types)] // reqwest::Client allowed here
+ fn new(client: reqwest::Client) -> Self {
Self {
client,
_unsend_marker: deno_core::unsync::UnsendMarker::default(),
}
}
- pub fn get(&self, url: Url) -> Result<RequestBuilder, http::Error> {
- let body = http_body_util::Empty::new()
- .map_err(|never| match never {})
- .boxed();
- let mut req = http::Request::new(body);
- *req.uri_mut() = url.as_str().parse()?;
- Ok(RequestBuilder {
- client: self.client.clone(),
- req,
- })
- }
-
- pub fn post(
- &self,
- url: Url,
- body: deno_fetch::ReqBody,
- ) -> Result<RequestBuilder, http::Error> {
- let mut req = http::Request::new(body);
- *req.method_mut() = http::Method::POST;
- *req.uri_mut() = url.as_str().parse()?;
- Ok(RequestBuilder {
- client: self.client.clone(),
- req,
- })
+ // todo(dsherret): don't expose `reqwest::RequestBuilder` because it
+ // is `Sync` and could accidentally be shared with multiple tokio runtimes
+ pub fn get(&self, url: impl reqwest::IntoUrl) -> reqwest::RequestBuilder {
+ self.client.get(url)
}
- pub fn post_json<S>(
- &self,
- url: Url,
- ser: &S,
- ) -> Result<RequestBuilder, DownloadError>
- where
- S: serde::Serialize,
- {
- let json = deno_core::serde_json::to_vec(ser)?;
- let body = http_body_util::Full::new(json.into())
- .map_err(|never| match never {})
- .boxed();
- let builder = self.post(url, body)?;
- Ok(builder.header(
- http::header::CONTENT_TYPE,
- "application/json".parse().map_err(http::Error::from)?,
- ))
+ pub fn post(&self, url: impl reqwest::IntoUrl) -> reqwest::RequestBuilder {
+ self.client.post(url)
}
/// Asynchronously fetches the given HTTP URL one pass only.
@@ -364,35 +322,27 @@ impl HttpClient {
&self,
args: FetchOnceArgs<'a>,
) -> Result<FetchOnceResult, AnyError> {
- let body = http_body_util::Empty::new()
- .map_err(|never| match never {})
- .boxed();
- let mut request = http::Request::new(body);
- *request.uri_mut() = args.url.as_str().parse()?;
+ let mut request = self.client.get(args.url.clone());
if let Some(etag) = args.maybe_etag {
let if_none_match_val = HeaderValue::from_str(&etag)?;
- request
- .headers_mut()
- .insert(IF_NONE_MATCH, if_none_match_val);
+ request = request.header(IF_NONE_MATCH, if_none_match_val);
}
if let Some(auth_token) = args.maybe_auth_token {
let authorization_val = HeaderValue::from_str(&auth_token.to_string())?;
- request
- .headers_mut()
- .insert(AUTHORIZATION, authorization_val);
+ request = request.header(AUTHORIZATION, authorization_val);
}
if let Some(accept) = args.maybe_accept {
let accepts_val = HeaderValue::from_str(&accept)?;
- request.headers_mut().insert(ACCEPT, accepts_val);
+ request = request.header(ACCEPT, accepts_val);
}
- let response = match self.client.clone().send(request).await {
+ let response = match request.send().await {
Ok(resp) => resp,
Err(err) => {
- if is_error_connect(&err) {
+ if err.is_connect() || err.is_timeout() {
return Ok(FetchOnceResult::RequestError(err.to_string()));
}
- return Err(err);
+ return Err(err.into());
}
};
@@ -456,12 +406,18 @@ impl HttpClient {
Ok(FetchOnceResult::Code(body, result_headers))
}
- pub async fn download_text(&self, url: Url) -> Result<String, AnyError> {
+ pub async fn download_text(
+ &self,
+ url: impl reqwest::IntoUrl,
+ ) -> Result<String, AnyError> {
let bytes = self.download(url).await?;
Ok(String::from_utf8(bytes)?)
}
- pub async fn download(&self, url: Url) -> Result<Vec<u8>, AnyError> {
+ pub async fn download(
+ &self,
+ url: impl reqwest::IntoUrl,
+ ) -> Result<Vec<u8>, AnyError> {
let maybe_bytes = self.download_inner(url, None, None).await?;
match maybe_bytes {
Some(bytes) => Ok(bytes),
@@ -471,7 +427,7 @@ impl HttpClient {
pub async fn download_with_progress(
&self,
- url: Url,
+ url: impl reqwest::IntoUrl,
maybe_header: Option<(HeaderName, HeaderValue)>,
progress_guard: &UpdateGuard,
) -> Result<Option<Vec<u8>>, DownloadError> {
@@ -482,26 +438,26 @@ impl HttpClient {
pub async fn get_redirected_url(
&self,
- url: Url,
+ url: impl reqwest::IntoUrl,
maybe_header: Option<(HeaderName, HeaderValue)>,
) -> Result<Url, AnyError> {
- let (_, url) = self.get_redirected_response(url, maybe_header).await?;
- Ok(url)
+ let response = self.get_redirected_response(url, maybe_header).await?;
+ Ok(response.url().clone())
}
async fn download_inner(
&self,
- url: Url,
+ url: impl reqwest::IntoUrl,
maybe_header: Option<(HeaderName, HeaderValue)>,
progress_guard: Option<&UpdateGuard>,
) -> Result<Option<Vec<u8>>, DownloadError> {
- let (response, _) = self.get_redirected_response(url, maybe_header).await?;
+ let response = self.get_redirected_response(url, maybe_header).await?;
if response.status() == 404 {
return Ok(None);
} else if !response.status().is_success() {
let status = response.status();
- let maybe_response_text = body_to_string(response).await.ok();
+ let maybe_response_text = response.text().await.ok();
return Err(DownloadError::BadResponse(BadResponseError {
status_code: status,
response_text: maybe_response_text
@@ -513,77 +469,60 @@ impl HttpClient {
get_response_body_with_progress(response, progress_guard)
.await
.map(Some)
- .map_err(DownloadError::Fetch)
+ .map_err(Into::into)
}
async fn get_redirected_response(
&self,
- mut url: Url,
+ url: impl reqwest::IntoUrl,
mut maybe_header: Option<(HeaderName, HeaderValue)>,
- ) -> Result<(http::Response<deno_fetch::ResBody>, Url), DownloadError> {
- let mut req = self.get(url.clone())?.build();
+ ) -> Result<reqwest::Response, DownloadError> {
+ let mut url = url.into_url()?;
+ let mut builder = self.get(url.clone());
if let Some((header_name, header_value)) = maybe_header.as_ref() {
- req.headers_mut().append(header_name, header_value.clone());
+ builder = builder.header(header_name, header_value);
}
- let mut response = self
- .client
- .clone()
- .send(req)
- .await
- .map_err(DownloadError::Fetch)?;
+ let mut response = builder.send().await?;
let status = response.status();
if status.is_redirection() {
for _ in 0..5 {
let new_url = resolve_redirect_from_response(&url, &response)?;
- let mut req = self.get(new_url.clone())?.build();
+ let mut builder = self.get(new_url.clone());
if new_url.origin() == url.origin() {
if let Some((header_name, header_value)) = maybe_header.as_ref() {
- req.headers_mut().append(header_name, header_value.clone());
+ builder = builder.header(header_name, header_value);
}
} else {
maybe_header = None;
}
- let new_response = self
- .client
- .clone()
- .send(req)
- .await
- .map_err(DownloadError::Fetch)?;
+ let new_response = builder.send().await?;
let status = new_response.status();
if status.is_redirection() {
response = new_response;
url = new_url;
} else {
- return Ok((new_response, new_url));
+ return Ok(new_response);
}
}
Err(DownloadError::TooManyRedirects)
} else {
- Ok((response, url))
+ Ok(response)
}
}
}
-fn is_error_connect(err: &AnyError) -> bool {
- err
- .downcast_ref::<hyper_util::client::legacy::Error>()
- .map(|err| err.is_connect())
- .unwrap_or(false)
-}
-
async fn get_response_body_with_progress(
- response: http::Response<deno_fetch::ResBody>,
+ response: reqwest::Response,
progress_guard: Option<&UpdateGuard>,
-) -> Result<Vec<u8>, AnyError> {
- use http_body::Body as _;
+) -> Result<Vec<u8>, reqwest::Error> {
if let Some(progress_guard) = progress_guard {
- if let Some(total_size) = response.body().size_hint().exact() {
+ if let Some(total_size) = response.content_length() {
progress_guard.set_total_size(total_size);
let mut current_size = 0;
let mut data = Vec::with_capacity(total_size as usize);
- let mut stream = response.into_body().into_data_stream();
+ let mut stream = response.bytes_stream();
while let Some(item) = stream.next().await {
let bytes = item?;
current_size += bytes.len() as u64;
@@ -593,7 +532,7 @@ async fn get_response_body_with_progress(
return Ok(data);
}
}
- let bytes = response.collect().await?.to_bytes();
+ let bytes = response.bytes().await?;
Ok(bytes.into())
}
@@ -624,9 +563,9 @@ fn resolve_url_from_location(base_url: &Url, location: &str) -> Url {
}
}
-fn resolve_redirect_from_response<B>(
+fn resolve_redirect_from_response(
request_url: &Url,
- response: &http::Response<B>,
+ response: &reqwest::Response,
) -> Result<Url, DownloadError> {
debug_assert!(response.status().is_redirection());
if let Some(location) = response.headers().get(LOCATION) {
@@ -641,49 +580,6 @@ fn resolve_redirect_from_response<B>(
}
}
-pub async fn body_to_string<B>(body: B) -> Result<String, AnyError>
-where
- B: http_body::Body,
- AnyError: From<B::Error>,
-{
- let bytes = body.collect().await?.to_bytes();
- let s = std::str::from_utf8(&bytes)?;
- Ok(s.into())
-}
-
-pub async fn body_to_json<B, D>(body: B) -> Result<D, AnyError>
-where
- B: http_body::Body,
- AnyError: From<B::Error>,
- D: serde::de::DeserializeOwned,
-{
- let bytes = body.collect().await?.to_bytes();
- let val = deno_core::serde_json::from_slice(&bytes)?;
- Ok(val)
-}
-
-pub struct RequestBuilder {
- client: deno_fetch::Client,
- req: http::Request<deno_fetch::ReqBody>,
-}
-
-impl RequestBuilder {
- pub fn header(mut self, name: HeaderName, value: HeaderValue) -> Self {
- self.req.headers_mut().append(name, value);
- self
- }
-
- pub async fn send(
- self,
- ) -> Result<http::Response<deno_fetch::ResBody>, AnyError> {
- self.client.send(self.req).await
- }
-
- pub fn build(self) -> http::Request<deno_fetch::ReqBody> {
- self.req
- }
-}
-
#[allow(clippy::print_stdout)]
#[allow(clippy::print_stderr)]
#[cfg(test)]
@@ -704,20 +600,14 @@ mod test {
// make a request to the redirect server
let text = client
- .download_text(
- Url::parse("http://localhost:4546/subdir/redirects/redirect1.js")
- .unwrap(),
- )
+ .download_text("http://localhost:4546/subdir/redirects/redirect1.js")
.await
.unwrap();
assert_eq!(text, "export const redirect = 1;\n");
// now make one to the infinite redirects server
let err = client
- .download_text(
- Url::parse("http://localhost:4549/subdir/redirects/redirect1.js")
- .unwrap(),
- )
+ .download_text("http://localhost:4549/subdir/redirects/redirect1.js")
.await
.err()
.unwrap();