diff options
author | Ryan Dahl <ry@tinyclouds.org> | 2024-07-13 17:08:23 -0400 |
---|---|---|
committer | GitHub <noreply@github.com> | 2024-07-13 21:08:23 +0000 |
commit | e0cfc9da39e1d05e6a95c89c41cff8ae34fcbd66 (patch) | |
tree | 97e291e29e8e0e49796f3929e9bf5f42d0e5f76c /cli/http_util.rs | |
parent | f6fd6619e708a515831f707438368d81b0c9aa56 (diff) |
Revert "refactor(fetch): reimplement fetch with hyper instead of reqwest (#24237)" (#24574)
This reverts commit f6fd6619e708a515831f707438368d81b0c9aa56.
I'm seeing a difference between canary and 1.45.2. In
`deno-docs/reference_gen` I can't download dax when running `deno task
types`
```
~/src/deno-docs/reference_gen# deno upgrade --canary
Looking up latest canary version
Found latest version f6fd6619e708a515831f707438368d81b0c9aa56
Downloading https://dl.deno.land/canary/f6fd6619e708a515831f707438368d81b0c9aa56/deno-aarch64-apple-darwin.zip
Deno is upgrading to version f6fd6619e708a515831f707438368d81b0c9aa56
Archive: /var/folders/9v/kys6gqns6kl8nksyn4l1f9v40000gn/T/.tmpb5lDnq/deno.zip
inflating: deno
Upgraded successfully
~/src/deno-docs/reference_gen# deno -v
deno 1.45.2+f6fd661
~/src/deno-docs/reference_gen# rm -rf /Users/ry/Library/Caches/deno
~/src/deno-docs/reference_gen# deno task types
Task types deno task types:deno && deno task types:node
Task types:deno deno run --allow-read --allow-write --allow-run --allow-env --allow-sys deno-docs.ts
error: JSR package manifest for '@david/dax' failed to load. expected value at line 1 column 1
at file:///Users/ry/src/deno-docs/reference_gen/deno-docs.ts:2:15
~/src/deno-docs/reference_gen# deno upgrade --version 1.45.2
Downloading https://github.com/denoland/deno/releases/download/v1.45.2/deno-aarch64-apple-darwin.zip
Deno is upgrading to version 1.45.2
Archive: /var/folders/9v/kys6gqns6kl8nksyn4l1f9v40000gn/T/.tmp3R7uhF/deno.zip
inflating: deno
Upgraded successfully
~/src/deno-docs/reference_gen# rm -rf /Users/ry/Library/Caches/deno
~/src/deno-docs/reference_gen# deno task types
Task types deno task types:deno && deno task types:node
Task types:deno deno run --allow-read --allow-write --allow-run --allow-env --allow-sys deno-docs.ts
Task types:node deno run --allow-read --allow-write=. --allow-env --allow-sys node-docs.ts
```
Diffstat (limited to 'cli/http_util.rs')
-rw-r--r-- | cli/http_util.rs | 240 |
1 files changed, 65 insertions, 175 deletions
diff --git a/cli/http_util.rs b/cli/http_util.rs index a8646c188..18c0687bd 100644 --- a/cli/http_util.rs +++ b/cli/http_util.rs @@ -12,22 +12,18 @@ use deno_core::error::generic_error; use deno_core::error::AnyError; use deno_core::futures::StreamExt; use deno_core::parking_lot::Mutex; -use deno_core::serde; -use deno_core::serde_json; use deno_core::url::Url; -use deno_runtime::deno_fetch; use deno_runtime::deno_fetch::create_http_client; +use deno_runtime::deno_fetch::reqwest; +use deno_runtime::deno_fetch::reqwest::header::HeaderName; +use deno_runtime::deno_fetch::reqwest::header::HeaderValue; +use deno_runtime::deno_fetch::reqwest::header::ACCEPT; +use deno_runtime::deno_fetch::reqwest::header::AUTHORIZATION; +use deno_runtime::deno_fetch::reqwest::header::IF_NONE_MATCH; +use deno_runtime::deno_fetch::reqwest::header::LOCATION; +use deno_runtime::deno_fetch::reqwest::StatusCode; use deno_runtime::deno_fetch::CreateHttpClientOptions; use deno_runtime::deno_tls::RootCertStoreProvider; -use http::header::HeaderName; -use http::header::HeaderValue; -use http::header::ACCEPT; -use http::header::AUTHORIZATION; -use http::header::IF_NONE_MATCH; -use http::header::LOCATION; -use http::StatusCode; -use http_body_util::BodyExt; - use std::collections::HashMap; use std::sync::Arc; use std::thread::ThreadId; @@ -212,7 +208,8 @@ pub struct HttpClientProvider { // it's not safe to share a reqwest::Client across tokio runtimes, // so we store these Clients keyed by thread id // https://github.com/seanmonstar/reqwest/issues/1148#issuecomment-910868788 - clients_by_thread_id: Mutex<HashMap<ThreadId, deno_fetch::Client>>, + #[allow(clippy::disallowed_types)] // reqwest::Client allowed here + clients_by_thread_id: Mutex<HashMap<ThreadId, reqwest::Client>>, } impl std::fmt::Debug for HttpClientProvider { @@ -273,15 +270,9 @@ pub struct BadResponseError { #[derive(Debug, Error)] pub enum DownloadError { #[error(transparent)] - Fetch(AnyError), - #[error(transparent)] - UrlParse(#[from] deno_core::url::ParseError), - #[error(transparent)] - HttpParse(#[from] http::Error), - #[error(transparent)] - Json(#[from] serde_json::Error), + Reqwest(#[from] reqwest::Error), #[error(transparent)] - ToStr(#[from] http::header::ToStrError), + ToStr(#[from] reqwest::header::ToStrError), #[error("Redirection from '{}' did not provide location header", .request_url)] NoRedirectHeader { request_url: Url }, #[error("Too many redirects.")] @@ -292,7 +283,8 @@ pub enum DownloadError { #[derive(Debug)] pub struct HttpClient { - client: deno_fetch::Client, + #[allow(clippy::disallowed_types)] // reqwest::Client allowed here + client: reqwest::Client, // don't allow sending this across threads because then // it might be shared accidentally across tokio runtimes // which will cause issues @@ -303,56 +295,22 @@ pub struct HttpClient { impl HttpClient { // DO NOT make this public. You should always be creating one of these from // the HttpClientProvider - fn new(client: deno_fetch::Client) -> Self { + #[allow(clippy::disallowed_types)] // reqwest::Client allowed here + fn new(client: reqwest::Client) -> Self { Self { client, _unsend_marker: deno_core::unsync::UnsendMarker::default(), } } - pub fn get(&self, url: Url) -> Result<RequestBuilder, http::Error> { - let body = http_body_util::Empty::new() - .map_err(|never| match never {}) - .boxed(); - let mut req = http::Request::new(body); - *req.uri_mut() = url.as_str().parse()?; - Ok(RequestBuilder { - client: self.client.clone(), - req, - }) - } - - pub fn post( - &self, - url: Url, - body: deno_fetch::ReqBody, - ) -> Result<RequestBuilder, http::Error> { - let mut req = http::Request::new(body); - *req.method_mut() = http::Method::POST; - *req.uri_mut() = url.as_str().parse()?; - Ok(RequestBuilder { - client: self.client.clone(), - req, - }) + // todo(dsherret): don't expose `reqwest::RequestBuilder` because it + // is `Sync` and could accidentally be shared with multiple tokio runtimes + pub fn get(&self, url: impl reqwest::IntoUrl) -> reqwest::RequestBuilder { + self.client.get(url) } - pub fn post_json<S>( - &self, - url: Url, - ser: &S, - ) -> Result<RequestBuilder, DownloadError> - where - S: serde::Serialize, - { - let json = deno_core::serde_json::to_vec(ser)?; - let body = http_body_util::Full::new(json.into()) - .map_err(|never| match never {}) - .boxed(); - let builder = self.post(url, body)?; - Ok(builder.header( - http::header::CONTENT_TYPE, - "application/json".parse().map_err(http::Error::from)?, - )) + pub fn post(&self, url: impl reqwest::IntoUrl) -> reqwest::RequestBuilder { + self.client.post(url) } /// Asynchronously fetches the given HTTP URL one pass only. @@ -364,35 +322,27 @@ impl HttpClient { &self, args: FetchOnceArgs<'a>, ) -> Result<FetchOnceResult, AnyError> { - let body = http_body_util::Empty::new() - .map_err(|never| match never {}) - .boxed(); - let mut request = http::Request::new(body); - *request.uri_mut() = args.url.as_str().parse()?; + let mut request = self.client.get(args.url.clone()); if let Some(etag) = args.maybe_etag { let if_none_match_val = HeaderValue::from_str(&etag)?; - request - .headers_mut() - .insert(IF_NONE_MATCH, if_none_match_val); + request = request.header(IF_NONE_MATCH, if_none_match_val); } if let Some(auth_token) = args.maybe_auth_token { let authorization_val = HeaderValue::from_str(&auth_token.to_string())?; - request - .headers_mut() - .insert(AUTHORIZATION, authorization_val); + request = request.header(AUTHORIZATION, authorization_val); } if let Some(accept) = args.maybe_accept { let accepts_val = HeaderValue::from_str(&accept)?; - request.headers_mut().insert(ACCEPT, accepts_val); + request = request.header(ACCEPT, accepts_val); } - let response = match self.client.clone().send(request).await { + let response = match request.send().await { Ok(resp) => resp, Err(err) => { - if is_error_connect(&err) { + if err.is_connect() || err.is_timeout() { return Ok(FetchOnceResult::RequestError(err.to_string())); } - return Err(err); + return Err(err.into()); } }; @@ -456,12 +406,18 @@ impl HttpClient { Ok(FetchOnceResult::Code(body, result_headers)) } - pub async fn download_text(&self, url: Url) -> Result<String, AnyError> { + pub async fn download_text( + &self, + url: impl reqwest::IntoUrl, + ) -> Result<String, AnyError> { let bytes = self.download(url).await?; Ok(String::from_utf8(bytes)?) } - pub async fn download(&self, url: Url) -> Result<Vec<u8>, AnyError> { + pub async fn download( + &self, + url: impl reqwest::IntoUrl, + ) -> Result<Vec<u8>, AnyError> { let maybe_bytes = self.download_inner(url, None, None).await?; match maybe_bytes { Some(bytes) => Ok(bytes), @@ -471,7 +427,7 @@ impl HttpClient { pub async fn download_with_progress( &self, - url: Url, + url: impl reqwest::IntoUrl, maybe_header: Option<(HeaderName, HeaderValue)>, progress_guard: &UpdateGuard, ) -> Result<Option<Vec<u8>>, DownloadError> { @@ -482,26 +438,26 @@ impl HttpClient { pub async fn get_redirected_url( &self, - url: Url, + url: impl reqwest::IntoUrl, maybe_header: Option<(HeaderName, HeaderValue)>, ) -> Result<Url, AnyError> { - let (_, url) = self.get_redirected_response(url, maybe_header).await?; - Ok(url) + let response = self.get_redirected_response(url, maybe_header).await?; + Ok(response.url().clone()) } async fn download_inner( &self, - url: Url, + url: impl reqwest::IntoUrl, maybe_header: Option<(HeaderName, HeaderValue)>, progress_guard: Option<&UpdateGuard>, ) -> Result<Option<Vec<u8>>, DownloadError> { - let (response, _) = self.get_redirected_response(url, maybe_header).await?; + let response = self.get_redirected_response(url, maybe_header).await?; if response.status() == 404 { return Ok(None); } else if !response.status().is_success() { let status = response.status(); - let maybe_response_text = body_to_string(response).await.ok(); + let maybe_response_text = response.text().await.ok(); return Err(DownloadError::BadResponse(BadResponseError { status_code: status, response_text: maybe_response_text @@ -513,77 +469,60 @@ impl HttpClient { get_response_body_with_progress(response, progress_guard) .await .map(Some) - .map_err(DownloadError::Fetch) + .map_err(Into::into) } async fn get_redirected_response( &self, - mut url: Url, + url: impl reqwest::IntoUrl, mut maybe_header: Option<(HeaderName, HeaderValue)>, - ) -> Result<(http::Response<deno_fetch::ResBody>, Url), DownloadError> { - let mut req = self.get(url.clone())?.build(); + ) -> Result<reqwest::Response, DownloadError> { + let mut url = url.into_url()?; + let mut builder = self.get(url.clone()); if let Some((header_name, header_value)) = maybe_header.as_ref() { - req.headers_mut().append(header_name, header_value.clone()); + builder = builder.header(header_name, header_value); } - let mut response = self - .client - .clone() - .send(req) - .await - .map_err(DownloadError::Fetch)?; + let mut response = builder.send().await?; let status = response.status(); if status.is_redirection() { for _ in 0..5 { let new_url = resolve_redirect_from_response(&url, &response)?; - let mut req = self.get(new_url.clone())?.build(); + let mut builder = self.get(new_url.clone()); if new_url.origin() == url.origin() { if let Some((header_name, header_value)) = maybe_header.as_ref() { - req.headers_mut().append(header_name, header_value.clone()); + builder = builder.header(header_name, header_value); } } else { maybe_header = None; } - let new_response = self - .client - .clone() - .send(req) - .await - .map_err(DownloadError::Fetch)?; + let new_response = builder.send().await?; let status = new_response.status(); if status.is_redirection() { response = new_response; url = new_url; } else { - return Ok((new_response, new_url)); + return Ok(new_response); } } Err(DownloadError::TooManyRedirects) } else { - Ok((response, url)) + Ok(response) } } } -fn is_error_connect(err: &AnyError) -> bool { - err - .downcast_ref::<hyper_util::client::legacy::Error>() - .map(|err| err.is_connect()) - .unwrap_or(false) -} - async fn get_response_body_with_progress( - response: http::Response<deno_fetch::ResBody>, + response: reqwest::Response, progress_guard: Option<&UpdateGuard>, -) -> Result<Vec<u8>, AnyError> { - use http_body::Body as _; +) -> Result<Vec<u8>, reqwest::Error> { if let Some(progress_guard) = progress_guard { - if let Some(total_size) = response.body().size_hint().exact() { + if let Some(total_size) = response.content_length() { progress_guard.set_total_size(total_size); let mut current_size = 0; let mut data = Vec::with_capacity(total_size as usize); - let mut stream = response.into_body().into_data_stream(); + let mut stream = response.bytes_stream(); while let Some(item) = stream.next().await { let bytes = item?; current_size += bytes.len() as u64; @@ -593,7 +532,7 @@ async fn get_response_body_with_progress( return Ok(data); } } - let bytes = response.collect().await?.to_bytes(); + let bytes = response.bytes().await?; Ok(bytes.into()) } @@ -624,9 +563,9 @@ fn resolve_url_from_location(base_url: &Url, location: &str) -> Url { } } -fn resolve_redirect_from_response<B>( +fn resolve_redirect_from_response( request_url: &Url, - response: &http::Response<B>, + response: &reqwest::Response, ) -> Result<Url, DownloadError> { debug_assert!(response.status().is_redirection()); if let Some(location) = response.headers().get(LOCATION) { @@ -641,49 +580,6 @@ fn resolve_redirect_from_response<B>( } } -pub async fn body_to_string<B>(body: B) -> Result<String, AnyError> -where - B: http_body::Body, - AnyError: From<B::Error>, -{ - let bytes = body.collect().await?.to_bytes(); - let s = std::str::from_utf8(&bytes)?; - Ok(s.into()) -} - -pub async fn body_to_json<B, D>(body: B) -> Result<D, AnyError> -where - B: http_body::Body, - AnyError: From<B::Error>, - D: serde::de::DeserializeOwned, -{ - let bytes = body.collect().await?.to_bytes(); - let val = deno_core::serde_json::from_slice(&bytes)?; - Ok(val) -} - -pub struct RequestBuilder { - client: deno_fetch::Client, - req: http::Request<deno_fetch::ReqBody>, -} - -impl RequestBuilder { - pub fn header(mut self, name: HeaderName, value: HeaderValue) -> Self { - self.req.headers_mut().append(name, value); - self - } - - pub async fn send( - self, - ) -> Result<http::Response<deno_fetch::ResBody>, AnyError> { - self.client.send(self.req).await - } - - pub fn build(self) -> http::Request<deno_fetch::ReqBody> { - self.req - } -} - #[allow(clippy::print_stdout)] #[allow(clippy::print_stderr)] #[cfg(test)] @@ -704,20 +600,14 @@ mod test { // make a request to the redirect server let text = client - .download_text( - Url::parse("http://localhost:4546/subdir/redirects/redirect1.js") - .unwrap(), - ) + .download_text("http://localhost:4546/subdir/redirects/redirect1.js") .await .unwrap(); assert_eq!(text, "export const redirect = 1;\n"); // now make one to the infinite redirects server let err = client - .download_text( - Url::parse("http://localhost:4549/subdir/redirects/redirect1.js") - .unwrap(), - ) + .download_text("http://localhost:4549/subdir/redirects/redirect1.js") .await .err() .unwrap(); |