diff options
Diffstat (limited to 'cli/http_util.rs')
| -rw-r--r-- | cli/http_util.rs | 303 |
1 files changed, 116 insertions, 187 deletions
diff --git a/cli/http_util.rs b/cli/http_util.rs index 511c67df0..ec035d6bf 100644 --- a/cli/http_util.rs +++ b/cli/http_util.rs @@ -2,76 +2,56 @@ use crate::deno_error; use crate::deno_error::DenoError; use deno::ErrBox; -#[cfg(test)] -use futures::future::{loop_fn, Loop}; -use futures::{future, Future, Stream}; -use hyper; -use hyper::client::{Client, HttpConnector}; -use hyper::header::CONTENT_TYPE; -use hyper::Uri; -use hyper_rustls; - -type Connector = hyper_rustls::HttpsConnector<HttpConnector>; - -lazy_static! { - static ref CONNECTOR: Connector = { - let num_dns_threads = 4; - Connector::new(num_dns_threads) - }; -} - -pub fn get_client() -> Client<Connector, hyper::Body> { - // TODO use Hyper's connection pool. - let c = CONNECTOR.clone(); - Client::builder().build(c) +use futures::{future, Future}; +use reqwest; +use reqwest::header::CONTENT_TYPE; +use reqwest::header::LOCATION; +use reqwest::r#async::Client; +use reqwest::RedirectPolicy; +use url::Url; + +/// Create new instance of async reqwest::Client. This client supports +/// proxies and doesn't follow redirects. +pub fn get_client() -> Client { + Client::builder() + .redirect(RedirectPolicy::none()) + .use_sys_proxy() + .build() + .unwrap() } /// Construct the next uri based on base uri and location header fragment /// See <https://tools.ietf.org/html/rfc3986#section-4.2> -fn resolve_uri_from_location(base_uri: &Uri, location: &str) -> Uri { +fn resolve_url_from_location(base_url: &Url, location: &str) -> Url { if location.starts_with("http://") || location.starts_with("https://") { // absolute uri - location - .parse::<Uri>() - .expect("provided redirect url should be a valid url") + Url::parse(location).expect("provided redirect url should be a valid url") } else if location.starts_with("//") { // "//" authority path-abempty - format!("{}:{}", base_uri.scheme_part().unwrap().as_str(), location) - .parse::<Uri>() + Url::parse(&format!("{}:{}", base_url.scheme(), location)) .expect("provided redirect url should be a valid url") } else if location.starts_with('/') { // path-absolute - let mut new_uri_parts = base_uri.clone().into_parts(); - new_uri_parts.path_and_query = Some(location.parse().unwrap()); - Uri::from_parts(new_uri_parts).unwrap() + base_url + .join(location) + .expect("provided redirect url should be a valid url") } else { // assuming path-noscheme | path-empty - let mut new_uri_parts = base_uri.clone().into_parts(); - let base_uri_path_str = base_uri.path().to_owned(); - let segs: Vec<&str> = base_uri_path_str.rsplitn(2, '/').collect(); - new_uri_parts.path_and_query = Some( - format!("{}/{}", segs.last().unwrap_or(&""), location) - .parse() - .unwrap(), - ); - Uri::from_parts(new_uri_parts).unwrap() + let base_url_path_str = base_url.path().to_owned(); + // Pop last part or url (after last slash) + let segs: Vec<&str> = base_url_path_str.rsplitn(2, '/').collect(); + let new_path = format!("{}/{}", segs.last().unwrap_or(&""), location); + base_url + .join(&new_path) + .expect("provided redirect url should be a valid url") } } -#[cfg(test)] -use crate::tokio_util; -#[cfg(test)] -/// Synchronously fetchs the given HTTP URL. Returns (content, media_type). -pub fn fetch_sync_string( - module_name: &str, -) -> Result<(String, String), ErrBox> { - tokio_util::block_on(fetch_string(module_name)) -} - +#[derive(Debug, PartialEq)] pub enum FetchOnceResult { // (code, maybe_content_type) Code(String, Option<String>), - Redirect(http::uri::Uri), + Redirect(Url), } /// Asynchronously fetchs the given HTTP URL one pass only. @@ -80,51 +60,57 @@ pub enum FetchOnceResult { /// If redirect occurs, does not follow and /// yields Redirect(url). pub fn fetch_string_once( - url: http::uri::Uri, + url: &Url, ) -> impl Future<Item = FetchOnceResult, Error = ErrBox> { type FetchAttempt = (Option<String>, Option<String>, Option<FetchOnceResult>); + + let url = url.clone(); let client = get_client(); + client .get(url.clone()) + .send() .map_err(ErrBox::from) .and_then( - move |response| -> Box< + move |mut response| -> Box< dyn Future<Item = FetchAttempt, Error = ErrBox> + Send, > { if response.status().is_redirection() { - let location_string = response - .headers() - .get("location") + let location_string = response.headers() + .get(LOCATION) .expect("url redirection should provide 'location' header") .to_str() - .unwrap() - .to_string(); - debug!("Redirecting to {}...", &location_string); - let new_url = resolve_uri_from_location(&url, &location_string); + .unwrap(); + + debug!("Redirecting to {:?}...", &location_string); + let new_url = resolve_url_from_location(&url, location_string); // Boxed trait object turns out to be the savior for 2+ types yielding same results. return Box::new(future::ok(None).join3( future::ok(None), future::ok(Some(FetchOnceResult::Redirect(new_url))), )); - } else if response.status().is_client_error() - || response.status().is_server_error() - { + } + + if response.status().is_client_error() || response.status().is_server_error() { return Box::new(future::err(DenoError::new( deno_error::ErrorKind::Other, format!("Import '{}' failed: {}", &url, response.status()), ).into())); } + let content_type = response .headers() .get(CONTENT_TYPE) .map(|content_type| content_type.to_str().unwrap().to_owned()); + let body = response - .into_body() - .concat2() - .map(|body| String::from_utf8(body.to_vec()).ok()) + .text() .map_err(ErrBox::from); - Box::new(body.join3(future::ok(content_type), future::ok(None))) - }, + + Box::new( + Some(body).join3(future::ok(content_type), future::ok(None)) + ) + } ) .and_then(move |(maybe_code, maybe_content_type, maybe_redirect)| { if let Some(redirect) = maybe_redirect { @@ -140,131 +126,74 @@ pub fn fetch_string_once( } #[cfg(test)] -/// Asynchronously fetchs the given HTTP URL. Returns (content, media_type). -pub fn fetch_string( - module_name: &str, -) -> impl Future<Item = (String, String), Error = ErrBox> { - let url = module_name.parse::<Uri>().unwrap(); - let client = get_client(); - // TODO(kevinkassimo): consider set a max redirection counter - // to avoid bouncing between 2 or more urls - loop_fn((client, url), |(client, url)| { - client - .get(url.clone()) - .map_err(ErrBox::from) - .and_then(move |response| { - if response.status().is_redirection() { - let location_string = response - .headers() - .get("location") - .expect("url redirection should provide 'location' header") - .to_str() - .unwrap() - .to_string(); - debug!("Redirecting to {}...", &location_string); - let new_url = resolve_uri_from_location(&url, &location_string); - return Ok(Loop::Continue((client, new_url))); - } - if !response.status().is_success() { - return Err( - DenoError::new( - deno_error::ErrorKind::NotFound, - "module not found".to_string(), - ) - .into(), - ); - } - Ok(Loop::Break(response)) - }) - }) - .and_then(|response| { - let content_type = response - .headers() - .get(CONTENT_TYPE) - .map(|content_type| content_type.to_str().unwrap().to_string()); - let body = response - .into_body() - .concat2() - .map(|body| String::from_utf8(body.to_vec()).unwrap()) - .map_err(ErrBox::from); - body.join(future::ok(content_type)) - }) - .and_then(|(body_string, maybe_content_type)| { - future::ok((body_string, maybe_content_type.unwrap())) - }) -} +mod tests { + use super::*; + use crate::tokio_util; -#[test] -fn test_fetch_sync_string() { - // Relies on external http server. See tools/http_server.py - tokio_util::init(|| { - let (p, m) = - fetch_sync_string("http://127.0.0.1:4545/package.json").unwrap(); - println!("package.json len {}", p.len()); - assert!(p.len() > 1); - assert!(m == "application/json") - }); -} + pub fn fetch_string_once_sync(url: &Url) -> Result<FetchOnceResult, ErrBox> { + tokio_util::block_on(fetch_string_once(url)) + } -#[test] -fn test_fetch_string() { - // Relies on external http server. See tools/http_server.py - tokio_util::init(|| { - let (p, m) = fetch_string("http://127.0.0.1:4545/package.json") - .wait() - .unwrap(); - println!("package.json len {}", p.len()); - assert!(p.len() > 1); - assert!(m == "application/json") - }); -} + #[test] + fn test_fetch_sync_string() { + // Relies on external http server. See tools/http_server.py + let url = Url::parse("http://127.0.0.1:4545/package.json").unwrap(); + tokio_util::init(|| match fetch_string_once_sync(&url).unwrap() { + FetchOnceResult::Code(code, maybe_content_type) => { + assert!(!code.is_empty()); + assert_eq!(maybe_content_type, Some("application/json".to_string())); + } + _ => unreachable!(), + }); + } -#[test] -fn test_fetch_sync_string_with_redirect() { - // Relies on external http server. See tools/http_server.py - tokio_util::init(|| { - let (p, m) = - fetch_sync_string("http://127.0.0.1:4546/package.json").unwrap(); - println!("package.json len {}", p.len()); - assert!(p.len() > 1); - assert!(m == "application/json") - }); -} + #[test] + fn test_fetch_string_once_with_redirect() { + // Relies on external http server. See tools/http_server.py + let url = Url::parse("http://127.0.0.1:4546/package.json").unwrap(); + // Dns resolver substitutes `127.0.0.1` with `localhost` + let target_url = Url::parse("http://localhost:4545/package.json").unwrap(); + tokio_util::init(|| { + let result = fetch_string_once_sync(&url).unwrap(); + assert_eq!(result, FetchOnceResult::Redirect(target_url)); + }); + } -#[test] -fn test_resolve_uri_from_location_full_1() { - let url = "http://deno.land".parse::<Uri>().unwrap(); - let new_uri = resolve_uri_from_location(&url, "http://golang.org"); - assert_eq!(new_uri.host().unwrap(), "golang.org"); -} + #[test] + fn test_resolve_url_from_location_full_1() { + let url = "http://deno.land".parse::<Url>().unwrap(); + let new_uri = resolve_url_from_location(&url, "http://golang.org"); + assert_eq!(new_uri.host_str().unwrap(), "golang.org"); + } -#[test] -fn test_resolve_uri_from_location_full_2() { - let url = "https://deno.land".parse::<Uri>().unwrap(); - let new_uri = resolve_uri_from_location(&url, "https://golang.org"); - assert_eq!(new_uri.host().unwrap(), "golang.org"); -} + #[test] + fn test_resolve_url_from_location_full_2() { + let url = "https://deno.land".parse::<Url>().unwrap(); + let new_uri = resolve_url_from_location(&url, "https://golang.org"); + assert_eq!(new_uri.host_str().unwrap(), "golang.org"); + } -#[test] -fn test_resolve_uri_from_location_relative_1() { - let url = "http://deno.land/x".parse::<Uri>().unwrap(); - let new_uri = resolve_uri_from_location(&url, "//rust-lang.org/en-US"); - assert_eq!(new_uri.host().unwrap(), "rust-lang.org"); - assert_eq!(new_uri.path(), "/en-US"); -} + #[test] + fn test_resolve_url_from_location_relative_1() { + let url = "http://deno.land/x".parse::<Url>().unwrap(); + let new_uri = resolve_url_from_location(&url, "//rust-lang.org/en-US"); + assert_eq!(new_uri.host_str().unwrap(), "rust-lang.org"); + assert_eq!(new_uri.path(), "/en-US"); + } -#[test] -fn test_resolve_uri_from_location_relative_2() { - let url = "http://deno.land/x".parse::<Uri>().unwrap(); - let new_uri = resolve_uri_from_location(&url, "/y"); - assert_eq!(new_uri.host().unwrap(), "deno.land"); - assert_eq!(new_uri.path(), "/y"); -} + #[test] + fn test_resolve_url_from_location_relative_2() { + let url = "http://deno.land/x".parse::<Url>().unwrap(); + let new_uri = resolve_url_from_location(&url, "/y"); + assert_eq!(new_uri.host_str().unwrap(), "deno.land"); + assert_eq!(new_uri.path(), "/y"); + } -#[test] -fn test_resolve_uri_from_location_relative_3() { - let url = "http://deno.land/x".parse::<Uri>().unwrap(); - let new_uri = resolve_uri_from_location(&url, "z"); - assert_eq!(new_uri.host().unwrap(), "deno.land"); - assert_eq!(new_uri.path(), "/z"); + #[test] + fn test_resolve_url_from_location_relative_3() { + let url = "http://deno.land/x".parse::<Url>().unwrap(); + let new_uri = resolve_url_from_location(&url, "z"); + assert_eq!(new_uri.host_str().unwrap(), "deno.land"); + assert_eq!(new_uri.path(), "/z"); + } } |