From 4f1b1903cfadeeba24e1b0448879fe12682effb9 Mon Sep 17 00:00:00 2001 From: Tomofumi Chiba Date: Tue, 22 Jun 2021 12:21:57 +0900 Subject: feat(fetch): add programmatic proxy (#10907) This commit adds new options to unstable "Deno.createHttpClient" API. "proxy" and "basicAuth" options were added that allow to use custom proxy when client instance is passed to "fetch" API. --- cli/dts/lib.deno.unstable.d.ts | 18 +++++++++++++- cli/tests/045_programmatic_proxy_client.ts | 16 ++++++++++++ cli/tests/045_proxy_test.ts | 24 ++++++++++++++++++ cli/tests/045_proxy_test.ts.out | 2 ++ extensions/fetch/lib.rs | 39 +++++++++++++++++++++++++++++- runtime/build.rs | 6 ++++- runtime/web_worker.rs | 1 + runtime/worker.rs | 1 + 8 files changed, 104 insertions(+), 3 deletions(-) create mode 100644 cli/tests/045_programmatic_proxy_client.ts diff --git a/cli/dts/lib.deno.unstable.d.ts b/cli/dts/lib.deno.unstable.d.ts index 828211c5b..1d01a748e 100644 --- a/cli/dts/lib.deno.unstable.d.ts +++ b/cli/dts/lib.deno.unstable.d.ts @@ -1089,6 +1089,17 @@ declare namespace Deno { /** A certificate authority to use when validating TLS certificates. Certificate data must be PEM encoded. */ caData?: string; + proxy?: Proxy; + } + + export interface Proxy { + url: string; + basicAuth?: BasicAuth; + } + + export interface BasicAuth { + username: string; + password: string; } /** **UNSTABLE**: New API, yet to be vetted. @@ -1096,7 +1107,12 @@ declare namespace Deno { * * ```ts * const client = Deno.createHttpClient({ caData: await Deno.readTextFile("./ca.pem") }); - * const req = await fetch("https://myserver.com", { client }); + * const response = await fetch("https://myserver.com", { client }); + * ``` + * + * ```ts + * const client = Deno.createHttpClient({ proxy: { url: "http://myproxy.com:8080" } }); + * const response = await fetch("https://myserver.com", { client }); * ``` */ export function createHttpClient( diff --git a/cli/tests/045_programmatic_proxy_client.ts b/cli/tests/045_programmatic_proxy_client.ts new file mode 100644 index 000000000..50884407d --- /dev/null +++ b/cli/tests/045_programmatic_proxy_client.ts @@ -0,0 +1,16 @@ +// Copyright 2018-2021 the Deno authors. All rights reserved. MIT license. + +const client = Deno.createHttpClient({ + proxy: { + url: "http://localhost:4555", + basicAuth: { username: "username", password: "password" }, + }, +}); + +const res = await fetch( + "http://localhost:4545/test_util/std/examples/colors.ts", + { client }, +); +console.log(`Response http: ${await res.text()}`); + +client.close(); diff --git a/cli/tests/045_proxy_test.ts b/cli/tests/045_proxy_test.ts index c7ba5e967..6e338f4fc 100644 --- a/cli/tests/045_proxy_test.ts +++ b/cli/tests/045_proxy_test.ts @@ -15,6 +15,11 @@ async function proxyServer(): Promise { async function proxyRequest(req: ServerRequest): Promise { console.log(`Proxy request to: ${req.url}`); + const proxyAuthorization = req.headers.get("proxy-authorization"); + if (proxyAuthorization) { + console.log(`proxy-authorization: ${proxyAuthorization}`); + req.headers.delete("proxy-authorization"); + } const resp = await fetch(req.url, { method: req.method, headers: req.headers, @@ -110,9 +115,28 @@ async function testModuleDownloadNoProxy(): Promise { http.close(); } +async function testFetchProgrammaticProxy(): Promise { + const c = Deno.run({ + cmd: [ + Deno.execPath(), + "run", + "--quiet", + "--reload", + "--allow-net=localhost:4545,localhost:4555", + "--unstable", + "045_programmatic_proxy_client.ts", + ], + stdout: "piped", + }); + const status = await c.status(); + assertEquals(status.code, 0); + c.close(); +} + proxyServer(); await testFetch(); await testModuleDownload(); await testFetchNoProxy(); await testModuleDownloadNoProxy(); +await testFetchProgrammaticProxy(); Deno.exit(0); diff --git a/cli/tests/045_proxy_test.ts.out b/cli/tests/045_proxy_test.ts.out index 4b07438ec..4957c9307 100644 --- a/cli/tests/045_proxy_test.ts.out +++ b/cli/tests/045_proxy_test.ts.out @@ -2,3 +2,5 @@ Proxy server listening on [WILDCARD] Proxy request to: http://localhost:4545/test_util/std/examples/colors.ts Proxy request to: http://localhost:4545/test_util/std/examples/colors.ts Proxy request to: http://localhost:4545/test_util/std/fmt/colors.ts +Proxy request to: http://localhost:4545/test_util/std/examples/colors.ts +proxy-authorization: Basic dXNlcm5hbWU6cGFzc3dvcmQ= diff --git a/extensions/fetch/lib.rs b/extensions/fetch/lib.rs index cdac0d64c..3652c8724 100644 --- a/extensions/fetch/lib.rs +++ b/extensions/fetch/lib.rs @@ -56,6 +56,7 @@ pub use reqwest; // Re-export reqwest pub fn init( user_agent: String, ca_data: Option>, + proxy: Option, ) -> Extension { Extension::builder() .js(include_js_files!( @@ -78,11 +79,13 @@ pub fn init( ]) .state(move |state| { state.put::({ - create_http_client(user_agent.clone(), ca_data.clone()).unwrap() + create_http_client(user_agent.clone(), ca_data.clone(), proxy.clone()) + .unwrap() }); state.put::(HttpClientDefaults { ca_data: ca_data.clone(), user_agent: user_agent.clone(), + proxy: proxy.clone(), }); Ok(()) }) @@ -92,6 +95,7 @@ pub fn init( pub struct HttpClientDefaults { pub user_agent: String, pub ca_data: Option>, + pub proxy: Option, } pub trait FetchPermissions { @@ -461,6 +465,22 @@ impl HttpClientResource { pub struct CreateHttpClientOptions { ca_file: Option, ca_data: Option, + proxy: Option, +} + +#[derive(Deserialize, Default, Debug, Clone)] +#[serde(rename_all = "camelCase")] +#[serde(default)] +pub struct Proxy { + pub url: String, + pub basic_auth: Option, +} + +#[derive(Deserialize, Default, Debug, Clone)] +#[serde(default)] +pub struct BasicAuth { + pub username: String, + pub password: String, } pub fn op_create_http_client( @@ -476,6 +496,12 @@ where permissions.check_read(&PathBuf::from(ca_file))?; } + if let Some(proxy) = args.proxy.clone() { + let permissions = state.borrow_mut::(); + let url = Url::parse(&proxy.url)?; + permissions.check_net_url(&url)?; + } + let defaults = state.borrow::(); let cert_data = @@ -483,6 +509,7 @@ where let client = create_http_client( defaults.user_agent.clone(), cert_data.or_else(|| defaults.ca_data.clone()), + args.proxy, ) .unwrap(); @@ -510,6 +537,7 @@ fn get_cert_data( pub fn create_http_client( user_agent: String, ca_data: Option>, + proxy: Option, ) -> Result { let mut headers = HeaderMap::new(); headers.insert(USER_AGENT, user_agent.parse().unwrap()); @@ -523,6 +551,15 @@ pub fn create_http_client( builder = builder.add_root_certificate(cert); } + if let Some(proxy) = proxy { + let mut reqwest_proxy = reqwest::Proxy::all(&proxy.url)?; + if let Some(basic_auth) = &proxy.basic_auth { + reqwest_proxy = + reqwest_proxy.basic_auth(&basic_auth.username, &basic_auth.password); + } + builder = builder.proxy(reqwest_proxy); + } + builder .build() .map_err(|e| generic_error(format!("Unable to build http client: {}", e))) diff --git a/runtime/build.rs b/runtime/build.rs index 84418af4e..7d086b045 100644 --- a/runtime/build.rs +++ b/runtime/build.rs @@ -42,7 +42,11 @@ fn create_runtime_snapshot(snapshot_path: &Path, files: Vec) { deno_console::init(), deno_url::init(), deno_web::init(Default::default(), Default::default()), - deno_fetch::init::("".to_owned(), None), + deno_fetch::init::( + "".to_owned(), + None, + None, + ), deno_websocket::init::( "".to_owned(), None, diff --git a/runtime/web_worker.rs b/runtime/web_worker.rs index 84d8157d6..753238052 100644 --- a/runtime/web_worker.rs +++ b/runtime/web_worker.rs @@ -257,6 +257,7 @@ impl WebWorker { deno_fetch::init::( options.user_agent.clone(), options.ca_data.clone(), + None, ), deno_websocket::init::( options.user_agent.clone(), diff --git a/runtime/worker.rs b/runtime/worker.rs index 7bfb1506b..9dfdcc825 100644 --- a/runtime/worker.rs +++ b/runtime/worker.rs @@ -99,6 +99,7 @@ impl MainWorker { deno_fetch::init::( options.user_agent.clone(), options.ca_data.clone(), + None, ), deno_websocket::init::( options.user_agent.clone(), -- cgit v1.2.3