From e92a05b5518e5fd30559c96c5990b08657bbc3e4 Mon Sep 17 00:00:00 2001 From: Nathan Whitaker <17734409+nathanwhit@users.noreply.github.com> Date: Wed, 14 Aug 2024 15:26:21 -0700 Subject: feat(serve): Opt-in parallelism for `deno serve` (#24920) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds a `parallel` flag to `deno serve`. When present, we spawn multiple workers to parallelize serving requests. ```bash deno serve --parallel main.ts ``` Currently on linux we use `SO_REUSEPORT` and rely on the fact that the kernel will distribute connections in a round-robin manner. On mac and windows, we sort of emulate this by cloning the underlying file descriptor and passing a handle to each worker. The connections will not be guaranteed to be fairly distributed (and in practice almost certainly won't be), but the distribution is still spread enough to provide a significant performance increase. --- (Run on an Macbook Pro with an M3 Max, serving `deno.com` baseline:: ``` ❯ wrk -d 30s -c 125 --latency http://127.0.0.1:8000 Running 30s test @ http://127.0.0.1:8000 2 threads and 125 connections Thread Stats Avg Stdev Max +/- Stdev Latency 239.78ms 13.56ms 330.54ms 79.12% Req/Sec 258.58 35.56 360.00 70.64% Latency Distribution 50% 236.72ms 75% 248.46ms 90% 256.84ms 99% 268.23ms 15458 requests in 30.02s, 2.47GB read Requests/sec: 514.89 Transfer/sec: 84.33MB ``` this PR (`with --parallel` flag) ``` ❯ wrk -d 30s -c 125 --latency http://127.0.0.1:8000 Running 30s test @ http://127.0.0.1:8000 2 threads and 125 connections Thread Stats Avg Stdev Max +/- Stdev Latency 117.40ms 142.84ms 590.45ms 79.07% Req/Sec 1.33k 175.19 1.77k 69.00% Latency Distribution 50% 22.34ms 75% 223.67ms 90% 357.32ms 99% 460.50ms 79636 requests in 30.07s, 12.74GB read Requests/sec: 2647.96 Transfer/sec: 433.71MB ``` --- ext/http/00_serve.ts | 26 ++++++++++++++++++-------- ext/net/01_net.js | 12 ++++++++---- ext/net/ops.rs | 7 ++++++- ext/net/ops_tls.rs | 8 +++++++- 4 files changed, 39 insertions(+), 14 deletions(-) (limited to 'ext') diff --git a/ext/http/00_serve.ts b/ext/http/00_serve.ts index 8ed1a1d04..9c6f80552 100644 --- a/ext/http/00_serve.ts +++ b/ext/http/00_serve.ts @@ -579,6 +579,8 @@ type RawServeOptions = { handler?: RawHandler; }; +const kLoadBalanced = Symbol("kLoadBalanced"); + function serve(arg1, arg2) { let options: RawServeOptions | undefined; let handler: RawHandler | undefined; @@ -634,6 +636,7 @@ function serve(arg1, arg2) { hostname: options.hostname ?? "0.0.0.0", port: options.port ?? 8000, reusePort: options.reusePort ?? false, + loadBalanced: options[kLoadBalanced] ?? false, }; if (options.certFile || options.keyFile) { @@ -842,18 +845,25 @@ function registerDeclarativeServer(exports) { "Invalid type for fetch: must be a function with a single or no parameter", ); } - return ({ servePort, serveHost }) => { + return ({ servePort, serveHost, serveIsMain, serveWorkerCount }) => { Deno.serve({ port: servePort, hostname: serveHost, + [kLoadBalanced]: (serveIsMain && serveWorkerCount > 1) || + (serveWorkerCount !== null), onListen: ({ port, hostname }) => { - console.debug( - `%cdeno serve%c: Listening on %chttp://${hostname}:${port}/%c`, - "color: green", - "color: inherit", - "color: yellow", - "color: inherit", - ); + if (serveIsMain) { + const nThreads = serveWorkerCount > 1 + ? ` with ${serveWorkerCount} threads` + : ""; + console.debug( + `%cdeno serve%c: Listening on %chttp://${hostname}:${port}/%c${nThreads}`, + "color: green", + "color: inherit", + "color: yellow", + "color: inherit", + ); + } }, handler: (req) => { return exports.fetch(req); diff --git a/ext/net/01_net.js b/ext/net/01_net.js index 517ab127e..536f79bbf 100644 --- a/ext/net/01_net.js +++ b/ext/net/01_net.js @@ -531,10 +531,14 @@ const listenOptionApiName = Symbol("listenOptionApiName"); function listen(args) { switch (args.transport ?? "tcp") { case "tcp": { - const { 0: rid, 1: addr } = op_net_listen_tcp({ - hostname: args.hostname ?? "0.0.0.0", - port: Number(args.port), - }, args.reusePort); + const { 0: rid, 1: addr } = op_net_listen_tcp( + { + hostname: args.hostname ?? "0.0.0.0", + port: Number(args.port), + }, + args.reusePort, + args.loadBalanced ?? false, + ); addr.transport = "tcp"; return new Listener(rid, addr); } diff --git a/ext/net/ops.rs b/ext/net/ops.rs index f28778d29..b74dc8d75 100644 --- a/ext/net/ops.rs +++ b/ext/net/ops.rs @@ -353,6 +353,7 @@ pub fn op_net_listen_tcp( state: &mut OpState, #[serde] addr: IpAddr, reuse_port: bool, + load_balanced: bool, ) -> Result<(ResourceId, IpAddr), AnyError> where NP: NetPermissions + 'static, @@ -367,7 +368,11 @@ where .next() .ok_or_else(|| generic_error("No resolved address found"))?; - let listener = TcpListener::bind_direct(addr, reuse_port)?; + let listener = if load_balanced { + TcpListener::bind_load_balanced(addr) + } else { + TcpListener::bind_direct(addr, reuse_port) + }?; let local_addr = listener.local_addr()?; let listener_resource = NetworkListenerResource::new(listener); let rid = state.resource_table.add(listener_resource); diff --git a/ext/net/ops_tls.rs b/ext/net/ops_tls.rs index a2a27c4ad..8483e7e66 100644 --- a/ext/net/ops_tls.rs +++ b/ext/net/ops_tls.rs @@ -475,6 +475,8 @@ fn load_private_keys_from_file( pub struct ListenTlsArgs { alpn_protocols: Option>, reuse_port: bool, + #[serde(default)] + load_balanced: bool, } #[op2] @@ -502,7 +504,11 @@ where .next() .ok_or_else(|| generic_error("No resolved address found"))?; - let tcp_listener = TcpListener::bind_direct(bind_addr, args.reuse_port)?; + let tcp_listener = if args.load_balanced { + TcpListener::bind_load_balanced(bind_addr) + } else { + TcpListener::bind_direct(bind_addr, args.reuse_port) + }?; let local_addr = tcp_listener.local_addr()?; let alpn = args .alpn_protocols -- cgit v1.2.3