refactor(ext/tls): Implement required functionality for later SNI support (#23686)

Precursor to #23236 

This implements the SNI features, but uses private symbols to avoid
exposing the functionality at this time. Note that to properly test this
feature, we need to add a way for `connectTls` to specify a hostname.
This is something that should be pushed into that API at a later time as
well.

```ts
Deno.test(
  { permissions: { net: true, read: true } },
  async function listenResolver() {
    let sniRequests = [];
    const listener = Deno.listenTls({
      hostname: "localhost",
      port: 0,
      [resolverSymbol]: (sni: string) => {
        sniRequests.push(sni);
        return {
          cert,
          key,
        };
      },
    });

    {
      const conn = await Deno.connectTls({
        hostname: "localhost",
        [serverNameSymbol]: "server-1",
        port: listener.addr.port,
      });
      const [_handshake, serverConn] = await Promise.all([
        conn.handshake(),
        listener.accept(),
      ]);
      conn.close();
      serverConn.close();
    }

    {
      const conn = await Deno.connectTls({
        hostname: "localhost",
        [serverNameSymbol]: "server-2",
        port: listener.addr.port,
      });
      const [_handshake, serverConn] = await Promise.all([
        conn.handshake(),
        listener.accept(),
      ]);
      conn.close();
      serverConn.close();
    }

    assertEquals(sniRequests, ["server-1", "server-2"]);
    listener.close();
  },
);
```

---------

Signed-off-by: Matt Mastracci <matthew@mastracci.com>

1 files changed, 49 insertions, 1 deletions

diff --git a/ext/net/02_tls.js b/ext/net/02_tls.js
index 0b775047f..e51df7424 100644
--- a/ext/net/02_tls.js
+++ b/ext/net/02_tls.js
@@ -6,6 +6,10 @@ import {
   op_net_accept_tls,
   op_net_connect_tls,
   op_net_listen_tls,
+  op_tls_cert_resolver_create,
+  op_tls_cert_resolver_poll,
+  op_tls_cert_resolver_resolve,
+  op_tls_cert_resolver_resolve_error,
   op_tls_handshake,
   op_tls_key_null,
   op_tls_key_static,
@@ -16,6 +20,7 @@ const {
   Number,
   ObjectDefineProperty,
   TypeError,
+  SymbolFor,
 } = primordials;
 
 import { Conn, Listener } from "ext:deno_net/01_net.js";
@@ -87,9 +92,12 @@ async function connectTls({
     keyFile,
     privateKey,
   });
+  // TODO(mmastrac): We only expose this feature via symbol for now. This should actually be a feature
+  // in Deno.connectTls, however.
+  const serverName = arguments[0][serverNameSymbol] ?? null;
   const { 0: rid, 1: localAddr, 2: remoteAddr } = await op_net_connect_tls(
     { hostname, port },
-    { certFile: deprecatedCertFile, caCerts, alpnProtocols },
+    { certFile: deprecatedCertFile, caCerts, alpnProtocols, serverName },
     keyPair,
   );
   localAddr.transport = "tcp";
@@ -133,6 +141,10 @@ class TlsListener extends Listener {
  * interfaces.
  */
 function hasTlsKeyPairOptions(options) {
+  // TODO(mmastrac): remove this temporary symbol when the API lands
+  if (options[resolverSymbol] !== undefined) {
+    return true;
+  }
   return (options.cert !== undefined || options.key !== undefined ||
     options.certFile !== undefined ||
     options.keyFile !== undefined || options.privateKey !== undefined ||
@@ -159,6 +171,11 @@ function loadTlsKeyPair(api, {
     privateKey = undefined;
   }
 
+  // TODO(mmastrac): remove this temporary symbol when the API lands
+  if (arguments[1][resolverSymbol] !== undefined) {
+    return createTlsKeyResolver(arguments[1][resolverSymbol]);
+  }
+
   // Check for "pem" format
   if (keyFormat !== undefined && keyFormat !== "pem") {
     throw new TypeError('If `keyFormat` is specified, it must be "pem"');
@@ -275,6 +292,37 @@ async function startTls(
   return new TlsConn(rid, remoteAddr, localAddr);
 }
 
+const resolverSymbol = SymbolFor("unstableSniResolver");
+const serverNameSymbol = SymbolFor("unstableServerName");
+
+function createTlsKeyResolver(callback) {
+  const { 0: resolver, 1: lookup } = op_tls_cert_resolver_create();
+  (async () => {
+    while (true) {
+      const sni = await op_tls_cert_resolver_poll(lookup);
+      if (typeof sni !== "string") {
+        break;
+      }
+      try {
+        const key = await callback(sni);
+        if (!hasTlsKeyPairOptions(key)) {
+          op_tls_cert_resolver_resolve_error(lookup, sni, "Invalid key");
+        } else {
+          const resolved = loadTlsKeyPair("Deno.listenTls", key);
+          op_tls_cert_resolver_resolve(lookup, sni, resolved);
+        }
+      } catch (e) {
+        op_tls_cert_resolver_resolve_error(lookup, sni, e.message);
+      }
+    }
+  })();
+  return resolver;
+}
+
+internals.resolverSymbol = resolverSymbol;
+internals.serverNameSymbol = serverNameSymbol;
+internals.createTlsKeyResolver = createTlsKeyResolver;
+
 export {
   connectTls,
   hasTlsKeyPairOptions,