refactor(ext/tls): Implement required functionality for later SNI support (#23686)

Precursor to #23236 

This implements the SNI features, but uses private symbols to avoid
exposing the functionality at this time. Note that to properly test this
feature, we need to add a way for `connectTls` to specify a hostname.
This is something that should be pushed into that API at a later time as
well.

```ts
Deno.test(
  { permissions: { net: true, read: true } },
  async function listenResolver() {
    let sniRequests = [];
    const listener = Deno.listenTls({
      hostname: "localhost",
      port: 0,
      [resolverSymbol]: (sni: string) => {
        sniRequests.push(sni);
        return {
          cert,
          key,
        };
      },
    });

    {
      const conn = await Deno.connectTls({
        hostname: "localhost",
        [serverNameSymbol]: "server-1",
        port: listener.addr.port,
      });
      const [_handshake, serverConn] = await Promise.all([
        conn.handshake(),
        listener.accept(),
      ]);
      conn.close();
      serverConn.close();
    }

    {
      const conn = await Deno.connectTls({
        hostname: "localhost",
        [serverNameSymbol]: "server-2",
        port: listener.addr.port,
      });
      const [_handshake, serverConn] = await Promise.all([
        conn.handshake(),
        listener.accept(),
      ]);
      conn.close();
      serverConn.close();
    }

    assertEquals(sniRequests, ["server-1", "server-2"]);
    listener.close();
  },
);
```

---------

Signed-off-by: Matt Mastracci <matthew@mastracci.com>

1 files changed, 60 insertions, 0 deletions

diff --git a/tests/unit/tls_sni_test.ts b/tests/unit/tls_sni_test.ts
new file mode 100644
index 000000000..404f8016e
--- /dev/null
+++ b/tests/unit/tls_sni_test.ts
@@ -0,0 +1,60 @@
+// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
+import { assertEquals, assertRejects } from "./test_util.ts";
+// @ts-expect-error TypeScript (as of 3.7) does not support indexing namespaces by symbol
+const { resolverSymbol, serverNameSymbol } = Deno[Deno.internal];
+
+const cert = Deno.readTextFileSync("tests/testdata/tls/localhost.crt");
+const key = Deno.readTextFileSync("tests/testdata/tls/localhost.key");
+const certEcc = Deno.readTextFileSync("tests/testdata/tls/localhost_ecc.crt");
+const keyEcc = Deno.readTextFileSync("tests/testdata/tls/localhost_ecc.key");
+
+Deno.test(
+  { permissions: { net: true, read: true } },
+  async function listenResolver() {
+    const sniRequests: string[] = [];
+    const keys: Record<string, { cert: string; key: string }> = {
+      "server-1": { cert, key },
+      "server-2": { cert: certEcc, key: keyEcc },
+      "fail-server-3": { cert: "(invalid)", key: "(bad)" },
+    };
+    const opts: unknown = {
+      hostname: "localhost",
+      port: 0,
+      [resolverSymbol]: (sni: string) => {
+        sniRequests.push(sni);
+        return keys[sni]!;
+      },
+    };
+    const listener = Deno.listenTls(
+      <Deno.ListenTlsOptions & Deno.TlsCertifiedKeyConnectTls> opts,
+    );
+
+    for (
+      const server of ["server-1", "server-2", "fail-server-3", "fail-server-4"]
+    ) {
+      const conn = await Deno.connectTls({
+        hostname: "localhost",
+        [serverNameSymbol]: server,
+        port: listener.addr.port,
+      });
+      const serverConn = await listener.accept();
+      if (server.startsWith("fail-")) {
+        await assertRejects(async () => await conn.handshake());
+        await assertRejects(async () => await serverConn.handshake());
+      } else {
+        await conn.handshake();
+        await serverConn.handshake();
+      }
+      conn.close();
+      serverConn.close();
+    }
+
+    assertEquals(sniRequests, [
+      "server-1",
+      "server-2",
+      "fail-server-3",
+      "fail-server-4",
+    ]);
+    listener.close();
+  },
+);