refactor(ext/tls): Implement required functionality for later SNI support (#23686)

Precursor to #23236 

This implements the SNI features, but uses private symbols to avoid
exposing the functionality at this time. Note that to properly test this
feature, we need to add a way for `connectTls` to specify a hostname.
This is something that should be pushed into that API at a later time as
well.

```ts
Deno.test(
  { permissions: { net: true, read: true } },
  async function listenResolver() {
    let sniRequests = [];
    const listener = Deno.listenTls({
      hostname: "localhost",
      port: 0,
      [resolverSymbol]: (sni: string) => {
        sniRequests.push(sni);
        return {
          cert,
          key,
        };
      },
    });

    {
      const conn = await Deno.connectTls({
        hostname: "localhost",
        [serverNameSymbol]: "server-1",
        port: listener.addr.port,
      });
      const [_handshake, serverConn] = await Promise.all([
        conn.handshake(),
        listener.accept(),
      ]);
      conn.close();
      serverConn.close();
    }

    {
      const conn = await Deno.connectTls({
        hostname: "localhost",
        [serverNameSymbol]: "server-2",
        port: listener.addr.port,
      });
      const [_handshake, serverConn] = await Promise.all([
        conn.handshake(),
        listener.accept(),
      ]);
      conn.close();
      serverConn.close();
    }

    assertEquals(sniRequests, ["server-1", "server-2"]);
    listener.close();
  },
);
```

---------

Signed-off-by: Matt Mastracci <matthew@mastracci.com>

3 files changed, 73 insertions, 5 deletions

diff --git a/tests/integration/js_unit_tests.rs b/tests/integration/js_unit_tests.rs
index 2bf78034e..cbae4a0b8 100644
--- a/tests/integration/js_unit_tests.rs
+++ b/tests/integration/js_unit_tests.rs
@@ -94,6 +94,7 @@ util::unit_test_factory!(
     text_encoding_test,
     timers_test,
     tls_test,
+    tls_sni_test,
     truncate_test,
     tty_color_test,
     tty_test,
@@ -129,7 +130,7 @@ fn js_unit_test(test: String) {
     .arg("--no-prompt");
 
   // TODO(mmastrac): it would be better to just load a test CA for all tests
-  let deno = if test == "websocket_test" {
+  let deno = if test == "websocket_test" || test == "tls_sni_test" {
     deno.arg("--unsafely-ignore-certificate-errors")
   } else {
     deno
diff --git a/tests/integration/run_tests.rs b/tests/integration/run_tests.rs
index 88ddfb318..8a24603b3 100644
--- a/tests/integration/run_tests.rs
+++ b/tests/integration/run_tests.rs
@@ -13,6 +13,7 @@ use deno_core::serde_json::json;
 use deno_core::url;
 use deno_fetch::reqwest;
 use deno_tls::rustls;
+use deno_tls::rustls::ClientConnection;
 use deno_tls::rustls_pemfile;
 use deno_tls::TlsStream;
 use pretty_assertions::assert_eq;
@@ -5388,8 +5389,11 @@ async fn listen_tls_alpn() {
   let tcp_stream = tokio::net::TcpStream::connect("localhost:4504")
     .await
     .unwrap();
-  let mut tls_stream =
-    TlsStream::new_client_side(tcp_stream, cfg, hostname, None);
+  let mut tls_stream = TlsStream::new_client_side(
+    tcp_stream,
+    ClientConnection::new(cfg, hostname).unwrap(),
+    None,
+  );
 
   let handshake = tls_stream.handshake().await.unwrap();
 
@@ -5437,8 +5441,11 @@ async fn listen_tls_alpn_fail() {
   let tcp_stream = tokio::net::TcpStream::connect("localhost:4505")
     .await
     .unwrap();
-  let mut tls_stream =
-    TlsStream::new_client_side(tcp_stream, cfg, hostname, None);
+  let mut tls_stream = TlsStream::new_client_side(
+    tcp_stream,
+    ClientConnection::new(cfg, hostname).unwrap(),
+    None,
+  );
 
   tls_stream.handshake().await.unwrap_err();
 
diff --git a/tests/unit/tls_sni_test.ts b/tests/unit/tls_sni_test.ts
new file mode 100644
index 000000000..404f8016e
--- /dev/null
+++ b/tests/unit/tls_sni_test.ts
@@ -0,0 +1,60 @@
+// Copyright 2018-2024 the Deno authors. All rights reserved. MIT license.
+import { assertEquals, assertRejects } from "./test_util.ts";
+// @ts-expect-error TypeScript (as of 3.7) does not support indexing namespaces by symbol
+const { resolverSymbol, serverNameSymbol } = Deno[Deno.internal];
+
+const cert = Deno.readTextFileSync("tests/testdata/tls/localhost.crt");
+const key = Deno.readTextFileSync("tests/testdata/tls/localhost.key");
+const certEcc = Deno.readTextFileSync("tests/testdata/tls/localhost_ecc.crt");
+const keyEcc = Deno.readTextFileSync("tests/testdata/tls/localhost_ecc.key");
+
+Deno.test(
+  { permissions: { net: true, read: true } },
+  async function listenResolver() {
+    const sniRequests: string[] = [];
+    const keys: Record<string, { cert: string; key: string }> = {
+      "server-1": { cert, key },
+      "server-2": { cert: certEcc, key: keyEcc },
+      "fail-server-3": { cert: "(invalid)", key: "(bad)" },
+    };
+    const opts: unknown = {
+      hostname: "localhost",
+      port: 0,
+      [resolverSymbol]: (sni: string) => {
+        sniRequests.push(sni);
+        return keys[sni]!;
+      },
+    };
+    const listener = Deno.listenTls(
+      <Deno.ListenTlsOptions & Deno.TlsCertifiedKeyConnectTls> opts,
+    );
+
+    for (
+      const server of ["server-1", "server-2", "fail-server-3", "fail-server-4"]
+    ) {
+      const conn = await Deno.connectTls({
+        hostname: "localhost",
+        [serverNameSymbol]: server,
+        port: listener.addr.port,
+      });
+      const serverConn = await listener.accept();
+      if (server.startsWith("fail-")) {
+        await assertRejects(async () => await conn.handshake());
+        await assertRejects(async () => await serverConn.handshake());
+      } else {
+        await conn.handshake();
+        await serverConn.handshake();
+      }
+      conn.close();
+      serverConn.close();
+    }
+
+    assertEquals(sniRequests, [
+      "server-1",
+      "server-2",
+      "fail-server-3",
+      "fail-server-4",
+    ]);
+    listener.close();
+  },
+);