refactor(ext/net): extract TLS key and certificate from interfaces (#23296)

Removes the certificate options from all the interfaces and replaces
them with a new `TlsCertifiedKeyOptions`. This allows us to centralize
the documentation for TLS key management for both client and server, and
will allow us to add key object support in the future.

Also adds an option `keyFormat` field to the cert/key that must be
omitted or set to `pem`. This will allow us to load other format keys in
the future `der`, `pfx`, etc.

In a future PR, we will add a way to load a certified key object, and we
will add another option to `TlsCertifiedKeyOptions` like so:

```ts
export interface TlsCertifiedKeyOptions =
    | TlsCertifiedKeyPem
    | TlsCertifiedKeyFromFile
    | TlsCertifiedKeyConnectTls
    | { key: Deno.CertifiedKey }
```

1 files changed, 110 insertions, 66 deletions

diff --git a/ext/net/02_tls.js b/ext/net/02_tls.js
index 8d43e8604..003c5336a 100644
--- a/ext/net/02_tls.js
+++ b/ext/net/02_tls.js
@@ -51,54 +51,46 @@ async function connectTls({
   port,
   hostname = "127.0.0.1",
   transport = "tcp",
-  certFile = undefined,
   caCerts = [],
-  certChain = undefined,
-  privateKey = undefined,
+  alpnProtocols = undefined,
+  keyFormat = undefined,
   cert = undefined,
+  certFile = undefined,
+  certChain = undefined,
   key = undefined,
-  alpnProtocols = undefined,
+  keyFile = undefined,
+  privateKey = undefined,
 }) {
-  if (certFile !== undefined) {
-    internals.warnOnDeprecatedApi(
-      "Deno.ConnectTlsOptions.certFile",
-      new Error().stack,
-      "Pass the cert file contents to the `Deno.ConnectTlsOptions.cert` option instead.",
-    );
-  }
-  if (certChain !== undefined) {
-    internals.warnOnDeprecatedApi(
-      "Deno.ConnectTlsOptions.certChain",
-      new Error().stack,
-      "Use the `Deno.ConnectTlsOptions.cert` option instead.",
-    );
-  }
-  if (privateKey !== undefined) {
-    internals.warnOnDeprecatedApi(
-      "Deno.ConnectTlsOptions.privateKey",
-      new Error().stack,
-      "Use the `Deno.ConnectTlsOptions.key` option instead.",
-    );
-  }
   if (transport !== "tcp") {
     throw new TypeError(`Unsupported transport: '${transport}'`);
   }
-  if (certChain !== undefined && cert !== undefined) {
-    throw new TypeError(
-      "Cannot specify both `certChain` and `cert`",
-    );
-  }
-  if (privateKey !== undefined && key !== undefined) {
-    throw new TypeError(
-      "Cannot specify both `privateKey` and `key`",
+  let deprecatedCertFile = undefined;
+
+  // Deno.connectTls has an irregular option where you can just pass `certFile` and
+  // not `keyFile`. In this case it's used for `caCerts` rather than the client key.
+  if (certFile !== undefined && keyFile === undefined) {
+    internals.warnOnDeprecatedApi(
+      "Deno.ConnectTlsOptions.certFile",
+      new Error().stack,
+      "Pass the cert file's contents to the `Deno.ConnectTlsOptions.caCerts` option instead.",
     );
+
+    deprecatedCertFile = certFile;
+    certFile = undefined;
   }
-  cert ??= certChain;
-  key ??= privateKey;
-  const keyPair = loadTlsKeyPair(cert, undefined, key, undefined);
+
+  const keyPair = loadTlsKeyPair("Deno.connectTls", {
+    keyFormat,
+    cert,
+    certFile,
+    certChain,
+    key,
+    keyFile,
+    privateKey,
+  });
   const { 0: rid, 1: localAddr, 2: remoteAddr } = await op_net_connect_tls(
     { hostname, port },
-    { certFile, caCerts, cert, key, alpnProtocols },
+    { certFile: deprecatedCertFile, caCerts, alpnProtocols },
     keyPair,
   );
   localAddr.transport = "tcp";
@@ -137,29 +129,96 @@ class TlsListener extends Listener {
   }
 }
 
+/**
+ * Returns true if this object has the shape of one of the certified key material
+ * interfaces.
+ */
 function hasTlsKeyPairOptions(options) {
   return (ReflectHas(options, "cert") || ReflectHas(options, "key") ||
     ReflectHas(options, "certFile") ||
-    ReflectHas(options, "keyFile"));
+    ReflectHas(options, "keyFile") || ReflectHas(options, "privateKey") ||
+    ReflectHas(options, "certChain"));
 }
 
-function loadTlsKeyPair(
+/**
+ * Loads a TLS keypair from one of the various options. If no key material is provided,
+ * returns a special Null keypair.
+ */
+function loadTlsKeyPair(api, {
+  keyFormat,
   cert,
   certFile,
+  certChain,
   key,
   keyFile,
-) {
-  if ((certFile !== undefined) ^ (keyFile !== undefined)) {
-    throw new TypeError(
-      "If certFile is specified, keyFile must also be specified",
-    );
+  privateKey,
+}) {
+  // Check for "pem" format
+  if (keyFormat !== undefined && keyFormat !== "pem") {
+    throw new TypeError('If `keyFormat` is specified, it must be "pem"');
+  }
+
+  function exclusive(a1, a1v, a2, a2v) {
+    if (a1v !== undefined && a2v !== undefined) {
+      throw new TypeError(
+        `Cannot specify both \`${a1}\` and \`${a2}\` for \`${api}\`.`,
+      );
+    }
   }
-  if ((cert !== undefined) ^ (key !== undefined)) {
-    throw new TypeError("If cert is specified, key must also be specified");
+
+  // Ensure that only one pair is valid
+  exclusive("certChain", certChain, "cert", cert);
+  exclusive("certChain", certChain, "certFile", certFile);
+  exclusive("key", key, "keyFile", keyFile);
+  exclusive("key", key, "privateKey", privateKey);
+
+  function both(a1, a1v, a2, a2v) {
+    if (a1v !== undefined && a2v === undefined) {
+      throw new TypeError(
+        `If \`${a1}\` is specified, \`${a2}\` must be specified as well for \`${api}\`.`,
+      );
+    }
+    if (a1v === undefined && a2v !== undefined) {
+      throw new TypeError(
+        `If \`${a2}\` is specified, \`${a1}\` must be specified as well for \`${api}\`.`,
+      );
+    }
   }
 
+  // Pick one pair of cert/key, certFile/keyFile or certChain/privateKey
+  both("cert", cert, "key", key);
+  both("certFile", certFile, "keyFile", keyFile);
+  both("certChain", certChain, "privateKey", privateKey);
+
   if (certFile !== undefined) {
-    return op_tls_key_static_from_file("Deno.listenTls", certFile, keyFile);
+    internals.warnOnDeprecatedApi(
+      "Deno.TlsCertifiedKeyOptions.keyFile",
+      new Error().stack,
+      "Pass the key file's contents to the `Deno.TlsCertifiedKeyPem.key` option instead.",
+    );
+    internals.warnOnDeprecatedApi(
+      "Deno.TlsCertifiedKeyOptions.certFile",
+      new Error().stack,
+      "Pass the cert file's contents to the `Deno.TlsCertifiedKeyPem.cert` option instead.",
+    );
+    return op_tls_key_static_from_file(api, certFile, keyFile);
+  } else if (certChain !== undefined) {
+    if (api !== "Deno.connectTls") {
+      throw new TypeError(
+        `Invalid options 'certChain' and 'privateKey' for ${api}`,
+      );
+    }
+    internals.warnOnDeprecatedApi(
+      "Deno.TlsCertifiedKeyOptions.privateKey",
+      new Error().stack,
+      "Use the `Deno.TlsCertifiedKeyPem.key` option instead.",
+    );
+    internals.warnOnDeprecatedApi(
+      "Deno.TlsCertifiedKeyOptions.certChain",
+      new Error().stack,
+      "Use the `Deno.TlsCertifiedKeyPem.cert` option instead.",
+    );
+    return op_tls_key_static(certChain, privateKey);
   } else if (cert !== undefined) {
     return op_tls_key_static(cert, key);
   } else {
@@ -169,10 +228,6 @@ function loadTlsKeyPair(
 
 function listenTls({
   port,
-  cert,
-  certFile,
-  key,
-  keyFile,
   hostname = "0.0.0.0",
   transport = "tcp",
   alpnProtocols = undefined,
@@ -181,22 +236,11 @@ function listenTls({
   if (transport !== "tcp") {
     throw new TypeError(`Unsupported transport: '${transport}'`);
   }
-  if (keyFile !== undefined) {
-    internals.warnOnDeprecatedApi(
-      "Deno.ListenTlsOptions.keyFile",
-      new Error().stack,
-      "Pass the key file contents to the `Deno.ListenTlsOptions.key` option instead.",
-    );
-  }
-  if (certFile !== undefined) {
-    internals.warnOnDeprecatedApi(
-      "Deno.ListenTlsOptions.certFile",
-      new Error().stack,
-      "Pass the cert file contents to the `Deno.ListenTlsOptions.cert` option instead.",
-    );
-  }
 
-  const keyPair = loadTlsKeyPair(cert, certFile, key, keyFile);
+  if (!hasTlsKeyPairOptions(arguments[0])) {
+    throw new TypeError("A key and certificate are required for `listenTls`");
+  }
+  const keyPair = loadTlsKeyPair("Deno.listenTls", arguments[0]);
   const { 0: rid, 1: localAddr } = op_net_listen_tls(
     { hostname, port: Number(port) },
     { alpnProtocols, reusePort },