fix(ext/node): sign with PEM private keys (#21287)

Add support for signing with a RSA PEM private key: `pkcs8` and `pkcs1`.

Fixes https://github.com/denoland/deno/issues/18972
Ref #21124 

Verified fix with `npm:sshpk`. Unverfied but fixes
`npm:google-auth-library`, `npm:web-push` & `oracle/oci-typescript-sdk`

---------

Signed-off-by: Divy Srivastava <dj.srivastava23@gmail.com>

3 files changed, 8 insertions, 20 deletions

diff --git a/ext/node/polyfills/internal/crypto/cipher.ts b/ext/node/polyfills/internal/crypto/cipher.ts
index 5fec98ff0..9b63db538 100644
--- a/ext/node/polyfills/internal/crypto/cipher.ts
+++ b/ext/node/polyfills/internal/crypto/cipher.ts
@@ -31,7 +31,8 @@ import {
 export function isStringOrBuffer(val) {
   return typeof val === "string" ||
     isArrayBufferView(val) ||
-    isAnyArrayBuffer(val);
+    isAnyArrayBuffer(val) ||
+    Buffer.isBuffer(val);
 }
 
 const { ops, encode } = globalThis.__bootstrap.core;
diff --git a/ext/node/polyfills/internal/crypto/keys.ts b/ext/node/polyfills/internal/crypto/keys.ts
index e0c44cbf9..6a4fb2149 100644
--- a/ext/node/polyfills/internal/crypto/keys.ts
+++ b/ext/node/polyfills/internal/crypto/keys.ts
@@ -210,7 +210,7 @@ export interface JsonWebKeyInput {
   format: "jwk";
 }
 
-function prepareAsymmetricKey(key) {
+export function prepareAsymmetricKey(key) {
   if (isStringOrBuffer(key)) {
     return { format: "pem", data: getArrayBufferOrView(key, "key") };
   } else if (typeof key == "object") {
diff --git a/ext/node/polyfills/internal/crypto/sig.ts b/ext/node/polyfills/internal/crypto/sig.ts
index c5eb34fae..9e8af8d08 100644
--- a/ext/node/polyfills/internal/crypto/sig.ts
+++ b/ext/node/polyfills/internal/crypto/sig.ts
@@ -20,8 +20,8 @@ import type {
   PublicKeyInput,
 } from "ext:deno_node/internal/crypto/types.ts";
 import {
-  getKeyMaterial,
   KeyObject,
+  prepareAsymmetricKey,
 } from "ext:deno_node/internal/crypto/keys.ts";
 import { createHash, Hash } from "ext:deno_node/internal/crypto/hash.ts";
 import { KeyFormat, KeyType } from "ext:deno_node/internal/crypto/types.ts";
@@ -80,26 +80,13 @@ export class SignImpl extends Writable {
     privateKey: BinaryLike | SignKeyObjectInput | SignPrivateKeyInput,
     encoding?: BinaryToTextEncoding,
   ): Buffer | string {
-    let keyData: Uint8Array;
-    let keyType: KeyType;
-    let keyFormat: KeyFormat;
-    if (typeof privateKey === "string" || isArrayBufferView(privateKey)) {
-      // if the key is BinaryLike, interpret it as a PEM encoded RSA key
-      // deno-lint-ignore no-explicit-any
-      keyData = privateKey as any;
-      keyType = "rsa";
-      keyFormat = "pem";
-    } else {
-      keyData = getKeyMaterial(privateKey);
-      keyType = "rsa";
-      keyFormat = "pem";
-    }
+    const { data, format, type } = prepareAsymmetricKey(privateKey);
     const ret = Buffer.from(ops.op_node_sign(
       this.hash.digest(),
       this.#digestType,
-      keyData!,
-      keyType,
-      keyFormat,
+      data!,
+      type,
+      format,
     ));
     return encoding ? ret.toString(encoding) : ret;
   }