fix(ext/node): support public key point encoding in ECDH.generateKeys() (#22976)

Towards https://github.com/denoland/deno/issues/22921

Co-authored-by: Divy Srivastava <dj.srivastava23@gmail.com>

4 files changed, 21 insertions, 8 deletions

diff --git a/ext/node/ops/crypto/mod.rs b/ext/node/ops/crypto/mod.rs
index 6b1ca9a38..cf44f48f7 100644
--- a/ext/node/ops/crypto/mod.rs
+++ b/ext/node/ops/crypto/mod.rs
@@ -10,6 +10,7 @@ use deno_core::OpState;
 use deno_core::ResourceId;
 use deno_core::StringOrBuffer;
 use deno_core::ToJsBuffer;
+use elliptic_curve::sec1::ToEncodedPoint;
 use hkdf::Hkdf;
 use num_bigint::BigInt;
 use num_bigint_dig::BigUint;
@@ -739,8 +740,6 @@ pub async fn op_node_dsa_generate_async(
 fn ec_generate(
   named_curve: &str,
 ) -> Result<(ToJsBuffer, ToJsBuffer), AnyError> {
-  use elliptic_curve::sec1::ToEncodedPoint;
-
   let mut rng = rand::thread_rng();
   // TODO(@littledivy): Support public key point encoding.
   // Default is uncompressed.
@@ -1054,14 +1053,16 @@ pub fn op_node_ecdh_generate_keys(
   #[string] curve: &str,
   #[buffer] pubbuf: &mut [u8],
   #[buffer] privbuf: &mut [u8],
+  #[string] format: &str,
 ) -> Result<ResourceId, AnyError> {
   let mut rng = rand::thread_rng();
+  let compress = format == "compressed";
   match curve {
     "secp256k1" => {
       let privkey =
         elliptic_curve::SecretKey::<k256::Secp256k1>::random(&mut rng);
       let pubkey = privkey.public_key();
-      pubbuf.copy_from_slice(pubkey.to_sec1_bytes().as_ref());
+      pubbuf.copy_from_slice(pubkey.to_encoded_point(compress).as_ref());
       privbuf.copy_from_slice(privkey.to_nonzero_scalar().to_bytes().as_ref());
 
       Ok(0)
@@ -1069,21 +1070,21 @@ pub fn op_node_ecdh_generate_keys(
     "prime256v1" | "secp256r1" => {
       let privkey = elliptic_curve::SecretKey::<NistP256>::random(&mut rng);
       let pubkey = privkey.public_key();
-      pubbuf.copy_from_slice(pubkey.to_sec1_bytes().as_ref());
+      pubbuf.copy_from_slice(pubkey.to_encoded_point(compress).as_ref());
       privbuf.copy_from_slice(privkey.to_nonzero_scalar().to_bytes().as_ref());
       Ok(0)
     }
     "secp384r1" => {
       let privkey = elliptic_curve::SecretKey::<NistP384>::random(&mut rng);
       let pubkey = privkey.public_key();
-      pubbuf.copy_from_slice(pubkey.to_sec1_bytes().as_ref());
+      pubbuf.copy_from_slice(pubkey.to_encoded_point(compress).as_ref());
       privbuf.copy_from_slice(privkey.to_nonzero_scalar().to_bytes().as_ref());
       Ok(0)
     }
     "secp224r1" => {
       let privkey = elliptic_curve::SecretKey::<NistP224>::random(&mut rng);
       let pubkey = privkey.public_key();
-      pubbuf.copy_from_slice(pubkey.to_sec1_bytes().as_ref());
+      pubbuf.copy_from_slice(pubkey.to_encoded_point(compress).as_ref());
       privbuf.copy_from_slice(privkey.to_nonzero_scalar().to_bytes().as_ref());
       Ok(0)
     }
diff --git a/ext/node/polyfills/internal/crypto/diffiehellman.ts b/ext/node/polyfills/internal/crypto/diffiehellman.ts
index 4b105e575..da7907734 100644
--- a/ext/node/polyfills/internal/crypto/diffiehellman.ts
+++ b/ext/node/polyfills/internal/crypto/diffiehellman.ts
@@ -1236,12 +1236,18 @@ export class ECDH {
   generateKeys(encoding: BinaryToTextEncoding, format?: ECDHKeyFormat): string;
   generateKeys(
     encoding?: BinaryToTextEncoding,
-    _format?: ECDHKeyFormat,
+    format: ECDHKeyFormat = "uncompressed",
   ): Buffer | string {
+    this.#pubbuf = Buffer.alloc(
+      format.trim() == "compressed"
+        ? this.#curve.publicKeySizeCompressed
+        : this.#curve.publicKeySize,
+    );
     op_node_ecdh_generate_keys(
       this.#curve.name,
       this.#pubbuf,
       this.#privbuf,
+      format,
     );
 
     if (encoding !== undefined) {
diff --git a/ext/node/polyfills/internal/crypto/types.ts b/ext/node/polyfills/internal/crypto/types.ts
index a6a366348..2b3ce34fe 100644
--- a/ext/node/polyfills/internal/crypto/types.ts
+++ b/ext/node/polyfills/internal/crypto/types.ts
@@ -16,7 +16,7 @@ export type Encoding =
   | CharacterEncoding
   | LegacyCharacterEncoding;
 
-export type ECDHKeyFormat = "compressed" | "uncompressed" | "hybrid";
+export type ECDHKeyFormat = "compressed" | "uncompressed";
 
 export type BinaryLike = string | ArrayBufferView;
 
diff --git a/ext/node/polyfills/internal/crypto/util.ts b/ext/node/polyfills/internal/crypto/util.ts
index a68ac3682..a39b031ee 100644
--- a/ext/node/polyfills/internal/crypto/util.ts
+++ b/ext/node/polyfills/internal/crypto/util.ts
@@ -25,6 +25,7 @@ export type EllipticCurve = {
   ephemeral: boolean;
   privateKeySize: number;
   publicKeySize: number;
+  publicKeySizeCompressed: number;
   sharedSecretSize: number;
 };
 
@@ -33,30 +34,35 @@ export const ellipticCurves: Array<EllipticCurve> = [
     name: "secp256k1",
     privateKeySize: 32,
     publicKeySize: 65,
+    publicKeySizeCompressed: 33,
     sharedSecretSize: 32,
   }, // Weierstrass-class EC used by Bitcoin
   {
     name: "prime256v1",
     privateKeySize: 32,
     publicKeySize: 65,
+    publicKeySizeCompressed: 33,
     sharedSecretSize: 32,
   }, // NIST P-256 EC
   {
     name: "secp256r1",
     privateKeySize: 32,
     publicKeySize: 65,
+    publicKeySizeCompressed: 33,
     sharedSecretSize: 32,
   }, // NIST P-256 EC (same as above)
   {
     name: "secp384r1",
     privateKeySize: 48,
     publicKeySize: 97,
+    publicKeySizeCompressed: 49,
     sharedSecretSize: 48,
   }, // NIST P-384 EC
   {
     name: "secp224r1",
     privateKeySize: 28,
     publicKeySize: 57,
+    publicKeySizeCompressed: 29,
     sharedSecretSize: 28,
   }, // NIST P-224 EC
 ];