fix(crypto): honor Sign and Verify encodings

changelog.d/11030-crypto-sign-encoding.md

@@ -0,0 +1,5 @@
+### Fixed
+
+`Sign.sign(key, encoding)` now returns the requested encoded string, and
+`Verify.verify(key, signature, encoding)` decodes string signatures using the
+same encoding rules as Node and Buffer.

crates/perry-stdlib/src/crypto/ecdh.rs

@@ -136,6 +136,13 @@ pub unsafe extern "C" fn js_crypto_ecdh_convert_key(
     )
 }
 
+unsafe fn signature_output(bytes: &[u8], encoding: Option<EncodingTag>) -> f64 {
+    match encoding {
+        Some(tag) => encode_bytes_with_tag(bytes, tag),
+        None => nanbox_ptr(alloc_buffer_from_slice(bytes)),
+    }
+}
+
 pub unsafe fn dispatch_sign(handle: i64, method: &str, args: &[f64]) -> f64 {
     let h = match get_handle_mut::<SignHandle>(handle) {
         Some(h) => h,
@@ -161,6 +168,7 @@ pub unsafe fn dispatch_sign(handle: i64, method: &str, args: &[f64]) -> f64 {
             // The handle is consumed by `.sign()` regardless of outcome.
             h.finalized
                 .store(true, std::sync::atomic::Ordering::Relaxed);
+            let output_encoding = encoding_tag_from_arg(args.get(1).copied());
             let key_bits = args[0].to_bits();
             let pem = match crypto_key_input_to_private_pem(key_bits) {
                 Some(pem) => pem,
@@ -171,16 +179,10 @@ pub unsafe fn dispatch_sign(handle: i64, method: &str, args: &[f64]) -> f64 {
                 let signature: P256EcdsaSignature = signing_key.sign(&data);
                 if key_input_uses_ieee_p1363(key_bits) {
                     let raw = signature.to_bytes();
-                    let buf = alloc_buffer_from_slice(raw.as_slice());
-                    return f64::from_bits(
-                        0x7FFD_0000_0000_0000u64 | ((buf as u64) & 0x0000_FFFF_FFFF_FFFF),
-                    );
+                    return signature_output(raw.as_slice(), output_encoding);
                 }
                 let der = signature.to_der();
-                let buf = alloc_buffer_from_slice(der.as_bytes());
-                return f64::from_bits(
-                    0x7FFD_0000_0000_0000u64 | ((buf as u64) & 0x0000_FFFF_FFFF_FFFF),
-                );
+                return signature_output(der.as_bytes(), output_encoding);
             }
             let private_key = match parse_rsa_private_key_pem(&pem) {
                 Some(key) => key,
@@ -193,8 +195,7 @@ pub unsafe fn dispatch_sign(handle: i64, method: &str, args: &[f64]) -> f64 {
             } else {
                 sign_rsa_data(h.alg, private_key, &data)
             };
-            let buf = alloc_buffer_from_slice(&signature);
-            f64::from_bits(0x7FFD_0000_0000_0000u64 | ((buf as u64) & 0x0000_FFFF_FFFF_FFFF))
+            signature_output(&signature, output_encoding)
         }
         _ => f64::from_bits(0x7FFC_0000_0000_0001),
     }
@@ -486,13 +487,21 @@ pub unsafe fn dispatch_verify(handle: i64, method: &str, args: &[f64]) -> f64 {
             h.finalized
                 .store(true, std::sync::atomic::Ordering::Relaxed);
             let key_bits = args[0].to_bits();
-            let sig_ptr = (args[1].to_bits() & 0x0000_FFFF_FFFF_FFFF) as i64;
+            // Node applies `signatureEncoding` only to string signatures.
+            // Buffer inputs keep their bytes and ignore even an invalid third
+            // argument. A string with no encoding defaults to UTF-8.
+            let sig_bytes = if let Some(signature) = string_from_jsvalue(args[1].to_bits()) {
+                let encoding =
+                    encoding_tag_from_arg(args.get(2).copied()).unwrap_or(EncodingTag(0));
+                decode_string_bytes_with_tag(signature.as_bytes(), encoding)
+            } else {
+                bytes_from_ptr(arg_ptr(args[1]))
+            };
             let pem = match crypto_key_input_to_public_pem(key_bits) {
                 Some(pem) => pem,
                 None => return js_bool(false),
             };
             if let Some(verifying_key) = parse_p256_verifying_key_pem(&pem) {
-                let sig_bytes = bytes_from_ptr(sig_ptr);
                 let signature = if key_input_uses_ieee_p1363(key_bits) {
                     P256EcdsaSignature::from_slice(&sig_bytes)
                 } else {
@@ -509,7 +518,6 @@ pub unsafe fn dispatch_verify(handle: i64, method: &str, args: &[f64]) -> f64 {
                 Some(key) => key,
                 None => return js_bool(false),
             };
-            let sig_bytes = bytes_from_ptr(sig_ptr);
             if key_input_uses_rsa_pss(key_bits) {
                 let signature = match RsaPssSignature::try_from(sig_bytes.as_slice()) {
                     Ok(sig) => sig,

test-files/test_gap_10472_crypto_sign_verify_encoding.ts

@@ -0,0 +1,141 @@
+// #10472 — Sign.sign must encode string output and Verify.verify must decode
+// string signatures with the optional encoding argument.
+import crypto from "node:crypto";
+
+const privateKey =
+  "-----BEGIN PRIVATE KEY-----\n" +
+  "MIGHAgEAMBMGByqGSM49AgEGCCqGSM49AwEHBG0wawIBAQQgzXENgrYISpXh8UGG\n" +
+  "n6gRdTvn03fTF16DgDQTADm0XV6hRANCAAS86N32ME7tVKj5oIMLOiYoElFNSXbJ\n" +
+  "wMQL3GyWDLKC996gWUP4WfQLYOJd6To9wdlomuiOFtVryzwKdMdFFd7G\n" +
+  "-----END PRIVATE KEY-----\n";
+const publicKey =
+  "-----BEGIN PUBLIC KEY-----\n" +
+  "MFkwEwYHKoZIzj0CAQYIKoZIzj0DAQcDQgAEvOjd9jBO7VSo+aCDCzomKBJRTUl2\n" +
+  "ycDEC9xslgyygvfeoFlD+Fn0C2DiXek6PcHZaJrojhbVa8s8CnTHRRXexg==\n" +
+  "-----END PUBLIC KEY-----\n";
+const rsaPrivateKey =
+  "-----BEGIN PRIVATE KEY-----\n" +
+  "MIICdwIBADANBgkqhkiG9w0BAQEFAASCAmEwggJdAgEAAoGBALA9qACs400Jgizt\n" +
+  "8uNV2sw/+Qj1V6/27b50gH4LBC81YPypipBTZ52mbb4Xfpr5OroUnWCxaibEj0rg\n" +
+  "2nlKuS6wSCOrEdsdRC40GoLeSnIDExDgYVTWlEiU2dZ2bAKqSO3l7JRBQyJkwfBG\n" +
+  "qJRHTtrZ3ycKmQTvlcZJw+p5r48fAgMBAAECgYA7hxfP4pWD18pYUqbPkpgslQ8Q\n" +
+  "r43Gqajzw3YDHMV1DJqNvNZImWNOJIC8zEK/JZ9oar4dgs9P+ORNblVc0phpVSQ4\n" +
+  "lKjOQguiFgZqjbEL1tQTpObQmf711ZcWOMiFweDKbT0foW1b+0BnzLVLQHsrnItv\n" +
+  "obASCIEv9vKytN1wwQJBANq7Q8811TaV5xrlzvITBIZxO/g8oneaHRfxLLGNBuWh\n" +
+  "TLPkzkJh2higWM0nKk6lcyKwCzAqI5DKMkaXpvmoBoMCQQDORQN37EXMed080eLI\n" +
+  "RwuyG+2ZGoxtyQtUyoznlIXHWsoE4uUBmQ4YjCNljhbqPz0RTvrDxPj0uzJnP2Vd\n" +
+  "+xI1AkEAzjuC0/yN68mq/VFwrg4AVkKtqICDLwHALLLY0Q+HUTukdnllgHGCkXWe\n" +
+  "RNCIs16MEEisQ913ay05+hVC+mHSwQJBAKl+4mu/9lcg6KBao+0JHF4+Ps6ply17\n" +
+  "n9kMHB8L16ZKP2kmfSIEACZBubBwwvm3/1liugL2r9CCptdaq9Q/ROUCQBWhzBBS\n" +
+  "LbtQCOQOxrzUW6ipqfEeoHWEEZI++krTqYFsZL62uZ86b6gLyQLLRSsngr7D4D/w\n" +
+  "+89KirPwQz+VOmA=\n" +
+  "-----END PRIVATE KEY-----\n";
+const rsaPublicKey =
+  "-----BEGIN PUBLIC KEY-----\n" +
+  "MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCwPagArONNCYIs7fLjVdrMP/kI\n" +
+  "9Vev9u2+dIB+CwQvNWD8qYqQU2edpm2+F36a+Tq6FJ1gsWomxI9K4Np5SrkusEgj\n" +
+  "qxHbHUQuNBqC3kpyAxMQ4GFU1pRIlNnWdmwCqkjt5eyUQUMiZMHwRqiUR07a2d8n\n" +
+  "CpkE75XGScPqea+PHwIDAQAB\n" +
+  "-----END PUBLIC KEY-----\n";
+const payload = "header.payload";
+
+function sign(key: any, encoding?: BufferEncoding): Buffer | string {
+  const signer = crypto.createSign("sha256");
+  signer.update(payload);
+  return encoding === undefined ? signer.sign(key) : signer.sign(key, encoding);
+}
+
+function verify(
+  key: any,
+  signature: Buffer | string,
+  encoding?: BufferEncoding,
+): boolean {
+  const verifier = crypto.createVerify("sha256");
+  verifier.update(payload);
+  return encoding === undefined
+    ? verifier.verify(key, signature)
+    : verifier.verify(key, signature, encoding);
+}
+
+for (const encoding of ["hex", "base64", "base64url", "latin1", "binary"] as const) {
+  const signature = sign(privateKey, encoding);
+  console.log(
+    "encoded",
+    encoding,
+    typeof signature,
+    Buffer.isBuffer(signature),
+    verify(publicKey, signature, encoding),
+  );
+}
+
+const p1363Private = { key: privateKey, dsaEncoding: "ieee-p1363" as const };
+const p1363Public = { key: publicKey, dsaEncoding: "ieee-p1363" as const };
+const p1363 = sign(p1363Private, "base64url");
+console.log(
+  "p1363",
+  typeof p1363,
+  Buffer.isBuffer(p1363),
+  verify(p1363Public, p1363, "base64url"),
+);
+
+const rsa = sign(rsaPrivateKey, "base64");
+console.log(
+  "rsa",
+  typeof rsa,
+  Buffer.isBuffer(rsa),
+  verify(rsaPublicKey, rsa, "base64"),
+);
+
+const rsaRaw = sign(rsaPrivateKey) as Buffer;
+for (const encoding of ["utf8", "ascii", "utf16le"] as const) {
+  const encoded = sign(rsaPrivateKey, encoding);
+  console.log(
+    "encoded-lossy",
+    encoding,
+    typeof encoded,
+    Buffer.isBuffer(encoded),
+    encoded === rsaRaw.toString(encoding),
+  );
+}
+
+const rsaPssPrivate = {
+  key: rsaPrivateKey,
+  padding: crypto.constants.RSA_PKCS1_PSS_PADDING,
+  saltLength: 32,
+};
+const rsaPssPublic = {
+  key: rsaPublicKey,
+  padding: crypto.constants.RSA_PKCS1_PSS_PADDING,
+  saltLength: 32,
+};
+const rsaPss = sign(rsaPssPrivate, "hex");
+console.log(
+  "rsa-pss",
+  typeof rsaPss,
+  Buffer.isBuffer(rsaPss),
+  verify(rsaPssPublic, rsaPss, "hex"),
+);
+
+const raw = sign(privateKey) as Buffer;
+console.log(
+  "buffer-invalid-encoding-ignored",
+  verify(publicKey, raw, "not-an-encoding" as any),
+);
+
+try {
+  sign(privateKey, "not-an-encoding" as any);
+  console.log("sign-invalid no-throw");
+} catch (error: any) {
+  console.log("sign-invalid", error.code);
+}
+
+try {
+  verify(
+    publicKey,
+    (sign(privateKey, "hex") as string),
+    "not-an-encoding" as any,
+  );
+  console.log("verify-invalid no-throw");
+} catch (error: any) {
+  console.log("verify-invalid", error.code);
+}