Webcrypto: fix RSA-OAEP digest being ignored for non-SHA-1 hashes

The issue was introduced in 8cd5ff2fa413 (0.7.0), which replaced
EVP_PKEY_CTX_set_rsa_oaep_md() with EVP_PKEY_CTX_set_signature_md()
to restore building with OpenSSL <= 1.0.1.

The signature digest control is not valid for encryption or decryption.
Its unchecked failure left the OAEP digest at the SHA-1 default while
MGF1 used the requested hash.  As a result, RSA-OAEP with a non-SHA-1
hash produced a non-standard digest pair that only interoperated with
njs itself.

Use the proper OAEP digest control and check all RSA context setters.
Preserve compatibility with older OpenSSL and LibreSSL versions by
supporting RSA-OAEP with SHA-1 and explicitly rejecting unsupported
digests.

Add externally generated SHA-256 and SHA-384 ciphertexts to ensure the
tests verify interoperability instead of an njs-only round trip.

Co-authored-by: Cursor <cursoragent@cursor.com>

diff --git a/external/njs_openssl.h b/external/njs_openssl.h
index c68f330b34375617b8bb8bb432cbc1cdab9184f2..4fada5b6ed218bd1daf957ab8a71f0bec9beba59 100644 (file)
--- a/external/njs_openssl.h
+++ b/external/njs_openssl.h
@@ -37,6 +37,15 @@
 #endif
 
 
+#if (OPENSSL_VERSION_NUMBER >= 0x10002000L                            \
+     && (!defined LIBRESSL_VERSION_NUMBER                            \
+         || LIBRESSL_VERSION_NUMBER >= 0x3010000fL))
+#define NJS_OPENSSL_HAS_RSA_OAEP_MD  1
+#else
+#define NJS_OPENSSL_HAS_RSA_OAEP_MD  0
+#endif
+
+
 #if (OPENSSL_VERSION_NUMBER >= 0x10100000L)
 #define njs_evp_md_ctx_new()  EVP_MD_CTX_new()
 #define njs_evp_md_ctx_free(_ctx)  EVP_MD_CTX_free(_ctx)

diff --git a/external/njs_webcrypto_module.c b/external/njs_webcrypto_module.c
index 7e2b950aaf9872d6082051ec57eb5de96c853266..9dfc3a7ed8baea06e08557174506e9dcf60f070a 100644 (file)
--- a/external/njs_webcrypto_module.c
+++ b/external/njs_webcrypto_module.c
@@ -847,11 +847,21 @@ njs_cipher_pkey(njs_vm_t *vm, njs_str_t *data, njs_webcrypto_key_t *key,
     u_char                  *dst;
     size_t                  outlen;
     njs_int_t               ret;
+#if (NJS_OPENSSL_HAS_RSA_OAEP_MD)
     const EVP_MD            *md;
+#endif
     EVP_PKEY_CTX            *ctx;
     EVP_PKEY_cipher_t       cipher;
     EVP_PKEY_cipher_init_t  init;
 
+#if (!NJS_OPENSSL_HAS_RSA_OAEP_MD)
+    if (key->hash != NJS_HASH_SHA1) {
+        njs_vm_type_error(vm, "RSA-OAEP with \"%V\" digest is not supported",
+                          njs_algorithm_hash_name(key->hash));
+        return NJS_ERROR;
+    }
+#endif
+
     ctx = EVP_PKEY_CTX_new(key->u.a.pkey, NULL);
     if (njs_slow_path(ctx == NULL)) {
         njs_webcrypto_error(vm, "EVP_PKEY_CTX_new() failed");
@@ -875,11 +885,30 @@ njs_cipher_pkey(njs_vm_t *vm, njs_str_t *data, njs_webcrypto_key_t *key,
         goto fail;
     }
 
+    ret = EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_OAEP_PADDING);
+    if (njs_slow_path(ret <= 0)) {
+        njs_webcrypto_error(vm, "EVP_PKEY_CTX_set_rsa_padding() failed");
+        ret = NJS_ERROR;
+        goto fail;
+    }
+
+#if (NJS_OPENSSL_HAS_RSA_OAEP_MD)
     md = njs_algorithm_hash_digest(key->hash);
 
-    EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_OAEP_PADDING);
-    EVP_PKEY_CTX_set_signature_md(ctx, md);
-    EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, md);
+    ret = EVP_PKEY_CTX_set_rsa_oaep_md(ctx, md);
+    if (njs_slow_path(ret <= 0)) {
+        njs_webcrypto_error(vm, "EVP_PKEY_CTX_set_rsa_oaep_md() failed");
+        ret = NJS_ERROR;
+        goto fail;
+    }
+
+    ret = EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, md);
+    if (njs_slow_path(ret <= 0)) {
+        njs_webcrypto_error(vm, "EVP_PKEY_CTX_set_rsa_mgf1_md() failed");
+        ret = NJS_ERROR;
+        goto fail;
+    }
+#endif
 
     ret = cipher(ctx, NULL, &outlen, data->start, data->length);
     if (njs_slow_path(ret <= 0)) {

diff --git a/external/qjs_webcrypto_module.c b/external/qjs_webcrypto_module.c
index 36d0d75f8905f90e0e555bb067947b1800ec09ff..3da69e19b6434f72f461b4259163e16c54183877 100644 (file)
--- a/external/qjs_webcrypto_module.c
+++ b/external/qjs_webcrypto_module.c
@@ -624,11 +624,21 @@ qjs_cipher_pkey(JSContext *cx, njs_str_t *data, qjs_webcrypto_key_t *key,
     u_char                  *dst;
     size_t                  outlen;
     JSValue                 ret;
+#if (NJS_OPENSSL_HAS_RSA_OAEP_MD)
     const EVP_MD            *md;
+#endif
     EVP_PKEY_CTX            *ctx;
     EVP_PKEY_cipher_t       cipher;
     EVP_PKEY_cipher_init_t  init;
 
+#if (!NJS_OPENSSL_HAS_RSA_OAEP_MD)
+    if (key->hash != QJS_HASH_SHA1) {
+        JS_ThrowTypeError(cx, "RSA-OAEP with \"%s\" digest is not supported",
+                          qjs_algorithm_hash_name(key->hash));
+        return JS_EXCEPTION;
+    }
+#endif
+
     ctx = EVP_PKEY_CTX_new(key->u.a.pkey, NULL);
     if (ctx == NULL) {
         qjs_webcrypto_error(cx, "EVP_PKEY_CTX_new() failed");
@@ -652,11 +662,30 @@ qjs_cipher_pkey(JSContext *cx, njs_str_t *data, qjs_webcrypto_key_t *key,
         goto fail;
     }
 
+    rc = EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_OAEP_PADDING);
+    if (rc <= 0) {
+        qjs_webcrypto_error(cx, "EVP_PKEY_CTX_set_rsa_padding() failed");
+        ret = JS_EXCEPTION;
+        goto fail;
+    }
+
+#if (NJS_OPENSSL_HAS_RSA_OAEP_MD)
     md = qjs_algorithm_hash_digest(key->hash);
 
-    EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_OAEP_PADDING);
-    EVP_PKEY_CTX_set_signature_md(ctx, md);
-    EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, md);
+    rc = EVP_PKEY_CTX_set_rsa_oaep_md(ctx, md);
+    if (rc <= 0) {
+        qjs_webcrypto_error(cx, "EVP_PKEY_CTX_set_rsa_oaep_md() failed");
+        ret = JS_EXCEPTION;
+        goto fail;
+    }
+
+    rc = EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, md);
+    if (rc <= 0) {
+        qjs_webcrypto_error(cx, "EVP_PKEY_CTX_set_rsa_mgf1_md() failed");
+        ret = JS_EXCEPTION;
+        goto fail;
+    }
+#endif
 
     rc = cipher(ctx, NULL, &outlen, data->start, data->length);
     if (rc <= 0) {

diff --git a/test/webcrypto/rsa_decoding.t.mjs b/test/webcrypto/rsa_decoding.t.mjs
index 179072ff7e172e073d6ebaec33d824dbbb4bf9fe..2dcff1276c2094b2c3fdec9964e347606b5433ec 100644 (file)
--- a/test/webcrypto/rsa_decoding.t.mjs
+++ b/test/webcrypto/rsa_decoding.t.mjs
@@ -12,7 +12,7 @@ async function test(params) {
     let enc = base64decode(fs.readFileSync(`test/webcrypto/${params.src}`));
 
     let key = await crypto.subtle.importKey("pkcs8", pem_to_der(pem, "PRIVATE"),
-                                            {name:"RSA-OAEP", hash:"SHA-1"},
+                                            {name:"RSA-OAEP", hash:params.hash},
                                             false, ["decrypt"]);
 
     let plaintext = await crypto.subtle.decrypt({name: "RSA-OAEP"}, key, enc);
@@ -28,10 +28,17 @@ async function test(params) {
 let rsa_tsuite = {
     name: "RSA-OAEP decoding",
     T: test,
-    prepare_args: (v) => v,
+
+    opts: {
+        pem: "rsa.pkcs8",
+        hash: "SHA-1",
+        expected: "WAKAWAKA",
+    },
 
     tests: [
-        { pem: "rsa.pkcs8", src: "text.base64.rsa-oaep.enc", expected: "WAKAWAKA" },
+        { src: "text.base64.rsa-oaep.enc" },
+        { src: "text.base64.rsa-oaep-sha256.enc", hash: "SHA-256" },
+        { src: "text.base64.rsa-oaep-sha384.enc", hash: "SHA-384" },
 ]};
 
 run([rsa_tsuite])

diff --git a/test/webcrypto/text.base64.rsa-oaep-sha256.enc b/test/webcrypto/text.base64.rsa-oaep-sha256.enc
new file mode 100644 (file)
index 0000000..ca55fc7
--- /dev/null
+++ b/test/webcrypto/text.base64.rsa-oaep-sha256.enc
@@ -0,0 +1,3 @@
+BEUBpWEnG1pXwCkYN5TIElS5Hl7KFNyjMVZuegLhy259b6owXYlbIwJ124IegEBYJ4xUA/DIoi4O
+HFWDNxoPJuTOijEsxRbbKJfzbWWi7OUvoFyloXx1SS0Sc4lLgzrdaxkYXH6rXmP/AmVxdhBhOrg/
+QfI1t5BNBB+vHhtsjXM=

diff --git a/test/webcrypto/text.base64.rsa-oaep-sha384.enc b/test/webcrypto/text.base64.rsa-oaep-sha384.enc
new file mode 100644 (file)
index 0000000..42e24f8
--- /dev/null
+++ b/test/webcrypto/text.base64.rsa-oaep-sha384.enc
@@ -0,0 +1,3 @@
+ZqEb024lPKdNsSifhHsyWboX1mm9rMIqXnA6lZIooRgApMkmTqOf6lonlHg1HKnjnIB4Nb76PFyh
+XaWRA9ZinSFQ4wz0U4NPFZEw8Ya4g0mCegDs152vkdDOeZ3Ps7OTkCWJBy/woKWndNCH5WTYEAUa
+SwIKSy+n/RgFKO0+xJE=