From: Tomas Psota Date: Mon, 27 Jul 2026 13:17:16 +0000 (+0200) Subject: Webcrypto: fix RSA-OAEP digest being ignored for non-SHA-1 hashes X-Git-Tag: 1.0.1~20 X-Git-Url: https://git.kaiwu.me/%7B@url%7D?a=commitdiff_plain;h=757d2302e54e494f808918a829cc1ba95a194dd8;p=njs.git Webcrypto: fix RSA-OAEP digest being ignored for non-SHA-1 hashes The issue was introduced in 8cd5ff2fa413 (0.7.0), which replaced EVP_PKEY_CTX_set_rsa_oaep_md() with EVP_PKEY_CTX_set_signature_md() to restore building with OpenSSL <= 1.0.1. The signature digest control is not valid for encryption or decryption. Its unchecked failure left the OAEP digest at the SHA-1 default while MGF1 used the requested hash. As a result, RSA-OAEP with a non-SHA-1 hash produced a non-standard digest pair that only interoperated with njs itself. Use the proper OAEP digest control and check all RSA context setters. Preserve compatibility with older OpenSSL and LibreSSL versions by supporting RSA-OAEP with SHA-1 and explicitly rejecting unsupported digests. Add externally generated SHA-256 and SHA-384 ciphertexts to ensure the tests verify interoperability instead of an njs-only round trip. Co-authored-by: Cursor --- diff --git a/external/njs_openssl.h b/external/njs_openssl.h index c68f330b..4fada5b6 100644 --- a/external/njs_openssl.h +++ b/external/njs_openssl.h @@ -37,6 +37,15 @@ #endif +#if (OPENSSL_VERSION_NUMBER >= 0x10002000L \ + && (!defined LIBRESSL_VERSION_NUMBER \ + || LIBRESSL_VERSION_NUMBER >= 0x3010000fL)) +#define NJS_OPENSSL_HAS_RSA_OAEP_MD 1 +#else +#define NJS_OPENSSL_HAS_RSA_OAEP_MD 0 +#endif + + #if (OPENSSL_VERSION_NUMBER >= 0x10100000L) #define njs_evp_md_ctx_new() EVP_MD_CTX_new() #define njs_evp_md_ctx_free(_ctx) EVP_MD_CTX_free(_ctx) diff --git a/external/njs_webcrypto_module.c b/external/njs_webcrypto_module.c index 7e2b950a..9dfc3a7e 100644 --- a/external/njs_webcrypto_module.c +++ b/external/njs_webcrypto_module.c @@ -847,11 +847,21 @@ njs_cipher_pkey(njs_vm_t *vm, njs_str_t *data, njs_webcrypto_key_t *key, u_char *dst; size_t outlen; njs_int_t ret; +#if (NJS_OPENSSL_HAS_RSA_OAEP_MD) const EVP_MD *md; +#endif EVP_PKEY_CTX *ctx; EVP_PKEY_cipher_t cipher; EVP_PKEY_cipher_init_t init; +#if (!NJS_OPENSSL_HAS_RSA_OAEP_MD) + if (key->hash != NJS_HASH_SHA1) { + njs_vm_type_error(vm, "RSA-OAEP with \"%V\" digest is not supported", + njs_algorithm_hash_name(key->hash)); + return NJS_ERROR; + } +#endif + ctx = EVP_PKEY_CTX_new(key->u.a.pkey, NULL); if (njs_slow_path(ctx == NULL)) { njs_webcrypto_error(vm, "EVP_PKEY_CTX_new() failed"); @@ -875,11 +885,30 @@ njs_cipher_pkey(njs_vm_t *vm, njs_str_t *data, njs_webcrypto_key_t *key, goto fail; } + ret = EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_OAEP_PADDING); + if (njs_slow_path(ret <= 0)) { + njs_webcrypto_error(vm, "EVP_PKEY_CTX_set_rsa_padding() failed"); + ret = NJS_ERROR; + goto fail; + } + +#if (NJS_OPENSSL_HAS_RSA_OAEP_MD) md = njs_algorithm_hash_digest(key->hash); - EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_OAEP_PADDING); - EVP_PKEY_CTX_set_signature_md(ctx, md); - EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, md); + ret = EVP_PKEY_CTX_set_rsa_oaep_md(ctx, md); + if (njs_slow_path(ret <= 0)) { + njs_webcrypto_error(vm, "EVP_PKEY_CTX_set_rsa_oaep_md() failed"); + ret = NJS_ERROR; + goto fail; + } + + ret = EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, md); + if (njs_slow_path(ret <= 0)) { + njs_webcrypto_error(vm, "EVP_PKEY_CTX_set_rsa_mgf1_md() failed"); + ret = NJS_ERROR; + goto fail; + } +#endif ret = cipher(ctx, NULL, &outlen, data->start, data->length); if (njs_slow_path(ret <= 0)) { diff --git a/external/qjs_webcrypto_module.c b/external/qjs_webcrypto_module.c index 36d0d75f..3da69e19 100644 --- a/external/qjs_webcrypto_module.c +++ b/external/qjs_webcrypto_module.c @@ -624,11 +624,21 @@ qjs_cipher_pkey(JSContext *cx, njs_str_t *data, qjs_webcrypto_key_t *key, u_char *dst; size_t outlen; JSValue ret; +#if (NJS_OPENSSL_HAS_RSA_OAEP_MD) const EVP_MD *md; +#endif EVP_PKEY_CTX *ctx; EVP_PKEY_cipher_t cipher; EVP_PKEY_cipher_init_t init; +#if (!NJS_OPENSSL_HAS_RSA_OAEP_MD) + if (key->hash != QJS_HASH_SHA1) { + JS_ThrowTypeError(cx, "RSA-OAEP with \"%s\" digest is not supported", + qjs_algorithm_hash_name(key->hash)); + return JS_EXCEPTION; + } +#endif + ctx = EVP_PKEY_CTX_new(key->u.a.pkey, NULL); if (ctx == NULL) { qjs_webcrypto_error(cx, "EVP_PKEY_CTX_new() failed"); @@ -652,11 +662,30 @@ qjs_cipher_pkey(JSContext *cx, njs_str_t *data, qjs_webcrypto_key_t *key, goto fail; } + rc = EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_OAEP_PADDING); + if (rc <= 0) { + qjs_webcrypto_error(cx, "EVP_PKEY_CTX_set_rsa_padding() failed"); + ret = JS_EXCEPTION; + goto fail; + } + +#if (NJS_OPENSSL_HAS_RSA_OAEP_MD) md = qjs_algorithm_hash_digest(key->hash); - EVP_PKEY_CTX_set_rsa_padding(ctx, RSA_PKCS1_OAEP_PADDING); - EVP_PKEY_CTX_set_signature_md(ctx, md); - EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, md); + rc = EVP_PKEY_CTX_set_rsa_oaep_md(ctx, md); + if (rc <= 0) { + qjs_webcrypto_error(cx, "EVP_PKEY_CTX_set_rsa_oaep_md() failed"); + ret = JS_EXCEPTION; + goto fail; + } + + rc = EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, md); + if (rc <= 0) { + qjs_webcrypto_error(cx, "EVP_PKEY_CTX_set_rsa_mgf1_md() failed"); + ret = JS_EXCEPTION; + goto fail; + } +#endif rc = cipher(ctx, NULL, &outlen, data->start, data->length); if (rc <= 0) { diff --git a/test/webcrypto/rsa_decoding.t.mjs b/test/webcrypto/rsa_decoding.t.mjs index 179072ff..2dcff127 100644 --- a/test/webcrypto/rsa_decoding.t.mjs +++ b/test/webcrypto/rsa_decoding.t.mjs @@ -12,7 +12,7 @@ async function test(params) { let enc = base64decode(fs.readFileSync(`test/webcrypto/${params.src}`)); let key = await crypto.subtle.importKey("pkcs8", pem_to_der(pem, "PRIVATE"), - {name:"RSA-OAEP", hash:"SHA-1"}, + {name:"RSA-OAEP", hash:params.hash}, false, ["decrypt"]); let plaintext = await crypto.subtle.decrypt({name: "RSA-OAEP"}, key, enc); @@ -28,10 +28,17 @@ async function test(params) { let rsa_tsuite = { name: "RSA-OAEP decoding", T: test, - prepare_args: (v) => v, + + opts: { + pem: "rsa.pkcs8", + hash: "SHA-1", + expected: "WAKAWAKA", + }, tests: [ - { pem: "rsa.pkcs8", src: "text.base64.rsa-oaep.enc", expected: "WAKAWAKA" }, + { src: "text.base64.rsa-oaep.enc" }, + { src: "text.base64.rsa-oaep-sha256.enc", hash: "SHA-256" }, + { src: "text.base64.rsa-oaep-sha384.enc", hash: "SHA-384" }, ]}; run([rsa_tsuite]) diff --git a/test/webcrypto/text.base64.rsa-oaep-sha256.enc b/test/webcrypto/text.base64.rsa-oaep-sha256.enc new file mode 100644 index 00000000..ca55fc78 --- /dev/null +++ b/test/webcrypto/text.base64.rsa-oaep-sha256.enc @@ -0,0 +1,3 @@ +BEUBpWEnG1pXwCkYN5TIElS5Hl7KFNyjMVZuegLhy259b6owXYlbIwJ124IegEBYJ4xUA/DIoi4O +HFWDNxoPJuTOijEsxRbbKJfzbWWi7OUvoFyloXx1SS0Sc4lLgzrdaxkYXH6rXmP/AmVxdhBhOrg/ +QfI1t5BNBB+vHhtsjXM= diff --git a/test/webcrypto/text.base64.rsa-oaep-sha384.enc b/test/webcrypto/text.base64.rsa-oaep-sha384.enc new file mode 100644 index 00000000..42e24f89 --- /dev/null +++ b/test/webcrypto/text.base64.rsa-oaep-sha384.enc @@ -0,0 +1,3 @@ +ZqEb024lPKdNsSifhHsyWboX1mm9rMIqXnA6lZIooRgApMkmTqOf6lonlHg1HKnjnIB4Nb76PFyh +XaWRA9ZinSFQ4wz0U4NPFZEw8Ya4g0mCegDs152vkdDOeZ3Ps7OTkCWJBy/woKWndNCH5WTYEAUa +SwIKSy+n/RgFKO0+xJE=