Add configurable OAEP hash algorithm

RSA.pm

@@ -384,11 +384,15 @@ C<use_pkcs1_oaep_padding()> for encryption.
 
 =item use_pkcs1_oaep_padding
 
-Use C<EME-OAEP> padding as defined in PKCS #1 v2.0 with SHA-1, MGF1 and
+Use C<EME-OAEP> padding as defined in PKCS #1 v2.0 with MGF1 and
 an empty encoding parameter. This mode of padding is recommended for
 all new applications.  It is the default mode used by
 C<Crypt::OpenSSL::RSA> but is only valid for encryption/decryption.
 
+The OAEP hash algorithm defaults to SHA-1 for backward compatibility.
+Use C<use_sha256_oaep_hash()> (or other C<use_*_oaep_hash()> methods)
+to select a stronger hash.  See L</OAEP Hash Methods> below.
+
 =item use_pkcs1_pss_padding
 
 Use C<RSA-PSS> padding as defined in PKCS#1 v2.1.  In general, RSA-PSS
@@ -411,6 +415,32 @@ C<use_pkcs1_pss_padding()> for signatures.
 
 =back
 
+=head1 OAEP Hash Methods
+
+These methods control the hash algorithm used for OAEP padding (both the
+label hash and the MGF1 mask generation function).  The OAEP hash is
+independent of the signature hash set by C<use_sha256_hash()> etc.
+
+B<On OpenSSL 3.x>, any of the methods below can be used.
+B<On pre-3.x OpenSSL>, only SHA-1 is supported for OAEP; calling
+C<encrypt()>/C<decrypt()> after setting a non-SHA1 OAEP hash will croak.
+
+=over
+
+=item use_sha1_oaep_hash
+
+Use SHA-1 for OAEP (the default).
+
+=item use_sha224_oaep_hash, use_sha256_oaep_hash, use_sha384_oaep_hash, use_sha512_oaep_hash
+
+Use the specified SHA-2 hash for OAEP.  C<use_sha256_oaep_hash> is
+recommended for new applications that need to move away from SHA-1.
+
+These are only available when OpenSSL was built with SHA-2 support
+(>= 0.9.8, which covers all supported versions).
+
+=back
+
 =head1 Hash/Digest Methods
 
 =over
@@ -455,7 +485,10 @@ the text to be encrypted should be:
 
 =item pkcs1_oaep_padding
 
-at most 42 bytes less than this size.
+at most C<2 * hash_length + 2> bytes less than this size, where
+C<hash_length> is the digest size of the OAEP hash algorithm.
+With the default SHA-1, this is 42 bytes; with SHA-256 it is 66 bytes;
+with SHA-512 it is 130 bytes.  See L</OAEP Hash Methods>.
 
 =item pkcs1_padding or sslv23_padding
 

RSA.xs

@@ -100,6 +100,7 @@ typedef struct
     EVP_PKEY* rsa;
     int padding;
     int hashMode;
+    int oaepHashMode;   /* hash for OAEP padding (label hash + MGF1); default NID_sha1 */
     int is_private_key;  /* cached once at construction; avoids per-call BIGNUM alloc on 3.x */
 } rsaData;
 
@@ -182,6 +183,7 @@ SV* make_rsa_obj(SV* p_proto, EVP_PKEY* p_rsa)
 #else
     rsa->hashMode = NID_sha1;
 #endif
+    rsa->oaepHashMode = NID_sha1;
     rsa->padding = RSA_PKCS1_OAEP_PADDING;
     rsa->is_private_key = _detect_private_key(p_rsa);
     return sv_bless(
@@ -434,7 +436,7 @@ static void check_max_message_length(rsaData* p_rsa, STRLEN from_length) {
     size = EVP_PKEY_get_size(p_rsa->rsa);
 
     if (p_rsa->padding == RSA_PKCS1_OAEP_PADDING) {
-        max_len = size - 42;  /* 2 * SHA1_DIGEST_LENGTH + 2 */
+        max_len = size - 2 * get_digest_length(p_rsa->oaepHashMode) - 2;
         pad_name = "OAEP";
     } else if (p_rsa->padding == RSA_PKCS1_PADDING) {
         max_len = size - 11;  /* PKCS#1 v1.5 overhead */
@@ -480,6 +482,7 @@ SV* rsa_crypt(rsaData* p_rsa, SV* p_from,
     }
 #if OPENSSL_VERSION_NUMBER >= 0x30000000L
     EVP_PKEY_CTX *ctx = NULL;
+    EVP_MD *oaep_md = NULL;
     int error = 0;
 
     if (is_encrypt) {
@@ -508,20 +511,34 @@ SV* rsa_crypt(rsaData* p_rsa, SV* p_from,
 
     THROW(init_crypt(ctx) == 1);
     THROW(EVP_PKEY_CTX_set_rsa_padding(ctx, p_rsa->padding) > 0);
+
+    if (is_encrypt && p_rsa->padding == RSA_PKCS1_OAEP_PADDING) {
+        oaep_md = get_md_bynid(p_rsa->oaepHashMode);
+        THROW(oaep_md);
+        THROW(EVP_PKEY_CTX_set_rsa_oaep_md(ctx, oaep_md) > 0);
+        THROW(EVP_PKEY_CTX_set_rsa_mgf1_md(ctx, oaep_md) > 0);
+    }
+
     THROW(p_crypt(ctx, NULL, &to_length, from, from_length) == 1);
     Newx(to, to_length, UNSIGNED_CHAR);
     THROW(to);
     THROW(p_crypt(ctx, to, &to_length, from, from_length) == 1);
 
     EVP_PKEY_CTX_free(ctx);
+    if (oaep_md) EVP_MD_free(oaep_md);
 
     goto crypt_done;
     err:
         if (ctx) EVP_PKEY_CTX_free(ctx);
+        if (oaep_md) EVP_MD_free(oaep_md);
         Safefree(to);
         CHECK_OPEN_SSL(0);
     crypt_done:
 #else
+    if (is_encrypt && p_rsa->padding == RSA_PKCS1_OAEP_PADDING
+        && p_rsa->oaepHashMode != NID_sha1) {
+        croak("OAEP with non-SHA1 hash requires OpenSSL 3.0 or later");
+    }
     size = EVP_PKEY_get_size(p_rsa->rsa);
     CHECK_NEW(to, size, UNSIGNED_CHAR);
     to_length = p_crypt(
@@ -1409,6 +1426,40 @@ use_pkcs1_pss_padding(p_rsa)
   CODE:
     p_rsa->padding = RSA_PKCS1_PSS_PADDING;
 
+void
+use_sha1_oaep_hash(p_rsa)
+    rsaData* p_rsa;
+  CODE:
+    p_rsa->oaepHashMode = NID_sha1;
+
+#ifdef SHA512_DIGEST_LENGTH
+
+void
+use_sha224_oaep_hash(p_rsa)
+    rsaData* p_rsa;
+  CODE:
+    p_rsa->oaepHashMode = NID_sha224;
+
+void
+use_sha256_oaep_hash(p_rsa)
+    rsaData* p_rsa;
+  CODE:
+    p_rsa->oaepHashMode = NID_sha256;
+
+void
+use_sha384_oaep_hash(p_rsa)
+    rsaData* p_rsa;
+  CODE:
+    p_rsa->oaepHashMode = NID_sha384;
+
+void
+use_sha512_oaep_hash(p_rsa)
+    rsaData* p_rsa;
+  CODE:
+    p_rsa->oaepHashMode = NID_sha512;
+
+#endif
+
 #if OPENSSL_VERSION_NUMBER < 0x30000000L
 
 void

t/oaep_hash.t

@@ -0,0 +1,120 @@
+use strict;
+use warnings;
+use Test::More;
+
+use Crypt::OpenSSL::Random;
+use Crypt::OpenSSL::RSA;
+use Crypt::OpenSSL::Guess qw(openssl_version);
+
+Crypt::OpenSSL::Random::random_seed("OpenSSL needs at least 32 bytes.");
+Crypt::OpenSSL::RSA->import_random_seed();
+
+my ($major, $minor, $patch) = openssl_version();
+my $is_3x = ($major ge '3.0' && defined $patch);
+
+my $rsa = Crypt::OpenSSL::RSA->generate_key(2048);
+my $key_size = $rsa->size();
+my $plaintext = "The quick brown fox jumps over the lazy dog";
+
+# --- SHA-1 OAEP (default) round-trip works on all versions ---
+
+$rsa->use_pkcs1_oaep_padding();
+$rsa->use_sha1_oaep_hash();
+
+my $ct_sha1 = $rsa->encrypt($plaintext);
+ok(defined $ct_sha1, "OAEP SHA-1 encrypt succeeds");
+is($rsa->decrypt($ct_sha1), $plaintext, "OAEP SHA-1 decrypt round-trips");
+
+# --- SHA-256 OAEP ---
+
+SKIP: {
+    skip "OAEP with non-SHA1 hash requires OpenSSL 3.x", 6 unless $is_3x;
+
+    $rsa->use_sha256_oaep_hash();
+
+    my $ct_sha256 = $rsa->encrypt($plaintext);
+    ok(defined $ct_sha256, "OAEP SHA-256 encrypt succeeds");
+    is($rsa->decrypt($ct_sha256), $plaintext, "OAEP SHA-256 decrypt round-trips");
+
+    # Mismatched hash: encrypt with SHA-256, decrypt with SHA-1
+    $rsa->use_sha1_oaep_hash();
+    eval { $rsa->decrypt($ct_sha256) };
+    ok($@, "decrypt with wrong OAEP hash croaks");
+
+    # Mismatched hash: encrypt with SHA-1, decrypt with SHA-256
+    my $ct2 = $rsa->encrypt($plaintext);
+    $rsa->use_sha256_oaep_hash();
+    eval { $rsa->decrypt($ct2) };
+    ok($@, "decrypt SHA-1 ciphertext with SHA-256 OAEP hash croaks");
+
+    # Cross-key: encrypt with key1 SHA-256, decrypt with key2 SHA-256
+    my $rsa2 = Crypt::OpenSSL::RSA->generate_key(2048);
+    $rsa2->use_pkcs1_oaep_padding();
+    $rsa2->use_sha256_oaep_hash();
+
+    my $ct_k1 = $rsa->encrypt($plaintext);
+    eval { $rsa2->decrypt($ct_k1) };
+    ok($@, "decrypt with different key croaks even with matching OAEP hash");
+
+    # SHA-256 with its own key pair round-trips
+    my $ct_k2 = $rsa2->encrypt("secret");
+    is($rsa2->decrypt($ct_k2), "secret", "SHA-256 OAEP round-trip with second key");
+}
+
+# --- SHA-512 OAEP (larger hash = smaller max message) ---
+
+SKIP: {
+    skip "OAEP with non-SHA1 hash requires OpenSSL 3.x", 3 unless $is_3x;
+
+    $rsa->use_sha512_oaep_hash();
+
+    my $max_sha512 = $key_size - 2 * 64 - 2;  # SHA-512 = 64 bytes
+    my $ct_512 = $rsa->encrypt("x" x $max_sha512);
+    ok(defined $ct_512, "OAEP SHA-512 encrypt at max size succeeds");
+    is($rsa->decrypt($ct_512), "x" x $max_sha512,
+       "OAEP SHA-512 round-trip at max size");
+
+    eval { $rsa->encrypt("x" x ($max_sha512 + 1)) };
+    like($@, qr/plaintext too long/,
+         "OAEP SHA-512 rejects plaintext exceeding max size");
+}
+
+# --- Max message length varies correctly with OAEP hash ---
+
+{
+    my $max_sha1 = $key_size - 2 * 20 - 2;     # 214 for 2048-bit
+    my $max_sha256 = $key_size - 2 * 32 - 2;    # 190 for 2048-bit
+    my $max_sha512 = $key_size - 2 * 64 - 2;    # 126 for 2048-bit
+
+    $rsa->use_sha1_oaep_hash();
+    my $msg_sha1 = "x" x $max_sha1;
+    my $ct = eval { $rsa->encrypt($msg_sha1) };
+    ok(!$@, "SHA-1 OAEP max ($max_sha1 bytes) accepted");
+
+    eval { $rsa->encrypt("x" x ($max_sha1 + 1)) };
+    ok($@, "SHA-1 OAEP max+1 rejected");
+
+    SKIP: {
+        skip "Length validation with SHA-256 OAEP requires 3.x", 2 unless $is_3x;
+
+        $rsa->use_sha256_oaep_hash();
+        $ct = eval { $rsa->encrypt("x" x $max_sha256) };
+        ok(!$@, "SHA-256 OAEP max ($max_sha256 bytes) accepted");
+
+        eval { $rsa->encrypt("x" x ($max_sha256 + 1)) };
+        ok($@, "SHA-256 OAEP max+1 rejected");
+    }
+}
+
+# --- Pre-3.x: non-SHA1 OAEP croaks at encrypt time ---
+
+SKIP: {
+    skip "Only relevant on pre-3.x OpenSSL", 1 if $is_3x;
+
+    $rsa->use_sha256_oaep_hash();
+    eval { $rsa->encrypt($plaintext) };
+    like($@, qr/OAEP with non-SHA1 hash requires OpenSSL 3\.0/,
+         "non-SHA1 OAEP croaks on pre-3.x");
+}
+
+done_testing;