Update RSA keygen to use sp800-56b by default

Fixes #11742
Fixes #11764

The newer RSA sp800-56b algorithm is being used for the normal case of a non multiprime key of at least length 2048.
Insecure key lengths and mutltiprime RSA will use the old method.

Bad public exponents are no longer allowed (i.e values less than 65537 or even). Values such as 2 that would cause a infinite loop now result in an error. The value of 3 has been marked as deprecated but is still allowed for legacy purposes.

Reviewed-by: Tomas Mraz <tmraz@fedoraproject.org>
(Merged from https://github.com/openssl/openssl/pull/11765)

diff --git a/apps/genrsa.c b/apps/genrsa.c
index 44ce42880c3e5cbddf1825000d3fe30b52a76b14..9a9130125e7e290e2cd7c00d909bf07ee5323c79 100644 (file)
--- a/apps/genrsa.c
+++ b/apps/genrsa.c
@@ -33,7 +33,10 @@ static int genrsa_cb(EVP_PKEY_CTX *ctx);
 
 typedef enum OPTION_choice {
     OPT_ERR = -1, OPT_EOF = 0, OPT_HELP,
-    OPT_3, OPT_F4, OPT_ENGINE,
+#ifndef OPENSSL_NO_DEPRECATED_3_0
+    OPT_3,
+#endif
+    OPT_F4, OPT_ENGINE,
     OPT_OUT, OPT_PASSOUT, OPT_CIPHER, OPT_PRIMES, OPT_VERBOSE,
     OPT_R_ENUM, OPT_PROV_ENUM
 } OPTION_CHOICE;
@@ -48,9 +51,11 @@ const OPTIONS genrsa_options[] = {
 #endif
 
     OPT_SECTION("Input"),
-    {"3", OPT_3, '-', "Use 3 for the E value"},
-    {"F4", OPT_F4, '-', "Use F4 (0x10001) for the E value"},
-    {"f4", OPT_F4, '-', "Use F4 (0x10001) for the E value"},
+#ifndef OPENSSL_NO_DEPRECATED_3_0
+    {"3", OPT_3, '-', "(deprecated) Use 3 for the E value"},
+#endif
+    {"F4", OPT_F4, '-', "Use the Fermat number F4 (0x10001) for the E value"},
+    {"f4", OPT_F4, '-', "Use the Fermat number F4 (0x10001) for the E value"},
 
     OPT_SECTION("Output"),
     {"out", OPT_OUT, '>', "Output the key to specified file"},
@@ -100,9 +105,11 @@ opthelp:
             ret = 0;
             opt_help(genrsa_options);
             goto end;
+#ifndef OPENSSL_NO_DEPRECATED_3_0
         case OPT_3:
             f4 = RSA_3;
             break;
+#endif
         case OPT_F4:
             f4 = RSA_F4;
             break;

diff --git a/crypto/rsa/rsa_gen.c b/crypto/rsa/rsa_gen.c
index 5712aa1791b973d6a57b688a5b8036fad26b53fe..e52bce63551d8e1ea121cdba268e59e2264d3f54 100644 (file)
--- a/crypto/rsa/rsa_gen.c
+++ b/crypto/rsa/rsa_gen.c
@@ -70,16 +70,10 @@ int RSA_generate_multi_prime_key(RSA *rsa, int bits, int primes,
     return rsa_keygen(NULL, rsa, bits, primes, e_value, cb, 0);
 }
 
-static int rsa_keygen(OPENSSL_CTX *libctx, RSA *rsa, int bits, int primes,
-                      BIGNUM *e_value, BN_GENCB *cb, int pairwise_test)
+#ifndef FIPS_MODULE
+static int rsa_multiprime_keygen(RSA *rsa, int bits, int primes,
+                                 BIGNUM *e_value, BN_GENCB *cb)
 {
-    int ok = -1;
-#ifdef FIPS_MODULE
-    if (primes != 2)
-        return 0;
-    ok = rsa_sp800_56b_generate_key(rsa, bits, e_value, cb);
-    pairwise_test = 1; /* FIPS MODE needs to always run the pairwise test */
-#else
     BIGNUM *r0 = NULL, *r1 = NULL, *r2 = NULL, *tmp, *prime;
     int n = 0, bitsr[RSA_MAX_PRIME_NUM], bitse = 0;
     int i = 0, quo = 0, rmd = 0, adj = 0, retries = 0;
@@ -88,6 +82,7 @@ static int rsa_keygen(OPENSSL_CTX *libctx, RSA *rsa, int bits, int primes,
     BN_CTX *ctx = NULL;
     BN_ULONG bitst = 0;
     unsigned long error = 0;
+    int ok = -1;
 
     if (bits < RSA_MIN_MODULUS_BITS) {
         ok = 0;             /* we set our own err */
@@ -95,6 +90,12 @@ static int rsa_keygen(OPENSSL_CTX *libctx, RSA *rsa, int bits, int primes,
         goto err;
     }
 
+    /* A bad value for e can cause infinite loops */
+    if (e_value != NULL && !rsa_check_public_exponent(e_value)) {
+        RSAerr(0, RSA_R_PUB_EXPONENT_OUT_OF_RANGE);
+        return 0;
+    }
+
     if (primes < RSA_DEFAULT_PRIME_NUM || primes > rsa_multip_cap(bits)) {
         ok = 0;             /* we set our own err */
         RSAerr(0, RSA_R_KEY_PRIME_NUM_INVALID);
@@ -407,8 +408,29 @@ static int rsa_keygen(OPENSSL_CTX *libctx, RSA *rsa, int bits, int primes,
     }
     BN_CTX_end(ctx);
     BN_CTX_free(ctx);
+    return ok;
+}
 #endif /* FIPS_MODULE */
 
+static int rsa_keygen(OPENSSL_CTX *libctx, RSA *rsa, int bits, int primes,
+                      BIGNUM *e_value, BN_GENCB *cb, int pairwise_test)
+{
+    int ok = 0;
+
+    /*
+     * Only multi-prime keys or insecure keys with a small key length will use
+     * the older rsa_multiprime_keygen().
+     */
+    if (primes == 2 && bits >= 2048)
+        ok = rsa_sp800_56b_generate_key(rsa, bits, e_value, cb);
+#ifndef FIPS_MODULE
+    else
+        ok = rsa_multiprime_keygen(rsa, bits, primes, e_value, cb);
+#endif /* FIPS_MODULE */
+
+#ifdef FIPS_MODULE
+    pairwise_test = 1; /* FIPS MODE needs to always run the pairwise test */
+#endif
     if (pairwise_test && ok > 0) {
         OSSL_CALLBACK *stcb = NULL;
         void *stcbarg = NULL;

diff --git a/crypto/rsa/rsa_sp800_56b_check.c b/crypto/rsa/rsa_sp800_56b_check.c
index 7cf6241dee6126177f06d95806dfc366690ad2ff..df9b7bf054546fc840c5dff14d471ba40cb86cd7 100644 (file)
--- a/crypto/rsa/rsa_sp800_56b_check.c
+++ b/crypto/rsa/rsa_sp800_56b_check.c
@@ -189,12 +189,30 @@ int rsa_check_private_exponent(const RSA *rsa, int nbits, BN_CTX *ctx)
     return ret;
 }
 
+#ifndef FIPS_MODULE
+static int bn_is_three(const BIGNUM *bn)
+{
+    BIGNUM *num = BN_dup(bn);
+    int ret = (num != NULL && BN_sub_word(num, 3) && BN_is_zero(num));
+
+    BN_free(num);
+    return ret;
+}
+#endif /* FIPS_MODULE */
+
 /* Check exponent is odd, and has a bitlen ranging from [17..256] */
 int rsa_check_public_exponent(const BIGNUM *e)
 {
-    int bitlen = BN_num_bits(e);
+    int bitlen;
+
+    /* For legacy purposes RSA_3 is allowed in non fips mode */
+#ifndef FIPS_MODULE
+    if (bn_is_three(e))
+        return 1;
+#endif /* FIPS_MODULE */
 
-    return (BN_is_odd(e) &&  bitlen > 16 && bitlen < 257);
+    bitlen = BN_num_bits(e);
+    return (BN_is_odd(e) && bitlen > 16 && bitlen < 257);
 }
 
 /*

diff --git a/crypto/rsa/rsa_sp800_56b_gen.c b/crypto/rsa/rsa_sp800_56b_gen.c
index c4c7c08e945df3cee1ab581efc75937ba85c35d0..d1673d5c9899d1986252b9a0b1534bcb54b64a08 100644 (file)
--- a/crypto/rsa/rsa_sp800_56b_gen.c
+++ b/crypto/rsa/rsa_sp800_56b_gen.c
@@ -65,7 +65,7 @@ int rsa_fips186_4_gen_prob_primes(RSA *rsa, BIGNUM *p1, BIGNUM *p2,
      * Signature Generation and Key Agree/Transport.
      */
     if (nbits < RSA_FIPS1864_MIN_KEYGEN_KEYSIZE) {
-        RSAerr(RSA_F_RSA_FIPS186_4_GEN_PROB_PRIMES, RSA_R_INVALID_KEY_LENGTH);
+        RSAerr(RSA_F_RSA_FIPS186_4_GEN_PROB_PRIMES, RSA_R_KEY_SIZE_TOO_SMALL);
         return 0;
     }
 
@@ -146,12 +146,13 @@ err:
 int rsa_sp800_56b_validate_strength(int nbits, int strength)
 {
     int s = (int)ifc_ffc_compute_security_bits(nbits);
-
+#ifdef FIPS_MODULE
     if (s < RSA_FIPS1864_MIN_KEYGEN_STRENGTH
             || s > RSA_FIPS1864_MAX_KEYGEN_STRENGTH) {
         RSAerr(RSA_F_RSA_SP800_56B_VALIDATE_STRENGTH, RSA_R_INVALID_MODULUS);
         return 0;
     }
+#endif
     if (strength != -1 && s != strength) {
         RSAerr(RSA_F_RSA_SP800_56B_VALIDATE_STRENGTH, RSA_R_INVALID_STRENGTH);
         return 0;

diff --git a/doc/man1/openssl-genrsa.pod.in b/doc/man1/openssl-genrsa.pod.in
index 89ae929ced50c67dd2840e829897b94d999044b0..33aa60ca4efc08842784ed1acdfc6c67997ff7fa 100644 (file)
--- a/doc/man1/openssl-genrsa.pod.in
+++ b/doc/man1/openssl-genrsa.pod.in
@@ -33,7 +33,7 @@ B<openssl> B<genrsa>
 {- $OpenSSL::safe::opt_provider_synopsis -}
 [B<numbits>]
 
-=for openssl ifdef engine
+=for openssl ifdef engine 3
 
 =head1 DESCRIPTION
 
@@ -70,6 +70,7 @@ for if it is not supplied via the B<-passout> argument.
 =item B<-F4>, B<-f4>, B<-3>
 
 The public exponent to use, either 65537 or 3. The default is 65537.
+The B<-3> option has been deprecated.
 
 =item B<-primes> I<num>
 

diff --git a/doc/man7/EVP_PKEY-RSA.pod b/doc/man7/EVP_PKEY-RSA.pod
index ad492844379a8df6d2f740dd16e5587cc7883618..28f61e406626425e6536d94d0bc48f403fef970b 100644 (file)
--- a/doc/man7/EVP_PKEY-RSA.pod
+++ b/doc/man7/EVP_PKEY-RSA.pod
@@ -123,6 +123,12 @@ default is 2.  It isn't permitted to specify a larger number of primes than
 being generated so the maximum number could be less.
 Some providers may only support a value of 2.
 
+=item "e" (B<OSSL_PKEY_PARAM_RSA_E>) <unsigned integer>
+
+The RSA "e" value. The value may be any odd number greater than or equal to
+65537. The default value is 65537.
+For legacy reasons a value of 3 is currently accepted but is deprecated.
+
 =back
 
 =head1 CONFORMING TO

diff --git a/include/openssl/rsa.h b/include/openssl/rsa.h
index bf12b90088027e90beeabd58f863dc688611eec2..140c0d4412619d7c553ed33ef916001ef8fb426a 100644 (file)
--- a/include/openssl/rsa.h
+++ b/include/openssl/rsa.h
@@ -40,7 +40,7 @@ extern "C" {
 #  ifndef OPENSSL_NO_DEPRECATED_3_0
 /* The types RSA and RSA_METHOD are defined in ossl_typ.h */
 
-#   define OPENSSL_RSA_FIPS_MIN_MODULUS_BITS 1024
+#   define OPENSSL_RSA_FIPS_MIN_MODULUS_BITS 2048
 
 #   ifndef OPENSSL_RSA_SMALL_MODULUS_BITS
 #    define OPENSSL_RSA_SMALL_MODULUS_BITS 3072

diff --git a/test/recipes/15-test_genrsa.t b/test/recipes/15-test_genrsa.t
index bfe000a26d8324c65dc6f488be67276328543c96..17b0cbc1a00c64651784b8a6b49fcc1160b597dc 100644 (file)
--- a/test/recipes/15-test_genrsa.t
+++ b/test/recipes/15-test_genrsa.t
@@ -16,7 +16,7 @@ use OpenSSL::Test::Utils;
 
 setup("test_genrsa");
 
-plan tests => 9;
+plan tests => 12;
 
 # We want to know that an absurdly small number of bits isn't support
 if (disabled("deprecated-3.0")) {
@@ -43,7 +43,7 @@ while ($good > $bad + 1) {
     my $bits = 2 ** $checked;
     if (disabled("deprecated-3.0")) {
         $fin = run(app([ 'openssl', 'genpkey', '-out', 'genrsatest.pem',
-                         '-algorithm', 'RSA', '-pkeyopt', 'rsa_keygen_pubexp:3',
+                         '-algorithm', 'RSA', '-pkeyopt', 'rsa_keygen_pubexp:65537',
                          '-pkeyopt', "rsa_keygen_bits:$bits",
                        ], stderr => undef));
     } else {
@@ -64,7 +64,7 @@ $good = 2 ** $good;
 note "Found lowest allowed amount of bits to be $good";
 
 ok(run(app([ 'openssl', 'genpkey', '-algorithm', 'RSA',
-             '-pkeyopt', 'rsa_keygen_pubexp:3',
+             '-pkeyopt', 'rsa_keygen_pubexp:65537',
              '-pkeyopt', "rsa_keygen_bits:$good",
              '-out', 'genrsatest.pem' ])),
    "genpkey -3 $good");
@@ -75,9 +75,24 @@ ok(run(app([ 'openssl', 'genpkey', '-algorithm', 'RSA',
              '-pkeyopt', "rsa_keygen_bits:$good",
              '-out', 'genrsatest.pem' ])),
    "genpkey -f4 $good");
-ok(run(app([ 'openssl', 'pkey', '-check', '-in', 'genrsatest.pem', '-noout' ])),
+
+ok(run(app([ 'openssl', 'genpkey', '-algorithm', 'RSA',
+             '-pkeyopt', 'rsa_keygen_bits:2048',
+             '-out', 'genrsatest2048.pem' ])),
+   "genpkey 2048 bits");
+ok(run(app([ 'openssl', 'pkey', '-check', '-in', 'genrsatest2048.pem', '-noout' ])),
    "pkey -check");
 
+ok(!run(app([ 'openssl', 'genpkey', '-algorithm', 'RSA',
+             '-pkeyopt', 'hexe:02',
+             '-out', 'genrsatest.pem' ])),
+   "genpkey with a bad public exponent should fail");
+ok(!run(app([ 'openssl', 'genpkey', '-algorithm', 'RSA',
+             '-pkeyopt', 'e:65538',
+             '-out', 'genrsatest.pem' ])),
+   "genpkey with a even public exponent should fail");
+
+
  SKIP: {
     skip "Skipping rsa command line test", 4 if disabled("deprecated-3.0");