RSA key generation: ensure BN_mod_inverse and BN_mod_exp_mont both get called with...

[openssl.git] / crypto / rsa / rsa_gen.c

diff --git a/crypto/rsa/rsa_gen.c b/crypto/rsa/rsa_gen.c
index 0d1d56bfe75fae77ef76c66f6f24d5627663e9c9..79f77e3eafdf561d810e7d44341ae53cbbf3cd43 100644 (file)
--- a/crypto/rsa/rsa_gen.c
+++ b/crypto/rsa/rsa_gen.c
@@ -1,5 +1,5 @@
 /*
- * Copyright 1995-2016 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 1995-2018 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the OpenSSL license (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -42,6 +42,17 @@ static int rsa_builtin_keygen(RSA *rsa, int bits, BIGNUM *e_value,
     BIGNUM *r0 = NULL, *r1 = NULL, *r2 = NULL, *r3 = NULL, *tmp;
     int bitsp, bitsq, ok = -1, n = 0;
     BN_CTX *ctx = NULL;
+    unsigned long error = 0;
+
+    /*
+     * When generating ridiculously small keys, we can get stuck
+     * continually regenerating the same prime values.
+     */
+    if (bits < 16) {
+        ok = 0;             /* we set our own err */
+        RSAerr(RSA_F_RSA_BUILTIN_KEYGEN, RSA_R_KEY_SIZE_TOO_SMALL);
+        goto err;
+    }
 
     ctx = BN_CTX_new();
     if (ctx == NULL)
@@ -78,43 +89,53 @@ static int rsa_builtin_keygen(RSA *rsa, int bits, BIGNUM *e_value,
     if (BN_copy(rsa->e, e_value) == NULL)
         goto err;
 
+    BN_set_flags(rsa->p, BN_FLG_CONSTTIME);
+    BN_set_flags(rsa->q, BN_FLG_CONSTTIME);
+    BN_set_flags(r2, BN_FLG_CONSTTIME);
     /* generate p and q */
     for (;;) {
         if (!BN_generate_prime_ex(rsa->p, bitsp, 0, NULL, NULL, cb))
             goto err;
         if (!BN_sub(r2, rsa->p, BN_value_one()))
             goto err;
-        if (!BN_gcd(r1, r2, rsa->e, ctx))
-            goto err;
-        if (BN_is_one(r1))
+        ERR_set_mark();
+        if (BN_mod_inverse(r1, r2, rsa->e, ctx) != NULL) {
+            /* GCD == 1 since inverse exists */
             break;
+        }
+        error = ERR_peek_last_error();
+        if (ERR_GET_LIB(error) == ERR_LIB_BN
+            && ERR_GET_REASON(error) == BN_R_NO_INVERSE) {
+            /* GCD != 1 */
+            ERR_pop_to_mark();
+        } else {
+            goto err;
+        }
         if (!BN_GENCB_call(cb, 2, n++))
             goto err;
     }
     if (!BN_GENCB_call(cb, 3, 0))
         goto err;
     for (;;) {
-        /*
-         * When generating ridiculously small keys, we can get stuck
-         * continually regenerating the same prime values. Check for this and
-         * bail if it happens 3 times.
-         */
-        unsigned int degenerate = 0;
         do {
             if (!BN_generate_prime_ex(rsa->q, bitsq, 0, NULL, NULL, cb))
                 goto err;
-        } while ((BN_cmp(rsa->p, rsa->q) == 0) && (++degenerate < 3));
-        if (degenerate == 3) {
-            ok = 0;             /* we set our own err */
-            RSAerr(RSA_F_RSA_BUILTIN_KEYGEN, RSA_R_KEY_SIZE_TOO_SMALL);
-            goto err;
-        }
+        } while (BN_cmp(rsa->p, rsa->q) == 0);
         if (!BN_sub(r2, rsa->q, BN_value_one()))
             goto err;
-        if (!BN_gcd(r1, r2, rsa->e, ctx))
-            goto err;
-        if (BN_is_one(r1))
+        ERR_set_mark();
+        if (BN_mod_inverse(r1, r2, rsa->e, ctx) != NULL) {
+            /* GCD == 1 since inverse exists */
             break;
+        }
+        error = ERR_peek_last_error();
+        if (ERR_GET_LIB(error) == ERR_LIB_BN
+            && ERR_GET_REASON(error) == BN_R_NO_INVERSE) {
+            /* GCD != 1 */
+            ERR_pop_to_mark();
+        } else {
+            goto err;
+        }
         if (!BN_GENCB_call(cb, 2, n++))
             goto err;
     }