The problem of rsa key-generation getting stuck in a loop for (pointlessly)

[openssl.git] / crypto / rsa / rsa_gen.c

diff --git a/crypto/rsa/rsa_gen.c b/crypto/rsa/rsa_gen.c
index a45b9aab5c3c169225119d63e564fbec2d0d0b80..6f4b8db2c12197f159f275d4b88539395070bd0f 100644 (file)
--- a/crypto/rsa/rsa_gen.c
+++ b/crypto/rsa/rsa_gen.c
@@ -68,10 +68,24 @@
 #include <openssl/bn.h>
 #include <openssl/rsa.h>
 
-int RSA_generate_key_ex(RSA *rsa, int bits, unsigned long e_value, BN_GENCB *cb)
+static int rsa_builtin_keygen(RSA *rsa, int bits, BIGNUM *e_value, BN_GENCB *cb);
+
+/* NB: this wrapper would normally be placed in rsa_lib.c and the static
+ * implementation would probably be in rsa_eay.c. Nonetheless, is kept here so
+ * that we don't introduce a new linker dependency. Eg. any application that
+ * wasn't previously linking object code related to key-generation won't have to
+ * now just because key-generation is part of RSA_METHOD. */
+int RSA_generate_key_ex(RSA *rsa, int bits, BIGNUM *e_value, BN_GENCB *cb)
+       {
+       if(rsa->meth->rsa_keygen)
+               return rsa->meth->rsa_keygen(rsa, bits, e_value, cb);
+       return rsa_builtin_keygen(rsa, bits, e_value, cb);
+       }
+
+static int rsa_builtin_keygen(RSA *rsa, int bits, BIGNUM *e_value, BN_GENCB *cb)
        {
        BIGNUM *r0=NULL,*r1=NULL,*r2=NULL,*r3=NULL,*tmp;
-       int bitsp,bitsq,ok= -1,n=0,i;
+       int bitsp,bitsq,ok= -1,n=0;
        BN_CTX *ctx=NULL,*ctx2=NULL;
 
        ctx=BN_CTX_new();
@@ -98,17 +112,7 @@ int RSA_generate_key_ex(RSA *rsa, int bits, unsigned long e_value, BN_GENCB *cb)
        if(!rsa->dmq1 && ((rsa->dmq1=BN_new()) == NULL)) goto err;
        if(!rsa->iqmp && ((rsa->iqmp=BN_new()) == NULL)) goto err;
 
-#if 1
-       /* The problem is when building with 8, 16, or 32 BN_ULONG,
-        * unsigned long can be larger */
-       for (i=0; i<sizeof(unsigned long)*8; i++)
-               {
-               if (e_value & (1UL<<i))
-                       BN_set_bit(rsa->e,i);
-               }
-#else
-       if (!BN_set_word(rsa->e,e_value)) goto err;
-#endif
+       BN_copy(rsa->e, e_value);
 
        /* generate p and q */
        for (;;)
@@ -125,11 +129,24 @@ int RSA_generate_key_ex(RSA *rsa, int bits, unsigned long e_value, BN_GENCB *cb)
                goto err;
        for (;;)
                {
-               if(!BN_generate_prime_ex(rsa->q, bitsq, 0, NULL, NULL, cb))
+               /* When generating ridiculously small keys, we can get stuck
+                * continually regenerating the same prime values. Check for
+                * this and bail if it happens 3 times. */
+               unsigned int degenerate = 0;
+               do
+                       {
+                       if(!BN_generate_prime_ex(rsa->q, bitsq, 0, NULL, NULL, cb))
+                               goto err;
+                       } while((BN_cmp(rsa->p, rsa->q) == 0) && (++degenerate < 3));
+               if(degenerate == 3)
+                       {
+                       ok = 0; /* we set our own err */
+                       RSAerr(RSA_F_RSA_GENERATE_KEY,RSA_R_KEY_SIZE_TOO_SMALL);
                        goto err;
+                       }
                if (!BN_sub(r2,rsa->q,BN_value_one())) goto err;
                if (!BN_gcd(r1,r2,rsa->e,ctx)) goto err;
-               if (BN_is_one(r1) && (BN_cmp(rsa->p,rsa->q) != 0))
+               if (BN_is_one(r1))
                        break;
                if(!BN_GENCB_call(cb, 2, n++))
                        goto err;
@@ -166,22 +183,16 @@ int RSA_generate_key_ex(RSA *rsa, int bits, unsigned long e_value, BN_GENCB *cb)
                goto err;
                }
 */
-       rsa->d=BN_mod_inverse(NULL,rsa->e,r0,ctx2);     /* d */
-       if (rsa->d == NULL) goto err;
+       if (!BN_mod_inverse(rsa->d,rsa->e,r0,ctx2)) goto err;   /* d */
 
        /* calculate d mod (p-1) */
-       rsa->dmp1=BN_new();
-       if (rsa->dmp1 == NULL) goto err;
        if (!BN_mod(rsa->dmp1,rsa->d,r1,ctx)) goto err;
 
        /* calculate d mod (q-1) */
-       rsa->dmq1=BN_new();
-       if (rsa->dmq1 == NULL) goto err;
        if (!BN_mod(rsa->dmq1,rsa->d,r2,ctx)) goto err;
 
        /* calculate inverse of q mod p */
-       rsa->iqmp=BN_mod_inverse(NULL,rsa->q,rsa->p,ctx2);
-       if (rsa->iqmp == NULL) goto err;
+       if (!BN_mod_inverse(rsa->iqmp,rsa->q,rsa->p,ctx2)) goto err;
 
        ok=1;
 err: