Check for malloc error in bn_x931p.c

[openssl.git] / crypto / bn / bn_x931p.c

diff --git a/crypto/bn/bn_x931p.c b/crypto/bn/bn_x931p.c
index 83170d49196c477548367723c430aa9d8d019a6b..40734cb2f69f09a657a775ae080ca3e3fa5fe576 100644 (file)
--- a/crypto/bn/bn_x931p.c
+++ b/crypto/bn/bn_x931p.c
@@ -21,7 +21,7 @@
 static int bn_x931_derive_pi(BIGNUM *pi, const BIGNUM *Xpi, BN_CTX *ctx,
                              BN_GENCB *cb)
 {
-    int i = 0;
+    int i = 0, is_prime;
     if (!BN_copy(pi, Xpi))
         return 0;
     if (!BN_is_odd(pi) && !BN_add_word(pi, 1))
@@ -30,7 +30,10 @@ static int bn_x931_derive_pi(BIGNUM *pi, const BIGNUM *Xpi, BN_CTX *ctx,
         i++;
         BN_GENCB_call(cb, 0, i);
         /* NB 27 MR is specified in X9.31 */
-        if (BN_is_prime_fasttest_ex(pi, 27, ctx, 1, cb))
+        is_prime = BN_is_prime_fasttest_ex(pi, 27, ctx, 1, cb);
+        if (is_prime < 0)
+            return 0;
+        if (is_prime)
             break;
         if (!BN_add_word(pi, 2))
             return 0;
@@ -71,6 +74,9 @@ int BN_X931_derive_prime_ex(BIGNUM *p, BIGNUM *p1, BIGNUM *p2,
 
     pm1 = BN_CTX_get(ctx);
 
+    if (pm1 == NULL)
+        goto err;
+
     if (!bn_x931_derive_pi(p1, Xp1, ctx, cb))
         goto err;
 
@@ -119,14 +125,18 @@ int BN_X931_derive_prime_ex(BIGNUM *p, BIGNUM *p1, BIGNUM *p2,
             goto err;
         if (!BN_gcd(t, pm1, e, ctx))
             goto err;
-        if (BN_is_one(t)
+        if (BN_is_one(t)) {
             /*
              * X9.31 specifies 8 MR and 1 Lucas test or any prime test
              * offering similar or better guarantees 50 MR is considerably
              * better.
              */
-            && BN_is_prime_fasttest_ex(p, 50, ctx, 1, cb))
-            break;
+            int r = BN_is_prime_fasttest_ex(p, 50, ctx, 1, cb);
+            if (r < 0)
+                goto err;
+            if (r)
+                break;
+        }
         if (!BN_add(p, p, p1p2))
             goto err;
     }
@@ -163,14 +173,14 @@ int BN_X931_generate_Xpq(BIGNUM *Xp, BIGNUM *Xq, int nbits, BN_CTX *ctx)
      * - 1. By setting the top two bits we ensure that the lower bound is
      * exceeded.
      */
-    if (!BN_rand(Xp, nbits, 1, 0))
+    if (!BN_rand(Xp, nbits, BN_RAND_TOP_TWO, BN_RAND_BOTTOM_ANY))
         goto err;
 
     BN_CTX_start(ctx);
     t = BN_CTX_get(ctx);
 
     for (i = 0; i < 1000; i++) {
-        if (!BN_rand(Xq, nbits, 1, 0))
+        if (!BN_rand(Xq, nbits, BN_RAND_TOP_TWO, BN_RAND_BOTTOM_ANY))
             goto err;
         /* Check that |Xp - Xq| > 2^(nbits - 100) */
         BN_sub(t, Xp, Xq);
@@ -211,9 +221,9 @@ int BN_X931_generate_prime_ex(BIGNUM *p, BIGNUM *p1, BIGNUM *p2,
     if (!Xp2)
         Xp2 = BN_CTX_get(ctx);
 
-    if (!BN_rand(Xp1, 101, 0, 0))
+    if (!BN_rand(Xp1, 101, BN_RAND_TOP_ONE, BN_RAND_BOTTOM_ANY))
         goto error;
-    if (!BN_rand(Xp2, 101, 0, 0))
+    if (!BN_rand(Xp2, 101, BN_RAND_TOP_ONE, BN_RAND_BOTTOM_ANY))
         goto error;
     if (!BN_X931_derive_prime_ex(p, p1, p2, Xp, Xp1, Xp2, e, ctx, cb))
         goto error;