Fix BN_is_prime* calls.

This function returns a tri-state -1 on error. See BoringSSL's
53409ee3d7595ed37da472bc73b010cd2c8a5ffd.

Signed-off-by: Kurt Roeckx <kurt@roeckx.be>
Reviewed-by: Rich Salz <rsalz@openssl.org>
GH: #1251

diff --git a/apps/s_client.c b/apps/s_client.c
index 56a7081cab8f2b6f69a35c72713155a99c6448f1..e79cf7e496e3161124f441699c6bf24091780a0c 100644 (file)
--- a/apps/s_client.c
+++ b/apps/s_client.c
@@ -249,10 +249,10 @@ static int srp_Verify_N_and_g(const BIGNUM *N, const BIGNUM *g)
     BIGNUM *r = BN_new();
     int ret =
         g != NULL && N != NULL && bn_ctx != NULL && BN_is_odd(N) &&
     BIGNUM *r = BN_new();
     int ret =
         g != NULL && N != NULL && bn_ctx != NULL && BN_is_odd(N) &&

-        BN_is_prime_ex(N, SRP_NUMBER_ITERATIONS_FOR_PRIME, bn_ctx, NULL) &&
+        BN_is_prime_ex(N, SRP_NUMBER_ITERATIONS_FOR_PRIME, bn_ctx, NULL) == 1 &&

         p != NULL && BN_rshift1(p, N) &&
         /* p = (N-1)/2 */
         p != NULL && BN_rshift1(p, N) &&
         /* p = (N-1)/2 */

-        BN_is_prime_ex(p, SRP_NUMBER_ITERATIONS_FOR_PRIME, bn_ctx, NULL) &&
+        BN_is_prime_ex(p, SRP_NUMBER_ITERATIONS_FOR_PRIME, bn_ctx, NULL) == 1 &&

         r != NULL &&
         /* verify g^((N-1)/2) == -1 (mod N) */
         BN_mod_exp(r, g, p, N, bn_ctx) &&
         r != NULL &&
         /* verify g^((N-1)/2) == -1 (mod N) */
         BN_mod_exp(r, g, p, N, bn_ctx) &&

diff --git a/crypto/bn/bn_x931p.c b/crypto/bn/bn_x931p.c
index 83170d49196c477548367723c430aa9d8d019a6b..d863386ef4212482a17296b085afda412609af40 100644 (file)
--- a/crypto/bn/bn_x931p.c
+++ b/crypto/bn/bn_x931p.c
@@ -21,7 +21,7 @@
 static int bn_x931_derive_pi(BIGNUM *pi, const BIGNUM *Xpi, BN_CTX *ctx,
                              BN_GENCB *cb)
 {
 static int bn_x931_derive_pi(BIGNUM *pi, const BIGNUM *Xpi, BN_CTX *ctx,
                              BN_GENCB *cb)
 {

-    int i = 0;
+    int i = 0, is_prime;

     if (!BN_copy(pi, Xpi))
         return 0;
     if (!BN_is_odd(pi) && !BN_add_word(pi, 1))
     if (!BN_copy(pi, Xpi))
         return 0;
     if (!BN_is_odd(pi) && !BN_add_word(pi, 1))


@@ -30,7 +30,10 @@ static int bn_x931_derive_pi(BIGNUM *pi, const BIGNUM *Xpi, BN_CTX *ctx,
         i++;
         BN_GENCB_call(cb, 0, i);
         /* NB 27 MR is specified in X9.31 */
         i++;
         BN_GENCB_call(cb, 0, i);
         /* NB 27 MR is specified in X9.31 */

-        if (BN_is_prime_fasttest_ex(pi, 27, ctx, 1, cb))
+        is_prime = BN_is_prime_fasttest_ex(pi, 27, ctx, 1, cb);
+        if (is_prime < 0)
+            return 0;
+        if (is_prime)

             break;
         if (!BN_add_word(pi, 2))
             return 0;
             break;
         if (!BN_add_word(pi, 2))
             return 0;


@@ -119,14 +122,18 @@ int BN_X931_derive_prime_ex(BIGNUM *p, BIGNUM *p1, BIGNUM *p2,
             goto err;
         if (!BN_gcd(t, pm1, e, ctx))
             goto err;
             goto err;
         if (!BN_gcd(t, pm1, e, ctx))
             goto err;

-        if (BN_is_one(t)
+        if (BN_is_one(t)) {

             /*
              * X9.31 specifies 8 MR and 1 Lucas test or any prime test
              * offering similar or better guarantees 50 MR is considerably
              * better.
              */
             /*
              * X9.31 specifies 8 MR and 1 Lucas test or any prime test
              * offering similar or better guarantees 50 MR is considerably
              * better.
              */

-            && BN_is_prime_fasttest_ex(p, 50, ctx, 1, cb))
-            break;
+            int r = BN_is_prime_fasttest_ex(p, 50, ctx, 1, cb);
+            if (r < 0)
+                goto err;
+            if (r)
+                break;
+        }

         if (!BN_add(p, p, p1p2))
             goto err;
     }
         if (!BN_add(p, p, p1p2))
             goto err;
     }

diff --git a/crypto/dh/dh_check.c b/crypto/dh/dh_check.c
index 523e31d05e1c1f7c35c596bf178d8ff4bd8ea183..8d2e096c08ed57bbea9ae31a69e18f81f8eef2ea 100644 (file)
--- a/crypto/dh/dh_check.c
+++ b/crypto/dh/dh_check.c
@@ -24,7 +24,7 @@
 
 int DH_check(const DH *dh, int *ret)
 {
 
 int DH_check(const DH *dh, int *ret)
 {

-    int ok = 0;
+    int ok = 0, r;

     BN_CTX *ctx = NULL;
     BN_ULONG l;
     BIGNUM *t1 = NULL, *t2 = NULL;
     BN_CTX *ctx = NULL;
     BN_ULONG l;
     BIGNUM *t1 = NULL, *t2 = NULL;


@@ -53,7 +53,10 @@ int DH_check(const DH *dh, int *ret)
             if (!BN_is_one(t1))
                 *ret |= DH_NOT_SUITABLE_GENERATOR;
         }
             if (!BN_is_one(t1))
                 *ret |= DH_NOT_SUITABLE_GENERATOR;
         }

-        if (!BN_is_prime_ex(dh->q, BN_prime_checks, ctx, NULL))
+        r = BN_is_prime_ex(dh->q, BN_prime_checks, ctx, NULL);
+        if (r < 0)
+            goto err;
+        if (!r)

             *ret |= DH_CHECK_Q_NOT_PRIME;
         /* Check p == 1 mod q  i.e. q divides p - 1 */
         if (!BN_div(t1, t2, dh->p, dh->q, ctx))
             *ret |= DH_CHECK_Q_NOT_PRIME;
         /* Check p == 1 mod q  i.e. q divides p - 1 */
         if (!BN_div(t1, t2, dh->p, dh->q, ctx))


@@ -74,12 +77,18 @@ int DH_check(const DH *dh, int *ret)
     } else
         *ret |= DH_UNABLE_TO_CHECK_GENERATOR;
 
     } else
         *ret |= DH_UNABLE_TO_CHECK_GENERATOR;
 

-    if (!BN_is_prime_ex(dh->p, BN_prime_checks, ctx, NULL))
+    r = BN_is_prime_ex(dh->p, BN_prime_checks, ctx, NULL);
+    if (r < 0)
+        goto err;
+    if (!r)

         *ret |= DH_CHECK_P_NOT_PRIME;
     else if (!dh->q) {
         if (!BN_rshift1(t1, dh->p))
             goto err;
         *ret |= DH_CHECK_P_NOT_PRIME;
     else if (!dh->q) {
         if (!BN_rshift1(t1, dh->p))
             goto err;

-        if (!BN_is_prime_ex(t1, BN_prime_checks, ctx, NULL))
+        r = BN_is_prime_ex(t1, BN_prime_checks, ctx, NULL);
+        if (r < 0)
+            goto err;
+       if (!r)

             *ret |= DH_CHECK_P_NOT_SAFE_PRIME;
     }
     ok = 1;
             *ret |= DH_CHECK_P_NOT_SAFE_PRIME;
     }
     ok = 1;