Fix probable_prime over large shift

[openssl.git] / crypto / bn / bn_prime.c

diff --git a/crypto/bn/bn_prime.c b/crypto/bn/bn_prime.c
index b12295e84e4d55bc185999dcbd425425751dba0a..2a7822ef1d907014f48b5db38981263b1cb30e8c 100644 (file)
--- a/crypto/bn/bn_prime.c
+++ b/crypto/bn/bn_prime.c
@@ -518,7 +518,17 @@ static int probable_prime(BIGNUM *rnd, int bits)
      * additionally don't want to exceed that many bits.
      */
     if (is_single_word) {
-        BN_ULONG size_limit = (((BN_ULONG)1) << bits) - BN_get_word(rnd) - 1;
+        BN_ULONG size_limit;
+        
+        if (bits == BN_BITS2) {
+            /*
+             * Shifting by this much has undefined behaviour so we do it a
+             * different way
+             */
+            size_limit = ~((BN_ULONG)0) - BN_get_word(rnd);
+        } else {
+            size_limit = (((BN_ULONG)1) << bits) - BN_get_word(rnd) - 1;
+        }
         if (size_limit < maxdelta)
             maxdelta = size_limit;
     }