evp/pbe_scrypt.c: add boundary condition for implicit cast.

[openssl.git] / crypto / evp / pbe_scrypt.c

diff --git a/crypto/evp/pbe_scrypt.c b/crypto/evp/pbe_scrypt.c
index 101bb1edbd482f0b93f703bc571bb7ecce85999a..fddabc9a3270442514683335ec0eab41e306a61e 100644 (file)
--- a/crypto/evp/pbe_scrypt.c
+++ b/crypto/evp/pbe_scrypt.c
@@ -12,7 +12,7 @@
 #include <string.h>
 #include <openssl/evp.h>
 #include <openssl/err.h>
-#include <internal/numbers.h>
+#include "internal/numbers.h"
 
 #ifndef OPENSSL_NO_SCRYPT
 
@@ -164,7 +164,6 @@ int EVP_PBE_scrypt(const char *pass, size_t passlen,
     unsigned char *B;
     uint32_t *X, *V, *T;
     uint64_t i, Blen, Vlen;
-    size_t allocsize;
 
     /* Sanity check parameters */
     /* initial check, r,p must be non zero, N >= 2 and a power of 2 */
@@ -192,10 +191,17 @@ int EVP_PBE_scrypt(const char *pass, size_t passlen,
      * p * r < SCRYPT_PR_MAX
      */
     Blen = p * 128 * r;
+    /*
+     * Yet we pass it as integer to PKCS5_PBKDF2_HMAC... [This would
+     * have to be revised when/if PKCS5_PBKDF2_HMAC accepts size_t.]
+     */
+    if (Blen > INT_MAX) {
+        EVPerr(EVP_F_EVP_PBE_SCRYPT, EVP_R_MEMORY_LIMIT_EXCEEDED);
+        return 0;
+    }
 
     /*
-     * Check 32 * r * (N + 2) * sizeof(uint32_t) fits in
-     * uint64_t and also size_t (their sizes are unrelated).
+     * Check 32 * r * (N + 2) * sizeof(uint32_t) fits in uint64_t
      * This is combined size V, X and T (section 4)
      */
     i = UINT64_MAX / (32 * sizeof(uint32_t));
@@ -206,16 +212,15 @@ int EVP_PBE_scrypt(const char *pass, size_t passlen,
     /* check total allocated size fits in uint64_t */
     if (Blen > UINT64_MAX - Vlen)
         return 0;
-    /* check total allocated size fits in size_t */
-    if (Blen > SIZE_MAX - Vlen)
-        return 0;
-
-    allocsize = (size_t)(Blen + Vlen);
 
     if (maxmem == 0)
         maxmem = SCRYPT_MAX_MEM;
 
-    if (allocsize > maxmem) {
+    /* Check that the maximum memory doesn't exceed a size_t limits */
+    if (maxmem > SIZE_MAX)
+        maxmem = SIZE_MAX;
+
+    if (Blen + Vlen > maxmem) {
         EVPerr(EVP_F_EVP_PBE_SCRYPT, EVP_R_MEMORY_LIMIT_EXCEEDED);
         return 0;
     }
@@ -224,25 +229,25 @@ int EVP_PBE_scrypt(const char *pass, size_t passlen,
     if (key == NULL)
         return 1;
 
-    B = OPENSSL_malloc(allocsize);
+    B = OPENSSL_malloc((size_t)(Blen + Vlen));
     if (B == NULL)
         return 0;
     X = (uint32_t *)(B + Blen);
     T = X + 32 * r;
     V = T + 32 * r;
     if (PKCS5_PBKDF2_HMAC(pass, passlen, salt, saltlen, 1, EVP_sha256(),
-                          Blen, B) == 0)
+                          (int)Blen, B) == 0)
         goto err;
 
     for (i = 0; i < p; i++)
         scryptROMix(B + 128 * r * i, r, N, X, T, V);
 
-    if (PKCS5_PBKDF2_HMAC(pass, passlen, B, Blen, 1, EVP_sha256(),
+    if (PKCS5_PBKDF2_HMAC(pass, passlen, B, (int)Blen, 1, EVP_sha256(),
                           keylen, key) == 0)
         goto err;
     rv = 1;
  err:
-    OPENSSL_clear_free(B, allocsize);
+    OPENSSL_clear_free(B, (size_t)(Blen + Vlen));
     return rv;
 }
 #endif