crypto: updates to pass size_t to RAND_bytes_ex()

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/15540)

diff --git a/crypto/asn1/p5_pbev2.c b/crypto/asn1/p5_pbev2.c
index d16fb8cfe3a64d5fce9c60d186300878e5a73965..162e31d7bab4b7c1b8e53f507a5fa2a1ef86ce6f 100644 (file)
--- a/crypto/asn1/p5_pbev2.c
+++ b/crypto/asn1/p5_pbev2.c
@@ -45,7 +45,7 @@ X509_ALGOR *PKCS5_pbe2_set_iv_ex(const EVP_CIPHER *cipher, int iter,
                                  OSSL_LIB_CTX *libctx)
 {
     X509_ALGOR *scheme = NULL, *ret = NULL;
-    int alg_nid, keylen;
+    int alg_nid, keylen, ivlen;
     EVP_CIPHER_CTX *ctx = NULL;
     unsigned char iv[EVP_MAX_IV_LENGTH];
     PBE2PARAM *pbe2 = NULL;
@@ -66,11 +66,11 @@ X509_ALGOR *PKCS5_pbe2_set_iv_ex(const EVP_CIPHER *cipher, int iter,
         goto merr;
 
     /* Create random IV */
-    if (EVP_CIPHER_iv_length(cipher)) {
+    ivlen = EVP_CIPHER_iv_length(cipher);
+    if (ivlen > 0) {
         if (aiv)
-            memcpy(iv, aiv, EVP_CIPHER_iv_length(cipher));
-        else if (RAND_bytes_ex(libctx, iv, EVP_CIPHER_iv_length(cipher),
-                               0) <= 0)
+            memcpy(iv, aiv, ivlen);
+        else if (RAND_bytes_ex(libctx, iv, ivlen, 0) <= 0)
             goto err;
     }
 

diff --git a/crypto/crmf/crmf_pbm.c b/crypto/crmf/crmf_pbm.c
index 21808d014bf73e0117e7cc3e56e772613835d33b..5641bee65a23dab1eb0dd908b0940dcb4cddcd85 100644 (file)
--- a/crypto/crmf/crmf_pbm.c
+++ b/crypto/crmf/crmf_pbm.c
@@ -55,7 +55,7 @@ OSSL_CRMF_PBMPARAMETER *OSSL_CRMF_pbmp_new(OSSL_LIB_CTX *libctx, size_t slen,
      */
     if ((salt = OPENSSL_malloc(slen)) == NULL)
         goto err;
-    if (RAND_bytes_ex(libctx, salt, (int)slen, 0) <= 0) {
+    if (RAND_bytes_ex(libctx, salt, slen, 0) <= 0) {
         ERR_raise(ERR_LIB_CRMF, CRMF_R_FAILURE_OBTAINING_RANDOM);
         goto err;
     }

diff --git a/crypto/ec/ec_err.c b/crypto/ec/ec_err.c
index 9e21968499ff239a128f47ce0a2c123268704fd2..9dc143c2ac6928e3bc56468d1acb369f5032c6c0 100644 (file)
--- a/crypto/ec/ec_err.c
+++ b/crypto/ec/ec_err.c
@@ -1,6 +1,6 @@
 /*
  * Generated by util/mkerr.pl DO NOT EDIT
- * Copyright 1995-2020 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 1995-2021 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the Apache License 2.0 (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -62,6 +62,7 @@ static const ERR_STRING_DATA EC_str_reasons[] = {
     {ERR_PACK(ERR_LIB_EC, 0, EC_R_INVALID_GENERATOR), "invalid generator"},
     {ERR_PACK(ERR_LIB_EC, 0, EC_R_INVALID_GROUP_ORDER), "invalid group order"},
     {ERR_PACK(ERR_LIB_EC, 0, EC_R_INVALID_KEY), "invalid key"},
+    {ERR_PACK(ERR_LIB_EC, 0, EC_R_INVALID_LENGTH), "invalid length"},
     {ERR_PACK(ERR_LIB_EC, 0, EC_R_INVALID_NAMED_GROUP_CONVERSION),
     "invalid named group conversion"},
     {ERR_PACK(ERR_LIB_EC, 0, EC_R_INVALID_OUTPUT_LENGTH),

diff --git a/crypto/ec/ecp_s390x_nistp.c b/crypto/ec/ecp_s390x_nistp.c
index 4a676c37adce48d94420f5cd665d6ddea5832abf..5c70b2d67840c22a455418a11ed37ecfccb5b7c5 100644 (file)
--- a/crypto/ec/ecp_s390x_nistp.c
+++ b/crypto/ec/ecp_s390x_nistp.c
@@ -173,6 +173,10 @@ static ECDSA_SIG *ecdsa_s390x_nistp_sign_sig(const unsigned char *dgst,
     }
 
     if (r == NULL || kinv == NULL) {
+        if (len < 0) {
+            ERR_raise(ERR_LIB_EC, EC_R_INVALID_LENGTH);
+            goto ret;
+        }
         /*
          * Generate random k and copy to param param block. RAND_priv_bytes_ex
          * is used instead of BN_priv_rand_range or BN_generate_dsa_nonce
@@ -180,7 +184,7 @@ static ECDSA_SIG *ecdsa_s390x_nistp_sign_sig(const unsigned char *dgst,
          * internally implementing counter-measures for RNG weakness.
          */
          if (RAND_priv_bytes_ex(eckey->libctx, param + S390X_OFF_RN(len),
-                                len, 0) != 1) {
+                                (size_t)len, 0) != 1) {
              ERR_raise(ERR_LIB_EC, EC_R_RANDOM_NUMBER_GENERATION_FAILED);
              goto ret;
          }

diff --git a/crypto/err/openssl.txt b/crypto/err/openssl.txt
index 48d1175bcee33842b270c0318b5b232eca605303..eb0ace54745fcaaf058ee6efb2a2c88d3741287d 100644 (file)
--- a/crypto/err/openssl.txt
+++ b/crypto/err/openssl.txt
@@ -555,6 +555,7 @@ EC_R_INVALID_FORM:104:invalid form
 EC_R_INVALID_GENERATOR:173:invalid generator
 EC_R_INVALID_GROUP_ORDER:122:invalid group order
 EC_R_INVALID_KEY:116:invalid key
+EC_R_INVALID_LENGTH:117:invalid length
 EC_R_INVALID_NAMED_GROUP_CONVERSION:174:invalid named group conversion
 EC_R_INVALID_OUTPUT_LENGTH:161:invalid output length
 EC_R_INVALID_P:172:invalid p
@@ -1152,6 +1153,7 @@ RSA_R_INVALID_HEADER:137:invalid header
 RSA_R_INVALID_KEYPAIR:171:invalid keypair
 RSA_R_INVALID_KEY_LENGTH:173:invalid key length
 RSA_R_INVALID_LABEL:160:invalid label
+RSA_R_INVALID_LENGTH:181:invalid length
 RSA_R_INVALID_MESSAGE_LENGTH:131:invalid message length
 RSA_R_INVALID_MGF1_MD:156:invalid mgf1 md
 RSA_R_INVALID_MODULUS:174:invalid modulus

diff --git a/crypto/ffc/ffc_params_generate.c b/crypto/ffc/ffc_params_generate.c
index 3c6f789c3e23e11dfa7f4dd552937a7d3aa3d970..85ae5240154f2ce9e38ce17171b471c1e2084108 100644 (file)
--- a/crypto/ffc/ffc_params_generate.c
+++ b/crypto/ffc/ffc_params_generate.c
@@ -329,7 +329,7 @@ static int generate_q_fips186_4(BN_CTX *ctx, BIGNUM *q, const EVP_MD *evpmd,
 
         /* A.1.1.2 Step (5) : generate seed with size seed_len */
         if (generate_seed
-                && RAND_bytes_ex(libctx, seed, (int)seedlen, 0) < 0)
+                && RAND_bytes_ex(libctx, seed, seedlen, 0) < 0)
             goto err;
         /*
          * A.1.1.2 Step (6) AND
@@ -399,7 +399,7 @@ static int generate_q_fips186_2(BN_CTX *ctx, BIGNUM *q, const EVP_MD *evpmd,
         if (!BN_GENCB_call(cb, 0, m++))
             goto err;
 
-        if (generate_seed && RAND_bytes_ex(libctx, seed, (int)qsize, 0) <= 0)
+        if (generate_seed && RAND_bytes_ex(libctx, seed, qsize, 0) <= 0)
             goto err;
 
         memcpy(buf, seed, qsize);

diff --git a/crypto/pkcs12/p12_mutl.c b/crypto/pkcs12/p12_mutl.c
index 041711d7d42dda91af68b944df929c11e9bab2b2..be4ed16ab72f9853f10b1e05d5da8a5f69a54679 100644 (file)
--- a/crypto/pkcs12/p12_mutl.c
+++ b/crypto/pkcs12/p12_mutl.c
@@ -259,8 +259,10 @@ int PKCS12_setup_mac(PKCS12 *p12, int iter, unsigned char *salt, int saltlen,
     }
     p12->mac->salt->length = saltlen;
     if (!salt) {
+        if (saltlen < 0)
+            return 0;
         if (RAND_bytes_ex(p12->authsafes->ctx.libctx, p12->mac->salt->data,
-                          saltlen, 0) <= 0)
+                          (size_t)saltlen, 0) <= 0)
             return 0;
     } else
         memcpy(p12->mac->salt->data, salt, saltlen);

diff --git a/crypto/rsa/rsa_err.c b/crypto/rsa/rsa_err.c
index 85bee965fcd2268a23335c173dd84b8e2ab417ea..269971c07b8da9b7089660bc298ab6cb3f527a18 100644 (file)
--- a/crypto/rsa/rsa_err.c
+++ b/crypto/rsa/rsa_err.c
@@ -57,6 +57,7 @@ static const ERR_STRING_DATA RSA_str_reasons[] = {
     {ERR_PACK(ERR_LIB_RSA, 0, RSA_R_INVALID_KEYPAIR), "invalid keypair"},
     {ERR_PACK(ERR_LIB_RSA, 0, RSA_R_INVALID_KEY_LENGTH), "invalid key length"},
     {ERR_PACK(ERR_LIB_RSA, 0, RSA_R_INVALID_LABEL), "invalid label"},
+    {ERR_PACK(ERR_LIB_RSA, 0, RSA_R_INVALID_LENGTH), "invalid length"},
     {ERR_PACK(ERR_LIB_RSA, 0, RSA_R_INVALID_MESSAGE_LENGTH),
     "invalid message length"},
     {ERR_PACK(ERR_LIB_RSA, 0, RSA_R_INVALID_MGF1_MD), "invalid mgf1 md"},

diff --git a/crypto/rsa/rsa_oaep.c b/crypto/rsa/rsa_oaep.c
index 5068057fd1e0284dfe072838ca753e37e7646e47..00646648c7654bdf8fea62e26607ce94d57f48b7 100644 (file)
--- a/crypto/rsa/rsa_oaep.c
+++ b/crypto/rsa/rsa_oaep.c
@@ -77,6 +77,10 @@ int ossl_rsa_padding_add_PKCS1_OAEP_mgf1_ex(OSSL_LIB_CTX *libctx,
         mgf1md = md;
 
     mdlen = EVP_MD_size(md);
+    if (mdlen <= 0) {
+        ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_LENGTH);
+        return 0;
+    }
 
     /* step 2b: check KLen > nLen - 2 HLen - 2 */
     if (flen > emlen - 2 * mdlen - 1) {

diff --git a/crypto/rsa/rsa_pk1.c b/crypto/rsa/rsa_pk1.c
index 9094b1ac50228f37adc17a5a258b5e5c3d0e2efe..f1eabf177c95ea3e4a40fc6457c40b142154b845 100644 (file)
--- a/crypto/rsa/rsa_pk1.c
+++ b/crypto/rsa/rsa_pk1.c
@@ -128,6 +128,9 @@ int ossl_rsa_padding_add_PKCS1_type_2_ex(OSSL_LIB_CTX *libctx, unsigned char *to
     if (flen > (tlen - RSA_PKCS1_PADDING_SIZE)) {
         ERR_raise(ERR_LIB_RSA, RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);
         return 0;
+    } else if (flen < 0) {
+        ERR_raise(ERR_LIB_RSA, RSA_R_INVALID_LENGTH);
+        return 0;
     }
 
     p = (unsigned char *)to;