Fix size_t/int mismatch in cms_ec.c and rsa_sig.c

Fixes #20435

Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/20457)

(cherry picked from commit 559e078d94f1213318105b03f4e88b848fc28314)

diff --git a/crypto/cms/cms_dh.c b/crypto/cms/cms_dh.c
index 7cc36f835f21dd693be71d0e3eaeed34ab41d689..2c0712b41a24f4ca7ef25edaf4c2d7eced3fde6d 100644 (file)
--- a/crypto/cms/cms_dh.c
+++ b/crypto/cms/cms_dh.c
@@ -309,7 +309,7 @@ static int dh_cms_encrypt(CMS_RecipientInfo *ri)
      */
     penc = NULL;
     penclen = i2d_X509_ALGOR(wrap_alg, &penc);
-    if (penc == NULL || penclen == 0)
+    if (penclen <= 0)
         goto err;
     wrap_str = ASN1_STRING_new();
     if (wrap_str == NULL)

diff --git a/crypto/cms/cms_ec.c b/crypto/cms/cms_ec.c
index 1ff83d36a80e2870654b6122c0ee82c055bdb297..da237e7877a809fb15cd0ca069f9dcb8f02c02a0 100644 (file)
--- a/crypto/cms/cms_ec.c
+++ b/crypto/cms/cms_ec.c
@@ -8,6 +8,7 @@
  */
 
 #include <assert.h>
+#include <limits.h>
 #include <openssl/cms.h>
 #include <openssl/err.h>
 #include <openssl/decoder.h>
@@ -257,7 +258,7 @@ static int ecdh_cms_encrypt(CMS_RecipientInfo *ri)
     ASN1_STRING *wrap_str;
     ASN1_OCTET_STRING *ukm;
     unsigned char *penc = NULL;
-    size_t penclen;
+    int penclen;
     int rv = 0;
     int ecdh_nid, kdf_type, kdf_nid, wrap_nid;
     const EVP_MD *kdf_md;
@@ -274,8 +275,11 @@ static int ecdh_cms_encrypt(CMS_RecipientInfo *ri)
     /* Is everything uninitialised? */
     if (aoid == OBJ_nid2obj(NID_undef)) {
         /* Set the key */
+        size_t enckeylen;
 
         penclen = EVP_PKEY_get1_encoded_public_key(pkey, &penc);
+        if (penclen > INT_MAX || penclen == 0)
+            goto err;
         ASN1_STRING_set0(pubkey, penc, penclen);
         pubkey->flags &= ~(ASN1_STRING_FLAG_BITS_LEFT | 0x07);
         pubkey->flags |= ASN1_STRING_FLAG_BITS_LEFT;
@@ -358,7 +362,7 @@ static int ecdh_cms_encrypt(CMS_RecipientInfo *ri)
      * of another AlgorithmIdentifier.
      */
     penclen = i2d_X509_ALGOR(wrap_alg, &penc);
-    if (penc == NULL || penclen == 0)
+    if (penclen <= 0)
         goto err;
     wrap_str = ASN1_STRING_new();
     if (wrap_str == NULL)

diff --git a/providers/implementations/signature/rsa_sig.c b/providers/implementations/signature/rsa_sig.c
index be7cf53b26f50ca313624c1c4377fbade2950607..cd5de6bd5183bf7b1ecbed9dc62aab080335ec90 100644 (file)
--- a/providers/implementations/signature/rsa_sig.c
+++ b/providers/implementations/signature/rsa_sig.c
@@ -837,14 +837,17 @@ static int rsa_verify(void *vprsactx, const unsigned char *sig, size_t siglen,
             return 0;
         }
     } else {
+        int ret;
+
         if (!setup_tbuf(prsactx))
             return 0;
-        rslen = RSA_public_decrypt(siglen, sig, prsactx->tbuf, prsactx->rsa,
-                                   prsactx->pad_mode);
-        if (rslen <= 0) {
+        ret = RSA_public_decrypt(siglen, sig, prsactx->tbuf, prsactx->rsa,
+                                 prsactx->pad_mode);
+        if (ret <= 0) {
             ERR_raise(ERR_LIB_PROV, ERR_R_RSA_LIB);
             return 0;
         }
+        rslen = (size_t)ret;
     }
 
     if ((rslen != tbslen) || memcmp(tbs, prsactx->tbuf, rslen))