Coverity fix in some crypto/asn1 code

Call to i2d method returns an int value.

Fix:
CID 1338183 (#1 of 1): Improper use of negative value (NEGATIVE_RETURNS)
CID 1371691 (#1 of 1): Improper use of negative value (NEGATIVE_RETURNS)
CID 1371692 (#1 of 1): Improper use of negative value (NEGATIVE_RETURNS)

[extended tests]

Reviewed-by: Richard Levitte <levitte@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/7359)

diff --git a/crypto/asn1/a_digest.c b/crypto/asn1/a_digest.c
index 0c00f18616bba2bd6ef948b2885c273e990c0515..ad6a17d245ca330929af056f0d89fd0c17b45ce2 100644 (file)
--- a/crypto/asn1/a_digest.c
+++ b/crypto/asn1/a_digest.c
@@ -23,18 +23,22 @@
 int ASN1_digest(i2d_of_void *i2d, const EVP_MD *type, char *data,
                 unsigned char *md, unsigned int *len)
 {
 int ASN1_digest(i2d_of_void *i2d, const EVP_MD *type, char *data,
                 unsigned char *md, unsigned int *len)
 {

-    int i;
+    int inl;

     unsigned char *str, *p;
 
     unsigned char *str, *p;
 

-    i = i2d(data, NULL);
-    if ((str = OPENSSL_malloc(i)) == NULL) {
+    inl = i2d(data, NULL);
+    if (inl <= 0) {
+        ASN1err(ASN1_F_ASN1_DIGEST, ERR_R_INTERNAL_ERROR);
+        return 0;
+    }
+    if ((str = OPENSSL_malloc(inl)) == NULL) {

         ASN1err(ASN1_F_ASN1_DIGEST, ERR_R_MALLOC_FAILURE);
         return 0;
     }
     p = str;
     i2d(data, &p);
 
         ASN1err(ASN1_F_ASN1_DIGEST, ERR_R_MALLOC_FAILURE);
         return 0;
     }
     p = str;
     i2d(data, &p);
 

-    if (!EVP_Digest(str, i, md, len, type, NULL)) {
+    if (!EVP_Digest(str, inl, md, len, type, NULL)) {

         OPENSSL_free(str);
         return 0;
     }
         OPENSSL_free(str);
         return 0;
     }

diff --git a/crypto/asn1/a_sign.c b/crypto/asn1/a_sign.c
index 9de4fb615866ebe8cdc7037a52ca97453b3a488c..97e8efcbe88939f30afc86e757623b5ebd19cc61 100644 (file)
--- a/crypto/asn1/a_sign.c
+++ b/crypto/asn1/a_sign.c
@@ -29,7 +29,8 @@ int ASN1_sign(i2d_of_void *i2d, X509_ALGOR *algor1, X509_ALGOR *algor2,
 {
     EVP_MD_CTX *ctx = EVP_MD_CTX_new();
     unsigned char *p, *buf_in = NULL, *buf_out = NULL;
 {
     EVP_MD_CTX *ctx = EVP_MD_CTX_new();
     unsigned char *p, *buf_in = NULL, *buf_out = NULL;

-    int i, inl = 0, outl = 0, outll = 0;
+    int i, inl = 0, outl = 0;
+    size_t inll = 0, outll = 0;

     X509_ALGOR *a;
 
     if (ctx == NULL) {
     X509_ALGOR *a;
 
     if (ctx == NULL) {


@@ -70,10 +71,15 @@ int ASN1_sign(i2d_of_void *i2d, X509_ALGOR *algor1, X509_ALGOR *algor2,
         }
     }
     inl = i2d(data, NULL);
         }
     }
     inl = i2d(data, NULL);

-    buf_in = OPENSSL_malloc((unsigned int)inl);
+    if (inl <= 0) {
+        ASN1err(ASN1_F_ASN1_SIGN, ERR_R_INTERNAL_ERROR);
+        goto err;
+    }
+    inll = (size_t)inl;
+    buf_in = OPENSSL_malloc(inll);

     outll = outl = EVP_PKEY_size(pkey);
     outll = outl = EVP_PKEY_size(pkey);

-    buf_out = OPENSSL_malloc((unsigned int)outl);
-    if ((buf_in == NULL) || (buf_out == NULL)) {
+    buf_out = OPENSSL_malloc(outll);
+    if (buf_in == NULL || buf_out == NULL) {

         outl = 0;
         ASN1err(ASN1_F_ASN1_SIGN, ERR_R_MALLOC_FAILURE);
         goto err;
         outl = 0;
         ASN1err(ASN1_F_ASN1_SIGN, ERR_R_MALLOC_FAILURE);
         goto err;


@@ -101,7 +107,7 @@ int ASN1_sign(i2d_of_void *i2d, X509_ALGOR *algor1, X509_ALGOR *algor2,
     signature->flags |= ASN1_STRING_FLAG_BITS_LEFT;
  err:
     EVP_MD_CTX_free(ctx);
     signature->flags |= ASN1_STRING_FLAG_BITS_LEFT;
  err:
     EVP_MD_CTX_free(ctx);

-    OPENSSL_clear_free((char *)buf_in, (unsigned int)inl);
+    OPENSSL_clear_free((char *)buf_in, inll);

     OPENSSL_clear_free((char *)buf_out, outll);
     return outl;
 }
     OPENSSL_clear_free((char *)buf_out, outll);
     return outl;
 }


@@ -138,7 +144,7 @@ int ASN1_item_sign_ctx(const ASN1_ITEM *it,
     EVP_PKEY *pkey;
     unsigned char *buf_in = NULL, *buf_out = NULL;
     size_t inl = 0, outl = 0, outll = 0;
     EVP_PKEY *pkey;
     unsigned char *buf_in = NULL, *buf_out = NULL;
     size_t inl = 0, outl = 0, outll = 0;

-    int signid, paramtype;
+    int signid, paramtype, buf_len = 0;

     int rv;
 
     type = EVP_MD_CTX_md(ctx);
     int rv;
 
     type = EVP_MD_CTX_md(ctx);


@@ -198,10 +204,16 @@ int ASN1_item_sign_ctx(const ASN1_ITEM *it,
 
     }
 
 
     }
 

-    inl = ASN1_item_i2d(asn, &buf_in, it);
+    buf_len = ASN1_item_i2d(asn, &buf_in, it);
+    if (buf_len <= 0) {
+        outl = 0;
+        ASN1err(ASN1_F_ASN1_ITEM_SIGN_CTX, ERR_R_INTERNAL_ERROR);
+        goto err;
+    }
+    inl = buf_len;

     outll = outl = EVP_PKEY_size(pkey);
     outll = outl = EVP_PKEY_size(pkey);

-    buf_out = OPENSSL_malloc((unsigned int)outl);
-    if ((buf_in == NULL) || (buf_out == NULL)) {
+    buf_out = OPENSSL_malloc(outll);
+    if (buf_in == NULL || buf_out == NULL) {

         outl = 0;
         ASN1err(ASN1_F_ASN1_ITEM_SIGN_CTX, ERR_R_MALLOC_FAILURE);
         goto err;
         outl = 0;
         ASN1err(ASN1_F_ASN1_ITEM_SIGN_CTX, ERR_R_MALLOC_FAILURE);
         goto err;


@@ -223,7 +235,7 @@ int ASN1_item_sign_ctx(const ASN1_ITEM *it,
     signature->flags &= ~(ASN1_STRING_FLAG_BITS_LEFT | 0x07);
     signature->flags |= ASN1_STRING_FLAG_BITS_LEFT;
  err:
     signature->flags &= ~(ASN1_STRING_FLAG_BITS_LEFT | 0x07);
     signature->flags |= ASN1_STRING_FLAG_BITS_LEFT;
  err:

-    OPENSSL_clear_free((char *)buf_in, (unsigned int)inl);
+    OPENSSL_clear_free((char *)buf_in, inl);

     OPENSSL_clear_free((char *)buf_out, outll);
     return outl;
 }
     OPENSSL_clear_free((char *)buf_out, outll);
     return outl;
 }

diff --git a/crypto/asn1/a_verify.c b/crypto/asn1/a_verify.c
index dc3e9720c7cf352a6f30fdad189d4385881abaf4..1f2458311307fcc977f26a3536f28f674c6340d3 100644 (file)
--- a/crypto/asn1/a_verify.c
+++ b/crypto/asn1/a_verify.c
@@ -48,6 +48,10 @@ int ASN1_verify(i2d_of_void *i2d, X509_ALGOR *a, ASN1_BIT_STRING *signature,
     }
 
     inl = i2d(data, NULL);
     }
 
     inl = i2d(data, NULL);

+    if (inl <= 0) {
+        ASN1err(ASN1_F_ASN1_VERIFY, ERR_R_INTERNAL_ERROR);
+        goto err;
+    }

     buf_in = OPENSSL_malloc((unsigned int)inl);
     if (buf_in == NULL) {
         ASN1err(ASN1_F_ASN1_VERIFY, ERR_R_MALLOC_FAILURE);
     buf_in = OPENSSL_malloc((unsigned int)inl);
     if (buf_in == NULL) {
         ASN1err(ASN1_F_ASN1_VERIFY, ERR_R_MALLOC_FAILURE);


@@ -87,8 +91,8 @@ int ASN1_item_verify(const ASN1_ITEM *it, X509_ALGOR *a,
     EVP_MD_CTX *ctx = NULL;
     unsigned char *buf_in = NULL;
     int ret = -1, inl = 0;
     EVP_MD_CTX *ctx = NULL;
     unsigned char *buf_in = NULL;
     int ret = -1, inl = 0;

-

     int mdnid, pknid;
     int mdnid, pknid;

+    size_t inll = 0;

 
     if (!pkey) {
         ASN1err(ASN1_F_ASN1_ITEM_VERIFY, ERR_R_PASSED_NULL_PARAMETER);
 
     if (!pkey) {
         ASN1err(ASN1_F_ASN1_ITEM_VERIFY, ERR_R_PASSED_NULL_PARAMETER);


@@ -127,8 +131,8 @@ int ASN1_item_verify(const ASN1_ITEM *it, X509_ALGOR *a,
             goto err;
         ret = -1;
     } else {
             goto err;
         ret = -1;
     } else {

-        const EVP_MD *type;
-        type = EVP_get_digestbynid(mdnid);
+        const EVP_MD *type = EVP_get_digestbynid(mdnid);
+

         if (type == NULL) {
             ASN1err(ASN1_F_ASN1_ITEM_VERIFY,
                     ASN1_R_UNKNOWN_MESSAGE_DIGEST_ALGORITHM);
         if (type == NULL) {
             ASN1err(ASN1_F_ASN1_ITEM_VERIFY,
                     ASN1_R_UNKNOWN_MESSAGE_DIGEST_ALGORITHM);


@@ -150,11 +154,15 @@ int ASN1_item_verify(const ASN1_ITEM *it, X509_ALGOR *a,
     }
 
     inl = ASN1_item_i2d(asn, &buf_in, it);
     }
 
     inl = ASN1_item_i2d(asn, &buf_in, it);

-
+    if (inl <= 0) {
+        ASN1err(ASN1_F_ASN1_ITEM_VERIFY, ERR_R_INTERNAL_ERROR);
+        goto err;
+    }

     if (buf_in == NULL) {
         ASN1err(ASN1_F_ASN1_ITEM_VERIFY, ERR_R_MALLOC_FAILURE);
         goto err;
     }
     if (buf_in == NULL) {
         ASN1err(ASN1_F_ASN1_ITEM_VERIFY, ERR_R_MALLOC_FAILURE);
         goto err;
     }

+    inll = inl;

 
     ret = EVP_DigestVerify(ctx, signature->data, (size_t)signature->length,
                            buf_in, inl);
 
     ret = EVP_DigestVerify(ctx, signature->data, (size_t)signature->length,
                            buf_in, inl);


@@ -164,7 +172,7 @@ int ASN1_item_verify(const ASN1_ITEM *it, X509_ALGOR *a,
     }
     ret = 1;
  err:
     }
     ret = 1;
  err:

-    OPENSSL_clear_free(buf_in, (unsigned int)inl);
+    OPENSSL_clear_free(buf_in, inll);

     EVP_MD_CTX_free(ctx);
     return ret;
 }
     EVP_MD_CTX_free(ctx);
     return ret;
 }