Coverity fix in some crypto/asn1 code

[openssl.git] / crypto / asn1 / a_sign.c

diff --git a/crypto/asn1/a_sign.c b/crypto/asn1/a_sign.c
index 4e93b5a7683d5cdf0eee6c96835ea20271f3baa0..146fdb9626281024cbeefcc78dface7da1a4a7c3 100644 (file)
--- a/crypto/asn1/a_sign.c
+++ b/crypto/asn1/a_sign.c
@@ -9,13 +9,10 @@
 
 #include <stdio.h>
 #include <time.h>
+#include <sys/types.h>
 
 #include "internal/cryptlib.h"
 
-#ifndef NO_SYS_TYPES_H
-# include <sys/types.h>
-#endif
-
 #include <openssl/bn.h>
 #include <openssl/evp.h>
 #include <openssl/x509.h>
@@ -32,7 +29,8 @@ int ASN1_sign(i2d_of_void *i2d, X509_ALGOR *algor1, X509_ALGOR *algor2,
 {
     EVP_MD_CTX *ctx = EVP_MD_CTX_new();
     unsigned char *p, *buf_in = NULL, *buf_out = NULL;
-    int i, inl = 0, outl = 0, outll = 0;
+    int i, inl = 0, outl = 0;
+    size_t inll = 0, outll = 0;
     X509_ALGOR *a;
 
     if (ctx == NULL) {
@@ -73,10 +71,15 @@ int ASN1_sign(i2d_of_void *i2d, X509_ALGOR *algor1, X509_ALGOR *algor2,
         }
     }
     inl = i2d(data, NULL);
-    buf_in = OPENSSL_malloc((unsigned int)inl);
+    if (inl <= 0) {
+        ASN1err(ASN1_F_ASN1_SIGN, ERR_R_INTERNAL_ERROR);
+        goto err;
+    }
+    inll = (size_t)inl;
+    buf_in = OPENSSL_malloc(inll);
     outll = outl = EVP_PKEY_size(pkey);
-    buf_out = OPENSSL_malloc((unsigned int)outl);
-    if ((buf_in == NULL) || (buf_out == NULL)) {
+    buf_out = OPENSSL_malloc(outll);
+    if (buf_in == NULL || buf_out == NULL) {
         outl = 0;
         ASN1err(ASN1_F_ASN1_SIGN, ERR_R_MALLOC_FAILURE);
         goto err;
@@ -104,9 +107,9 @@ int ASN1_sign(i2d_of_void *i2d, X509_ALGOR *algor1, X509_ALGOR *algor2,
     signature->flags |= ASN1_STRING_FLAG_BITS_LEFT;
  err:
     EVP_MD_CTX_free(ctx);
-    OPENSSL_clear_free((char *)buf_in, (unsigned int)inl);
+    OPENSSL_clear_free((char *)buf_in, inll);
     OPENSSL_clear_free((char *)buf_out, outll);
-    return (outl);
+    return outl;
 }
 
 #endif
@@ -141,13 +144,13 @@ int ASN1_item_sign_ctx(const ASN1_ITEM *it,
     EVP_PKEY *pkey;
     unsigned char *buf_in = NULL, *buf_out = NULL;
     size_t inl = 0, outl = 0, outll = 0;
-    int signid, paramtype;
+    int signid, paramtype, buf_len = 0;
     int rv;
 
     type = EVP_MD_CTX_md(ctx);
     pkey = EVP_PKEY_CTX_get0_pkey(EVP_MD_CTX_pkey_ctx(ctx));
 
-    if (type == NULL || pkey == NULL) {
+    if (pkey == NULL) {
         ASN1err(ASN1_F_ASN1_ITEM_SIGN_CTX, ASN1_R_CONTEXT_NOT_INITIALISED);
         goto err;
     }
@@ -172,10 +175,15 @@ int ASN1_item_sign_ctx(const ASN1_ITEM *it,
             ASN1err(ASN1_F_ASN1_ITEM_SIGN_CTX, ERR_R_EVP_LIB);
         if (rv <= 1)
             goto err;
-    } else
+    } else {
         rv = 2;
+    }
 
     if (rv == 2) {
+        if (type == NULL) {
+            ASN1err(ASN1_F_ASN1_ITEM_SIGN_CTX, ASN1_R_CONTEXT_NOT_INITIALISED);
+            goto err;
+        }
         if (!OBJ_find_sigid_by_algs(&signid,
                                     EVP_MD_nid(type),
                                     pkey->ameth->pkey_id)) {
@@ -196,10 +204,16 @@ int ASN1_item_sign_ctx(const ASN1_ITEM *it,
 
     }
 
-    inl = ASN1_item_i2d(asn, &buf_in, it);
+    buf_len = ASN1_item_i2d(asn, &buf_in, it);
+    if (buf_len <= 0) {
+        outl = 0;
+        ASN1err(ASN1_F_ASN1_ITEM_SIGN_CTX, ERR_R_INTERNAL_ERROR);
+        goto err;
+    }
+    inl = buf_len;
     outll = outl = EVP_PKEY_size(pkey);
-    buf_out = OPENSSL_malloc((unsigned int)outl);
-    if ((buf_in == NULL) || (buf_out == NULL)) {
+    buf_out = OPENSSL_malloc(outll);
+    if (buf_in == NULL || buf_out == NULL) {
         outl = 0;
         ASN1err(ASN1_F_ASN1_ITEM_SIGN_CTX, ERR_R_MALLOC_FAILURE);
         goto err;
@@ -221,7 +235,7 @@ int ASN1_item_sign_ctx(const ASN1_ITEM *it,
     signature->flags &= ~(ASN1_STRING_FLAG_BITS_LEFT | 0x07);
     signature->flags |= ASN1_STRING_FLAG_BITS_LEFT;
  err:
-    OPENSSL_clear_free((char *)buf_in, (unsigned int)inl);
+    OPENSSL_clear_free((char *)buf_in, inl);
     OPENSSL_clear_free((char *)buf_out, outll);
-    return (outl);
+    return outl;
 }