Fix coverity CID #1465531 - Negative return passed to a function param using size_t...

Reviewed-by: Tomas Mraz <tmraz@fedoraproject.org>
(Merged from https://github.com/openssl/openssl/pull/12628)

diff --git a/crypto/asn1/a_digest.c b/crypto/asn1/a_digest.c
index 3e7b418a1945a3bf8d74791042bf40af6395f015..9211d7a9682f98c06a7aa5eb55435d3b77913529 100644 (file)
--- a/crypto/asn1/a_digest.c
+++ b/crypto/asn1/a_digest.c
@@ -63,7 +63,7 @@ int asn1_item_digest_with_libctx(const ASN1_ITEM *it, const EVP_MD *md,
     EVP_MD *fetched_md = (EVP_MD *)md;
 
     i = ASN1_item_i2d(asn, &str, it);
-    if (str == NULL)
+    if (i < 0 || str == NULL)
         return 0;
 
     if (EVP_MD_provider(md) == NULL) {