check i2d_ASN1_TYPE return value

add a length check to the return value of function i2d_ASN1_TYPE. Return an error instead of trying to malloc a negative number.

CLA: trivial

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Paul Dale <pauli@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/14828)

(cherry picked from commit c65abf2213117eb5348a46fbc18f706aca052e85)

diff --git a/crypto/asn1/a_strex.c b/crypto/asn1/a_strex.c
index 4879b33785e97e9a09e96eaa3f3e8b9a7f088ae3..7cd18b4b855aad8586eeac67318a063d1338aae4 100644 (file)
--- a/crypto/asn1/a_strex.c
+++ b/crypto/asn1/a_strex.c
@@ -280,6 +280,8 @@ static int do_dump(unsigned long lflags, char_io *io_ch, void *arg,
     t.type = str->type;
     t.value.ptr = (char *)str;
     der_len = i2d_ASN1_TYPE(&t, NULL);
+    if (der_len <= 0)
+        return -1;
     if ((der_buf = OPENSSL_malloc(der_len)) == NULL) {
         ASN1err(ASN1_F_DO_DUMP, ERR_R_MALLOC_FAILURE);
         return -1;