Initialize outl in evp_enc.c to 0, protect against NULL

Fixes #12734

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Tomas Mraz <tmraz@fedoraproject.org>
(Merged from https://github.com/openssl/openssl/pull/13268)

diff --git a/crypto/evp/evp_enc.c b/crypto/evp/evp_enc.c
index 929c95eed82e67ff1b411fce9c2077c1538674a8..d8fc3ab7ad7ee19159159e9928e037de5495a560 100644 (file)
--- a/crypto/evp/evp_enc.c
+++ b/crypto/evp/evp_enc.c
@@ -535,6 +535,13 @@ int EVP_EncryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
     size_t soutl;
     int blocksize;
 
+    if (outl != NULL) {
+        *outl = 0;
+    } else {
+        EVPerr(EVP_F_EVP_ENCRYPTUPDATE, ERR_R_PASSED_NULL_PARAMETER);
+        return 0;
+    }
+
     /* Prevent accidental use of decryption context when encrypting */
     if (!ctx->encrypt) {
         EVPerr(EVP_F_EVP_ENCRYPTUPDATE, EVP_R_INVALID_OPERATION);
@@ -589,6 +596,13 @@ int EVP_EncryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl)
     size_t soutl;
     int blocksize;
 
+    if (outl != NULL) {
+        *outl = 0;
+    } else {
+        EVPerr(EVP_F_EVP_ENCRYPTFINAL_EX, ERR_R_PASSED_NULL_PARAMETER);
+        return 0;
+    }
+
     /* Prevent accidental use of decryption context when encrypting */
     if (!ctx->encrypt) {
         EVPerr(EVP_F_EVP_ENCRYPTFINAL_EX, EVP_R_INVALID_OPERATION);
@@ -670,6 +684,13 @@ int EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
     size_t soutl;
     int blocksize;
 
+    if (outl != NULL) {
+        *outl = 0;
+    } else {
+        EVPerr(EVP_F_EVP_DECRYPTUPDATE, ERR_R_PASSED_NULL_PARAMETER);
+        return 0;
+    }
+
     /* Prevent accidental use of encryption context when decrypting */
     if (ctx->encrypt) {
         EVPerr(EVP_F_EVP_DECRYPTUPDATE, EVP_R_INVALID_OPERATION);
@@ -784,6 +805,13 @@ int EVP_DecryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl)
     int ret;
     int blocksize;
 
+    if (outl != NULL) {
+        *outl = 0;
+    } else {
+        EVPerr(EVP_F_EVP_DECRYPTFINAL_EX, ERR_R_PASSED_NULL_PARAMETER);
+        return 0;
+    }
+
     /* Prevent accidental use of encryption context when decrypting */
     if (ctx->encrypt) {
         EVPerr(EVP_F_EVP_DECRYPTFINAL_EX, EVP_R_INVALID_OPERATION);