Fix signed integer overflow in evp_enc

Fixes #17869.

Reviewed-by: Paul Dale <pauli@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/17870)

diff --git a/crypto/evp/evp_enc.c b/crypto/evp/evp_enc.c
index 02566ae949ab0595773128a2cbcb4f794af0c7bc..d0a62a6d4684035c8212c27b55d58072f65377ba 100644 (file)
--- a/crypto/evp/evp_enc.c
+++ b/crypto/evp/evp_enc.c
@@ -605,7 +605,7 @@ int EVP_EncryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
                       const unsigned char *in, int inl)
 {
     int ret;
-    size_t soutl;
+    size_t soutl, inl_ = (size_t)inl;
     int blocksize;
 
     if (outl != NULL) {
@@ -635,9 +635,10 @@ int EVP_EncryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
         ERR_raise(ERR_LIB_EVP, EVP_R_UPDATE_ERROR);
         return 0;
     }
+
     ret = ctx->cipher->cupdate(ctx->algctx, out, &soutl,
-                               inl + (blocksize == 1 ? 0 : blocksize), in,
-                               (size_t)inl);
+                               inl_ + (size_t)(blocksize == 1 ? 0 : blocksize),
+                               in, inl_);
 
     if (ret) {
         if (soutl > INT_MAX) {
@@ -753,7 +754,7 @@ int EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
 {
     int fix_len, cmpl = inl, ret;
     unsigned int b;
-    size_t soutl;
+    size_t soutl, inl_ = (size_t)inl;
     int blocksize;
 
     if (outl != NULL) {
@@ -783,8 +784,8 @@ int EVP_DecryptUpdate(EVP_CIPHER_CTX *ctx, unsigned char *out, int *outl,
         return 0;
     }
     ret = ctx->cipher->cupdate(ctx->algctx, out, &soutl,
-                               inl + (blocksize == 1 ? 0 : blocksize), in,
-                               (size_t)inl);
+                               inl_ + (size_t)(blocksize == 1 ? 0 : blocksize),
+                               in, inl_);
 
     if (ret) {
         if (soutl > INT_MAX) {

diff --git a/test/sanitytest.c b/test/sanitytest.c
index b1aadc4aa8333f2d8bd98b9c4e6f1e24431bdefb..ec850651d27b44d40be6cfa89239677e3019a8a6 100644 (file)
--- a/test/sanitytest.c
+++ b/test/sanitytest.c
@@ -114,6 +114,13 @@ static int test_sanity_range(void)
         TEST_error("int must not be wider than size_t");
         return 0;
     }
+
+    /* SIZE_MAX is always greater than 2*INT_MAX */
+    if (SIZE_MAX - INT_MAX <= INT_MAX) {
+        TEST_error("SIZE_MAX must exceed 2*INT_MAX");
+        return 0;
+    }
+
     return 1;
 }