Ensure GCM "update" failures return 0 on error

EVP_CipherUpdate is supposed to return 1 for success or 0 for error.
However for GCM ciphers it was sometimes returning -1 for error.

Reviewed-by: Shane Lontis <shane.lontis@oracle.com>
(Merged from https://github.com/openssl/openssl/pull/12288)

diff --git a/providers/implementations/ciphers/ciphercommon_gcm.c b/providers/implementations/ciphers/ciphercommon_gcm.c
index 7daa8dce5b1a004c3f9c099c3e776448acb2018f..080fcc9bc23bb5450e723222cb7d8d3f73289e6d 100644 (file)
--- a/providers/implementations/ciphers/ciphercommon_gcm.c
+++ b/providers/implementations/ciphers/ciphercommon_gcm.c
@@ -280,12 +280,12 @@ int gcm_stream_update(void *vctx, unsigned char *out, size_t *outl,
 
     if (outsize < inl) {
         ERR_raise(ERR_LIB_PROV, PROV_R_OUTPUT_BUFFER_TOO_SMALL);
-        return -1;
+        return 0;
     }
 
     if (gcm_cipher_internal(ctx, out, outl, in, inl) <= 0) {
         ERR_raise(ERR_LIB_PROV, PROV_R_CIPHER_OPERATION_FAILED);
-        return -1;
+        return 0;
     }
     return 1;
 }