Fix error check in GOST engine

The return value of i2d functions can be negative if an error occurs.
Therefore don't assign the return value to an unsigned type and *then*
check if it is negative.

RT#3862

Reviewed-by: Richard Levitte <levitte@openssl.org>
(cherry picked from commit 90e7cdff3aa66779486914f88333f6601f0c1cf4)

diff --git a/engines/ccgost/gost94_keyx.c b/engines/ccgost/gost94_keyx.c
index 85f4bc89982d3f0e57d947395253ff8a88d106ef..ce57f17cbf32f111061c9aca7224565d3da23664 100644 (file)
--- a/engines/ccgost/gost94_keyx.c
+++ b/engines/ccgost/gost94_keyx.c
@@ -104,6 +104,7 @@ int pkey_GOST94cp_encrypt(EVP_PKEY_CTX *ctx, unsigned char *out,
     struct gost_pmeth_data *data = EVP_PKEY_CTX_get_data(ctx);
     gost_ctx cctx;
     int key_is_ephemeral = 1;
+    int tmp_outlen;
     EVP_PKEY *mykey = EVP_PKEY_CTX_get0_peerkey(ctx);
 
     /* Do not use vizir cipher parameters with cryptopro */
@@ -174,12 +175,13 @@ int pkey_GOST94cp_encrypt(EVP_PKEY_CTX *ctx, unsigned char *out,
     }
     ASN1_OBJECT_free(gkt->key_agreement_info->cipher);
     gkt->key_agreement_info->cipher = OBJ_nid2obj(param->nid);
-    *outlen = i2d_GOST_KEY_TRANSPORT(gkt, out ? &out : NULL);
-    if (*outlen <= 0) {
+    tmp_outlen = i2d_GOST_KEY_TRANSPORT(gkt, out ? &out : NULL);
+    if (tmp_outlen <= 0) {
         GOSTerr(GOST_F_PKEY_GOST94CP_ENCRYPT,
                 GOST_R_ERROR_PACKING_KEY_TRANSPORT_INFO);
         goto err;
     }
+    *outlen = tmp_outlen;
     if (!key_is_ephemeral) {
         /* Set control "public key from client certificate used" */
         if (EVP_PKEY_CTX_ctrl(ctx, -1, -1, EVP_PKEY_CTRL_PEER_KEY, 3, NULL) <=