crypto/asn1/i2d_evp.c: Fix i2d_provided() to return a proper length

Fixes #14258

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Ben Kaduk <kaduk@mit.edu>
(Merged from https://github.com/openssl/openssl/pull/14291)

diff --git a/crypto/asn1/i2d_evp.c b/crypto/asn1/i2d_evp.c
index ffcb34aa2075646a0005f812362963d417ae59a5..2a101a6fa31a609fd3950d1157f9bc9c2f091ccb 100644 (file)
--- a/crypto/asn1/i2d_evp.c
+++ b/crypto/asn1/i2d_evp.c
@@ -42,8 +42,10 @@ static int i2d_provided(const EVP_PKEY *a, int selection,
          output_info++) {
         /*
          * The i2d_ calls don't take a boundary length for *pp.  However,
-         * OSSL_ENCODER_CTX_get_num_encoders() needs one, so we make one
-         * up.
+         * OSSL_ENCODER_to_data() needs one, so we make one up.  Because
+         * OSSL_ENCODER_to_data() decrements this number by the amount of
+         * bytes written, we need to calculate the length written further
+         * down, when pp != NULL.
          */
         size_t len = INT_MAX;
 
@@ -53,8 +55,12 @@ static int i2d_provided(const EVP_PKEY *a, int selection,
                                             NULL);
         if (ctx == NULL)
             return -1;
-        if (OSSL_ENCODER_to_data(ctx, pp, &len))
-            ret = (int)len;
+        if (OSSL_ENCODER_to_data(ctx, pp, &len)) {
+            if (pp == NULL)
+                ret = (int)len;
+            else
+                ret = INT_MAX - (int)len;
+        }
         OSSL_ENCODER_CTX_free(ctx);
         ctx = NULL;
     }