Fix a possible memleak in rsa_pub_encode

That seems to be only an issue for RSA-PSS with parameters.
Spotted by code review, so it looks like there is no test coverage for this.

Reviewed-by: Dmitry Belyavskiy <beldmit@gmail.com>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/22032)

(cherry picked from commit 285eb1688f05ad477fefc681bf05d0afedc46d40)

diff --git a/crypto/rsa/rsa_ameth.c b/crypto/rsa/rsa_ameth.c
index e819780e7d9439c392df71ed0fd13db339756743..07734077e3228a7930669f843278a351f9222885 100644 (file)
--- a/crypto/rsa/rsa_ameth.c
+++ b/crypto/rsa/rsa_ameth.c
@@ -60,13 +60,16 @@ static int rsa_pub_encode(X509_PUBKEY *pk, const EVP_PKEY *pkey)
     if (!rsa_param_encode(pkey, &str, &strtype))
         return 0;
     penclen = i2d_RSAPublicKey(pkey->pkey.rsa, &penc);
-    if (penclen <= 0)
+    if (penclen <= 0) {
+        ASN1_STRING_free(str);
         return 0;
+    }
     if (X509_PUBKEY_set0_param(pk, OBJ_nid2obj(pkey->ameth->pkey_id),
                                strtype, str, penc, penclen))
         return 1;
 
     OPENSSL_free(penc);
+    ASN1_STRING_free(str);
     return 0;
 }
 

diff --git a/test/recipes/15-test_rsapss.t b/test/recipes/15-test_rsapss.t
index ea1d9aceaed0f54752d2c91faae223fa995053e2..4773252185b91b99d1cc969d56cb777c2bd49d0f 100644 (file)
--- a/test/recipes/15-test_rsapss.t
+++ b/test/recipes/15-test_rsapss.t
@@ -16,7 +16,7 @@ use OpenSSL::Test::Utils;
 
 setup("test_rsapss");
 
-plan tests => 11;
+plan tests => 13;
 
 #using test/testrsa.pem which happens to be a 512 bit RSA
 ok(run(app(['openssl', 'dgst', '-sign', srctop_file('test', 'testrsa.pem'), '-sha1',
@@ -89,3 +89,11 @@ ok(run(app(['openssl', 'dgst', '-prverify', srctop_file('test', 'testrsa.pem'),
 ok(!run(app([ 'openssl', 'rsa',
              '-in' => data_file('negativesaltlen.pem')],
              '-out' => 'badout')));
+
+ok(run(app(['openssl', 'genpkey', '-algorithm', 'RSA-PSS', '-pkeyopt', 'rsa_keygen_bits:1024',
+            '-pkeyopt', 'rsa_pss_keygen_md:SHA256', '-pkeyopt', 'rsa_pss_keygen_saltlen:10',
+            '-out', 'testrsapss.pem'])),
+   "openssl genpkey RSA-PSS with pss parameters");
+ok(run(app(['openssl', 'pkey', '-in', 'testrsapss.pem', '-pubout', '-text'])),
+   "openssl pkey, execute rsa_pub_encode with pss parameters");
+unlink 'testrsapss.pem';