Fix a possbile memleak in rsa_pub_encode

That seems to be only an issue for RSA-PSS with parameters.
Spotted by code review, so it looks like there is no test coverage for this.

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Hugo Landau <hlandau@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/22007)

diff --git a/crypto/rsa/rsa_ameth.c b/crypto/rsa/rsa_ameth.c
index 2c9c46ea53c8becaf1f43e9eff467e135e2e760e..63efd93798811a940951fb59ea52e3c4d492617c 100644 (file)
--- a/crypto/rsa/rsa_ameth.c
+++ b/crypto/rsa/rsa_ameth.c
@@ -82,13 +82,16 @@ static int rsa_pub_encode(X509_PUBKEY *pk, const EVP_PKEY *pkey)
     if (!rsa_param_encode(pkey, &str, &strtype))
         return 0;
     penclen = i2d_RSAPublicKey(pkey->pkey.rsa, &penc);
-    if (penclen <= 0)
+    if (penclen <= 0) {
+        ASN1_STRING_free(str);
         return 0;
+    }
     if (X509_PUBKEY_set0_param(pk, OBJ_nid2obj(pkey->ameth->pkey_id),
                                strtype, str, penc, penclen))
         return 1;
 
     OPENSSL_free(penc);
+    ASN1_STRING_free(str);
     return 0;
 }
 

diff --git a/test/recipes/15-test_rsapss.t b/test/recipes/15-test_rsapss.t
index 65ec6f3d75289d5d8f53d95f6818ecefdb925642..61c13b6a49a58bceb9fde72e01a728650e547239 100644 (file)
--- a/test/recipes/15-test_rsapss.t
+++ b/test/recipes/15-test_rsapss.t
@@ -16,7 +16,7 @@ use OpenSSL::Test::Utils;
 
 setup("test_rsapss");
 
-plan tests => 5;
+plan tests => 7;
 
 #using test/testrsa.pem which happens to be a 512 bit RSA
 ok(run(app(['openssl', 'dgst', '-sign', srctop_file('test', 'testrsa.pem'), '-sha1',
@@ -47,3 +47,11 @@ ok(run(app(['openssl', 'dgst', '-prverify', srctop_file('test', 'testrsa.pem'),
             srctop_file('test', 'testrsa.pem')])),
    "openssl dgst -prverify");
 unlink 'testrsapss.sig';
+
+ok(run(app(['openssl', 'genpkey', '-algorithm', 'RSA-PSS', '-pkeyopt', 'rsa_keygen_bits:1024',
+            '-pkeyopt', 'rsa_pss_keygen_md:SHA256', '-pkeyopt', 'rsa_pss_keygen_saltlen:10',
+            '-out', 'testrsapss.pem'])),
+   "openssl genpkey RSA-PSS with pss parameters");
+ok(run(app(['openssl', 'pkey', '-in', 'testrsapss.pem', '-pubout', '-text'])),
+   "openssl pkey, execute rsa_pub_encode with pss parameters");
+unlink 'testrsapss.pem';