Add macros to determine if key or ctx is PSS.

Reviewed-by: Rich Salz <rsalz@openssl.org>
Reviewed-by: Matt Caswell <matt@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/2177)

diff --git a/crypto/rsa/rsa_ameth.c b/crypto/rsa/rsa_ameth.c
index 7268d1198d4156dc0f3a8ed7e6c8524a133166c3..b091746b1c83433178a1a3a85aaaa9eb464d8ed0 100644 (file)
--- a/crypto/rsa/rsa_ameth.c
+++ b/crypto/rsa/rsa_ameth.c
@@ -305,7 +305,6 @@ static int pkey_rsa_print(BIO *bp, const EVP_PKEY *pkey, int off, int priv)
     char *str;
     const char *s;
     int ret = 0, mod_len = 0;
-    int is_pss = pkey->ameth->pkey_id == EVP_PKEY_RSA_PSS;
 
     if (x->n != NULL)
         mod_len = BN_num_bits(x->n);
@@ -313,7 +312,7 @@ static int pkey_rsa_print(BIO *bp, const EVP_PKEY *pkey, int off, int priv)
     if (!BIO_indent(bp, off, 128))
         goto err;
 
-    if (BIO_printf(bp, "%s ", is_pss ?  "RSA-PSS" : "RSA") <= 0)
+    if (BIO_printf(bp, "%s ", pkey_is_pss(pkey) ?  "RSA-PSS" : "RSA") <= 0)
         goto err;
 
     if (priv && x->d) {
@@ -345,7 +344,7 @@ static int pkey_rsa_print(BIO *bp, const EVP_PKEY *pkey, int off, int priv)
         if (!ASN1_bn_print(bp, "coefficient:", x->iqmp, NULL, off))
             goto err;
     }
-    if (is_pss && !rsa_pss_param_print(bp, 1, x->pss, off))
+    if (pkey_is_pss(pkey) && !rsa_pss_param_print(bp, 1, x->pss, off))
         goto err;
     ret = 1;
  err:

diff --git a/crypto/rsa/rsa_locl.h b/crypto/rsa/rsa_locl.h
index 7171253a00758fbfb6daa949195fda31eb615453..51916084fed38f1f671fbf0263a8be5da488c9ad 100644 (file)
--- a/crypto/rsa/rsa_locl.h
+++ b/crypto/rsa/rsa_locl.h
@@ -97,6 +97,9 @@ extern int int_rsa_verify(int dtype, const unsigned char *m,
                           unsigned int m_len, unsigned char *rm,
                           size_t *prm_len, const unsigned char *sigbuf,
                           size_t siglen, RSA *rsa);
+/* Macros to test if a pkey or ctx is for a PSS key */
+#define pkey_is_pss(pkey) (pkey->ameth->pkey_id == EVP_PKEY_RSA_PSS)
+#define pkey_ctx_is_pss(ctx) (ctx->pmeth->pkey_id == EVP_PKEY_RSA_PSS)
 
 RSA_PSS_PARAMS *rsa_pss_params_create(const EVP_MD *sigmd,
                                       const EVP_MD *mgf1md, int saltlen);

diff --git a/crypto/rsa/rsa_pmeth.c b/crypto/rsa/rsa_pmeth.c
index a1c65ef87ecf5c52af531cac05865acb359da25c..80b1e210efc902685281551ed5ba318d6a83bc57 100644 (file)
--- a/crypto/rsa/rsa_pmeth.c
+++ b/crypto/rsa/rsa_pmeth.c
@@ -49,7 +49,7 @@ static int pkey_rsa_init(EVP_PKEY_CTX *ctx)
     if (rctx == NULL)
         return 0;
     rctx->nbits = 1024;
-    if (ctx->pmeth->pkey_id == EVP_PKEY_RSA_PSS)
+    if (pkey_ctx_is_pss(ctx))
         rctx->pad_mode = RSA_PKCS1_PSS_PADDING;
     else
         rctx->pad_mode = RSA_PKCS1_PADDING;
@@ -388,7 +388,7 @@ static int pkey_rsa_ctrl(EVP_PKEY_CTX *ctx, int type, int p1, void *p2)
                     goto bad_pad;
                 if (!rctx->md)
                     rctx->md = EVP_sha1();
-            } else if (ctx->pmeth->pkey_id == EVP_PKEY_RSA_PSS) {
+            } else if (pkey_ctx_is_pss(ctx)) {
                 goto bad_pad;
             }
             if (p1 == RSA_PKCS1_OAEP_PADDING) {
@@ -582,7 +582,7 @@ static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
                                EVP_PKEY_OP_TYPE_SIG | EVP_PKEY_OP_TYPE_CRYPT,
                                EVP_PKEY_CTRL_RSA_MGF1_MD, value);
 
-    if (ctx->pmeth->pkey_id == EVP_PKEY_RSA_PSS) {
+    if (pkey_ctx_is_pss(ctx)) {
 
         if (strcmp(type, "rsa_pss_keygen_mgf1_md") == 0)
             return EVP_PKEY_CTX_md(ctx, EVP_PKEY_OP_KEYGEN,
@@ -623,8 +623,9 @@ static int pkey_rsa_ctrl_str(EVP_PKEY_CTX *ctx,
 static int rsa_set_pss_param(RSA *rsa, EVP_PKEY_CTX *ctx)
 {
     RSA_PKEY_CTX *rctx = ctx->data;
-    if (ctx->pmeth->pkey_id != EVP_PKEY_RSA_PSS)
+    if (!pkey_ctx_is_pss(ctx))
         return 1;
+    /* If all parameters are default values don't set pss */
     if (rctx->md == NULL && rctx->mgf1md == NULL && rctx->saltlen == -2)
         return 1;
     rsa->pss = rsa_pss_params_create(rctx->md, rctx->mgf1md,