Support keys with RSA_METHOD_FLAG_NO_CHECK with OCSP sign

OCSP_basic_sign_ctx() in ocsp_srv.c , does not check for RSA_METHOD_FLAG_NO_CHECK.
If a key has RSA_METHOD_FLAG_NO_CHECK set, OCSP sign operations can fail
because the X509_check_private_key() can fail.

The check for the RSA_METHOD_FLAG_NO_CHECK was moved to crypto/rsa/rsa_ameth.c
as a common place to check. Checks in ssl_rsa.c were removed.

Reviewed-by: Matt Caswell <matt@openssl.org>
Reviewed-by: Tim Hudson <tjh@openssl.org>
Reviewed-by: Tomas Mraz <tmraz@fedoraproject.org>
(Merged from https://github.com/openssl/openssl/pull/12419)

(cherry picked from commit 56e8fe0b4efbf582e40ae91319727c9d176c5e1e)

diff --git a/crypto/rsa/rsa_ameth.c b/crypto/rsa/rsa_ameth.c
index 130f6156c5d7b5ee7f86cfa9ce7e2ff86fb78b48..6558e1c6626a0d0cd862732de841651bd6adfb57 100644 (file)
--- a/crypto/rsa/rsa_ameth.c
+++ b/crypto/rsa/rsa_ameth.c
@@ -144,6 +144,15 @@ static int rsa_pub_decode(EVP_PKEY *pkey, const X509_PUBKEY *pubkey)
 
 static int rsa_pub_cmp(const EVP_PKEY *a, const EVP_PKEY *b)
 {
+    /*
+     * Don't check the public/private key, this is mostly for smart
+     * cards.
+     */
+    if (((RSA_flags(a->pkey.rsa) & RSA_METHOD_FLAG_NO_CHECK))
+            || (RSA_flags(b->pkey.rsa) & RSA_METHOD_FLAG_NO_CHECK)) {
+        return 1;
+    }
+
     if (BN_cmp(b->pkey.rsa->n, a->pkey.rsa->n) != 0
         || BN_cmp(b->pkey.rsa->e, a->pkey.rsa->e) != 0)
         return 0;

diff --git a/ssl/ssl_rsa.c b/ssl/ssl_rsa.c
index 3a28b60ba60effceaca98ea680feab6f66928d2d..76270b677e18de13d13c437e87afd80204ff3bd3 100644 (file)
--- a/ssl/ssl_rsa.c
+++ b/ssl/ssl_rsa.c
@@ -166,15 +166,6 @@ static int ssl_set_pkey(CERT *c, EVP_PKEY *pkey)
         EVP_PKEY_copy_parameters(pktmp, pkey);
         ERR_clear_error();
 
-#ifndef OPENSSL_NO_RSA
-        /*
-         * Don't check the public/private key, this is mostly for smart
-         * cards.
-         */
-        if (EVP_PKEY_id(pkey) == EVP_PKEY_RSA
-            && RSA_flags(EVP_PKEY_get0_RSA(pkey)) & RSA_METHOD_FLAG_NO_CHECK) ;
-        else
-#endif
         if (!X509_check_private_key(c->pkeys[i].x509, pkey)) {
             X509_free(c->pkeys[i].x509);
             c->pkeys[i].x509 = NULL;
@@ -365,16 +356,6 @@ static int ssl_set_cert(CERT *c, X509 *x)
         EVP_PKEY_copy_parameters(pkey, c->pkeys[i].privatekey);
         ERR_clear_error();
 
-#ifndef OPENSSL_NO_RSA
-        /*
-         * Don't check the public/private key, this is mostly for smart
-         * cards.
-         */
-        if (EVP_PKEY_id(c->pkeys[i].privatekey) == EVP_PKEY_RSA
-            && RSA_flags(EVP_PKEY_get0_RSA(c->pkeys[i].privatekey)) &
-            RSA_METHOD_FLAG_NO_CHECK) ;
-        else
-#endif                          /* OPENSSL_NO_RSA */
         if (!X509_check_private_key(x, c->pkeys[i].privatekey)) {
             /*
              * don't fail for a cert/key mismatch, just free current private
@@ -1134,13 +1115,6 @@ static int ssl_set_cert_and_key(SSL *ssl, SSL_CTX *ctx, X509 *x509, EVP_PKEY *pr
             EVP_PKEY_copy_parameters(pubkey, privatekey);
         } /* else both have parameters */
 
-        /* Copied from ssl_set_cert/pkey */
-#ifndef OPENSSL_NO_RSA
-        if ((EVP_PKEY_id(privatekey) == EVP_PKEY_RSA) &&
-            ((RSA_flags(EVP_PKEY_get0_RSA(privatekey)) & RSA_METHOD_FLAG_NO_CHECK)))
-            /* no-op */ ;
-        else
-#endif
         /* check that key <-> cert match */
         if (EVP_PKEY_eq(pubkey, privatekey) != 1) {
             SSLerr(SSL_F_SSL_SET_CERT_AND_KEY, SSL_R_PRIVATE_KEY_MISMATCH);