Fix the checks of EVP_PKEY_check

Reviewed-by: Tomas Mraz <tomas@openssl.org>
Reviewed-by: Paul Dale <pauli@openssl.org>
Reviewed-by: Todd Short <todd.short@me.com>
(Merged from https://github.com/openssl/openssl/pull/18424)

diff --git a/apps/ec.c b/apps/ec.c
index fe9a774ab4d524f96655f2d4b4326efcb454471f..0962d6a50037326d549acb17e0cfb1ab471dd69b 100644 (file)
--- a/apps/ec.c
+++ b/apps/ec.c
@@ -230,7 +230,7 @@ int ec_main(int argc, char **argv)
             BIO_printf(bio_err, "unable to check EC key\n");
             goto end;
         }
-        if (!EVP_PKEY_check(pctx))
+        if (EVP_PKEY_check(pctx) <= 0)
             BIO_printf(bio_err, "EC Key Invalid!\n");
         else
             BIO_printf(bio_err, "EC Key valid.\n");

diff --git a/apps/rsa.c b/apps/rsa.c
index 11a2adbc3eb4b324d15d8b7fefebf19fccc5d22d..9df07aba5bc9b1a0170084f5035a3f6b7f26db49 100644 (file)
--- a/apps/rsa.c
+++ b/apps/rsa.c
@@ -302,7 +302,7 @@ int rsa_main(int argc, char **argv)
         } else if (r == 0) {
             BIO_printf(bio_err, "RSA key not ok\n");
             ERR_print_errors(bio_err);
-        } else if (r == -1) {
+        } else if (r < 0) {
             ERR_print_errors(bio_err);
             goto end;
         }

diff --git a/test/evp_pkey_provided_test.c b/test/evp_pkey_provided_test.c
index 52228db4e31ccadb34852d4521c28a8caa13fa4b..7af5796113ddd71d0b43467a5a21f428d233e677 100644 (file)
--- a/test/evp_pkey_provided_test.c
+++ b/test/evp_pkey_provided_test.c
@@ -396,7 +396,7 @@ static int test_fromdata_rsa(void)
         if (!TEST_ptr(key_ctx = EVP_PKEY_CTX_new_from_pkey(NULL, pk, "")))
             goto err;
 
-        if (!TEST_true(EVP_PKEY_check(key_ctx))
+        if (!TEST_int_gt(EVP_PKEY_check(key_ctx), 0)
             || !TEST_true(EVP_PKEY_public_check(key_ctx))
             || !TEST_true(EVP_PKEY_private_check(key_ctx))
             || !TEST_true(EVP_PKEY_pairwise_check(key_ctx)))
@@ -660,7 +660,7 @@ static int test_fromdata_dh_named_group(void)
         if (!TEST_ptr(key_ctx = EVP_PKEY_CTX_new_from_pkey(NULL, pk, "")))
             goto err;
 
-        if (!TEST_true(EVP_PKEY_check(key_ctx))
+        if (!TEST_int_gt(EVP_PKEY_check(key_ctx), 0)
             || !TEST_true(EVP_PKEY_public_check(key_ctx))
             || !TEST_true(EVP_PKEY_private_check(key_ctx))
             || !TEST_true(EVP_PKEY_pairwise_check(key_ctx)))
@@ -841,7 +841,7 @@ static int test_fromdata_dh_fips186_4(void)
         if (!TEST_ptr(key_ctx = EVP_PKEY_CTX_new_from_pkey(NULL, pk, "")))
             goto err;
 
-        if (!TEST_true(EVP_PKEY_check(key_ctx))
+        if (!TEST_int_gt(EVP_PKEY_check(key_ctx), 0)
             || !TEST_true(EVP_PKEY_public_check(key_ctx))
             || !TEST_true(EVP_PKEY_private_check(key_ctx))
             || !TEST_true(EVP_PKEY_pairwise_check(key_ctx)))
@@ -1096,7 +1096,7 @@ static int test_fromdata_ecx(int tst)
         if (!TEST_ptr(ctx2 = EVP_PKEY_CTX_new_from_pkey(NULL, pk, NULL)))
             goto err;
         if (tst <= 7) {
-            if (!TEST_true(EVP_PKEY_check(ctx2)))
+            if (!TEST_int_gt(EVP_PKEY_check(ctx2), 0))
                 goto err;
             if (!TEST_true(EVP_PKEY_get_octet_string_param(
                                pk, orig_fromdata_params[PRIV_KEY].key,
@@ -1115,7 +1115,7 @@ static int test_fromdata_ecx(int tst)
             /* The private key check should fail if there is only a public key */
             if (!TEST_true(EVP_PKEY_public_check(ctx2))
                 || !TEST_false(EVP_PKEY_private_check(ctx2))
-                || !TEST_false(EVP_PKEY_check(ctx2)))
+                || !TEST_int_le(EVP_PKEY_check(ctx2), 0))
                 goto err;
         }
         EVP_PKEY_CTX_free(ctx2);
@@ -1606,7 +1606,7 @@ static int test_fromdata_dsa_fips186_4(void)
         if (!TEST_ptr(key_ctx = EVP_PKEY_CTX_new_from_pkey(NULL, pk, "")))
             goto err;
 
-        if (!TEST_true(EVP_PKEY_check(key_ctx))
+        if (!TEST_int_gt(EVP_PKEY_check(key_ctx), 0)
             || !TEST_true(EVP_PKEY_public_check(key_ctx))
             || !TEST_true(EVP_PKEY_private_check(key_ctx))
             || !TEST_true(EVP_PKEY_pairwise_check(key_ctx)))
@@ -1660,7 +1660,7 @@ static int test_check_dsa(void)
     EVP_PKEY_CTX *ctx = NULL;
 
     if (!TEST_ptr(ctx = EVP_PKEY_CTX_new_from_name(NULL, "DSA", NULL))
-        || !TEST_false(EVP_PKEY_check(ctx))
+        || !TEST_int_le(EVP_PKEY_check(ctx), 0)
         || !TEST_false(EVP_PKEY_public_check(ctx))
         || !TEST_false(EVP_PKEY_private_check(ctx))
         || !TEST_false(EVP_PKEY_pairwise_check(ctx)))