Fix: invoking X509_self_signed improperly

Reviewed-by: Paul Dale <pauli@openssl.org>
Reviewed-by: Tomas Mraz <tomas@openssl.org>
(Merged from https://github.com/openssl/openssl/pull/16976)

diff --git a/crypto/x509/x509_cmp.c b/crypto/x509/x509_cmp.c
index 8b4e46a5895618c967efae990281a7d6521f3319..f3d58cdfa61331a5fe6f01b3a4ed8fd39cc49a88 100644 (file)
--- a/crypto/x509/x509_cmp.c
+++ b/crypto/x509/x509_cmp.c
@@ -208,8 +208,12 @@ int X509_add_cert(STACK_OF(X509) *sk, X509 *cert, int flags)
                 return 1;
         }
     }
-    if ((flags & X509_ADD_FLAG_NO_SS) != 0 && X509_self_signed(cert, 0))
-        return 1;
+    if ((flags & X509_ADD_FLAG_NO_SS) != 0) {
+        int ret = X509_self_signed(cert, 0);
+
+        if (ret != 0)
+            return ret > 0 ? 1 : 0;
+    }
     if (!sk_X509_insert(sk, cert,
                         (flags & X509_ADD_FLAG_PREPEND) != 0 ? 0 : -1)) {
         ERR_raise(ERR_LIB_X509, ERR_R_MALLOC_FAILURE);

diff --git a/crypto/x509/x509_vfy.c b/crypto/x509/x509_vfy.c
index 1039bad305fe57caa80426f1c32ad0a44bd5afd6..7221bbe050e3652510dab3ce6e2b35451807c62d 100644 (file)
--- a/crypto/x509/x509_vfy.c
+++ b/crypto/x509/x509_vfy.c
@@ -3231,7 +3231,7 @@ static int build_chain(X509_STORE_CTX *ctx)
             if (!ossl_assert(num == ctx->num_untrusted))
                 goto int_err;
             curr = sk_X509_value(ctx->chain, num - 1);
-            issuer = (X509_self_signed(curr, 0) || num > max_depth) ?
+            issuer = (X509_self_signed(curr, 0) > 0 || num > max_depth) ?
                 NULL : find_issuer(ctx, sk_untrusted, curr);
             if (issuer == NULL) {
                 /*
@@ -3302,7 +3302,7 @@ static int build_chain(X509_STORE_CTX *ctx)
         CB_FAIL_IF(DANETLS_ENABLED(dane)
                        && (!DANETLS_HAS_PKIX(dane) || dane->pdpth >= 0),
                    ctx, NULL, num - 1, X509_V_ERR_DANE_NO_MATCH);
-        if (X509_self_signed(sk_X509_value(ctx->chain, num - 1), 0))
+        if (X509_self_signed(sk_X509_value(ctx->chain, num - 1), 0) > 0)
             return verify_cb_cert(ctx, NULL, num - 1,
                                   num == 1
                                   ? X509_V_ERR_DEPTH_ZERO_SELF_SIGNED_CERT