Avoid overflow issues in X509_cmp.

[openssl.git] / crypto / x509 / x509_cmp.c

diff --git a/crypto/x509/x509_cmp.c b/crypto/x509/x509_cmp.c
index 9d9ea4b6050d396f36ae268bd5ae85d256d18207..831cfb70f02827359760909f4ac412dbb1dd3aa3 100644 (file)
--- a/crypto/x509/x509_cmp.c
+++ b/crypto/x509/x509_cmp.c
@@ -1,4 +1,3 @@
-/* crypto/x509/x509_cmp.c */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -188,9 +187,10 @@ int X509_cmp(const X509 *a, const X509 *b)
         return rv;
     /* Check for match against stored encoding too */
     if (!a->cert_info.enc.modified && !b->cert_info.enc.modified) {
-        rv = (int)(a->cert_info.enc.len - b->cert_info.enc.len);
-        if (rv)
-            return rv;
+        if (a->cert_info.enc.len < b->cert_info.enc.len)
+            return -1;
+        if (a->cert_info.enc.len > b->cert_info.enc.len)
+            return 1;
         return memcmp(a->cert_info.enc.enc, b->cert_info.enc.enc,
                       a->cert_info.enc.len);
     }
@@ -319,13 +319,6 @@ EVP_PKEY *X509_get_pubkey(X509 *x)
     return X509_PUBKEY_get(x->cert_info.key);
 }
 
-ASN1_BIT_STRING *X509_get0_pubkey_bitstr(const X509 *x)
-{
-    if (!x)
-        return NULL;
-    return x->cert_info.key->public_key;
-}
-
 int X509_check_private_key(X509 *x, EVP_PKEY *k)
 {
     EVP_PKEY *xk;
@@ -367,8 +360,8 @@ static int check_suite_b(EVP_PKEY *pkey, int sign_nid, unsigned long *pflags)
 {
     const EC_GROUP *grp = NULL;
     int curve_nid;
-    if (pkey && pkey->type == EVP_PKEY_EC)
-        grp = EC_KEY_get0_group(pkey->pkey.ec);
+    if (pkey && EVP_PKEY_id(pkey) == EVP_PKEY_EC)
+        grp = EC_KEY_get0_group(EVP_PKEY_get0_EC_KEY(pkey));
     if (!grp)
         return X509_V_ERR_SUITE_B_INVALID_ALGORITHM;
     curve_nid = EC_GROUP_get_curve_name(grp);
@@ -398,11 +391,12 @@ int X509_chain_check_suiteb(int *perror_depth, X509 *x, STACK_OF(X509) *chain,
                             unsigned long flags)
 {
     int rv, i, sign_nid;
-    EVP_PKEY *pk = NULL;
-    unsigned long tflags;
+    EVP_PKEY *pk;
+    unsigned long tflags = flags;
+
     if (!(flags & X509_V_FLAG_SUITEB_128_LOS))
         return X509_V_OK;
-    tflags = flags;
+
     /* If no EE certificate passed in must be first in chain */
     if (x == NULL) {
         x = sk_X509_value(chain, 0);
@@ -410,6 +404,17 @@ int X509_chain_check_suiteb(int *perror_depth, X509 *x, STACK_OF(X509) *chain,
     } else
         i = 0;
 
+    pk = X509_get0_pubkey(x);
+
+    /*
+     * With DANE-EE(3) success, or DANE-EE(3)/PKIX-EE(1) failure we don't build
+     * a chain all, just report trust success or failure, but must also report
+     * Suite-B errors if applicable.  This is indicated via a NULL chain
+     * pointer.  All we need to do is check the leaf key algorithm.
+     */
+    if (chain == NULL)
+        return check_suite_b(pk, -1, &tflags);
+
     if (X509_get_version(x) != 2) {
         rv = X509_V_ERR_SUITE_B_INVALID_VERSION;
         /* Correct error depth */
@@ -417,7 +422,6 @@ int X509_chain_check_suiteb(int *perror_depth, X509 *x, STACK_OF(X509) *chain,
         goto end;
     }
 
-    pk = X509_get0_pubkey(x);
     /* Check EE key only */
     rv = check_suite_b(pk, -1, &tflags);
     if (rv != X509_V_OK) {
@@ -432,7 +436,7 @@ int X509_chain_check_suiteb(int *perror_depth, X509 *x, STACK_OF(X509) *chain,
             rv = X509_V_ERR_SUITE_B_INVALID_VERSION;
             goto end;
         }
-        pk = X509_get_pubkey(x);
+        pk = X509_get0_pubkey(x);
         rv = check_suite_b(pk, sign_nid, &tflags);
         if (rv != X509_V_OK)
             goto end;
@@ -448,7 +452,7 @@ int X509_chain_check_suiteb(int *perror_depth, X509 *x, STACK_OF(X509) *chain,
             i--;
         /*
          * If we have LOS error and flags changed then we are signing P-384
-         * with P-256. Use more meaninggul error.
+         * with P-256. Use more meaningful error.
          */
         if (rv == X509_V_ERR_SUITE_B_LOS_NOT_ALLOWED && flags != tflags)
             rv = X509_V_ERR_SUITE_B_CANNOT_SIGN_P_384_WITH_P_256;