Avoid overflow issues in X509_cmp.
[openssl.git] / crypto / x509 / x509_cmp.c
index c7b0fe1e2955c1baf590077b6928a2ddc2104c23..831cfb70f02827359760909f4ac412dbb1dd3aa3 100644 (file)
@@ -1,4 +1,3 @@
-/* crypto/x509/x509_cmp.c */
 /* Copyright (C) 1995-1998 Eric Young (eay@cryptsoft.com)
  * All rights reserved.
  *
@@ -58,7 +57,7 @@
 
 #include <stdio.h>
 #include <ctype.h>
-#include "cryptlib.h"
+#include "internal/cryptlib.h"
 #include <openssl/asn1.h>
 #include <openssl/objects.h>
 #include <openssl/x509.h>
 int X509_issuer_and_serial_cmp(const X509 *a, const X509 *b)
 {
     int i;
-    X509_CINF *ai, *bi;
+    const X509_CINF *ai, *bi;
 
-    ai = a->cert_info;
-    bi = b->cert_info;
-    i = ASN1_INTEGER_cmp(ai->serialNumber, bi->serialNumber);
+    ai = &a->cert_info;
+    bi = &b->cert_info;
+    i = ASN1_INTEGER_cmp(&ai->serialNumber, &bi->serialNumber);
     if (i)
         return (i);
     return (X509_NAME_cmp(ai->issuer, bi->issuer));
@@ -82,45 +81,46 @@ int X509_issuer_and_serial_cmp(const X509 *a, const X509 *b)
 unsigned long X509_issuer_and_serial_hash(X509 *a)
 {
     unsigned long ret = 0;
-    EVP_MD_CTX ctx;
+    EVP_MD_CTX *ctx = EVP_MD_CTX_new();
     unsigned char md[16];
     char *f;
 
-    EVP_MD_CTX_init(&ctx);
-    f = X509_NAME_oneline(a->cert_info->issuer, NULL, 0);
-    if (!EVP_DigestInit_ex(&ctx, EVP_md5(), NULL))
+    if (ctx == NULL)
         goto err;
-    if (!EVP_DigestUpdate(&ctx, (unsigned char *)f, strlen(f)))
+    f = X509_NAME_oneline(a->cert_info.issuer, NULL, 0);
+    if (!EVP_DigestInit_ex(ctx, EVP_md5(), NULL))
+        goto err;
+    if (!EVP_DigestUpdate(ctx, (unsigned char *)f, strlen(f)))
         goto err;
     OPENSSL_free(f);
     if (!EVP_DigestUpdate
-        (&ctx, (unsigned char *)a->cert_info->serialNumber->data,
-         (unsigned long)a->cert_info->serialNumber->length))
+        (ctx, (unsigned char *)a->cert_info.serialNumber.data,
+         (unsigned long)a->cert_info.serialNumber.length))
         goto err;
-    if (!EVP_DigestFinal_ex(&ctx, &(md[0]), NULL))
+    if (!EVP_DigestFinal_ex(ctx, &(md[0]), NULL))
         goto err;
     ret = (((unsigned long)md[0]) | ((unsigned long)md[1] << 8L) |
            ((unsigned long)md[2] << 16L) | ((unsigned long)md[3] << 24L)
         ) & 0xffffffffL;
  err:
-    EVP_MD_CTX_cleanup(&ctx);
+    EVP_MD_CTX_free(ctx);
     return (ret);
 }
 #endif
 
 int X509_issuer_name_cmp(const X509 *a, const X509 *b)
 {
-    return (X509_NAME_cmp(a->cert_info->issuer, b->cert_info->issuer));
+    return (X509_NAME_cmp(a->cert_info.issuer, b->cert_info.issuer));
 }
 
 int X509_subject_name_cmp(const X509 *a, const X509 *b)
 {
-    return (X509_NAME_cmp(a->cert_info->subject, b->cert_info->subject));
+    return (X509_NAME_cmp(a->cert_info.subject, b->cert_info.subject));
 }
 
 int X509_CRL_cmp(const X509_CRL *a, const X509_CRL *b)
 {
-    return (X509_NAME_cmp(a->crl->issuer, b->crl->issuer));
+    return (X509_NAME_cmp(a->crl.issuer, b->crl.issuer));
 }
 
 int X509_CRL_match(const X509_CRL *a, const X509_CRL *b)
@@ -130,40 +130,40 @@ int X509_CRL_match(const X509_CRL *a, const X509_CRL *b)
 
 X509_NAME *X509_get_issuer_name(X509 *a)
 {
-    return (a->cert_info->issuer);
+    return (a->cert_info.issuer);
 }
 
 unsigned long X509_issuer_name_hash(X509 *x)
 {
-    return (X509_NAME_hash(x->cert_info->issuer));
+    return (X509_NAME_hash(x->cert_info.issuer));
 }
 
 #ifndef OPENSSL_NO_MD5
 unsigned long X509_issuer_name_hash_old(X509 *x)
 {
-    return (X509_NAME_hash_old(x->cert_info->issuer));
+    return (X509_NAME_hash_old(x->cert_info.issuer));
 }
 #endif
 
 X509_NAME *X509_get_subject_name(X509 *a)
 {
-    return (a->cert_info->subject);
+    return (a->cert_info.subject);
 }
 
 ASN1_INTEGER *X509_get_serialNumber(X509 *a)
 {
-    return (a->cert_info->serialNumber);
+    return &a->cert_info.serialNumber;
 }
 
 unsigned long X509_subject_name_hash(X509 *x)
 {
-    return (X509_NAME_hash(x->cert_info->subject));
+    return (X509_NAME_hash(x->cert_info.subject));
 }
 
 #ifndef OPENSSL_NO_MD5
 unsigned long X509_subject_name_hash_old(X509 *x)
 {
-    return (X509_NAME_hash_old(x->cert_info->subject));
+    return (X509_NAME_hash_old(x->cert_info.subject));
 }
 #endif
 
@@ -186,12 +186,13 @@ int X509_cmp(const X509 *a, const X509 *b)
     if (rv)
         return rv;
     /* Check for match against stored encoding too */
-    if (!a->cert_info->enc.modified && !b->cert_info->enc.modified) {
-        rv = (int)(a->cert_info->enc.len - b->cert_info->enc.len);
-        if (rv)
-            return rv;
-        return memcmp(a->cert_info->enc.enc, b->cert_info->enc.enc,
-                      a->cert_info->enc.len);
+    if (!a->cert_info.enc.modified && !b->cert_info.enc.modified) {
+        if (a->cert_info.enc.len < b->cert_info.enc.len)
+            return -1;
+        if (a->cert_info.enc.len > b->cert_info.enc.len)
+            return 1;
+        return memcmp(a->cert_info.enc.enc, b->cert_info.enc.enc,
+                      a->cert_info.enc.len);
     }
     return rv;
 }
@@ -248,21 +249,23 @@ unsigned long X509_NAME_hash(X509_NAME *x)
 
 unsigned long X509_NAME_hash_old(X509_NAME *x)
 {
-    EVP_MD_CTX md_ctx;
+    EVP_MD_CTX *md_ctx = EVP_MD_CTX_new();
     unsigned long ret = 0;
     unsigned char md[16];
 
+    if (md_ctx == NULL)
+        return ret;
+
     /* Make sure X509_NAME structure contains valid cached encoding */
     i2d_X509_NAME(x, NULL);
-    EVP_MD_CTX_init(&md_ctx);
-    EVP_MD_CTX_set_flags(&md_ctx, EVP_MD_CTX_FLAG_NON_FIPS_ALLOW);
-    if (EVP_DigestInit_ex(&md_ctx, EVP_md5(), NULL)
-        && EVP_DigestUpdate(&md_ctx, x->bytes->data, x->bytes->length)
-        && EVP_DigestFinal_ex(&md_ctx, md, NULL))
+    EVP_MD_CTX_set_flags(md_ctx, EVP_MD_CTX_FLAG_NON_FIPS_ALLOW);
+    if (EVP_DigestInit_ex(md_ctx, EVP_md5(), NULL)
+        && EVP_DigestUpdate(md_ctx, x->bytes->data, x->bytes->length)
+        && EVP_DigestFinal_ex(md_ctx, md, NULL))
         ret = (((unsigned long)md[0]) | ((unsigned long)md[1] << 8L) |
                ((unsigned long)md[2] << 16L) | ((unsigned long)md[3] << 24L)
             ) & 0xffffffffL;
-    EVP_MD_CTX_cleanup(&md_ctx);
+    EVP_MD_CTX_free(md_ctx);
 
     return (ret);
 }
@@ -273,15 +276,13 @@ X509 *X509_find_by_issuer_and_serial(STACK_OF(X509) *sk, X509_NAME *name,
                                      ASN1_INTEGER *serial)
 {
     int i;
-    X509_CINF cinf;
     X509 x, *x509 = NULL;
 
     if (!sk)
         return NULL;
 
-    x.cert_info = &cinf;
-    cinf.serialNumber = serial;
-    cinf.issuer = name;
+    x.cert_info.serialNumber = *serial;
+    x.cert_info.issuer = name;
 
     for (i = 0; i < sk_X509_num(sk); i++) {
         x509 = sk_X509_value(sk, i);
@@ -304,18 +305,18 @@ X509 *X509_find_by_subject(STACK_OF(X509) *sk, X509_NAME *name)
     return (NULL);
 }
 
-EVP_PKEY *X509_get_pubkey(X509 *x)
+EVP_PKEY *X509_get0_pubkey(X509 *x)
 {
-    if ((x == NULL) || (x->cert_info == NULL))
-        return (NULL);
-    return (X509_PUBKEY_get(x->cert_info->key));
+    if (x == NULL)
+        return NULL;
+    return X509_PUBKEY_get0(x->cert_info.key);
 }
 
-ASN1_BIT_STRING *X509_get0_pubkey_bitstr(const X509 *x)
+EVP_PKEY *X509_get_pubkey(X509 *x)
 {
-    if (!x)
+    if (x == NULL)
         return NULL;
-    return x->cert_info->key->public_key;
+    return X509_PUBKEY_get(x->cert_info.key);
 }
 
 int X509_check_private_key(X509 *x, EVP_PKEY *k)
@@ -323,7 +324,7 @@ int X509_check_private_key(X509 *x, EVP_PKEY *k)
     EVP_PKEY *xk;
     int ret;
 
-    xk = X509_get_pubkey(x);
+    xk = X509_get0_pubkey(x);
 
     if (xk)
         ret = EVP_PKEY_cmp(xk, k);
@@ -342,7 +343,6 @@ int X509_check_private_key(X509 *x, EVP_PKEY *k)
     case -2:
         X509err(X509_F_X509_CHECK_PRIVATE_KEY, X509_R_UNKNOWN_KEY_TYPE);
     }
-    EVP_PKEY_free(xk);
     if (ret > 0)
         return 1;
     return 0;
@@ -360,8 +360,8 @@ static int check_suite_b(EVP_PKEY *pkey, int sign_nid, unsigned long *pflags)
 {
     const EC_GROUP *grp = NULL;
     int curve_nid;
-    if (pkey && pkey->type == EVP_PKEY_EC)
-        grp = EC_KEY_get0_group(pkey->pkey.ec);
+    if (pkey && EVP_PKEY_id(pkey) == EVP_PKEY_EC)
+        grp = EC_KEY_get0_group(EVP_PKEY_get0_EC_KEY(pkey));
     if (!grp)
         return X509_V_ERR_SUITE_B_INVALID_ALGORITHM;
     curve_nid = EC_GROUP_get_curve_name(grp);
@@ -391,11 +391,12 @@ int X509_chain_check_suiteb(int *perror_depth, X509 *x, STACK_OF(X509) *chain,
                             unsigned long flags)
 {
     int rv, i, sign_nid;
-    EVP_PKEY *pk = NULL;
-    unsigned long tflags;
+    EVP_PKEY *pk;
+    unsigned long tflags = flags;
+
     if (!(flags & X509_V_FLAG_SUITEB_128_LOS))
         return X509_V_OK;
-    tflags = flags;
+
     /* If no EE certificate passed in must be first in chain */
     if (x == NULL) {
         x = sk_X509_value(chain, 0);
@@ -403,6 +404,17 @@ int X509_chain_check_suiteb(int *perror_depth, X509 *x, STACK_OF(X509) *chain,
     } else
         i = 0;
 
+    pk = X509_get0_pubkey(x);
+
+    /*
+     * With DANE-EE(3) success, or DANE-EE(3)/PKIX-EE(1) failure we don't build
+     * a chain all, just report trust success or failure, but must also report
+     * Suite-B errors if applicable.  This is indicated via a NULL chain
+     * pointer.  All we need to do is check the leaf key algorithm.
+     */
+    if (chain == NULL)
+        return check_suite_b(pk, -1, &tflags);
+
     if (X509_get_version(x) != 2) {
         rv = X509_V_ERR_SUITE_B_INVALID_VERSION;
         /* Correct error depth */
@@ -410,7 +422,6 @@ int X509_chain_check_suiteb(int *perror_depth, X509 *x, STACK_OF(X509) *chain,
         goto end;
     }
 
-    pk = X509_get_pubkey(x);
     /* Check EE key only */
     rv = check_suite_b(pk, -1, &tflags);
     if (rv != X509_V_OK) {
@@ -425,8 +436,7 @@ int X509_chain_check_suiteb(int *perror_depth, X509 *x, STACK_OF(X509) *chain,
             rv = X509_V_ERR_SUITE_B_INVALID_VERSION;
             goto end;
         }
-        EVP_PKEY_free(pk);
-        pk = X509_get_pubkey(x);
+        pk = X509_get0_pubkey(x);
         rv = check_suite_b(pk, sign_nid, &tflags);
         if (rv != X509_V_OK)
             goto end;
@@ -435,7 +445,6 @@ int X509_chain_check_suiteb(int *perror_depth, X509 *x, STACK_OF(X509) *chain,
     /* Final check: root CA signature */
     rv = check_suite_b(pk, X509_get_signature_nid(x), &tflags);
  end:
-    EVP_PKEY_free(pk);
     if (rv != X509_V_OK) {
         /* Invalid signature or LOS errors are for previous cert */
         if ((rv == X509_V_ERR_SUITE_B_INVALID_SIGNATURE_ALGORITHM
@@ -443,7 +452,7 @@ int X509_chain_check_suiteb(int *perror_depth, X509 *x, STACK_OF(X509) *chain,
             i--;
         /*
          * If we have LOS error and flags changed then we are signing P-384
-         * with P-256. Use more meaninggul error.
+         * with P-256. Use more meaningful error.
          */
         if (rv == X509_V_ERR_SUITE_B_LOS_NOT_ALLOWED && flags != tflags)
             rv = X509_V_ERR_SUITE_B_CANNOT_SIGN_P_384_WITH_P_256;
@@ -458,7 +467,7 @@ int X509_CRL_check_suiteb(X509_CRL *crl, EVP_PKEY *pk, unsigned long flags)
     int sign_nid;
     if (!(flags & X509_V_FLAG_SUITEB_128_LOS))
         return X509_V_OK;
-    sign_nid = OBJ_obj2nid(crl->crl->sig_alg->algorithm);
+    sign_nid = OBJ_obj2nid(crl->crl.sig_alg.algorithm);
     return check_suite_b(pk, sign_nid, &flags);
 }
 
@@ -487,7 +496,7 @@ STACK_OF(X509) *X509_chain_up_ref(STACK_OF(X509) *chain)
     ret = sk_X509_dup(chain);
     for (i = 0; i < sk_X509_num(ret); i++) {
         X509 *x = sk_X509_value(ret, i);
-        CRYPTO_add(&x->references, 1, CRYPTO_LOCK_X509);
+        X509_up_ref(x);
     }
     return ret;
 }