X509: add more error codes on malloc or sk_TYP_push failure
[openssl.git] / crypto / x509 / x509_lu.c
index f40f3fc0810df9d6424b5f550ab3c8a90ce0a098..fbeed018f57e38cc2eaca93c21e23d1e0ef04533 100644 (file)
@@ -1,5 +1,5 @@
 /*
- * Copyright 1995-2016 The OpenSSL Project Authors. All Rights Reserved.
+ * Copyright 1995-2018 The OpenSSL Project Authors. All Rights Reserved.
  *
  * Licensed under the OpenSSL license (the "License").  You may not use
  * this file except in compliance with the License.  You can obtain a copy
@@ -9,7 +9,7 @@
 
 #include <stdio.h>
 #include "internal/cryptlib.h"
-#include <openssl/lhash.h>
+#include "internal/refcount.h"
 #include <openssl/x509.h>
 #include "internal/x509_int.h"
 #include <openssl/x509v3.h>
 
 X509_LOOKUP *X509_LOOKUP_new(X509_LOOKUP_METHOD *method)
 {
-    X509_LOOKUP *ret;
+    X509_LOOKUP *ret = OPENSSL_zalloc(sizeof(*ret));
 
-    ret = OPENSSL_zalloc(sizeof(*ret));
-    if (ret == NULL)
+    if (ret == NULL) {
+        X509err(X509_F_X509_LOOKUP_NEW, ERR_R_MALLOC_FAILURE);
         return NULL;
+    }
 
     ret->method = method;
-    if ((method->new_item != NULL) && !method->new_item(ret)) {
+    if (method->new_item != NULL && method->new_item(ret) == 0) {
         OPENSSL_free(ret);
         return NULL;
     }
@@ -81,38 +82,39 @@ int X509_LOOKUP_ctrl(X509_LOOKUP *ctx, int cmd, const char *argc, long argl,
         return 1;
 }
 
-int X509_LOOKUP_by_subject(X509_LOOKUP *ctx, int type, X509_NAME *name,
-                           X509_OBJECT *ret)
+int X509_LOOKUP_by_subject(X509_LOOKUP *ctx, X509_LOOKUP_TYPE type,
+                           X509_NAME *name, X509_OBJECT *ret)
 {
     if ((ctx->method == NULL) || (ctx->method->get_by_subject == NULL))
-        return X509_LU_FAIL;
+        return 0;
     if (ctx->skip)
         return 0;
     return ctx->method->get_by_subject(ctx, type, name, ret);
 }
 
-int X509_LOOKUP_by_issuer_serial(X509_LOOKUP *ctx, int type, X509_NAME *name,
-                                 ASN1_INTEGER *serial, X509_OBJECT *ret)
+int X509_LOOKUP_by_issuer_serial(X509_LOOKUP *ctx, X509_LOOKUP_TYPE type,
+                                 X509_NAME *name, ASN1_INTEGER *serial,
+                                 X509_OBJECT *ret)
 {
     if ((ctx->method == NULL) || (ctx->method->get_by_issuer_serial == NULL))
-        return X509_LU_FAIL;
+        return 0;
     return ctx->method->get_by_issuer_serial(ctx, type, name, serial, ret);
 }
 
-int X509_LOOKUP_by_fingerprint(X509_LOOKUP *ctx, int type,
-                               unsigned char *bytes, int len,
+int X509_LOOKUP_by_fingerprint(X509_LOOKUP *ctx, X509_LOOKUP_TYPE type,
+                               const unsigned char *bytes, int len,
                                X509_OBJECT *ret)
 {
     if ((ctx->method == NULL) || (ctx->method->get_by_fingerprint == NULL))
-        return X509_LU_FAIL;
+        return 0;
     return ctx->method->get_by_fingerprint(ctx, type, bytes, len, ret);
 }
 
-int X509_LOOKUP_by_alias(X509_LOOKUP *ctx, int type, char *str, int len,
-                         X509_OBJECT *ret)
+int X509_LOOKUP_by_alias(X509_LOOKUP *ctx, X509_LOOKUP_TYPE type,
+                         const char *str, int len, X509_OBJECT *ret)
 {
     if ((ctx->method == NULL) || (ctx->method->get_by_alias == NULL))
-        return X509_LU_FAIL;
+        return 0;
     return ctx->method->get_by_alias(ctx, type, str, len, ret);
 }
 
@@ -131,7 +133,7 @@ static int x509_object_cmp(const X509_OBJECT *const *a,
     case X509_LU_CRL:
         ret = X509_CRL_cmp((*a)->data.crl, (*b)->data.crl);
         break;
-    default:
+    case X509_LU_NONE:
         /* abort(); */
         return 0;
     }
@@ -140,25 +142,36 @@ static int x509_object_cmp(const X509_OBJECT *const *a,
 
 X509_STORE *X509_STORE_new(void)
 {
-    X509_STORE *ret;
+    X509_STORE *ret = OPENSSL_zalloc(sizeof(*ret));
 
-    if ((ret = OPENSSL_zalloc(sizeof(*ret))) == NULL)
+    if (ret == NULL) {
+        X509err(X509_F_X509_STORE_NEW, ERR_R_MALLOC_FAILURE);
         return NULL;
-    if ((ret->objs = sk_X509_OBJECT_new(x509_object_cmp)) == NULL)
+    }
+    if ((ret->objs = sk_X509_OBJECT_new(x509_object_cmp)) == NULL) {
+        X509err(X509_F_X509_STORE_NEW, ERR_R_MALLOC_FAILURE);
         goto err;
+    }
     ret->cache = 1;
-    if ((ret->get_cert_methods = sk_X509_LOOKUP_new_null()) == NULL)
+    if ((ret->get_cert_methods = sk_X509_LOOKUP_new_null()) == NULL) {
+        X509err(X509_F_X509_STORE_NEW, ERR_R_MALLOC_FAILURE);
         goto err;
+    }
 
-    if ((ret->param = X509_VERIFY_PARAM_new()) == NULL)
+    if ((ret->param = X509_VERIFY_PARAM_new()) == NULL) {
+        X509err(X509_F_X509_STORE_NEW, ERR_R_MALLOC_FAILURE);
         goto err;
-
-    if (!CRYPTO_new_ex_data(CRYPTO_EX_INDEX_X509_STORE, ret, &ret->ex_data))
+    }
+    if (!CRYPTO_new_ex_data(CRYPTO_EX_INDEX_X509_STORE, ret, &ret->ex_data)) {
+        X509err(X509_F_X509_STORE_NEW, ERR_R_MALLOC_FAILURE);
         goto err;
+    }
 
     ret->lock = CRYPTO_THREAD_lock_new();
-    if (ret->lock == NULL)
+    if (ret->lock == NULL) {
+        X509err(X509_F_X509_STORE_NEW, ERR_R_MALLOC_FAILURE);
         goto err;
+    }
 
     ret->references = 1;
     return ret;
@@ -171,21 +184,6 @@ err:
     return NULL;
 }
 
-static void cleanup(X509_OBJECT *a)
-{
-    if (!a)
-        return;
-    if (a->type == X509_LU_X509) {
-        X509_free(a->data.x509);
-    } else if (a->type == X509_LU_CRL) {
-        X509_CRL_free(a->data.crl);
-    } else {
-        /* abort(); */
-    }
-
-    OPENSSL_free(a);
-}
-
 void X509_STORE_free(X509_STORE *vfy)
 {
     int i;
@@ -194,8 +192,7 @@ void X509_STORE_free(X509_STORE *vfy)
 
     if (vfy == NULL)
         return;
-
-    CRYPTO_atomic_add(&vfy->references, -1, &i, vfy->lock);
+    CRYPTO_DOWN_REF(&vfy->references, &i, vfy->lock);
     REF_PRINT_COUNT("X509_STORE", vfy);
     if (i > 0)
         return;
@@ -208,7 +205,7 @@ void X509_STORE_free(X509_STORE *vfy)
         X509_LOOKUP_free(lu);
     }
     sk_X509_LOOKUP_free(sk);
-    sk_X509_OBJECT_pop_free(vfy->objs, cleanup);
+    sk_X509_OBJECT_pop_free(vfy->objs, X509_OBJECT_free);
 
     CRYPTO_free_ex_data(CRYPTO_EX_INDEX_X509_STORE, vfy, &vfy->ex_data);
     X509_VERIFY_PARAM_free(vfy->param);
@@ -220,7 +217,7 @@ int X509_STORE_up_ref(X509_STORE *vfy)
 {
     int i;
 
-    if (CRYPTO_atomic_add(&vfy->references, 1, &i, vfy->lock) <= 0)
+    if (CRYPTO_UP_REF(&vfy->references, &i, vfy->lock) <= 0)
         return 0;
 
     REF_PRINT_COUNT("X509_STORE", a);
@@ -243,20 +240,22 @@ X509_LOOKUP *X509_STORE_add_lookup(X509_STORE *v, X509_LOOKUP_METHOD *m)
     }
     /* a new one */
     lu = X509_LOOKUP_new(m);
-    if (lu == NULL)
+    if (lu == NULL) {
+        X509err(X509_F_X509_STORE_ADD_LOOKUP, ERR_R_MALLOC_FAILURE);
         return NULL;
-    else {
-        lu->store_ctx = v;
-        if (sk_X509_LOOKUP_push(v->get_cert_methods, lu))
-            return lu;
-        else {
-            X509_LOOKUP_free(lu);
-            return NULL;
-        }
     }
+
+    lu->store_ctx = v;
+    if (sk_X509_LOOKUP_push(v->get_cert_methods, lu))
+        return lu;
+    /* malloc failed */
+    X509err(X509_F_X509_STORE_ADD_LOOKUP, ERR_R_MALLOC_FAILURE);
+    X509_LOOKUP_free(lu);
+    return NULL;
 }
 
-X509_OBJECT *X509_STORE_CTX_get_obj_by_subject(X509_STORE_CTX *vs, int type,
+X509_OBJECT *X509_STORE_CTX_get_obj_by_subject(X509_STORE_CTX *vs,
+                                               X509_LOOKUP_TYPE type,
                                                X509_NAME *name)
 {
     X509_OBJECT *ret = X509_OBJECT_new();
@@ -278,6 +277,9 @@ int X509_STORE_CTX_get_by_subject(X509_STORE_CTX *vs, X509_LOOKUP_TYPE type,
     X509_OBJECT stmp, *tmp;
     int i, j;
 
+    if (ctx == NULL)
+        return 0;
+
     CRYPTO_THREAD_write_lock(ctx->lock);
     tmp = X509_OBJECT_retrieve_by_subject(ctx->objs, type, name);
     CRYPTO_THREAD_unlock(ctx->lock);
@@ -303,26 +305,29 @@ int X509_STORE_CTX_get_by_subject(X509_STORE_CTX *vs, X509_LOOKUP_TYPE type,
     return 1;
 }
 
-int X509_STORE_add_cert(X509_STORE *ctx, X509 *x)
-{
+static int x509_store_add(X509_STORE *ctx, void *x, int crl) {
     X509_OBJECT *obj;
-    int ret = 1, added = 1;
+    int ret = 0, added = 0;
 
     if (x == NULL)
         return 0;
     obj = X509_OBJECT_new();
     if (obj == NULL)
         return 0;
-    obj->type = X509_LU_X509;
-    obj->data.x509 = x;
+
+    if (crl) {
+        obj->type = X509_LU_CRL;
+        obj->data.crl = (X509_CRL *)x;
+    } else {
+        obj->type = X509_LU_X509;
+        obj->data.x509 = (X509 *)x;
+    }
     X509_OBJECT_up_ref_count(obj);
 
     CRYPTO_THREAD_write_lock(ctx->lock);
 
     if (X509_OBJECT_retrieve_match(ctx->objs, obj)) {
-        X509err(X509_F_X509_STORE_ADD_CERT,
-                X509_R_CERT_ALREADY_IN_HASH_TABLE);
-        ret = 0;
+        ret = 1;
     } else {
         added = sk_X509_OBJECT_push(ctx->objs, obj);
         ret = added != 0;
@@ -330,52 +335,34 @@ int X509_STORE_add_cert(X509_STORE *ctx, X509 *x)
 
     CRYPTO_THREAD_unlock(ctx->lock);
 
-    if (!ret)                   /* obj not pushed */
+    if (added == 0)             /* obj not pushed */
         X509_OBJECT_free(obj);
-    if (!added)                 /* on push failure */
-        X509err(X509_F_X509_STORE_ADD_CERT, ERR_R_MALLOC_FAILURE);
 
     return ret;
 }
 
-int X509_STORE_add_crl(X509_STORE *ctx, X509_CRL *x)
+int X509_STORE_add_cert(X509_STORE *ctx, X509 *x)
 {
-    X509_OBJECT *obj;
-    int ret = 1, added = 1;
-
-    if (x == NULL)
-        return 0;
-    obj = X509_OBJECT_new();
-    if (obj == NULL)
+    if (!x509_store_add(ctx, x, 0)) {
+        X509err(X509_F_X509_STORE_ADD_CERT, ERR_R_MALLOC_FAILURE);
         return 0;
-    obj->type = X509_LU_CRL;
-    obj->data.crl = x;
-    X509_OBJECT_up_ref_count(obj);
-
-    CRYPTO_THREAD_write_lock(ctx->lock);
-
-    if (X509_OBJECT_retrieve_match(ctx->objs, obj)) {
-        X509err(X509_F_X509_STORE_ADD_CRL, X509_R_CERT_ALREADY_IN_HASH_TABLE);
-        ret = 0;
-    } else {
-        added = sk_X509_OBJECT_push(ctx->objs, obj);
-        ret = added != 0;
     }
+    return 1;
+}
 
-    CRYPTO_THREAD_unlock(ctx->lock);
-
-    if (!ret)                   /* obj not pushed */
-        X509_OBJECT_free(obj);
-    if (!added)                 /* on push failure */
+int X509_STORE_add_crl(X509_STORE *ctx, X509_CRL *x)
+{
+    if (!x509_store_add(ctx, x, 1)) {
         X509err(X509_F_X509_STORE_ADD_CRL, ERR_R_MALLOC_FAILURE);
-
-    return ret;
+        return 0;
+    }
+    return 1;
 }
 
 int X509_OBJECT_up_ref_count(X509_OBJECT *a)
 {
     switch (a->type) {
-    default:
+    case X509_LU_NONE:
         break;
     case X509_LU_X509:
         return X509_up_ref(a->data.x509);
@@ -399,7 +386,7 @@ X509_CRL *X509_OBJECT_get0_X509_CRL(X509_OBJECT *a)
     return a->data.crl;
 }
 
-int X509_OBJECT_get_type(const X509_OBJECT *a)
+X509_LOOKUP_TYPE X509_OBJECT_get_type(const X509_OBJECT *a)
 {
     return a->type;
 }
@@ -412,7 +399,7 @@ X509_OBJECT *X509_OBJECT_new()
         X509err(X509_F_X509_OBJECT_NEW, ERR_R_MALLOC_FAILURE);
         return NULL;
     }
-    ret->type = X509_LU_FAIL;
+    ret->type = X509_LU_NONE;
     return ret;
 }
 
@@ -422,7 +409,7 @@ void X509_OBJECT_free(X509_OBJECT *a)
     if (a == NULL)
         return;
     switch (a->type) {
-    default:
+    case X509_LU_NONE:
         break;
     case X509_LU_X509:
         X509_free(a->data.x509);
@@ -434,7 +421,7 @@ void X509_OBJECT_free(X509_OBJECT *a)
     OPENSSL_free(a);
 }
 
-static int x509_object_idx_cnt(STACK_OF(X509_OBJECT) *h, int type,
+static int x509_object_idx_cnt(STACK_OF(X509_OBJECT) *h, X509_LOOKUP_TYPE type,
                                X509_NAME *name, int *pnmatch)
 {
     X509_OBJECT stmp;
@@ -452,7 +439,7 @@ static int x509_object_idx_cnt(STACK_OF(X509_OBJECT) *h, int type,
         stmp.data.crl = &crl_s;
         crl_s.crl.issuer = name;
         break;
-    default:
+    case X509_LU_NONE:
         /* abort(); */
         return -1;
     }
@@ -473,14 +460,15 @@ static int x509_object_idx_cnt(STACK_OF(X509_OBJECT) *h, int type,
     return idx;
 }
 
-int X509_OBJECT_idx_by_subject(STACK_OF(X509_OBJECT) *h, int type,
+int X509_OBJECT_idx_by_subject(STACK_OF(X509_OBJECT) *h, X509_LOOKUP_TYPE type,
                                X509_NAME *name)
 {
     return x509_object_idx_cnt(h, type, name, NULL);
 }
 
 X509_OBJECT *X509_OBJECT_retrieve_by_subject(STACK_OF(X509_OBJECT) *h,
-                                             int type, X509_NAME *name)
+                                             X509_LOOKUP_TYPE type,
+                                             X509_NAME *name)
 {
     int idx;
     idx = X509_OBJECT_idx_by_subject(h, type, name);
@@ -501,6 +489,9 @@ STACK_OF(X509) *X509_STORE_CTX_get1_certs(X509_STORE_CTX *ctx, X509_NAME *nm)
     X509 *x;
     X509_OBJECT *obj;
 
+    if (ctx->ctx == NULL)
+        return NULL;
+
     CRYPTO_THREAD_write_lock(ctx->ctx->lock);
     idx = x509_object_idx_cnt(ctx->ctx->objs, X509_LU_X509, nm, &cnt);
     if (idx < 0) {
@@ -550,8 +541,10 @@ STACK_OF(X509_CRL) *X509_STORE_CTX_get1_crls(X509_STORE_CTX *ctx, X509_NAME *nm)
     X509_OBJECT *obj, *xobj = X509_OBJECT_new();
 
     /* Always do lookup to possibly add new CRLs to cache */
-    if (sk == NULL || xobj == NULL ||
-            !X509_STORE_CTX_get_by_subject(ctx, X509_LU_CRL, nm, xobj)) {
+    if (sk == NULL
+            || xobj == NULL
+            || ctx->ctx == NULL
+            || !X509_STORE_CTX_get_by_subject(ctx, X509_LU_CRL, nm, xobj)) {
         X509_OBJECT_free(xobj);
         sk_X509_CRL_free(sk);
         return NULL;
@@ -630,16 +623,8 @@ int X509_STORE_CTX_get1_issuer(X509 **issuer, X509_STORE_CTX *ctx, X509 *x)
     *issuer = NULL;
     xn = X509_get_issuer_name(x);
     ok = X509_STORE_CTX_get_by_subject(ctx, X509_LU_X509, xn, obj);
-    if (ok != X509_LU_X509) {
+    if (ok != 1) {
         X509_OBJECT_free(obj);
-        if (ok == X509_LU_RETRY) {
-            X509err(X509_F_X509_STORE_CTX_GET1_ISSUER, X509_R_SHOULD_RETRY);
-            return -1;
-        }
-        if (ok != X509_LU_FAIL) {
-            /* not good :-(, break anyway */
-            return -1;
-        }
         return 0;
     }
     /* If certificate matches all OK */
@@ -653,6 +638,9 @@ int X509_STORE_CTX_get1_issuer(X509 **issuer, X509_STORE_CTX *ctx, X509 *x)
     }
     X509_OBJECT_free(obj);
 
+    if (ctx->ctx == NULL)
+        return 0;
+
     /* Else find index of first cert accepted by 'check_issued' */
     ret = 0;
     CRYPTO_THREAD_write_lock(ctx->ctx->lock);