X-Git-Url: https://git.openssl.org/gitweb/?p=openssl.git;a=blobdiff_plain;f=crypto%2Fthreads_pthread.c;h=ecb53299c48459eba8a026ef9f3b18971c9d893d;hp=a2c1707865283282be5987713a2a85e094b92225;hb=HEAD;hpb=7de2b9c4afd90359e47d81a5fa70bcb8506fbf91 diff --git a/crypto/threads_pthread.c b/crypto/threads_pthread.c index a2c1707865..8e411671d9 100644 --- a/crypto/threads_pthread.c +++ b/crypto/threads_pthread.c @@ -1,30 +1,705 @@ /* - * Copyright 2016 The OpenSSL Project Authors. All Rights Reserved. + * Copyright 2016-2024 The OpenSSL Project Authors. All Rights Reserved. * - * Licensed under the OpenSSL license (the "License"). You may not use + * Licensed under the Apache License 2.0 (the "License"). You may not use * this file except in compliance with the License. You can obtain a copy * in the file LICENSE in the source distribution or at * https://www.openssl.org/source/license.html */ +/* We need to use the OPENSSL_fork_*() deprecated APIs */ +#define OPENSSL_SUPPRESS_DEPRECATED + #include +#include #include "internal/cryptlib.h" +#include "internal/rcu.h" +#include "rcu_internal.h" + +#if defined(__clang__) && defined(__has_feature) +# if __has_feature(thread_sanitizer) +# define __SANITIZE_THREAD__ +# endif +#endif + +#if defined(__SANITIZE_THREAD__) +# include +# define TSAN_FAKE_UNLOCK(x) __tsan_mutex_pre_unlock((x), 0); \ +__tsan_mutex_post_unlock((x), 0) + +# define TSAN_FAKE_LOCK(x) __tsan_mutex_pre_lock((x), 0); \ +__tsan_mutex_post_lock((x), 0, 0) +#else +# define TSAN_FAKE_UNLOCK(x) +# define TSAN_FAKE_LOCK(x) +#endif + +#if defined(__sun) +# include +#endif + +#if defined(__apple_build_version__) && __apple_build_version__ < 6000000 +/* + * OS/X 10.7 and 10.8 had a weird version of clang which has __ATOMIC_ACQUIRE and + * __ATOMIC_ACQ_REL but which expects only one parameter for __atomic_is_lock_free() + * rather than two which has signature __atomic_is_lock_free(sizeof(_Atomic(T))). + * All of this makes impossible to use __atomic_is_lock_free here. + * + * See: https://github.com/llvm/llvm-project/commit/a4c2602b714e6c6edb98164550a5ae829b2de760 + */ +# define BROKEN_CLANG_ATOMICS +#endif #if defined(OPENSSL_THREADS) && !defined(CRYPTO_TDEBUG) && !defined(OPENSSL_SYS_WINDOWS) +# if defined(OPENSSL_SYS_UNIX) +# include +# include +# endif + +# include + # ifdef PTHREAD_RWLOCK_INITIALIZER # define USE_RWLOCK # endif +/* + * For all GNU/clang atomic builtins, we also need fallbacks, to cover all + * other compilers. + + * Unfortunately, we can't do that with some "generic type", because there's no + * guarantee that the chosen generic type is large enough to cover all cases. + * Therefore, we implement fallbacks for each applicable type, with composed + * names that include the type they handle. + * + * (an anecdote: we previously tried to use |void *| as the generic type, with + * the thought that the pointer itself is the largest type. However, this is + * not true on 32-bit pointer platforms, as a |uint64_t| is twice as large) + * + * All applicable ATOMIC_ macros take the intended type as first parameter, so + * they can map to the correct fallback function. In the GNU/clang case, that + * parameter is simply ignored. + */ + +/* + * Internal types used with the ATOMIC_ macros, to make it possible to compose + * fallback function names. + */ +typedef void *pvoid; +typedef struct rcu_cb_item *prcu_cb_item; + +# if defined(__GNUC__) && defined(__ATOMIC_ACQUIRE) && !defined(BROKEN_CLANG_ATOMICS) \ + && !defined(USE_ATOMIC_FALLBACKS) +# if defined(__APPLE__) && defined(__clang__) && defined(__aarch64__) +/* + * For pointers, Apple M1 virtualized cpu seems to have some problem using the + * ldapr instruction (see https://github.com/openssl/openssl/pull/23974) + * When using the native apple clang compiler, this instruction is emitted for + * atomic loads, which is bad. So, if + * 1) We are building on a target that defines __APPLE__ AND + * 2) We are building on a target using clang (__clang__) AND + * 3) We are building for an M1 processor (__aarch64__) + * Then we should not use __atomic_load_n and instead implement our own + * function to issue the ldar instruction instead, which produces the proper + * sequencing guarantees + */ +static inline void *apple_atomic_load_n_pvoid(void **p, + ossl_unused int memorder) +{ + void *ret; + + __asm volatile("ldar %0, [%1]" : "=r" (ret): "r" (p):); + + return ret; +} + +/* For uint64_t, we should be fine, though */ +# define apple_atomic_load_n_uint64_t(p, o) __atomic_load_n(p, o) + +# define ATOMIC_LOAD_N(t, p, o) apple_atomic_load_n_##t(p, o) +# else +# define ATOMIC_LOAD_N(t, p, o) __atomic_load_n(p, o) +# endif +# define ATOMIC_STORE_N(t, p, v, o) __atomic_store_n(p, v, o) +# define ATOMIC_STORE(t, p, v, o) __atomic_store(p, v, o) +# define ATOMIC_EXCHANGE_N(t, p, v, o) __atomic_exchange_n(p, v, o) +# define ATOMIC_ADD_FETCH(p, v, o) __atomic_add_fetch(p, v, o) +# define ATOMIC_FETCH_ADD(p, v, o) __atomic_fetch_add(p, v, o) +# define ATOMIC_SUB_FETCH(p, v, o) __atomic_sub_fetch(p, v, o) +# define ATOMIC_AND_FETCH(p, m, o) __atomic_and_fetch(p, m, o) +# define ATOMIC_OR_FETCH(p, m, o) __atomic_or_fetch(p, m, o) +# else +static pthread_mutex_t atomic_sim_lock = PTHREAD_MUTEX_INITIALIZER; + +# define IMPL_fallback_atomic_load_n(t) \ + static inline t fallback_atomic_load_n_##t(t *p) \ + { \ + t ret; \ + \ + pthread_mutex_lock(&atomic_sim_lock); \ + ret = *p; \ + pthread_mutex_unlock(&atomic_sim_lock); \ + return ret; \ + } +IMPL_fallback_atomic_load_n(uint64_t) +IMPL_fallback_atomic_load_n(pvoid) + +# define ATOMIC_LOAD_N(t, p, o) fallback_atomic_load_n_##t(p) + +# define IMPL_fallback_atomic_store_n(t) \ + static inline t fallback_atomic_store_n_##t(t *p, t v) \ + { \ + t ret; \ + \ + pthread_mutex_lock(&atomic_sim_lock); \ + ret = *p; \ + *p = v; \ + pthread_mutex_unlock(&atomic_sim_lock); \ + return ret; \ + } +IMPL_fallback_atomic_store_n(uint64_t) + +# define ATOMIC_STORE_N(t, p, v, o) fallback_atomic_store_n_##t(p, v) + +# define IMPL_fallback_atomic_store(t) \ + static inline void fallback_atomic_store_##t(t *p, t *v) \ + { \ + pthread_mutex_lock(&atomic_sim_lock); \ + *p = *v; \ + pthread_mutex_unlock(&atomic_sim_lock); \ + } +IMPL_fallback_atomic_store(uint64_t) +IMPL_fallback_atomic_store(pvoid) + +# define ATOMIC_STORE(t, p, v, o) fallback_atomic_store_##t(p, v) + +# define IMPL_fallback_atomic_exchange_n(t) \ + static inline t fallback_atomic_exchange_n_##t(t *p, t v) \ + { \ + t ret; \ + \ + pthread_mutex_lock(&atomic_sim_lock); \ + ret = *p; \ + *p = v; \ + pthread_mutex_unlock(&atomic_sim_lock); \ + return ret; \ + } +IMPL_fallback_atomic_exchange_n(uint64_t) +IMPL_fallback_atomic_exchange_n(prcu_cb_item) + +# define ATOMIC_EXCHANGE_N(t, p, v, o) fallback_atomic_exchange_n_##t(p, v) + +/* + * The fallbacks that follow don't need any per type implementation, as + * they are designed for uint64_t only. If there comes a time when multiple + * types need to be covered, it's relatively easy to refactor them the same + * way as the fallbacks above. + */ + +static inline uint64_t fallback_atomic_add_fetch(uint64_t *p, uint64_t v) +{ + uint64_t ret; + + pthread_mutex_lock(&atomic_sim_lock); + *p += v; + ret = *p; + pthread_mutex_unlock(&atomic_sim_lock); + return ret; +} + +# define ATOMIC_ADD_FETCH(p, v, o) fallback_atomic_add_fetch(p, v) + +static inline uint64_t fallback_atomic_fetch_add(uint64_t *p, uint64_t v) +{ + uint64_t ret; + + pthread_mutex_lock(&atomic_sim_lock); + ret = *p; + *p += v; + pthread_mutex_unlock(&atomic_sim_lock); + return ret; +} + +# define ATOMIC_FETCH_ADD(p, v, o) fallback_atomic_fetch_add(p, v) + +static inline uint64_t fallback_atomic_sub_fetch(uint64_t *p, uint64_t v) +{ + uint64_t ret; + + pthread_mutex_lock(&atomic_sim_lock); + *p -= v; + ret = *p; + pthread_mutex_unlock(&atomic_sim_lock); + return ret; +} + +# define ATOMIC_SUB_FETCH(p, v, o) fallback_atomic_sub_fetch(p, v) + +static inline uint64_t fallback_atomic_and_fetch(uint64_t *p, uint64_t m) +{ + uint64_t ret; + + pthread_mutex_lock(&atomic_sim_lock); + *p &= m; + ret = *p; + pthread_mutex_unlock(&atomic_sim_lock); + return ret; +} + +# define ATOMIC_AND_FETCH(p, v, o) fallback_atomic_and_fetch(p, v) + +static inline uint64_t fallback_atomic_or_fetch(uint64_t *p, uint64_t m) +{ + uint64_t ret; + + pthread_mutex_lock(&atomic_sim_lock); + *p |= m; + ret = *p; + pthread_mutex_unlock(&atomic_sim_lock); + return ret; +} + +# define ATOMIC_OR_FETCH(p, v, o) fallback_atomic_or_fetch(p, v) +# endif + +/* + * users is broken up into 2 parts + * bits 0-15 current readers + * bit 32-63 - ID + */ +# define READER_SHIFT 0 +# define ID_SHIFT 32 +# define READER_SIZE 16 +# define ID_SIZE 32 + +# define READER_MASK (((uint64_t)1 << READER_SIZE) - 1) +# define ID_MASK (((uint64_t)1 << ID_SIZE) - 1) +# define READER_COUNT(x) (((uint64_t)(x) >> READER_SHIFT) & READER_MASK) +# define ID_VAL(x) (((uint64_t)(x) >> ID_SHIFT) & ID_MASK) +# define VAL_READER ((uint64_t)1 << READER_SHIFT) +# define VAL_ID(x) ((uint64_t)x << ID_SHIFT) + +/* + * This is the core of an rcu lock. It tracks the readers and writers for the + * current quiescence point for a given lock. Users is the 64 bit value that + * stores the READERS/ID as defined above + * + */ +struct rcu_qp { + uint64_t users; +}; + +struct thread_qp { + struct rcu_qp *qp; + unsigned int depth; + CRYPTO_RCU_LOCK *lock; +}; + +# define MAX_QPS 10 +/* + * This is the per thread tracking data + * that is assigned to each thread participating + * in an rcu qp + * + * qp points to the qp that it last acquired + * + */ +struct rcu_thr_data { + struct thread_qp thread_qps[MAX_QPS]; +}; + +/* + * This is the internal version of a CRYPTO_RCU_LOCK + * it is cast from CRYPTO_RCU_LOCK + */ +struct rcu_lock_st { + /* Callbacks to call for next ossl_synchronize_rcu */ + struct rcu_cb_item *cb_items; + + /* The context we are being created against */ + OSSL_LIB_CTX *ctx; + + /* rcu generation counter for in-order retirement */ + uint32_t id_ctr; + + /* Array of quiescent points for synchronization */ + struct rcu_qp *qp_group; + + /* Number of elements in qp_group array */ + size_t group_count; + + /* Index of the current qp in the qp_group array */ + uint64_t reader_idx; + + /* value of the next id_ctr value to be retired */ + uint32_t next_to_retire; + + /* index of the next free rcu_qp in the qp_group */ + uint64_t current_alloc_idx; + + /* number of qp's in qp_group array currently being retired */ + uint32_t writers_alloced; + + /* lock protecting write side operations */ + pthread_mutex_t write_lock; + + /* lock protecting updates to writers_alloced/current_alloc_idx */ + pthread_mutex_t alloc_lock; + + /* signal to wake threads waiting on alloc_lock */ + pthread_cond_t alloc_signal; + + /* lock to enforce in-order retirement */ + pthread_mutex_t prior_lock; + + /* signal to wake threads waiting on prior_lock */ + pthread_cond_t prior_signal; +}; + +/* Read side acquisition of the current qp */ +static struct rcu_qp *get_hold_current_qp(struct rcu_lock_st *lock) +{ + uint64_t qp_idx; + + /* get the current qp index */ + for (;;) { + /* + * Notes on use of __ATOMIC_ACQUIRE + * We need to ensure the following: + * 1) That subsequent operations aren't optimized by hoisting them above + * this operation. Specifically, we don't want the below re-load of + * qp_idx to get optimized away + * 2) We want to ensure that any updating of reader_idx on the write side + * of the lock is flushed from a local cpu cache so that we see any + * updates prior to the load. This is a non-issue on cache coherent + * systems like x86, but is relevant on other arches + * Note: This applies to the reload below as well + */ + qp_idx = ATOMIC_LOAD_N(uint64_t, &lock->reader_idx, __ATOMIC_ACQUIRE); + + /* + * Notes of use of __ATOMIC_RELEASE + * This counter is only read by the write side of the lock, and so we + * specify __ATOMIC_RELEASE here to ensure that the write side of the + * lock see this during the spin loop read of users, as it waits for the + * reader count to approach zero + */ + ATOMIC_ADD_FETCH(&lock->qp_group[qp_idx].users, VAL_READER, + __ATOMIC_RELEASE); + + /* if the idx hasn't changed, we're good, else try again */ + if (qp_idx == ATOMIC_LOAD_N(uint64_t, &lock->reader_idx, __ATOMIC_ACQUIRE)) + break; + + /* + * Notes on use of __ATOMIC_RELEASE + * As with the add above, we want to ensure that this decrement is + * seen by the write side of the lock as soon as it happens to prevent + * undue spinning waiting for write side completion + */ + ATOMIC_SUB_FETCH(&lock->qp_group[qp_idx].users, VAL_READER, + __ATOMIC_RELEASE); + } + + return &lock->qp_group[qp_idx]; +} + +static void ossl_rcu_free_local_data(void *arg) +{ + OSSL_LIB_CTX *ctx = arg; + CRYPTO_THREAD_LOCAL *lkey = ossl_lib_ctx_get_rcukey(ctx); + struct rcu_thr_data *data = CRYPTO_THREAD_get_local(lkey); + OPENSSL_free(data); +} + +void ossl_rcu_read_lock(CRYPTO_RCU_LOCK *lock) +{ + struct rcu_thr_data *data; + int i, available_qp = -1; + CRYPTO_THREAD_LOCAL *lkey = ossl_lib_ctx_get_rcukey(lock->ctx); + + /* + * we're going to access current_qp here so ask the + * processor to fetch it + */ + data = CRYPTO_THREAD_get_local(lkey); + + if (data == NULL) { + data = OPENSSL_zalloc(sizeof(*data)); + OPENSSL_assert(data != NULL); + CRYPTO_THREAD_set_local(lkey, data); + ossl_init_thread_start(NULL, lock->ctx, ossl_rcu_free_local_data); + } + + for (i = 0; i < MAX_QPS; i++) { + if (data->thread_qps[i].qp == NULL && available_qp == -1) + available_qp = i; + /* If we have a hold on this lock already, we're good */ + if (data->thread_qps[i].lock == lock) { + data->thread_qps[i].depth++; + return; + } + } + + /* + * if we get here, then we don't have a hold on this lock yet + */ + assert(available_qp != -1); + + data->thread_qps[available_qp].qp = get_hold_current_qp(lock); + data->thread_qps[available_qp].depth = 1; + data->thread_qps[available_qp].lock = lock; +} + +void ossl_rcu_read_unlock(CRYPTO_RCU_LOCK *lock) +{ + int i; + CRYPTO_THREAD_LOCAL *lkey = ossl_lib_ctx_get_rcukey(lock->ctx); + struct rcu_thr_data *data = CRYPTO_THREAD_get_local(lkey); + uint64_t ret; + + assert(data != NULL); + + for (i = 0; i < MAX_QPS; i++) { + if (data->thread_qps[i].lock == lock) { + /* + * As with read side acquisition, we use __ATOMIC_RELEASE here + * to ensure that the decrement is published immediately + * to any write side waiters + */ + data->thread_qps[i].depth--; + if (data->thread_qps[i].depth == 0) { + ret = ATOMIC_SUB_FETCH(&data->thread_qps[i].qp->users, VAL_READER, + __ATOMIC_RELEASE); + OPENSSL_assert(ret != UINT64_MAX); + data->thread_qps[i].qp = NULL; + data->thread_qps[i].lock = NULL; + } + return; + } + } + /* + * If we get here, we're trying to unlock a lock that we never acquired - + * that's fatal. + */ + assert(0); +} + +/* + * Write side allocation routine to get the current qp + * and replace it with a new one + */ +static struct rcu_qp *update_qp(CRYPTO_RCU_LOCK *lock) +{ + uint64_t new_id; + uint64_t current_idx; + + pthread_mutex_lock(&lock->alloc_lock); + + /* + * we need at least one qp to be available with one + * left over, so that readers can start working on + * one that isn't yet being waited on + */ + while (lock->group_count - lock->writers_alloced < 2) + /* we have to wait for one to be free */ + pthread_cond_wait(&lock->alloc_signal, &lock->alloc_lock); + + current_idx = lock->current_alloc_idx; + + /* Allocate the qp */ + lock->writers_alloced++; + + /* increment the allocation index */ + lock->current_alloc_idx = + (lock->current_alloc_idx + 1) % lock->group_count; + + /* get and insert a new id */ + new_id = lock->id_ctr; + lock->id_ctr++; + + new_id = VAL_ID(new_id); + /* + * Even though we are under a write side lock here + * We need to use atomic instructions to ensure that the results + * of this update are published to the read side prior to updating the + * reader idx below + */ + ATOMIC_AND_FETCH(&lock->qp_group[current_idx].users, ID_MASK, + __ATOMIC_RELEASE); + ATOMIC_OR_FETCH(&lock->qp_group[current_idx].users, new_id, + __ATOMIC_RELEASE); + + /* + * Update the reader index to be the prior qp. + * Note the use of __ATOMIC_RELEASE here is based on the corresponding use + * of __ATOMIC_ACQUIRE in get_hold_current_qp, as we want any publication + * of this value to be seen on the read side immediately after it happens + */ + ATOMIC_STORE_N(uint64_t, &lock->reader_idx, lock->current_alloc_idx, + __ATOMIC_RELEASE); + + /* wake up any waiters */ + pthread_cond_signal(&lock->alloc_signal); + pthread_mutex_unlock(&lock->alloc_lock); + return &lock->qp_group[current_idx]; +} + +static void retire_qp(CRYPTO_RCU_LOCK *lock, struct rcu_qp *qp) +{ + pthread_mutex_lock(&lock->alloc_lock); + lock->writers_alloced--; + pthread_cond_signal(&lock->alloc_signal); + pthread_mutex_unlock(&lock->alloc_lock); +} + +static struct rcu_qp *allocate_new_qp_group(CRYPTO_RCU_LOCK *lock, + int count) +{ + struct rcu_qp *new = + OPENSSL_zalloc(sizeof(*new) * count); + + lock->group_count = count; + return new; +} + +void ossl_rcu_write_lock(CRYPTO_RCU_LOCK *lock) +{ + pthread_mutex_lock(&lock->write_lock); + TSAN_FAKE_UNLOCK(&lock->write_lock); +} + +void ossl_rcu_write_unlock(CRYPTO_RCU_LOCK *lock) +{ + TSAN_FAKE_LOCK(&lock->write_lock); + pthread_mutex_unlock(&lock->write_lock); +} + +void ossl_synchronize_rcu(CRYPTO_RCU_LOCK *lock) +{ + struct rcu_qp *qp; + uint64_t count; + struct rcu_cb_item *cb_items, *tmpcb; + + pthread_mutex_lock(&lock->write_lock); + cb_items = lock->cb_items; + lock->cb_items = NULL; + pthread_mutex_unlock(&lock->write_lock); + + qp = update_qp(lock); + + /* + * wait for the reader count to reach zero + * Note the use of __ATOMIC_ACQUIRE here to ensure that any + * prior __ATOMIC_RELEASE write operation in get_hold_current_qp + * is visible prior to our read + */ + do { + count = ATOMIC_LOAD_N(uint64_t, &qp->users, __ATOMIC_ACQUIRE); + } while (READER_COUNT(count) != 0); + + /* retire in order */ + pthread_mutex_lock(&lock->prior_lock); + while (lock->next_to_retire != ID_VAL(count)) + pthread_cond_wait(&lock->prior_signal, &lock->prior_lock); + lock->next_to_retire++; + pthread_cond_broadcast(&lock->prior_signal); + pthread_mutex_unlock(&lock->prior_lock); + + retire_qp(lock, qp); + + /* handle any callbacks that we have */ + while (cb_items != NULL) { + tmpcb = cb_items; + cb_items = cb_items->next; + tmpcb->fn(tmpcb->data); + OPENSSL_free(tmpcb); + } +} + +int ossl_rcu_call(CRYPTO_RCU_LOCK *lock, rcu_cb_fn cb, void *data) +{ + struct rcu_cb_item *new = + OPENSSL_zalloc(sizeof(*new)); + + if (new == NULL) + return 0; + + new->data = data; + new->fn = cb; + /* + * Use __ATOMIC_ACQ_REL here to indicate that any prior writes to this + * list are visible to us prior to reading, and publish the new value + * immediately + */ + new->next = ATOMIC_EXCHANGE_N(prcu_cb_item, &lock->cb_items, new, + __ATOMIC_ACQ_REL); + + return 1; +} + +void *ossl_rcu_uptr_deref(void **p) +{ + return ATOMIC_LOAD_N(pvoid, p, __ATOMIC_ACQUIRE); +} + +void ossl_rcu_assign_uptr(void **p, void **v) +{ + ATOMIC_STORE(pvoid, p, v, __ATOMIC_RELEASE); +} + +CRYPTO_RCU_LOCK *ossl_rcu_lock_new(int num_writers, OSSL_LIB_CTX *ctx) +{ + struct rcu_lock_st *new; + + if (num_writers < 1) + num_writers = 1; + + ctx = ossl_lib_ctx_get_concrete(ctx); + if (ctx == NULL) + return 0; + + new = OPENSSL_zalloc(sizeof(*new)); + if (new == NULL) + return NULL; + + new->ctx = ctx; + pthread_mutex_init(&new->write_lock, NULL); + pthread_mutex_init(&new->prior_lock, NULL); + pthread_mutex_init(&new->alloc_lock, NULL); + pthread_cond_init(&new->prior_signal, NULL); + pthread_cond_init(&new->alloc_signal, NULL); + new->qp_group = allocate_new_qp_group(new, num_writers + 1); + if (new->qp_group == NULL) { + OPENSSL_free(new); + new = NULL; + } + return new; +} + +void ossl_rcu_lock_free(CRYPTO_RCU_LOCK *lock) +{ + struct rcu_lock_st *rlock = (struct rcu_lock_st *)lock; + + if (lock == NULL) + return; + + /* make sure we're synchronized */ + ossl_synchronize_rcu(rlock); + + OPENSSL_free(rlock->qp_group); + /* There should only be a single qp left now */ + OPENSSL_free(rlock); +} + CRYPTO_RWLOCK *CRYPTO_THREAD_lock_new(void) { # ifdef USE_RWLOCK CRYPTO_RWLOCK *lock; - if ((lock = OPENSSL_zalloc(sizeof(pthread_rwlock_t))) == NULL) { + if ((lock = OPENSSL_zalloc(sizeof(pthread_rwlock_t))) == NULL) /* Don't set error, to avoid recursion blowup. */ return NULL; - } if (pthread_rwlock_init(lock, NULL) != 0) { OPENSSL_free(lock); @@ -34,13 +709,21 @@ CRYPTO_RWLOCK *CRYPTO_THREAD_lock_new(void) pthread_mutexattr_t attr; CRYPTO_RWLOCK *lock; - if ((lock = OPENSSL_zalloc(sizeof(pthread_mutex_t))) == NULL) { + if ((lock = OPENSSL_zalloc(sizeof(pthread_mutex_t))) == NULL) /* Don't set error, to avoid recursion blowup. */ return NULL; - } + /* + * We don't use recursive mutexes, but try to catch errors if we do. + */ pthread_mutexattr_init(&attr); - pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); +# if !defined (__TANDEM) && !defined (_SPT_MODEL_) +# if !defined(NDEBUG) && !defined(OPENSSL_NO_MUTEX_ERRORCHECK) + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK); +# endif +# else + /* The SPT Thread Library does not define MUTEX attributes. */ +# endif if (pthread_mutex_init(lock, &attr) != 0) { pthread_mutexattr_destroy(&attr); @@ -54,27 +737,31 @@ CRYPTO_RWLOCK *CRYPTO_THREAD_lock_new(void) return lock; } -int CRYPTO_THREAD_read_lock(CRYPTO_RWLOCK *lock) +__owur int CRYPTO_THREAD_read_lock(CRYPTO_RWLOCK *lock) { # ifdef USE_RWLOCK if (pthread_rwlock_rdlock(lock) != 0) return 0; # else - if (pthread_mutex_lock(lock) != 0) + if (pthread_mutex_lock(lock) != 0) { + assert(errno != EDEADLK && errno != EBUSY); return 0; + } # endif return 1; } -int CRYPTO_THREAD_write_lock(CRYPTO_RWLOCK *lock) +__owur int CRYPTO_THREAD_write_lock(CRYPTO_RWLOCK *lock) { # ifdef USE_RWLOCK if (pthread_rwlock_wrlock(lock) != 0) return 0; # else - if (pthread_mutex_lock(lock) != 0) + if (pthread_mutex_lock(lock) != 0) { + assert(errno != EDEADLK && errno != EBUSY); return 0; + } # endif return 1; @@ -86,8 +773,10 @@ int CRYPTO_THREAD_unlock(CRYPTO_RWLOCK *lock) if (pthread_rwlock_unlock(lock) != 0) return 0; # else - if (pthread_mutex_unlock(lock) != 0) + if (pthread_mutex_unlock(lock) != 0) { + assert(errno != EPERM); return 0; + } # endif return 1; @@ -157,13 +846,19 @@ int CRYPTO_THREAD_compare_id(CRYPTO_THREAD_ID a, CRYPTO_THREAD_ID b) int CRYPTO_atomic_add(int *val, int amount, int *ret, CRYPTO_RWLOCK *lock) { -# if defined(__GNUC__) && defined(__ATOMIC_ACQ_REL) +# if defined(__GNUC__) && defined(__ATOMIC_ACQ_REL) && !defined(BROKEN_CLANG_ATOMICS) if (__atomic_is_lock_free(sizeof(*val), val)) { *ret = __atomic_add_fetch(val, amount, __ATOMIC_ACQ_REL); return 1; } +# elif defined(__sun) && (defined(__SunOS_5_10) || defined(__SunOS_5_11)) + /* This will work for all future Solaris versions. */ + if (ret != NULL) { + *ret = atomic_add_int_nv((volatile unsigned int *)val, amount); + return 1; + } # endif - if (!CRYPTO_THREAD_write_lock(lock)) + if (lock == NULL || !CRYPTO_THREAD_write_lock(lock)) return 0; *val += amount; @@ -175,17 +870,24 @@ int CRYPTO_atomic_add(int *val, int amount, int *ret, CRYPTO_RWLOCK *lock) return 1; } -int CRYPTO_atomic_read(int *val, int *ret, CRYPTO_RWLOCK *lock) +int CRYPTO_atomic_or(uint64_t *val, uint64_t op, uint64_t *ret, + CRYPTO_RWLOCK *lock) { -# if defined(__GNUC__) && defined(__ATOMIC_ACQUIRE) +# if defined(__GNUC__) && defined(__ATOMIC_ACQ_REL) && !defined(BROKEN_CLANG_ATOMICS) if (__atomic_is_lock_free(sizeof(*val), val)) { - __atomic_load(val, ret, __ATOMIC_ACQUIRE); + *ret = __atomic_or_fetch(val, op, __ATOMIC_ACQ_REL); + return 1; + } +# elif defined(__sun) && (defined(__SunOS_5_10) || defined(__SunOS_5_11)) + /* This will work for all future Solaris versions. */ + if (ret != NULL) { + *ret = atomic_or_64_nv(val, op); return 1; } # endif - if (!CRYPTO_THREAD_read_lock(lock)) + if (lock == NULL || !CRYPTO_THREAD_write_lock(lock)) return 0; - + *val |= op; *ret = *val; if (!CRYPTO_THREAD_unlock(lock)) @@ -194,41 +896,84 @@ int CRYPTO_atomic_read(int *val, int *ret, CRYPTO_RWLOCK *lock) return 1; } -int CRYPTO_atomic_write(int *val, int n, CRYPTO_RWLOCK *lock) +int CRYPTO_atomic_load(uint64_t *val, uint64_t *ret, CRYPTO_RWLOCK *lock) { -# if defined(__GNUC__) && defined(__ATOMIC_RELEASE) +# if defined(__GNUC__) && defined(__ATOMIC_ACQUIRE) && !defined(BROKEN_CLANG_ATOMICS) if (__atomic_is_lock_free(sizeof(*val), val)) { - __atomic_store(val, &n, __ATOMIC_RELEASE); + __atomic_load(val, ret, __ATOMIC_ACQUIRE); + return 1; + } +# elif defined(__sun) && (defined(__SunOS_5_10) || defined(__SunOS_5_11)) + /* This will work for all future Solaris versions. */ + if (ret != NULL) { + *ret = atomic_or_64_nv(val, 0); return 1; } # endif - if (!CRYPTO_THREAD_write_lock(lock)) + if (lock == NULL || !CRYPTO_THREAD_read_lock(lock)) + return 0; + *ret = *val; + if (!CRYPTO_THREAD_unlock(lock)) return 0; - *val = n; + return 1; +} +int CRYPTO_atomic_store(uint64_t *dst, uint64_t val, CRYPTO_RWLOCK *lock) +{ +# if defined(__GNUC__) && defined(__ATOMIC_ACQUIRE) && !defined(BROKEN_CLANG_ATOMICS) + if (__atomic_is_lock_free(sizeof(*dst), dst)) { + __atomic_store(dst, &val, __ATOMIC_RELEASE); + return 1; + } +# elif defined(__sun) && (defined(__SunOS_5_10) || defined(__SunOS_5_11)) + /* This will work for all future Solaris versions. */ + if (ret != NULL) { + atomic_swap_64(dst, val); + return 1; + } +# endif + if (lock == NULL || !CRYPTO_THREAD_read_lock(lock)) + return 0; + *dst = val; if (!CRYPTO_THREAD_unlock(lock)) return 0; return 1; } -# ifdef OPENSSL_SYS_UNIX -static pthread_once_t fork_once_control = PTHREAD_ONCE_INIT; - -static void fork_once_func(void) +int CRYPTO_atomic_load_int(int *val, int *ret, CRYPTO_RWLOCK *lock) { - pthread_atfork(OPENSSL_fork_prepare, - OPENSSL_fork_parent, OPENSSL_fork_child); -} +# if defined(__GNUC__) && defined(__ATOMIC_ACQUIRE) && !defined(BROKEN_CLANG_ATOMICS) + if (__atomic_is_lock_free(sizeof(*val), val)) { + __atomic_load(val, ret, __ATOMIC_ACQUIRE); + return 1; + } +# elif defined(__sun) && (defined(__SunOS_5_10) || defined(__SunOS_5_11)) + /* This will work for all future Solaris versions. */ + if (ret != NULL) { + *ret = (int *)atomic_or_uint_nv((unsigned int *)val, 0); + return 1; + } # endif + if (lock == NULL || !CRYPTO_THREAD_read_lock(lock)) + return 0; + *ret = *val; + if (!CRYPTO_THREAD_unlock(lock)) + return 0; + + return 1; +} +# ifndef FIPS_MODULE int openssl_init_fork_handlers(void) { -# ifdef OPENSSL_SYS_UNIX - if (pthread_once(&fork_once_control, fork_once_func) == 0) - return 1; -# endif - return 0; + return 1; +} +# endif /* FIPS_MODULE */ + +int openssl_get_fork_id(void) +{ + return getpid(); } #endif