#include "internal/rcu.h"
#include "rcu_internal.h"
+#if defined(__clang__) && defined(__has_feature)
+# if __has_feature(thread_sanitizer)
+# define __SANITIZE_THREAD__
+# endif
+#endif
+
+#if defined(__SANITIZE_THREAD__)
+# include <sanitizer/tsan_interface.h>
+# define TSAN_FAKE_UNLOCK(x) __tsan_mutex_pre_unlock((x), 0); \
+__tsan_mutex_post_unlock((x), 0)
+
+# define TSAN_FAKE_LOCK(x) __tsan_mutex_pre_lock((x), 0); \
+__tsan_mutex_post_lock((x), 0, 0)
+#else
+# define TSAN_FAKE_UNLOCK(x)
+# define TSAN_FAKE_LOCK(x)
+#endif
+
#if defined(__sun)
# include <atomic.h>
#endif
* 1) We are building on a target that defines __APPLE__ AND
* 2) We are building on a target using clang (__clang__) AND
* 3) We are building for an M1 processor (__aarch64__)
- * Then we shold not use __atomic_load_n and instead implement our own
- * function to issue the ldar instruction instead, which procuces the proper
+ * Then we should not use __atomic_load_n and instead implement our own
+ * function to issue the ldar instruction instead, which produces the proper
* sequencing guarantees
*/
static inline void *apple_atomic_load_n_pvoid(void **p,
# define ATOMIC_OR_FETCH(p, v, o) fallback_atomic_or_fetch(p, v)
# endif
-static CRYPTO_THREAD_LOCAL rcu_thr_key;
-
/*
* users is broken up into 2 parts
* bits 0-15 current readers
/* Callbacks to call for next ossl_synchronize_rcu */
struct rcu_cb_item *cb_items;
+ /* The context we are being created against */
+ OSSL_LIB_CTX *ctx;
+
/* rcu generation counter for in-order retirement */
uint32_t id_ctr;
pthread_cond_t prior_signal;
};
-/*
- * Called on thread exit to free the pthread key
- * associated with this thread, if any
- */
-static void free_rcu_thr_data(void *ptr)
-{
- struct rcu_thr_data *data =
- (struct rcu_thr_data *)CRYPTO_THREAD_get_local(&rcu_thr_key);
-
- OPENSSL_free(data);
- CRYPTO_THREAD_set_local(&rcu_thr_key, NULL);
-}
-
-static void ossl_rcu_init(void)
-{
- CRYPTO_THREAD_init_local(&rcu_thr_key, NULL);
-}
-
/* Read side acquisition of the current qp */
static struct rcu_qp *get_hold_current_qp(struct rcu_lock_st *lock)
{
return &lock->qp_group[qp_idx];
}
+static void ossl_rcu_free_local_data(void *arg)
+{
+ OSSL_LIB_CTX *ctx = arg;
+ CRYPTO_THREAD_LOCAL *lkey = ossl_lib_ctx_get_rcukey(ctx);
+ struct rcu_thr_data *data = CRYPTO_THREAD_get_local(lkey);
+ OPENSSL_free(data);
+}
+
void ossl_rcu_read_lock(CRYPTO_RCU_LOCK *lock)
{
struct rcu_thr_data *data;
int i, available_qp = -1;
+ CRYPTO_THREAD_LOCAL *lkey = ossl_lib_ctx_get_rcukey(lock->ctx);
/*
* we're going to access current_qp here so ask the
* processor to fetch it
*/
- data = CRYPTO_THREAD_get_local(&rcu_thr_key);
+ data = CRYPTO_THREAD_get_local(lkey);
if (data == NULL) {
data = OPENSSL_zalloc(sizeof(*data));
OPENSSL_assert(data != NULL);
- CRYPTO_THREAD_set_local(&rcu_thr_key, data);
- ossl_init_thread_start(NULL, NULL, free_rcu_thr_data);
+ CRYPTO_THREAD_set_local(lkey, data);
+ ossl_init_thread_start(NULL, lock->ctx, ossl_rcu_free_local_data);
}
for (i = 0; i < MAX_QPS; i++) {
void ossl_rcu_read_unlock(CRYPTO_RCU_LOCK *lock)
{
int i;
- struct rcu_thr_data *data = CRYPTO_THREAD_get_local(&rcu_thr_key);
+ CRYPTO_THREAD_LOCAL *lkey = ossl_lib_ctx_get_rcukey(lock->ctx);
+ struct rcu_thr_data *data = CRYPTO_THREAD_get_local(lkey);
uint64_t ret;
assert(data != NULL);
void ossl_rcu_write_lock(CRYPTO_RCU_LOCK *lock)
{
pthread_mutex_lock(&lock->write_lock);
+ TSAN_FAKE_UNLOCK(&lock->write_lock);
}
void ossl_rcu_write_unlock(CRYPTO_RCU_LOCK *lock)
{
+ TSAN_FAKE_LOCK(&lock->write_lock);
pthread_mutex_unlock(&lock->write_lock);
}
uint64_t count;
struct rcu_cb_item *cb_items, *tmpcb;
- /*
- * __ATOMIC_ACQ_REL is used here to ensure that we get any prior published
- * writes before we read, and publish our write immediately
- */
- cb_items = ATOMIC_EXCHANGE_N(prcu_cb_item, &lock->cb_items, NULL,
- __ATOMIC_ACQ_REL);
+ pthread_mutex_lock(&lock->write_lock);
+ cb_items = lock->cb_items;
+ lock->cb_items = NULL;
+ pthread_mutex_unlock(&lock->write_lock);
qp = update_qp(lock);
ATOMIC_STORE(pvoid, p, v, __ATOMIC_RELEASE);
}
-static CRYPTO_ONCE rcu_init_once = CRYPTO_ONCE_STATIC_INIT;
-
-CRYPTO_RCU_LOCK *ossl_rcu_lock_new(int num_writers)
+CRYPTO_RCU_LOCK *ossl_rcu_lock_new(int num_writers, OSSL_LIB_CTX *ctx)
{
struct rcu_lock_st *new;
- if (!CRYPTO_THREAD_run_once(&rcu_init_once, ossl_rcu_init))
- return NULL;
-
if (num_writers < 1)
num_writers = 1;
+ ctx = ossl_lib_ctx_get_concrete(ctx);
+ if (ctx == NULL)
+ return 0;
+
new = OPENSSL_zalloc(sizeof(*new));
if (new == NULL)
return NULL;
+ new->ctx = ctx;
pthread_mutex_init(&new->write_lock, NULL);
pthread_mutex_init(&new->prior_lock, NULL);
pthread_mutex_init(&new->alloc_lock, NULL);
return 1;
}
+int CRYPTO_atomic_store(uint64_t *dst, uint64_t val, CRYPTO_RWLOCK *lock)
+{
+# if defined(__GNUC__) && defined(__ATOMIC_ACQUIRE) && !defined(BROKEN_CLANG_ATOMICS)
+ if (__atomic_is_lock_free(sizeof(*dst), dst)) {
+ __atomic_store(dst, &val, __ATOMIC_RELEASE);
+ return 1;
+ }
+# elif defined(__sun) && (defined(__SunOS_5_10) || defined(__SunOS_5_11))
+ /* This will work for all future Solaris versions. */
+ if (ret != NULL) {
+ atomic_swap_64(dst, val);
+ return 1;
+ }
+# endif
+ if (lock == NULL || !CRYPTO_THREAD_read_lock(lock))
+ return 0;
+ *dst = val;
+ if (!CRYPTO_THREAD_unlock(lock))
+ return 0;
+
+ return 1;
+}
+
int CRYPTO_atomic_load_int(int *val, int *ret, CRYPTO_RWLOCK *lock)
{
# if defined(__GNUC__) && defined(__ATOMIC_ACQUIRE) && !defined(BROKEN_CLANG_ATOMICS)
projects / openssl.git / blobdiff