Lines Matching refs:lock
304 CRYPTO_RCU_LOCK *lock; member
370 static struct rcu_qp *get_hold_current_qp(struct rcu_lock_st *lock) in get_hold_current_qp() argument
388 qp_idx = ATOMIC_LOAD_N(uint32_t, &lock->reader_idx, __ATOMIC_ACQUIRE); in get_hold_current_qp()
397 ATOMIC_ADD_FETCH(&lock->qp_group[qp_idx].users, VAL_READER, in get_hold_current_qp()
401 if (qp_idx == ATOMIC_LOAD_N(uint32_t, &lock->reader_idx, in get_hold_current_qp()
411 ATOMIC_SUB_FETCH(&lock->qp_group[qp_idx].users, VAL_READER, in get_hold_current_qp()
415 return &lock->qp_group[qp_idx]; in get_hold_current_qp()
428 void ossl_rcu_read_lock(CRYPTO_RCU_LOCK *lock) in ossl_rcu_read_lock() argument
432 CRYPTO_THREAD_LOCAL *lkey = ossl_lib_ctx_get_rcukey(lock->ctx); in ossl_rcu_read_lock()
444 ossl_init_thread_start(NULL, lock->ctx, ossl_rcu_free_local_data); in ossl_rcu_read_lock()
451 if (data->thread_qps[i].lock == lock) { in ossl_rcu_read_lock()
462 data->thread_qps[available_qp].qp = get_hold_current_qp(lock); in ossl_rcu_read_lock()
464 data->thread_qps[available_qp].lock = lock; in ossl_rcu_read_lock()
467 void ossl_rcu_read_unlock(CRYPTO_RCU_LOCK *lock) in ossl_rcu_read_unlock() argument
470 CRYPTO_THREAD_LOCAL *lkey = ossl_lib_ctx_get_rcukey(lock->ctx); in ossl_rcu_read_unlock()
477 if (data->thread_qps[i].lock == lock) { in ossl_rcu_read_unlock()
489 data->thread_qps[i].lock = NULL; in ossl_rcu_read_unlock()
505 static struct rcu_qp *update_qp(CRYPTO_RCU_LOCK *lock) in update_qp() argument
510 pthread_mutex_lock(&lock->alloc_lock); in update_qp()
517 while (lock->group_count - lock->writers_alloced < 2) in update_qp()
519 pthread_cond_wait(&lock->alloc_signal, &lock->alloc_lock); in update_qp()
521 current_idx = lock->current_alloc_idx; in update_qp()
524 lock->writers_alloced++; in update_qp()
527 lock->current_alloc_idx = in update_qp()
528 (lock->current_alloc_idx + 1) % lock->group_count; in update_qp()
531 new_id = VAL_ID(lock->id_ctr); in update_qp()
532 lock->id_ctr++; in update_qp()
540 ATOMIC_AND_FETCH(&lock->qp_group[current_idx].users, ID_MASK, in update_qp()
542 ATOMIC_OR_FETCH(&lock->qp_group[current_idx].users, new_id, in update_qp()
551 ATOMIC_STORE_N(uint32_t, &lock->reader_idx, lock->current_alloc_idx, in update_qp()
555 pthread_cond_signal(&lock->alloc_signal); in update_qp()
556 pthread_mutex_unlock(&lock->alloc_lock); in update_qp()
557 return &lock->qp_group[current_idx]; in update_qp()
560 static void retire_qp(CRYPTO_RCU_LOCK *lock, struct rcu_qp *qp) in retire_qp() argument
562 pthread_mutex_lock(&lock->alloc_lock); in retire_qp()
563 lock->writers_alloced--; in retire_qp()
564 pthread_cond_signal(&lock->alloc_signal); in retire_qp()
565 pthread_mutex_unlock(&lock->alloc_lock); in retire_qp()
570 static struct rcu_qp *allocate_new_qp_group(CRYPTO_RCU_LOCK *lock, in allocate_new_qp_group() argument
576 lock->group_count = count; in allocate_new_qp_group()
580 void ossl_rcu_write_lock(CRYPTO_RCU_LOCK *lock) in ossl_rcu_write_lock() argument
582 pthread_mutex_lock(&lock->write_lock); in ossl_rcu_write_lock()
583 TSAN_FAKE_UNLOCK(&lock->write_lock); in ossl_rcu_write_lock()
586 void ossl_rcu_write_unlock(CRYPTO_RCU_LOCK *lock) in ossl_rcu_write_unlock() argument
588 TSAN_FAKE_LOCK(&lock->write_lock); in ossl_rcu_write_unlock()
589 pthread_mutex_unlock(&lock->write_lock); in ossl_rcu_write_unlock()
592 void ossl_synchronize_rcu(CRYPTO_RCU_LOCK *lock) in ossl_synchronize_rcu() argument
598 pthread_mutex_lock(&lock->write_lock); in ossl_synchronize_rcu()
599 cb_items = lock->cb_items; in ossl_synchronize_rcu()
600 lock->cb_items = NULL; in ossl_synchronize_rcu()
601 pthread_mutex_unlock(&lock->write_lock); in ossl_synchronize_rcu()
603 qp = update_qp(lock); in ossl_synchronize_rcu()
616 pthread_mutex_lock(&lock->prior_lock); in ossl_synchronize_rcu()
617 while (lock->next_to_retire != ID_VAL(count)) in ossl_synchronize_rcu()
618 pthread_cond_wait(&lock->prior_signal, &lock->prior_lock); in ossl_synchronize_rcu()
619 lock->next_to_retire++; in ossl_synchronize_rcu()
620 pthread_cond_broadcast(&lock->prior_signal); in ossl_synchronize_rcu()
621 pthread_mutex_unlock(&lock->prior_lock); in ossl_synchronize_rcu()
623 retire_qp(lock, qp); in ossl_synchronize_rcu()
634 int ossl_rcu_call(CRYPTO_RCU_LOCK *lock, rcu_cb_fn cb, void *data) in ossl_rcu_call() argument
649 new->next = ATOMIC_EXCHANGE_N(prcu_cb_item, &lock->cb_items, new, in ossl_rcu_call()
694 void ossl_rcu_lock_free(CRYPTO_RCU_LOCK *lock) in ossl_rcu_lock_free() argument
696 struct rcu_lock_st *rlock = (struct rcu_lock_st *)lock; in ossl_rcu_lock_free()
698 if (lock == NULL) in ossl_rcu_lock_free()
712 CRYPTO_RWLOCK *lock; in CRYPTO_THREAD_lock_new() local
714 if ((lock = OPENSSL_zalloc(sizeof(pthread_rwlock_t))) == NULL) in CRYPTO_THREAD_lock_new()
718 if (pthread_rwlock_init(lock, NULL) != 0) { in CRYPTO_THREAD_lock_new()
719 OPENSSL_free(lock); in CRYPTO_THREAD_lock_new()
724 CRYPTO_RWLOCK *lock; in CRYPTO_THREAD_lock_new()
726 if ((lock = OPENSSL_zalloc(sizeof(pthread_mutex_t))) == NULL) in CRYPTO_THREAD_lock_new()
742 if (pthread_mutex_init(lock, &attr) != 0) { in CRYPTO_THREAD_lock_new()
744 OPENSSL_free(lock); in CRYPTO_THREAD_lock_new()
751 return lock; in CRYPTO_THREAD_lock_new()
754 __owur int CRYPTO_THREAD_read_lock(CRYPTO_RWLOCK *lock) in CRYPTO_THREAD_read_lock() argument
757 if (pthread_rwlock_rdlock(lock) != 0) in CRYPTO_THREAD_read_lock()
760 if (pthread_mutex_lock(lock) != 0) { in CRYPTO_THREAD_read_lock()
769 __owur int CRYPTO_THREAD_write_lock(CRYPTO_RWLOCK *lock) in CRYPTO_THREAD_write_lock() argument
772 if (pthread_rwlock_wrlock(lock) != 0) in CRYPTO_THREAD_write_lock()
775 if (pthread_mutex_lock(lock) != 0) { in CRYPTO_THREAD_write_lock()
784 int CRYPTO_THREAD_unlock(CRYPTO_RWLOCK *lock) in CRYPTO_THREAD_unlock() argument
787 if (pthread_rwlock_unlock(lock) != 0) in CRYPTO_THREAD_unlock()
790 if (pthread_mutex_unlock(lock) != 0) { in CRYPTO_THREAD_unlock()
799 void CRYPTO_THREAD_lock_free(CRYPTO_RWLOCK *lock) in CRYPTO_THREAD_lock_free() argument
801 if (lock == NULL) in CRYPTO_THREAD_lock_free()
805 pthread_rwlock_destroy(lock); in CRYPTO_THREAD_lock_free()
807 pthread_mutex_destroy(lock); in CRYPTO_THREAD_lock_free()
809 OPENSSL_free(lock); in CRYPTO_THREAD_lock_free()
861 int CRYPTO_atomic_add(int *val, int amount, int *ret, CRYPTO_RWLOCK *lock) in CRYPTO_atomic_add() argument
875 if (lock == NULL || !CRYPTO_THREAD_write_lock(lock)) in CRYPTO_atomic_add()
881 if (!CRYPTO_THREAD_unlock(lock)) in CRYPTO_atomic_add()
888 CRYPTO_RWLOCK *lock) in CRYPTO_atomic_add64() argument
902 if (lock == NULL || !CRYPTO_THREAD_write_lock(lock)) in CRYPTO_atomic_add64()
907 if (!CRYPTO_THREAD_unlock(lock)) in CRYPTO_atomic_add64()
914 CRYPTO_RWLOCK *lock) in CRYPTO_atomic_and() argument
928 if (lock == NULL || !CRYPTO_THREAD_write_lock(lock)) in CRYPTO_atomic_and()
933 if (!CRYPTO_THREAD_unlock(lock)) in CRYPTO_atomic_and()
940 CRYPTO_RWLOCK *lock) in CRYPTO_atomic_or() argument
954 if (lock == NULL || !CRYPTO_THREAD_write_lock(lock)) in CRYPTO_atomic_or()
959 if (!CRYPTO_THREAD_unlock(lock)) in CRYPTO_atomic_or()
965 int CRYPTO_atomic_load(uint64_t *val, uint64_t *ret, CRYPTO_RWLOCK *lock) in CRYPTO_atomic_load() argument
979 if (lock == NULL || !CRYPTO_THREAD_read_lock(lock)) in CRYPTO_atomic_load()
982 if (!CRYPTO_THREAD_unlock(lock)) in CRYPTO_atomic_load()
988 int CRYPTO_atomic_store(uint64_t *dst, uint64_t val, CRYPTO_RWLOCK *lock) in CRYPTO_atomic_store() argument
1002 if (lock == NULL || !CRYPTO_THREAD_write_lock(lock)) in CRYPTO_atomic_store()
1005 if (!CRYPTO_THREAD_unlock(lock)) in CRYPTO_atomic_store()
1011 int CRYPTO_atomic_load_int(int *val, int *ret, CRYPTO_RWLOCK *lock) in CRYPTO_atomic_load_int() argument
1025 if (lock == NULL || !CRYPTO_THREAD_read_lock(lock)) in CRYPTO_atomic_load_int()
1028 if (!CRYPTO_THREAD_unlock(lock)) in CRYPTO_atomic_load_int()
