Lines Matching refs:lock
295 CRYPTO_RCU_LOCK *lock; member
360 static struct rcu_qp *get_hold_current_qp(struct rcu_lock_st *lock) in get_hold_current_qp() argument
378 qp_idx = ATOMIC_LOAD_N(uint64_t, &lock->reader_idx, __ATOMIC_ACQUIRE); in get_hold_current_qp()
387 ATOMIC_ADD_FETCH(&lock->qp_group[qp_idx].users, VAL_READER, in get_hold_current_qp()
391 if (qp_idx == ATOMIC_LOAD_N(uint64_t, &lock->reader_idx, __ATOMIC_ACQUIRE)) in get_hold_current_qp()
400 ATOMIC_SUB_FETCH(&lock->qp_group[qp_idx].users, VAL_READER, in get_hold_current_qp()
404 return &lock->qp_group[qp_idx]; in get_hold_current_qp()
417 void ossl_rcu_read_lock(CRYPTO_RCU_LOCK *lock) in ossl_rcu_read_lock() argument
421 CRYPTO_THREAD_LOCAL *lkey = ossl_lib_ctx_get_rcukey(lock->ctx); in ossl_rcu_read_lock()
433 ossl_init_thread_start(NULL, lock->ctx, ossl_rcu_free_local_data); in ossl_rcu_read_lock()
440 if (data->thread_qps[i].lock == lock) { in ossl_rcu_read_lock()
451 data->thread_qps[available_qp].qp = get_hold_current_qp(lock); in ossl_rcu_read_lock()
453 data->thread_qps[available_qp].lock = lock; in ossl_rcu_read_lock()
456 void ossl_rcu_read_unlock(CRYPTO_RCU_LOCK *lock) in ossl_rcu_read_unlock() argument
459 CRYPTO_THREAD_LOCAL *lkey = ossl_lib_ctx_get_rcukey(lock->ctx); in ossl_rcu_read_unlock()
466 if (data->thread_qps[i].lock == lock) { in ossl_rcu_read_unlock()
478 data->thread_qps[i].lock = NULL; in ossl_rcu_read_unlock()
494 static struct rcu_qp *update_qp(CRYPTO_RCU_LOCK *lock) in update_qp() argument
499 pthread_mutex_lock(&lock->alloc_lock); in update_qp()
506 while (lock->group_count - lock->writers_alloced < 2) in update_qp()
508 pthread_cond_wait(&lock->alloc_signal, &lock->alloc_lock); in update_qp()
510 current_idx = lock->current_alloc_idx; in update_qp()
513 lock->writers_alloced++; in update_qp()
516 lock->current_alloc_idx = in update_qp()
517 (lock->current_alloc_idx + 1) % lock->group_count; in update_qp()
520 new_id = lock->id_ctr; in update_qp()
521 lock->id_ctr++; in update_qp()
530 ATOMIC_AND_FETCH(&lock->qp_group[current_idx].users, ID_MASK, in update_qp()
532 ATOMIC_OR_FETCH(&lock->qp_group[current_idx].users, new_id, in update_qp()
541 ATOMIC_STORE_N(uint64_t, &lock->reader_idx, lock->current_alloc_idx, in update_qp()
545 pthread_cond_signal(&lock->alloc_signal); in update_qp()
546 pthread_mutex_unlock(&lock->alloc_lock); in update_qp()
547 return &lock->qp_group[current_idx]; in update_qp()
550 static void retire_qp(CRYPTO_RCU_LOCK *lock, struct rcu_qp *qp) in retire_qp() argument
552 pthread_mutex_lock(&lock->alloc_lock); in retire_qp()
553 lock->writers_alloced--; in retire_qp()
554 pthread_cond_signal(&lock->alloc_signal); in retire_qp()
555 pthread_mutex_unlock(&lock->alloc_lock); in retire_qp()
558 static struct rcu_qp *allocate_new_qp_group(CRYPTO_RCU_LOCK *lock, in allocate_new_qp_group() argument
564 lock->group_count = count; in allocate_new_qp_group()
568 void ossl_rcu_write_lock(CRYPTO_RCU_LOCK *lock) in ossl_rcu_write_lock() argument
570 pthread_mutex_lock(&lock->write_lock); in ossl_rcu_write_lock()
571 TSAN_FAKE_UNLOCK(&lock->write_lock); in ossl_rcu_write_lock()
574 void ossl_rcu_write_unlock(CRYPTO_RCU_LOCK *lock) in ossl_rcu_write_unlock() argument
576 TSAN_FAKE_LOCK(&lock->write_lock); in ossl_rcu_write_unlock()
577 pthread_mutex_unlock(&lock->write_lock); in ossl_rcu_write_unlock()
580 void ossl_synchronize_rcu(CRYPTO_RCU_LOCK *lock) in ossl_synchronize_rcu() argument
586 pthread_mutex_lock(&lock->write_lock); in ossl_synchronize_rcu()
587 cb_items = lock->cb_items; in ossl_synchronize_rcu()
588 lock->cb_items = NULL; in ossl_synchronize_rcu()
589 pthread_mutex_unlock(&lock->write_lock); in ossl_synchronize_rcu()
591 qp = update_qp(lock); in ossl_synchronize_rcu()
604 pthread_mutex_lock(&lock->prior_lock); in ossl_synchronize_rcu()
605 while (lock->next_to_retire != ID_VAL(count)) in ossl_synchronize_rcu()
606 pthread_cond_wait(&lock->prior_signal, &lock->prior_lock); in ossl_synchronize_rcu()
607 lock->next_to_retire++; in ossl_synchronize_rcu()
608 pthread_cond_broadcast(&lock->prior_signal); in ossl_synchronize_rcu()
609 pthread_mutex_unlock(&lock->prior_lock); in ossl_synchronize_rcu()
611 retire_qp(lock, qp); in ossl_synchronize_rcu()
622 int ossl_rcu_call(CRYPTO_RCU_LOCK *lock, rcu_cb_fn cb, void *data) in ossl_rcu_call() argument
637 new->next = ATOMIC_EXCHANGE_N(prcu_cb_item, &lock->cb_items, new, in ossl_rcu_call()
682 void ossl_rcu_lock_free(CRYPTO_RCU_LOCK *lock) in ossl_rcu_lock_free() argument
684 struct rcu_lock_st *rlock = (struct rcu_lock_st *)lock; in ossl_rcu_lock_free()
686 if (lock == NULL) in ossl_rcu_lock_free()
700 CRYPTO_RWLOCK *lock; in CRYPTO_THREAD_lock_new() local
702 if ((lock = OPENSSL_zalloc(sizeof(pthread_rwlock_t))) == NULL) in CRYPTO_THREAD_lock_new()
706 if (pthread_rwlock_init(lock, NULL) != 0) { in CRYPTO_THREAD_lock_new()
707 OPENSSL_free(lock); in CRYPTO_THREAD_lock_new()
712 CRYPTO_RWLOCK *lock; in CRYPTO_THREAD_lock_new()
714 if ((lock = OPENSSL_zalloc(sizeof(pthread_mutex_t))) == NULL) in CRYPTO_THREAD_lock_new()
730 if (pthread_mutex_init(lock, &attr) != 0) { in CRYPTO_THREAD_lock_new()
732 OPENSSL_free(lock); in CRYPTO_THREAD_lock_new()
739 return lock; in CRYPTO_THREAD_lock_new()
742 __owur int CRYPTO_THREAD_read_lock(CRYPTO_RWLOCK *lock) in CRYPTO_THREAD_read_lock() argument
745 if (pthread_rwlock_rdlock(lock) != 0) in CRYPTO_THREAD_read_lock()
748 if (pthread_mutex_lock(lock) != 0) { in CRYPTO_THREAD_read_lock()
757 __owur int CRYPTO_THREAD_write_lock(CRYPTO_RWLOCK *lock) in CRYPTO_THREAD_write_lock() argument
760 if (pthread_rwlock_wrlock(lock) != 0) in CRYPTO_THREAD_write_lock()
763 if (pthread_mutex_lock(lock) != 0) { in CRYPTO_THREAD_write_lock()
772 int CRYPTO_THREAD_unlock(CRYPTO_RWLOCK *lock) in CRYPTO_THREAD_unlock() argument
775 if (pthread_rwlock_unlock(lock) != 0) in CRYPTO_THREAD_unlock()
778 if (pthread_mutex_unlock(lock) != 0) { in CRYPTO_THREAD_unlock()
787 void CRYPTO_THREAD_lock_free(CRYPTO_RWLOCK *lock) in CRYPTO_THREAD_lock_free() argument
789 if (lock == NULL) in CRYPTO_THREAD_lock_free()
793 pthread_rwlock_destroy(lock); in CRYPTO_THREAD_lock_free()
795 pthread_mutex_destroy(lock); in CRYPTO_THREAD_lock_free()
797 OPENSSL_free(lock); in CRYPTO_THREAD_lock_free()
849 int CRYPTO_atomic_add(int *val, int amount, int *ret, CRYPTO_RWLOCK *lock) in CRYPTO_atomic_add() argument
863 if (lock == NULL || !CRYPTO_THREAD_write_lock(lock)) in CRYPTO_atomic_add()
869 if (!CRYPTO_THREAD_unlock(lock)) in CRYPTO_atomic_add()
876 CRYPTO_RWLOCK *lock) in CRYPTO_atomic_or() argument
890 if (lock == NULL || !CRYPTO_THREAD_write_lock(lock)) in CRYPTO_atomic_or()
895 if (!CRYPTO_THREAD_unlock(lock)) in CRYPTO_atomic_or()
901 int CRYPTO_atomic_load(uint64_t *val, uint64_t *ret, CRYPTO_RWLOCK *lock) in CRYPTO_atomic_load() argument
915 if (lock == NULL || !CRYPTO_THREAD_read_lock(lock)) in CRYPTO_atomic_load()
918 if (!CRYPTO_THREAD_unlock(lock)) in CRYPTO_atomic_load()
924 int CRYPTO_atomic_store(uint64_t *dst, uint64_t val, CRYPTO_RWLOCK *lock) in CRYPTO_atomic_store() argument
938 if (lock == NULL || !CRYPTO_THREAD_read_lock(lock)) in CRYPTO_atomic_store()
941 if (!CRYPTO_THREAD_unlock(lock)) in CRYPTO_atomic_store()
947 int CRYPTO_atomic_load_int(int *val, int *ret, CRYPTO_RWLOCK *lock) in CRYPTO_atomic_load_int() argument
961 if (lock == NULL || !CRYPTO_THREAD_read_lock(lock)) in CRYPTO_atomic_load_int()
964 if (!CRYPTO_THREAD_unlock(lock)) in CRYPTO_atomic_load_int()
