xref: /openssl/include/internal/refcount.h (revision 1cf2f823)
1 /*
2  * Copyright 2016-2024 The OpenSSL Project Authors. All Rights Reserved.
3  *
4  * Licensed under the Apache License 2.0 (the "License").  You may not use
5  * this file except in compliance with the License.  You can obtain a copy
6  * in the file LICENSE in the source distribution or at
7  * https://www.openssl.org/source/license.html
8  */
9 #ifndef OSSL_INTERNAL_REFCOUNT_H
10 # define OSSL_INTERNAL_REFCOUNT_H
11 # pragma once
12 
13 # include <openssl/e_os2.h>
14 # include <openssl/trace.h>
15 # include <openssl/err.h>
16 
17 # if defined(OPENSSL_THREADS) && !defined(OPENSSL_DEV_NO_ATOMICS)
18 #  if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L \
19       && !defined(__STDC_NO_ATOMICS__)
20 #   include <stdatomic.h>
21 #   define HAVE_C11_ATOMICS
22 #  endif
23 
24 #  if defined(HAVE_C11_ATOMICS) && defined(ATOMIC_INT_LOCK_FREE) \
25       && ATOMIC_INT_LOCK_FREE > 0
26 
27 #   define HAVE_ATOMICS 1
28 
29 typedef struct {
30     _Atomic int val;
31 } CRYPTO_REF_COUNT;
32 
CRYPTO_UP_REF(CRYPTO_REF_COUNT * refcnt,int * ret)33 static inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, int *ret)
34 {
35     *ret = atomic_fetch_add_explicit(&refcnt->val, 1, memory_order_relaxed) + 1;
36     return 1;
37 }
38 
39 /*
40  * Changes to shared structure other than reference counter have to be
41  * serialized. And any kind of serialization implies a release fence. This
42  * means that by the time reference counter is decremented all other
43  * changes are visible on all processors. Hence decrement itself can be
44  * relaxed. In case it hits zero, object will be destructed. Since it's
45  * last use of the object, destructor programmer might reason that access
46  * to mutable members doesn't have to be serialized anymore, which would
47  * otherwise imply an acquire fence. Hence conditional acquire fence...
48  */
CRYPTO_DOWN_REF(CRYPTO_REF_COUNT * refcnt,int * ret)49 static inline int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt, int *ret)
50 {
51     *ret = atomic_fetch_sub_explicit(&refcnt->val, 1, memory_order_relaxed) - 1;
52     if (*ret == 0)
53         atomic_thread_fence(memory_order_acquire);
54     return 1;
55 }
56 
CRYPTO_GET_REF(CRYPTO_REF_COUNT * refcnt,int * ret)57 static inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, int *ret)
58 {
59     *ret = atomic_load_explicit(&refcnt->val, memory_order_relaxed);
60     return 1;
61 }
62 
63 #  elif defined(__GNUC__) && defined(__ATOMIC_RELAXED) && __GCC_ATOMIC_INT_LOCK_FREE > 0
64 
65 #   define HAVE_ATOMICS 1
66 
67 typedef struct {
68     int val;
69 } CRYPTO_REF_COUNT;
70 
CRYPTO_UP_REF(CRYPTO_REF_COUNT * refcnt,int * ret)71 static __inline__ int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, int *ret)
72 {
73     *ret = __atomic_fetch_add(&refcnt->val, 1, __ATOMIC_RELAXED) + 1;
74     return 1;
75 }
76 
CRYPTO_DOWN_REF(CRYPTO_REF_COUNT * refcnt,int * ret)77 static __inline__ int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt, int *ret)
78 {
79     *ret = __atomic_fetch_sub(&refcnt->val, 1, __ATOMIC_RELAXED) - 1;
80     if (*ret == 0)
81         __atomic_thread_fence(__ATOMIC_ACQUIRE);
82     return 1;
83 }
84 
CRYPTO_GET_REF(CRYPTO_REF_COUNT * refcnt,int * ret)85 static __inline__ int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, int *ret)
86 {
87     *ret = __atomic_load_n(&refcnt->val, __ATOMIC_RELAXED);
88     return 1;
89 }
90 
91 #  elif defined(__ICL) && defined(_WIN32)
92 #   define HAVE_ATOMICS 1
93 
94 typedef struct {
95     volatile int val;
96 } CRYPTO_REF_COUNT;
97 
CRYPTO_UP_REF(CRYPTO_REF_COUNT * refcnt,int * ret)98 static __inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, int *ret)
99 {
100     *ret = _InterlockedExchangeAdd((void *)&refcnt->val, 1) + 1;
101     return 1;
102 }
103 
CRYPTO_DOWN_REF(CRYPTO_REF_COUNT * refcnt,int * ret)104 static __inline int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt, int *ret)
105 {
106     *ret = _InterlockedExchangeAdd((void *)&refcnt->val, -1) - 1;
107     return 1;
108 }
109 
CRYPTO_GET_REF(CRYPTO_REF_COUNT * refcnt,int * ret)110 static __inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, int *ret)
111 {
112     *ret = _InterlockedOr((void *)&refcnt->val, 0);
113     return 1;
114 }
115 
116 #  elif defined(_MSC_VER) && _MSC_VER>=1200
117 
118 #   define HAVE_ATOMICS 1
119 
120 typedef struct {
121     volatile int val;
122 } CRYPTO_REF_COUNT;
123 
124 #   if (defined(_M_ARM) && _M_ARM>=7 && !defined(_WIN32_WCE)) || defined(_M_ARM64)
125 #    include <intrin.h>
126 #    if defined(_M_ARM64) && !defined(_ARM_BARRIER_ISH)
127 #     define _ARM_BARRIER_ISH _ARM64_BARRIER_ISH
128 #    endif
129 
CRYPTO_UP_REF(CRYPTO_REF_COUNT * refcnt,int * ret)130 static __inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, int *ret)
131 {
132     *ret = _InterlockedExchangeAdd_nf(&refcnt->val, 1) + 1;
133     return 1;
134 }
135 
CRYPTO_DOWN_REF(CRYPTO_REF_COUNT * refcnt,int * ret)136 static __inline int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt, int *ret)
137 {
138     *ret = _InterlockedExchangeAdd_nf(&refcnt->val, -1) - 1;
139     if (*ret == 0)
140         __dmb(_ARM_BARRIER_ISH);
141     return 1;
142 }
143 
CRYPTO_GET_REF(CRYPTO_REF_COUNT * refcnt,int * ret)144 static __inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, int *ret)
145 {
146     *ret = _InterlockedOr_nf((void *)&refcnt->val, 0);
147     return 1;
148 }
149 
150 #   else
151 #    if !defined(_WIN32_WCE)
152 #     pragma intrinsic(_InterlockedExchangeAdd)
153 #    else
154 #     if _WIN32_WCE >= 0x600
155        extern long __cdecl _InterlockedExchangeAdd(long volatile*, long);
156 #     else
157        /* under Windows CE we still have old-style Interlocked* functions */
158        extern long __cdecl InterlockedExchangeAdd(long volatile*, long);
159 #      define _InterlockedExchangeAdd InterlockedExchangeAdd
160 #     endif
161 #    endif
162 
CRYPTO_UP_REF(CRYPTO_REF_COUNT * refcnt,int * ret)163 static __inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt, int *ret)
164 {
165     *ret = _InterlockedExchangeAdd(&refcnt->val, 1) + 1;
166     return 1;
167 }
168 
CRYPTO_DOWN_REF(CRYPTO_REF_COUNT * refcnt,int * ret)169 static __inline int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt, int *ret)
170 {
171     *ret = _InterlockedExchangeAdd(&refcnt->val, -1) - 1;
172     return 1;
173 }
174 
CRYPTO_GET_REF(CRYPTO_REF_COUNT * refcnt,int * ret)175 static __inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt, int *ret)
176 {
177     *ret = _InterlockedExchangeAdd(&refcnt->val, 0);
178     return 1;
179 }
180 
181 #   endif
182 
183 #  endif
184 # endif  /* !OPENSSL_DEV_NO_ATOMICS */
185 
186 /*
187  * All the refcounting implementations above define HAVE_ATOMICS, so if it's
188  * still undefined here (such as when OPENSSL_DEV_NO_ATOMICS is defined), it
189  * means we need to implement a fallback.  This fallback uses locks.
190  */
191 # ifndef HAVE_ATOMICS
192 
193 typedef struct {
194     int val;
195 #  ifdef OPENSSL_THREADS
196     CRYPTO_RWLOCK *lock;
197 #  endif
198 } CRYPTO_REF_COUNT;
199 
200 #  ifdef OPENSSL_THREADS
201 
CRYPTO_UP_REF(CRYPTO_REF_COUNT * refcnt,int * ret)202 static ossl_unused ossl_inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt,
203                                                  int *ret)
204 {
205     return CRYPTO_atomic_add(&refcnt->val, 1, ret, refcnt->lock);
206 }
207 
CRYPTO_DOWN_REF(CRYPTO_REF_COUNT * refcnt,int * ret)208 static ossl_unused ossl_inline int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt,
209                                                    int *ret)
210 {
211     return CRYPTO_atomic_add(&refcnt->val, -1, ret, refcnt->lock);
212 }
213 
CRYPTO_GET_REF(CRYPTO_REF_COUNT * refcnt,int * ret)214 static ossl_unused ossl_inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt,
215                                                    int *ret)
216 {
217     return CRYPTO_atomic_load_int(&refcnt->val, ret, refcnt->lock);
218 }
219 
220 #   define CRYPTO_NEW_FREE_DEFINED  1
CRYPTO_NEW_REF(CRYPTO_REF_COUNT * refcnt,int n)221 static ossl_unused ossl_inline int CRYPTO_NEW_REF(CRYPTO_REF_COUNT *refcnt, int n)
222 {
223     refcnt->val = n;
224     refcnt->lock = CRYPTO_THREAD_lock_new();
225     if (refcnt->lock == NULL) {
226         ERR_raise(ERR_LIB_CRYPTO, ERR_R_CRYPTO_LIB);
227         return 0;
228     }
229     return 1;
230 }
231 
CRYPTO_FREE_REF(CRYPTO_REF_COUNT * refcnt)232 static ossl_unused ossl_inline void CRYPTO_FREE_REF(CRYPTO_REF_COUNT *refcnt)                                  \
233 {
234     if (refcnt != NULL)
235         CRYPTO_THREAD_lock_free(refcnt->lock);
236 }
237 
238 #  else     /* OPENSSL_THREADS */
239 
CRYPTO_UP_REF(CRYPTO_REF_COUNT * refcnt,int * ret)240 static ossl_unused ossl_inline int CRYPTO_UP_REF(CRYPTO_REF_COUNT *refcnt,
241                                                  int *ret)
242 {
243     refcnt->val++;
244     *ret = refcnt->val;
245     return 1;
246 }
247 
CRYPTO_DOWN_REF(CRYPTO_REF_COUNT * refcnt,int * ret)248 static ossl_unused ossl_inline int CRYPTO_DOWN_REF(CRYPTO_REF_COUNT *refcnt,
249                                                    int *ret)
250 {
251     refcnt->val--;
252     *ret = refcnt->val;
253     return 1;
254 }
255 
CRYPTO_GET_REF(CRYPTO_REF_COUNT * refcnt,int * ret)256 static ossl_unused ossl_inline int CRYPTO_GET_REF(CRYPTO_REF_COUNT *refcnt,
257                                                    int *ret)
258 {
259     *ret = refcnt->val;
260     return 1;
261 }
262 
263 #  endif    /* OPENSSL_THREADS */
264 # endif
265 
266 # ifndef CRYPTO_NEW_FREE_DEFINED
CRYPTO_NEW_REF(CRYPTO_REF_COUNT * refcnt,int n)267 static ossl_unused ossl_inline int CRYPTO_NEW_REF(CRYPTO_REF_COUNT *refcnt, int n)
268 {
269     refcnt->val = n;
270     return 1;
271 }
272 
CRYPTO_FREE_REF(CRYPTO_REF_COUNT * refcnt)273 static ossl_unused ossl_inline void CRYPTO_FREE_REF(CRYPTO_REF_COUNT *refcnt)                                  \
274 {
275 }
276 # endif /* CRYPTO_NEW_FREE_DEFINED */
277 #undef CRYPTO_NEW_FREE_DEFINED
278 
279 # if !defined(NDEBUG) && !defined(OPENSSL_NO_STDIO)
280 #  define REF_ASSERT_ISNT(test) \
281     (void)((test) ? (OPENSSL_die("refcount error", __FILE__, __LINE__), 1) : 0)
282 # else
283 #  define REF_ASSERT_ISNT(i)
284 # endif
285 
286 # define REF_PRINT_EX(text, count, object) \
287     OSSL_TRACE3(REF_COUNT, "%p:%4d:%s\n", (object), (count), (text));
288 # define REF_PRINT_COUNT(text, object) \
289     REF_PRINT_EX(text, object->references.val, (void *)object)
290 
291 #endif
292