xref: /openssl/include/internal/tsan_assist.h (revision fecb3aae)
1 /*
2  * Copyright 2018-2022 The OpenSSL Project Authors. All Rights Reserved.
3  *
4  * Licensed under the Apache License 2.0 (the "License").  You may not use
5  * this file except in compliance with the License.  You can obtain a copy
6  * in the file LICENSE in the source distribution or at
7  * https://www.openssl.org/source/license.html
8  */
9 
10 /*
11  * Contemporary compilers implement lock-free atomic memory access
12  * primitives that facilitate writing "thread-opportunistic" or even real
13  * multi-threading low-overhead code. "Thread-opportunistic" is when
14  * exact result is not required, e.g. some statistics, or execution flow
15  * doesn't have to be unambiguous. Simplest example is lazy "constant"
16  * initialization when one can synchronize on variable itself, e.g.
17  *
18  * if (var == NOT_YET_INITIALIZED)
19  *     var = function_returning_same_value();
20  *
21  * This does work provided that loads and stores are single-instruction
22  * operations (and integer ones are on *all* supported platforms), but
23  * it upsets Thread Sanitizer. Suggested solution is
24  *
25  * if (tsan_load(&var) == NOT_YET_INITIALIZED)
26  *     tsan_store(&var, function_returning_same_value());
27  *
28  * Production machine code would be the same, so one can wonder why
29  * bother. Having Thread Sanitizer accept "thread-opportunistic" code
30  * allows to move on trouble-shooting real bugs.
31  *
32  * Resolving Thread Sanitizer nits was the initial purpose for this module,
33  * but it was later extended with more nuanced primitives that are useful
34  * even in "non-opportunistic" scenarios. Most notably verifying if a shared
35  * structure is fully initialized and bypassing the initialization lock.
36  * It's suggested to view macros defined in this module as "annotations" for
37  * thread-safe lock-free code, "Thread-Safe ANnotations"...
38  *
39  * It's assumed that ATOMIC_{LONG|INT}_LOCK_FREE are assigned same value as
40  * ATOMIC_POINTER_LOCK_FREE. And check for >= 2 ensures that corresponding
41  * code is inlined. It should be noted that statistics counters become
42  * accurate in such case.
43  *
44  * Special note about TSAN_QUALIFIER. It might be undesired to use it in
45  * a shared header. Because whether operation on specific variable or member
46  * is atomic or not might be irrelevant in other modules. In such case one
47  * can use TSAN_QUALIFIER in cast specifically when it has to count.
48  */
49 
50 #ifndef OSSL_INTERNAL_TSAN_ASSIST_H
51 # define OSSL_INTERNAL_TSAN_ASSIST_H
52 # pragma once
53 
54 # if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112L \
55     && !defined(__STDC_NO_ATOMICS__)
56 #  include <stdatomic.h>
57 
58 #  if defined(ATOMIC_POINTER_LOCK_FREE) \
59           && ATOMIC_POINTER_LOCK_FREE >= 2
60 #   define TSAN_QUALIFIER _Atomic
61 #   define tsan_load(ptr) atomic_load_explicit((ptr), memory_order_relaxed)
62 #   define tsan_store(ptr, val) atomic_store_explicit((ptr), (val), memory_order_relaxed)
63 #   define tsan_add(ptr, n) atomic_fetch_add_explicit((ptr), (n), memory_order_relaxed)
64 #   define tsan_ld_acq(ptr) atomic_load_explicit((ptr), memory_order_acquire)
65 #   define tsan_st_rel(ptr, val) atomic_store_explicit((ptr), (val), memory_order_release)
66 #  endif
67 
68 # elif defined(__GNUC__) && defined(__ATOMIC_RELAXED)
69 
70 #  if defined(__GCC_ATOMIC_POINTER_LOCK_FREE) \
71           && __GCC_ATOMIC_POINTER_LOCK_FREE >= 2
72 #   define TSAN_QUALIFIER volatile
73 #   define tsan_load(ptr) __atomic_load_n((ptr), __ATOMIC_RELAXED)
74 #   define tsan_store(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_RELAXED)
75 #   define tsan_add(ptr, n) __atomic_fetch_add((ptr), (n), __ATOMIC_RELAXED)
76 #   define tsan_ld_acq(ptr) __atomic_load_n((ptr), __ATOMIC_ACQUIRE)
77 #   define tsan_st_rel(ptr, val) __atomic_store_n((ptr), (val), __ATOMIC_RELEASE)
78 #  endif
79 
80 # elif defined(_MSC_VER) && _MSC_VER>=1200 \
81       && (defined(_M_IX86) || defined(_M_AMD64) || defined(_M_X64) || \
82           defined(_M_ARM64) || (defined(_M_ARM) && _M_ARM >= 7 && !defined(_WIN32_WCE)))
83 /*
84  * There is subtle dependency on /volatile:<iso|ms> command-line option.
85  * "ms" implies same semantic as memory_order_acquire for loads and
86  * memory_order_release for stores, while "iso" - memory_order_relaxed for
87  * either. Real complication is that defaults are different on x86 and ARM.
88  * There is explanation for that, "ms" is backward compatible with earlier
89  * compiler versions, while multi-processor ARM can be viewed as brand new
90  * platform to MSC and its users, and with non-relaxed semantic taking toll
91  * with additional instructions and penalties, it kind of makes sense to
92  * default to "iso"...
93  */
94 #  define TSAN_QUALIFIER volatile
95 #  if defined(_M_ARM) || defined(_M_ARM64)
96 #   define _InterlockedExchangeAdd _InterlockedExchangeAdd_nf
97 #   pragma intrinsic(_InterlockedExchangeAdd_nf)
98 #   pragma intrinsic(__iso_volatile_load32, __iso_volatile_store32)
99 #   ifdef _WIN64
100 #    define _InterlockedExchangeAdd64 _InterlockedExchangeAdd64_nf
101 #    pragma intrinsic(_InterlockedExchangeAdd64_nf)
102 #    pragma intrinsic(__iso_volatile_load64, __iso_volatile_store64)
103 #    define tsan_load(ptr) (sizeof(*(ptr)) == 8 ? __iso_volatile_load64(ptr) \
104                                                : __iso_volatile_load32(ptr))
105 #    define tsan_store(ptr, val) (sizeof(*(ptr)) == 8 ? __iso_volatile_store64((ptr), (val)) \
106                                                      : __iso_volatile_store32((ptr), (val)))
107 #   else
108 #    define tsan_load(ptr) __iso_volatile_load32(ptr)
109 #    define tsan_store(ptr, val) __iso_volatile_store32((ptr), (val))
110 #   endif
111 #  else
112 #   define tsan_load(ptr) (*(ptr))
113 #   define tsan_store(ptr, val) (*(ptr) = (val))
114 #  endif
115 #  pragma intrinsic(_InterlockedExchangeAdd)
116 #  ifdef _WIN64
117 #   pragma intrinsic(_InterlockedExchangeAdd64)
118 #   define tsan_add(ptr, n) (sizeof(*(ptr)) == 8 ? _InterlockedExchangeAdd64((ptr), (n)) \
119                                                 : _InterlockedExchangeAdd((ptr), (n)))
120 #  else
121 #   define tsan_add(ptr, n) _InterlockedExchangeAdd((ptr), (n))
122 #  endif
123 #  if !defined(_ISO_VOLATILE)
124 #   define tsan_ld_acq(ptr) (*(ptr))
125 #   define tsan_st_rel(ptr, val) (*(ptr) = (val))
126 #  endif
127 
128 # endif
129 
130 # ifndef TSAN_QUALIFIER
131 
132 #  ifdef OPENSSL_THREADS
133 #   define TSAN_QUALIFIER volatile
134 #   define TSAN_REQUIRES_LOCKING
135 #  else  /* OPENSSL_THREADS */
136 #   define TSAN_QUALIFIER
137 #  endif /* OPENSSL_THREADS */
138 
139 #  define tsan_load(ptr) (*(ptr))
140 #  define tsan_store(ptr, val) (*(ptr) = (val))
141 #  define tsan_add(ptr, n) (*(ptr) += (n))
142 /*
143  * Lack of tsan_ld_acq and tsan_ld_rel means that compiler support is not
144  * sophisticated enough to support them. Code that relies on them should be
145  * protected with #ifdef tsan_ld_acq with locked fallback.
146  */
147 
148 # endif
149 
150 # define tsan_counter(ptr) tsan_add((ptr), 1)
151 # define tsan_decr(ptr) tsan_add((ptr), -1)
152 
153 #endif
154