Atomic.h 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449
  1. #pragma once
  2. #include "../C/Baselib_Atomic.h"
  3. #include "Internal/TypeTraits.h"
  4. // Note that aligning by type is not possible with the C compatible COMPILER_ALIGN_AS as MSVC's own alignment attribute does not allow evaluation of sizeof
  5. #define ALIGN_ATOMIC(TYPE_) alignas(sizeof(TYPE_))
  6. #define ALIGNED_ATOMIC(TYPE_) ALIGN_ATOMIC(TYPE_) TYPE_
  7. // Atomic interface that sticks closely to std::atomic
  8. // Major differences:
  9. // * free functions that operate on types other than baselib::atomic
  10. // * baselib::atomic allows access to its internal value
  11. // * no zero initialization on baselib::atomic
  12. // * no single parameter versions of compare_exchange
  13. namespace baselib
  14. {
  15. BASELIB_CPP_INTERFACE
  16. {
  17. enum memory_order_relaxed_t { memory_order_relaxed = 0 }; // Equal to std::memory_order_relaxed
  18. enum memory_order_acquire_t { memory_order_acquire = 2 }; // Equal to std::memory_order_acquire
  19. enum memory_order_release_t { memory_order_release = 3 }; // Equal to std::memory_order_release
  20. enum memory_order_acq_rel_t { memory_order_acq_rel = 4 }; // Equal to std::memory_order_acq_rel
  21. enum memory_order_seq_cst_t { memory_order_seq_cst = 5 }; // Equal to std::memory_order_seq_cst
  22. namespace detail
  23. {
  24. template<typename T, typename ... Rest>
  25. struct is_any : std::false_type {};
  26. template<typename T, typename First>
  27. struct is_any<T, First> : std::is_same<T, First> {};
  28. template<typename T, typename First, typename ... Rest>
  29. struct is_any<T, First, Rest...>
  30. : std::integral_constant<bool, std::is_same<T, First>::value || is_any<T, Rest...>::value>
  31. {};
  32. #define TEST_ATOMICS_PREREQUISITES(_TYPE) \
  33. static_assert(baselib::is_trivially_copyable<_TYPE>::value, "atomic operation operands needs to be trivially copyable"); \
  34. static_assert(sizeof(_TYPE) <= sizeof(void*) * 2, "atomic operation operands need to be smaller or equal than two pointers");
  35. template<typename T> static inline T fail();
  36. template<typename T, typename MemoryOrder, typename ... AllowedMemoryOrders> static inline T fail_prerequisites()
  37. {
  38. TEST_ATOMICS_PREREQUISITES(T);
  39. static_assert(is_any<MemoryOrder, AllowedMemoryOrders...>::value, "the specified memory ordering is invalid for this atomic operation");
  40. return fail<T>();
  41. }
  42. template<typename T, typename MemoryOrderSuccess, typename MemoryOrderFailure> static inline T fail_prerequisites_cmpxchg()
  43. {
  44. TEST_ATOMICS_PREREQUISITES(T);
  45. static_assert(
  46. // fail: relaxed, success: relaxed/acquire/release/seq_cst
  47. (std::is_same<MemoryOrderFailure, baselib::memory_order_relaxed_t>::value &&
  48. is_any<MemoryOrderSuccess, baselib::memory_order_relaxed_t, baselib::memory_order_acquire_t, baselib::memory_order_release_t, baselib::memory_order_seq_cst_t>::value) ||
  49. // fail: acquire, success acquire/release/seq_cst
  50. (std::is_same<MemoryOrderFailure, baselib::memory_order_relaxed_t>::value &&
  51. is_any<MemoryOrderSuccess, baselib::memory_order_acquire_t, baselib::memory_order_release_t, baselib::memory_order_seq_cst_t>::value) ||
  52. // fail: seq_cst, success: seq_cst
  53. (std::is_same<MemoryOrderSuccess, baselib::memory_order_seq_cst_t>::value && std::is_same<MemoryOrderFailure, baselib::memory_order_seq_cst_t>::value),
  54. "the specified combination of memory ordering is invalid for compare exchange operations");
  55. return fail<T>();
  56. }
  57. template<typename T, typename MemoryOrder> static inline T fail_prerequisites_alu()
  58. {
  59. static_assert(std::is_integral<T>::value, "operands of arithmetic atomic operations need to be integral");
  60. return fail_prerequisites<T, MemoryOrder,
  61. baselib::memory_order_relaxed_t,
  62. baselib::memory_order_acquire_t,
  63. baselib::memory_order_release_t,
  64. baselib::memory_order_acq_rel_t,
  65. baselib::memory_order_seq_cst_t>();
  66. }
  67. }
  68. // MACRO generated impl
  69. // re-directs to Baselib_atomic_ API
  70. // ----------------------------------------------------------------------------------------------------------------------------------
  71. #define detail_THREAD_FENCE(order, ...) \
  72. static FORCE_INLINE void atomic_thread_fence(memory_order_##order##_t order) \
  73. { \
  74. return Baselib_atomic_thread_fence_##order(); \
  75. }
  76. #define detail_LOAD(op, order, id, bits, ...) \
  77. template<typename T, typename std::enable_if<baselib::is_trivial_of_size<T, bits/8>::value, int>::type = 0> \
  78. static FORCE_INLINE T atomic_load_explicit(const T& obj, memory_order_##order##_t order) \
  79. { \
  80. T ret; \
  81. Baselib_atomic_load_##id##_##order##_v(&obj, &ret); \
  82. return ret; \
  83. }
  84. #define detail_LOAD128(op, order, id, bits, ...) \
  85. template<typename T, typename std::enable_if<baselib::is_trivial_of_size<T, bits/8>::value, int>::type = 0> \
  86. static FORCE_INLINE T atomic_load_explicit(const T& obj, memory_order_##order##_t order) \
  87. { \
  88. T ret; \
  89. Baselib_atomic_load_##id##_##order##_v(const_cast<T*>(&obj), &ret); \
  90. return ret; \
  91. }
  92. #define detail_STORE(op, order, id, bits, ...) \
  93. template<typename T, typename std::enable_if<baselib::is_trivial_of_size<T, bits/8>::value, int>::type = 0> \
  94. static FORCE_INLINE void atomic_store_explicit(T& obj, typename std::common_type<T>::type value, memory_order_##order##_t order)\
  95. { \
  96. return Baselib_atomic_store_##id##_##order##_v(&obj, &value); \
  97. }
  98. #define detail_LOAD_STORE(op, order, id, bits, ...) \
  99. template<typename T, typename std::enable_if<baselib::is_trivial_of_size<T, bits/8>::value, int>::type = 0> \
  100. static FORCE_INLINE T atomic_##op##_explicit(T& obj, typename std::common_type<T>::type value, memory_order_##order##_t order) \
  101. { \
  102. T ret; \
  103. Baselib_atomic_##op##_##id##_##order##_v(&obj, &value, &ret); \
  104. return ret; \
  105. }
  106. #define detail_ALU(op, order, id, bits, ...) \
  107. template<typename T, typename std::enable_if<baselib::is_integral_of_size<T, bits/8>::value, int>::type = 0> \
  108. static FORCE_INLINE T atomic_##op##_explicit(T& obj, typename std::common_type<T>::type value, memory_order_##order##_t order) \
  109. { \
  110. T ret; \
  111. Baselib_atomic_##op##_##id##_##order##_v(&obj, &value, &ret); \
  112. return ret; \
  113. }
  114. #define detail_CMP_XCHG(op, order1, order2, id, bits, ...) \
  115. template<typename T, typename std::enable_if<baselib::is_trivial_of_size<T, bits/8>::value, int>::type = 0> \
  116. static FORCE_INLINE bool atomic_##op##_explicit(T& obj, \
  117. typename std::common_type<T>::type& expected, \
  118. typename std::common_type<T>::type desired, \
  119. memory_order_##order1##_t order_success, \
  120. memory_order_##order2##_t order_failure) \
  121. { \
  122. return Baselib_atomic_##op##_##id##_##order1##_##order2##_v(&obj, &expected, &desired); \
  123. }
  124. #define detail_NOT_SUPPORTED(...)
  125. Baselib_Atomic_FOR_EACH_MEMORY_ORDER(
  126. detail_THREAD_FENCE
  127. )
  128. Baselib_Atomic_FOR_EACH_ATOMIC_OP_MEMORY_ORDER_AND_INT_TYPE(
  129. detail_LOAD, // load
  130. detail_STORE, // store
  131. detail_ALU, // add
  132. detail_ALU, // and
  133. detail_ALU, // or
  134. detail_ALU, // xor
  135. detail_LOAD_STORE, // exchange
  136. detail_CMP_XCHG, // compare_exchange_weak
  137. detail_CMP_XCHG // compare_exchange_strong
  138. )
  139. #if PLATFORM_ARCH_64
  140. // 128bit atomics
  141. Baselib_Atomic_FOR_EACH_ATOMIC_OP_AND_MEMORY_ORDER(
  142. detail_LOAD128, // load
  143. detail_STORE, // store
  144. detail_NOT_SUPPORTED, // add
  145. detail_NOT_SUPPORTED, // and
  146. detail_NOT_SUPPORTED, // or
  147. detail_NOT_SUPPORTED, // xor
  148. detail_LOAD_STORE, // exchange
  149. detail_CMP_XCHG, // compare_exchange_weak
  150. detail_CMP_XCHG, // compare_exchange_strong
  151. 128, 128)
  152. #endif
  153. #undef detail_THREAD_FENCE
  154. #undef detail_LOAD128
  155. #undef detail_LOAD
  156. #undef detail_STORE
  157. #undef detail_LOAD_STORE
  158. #undef detail_ALU
  159. #undef detail_CMP_XCHG
  160. #undef detail_NOT_SUPPORTED
  161. template<typename T, typename MemoryOrder>
  162. static FORCE_INLINE T atomic_fetch_sub_explicit(T& obj, typename std::common_type<T>::type value, MemoryOrder order)
  163. {
  164. return atomic_fetch_add_explicit(obj, 0 - value, order);
  165. }
  166. // API documentation and default fallback for non-matching types
  167. // ----------------------------------------------------------------------------------------------------------------------
  168. template<typename T, typename MemoryOrder>
  169. static FORCE_INLINE T atomic_load_explicit(const T& obj, MemoryOrder order)
  170. {
  171. return detail::fail_prerequisites<T, MemoryOrder, baselib::memory_order_relaxed_t, baselib::memory_order_acquire_t, baselib::memory_order_seq_cst_t>();
  172. }
  173. template<typename T, typename MemoryOrder>
  174. static FORCE_INLINE void atomic_store_explicit(T& obj, typename std::common_type<T>::type value, MemoryOrder order)
  175. {
  176. detail::fail_prerequisites<T, MemoryOrder, baselib::memory_order_relaxed_t, baselib::memory_order_release_t, baselib::memory_order_seq_cst_t>();
  177. }
  178. template<typename T, typename MemoryOrder>
  179. static FORCE_INLINE T atomic_fetch_add_explicit(T& obj, typename std::common_type<T>::type value, MemoryOrder order)
  180. {
  181. return detail::fail_prerequisites_alu<T, MemoryOrder>();
  182. }
  183. template<typename T, typename MemoryOrder>
  184. static FORCE_INLINE T atomic_fetch_and_explicit(T& obj, typename std::common_type<T>::type value, MemoryOrder order)
  185. {
  186. return detail::fail_prerequisites_alu<T, MemoryOrder>();
  187. }
  188. template<typename T, typename MemoryOrder>
  189. static FORCE_INLINE T atomic_fetch_or_explicit(T& obj, typename std::common_type<T>::type value, MemoryOrder order)
  190. {
  191. return detail::fail_prerequisites_alu<T, MemoryOrder>();
  192. }
  193. template<typename T, typename MemoryOrder>
  194. static FORCE_INLINE T atomic_fetch_xor_explicit(T& obj, typename std::common_type<T>::type value, MemoryOrder order)
  195. {
  196. return detail::fail_prerequisites_alu<T, MemoryOrder>();
  197. }
  198. template<typename T, typename MemoryOrder>
  199. static FORCE_INLINE T atomic_exchange_explicit(T& obj, typename std::common_type<T>::type value, MemoryOrder order)
  200. {
  201. return detail::fail_prerequisites<T, MemoryOrder>();
  202. }
  203. template<typename T, typename MemoryOrderSuccess, typename MemoryOrderFailure>
  204. static FORCE_INLINE bool atomic_compare_exchange_weak_explicit(T& obj,
  205. typename std::common_type<T>::type& expected,
  206. typename std::common_type<T>::type desired,
  207. MemoryOrderSuccess order_success,
  208. MemoryOrderFailure order_failure)
  209. {
  210. detail::fail_prerequisites_cmpxchg<T, MemoryOrderSuccess, MemoryOrderFailure>();
  211. return false;
  212. }
  213. template<typename T, typename MemoryOrderSuccess, typename MemoryOrderFailure>
  214. static FORCE_INLINE bool atomic_compare_exchange_strong_explicit(T& obj,
  215. typename std::common_type<T>::type& expected,
  216. typename std::common_type<T>::type desired,
  217. MemoryOrderSuccess order_success,
  218. MemoryOrderFailure order_failure)
  219. {
  220. detail::fail_prerequisites_cmpxchg<T, MemoryOrderSuccess, MemoryOrderFailure>();
  221. return false;
  222. }
  223. // default memory order (memory_order_seq_cst)
  224. // ----------------------------------------------------------------------------------------------------------------------
  225. template<typename T>
  226. static FORCE_INLINE T atomic_load(const T& obj)
  227. {
  228. return atomic_load_explicit(obj, memory_order_seq_cst);
  229. }
  230. template<typename T>
  231. static FORCE_INLINE void atomic_store(T& obj, typename std::common_type<T>::type value)
  232. {
  233. return atomic_store_explicit(obj, value, memory_order_seq_cst);
  234. }
  235. template<typename T>
  236. static FORCE_INLINE T atomic_fetch_add(T& obj, typename std::common_type<T>::type value)
  237. {
  238. return atomic_fetch_add_explicit(obj, value, memory_order_seq_cst);
  239. }
  240. template<typename T>
  241. static FORCE_INLINE T atomic_fetch_sub(T& obj, typename std::common_type<T>::type value)
  242. {
  243. return atomic_fetch_sub_explicit(obj, value, memory_order_seq_cst);
  244. }
  245. template<typename T>
  246. static FORCE_INLINE T atomic_fetch_and(T& obj, typename std::common_type<T>::type value)
  247. {
  248. return atomic_fetch_and_explicit(obj, value, memory_order_seq_cst);
  249. }
  250. template<typename T>
  251. static FORCE_INLINE T atomic_fetch_or(T& obj, typename std::common_type<T>::type value)
  252. {
  253. return atomic_fetch_or_explicit(obj, value, memory_order_seq_cst);
  254. }
  255. template<typename T>
  256. static FORCE_INLINE T atomic_fetch_xor(T& obj, typename std::common_type<T>::type value)
  257. {
  258. return atomic_fetch_xor_explicit(obj, value, memory_order_seq_cst);
  259. }
  260. template<typename T>
  261. static FORCE_INLINE T atomic_exchange(T& obj, typename std::common_type<T>::type value)
  262. {
  263. return atomic_exchange_explicit(obj, value, memory_order_seq_cst);
  264. }
  265. template<typename T>
  266. static FORCE_INLINE bool atomic_compare_exchange_weak(T& obj,
  267. typename std::common_type<T>::type& expected,
  268. typename std::common_type<T>::type desired)
  269. {
  270. return atomic_compare_exchange_weak_explicit(obj, expected, desired, memory_order_seq_cst, memory_order_seq_cst);
  271. }
  272. template<typename T>
  273. static FORCE_INLINE bool atomic_compare_exchange_strong(T& obj,
  274. typename std::common_type<T>::type& expected,
  275. typename std::common_type<T>::type desired)
  276. {
  277. return atomic_compare_exchange_strong_explicit(obj, expected, desired, memory_order_seq_cst, memory_order_seq_cst);
  278. }
  279. template<typename T>
  280. struct atomic_common
  281. {
  282. using value_type = T;
  283. TEST_ATOMICS_PREREQUISITES(T);
  284. ALIGNED_ATOMIC(T) obj;
  285. FORCE_INLINE atomic_common() = default;
  286. // Initializes atomic with a given value. Initialization is not atomic!
  287. FORCE_INLINE atomic_common(T value)
  288. {
  289. obj = value;
  290. }
  291. FORCE_INLINE operator T() const { return atomic_load_explicit(obj, memory_order_seq_cst); }
  292. FORCE_INLINE T operator=(T value) { atomic_store_explicit(obj, value, memory_order_seq_cst); return value; }
  293. template<typename TMemoryOrder = memory_order_seq_cst_t>
  294. FORCE_INLINE T load(TMemoryOrder order = memory_order_seq_cst) const
  295. {
  296. return atomic_load_explicit(obj, order);
  297. }
  298. template<typename TMemoryOrder = memory_order_seq_cst_t>
  299. FORCE_INLINE void store(T value, TMemoryOrder order = memory_order_seq_cst)
  300. {
  301. return atomic_store_explicit(obj, value, order);
  302. }
  303. template<typename TMemoryOrder = memory_order_seq_cst_t>
  304. FORCE_INLINE T exchange(T value, TMemoryOrder order = memory_order_seq_cst)
  305. {
  306. return atomic_exchange_explicit(obj, value, order);
  307. }
  308. template<typename TMemoryOrderSuccess, typename TMemoryOrderFailure>
  309. FORCE_INLINE bool compare_exchange_weak(T& expected, T desired, TMemoryOrderSuccess order_success, TMemoryOrderFailure order_failure)
  310. {
  311. return atomic_compare_exchange_weak_explicit(obj, expected, desired, order_success, order_failure);
  312. }
  313. FORCE_INLINE bool compare_exchange_weak(T& expected, T desired)
  314. {
  315. return atomic_compare_exchange_weak_explicit(obj, expected, desired, memory_order_seq_cst, memory_order_seq_cst);
  316. }
  317. template<typename TMemoryOrderSuccess, typename TMemoryOrderFailure>
  318. FORCE_INLINE bool compare_exchange_strong(T& expected, T desired, TMemoryOrderSuccess order_success, TMemoryOrderFailure order_failure)
  319. {
  320. return atomic_compare_exchange_strong_explicit(obj, expected, desired, order_success, order_failure);
  321. }
  322. FORCE_INLINE bool compare_exchange_strong(T& expected, T desired)
  323. {
  324. return atomic_compare_exchange_strong_explicit(obj, expected, desired, memory_order_seq_cst, memory_order_seq_cst);
  325. }
  326. };
  327. template<typename T, bool IsIntegral>
  328. struct atomic_base {};
  329. // Atomic type for integral types.
  330. template<typename T>
  331. struct atomic_base<T, true> : atomic_common<T>
  332. {
  333. using atomic_common<T>::atomic_common;
  334. template<typename TMemoryOrder = memory_order_seq_cst_t>
  335. FORCE_INLINE T fetch_add(T value, TMemoryOrder order = memory_order_seq_cst)
  336. {
  337. return atomic_fetch_add_explicit(atomic_common<T>::obj, value, order);
  338. }
  339. template<typename TMemoryOrder = memory_order_seq_cst_t>
  340. FORCE_INLINE T fetch_sub(T value, TMemoryOrder order = memory_order_seq_cst)
  341. {
  342. return atomic_fetch_sub_explicit(atomic_common<T>::obj, value, order);
  343. }
  344. template<typename TMemoryOrder = memory_order_seq_cst_t>
  345. FORCE_INLINE T fetch_and(T value, TMemoryOrder order = memory_order_seq_cst)
  346. {
  347. return atomic_fetch_and_explicit(atomic_common<T>::obj, value, order);
  348. }
  349. template<typename TMemoryOrder = memory_order_seq_cst_t>
  350. FORCE_INLINE T fetch_or(T value, TMemoryOrder order = memory_order_seq_cst)
  351. {
  352. return atomic_fetch_or_explicit(atomic_common<T>::obj, value, order);
  353. }
  354. template<typename TMemoryOrder = memory_order_seq_cst_t>
  355. FORCE_INLINE T fetch_xor(T value, TMemoryOrder order = memory_order_seq_cst)
  356. {
  357. return atomic_fetch_xor_explicit(atomic_common<T>::obj, value, order);
  358. }
  359. FORCE_INLINE T operator++(int) { return atomic_fetch_add_explicit(atomic_common<T>::obj, T(1), memory_order_seq_cst); }
  360. FORCE_INLINE T operator--(int) { return atomic_fetch_sub_explicit(atomic_common<T>::obj, T(1), memory_order_seq_cst); }
  361. FORCE_INLINE T operator++() { return atomic_fetch_add_explicit(atomic_common<T>::obj, T(1), memory_order_seq_cst) + T(1); }
  362. FORCE_INLINE T operator--() { return atomic_fetch_sub_explicit(atomic_common<T>::obj, T(1), memory_order_seq_cst) - T(1); }
  363. FORCE_INLINE T operator+=(T value) { return atomic_fetch_add_explicit(atomic_common<T>::obj, value, memory_order_seq_cst) + value; }
  364. FORCE_INLINE T operator-=(T value) { return atomic_fetch_sub_explicit(atomic_common<T>::obj, value, memory_order_seq_cst) - value; }
  365. FORCE_INLINE T operator&=(T value) { return atomic_fetch_and_explicit(atomic_common<T>::obj, value, memory_order_seq_cst) & value; }
  366. FORCE_INLINE T operator|=(T value) { return atomic_fetch_or_explicit(atomic_common<T>::obj, value, memory_order_seq_cst) | value; }
  367. FORCE_INLINE T operator^=(T value) { return atomic_fetch_xor_explicit(atomic_common<T>::obj, value, memory_order_seq_cst) ^ value; }
  368. };
  369. // Atomic type for non-integral types.
  370. template<typename T>
  371. struct atomic_base<T, false> : atomic_common<T>
  372. {
  373. using atomic_common<T>::atomic_common;
  374. };
  375. template<typename T>
  376. struct atomic : atomic_base<T, std::is_integral<T>::value>
  377. {
  378. using atomic_base<T, std::is_integral<T>::value>::atomic_base;
  379. };
  380. #undef TEST_ATOMICS_PREREQUISITES
  381. }
  382. }