extra_ops_generic.hpp 17 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402
  1. /*
  2. * Distributed under the Boost Software License, Version 1.0.
  3. * (See accompanying file LICENSE_1_0.txt or copy at
  4. * http://www.boost.org/LICENSE_1_0.txt)
  5. *
  6. * Copyright (c) 2015 Andrey Semashev
  7. */
  8. /*!
  9. * \file atomic/detail/extra_ops_generic.hpp
  10. *
  11. * This header contains generic implementation of the extra atomic operations.
  12. */
  13. #ifndef BOOST_ATOMIC_DETAIL_EXTRA_OPS_GENERIC_HPP_INCLUDED_
  14. #define BOOST_ATOMIC_DETAIL_EXTRA_OPS_GENERIC_HPP_INCLUDED_
  15. #include <cstddef>
  16. #include <boost/memory_order.hpp>
  17. #include <boost/atomic/detail/config.hpp>
  18. #include <boost/atomic/detail/storage_type.hpp>
  19. #include <boost/atomic/detail/integral_extend.hpp>
  20. #include <boost/atomic/detail/extra_operations_fwd.hpp>
  21. #include <boost/atomic/capabilities.hpp>
  22. #ifdef BOOST_HAS_PRAGMA_ONCE
  23. #pragma once
  24. #endif
  25. #if defined(BOOST_MSVC)
  26. #pragma warning(push)
  27. // unary minus operator applied to unsigned type, result still unsigned
  28. #pragma warning(disable: 4146)
  29. #endif
  30. namespace boost {
  31. namespace atomics {
  32. namespace detail {
  33. //! Generic implementation of extra operations
  34. template< typename Base, std::size_t Size, bool Signed, bool = Base::full_cas_based >
  35. struct generic_extra_operations :
  36. public Base
  37. {
  38. typedef Base base_type;
  39. typedef typename base_type::storage_type storage_type;
  40. typedef typename make_storage_type< Size >::type emulated_storage_type;
  41. static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  42. {
  43. storage_type old_val;
  44. atomics::detail::non_atomic_load(storage, old_val);
  45. while (!base_type::compare_exchange_weak(storage, old_val, atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(-old_val)), order, memory_order_relaxed)) {}
  46. return old_val;
  47. }
  48. static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  49. {
  50. storage_type old_val, new_val;
  51. atomics::detail::non_atomic_load(storage, old_val);
  52. do
  53. {
  54. new_val = atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(-old_val));
  55. }
  56. while (!base_type::compare_exchange_weak(storage, old_val, new_val, order, memory_order_relaxed));
  57. return new_val;
  58. }
  59. static BOOST_FORCEINLINE storage_type add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  60. {
  61. return base_type::fetch_add(storage, v, order) + v;
  62. }
  63. static BOOST_FORCEINLINE storage_type sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  64. {
  65. return base_type::fetch_sub(storage, v, order) - v;
  66. }
  67. static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  68. {
  69. return base_type::fetch_and(storage, v, order) & v;
  70. }
  71. static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  72. {
  73. return base_type::fetch_or(storage, v, order) | v;
  74. }
  75. static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  76. {
  77. return base_type::fetch_xor(storage, v, order) ^ v;
  78. }
  79. static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  80. {
  81. return base_type::fetch_xor(storage, atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(~static_cast< emulated_storage_type >(0u))), order);
  82. }
  83. static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  84. {
  85. const storage_type mask = atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(~static_cast< emulated_storage_type >(0u)));
  86. return base_type::fetch_xor(storage, mask, order) ^ mask;
  87. }
  88. static BOOST_FORCEINLINE void opaque_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  89. {
  90. base_type::fetch_add(storage, v, order);
  91. }
  92. static BOOST_FORCEINLINE void opaque_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  93. {
  94. base_type::fetch_sub(storage, v, order);
  95. }
  96. static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  97. {
  98. fetch_negate(storage, order);
  99. }
  100. static BOOST_FORCEINLINE void opaque_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  101. {
  102. base_type::fetch_and(storage, v, order);
  103. }
  104. static BOOST_FORCEINLINE void opaque_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  105. {
  106. base_type::fetch_or(storage, v, order);
  107. }
  108. static BOOST_FORCEINLINE void opaque_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  109. {
  110. base_type::fetch_xor(storage, v, order);
  111. }
  112. static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  113. {
  114. fetch_complement(storage, order);
  115. }
  116. static BOOST_FORCEINLINE bool add_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  117. {
  118. return !!static_cast< emulated_storage_type >(add(storage, v, order));
  119. }
  120. static BOOST_FORCEINLINE bool sub_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  121. {
  122. return !!static_cast< emulated_storage_type >(sub(storage, v, order));
  123. }
  124. static BOOST_FORCEINLINE bool negate_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  125. {
  126. return !!negate(storage, order);
  127. }
  128. static BOOST_FORCEINLINE bool and_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  129. {
  130. return !!bitwise_and(storage, v, order);
  131. }
  132. static BOOST_FORCEINLINE bool or_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  133. {
  134. return !!bitwise_or(storage, v, order);
  135. }
  136. static BOOST_FORCEINLINE bool xor_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  137. {
  138. return !!bitwise_xor(storage, v, order);
  139. }
  140. static BOOST_FORCEINLINE bool complement_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  141. {
  142. return !!static_cast< emulated_storage_type >(bitwise_complement(storage, order));
  143. }
  144. static BOOST_FORCEINLINE bool bit_test_and_set(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT
  145. {
  146. const storage_type mask = atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(static_cast< emulated_storage_type >(1u) << bit_number));
  147. storage_type old_val = base_type::fetch_or(storage, mask, order);
  148. return !!(old_val & mask);
  149. }
  150. static BOOST_FORCEINLINE bool bit_test_and_reset(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT
  151. {
  152. const storage_type mask = atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(static_cast< emulated_storage_type >(1u) << bit_number));
  153. storage_type old_val = base_type::fetch_and(storage, ~mask, order);
  154. return !!(old_val & mask);
  155. }
  156. static BOOST_FORCEINLINE bool bit_test_and_complement(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT
  157. {
  158. const storage_type mask = atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(static_cast< emulated_storage_type >(1u) << bit_number));
  159. storage_type old_val = base_type::fetch_xor(storage, mask, order);
  160. return !!(old_val & mask);
  161. }
  162. };
  163. //! Specialization for cases when the platform only natively supports CAS
  164. template< typename Base, std::size_t Size, bool Signed >
  165. struct generic_extra_operations< Base, Size, Signed, true > :
  166. public Base
  167. {
  168. typedef Base base_type;
  169. typedef typename base_type::storage_type storage_type;
  170. typedef typename make_storage_type< Size >::type emulated_storage_type;
  171. static BOOST_FORCEINLINE storage_type fetch_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  172. {
  173. storage_type old_val;
  174. atomics::detail::non_atomic_load(storage, old_val);
  175. while (!base_type::compare_exchange_weak(storage, old_val, atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(-old_val)), order, memory_order_relaxed)) {}
  176. return old_val;
  177. }
  178. static BOOST_FORCEINLINE storage_type negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  179. {
  180. storage_type old_val, new_val;
  181. atomics::detail::non_atomic_load(storage, old_val);
  182. do
  183. {
  184. new_val = atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(-old_val));
  185. }
  186. while (!base_type::compare_exchange_weak(storage, old_val, new_val, order, memory_order_relaxed));
  187. return new_val;
  188. }
  189. static BOOST_FORCEINLINE storage_type add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  190. {
  191. storage_type old_val, new_val;
  192. atomics::detail::non_atomic_load(storage, old_val);
  193. do
  194. {
  195. new_val = atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(old_val + v));
  196. }
  197. while (!base_type::compare_exchange_weak(storage, old_val, new_val, order, memory_order_relaxed));
  198. return new_val;
  199. }
  200. static BOOST_FORCEINLINE storage_type sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  201. {
  202. storage_type old_val, new_val;
  203. atomics::detail::non_atomic_load(storage, old_val);
  204. do
  205. {
  206. new_val = atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(old_val - v));
  207. }
  208. while (!base_type::compare_exchange_weak(storage, old_val, new_val, order, memory_order_relaxed));
  209. return new_val;
  210. }
  211. static BOOST_FORCEINLINE storage_type bitwise_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  212. {
  213. storage_type old_val, new_val;
  214. atomics::detail::non_atomic_load(storage, old_val);
  215. do
  216. {
  217. new_val = atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(old_val & v));
  218. }
  219. while (!base_type::compare_exchange_weak(storage, old_val, new_val, order, memory_order_relaxed));
  220. return new_val;
  221. }
  222. static BOOST_FORCEINLINE storage_type bitwise_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  223. {
  224. storage_type old_val, new_val;
  225. atomics::detail::non_atomic_load(storage, old_val);
  226. do
  227. {
  228. new_val = atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(old_val | v));
  229. }
  230. while (!base_type::compare_exchange_weak(storage, old_val, new_val, order, memory_order_relaxed));
  231. return new_val;
  232. }
  233. static BOOST_FORCEINLINE storage_type bitwise_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  234. {
  235. storage_type old_val, new_val;
  236. atomics::detail::non_atomic_load(storage, old_val);
  237. do
  238. {
  239. new_val = atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(old_val ^ v));
  240. }
  241. while (!base_type::compare_exchange_weak(storage, old_val, new_val, order, memory_order_relaxed));
  242. return new_val;
  243. }
  244. static BOOST_FORCEINLINE storage_type fetch_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  245. {
  246. return base_type::fetch_xor(storage, atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(~static_cast< emulated_storage_type >(0u))), order);
  247. }
  248. static BOOST_FORCEINLINE storage_type bitwise_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  249. {
  250. return bitwise_xor(storage, atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(~static_cast< emulated_storage_type >(0u))), order);
  251. }
  252. static BOOST_FORCEINLINE void opaque_add(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  253. {
  254. base_type::fetch_add(storage, v, order);
  255. }
  256. static BOOST_FORCEINLINE void opaque_sub(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  257. {
  258. base_type::fetch_sub(storage, v, order);
  259. }
  260. static BOOST_FORCEINLINE void opaque_negate(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  261. {
  262. fetch_negate(storage, order);
  263. }
  264. static BOOST_FORCEINLINE void opaque_and(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  265. {
  266. base_type::fetch_and(storage, v, order);
  267. }
  268. static BOOST_FORCEINLINE void opaque_or(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  269. {
  270. base_type::fetch_or(storage, v, order);
  271. }
  272. static BOOST_FORCEINLINE void opaque_xor(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  273. {
  274. base_type::fetch_xor(storage, v, order);
  275. }
  276. static BOOST_FORCEINLINE void opaque_complement(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  277. {
  278. fetch_complement(storage, order);
  279. }
  280. static BOOST_FORCEINLINE bool add_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  281. {
  282. return !!static_cast< emulated_storage_type >(add(storage, v, order));
  283. }
  284. static BOOST_FORCEINLINE bool sub_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  285. {
  286. return !!static_cast< emulated_storage_type >(sub(storage, v, order));
  287. }
  288. static BOOST_FORCEINLINE bool negate_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  289. {
  290. return !!negate(storage, order);
  291. }
  292. static BOOST_FORCEINLINE bool and_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  293. {
  294. return !!bitwise_and(storage, v, order);
  295. }
  296. static BOOST_FORCEINLINE bool or_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  297. {
  298. return !!bitwise_or(storage, v, order);
  299. }
  300. static BOOST_FORCEINLINE bool xor_and_test(storage_type volatile& storage, storage_type v, memory_order order) BOOST_NOEXCEPT
  301. {
  302. return !!bitwise_xor(storage, v, order);
  303. }
  304. static BOOST_FORCEINLINE bool complement_and_test(storage_type volatile& storage, memory_order order) BOOST_NOEXCEPT
  305. {
  306. return !!static_cast< emulated_storage_type >(bitwise_complement(storage, order));
  307. }
  308. static BOOST_FORCEINLINE bool bit_test_and_set(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT
  309. {
  310. const storage_type mask = atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(static_cast< emulated_storage_type >(1u) << bit_number));
  311. storage_type old_val = base_type::fetch_or(storage, mask, order);
  312. return !!(old_val & mask);
  313. }
  314. static BOOST_FORCEINLINE bool bit_test_and_reset(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT
  315. {
  316. const storage_type mask = atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(static_cast< emulated_storage_type >(1u) << bit_number));
  317. storage_type old_val = base_type::fetch_and(storage, ~mask, order);
  318. return !!(old_val & mask);
  319. }
  320. static BOOST_FORCEINLINE bool bit_test_and_complement(storage_type volatile& storage, unsigned int bit_number, memory_order order) BOOST_NOEXCEPT
  321. {
  322. const storage_type mask = atomics::detail::integral_extend< Signed, storage_type >(static_cast< emulated_storage_type >(static_cast< emulated_storage_type >(1u) << bit_number));
  323. storage_type old_val = base_type::fetch_xor(storage, mask, order);
  324. return !!(old_val & mask);
  325. }
  326. };
  327. // Default extra_operations template definition will be used unless specialized for a specific platform
  328. template< typename Base, std::size_t Size, bool Signed >
  329. struct extra_operations< Base, Size, Signed, true > :
  330. public generic_extra_operations< Base, Size, Signed >
  331. {
  332. };
  333. } // namespace detail
  334. } // namespace atomics
  335. } // namespace boost
  336. #if defined(BOOST_MSVC)
  337. #pragma warning(pop)
  338. #endif
  339. #endif // BOOST_ATOMIC_DETAIL_EXTRA_OPS_GENERIC_HPP_INCLUDED_