123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145 |
- #ifndef ATOMIC_X86_GCC_INCLUDED
- #define ATOMIC_X86_GCC_INCLUDED
- /* Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
- This program is free software; you can redistribute it and/or modify
- it under the terms of the GNU General Public License as published by
- the Free Software Foundation; version 2 of the License.
- This program is distributed in the hope that it will be useful,
- but WITHOUT ANY WARRANTY; without even the implied warranty of
- MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
- GNU General Public License for more details.
- You should have received a copy of the GNU General Public License
- along with this program; if not, write to the Free Software
- Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */
- /*
- XXX 64-bit atomic operations can be implemented using
- cmpxchg8b, if necessary. Though I've heard that not all 64-bit
- architectures support double-word (128-bit) cas.
- */
- /*
- No special support of 8 and 16 bit operations are implemented here
- currently.
- */
- #undef MY_ATOMIC_HAS_8_AND_16
- #ifdef __x86_64__
- # ifdef MY_ATOMIC_NO_XADD
- # define MY_ATOMIC_MODE "gcc-amd64" LOCK_prefix "-no-xadd"
- # else
- # define MY_ATOMIC_MODE "gcc-amd64" LOCK_prefix
- # endif
- #else
- # ifdef MY_ATOMIC_NO_XADD
- # define MY_ATOMIC_MODE "gcc-x86" LOCK_prefix "-no-xadd"
- # else
- # define MY_ATOMIC_MODE "gcc-x86" LOCK_prefix
- # endif
- #endif
- /* fix -ansi errors while maintaining readability */
- #ifndef asm
- #define asm __asm__
- #endif
- #ifndef MY_ATOMIC_NO_XADD
- #define make_atomic_add_body(S) make_atomic_add_body ## S
- #define make_atomic_cas_body(S) make_atomic_cas_body ## S
- #endif
- #define make_atomic_add_body32 \
- asm volatile (LOCK_prefix "; xadd %0, %1;" \
- : "+r" (v), "=m" (*a) \
- : "m" (*a) \
- : "memory")
- #define make_atomic_cas_body32 \
- __typeof__(*cmp) sav; \
- asm volatile (LOCK_prefix "; cmpxchg %3, %0; setz %2;" \
- : "=m" (*a), "=a" (sav), "=q" (ret) \
- : "r" (set), "m" (*a), "a" (*cmp) \
- : "memory"); \
- if (!ret) \
- *cmp= sav
- #ifdef __x86_64__
- #define make_atomic_add_body64 make_atomic_add_body32
- #define make_atomic_cas_body64 make_atomic_cas_body32
- #define make_atomic_fas_body(S) \
- asm volatile ("xchg %0, %1;" \
- : "+r" (v), "=m" (*a) \
- : "m" (*a) \
- : "memory")
- /*
- Actually 32/64-bit reads/writes are always atomic on x86_64,
- nonetheless issue memory barriers as appropriate.
- */
- #define make_atomic_load_body(S) \
- /* Serialize prior load and store operations. */ \
- asm volatile ("mfence" ::: "memory"); \
- ret= *a; \
- /* Prevent compiler from reordering instructions. */ \
- asm volatile ("" ::: "memory")
- #define make_atomic_store_body(S) \
- asm volatile ("; xchg %0, %1;" \
- : "=m" (*a), "+r" (v) \
- : "m" (*a) \
- : "memory")
- #else
- /*
- Use default implementations of 64-bit operations since we solved
- the 64-bit problem on 32-bit platforms for CAS, no need to solve it
- once more for ADD, LOAD, STORE and FAS as well.
- Since we already added add32 support, we need to define add64
- here, but we haven't defined fas, load and store at all, so
- we can fallback on default implementations.
- */
- #define make_atomic_add_body64 \
- int64 tmp=*a; \
- while (!my_atomic_cas64(a, &tmp, tmp+v)) ; \
- v=tmp;
- /*
- On some platforms (e.g. Mac OS X and Solaris) the ebx register
- is held as a pointer to the global offset table. Thus we're not
- allowed to use the b-register on those platforms when compiling
- PIC code, to avoid this we push ebx and pop ebx. The new value
- is copied directly from memory to avoid problems with a implicit
- manipulation of the stack pointer by the push.
- cmpxchg8b works on both 32-bit platforms and 64-bit platforms but
- the code here is only used on 32-bit platforms, on 64-bit
- platforms the much simpler make_atomic_cas_body32 will work
- fine.
- */
- #define make_atomic_cas_body64 \
- asm volatile ("push %%ebx;" \
- "movl (%%ecx), %%ebx;" \
- "movl 4(%%ecx), %%ecx;" \
- LOCK_prefix "; cmpxchg8b (%%esi);" \
- "setz %2; pop %%ebx" \
- : "+S" (a), "+A" (*cmp), "=c" (ret) \
- : "c" (&set) \
- : "memory", "esp")
- #endif
- /*
- The implementation of make_atomic_cas_body32 is adaptable to
- the OS word size, so on 64-bit platforms it will automatically
- adapt to 64-bits and so it will work also on 64-bit platforms
- */
- #define make_atomic_cas_bodyptr make_atomic_cas_body32
- #ifdef MY_ATOMIC_MODE_DUMMY
- #define make_atomic_load_body(S) ret=*a
- #define make_atomic_store_body(S) *a=v
- #endif
- #endif /* ATOMIC_X86_GCC_INCLUDED */
|