From 7893248c5ab102f70f4dedd1d18e2531cc029e40 Mon Sep 17 00:00:00 2001 From: Nathan Hjelm Date: Thu, 30 Nov 2017 09:25:21 -0700 Subject: [PATCH] opal/asm: add fetch-and-op atomics This commit adds support for fetch-and-op atomics. This is needed because and and or are irreversible operations so there needs to be a way to get the old value atomically. These are also the only semantics supported by C11 (there is not atomic_op_fetch, just atomic_fetch_op). The old op-and-fetch atomics have been defined in terms of fetch-and-op. Signed-off-by: Nathan Hjelm --- ompi/mca/coll/sm/coll_sm.h | 2 +- ompi/mca/coll/sm/coll_sm_barrier.c | 2 +- ompi/mca/coll/sm/coll_sm_module.c | 2 +- ompi/mca/osc/sm/osc_sm_active_target.c | 8 +- opal/class/opal_list.c | 2 +- opal/include/opal/sys/arm/atomic.h | 40 ++--- opal/include/opal/sys/arm64/atomic.h | 16 +- opal/include/opal/sys/atomic.h | 145 ++++++++---------- opal/include/opal/sys/atomic_impl.h | 116 ++++++++------ opal/include/opal/sys/gcc_builtin/atomic.h | 40 ++--- opal/include/opal/sys/ia32/atomic.h | 8 +- opal/include/opal/sys/powerpc/atomic.h | 26 ++-- opal/include/opal/sys/sync_builtin/atomic.h | 40 ++--- opal/include/opal/sys/x86_64/atomic.h | 16 +- opal/mca/btl/vader/btl_vader_xpmem.c | 2 +- .../mpool/hugepage/mpool_hugepage_module.c | 4 +- opal/threads/thread_usage.h | 106 +++++++------ test/asm/atomic_cmpset.c | 4 +- test/asm/atomic_math.c | 142 ++++++++++++++++- test/threads/opal_thread.c | 4 +- 20 files changed, 441 insertions(+), 284 deletions(-) diff --git a/ompi/mca/coll/sm/coll_sm.h b/ompi/mca/coll/sm/coll_sm.h index eaff4518b1..b2da6ede42 100644 --- a/ompi/mca/coll/sm/coll_sm.h +++ b/ompi/mca/coll/sm/coll_sm.h @@ -358,7 +358,7 @@ extern uint32_t mca_coll_sm_one; * Macro to release an in-use flag from this process */ #define FLAG_RELEASE(flag) \ - (void)opal_atomic_add_fetch(&(flag)->mcsiuf_num_procs_using, -1) + opal_atomic_add(&(flag)->mcsiuf_num_procs_using, -1) /** * Macro to copy a single segment in from a user buffer to a shared diff --git a/ompi/mca/coll/sm/coll_sm_barrier.c b/ompi/mca/coll/sm/coll_sm_barrier.c index b29199271d..2722bbf09f 100644 --- a/ompi/mca/coll/sm/coll_sm_barrier.c +++ b/ompi/mca/coll/sm/coll_sm_barrier.c @@ -101,7 +101,7 @@ int mca_coll_sm_barrier_intra(struct ompi_communicator_t *comm, if (0 != rank) { /* Get parent *in* buffer */ parent = &data->mcb_barrier_control_parent[buffer_set]; - (void)opal_atomic_add_fetch(parent, 1); + opal_atomic_add (parent, 1); SPIN_CONDITION(0 != *me_out, exit_label2); *me_out = 0; diff --git a/ompi/mca/coll/sm/coll_sm_module.c b/ompi/mca/coll/sm/coll_sm_module.c index 88393bebf0..8922a70eaf 100644 --- a/ompi/mca/coll/sm/coll_sm_module.c +++ b/ompi/mca/coll/sm/coll_sm_module.c @@ -463,7 +463,7 @@ int ompi_coll_sm_lazy_enable(mca_coll_base_module_t *module, OBJ_RETAIN(sm_module->previous_reduce_module); /* Indicate that we have successfully attached and setup */ - (void)opal_atomic_add_fetch(&(data->sm_bootstrap_meta->module_seg->seg_inited), 1); + opal_atomic_add (&(data->sm_bootstrap_meta->module_seg->seg_inited), 1); /* Wait for everyone in this communicator to attach and setup */ opal_output_verbose(10, ompi_coll_base_framework.framework_output, diff --git a/ompi/mca/osc/sm/osc_sm_active_target.c b/ompi/mca/osc/sm/osc_sm_active_target.c index 31a6ee645e..ab0f73f87c 100644 --- a/ompi/mca/osc/sm/osc_sm_active_target.c +++ b/ompi/mca/osc/sm/osc_sm_active_target.c @@ -151,7 +151,7 @@ ompi_osc_sm_start(struct ompi_group_t *group, for (int i = 0 ; i < size ; ++i) { int rank_byte = ranks[i] >> OSC_SM_POST_BITS; - osc_sm_post_type_t old, rank_bit = ((osc_sm_post_type_t) 1) << (ranks[i] & 0x3f); + osc_sm_post_type_t rank_bit = ((osc_sm_post_type_t) 1) << (ranks[i] & 0x3f); /* wait for rank to post */ while (!(module->posts[my_rank][rank_byte] & rank_bit)) { @@ -162,9 +162,9 @@ ompi_osc_sm_start(struct ompi_group_t *group, opal_atomic_rmb (); #if OPAL_HAVE_ATOMIC_MATH_64 - opal_atomic_xor_fetch_64 ((volatile osc_sm_post_type_t *) module->posts[my_rank] + rank_byte, rank_bit); + (void) opal_atomic_fetch_xor_64 ((volatile int64_t *) module->posts[my_rank] + rank_byte, rank_bit); #else - opal_atomic_xor_fetch_32 ((volatile osc_sm_post_type_t *) module->posts[my_rank] + rank_byte, rank_bit); + (void) opal_atomic_fetch_xor_32 ((volatile int32_t *) module->posts[my_rank] + rank_byte, rank_bit); #endif } @@ -247,7 +247,7 @@ ompi_osc_sm_post(struct ompi_group_t *group, gsize = ompi_group_size(module->post_group); for (int i = 0 ; i < gsize ; ++i) { - (void) opal_atomic_add_fetch ((volatile osc_sm_post_type_t *) module->posts[ranks[i]] + my_byte, my_bit); + opal_atomic_add ((volatile osc_sm_post_type_t *) module->posts[ranks[i]] + my_byte, my_bit); } opal_atomic_wmb (); diff --git a/opal/class/opal_list.c b/opal/class/opal_list.c index dd0f654fd8..87cb1192b1 100644 --- a/opal/class/opal_list.c +++ b/opal/class/opal_list.c @@ -144,7 +144,7 @@ bool opal_list_insert(opal_list_t *list, opal_list_item_t *item, long long idx) /* Spot check: ensure this item is only on the list that we just insertted it into */ - (void)opal_atomic_add_fetch( &(item->opal_list_item_refcount), 1 ); + opal_atomic_add ( &(item->opal_list_item_refcount), 1 ); assert(1 == item->opal_list_item_refcount); item->opal_list_item_belong_to = list; #endif diff --git a/opal/include/opal/sys/arm/atomic.h b/opal/include/opal/sys/arm/atomic.h index 94576b6ddc..6d4db3ad7a 100644 --- a/opal/include/opal/sys/arm/atomic.h +++ b/opal/include/opal/sys/arm/atomic.h @@ -209,44 +209,44 @@ static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t #define OPAL_HAVE_ATOMIC_ADD_32 1 -static inline int32_t opal_atomic_add_fetch_32(volatile int32_t* v, int inc) +static inline int32_t opal_atomic_fetch_add_32(volatile int32_t* v, int inc) { - int32_t t; - int tmp; + int32_t t, old; + int tmp; - __asm__ __volatile__( - "1: ldrex %0, [%2] \n" - " add %0, %0, %3 \n" - " strex %1, %0, [%2] \n" - " cmp %1, #0 \n" + __asm__ __volatile__( + "1: ldrex %1, [%3] \n" + " add %0, %1, %4 \n" + " strex %2, %0, [%3] \n" + " cmp %2, #0 \n" " bne 1b \n" - : "=&r" (t), "=&r" (tmp) + : "=&r" (t), "=&r" (old), "=&r" (tmp) : "r" (v), "r" (inc) : "cc", "memory"); - return t; + return old; } #define OPAL_HAVE_ATOMIC_SUB_32 1 -static inline int32_t opal_atomic_sub_fetch_32(volatile int32_t* v, int dec) +static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t* v, int dec) { - int32_t t; - int tmp; + int32_t t, old; + int tmp; - __asm__ __volatile__( - "1: ldrex %0, [%2] \n" - " sub %0, %0, %3 \n" - " strex %1, %0, [%2] \n" - " cmp %1, #0 \n" + __asm__ __volatile__( + "1: ldrex %1, [%3] \n" + " sub %0, %1, %4 \n" + " strex %2, %0, [%3] \n" + " cmp %2, #0 \n" " bne 1b \n" - : "=&r" (t), "=&r" (tmp) + : "=&r" (t), "=&r" (old), "=&r" (tmp) : "r" (v), "r" (dec) : "cc", "memory"); - return t; + return t; } #endif diff --git a/opal/include/opal/sys/arm64/atomic.h b/opal/include/opal/sys/arm64/atomic.h index 8cc9313e14..fd5a773a4f 100644 --- a/opal/include/opal/sys/arm64/atomic.h +++ b/opal/include/opal/sys/arm64/atomic.h @@ -293,20 +293,20 @@ static inline int opal_atomic_sc_64 (volatile int64_t *addr, int64_t newval) } #define OPAL_ASM_MAKE_ATOMIC(type, bits, name, inst, reg) \ - static inline type opal_atomic_ ## name ## _fetch_ ## bits (volatile type *addr, type value) \ + static inline type opal_atomic_fetch_ ## name ## _ ## bits (volatile type *addr, type value) \ { \ - type newval; \ + type newval, old; \ int32_t tmp; \ \ - __asm__ __volatile__("1: ldxr %" reg "0, [%2] \n" \ - " " inst " %" reg "0, %" reg "0, %" reg "3 \n" \ - " stxr %w1, %" reg "0, [%2] \n" \ - " cbnz %w1, 1b \n" \ - : "=&r" (newval), "=&r" (tmp) \ + __asm__ __volatile__("1: ldxr %" reg "1, [%3] \n" \ + " " inst " %" reg "0, %" reg "1, %" reg "4 \n" \ + " stxr %w2, %" reg "0, [%3] \n" \ + " cbnz %w2, 1b \n" \ + : "=&r" (newval), "=&r" (old), "=&r" (tmp) \ : "r" (addr), "r" (value) \ : "cc", "memory"); \ \ - return newval; \ + return old; \ } OPAL_ASM_MAKE_ATOMIC(int32_t, 32, add, "add", "w") diff --git a/opal/include/opal/sys/atomic.h b/opal/include/opal/sys/atomic.h index 2a64308f81..53e34333d8 100644 --- a/opal/include/opal/sys/atomic.h +++ b/opal/include/opal/sys/atomic.h @@ -399,36 +399,16 @@ bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t #if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 -/* OPAL_HAVE_INLINE_ATOMIC_*_32 will be 1 if /atomic.h provides - a static inline version of it (in assembly). If we have to fall - back on compare-exchange 32, that too will be inline. */ -#if OPAL_HAVE_INLINE_ATOMIC_ADD_32 || (!defined(OPAL_HAVE_ATOMIC_ADD_32) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32) -static inline -#endif -int32_t opal_atomic_add_fetch_32(volatile int32_t *addr, int delta); - -#if OPAL_HAVE_INLINE_ATOMIC_AND_32 || (!defined(OPAL_HAVE_ATOMIC_AND_32) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32) -static inline -#endif -int32_t opal_atomic_and_fetch_32(volatile int32_t *addr, int32_t value); - -#if OPAL_HAVE_INLINE_ATOMIC_OR_32 || (!defined(OPAL_HAVE_ATOMIC_OR_32) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32) -static inline -#endif -int32_t opal_atomic_or_fetch_32(volatile int32_t *addr, int32_t value); - -#if OPAL_HAVE_INLINE_ATOMIC_XOR_32 || (!defined(OPAL_HAVE_ATOMIC_XOR_32) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32) -static inline -#endif -int32_t opal_atomic_xor_fetch_32(volatile int32_t *addr, int32_t value); - -/* OPAL_HAVE_INLINE_ATOMIC_*_32 will be 1 if /atomic.h provides - a static inline version of it (in assembly). If we have to fall - back to compare-exchange 32, that too will be inline. */ -#if OPAL_HAVE_INLINE_ATOMIC_SUB_32 || (!defined(OPAL_HAVE_ATOMIC_ADD_32) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32) -static inline -#endif -int32_t opal_atomic_sub_fetch_32(volatile int32_t *addr, int delta); +static inline int32_t opal_atomic_add_fetch_32(volatile int32_t *addr, int delta); +static inline int32_t opal_atomic_fetch_add_32(volatile int32_t *addr, int delta); +static inline int32_t opal_atomic_and_fetch_32(volatile int32_t *addr, int32_t value); +static inline int32_t opal_atomic_fetch_and_32(volatile int32_t *addr, int32_t value); +static inline int32_t opal_atomic_or_fetch_32(volatile int32_t *addr, int32_t value); +static inline int32_t opal_atomic_fetch_or_32(volatile int32_t *addr, int32_t value); +static inline int32_t opal_atomic_xor_fetch_32(volatile int32_t *addr, int32_t value); +static inline int32_t opal_atomic_fetch_xor_32(volatile int32_t *addr, int32_t value); +static inline int32_t opal_atomic_sub_fetch_32(volatile int32_t *addr, int delta); +static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t *addr, int delta); #endif /* OPAL_HAVE_ATOMIC_MATH_32 */ @@ -445,36 +425,15 @@ int32_t opal_atomic_sub_fetch_32(volatile int32_t *addr, int delta); #if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MATH_64 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 -/* OPAL_HAVE_INLINE_ATOMIC_*_64 will be 1 if /atomic.h provides - a static inline version of it (in assembly). If we have to fall - back to compare-exchange 64, that too will be inline */ -#if OPAL_HAVE_INLINE_ATOMIC_ADD_64 || (!defined(OPAL_HAVE_ATOMIC_ADD_64) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64) -static inline -#endif -int64_t opal_atomic_add_fetch_64(volatile int64_t *addr, int64_t delta); - -#if OPAL_HAVE_INLINE_ATOMIC_AND_64 || (!defined(OPAL_HAVE_ATOMIC_AND_64) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64) -static inline -#endif -int64_t opal_atomic_and_fetch_64(volatile int64_t *addr, int64_t value); - -#if OPAL_HAVE_INLINE_ATOMIC_OR_64 || (!defined(OPAL_HAVE_ATOMIC_OR_64) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64) -static inline -#endif -int64_t opal_atomic_or_fetch_64(volatile int64_t *addr, int64_t value); - -#if OPAL_HAVE_INLINE_ATOMIC_XOR_64 || (!defined(OPAL_HAVE_ATOMIC_XOR_64) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64) -static inline -#endif -int64_t opal_atomic_xor_fetch_64(volatile int64_t *addr, int64_t value); - -/* OPAL_HAVE_INLINE_ATOMIC_*_64 will be 1 if /atomic.h provides - a static inline version of it (in assembly). If we have to fall - back to compare-exchange 64, that too will be inline */ -#if OPAL_HAVE_INLINE_ATOMIC_SUB_64 || (!defined(OPAL_HAVE_ATOMIC_ADD_64) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64) -static inline -#endif -int64_t opal_atomic_sub_fetch_64(volatile int64_t *addr, int64_t delta); +static inline int64_t opal_atomic_add_fetch_64(volatile int64_t *addr, int64_t delta); +static inline int64_t opal_atomic_fetch_add_64(volatile int64_t *addr, int64_t delta); +static inline int64_t opal_atomic_and_fetch_64(volatile int64_t *addr, int64_t value); +static inline int64_t opal_atomic_fetch_and_64(volatile int64_t *addr, int64_t value); +static inline int64_t opal_atomic_or_fetch_64(volatile int64_t *addr, int64_t value); +static inline int64_t opal_atomic_fetch_or_64(volatile int64_t *addr, int64_t value); +static inline int64_t opal_atomic_fetch_xor_64(volatile int64_t *addr, int64_t value); +static inline int64_t opal_atomic_sub_fetch_64(volatile int64_t *addr, int64_t delta); +static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t *addr, int64_t delta); #endif /* OPAL_HAVE_ATOMIC_MATH_32 */ @@ -501,6 +460,19 @@ opal_atomic_add_fetch_size_t(volatile size_t *addr, size_t delta) #error "Unknown size_t size" #endif } + +static inline size_t +opal_atomic_fetch_add_size_t(volatile size_t *addr, size_t delta) +{ +#if SIZEOF_SIZE_T == 4 + return (size_t) opal_atomic_fetch_add_32((int32_t*) addr, delta); +#elif SIZEOF_SIZE_T == 8 + return (size_t) opal_atomic_fetch_add_64((int64_t*) addr, delta); +#else +#error "Unknown size_t size" +#endif +} + static inline size_t opal_atomic_sub_fetch_size_t(volatile size_t *addr, size_t delta) { @@ -512,13 +484,30 @@ opal_atomic_sub_fetch_size_t(volatile size_t *addr, size_t delta) #error "Unknown size_t size" #endif } + +static inline size_t +opal_atomic_fetch_sub_size_t(volatile size_t *addr, size_t delta) +{ +#if SIZEOF_SIZE_T == 4 + return (size_t) opal_atomic_fetch_sub_32((int32_t*) addr, delta); +#elif SIZEOF_SIZE_T == 8 + return (size_t) opal_atomic_fetch_sub_64((int64_t*) addr, delta); +#else +#error "Unknown size_t size" +#endif +} + #else #if SIZEOF_SIZE_T == 4 -#define opal_atomic_add_fetch_size_t(addr, delta) ((size_t) opal_atomic_add_fetch_32((int32_t*) addr, delta)) -#define opal_atomic_sub_fetch_size_t(addr, delta) ((size_t) opal_atomic_sub_fetch_32((int32_t*) addr, delta)) -#elif SIZEOF_SIZE_T ==8 -#define opal_atomic_add_fetch_size_t(addr, delta) ((size_t) opal_atomic_add_fetch_64((int64_t*) addr, delta)) -#define opal_atomic_sub_fetch_size_t(addr, delta) ((size_t) opal_atomic_sub_fetch_64((int64_t*) addr, delta)) +#define opal_atomic_add_fetch_size_t(addr, delta) ((size_t) opal_atomic_add_fetch_32((volatile int32_t *) addr, delta)) +#define opal_atomic_fetch_add_size_t(addr, delta) ((size_t) opal_atomic_fetch_add_32((volatile int32_t *) addr, delta)) +#define opal_atomic_sub_fetch_size_t(addr, delta) ((size_t) opal_atomic_sub_fetch_32((volatile int32_t *) addr, delta)) +#define opal_atomic_fetch_sub_size_t(addr, delta) ((size_t) opal_atomic_fetch_sub_32((volatile int32_t *) addr, delta)) +#elif SIZEOF_SIZE_T == 8 +#define opal_atomic_add_fetch_size_t(addr, delta) ((size_t) opal_atomic_add_fetch_64((volatile int64_t *) addr, delta)) +#define opal_atomic_fetch_add_size_t(addr, delta) ((size_t) opal_atomic_fetch_add_64((volatile int64_t *) addr, delta)) +#define opal_atomic_sub_fetch_size_t(addr, delta) ((size_t) opal_atomic_sub_fetch_64((volatile int64_t *) addr, delta)) +#define opal_atomic_fetch_sub_size_t(addr, delta) ((size_t) opal_atomic_fetch_sub_64((volatile int64_t *) addr, delta)) #else #error "Unknown size_t size" #endif @@ -599,19 +588,15 @@ static inline bool opal_atomic_compare_exchange_strong_rel_ptr (volatile void* a #if defined(DOXYGEN) || (OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64) -static inline void opal_atomic_add_fetch_xx(volatile void* addr, +static inline void opal_atomic_add_xx(volatile void* addr, int32_t value, size_t length); -static inline void opal_atomic_sub_fetch_xx(volatile void* addr, +static inline void opal_atomic_sub_xx(volatile void* addr, int32_t value, size_t length); -#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 -static inline int32_t opal_atomic_add_fetch_ptr( volatile void* addr, void* delta ); -static inline int32_t opal_atomic_sub_fetch_ptr( volatile void* addr, void* delta ); -#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 -static inline int64_t opal_atomic_add_fetch_ptr( volatile void* addr, void* delta ); -static inline int64_t opal_atomic_sub_fetch_ptr( volatile void* addr, void* delta ); -#else -#error Atomic arithmetic on pointers not supported -#endif + +static inline intptr_t opal_atomic_add_fetch_ptr( volatile void* addr, void* delta ); +static inline intptr_t opal_atomic_fetch_add_ptr( volatile void* addr, void* delta ); +static inline intptr_t opal_atomic_sub_fetch_ptr( volatile void* addr, void* delta ); +static inline intptr_t opal_atomic_fetch_sub_ptr( volatile void* addr, void* delta ); /** * Atomically increment the content depending on the type. This @@ -623,8 +608,8 @@ static inline int64_t opal_atomic_sub_fetch_ptr( volatile void* addr, void* delt * @param addr Address of * @param delta Value to add (converted to ). */ -#define opal_atomic_add_fetch( ADDR, VALUE ) \ - opal_atomic_add_fetch_xx( (volatile void*)(ADDR), (int32_t)(VALUE), \ +#define opal_atomic_add( ADDR, VALUE ) \ + opal_atomic_add_xx( (volatile void*)(ADDR), (int32_t)(VALUE), \ sizeof(*(ADDR)) ) /** @@ -637,8 +622,8 @@ static inline int64_t opal_atomic_sub_fetch_ptr( volatile void* addr, void* delt * @param addr Address of * @param delta Value to substract (converted to ). */ -#define opal_atomic_sub_fetch( ADDR, VALUE ) \ - opal_atomic_sub_fetch_xx( (volatile void*)(ADDR), (int32_t)(VALUE), \ +#define opal_atomic_sub( ADDR, VALUE ) \ + opal_atomic_sub_xx( (volatile void*)(ADDR), (int32_t)(VALUE), \ sizeof(*(ADDR)) ) #endif /* OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64 */ diff --git a/opal/include/opal/sys/atomic_impl.h b/opal/include/opal/sys/atomic_impl.h index 8c54a62b62..b3aba9af66 100644 --- a/opal/include/opal/sys/atomic_impl.h +++ b/opal/include/opal/sys/atomic_impl.h @@ -39,16 +39,15 @@ *********************************************************************/ #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 -#define OPAL_ATOMIC_DEFINE_CMPXCG_OP(type, bits, operand, name) \ - static inline type opal_atomic_ ## name ## _fetch_ ## bits (volatile type *addr, type value) \ +#define OPAL_ATOMIC_DEFINE_CMPXCG_OP(type, bits, operation, name) \ + static inline type opal_atomic_fetch_ ## name ## _ ## bits (volatile type *addr, type value) \ { \ - type oldval, newval; \ + type oldval; \ do { \ oldval = *addr; \ - newval = oldval operand value; \ - } while (!opal_atomic_compare_exchange_strong_ ## bits (addr, &oldval, newval)); \ + } while (!opal_atomic_compare_exchange_strong_ ## bits (addr, &oldval, oldval operation value)); \ \ - return newval; \ + return oldval; \ } #if !defined(OPAL_HAVE_ATOMIC_SWAP_32) @@ -264,20 +263,19 @@ OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(_rel_) #if OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64 - static inline void -opal_atomic_add_fetch_xx(volatile void* addr, int32_t value, size_t length) + opal_atomic_add_xx(volatile void* addr, int32_t value, size_t length) { switch( length ) { #if OPAL_HAVE_ATOMIC_ADD_32 case 4: - opal_atomic_add_fetch_32( (volatile int32_t*)addr, (int32_t)value ); + (void) opal_atomic_fetch_add_32( (volatile int32_t*)addr, (int32_t)value ); break; #endif /* OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 */ #if OPAL_HAVE_ATOMIC_ADD_64 case 8: - opal_atomic_add_fetch_64( (volatile int64_t*)addr, (int64_t)value ); + (void) opal_atomic_fetch_add_64( (volatile int64_t*)addr, (int64_t)value ); break; #endif /* OPAL_HAVE_ATOMIC_ADD_64 */ default: @@ -289,18 +287,18 @@ opal_atomic_add_fetch_xx(volatile void* addr, int32_t value, size_t length) static inline void -opal_atomic_sub_fetch_xx(volatile void* addr, int32_t value, size_t length) +opal_atomic_sub_xx(volatile void* addr, int32_t value, size_t length) { switch( length ) { #if OPAL_HAVE_ATOMIC_SUB_32 case 4: - opal_atomic_sub_fetch_32( (volatile int32_t*)addr, (int32_t)value ); + (void) opal_atomic_fetch_sub_32( (volatile int32_t*)addr, (int32_t)value ); break; #endif /* OPAL_HAVE_ATOMIC_SUB_32 */ #if OPAL_HAVE_ATOMIC_SUB_64 case 8: - opal_atomic_sub_fetch_64( (volatile int64_t*)addr, (int64_t)value ); + (void) opal_atomic_fetch_sub_64( (volatile int64_t*)addr, (int64_t)value ); break; #endif /* OPAL_HAVE_ATOMIC_SUB_64 */ default: @@ -310,47 +308,77 @@ opal_atomic_sub_fetch_xx(volatile void* addr, int32_t value, size_t length) } } -#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_ADD_32 -static inline int32_t opal_atomic_add_fetch_ptr( volatile void* addr, - void* delta ) -{ - return opal_atomic_add_fetch_32((int32_t*) addr, (unsigned long) delta); -} -#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_ADD_64 -static inline int64_t opal_atomic_add_fetch_ptr( volatile void* addr, - void* delta ) -{ - return opal_atomic_add_fetch_64((int64_t*) addr, (unsigned long) delta); -} -#else -static inline int32_t opal_atomic_add_fetch_ptr( volatile void* addr, - void* delta ) -{ - abort(); - return 0; -} +#define OPAL_ATOMIC_DEFINE_OP_FETCH(op, operation, type, ptr_type, suffix) \ + static inline type opal_atomic_ ## op ## _fetch_ ## suffix (volatile ptr_type *addr, type value) \ + { \ + return opal_atomic_fetch_ ## op ## _ ## suffix (addr, value) operation value; \ + } + +OPAL_ATOMIC_DEFINE_OP_FETCH(add, +, int32_t, int32_t, 32) +OPAL_ATOMIC_DEFINE_OP_FETCH(and, &, int32_t, int32_t, 32) +OPAL_ATOMIC_DEFINE_OP_FETCH(or, |, int32_t, int32_t, 32) +OPAL_ATOMIC_DEFINE_OP_FETCH(xor, ^, int32_t, int32_t, 32) +OPAL_ATOMIC_DEFINE_OP_FETCH(sub, -, int32_t, int32_t, 32) + +#if OPAL_HAVE_ATOMIC_MATH_64 +OPAL_ATOMIC_DEFINE_OP_FETCH(add, +, int64_t, int64_t, 64) +OPAL_ATOMIC_DEFINE_OP_FETCH(and, &, int64_t, int64_t, 64) +OPAL_ATOMIC_DEFINE_OP_FETCH(or, |, int64_t, int64_t, 64) +OPAL_ATOMIC_DEFINE_OP_FETCH(xor, ^, int64_t, int64_t, 64) +OPAL_ATOMIC_DEFINE_OP_FETCH(sub, -, int64_t, int64_t, 64) #endif -#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_SUB_32 -static inline int32_t opal_atomic_sub_fetch_ptr( volatile void* addr, +static inline intptr_t opal_atomic_fetch_add_ptr( volatile void* addr, void* delta ) { - return opal_atomic_sub_fetch_32((int32_t*) addr, (unsigned long) delta); -} -#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_SUB_32 -static inline int64_t opal_atomic_sub_fetch_ptr( volatile void* addr, - void* delta ) -{ - return opal_atomic_sub_fetch_64((int64_t*) addr, (unsigned long) delta); -} +#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_ADD_32 + return opal_atomic_fetch_add_32((int32_t*) addr, (unsigned long) delta); +#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_ADD_64 + return opal_atomic_fetch_add_64((int64_t*) addr, (unsigned long) delta); #else -static inline int32_t opal_atomic_sub_fetch_ptr( volatile void* addr, + abort (); + return 0; +#endif +} + +static inline intptr_t opal_atomic_add_fetch_ptr( volatile void* addr, void* delta ) { +#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_ADD_32 + return opal_atomic_add_fetch_32((int32_t*) addr, (unsigned long) delta); +#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_ADD_64 + return opal_atomic_add_fetch_64((int64_t*) addr, (unsigned long) delta); +#else + abort (); + return 0; +#endif +} + +static inline intptr_t opal_atomic_fetch_sub_ptr( volatile void* addr, + void* delta ) +{ +#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_SUB_32 + return opal_atomic_fetch_sub_32((int32_t*) addr, (unsigned long) delta); +#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_SUB_32 + return opal_atomic_fetch_sub_64((int64_t*) addr, (unsigned long) delta); +#else abort(); return 0; -} #endif +} + +static inline intptr_t opal_atomic_sub_fetch_ptr( volatile void* addr, + void* delta ) +{ +#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_SUB_32 + return opal_atomic_sub_fetch_32((int32_t*) addr, (unsigned long) delta); +#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_SUB_32 + return opal_atomic_sub_fetch_64((int64_t*) addr, (unsigned long) delta); +#else + abort(); + return 0; +#endif +} #endif /* OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64 */ diff --git a/opal/include/opal/sys/gcc_builtin/atomic.h b/opal/include/opal/sys/gcc_builtin/atomic.h index ecef65f3d0..c6ef6eb9c3 100644 --- a/opal/include/opal/sys/gcc_builtin/atomic.h +++ b/opal/include/opal/sys/gcc_builtin/atomic.h @@ -104,29 +104,29 @@ static inline int32_t opal_atomic_swap_32 (volatile int32_t *addr, int32_t newva return oldval; } -static inline int32_t opal_atomic_add_fetch_32(volatile int32_t *addr, int32_t delta) +static inline int32_t opal_atomic_fetch_add_32(volatile int32_t *addr, int32_t delta) { - return __atomic_add_fetch (addr, delta, __ATOMIC_RELAXED); + return __atomic_fetch_add (addr, delta, __ATOMIC_RELAXED); } -static inline int32_t opal_atomic_and_fetch_32(volatile int32_t *addr, int32_t value) +static inline int32_t opal_atomic_fetch_and_32(volatile int32_t *addr, int32_t value) { - return __atomic_and_fetch (addr, value, __ATOMIC_RELAXED); + return __atomic_fetch_and (addr, value, __ATOMIC_RELAXED); } -static inline int32_t opal_atomic_or_fetch_32(volatile int32_t *addr, int32_t value) +static inline int32_t opal_atomic_fetch_or_32(volatile int32_t *addr, int32_t value) { - return __atomic_or_fetch (addr, value, __ATOMIC_RELAXED); + return __atomic_fetch_or (addr, value, __ATOMIC_RELAXED); } -static inline int32_t opal_atomic_xor_fetch_32(volatile int32_t *addr, int32_t value) +static inline int32_t opal_atomic_fetch_xor_32(volatile int32_t *addr, int32_t value) { - return __atomic_xor_fetch (addr, value, __ATOMIC_RELAXED); + return __atomic_fetch_xor (addr, value, __ATOMIC_RELAXED); } -static inline int32_t opal_atomic_sub_fetch_32(volatile int32_t *addr, int32_t delta) +static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t *addr, int32_t delta) { - return __atomic_sub_fetch (addr, delta, __ATOMIC_RELAXED); + return __atomic_fetch_sub (addr, delta, __ATOMIC_RELAXED); } static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) @@ -152,29 +152,29 @@ static inline int64_t opal_atomic_swap_64 (volatile int64_t *addr, int64_t newva return oldval; } -static inline int64_t opal_atomic_add_fetch_64(volatile int64_t *addr, int64_t delta) +static inline int64_t opal_atomic_fetch_add_64(volatile int64_t *addr, int64_t delta) { - return __atomic_add_fetch (addr, delta, __ATOMIC_RELAXED); + return __atomic_fetch_add (addr, delta, __ATOMIC_RELAXED); } -static inline int64_t opal_atomic_and_fetch_64(volatile int64_t *addr, int64_t value) +static inline int64_t opal_atomic_fetch_and_64(volatile int64_t *addr, int64_t value) { - return __atomic_and_fetch (addr, value, __ATOMIC_RELAXED); + return __atomic_fetch_and (addr, value, __ATOMIC_RELAXED); } -static inline int64_t opal_atomic_or_fetch_64(volatile int64_t *addr, int64_t value) +static inline int64_t opal_atomic_fetch_or_64(volatile int64_t *addr, int64_t value) { - return __atomic_or_fetch (addr, value, __ATOMIC_RELAXED); + return __atomic_fetch_or (addr, value, __ATOMIC_RELAXED); } -static inline int64_t opal_atomic_xor_fetch_64(volatile int64_t *addr, int64_t value) +static inline int64_t opal_atomic_fetch_xor_64(volatile int64_t *addr, int64_t value) { - return __atomic_xor_fetch (addr, value, __ATOMIC_RELAXED); + return __atomic_fetch_xor (addr, value, __ATOMIC_RELAXED); } -static inline int64_t opal_atomic_sub_fetch_64(volatile int64_t *addr, int64_t delta) +static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t *addr, int64_t delta) { - return __atomic_sub_fetch (addr, delta, __ATOMIC_RELAXED); + return __atomic_fetch_sub (addr, delta, __ATOMIC_RELAXED); } #if OPAL_HAVE_GCC_BUILTIN_CSWAP_INT128 diff --git a/opal/include/opal/sys/ia32/atomic.h b/opal/include/opal/sys/ia32/atomic.h index 6b71dd4f2e..bb863dec14 100644 --- a/opal/include/opal/sys/ia32/atomic.h +++ b/opal/include/opal/sys/ia32/atomic.h @@ -130,7 +130,7 @@ static inline int32_t opal_atomic_swap_32( volatile int32_t *addr, * * Atomically adds @i to @v. */ -static inline int32_t opal_atomic_add_fetch_32(volatile int32_t* v, int i) +static inline int32_t opal_atomic_fetch_add_32(volatile int32_t* v, int i) { int ret = i; __asm__ __volatile__( @@ -139,7 +139,7 @@ static inline int32_t opal_atomic_add_fetch_32(volatile int32_t* v, int i) : :"memory", "cc" ); - return (ret+i); + return ret; } @@ -150,7 +150,7 @@ static inline int32_t opal_atomic_add_fetch_32(volatile int32_t* v, int i) * * Atomically subtracts @i from @v. */ -static inline int32_t opal_atomic_sub_fetch_32(volatile int32_t* v, int i) +static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t* v, int i) { int ret = -i; __asm__ __volatile__( @@ -159,7 +159,7 @@ static inline int32_t opal_atomic_sub_fetch_32(volatile int32_t* v, int i) : :"memory", "cc" ); - return (ret-i); + return ret; } #endif /* OPAL_GCC_INLINE_ASSEMBLY */ diff --git a/opal/include/opal/sys/powerpc/atomic.h b/opal/include/opal/sys/powerpc/atomic.h index 4aebb43fdd..bf6978aa85 100644 --- a/opal/include/opal/sys/powerpc/atomic.h +++ b/opal/include/opal/sys/powerpc/atomic.h @@ -235,20 +235,20 @@ static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, int32_t newval #if OPAL_GCC_INLINE_ASSEMBLY #define OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(type, instr) \ -static inline int64_t opal_atomic_ ## type ## _fetch_64(volatile int64_t* v, int64_t val) \ +static inline int64_t opal_atomic_fetch_ ## type ## _64(volatile int64_t* v, int64_t val) \ { \ - int64_t t; \ + int64_t t, old; \ \ __asm__ __volatile__( \ - "1: ldarx %0, 0, %3 \n\t" \ - " " #instr " %0, %2, %0 \n\t" \ - " stdcx. %0, 0, %3 \n\t" \ + "1: ldarx %1, 0, %4 \n\t" \ + " " #instr " %0, %3, %1 \n\t" \ + " stdcx. %0, 0, %4 \n\t" \ " bne- 1b \n\t" \ - : "=&r" (t), "=m" (*v) \ + : "=&r" (t), "=&r" (old), "=m" (*v) \ : "r" (OPAL_ASM_VALUE64(val)), "r" OPAL_ASM_ADDR(v), "m" (*v) \ : "cc"); \ \ - return t; \ + return old; \ } OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(add, add) @@ -396,16 +396,16 @@ static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t #define OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_32(type, instr) \ -static inline int32_t opal_atomic_ ## type ## _fetch_32(volatile int32_t* v, int val) \ +static inline int32_t opal_atomic_fetch_ ## type ## _32(volatile int32_t* v, int val) \ { \ - int32_t t; \ + int32_t t, old; \ \ __asm__ __volatile__( \ - "1: lwarx %0, 0, %3 \n\t" \ - " " #instr " %0, %2, %0 \n\t" \ - " stwcx. %0, 0, %3 \n\t" \ + "1: lwarx %1, 0, %4 \n\t" \ + " " #instr " %0, %3, %1 \n\t" \ + " stwcx. %0, 0, %4 \n\t" \ " bne- 1b \n\t" \ - : "=&r" (t), "=m" (*v) \ + : "=&r" (t), "=&r" (old), "=m" (*v) \ : "r" (val), "r" OPAL_ASM_ADDR(v), "m" (*v) \ : "cc"); \ \ diff --git a/opal/include/opal/sys/sync_builtin/atomic.h b/opal/include/opal/sys/sync_builtin/atomic.h index f80a29684c..4a6cfbfbe0 100644 --- a/opal/include/opal/sys/sync_builtin/atomic.h +++ b/opal/include/opal/sys/sync_builtin/atomic.h @@ -69,33 +69,33 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add #define OPAL_HAVE_ATOMIC_MATH_32 1 #define OPAL_HAVE_ATOMIC_ADD_32 1 -static inline int32_t opal_atomic_add_fetch_32(volatile int32_t *addr, int32_t delta) +static inline int32_t opal_atomic_fetch_add_32(volatile int32_t *addr, int32_t delta) { - return __sync_add_and_fetch(addr, delta); + return __sync_fetch_and_add(addr, delta); } #define OPAL_HAVE_ATOMIC_AND_32 1 -static inline int32_t opal_atomic_and_fetch_32(volatile int32_t *addr, int32_t value) +static inline int32_t opal_atomic_fetch_and_32(volatile int32_t *addr, int32_t value) { - return __sync_and_and_fetch(addr, value); + return __sync_fetch_and_and(addr, value); } #define OPAL_HAVE_ATOMIC_OR_32 1 -static inline int32_t opal_atomic_or_fetch_32(volatile int32_t *addr, int32_t value) +static inline int32_t opal_atomic_fetch_or_32(volatile int32_t *addr, int32_t value) { - return __sync_or_and_fetch(addr, value); + return __sync_fetch_and_or(addr, value); } #define OPAL_HAVE_ATOMIC_XOR_32 1 -static inline int32_t opal_atomic_xor_fetch_32(volatile int32_t *addr, int32_t value) +static inline int32_t opal_atomic_fetch_xor_32(volatile int32_t *addr, int32_t value) { - return __sync_xor_and_fetch(addr, value); + return __sync_fetch_and_xor(addr, value); } #define OPAL_HAVE_ATOMIC_SUB_32 1 -static inline int32_t opal_atomic_sub_fetch_32(volatile int32_t *addr, int32_t delta) +static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t *addr, int32_t delta) { - return __sync_sub_and_fetch(addr, delta); + return __sync_fetch_and_sub(addr, delta); } #if OPAL_ASM_SYNC_HAVE_64BIT @@ -115,33 +115,33 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add #define OPAL_HAVE_ATOMIC_MATH_64 1 #define OPAL_HAVE_ATOMIC_ADD_64 1 -static inline int64_t opal_atomic_add_fetch_64(volatile int64_t *addr, int64_t delta) +static inline int64_t opal_atomic_fetch_add_64(volatile int64_t *addr, int64_t delta) { - return __sync_add_and_fetch(addr, delta); + return __sync_fetch_and_add(addr, delta); } #define OPAL_HAVE_ATOMIC_AND_64 1 -static inline int64_t opal_atomic_and_fetch_64(volatile int64_t *addr, int64_t value) +static inline int64_t opal_atomic_fetch_and_64(volatile int64_t *addr, int64_t value) { - return __sync_and_and_fetch(addr, value); + return __sync_fetch_and_and(addr, value); } #define OPAL_HAVE_ATOMIC_OR_64 1 -static inline int64_t opal_atomic_or_fetch_64(volatile int64_t *addr, int64_t value) +static inline int64_t opal_atomic_fetch_or_64(volatile int64_t *addr, int64_t value) { - return __sync_or_and_fetch(addr, value); + return __sync_fetch_and_or(addr, value); } #define OPAL_HAVE_ATOMIC_XOR_64 1 -static inline int64_t opal_atomic_xor_fetch_64(volatile int64_t *addr, int64_t value) +static inline int64_t opal_atomic_fetch_xor_64(volatile int64_t *addr, int64_t value) { - return __sync_xor_and_fetch(addr, value); + return __sync_fetch_and_xor(addr, value); } #define OPAL_HAVE_ATOMIC_SUB_64 1 -static inline int64_t opal_atomic_sub_fetch_64(volatile int64_t *addr, int64_t delta) +static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t *addr, int64_t delta) { - return __sync_sub_and_fetch(addr, delta); + return __sync_fetch_and_sub(addr, delta); } #endif diff --git a/opal/include/opal/sys/x86_64/atomic.h b/opal/include/opal/sys/x86_64/atomic.h index 046b4ad7d2..9590ada081 100644 --- a/opal/include/opal/sys/x86_64/atomic.h +++ b/opal/include/opal/sys/x86_64/atomic.h @@ -196,7 +196,7 @@ static inline int64_t opal_atomic_swap_64( volatile int64_t *addr, * * Atomically adds @i to @v. */ -static inline int32_t opal_atomic_add_fetch_32(volatile int32_t* v, int i) +static inline int32_t opal_atomic_fetch_add_32(volatile int32_t* v, int i) { int ret = i; __asm__ __volatile__( @@ -205,7 +205,7 @@ static inline int32_t opal_atomic_add_fetch_32(volatile int32_t* v, int i) : :"memory", "cc" ); - return (ret+i); + return ret; } #define OPAL_HAVE_ATOMIC_ADD_64 1 @@ -217,7 +217,7 @@ static inline int32_t opal_atomic_add_fetch_32(volatile int32_t* v, int i) * * Atomically adds @i to @v. */ -static inline int64_t opal_atomic_add_fetch_64(volatile int64_t* v, int64_t i) +static inline int64_t opal_atomic_fetch_add_64(volatile int64_t* v, int64_t i) { int64_t ret = i; __asm__ __volatile__( @@ -226,7 +226,7 @@ static inline int64_t opal_atomic_add_fetch_64(volatile int64_t* v, int64_t i) : :"memory", "cc" ); - return (ret+i); + return ret; } #define OPAL_HAVE_ATOMIC_SUB_32 1 @@ -238,7 +238,7 @@ static inline int64_t opal_atomic_add_fetch_64(volatile int64_t* v, int64_t i) * * Atomically subtracts @i from @v. */ -static inline int32_t opal_atomic_sub_fetch_32(volatile int32_t* v, int i) +static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t* v, int i) { int ret = -i; __asm__ __volatile__( @@ -247,7 +247,7 @@ static inline int32_t opal_atomic_sub_fetch_32(volatile int32_t* v, int i) : :"memory", "cc" ); - return (ret-i); + return ret; } #define OPAL_HAVE_ATOMIC_SUB_64 1 @@ -259,7 +259,7 @@ static inline int32_t opal_atomic_sub_fetch_32(volatile int32_t* v, int i) * * Atomically subtracts @i from @v. */ -static inline int64_t opal_atomic_sub_fetch_64(volatile int64_t* v, int64_t i) +static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t* v, int64_t i) { int64_t ret = -i; __asm__ __volatile__( @@ -268,7 +268,7 @@ static inline int64_t opal_atomic_sub_fetch_64(volatile int64_t* v, int64_t i) : :"memory", "cc" ); - return (ret-i); + return ret; } #endif /* OPAL_GCC_INLINE_ASSEMBLY */ diff --git a/opal/mca/btl/vader/btl_vader_xpmem.c b/opal/mca/btl/vader/btl_vader_xpmem.c index 7f8cd7da73..00275df48c 100644 --- a/opal/mca/btl/vader/btl_vader_xpmem.c +++ b/opal/mca/btl/vader/btl_vader_xpmem.c @@ -54,7 +54,7 @@ static int vader_check_reg (mca_rcache_base_registration_t *reg, void *ctx) vader_ctx->reg[0] = reg; if (vader_ctx->bound <= (uintptr_t) reg->bound && vader_ctx->base >= (uintptr_t) reg->base) { - (void)opal_atomic_add_fetch (®->ref_count, 1); + opal_atomic_add (®->ref_count, 1); return 1; } diff --git a/opal/mca/mpool/hugepage/mpool_hugepage_module.c b/opal/mca/mpool/hugepage/mpool_hugepage_module.c index 6af7003563..89a8b7eb6d 100644 --- a/opal/mca/mpool/hugepage/mpool_hugepage_module.c +++ b/opal/mca/mpool/hugepage/mpool_hugepage_module.c @@ -183,7 +183,7 @@ void *mca_mpool_hugepage_seg_alloc (void *ctx, size_t *sizep) opal_mutex_lock (&hugepage_module->lock); opal_rb_tree_insert (&hugepage_module->allocation_tree, base, (void *) (intptr_t) size); - opal_atomic_add_fetch (&mca_mpool_hugepage_component.bytes_allocated, (int64_t) size); + opal_atomic_add (&mca_mpool_hugepage_component.bytes_allocated, (int64_t) size); opal_mutex_unlock (&hugepage_module->lock); OPAL_OUTPUT_VERBOSE((MCA_BASE_VERBOSE_TRACE, opal_mpool_base_framework.framework_verbose, @@ -207,7 +207,7 @@ void mca_mpool_hugepage_seg_free (void *ctx, void *addr) OPAL_OUTPUT_VERBOSE((MCA_BASE_VERBOSE_TRACE, opal_mpool_base_framework.framework_verbose, "freeing segment %p of size %lu bytes", addr, size)); munmap (addr, size); - opal_atomic_add_fetch (&mca_mpool_hugepage_component.bytes_allocated, -(int64_t) size); + opal_atomic_add (&mca_mpool_hugepage_component.bytes_allocated, -(int64_t) size); } opal_mutex_unlock (&hugepage_module->lock); diff --git a/opal/threads/thread_usage.h b/opal/threads/thread_usage.h index 00752f0605..85492d5f89 100644 --- a/opal/threads/thread_usage.h +++ b/opal/threads/thread_usage.h @@ -93,54 +93,26 @@ static inline bool opal_set_using_threads(bool have) * indicates that threads are in use by the application or library. */ -#define OPAL_THREAD_DEFINE_ATOMIC_ADD(type, suffix) \ -static inline type opal_thread_add_fetch_ ## suffix (volatile type *addr, type delta) \ +#define OPAL_THREAD_DEFINE_ATOMIC_OP(type, name, operator, suffix) \ +static inline type opal_thread_ ## name ## _fetch_ ## suffix (volatile type *addr, type delta) \ { \ if (OPAL_UNLIKELY(opal_using_threads())) { \ - return opal_atomic_add_fetch_ ## suffix (addr, delta); \ + return opal_atomic_ ## name ## _fetch_ ## suffix (addr, delta); \ } \ \ - return (*addr += delta); \ -} - -#define OPAL_THREAD_DEFINE_ATOMIC_AND(type, suffix) \ -static inline type opal_thread_and_fetch_ ## suffix (volatile type *addr, type delta) \ + *addr = *addr operator delta; \ + return *addr; \ +} \ + \ +static inline type opal_thread_fetch_ ## name ## _ ## suffix (volatile type *addr, type delta) \ { \ if (OPAL_UNLIKELY(opal_using_threads())) { \ - return opal_atomic_and_fetch_ ## suffix (addr, delta); \ + return opal_atomic_fetch_ ## name ## _ ## suffix (addr, delta); \ } \ \ - return (*addr &= delta); \ -} - -#define OPAL_THREAD_DEFINE_ATOMIC_OR(type, suffix) \ -static inline type opal_thread_or_fetch_ ## suffix (volatile type *addr, type delta) \ -{ \ - if (OPAL_UNLIKELY(opal_using_threads())) { \ - return opal_atomic_or_fetch_ ## suffix (addr, delta); \ - } \ - \ - return (*addr |= delta); \ -} - -#define OPAL_THREAD_DEFINE_ATOMIC_XOR(type, suffix) \ -static inline type opal_thread_xor_fetch_ ## suffix (volatile type *addr, type delta) \ -{ \ - if (OPAL_UNLIKELY(opal_using_threads())) { \ - return opal_atomic_xor_fetch_ ## suffix (addr, delta); \ - } \ - \ - return (*addr ^= delta); \ -} - -#define OPAL_THREAD_DEFINE_ATOMIC_SUB(type, suffix) \ -static inline type opal_thread_sub_fetch_ ## suffix (volatile type *addr, type delta) \ -{ \ - if (OPAL_UNLIKELY(opal_using_threads())) { \ - return opal_atomic_sub_fetch_ ## suffix (addr, delta); \ - } \ - \ - return (*addr -= delta); \ + type old = *addr; \ + *addr = old operator delta; \ + return old; \ } #define OPAL_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(type, addr_type, suffix) \ @@ -173,13 +145,14 @@ static inline type opal_thread_swap_ ## suffix (volatile addr_type *ptr, type ne return old; \ } -OPAL_THREAD_DEFINE_ATOMIC_ADD(int32_t, 32) -OPAL_THREAD_DEFINE_ATOMIC_ADD(size_t, size_t) -OPAL_THREAD_DEFINE_ATOMIC_AND(int32_t, 32) -OPAL_THREAD_DEFINE_ATOMIC_OR(int32_t, 32) -OPAL_THREAD_DEFINE_ATOMIC_XOR(int32_t, 32) -OPAL_THREAD_DEFINE_ATOMIC_SUB(int32_t, 32) -OPAL_THREAD_DEFINE_ATOMIC_SUB(size_t, size_t) +OPAL_THREAD_DEFINE_ATOMIC_OP(int32_t, add, +, 32) +OPAL_THREAD_DEFINE_ATOMIC_OP(size_t, add, +, size_t) +OPAL_THREAD_DEFINE_ATOMIC_OP(int32_t, and, &, 32) +OPAL_THREAD_DEFINE_ATOMIC_OP(int32_t, or, |, 32) +OPAL_THREAD_DEFINE_ATOMIC_OP(int32_t, xor, ^, 32) +OPAL_THREAD_DEFINE_ATOMIC_OP(int32_t, sub, -, 32) +OPAL_THREAD_DEFINE_ATOMIC_OP(size_t, sub, -, size_t) + OPAL_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(int32_t, int32_t, 32) OPAL_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(void *, intptr_t, ptr) OPAL_THREAD_DEFINE_ATOMIC_SWAP(int32_t, int32_t, 32) @@ -203,6 +176,24 @@ OPAL_THREAD_DEFINE_ATOMIC_SWAP(void *, intptr_t, ptr) #define OPAL_THREAD_SUB_FETCH_SIZE_T opal_thread_sub_fetch_size_t #define OPAL_ATOMIC_SUB_FETCH_SIZE_T opal_thread_sub_fetch_size_t +#define OPAL_THREAD_FETCH_ADD32 opal_thread_fetch_add_32 +#define OPAL_ATOMIC_FETCH_ADD32 opal_thread_fetch_add_32 + +#define OPAL_THREAD_FETCH_AND32 opal_thread_fetch_and_32 +#define OPAL_ATOMIC_FETCH_AND32 opal_thread_fetch_and_32 + +#define OPAL_THREAD_FETCH_OR32 opal_thread_fetch_or_32 +#define OPAL_ATOMIC_FETCH_OR32 opal_thread_fetch_or_32 + +#define OPAL_THREAD_FETCH_XOR32 opal_thread_fetch_xor_32 +#define OPAL_ATOMIC_FETCH_XOR32 opal_thread_fetch_xor_32 + +#define OPAL_THREAD_FETCH_ADD_SIZE_T opal_thread_fetch_add_size_t +#define OPAL_ATOMIC_FETCH_ADD_SIZE_T opal_thread_fetch_add_size_t + +#define OPAL_THREAD_FETCH_SUB_SIZE_T opal_thread_fetch_sub_size_t +#define OPAL_ATOMIC_FETCH_SUB_SIZE_T opal_thread_fetch_sub_size_t + #define OPAL_THREAD_COMPARE_EXCHANGE_STRONG_32 opal_thread_compare_exchange_strong_32 #define OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_32 opal_thread_compare_exchange_strong_32 @@ -218,10 +209,11 @@ OPAL_THREAD_DEFINE_ATOMIC_SWAP(void *, intptr_t, ptr) /* define 64-bit macros is 64-bit atomic math is available */ #if OPAL_HAVE_ATOMIC_MATH_64 -OPAL_THREAD_DEFINE_ATOMIC_ADD(int64_t, 64) -OPAL_THREAD_DEFINE_ATOMIC_AND(int64_t, 64) -OPAL_THREAD_DEFINE_ATOMIC_OR(int64_t, 64) -OPAL_THREAD_DEFINE_ATOMIC_XOR(int64_t, 64) +OPAL_THREAD_DEFINE_ATOMIC_OP(int64_t, add, +, 64) +OPAL_THREAD_DEFINE_ATOMIC_OP(int64_t, and, &, 64) +OPAL_THREAD_DEFINE_ATOMIC_OP(int64_t, or, |, 64) +OPAL_THREAD_DEFINE_ATOMIC_OP(int64_t, xor, ^, 64) +OPAL_THREAD_DEFINE_ATOMIC_OP(int64_t, sub, -, 64) OPAL_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(int64_t, int64_t, 64) OPAL_THREAD_DEFINE_ATOMIC_SWAP(int64_t, int64_t, 64) @@ -237,6 +229,18 @@ OPAL_THREAD_DEFINE_ATOMIC_SWAP(int64_t, int64_t, 64) #define OPAL_THREAD_XOR_FETCH64 opal_thread_xor_fetch_64 #define OPAL_ATOMIC_XOR_FETCH64 opal_thread_xor_fetch_64 +#define OPAL_THREAD_FETCH_ADD64 opal_thread_fetch_add_64 +#define OPAL_ATOMIC_FETCH_ADD64 opal_thread_fetch_add_64 + +#define OPAL_THREAD_FETCH_AND64 opal_thread_fetch_and_64 +#define OPAL_ATOMIC_FETCH_AND64 opal_thread_fetch_and_64 + +#define OPAL_THREAD_FETCH_OR64 opal_thread_fetch_or_64 +#define OPAL_ATOMIC_FETCH_OR64 opal_thread_fetch_or_64 + +#define OPAL_THREAD_FETCH_XOR64 opal_thread_fetch_xor_64 +#define OPAL_ATOMIC_FETCH_XOR64 opal_thread_fetch_xor_64 + #define OPAL_THREAD_COMPARE_EXCHANGE_STRONG_64 opal_thread_compare_exchange_strong_64 #define OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_64 opal_thread_compare_exchange_strong_64 diff --git a/test/asm/atomic_cmpset.c b/test/asm/atomic_cmpset.c index d19847cb19..4a06847703 100644 --- a/test/asm/atomic_cmpset.c +++ b/test/asm/atomic_cmpset.c @@ -86,7 +86,7 @@ static void *thread_main(void *arg) #if OPAL_HAVE_ATOMIC_MATH_64 opal_atomic_add_fetch_64(&val64, 5); #endif - opal_atomic_add_fetch(&valint, 5); + opal_atomic_add (&valint, 5); } return (void *) (unsigned long) (rank + 1000); @@ -286,7 +286,7 @@ int main(int argc, char *argv[]) /* -- add_int tests -- */ valint = 42; - opal_atomic_add_fetch(&valint, 5); + opal_atomic_add (&valint, 5); opal_atomic_rmb(); assert((42 + 5) == valint); diff --git a/test/asm/atomic_math.c b/test/asm/atomic_math.c index e489553d4a..54f771cc26 100644 --- a/test/asm/atomic_math.c +++ b/test/asm/atomic_math.c @@ -48,7 +48,7 @@ static void* atomic_math_test(void* arg) #if OPAL_HAVE_ATOMIC_MATH_64 (void)opal_atomic_add_fetch_64(&val64, 6); #endif - (void)opal_atomic_add_fetch(&valint, 4); + opal_atomic_add (&valint, 4); } return NULL; @@ -100,6 +100,10 @@ atomic_math_test_th(int count, int thr_count) int main(int argc, char *argv[]) { + int32_t test32; +#if OPAL_HAVE_ATOMIC_MATH_64 + int64_t test64; +#endif int ret = 77; int num_threads = 1; @@ -109,6 +113,142 @@ main(int argc, char *argv[]) } num_threads = atoi(argv[1]); + test32 = opal_atomic_add_fetch_32 (&val32, 17); + if (test32 != 17 || val32 != 17) { + fprintf (stderr, "error in opal_atomic_add_fetch_32. expected (17, 17), got (%d, %d)\n", test32, val32); + exit(EXIT_FAILURE); + } + + test32 = opal_atomic_fetch_add_32 (&val32, 13); + if (test32 != 17 || val32 != 30) { + fprintf (stderr, "error in opal_atomic_fetch_add_32. expected (17, 30), got (%d, %d)\n", test32, val32); + exit(EXIT_FAILURE); + } + + + + test32 = opal_atomic_and_fetch_32 (&val32, 0x18); + if (test32 != 24 || val32 != 24) { + fprintf (stderr, "error in opal_atomic_and_fetch_32. expected (24, 24), got (%d, %d)\n", test32, val32); + exit(EXIT_FAILURE); + } + + test32 = opal_atomic_fetch_and_32 (&val32, 0x10); + if (test32 != 24 || val32 != 16) { + fprintf (stderr, "error in opal_atomic_fetch_and_32. expected (24, 16), got (%d, %d)\n", test32, val32); + exit(EXIT_FAILURE); + } + + + + test32 = opal_atomic_or_fetch_32 (&val32, 0x03); + if (test32 != 19 || val32 != 19) { + fprintf (stderr, "error in opal_atomic_or_fetch_32. expected (19, 19), got (%d, %d)\n", test32, val32); + exit(EXIT_FAILURE); + } + + test32 = opal_atomic_fetch_or_32 (&val32, 0x04); + if (test32 != 19 || val32 != 23) { + fprintf (stderr, "error in opal_atomic_fetch_or_32. expected (19, 23), got (%d, %d)\n", test32, val32); + exit(EXIT_FAILURE); + } + + + test32 = opal_atomic_xor_fetch_32 (&val32, 0x03); + if (test32 != 20 || val32 != 20) { + fprintf (stderr, "error in opal_atomic_xor_fetch_32. expected (20, 20), got (%d, %d)\n", test32, val32); + exit(EXIT_FAILURE); + } + + test32 = opal_atomic_fetch_xor_32 (&val32, 0x05); + if (test32 != 20 || val32 != 17) { + fprintf (stderr, "error in opal_atomic_fetch_xor_32. expected (20, 17), got (%d, %d)\n", test32, val32); + exit(EXIT_FAILURE); + } + + + + test32 = opal_atomic_sub_fetch_32 (&val32, 14); + if (test32 != 3 || val32 != 3) { + fprintf (stderr, "error in opal_atomic_sub_fetch_32. expected (3, 3), got (%d, %d)\n", test32, val32); + exit(EXIT_FAILURE); + } + + test32 = opal_atomic_fetch_xor_32 (&val32, 3); + if (test32 != 3 || val32 != 0) { + fprintf (stderr, "error in opal_atomic_fetch_sub_32. expected (3, 0), got (%d, %d)\n", test32, val32); + exit(EXIT_FAILURE); + } + +#if OPAL_HAVE_ATOMIC_MATH_64 + test64 = opal_atomic_add_fetch_64 (&val64, 17); + if (test64 != 17 || val64 != 17) { + fprintf (stderr, "error in opal_atomic_add_fetch_64. expected (17, 17), got (%" PRId64 ", %" PRId64 ")\n", test64, val64); + exit(EXIT_FAILURE); + } + + test64 = opal_atomic_fetch_add_64 (&val64, 13); + if (test64 != 17 || val64 != 30) { + fprintf (stderr, "error in opal_atomic_fetch_add_64. expected (17, 30), got (%" PRId64 ", %" PRId64 ")\n", test64, val64); + exit(EXIT_FAILURE); + } + + + + test64 = opal_atomic_and_fetch_64 (&val64, 0x18); + if (test64 != 24 || val64 != 24) { + fprintf (stderr, "error in opal_atomic_and_fetch_64. expected (24, 24), got (%" PRId64 ", %" PRId64 ")\n", test64, val64); + exit(EXIT_FAILURE); + } + + test64 = opal_atomic_fetch_and_64 (&val64, 0x10); + if (test64 != 24 || val64 != 16) { + fprintf (stderr, "error in opal_atomic_fetch_and_64. expected (24, 16), got (%" PRId64 ", %" PRId64 ")\n", test64, val64); + exit(EXIT_FAILURE); + } + + + + test64 = opal_atomic_or_fetch_64 (&val64, 0x03); + if (test64 != 19 || val64 != 19) { + fprintf (stderr, "error in opal_atomic_or_fetch_64. expected (19, 19), got (%" PRId64 ", %" PRId64 ")\n", test64, val64); + exit(EXIT_FAILURE); + } + + test64 = opal_atomic_fetch_or_64 (&val64, 0x04); + if (test64 != 19 || val64 != 23) { + fprintf (stderr, "error in opal_atomic_fetch_or_64. expected (19, 23), got (%" PRId64 ", %" PRId64 ")\n", test64, val64); + exit(EXIT_FAILURE); + } + + + test64 = opal_atomic_xor_fetch_64 (&val64, 0x03); + if (test64 != 20 || val64 != 20) { + fprintf (stderr, "error in opal_atomic_xor_fetch_64. expected (20, 20), got (%" PRId64 ", %" PRId64 ")\n", test64, val64); + exit(EXIT_FAILURE); + } + + test64 = opal_atomic_fetch_xor_64 (&val64, 0x05); + if (test64 != 20 || val64 != 17) { + fprintf (stderr, "error in opal_atomic_fetch_xor_64. expected (20, 17), got (%" PRId64 ", %" PRId64 ")\n", test64, val64); + exit(EXIT_FAILURE); + } + + + + test64 = opal_atomic_sub_fetch_64 (&val64, 14); + if (test64 != 3 || val64 != 3) { + fprintf (stderr, "error in opal_atomic_sub_fetch_64. expected (3, 3), got (%" PRId64 ", %" PRId64 ")\n", test64, val64); + exit(EXIT_FAILURE); + } + + test64 = opal_atomic_fetch_xor_64 (&val64, 3); + if (test64 != 3 || val64 != 0) { + fprintf (stderr, "error in opal_atomic_fetch_sub_64. expected (3, 0), got (%" PRId64 ", %" PRId64 ")\n", test64, val64); + exit(EXIT_FAILURE); + } +#endif + ret = atomic_math_test_th(TEST_REPS, num_threads); if (ret == 77) return ret; opal_atomic_mb(); diff --git a/test/threads/opal_thread.c b/test/threads/opal_thread.c index f8a743a535..169c8b5984 100644 --- a/test/threads/opal_thread.c +++ b/test/threads/opal_thread.c @@ -36,13 +36,13 @@ static volatile int count = 0; static void* thr1_run(opal_object_t* obj) { - (void)opal_atomic_add_fetch(&count, 1); + opal_atomic_add (&count, 1); return NULL; } static void* thr2_run(opal_object_t* obj) { - (void)opal_atomic_add_fetch(&count, 2); + opal_atomic_add (&count, 2); return NULL; }