ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

opal/asm: add fetch-and-op atomics

Просмотр исходного кода 
This commit adds support for fetch-and-op atomics. This is needed
because and and or are irreversible operations so there needs to be a
way to get the old value atomically. These are also the only semantics
supported by C11 (there is not atomic_op_fetch, just
atomic_fetch_op). The old op-and-fetch atomics have been defined in
terms of fetch-and-op.

Signed-off-by: Nathan Hjelm <hjelmn@lanl.gov>
Этот коммит содержится в:
Nathan Hjelm 2017-11-30 09:25:21 -07:00
родитель 1282e98a01
Коммит 7893248c5a
20 изменённых файлов: 441 добавлений и 284 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

2
ompi/mca/coll/sm/coll_sm.h
Просмотреть файл

@ -358,7 +358,7 @@ extern uint32_t mca_coll_sm_one;
* Macro to release an in-use flag from this process
*/
#define FLAG_RELEASE(flag) \
(void)opal_atomic_add_fetch(&(flag)->mcsiuf_num_procs_using, -1)
opal_atomic_add(&(flag)->mcsiuf_num_procs_using, -1)
/**
* Macro to copy a single segment in from a user buffer to a shared

2
ompi/mca/coll/sm/coll_sm_barrier.c
Просмотреть файл

@ -101,7 +101,7 @@ int mca_coll_sm_barrier_intra(struct ompi_communicator_t *comm,
if (0 != rank) {
/* Get parent *in* buffer */
parent = &data->mcb_barrier_control_parent[buffer_set];
(void)opal_atomic_add_fetch(parent, 1);
opal_atomic_add (parent, 1);
SPIN_CONDITION(0 != *me_out, exit_label2);
*me_out = 0;

2
ompi/mca/coll/sm/coll_sm_module.c
Просмотреть файл

@ -463,7 +463,7 @@ int ompi_coll_sm_lazy_enable(mca_coll_base_module_t *module,
OBJ_RETAIN(sm_module->previous_reduce_module);
/* Indicate that we have successfully attached and setup */
(void)opal_atomic_add_fetch(&(data->sm_bootstrap_meta->module_seg->seg_inited), 1);
opal_atomic_add (&(data->sm_bootstrap_meta->module_seg->seg_inited), 1);
/* Wait for everyone in this communicator to attach and setup */
opal_output_verbose(10, ompi_coll_base_framework.framework_output,

8
ompi/mca/osc/sm/osc_sm_active_target.c
Просмотреть файл

@ -151,7 +151,7 @@ ompi_osc_sm_start(struct ompi_group_t *group,
for (int i = 0 ; i < size ; ++i) {
int rank_byte = ranks[i] >> OSC_SM_POST_BITS;
osc_sm_post_type_t old, rank_bit = ((osc_sm_post_type_t) 1) << (ranks[i] & 0x3f);
osc_sm_post_type_t rank_bit = ((osc_sm_post_type_t) 1) << (ranks[i] & 0x3f);
/* wait for rank to post */
while (!(module->posts[my_rank][rank_byte] & rank_bit)) {
@ -162,9 +162,9 @@ ompi_osc_sm_start(struct ompi_group_t *group,
opal_atomic_rmb ();
#if OPAL_HAVE_ATOMIC_MATH_64
opal_atomic_xor_fetch_64 ((volatile osc_sm_post_type_t *) module->posts[my_rank] + rank_byte, rank_bit);
(void) opal_atomic_fetch_xor_64 ((volatile int64_t *) module->posts[my_rank] + rank_byte, rank_bit);
#else
opal_atomic_xor_fetch_32 ((volatile osc_sm_post_type_t *) module->posts[my_rank] + rank_byte, rank_bit);
(void) opal_atomic_fetch_xor_32 ((volatile int32_t *) module->posts[my_rank] + rank_byte, rank_bit);
#endif
}
@ -247,7 +247,7 @@ ompi_osc_sm_post(struct ompi_group_t *group,
gsize = ompi_group_size(module->post_group);
for (int i = 0 ; i < gsize ; ++i) {
(void) opal_atomic_add_fetch ((volatile osc_sm_post_type_t *) module->posts[ranks[i]] + my_byte, my_bit);
opal_atomic_add ((volatile osc_sm_post_type_t *) module->posts[ranks[i]] + my_byte, my_bit);
}
opal_atomic_wmb ();

2
opal/class/opal_list.c
Просмотреть файл

@ -144,7 +144,7 @@ bool opal_list_insert(opal_list_t *list, opal_list_item_t *item, long long idx)
/* Spot check: ensure this item is only on the list that we
just insertted it into */
(void)opal_atomic_add_fetch( &(item->opal_list_item_refcount), 1 );
opal_atomic_add ( &(item->opal_list_item_refcount), 1 );
assert(1 == item->opal_list_item_refcount);
item->opal_list_item_belong_to = list;
#endif

40
opal/include/opal/sys/arm/atomic.h
Просмотреть файл

@ -209,44 +209,44 @@ static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t
#define OPAL_HAVE_ATOMIC_ADD_32 1
static inline int32_t opal_atomic_add_fetch_32(volatile int32_t* v, int inc)
static inline int32_t opal_atomic_fetch_add_32(volatile int32_t* v, int inc)
{
int32_t t;
int tmp;
int32_t t, old;
int tmp;
__asm__ __volatile__(
"1: ldrex %0, [%2] \n"
" add %0, %0, %3 \n"
" strex %1, %0, [%2] \n"
" cmp %1, #0 \n"
__asm__ __volatile__(
"1: ldrex %1, [%3] \n"
" add %0, %1, %4 \n"
" strex %2, %0, [%3] \n"
" cmp %2, #0 \n"
" bne 1b \n"
: "=&r" (t), "=&r" (tmp)
: "=&r" (t), "=&r" (old), "=&r" (tmp)
: "r" (v), "r" (inc)
: "cc", "memory");
return t;
return old;
}
#define OPAL_HAVE_ATOMIC_SUB_32 1
static inline int32_t opal_atomic_sub_fetch_32(volatile int32_t* v, int dec)
static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t* v, int dec)
{
int32_t t;
int tmp;
int32_t t, old;
int tmp;
__asm__ __volatile__(
"1: ldrex %0, [%2] \n"
" sub %0, %0, %3 \n"
" strex %1, %0, [%2] \n"
" cmp %1, #0 \n"
__asm__ __volatile__(
"1: ldrex %1, [%3] \n"
" sub %0, %1, %4 \n"
" strex %2, %0, [%3] \n"
" cmp %2, #0 \n"
" bne 1b \n"
: "=&r" (t), "=&r" (tmp)
: "=&r" (t), "=&r" (old), "=&r" (tmp)
: "r" (v), "r" (dec)
: "cc", "memory");
return t;
return t;
}
#endif

16
opal/include/opal/sys/arm64/atomic.h
Просмотреть файл

@ -293,20 +293,20 @@ static inline int opal_atomic_sc_64 (volatile int64_t *addr, int64_t newval)
}
#define OPAL_ASM_MAKE_ATOMIC(type, bits, name, inst, reg) \
static inline type opal_atomic_ ## name ## _fetch_ ## bits (volatile type *addr, type value) \
static inline type opal_atomic_fetch_ ## name ## _ ## bits (volatile type *addr, type value) \
{ \
type newval; \
type newval, old; \
int32_t tmp; \
\
__asm__ __volatile__("1: ldxr %" reg "0, [%2] \n" \
" " inst " %" reg "0, %" reg "0, %" reg "3 \n" \
" stxr %w1, %" reg "0, [%2] \n" \
" cbnz %w1, 1b \n" \
: "=&r" (newval), "=&r" (tmp) \
__asm__ __volatile__("1: ldxr %" reg "1, [%3] \n" \
" " inst " %" reg "0, %" reg "1, %" reg "4 \n" \
" stxr %w2, %" reg "0, [%3] \n" \
" cbnz %w2, 1b \n" \
: "=&r" (newval), "=&r" (old), "=&r" (tmp) \
: "r" (addr), "r" (value) \
: "cc", "memory"); \
\
return newval; \
return old; \
}
OPAL_ASM_MAKE_ATOMIC(int32_t, 32, add, "add", "w")

145
opal/include/opal/sys/atomic.h
Просмотреть файл

@ -399,36 +399,16 @@ bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t
#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
/* OPAL_HAVE_INLINE_ATOMIC_*_32 will be 1 if <arch>/atomic.h provides
a static inline version of it (in assembly). If we have to fall
back on compare-exchange 32, that too will be inline. */
#if OPAL_HAVE_INLINE_ATOMIC_ADD_32 || (!defined(OPAL_HAVE_ATOMIC_ADD_32) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32)
static inline
#endif
int32_t opal_atomic_add_fetch_32(volatile int32_t *addr, int delta);
#if OPAL_HAVE_INLINE_ATOMIC_AND_32 || (!defined(OPAL_HAVE_ATOMIC_AND_32) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32)
static inline
#endif
int32_t opal_atomic_and_fetch_32(volatile int32_t *addr, int32_t value);
#if OPAL_HAVE_INLINE_ATOMIC_OR_32 || (!defined(OPAL_HAVE_ATOMIC_OR_32) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32)
static inline
#endif
int32_t opal_atomic_or_fetch_32(volatile int32_t *addr, int32_t value);
#if OPAL_HAVE_INLINE_ATOMIC_XOR_32 || (!defined(OPAL_HAVE_ATOMIC_XOR_32) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32)
static inline
#endif
int32_t opal_atomic_xor_fetch_32(volatile int32_t *addr, int32_t value);
/* OPAL_HAVE_INLINE_ATOMIC_*_32 will be 1 if <arch>/atomic.h provides
a static inline version of it (in assembly). If we have to fall
back to compare-exchange 32, that too will be inline. */
#if OPAL_HAVE_INLINE_ATOMIC_SUB_32 || (!defined(OPAL_HAVE_ATOMIC_ADD_32) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32)
static inline
#endif
int32_t opal_atomic_sub_fetch_32(volatile int32_t *addr, int delta);
static inline int32_t opal_atomic_add_fetch_32(volatile int32_t *addr, int delta);
static inline int32_t opal_atomic_fetch_add_32(volatile int32_t *addr, int delta);
static inline int32_t opal_atomic_and_fetch_32(volatile int32_t *addr, int32_t value);
static inline int32_t opal_atomic_fetch_and_32(volatile int32_t *addr, int32_t value);
static inline int32_t opal_atomic_or_fetch_32(volatile int32_t *addr, int32_t value);
static inline int32_t opal_atomic_fetch_or_32(volatile int32_t *addr, int32_t value);
static inline int32_t opal_atomic_xor_fetch_32(volatile int32_t *addr, int32_t value);
static inline int32_t opal_atomic_fetch_xor_32(volatile int32_t *addr, int32_t value);
static inline int32_t opal_atomic_sub_fetch_32(volatile int32_t *addr, int delta);
static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t *addr, int delta);
#endif /* OPAL_HAVE_ATOMIC_MATH_32 */
@ -445,36 +425,15 @@ int32_t opal_atomic_sub_fetch_32(volatile int32_t *addr, int delta);
#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MATH_64 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
/* OPAL_HAVE_INLINE_ATOMIC_*_64 will be 1 if <arch>/atomic.h provides
a static inline version of it (in assembly). If we have to fall
back to compare-exchange 64, that too will be inline */
#if OPAL_HAVE_INLINE_ATOMIC_ADD_64 || (!defined(OPAL_HAVE_ATOMIC_ADD_64) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
static inline
#endif
int64_t opal_atomic_add_fetch_64(volatile int64_t *addr, int64_t delta);
#if OPAL_HAVE_INLINE_ATOMIC_AND_64 || (!defined(OPAL_HAVE_ATOMIC_AND_64) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
static inline
#endif
int64_t opal_atomic_and_fetch_64(volatile int64_t *addr, int64_t value);
#if OPAL_HAVE_INLINE_ATOMIC_OR_64 || (!defined(OPAL_HAVE_ATOMIC_OR_64) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
static inline
#endif
int64_t opal_atomic_or_fetch_64(volatile int64_t *addr, int64_t value);
#if OPAL_HAVE_INLINE_ATOMIC_XOR_64 || (!defined(OPAL_HAVE_ATOMIC_XOR_64) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
static inline
#endif
int64_t opal_atomic_xor_fetch_64(volatile int64_t *addr, int64_t value);
/* OPAL_HAVE_INLINE_ATOMIC_*_64 will be 1 if <arch>/atomic.h provides
a static inline version of it (in assembly). If we have to fall
back to compare-exchange 64, that too will be inline */
#if OPAL_HAVE_INLINE_ATOMIC_SUB_64 || (!defined(OPAL_HAVE_ATOMIC_ADD_64) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
static inline
#endif
int64_t opal_atomic_sub_fetch_64(volatile int64_t *addr, int64_t delta);
static inline int64_t opal_atomic_add_fetch_64(volatile int64_t *addr, int64_t delta);
static inline int64_t opal_atomic_fetch_add_64(volatile int64_t *addr, int64_t delta);
static inline int64_t opal_atomic_and_fetch_64(volatile int64_t *addr, int64_t value);
static inline int64_t opal_atomic_fetch_and_64(volatile int64_t *addr, int64_t value);
static inline int64_t opal_atomic_or_fetch_64(volatile int64_t *addr, int64_t value);
static inline int64_t opal_atomic_fetch_or_64(volatile int64_t *addr, int64_t value);
static inline int64_t opal_atomic_fetch_xor_64(volatile int64_t *addr, int64_t value);
static inline int64_t opal_atomic_sub_fetch_64(volatile int64_t *addr, int64_t delta);
static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t *addr, int64_t delta);
#endif /* OPAL_HAVE_ATOMIC_MATH_32 */
@ -501,6 +460,19 @@ opal_atomic_add_fetch_size_t(volatile size_t *addr, size_t delta)
#error "Unknown size_t size"
#endif
}
static inline size_t
opal_atomic_fetch_add_size_t(volatile size_t *addr, size_t delta)
{
#if SIZEOF_SIZE_T == 4
return (size_t) opal_atomic_fetch_add_32((int32_t*) addr, delta);
#elif SIZEOF_SIZE_T == 8
return (size_t) opal_atomic_fetch_add_64((int64_t*) addr, delta);
#else
#error "Unknown size_t size"
#endif
}
static inline size_t
opal_atomic_sub_fetch_size_t(volatile size_t *addr, size_t delta)
{
@ -512,13 +484,30 @@ opal_atomic_sub_fetch_size_t(volatile size_t *addr, size_t delta)
#error "Unknown size_t size"
#endif
}
static inline size_t
opal_atomic_fetch_sub_size_t(volatile size_t *addr, size_t delta)
{
#if SIZEOF_SIZE_T == 4
return (size_t) opal_atomic_fetch_sub_32((int32_t*) addr, delta);
#elif SIZEOF_SIZE_T == 8
return (size_t) opal_atomic_fetch_sub_64((int64_t*) addr, delta);
#else
#error "Unknown size_t size"
#endif
}
#else
#if SIZEOF_SIZE_T == 4
#define opal_atomic_add_fetch_size_t(addr, delta) ((size_t) opal_atomic_add_fetch_32((int32_t*) addr, delta))
#define opal_atomic_sub_fetch_size_t(addr, delta) ((size_t) opal_atomic_sub_fetch_32((int32_t*) addr, delta))
#elif SIZEOF_SIZE_T ==8
#define opal_atomic_add_fetch_size_t(addr, delta) ((size_t) opal_atomic_add_fetch_64((int64_t*) addr, delta))
#define opal_atomic_sub_fetch_size_t(addr, delta) ((size_t) opal_atomic_sub_fetch_64((int64_t*) addr, delta))
#define opal_atomic_add_fetch_size_t(addr, delta) ((size_t) opal_atomic_add_fetch_32((volatile int32_t *) addr, delta))
#define opal_atomic_fetch_add_size_t(addr, delta) ((size_t) opal_atomic_fetch_add_32((volatile int32_t *) addr, delta))
#define opal_atomic_sub_fetch_size_t(addr, delta) ((size_t) opal_atomic_sub_fetch_32((volatile int32_t *) addr, delta))
#define opal_atomic_fetch_sub_size_t(addr, delta) ((size_t) opal_atomic_fetch_sub_32((volatile int32_t *) addr, delta))
#elif SIZEOF_SIZE_T == 8
#define opal_atomic_add_fetch_size_t(addr, delta) ((size_t) opal_atomic_add_fetch_64((volatile int64_t *) addr, delta))
#define opal_atomic_fetch_add_size_t(addr, delta) ((size_t) opal_atomic_fetch_add_64((volatile int64_t *) addr, delta))
#define opal_atomic_sub_fetch_size_t(addr, delta) ((size_t) opal_atomic_sub_fetch_64((volatile int64_t *) addr, delta))
#define opal_atomic_fetch_sub_size_t(addr, delta) ((size_t) opal_atomic_fetch_sub_64((volatile int64_t *) addr, delta))
#else
#error "Unknown size_t size"
#endif
@ -599,19 +588,15 @@ static inline bool opal_atomic_compare_exchange_strong_rel_ptr (volatile void* a
#if defined(DOXYGEN) || (OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64)
static inline void opal_atomic_add_fetch_xx(volatile void* addr,
static inline void opal_atomic_add_xx(volatile void* addr,
int32_t value, size_t length);
static inline void opal_atomic_sub_fetch_xx(volatile void* addr,
static inline void opal_atomic_sub_xx(volatile void* addr,
int32_t value, size_t length);
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
static inline int32_t opal_atomic_add_fetch_ptr( volatile void* addr, void* delta );
static inline int32_t opal_atomic_sub_fetch_ptr( volatile void* addr, void* delta );
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
static inline int64_t opal_atomic_add_fetch_ptr( volatile void* addr, void* delta );
static inline int64_t opal_atomic_sub_fetch_ptr( volatile void* addr, void* delta );
#else
#error Atomic arithmetic on pointers not supported
#endif
static inline intptr_t opal_atomic_add_fetch_ptr( volatile void* addr, void* delta );
static inline intptr_t opal_atomic_fetch_add_ptr( volatile void* addr, void* delta );
static inline intptr_t opal_atomic_sub_fetch_ptr( volatile void* addr, void* delta );
static inline intptr_t opal_atomic_fetch_sub_ptr( volatile void* addr, void* delta );
/**
* Atomically increment the content depending on the type. This
@ -623,8 +608,8 @@ static inline int64_t opal_atomic_sub_fetch_ptr( volatile void* addr, void* delt
* @param addr Address of <TYPE>
* @param delta Value to add (converted to <TYPE>).
*/
#define opal_atomic_add_fetch( ADDR, VALUE ) \
opal_atomic_add_fetch_xx( (volatile void*)(ADDR), (int32_t)(VALUE), \
#define opal_atomic_add( ADDR, VALUE ) \
opal_atomic_add_xx( (volatile void*)(ADDR), (int32_t)(VALUE), \
sizeof(*(ADDR)) )
/**
@ -637,8 +622,8 @@ static inline int64_t opal_atomic_sub_fetch_ptr( volatile void* addr, void* delt
* @param addr Address of <TYPE>
* @param delta Value to substract (converted to <TYPE>).
*/
#define opal_atomic_sub_fetch( ADDR, VALUE ) \
opal_atomic_sub_fetch_xx( (volatile void*)(ADDR), (int32_t)(VALUE), \
#define opal_atomic_sub( ADDR, VALUE ) \
opal_atomic_sub_xx( (volatile void*)(ADDR), (int32_t)(VALUE), \
sizeof(*(ADDR)) )
#endif /* OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64 */

116
opal/include/opal/sys/atomic_impl.h
Просмотреть файл

@ -39,16 +39,15 @@
*********************************************************************/
#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
#define OPAL_ATOMIC_DEFINE_CMPXCG_OP(type, bits, operand, name) \
static inline type opal_atomic_ ## name ## _fetch_ ## bits (volatile type *addr, type value) \
#define OPAL_ATOMIC_DEFINE_CMPXCG_OP(type, bits, operation, name) \
static inline type opal_atomic_fetch_ ## name ## _ ## bits (volatile type *addr, type value) \
{ \
type oldval, newval; \
type oldval; \
do { \
oldval = *addr; \
newval = oldval operand value; \
} while (!opal_atomic_compare_exchange_strong_ ## bits (addr, &oldval, newval)); \
} while (!opal_atomic_compare_exchange_strong_ ## bits (addr, &oldval, oldval operation value)); \
\
return newval; \
return oldval; \
}
#if !defined(OPAL_HAVE_ATOMIC_SWAP_32)
@ -264,20 +263,19 @@ OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(_rel_)
#if OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64
static inline void
opal_atomic_add_fetch_xx(volatile void* addr, int32_t value, size_t length)
opal_atomic_add_xx(volatile void* addr, int32_t value, size_t length)
{
switch( length ) {
#if OPAL_HAVE_ATOMIC_ADD_32
case 4:
opal_atomic_add_fetch_32( (volatile int32_t*)addr, (int32_t)value );
(void) opal_atomic_fetch_add_32( (volatile int32_t*)addr, (int32_t)value );
break;
#endif /* OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 */
#if OPAL_HAVE_ATOMIC_ADD_64
case 8:
opal_atomic_add_fetch_64( (volatile int64_t*)addr, (int64_t)value );
(void) opal_atomic_fetch_add_64( (volatile int64_t*)addr, (int64_t)value );
break;
#endif /* OPAL_HAVE_ATOMIC_ADD_64 */
default:
@ -289,18 +287,18 @@ opal_atomic_add_fetch_xx(volatile void* addr, int32_t value, size_t length)
static inline void
opal_atomic_sub_fetch_xx(volatile void* addr, int32_t value, size_t length)
opal_atomic_sub_xx(volatile void* addr, int32_t value, size_t length)
{
switch( length ) {
#if OPAL_HAVE_ATOMIC_SUB_32
case 4:
opal_atomic_sub_fetch_32( (volatile int32_t*)addr, (int32_t)value );
(void) opal_atomic_fetch_sub_32( (volatile int32_t*)addr, (int32_t)value );
break;
#endif /* OPAL_HAVE_ATOMIC_SUB_32 */
#if OPAL_HAVE_ATOMIC_SUB_64
case 8:
opal_atomic_sub_fetch_64( (volatile int64_t*)addr, (int64_t)value );
(void) opal_atomic_fetch_sub_64( (volatile int64_t*)addr, (int64_t)value );
break;
#endif /* OPAL_HAVE_ATOMIC_SUB_64 */
default:
@ -310,47 +308,77 @@ opal_atomic_sub_fetch_xx(volatile void* addr, int32_t value, size_t length)
}
}
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_ADD_32
static inline int32_t opal_atomic_add_fetch_ptr( volatile void* addr,
void* delta )
{
return opal_atomic_add_fetch_32((int32_t*) addr, (unsigned long) delta);
}
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_ADD_64
static inline int64_t opal_atomic_add_fetch_ptr( volatile void* addr,
void* delta )
{
return opal_atomic_add_fetch_64((int64_t*) addr, (unsigned long) delta);
}
#else
static inline int32_t opal_atomic_add_fetch_ptr( volatile void* addr,
void* delta )
{
abort();
return 0;
}
#define OPAL_ATOMIC_DEFINE_OP_FETCH(op, operation, type, ptr_type, suffix) \
static inline type opal_atomic_ ## op ## _fetch_ ## suffix (volatile ptr_type *addr, type value) \
{ \
return opal_atomic_fetch_ ## op ## _ ## suffix (addr, value) operation value; \
}
OPAL_ATOMIC_DEFINE_OP_FETCH(add, +, int32_t, int32_t, 32)
OPAL_ATOMIC_DEFINE_OP_FETCH(and, &, int32_t, int32_t, 32)
OPAL_ATOMIC_DEFINE_OP_FETCH(or, |, int32_t, int32_t, 32)
OPAL_ATOMIC_DEFINE_OP_FETCH(xor, ^, int32_t, int32_t, 32)
OPAL_ATOMIC_DEFINE_OP_FETCH(sub, -, int32_t, int32_t, 32)
#if OPAL_HAVE_ATOMIC_MATH_64
OPAL_ATOMIC_DEFINE_OP_FETCH(add, +, int64_t, int64_t, 64)
OPAL_ATOMIC_DEFINE_OP_FETCH(and, &, int64_t, int64_t, 64)
OPAL_ATOMIC_DEFINE_OP_FETCH(or, |, int64_t, int64_t, 64)
OPAL_ATOMIC_DEFINE_OP_FETCH(xor, ^, int64_t, int64_t, 64)
OPAL_ATOMIC_DEFINE_OP_FETCH(sub, -, int64_t, int64_t, 64)
#endif
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_SUB_32
static inline int32_t opal_atomic_sub_fetch_ptr( volatile void* addr,
static inline intptr_t opal_atomic_fetch_add_ptr( volatile void* addr,
void* delta )
{
return opal_atomic_sub_fetch_32((int32_t*) addr, (unsigned long) delta);
}
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_SUB_32
static inline int64_t opal_atomic_sub_fetch_ptr( volatile void* addr,
void* delta )
{
return opal_atomic_sub_fetch_64((int64_t*) addr, (unsigned long) delta);
}
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_ADD_32
return opal_atomic_fetch_add_32((int32_t*) addr, (unsigned long) delta);
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_ADD_64
return opal_atomic_fetch_add_64((int64_t*) addr, (unsigned long) delta);
#else
static inline int32_t opal_atomic_sub_fetch_ptr( volatile void* addr,
abort ();
return 0;
#endif
}
static inline intptr_t opal_atomic_add_fetch_ptr( volatile void* addr,
void* delta )
{
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_ADD_32
return opal_atomic_add_fetch_32((int32_t*) addr, (unsigned long) delta);
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_ADD_64
return opal_atomic_add_fetch_64((int64_t*) addr, (unsigned long) delta);
#else
abort ();
return 0;
#endif
}
static inline intptr_t opal_atomic_fetch_sub_ptr( volatile void* addr,
void* delta )
{
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_SUB_32
return opal_atomic_fetch_sub_32((int32_t*) addr, (unsigned long) delta);
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_SUB_32
return opal_atomic_fetch_sub_64((int64_t*) addr, (unsigned long) delta);
#else
abort();
return 0;
}
#endif
}
static inline intptr_t opal_atomic_sub_fetch_ptr( volatile void* addr,
void* delta )
{
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_SUB_32
return opal_atomic_sub_fetch_32((int32_t*) addr, (unsigned long) delta);
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_SUB_32
return opal_atomic_sub_fetch_64((int64_t*) addr, (unsigned long) delta);
#else
abort();
return 0;
#endif
}
#endif /* OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64 */

40
opal/include/opal/sys/gcc_builtin/atomic.h
Просмотреть файл

@ -104,29 +104,29 @@ static inline int32_t opal_atomic_swap_32 (volatile int32_t *addr, int32_t newva
return oldval;
}
static inline int32_t opal_atomic_add_fetch_32(volatile int32_t *addr, int32_t delta)
static inline int32_t opal_atomic_fetch_add_32(volatile int32_t *addr, int32_t delta)
{
return __atomic_add_fetch (addr, delta, __ATOMIC_RELAXED);
return __atomic_fetch_add (addr, delta, __ATOMIC_RELAXED);
}
static inline int32_t opal_atomic_and_fetch_32(volatile int32_t *addr, int32_t value)
static inline int32_t opal_atomic_fetch_and_32(volatile int32_t *addr, int32_t value)
{
return __atomic_and_fetch (addr, value, __ATOMIC_RELAXED);
return __atomic_fetch_and (addr, value, __ATOMIC_RELAXED);
}
static inline int32_t opal_atomic_or_fetch_32(volatile int32_t *addr, int32_t value)
static inline int32_t opal_atomic_fetch_or_32(volatile int32_t *addr, int32_t value)
{
return __atomic_or_fetch (addr, value, __ATOMIC_RELAXED);
return __atomic_fetch_or (addr, value, __ATOMIC_RELAXED);
}
static inline int32_t opal_atomic_xor_fetch_32(volatile int32_t *addr, int32_t value)
static inline int32_t opal_atomic_fetch_xor_32(volatile int32_t *addr, int32_t value)
{
return __atomic_xor_fetch (addr, value, __ATOMIC_RELAXED);
return __atomic_fetch_xor (addr, value, __ATOMIC_RELAXED);
}
static inline int32_t opal_atomic_sub_fetch_32(volatile int32_t *addr, int32_t delta)
static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t *addr, int32_t delta)
{
return __atomic_sub_fetch (addr, delta, __ATOMIC_RELAXED);
return __atomic_fetch_sub (addr, delta, __ATOMIC_RELAXED);
}
static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
@ -152,29 +152,29 @@ static inline int64_t opal_atomic_swap_64 (volatile int64_t *addr, int64_t newva
return oldval;
}
static inline int64_t opal_atomic_add_fetch_64(volatile int64_t *addr, int64_t delta)
static inline int64_t opal_atomic_fetch_add_64(volatile int64_t *addr, int64_t delta)
{
return __atomic_add_fetch (addr, delta, __ATOMIC_RELAXED);
return __atomic_fetch_add (addr, delta, __ATOMIC_RELAXED);
}
static inline int64_t opal_atomic_and_fetch_64(volatile int64_t *addr, int64_t value)
static inline int64_t opal_atomic_fetch_and_64(volatile int64_t *addr, int64_t value)
{
return __atomic_and_fetch (addr, value, __ATOMIC_RELAXED);
return __atomic_fetch_and (addr, value, __ATOMIC_RELAXED);
}
static inline int64_t opal_atomic_or_fetch_64(volatile int64_t *addr, int64_t value)
static inline int64_t opal_atomic_fetch_or_64(volatile int64_t *addr, int64_t value)
{
return __atomic_or_fetch (addr, value, __ATOMIC_RELAXED);
return __atomic_fetch_or (addr, value, __ATOMIC_RELAXED);
}
static inline int64_t opal_atomic_xor_fetch_64(volatile int64_t *addr, int64_t value)
static inline int64_t opal_atomic_fetch_xor_64(volatile int64_t *addr, int64_t value)
{
return __atomic_xor_fetch (addr, value, __ATOMIC_RELAXED);
return __atomic_fetch_xor (addr, value, __ATOMIC_RELAXED);
}
static inline int64_t opal_atomic_sub_fetch_64(volatile int64_t *addr, int64_t delta)
static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t *addr, int64_t delta)
{
return __atomic_sub_fetch (addr, delta, __ATOMIC_RELAXED);
return __atomic_fetch_sub (addr, delta, __ATOMIC_RELAXED);
}
#if OPAL_HAVE_GCC_BUILTIN_CSWAP_INT128

8
opal/include/opal/sys/ia32/atomic.h
Просмотреть файл

@ -130,7 +130,7 @@ static inline int32_t opal_atomic_swap_32( volatile int32_t *addr,
*
* Atomically adds @i to @v.
*/
static inline int32_t opal_atomic_add_fetch_32(volatile int32_t* v, int i)
static inline int32_t opal_atomic_fetch_add_32(volatile int32_t* v, int i)
{
int ret = i;
__asm__ __volatile__(
@ -139,7 +139,7 @@ static inline int32_t opal_atomic_add_fetch_32(volatile int32_t* v, int i)
:
:"memory", "cc"
);
return (ret+i);
return ret;
}
@ -150,7 +150,7 @@ static inline int32_t opal_atomic_add_fetch_32(volatile int32_t* v, int i)
*
* Atomically subtracts @i from @v.
*/
static inline int32_t opal_atomic_sub_fetch_32(volatile int32_t* v, int i)
static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t* v, int i)
{
int ret = -i;
__asm__ __volatile__(
@ -159,7 +159,7 @@ static inline int32_t opal_atomic_sub_fetch_32(volatile int32_t* v, int i)
:
:"memory", "cc"
);
return (ret-i);
return ret;
}
#endif /* OPAL_GCC_INLINE_ASSEMBLY */

26
opal/include/opal/sys/powerpc/atomic.h
Просмотреть файл

@ -235,20 +235,20 @@ static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, int32_t newval
#if OPAL_GCC_INLINE_ASSEMBLY
#define OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(type, instr) \
static inline int64_t opal_atomic_ ## type ## _fetch_64(volatile int64_t* v, int64_t val) \
static inline int64_t opal_atomic_fetch_ ## type ## _64(volatile int64_t* v, int64_t val) \
{ \
int64_t t; \
int64_t t, old; \
\
__asm__ __volatile__( \
"1: ldarx %0, 0, %3 \n\t" \
" " #instr " %0, %2, %0 \n\t" \
" stdcx. %0, 0, %3 \n\t" \
"1: ldarx %1, 0, %4 \n\t" \
" " #instr " %0, %3, %1 \n\t" \
" stdcx. %0, 0, %4 \n\t" \
" bne- 1b \n\t" \
: "=&r" (t), "=m" (*v) \
: "=&r" (t), "=&r" (old), "=m" (*v) \
: "r" (OPAL_ASM_VALUE64(val)), "r" OPAL_ASM_ADDR(v), "m" (*v) \
: "cc"); \
\
return t; \
return old; \
}
OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(add, add)
@ -396,16 +396,16 @@ static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t
#define OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_32(type, instr) \
static inline int32_t opal_atomic_ ## type ## _fetch_32(volatile int32_t* v, int val) \
static inline int32_t opal_atomic_fetch_ ## type ## _32(volatile int32_t* v, int val) \
{ \
int32_t t; \
int32_t t, old; \
\
__asm__ __volatile__( \
"1: lwarx %0, 0, %3 \n\t" \
" " #instr " %0, %2, %0 \n\t" \
" stwcx. %0, 0, %3 \n\t" \
"1: lwarx %1, 0, %4 \n\t" \
" " #instr " %0, %3, %1 \n\t" \
" stwcx. %0, 0, %4 \n\t" \
" bne- 1b \n\t" \
: "=&r" (t), "=m" (*v) \
: "=&r" (t), "=&r" (old), "=m" (*v) \
: "r" (val), "r" OPAL_ASM_ADDR(v), "m" (*v) \
: "cc"); \
\

40
opal/include/opal/sys/sync_builtin/atomic.h
Просмотреть файл

@ -69,33 +69,33 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add
#define OPAL_HAVE_ATOMIC_MATH_32 1
#define OPAL_HAVE_ATOMIC_ADD_32 1
static inline int32_t opal_atomic_add_fetch_32(volatile int32_t *addr, int32_t delta)
static inline int32_t opal_atomic_fetch_add_32(volatile int32_t *addr, int32_t delta)
{
return __sync_add_and_fetch(addr, delta);
return __sync_fetch_and_add(addr, delta);
}
#define OPAL_HAVE_ATOMIC_AND_32 1
static inline int32_t opal_atomic_and_fetch_32(volatile int32_t *addr, int32_t value)
static inline int32_t opal_atomic_fetch_and_32(volatile int32_t *addr, int32_t value)
{
return __sync_and_and_fetch(addr, value);
return __sync_fetch_and_and(addr, value);
}
#define OPAL_HAVE_ATOMIC_OR_32 1
static inline int32_t opal_atomic_or_fetch_32(volatile int32_t *addr, int32_t value)
static inline int32_t opal_atomic_fetch_or_32(volatile int32_t *addr, int32_t value)
{
return __sync_or_and_fetch(addr, value);
return __sync_fetch_and_or(addr, value);
}
#define OPAL_HAVE_ATOMIC_XOR_32 1
static inline int32_t opal_atomic_xor_fetch_32(volatile int32_t *addr, int32_t value)
static inline int32_t opal_atomic_fetch_xor_32(volatile int32_t *addr, int32_t value)
{
return __sync_xor_and_fetch(addr, value);
return __sync_fetch_and_xor(addr, value);
}
#define OPAL_HAVE_ATOMIC_SUB_32 1
static inline int32_t opal_atomic_sub_fetch_32(volatile int32_t *addr, int32_t delta)
static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t *addr, int32_t delta)
{
return __sync_sub_and_fetch(addr, delta);
return __sync_fetch_and_sub(addr, delta);
}
#if OPAL_ASM_SYNC_HAVE_64BIT
@ -115,33 +115,33 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add
#define OPAL_HAVE_ATOMIC_MATH_64 1
#define OPAL_HAVE_ATOMIC_ADD_64 1
static inline int64_t opal_atomic_add_fetch_64(volatile int64_t *addr, int64_t delta)
static inline int64_t opal_atomic_fetch_add_64(volatile int64_t *addr, int64_t delta)
{
return __sync_add_and_fetch(addr, delta);
return __sync_fetch_and_add(addr, delta);
}
#define OPAL_HAVE_ATOMIC_AND_64 1
static inline int64_t opal_atomic_and_fetch_64(volatile int64_t *addr, int64_t value)
static inline int64_t opal_atomic_fetch_and_64(volatile int64_t *addr, int64_t value)
{
return __sync_and_and_fetch(addr, value);
return __sync_fetch_and_and(addr, value);
}
#define OPAL_HAVE_ATOMIC_OR_64 1
static inline int64_t opal_atomic_or_fetch_64(volatile int64_t *addr, int64_t value)
static inline int64_t opal_atomic_fetch_or_64(volatile int64_t *addr, int64_t value)
{
return __sync_or_and_fetch(addr, value);
return __sync_fetch_and_or(addr, value);
}
#define OPAL_HAVE_ATOMIC_XOR_64 1
static inline int64_t opal_atomic_xor_fetch_64(volatile int64_t *addr, int64_t value)
static inline int64_t opal_atomic_fetch_xor_64(volatile int64_t *addr, int64_t value)
{
return __sync_xor_and_fetch(addr, value);
return __sync_fetch_and_xor(addr, value);
}
#define OPAL_HAVE_ATOMIC_SUB_64 1
static inline int64_t opal_atomic_sub_fetch_64(volatile int64_t *addr, int64_t delta)
static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t *addr, int64_t delta)
{
return __sync_sub_and_fetch(addr, delta);
return __sync_fetch_and_sub(addr, delta);
}
#endif

16
opal/include/opal/sys/x86_64/atomic.h
Просмотреть файл

@ -196,7 +196,7 @@ static inline int64_t opal_atomic_swap_64( volatile int64_t *addr,
*
* Atomically adds @i to @v.
*/
static inline int32_t opal_atomic_add_fetch_32(volatile int32_t* v, int i)
static inline int32_t opal_atomic_fetch_add_32(volatile int32_t* v, int i)
{
int ret = i;
__asm__ __volatile__(
@ -205,7 +205,7 @@ static inline int32_t opal_atomic_add_fetch_32(volatile int32_t* v, int i)
:
:"memory", "cc"
);
return (ret+i);
return ret;
}
#define OPAL_HAVE_ATOMIC_ADD_64 1
@ -217,7 +217,7 @@ static inline int32_t opal_atomic_add_fetch_32(volatile int32_t* v, int i)
*
* Atomically adds @i to @v.
*/
static inline int64_t opal_atomic_add_fetch_64(volatile int64_t* v, int64_t i)
static inline int64_t opal_atomic_fetch_add_64(volatile int64_t* v, int64_t i)
{
int64_t ret = i;
__asm__ __volatile__(
@ -226,7 +226,7 @@ static inline int64_t opal_atomic_add_fetch_64(volatile int64_t* v, int64_t i)
:
:"memory", "cc"
);
return (ret+i);
return ret;
}
#define OPAL_HAVE_ATOMIC_SUB_32 1
@ -238,7 +238,7 @@ static inline int64_t opal_atomic_add_fetch_64(volatile int64_t* v, int64_t i)
*
* Atomically subtracts @i from @v.
*/
static inline int32_t opal_atomic_sub_fetch_32(volatile int32_t* v, int i)
static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t* v, int i)
{
int ret = -i;
__asm__ __volatile__(
@ -247,7 +247,7 @@ static inline int32_t opal_atomic_sub_fetch_32(volatile int32_t* v, int i)
:
:"memory", "cc"
);
return (ret-i);
return ret;
}
#define OPAL_HAVE_ATOMIC_SUB_64 1
@ -259,7 +259,7 @@ static inline int32_t opal_atomic_sub_fetch_32(volatile int32_t* v, int i)
*
* Atomically subtracts @i from @v.
*/
static inline int64_t opal_atomic_sub_fetch_64(volatile int64_t* v, int64_t i)
static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t* v, int64_t i)
{
int64_t ret = -i;
__asm__ __volatile__(
@ -268,7 +268,7 @@ static inline int64_t opal_atomic_sub_fetch_64(volatile int64_t* v, int64_t i)
:
:"memory", "cc"
);
return (ret-i);
return ret;
}
#endif /* OPAL_GCC_INLINE_ASSEMBLY */

2
opal/mca/btl/vader/btl_vader_xpmem.c
Просмотреть файл

@ -54,7 +54,7 @@ static int vader_check_reg (mca_rcache_base_registration_t *reg, void *ctx)
vader_ctx->reg[0] = reg;
if (vader_ctx->bound <= (uintptr_t) reg->bound && vader_ctx->base >= (uintptr_t) reg->base) {
(void)opal_atomic_add_fetch (&reg->ref_count, 1);
opal_atomic_add (&reg->ref_count, 1);
return 1;
}

4
opal/mca/mpool/hugepage/mpool_hugepage_module.c
Просмотреть файл

@ -183,7 +183,7 @@ void *mca_mpool_hugepage_seg_alloc (void *ctx, size_t *sizep)
opal_mutex_lock (&hugepage_module->lock);
opal_rb_tree_insert (&hugepage_module->allocation_tree, base, (void *) (intptr_t) size);
opal_atomic_add_fetch (&mca_mpool_hugepage_component.bytes_allocated, (int64_t) size);
opal_atomic_add (&mca_mpool_hugepage_component.bytes_allocated, (int64_t) size);
opal_mutex_unlock (&hugepage_module->lock);
OPAL_OUTPUT_VERBOSE((MCA_BASE_VERBOSE_TRACE, opal_mpool_base_framework.framework_verbose,
@ -207,7 +207,7 @@ void mca_mpool_hugepage_seg_free (void *ctx, void *addr)
OPAL_OUTPUT_VERBOSE((MCA_BASE_VERBOSE_TRACE, opal_mpool_base_framework.framework_verbose,
"freeing segment %p of size %lu bytes", addr, size));
munmap (addr, size);
opal_atomic_add_fetch (&mca_mpool_hugepage_component.bytes_allocated, -(int64_t) size);
opal_atomic_add (&mca_mpool_hugepage_component.bytes_allocated, -(int64_t) size);
}
opal_mutex_unlock (&hugepage_module->lock);

106
opal/threads/thread_usage.h
Просмотреть файл

@ -93,54 +93,26 @@ static inline bool opal_set_using_threads(bool have)
* indicates that threads are in use by the application or library.
*/
#define OPAL_THREAD_DEFINE_ATOMIC_ADD(type, suffix) \
static inline type opal_thread_add_fetch_ ## suffix (volatile type *addr, type delta) \
#define OPAL_THREAD_DEFINE_ATOMIC_OP(type, name, operator, suffix) \
static inline type opal_thread_ ## name ## _fetch_ ## suffix (volatile type *addr, type delta) \
{ \
if (OPAL_UNLIKELY(opal_using_threads())) { \
return opal_atomic_add_fetch_ ## suffix (addr, delta); \
return opal_atomic_ ## name ## _fetch_ ## suffix (addr, delta); \
} \
\
return (*addr += delta); \
}
#define OPAL_THREAD_DEFINE_ATOMIC_AND(type, suffix) \
static inline type opal_thread_and_fetch_ ## suffix (volatile type *addr, type delta) \
*addr = *addr operator delta; \
return *addr; \
} \
\
static inline type opal_thread_fetch_ ## name ## _ ## suffix (volatile type *addr, type delta) \
{ \
if (OPAL_UNLIKELY(opal_using_threads())) { \
return opal_atomic_and_fetch_ ## suffix (addr, delta); \
return opal_atomic_fetch_ ## name ## _ ## suffix (addr, delta); \
} \
\
return (*addr &= delta); \
}
#define OPAL_THREAD_DEFINE_ATOMIC_OR(type, suffix) \
static inline type opal_thread_or_fetch_ ## suffix (volatile type *addr, type delta) \
{ \
if (OPAL_UNLIKELY(opal_using_threads())) { \
return opal_atomic_or_fetch_ ## suffix (addr, delta); \
} \
\
return (*addr |= delta); \
}
#define OPAL_THREAD_DEFINE_ATOMIC_XOR(type, suffix) \
static inline type opal_thread_xor_fetch_ ## suffix (volatile type *addr, type delta) \
{ \
if (OPAL_UNLIKELY(opal_using_threads())) { \
return opal_atomic_xor_fetch_ ## suffix (addr, delta); \
} \
\
return (*addr ^= delta); \
}
#define OPAL_THREAD_DEFINE_ATOMIC_SUB(type, suffix) \
static inline type opal_thread_sub_fetch_ ## suffix (volatile type *addr, type delta) \
{ \
if (OPAL_UNLIKELY(opal_using_threads())) { \
return opal_atomic_sub_fetch_ ## suffix (addr, delta); \
} \
\
return (*addr -= delta); \
type old = *addr; \
*addr = old operator delta; \
return old; \
}
#define OPAL_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(type, addr_type, suffix) \
@ -173,13 +145,14 @@ static inline type opal_thread_swap_ ## suffix (volatile addr_type *ptr, type ne
return old; \
}
OPAL_THREAD_DEFINE_ATOMIC_ADD(int32_t, 32)
OPAL_THREAD_DEFINE_ATOMIC_ADD(size_t, size_t)
OPAL_THREAD_DEFINE_ATOMIC_AND(int32_t, 32)
OPAL_THREAD_DEFINE_ATOMIC_OR(int32_t, 32)
OPAL_THREAD_DEFINE_ATOMIC_XOR(int32_t, 32)
OPAL_THREAD_DEFINE_ATOMIC_SUB(int32_t, 32)
OPAL_THREAD_DEFINE_ATOMIC_SUB(size_t, size_t)
OPAL_THREAD_DEFINE_ATOMIC_OP(int32_t, add, +, 32)
OPAL_THREAD_DEFINE_ATOMIC_OP(size_t, add, +, size_t)
OPAL_THREAD_DEFINE_ATOMIC_OP(int32_t, and, &, 32)
OPAL_THREAD_DEFINE_ATOMIC_OP(int32_t, or, |, 32)
OPAL_THREAD_DEFINE_ATOMIC_OP(int32_t, xor, ^, 32)
OPAL_THREAD_DEFINE_ATOMIC_OP(int32_t, sub, -, 32)
OPAL_THREAD_DEFINE_ATOMIC_OP(size_t, sub, -, size_t)
OPAL_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(int32_t, int32_t, 32)
OPAL_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(void *, intptr_t, ptr)
OPAL_THREAD_DEFINE_ATOMIC_SWAP(int32_t, int32_t, 32)
@ -203,6 +176,24 @@ OPAL_THREAD_DEFINE_ATOMIC_SWAP(void *, intptr_t, ptr)
#define OPAL_THREAD_SUB_FETCH_SIZE_T opal_thread_sub_fetch_size_t
#define OPAL_ATOMIC_SUB_FETCH_SIZE_T opal_thread_sub_fetch_size_t
#define OPAL_THREAD_FETCH_ADD32 opal_thread_fetch_add_32
#define OPAL_ATOMIC_FETCH_ADD32 opal_thread_fetch_add_32
#define OPAL_THREAD_FETCH_AND32 opal_thread_fetch_and_32
#define OPAL_ATOMIC_FETCH_AND32 opal_thread_fetch_and_32
#define OPAL_THREAD_FETCH_OR32 opal_thread_fetch_or_32
#define OPAL_ATOMIC_FETCH_OR32 opal_thread_fetch_or_32
#define OPAL_THREAD_FETCH_XOR32 opal_thread_fetch_xor_32
#define OPAL_ATOMIC_FETCH_XOR32 opal_thread_fetch_xor_32
#define OPAL_THREAD_FETCH_ADD_SIZE_T opal_thread_fetch_add_size_t
#define OPAL_ATOMIC_FETCH_ADD_SIZE_T opal_thread_fetch_add_size_t
#define OPAL_THREAD_FETCH_SUB_SIZE_T opal_thread_fetch_sub_size_t
#define OPAL_ATOMIC_FETCH_SUB_SIZE_T opal_thread_fetch_sub_size_t
#define OPAL_THREAD_COMPARE_EXCHANGE_STRONG_32 opal_thread_compare_exchange_strong_32
#define OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_32 opal_thread_compare_exchange_strong_32
@ -218,10 +209,11 @@ OPAL_THREAD_DEFINE_ATOMIC_SWAP(void *, intptr_t, ptr)
/* define 64-bit macros is 64-bit atomic math is available */
#if OPAL_HAVE_ATOMIC_MATH_64
OPAL_THREAD_DEFINE_ATOMIC_ADD(int64_t, 64)
OPAL_THREAD_DEFINE_ATOMIC_AND(int64_t, 64)
OPAL_THREAD_DEFINE_ATOMIC_OR(int64_t, 64)
OPAL_THREAD_DEFINE_ATOMIC_XOR(int64_t, 64)
OPAL_THREAD_DEFINE_ATOMIC_OP(int64_t, add, +, 64)
OPAL_THREAD_DEFINE_ATOMIC_OP(int64_t, and, &, 64)
OPAL_THREAD_DEFINE_ATOMIC_OP(int64_t, or, |, 64)
OPAL_THREAD_DEFINE_ATOMIC_OP(int64_t, xor, ^, 64)
OPAL_THREAD_DEFINE_ATOMIC_OP(int64_t, sub, -, 64)
OPAL_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(int64_t, int64_t, 64)
OPAL_THREAD_DEFINE_ATOMIC_SWAP(int64_t, int64_t, 64)
@ -237,6 +229,18 @@ OPAL_THREAD_DEFINE_ATOMIC_SWAP(int64_t, int64_t, 64)
#define OPAL_THREAD_XOR_FETCH64 opal_thread_xor_fetch_64
#define OPAL_ATOMIC_XOR_FETCH64 opal_thread_xor_fetch_64
#define OPAL_THREAD_FETCH_ADD64 opal_thread_fetch_add_64
#define OPAL_ATOMIC_FETCH_ADD64 opal_thread_fetch_add_64
#define OPAL_THREAD_FETCH_AND64 opal_thread_fetch_and_64
#define OPAL_ATOMIC_FETCH_AND64 opal_thread_fetch_and_64
#define OPAL_THREAD_FETCH_OR64 opal_thread_fetch_or_64
#define OPAL_ATOMIC_FETCH_OR64 opal_thread_fetch_or_64
#define OPAL_THREAD_FETCH_XOR64 opal_thread_fetch_xor_64
#define OPAL_ATOMIC_FETCH_XOR64 opal_thread_fetch_xor_64
#define OPAL_THREAD_COMPARE_EXCHANGE_STRONG_64 opal_thread_compare_exchange_strong_64
#define OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_64 opal_thread_compare_exchange_strong_64

4
test/asm/atomic_cmpset.c
Просмотреть файл

@ -86,7 +86,7 @@ static void *thread_main(void *arg)
#if OPAL_HAVE_ATOMIC_MATH_64
opal_atomic_add_fetch_64(&val64, 5);
#endif
opal_atomic_add_fetch(&valint, 5);
opal_atomic_add (&valint, 5);
}
return (void *) (unsigned long) (rank + 1000);
@ -286,7 +286,7 @@ int main(int argc, char *argv[])
/* -- add_int tests -- */
valint = 42;
opal_atomic_add_fetch(&valint, 5);
opal_atomic_add (&valint, 5);
opal_atomic_rmb();
assert((42 + 5) == valint);

142
test/asm/atomic_math.c
Просмотреть файл

@ -48,7 +48,7 @@ static void* atomic_math_test(void* arg)
#if OPAL_HAVE_ATOMIC_MATH_64
(void)opal_atomic_add_fetch_64(&val64, 6);
#endif
(void)opal_atomic_add_fetch(&valint, 4);
opal_atomic_add (&valint, 4);
}
return NULL;
@ -100,6 +100,10 @@ atomic_math_test_th(int count, int thr_count)
int
main(int argc, char *argv[])
{
int32_t test32;
#if OPAL_HAVE_ATOMIC_MATH_64
int64_t test64;
#endif
int ret = 77;
int num_threads = 1;
@ -109,6 +113,142 @@ main(int argc, char *argv[])
}
num_threads = atoi(argv[1]);
test32 = opal_atomic_add_fetch_32 (&val32, 17);
if (test32 != 17 || val32 != 17) {
fprintf (stderr, "error in opal_atomic_add_fetch_32. expected (17, 17), got (%d, %d)\n", test32, val32);
exit(EXIT_FAILURE);
}
test32 = opal_atomic_fetch_add_32 (&val32, 13);
if (test32 != 17 || val32 != 30) {
fprintf (stderr, "error in opal_atomic_fetch_add_32. expected (17, 30), got (%d, %d)\n", test32, val32);
exit(EXIT_FAILURE);
}
test32 = opal_atomic_and_fetch_32 (&val32, 0x18);
if (test32 != 24 || val32 != 24) {
fprintf (stderr, "error in opal_atomic_and_fetch_32. expected (24, 24), got (%d, %d)\n", test32, val32);
exit(EXIT_FAILURE);
}
test32 = opal_atomic_fetch_and_32 (&val32, 0x10);
if (test32 != 24 || val32 != 16) {
fprintf (stderr, "error in opal_atomic_fetch_and_32. expected (24, 16), got (%d, %d)\n", test32, val32);
exit(EXIT_FAILURE);
}
test32 = opal_atomic_or_fetch_32 (&val32, 0x03);
if (test32 != 19 || val32 != 19) {
fprintf (stderr, "error in opal_atomic_or_fetch_32. expected (19, 19), got (%d, %d)\n", test32, val32);
exit(EXIT_FAILURE);
}
test32 = opal_atomic_fetch_or_32 (&val32, 0x04);
if (test32 != 19 || val32 != 23) {
fprintf (stderr, "error in opal_atomic_fetch_or_32. expected (19, 23), got (%d, %d)\n", test32, val32);
exit(EXIT_FAILURE);
}
test32 = opal_atomic_xor_fetch_32 (&val32, 0x03);
if (test32 != 20 || val32 != 20) {
fprintf (stderr, "error in opal_atomic_xor_fetch_32. expected (20, 20), got (%d, %d)\n", test32, val32);
exit(EXIT_FAILURE);
}
test32 = opal_atomic_fetch_xor_32 (&val32, 0x05);
if (test32 != 20 || val32 != 17) {
fprintf (stderr, "error in opal_atomic_fetch_xor_32. expected (20, 17), got (%d, %d)\n", test32, val32);
exit(EXIT_FAILURE);
}
test32 = opal_atomic_sub_fetch_32 (&val32, 14);
if (test32 != 3 || val32 != 3) {
fprintf (stderr, "error in opal_atomic_sub_fetch_32. expected (3, 3), got (%d, %d)\n", test32, val32);
exit(EXIT_FAILURE);
}
test32 = opal_atomic_fetch_xor_32 (&val32, 3);
if (test32 != 3 || val32 != 0) {
fprintf (stderr, "error in opal_atomic_fetch_sub_32. expected (3, 0), got (%d, %d)\n", test32, val32);
exit(EXIT_FAILURE);
}
#if OPAL_HAVE_ATOMIC_MATH_64
test64 = opal_atomic_add_fetch_64 (&val64, 17);
if (test64 != 17 || val64 != 17) {
fprintf (stderr, "error in opal_atomic_add_fetch_64. expected (17, 17), got (%" PRId64 ", %" PRId64 ")\n", test64, val64);
exit(EXIT_FAILURE);
}
test64 = opal_atomic_fetch_add_64 (&val64, 13);
if (test64 != 17 || val64 != 30) {
fprintf (stderr, "error in opal_atomic_fetch_add_64. expected (17, 30), got (%" PRId64 ", %" PRId64 ")\n", test64, val64);
exit(EXIT_FAILURE);
}
test64 = opal_atomic_and_fetch_64 (&val64, 0x18);
if (test64 != 24 || val64 != 24) {
fprintf (stderr, "error in opal_atomic_and_fetch_64. expected (24, 24), got (%" PRId64 ", %" PRId64 ")\n", test64, val64);
exit(EXIT_FAILURE);
}
test64 = opal_atomic_fetch_and_64 (&val64, 0x10);
if (test64 != 24 || val64 != 16) {
fprintf (stderr, "error in opal_atomic_fetch_and_64. expected (24, 16), got (%" PRId64 ", %" PRId64 ")\n", test64, val64);
exit(EXIT_FAILURE);
}
test64 = opal_atomic_or_fetch_64 (&val64, 0x03);
if (test64 != 19 || val64 != 19) {
fprintf (stderr, "error in opal_atomic_or_fetch_64. expected (19, 19), got (%" PRId64 ", %" PRId64 ")\n", test64, val64);
exit(EXIT_FAILURE);
}
test64 = opal_atomic_fetch_or_64 (&val64, 0x04);
if (test64 != 19 || val64 != 23) {
fprintf (stderr, "error in opal_atomic_fetch_or_64. expected (19, 23), got (%" PRId64 ", %" PRId64 ")\n", test64, val64);
exit(EXIT_FAILURE);
}
test64 = opal_atomic_xor_fetch_64 (&val64, 0x03);
if (test64 != 20 || val64 != 20) {
fprintf (stderr, "error in opal_atomic_xor_fetch_64. expected (20, 20), got (%" PRId64 ", %" PRId64 ")\n", test64, val64);
exit(EXIT_FAILURE);
}
test64 = opal_atomic_fetch_xor_64 (&val64, 0x05);
if (test64 != 20 || val64 != 17) {
fprintf (stderr, "error in opal_atomic_fetch_xor_64. expected (20, 17), got (%" PRId64 ", %" PRId64 ")\n", test64, val64);
exit(EXIT_FAILURE);
}
test64 = opal_atomic_sub_fetch_64 (&val64, 14);
if (test64 != 3 || val64 != 3) {
fprintf (stderr, "error in opal_atomic_sub_fetch_64. expected (3, 3), got (%" PRId64 ", %" PRId64 ")\n", test64, val64);
exit(EXIT_FAILURE);
}
test64 = opal_atomic_fetch_xor_64 (&val64, 3);
if (test64 != 3 || val64 != 0) {
fprintf (stderr, "error in opal_atomic_fetch_sub_64. expected (3, 0), got (%" PRId64 ", %" PRId64 ")\n", test64, val64);
exit(EXIT_FAILURE);
}
#endif
ret = atomic_math_test_th(TEST_REPS, num_threads);
if (ret == 77) return ret;
opal_atomic_mb();

4
test/threads/opal_thread.c
Просмотреть файл

@ -36,13 +36,13 @@ static volatile int count = 0;
static void* thr1_run(opal_object_t* obj)
{
(void)opal_atomic_add_fetch(&count, 1);
opal_atomic_add (&count, 1);
return NULL;
}
static void* thr2_run(opal_object_t* obj)
{
(void)opal_atomic_add_fetch(&count, 2);
opal_atomic_add (&count, 2);
return NULL;
}