ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

opal/asm: add opal_atomic_compare_exchange_strong functions

Просмотр исходного кода 
This commit adds a new set of compare-and-exchange functions. These
functions have a signature similar to the functions found in C11. The
old cmpset functions are now deprecated and defined in terms of the
new compare-and-exchange functions. All asm backends have been
updated.

Signed-off-by: Nathan Hjelm <hjelmn@lanl.gov>
Этот коммит содержится в:
Nathan Hjelm 2017-11-08 09:54:54 -07:00
родитель 45db3637af
Коммит 84f63d0aca
14 изменённых файлов: 714 добавлений и 547 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

2
opal/class/opal_fifo.h
Просмотреть файл

@ -76,7 +76,7 @@ static inline bool opal_fifo_is_empty( opal_fifo_t* fifo )
return opal_fifo_head (fifo) == &fifo->opal_fifo_ghost; return opal_fifo_head (fifo) == &fifo->opal_fifo_ghost;
} }
#if OPAL_HAVE_ATOMIC_CMPSET_128 #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128
/* Add one element to the FIFO. We will return the last head of the list /* Add one element to the FIFO. We will return the last head of the list
* to allow the upper level to detect if this element is the first one in the * to allow the upper level to detect if this element is the first one in the

10
opal/class/opal_lifo.h
Просмотреть файл

@ -36,8 +36,8 @@
BEGIN_C_DECLS BEGIN_C_DECLS
/* NTH: temporarily suppress warnings about this not being defined */ /* NTH: temporarily suppress warnings about this not being defined */
#if !defined(OPAL_HAVE_ATOMIC_CMPSET_128) #if !defined(OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128)
#define OPAL_HAVE_ATOMIC_CMPSET_128 0 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 0
#endif #endif
/** /**
@ -50,7 +50,7 @@ union opal_counted_pointer_t {
/** list item pointer */ /** list item pointer */
volatile opal_list_item_t * volatile item; volatile opal_list_item_t * volatile item;
} data; } data;
#if OPAL_HAVE_ATOMIC_CMPSET_128 && HAVE_OPAL_INT128_T #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 && HAVE_OPAL_INT128_T
/** used for atomics when there is a cmpset that can operate on /** used for atomics when there is a cmpset that can operate on
* two 64-bit values */ * two 64-bit values */
opal_int128_t value; opal_int128_t value;
@ -59,7 +59,7 @@ union opal_counted_pointer_t {
typedef union opal_counted_pointer_t opal_counted_pointer_t; typedef union opal_counted_pointer_t opal_counted_pointer_t;
#if OPAL_HAVE_ATOMIC_CMPSET_128 #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128
/* Add one element to the FIFO. We will return the last head of the list /* Add one element to the FIFO. We will return the last head of the list
* to allow the upper level to detect if this element is the first one in the * to allow the upper level to detect if this element is the first one in the
@ -110,7 +110,7 @@ static inline bool opal_lifo_is_empty( opal_lifo_t* lifo )
} }
#if OPAL_HAVE_ATOMIC_CMPSET_128 #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128
/* Add one element to the LIFO. We will return the last head of the list /* Add one element to the LIFO. We will return the last head of the list
* to allow the upper level to detect if this element is the first one in the * to allow the upper level to detect if this element is the first one in the

104
opal/include/opal/sys/arm/atomic.h
Просмотреть файл

@ -1,3 +1,4 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/* /*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology * University Research and Technology
@ -11,6 +12,8 @@
* All rights reserved. * All rights reserved.
* Copyright (c) 2010 IBM Corporation. All rights reserved. * Copyright (c) 2010 IBM Corporation. All rights reserved.
* Copyright (c) 2010 ARM ltd. All rights reserved. * Copyright (c) 2010 ARM ltd. All rights reserved.
* Copyright (c) 2017 Los Alamos National Security, LLC. All rights
* reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
* Additional copyrights may follow * Additional copyrights may follow
@ -104,12 +107,12 @@ void opal_atomic_isync(void)
#if (OPAL_GCC_INLINE_ASSEMBLY && (OPAL_ASM_ARM_VERSION >= 6)) #if (OPAL_GCC_INLINE_ASSEMBLY && (OPAL_ASM_ARM_VERSION >= 6))
#define OPAL_HAVE_ATOMIC_CMPSET_32 1 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
#define OPAL_HAVE_ATOMIC_MATH_32 1 #define OPAL_HAVE_ATOMIC_MATH_32 1
static inline bool opal_atomic_bool_cmpset_32(volatile int32_t *addr, static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
int32_t oldval, int32_t newval)
{ {
int32_t ret, tmp; int32_t prev, tmp;
bool ret;
__asm__ __volatile__ ( __asm__ __volatile__ (
"1: ldrex %0, [%2] \n" "1: ldrex %0, [%2] \n"
@ -120,11 +123,13 @@ static inline bool opal_atomic_bool_cmpset_32(volatile int32_t *addr,
" bne 1b \n" " bne 1b \n"
"2: \n" "2: \n"
: "=&r" (ret), "=&r" (tmp) : "=&r" (prev), "=&r" (tmp)
: "r" (addr), "r" (oldval), "r" (newval) : "r" (addr), "r" (*oldval), "r" (newval)
: "cc", "memory"); : "cc", "memory");
return (ret == oldval); ret = (prev == *oldval);
*oldval = prev;
return ret;
} }
/* these two functions aren't inlined in the non-gcc case because then /* these two functions aren't inlined in the non-gcc case because then
@ -132,51 +137,50 @@ static inline bool opal_atomic_bool_cmpset_32(volatile int32_t *addr,
atomic_?mb can be inlined). Instead, we "inline" them by hand in atomic_?mb can be inlined). Instead, we "inline" them by hand in
the assembly, meaning there is one function call overhead instead the assembly, meaning there is one function call overhead instead
of two */ of two */
static inline bool opal_atomic_bool_cmpset_acq_32(volatile int32_t *addr, static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
int32_t oldval, int32_t newval)
{ {
bool rc; bool rc;
rc = opal_atomic_bool_cmpset_32(addr, oldval, newval); rc = opal_atomic_compare_exchange_strong_32 (addr, oldval, newval);
opal_atomic_rmb(); opal_atomic_rmb();
return rc; return rc;
} }
static inline bool opal_atomic_bool_cmpset_rel_32(volatile int32_t *addr, static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
int32_t oldval, int32_t newval)
{ {
opal_atomic_wmb(); opal_atomic_wmb();
return opal_atomic_bool_cmpset_32(addr, oldval, newval); return opal_atomic_compare_exchange_strong_32 (addr, oldval, newval);
} }
#if (OPAL_ASM_SUPPORT_64BIT == 1) #if (OPAL_ASM_SUPPORT_64BIT == 1)
#define OPAL_HAVE_ATOMIC_CMPSET_64 1 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
static inline bool opal_atomic_bool_cmpset_64(volatile int64_t *addr, static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
int64_t oldval, int64_t newval)
{ {
int64_t ret; int64_t prev;
int tmp; int tmp;
bool ret;
__asm__ __volatile__ (
"1: ldrexd %0, %H0, [%2] \n"
" cmp %0, %3 \n"
" it eq \n"
" cmpeq %H0, %H3 \n"
" bne 2f \n"
" strexd %1, %4, %H4, [%2] \n"
" cmp %1, #0 \n"
" bne 1b \n"
"2: \n"
__asm__ __volatile__ ( : "=&r" (prev), "=&r" (tmp)
"1: ldrexd %0, %H0, [%2] \n" : "r" (addr), "r" (*oldval), "r" (newval)
" cmp %0, %3 \n" : "cc", "memory");
" it eq \n"
" cmpeq %H0, %H3 \n"
" bne 2f \n"
" strexd %1, %4, %H4, [%2] \n"
" cmp %1, #0 \n"
" bne 1b \n"
"2: \n"
: "=&r" (ret), "=&r" (tmp) ret = (prev == *oldval);
: "r" (addr), "r" (oldval), "r" (newval) *oldval = prev;
: "cc", "memory"); return ret;
return (ret == oldval);
} }
/* these two functions aren't inlined in the non-gcc case because then /* these two functions aren't inlined in the non-gcc case because then
@ -184,23 +188,21 @@ static inline bool opal_atomic_bool_cmpset_64(volatile int64_t *addr,
atomic_?mb can be inlined). Instead, we "inline" them by hand in atomic_?mb can be inlined). Instead, we "inline" them by hand in
the assembly, meaning there is one function call overhead instead the assembly, meaning there is one function call overhead instead
of two */ of two */
static inline bool opal_atomic_bool_cmpset_acq_64(volatile int64_t *addr, static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
int64_t oldval, int64_t newval)
{ {
bool rc; bool rc;
rc = opal_atomic_bool_cmpset_64(addr, oldval, newval); rc = opal_atomic_compare_exchange_strong_64 (addr, oldval, newval);
opal_atomic_rmb(); opal_atomic_rmb();
return rc; return rc;
} }
static inline bool opal_atomic_bool_cmpset_rel_64(volatile int64_t *addr, static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
int64_t oldval, int64_t newval)
{ {
opal_atomic_wmb(); opal_atomic_wmb();
return opal_atomic_bool_cmpset_64(addr, oldval, newval); return opal_atomic_compare_exchange_strong_64 (addr, oldval, newval);
} }
#endif #endif
@ -247,30 +249,6 @@ static inline int32_t opal_atomic_sub_32(volatile int32_t* v, int dec)
return t; return t;
} }
#else /* OPAL_ASM_ARM_VERSION <=5 or no GCC inline assembly */
#define OPAL_HAVE_ATOMIC_CMPSET_32 1
#define __kuser_cmpxchg (*((int (*)(int, int, volatile int*))(0xffff0fc0)))
static inline bool opal_atomic_bool_cmpset_32(volatile int32_t *addr,
int32_t oldval, int32_t newval)
{
return !(__kuser_cmpxchg(oldval, newval, addr));
}
static inline bool opal_atomic_bool_cmpset_acq_32(volatile int32_t *addr,
int32_t oldval, int32_t newval)
{
/* kernel function includes all necessary memory barriers */
return opal_atomic_bool_cmpset_32(addr, oldval, newval);
}
static inline bool opal_atomic_bool_cmpset_rel_32(volatile int32_t *addr,
int32_t oldval, int32_t newval)
{
/* kernel function includes all necessary memory barriers */
return opal_atomic_bool_cmpset_32(addr, oldval, newval);
}
#endif #endif
#endif /* ! OPAL_SYS_ARCH_ATOMIC_H */ #endif /* ! OPAL_SYS_ARCH_ATOMIC_H */

88
opal/include/opal/sys/arm64/atomic.h
Просмотреть файл

@ -29,10 +29,10 @@
#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1 #define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
#define OPAL_HAVE_ATOMIC_LLSC_32 1 #define OPAL_HAVE_ATOMIC_LLSC_32 1
#define OPAL_HAVE_ATOMIC_CMPSET_32 1 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
#define OPAL_HAVE_ATOMIC_SWAP_32 1 #define OPAL_HAVE_ATOMIC_SWAP_32 1
#define OPAL_HAVE_ATOMIC_MATH_32 1 #define OPAL_HAVE_ATOMIC_MATH_32 1
#define OPAL_HAVE_ATOMIC_CMPSET_64 1 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
#define OPAL_HAVE_ATOMIC_SWAP_64 1 #define OPAL_HAVE_ATOMIC_SWAP_64 1
#define OPAL_HAVE_ATOMIC_LLSC_64 1 #define OPAL_HAVE_ATOMIC_LLSC_64 1
#define OPAL_HAVE_ATOMIC_ADD_32 1 #define OPAL_HAVE_ATOMIC_ADD_32 1
@ -82,10 +82,10 @@ static inline void opal_atomic_isync (void)
* *
*********************************************************************/ *********************************************************************/
static inline bool opal_atomic_bool_cmpset_32(volatile int32_t *addr, static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
int32_t oldval, int32_t newval)
{ {
int32_t ret, tmp; int32_t prev, tmp;
bool ret;
__asm__ __volatile__ ("1: ldaxr %w0, [%2] \n" __asm__ __volatile__ ("1: ldaxr %w0, [%2] \n"
" cmp %w0, %w3 \n" " cmp %w0, %w3 \n"
@ -93,11 +93,13 @@ static inline bool opal_atomic_bool_cmpset_32(volatile int32_t *addr,
" stxr %w1, %w4, [%2] \n" " stxr %w1, %w4, [%2] \n"
" cbnz %w1, 1b \n" " cbnz %w1, 1b \n"
"2: \n" "2: \n"
: "=&r" (ret), "=&r" (tmp) : "=&r" (prev), "=&r" (tmp)
: "r" (addr), "r" (oldval), "r" (newval) : "r" (addr), "r" (*oldval), "r" (newval)
: "cc", "memory"); : "cc", "memory");
return (ret == oldval); ret = (prev == *oldval);
*oldval = prev;
return ret;
} }
static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, int32_t newval) static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, int32_t newval)
@ -119,10 +121,10 @@ static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, int32_t newval
atomic_?mb can be inlined). Instead, we "inline" them by hand in atomic_?mb can be inlined). Instead, we "inline" them by hand in
the assembly, meaning there is one function call overhead instead the assembly, meaning there is one function call overhead instead
of two */ of two */
static inline bool opal_atomic_bool_cmpset_acq_32(volatile int32_t *addr, static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
int32_t oldval, int32_t newval)
{ {
int32_t ret, tmp; int32_t prev, tmp;
bool ret;
__asm__ __volatile__ ("1: ldaxr %w0, [%2] \n" __asm__ __volatile__ ("1: ldaxr %w0, [%2] \n"
" cmp %w0, %w3 \n" " cmp %w0, %w3 \n"
@ -130,18 +132,20 @@ static inline bool opal_atomic_bool_cmpset_acq_32(volatile int32_t *addr,
" stxr %w1, %w4, [%2] \n" " stxr %w1, %w4, [%2] \n"
" cbnz %w1, 1b \n" " cbnz %w1, 1b \n"
"2: \n" "2: \n"
: "=&r" (ret), "=&r" (tmp) : "=&r" (prev), "=&r" (tmp)
: "r" (addr), "r" (oldval), "r" (newval) : "r" (addr), "r" (*oldval), "r" (newval)
: "cc", "memory"); : "cc", "memory");
return (ret == oldval); ret = (prev == *oldval);
*oldval = prev;
return ret;
} }
static inline bool opal_atomic_bool_cmpset_rel_32(volatile int32_t *addr, static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
int32_t oldval, int32_t newval)
{ {
int32_t ret, tmp; int32_t prev, tmp;
bool ret;
__asm__ __volatile__ ("1: ldxr %w0, [%2] \n" __asm__ __volatile__ ("1: ldxr %w0, [%2] \n"
" cmp %w0, %w3 \n" " cmp %w0, %w3 \n"
@ -149,11 +153,13 @@ static inline bool opal_atomic_bool_cmpset_rel_32(volatile int32_t *addr,
" stlxr %w1, %w4, [%2] \n" " stlxr %w1, %w4, [%2] \n"
" cbnz %w1, 1b \n" " cbnz %w1, 1b \n"
"2: \n" "2: \n"
: "=&r" (ret), "=&r" (tmp) : "=&r" (prev), "=&r" (tmp)
: "r" (addr), "r" (oldval), "r" (newval) : "r" (addr), "r" (*oldval), "r" (newval)
: "cc", "memory"); : "cc", "memory");
return (ret == oldval); ret = (prev == *oldval);
*oldval = prev;
return ret;
} }
static inline int32_t opal_atomic_ll_32 (volatile int32_t *addr) static inline int32_t opal_atomic_ll_32 (volatile int32_t *addr)
@ -179,11 +185,11 @@ static inline int opal_atomic_sc_32 (volatile int32_t *addr, int32_t newval)
return ret == 0; return ret == 0;
} }
static inline bool opal_atomic_bool_cmpset_64(volatile int64_t *addr, static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
int64_t oldval, int64_t newval)
{ {
int64_t ret; int64_t prev;
int tmp; int tmp;
bool ret;
__asm__ __volatile__ ("1: ldaxr %0, [%2] \n" __asm__ __volatile__ ("1: ldaxr %0, [%2] \n"
" cmp %0, %3 \n" " cmp %0, %3 \n"
@ -191,11 +197,13 @@ static inline bool opal_atomic_bool_cmpset_64(volatile int64_t *addr,
" stxr %w1, %4, [%2] \n" " stxr %w1, %4, [%2] \n"
" cbnz %w1, 1b \n" " cbnz %w1, 1b \n"
"2: \n" "2: \n"
: "=&r" (ret), "=&r" (tmp) : "=&r" (prev), "=&r" (tmp)
: "r" (addr), "r" (oldval), "r" (newval) : "r" (addr), "r" (*oldval), "r" (newval)
: "cc", "memory"); : "cc", "memory");
return (ret == oldval); ret = (prev == oldval);
*oldval = prev;
return ret;
} }
static inline int64_t opal_atomic_swap_64 (volatile int64_t *addr, int64_t newval) static inline int64_t opal_atomic_swap_64 (volatile int64_t *addr, int64_t newval)
@ -218,11 +226,11 @@ static inline int64_t opal_atomic_swap_64 (volatile int64_t *addr, int64_t newva
atomic_?mb can be inlined). Instead, we "inline" them by hand in atomic_?mb can be inlined). Instead, we "inline" them by hand in
the assembly, meaning there is one function call overhead instead the assembly, meaning there is one function call overhead instead
of two */ of two */
static inline bool opal_atomic_bool_cmpset_acq_64(volatile int64_t *addr, static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
int64_t oldval, int64_t newval)
{ {
int64_t ret; int64_t prev;
int tmp; int tmp;
bool ret;
__asm__ __volatile__ ("1: ldaxr %0, [%2] \n" __asm__ __volatile__ ("1: ldaxr %0, [%2] \n"
" cmp %0, %3 \n" " cmp %0, %3 \n"
@ -230,19 +238,21 @@ static inline bool opal_atomic_bool_cmpset_acq_64(volatile int64_t *addr,
" stxr %w1, %4, [%2] \n" " stxr %w1, %4, [%2] \n"
" cbnz %w1, 1b \n" " cbnz %w1, 1b \n"
"2: \n" "2: \n"
: "=&r" (ret), "=&r" (tmp) : "=&r" (prev), "=&r" (tmp)
: "r" (addr), "r" (oldval), "r" (newval) : "r" (addr), "r" (*oldval), "r" (newval)
: "cc", "memory"); : "cc", "memory");
return (ret == oldval); ret = (prev == oldval);
*oldval = prev;
return ret;
} }
static inline bool opal_atomic_bool_cmpset_rel_64(volatile int64_t *addr, static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
int64_t oldval, int64_t newval)
{ {
int64_t ret; int64_t prev;
int tmp; int tmp;
bool ret;
__asm__ __volatile__ ("1: ldxr %0, [%2] \n" __asm__ __volatile__ ("1: ldxr %0, [%2] \n"
" cmp %0, %3 \n" " cmp %0, %3 \n"
@ -250,11 +260,13 @@ static inline bool opal_atomic_bool_cmpset_rel_64(volatile int64_t *addr,
" stlxr %w1, %4, [%2] \n" " stlxr %w1, %4, [%2] \n"
" cbnz %w1, 1b \n" " cbnz %w1, 1b \n"
"2: \n" "2: \n"
: "=&r" (ret), "=&r" (tmp) : "=&r" (prev), "=&r" (tmp)
: "r" (addr), "r" (oldval), "r" (newval) : "r" (addr), "r" (*oldval), "r" (newval)
: "cc", "memory"); : "cc", "memory");
return (ret == oldval); ret = (prev == oldval);
*oldval = prev;
return ret;
} }
static inline int64_t opal_atomic_ll_64 (volatile int64_t *addr) static inline int64_t opal_atomic_ll_64 (volatile int64_t *addr)

177
opal/include/opal/sys/atomic.h
Просмотреть файл

@ -40,11 +40,11 @@
* *
* - \c OPAL_HAVE_ATOMIC_MEM_BARRIER atomic memory barriers * - \c OPAL_HAVE_ATOMIC_MEM_BARRIER atomic memory barriers
* - \c OPAL_HAVE_ATOMIC_SPINLOCKS atomic spinlocks * - \c OPAL_HAVE_ATOMIC_SPINLOCKS atomic spinlocks
* - \c OPAL_HAVE_ATOMIC_MATH_32 if 32 bit add/sub/cmpset can be done "atomicly" * - \c OPAL_HAVE_ATOMIC_MATH_32 if 32 bit add/sub/compare-exchange can be done "atomicly"
* - \c OPAL_HAVE_ATOMIC_MATH_64 if 64 bit add/sub/cmpset can be done "atomicly" * - \c OPAL_HAVE_ATOMIC_MATH_64 if 64 bit add/sub/compare-exchange can be done "atomicly"
* *
* Note that for the Atomic math, atomic add/sub may be implemented as * Note that for the Atomic math, atomic add/sub may be implemented as
* C code using opal_atomic_bool_cmpset. The appearance of atomic * C code using opal_atomic_compare_exchange. The appearance of atomic
* operation will be upheld in these cases. * operation will be upheld in these cases.
*/ */
@ -107,8 +107,8 @@ typedef struct opal_atomic_lock_t opal_atomic_lock_t;
*********************************************************************/ *********************************************************************/
#if !OPAL_GCC_INLINE_ASSEMBLY #if !OPAL_GCC_INLINE_ASSEMBLY
#define OPAL_HAVE_INLINE_ATOMIC_MEM_BARRIER 0 #define OPAL_HAVE_INLINE_ATOMIC_MEM_BARRIER 0
#define OPAL_HAVE_INLINE_ATOMIC_CMPSET_32 0 #define OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32 0
#define OPAL_HAVE_INLINE_ATOMIC_CMPSET_64 0 #define OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64 0
#define OPAL_HAVE_INLINE_ATOMIC_ADD_32 0 #define OPAL_HAVE_INLINE_ATOMIC_ADD_32 0
#define OPAL_HAVE_INLINE_ATOMIC_AND_32 0 #define OPAL_HAVE_INLINE_ATOMIC_AND_32 0
#define OPAL_HAVE_INLINE_ATOMIC_OR_32 0 #define OPAL_HAVE_INLINE_ATOMIC_OR_32 0
@ -123,8 +123,8 @@ typedef struct opal_atomic_lock_t opal_atomic_lock_t;
#define OPAL_HAVE_INLINE_ATOMIC_SWAP_64 0 #define OPAL_HAVE_INLINE_ATOMIC_SWAP_64 0
#else #else
#define OPAL_HAVE_INLINE_ATOMIC_MEM_BARRIER 1 #define OPAL_HAVE_INLINE_ATOMIC_MEM_BARRIER 1
#define OPAL_HAVE_INLINE_ATOMIC_CMPSET_32 1 #define OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32 1
#define OPAL_HAVE_INLINE_ATOMIC_CMPSET_64 1 #define OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64 1
#define OPAL_HAVE_INLINE_ATOMIC_ADD_32 1 #define OPAL_HAVE_INLINE_ATOMIC_ADD_32 1
#define OPAL_HAVE_INLINE_ATOMIC_AND_32 1 #define OPAL_HAVE_INLINE_ATOMIC_AND_32 1
#define OPAL_HAVE_INLINE_ATOMIC_OR_32 1 #define OPAL_HAVE_INLINE_ATOMIC_OR_32 1
@ -187,14 +187,14 @@ enum {
/* compare and set operations can't really be emulated from software, /* compare and set operations can't really be emulated from software,
so if these defines aren't already set, they should be set to 0 so if these defines aren't already set, they should be set to 0
now */ now */
#ifndef OPAL_HAVE_ATOMIC_CMPSET_32 #ifndef OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
#define OPAL_HAVE_ATOMIC_CMPSET_32 0 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 0
#endif #endif
#ifndef OPAL_HAVE_ATOMIC_CMPSET_64 #ifndef OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
#define OPAL_HAVE_ATOMIC_CMPSET_64 0 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 0
#endif #endif
#ifndef OPAL_HAVE_ATOMIC_CMPSET_128 #ifndef OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128
#define OPAL_HAVE_ATOMIC_CMPSET_128 0 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 0
#endif #endif
#ifndef OPAL_HAVE_ATOMIC_LLSC_32 #ifndef OPAL_HAVE_ATOMIC_LLSC_32
#define OPAL_HAVE_ATOMIC_LLSC_32 0 #define OPAL_HAVE_ATOMIC_LLSC_32 0
@ -270,7 +270,7 @@ void opal_atomic_wmb(void);
/********************************************************************** /**********************************************************************
* *
* Atomic spinlocks - always inlined, if have atomic cmpset * Atomic spinlocks - always inlined, if have atomic compare-and-swap
* *
*********************************************************************/ *********************************************************************/
@ -280,7 +280,7 @@ void opal_atomic_wmb(void);
#define OPAL_HAVE_ATOMIC_SPINLOCKS 0 #define OPAL_HAVE_ATOMIC_SPINLOCKS 0
#endif #endif
#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_SPINLOCKS || (OPAL_HAVE_ATOMIC_CMPSET_32 || OPAL_HAVE_ATOMIC_CMPSET_64) #if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_SPINLOCKS || (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
/** /**
* Initialize a lock to value * Initialize a lock to value
@ -330,7 +330,7 @@ void opal_atomic_unlock(opal_atomic_lock_t *lock);
#if OPAL_HAVE_ATOMIC_SPINLOCKS == 0 #if OPAL_HAVE_ATOMIC_SPINLOCKS == 0
#undef OPAL_HAVE_ATOMIC_SPINLOCKS #undef OPAL_HAVE_ATOMIC_SPINLOCKS
#define OPAL_HAVE_ATOMIC_SPINLOCKS (OPAL_HAVE_ATOMIC_CMPSET_32 || OPAL_HAVE_ATOMIC_CMPSET_64) #define OPAL_HAVE_ATOMIC_SPINLOCKS (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
#define OPAL_NEED_INLINE_ATOMIC_SPINLOCKS 1 #define OPAL_NEED_INLINE_ATOMIC_SPINLOCKS 1
#endif #endif
@ -353,25 +353,40 @@ static inline
bool opal_atomic_bool_cmpset_32(volatile int32_t *addr, int32_t oldval, bool opal_atomic_bool_cmpset_32(volatile int32_t *addr, int32_t oldval,
int32_t newval); int32_t newval);
#if OPAL_HAVE_INLINE_ATOMIC_CMPSET_32
static inline static inline
#endif
bool opal_atomic_bool_cmpset_acq_32(volatile int32_t *addr, int32_t oldval, bool opal_atomic_bool_cmpset_acq_32(volatile int32_t *addr, int32_t oldval,
int32_t newval); int32_t newval);
#if OPAL_HAVE_INLINE_ATOMIC_CMPSET_32
static inline static inline
#endif
bool opal_atomic_bool_cmpset_rel_32(volatile int32_t *addr, int32_t oldval, bool opal_atomic_bool_cmpset_rel_32(volatile int32_t *addr, int32_t oldval,
int32_t newval); int32_t newval);
#endif #endif
#if !defined(OPAL_HAVE_ATOMIC_CMPSET_64) && !defined(DOXYGEN) #if !defined(OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64) && !defined(DOXYGEN)
#define OPAL_HAVE_ATOMIC_CMPSET_64 0 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 0
#endif #endif
#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_CMPSET_64 #if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
#if OPAL_HAVE_INLINE_ATOMIC_CMPSET_64
static inline
#endif
bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval,
int64_t newval);
#if OPAL_HAVE_INLINE_ATOMIC_CMPSET_64
static inline
#endif
bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval,
int64_t newval);
#if OPAL_HAVE_INLINE_ATOMIC_CMPSET_64
static inline
#endif
bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval,
int64_t newval);
/* XXX -- DEPRECATED -- XXX -- Legacy cmpset functions */
#if OPAL_HAVE_INLINE_ATOMIC_CMPSET_64 #if OPAL_HAVE_INLINE_ATOMIC_CMPSET_64
static inline static inline
#endif #endif
@ -397,35 +412,35 @@ bool opal_atomic_bool_cmpset_rel_64(volatile int64_t *addr, int64_t oldval,
#define OPAL_HAVE_ATOMIC_MATH_32 0 #define OPAL_HAVE_ATOMIC_MATH_32 0
#endif #endif
#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_CMPSET_32 #if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
/* OPAL_HAVE_INLINE_ATOMIC_*_32 will be 1 if <arch>/atomic.h provides /* OPAL_HAVE_INLINE_ATOMIC_*_32 will be 1 if <arch>/atomic.h provides
a static inline version of it (in assembly). If we have to fall a static inline version of it (in assembly). If we have to fall
back on cmpset 32, that too will be inline. */ back on compare-exchange 32, that too will be inline. */
#if OPAL_HAVE_INLINE_ATOMIC_ADD_32 || (!defined(OPAL_HAVE_ATOMIC_ADD_32) && OPAL_HAVE_ATOMIC_CMPSET_32) #if OPAL_HAVE_INLINE_ATOMIC_ADD_32 || (!defined(OPAL_HAVE_ATOMIC_ADD_32) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32)
static inline static inline
#endif #endif
int32_t opal_atomic_add_32(volatile int32_t *addr, int delta); int32_t opal_atomic_add_32(volatile int32_t *addr, int delta);
#if OPAL_HAVE_INLINE_ATOMIC_AND_32 || (!defined(OPAL_HAVE_ATOMIC_AND_32) && OPAL_HAVE_ATOMIC_CMPSET_32) #if OPAL_HAVE_INLINE_ATOMIC_AND_32 || (!defined(OPAL_HAVE_ATOMIC_AND_32) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32)
static inline static inline
#endif #endif
int32_t opal_atomic_and_32(volatile int32_t *addr, int32_t value); int32_t opal_atomic_and_32(volatile int32_t *addr, int32_t value);
#if OPAL_HAVE_INLINE_ATOMIC_OR_32 || (!defined(OPAL_HAVE_ATOMIC_OR_32) && OPAL_HAVE_ATOMIC_CMPSET_32) #if OPAL_HAVE_INLINE_ATOMIC_OR_32 || (!defined(OPAL_HAVE_ATOMIC_OR_32) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32)
static inline static inline
#endif #endif
int32_t opal_atomic_or_32(volatile int32_t *addr, int32_t value); int32_t opal_atomic_or_32(volatile int32_t *addr, int32_t value);
#if OPAL_HAVE_INLINE_ATOMIC_XOR_32 || (!defined(OPAL_HAVE_ATOMIC_XOR_32) && OPAL_HAVE_ATOMIC_CMPSET_32) #if OPAL_HAVE_INLINE_ATOMIC_XOR_32 || (!defined(OPAL_HAVE_ATOMIC_XOR_32) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32)
static inline static inline
#endif #endif
int32_t opal_atomic_xor_32(volatile int32_t *addr, int32_t value); int32_t opal_atomic_xor_32(volatile int32_t *addr, int32_t value);
/* OPAL_HAVE_INLINE_ATOMIC_*_32 will be 1 if <arch>/atomic.h provides /* OPAL_HAVE_INLINE_ATOMIC_*_32 will be 1 if <arch>/atomic.h provides
a static inline version of it (in assembly). If we have to fall a static inline version of it (in assembly). If we have to fall
back to cmpset 32, that too will be inline. */ back to compare-exchange 32, that too will be inline. */
#if OPAL_HAVE_INLINE_ATOMIC_SUB_32 || (!defined(OPAL_HAVE_ATOMIC_ADD_32) && OPAL_HAVE_ATOMIC_CMPSET_32) #if OPAL_HAVE_INLINE_ATOMIC_SUB_32 || (!defined(OPAL_HAVE_ATOMIC_ADD_32) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32)
static inline static inline
#endif #endif
int32_t opal_atomic_sub_32(volatile int32_t *addr, int delta); int32_t opal_atomic_sub_32(volatile int32_t *addr, int delta);
@ -435,7 +450,7 @@ int32_t opal_atomic_sub_32(volatile int32_t *addr, int delta);
#if ! OPAL_HAVE_ATOMIC_MATH_32 #if ! OPAL_HAVE_ATOMIC_MATH_32
/* fix up the value of opal_have_atomic_math_32 to allow for C versions */ /* fix up the value of opal_have_atomic_math_32 to allow for C versions */
#undef OPAL_HAVE_ATOMIC_MATH_32 #undef OPAL_HAVE_ATOMIC_MATH_32
#define OPAL_HAVE_ATOMIC_MATH_32 OPAL_HAVE_ATOMIC_CMPSET_32 #define OPAL_HAVE_ATOMIC_MATH_32 OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
#endif #endif
#ifndef OPAL_HAVE_ATOMIC_MATH_64 #ifndef OPAL_HAVE_ATOMIC_MATH_64
@ -443,35 +458,35 @@ int32_t opal_atomic_sub_32(volatile int32_t *addr, int delta);
#define OPAL_HAVE_ATOMIC_MATH_64 0 #define OPAL_HAVE_ATOMIC_MATH_64 0
#endif #endif
#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MATH_64 || OPAL_HAVE_ATOMIC_CMPSET_64 #if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MATH_64 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
/* OPAL_HAVE_INLINE_ATOMIC_*_64 will be 1 if <arch>/atomic.h provides /* OPAL_HAVE_INLINE_ATOMIC_*_64 will be 1 if <arch>/atomic.h provides
a static inline version of it (in assembly). If we have to fall a static inline version of it (in assembly). If we have to fall
back to cmpset 64, that too will be inline */ back to compare-exchange 64, that too will be inline */
#if OPAL_HAVE_INLINE_ATOMIC_ADD_64 || (!defined(OPAL_HAVE_ATOMIC_ADD_64) && OPAL_HAVE_ATOMIC_CMPSET_64) #if OPAL_HAVE_INLINE_ATOMIC_ADD_64 || (!defined(OPAL_HAVE_ATOMIC_ADD_64) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
static inline static inline
#endif #endif
int64_t opal_atomic_add_64(volatile int64_t *addr, int64_t delta); int64_t opal_atomic_add_64(volatile int64_t *addr, int64_t delta);
#if OPAL_HAVE_INLINE_ATOMIC_AND_64 || (!defined(OPAL_HAVE_ATOMIC_AND_64) && OPAL_HAVE_ATOMIC_CMPSET_64) #if OPAL_HAVE_INLINE_ATOMIC_AND_64 || (!defined(OPAL_HAVE_ATOMIC_AND_64) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
static inline static inline
#endif #endif
int64_t opal_atomic_and_64(volatile int64_t *addr, int64_t value); int64_t opal_atomic_and_64(volatile int64_t *addr, int64_t value);
#if OPAL_HAVE_INLINE_ATOMIC_OR_64 || (!defined(OPAL_HAVE_ATOMIC_OR_64) && OPAL_HAVE_ATOMIC_CMPSET_64) #if OPAL_HAVE_INLINE_ATOMIC_OR_64 || (!defined(OPAL_HAVE_ATOMIC_OR_64) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
static inline static inline
#endif #endif
int64_t opal_atomic_or_64(volatile int64_t *addr, int64_t value); int64_t opal_atomic_or_64(volatile int64_t *addr, int64_t value);
#if OPAL_HAVE_INLINE_ATOMIC_XOR_64 || (!defined(OPAL_HAVE_ATOMIC_XOR_64) && OPAL_HAVE_ATOMIC_CMPSET_64) #if OPAL_HAVE_INLINE_ATOMIC_XOR_64 || (!defined(OPAL_HAVE_ATOMIC_XOR_64) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
static inline static inline
#endif #endif
int64_t opal_atomic_xor_64(volatile int64_t *addr, int64_t value); int64_t opal_atomic_xor_64(volatile int64_t *addr, int64_t value);
/* OPAL_HAVE_INLINE_ATOMIC_*_64 will be 1 if <arch>/atomic.h provides /* OPAL_HAVE_INLINE_ATOMIC_*_64 will be 1 if <arch>/atomic.h provides
a static inline version of it (in assembly). If we have to fall a static inline version of it (in assembly). If we have to fall
back to cmpset 64, that too will be inline */ back to compare-exchange 64, that too will be inline */
#if OPAL_HAVE_INLINE_ATOMIC_SUB_64 || (!defined(OPAL_HAVE_ATOMIC_ADD_64) && OPAL_HAVE_ATOMIC_CMPSET_64) #if OPAL_HAVE_INLINE_ATOMIC_SUB_64 || (!defined(OPAL_HAVE_ATOMIC_ADD_64) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
static inline static inline
#endif #endif
int64_t opal_atomic_sub_64(volatile int64_t *addr, int64_t delta); int64_t opal_atomic_sub_64(volatile int64_t *addr, int64_t delta);
@ -481,7 +496,7 @@ int64_t opal_atomic_sub_64(volatile int64_t *addr, int64_t delta);
#if ! OPAL_HAVE_ATOMIC_MATH_64 #if ! OPAL_HAVE_ATOMIC_MATH_64
/* fix up the value of opal_have_atomic_math_64 to allow for C versions */ /* fix up the value of opal_have_atomic_math_64 to allow for C versions */
#undef OPAL_HAVE_ATOMIC_MATH_64 #undef OPAL_HAVE_ATOMIC_MATH_64
#define OPAL_HAVE_ATOMIC_MATH_64 OPAL_HAVE_ATOMIC_CMPSET_64 #define OPAL_HAVE_ATOMIC_MATH_64 OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
#endif #endif
/* provide a size_t add/subtract. When in debug mode, make it an /* provide a size_t add/subtract. When in debug mode, make it an
@ -524,9 +539,26 @@ opal_atomic_sub_size_t(volatile size_t *addr, size_t delta)
#endif #endif
#endif #endif
#if defined(DOXYGEN) || (OPAL_HAVE_ATOMIC_CMPSET_32 || OPAL_HAVE_ATOMIC_CMPSET_64) #if defined(DOXYGEN) || (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
/* these are always done with inline functions, so always mark as /* these are always done with inline functions, so always mark as
static inline */ static inline */
static inline bool opal_atomic_compare_exchange_strong_xx (volatile void *addr, void *oldval,
int64_t newval, size_t length);
static inline bool opal_atomic_compare_exchange_strong_acq_xx (volatile void *addr, void *oldval,
int64_t newval, size_t length);
static inline bool opal_atomic_compare_exchange_strong_rel_xx (volatile void *addr, void *oldval,
int64_t newval, size_t length);
static inline bool opal_atomic_compare_exchange_strong_ptr (volatile void* addr, void *oldval,
void *newval);
static inline bool opal_atomic_compare_exchange_strong_acq_ptr (volatile void* addr, void *oldval,
void *newval);
static inline bool opal_atomic_compare_exchange_strong_rel_ptr (volatile void* addr, void *oldval,
void *newval);
/* XXX -- DEPRECATED -- XXX -- Define legacy cmpset functions */
static inline bool opal_atomic_bool_cmpset_xx(volatile void* addr, int64_t oldval, static inline bool opal_atomic_bool_cmpset_xx(volatile void* addr, int64_t oldval,
int64_t newval, size_t length); int64_t newval, size_t length);
static inline bool opal_atomic_bool_cmpset_acq_xx(volatile void* addr, static inline bool opal_atomic_bool_cmpset_acq_xx(volatile void* addr,
@ -546,6 +578,61 @@ static inline bool opal_atomic_bool_cmpset_rel_ptr(volatile void* addr,
void* oldval, void* oldval,
void* newval); void* newval);
/**
* Atomic compare and set of generic type with relaxed semantics. This
* macro detect at compile time the type of the first argument and
* choose the correct function to be called.
*
* \note This macro should only be used for integer types.
*
* @param addr Address of <TYPE>.
* @param oldval Comparison value address of <TYPE>.
* @param newval New value to set if comparision is true <TYPE>.
*
* See opal_atomic_compare_exchange_* for pseudo-code.
*/
#define opal_atomic_compare_exchange_strong( ADDR, OLDVAL, NEWVAL ) \
opal_atomic_compare_exchange_strong_xx( (volatile void*)(ADDR), (void *)(OLDVAL), \
(intptr_t)(NEWVAL), sizeof(*(ADDR)) )
/**
* Atomic compare and set of generic type with acquire semantics. This
* macro detect at compile time the type of the first argument and
* choose the correct function to be called.
*
* \note This macro should only be used for integer types.
*
* @param addr Address of <TYPE>.
* @param oldval Comparison value address of <TYPE>.
* @param newval New value to set if comparision is true <TYPE>.
*
* See opal_atomic_compare_exchange_acq_* for pseudo-code.
*/
#define opal_atomic_compare_exchange_strong_acq( ADDR, OLDVAL, NEWVAL ) \
opal_atomic_compare_exchange_strong_acq_xx( (volatile void*)(ADDR), (void *)(OLDVAL), \
(intptr_t)(NEWVAL), sizeof(*(ADDR)) )
/**
* Atomic compare and set of generic type with release semantics. This
* macro detect at compile time the type of the first argument and
* choose the correct function to be called.
*
* \note This macro should only be used for integer types.
*
* @param addr Address of <TYPE>.
* @param oldval Comparison value address of <TYPE>.
* @param newval New value to set if comparision is true <TYPE>.
*
* See opal_atomic_compare_exchange_rel_* for pseudo-code.
*/
#define opal_atomic_compare_exchange_strong_rel( ADDR, OLDVAL, NEWVAL ) \
opal_atomic_compare_exchange_strong_rel_xx( (volatile void*)(ADDR), (void *)(OLDVAL), \
(intptr_t)(NEWVAL), sizeof(*(ADDR)) )
/* XXX -- DEPRECATED -- XXX -- Define legacy cmpset functions */
/** /**
* Atomic compare and set of pointer with relaxed semantics. This * Atomic compare and set of pointer with relaxed semantics. This
* macro detect at compile time the type of the first argument and * macro detect at compile time the type of the first argument and
@ -598,7 +685,7 @@ static inline bool opal_atomic_bool_cmpset_rel_ptr(volatile void* addr,
opal_atomic_bool_cmpset_rel_xx( (volatile void*)(ADDR), (int64_t)(OLDVAL), \ opal_atomic_bool_cmpset_rel_xx( (volatile void*)(ADDR), (int64_t)(OLDVAL), \
(int64_t)(NEWVAL), sizeof(*(ADDR)) ) (int64_t)(NEWVAL), sizeof(*(ADDR)) )
#endif /* (OPAL_HAVE_ATOMIC_CMPSET_32 || OPAL_HAVE_ATOMIC_CMPSET_64) */ #endif /* (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64) */
#if defined(DOXYGEN) || (OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64) #if defined(DOXYGEN) || (OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64)
@ -606,10 +693,10 @@ static inline void opal_atomic_add_xx(volatile void* addr,
int32_t value, size_t length); int32_t value, size_t length);
static inline void opal_atomic_sub_xx(volatile void* addr, static inline void opal_atomic_sub_xx(volatile void* addr,
int32_t value, size_t length); int32_t value, size_t length);
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_CMPSET_32 #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
static inline int32_t opal_atomic_add_ptr( volatile void* addr, void* delta ); static inline int32_t opal_atomic_add_ptr( volatile void* addr, void* delta );
static inline int32_t opal_atomic_sub_ptr( volatile void* addr, void* delta ); static inline int32_t opal_atomic_sub_ptr( volatile void* addr, void* delta );
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_CMPSET_64 #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
static inline int64_t opal_atomic_add_ptr( volatile void* addr, void* delta ); static inline int64_t opal_atomic_add_ptr( volatile void* addr, void* delta );
static inline int64_t opal_atomic_sub_ptr( volatile void* addr, void* delta ); static inline int64_t opal_atomic_sub_ptr( volatile void* addr, void* delta );
#else #else

285
opal/include/opal/sys/atomic_impl.h
Просмотреть файл

@ -34,10 +34,22 @@
* *
* Some architectures do not provide support for the 64 bits * Some architectures do not provide support for the 64 bits
* atomic operations. Until we find a better solution let's just * atomic operations. Until we find a better solution let's just
* undefine all those functions if there is no 64 bit cmpset * undefine all those functions if there is no 64 bit compare-exchange
* *
*********************************************************************/ *********************************************************************/
#if OPAL_HAVE_ATOMIC_CMPSET_32 #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
#define OPAL_ATOMIC_DEFINE_CMPXCG_OP(type, bits, operand, name) \
static inline type opal_atomic_ ## name ## _ ## bits (volatile type *addr, type value) \
{ \
type oldval, newval; \
do { \
oldval = *addr; \
newval = oldval operand value; \
} while (!opal_atomic_compare_exchange_strong_ ## bits (addr, &oldval, newval)); \
\
return newval; \
}
#if !defined(OPAL_HAVE_ATOMIC_SWAP_32) #if !defined(OPAL_HAVE_ATOMIC_SWAP_32)
#define OPAL_HAVE_ATOMIC_SWAP_32 1 #define OPAL_HAVE_ATOMIC_SWAP_32 1
@ -55,79 +67,44 @@ static inline int32_t opal_atomic_swap_32(volatile int32_t *addr,
#if !defined(OPAL_HAVE_ATOMIC_ADD_32) #if !defined(OPAL_HAVE_ATOMIC_ADD_32)
#define OPAL_HAVE_ATOMIC_ADD_32 1 #define OPAL_HAVE_ATOMIC_ADD_32 1
static inline int32_t
opal_atomic_add_32(volatile int32_t *addr, int delta)
{
int32_t oldval;
do { OPAL_ATOMIC_DEFINE_CMPXCG_OP(int32_t, 32, +, add)
oldval = *addr;
} while (!opal_atomic_bool_cmpset_32(addr, oldval, oldval + delta));
return (oldval + delta);
}
#endif /* OPAL_HAVE_ATOMIC_ADD_32 */ #endif /* OPAL_HAVE_ATOMIC_ADD_32 */
#if !defined(OPAL_HAVE_ATOMIC_AND_32) #if !defined(OPAL_HAVE_ATOMIC_AND_32)
#define OPAL_HAVE_ATOMIC_AND_32 1 #define OPAL_HAVE_ATOMIC_AND_32 1
static inline int32_t
opal_atomic_and_32(volatile int32_t *addr, int32_t value)
{
int32_t oldval;
do { OPAL_ATOMIC_DEFINE_CMPXCG_OP(int32_t, 32, &, and)
oldval = *addr;
} while (!opal_atomic_bool_cmpset_32(addr, oldval, oldval & value));
return (oldval & value);
}
#endif /* OPAL_HAVE_ATOMIC_AND_32 */ #endif /* OPAL_HAVE_ATOMIC_AND_32 */
#if !defined(OPAL_HAVE_ATOMIC_OR_32) #if !defined(OPAL_HAVE_ATOMIC_OR_32)
#define OPAL_HAVE_ATOMIC_OR_32 1 #define OPAL_HAVE_ATOMIC_OR_32 1
static inline int32_t
opal_atomic_or_32(volatile int32_t *addr, int32_t value)
{
int32_t oldval;
do { OPAL_ATOMIC_DEFINE_CMPXCG_OP(int32_t, 32, |, or)
oldval = *addr;
} while (!opal_atomic_bool_cmpset_32(addr, oldval, oldval | value));
return (oldval | value);
}
#endif /* OPAL_HAVE_ATOMIC_OR_32 */ #endif /* OPAL_HAVE_ATOMIC_OR_32 */
#if !defined(OPAL_HAVE_ATOMIC_XOR_32) #if !defined(OPAL_HAVE_ATOMIC_XOR_32)
#define OPAL_HAVE_ATOMIC_XOR_32 1 #define OPAL_HAVE_ATOMIC_XOR_32 1
static inline int32_t
opal_atomic_xor_32(volatile int32_t *addr, int32_t value)
{
int32_t oldval;
do { OPAL_ATOMIC_DEFINE_CMPXCG_OP(int32_t, 32, ^, xor)
oldval = *addr;
} while (!opal_atomic_bool_cmpset_32(addr, oldval, oldval ^ value));
return (oldval ^ value);
}
#endif /* OPAL_HAVE_ATOMIC_XOR_32 */ #endif /* OPAL_HAVE_ATOMIC_XOR_32 */
#if !defined(OPAL_HAVE_ATOMIC_SUB_32) #if !defined(OPAL_HAVE_ATOMIC_SUB_32)
#define OPAL_HAVE_ATOMIC_SUB_32 1 #define OPAL_HAVE_ATOMIC_SUB_32 1
static inline int32_t
opal_atomic_sub_32(volatile int32_t *addr, int delta)
{
int32_t oldval;
do { OPAL_ATOMIC_DEFINE_CMPXCG_OP(int32_t, 32, -, sub)
oldval = *addr;
} while (!opal_atomic_bool_cmpset_32(addr, oldval, oldval - delta));
return (oldval - delta);
}
#endif /* OPAL_HAVE_ATOMIC_SUB_32 */ #endif /* OPAL_HAVE_ATOMIC_SUB_32 */
#endif /* OPAL_HAVE_ATOMIC_CMPSET_32 */ #endif /* OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 */
#if OPAL_HAVE_ATOMIC_CMPSET_64 #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
#if !defined(OPAL_HAVE_ATOMIC_SWAP_64) #if !defined(OPAL_HAVE_ATOMIC_SWAP_64)
#define OPAL_HAVE_ATOMIC_SWAP_64 1 #define OPAL_HAVE_ATOMIC_SWAP_64 1
@ -144,72 +121,37 @@ static inline int64_t opal_atomic_swap_64(volatile int64_t *addr,
#if !defined(OPAL_HAVE_ATOMIC_ADD_64) #if !defined(OPAL_HAVE_ATOMIC_ADD_64)
#define OPAL_HAVE_ATOMIC_ADD_64 1 #define OPAL_HAVE_ATOMIC_ADD_64 1
static inline int64_t
opal_atomic_add_64(volatile int64_t *addr, int64_t delta)
{
int64_t oldval;
do { OPAL_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, +, add)
oldval = *addr;
} while (!opal_atomic_bool_cmpset_64(addr, oldval, oldval + delta));
return (oldval + delta);
}
#endif /* OPAL_HAVE_ATOMIC_ADD_64 */ #endif /* OPAL_HAVE_ATOMIC_ADD_64 */
#if !defined(OPAL_HAVE_ATOMIC_AND_64) #if !defined(OPAL_HAVE_ATOMIC_AND_64)
#define OPAL_HAVE_ATOMIC_AND_64 1 #define OPAL_HAVE_ATOMIC_AND_64 1
static inline int64_t
opal_atomic_and_64(volatile int64_t *addr, int64_t value)
{
int64_t oldval;
do { OPAL_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, &, and)
oldval = *addr;
} while (!opal_atomic_bool_cmpset_64(addr, oldval, oldval & value));
return (oldval & value);
}
#endif /* OPAL_HAVE_ATOMIC_AND_64 */ #endif /* OPAL_HAVE_ATOMIC_AND_64 */
#if !defined(OPAL_HAVE_ATOMIC_OR_64) #if !defined(OPAL_HAVE_ATOMIC_OR_64)
#define OPAL_HAVE_ATOMIC_OR_64 1 #define OPAL_HAVE_ATOMIC_OR_64 1
static inline int64_t
opal_atomic_or_64(volatile int64_t *addr, int64_t value)
{
int64_t oldval;
do { OPAL_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, |, or)
oldval = *addr;
} while (!opal_atomic_bool_cmpset_64(addr, oldval, oldval | value));
return (oldval | value);
}
#endif /* OPAL_HAVE_ATOMIC_OR_64 */ #endif /* OPAL_HAVE_ATOMIC_OR_64 */
#if !defined(OPAL_HAVE_ATOMIC_XOR_64) #if !defined(OPAL_HAVE_ATOMIC_XOR_64)
#define OPAL_HAVE_ATOMIC_XOR_64 1 #define OPAL_HAVE_ATOMIC_XOR_64 1
static inline int64_t
opal_atomic_xor_64(volatile int64_t *addr, int64_t value)
{
int64_t oldval;
do { OPAL_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, ^, xor)
oldval = *addr;
} while (!opal_atomic_bool_cmpset_64(addr, oldval, oldval ^ value));
return (oldval ^ value);
}
#endif /* OPAL_HAVE_ATOMIC_XOR_64 */ #endif /* OPAL_HAVE_ATOMIC_XOR_64 */
#if !defined(OPAL_HAVE_ATOMIC_SUB_64) #if !defined(OPAL_HAVE_ATOMIC_SUB_64)
#define OPAL_HAVE_ATOMIC_SUB_64 1 #define OPAL_HAVE_ATOMIC_SUB_64 1
static inline int64_t
opal_atomic_sub_64(volatile int64_t *addr, int64_t delta)
{
int64_t oldval;
do { OPAL_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, -, sub)
oldval = *addr;
} while (!opal_atomic_bool_cmpset_64(addr, oldval, oldval - delta));
return (oldval - delta);
}
#endif /* OPAL_HAVE_ATOMIC_SUB_64 */ #endif /* OPAL_HAVE_ATOMIC_SUB_64 */
#else #else
@ -222,27 +164,138 @@ opal_atomic_sub_64(volatile int64_t *addr, int64_t delta)
#define OPAL_HAVE_ATOMIC_SUB_64 0 #define OPAL_HAVE_ATOMIC_SUB_64 0
#endif #endif
#endif /* OPAL_HAVE_ATOMIC_CMPSET_64 */ #endif /* OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 */
#if (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
#define OPAL_ATOMIC_DEFINE_CMPXCG_XX(semantics) \
static inline bool \
opal_atomic_compare_exchange_strong ## semantics ## xx (volatile void* addr, void *oldval, \
int64_t newval, const size_t length) \
{ \
switch (length) { \
case 4: \
return opal_atomic_compare_exchange_strong_32 ((volatile int32_t *) addr, \
(int32_t *) oldval, (int32_t) newval); \
case 8: \
return opal_atomic_compare_exchange_strong_64 ((volatile int64_t *) addr, \
(int64_t *) oldval, (int64_t) newval); \
} \
abort(); \
}
#elif OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
#define OPAL_ATOMIC_DEFINE_CMPXCG_XX(semantics) \
static inline bool \
opal_atomic_compare_exchange_strong ## semantics ## xx (volatile void* addr, void *oldval, \
int64_t newval, const size_t length) \
{ \
switch (length) { \
case 4: \
return opal_atomic_compare_exchange_strong_32 ((volatile int32_t *) addr, \
(int32_t *) oldval, (int32_t) newval); \
abort(); \
}
#else
#error "Platform does not have required atomic compare-and-swap functionality"
#endif
OPAL_ATOMIC_DEFINE_CMPXCG_XX(_)
OPAL_ATOMIC_DEFINE_CMPXCG_XX(_acq_)
OPAL_ATOMIC_DEFINE_CMPXCG_XX(_rel_)
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
#define OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(semantics) \
static inline bool \
opal_atomic_compare_exchange_strong ## semantics ## ptr (volatile void* addr, void *oldval, void *newval) \
{ \
return opal_atomic_compare_exchange_strong_32 ((volatile int32_t *) addr, (int32_t *) oldval, (int32_t) newval); \
}
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
#define OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(semantics) \
static inline bool \
opal_atomic_compare_exchange_strong ## semantics ## ptr (volatile void* addr, void *oldval, void *newval) \
{ \
return opal_atomic_compare_exchange_strong_64 ((volatile int64_t *) addr, (int64_t *) oldval, (int64_t) newval); \
}
#else
#error "Can not define opal_atomic_compare_exchange_strong_ptr with existing atomics"
#endif
OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(_)
OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(_acq_)
OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(_rel_)
#endif /* (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64) */
#if (OPAL_HAVE_ATOMIC_CMPSET_32 || OPAL_HAVE_ATOMIC_CMPSET_64) /* XXX -- DEPRECATED -- XXX -- Define legacy cmpset functions */
#if (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32)
static inline bool opal_atomic_bool_cmpset_32 (volatile int32_t *addr, int32_t oldval,
int32_t newval)
{
return opal_atomic_compare_exchange_strong_32 (addr, &oldval, newval);
}
static inline bool opal_atomic_bool_cmpset_acq_32 (volatile int32_t *addr, int32_t oldval,
int32_t newval)
{
return opal_atomic_compare_exchange_strong_acq_32 (addr, &oldval, newval);
}
static inline bool opal_atomic_bool_cmpset_rel_32 (volatile int32_t *addr, int32_t oldval,
int32_t newval)
{
return opal_atomic_compare_exchange_strong_rel_32 (addr, &oldval, newval);
}
#endif
#if (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
static inline bool opal_atomic_bool_cmpset_64 (volatile int64_t *addr, int64_t oldval,
int64_t newval)
{
return opal_atomic_compare_exchange_strong_64 (addr, &oldval, newval);
}
static inline bool opal_atomic_bool_cmpset_acq_64 (volatile int64_t *addr, int64_t oldval,
int64_t newval)
{
return opal_atomic_compare_exchange_strong_acq_64 (addr, &oldval, newval);
}
static inline bool opal_atomic_bool_cmpset_rel_64 (volatile int64_t *addr, int64_t oldval,
int64_t newval)
{
return opal_atomic_compare_exchange_strong_rel_64 (addr, &oldval, newval);
}
#endif
#if (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128)
static inline bool opal_atomic_bool_cmpset_128 (volatile opal_int128_t *addr, opal_int128_t oldval,
opal_int128_t newval)
{
return opal_atomic_compare_exchange_strong_128 (addr, &oldval, newval);
}
#endif
#if (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
static inline bool static inline bool
opal_atomic_bool_cmpset_xx(volatile void* addr, int64_t oldval, opal_atomic_bool_cmpset_xx(volatile void* addr, int64_t oldval,
int64_t newval, size_t length) int64_t newval, size_t length)
{ {
switch( length ) { switch( length ) {
#if OPAL_HAVE_ATOMIC_CMPSET_32 #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
case 4: case 4:
return opal_atomic_bool_cmpset_32( (volatile int32_t*)addr, return opal_atomic_bool_cmpset_32( (volatile int32_t*)addr,
(int32_t)oldval, (int32_t)newval ); (int32_t)oldval, (int32_t)newval );
#endif /* OPAL_HAVE_ATOMIC_CMPSET_32 */ #endif /* OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 */
#if OPAL_HAVE_ATOMIC_CMPSET_64 #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
case 8: case 8:
return opal_atomic_bool_cmpset_64( (volatile int64_t*)addr, return opal_atomic_bool_cmpset_64( (volatile int64_t*)addr,
(int64_t)oldval, (int64_t)newval ); (int64_t)oldval, (int64_t)newval );
#endif /* OPAL_HAVE_ATOMIC_CMPSET_64 */ #endif /* OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 */
} }
abort(); abort();
/* This should never happen, so deliberately abort (hopefully /* This should never happen, so deliberately abort (hopefully
@ -255,17 +308,17 @@ opal_atomic_bool_cmpset_acq_xx(volatile void* addr, int64_t oldval,
int64_t newval, size_t length) int64_t newval, size_t length)
{ {
switch( length ) { switch( length ) {
#if OPAL_HAVE_ATOMIC_CMPSET_32 #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
case 4: case 4:
return opal_atomic_bool_cmpset_acq_32( (volatile int32_t*)addr, return opal_atomic_bool_cmpset_acq_32( (volatile int32_t*)addr,
(int32_t)oldval, (int32_t)newval ); (int32_t)oldval, (int32_t)newval );
#endif /* OPAL_HAVE_ATOMIC_CMPSET_32 */ #endif /* OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 */
#if OPAL_HAVE_ATOMIC_CMPSET_64 #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
case 8: case 8:
return opal_atomic_bool_cmpset_acq_64( (volatile int64_t*)addr, return opal_atomic_bool_cmpset_acq_64( (volatile int64_t*)addr,
(int64_t)oldval, (int64_t)newval ); (int64_t)oldval, (int64_t)newval );
#endif /* OPAL_HAVE_ATOMIC_CMPSET_64 */ #endif /* OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 */
} }
/* This should never happen, so deliberately abort (hopefully /* This should never happen, so deliberately abort (hopefully
leaving a corefile for analysis) */ leaving a corefile for analysis) */
@ -278,17 +331,17 @@ opal_atomic_bool_cmpset_rel_xx(volatile void* addr, int64_t oldval,
int64_t newval, size_t length) int64_t newval, size_t length)
{ {
switch( length ) { switch( length ) {
#if OPAL_HAVE_ATOMIC_CMPSET_32 #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
case 4: case 4:
return opal_atomic_bool_cmpset_rel_32( (volatile int32_t*)addr, return opal_atomic_bool_cmpset_rel_32( (volatile int32_t*)addr,
(int32_t)oldval, (int32_t)newval ); (int32_t)oldval, (int32_t)newval );
#endif /* OPAL_HAVE_ATOMIC_CMPSET_32 */ #endif /* OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 */
#if OPAL_HAVE_ATOMIC_CMPSET_64 #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
case 8: case 8:
return opal_atomic_bool_cmpset_rel_64( (volatile int64_t*)addr, return opal_atomic_bool_cmpset_rel_64( (volatile int64_t*)addr,
(int64_t)oldval, (int64_t)newval ); (int64_t)oldval, (int64_t)newval );
#endif /* OPAL_HAVE_ATOMIC_CMPSET_64 */ #endif /* OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 */
} }
/* This should never happen, so deliberately abort (hopefully /* This should never happen, so deliberately abort (hopefully
leaving a corefile for analysis) */ leaving a corefile for analysis) */
@ -301,10 +354,10 @@ opal_atomic_bool_cmpset_ptr(volatile void* addr,
void* oldval, void* oldval,
void* newval) void* newval)
{ {
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_CMPSET_32 #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
return opal_atomic_bool_cmpset_32((int32_t*) addr, (unsigned long) oldval, return opal_atomic_bool_cmpset_32((int32_t*) addr, (unsigned long) oldval,
(unsigned long) newval); (unsigned long) newval);
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_CMPSET_64 #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
return opal_atomic_bool_cmpset_64((int64_t*) addr, (unsigned long) oldval, return opal_atomic_bool_cmpset_64((int64_t*) addr, (unsigned long) oldval,
(unsigned long) newval); (unsigned long) newval);
#else #else
@ -318,10 +371,10 @@ opal_atomic_bool_cmpset_acq_ptr(volatile void* addr,
void* oldval, void* oldval,
void* newval) void* newval)
{ {
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_CMPSET_32 #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
return opal_atomic_bool_cmpset_acq_32((int32_t*) addr, (unsigned long) oldval, return opal_atomic_bool_cmpset_acq_32((int32_t*) addr, (unsigned long) oldval,
(unsigned long) newval); (unsigned long) newval);
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_CMPSET_64 #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
return opal_atomic_bool_cmpset_acq_64((int64_t*) addr, (unsigned long) oldval, return opal_atomic_bool_cmpset_acq_64((int64_t*) addr, (unsigned long) oldval,
(unsigned long) newval); (unsigned long) newval);
#else #else
@ -334,10 +387,10 @@ static inline bool opal_atomic_bool_cmpset_rel_ptr(volatile void* addr,
void* oldval, void* oldval,
void* newval) void* newval)
{ {
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_CMPSET_32 #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
return opal_atomic_bool_cmpset_rel_32((int32_t*) addr, (unsigned long) oldval, return opal_atomic_bool_cmpset_rel_32((int32_t*) addr, (unsigned long) oldval,
(unsigned long) newval); (unsigned long) newval);
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_CMPSET_64 #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
return opal_atomic_bool_cmpset_rel_64((int64_t*) addr, (unsigned long) oldval, return opal_atomic_bool_cmpset_rel_64((int64_t*) addr, (unsigned long) oldval,
(unsigned long) newval); (unsigned long) newval);
#else #else
@ -345,7 +398,7 @@ static inline bool opal_atomic_bool_cmpset_rel_ptr(volatile void* addr,
#endif #endif
} }
#endif /* (OPAL_HAVE_ATOMIC_CMPSET_32 || OPAL_HAVE_ATOMIC_CMPSET_64) */ #endif /* (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64) */
#if (OPAL_HAVE_ATOMIC_SWAP_32 || OPAL_HAVE_ATOMIC_SWAP_64) #if (OPAL_HAVE_ATOMIC_SWAP_32 || OPAL_HAVE_ATOMIC_SWAP_64)
@ -392,7 +445,7 @@ opal_atomic_add_xx(volatile void* addr, int32_t value, size_t length)
case 4: case 4:
opal_atomic_add_32( (volatile int32_t*)addr, (int32_t)value ); opal_atomic_add_32( (volatile int32_t*)addr, (int32_t)value );
break; break;
#endif /* OPAL_HAVE_ATOMIC_CMPSET_32 */ #endif /* OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 */
#if OPAL_HAVE_ATOMIC_ADD_64 #if OPAL_HAVE_ATOMIC_ADD_64
case 8: case 8:

59
opal/include/opal/sys/gcc_builtin/atomic.h
Просмотреть файл

@ -33,7 +33,7 @@
#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1 #define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
#define OPAL_HAVE_ATOMIC_MATH_32 1 #define OPAL_HAVE_ATOMIC_MATH_32 1
#define OPAL_HAVE_ATOMIC_CMPSET_32 1 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
#define OPAL_HAVE_ATOMIC_ADD_32 1 #define OPAL_HAVE_ATOMIC_ADD_32 1
#define OPAL_HAVE_ATOMIC_AND_32 1 #define OPAL_HAVE_ATOMIC_AND_32 1
#define OPAL_HAVE_ATOMIC_OR_32 1 #define OPAL_HAVE_ATOMIC_OR_32 1
@ -41,7 +41,7 @@
#define OPAL_HAVE_ATOMIC_SUB_32 1 #define OPAL_HAVE_ATOMIC_SUB_32 1
#define OPAL_HAVE_ATOMIC_SWAP_32 1 #define OPAL_HAVE_ATOMIC_SWAP_32 1
#define OPAL_HAVE_ATOMIC_MATH_64 1 #define OPAL_HAVE_ATOMIC_MATH_64 1
#define OPAL_HAVE_ATOMIC_CMPSET_64 1 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
#define OPAL_HAVE_ATOMIC_ADD_64 1 #define OPAL_HAVE_ATOMIC_ADD_64 1
#define OPAL_HAVE_ATOMIC_AND_64 1 #define OPAL_HAVE_ATOMIC_AND_64 1
#define OPAL_HAVE_ATOMIC_OR_64 1 #define OPAL_HAVE_ATOMIC_OR_64 1
@ -81,26 +81,20 @@ static inline void opal_atomic_wmb(void)
#pragma error_messages(off, E_ARG_INCOMPATIBLE_WITH_ARG_L) #pragma error_messages(off, E_ARG_INCOMPATIBLE_WITH_ARG_L)
#endif #endif
static inline bool opal_atomic_bool_cmpset_acq_32( volatile int32_t *addr, static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
int32_t oldval, int32_t newval)
{ {
return __atomic_compare_exchange_n (addr, &oldval, newval, false, return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
} }
static inline bool opal_atomic_bool_cmpset_rel_32( volatile int32_t *addr, static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
int32_t oldval, int32_t newval)
{ {
return __atomic_compare_exchange_n (addr, &oldval, newval, false, return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
__ATOMIC_RELEASE, __ATOMIC_RELAXED);
} }
static inline bool opal_atomic_bool_cmpset_32( volatile int32_t *addr, static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
int32_t oldval, int32_t newval)
{ {
return __atomic_compare_exchange_n (addr, &oldval, newval, false, return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
} }
static inline int32_t opal_atomic_swap_32 (volatile int32_t *addr, int32_t newval) static inline int32_t opal_atomic_swap_32 (volatile int32_t *addr, int32_t newval)
@ -135,26 +129,20 @@ static inline int32_t opal_atomic_sub_32(volatile int32_t *addr, int32_t delta)
return __atomic_sub_fetch (addr, delta, __ATOMIC_RELAXED); return __atomic_sub_fetch (addr, delta, __ATOMIC_RELAXED);
} }
static inline bool opal_atomic_bool_cmpset_acq_64( volatile int64_t *addr, static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
int64_t oldval, int64_t newval)
{ {
return __atomic_compare_exchange_n (addr, &oldval, newval, false, return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
} }
static inline bool opal_atomic_bool_cmpset_rel_64( volatile int64_t *addr, static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
int64_t oldval, int64_t newval)
{ {
return __atomic_compare_exchange_n (addr, &oldval, newval, false, return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
__ATOMIC_RELEASE, __ATOMIC_RELAXED);
} }
static inline bool opal_atomic_bool_cmpset_64( volatile int64_t *addr, static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
int64_t oldval, int64_t newval)
{ {
return __atomic_compare_exchange_n (addr, &oldval, newval, false, return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
} }
static inline int64_t opal_atomic_swap_64 (volatile int64_t *addr, int64_t newval) static inline int64_t opal_atomic_swap_64 (volatile int64_t *addr, int64_t newval)
@ -191,25 +179,28 @@ static inline int64_t opal_atomic_sub_64(volatile int64_t *addr, int64_t delta)
#if OPAL_HAVE_GCC_BUILTIN_CSWAP_INT128 #if OPAL_HAVE_GCC_BUILTIN_CSWAP_INT128
#define OPAL_HAVE_ATOMIC_CMPSET_128 1 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
static inline bool opal_atomic_bool_cmpset_128 (volatile opal_int128_t *addr, static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128_t *addr,
opal_int128_t oldval, opal_int128_t newval) opal_int128_t *oldval, opal_int128_t newval)
{ {
return __atomic_compare_exchange_n (addr, &oldval, newval, false, return __atomic_compare_exchange_n (addr, oldval, newval, false,
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED); __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
} }
#elif defined(OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128) && OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128 #elif defined(OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128) && OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128
#define OPAL_HAVE_ATOMIC_CMPSET_128 1 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
/* __atomic version is not lock-free so use legacy __sync version */ /* __atomic version is not lock-free so use legacy __sync version */
static inline bool opal_atomic_bool_cmpset_128 (volatile opal_int128_t *addr, static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128_t *addr,
opal_int128_t oldval, opal_int128_t newval) opal_int128_t *oldval, opal_int128_t newval)
{ {
return __sync_bool_compare_and_swap (addr, oldval, newval); opal_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval);
bool ret = prev == *oldval;
*oldval = prev;
return ret;
} }
#endif #endif

12
opal/include/opal/sys/ia32/atomic.h
Просмотреть файл

@ -40,7 +40,7 @@
*********************************************************************/ *********************************************************************/
#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1 #define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
#define OPAL_HAVE_ATOMIC_CMPSET_32 1 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
#define OPAL_HAVE_ATOMIC_MATH_32 1 #define OPAL_HAVE_ATOMIC_MATH_32 1
#define OPAL_HAVE_ATOMIC_ADD_32 1 #define OPAL_HAVE_ATOMIC_ADD_32 1
@ -84,15 +84,13 @@ static inline void opal_atomic_isync(void)
*********************************************************************/ *********************************************************************/
#if OPAL_GCC_INLINE_ASSEMBLY #if OPAL_GCC_INLINE_ASSEMBLY
static inline bool opal_atomic_bool_cmpset_32(volatile int32_t *addr, static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
int32_t oldval,
int32_t newval)
{ {
unsigned char ret; unsigned char ret;
__asm__ __volatile__ ( __asm__ __volatile__ (
SMPLOCK "cmpxchgl %3,%2 \n\t" SMPLOCK "cmpxchgl %3,%2 \n\t"
"sete %0 \n\t" "sete %0 \n\t"
: "=qm" (ret), "+a" (oldval), "+m" (*addr) : "=qm" (ret), "+a" (*oldval), "+m" (*addr)
: "q"(newval) : "q"(newval)
: "memory", "cc"); : "memory", "cc");
@ -101,8 +99,8 @@ static inline bool opal_atomic_bool_cmpset_32(volatile int32_t *addr,
#endif /* OPAL_GCC_INLINE_ASSEMBLY */ #endif /* OPAL_GCC_INLINE_ASSEMBLY */
#define opal_atomic_bool_cmpset_acq_32 opal_atomic_bool_cmpset_32 #define opal_atomic_compare_exchange_strong_acq_32 opal_atomic_compare_exchange_strong_32
#define opal_atomic_bool_cmpset_rel_32 opal_atomic_bool_cmpset_32 #define opal_atomic_compare_exchange_strong_rel_32 opal_atomic_compare_exchange_strong_32
#if OPAL_GCC_INLINE_ASSEMBLY #if OPAL_GCC_INLINE_ASSEMBLY

170
opal/include/opal/sys/powerpc/atomic.h
Просмотреть файл

@ -40,7 +40,7 @@
*********************************************************************/ *********************************************************************/
#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1 #define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
#define OPAL_HAVE_ATOMIC_CMPSET_32 1 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
#define OPAL_HAVE_ATOMIC_SWAP_32 1 #define OPAL_HAVE_ATOMIC_SWAP_32 1
#define OPAL_HAVE_ATOMIC_LLSC_32 1 #define OPAL_HAVE_ATOMIC_LLSC_32 1
@ -53,7 +53,7 @@
#if (OPAL_ASSEMBLY_ARCH == OPAL_POWERPC64) || OPAL_ASM_SUPPORT_64BIT #if (OPAL_ASSEMBLY_ARCH == OPAL_POWERPC64) || OPAL_ASM_SUPPORT_64BIT
#define OPAL_HAVE_ATOMIC_CMPSET_64 1 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
#define OPAL_HAVE_ATOMIC_SWAP_64 1 #define OPAL_HAVE_ATOMIC_SWAP_64 1
#define OPAL_HAVE_ATOMIC_LLSC_64 1 #define OPAL_HAVE_ATOMIC_LLSC_64 1
#define OPAL_HAVE_ATOMIC_MATH_64 1 #define OPAL_HAVE_ATOMIC_MATH_64 1
@ -144,24 +144,25 @@ void opal_atomic_isync(void)
#define OPAL_ASM_VALUE64(x) x #define OPAL_ASM_VALUE64(x) x
#endif #endif
static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
static inline bool opal_atomic_bool_cmpset_32(volatile int32_t *addr,
int32_t oldval, int32_t newval)
{ {
int32_t ret; int32_t prev;
bool ret;
__asm__ __volatile__ ( __asm__ __volatile__ (
"1: lwarx %0, 0, %2 \n\t" "1: lwarx %0, 0, %2 \n\t"
" cmpw 0, %0, %3 \n\t" " cmpw 0, %0, %3 \n\t"
" bne- 2f \n\t" " bne- 2f \n\t"
" stwcx. %4, 0, %2 \n\t" " stwcx. %4, 0, %2 \n\t"
" bne- 1b \n\t" " bne- 1b \n\t"
"2:" "2:"
: "=&r" (ret), "=m" (*addr) : "=&r" (prev), "=m" (*addr)
: "r" OPAL_ASM_ADDR(addr), "r" (oldval), "r" (newval), "m" (*addr) : "r" OPAL_ASM_ADDR(addr), "r" (*oldval), "r" (newval), "m" (*addr)
: "cc", "memory"); : "cc", "memory");
return (ret == oldval); ret = (prev == *oldval);
*oldval = prev;
return ret;
} }
static inline int32_t opal_atomic_ll_32 (volatile int32_t *addr) static inline int32_t opal_atomic_ll_32 (volatile int32_t *addr)
@ -195,23 +196,21 @@ static inline int opal_atomic_sc_32 (volatile int32_t *addr, int32_t newval)
atomic_?mb can be inlined). Instead, we "inline" them by hand in atomic_?mb can be inlined). Instead, we "inline" them by hand in
the assembly, meaning there is one function call overhead instead the assembly, meaning there is one function call overhead instead
of two */ of two */
static inline bool opal_atomic_bool_cmpset_acq_32(volatile int32_t *addr, static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
int32_t oldval, int32_t newval)
{ {
bool rc; bool rc;
rc = opal_atomic_bool_cmpset_32(addr, oldval, newval); rc = opal_atomic_compare_exchange_strong_32 (addr, oldval, newval);
opal_atomic_rmb(); opal_atomic_rmb();
return rc; return rc;
} }
static inline bool opal_atomic_bool_cmpset_rel_32(volatile int32_t *addr, static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
int32_t oldval, int32_t newval)
{ {
opal_atomic_wmb(); opal_atomic_wmb();
return opal_atomic_bool_cmpset_32(addr, oldval, newval); return opal_atomic_compare_exchange_strong_32 (addr, oldval, newval);
} }
static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, int32_t newval) static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, int32_t newval)
@ -258,23 +257,25 @@ OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(or, or)
OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(xor, xor) OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(xor, xor)
OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(sub, subf) OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(sub, subf)
static inline bool opal_atomic_bool_cmpset_64(volatile int64_t *addr, static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
int64_t oldval, int64_t newval)
{ {
int64_t ret; int64_t prev;
bool ret;
__asm__ __volatile__ ( __asm__ __volatile__ (
"1: ldarx %0, 0, %2 \n\t" "1: ldarx %0, 0, %2 \n\t"
" cmpd 0, %0, %3 \n\t" " cmpd 0, %0, %3 \n\t"
" bne- 2f \n\t" " bne- 2f \n\t"
" stdcx. %4, 0, %2 \n\t" " stdcx. %4, 0, %2 \n\t"
" bne- 1b \n\t" " bne- 1b \n\t"
"2:" "2:"
: "=&r" (ret), "=m" (*addr) : "=&r" (prev), "=m" (*addr)
: "r" (addr), "r" (OPAL_ASM_VALUE64(oldval)), "r" (OPAL_ASM_VALUE64(newval)), "m" (*addr) : "r" (addr), "r" (OPAL_ASM_VALUE64(*oldval)), "r" (OPAL_ASM_VALUE64(newval)), "m" (*addr)
: "cc", "memory"); : "cc", "memory");
return (ret == oldval); ret = (prev == *oldval);
*oldval = prev;
return ret;
} }
static inline int64_t opal_atomic_ll_64(volatile int64_t *addr) static inline int64_t opal_atomic_ll_64(volatile int64_t *addr)
@ -303,29 +304,6 @@ static inline int opal_atomic_sc_64(volatile int64_t *addr, int64_t newval)
return ret; return ret;
} }
/* these two functions aren't inlined in the non-gcc case because then
there would be two function calls (since neither cmpset_64 nor
atomic_?mb can be inlined). Instead, we "inline" them by hand in
the assembly, meaning there is one function call overhead instead
of two */
static inline bool opal_atomic_bool_cmpset_acq_64(volatile int64_t *addr,
int64_t oldval, int64_t newval)
{
bool rc;
rc = opal_atomic_bool_cmpset_64(addr, oldval, newval);
opal_atomic_rmb();
return rc;
}
static inline bool opal_atomic_bool_cmpset_rel_64(volatile int64_t *addr,
int64_t oldval, int64_t newval)
{
opal_atomic_wmb();
return opal_atomic_bool_cmpset_64(addr, oldval, newval);
}
static inline int64_t opal_atomic_swap_64(volatile int64_t *addr, int64_t newval) static inline int64_t opal_atomic_swap_64(volatile int64_t *addr, int64_t newval)
{ {
@ -352,9 +330,9 @@ static inline int64_t opal_atomic_swap_64(volatile int64_t *addr, int64_t newval
#if OPAL_GCC_INLINE_ASSEMBLY #if OPAL_GCC_INLINE_ASSEMBLY
static inline int opal_atomic_bool_cmpset_64(volatile int64_t *addr, static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
int64_t oldval, int64_t newval)
{ {
int64_t prev;
int ret; int ret;
/* /*
@ -369,56 +347,54 @@ static inline int opal_atomic_bool_cmpset_64(volatile int64_t *addr,
* is very similar to the pure 64 bit version. * is very similar to the pure 64 bit version.
*/ */
__asm__ __volatile__ ( __asm__ __volatile__ (
"ld r4,%2 \n\t" "ld r4,%3 \n\t"
"ld r5,%3 \n\t" "ld r5,%4 \n\t"
"1: ldarx r9, 0, %1 \n\t" "1: ldarx %1, 0, %2 \n\t"
" cmpd 0, r9, r4 \n\t" " cmpd 0, %1, r4 \n\t"
" bne- 2f \n\t" " bne- 2f \n\t"
" stdcx. r5, 0, %1 \n\t" " stdcx. r5, 0, %2 \n\t"
" bne- 1b \n\t" " bne- 1b \n\t"
"2: \n\t" "2: \n\t"
"xor r5,r4,r9 \n\t" "xor r5,r4,%1 \n\t"
"subfic r9,r5,0 \n\t" "subfic r9,r5,0 \n\t"
"adde %0,r9,r5 \n\t" "adde %0,r9,r5 \n\t"
: "=&r" (ret) : "=&r" (ret), "+r" (prev)
: "r"OPAL_ASM_ADDR(addr), : "r"OPAL_ASM_ADDR(addr),
"m"(oldval), "m"(newval) "m"(*oldval), "m"(newval)
: "r4", "r5", "r9", "cc", "memory"); : "r4", "r5", "r9", "cc", "memory");
*oldval = prev;
return ret; return (bool) ret;
}
/* these two functions aren't inlined in the non-gcc case because then
there would be two function calls (since neither cmpset_64 nor
atomic_?mb can be inlined). Instead, we "inline" them by hand in
the assembly, meaning there is one function call overhead instead
of two */
static inline bool opal_atomic_bool_cmpset_acq_64(volatile int64_t *addr,
int64_t oldval, int64_t newval)
{
int rc;
rc = opal_atomic_bool_cmpset_64(addr, oldval, newval);
opal_atomic_rmb();
return rc;
}
static inline bool opal_atomic_bool_cmpset_rel_64(volatile int64_t *addr,
int64_t oldval, int64_t newval)
{
opal_atomic_wmb();
return opal_atomic_bool_cmpset_64(addr, oldval, newval);
} }
#endif /* OPAL_GCC_INLINE_ASSEMBLY */ #endif /* OPAL_GCC_INLINE_ASSEMBLY */
#endif /* OPAL_ASM_SUPPORT_64BIT */ #endif /* OPAL_ASM_SUPPORT_64BIT */
#if OPAL_GCC_INLINE_ASSEMBLY #if OPAL_GCC_INLINE_ASSEMBLY
/* these two functions aren't inlined in the non-gcc case because then
there would be two function calls (since neither cmpset_64 nor
atomic_?mb can be inlined). Instead, we "inline" them by hand in
the assembly, meaning there is one function call overhead instead
of two */
static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
{
bool rc;
rc = opal_atomic_compare_exchange_strong_64 (addr, oldval, newval);
opal_atomic_rmb();
return rc;
}
static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
{
opal_atomic_wmb();
return opal_atomic_compare_exchange_strong_64 (addr, oldval, newval);
}
#define OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_32(type, instr) \ #define OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_32(type, instr) \
static inline int32_t opal_atomic_ ## type ## _32(volatile int32_t* v, int val) \ static inline int32_t opal_atomic_ ## type ## _32(volatile int32_t* v, int val) \
{ \ { \

103
opal/include/opal/sys/sparcv9/atomic.h
Просмотреть файл

@ -1,3 +1,4 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/* /*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology * University Research and Technology
@ -12,6 +13,8 @@
* Copyright (c) 2007 Sun Microsystems, Inc. All rights reserverd. * Copyright (c) 2007 Sun Microsystems, Inc. All rights reserverd.
* Copyright (c) 2016 Research Organization for Information Science * Copyright (c) 2016 Research Organization for Information Science
* and Technology (RIST). All rights reserved. * and Technology (RIST). All rights reserved.
* Copyright (c) 2017 Los Alamos National Security, LLC. All rights
* reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
* Additional copyrights may follow * Additional copyrights may follow
@ -38,9 +41,9 @@
*********************************************************************/ *********************************************************************/
#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1 #define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
#define OPAL_HAVE_ATOMIC_CMPSET_32 1 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
#define OPAL_HAVE_ATOMIC_CMPSET_64 1 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
/********************************************************************** /**********************************************************************
@ -82,50 +85,49 @@ static inline void opal_atomic_isync(void)
*********************************************************************/ *********************************************************************/
#if OPAL_GCC_INLINE_ASSEMBLY #if OPAL_GCC_INLINE_ASSEMBLY
static inline bool opal_atomic_bool_cmpset_32( volatile int32_t *addr, static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
int32_t oldval, int32_t newval)
{ {
/* casa [reg(rs1)] %asi, reg(rs2), reg(rd) /* casa [reg(rs1)] %asi, reg(rs2), reg(rd)
* *
* if (*(reg(rs1)) == reg(rs2) ) * if (*(reg(rs1)) == reg(rs2) )
* swap reg(rd), *(reg(rs1)) * swap reg(rd), *(reg(rs1))
* else * else
* reg(rd) = *(reg(rs1)) * reg(rd) = *(reg(rs1))
*/ */
int32_t ret = newval; int32_t prev = newval;
bool ret;
__asm__ __volatile__("casa [%1] " ASI_P ", %2, %0" __asm__ __volatile__("casa [%1] " ASI_P ", %2, %0"
: "+r" (ret) : "+r" (prev)
: "r" (addr), "r" (oldval)); : "r" (addr), "r" (*oldval));
return (ret == oldval); ret = (prev == *oldval);
*oldval = prev;
return ret;
} }
static inline bool opal_atomic_bool_cmpset_acq_32( volatile int32_t *addr, static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
int32_t oldval, int32_t newval)
{ {
bool rc; bool rc;
rc = opal_atomic_bool_cmpset_32(addr, oldval, newval); rc = opal_atomic_compare_exchange_strong_32 (addr, oldval, newval);
opal_atomic_rmb(); opal_atomic_rmb();
return rc; return rc;
} }
static inline bool opal_atomic_bool_cmpset_rel_32( volatile int32_t *addr, static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
int32_t oldval, int32_t newval)
{ {
opal_atomic_wmb(); opal_atomic_wmb();
return opal_atomic_bool_cmpset_32(addr, oldval, newval); return opal_atomic_compare_exchange_strong_32 (addr, oldval, newval);
} }
#if OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64 #if OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64
static inline bool opal_atomic_bool_cmpset_64( volatile int64_t *addr, static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
int64_t oldval, int64_t newval)
{ {
/* casa [reg(rs1)] %asi, reg(rs2), reg(rd) /* casa [reg(rs1)] %asi, reg(rs2), reg(rd)
* *
@ -134,18 +136,20 @@ static inline bool opal_atomic_bool_cmpset_64( volatile int64_t *addr,
* else * else
* reg(rd) = *(reg(rs1)) * reg(rd) = *(reg(rs1))
*/ */
int64_t ret = newval; int64_t prev = newval;
bool ret;
__asm__ __volatile__("casxa [%1] " ASI_P ", %2, %0" __asm__ __volatile__("casxa [%1] " ASI_P ", %2, %0"
: "+r" (ret) : "+r" (prev)
: "r" (addr), "r" (oldval)); : "r" (addr), "r" (*oldval));
return (ret == oldval); ret = (prev == *oldval);
*oldval = prev;
return ret;
} }
#else /* OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64 */ #else /* OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64 */
static inline bool opal_atomic_bool_cmpset_64( volatile int64_t *addr, static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
int64_t oldval, int64_t newval)
{ {
/* casa [reg(rs1)] %asi, reg(rs2), reg(rd) /* casa [reg(rs1)] %asi, reg(rs2), reg(rd)
* *
@ -155,40 +159,41 @@ static inline bool opal_atomic_bool_cmpset_64( volatile int64_t *addr,
* reg(rd) = *(reg(rs1)) * reg(rd) = *(reg(rs1))
* *
*/ */
long long ret = newval; int64_t prev = newval;
bool ret;
__asm__ __volatile__( __asm__ __volatile__(
"ldx %0, %%g1 \n\t" /* g1 = ret */ "ldx %0, %%g1 \n\t" /* g1 = ret */
"ldx %2, %%g2 \n\t" /* g2 = oldval */ "ldx %2, %%g2 \n\t" /* g2 = oldval */
"casxa [%1] " ASI_P ", %%g2, %%g1 \n\t" "casxa [%1] " ASI_P ", %%g2, %%g1 \n\t"
"stx %%g1, %0 \n" "stx %%g1, %0 \n"
: "+m"(ret) : "+m"(prev)
: "r"(addr), "m"(oldval) : "r"(addr), "m"(*oldval)
: "%g1", "%g2" : "%g1", "%g2"
); );
return (ret == oldval); ret = (prev == *oldval);
*oldval = prev;
return ret;
} }
#endif /* OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64 */ #endif /* OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64 */
static inline bool opal_atomic_bool_cmpset_acq_64( volatile int64_t *addr, static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
int64_t oldval, int64_t newval)
{ {
bool rc; bool rc;
rc = opal_atomic_bool_cmpset_64(addr, oldval, newval); rc = opal_atomic_compare_exchange_strong_64 (addr, oldval, newval);
opal_atomic_rmb(); opal_atomic_rmb();
return rc; return rc;
} }
static inline bool opal_atomic_bool_cmpset_rel_64( volatile int64_t *addr, static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
int64_t oldval, int64_t newval)
{ {
opal_atomic_wmb(); opal_atomic_wmb();
return opal_atomic_bool_cmpset_64(addr, oldval, newval); return opal_atomic_compare_exchange_strong_64 (addr, oldval, newval);
} }
#endif /* OPAL_GCC_INLINE_ASSEMBLY */ #endif /* OPAL_GCC_INLINE_ASSEMBLY */

59
opal/include/opal/sys/sync_builtin/atomic.h
Просмотреть файл

@ -53,24 +53,18 @@ static inline void opal_atomic_wmb(void)
* *
*********************************************************************/ *********************************************************************/
#define OPAL_HAVE_ATOMIC_CMPSET_32 1 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
static inline bool opal_atomic_bool_cmpset_acq_32( volatile int32_t *addr,
int32_t oldval, int32_t newval) static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
{ {
return __sync_bool_compare_and_swap(addr, oldval, newval); int32_t prev = __sync_val_compare_and_swap (add, *oldval, newval);
bool ret = prev == *oldval;
*oldval = prev;
return ret;
} }
#define opal_atomic_compare_exchange_strong_acq_32 opal_atomic_compare_exchange_strong_32
static inline bool opal_atomic_bool_cmpset_rel_32( volatile int32_t *addr, #define opal_atomic_compare_exchange_strong_rel_32 opal_atomic_compare_exchange_strong_32
int32_t oldval, int32_t newval)
{
return __sync_bool_compare_and_swap(addr, oldval, newval);}
static inline bool opal_atomic_bool_cmpset_32( volatile int32_t *addr,
int32_t oldval, int32_t newval)
{
return __sync_bool_compare_and_swap(addr, oldval, newval);
}
#define OPAL_HAVE_ATOMIC_MATH_32 1 #define OPAL_HAVE_ATOMIC_MATH_32 1
@ -106,24 +100,18 @@ static inline int32_t opal_atomic_sub_32(volatile int32_t *addr, int32_t delta)
#if OPAL_ASM_SYNC_HAVE_64BIT #if OPAL_ASM_SYNC_HAVE_64BIT
#define OPAL_HAVE_ATOMIC_CMPSET_64 1 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
static inline bool opal_atomic_bool_cmpset_acq_64( volatile int64_t *addr,
int64_t oldval, int64_t newval) static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
{ {
return __sync_bool_compare_and_swap(addr, oldval, newval); int64_t prev = __sync_val_compare_and_swap (add, *oldval, newval);
bool ret = prev == *oldval;
*oldval = prev;
return ret;
} }
static inline bool opal_atomic_bool_cmpset_rel_64( volatile int64_t *addr, #define opal_atomic_compare_exchange_strong_acq_64 opal_atomic_compare_exchange_strong_64
int64_t oldval, int64_t newval) #define opal_atomic_compare_exchange_strong_rel_64 opal_atomic_compare_exchange_strong_64
{
return __sync_bool_compare_and_swap(addr, oldval, newval);}
static inline bool opal_atomic_bool_cmpset_64( volatile int64_t *addr,
int64_t oldval, int64_t newval)
{
return __sync_bool_compare_and_swap(addr, oldval, newval);
}
#define OPAL_HAVE_ATOMIC_MATH_64 1 #define OPAL_HAVE_ATOMIC_MATH_64 1
#define OPAL_HAVE_ATOMIC_ADD_64 1 #define OPAL_HAVE_ATOMIC_ADD_64 1
@ -159,13 +147,16 @@ static inline int64_t opal_atomic_sub_64(volatile int64_t *addr, int64_t delta)
#endif #endif
#if OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128 #if OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128
static inline bool opal_atomic_bool_cmpset_128 (volatile opal_int128_t *addr, static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128_t *addr,
opal_int128_t oldval, opal_int128_t newval) opal_int128_t *oldval, opal_int128_t newval)
{ {
return __sync_bool_compare_and_swap(addr, oldval, newval); opal_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval);
bool ret = prev == *oldval;
*oldval = prev;
return ret;
} }
#define OPAL_HAVE_ATOMIC_CMPSET_128 1 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
#endif #endif

34
opal/include/opal/sys/x86_64/atomic.h
Просмотреть файл

@ -40,9 +40,9 @@
*********************************************************************/ *********************************************************************/
#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1 #define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
#define OPAL_HAVE_ATOMIC_CMPSET_32 1 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
#define OPAL_HAVE_ATOMIC_CMPSET_64 1 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
/********************************************************************** /**********************************************************************
* *
@ -82,14 +82,13 @@ static inline void opal_atomic_isync(void)
*********************************************************************/ *********************************************************************/
#if OPAL_GCC_INLINE_ASSEMBLY #if OPAL_GCC_INLINE_ASSEMBLY
static inline bool opal_atomic_bool_cmpset_32( volatile int32_t *addr, static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
int32_t oldval, int32_t newval)
{ {
unsigned char ret; unsigned char ret;
__asm__ __volatile__ ( __asm__ __volatile__ (
SMPLOCK "cmpxchgl %3,%2 \n\t" SMPLOCK "cmpxchgl %3,%2 \n\t"
"sete %0 \n\t" "sete %0 \n\t"
: "=qm" (ret), "+a" (oldval), "+m" (*addr) : "=qm" (ret), "+a" (*oldval), "+m" (*addr)
: "q"(newval) : "q"(newval)
: "memory", "cc"); : "memory", "cc");
@ -98,19 +97,18 @@ static inline bool opal_atomic_bool_cmpset_32( volatile int32_t *addr,
#endif /* OPAL_GCC_INLINE_ASSEMBLY */ #endif /* OPAL_GCC_INLINE_ASSEMBLY */
#define opal_atomic_bool_cmpset_acq_32 opal_atomic_bool_cmpset_32 #define opal_atomic_compare_exchange_strong_acq_32 opal_atomic_compare_exchange_strong_32
#define opal_atomic_bool_cmpset_rel_32 opal_atomic_bool_cmpset_32 #define opal_atomic_compare_exchange_strong_rel_32 opal_atomic_compare_exchange_strong_32
#if OPAL_GCC_INLINE_ASSEMBLY #if OPAL_GCC_INLINE_ASSEMBLY
static inline bool opal_atomic_bool_cmpset_64( volatile int64_t *addr, static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
int64_t oldval, int64_t newval)
{ {
unsigned char ret; unsigned char ret;
__asm__ __volatile__ ( __asm__ __volatile__ (
SMPLOCK "cmpxchgq %3,%2 \n\t" SMPLOCK "cmpxchgq %3,%2 \n\t"
"sete %0 \n\t" "sete %0 \n\t"
: "=qm" (ret), "+a" (oldval), "+m" (*((volatile long*)addr)) : "=qm" (ret), "+a" (*oldval), "+m" (*((volatile long*)addr))
: "q"(newval) : "q"(newval)
: "memory", "cc" : "memory", "cc"
); );
@ -120,13 +118,12 @@ static inline bool opal_atomic_bool_cmpset_64( volatile int64_t *addr,
#endif /* OPAL_GCC_INLINE_ASSEMBLY */ #endif /* OPAL_GCC_INLINE_ASSEMBLY */
#define opal_atomic_bool_cmpset_acq_64 opal_atomic_bool_cmpset_64 #define opal_atomic_compare_exchange_strong_acq_64 opal_atomic_compare_exchange_strong_64
#define opal_atomic_bool_cmpset_rel_64 opal_atomic_bool_cmpset_64 #define opal_atomic_compare_exchange_strong_rel_64 opal_atomic_compare_exchange_strong_64
#if OPAL_GCC_INLINE_ASSEMBLY && OPAL_HAVE_CMPXCHG16B && HAVE_OPAL_INT128_T #if OPAL_GCC_INLINE_ASSEMBLY && OPAL_HAVE_CMPXCHG16B && HAVE_OPAL_INT128_T
static inline bool opal_atomic_bool_cmpset_128 (volatile opal_int128_t *addr, opal_int128_t oldval, static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128_t *addr, opal_int128_t *oldval, opal_int128_t newval)
opal_int128_t newval)
{ {
unsigned char ret; unsigned char ret;
@ -135,15 +132,14 @@ static inline bool opal_atomic_bool_cmpset_128 (volatile opal_int128_t *addr, op
* at the address is returned in eax:edx. */ * at the address is returned in eax:edx. */
__asm__ __volatile__ (SMPLOCK "cmpxchg16b (%%rsi) \n\t" __asm__ __volatile__ (SMPLOCK "cmpxchg16b (%%rsi) \n\t"
"sete %0 \n\t" "sete %0 \n\t"
: "=qm" (ret) : "=qm" (ret), "+a" (((int64_t *)oldval)[0]), "+d" (((int64_t *)oldval)[1])
: "S" (addr), "b" (((int64_t *)&newval)[0]), "c" (((int64_t *)&newval)[1]), : "S" (addr), "b" (((int64_t *)&newval)[0]), "c" (((int64_t *)&newval)[1])
"a" (((int64_t *)&oldval)[0]), "d" (((int64_t *)&oldval)[1]) : "memory", "cc", "eax", "edx");
: "memory", "cc");
return (bool) ret; return (bool) ret;
} }
#define OPAL_HAVE_ATOMIC_CMPSET_128 1 #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
#endif /* OPAL_GCC_INLINE_ASSEMBLY */ #endif /* OPAL_GCC_INLINE_ASSEMBLY */

31
opal/threads/thread_usage.h
Просмотреть файл

@ -13,7 +13,7 @@
* Copyright (c) 2007-2014 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2007-2014 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2014-2016 Research Organization for Information Science * Copyright (c) 2014-2016 Research Organization for Information Science
* and Technology (RIST). All rights reserved. * and Technology (RIST). All rights reserved.
* Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights * Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights
* reserved. * reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
@ -158,6 +158,23 @@ static inline bool opal_thread_cmpset_bool_ ## suffix (volatile addr_type *addr,
return false; \ return false; \
} }
#define OPAL_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(type, addr_type, suffix) \
static inline bool opal_thread_compare_exchange_strong_ ## suffix (volatile addr_type *addr, type *compare, type value) \
{ \
if (OPAL_UNLIKELY(opal_using_threads())) { \
return opal_atomic_compare_exchange_strong_ ## suffix ((volatile type *) addr, compare, value); \
} \
\
if ((type) *addr == *compare) { \
((type *) addr)[0] = value; \
return true; \
} \
\
*compare = ((type *) addr)[0]; \
\
return false; \
}
#define OPAL_THREAD_DEFINE_ATOMIC_SWAP(type, addr_type, suffix) \ #define OPAL_THREAD_DEFINE_ATOMIC_SWAP(type, addr_type, suffix) \
static inline type opal_thread_swap_ ## suffix (volatile addr_type *ptr, type newvalue) \ static inline type opal_thread_swap_ ## suffix (volatile addr_type *ptr, type newvalue) \
{ \ { \
@ -180,6 +197,8 @@ OPAL_THREAD_DEFINE_ATOMIC_SUB(int32_t, 32)
OPAL_THREAD_DEFINE_ATOMIC_SUB(size_t, size_t) OPAL_THREAD_DEFINE_ATOMIC_SUB(size_t, size_t)
OPAL_THREAD_DEFINE_ATOMIC_CMPSET(int32_t, int32_t, 32) OPAL_THREAD_DEFINE_ATOMIC_CMPSET(int32_t, int32_t, 32)
OPAL_THREAD_DEFINE_ATOMIC_CMPSET(void *, intptr_t, ptr) OPAL_THREAD_DEFINE_ATOMIC_CMPSET(void *, intptr_t, ptr)
OPAL_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(int32_t, int32_t, 32)
OPAL_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(void *, intptr_t, ptr)
OPAL_THREAD_DEFINE_ATOMIC_SWAP(int32_t, int32_t, 32) OPAL_THREAD_DEFINE_ATOMIC_SWAP(int32_t, int32_t, 32)
OPAL_THREAD_DEFINE_ATOMIC_SWAP(void *, intptr_t, ptr) OPAL_THREAD_DEFINE_ATOMIC_SWAP(void *, intptr_t, ptr)
@ -204,9 +223,15 @@ OPAL_THREAD_DEFINE_ATOMIC_SWAP(void *, intptr_t, ptr)
#define OPAL_THREAD_BOOL_CMPSET_32 opal_thread_cmpset_bool_32 #define OPAL_THREAD_BOOL_CMPSET_32 opal_thread_cmpset_bool_32
#define OPAL_ATOMIC_BOOL_CMPSET_32 opal_thread_cmpset_bool_32 #define OPAL_ATOMIC_BOOL_CMPSET_32 opal_thread_cmpset_bool_32
#define OPAL_THREAD_BOOL_COMPARE_EXCHANGE_STRONG_32 opal_thread_compare_exchange_strong_32
#define OPAL_ATOMIC_BOOL_COMPARE_EXCHANGE_STRONG_32 opal_thread_compare_exchange_strong_32
#define OPAL_THREAD_BOOL_CMPSET_PTR(x, y, z) opal_thread_cmpset_bool_ptr ((volatile intptr_t *) x, (void *) y, (void *) z) #define OPAL_THREAD_BOOL_CMPSET_PTR(x, y, z) opal_thread_cmpset_bool_ptr ((volatile intptr_t *) x, (void *) y, (void *) z)
#define OPAL_ATOMIC_BOOL_CMPSET_PTR OPAL_THREAD_BOOL_CMPSET_PTR #define OPAL_ATOMIC_BOOL_CMPSET_PTR OPAL_THREAD_BOOL_CMPSET_PTR
#define OPAL_THREAD_COMPARE_EXCHANGE_STRONG_PTR(x, y, z) opal_thread_compare_exchange_strong_ptr ((volatile intptr_t *) x, (void *) y, (void *) z)
#define OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR OPAL_THREAD_COMPARE_EXCHANGE_STRONG_PTR
#define OPAL_THREAD_SWAP_32 opal_thread_swap_32 #define OPAL_THREAD_SWAP_32 opal_thread_swap_32
#define OPAL_ATOMIC_SWAP_32 opal_thread_swap_32 #define OPAL_ATOMIC_SWAP_32 opal_thread_swap_32
@ -221,6 +246,7 @@ OPAL_THREAD_DEFINE_ATOMIC_AND(int64_t, 64)
OPAL_THREAD_DEFINE_ATOMIC_OR(int64_t, 64) OPAL_THREAD_DEFINE_ATOMIC_OR(int64_t, 64)
OPAL_THREAD_DEFINE_ATOMIC_XOR(int64_t, 64) OPAL_THREAD_DEFINE_ATOMIC_XOR(int64_t, 64)
OPAL_THREAD_DEFINE_ATOMIC_CMPSET(int64_t, int64_t, 64) OPAL_THREAD_DEFINE_ATOMIC_CMPSET(int64_t, int64_t, 64)
OPAL_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(int64_t, int64_t, 64)
OPAL_THREAD_DEFINE_ATOMIC_SWAP(int64_t, int64_t, 64) OPAL_THREAD_DEFINE_ATOMIC_SWAP(int64_t, int64_t, 64)
#define OPAL_THREAD_ADD64 opal_thread_add_64 #define OPAL_THREAD_ADD64 opal_thread_add_64
@ -238,6 +264,9 @@ OPAL_THREAD_DEFINE_ATOMIC_SWAP(int64_t, int64_t, 64)
#define OPAL_THREAD_BOOL_CMPSET_64 opal_thread_cmpset_bool_64 #define OPAL_THREAD_BOOL_CMPSET_64 opal_thread_cmpset_bool_64
#define OPAL_ATOMIC_BOOL_CMPSET_64 opal_thread_cmpset_bool_64 #define OPAL_ATOMIC_BOOL_CMPSET_64 opal_thread_cmpset_bool_64
#define OPAL_THREAD_COMPARE_EXCHANGE_STRONG_64 opal_thread_compare_exchange_strong_64
#define OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_64 opal_thread_compare_exchange_strong_64
#define OPAL_THREAD_SWAP_64 opal_thread_swap_64 #define OPAL_THREAD_SWAP_64 opal_thread_swap_64
#define OPAL_ATOMIC_SWAP_64 opal_thread_swap_64 #define OPAL_ATOMIC_SWAP_64 opal_thread_swap_64

127
test/asm/atomic_cmpset.c
Просмотреть файл

@ -1,3 +1,4 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/* /*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology * University Research and Technology
@ -12,6 +13,8 @@
* Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2015 Research Organization for Information Science * Copyright (c) 2015 Research Organization for Information Science
* and Technology (RIST). All rights reserved. * and Technology (RIST). All rights reserved.
* Copyright (c) 2017 Los Alamos National Security, LLC. All rights
* reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
* Additional copyrights may follow * Additional copyrights may follow
@ -54,6 +57,13 @@ int64_t old64 = 0;
int64_t new64 = 0; int64_t new64 = 0;
#endif #endif
#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128
volatile opal_int128_t vol128;
opal_int128_t val128;
opal_int128_t old128;
opal_int128_t new128;
#endif
volatile int volint = 0; volatile int volint = 0;
int valint = 0; int valint = 0;
int oldint = 0; int oldint = 0;
@ -99,124 +109,165 @@ int main(int argc, char *argv[])
/* -- cmpset 32-bit tests -- */ /* -- cmpset 32-bit tests -- */
vol32 = 42, old32 = 42, new32 = 50; vol32 = 42, old32 = 42, new32 = 50;
assert(opal_atomic_bool_cmpset_32(&vol32, old32, new32) == 1); assert(opal_atomic_compare_exchange_strong_32 (&vol32, &old32, new32) == true);
opal_atomic_rmb(); opal_atomic_rmb();
assert(vol32 == new32); assert(vol32 == new32);
assert(old32 == 42);
vol32 = 42, old32 = 420, new32 = 50; vol32 = 42, old32 = 420, new32 = 50;
assert(opal_atomic_bool_cmpset_32(&vol32, old32, new32) == 0); assert(opal_atomic_compare_exchange_strong_32 (&vol32, &old32, new32) == false);
opal_atomic_rmb(); opal_atomic_rmb();
assert(vol32 == 42); assert(vol32 == 42);
assert(old32 == 42);
vol32 = 42, old32 = 42, new32 = 50; vol32 = 42, old32 = 42, new32 = 50;
assert(opal_atomic_bool_cmpset_acq_32(&vol32, old32, new32) == 1); assert(opal_atomic_compare_exchange_strong_32 (&vol32, &old32, new32) == true);
assert(vol32 == new32); assert(vol32 == new32);
assert(old32 == 42);
vol32 = 42, old32 = 420, new32 = 50; vol32 = 42, old32 = 420, new32 = 50;
assert(opal_atomic_bool_cmpset_acq_32(&vol32, old32, new32) == 0); assert(opal_atomic_compare_exchange_strong_acq_32 (&vol32, &old32, new32) == false);
assert(vol32 == 42); assert(vol32 == 42);
assert(old32 == 42);
vol32 = 42, old32 = 42, new32 = 50; vol32 = 42, old32 = 42, new32 = 50;
assert(opal_atomic_bool_cmpset_rel_32(&vol32, old32, new32) == 1); assert(opal_atomic_compare_exchange_strong_rel_32 (&vol32, &old32, new32) == true);
opal_atomic_rmb(); opal_atomic_rmb();
assert(vol32 == new32); assert(vol32 == new32);
assert(old32 == 42);
vol32 = 42, old32 = 420, new32 = 50; vol32 = 42, old32 = 420, new32 = 50;
assert(opal_atomic_bool_cmpset_rel_32(&vol32, old32, new32) == 0); assert(opal_atomic_compare_exchange_strong_rel_32 (&vol32, &old32, new32) == false);
opal_atomic_rmb(); opal_atomic_rmb();
assert(vol32 == 42); assert(vol32 == 42);
assert(old32 == 42);
/* -- cmpset 64-bit tests -- */ /* -- cmpset 64-bit tests -- */
#if OPAL_HAVE_ATOMIC_MATH_64 #if OPAL_HAVE_ATOMIC_MATH_64
vol64 = 42, old64 = 42, new64 = 50; vol64 = 42, old64 = 42, new64 = 50;
assert(1 == opal_atomic_bool_cmpset_64(&vol64, old64, new64)); assert(opal_atomic_compare_exchange_strong_64 (&vol64, &old64, new64) == true);
opal_atomic_rmb(); opal_atomic_rmb();
assert(new64 == vol64); assert(new64 == vol64);
assert(old64 == 42);
vol64 = 42, old64 = 420, new64 = 50; vol64 = 42, old64 = 420, new64 = 50;
assert(opal_atomic_bool_cmpset_64(&vol64, old64, new64) == 0); assert(opal_atomic_compare_exchange_strong_64 (&vol64, &old64, new64) == false);
opal_atomic_rmb(); opal_atomic_rmb();
assert(vol64 == 42); assert(vol64 == 42);
assert(old64 == 42);
vol64 = 42, old64 = 42, new64 = 50; vol64 = 42, old64 = 42, new64 = 50;
assert(opal_atomic_bool_cmpset_acq_64(&vol64, old64, new64) == 1); assert(opal_atomic_compare_exchange_strong_acq_64 (&vol64, &old64, new64) == true);
assert(vol64 == new64); assert(vol64 == new64);
assert(old64 == 42);
vol64 = 42, old64 = 420, new64 = 50; vol64 = 42, old64 = 420, new64 = 50;
assert(opal_atomic_bool_cmpset_acq_64(&vol64, old64, new64) == 0); assert(opal_atomic_compare_exchange_strong_acq_64 (&vol64, &old64, new64) == false);
assert(vol64 == 42); assert(vol64 == 42);
assert(old64 == 42);
vol64 = 42, old64 = 42, new64 = 50; vol64 = 42, old64 = 42, new64 = 50;
assert(opal_atomic_bool_cmpset_rel_64(&vol64, old64, new64) == 1); assert(opal_atomic_compare_exchange_strong_rel_64 (&vol64, &old64, new64) == true);
opal_atomic_rmb(); opal_atomic_rmb();
assert(vol64 == new64); assert(vol64 == new64);
assert(old64 == 42);
vol64 = 42, old64 = 420, new64 = 50; vol64 = 42, old64 = 420, new64 = 50;
assert(opal_atomic_bool_cmpset_rel_64(&vol64, old64, new64) == 0); assert(opal_atomic_compare_exchange_strong_rel_64 (&vol64, &old64, new64) == false);
opal_atomic_rmb(); opal_atomic_rmb();
assert(vol64 == 42); assert(vol64 == 42);
assert(old64 == 42);
#endif #endif
/* -- cmpset 128-bit tests -- */
#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128
vol128 = 42, old128 = 42, new128 = 50;
assert(opal_atomic_compare_exchange_strong_128 (&vol128, &old128, new128) == true);
opal_atomic_rmb();
assert(new128 == vol128);
assert(old128 == 42);
vol128 = 42, old128 = 420, new128 = 50;
assert(opal_atomic_compare_exchange_strong_128 (&vol128, &old128, new128) == false);
opal_atomic_rmb();
assert(vol128 == 42);
assert(old128 == 42);
#endif
/* -- cmpset int tests -- */ /* -- cmpset int tests -- */
volint = 42, oldint = 42, newint = 50; volint = 42, oldint = 42, newint = 50;
assert(opal_atomic_bool_cmpset(&volint, oldint, newint) == 1); assert(opal_atomic_compare_exchange_strong (&volint, &oldint, newint) == true);
opal_atomic_rmb();
assert(volint ==newint);
volint = 42, oldint = 420, newint = 50;
assert(opal_atomic_bool_cmpset(&volint, oldint, newint) == 0);
opal_atomic_rmb();
assert(volint == 42);
volint = 42, oldint = 42, newint = 50;
assert(opal_atomic_bool_cmpset_acq(&volint, oldint, newint) == 1);
assert(volint == newint);
volint = 42, oldint = 420, newint = 50;
assert(opal_atomic_bool_cmpset_acq(&volint, oldint, newint) == 0);
assert(volint == 42);
volint = 42, oldint = 42, newint = 50;
assert(opal_atomic_bool_cmpset_rel(&volint, oldint, newint) == 1);
opal_atomic_rmb(); opal_atomic_rmb();
assert(volint == newint); assert(volint == newint);
assert(oldint == 42);
volint = 42, oldint = 420, newint = 50; volint = 42, oldint = 420, newint = 50;
assert(opal_atomic_bool_cmpset_rel(&volint, oldint, newint) == 0); assert(opal_atomic_compare_exchange_strong (&volint, &oldint, newint) == false);
opal_atomic_rmb(); opal_atomic_rmb();
assert(volint == 42); assert(volint == 42);
assert(oldint == 42);
volint = 42, oldint = 42, newint = 50;
assert(opal_atomic_compare_exchange_strong_acq (&volint, &oldint, newint) == true);
assert(volint == newint);
assert(oldint == 42);
volint = 42, oldint = 420, newint = 50;
assert(opal_atomic_compare_exchange_strong_acq (&volint, &oldint, newint) == false);
assert(volint == 42);
assert(oldint == 42);
volint = 42, oldint = 42, newint = 50;
assert(opal_atomic_compare_exchange_strong_rel (&volint, &oldint, newint) == true);
opal_atomic_rmb();
assert(volint == newint);
assert(oldint == 42);
volint = 42, oldint = 420, newint = 50;
assert(opal_atomic_compare_exchange_strong_rel (&volint, &oldint, newint) == false);
opal_atomic_rmb();
assert(volint == 42);
assert(oldint == 42);
/* -- cmpset ptr tests -- */ /* -- cmpset ptr tests -- */
volptr = (void *) 42, oldptr = (void *) 42, newptr = (void *) 50; volptr = (void *) 42, oldptr = (void *) 42, newptr = (void *) 50;
assert(opal_atomic_bool_cmpset_ptr(&volptr, oldptr, newptr) == 1); assert(opal_atomic_compare_exchange_strong_ptr (&volptr, &oldptr, newptr) == true);
opal_atomic_rmb(); opal_atomic_rmb();
assert(volptr == newptr); assert(volptr == newptr);
assert(oldptr == (void *) 42);
volptr = (void *) 42, oldptr = (void *) 420, newptr = (void *) 50; volptr = (void *) 42, oldptr = (void *) 420, newptr = (void *) 50;
assert(opal_atomic_bool_cmpset_ptr(&volptr, oldptr, newptr) == 0); assert(opal_atomic_compare_exchange_strong_ptr (&volptr, &oldptr, newptr) == false);
opal_atomic_rmb(); opal_atomic_rmb();
assert(volptr == (void *) 42); assert(volptr == (void *) 42);
assert(oldptr == (void *) 42);
volptr = (void *) 42, oldptr = (void *) 42, newptr = (void *) 50; volptr = (void *) 42, oldptr = (void *) 42, newptr = (void *) 50;
assert(opal_atomic_bool_cmpset_acq_ptr(&volptr, oldptr, newptr) == 1); assert(opal_atomic_compare_exchange_strong_acq_ptr (&volptr, &oldptr, newptr) == true);
assert(volptr == newptr); assert(volptr == newptr);
assert(oldptr == (void *) 42);
volptr = (void *) 42, oldptr = (void *) 420, newptr = (void *) 50; volptr = (void *) 42, oldptr = (void *) 420, newptr = (void *) 50;
assert(opal_atomic_bool_cmpset_acq_ptr(&volptr, oldptr, newptr) == 0); assert(opal_atomic_compare_exchange_strong_acq_ptr (&volptr, &oldptr, newptr) == false);
assert(volptr == (void *) 42); assert(volptr == (void *) 42);
assert(oldptr == (void *) 42);
volptr = (void *) 42, oldptr = (void *) 42, newptr = (void *) 50; volptr = (void *) 42, oldptr = (void *) 42, newptr = (void *) 50;
assert(opal_atomic_bool_cmpset_rel_ptr(&volptr, oldptr, newptr) == 1); assert(opal_atomic_compare_exchange_strong_rel_ptr (&volptr, &oldptr, newptr) == true);
opal_atomic_rmb(); opal_atomic_rmb();
assert(volptr == newptr); assert(volptr == newptr);
assert(oldptr == (void *) 42);
volptr = (void *) 42, oldptr = (void *) 420, newptr = (void *) 50; volptr = (void *) 42, oldptr = (void *) 420, newptr = (void *) 50;
assert(opal_atomic_bool_cmpset_rel_ptr(&volptr, oldptr, newptr) == 0); assert(opal_atomic_compare_exchange_strong_rel_ptr (&volptr, &oldptr, newptr) == false);
opal_atomic_rmb(); opal_atomic_rmb();
assert(volptr == (void *) 42); assert(volptr == (void *) 42);
assert(oldptr == (void *) 42);
/* -- add_32 tests -- */ /* -- add_32 tests -- */