ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

opal/asm: add opal_atomic_compare_exchange_strong functions

Просмотр исходного кода 
This commit adds a new set of compare-and-exchange functions. These
functions have a signature similar to the functions found in C11. The
old cmpset functions are now deprecated and defined in terms of the
new compare-and-exchange functions. All asm backends have been
updated.

Signed-off-by: Nathan Hjelm <hjelmn@lanl.gov>
Этот коммит содержится в:
Nathan Hjelm 2017-11-08 09:54:54 -07:00
родитель 45db3637af
Коммит 84f63d0aca
14 изменённых файлов: 714 добавлений и 547 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

2
opal/class/opal_fifo.h
Просмотреть файл

@ -76,7 +76,7 @@ static inline bool opal_fifo_is_empty( opal_fifo_t* fifo )
return opal_fifo_head (fifo) == &fifo->opal_fifo_ghost;
}
#if OPAL_HAVE_ATOMIC_CMPSET_128
#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128
/* Add one element to the FIFO. We will return the last head of the list
* to allow the upper level to detect if this element is the first one in the

10
opal/class/opal_lifo.h
Просмотреть файл

@ -36,8 +36,8 @@
BEGIN_C_DECLS
/* NTH: temporarily suppress warnings about this not being defined */
#if !defined(OPAL_HAVE_ATOMIC_CMPSET_128)
#define OPAL_HAVE_ATOMIC_CMPSET_128 0
#if !defined(OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128)
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 0
#endif
/**
@ -50,7 +50,7 @@ union opal_counted_pointer_t {
/** list item pointer */
volatile opal_list_item_t * volatile item;
} data;
#if OPAL_HAVE_ATOMIC_CMPSET_128 && HAVE_OPAL_INT128_T
#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 && HAVE_OPAL_INT128_T
/** used for atomics when there is a cmpset that can operate on
* two 64-bit values */
opal_int128_t value;
@ -59,7 +59,7 @@ union opal_counted_pointer_t {
typedef union opal_counted_pointer_t opal_counted_pointer_t;
#if OPAL_HAVE_ATOMIC_CMPSET_128
#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128
/* Add one element to the FIFO. We will return the last head of the list
* to allow the upper level to detect if this element is the first one in the
@ -110,7 +110,7 @@ static inline bool opal_lifo_is_empty( opal_lifo_t* lifo )
}
#if OPAL_HAVE_ATOMIC_CMPSET_128
#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128
/* Add one element to the LIFO. We will return the last head of the list
* to allow the upper level to detect if this element is the first one in the

104
opal/include/opal/sys/arm/atomic.h
Просмотреть файл

@ -1,3 +1,4 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
@ -11,6 +12,8 @@
* All rights reserved.
* Copyright (c) 2010 IBM Corporation. All rights reserved.
* Copyright (c) 2010 ARM ltd. All rights reserved.
* Copyright (c) 2017 Los Alamos National Security, LLC. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -104,12 +107,12 @@ void opal_atomic_isync(void)
#if (OPAL_GCC_INLINE_ASSEMBLY && (OPAL_ASM_ARM_VERSION >= 6))
#define OPAL_HAVE_ATOMIC_CMPSET_32 1
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
#define OPAL_HAVE_ATOMIC_MATH_32 1
static inline bool opal_atomic_bool_cmpset_32(volatile int32_t *addr,
int32_t oldval, int32_t newval)
static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
{
int32_t ret, tmp;
int32_t prev, tmp;
bool ret;
__asm__ __volatile__ (
"1: ldrex %0, [%2] \n"
@ -120,11 +123,13 @@ static inline bool opal_atomic_bool_cmpset_32(volatile int32_t *addr,
" bne 1b \n"
"2: \n"
: "=&r" (ret), "=&r" (tmp)
: "r" (addr), "r" (oldval), "r" (newval)
: "=&r" (prev), "=&r" (tmp)
: "r" (addr), "r" (*oldval), "r" (newval)
: "cc", "memory");
return (ret == oldval);
ret = (prev == *oldval);
*oldval = prev;
return ret;
}
/* these two functions aren't inlined in the non-gcc case because then
@ -132,51 +137,50 @@ static inline bool opal_atomic_bool_cmpset_32(volatile int32_t *addr,
atomic_?mb can be inlined). Instead, we "inline" them by hand in
the assembly, meaning there is one function call overhead instead
of two */
static inline bool opal_atomic_bool_cmpset_acq_32(volatile int32_t *addr,
int32_t oldval, int32_t newval)
static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
{
bool rc;
rc = opal_atomic_bool_cmpset_32(addr, oldval, newval);
rc = opal_atomic_compare_exchange_strong_32 (addr, oldval, newval);
opal_atomic_rmb();
return rc;
}
static inline bool opal_atomic_bool_cmpset_rel_32(volatile int32_t *addr,
int32_t oldval, int32_t newval)
static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
{
opal_atomic_wmb();
return opal_atomic_bool_cmpset_32(addr, oldval, newval);
return opal_atomic_compare_exchange_strong_32 (addr, oldval, newval);
}
#if (OPAL_ASM_SUPPORT_64BIT == 1)
#define OPAL_HAVE_ATOMIC_CMPSET_64 1
static inline bool opal_atomic_bool_cmpset_64(volatile int64_t *addr,
int64_t oldval, int64_t newval)
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
{
int64_t ret;
int tmp;
int64_t prev;
int tmp;
bool ret;
__asm__ __volatile__ (
"1: ldrexd %0, %H0, [%2] \n"
" cmp %0, %3 \n"
" it eq \n"
" cmpeq %H0, %H3 \n"
" bne 2f \n"
" strexd %1, %4, %H4, [%2] \n"
" cmp %1, #0 \n"
" bne 1b \n"
"2: \n"
__asm__ __volatile__ (
"1: ldrexd %0, %H0, [%2] \n"
" cmp %0, %3 \n"
" it eq \n"
" cmpeq %H0, %H3 \n"
" bne 2f \n"
" strexd %1, %4, %H4, [%2] \n"
" cmp %1, #0 \n"
" bne 1b \n"
"2: \n"
: "=&r" (prev), "=&r" (tmp)
: "r" (addr), "r" (*oldval), "r" (newval)
: "cc", "memory");
: "=&r" (ret), "=&r" (tmp)
: "r" (addr), "r" (oldval), "r" (newval)
: "cc", "memory");
return (ret == oldval);
ret = (prev == *oldval);
*oldval = prev;
return ret;
}
/* these two functions aren't inlined in the non-gcc case because then
@ -184,23 +188,21 @@ static inline bool opal_atomic_bool_cmpset_64(volatile int64_t *addr,
atomic_?mb can be inlined). Instead, we "inline" them by hand in
the assembly, meaning there is one function call overhead instead
of two */
static inline bool opal_atomic_bool_cmpset_acq_64(volatile int64_t *addr,
int64_t oldval, int64_t newval)
static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
{
bool rc;
rc = opal_atomic_bool_cmpset_64(addr, oldval, newval);
rc = opal_atomic_compare_exchange_strong_64 (addr, oldval, newval);
opal_atomic_rmb();
return rc;
}
static inline bool opal_atomic_bool_cmpset_rel_64(volatile int64_t *addr,
int64_t oldval, int64_t newval)
static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
{
opal_atomic_wmb();
return opal_atomic_bool_cmpset_64(addr, oldval, newval);
return opal_atomic_compare_exchange_strong_64 (addr, oldval, newval);
}
#endif
@ -247,30 +249,6 @@ static inline int32_t opal_atomic_sub_32(volatile int32_t* v, int dec)
return t;
}
#else /* OPAL_ASM_ARM_VERSION <=5 or no GCC inline assembly */
#define OPAL_HAVE_ATOMIC_CMPSET_32 1
#define __kuser_cmpxchg (*((int (*)(int, int, volatile int*))(0xffff0fc0)))
static inline bool opal_atomic_bool_cmpset_32(volatile int32_t *addr,
int32_t oldval, int32_t newval)
{
return !(__kuser_cmpxchg(oldval, newval, addr));
}
static inline bool opal_atomic_bool_cmpset_acq_32(volatile int32_t *addr,
int32_t oldval, int32_t newval)
{
/* kernel function includes all necessary memory barriers */
return opal_atomic_bool_cmpset_32(addr, oldval, newval);
}
static inline bool opal_atomic_bool_cmpset_rel_32(volatile int32_t *addr,
int32_t oldval, int32_t newval)
{
/* kernel function includes all necessary memory barriers */
return opal_atomic_bool_cmpset_32(addr, oldval, newval);
}
#endif
#endif /* ! OPAL_SYS_ARCH_ATOMIC_H */

88
opal/include/opal/sys/arm64/atomic.h
Просмотреть файл

@ -29,10 +29,10 @@
#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
#define OPAL_HAVE_ATOMIC_LLSC_32 1
#define OPAL_HAVE_ATOMIC_CMPSET_32 1
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
#define OPAL_HAVE_ATOMIC_SWAP_32 1
#define OPAL_HAVE_ATOMIC_MATH_32 1
#define OPAL_HAVE_ATOMIC_CMPSET_64 1
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
#define OPAL_HAVE_ATOMIC_SWAP_64 1
#define OPAL_HAVE_ATOMIC_LLSC_64 1
#define OPAL_HAVE_ATOMIC_ADD_32 1
@ -82,10 +82,10 @@ static inline void opal_atomic_isync (void)
*
*********************************************************************/
static inline bool opal_atomic_bool_cmpset_32(volatile int32_t *addr,
int32_t oldval, int32_t newval)
static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
{
int32_t ret, tmp;
int32_t prev, tmp;
bool ret;
__asm__ __volatile__ ("1: ldaxr %w0, [%2] \n"
" cmp %w0, %w3 \n"
@ -93,11 +93,13 @@ static inline bool opal_atomic_bool_cmpset_32(volatile int32_t *addr,
" stxr %w1, %w4, [%2] \n"
" cbnz %w1, 1b \n"
"2: \n"
: "=&r" (ret), "=&r" (tmp)
: "r" (addr), "r" (oldval), "r" (newval)
: "=&r" (prev), "=&r" (tmp)
: "r" (addr), "r" (*oldval), "r" (newval)
: "cc", "memory");
return (ret == oldval);
ret = (prev == *oldval);
*oldval = prev;
return ret;
}
static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, int32_t newval)
@ -119,10 +121,10 @@ static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, int32_t newval
atomic_?mb can be inlined). Instead, we "inline" them by hand in
the assembly, meaning there is one function call overhead instead
of two */
static inline bool opal_atomic_bool_cmpset_acq_32(volatile int32_t *addr,
int32_t oldval, int32_t newval)
static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
{
int32_t ret, tmp;
int32_t prev, tmp;
bool ret;
__asm__ __volatile__ ("1: ldaxr %w0, [%2] \n"
" cmp %w0, %w3 \n"
@ -130,18 +132,20 @@ static inline bool opal_atomic_bool_cmpset_acq_32(volatile int32_t *addr,
" stxr %w1, %w4, [%2] \n"
" cbnz %w1, 1b \n"
"2: \n"
: "=&r" (ret), "=&r" (tmp)
: "r" (addr), "r" (oldval), "r" (newval)
: "=&r" (prev), "=&r" (tmp)
: "r" (addr), "r" (*oldval), "r" (newval)
: "cc", "memory");
return (ret == oldval);
ret = (prev == *oldval);
*oldval = prev;
return ret;
}
static inline bool opal_atomic_bool_cmpset_rel_32(volatile int32_t *addr,
int32_t oldval, int32_t newval)
static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
{
int32_t ret, tmp;
int32_t prev, tmp;
bool ret;
__asm__ __volatile__ ("1: ldxr %w0, [%2] \n"
" cmp %w0, %w3 \n"
@ -149,11 +153,13 @@ static inline bool opal_atomic_bool_cmpset_rel_32(volatile int32_t *addr,
" stlxr %w1, %w4, [%2] \n"
" cbnz %w1, 1b \n"
"2: \n"
: "=&r" (ret), "=&r" (tmp)
: "r" (addr), "r" (oldval), "r" (newval)
: "=&r" (prev), "=&r" (tmp)
: "r" (addr), "r" (*oldval), "r" (newval)
: "cc", "memory");
return (ret == oldval);
ret = (prev == *oldval);
*oldval = prev;
return ret;
}
static inline int32_t opal_atomic_ll_32 (volatile int32_t *addr)
@ -179,11 +185,11 @@ static inline int opal_atomic_sc_32 (volatile int32_t *addr, int32_t newval)
return ret == 0;
}
static inline bool opal_atomic_bool_cmpset_64(volatile int64_t *addr,
int64_t oldval, int64_t newval)
static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
{
int64_t ret;
int64_t prev;
int tmp;
bool ret;
__asm__ __volatile__ ("1: ldaxr %0, [%2] \n"
" cmp %0, %3 \n"
@ -191,11 +197,13 @@ static inline bool opal_atomic_bool_cmpset_64(volatile int64_t *addr,
" stxr %w1, %4, [%2] \n"
" cbnz %w1, 1b \n"
"2: \n"
: "=&r" (ret), "=&r" (tmp)
: "r" (addr), "r" (oldval), "r" (newval)
: "=&r" (prev), "=&r" (tmp)
: "r" (addr), "r" (*oldval), "r" (newval)
: "cc", "memory");
return (ret == oldval);
ret = (prev == oldval);
*oldval = prev;
return ret;
}
static inline int64_t opal_atomic_swap_64 (volatile int64_t *addr, int64_t newval)
@ -218,11 +226,11 @@ static inline int64_t opal_atomic_swap_64 (volatile int64_t *addr, int64_t newva
atomic_?mb can be inlined). Instead, we "inline" them by hand in
the assembly, meaning there is one function call overhead instead
of two */
static inline bool opal_atomic_bool_cmpset_acq_64(volatile int64_t *addr,
int64_t oldval, int64_t newval)
static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
{
int64_t ret;
int64_t prev;
int tmp;
bool ret;
__asm__ __volatile__ ("1: ldaxr %0, [%2] \n"
" cmp %0, %3 \n"
@ -230,19 +238,21 @@ static inline bool opal_atomic_bool_cmpset_acq_64(volatile int64_t *addr,
" stxr %w1, %4, [%2] \n"
" cbnz %w1, 1b \n"
"2: \n"
: "=&r" (ret), "=&r" (tmp)
: "r" (addr), "r" (oldval), "r" (newval)
: "=&r" (prev), "=&r" (tmp)
: "r" (addr), "r" (*oldval), "r" (newval)
: "cc", "memory");
return (ret == oldval);
ret = (prev == oldval);
*oldval = prev;
return ret;
}
static inline bool opal_atomic_bool_cmpset_rel_64(volatile int64_t *addr,
int64_t oldval, int64_t newval)
static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
{
int64_t ret;
int64_t prev;
int tmp;
bool ret;
__asm__ __volatile__ ("1: ldxr %0, [%2] \n"
" cmp %0, %3 \n"
@ -250,11 +260,13 @@ static inline bool opal_atomic_bool_cmpset_rel_64(volatile int64_t *addr,
" stlxr %w1, %4, [%2] \n"
" cbnz %w1, 1b \n"
"2: \n"
: "=&r" (ret), "=&r" (tmp)
: "r" (addr), "r" (oldval), "r" (newval)
: "=&r" (prev), "=&r" (tmp)
: "r" (addr), "r" (*oldval), "r" (newval)
: "cc", "memory");
return (ret == oldval);
ret = (prev == oldval);
*oldval = prev;
return ret;
}
static inline int64_t opal_atomic_ll_64 (volatile int64_t *addr)

177
opal/include/opal/sys/atomic.h
Просмотреть файл

@ -40,11 +40,11 @@
*
* - \c OPAL_HAVE_ATOMIC_MEM_BARRIER atomic memory barriers
* - \c OPAL_HAVE_ATOMIC_SPINLOCKS atomic spinlocks
* - \c OPAL_HAVE_ATOMIC_MATH_32 if 32 bit add/sub/cmpset can be done "atomicly"
* - \c OPAL_HAVE_ATOMIC_MATH_64 if 64 bit add/sub/cmpset can be done "atomicly"
* - \c OPAL_HAVE_ATOMIC_MATH_32 if 32 bit add/sub/compare-exchange can be done "atomicly"
* - \c OPAL_HAVE_ATOMIC_MATH_64 if 64 bit add/sub/compare-exchange can be done "atomicly"
*
* Note that for the Atomic math, atomic add/sub may be implemented as
* C code using opal_atomic_bool_cmpset. The appearance of atomic
* C code using opal_atomic_compare_exchange. The appearance of atomic
* operation will be upheld in these cases.
*/
@ -107,8 +107,8 @@ typedef struct opal_atomic_lock_t opal_atomic_lock_t;
*********************************************************************/
#if !OPAL_GCC_INLINE_ASSEMBLY
#define OPAL_HAVE_INLINE_ATOMIC_MEM_BARRIER 0
#define OPAL_HAVE_INLINE_ATOMIC_CMPSET_32 0
#define OPAL_HAVE_INLINE_ATOMIC_CMPSET_64 0
#define OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32 0
#define OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64 0
#define OPAL_HAVE_INLINE_ATOMIC_ADD_32 0
#define OPAL_HAVE_INLINE_ATOMIC_AND_32 0
#define OPAL_HAVE_INLINE_ATOMIC_OR_32 0
@ -123,8 +123,8 @@ typedef struct opal_atomic_lock_t opal_atomic_lock_t;
#define OPAL_HAVE_INLINE_ATOMIC_SWAP_64 0
#else
#define OPAL_HAVE_INLINE_ATOMIC_MEM_BARRIER 1
#define OPAL_HAVE_INLINE_ATOMIC_CMPSET_32 1
#define OPAL_HAVE_INLINE_ATOMIC_CMPSET_64 1
#define OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32 1
#define OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64 1
#define OPAL_HAVE_INLINE_ATOMIC_ADD_32 1
#define OPAL_HAVE_INLINE_ATOMIC_AND_32 1
#define OPAL_HAVE_INLINE_ATOMIC_OR_32 1
@ -187,14 +187,14 @@ enum {
/* compare and set operations can't really be emulated from software,
so if these defines aren't already set, they should be set to 0
now */
#ifndef OPAL_HAVE_ATOMIC_CMPSET_32
#define OPAL_HAVE_ATOMIC_CMPSET_32 0
#ifndef OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 0
#endif
#ifndef OPAL_HAVE_ATOMIC_CMPSET_64
#define OPAL_HAVE_ATOMIC_CMPSET_64 0
#ifndef OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 0
#endif
#ifndef OPAL_HAVE_ATOMIC_CMPSET_128
#define OPAL_HAVE_ATOMIC_CMPSET_128 0
#ifndef OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 0
#endif
#ifndef OPAL_HAVE_ATOMIC_LLSC_32
#define OPAL_HAVE_ATOMIC_LLSC_32 0
@ -270,7 +270,7 @@ void opal_atomic_wmb(void);
/**********************************************************************
*
* Atomic spinlocks - always inlined, if have atomic cmpset
* Atomic spinlocks - always inlined, if have atomic compare-and-swap
*
*********************************************************************/
@ -280,7 +280,7 @@ void opal_atomic_wmb(void);
#define OPAL_HAVE_ATOMIC_SPINLOCKS 0
#endif
#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_SPINLOCKS || (OPAL_HAVE_ATOMIC_CMPSET_32 || OPAL_HAVE_ATOMIC_CMPSET_64)
#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_SPINLOCKS || (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
/**
* Initialize a lock to value
@ -330,7 +330,7 @@ void opal_atomic_unlock(opal_atomic_lock_t *lock);
#if OPAL_HAVE_ATOMIC_SPINLOCKS == 0
#undef OPAL_HAVE_ATOMIC_SPINLOCKS
#define OPAL_HAVE_ATOMIC_SPINLOCKS (OPAL_HAVE_ATOMIC_CMPSET_32 || OPAL_HAVE_ATOMIC_CMPSET_64)
#define OPAL_HAVE_ATOMIC_SPINLOCKS (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
#define OPAL_NEED_INLINE_ATOMIC_SPINLOCKS 1
#endif
@ -353,25 +353,40 @@ static inline
bool opal_atomic_bool_cmpset_32(volatile int32_t *addr, int32_t oldval,
int32_t newval);
#if OPAL_HAVE_INLINE_ATOMIC_CMPSET_32
static inline
#endif
bool opal_atomic_bool_cmpset_acq_32(volatile int32_t *addr, int32_t oldval,
int32_t newval);
#if OPAL_HAVE_INLINE_ATOMIC_CMPSET_32
static inline
#endif
bool opal_atomic_bool_cmpset_rel_32(volatile int32_t *addr, int32_t oldval,
int32_t newval);
#endif
#if !defined(OPAL_HAVE_ATOMIC_CMPSET_64) && !defined(DOXYGEN)
#define OPAL_HAVE_ATOMIC_CMPSET_64 0
#if !defined(OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64) && !defined(DOXYGEN)
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 0
#endif
#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_CMPSET_64
#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
#if OPAL_HAVE_INLINE_ATOMIC_CMPSET_64
static inline
#endif
bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval,
int64_t newval);
#if OPAL_HAVE_INLINE_ATOMIC_CMPSET_64
static inline
#endif
bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval,
int64_t newval);
#if OPAL_HAVE_INLINE_ATOMIC_CMPSET_64
static inline
#endif
bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval,
int64_t newval);
/* XXX -- DEPRECATED -- XXX -- Legacy cmpset functions */
#if OPAL_HAVE_INLINE_ATOMIC_CMPSET_64
static inline
#endif
@ -397,35 +412,35 @@ bool opal_atomic_bool_cmpset_rel_64(volatile int64_t *addr, int64_t oldval,
#define OPAL_HAVE_ATOMIC_MATH_32 0
#endif
#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_CMPSET_32
#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
/* OPAL_HAVE_INLINE_ATOMIC_*_32 will be 1 if <arch>/atomic.h provides
a static inline version of it (in assembly). If we have to fall
back on cmpset 32, that too will be inline. */
#if OPAL_HAVE_INLINE_ATOMIC_ADD_32 || (!defined(OPAL_HAVE_ATOMIC_ADD_32) && OPAL_HAVE_ATOMIC_CMPSET_32)
back on compare-exchange 32, that too will be inline. */
#if OPAL_HAVE_INLINE_ATOMIC_ADD_32 || (!defined(OPAL_HAVE_ATOMIC_ADD_32) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32)
static inline
#endif
int32_t opal_atomic_add_32(volatile int32_t *addr, int delta);
#if OPAL_HAVE_INLINE_ATOMIC_AND_32 || (!defined(OPAL_HAVE_ATOMIC_AND_32) && OPAL_HAVE_ATOMIC_CMPSET_32)
#if OPAL_HAVE_INLINE_ATOMIC_AND_32 || (!defined(OPAL_HAVE_ATOMIC_AND_32) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32)
static inline
#endif
int32_t opal_atomic_and_32(volatile int32_t *addr, int32_t value);
#if OPAL_HAVE_INLINE_ATOMIC_OR_32 || (!defined(OPAL_HAVE_ATOMIC_OR_32) && OPAL_HAVE_ATOMIC_CMPSET_32)
#if OPAL_HAVE_INLINE_ATOMIC_OR_32 || (!defined(OPAL_HAVE_ATOMIC_OR_32) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32)
static inline
#endif
int32_t opal_atomic_or_32(volatile int32_t *addr, int32_t value);
#if OPAL_HAVE_INLINE_ATOMIC_XOR_32 || (!defined(OPAL_HAVE_ATOMIC_XOR_32) && OPAL_HAVE_ATOMIC_CMPSET_32)
#if OPAL_HAVE_INLINE_ATOMIC_XOR_32 || (!defined(OPAL_HAVE_ATOMIC_XOR_32) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32)
static inline
#endif
int32_t opal_atomic_xor_32(volatile int32_t *addr, int32_t value);
/* OPAL_HAVE_INLINE_ATOMIC_*_32 will be 1 if <arch>/atomic.h provides
a static inline version of it (in assembly). If we have to fall
back to cmpset 32, that too will be inline. */
#if OPAL_HAVE_INLINE_ATOMIC_SUB_32 || (!defined(OPAL_HAVE_ATOMIC_ADD_32) && OPAL_HAVE_ATOMIC_CMPSET_32)
back to compare-exchange 32, that too will be inline. */
#if OPAL_HAVE_INLINE_ATOMIC_SUB_32 || (!defined(OPAL_HAVE_ATOMIC_ADD_32) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32)
static inline
#endif
int32_t opal_atomic_sub_32(volatile int32_t *addr, int delta);
@ -435,7 +450,7 @@ int32_t opal_atomic_sub_32(volatile int32_t *addr, int delta);
#if ! OPAL_HAVE_ATOMIC_MATH_32
/* fix up the value of opal_have_atomic_math_32 to allow for C versions */
#undef OPAL_HAVE_ATOMIC_MATH_32
#define OPAL_HAVE_ATOMIC_MATH_32 OPAL_HAVE_ATOMIC_CMPSET_32
#define OPAL_HAVE_ATOMIC_MATH_32 OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
#endif
#ifndef OPAL_HAVE_ATOMIC_MATH_64
@ -443,35 +458,35 @@ int32_t opal_atomic_sub_32(volatile int32_t *addr, int delta);
#define OPAL_HAVE_ATOMIC_MATH_64 0
#endif
#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MATH_64 || OPAL_HAVE_ATOMIC_CMPSET_64
#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MATH_64 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
/* OPAL_HAVE_INLINE_ATOMIC_*_64 will be 1 if <arch>/atomic.h provides
a static inline version of it (in assembly). If we have to fall
back to cmpset 64, that too will be inline */
#if OPAL_HAVE_INLINE_ATOMIC_ADD_64 || (!defined(OPAL_HAVE_ATOMIC_ADD_64) && OPAL_HAVE_ATOMIC_CMPSET_64)
back to compare-exchange 64, that too will be inline */
#if OPAL_HAVE_INLINE_ATOMIC_ADD_64 || (!defined(OPAL_HAVE_ATOMIC_ADD_64) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
static inline
#endif
int64_t opal_atomic_add_64(volatile int64_t *addr, int64_t delta);
#if OPAL_HAVE_INLINE_ATOMIC_AND_64 || (!defined(OPAL_HAVE_ATOMIC_AND_64) && OPAL_HAVE_ATOMIC_CMPSET_64)
#if OPAL_HAVE_INLINE_ATOMIC_AND_64 || (!defined(OPAL_HAVE_ATOMIC_AND_64) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
static inline
#endif
int64_t opal_atomic_and_64(volatile int64_t *addr, int64_t value);
#if OPAL_HAVE_INLINE_ATOMIC_OR_64 || (!defined(OPAL_HAVE_ATOMIC_OR_64) && OPAL_HAVE_ATOMIC_CMPSET_64)
#if OPAL_HAVE_INLINE_ATOMIC_OR_64 || (!defined(OPAL_HAVE_ATOMIC_OR_64) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
static inline
#endif
int64_t opal_atomic_or_64(volatile int64_t *addr, int64_t value);
#if OPAL_HAVE_INLINE_ATOMIC_XOR_64 || (!defined(OPAL_HAVE_ATOMIC_XOR_64) && OPAL_HAVE_ATOMIC_CMPSET_64)
#if OPAL_HAVE_INLINE_ATOMIC_XOR_64 || (!defined(OPAL_HAVE_ATOMIC_XOR_64) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
static inline
#endif
int64_t opal_atomic_xor_64(volatile int64_t *addr, int64_t value);
/* OPAL_HAVE_INLINE_ATOMIC_*_64 will be 1 if <arch>/atomic.h provides
a static inline version of it (in assembly). If we have to fall
back to cmpset 64, that too will be inline */
#if OPAL_HAVE_INLINE_ATOMIC_SUB_64 || (!defined(OPAL_HAVE_ATOMIC_ADD_64) && OPAL_HAVE_ATOMIC_CMPSET_64)
back to compare-exchange 64, that too will be inline */
#if OPAL_HAVE_INLINE_ATOMIC_SUB_64 || (!defined(OPAL_HAVE_ATOMIC_ADD_64) && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
static inline
#endif
int64_t opal_atomic_sub_64(volatile int64_t *addr, int64_t delta);
@ -481,7 +496,7 @@ int64_t opal_atomic_sub_64(volatile int64_t *addr, int64_t delta);
#if ! OPAL_HAVE_ATOMIC_MATH_64
/* fix up the value of opal_have_atomic_math_64 to allow for C versions */
#undef OPAL_HAVE_ATOMIC_MATH_64
#define OPAL_HAVE_ATOMIC_MATH_64 OPAL_HAVE_ATOMIC_CMPSET_64
#define OPAL_HAVE_ATOMIC_MATH_64 OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
#endif
/* provide a size_t add/subtract. When in debug mode, make it an
@ -524,9 +539,26 @@ opal_atomic_sub_size_t(volatile size_t *addr, size_t delta)
#endif
#endif
#if defined(DOXYGEN) || (OPAL_HAVE_ATOMIC_CMPSET_32 || OPAL_HAVE_ATOMIC_CMPSET_64)
#if defined(DOXYGEN) || (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
/* these are always done with inline functions, so always mark as
static inline */
static inline bool opal_atomic_compare_exchange_strong_xx (volatile void *addr, void *oldval,
int64_t newval, size_t length);
static inline bool opal_atomic_compare_exchange_strong_acq_xx (volatile void *addr, void *oldval,
int64_t newval, size_t length);
static inline bool opal_atomic_compare_exchange_strong_rel_xx (volatile void *addr, void *oldval,
int64_t newval, size_t length);
static inline bool opal_atomic_compare_exchange_strong_ptr (volatile void* addr, void *oldval,
void *newval);
static inline bool opal_atomic_compare_exchange_strong_acq_ptr (volatile void* addr, void *oldval,
void *newval);
static inline bool opal_atomic_compare_exchange_strong_rel_ptr (volatile void* addr, void *oldval,
void *newval);
/* XXX -- DEPRECATED -- XXX -- Define legacy cmpset functions */
static inline bool opal_atomic_bool_cmpset_xx(volatile void* addr, int64_t oldval,
int64_t newval, size_t length);
static inline bool opal_atomic_bool_cmpset_acq_xx(volatile void* addr,
@ -546,6 +578,61 @@ static inline bool opal_atomic_bool_cmpset_rel_ptr(volatile void* addr,
void* oldval,
void* newval);
/**
* Atomic compare and set of generic type with relaxed semantics. This
* macro detect at compile time the type of the first argument and
* choose the correct function to be called.
*
* \note This macro should only be used for integer types.
*
* @param addr Address of <TYPE>.
* @param oldval Comparison value address of <TYPE>.
* @param newval New value to set if comparision is true <TYPE>.
*
* See opal_atomic_compare_exchange_* for pseudo-code.
*/
#define opal_atomic_compare_exchange_strong( ADDR, OLDVAL, NEWVAL ) \
opal_atomic_compare_exchange_strong_xx( (volatile void*)(ADDR), (void *)(OLDVAL), \
(intptr_t)(NEWVAL), sizeof(*(ADDR)) )
/**
* Atomic compare and set of generic type with acquire semantics. This
* macro detect at compile time the type of the first argument and
* choose the correct function to be called.
*
* \note This macro should only be used for integer types.
*
* @param addr Address of <TYPE>.
* @param oldval Comparison value address of <TYPE>.
* @param newval New value to set if comparision is true <TYPE>.
*
* See opal_atomic_compare_exchange_acq_* for pseudo-code.
*/
#define opal_atomic_compare_exchange_strong_acq( ADDR, OLDVAL, NEWVAL ) \
opal_atomic_compare_exchange_strong_acq_xx( (volatile void*)(ADDR), (void *)(OLDVAL), \
(intptr_t)(NEWVAL), sizeof(*(ADDR)) )
/**
* Atomic compare and set of generic type with release semantics. This
* macro detect at compile time the type of the first argument and
* choose the correct function to be called.
*
* \note This macro should only be used for integer types.
*
* @param addr Address of <TYPE>.
* @param oldval Comparison value address of <TYPE>.
* @param newval New value to set if comparision is true <TYPE>.
*
* See opal_atomic_compare_exchange_rel_* for pseudo-code.
*/
#define opal_atomic_compare_exchange_strong_rel( ADDR, OLDVAL, NEWVAL ) \
opal_atomic_compare_exchange_strong_rel_xx( (volatile void*)(ADDR), (void *)(OLDVAL), \
(intptr_t)(NEWVAL), sizeof(*(ADDR)) )
/* XXX -- DEPRECATED -- XXX -- Define legacy cmpset functions */
/**
* Atomic compare and set of pointer with relaxed semantics. This
* macro detect at compile time the type of the first argument and
@ -598,7 +685,7 @@ static inline bool opal_atomic_bool_cmpset_rel_ptr(volatile void* addr,
opal_atomic_bool_cmpset_rel_xx( (volatile void*)(ADDR), (int64_t)(OLDVAL), \
(int64_t)(NEWVAL), sizeof(*(ADDR)) )
#endif /* (OPAL_HAVE_ATOMIC_CMPSET_32 || OPAL_HAVE_ATOMIC_CMPSET_64) */
#endif /* (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64) */
#if defined(DOXYGEN) || (OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64)
@ -606,10 +693,10 @@ static inline void opal_atomic_add_xx(volatile void* addr,
int32_t value, size_t length);
static inline void opal_atomic_sub_xx(volatile void* addr,
int32_t value, size_t length);
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_CMPSET_32
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
static inline int32_t opal_atomic_add_ptr( volatile void* addr, void* delta );
static inline int32_t opal_atomic_sub_ptr( volatile void* addr, void* delta );
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_CMPSET_64
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
static inline int64_t opal_atomic_add_ptr( volatile void* addr, void* delta );
static inline int64_t opal_atomic_sub_ptr( volatile void* addr, void* delta );
#else

285
opal/include/opal/sys/atomic_impl.h
Просмотреть файл

@ -34,10 +34,22 @@
*
* Some architectures do not provide support for the 64 bits
* atomic operations. Until we find a better solution let's just
* undefine all those functions if there is no 64 bit cmpset
* undefine all those functions if there is no 64 bit compare-exchange
*
*********************************************************************/
#if OPAL_HAVE_ATOMIC_CMPSET_32
#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
#define OPAL_ATOMIC_DEFINE_CMPXCG_OP(type, bits, operand, name) \
static inline type opal_atomic_ ## name ## _ ## bits (volatile type *addr, type value) \
{ \
type oldval, newval; \
do { \
oldval = *addr; \
newval = oldval operand value; \
} while (!opal_atomic_compare_exchange_strong_ ## bits (addr, &oldval, newval)); \
\
return newval; \
}
#if !defined(OPAL_HAVE_ATOMIC_SWAP_32)
#define OPAL_HAVE_ATOMIC_SWAP_32 1
@ -55,79 +67,44 @@ static inline int32_t opal_atomic_swap_32(volatile int32_t *addr,
#if !defined(OPAL_HAVE_ATOMIC_ADD_32)
#define OPAL_HAVE_ATOMIC_ADD_32 1
static inline int32_t
opal_atomic_add_32(volatile int32_t *addr, int delta)
{
int32_t oldval;
do {
oldval = *addr;
} while (!opal_atomic_bool_cmpset_32(addr, oldval, oldval + delta));
return (oldval + delta);
}
OPAL_ATOMIC_DEFINE_CMPXCG_OP(int32_t, 32, +, add)
#endif /* OPAL_HAVE_ATOMIC_ADD_32 */
#if !defined(OPAL_HAVE_ATOMIC_AND_32)
#define OPAL_HAVE_ATOMIC_AND_32 1
static inline int32_t
opal_atomic_and_32(volatile int32_t *addr, int32_t value)
{
int32_t oldval;
do {
oldval = *addr;
} while (!opal_atomic_bool_cmpset_32(addr, oldval, oldval & value));
return (oldval & value);
}
OPAL_ATOMIC_DEFINE_CMPXCG_OP(int32_t, 32, &, and)
#endif /* OPAL_HAVE_ATOMIC_AND_32 */
#if !defined(OPAL_HAVE_ATOMIC_OR_32)
#define OPAL_HAVE_ATOMIC_OR_32 1
static inline int32_t
opal_atomic_or_32(volatile int32_t *addr, int32_t value)
{
int32_t oldval;
do {
oldval = *addr;
} while (!opal_atomic_bool_cmpset_32(addr, oldval, oldval | value));
return (oldval | value);
}
OPAL_ATOMIC_DEFINE_CMPXCG_OP(int32_t, 32, |, or)
#endif /* OPAL_HAVE_ATOMIC_OR_32 */
#if !defined(OPAL_HAVE_ATOMIC_XOR_32)
#define OPAL_HAVE_ATOMIC_XOR_32 1
static inline int32_t
opal_atomic_xor_32(volatile int32_t *addr, int32_t value)
{
int32_t oldval;
do {
oldval = *addr;
} while (!opal_atomic_bool_cmpset_32(addr, oldval, oldval ^ value));
return (oldval ^ value);
}
OPAL_ATOMIC_DEFINE_CMPXCG_OP(int32_t, 32, ^, xor)
#endif /* OPAL_HAVE_ATOMIC_XOR_32 */
#if !defined(OPAL_HAVE_ATOMIC_SUB_32)
#define OPAL_HAVE_ATOMIC_SUB_32 1
static inline int32_t
opal_atomic_sub_32(volatile int32_t *addr, int delta)
{
int32_t oldval;
do {
oldval = *addr;
} while (!opal_atomic_bool_cmpset_32(addr, oldval, oldval - delta));
return (oldval - delta);
}
OPAL_ATOMIC_DEFINE_CMPXCG_OP(int32_t, 32, -, sub)
#endif /* OPAL_HAVE_ATOMIC_SUB_32 */
#endif /* OPAL_HAVE_ATOMIC_CMPSET_32 */
#endif /* OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 */
#if OPAL_HAVE_ATOMIC_CMPSET_64
#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
#if !defined(OPAL_HAVE_ATOMIC_SWAP_64)
#define OPAL_HAVE_ATOMIC_SWAP_64 1
@ -144,72 +121,37 @@ static inline int64_t opal_atomic_swap_64(volatile int64_t *addr,
#if !defined(OPAL_HAVE_ATOMIC_ADD_64)
#define OPAL_HAVE_ATOMIC_ADD_64 1
static inline int64_t
opal_atomic_add_64(volatile int64_t *addr, int64_t delta)
{
int64_t oldval;
do {
oldval = *addr;
} while (!opal_atomic_bool_cmpset_64(addr, oldval, oldval + delta));
return (oldval + delta);
}
OPAL_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, +, add)
#endif /* OPAL_HAVE_ATOMIC_ADD_64 */
#if !defined(OPAL_HAVE_ATOMIC_AND_64)
#define OPAL_HAVE_ATOMIC_AND_64 1
static inline int64_t
opal_atomic_and_64(volatile int64_t *addr, int64_t value)
{
int64_t oldval;
do {
oldval = *addr;
} while (!opal_atomic_bool_cmpset_64(addr, oldval, oldval & value));
return (oldval & value);
}
OPAL_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, &, and)
#endif /* OPAL_HAVE_ATOMIC_AND_64 */
#if !defined(OPAL_HAVE_ATOMIC_OR_64)
#define OPAL_HAVE_ATOMIC_OR_64 1
static inline int64_t
opal_atomic_or_64(volatile int64_t *addr, int64_t value)
{
int64_t oldval;
do {
oldval = *addr;
} while (!opal_atomic_bool_cmpset_64(addr, oldval, oldval | value));
return (oldval | value);
}
OPAL_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, |, or)
#endif /* OPAL_HAVE_ATOMIC_OR_64 */
#if !defined(OPAL_HAVE_ATOMIC_XOR_64)
#define OPAL_HAVE_ATOMIC_XOR_64 1
static inline int64_t
opal_atomic_xor_64(volatile int64_t *addr, int64_t value)
{
int64_t oldval;
do {
oldval = *addr;
} while (!opal_atomic_bool_cmpset_64(addr, oldval, oldval ^ value));
return (oldval ^ value);
}
OPAL_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, ^, xor)
#endif /* OPAL_HAVE_ATOMIC_XOR_64 */
#if !defined(OPAL_HAVE_ATOMIC_SUB_64)
#define OPAL_HAVE_ATOMIC_SUB_64 1
static inline int64_t
opal_atomic_sub_64(volatile int64_t *addr, int64_t delta)
{
int64_t oldval;
do {
oldval = *addr;
} while (!opal_atomic_bool_cmpset_64(addr, oldval, oldval - delta));
return (oldval - delta);
}
OPAL_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, -, sub)
#endif /* OPAL_HAVE_ATOMIC_SUB_64 */
#else
@ -222,27 +164,138 @@ opal_atomic_sub_64(volatile int64_t *addr, int64_t delta)
#define OPAL_HAVE_ATOMIC_SUB_64 0
#endif
#endif /* OPAL_HAVE_ATOMIC_CMPSET_64 */
#endif /* OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 */
#if (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
#define OPAL_ATOMIC_DEFINE_CMPXCG_XX(semantics) \
static inline bool \
opal_atomic_compare_exchange_strong ## semantics ## xx (volatile void* addr, void *oldval, \
int64_t newval, const size_t length) \
{ \
switch (length) { \
case 4: \
return opal_atomic_compare_exchange_strong_32 ((volatile int32_t *) addr, \
(int32_t *) oldval, (int32_t) newval); \
case 8: \
return opal_atomic_compare_exchange_strong_64 ((volatile int64_t *) addr, \
(int64_t *) oldval, (int64_t) newval); \
} \
abort(); \
}
#elif OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
#define OPAL_ATOMIC_DEFINE_CMPXCG_XX(semantics) \
static inline bool \
opal_atomic_compare_exchange_strong ## semantics ## xx (volatile void* addr, void *oldval, \
int64_t newval, const size_t length) \
{ \
switch (length) { \
case 4: \
return opal_atomic_compare_exchange_strong_32 ((volatile int32_t *) addr, \
(int32_t *) oldval, (int32_t) newval); \
abort(); \
}
#else
#error "Platform does not have required atomic compare-and-swap functionality"
#endif
OPAL_ATOMIC_DEFINE_CMPXCG_XX(_)
OPAL_ATOMIC_DEFINE_CMPXCG_XX(_acq_)
OPAL_ATOMIC_DEFINE_CMPXCG_XX(_rel_)
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
#define OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(semantics) \
static inline bool \
opal_atomic_compare_exchange_strong ## semantics ## ptr (volatile void* addr, void *oldval, void *newval) \
{ \
return opal_atomic_compare_exchange_strong_32 ((volatile int32_t *) addr, (int32_t *) oldval, (int32_t) newval); \
}
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
#define OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(semantics) \
static inline bool \
opal_atomic_compare_exchange_strong ## semantics ## ptr (volatile void* addr, void *oldval, void *newval) \
{ \
return opal_atomic_compare_exchange_strong_64 ((volatile int64_t *) addr, (int64_t *) oldval, (int64_t) newval); \
}
#else
#error "Can not define opal_atomic_compare_exchange_strong_ptr with existing atomics"
#endif
OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(_)
OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(_acq_)
OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(_rel_)
#endif /* (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64) */
#if (OPAL_HAVE_ATOMIC_CMPSET_32 || OPAL_HAVE_ATOMIC_CMPSET_64)
/* XXX -- DEPRECATED -- XXX -- Define legacy cmpset functions */
#if (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32)
static inline bool opal_atomic_bool_cmpset_32 (volatile int32_t *addr, int32_t oldval,
int32_t newval)
{
return opal_atomic_compare_exchange_strong_32 (addr, &oldval, newval);
}
static inline bool opal_atomic_bool_cmpset_acq_32 (volatile int32_t *addr, int32_t oldval,
int32_t newval)
{
return opal_atomic_compare_exchange_strong_acq_32 (addr, &oldval, newval);
}
static inline bool opal_atomic_bool_cmpset_rel_32 (volatile int32_t *addr, int32_t oldval,
int32_t newval)
{
return opal_atomic_compare_exchange_strong_rel_32 (addr, &oldval, newval);
}
#endif
#if (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
static inline bool opal_atomic_bool_cmpset_64 (volatile int64_t *addr, int64_t oldval,
int64_t newval)
{
return opal_atomic_compare_exchange_strong_64 (addr, &oldval, newval);
}
static inline bool opal_atomic_bool_cmpset_acq_64 (volatile int64_t *addr, int64_t oldval,
int64_t newval)
{
return opal_atomic_compare_exchange_strong_acq_64 (addr, &oldval, newval);
}
static inline bool opal_atomic_bool_cmpset_rel_64 (volatile int64_t *addr, int64_t oldval,
int64_t newval)
{
return opal_atomic_compare_exchange_strong_rel_64 (addr, &oldval, newval);
}
#endif
#if (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128)
static inline bool opal_atomic_bool_cmpset_128 (volatile opal_int128_t *addr, opal_int128_t oldval,
opal_int128_t newval)
{
return opal_atomic_compare_exchange_strong_128 (addr, &oldval, newval);
}
#endif
#if (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
static inline bool
opal_atomic_bool_cmpset_xx(volatile void* addr, int64_t oldval,
int64_t newval, size_t length)
{
switch( length ) {
#if OPAL_HAVE_ATOMIC_CMPSET_32
#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
case 4:
return opal_atomic_bool_cmpset_32( (volatile int32_t*)addr,
(int32_t)oldval, (int32_t)newval );
#endif /* OPAL_HAVE_ATOMIC_CMPSET_32 */
#endif /* OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 */
#if OPAL_HAVE_ATOMIC_CMPSET_64
#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
case 8:
return opal_atomic_bool_cmpset_64( (volatile int64_t*)addr,
(int64_t)oldval, (int64_t)newval );
#endif /* OPAL_HAVE_ATOMIC_CMPSET_64 */
#endif /* OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 */
}
abort();
/* This should never happen, so deliberately abort (hopefully
@ -255,17 +308,17 @@ opal_atomic_bool_cmpset_acq_xx(volatile void* addr, int64_t oldval,
int64_t newval, size_t length)
{
switch( length ) {
#if OPAL_HAVE_ATOMIC_CMPSET_32
#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
case 4:
return opal_atomic_bool_cmpset_acq_32( (volatile int32_t*)addr,
(int32_t)oldval, (int32_t)newval );
#endif /* OPAL_HAVE_ATOMIC_CMPSET_32 */
#endif /* OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 */
#if OPAL_HAVE_ATOMIC_CMPSET_64
#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
case 8:
return opal_atomic_bool_cmpset_acq_64( (volatile int64_t*)addr,
(int64_t)oldval, (int64_t)newval );
#endif /* OPAL_HAVE_ATOMIC_CMPSET_64 */
#endif /* OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 */
}
/* This should never happen, so deliberately abort (hopefully
leaving a corefile for analysis) */
@ -278,17 +331,17 @@ opal_atomic_bool_cmpset_rel_xx(volatile void* addr, int64_t oldval,
int64_t newval, size_t length)
{
switch( length ) {
#if OPAL_HAVE_ATOMIC_CMPSET_32
#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
case 4:
return opal_atomic_bool_cmpset_rel_32( (volatile int32_t*)addr,
(int32_t)oldval, (int32_t)newval );
#endif /* OPAL_HAVE_ATOMIC_CMPSET_32 */
#endif /* OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 */
#if OPAL_HAVE_ATOMIC_CMPSET_64
#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
case 8:
return opal_atomic_bool_cmpset_rel_64( (volatile int64_t*)addr,
(int64_t)oldval, (int64_t)newval );
#endif /* OPAL_HAVE_ATOMIC_CMPSET_64 */
#endif /* OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 */
}
/* This should never happen, so deliberately abort (hopefully
leaving a corefile for analysis) */
@ -301,10 +354,10 @@ opal_atomic_bool_cmpset_ptr(volatile void* addr,
void* oldval,
void* newval)
{
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_CMPSET_32
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
return opal_atomic_bool_cmpset_32((int32_t*) addr, (unsigned long) oldval,
(unsigned long) newval);
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_CMPSET_64
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
return opal_atomic_bool_cmpset_64((int64_t*) addr, (unsigned long) oldval,
(unsigned long) newval);
#else
@ -318,10 +371,10 @@ opal_atomic_bool_cmpset_acq_ptr(volatile void* addr,
void* oldval,
void* newval)
{
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_CMPSET_32
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
return opal_atomic_bool_cmpset_acq_32((int32_t*) addr, (unsigned long) oldval,
(unsigned long) newval);
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_CMPSET_64
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
return opal_atomic_bool_cmpset_acq_64((int64_t*) addr, (unsigned long) oldval,
(unsigned long) newval);
#else
@ -334,10 +387,10 @@ static inline bool opal_atomic_bool_cmpset_rel_ptr(volatile void* addr,
void* oldval,
void* newval)
{
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_CMPSET_32
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
return opal_atomic_bool_cmpset_rel_32((int32_t*) addr, (unsigned long) oldval,
(unsigned long) newval);
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_CMPSET_64
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
return opal_atomic_bool_cmpset_rel_64((int64_t*) addr, (unsigned long) oldval,
(unsigned long) newval);
#else
@ -345,7 +398,7 @@ static inline bool opal_atomic_bool_cmpset_rel_ptr(volatile void* addr,
#endif
}
#endif /* (OPAL_HAVE_ATOMIC_CMPSET_32 || OPAL_HAVE_ATOMIC_CMPSET_64) */
#endif /* (OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64) */
#if (OPAL_HAVE_ATOMIC_SWAP_32 || OPAL_HAVE_ATOMIC_SWAP_64)
@ -392,7 +445,7 @@ opal_atomic_add_xx(volatile void* addr, int32_t value, size_t length)
case 4:
opal_atomic_add_32( (volatile int32_t*)addr, (int32_t)value );
break;
#endif /* OPAL_HAVE_ATOMIC_CMPSET_32 */
#endif /* OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 */
#if OPAL_HAVE_ATOMIC_ADD_64
case 8:

59
opal/include/opal/sys/gcc_builtin/atomic.h
Просмотреть файл

@ -33,7 +33,7 @@
#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
#define OPAL_HAVE_ATOMIC_MATH_32 1
#define OPAL_HAVE_ATOMIC_CMPSET_32 1
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
#define OPAL_HAVE_ATOMIC_ADD_32 1
#define OPAL_HAVE_ATOMIC_AND_32 1
#define OPAL_HAVE_ATOMIC_OR_32 1
@ -41,7 +41,7 @@
#define OPAL_HAVE_ATOMIC_SUB_32 1
#define OPAL_HAVE_ATOMIC_SWAP_32 1
#define OPAL_HAVE_ATOMIC_MATH_64 1
#define OPAL_HAVE_ATOMIC_CMPSET_64 1
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
#define OPAL_HAVE_ATOMIC_ADD_64 1
#define OPAL_HAVE_ATOMIC_AND_64 1
#define OPAL_HAVE_ATOMIC_OR_64 1
@ -81,26 +81,20 @@ static inline void opal_atomic_wmb(void)
#pragma error_messages(off, E_ARG_INCOMPATIBLE_WITH_ARG_L)
#endif
static inline bool opal_atomic_bool_cmpset_acq_32( volatile int32_t *addr,
int32_t oldval, int32_t newval)
static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
{
return __atomic_compare_exchange_n (addr, &oldval, newval, false,
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
}
static inline bool opal_atomic_bool_cmpset_rel_32( volatile int32_t *addr,
int32_t oldval, int32_t newval)
static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
{
return __atomic_compare_exchange_n (addr, &oldval, newval, false,
__ATOMIC_RELEASE, __ATOMIC_RELAXED);
return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
}
static inline bool opal_atomic_bool_cmpset_32( volatile int32_t *addr,
int32_t oldval, int32_t newval)
static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
{
return __atomic_compare_exchange_n (addr, &oldval, newval, false,
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
}
static inline int32_t opal_atomic_swap_32 (volatile int32_t *addr, int32_t newval)
@ -135,26 +129,20 @@ static inline int32_t opal_atomic_sub_32(volatile int32_t *addr, int32_t delta)
return __atomic_sub_fetch (addr, delta, __ATOMIC_RELAXED);
}
static inline bool opal_atomic_bool_cmpset_acq_64( volatile int64_t *addr,
int64_t oldval, int64_t newval)
static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
{
return __atomic_compare_exchange_n (addr, &oldval, newval, false,
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
}
static inline bool opal_atomic_bool_cmpset_rel_64( volatile int64_t *addr,
int64_t oldval, int64_t newval)
static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
{
return __atomic_compare_exchange_n (addr, &oldval, newval, false,
__ATOMIC_RELEASE, __ATOMIC_RELAXED);
return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
}
static inline bool opal_atomic_bool_cmpset_64( volatile int64_t *addr,
int64_t oldval, int64_t newval)
static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
{
return __atomic_compare_exchange_n (addr, &oldval, newval, false,
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
}
static inline int64_t opal_atomic_swap_64 (volatile int64_t *addr, int64_t newval)
@ -191,25 +179,28 @@ static inline int64_t opal_atomic_sub_64(volatile int64_t *addr, int64_t delta)
#if OPAL_HAVE_GCC_BUILTIN_CSWAP_INT128
#define OPAL_HAVE_ATOMIC_CMPSET_128 1
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
static inline bool opal_atomic_bool_cmpset_128 (volatile opal_int128_t *addr,
opal_int128_t oldval, opal_int128_t newval)
static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128_t *addr,
opal_int128_t *oldval, opal_int128_t newval)
{
return __atomic_compare_exchange_n (addr, &oldval, newval, false,
return __atomic_compare_exchange_n (addr, oldval, newval, false,
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
}
#elif defined(OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128) && OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128
#define OPAL_HAVE_ATOMIC_CMPSET_128 1
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
/* __atomic version is not lock-free so use legacy __sync version */
static inline bool opal_atomic_bool_cmpset_128 (volatile opal_int128_t *addr,
opal_int128_t oldval, opal_int128_t newval)
static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128_t *addr,
opal_int128_t *oldval, opal_int128_t newval)
{
return __sync_bool_compare_and_swap (addr, oldval, newval);
opal_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval);
bool ret = prev == *oldval;
*oldval = prev;
return ret;
}
#endif

12
opal/include/opal/sys/ia32/atomic.h
Просмотреть файл

@ -40,7 +40,7 @@
*********************************************************************/
#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
#define OPAL_HAVE_ATOMIC_CMPSET_32 1
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
#define OPAL_HAVE_ATOMIC_MATH_32 1
#define OPAL_HAVE_ATOMIC_ADD_32 1
@ -84,15 +84,13 @@ static inline void opal_atomic_isync(void)
*********************************************************************/
#if OPAL_GCC_INLINE_ASSEMBLY
static inline bool opal_atomic_bool_cmpset_32(volatile int32_t *addr,
int32_t oldval,
int32_t newval)
static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
{
unsigned char ret;
__asm__ __volatile__ (
SMPLOCK "cmpxchgl %3,%2 \n\t"
"sete %0 \n\t"
: "=qm" (ret), "+a" (oldval), "+m" (*addr)
: "=qm" (ret), "+a" (*oldval), "+m" (*addr)
: "q"(newval)
: "memory", "cc");
@ -101,8 +99,8 @@ static inline bool opal_atomic_bool_cmpset_32(volatile int32_t *addr,
#endif /* OPAL_GCC_INLINE_ASSEMBLY */
#define opal_atomic_bool_cmpset_acq_32 opal_atomic_bool_cmpset_32
#define opal_atomic_bool_cmpset_rel_32 opal_atomic_bool_cmpset_32
#define opal_atomic_compare_exchange_strong_acq_32 opal_atomic_compare_exchange_strong_32
#define opal_atomic_compare_exchange_strong_rel_32 opal_atomic_compare_exchange_strong_32
#if OPAL_GCC_INLINE_ASSEMBLY

170
opal/include/opal/sys/powerpc/atomic.h
Просмотреть файл

@ -40,7 +40,7 @@
*********************************************************************/
#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
#define OPAL_HAVE_ATOMIC_CMPSET_32 1
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
#define OPAL_HAVE_ATOMIC_SWAP_32 1
#define OPAL_HAVE_ATOMIC_LLSC_32 1
@ -53,7 +53,7 @@
#if (OPAL_ASSEMBLY_ARCH == OPAL_POWERPC64) || OPAL_ASM_SUPPORT_64BIT
#define OPAL_HAVE_ATOMIC_CMPSET_64 1
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
#define OPAL_HAVE_ATOMIC_SWAP_64 1
#define OPAL_HAVE_ATOMIC_LLSC_64 1
#define OPAL_HAVE_ATOMIC_MATH_64 1
@ -144,24 +144,25 @@ void opal_atomic_isync(void)
#define OPAL_ASM_VALUE64(x) x
#endif
static inline bool opal_atomic_bool_cmpset_32(volatile int32_t *addr,
int32_t oldval, int32_t newval)
static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
{
int32_t ret;
int32_t prev;
bool ret;
__asm__ __volatile__ (
"1: lwarx %0, 0, %2 \n\t"
" cmpw 0, %0, %3 \n\t"
" bne- 2f \n\t"
" stwcx. %4, 0, %2 \n\t"
" bne- 1b \n\t"
"2:"
: "=&r" (ret), "=m" (*addr)
: "r" OPAL_ASM_ADDR(addr), "r" (oldval), "r" (newval), "m" (*addr)
: "cc", "memory");
__asm__ __volatile__ (
"1: lwarx %0, 0, %2 \n\t"
" cmpw 0, %0, %3 \n\t"
" bne- 2f \n\t"
" stwcx. %4, 0, %2 \n\t"
" bne- 1b \n\t"
"2:"
: "=&r" (prev), "=m" (*addr)
: "r" OPAL_ASM_ADDR(addr), "r" (*oldval), "r" (newval), "m" (*addr)
: "cc", "memory");
return (ret == oldval);
ret = (prev == *oldval);
*oldval = prev;
return ret;
}
static inline int32_t opal_atomic_ll_32 (volatile int32_t *addr)
@ -195,23 +196,21 @@ static inline int opal_atomic_sc_32 (volatile int32_t *addr, int32_t newval)
atomic_?mb can be inlined). Instead, we "inline" them by hand in
the assembly, meaning there is one function call overhead instead
of two */
static inline bool opal_atomic_bool_cmpset_acq_32(volatile int32_t *addr,
int32_t oldval, int32_t newval)
static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
{
bool rc;
rc = opal_atomic_bool_cmpset_32(addr, oldval, newval);
rc = opal_atomic_compare_exchange_strong_32 (addr, oldval, newval);
opal_atomic_rmb();
return rc;
}
static inline bool opal_atomic_bool_cmpset_rel_32(volatile int32_t *addr,
int32_t oldval, int32_t newval)
static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
{
opal_atomic_wmb();
return opal_atomic_bool_cmpset_32(addr, oldval, newval);
return opal_atomic_compare_exchange_strong_32 (addr, oldval, newval);
}
static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, int32_t newval)
@ -258,23 +257,25 @@ OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(or, or)
OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(xor, xor)
OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(sub, subf)
static inline bool opal_atomic_bool_cmpset_64(volatile int64_t *addr,
int64_t oldval, int64_t newval)
static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
{
int64_t ret;
int64_t prev;
bool ret;
__asm__ __volatile__ (
"1: ldarx %0, 0, %2 \n\t"
" cmpd 0, %0, %3 \n\t"
" bne- 2f \n\t"
" stdcx. %4, 0, %2 \n\t"
" bne- 1b \n\t"
"2:"
: "=&r" (ret), "=m" (*addr)
: "r" (addr), "r" (OPAL_ASM_VALUE64(oldval)), "r" (OPAL_ASM_VALUE64(newval)), "m" (*addr)
: "cc", "memory");
__asm__ __volatile__ (
"1: ldarx %0, 0, %2 \n\t"
" cmpd 0, %0, %3 \n\t"
" bne- 2f \n\t"
" stdcx. %4, 0, %2 \n\t"
" bne- 1b \n\t"
"2:"
: "=&r" (prev), "=m" (*addr)
: "r" (addr), "r" (OPAL_ASM_VALUE64(*oldval)), "r" (OPAL_ASM_VALUE64(newval)), "m" (*addr)
: "cc", "memory");
return (ret == oldval);
ret = (prev == *oldval);
*oldval = prev;
return ret;
}
static inline int64_t opal_atomic_ll_64(volatile int64_t *addr)
@ -303,29 +304,6 @@ static inline int opal_atomic_sc_64(volatile int64_t *addr, int64_t newval)
return ret;
}
/* these two functions aren't inlined in the non-gcc case because then
there would be two function calls (since neither cmpset_64 nor
atomic_?mb can be inlined). Instead, we "inline" them by hand in
the assembly, meaning there is one function call overhead instead
of two */
static inline bool opal_atomic_bool_cmpset_acq_64(volatile int64_t *addr,
int64_t oldval, int64_t newval)
{
bool rc;
rc = opal_atomic_bool_cmpset_64(addr, oldval, newval);
opal_atomic_rmb();
return rc;
}
static inline bool opal_atomic_bool_cmpset_rel_64(volatile int64_t *addr,
int64_t oldval, int64_t newval)
{
opal_atomic_wmb();
return opal_atomic_bool_cmpset_64(addr, oldval, newval);
}
static inline int64_t opal_atomic_swap_64(volatile int64_t *addr, int64_t newval)
{
@ -352,9 +330,9 @@ static inline int64_t opal_atomic_swap_64(volatile int64_t *addr, int64_t newval
#if OPAL_GCC_INLINE_ASSEMBLY
static inline int opal_atomic_bool_cmpset_64(volatile int64_t *addr,
int64_t oldval, int64_t newval)
static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
{
int64_t prev;
int ret;
/*
@ -369,56 +347,54 @@ static inline int opal_atomic_bool_cmpset_64(volatile int64_t *addr,
* is very similar to the pure 64 bit version.
*/
__asm__ __volatile__ (
"ld r4,%2 \n\t"
"ld r5,%3 \n\t"
"1: ldarx r9, 0, %1 \n\t"
" cmpd 0, r9, r4 \n\t"
"ld r4,%3 \n\t"
"ld r5,%4 \n\t"
"1: ldarx %1, 0, %2 \n\t"
" cmpd 0, %1, r4 \n\t"
" bne- 2f \n\t"
" stdcx. r5, 0, %1 \n\t"
" stdcx. r5, 0, %2 \n\t"
" bne- 1b \n\t"
"2: \n\t"
"xor r5,r4,r9 \n\t"
"xor r5,r4,%1 \n\t"
"subfic r9,r5,0 \n\t"
"adde %0,r9,r5 \n\t"
: "=&r" (ret)
: "=&r" (ret), "+r" (prev)
: "r"OPAL_ASM_ADDR(addr),
"m"(oldval), "m"(newval)
"m"(*oldval), "m"(newval)
: "r4", "r5", "r9", "cc", "memory");
return ret;
}
/* these two functions aren't inlined in the non-gcc case because then
there would be two function calls (since neither cmpset_64 nor
atomic_?mb can be inlined). Instead, we "inline" them by hand in
the assembly, meaning there is one function call overhead instead
of two */
static inline bool opal_atomic_bool_cmpset_acq_64(volatile int64_t *addr,
int64_t oldval, int64_t newval)
{
int rc;
rc = opal_atomic_bool_cmpset_64(addr, oldval, newval);
opal_atomic_rmb();
return rc;
}
static inline bool opal_atomic_bool_cmpset_rel_64(volatile int64_t *addr,
int64_t oldval, int64_t newval)
{
opal_atomic_wmb();
return opal_atomic_bool_cmpset_64(addr, oldval, newval);
*oldval = prev;
return (bool) ret;
}
#endif /* OPAL_GCC_INLINE_ASSEMBLY */
#endif /* OPAL_ASM_SUPPORT_64BIT */
#if OPAL_GCC_INLINE_ASSEMBLY
/* these two functions aren't inlined in the non-gcc case because then
there would be two function calls (since neither cmpset_64 nor
atomic_?mb can be inlined). Instead, we "inline" them by hand in
the assembly, meaning there is one function call overhead instead
of two */
static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
{
bool rc;
rc = opal_atomic_compare_exchange_strong_64 (addr, oldval, newval);
opal_atomic_rmb();
return rc;
}
static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
{
opal_atomic_wmb();
return opal_atomic_compare_exchange_strong_64 (addr, oldval, newval);
}
#define OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_32(type, instr) \
static inline int32_t opal_atomic_ ## type ## _32(volatile int32_t* v, int val) \
{ \

103
opal/include/opal/sys/sparcv9/atomic.h
Просмотреть файл

@ -1,3 +1,4 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
@ -12,6 +13,8 @@
* Copyright (c) 2007 Sun Microsystems, Inc. All rights reserverd.
* Copyright (c) 2016 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* Copyright (c) 2017 Los Alamos National Security, LLC. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -38,9 +41,9 @@
*********************************************************************/
#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
#define OPAL_HAVE_ATOMIC_CMPSET_32 1
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
#define OPAL_HAVE_ATOMIC_CMPSET_64 1
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
/**********************************************************************
@ -82,50 +85,49 @@ static inline void opal_atomic_isync(void)
*********************************************************************/
#if OPAL_GCC_INLINE_ASSEMBLY
static inline bool opal_atomic_bool_cmpset_32( volatile int32_t *addr,
int32_t oldval, int32_t newval)
static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
{
/* casa [reg(rs1)] %asi, reg(rs2), reg(rd)
*
* if (*(reg(rs1)) == reg(rs2) )
* swap reg(rd), *(reg(rs1))
* else
* reg(rd) = *(reg(rs1))
*/
/* casa [reg(rs1)] %asi, reg(rs2), reg(rd)
*
* if (*(reg(rs1)) == reg(rs2) )
* swap reg(rd), *(reg(rs1))
* else
* reg(rd) = *(reg(rs1))
*/
int32_t ret = newval;
int32_t prev = newval;
bool ret;
__asm__ __volatile__("casa [%1] " ASI_P ", %2, %0"
: "+r" (ret)
: "r" (addr), "r" (oldval));
return (ret == oldval);
__asm__ __volatile__("casa [%1] " ASI_P ", %2, %0"
: "+r" (prev)
: "r" (addr), "r" (*oldval));
ret = (prev == *oldval);
*oldval = prev;
return ret;
}
static inline bool opal_atomic_bool_cmpset_acq_32( volatile int32_t *addr,
int32_t oldval, int32_t newval)
static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
{
bool rc;
bool rc;
rc = opal_atomic_bool_cmpset_32(addr, oldval, newval);
opal_atomic_rmb();
rc = opal_atomic_compare_exchange_strong_32 (addr, oldval, newval);
opal_atomic_rmb();
return rc;
return rc;
}
static inline bool opal_atomic_bool_cmpset_rel_32( volatile int32_t *addr,
int32_t oldval, int32_t newval)
static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
{
opal_atomic_wmb();
return opal_atomic_bool_cmpset_32(addr, oldval, newval);
opal_atomic_wmb();
return opal_atomic_compare_exchange_strong_32 (addr, oldval, newval);
}
#if OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64
static inline bool opal_atomic_bool_cmpset_64( volatile int64_t *addr,
int64_t oldval, int64_t newval)
static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
{
/* casa [reg(rs1)] %asi, reg(rs2), reg(rd)
*
@ -134,18 +136,20 @@ static inline bool opal_atomic_bool_cmpset_64( volatile int64_t *addr,
* else
* reg(rd) = *(reg(rs1))
*/
int64_t ret = newval;
int64_t prev = newval;
bool ret;
__asm__ __volatile__("casxa [%1] " ASI_P ", %2, %0"
: "+r" (ret)
: "r" (addr), "r" (oldval));
return (ret == oldval);
__asm__ __volatile__("casxa [%1] " ASI_P ", %2, %0"
: "+r" (prev)
: "r" (addr), "r" (*oldval));
ret = (prev == *oldval);
*oldval = prev;
return ret;
}
#else /* OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64 */
static inline bool opal_atomic_bool_cmpset_64( volatile int64_t *addr,
int64_t oldval, int64_t newval)
static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
{
/* casa [reg(rs1)] %asi, reg(rs2), reg(rd)
*
@ -155,40 +159,41 @@ static inline bool opal_atomic_bool_cmpset_64( volatile int64_t *addr,
* reg(rd) = *(reg(rs1))
*
*/
long long ret = newval;
int64_t prev = newval;
bool ret;
__asm__ __volatile__(
"ldx %0, %%g1 \n\t" /* g1 = ret */
"ldx %2, %%g2 \n\t" /* g2 = oldval */
"casxa [%1] " ASI_P ", %%g2, %%g1 \n\t"
"stx %%g1, %0 \n"
: "+m"(ret)
: "r"(addr), "m"(oldval)
: "+m"(prev)
: "r"(addr), "m"(*oldval)
: "%g1", "%g2"
);
return (ret == oldval);
ret = (prev == *oldval);
*oldval = prev;
return ret;
}
#endif /* OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64 */
static inline bool opal_atomic_bool_cmpset_acq_64( volatile int64_t *addr,
int64_t oldval, int64_t newval)
static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
{
bool rc;
bool rc;
rc = opal_atomic_bool_cmpset_64(addr, oldval, newval);
opal_atomic_rmb();
rc = opal_atomic_compare_exchange_strong_64 (addr, oldval, newval);
opal_atomic_rmb();
return rc;
return rc;
}
static inline bool opal_atomic_bool_cmpset_rel_64( volatile int64_t *addr,
int64_t oldval, int64_t newval)
static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
{
opal_atomic_wmb();
return opal_atomic_bool_cmpset_64(addr, oldval, newval);
opal_atomic_wmb();
return opal_atomic_compare_exchange_strong_64 (addr, oldval, newval);
}
#endif /* OPAL_GCC_INLINE_ASSEMBLY */

59
opal/include/opal/sys/sync_builtin/atomic.h
Просмотреть файл

@ -53,24 +53,18 @@ static inline void opal_atomic_wmb(void)
*
*********************************************************************/
#define OPAL_HAVE_ATOMIC_CMPSET_32 1
static inline bool opal_atomic_bool_cmpset_acq_32( volatile int32_t *addr,
int32_t oldval, int32_t newval)
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
{
return __sync_bool_compare_and_swap(addr, oldval, newval);
int32_t prev = __sync_val_compare_and_swap (add, *oldval, newval);
bool ret = prev == *oldval;
*oldval = prev;
return ret;
}
static inline bool opal_atomic_bool_cmpset_rel_32( volatile int32_t *addr,
int32_t oldval, int32_t newval)
{
return __sync_bool_compare_and_swap(addr, oldval, newval);}
static inline bool opal_atomic_bool_cmpset_32( volatile int32_t *addr,
int32_t oldval, int32_t newval)
{
return __sync_bool_compare_and_swap(addr, oldval, newval);
}
#define opal_atomic_compare_exchange_strong_acq_32 opal_atomic_compare_exchange_strong_32
#define opal_atomic_compare_exchange_strong_rel_32 opal_atomic_compare_exchange_strong_32
#define OPAL_HAVE_ATOMIC_MATH_32 1
@ -106,24 +100,18 @@ static inline int32_t opal_atomic_sub_32(volatile int32_t *addr, int32_t delta)
#if OPAL_ASM_SYNC_HAVE_64BIT
#define OPAL_HAVE_ATOMIC_CMPSET_64 1
static inline bool opal_atomic_bool_cmpset_acq_64( volatile int64_t *addr,
int64_t oldval, int64_t newval)
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
{
return __sync_bool_compare_and_swap(addr, oldval, newval);
int64_t prev = __sync_val_compare_and_swap (add, *oldval, newval);
bool ret = prev == *oldval;
*oldval = prev;
return ret;
}
static inline bool opal_atomic_bool_cmpset_rel_64( volatile int64_t *addr,
int64_t oldval, int64_t newval)
{
return __sync_bool_compare_and_swap(addr, oldval, newval);}
static inline bool opal_atomic_bool_cmpset_64( volatile int64_t *addr,
int64_t oldval, int64_t newval)
{
return __sync_bool_compare_and_swap(addr, oldval, newval);
}
#define opal_atomic_compare_exchange_strong_acq_64 opal_atomic_compare_exchange_strong_64
#define opal_atomic_compare_exchange_strong_rel_64 opal_atomic_compare_exchange_strong_64
#define OPAL_HAVE_ATOMIC_MATH_64 1
#define OPAL_HAVE_ATOMIC_ADD_64 1
@ -159,13 +147,16 @@ static inline int64_t opal_atomic_sub_64(volatile int64_t *addr, int64_t delta)
#endif
#if OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128
static inline bool opal_atomic_bool_cmpset_128 (volatile opal_int128_t *addr,
opal_int128_t oldval, opal_int128_t newval)
static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128_t *addr,
opal_int128_t *oldval, opal_int128_t newval)
{
return __sync_bool_compare_and_swap(addr, oldval, newval);
opal_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval);
bool ret = prev == *oldval;
*oldval = prev;
return ret;
}
#define OPAL_HAVE_ATOMIC_CMPSET_128 1
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
#endif

34
opal/include/opal/sys/x86_64/atomic.h
Просмотреть файл

@ -40,9 +40,9 @@
*********************************************************************/
#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
#define OPAL_HAVE_ATOMIC_CMPSET_32 1
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
#define OPAL_HAVE_ATOMIC_CMPSET_64 1
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
/**********************************************************************
*
@ -82,14 +82,13 @@ static inline void opal_atomic_isync(void)
*********************************************************************/
#if OPAL_GCC_INLINE_ASSEMBLY
static inline bool opal_atomic_bool_cmpset_32( volatile int32_t *addr,
int32_t oldval, int32_t newval)
static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
{
unsigned char ret;
__asm__ __volatile__ (
SMPLOCK "cmpxchgl %3,%2 \n\t"
"sete %0 \n\t"
: "=qm" (ret), "+a" (oldval), "+m" (*addr)
: "=qm" (ret), "+a" (*oldval), "+m" (*addr)
: "q"(newval)
: "memory", "cc");
@ -98,19 +97,18 @@ static inline bool opal_atomic_bool_cmpset_32( volatile int32_t *addr,
#endif /* OPAL_GCC_INLINE_ASSEMBLY */
#define opal_atomic_bool_cmpset_acq_32 opal_atomic_bool_cmpset_32
#define opal_atomic_bool_cmpset_rel_32 opal_atomic_bool_cmpset_32
#define opal_atomic_compare_exchange_strong_acq_32 opal_atomic_compare_exchange_strong_32
#define opal_atomic_compare_exchange_strong_rel_32 opal_atomic_compare_exchange_strong_32
#if OPAL_GCC_INLINE_ASSEMBLY
static inline bool opal_atomic_bool_cmpset_64( volatile int64_t *addr,
int64_t oldval, int64_t newval)
static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
{
unsigned char ret;
__asm__ __volatile__ (
SMPLOCK "cmpxchgq %3,%2 \n\t"
"sete %0 \n\t"
: "=qm" (ret), "+a" (oldval), "+m" (*((volatile long*)addr))
: "=qm" (ret), "+a" (*oldval), "+m" (*((volatile long*)addr))
: "q"(newval)
: "memory", "cc"
);
@ -120,13 +118,12 @@ static inline bool opal_atomic_bool_cmpset_64( volatile int64_t *addr,
#endif /* OPAL_GCC_INLINE_ASSEMBLY */
#define opal_atomic_bool_cmpset_acq_64 opal_atomic_bool_cmpset_64
#define opal_atomic_bool_cmpset_rel_64 opal_atomic_bool_cmpset_64
#define opal_atomic_compare_exchange_strong_acq_64 opal_atomic_compare_exchange_strong_64
#define opal_atomic_compare_exchange_strong_rel_64 opal_atomic_compare_exchange_strong_64
#if OPAL_GCC_INLINE_ASSEMBLY && OPAL_HAVE_CMPXCHG16B && HAVE_OPAL_INT128_T
static inline bool opal_atomic_bool_cmpset_128 (volatile opal_int128_t *addr, opal_int128_t oldval,
opal_int128_t newval)
static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128_t *addr, opal_int128_t *oldval, opal_int128_t newval)
{
unsigned char ret;
@ -135,15 +132,14 @@ static inline bool opal_atomic_bool_cmpset_128 (volatile opal_int128_t *addr, op
* at the address is returned in eax:edx. */
__asm__ __volatile__ (SMPLOCK "cmpxchg16b (%%rsi) \n\t"
"sete %0 \n\t"
: "=qm" (ret)
: "S" (addr), "b" (((int64_t *)&newval)[0]), "c" (((int64_t *)&newval)[1]),
"a" (((int64_t *)&oldval)[0]), "d" (((int64_t *)&oldval)[1])
: "memory", "cc");
: "=qm" (ret), "+a" (((int64_t *)oldval)[0]), "+d" (((int64_t *)oldval)[1])
: "S" (addr), "b" (((int64_t *)&newval)[0]), "c" (((int64_t *)&newval)[1])
: "memory", "cc", "eax", "edx");
return (bool) ret;
}
#define OPAL_HAVE_ATOMIC_CMPSET_128 1
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
#endif /* OPAL_GCC_INLINE_ASSEMBLY */

31
opal/threads/thread_usage.h
Просмотреть файл

@ -13,7 +13,7 @@
* Copyright (c) 2007-2014 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2014-2016 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights
* Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights
* reserved.
* $COPYRIGHT$
*
@ -158,6 +158,23 @@ static inline bool opal_thread_cmpset_bool_ ## suffix (volatile addr_type *addr,
return false; \
}
#define OPAL_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(type, addr_type, suffix) \
static inline bool opal_thread_compare_exchange_strong_ ## suffix (volatile addr_type *addr, type *compare, type value) \
{ \
if (OPAL_UNLIKELY(opal_using_threads())) { \
return opal_atomic_compare_exchange_strong_ ## suffix ((volatile type *) addr, compare, value); \
} \
\
if ((type) *addr == *compare) { \
((type *) addr)[0] = value; \
return true; \
} \
\
*compare = ((type *) addr)[0]; \
\
return false; \
}
#define OPAL_THREAD_DEFINE_ATOMIC_SWAP(type, addr_type, suffix) \
static inline type opal_thread_swap_ ## suffix (volatile addr_type *ptr, type newvalue) \
{ \
@ -180,6 +197,8 @@ OPAL_THREAD_DEFINE_ATOMIC_SUB(int32_t, 32)
OPAL_THREAD_DEFINE_ATOMIC_SUB(size_t, size_t)
OPAL_THREAD_DEFINE_ATOMIC_CMPSET(int32_t, int32_t, 32)
OPAL_THREAD_DEFINE_ATOMIC_CMPSET(void *, intptr_t, ptr)
OPAL_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(int32_t, int32_t, 32)
OPAL_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(void *, intptr_t, ptr)
OPAL_THREAD_DEFINE_ATOMIC_SWAP(int32_t, int32_t, 32)
OPAL_THREAD_DEFINE_ATOMIC_SWAP(void *, intptr_t, ptr)
@ -204,9 +223,15 @@ OPAL_THREAD_DEFINE_ATOMIC_SWAP(void *, intptr_t, ptr)
#define OPAL_THREAD_BOOL_CMPSET_32 opal_thread_cmpset_bool_32
#define OPAL_ATOMIC_BOOL_CMPSET_32 opal_thread_cmpset_bool_32
#define OPAL_THREAD_BOOL_COMPARE_EXCHANGE_STRONG_32 opal_thread_compare_exchange_strong_32
#define OPAL_ATOMIC_BOOL_COMPARE_EXCHANGE_STRONG_32 opal_thread_compare_exchange_strong_32
#define OPAL_THREAD_BOOL_CMPSET_PTR(x, y, z) opal_thread_cmpset_bool_ptr ((volatile intptr_t *) x, (void *) y, (void *) z)
#define OPAL_ATOMIC_BOOL_CMPSET_PTR OPAL_THREAD_BOOL_CMPSET_PTR
#define OPAL_THREAD_COMPARE_EXCHANGE_STRONG_PTR(x, y, z) opal_thread_compare_exchange_strong_ptr ((volatile intptr_t *) x, (void *) y, (void *) z)
#define OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR OPAL_THREAD_COMPARE_EXCHANGE_STRONG_PTR
#define OPAL_THREAD_SWAP_32 opal_thread_swap_32
#define OPAL_ATOMIC_SWAP_32 opal_thread_swap_32
@ -221,6 +246,7 @@ OPAL_THREAD_DEFINE_ATOMIC_AND(int64_t, 64)
OPAL_THREAD_DEFINE_ATOMIC_OR(int64_t, 64)
OPAL_THREAD_DEFINE_ATOMIC_XOR(int64_t, 64)
OPAL_THREAD_DEFINE_ATOMIC_CMPSET(int64_t, int64_t, 64)
OPAL_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(int64_t, int64_t, 64)
OPAL_THREAD_DEFINE_ATOMIC_SWAP(int64_t, int64_t, 64)
#define OPAL_THREAD_ADD64 opal_thread_add_64
@ -238,6 +264,9 @@ OPAL_THREAD_DEFINE_ATOMIC_SWAP(int64_t, int64_t, 64)
#define OPAL_THREAD_BOOL_CMPSET_64 opal_thread_cmpset_bool_64
#define OPAL_ATOMIC_BOOL_CMPSET_64 opal_thread_cmpset_bool_64
#define OPAL_THREAD_COMPARE_EXCHANGE_STRONG_64 opal_thread_compare_exchange_strong_64
#define OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_64 opal_thread_compare_exchange_strong_64
#define OPAL_THREAD_SWAP_64 opal_thread_swap_64
#define OPAL_ATOMIC_SWAP_64 opal_thread_swap_64

127
test/asm/atomic_cmpset.c
Просмотреть файл

@ -1,3 +1,4 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
@ -12,6 +13,8 @@
* Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2015 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* Copyright (c) 2017 Los Alamos National Security, LLC. All rights
* reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -54,6 +57,13 @@ int64_t old64 = 0;
int64_t new64 = 0;
#endif
#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128
volatile opal_int128_t vol128;
opal_int128_t val128;
opal_int128_t old128;
opal_int128_t new128;
#endif
volatile int volint = 0;
int valint = 0;
int oldint = 0;
@ -99,124 +109,165 @@ int main(int argc, char *argv[])
/* -- cmpset 32-bit tests -- */
vol32 = 42, old32 = 42, new32 = 50;
assert(opal_atomic_bool_cmpset_32(&vol32, old32, new32) == 1);
assert(opal_atomic_compare_exchange_strong_32 (&vol32, &old32, new32) == true);
opal_atomic_rmb();
assert(vol32 == new32);
assert(old32 == 42);
vol32 = 42, old32 = 420, new32 = 50;
assert(opal_atomic_bool_cmpset_32(&vol32, old32, new32) == 0);
assert(opal_atomic_compare_exchange_strong_32 (&vol32, &old32, new32) == false);
opal_atomic_rmb();
assert(vol32 == 42);
assert(old32 == 42);
vol32 = 42, old32 = 42, new32 = 50;
assert(opal_atomic_bool_cmpset_acq_32(&vol32, old32, new32) == 1);
assert(opal_atomic_compare_exchange_strong_32 (&vol32, &old32, new32) == true);
assert(vol32 == new32);
assert(old32 == 42);
vol32 = 42, old32 = 420, new32 = 50;
assert(opal_atomic_bool_cmpset_acq_32(&vol32, old32, new32) == 0);
assert(opal_atomic_compare_exchange_strong_acq_32 (&vol32, &old32, new32) == false);
assert(vol32 == 42);
assert(old32 == 42);
vol32 = 42, old32 = 42, new32 = 50;
assert(opal_atomic_bool_cmpset_rel_32(&vol32, old32, new32) == 1);
assert(opal_atomic_compare_exchange_strong_rel_32 (&vol32, &old32, new32) == true);
opal_atomic_rmb();
assert(vol32 == new32);
assert(old32 == 42);
vol32 = 42, old32 = 420, new32 = 50;
assert(opal_atomic_bool_cmpset_rel_32(&vol32, old32, new32) == 0);
assert(opal_atomic_compare_exchange_strong_rel_32 (&vol32, &old32, new32) == false);
opal_atomic_rmb();
assert(vol32 == 42);
assert(old32 == 42);
/* -- cmpset 64-bit tests -- */
#if OPAL_HAVE_ATOMIC_MATH_64
vol64 = 42, old64 = 42, new64 = 50;
assert(1 == opal_atomic_bool_cmpset_64(&vol64, old64, new64));
assert(opal_atomic_compare_exchange_strong_64 (&vol64, &old64, new64) == true);
opal_atomic_rmb();
assert(new64 == vol64);
assert(old64 == 42);
vol64 = 42, old64 = 420, new64 = 50;
assert(opal_atomic_bool_cmpset_64(&vol64, old64, new64) == 0);
assert(opal_atomic_compare_exchange_strong_64 (&vol64, &old64, new64) == false);
opal_atomic_rmb();
assert(vol64 == 42);
assert(old64 == 42);
vol64 = 42, old64 = 42, new64 = 50;
assert(opal_atomic_bool_cmpset_acq_64(&vol64, old64, new64) == 1);
assert(opal_atomic_compare_exchange_strong_acq_64 (&vol64, &old64, new64) == true);
assert(vol64 == new64);
assert(old64 == 42);
vol64 = 42, old64 = 420, new64 = 50;
assert(opal_atomic_bool_cmpset_acq_64(&vol64, old64, new64) == 0);
assert(opal_atomic_compare_exchange_strong_acq_64 (&vol64, &old64, new64) == false);
assert(vol64 == 42);
assert(old64 == 42);
vol64 = 42, old64 = 42, new64 = 50;
assert(opal_atomic_bool_cmpset_rel_64(&vol64, old64, new64) == 1);
assert(opal_atomic_compare_exchange_strong_rel_64 (&vol64, &old64, new64) == true);
opal_atomic_rmb();
assert(vol64 == new64);
assert(old64 == 42);
vol64 = 42, old64 = 420, new64 = 50;
assert(opal_atomic_bool_cmpset_rel_64(&vol64, old64, new64) == 0);
assert(opal_atomic_compare_exchange_strong_rel_64 (&vol64, &old64, new64) == false);
opal_atomic_rmb();
assert(vol64 == 42);
assert(old64 == 42);
#endif
/* -- cmpset 128-bit tests -- */
#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128
vol128 = 42, old128 = 42, new128 = 50;
assert(opal_atomic_compare_exchange_strong_128 (&vol128, &old128, new128) == true);
opal_atomic_rmb();
assert(new128 == vol128);
assert(old128 == 42);
vol128 = 42, old128 = 420, new128 = 50;
assert(opal_atomic_compare_exchange_strong_128 (&vol128, &old128, new128) == false);
opal_atomic_rmb();
assert(vol128 == 42);
assert(old128 == 42);
#endif
/* -- cmpset int tests -- */
volint = 42, oldint = 42, newint = 50;
assert(opal_atomic_bool_cmpset(&volint, oldint, newint) == 1);
opal_atomic_rmb();
assert(volint ==newint);
volint = 42, oldint = 420, newint = 50;
assert(opal_atomic_bool_cmpset(&volint, oldint, newint) == 0);
opal_atomic_rmb();
assert(volint == 42);
volint = 42, oldint = 42, newint = 50;
assert(opal_atomic_bool_cmpset_acq(&volint, oldint, newint) == 1);
assert(volint == newint);
volint = 42, oldint = 420, newint = 50;
assert(opal_atomic_bool_cmpset_acq(&volint, oldint, newint) == 0);
assert(volint == 42);
volint = 42, oldint = 42, newint = 50;
assert(opal_atomic_bool_cmpset_rel(&volint, oldint, newint) == 1);
assert(opal_atomic_compare_exchange_strong (&volint, &oldint, newint) == true);
opal_atomic_rmb();
assert(volint == newint);
assert(oldint == 42);
volint = 42, oldint = 420, newint = 50;
assert(opal_atomic_bool_cmpset_rel(&volint, oldint, newint) == 0);
assert(opal_atomic_compare_exchange_strong (&volint, &oldint, newint) == false);
opal_atomic_rmb();
assert(volint == 42);
assert(oldint == 42);
volint = 42, oldint = 42, newint = 50;
assert(opal_atomic_compare_exchange_strong_acq (&volint, &oldint, newint) == true);
assert(volint == newint);
assert(oldint == 42);
volint = 42, oldint = 420, newint = 50;
assert(opal_atomic_compare_exchange_strong_acq (&volint, &oldint, newint) == false);
assert(volint == 42);
assert(oldint == 42);
volint = 42, oldint = 42, newint = 50;
assert(opal_atomic_compare_exchange_strong_rel (&volint, &oldint, newint) == true);
opal_atomic_rmb();
assert(volint == newint);
assert(oldint == 42);
volint = 42, oldint = 420, newint = 50;
assert(opal_atomic_compare_exchange_strong_rel (&volint, &oldint, newint) == false);
opal_atomic_rmb();
assert(volint == 42);
assert(oldint == 42);
/* -- cmpset ptr tests -- */
volptr = (void *) 42, oldptr = (void *) 42, newptr = (void *) 50;
assert(opal_atomic_bool_cmpset_ptr(&volptr, oldptr, newptr) == 1);
assert(opal_atomic_compare_exchange_strong_ptr (&volptr, &oldptr, newptr) == true);
opal_atomic_rmb();
assert(volptr == newptr);
assert(oldptr == (void *) 42);
volptr = (void *) 42, oldptr = (void *) 420, newptr = (void *) 50;
assert(opal_atomic_bool_cmpset_ptr(&volptr, oldptr, newptr) == 0);
assert(opal_atomic_compare_exchange_strong_ptr (&volptr, &oldptr, newptr) == false);
opal_atomic_rmb();
assert(volptr == (void *) 42);
assert(oldptr == (void *) 42);
volptr = (void *) 42, oldptr = (void *) 42, newptr = (void *) 50;
assert(opal_atomic_bool_cmpset_acq_ptr(&volptr, oldptr, newptr) == 1);
assert(opal_atomic_compare_exchange_strong_acq_ptr (&volptr, &oldptr, newptr) == true);
assert(volptr == newptr);
assert(oldptr == (void *) 42);
volptr = (void *) 42, oldptr = (void *) 420, newptr = (void *) 50;
assert(opal_atomic_bool_cmpset_acq_ptr(&volptr, oldptr, newptr) == 0);
assert(opal_atomic_compare_exchange_strong_acq_ptr (&volptr, &oldptr, newptr) == false);
assert(volptr == (void *) 42);
assert(oldptr == (void *) 42);
volptr = (void *) 42, oldptr = (void *) 42, newptr = (void *) 50;
assert(opal_atomic_bool_cmpset_rel_ptr(&volptr, oldptr, newptr) == 1);
assert(opal_atomic_compare_exchange_strong_rel_ptr (&volptr, &oldptr, newptr) == true);
opal_atomic_rmb();
assert(volptr == newptr);
assert(oldptr == (void *) 42);
volptr = (void *) 42, oldptr = (void *) 420, newptr = (void *) 50;
assert(opal_atomic_bool_cmpset_rel_ptr(&volptr, oldptr, newptr) == 0);
assert(opal_atomic_compare_exchange_strong_rel_ptr (&volptr, &oldptr, newptr) == false);
opal_atomic_rmb();
assert(volptr == (void *) 42);
assert(oldptr == (void *) 42);
/* -- add_32 tests -- */