diff --git a/config/opal_config_asm.m4 b/config/opal_config_asm.m4 index 65675d16b5..9602a53c36 100644 --- a/config/opal_config_asm.m4 +++ b/config/opal_config_asm.m4 @@ -13,7 +13,7 @@ dnl Copyright (c) 2008-2018 Cisco Systems, Inc. All rights reserved. dnl Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. dnl Copyright (c) 2015-2017 Research Organization for Information Science dnl and Technology (RIST). All rights reserved. -dnl Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights +dnl Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights dnl reserved. dnl Copyright (c) 2017 Amazon.com, Inc. or its affiliates. All Rights dnl reserved. @@ -122,6 +122,58 @@ int main(int argc, char** argv) } ]]) +dnl This is a C test to see if 128-bit __atomic_compare_exchange_n() +dnl actually works (e.g., it compiles and links successfully on +dnl ARM64+clang, but returns incorrect answers as of August 2018). +AC_DEFUN([OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_TEST_SOURCE],[[ +#include +#include +#include +#include + +typedef union { + uint64_t fake@<:@2@:>@; + _Atomic __int128 real; +} ompi128; + +static void test1(void) +{ + // As of Aug 2018, we could not figure out a way to assign 128-bit + // constants -- the compilers would not accept it. So use a fake + // union to assign 2 uin64_t's to make a single __int128. + ompi128 ptr = { .fake = { 0xFFEEDDCCBBAA0099, 0x8877665544332211 }}; + ompi128 expected = { .fake = { 0x11EEDDCCBBAA0099, 0x88776655443322FF }}; + ompi128 desired = { .fake = { 0x1122DDCCBBAA0099, 0x887766554433EEFF }}; + bool r = atomic_compare_exchange_strong (&ptr.real, &expected.real, + desired.real, true, + atomic_relaxed, atomic_relaxed); + if ( !(r == false && ptr.real == expected.real)) { + exit(1); + } +} + +static void test2(void) +{ + ompi128 ptr = { .fake = { 0xFFEEDDCCBBAA0099, 0x8877665544332211 }}; + ompi128 expected = ptr; + ompi128 desired = { .fake = { 0x1122DDCCBBAA0099, 0x887766554433EEFF }}; + bool r = atomic_compare_exchange_strong (&ptr.real, &expected.real, + desired.real, true, + atomic_relaxed, atomic_relaxed); + if (!(r == true && ptr.real == desired.real)) { + exit(2); + } +} + +vvvvvvvvvvvvvvvvvvvv +int main(int argc, char** argv) +{ + test1(); + test2(); + return 0; +} +]]) + dnl ------------------------------------------------------------------ dnl @@ -329,6 +381,71 @@ __atomic_add_fetch(&tmp64, 1, __ATOMIC_RELAXED);], OPAL_CHECK_GCC_BUILTIN_CSWAP_INT128 ]) +AC_DEFUN([OPAL_CHECK_C11_CSWAP_INT128], [ + OPAL_VAR_SCOPE_PUSH([atomic_compare_exchange_result atomic_compare_exchange_CFLAGS_save atomic_compare_exchange_LIBS_save]) + + atomic_compare_exchange_CFLAGS_save=$CFLAGS + atomic_compare_exchange_LIBS_save=$LIBS + + # Do we have C11 atomics on 128-bit integers? + # Use a special macro because we need to check with a few different + # CFLAGS/LIBS. + OPAL_ASM_CHECK_ATOMIC_FUNC([atomic_compare_exchange_strong_16], + [AC_LANG_SOURCE(OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_TEST_SOURCE)], + [atomic_compare_exchange_result=1], + [atomic_compare_exchange_result=0]) + + # If we have it and it works, check to make sure it is always lock + # free. + AS_IF([test $atomic_compare_exchange_result -eq 1], + [AC_MSG_CHECKING([if C11 __int128 atomic compare-and-swap is always lock-free]) + AC_RUN_IFELSE([AC_LANG_PROGRAM([#include ], [_Atomic __int128_t x; if (!atomic_is_lock_free(&x)) { return 1; }])], + [AC_MSG_RESULT([yes])], + [atomic_compare_exchange_result=0 + # If this test fails, need to reset CFLAGS/LIBS (the + # above tests atomically set CFLAGS/LIBS or not; this + # test is running after the fact, so we have to undo + # the side-effects of setting CFLAGS/LIBS if the above + # tests passed). + CFLAGS=$atomic_compare_exchange_CFLAGS_save + LIBS=$atomic_compare_exchange_LIBS_save + AC_MSG_RESULT([no])], + [AC_MSG_RESULT([cannot test -- assume yes (cross compiling)])]) + ]) + + AC_DEFINE_UNQUOTED([OPAL_HAVE_C11_CSWAP_INT128], + [$atomic_compare_exchange_result], + [Whether C11 atomic compare swap is both supported and lock-free on 128-bit values]) + + dnl If we could not find decent support for 128-bits atomic let's + dnl try the GCC _sync + AS_IF([test $atomic_compare_exchange_result -eq 0], + [OPAL_CHECK_SYNC_BUILTIN_CSWAP_INT128]) + + OPAL_VAR_SCOPE_POP +]) + +AC_DEFUN([OPAL_CHECK_GCC_ATOMIC_BUILTINS], [ + AC_MSG_CHECKING([for __atomic builtin atomics]) + + AC_TRY_LINK([ +#include +uint32_t tmp, old = 0; +uint64_t tmp64, old64 = 0;], [ +__atomic_thread_fence(__ATOMIC_SEQ_CST); +__atomic_compare_exchange_n(&tmp, &old, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED); +__atomic_add_fetch(&tmp, 1, __ATOMIC_RELAXED); +__atomic_compare_exchange_n(&tmp64, &old64, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED); +__atomic_add_fetch(&tmp64, 1, __ATOMIC_RELAXED);], + [AC_MSG_RESULT([yes]) + $1], + [AC_MSG_RESULT([no]) + $2]) + + # Check for 128-bit support + OPAL_CHECK_GCC_BUILTIN_CSWAP_INT128 +]) + dnl ################################################################# dnl @@ -1020,17 +1137,27 @@ AC_DEFUN([OPAL_CONFIG_ASM],[ AC_REQUIRE([OPAL_SETUP_CC]) AC_REQUIRE([AM_PROG_AS]) + AC_ARG_ENABLE([c11-atomics],[AC_HELP_STRING([--enable-c11-atomics], + [Enable use of C11 atomics if available (default: enabled)])]) + AC_ARG_ENABLE([builtin-atomics], [AC_HELP_STRING([--enable-builtin-atomics], - [Enable use of __sync builtin atomics (default: enabled)])]) + [Enable use of __sync builtin atomics (default: disabled)])]) - opal_cv_asm_builtin="BUILTIN_NO" - AS_IF([test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" != "no"], - [OPAL_CHECK_GCC_ATOMIC_BUILTINS([opal_cv_asm_builtin="BUILTIN_GCC"], [])]) - AS_IF([test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" != "no"], - [OPAL_CHECK_SYNC_BUILTINS([opal_cv_asm_builtin="BUILTIN_SYNC"], [])]) - AS_IF([test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes"], - [AC_MSG_ERROR([__sync builtin atomics requested but not found.])]) + OPAL_CHECK_C11_CSWAP_INT128 + + if test "x$enable_c11_atomics" != "xno" && test "$opal_cv_c11_supported" = "yes" ; then + opal_cv_asm_builtin="BUILTIN_C11" + OPAL_CHECK_C11_CSWAP_INT128 + else + opal_cv_asm_builtin="BUILTIN_NO" + AS_IF([test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes"], + [OPAL_CHECK_GCC_ATOMIC_BUILTINS([opal_cv_asm_builtin="BUILTIN_GCC"], [])]) + AS_IF([test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes"], + [OPAL_CHECK_SYNC_BUILTINS([opal_cv_asm_builtin="BUILTIN_SYNC"], [])]) + AS_IF([test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes"], + [AC_MSG_ERROR([__sync builtin atomics requested but not found.])]) + fi OPAL_CHECK_ASM_PROC OPAL_CHECK_ASM_TEXT diff --git a/opal/class/opal_fifo.h b/opal/class/opal_fifo.h index 2a0607ac34..cd3f8320a3 100644 --- a/opal/class/opal_fifo.h +++ b/opal/class/opal_fifo.h @@ -155,11 +155,9 @@ static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo) * update the head */ /* wait for next pointer to be updated by push */ - while (ghost == item->opal_list_next) { + do { opal_atomic_rmb (); - } - - opal_atomic_rmb (); + } while (ghost == item->opal_list_next); /* update the head with the real next value. note that no other thread * will be attempting to update the head until after it has been updated diff --git a/opal/class/opal_object.c b/opal/class/opal_object.c index cd09d647f6..64fef5712f 100644 --- a/opal/class/opal_object.c +++ b/opal/class/opal_object.c @@ -55,7 +55,7 @@ int opal_class_init_epoch = 1; /* * Local variables */ -static opal_atomic_lock_t class_lock = { { OPAL_ATOMIC_LOCK_UNLOCKED } }; +static opal_atomic_lock_t class_lock = OPAL_ATOMIC_LOCK_INIT; static void** classes = NULL; static int num_classes = 0; static int max_classes = 0; diff --git a/opal/include/opal/sys/Makefile.am b/opal/include/opal/sys/Makefile.am index 9387ed6da1..cb5c471ca5 100644 --- a/opal/include/opal/sys/Makefile.am +++ b/opal/include/opal/sys/Makefile.am @@ -11,7 +11,7 @@ # All rights reserved. # Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2011 Sandia National Laboratories. All rights reserved. -# Copyright (c) 2016 Los Alamos National Security, LLC. All rights +# Copyright (c) 2016-2018 Los Alamos National Security, LLC. All rights # reserved. # Copyright (c) 2017 Research Organization for Information Science # and Technology (RIST). All rights reserved. @@ -27,6 +27,7 @@ headers += \ opal/sys/architecture.h \ opal/sys/atomic.h \ + opal/sys/atomic_stdc.h \ opal/sys/atomic_impl.h \ opal/sys/timer.h \ opal/sys/cma.h diff --git a/opal/include/opal/sys/architecture.h b/opal/include/opal/sys/architecture.h index ee9aa96901..2e61f0d795 100644 --- a/opal/include/opal/sys/architecture.h +++ b/opal/include/opal/sys/architecture.h @@ -47,6 +47,7 @@ #define OPAL_BUILTIN_SYNC 0200 #define OPAL_BUILTIN_GCC 0202 #define OPAL_BUILTIN_NO 0203 +#define OPAL_BUILTIN_C11 0204 /* Formats */ #define OPAL_DEFAULT 1000 /* standard for given architecture */ diff --git a/opal/include/opal/sys/atomic.h b/opal/include/opal/sys/atomic.h index 49cb18ac17..cf0115622e 100644 --- a/opal/include/opal/sys/atomic.h +++ b/opal/include/opal/sys/atomic.h @@ -58,6 +58,12 @@ #include "opal/sys/architecture.h" #include "opal_stdatomic.h" +#if OPAL_ASSEMBLY_BUILTIN == OPAL_BUILTIN_C11 + +#include "atomic_stdc.h" + +#else /* !OPAL_C_HAVE__ATOMIC */ + /* do some quick #define cleanup in cases where we are doing testing... */ #ifdef OPAL_DISABLE_INLINE_ASM @@ -147,6 +153,8 @@ enum { OPAL_ATOMIC_LOCK_LOCKED = 1 }; +#define OPAL_ATOMIC_LOCK_INIT {.u = {.lock = OPAL_ATOMIC_LOCK_UNLOCKED}} + /********************************************************************** * * Load the appropriate architecture files and set some reasonable @@ -643,6 +651,8 @@ static inline intptr_t opal_atomic_fetch_sub_ptr( opal_atomic_intptr_t* addr, vo */ #include "opal/sys/atomic_impl.h" +#endif /* !OPAL_C_HAVE__ATOMIC */ + END_C_DECLS #endif /* OPAL_SYS_ATOMIC_H */ diff --git a/opal/include/opal/sys/atomic_stdc.h b/opal/include/opal/sys/atomic_stdc.h new file mode 100644 index 0000000000..d545ed8f16 --- /dev/null +++ b/opal/include/opal/sys/atomic_stdc.h @@ -0,0 +1,261 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2018 Los Alamos National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +/* This file provides shims between the opal atomics interface and the C11 atomics interface. It + * is intended as the first step in moving to using C11 atomics across the entire codebase. Once + * all officially supported compilers offer C11 atomic (GCC 4.9.0+, icc 2018+, pgi, xlc, etc) then + * this shim will go away and the codebase will be updated to use C11's atomic support + * directly. + * This shim contains some functions already present in atomic_impl.h because we do not include + * atomic_impl.h when using C11 atomics. It would require alot of #ifdefs to avoid duplicate + * definitions to be worthwhile. */ + +#if !defined(OPAL_ATOMIC_STDC_H) +#define OPAL_ATOMIC_STDC_H + +#include +#include +#include "opal/include/opal_stdint.h" + +#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1 + +#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1 +#define OPAL_HAVE_ATOMIC_SWAP_32 1 + +#define OPAL_HAVE_ATOMIC_MATH_32 1 +#define OPAL_HAVE_ATOMIC_ADD_32 1 +#define OPAL_HAVE_ATOMIC_AND_32 1 +#define OPAL_HAVE_ATOMIC_OR_32 1 +#define OPAL_HAVE_ATOMIC_XOR_32 1 +#define OPAL_HAVE_ATOMIC_SUB_32 1 + +#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1 +#define OPAL_HAVE_ATOMIC_SWAP_64 1 + +#define OPAL_HAVE_ATOMIC_MATH_64 1 +#define OPAL_HAVE_ATOMIC_ADD_64 1 +#define OPAL_HAVE_ATOMIC_AND_64 1 +#define OPAL_HAVE_ATOMIC_OR_64 1 +#define OPAL_HAVE_ATOMIC_XOR_64 1 +#define OPAL_HAVE_ATOMIC_SUB_64 1 + +#define OPAL_HAVE_ATOMIC_LLSC_32 0 +#define OPAL_HAVE_ATOMIC_LLSC_64 0 +#define OPAL_HAVE_ATOMIC_LLSC_PTR 0 + +#define OPAL_HAVE_ATOMIC_MIN_32 1 +#define OPAL_HAVE_ATOMIC_MAX_32 1 + +#define OPAL_HAVE_ATOMIC_MIN_64 1 +#define OPAL_HAVE_ATOMIC_MAX_64 1 + +#define OPAL_HAVE_ATOMIC_SPINLOCKS 1 + +static inline void opal_atomic_mb (void) +{ + atomic_thread_fence (memory_order_seq_cst); +} + +static inline void opal_atomic_wmb (void) +{ + atomic_thread_fence (memory_order_release); +} + +static inline void opal_atomic_rmb (void) +{ + atomic_thread_fence (memory_order_acquire); +} + +#define opal_atomic_compare_exchange_strong_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_relaxed, memory_order_relaxed) +#define opal_atomic_compare_exchange_strong_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_relaxed, memory_order_relaxed) +#define opal_atomic_compare_exchange_strong_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_relaxed, memory_order_relaxed) +#define opal_atomic_compare_exchange_strong_acq_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed) +#define opal_atomic_compare_exchange_strong_acq_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed) +#define opal_atomic_compare_exchange_strong_acq_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed) + +#define opal_atomic_compare_exchange_strong_rel_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed) +#define opal_atomic_compare_exchange_strong_rel_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed) +#define opal_atomic_compare_exchange_strong_rel_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed) + +#define opal_atomic_compare_exchange_strong(addr, oldval, newval) atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_relaxed, memory_order_relaxed) +#define opal_atomic_compare_exchange_strong_acq(addr, oldval, newval) atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_acquire, memory_order_relaxed) +#define opal_atomic_compare_exchange_strong_rel(addr, oldval, newval) atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_release, memory_order_relaxed) + +#define opal_atomic_swap_32(addr, value) atomic_exchange_explicit (addr, value, memory_order_relaxed) +#define opal_atomic_swap_64(addr, value) atomic_exchange_explicit (addr, value, memory_order_relaxed) +#define opal_atomic_swap_ptr(addr, value) atomic_exchange_explicit (addr, value, memory_order_relaxed) + +#define OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(op, bits, type, operator) \ + static inline type opal_atomic_fetch_ ## op ##_## bits (opal_atomic_ ## type *addr, type value) \ + { \ + return atomic_fetch_ ## op ## _explicit (addr, value, memory_order_relaxed); \ + } \ + \ + static inline type opal_atomic_## op ## _fetch_ ## bits (opal_atomic_ ## type *addr, type value) \ + { \ + return atomic_fetch_ ## op ## _explicit (addr, value, memory_order_relaxed) operator value; \ + } + +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, 32, int32_t, +) +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, 64, int64_t, +) +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, size_t, size_t, +) + +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, 32, int32_t, -) +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, 64, int64_t, -) +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, size_t, size_t, -) + +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(or, 32, int32_t, |) +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(or, 64, int64_t, |) + +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(xor, 32, int32_t, ^) +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(xor, 64, int64_t, ^) + +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(and, 32, int32_t, &) +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(and, 64, int64_t, &) + +#define opal_atomic_add(addr, value) (void) atomic_fetch_add_explicit (addr, value, memory_order_relaxed) + +static inline int32_t opal_atomic_fetch_min_32 (opal_atomic_int32_t *addr, int32_t value) +{ + int32_t old = *addr; + do { + if (old <= value) { + break; + } + } while (!opal_atomic_compare_exchange_strong_32 (addr, &old, value)); + + return old; +} + +static inline int32_t opal_atomic_fetch_max_32 (opal_atomic_int32_t *addr, int32_t value) +{ + int32_t old = *addr; + do { + if (old >= value) { + break; + } + } while (!opal_atomic_compare_exchange_strong_32 (addr, &old, value)); + + return old; +} + +static inline int64_t opal_atomic_fetch_min_64 (opal_atomic_int64_t *addr, int64_t value) +{ + int64_t old = *addr; + do { + if (old <= value) { + break; + } + } while (!opal_atomic_compare_exchange_strong_64 (addr, &old, value)); + + return old; +} + +static inline int64_t opal_atomic_fetch_max_64 (opal_atomic_int64_t *addr, int64_t value) +{ + int64_t old = *addr; + do { + if (old >= value) { + break; + } + } while (!opal_atomic_compare_exchange_strong_64 (addr, &old, value)); + + return old; +} + +static inline int32_t opal_atomic_min_fetch_32 (opal_atomic_int32_t *addr, int32_t value) +{ + int32_t old = opal_atomic_fetch_min_32 (addr, value); + return old <= value ? old : value; +} + +static inline int32_t opal_atomic_max_fetch_32 (opal_atomic_int32_t *addr, int32_t value) +{ + int32_t old = opal_atomic_fetch_max_32 (addr, value); + return old >= value ? old : value; +} + +static inline int64_t opal_atomic_min_fetch_64 (opal_atomic_int64_t *addr, int64_t value) +{ + int64_t old = opal_atomic_fetch_min_64 (addr, value); + return old <= value ? old : value; +} + +static inline int64_t opal_atomic_max_fetch_64 (opal_atomic_int64_t *addr, int64_t value) +{ + int64_t old = opal_atomic_fetch_max_64 (addr, value); + return old >= value ? old : value; +} + +#define OPAL_ATOMIC_LOCK_UNLOCKED false +#define OPAL_ATOMIC_LOCK_LOCKED true + +#define OPAL_ATOMIC_LOCK_INIT ATOMIC_FLAG_INIT + +typedef atomic_flag opal_atomic_lock_t; + +/* + * Lock initialization function. It set the lock to UNLOCKED. + */ +static inline void opal_atomic_lock_init (opal_atomic_lock_t *lock, bool value) +{ + atomic_flag_clear (lock); +} + + +static inline int opal_atomic_trylock (opal_atomic_lock_t *lock) +{ + return (int) atomic_flag_test_and_set (lock); +} + + +static inline void opal_atomic_lock(opal_atomic_lock_t *lock) +{ + while (opal_atomic_trylock (lock)) { + } +} + + +static inline void opal_atomic_unlock (opal_atomic_lock_t *lock) +{ + atomic_flag_clear (lock); +} + + +#if OPAL_HAVE_C11_CSWAP_INT128 + +/* the C11 atomic compare-exchange is lock free so use it */ +#define opal_atomic_compare_exchange_strong_128 atomic_compare_exchange_strong + +#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1 + +#elif OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128 + +/* fall back on the __sync builtin if available since it will emit the expected instruction on x86_64 (cmpxchng16b) */ +__opal_attribute_always_inline__ +static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_int128_t *addr, + opal_int128_t *oldval, opal_int128_t newval) +{ + opal_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval); + bool ret = prev == *oldval; + *oldval = prev; + return ret; +} + +#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1 + +#else + +#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 0 + +#endif + +#endif /* !defined(OPAL_ATOMIC_STDC_H) */ diff --git a/opal/include/opal/sys/sync_builtin/atomic.h b/opal/include/opal/sys/sync_builtin/atomic.h index 4d5e3f5227..6a9c5386ed 100644 --- a/opal/include/opal/sys/sync_builtin/atomic.h +++ b/opal/include/opal/sys/sync_builtin/atomic.h @@ -149,7 +149,7 @@ static inline int64_t opal_atomic_fetch_sub_64(opal_atomic_int64_t *addr, int64_ #endif #if OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128 -static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_opal_int128_t *addr, +static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_int128_t *addr, opal_int128_t *oldval, opal_int128_t newval) { opal_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval); diff --git a/opal/include/opal_stdatomic.h b/opal/include/opal_stdatomic.h index 6617e63de8..854413c87f 100644 --- a/opal/include/opal_stdatomic.h +++ b/opal/include/opal_stdatomic.h @@ -14,6 +14,9 @@ #include "opal_stdint.h" +#if OPAL_ASSEMBLY_BUILTIN != OPAL_BUILTIN_C11 + +typedef volatile int opal_atomic_int_t; typedef volatile long opal_atomic_long_t; typedef volatile int32_t opal_atomic_int32_t; @@ -22,14 +25,42 @@ typedef volatile int64_t opal_atomic_int64_t; typedef volatile uint64_t opal_atomic_uint64_t; typedef volatile size_t opal_atomic_size_t; +typedef volatile ssize_t opal_atomic_ssize_t; typedef volatile intptr_t opal_atomic_intptr_t; typedef volatile uintptr_t opal_atomic_uintptr_t; +#else /* OPAL_HAVE_C__ATOMIC */ + +#include + +typedef atomic_int opal_atomic_int_t; +typedef atomic_long opal_atomic_long_t; + +typedef _Atomic int32_t opal_atomic_int32_t; +typedef _Atomic uint32_t opal_atomic_uint32_t; +typedef _Atomic int64_t opal_atomic_int64_t; +typedef _Atomic uint64_t opal_atomic_uint64_t; + +typedef _Atomic size_t opal_atomic_size_t; +typedef _Atomic ssize_t opal_atomic_ssize_t; +typedef _Atomic intptr_t opal_atomic_intptr_t; +typedef _Atomic uintptr_t opal_atomic_uintptr_t; + +#endif /* OPAL_HAVE_C__ATOMIC */ + #if HAVE_OPAL_INT128_T +/* do not use C11 atomics for __int128 if they are not lock free */ +#if OPAL_HAVE_C11_CSWAP_INT128 + +typedef _Atomic opal_int128_t opal_atomic_int128_t; + +#else + typedef volatile opal_int128_t opal_atomic_int128_t; #endif +#endif #endif /* !defined(OPAL_STDATOMIC_H) */ diff --git a/opal/include/opal_stdint.h b/opal/include/opal_stdint.h index 4089cb55a9..efbd72c645 100644 --- a/opal/include/opal_stdint.h +++ b/opal/include/opal_stdint.h @@ -28,6 +28,8 @@ #ifndef OPAL_STDINT_H #define OPAL_STDINT_H 1 +#include "opal_config.h" + /* * Include what we can and define what is missing. */ diff --git a/opal/threads/mutex_unix.h b/opal/threads/mutex_unix.h index a61623826c..1cafdedd4e 100644 --- a/opal/threads/mutex_unix.h +++ b/opal/threads/mutex_unix.h @@ -76,14 +76,14 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_recursive_mutex_t); .m_lock_debug = 0, \ .m_lock_file = NULL, \ .m_lock_line = 0, \ - .m_lock_atomic = { .u = { .lock = OPAL_ATOMIC_LOCK_UNLOCKED } }, \ + .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \ } #else #define OPAL_MUTEX_STATIC_INIT \ { \ .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), \ .m_lock_pthread = PTHREAD_MUTEX_INITIALIZER, \ - .m_lock_atomic = { .u = { .lock = OPAL_ATOMIC_LOCK_UNLOCKED } }, \ + .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \ } #endif @@ -97,14 +97,14 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_recursive_mutex_t); .m_lock_debug = 0, \ .m_lock_file = NULL, \ .m_lock_line = 0, \ - .m_lock_atomic = { .u = { .lock = OPAL_ATOMIC_LOCK_UNLOCKED } }, \ + .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \ } #else #define OPAL_RECURSIVE_MUTEX_STATIC_INIT \ { \ .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), \ .m_lock_pthread = OPAL_PTHREAD_RECURSIVE_MUTEX_INITIALIZER, \ - .m_lock_atomic = { .u = { .lock = OPAL_ATOMIC_LOCK_UNLOCKED } }, \ + .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \ } #endif diff --git a/orte/runtime/orte_locks.c b/orte/runtime/orte_locks.c index 9cf31a4889..386bc472b8 100644 --- a/orte/runtime/orte_locks.c +++ b/orte/runtime/orte_locks.c @@ -25,12 +25,12 @@ #include "orte/runtime/orte_locks.h" /* for everyone */ -opal_atomic_lock_t orte_finalize_lock = {{0}}; +opal_atomic_lock_t orte_finalize_lock = OPAL_ATOMIC_LOCK_INIT; /* for HNPs */ -opal_atomic_lock_t orte_abort_inprogress_lock = {{0}}; -opal_atomic_lock_t orte_jobs_complete_lock = {{0}}; -opal_atomic_lock_t orte_quit_lock = {{0}}; +opal_atomic_lock_t orte_abort_inprogress_lock = OPAL_ATOMIC_LOCK_INIT; +opal_atomic_lock_t orte_jobs_complete_lock = OPAL_ATOMIC_LOCK_INIT; +opal_atomic_lock_t orte_quit_lock = OPAL_ATOMIC_LOCK_INIT; int orte_locks_init(void) {