opal/atomic: always use C11 atomics if available

This commit disables the use of both the builtin and hand-written
atomics if proper C11 atomic support is detected. This is the first
step towards requiring the availability of C11 atomics for the C
compiler used to build Open MPI.

Signed-off-by: Nathan Hjelm <hjelmn@lanl.gov>

config/opal_config_asm.m4

@@ -13,7 +13,7 @@ dnl Copyright (c) 2008-2018 Cisco Systems, Inc.  All rights reserved.
 dnl Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved.
 dnl Copyright (c) 2015-2017 Research Organization for Information Science
 dnl                         and Technology (RIST). All rights reserved.
-dnl Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights
+dnl Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights
 dnl                         reserved.
 dnl Copyright (c) 2017      Amazon.com, Inc. or its affiliates.  All Rights
 dnl                         reserved.
@@ -122,6 +122,58 @@ int main(int argc, char** argv)
 }
 ]])
 
+dnl This is a C test to see if 128-bit __atomic_compare_exchange_n()
+dnl actually works (e.g., it compiles and links successfully on
+dnl ARM64+clang, but returns incorrect answers as of August 2018).
+AC_DEFUN([OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_TEST_SOURCE],[[
+#include <stdint.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <stdatomic.h>
+
+typedef union {
+    uint64_t fake@<:@2@:>@;
+    _Atomic __int128 real;
+} ompi128;
+
+static void test1(void)
+{
+    // As of Aug 2018, we could not figure out a way to assign 128-bit
+    // constants -- the compilers would not accept it.  So use a fake
+    // union to assign 2 uin64_t's to make a single __int128.
+    ompi128 ptr      = { .fake = { 0xFFEEDDCCBBAA0099, 0x8877665544332211 }};
+    ompi128 expected = { .fake = { 0x11EEDDCCBBAA0099, 0x88776655443322FF }};
+    ompi128 desired  = { .fake = { 0x1122DDCCBBAA0099, 0x887766554433EEFF }};
+    bool r = atomic_compare_exchange_strong (&ptr.real, &expected.real,
+                                             desired.real, true,
+					     atomic_relaxed, atomic_relaxed);
+    if ( !(r == false && ptr.real == expected.real)) {
+        exit(1);
+    }
+}
+
+static void test2(void)
+{
+    ompi128 ptr =      { .fake = { 0xFFEEDDCCBBAA0099, 0x8877665544332211 }};
+    ompi128 expected = ptr;
+    ompi128 desired =  { .fake = { 0x1122DDCCBBAA0099, 0x887766554433EEFF }};
+    bool r = atomic_compare_exchange_strong (&ptr.real, &expected.real,
+                                             desired.real, true,
+					     atomic_relaxed, atomic_relaxed);
+    if (!(r == true && ptr.real == desired.real)) {
+        exit(2);
+    }
+}
+
+vvvvvvvvvvvvvvvvvvvv
+int main(int argc, char** argv)
+{
+    test1();
+    test2();
+    return 0;
+}
+]])
+
 dnl ------------------------------------------------------------------
 
 dnl
@@ -329,6 +381,71 @@ __atomic_add_fetch(&tmp64, 1, __ATOMIC_RELAXED);],
   OPAL_CHECK_GCC_BUILTIN_CSWAP_INT128
 ])
 
+AC_DEFUN([OPAL_CHECK_C11_CSWAP_INT128], [
+  OPAL_VAR_SCOPE_PUSH([atomic_compare_exchange_result atomic_compare_exchange_CFLAGS_save atomic_compare_exchange_LIBS_save])
+
+  atomic_compare_exchange_CFLAGS_save=$CFLAGS
+  atomic_compare_exchange_LIBS_save=$LIBS
+
+  # Do we have C11 atomics on 128-bit integers?
+  # Use a special macro because we need to check with a few different
+  # CFLAGS/LIBS.
+  OPAL_ASM_CHECK_ATOMIC_FUNC([atomic_compare_exchange_strong_16],
+      [AC_LANG_SOURCE(OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_TEST_SOURCE)],
+      [atomic_compare_exchange_result=1],
+      [atomic_compare_exchange_result=0])
+
+  # If we have it and it works, check to make sure it is always lock
+  # free.
+  AS_IF([test $atomic_compare_exchange_result -eq 1],
+        [AC_MSG_CHECKING([if C11 __int128 atomic compare-and-swap is always lock-free])
+         AC_RUN_IFELSE([AC_LANG_PROGRAM([#include <stdatomic.h>], [_Atomic __int128_t x; if (!atomic_is_lock_free(&x)) { return 1; }])],
+              [AC_MSG_RESULT([yes])],
+              [atomic_compare_exchange_result=0
+               # If this test fails, need to reset CFLAGS/LIBS (the
+               # above tests atomically set CFLAGS/LIBS or not; this
+               # test is running after the fact, so we have to undo
+               # the side-effects of setting CFLAGS/LIBS if the above
+               # tests passed).
+               CFLAGS=$atomic_compare_exchange_CFLAGS_save
+               LIBS=$atomic_compare_exchange_LIBS_save
+               AC_MSG_RESULT([no])],
+              [AC_MSG_RESULT([cannot test -- assume yes (cross compiling)])])
+        ])
+
+  AC_DEFINE_UNQUOTED([OPAL_HAVE_C11_CSWAP_INT128],
+        [$atomic_compare_exchange_result],
+        [Whether C11 atomic compare swap is both supported and lock-free on 128-bit values])
+
+  dnl If we could not find decent support for 128-bits atomic let's
+  dnl try the GCC _sync
+  AS_IF([test $atomic_compare_exchange_result -eq 0],
+      [OPAL_CHECK_SYNC_BUILTIN_CSWAP_INT128])
+
+  OPAL_VAR_SCOPE_POP
+])
+
+AC_DEFUN([OPAL_CHECK_GCC_ATOMIC_BUILTINS], [
+  AC_MSG_CHECKING([for __atomic builtin atomics])
+
+  AC_TRY_LINK([
+#include <stdint.h>
+uint32_t tmp, old = 0;
+uint64_t tmp64, old64 = 0;], [
+__atomic_thread_fence(__ATOMIC_SEQ_CST);
+__atomic_compare_exchange_n(&tmp, &old, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+__atomic_add_fetch(&tmp, 1, __ATOMIC_RELAXED);
+__atomic_compare_exchange_n(&tmp64, &old64, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
+__atomic_add_fetch(&tmp64, 1, __ATOMIC_RELAXED);],
+    [AC_MSG_RESULT([yes])
+     $1],
+    [AC_MSG_RESULT([no])
+     $2])
+
+  # Check for 128-bit support
+  OPAL_CHECK_GCC_BUILTIN_CSWAP_INT128
+])
+
 
 dnl #################################################################
 dnl
@@ -1020,17 +1137,27 @@ AC_DEFUN([OPAL_CONFIG_ASM],[
     AC_REQUIRE([OPAL_SETUP_CC])
     AC_REQUIRE([AM_PROG_AS])
 
+    AC_ARG_ENABLE([c11-atomics],[AC_HELP_STRING([--enable-c11-atomics],
+                  [Enable use of C11 atomics if available (default: enabled)])])
+
     AC_ARG_ENABLE([builtin-atomics],
       [AC_HELP_STRING([--enable-builtin-atomics],
-         [Enable use of __sync builtin atomics (default: enabled)])])
+         [Enable use of __sync builtin atomics (default: disabled)])])
 
-    opal_cv_asm_builtin="BUILTIN_NO"
+    OPAL_CHECK_C11_CSWAP_INT128
-    AS_IF([test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" != "no"],
+
-          [OPAL_CHECK_GCC_ATOMIC_BUILTINS([opal_cv_asm_builtin="BUILTIN_GCC"], [])])
+    if test "x$enable_c11_atomics" != "xno" && test "$opal_cv_c11_supported" = "yes" ; then
-    AS_IF([test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" != "no"],
+        opal_cv_asm_builtin="BUILTIN_C11"
-          [OPAL_CHECK_SYNC_BUILTINS([opal_cv_asm_builtin="BUILTIN_SYNC"], [])])
+        OPAL_CHECK_C11_CSWAP_INT128
-    AS_IF([test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes"],
+    else
-          [AC_MSG_ERROR([__sync builtin atomics requested but not found.])])
+        opal_cv_asm_builtin="BUILTIN_NO"
+        AS_IF([test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes"],
+              [OPAL_CHECK_GCC_ATOMIC_BUILTINS([opal_cv_asm_builtin="BUILTIN_GCC"], [])])
+        AS_IF([test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes"],
+              [OPAL_CHECK_SYNC_BUILTINS([opal_cv_asm_builtin="BUILTIN_SYNC"], [])])
+        AS_IF([test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes"],
+              [AC_MSG_ERROR([__sync builtin atomics requested but not found.])])
+    fi
 
         OPAL_CHECK_ASM_PROC
         OPAL_CHECK_ASM_TEXT

opal/class/opal_fifo.h

@@ -155,11 +155,9 @@ static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo)
              * update the head */
 
             /* wait for next pointer to be updated by push */
-            while (ghost == item->opal_list_next) {
+            do {
                 opal_atomic_rmb ();
-            }
+            } while (ghost == item->opal_list_next);
-
-            opal_atomic_rmb ();
 
             /* update the head with the real next value. note that no other thread
              * will be attempting to update the head until after it has been updated

opal/class/opal_object.c

@@ -55,7 +55,7 @@ int opal_class_init_epoch = 1;
 /*
  * Local variables
  */
-static opal_atomic_lock_t class_lock = { { OPAL_ATOMIC_LOCK_UNLOCKED } };
+static opal_atomic_lock_t class_lock = OPAL_ATOMIC_LOCK_INIT;
 static void** classes = NULL;
 static int num_classes = 0;
 static int max_classes = 0;

opal/include/opal/sys/Makefile.am

@@ -11,7 +11,7 @@
 #                         All rights reserved.
 # Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
-# Copyright (c) 2016      Los Alamos National Security, LLC. All rights
+# Copyright (c) 2016-2018 Los Alamos National Security, LLC. All rights
 #                         reserved.
 # Copyright (c) 2017      Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
@@ -27,6 +27,7 @@
 headers += \
 	opal/sys/architecture.h \
 	opal/sys/atomic.h \
+	opal/sys/atomic_stdc.h \
 	opal/sys/atomic_impl.h \
 	opal/sys/timer.h \
         opal/sys/cma.h

opal/include/opal/sys/architecture.h

@@ -47,6 +47,7 @@
 #define OPAL_BUILTIN_SYNC   0200
 #define OPAL_BUILTIN_GCC    0202
 #define OPAL_BUILTIN_NO     0203
+#define OPAL_BUILTIN_C11    0204
 
 /* Formats */
 #define OPAL_DEFAULT        1000  /* standard for given architecture */

opal/include/opal/sys/atomic.h

@@ -58,6 +58,12 @@
 #include "opal/sys/architecture.h"
 #include "opal_stdatomic.h"
 
+#if OPAL_ASSEMBLY_BUILTIN == OPAL_BUILTIN_C11
+
+#include "atomic_stdc.h"
+
+#else /* !OPAL_C_HAVE__ATOMIC */
+
 /* do some quick #define cleanup in cases where we are doing
    testing... */
 #ifdef OPAL_DISABLE_INLINE_ASM
@@ -147,6 +153,8 @@ enum {
     OPAL_ATOMIC_LOCK_LOCKED = 1
 };
 
+#define OPAL_ATOMIC_LOCK_INIT {.u = {.lock = OPAL_ATOMIC_LOCK_UNLOCKED}}
+
 /**********************************************************************
  *
  * Load the appropriate architecture files and set some reasonable
@@ -643,6 +651,8 @@ static inline intptr_t opal_atomic_fetch_sub_ptr( opal_atomic_intptr_t* addr, vo
  */
 #include "opal/sys/atomic_impl.h"
 
+#endif /* !OPAL_C_HAVE__ATOMIC */
+
 END_C_DECLS
 
 #endif /* OPAL_SYS_ATOMIC_H */

opal/include/opal/sys/atomic_stdc.h

@@ -0,0 +1,261 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2018      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+/* This file provides shims between the opal atomics interface and the C11 atomics interface. It
+ * is intended as the first step in moving to using C11 atomics across the entire codebase. Once
+ * all officially supported compilers offer C11 atomic (GCC 4.9.0+, icc 2018+, pgi, xlc, etc) then
+ * this shim will go away and the codebase will be updated to use C11's atomic support
+ * directly.
+ * This shim contains some functions already present in atomic_impl.h because we do not include
+ * atomic_impl.h when using C11 atomics. It would require alot of #ifdefs to avoid duplicate
+ * definitions to be worthwhile. */
+
+#if !defined(OPAL_ATOMIC_STDC_H)
+#define OPAL_ATOMIC_STDC_H
+
+#include <stdatomic.h>
+#include <stdint.h>
+#include "opal/include/opal_stdint.h"
+
+#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
+
+#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
+#define OPAL_HAVE_ATOMIC_SWAP_32 1
+
+#define OPAL_HAVE_ATOMIC_MATH_32 1
+#define OPAL_HAVE_ATOMIC_ADD_32 1
+#define OPAL_HAVE_ATOMIC_AND_32 1
+#define OPAL_HAVE_ATOMIC_OR_32 1
+#define OPAL_HAVE_ATOMIC_XOR_32 1
+#define OPAL_HAVE_ATOMIC_SUB_32 1
+
+#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
+#define OPAL_HAVE_ATOMIC_SWAP_64 1
+
+#define OPAL_HAVE_ATOMIC_MATH_64 1
+#define OPAL_HAVE_ATOMIC_ADD_64 1
+#define OPAL_HAVE_ATOMIC_AND_64 1
+#define OPAL_HAVE_ATOMIC_OR_64 1
+#define OPAL_HAVE_ATOMIC_XOR_64 1
+#define OPAL_HAVE_ATOMIC_SUB_64 1
+
+#define OPAL_HAVE_ATOMIC_LLSC_32 0
+#define OPAL_HAVE_ATOMIC_LLSC_64 0
+#define OPAL_HAVE_ATOMIC_LLSC_PTR 0
+
+#define OPAL_HAVE_ATOMIC_MIN_32 1
+#define OPAL_HAVE_ATOMIC_MAX_32 1
+
+#define OPAL_HAVE_ATOMIC_MIN_64 1
+#define OPAL_HAVE_ATOMIC_MAX_64 1
+
+#define OPAL_HAVE_ATOMIC_SPINLOCKS 1
+
+static inline void opal_atomic_mb (void)
+{
+    atomic_thread_fence (memory_order_seq_cst);
+}
+
+static inline void opal_atomic_wmb (void)
+{
+    atomic_thread_fence (memory_order_release);
+}
+
+static inline void opal_atomic_rmb (void)
+{
+    atomic_thread_fence (memory_order_acquire);
+}
+
+#define opal_atomic_compare_exchange_strong_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_relaxed, memory_order_relaxed)
+#define opal_atomic_compare_exchange_strong_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_relaxed, memory_order_relaxed)
+#define opal_atomic_compare_exchange_strong_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_relaxed, memory_order_relaxed)
+#define opal_atomic_compare_exchange_strong_acq_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed)
+#define opal_atomic_compare_exchange_strong_acq_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed)
+#define opal_atomic_compare_exchange_strong_acq_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed)
+
+#define opal_atomic_compare_exchange_strong_rel_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed)
+#define opal_atomic_compare_exchange_strong_rel_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed)
+#define opal_atomic_compare_exchange_strong_rel_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed)
+
+#define opal_atomic_compare_exchange_strong(addr, oldval, newval) atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_relaxed, memory_order_relaxed)
+#define opal_atomic_compare_exchange_strong_acq(addr, oldval, newval)  atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_acquire, memory_order_relaxed)
+#define opal_atomic_compare_exchange_strong_rel(addr, oldval, newval)  atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_release, memory_order_relaxed)
+
+#define opal_atomic_swap_32(addr, value) atomic_exchange_explicit (addr, value, memory_order_relaxed)
+#define opal_atomic_swap_64(addr, value) atomic_exchange_explicit (addr, value, memory_order_relaxed)
+#define opal_atomic_swap_ptr(addr, value) atomic_exchange_explicit (addr, value, memory_order_relaxed)
+
+#define OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(op, bits, type, operator)      \
+    static inline type opal_atomic_fetch_ ## op ##_## bits (opal_atomic_ ## type *addr, type value) \
+    {                                                                   \
+        return atomic_fetch_ ## op ## _explicit (addr, value, memory_order_relaxed); \
+    }                                                                   \
+                                                                        \
+    static inline type opal_atomic_## op ## _fetch_ ## bits (opal_atomic_ ## type *addr, type value) \
+    {                                                                   \
+        return atomic_fetch_ ## op ## _explicit (addr, value, memory_order_relaxed) operator value; \
+    }
+
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, 32, int32_t, +)
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, 64, int64_t, +)
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, size_t, size_t, +)
+
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, 32, int32_t, -)
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, 64, int64_t, -)
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, size_t, size_t, -)
+
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(or, 32, int32_t, |)
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(or, 64, int64_t, |)
+
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(xor, 32, int32_t, ^)
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(xor, 64, int64_t, ^)
+
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(and, 32, int32_t, &)
+OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(and, 64, int64_t, &)
+
+#define opal_atomic_add(addr, value) (void) atomic_fetch_add_explicit (addr, value, memory_order_relaxed)
+
+static inline int32_t opal_atomic_fetch_min_32 (opal_atomic_int32_t *addr, int32_t value)
+{
+    int32_t old = *addr;
+    do {
+        if (old <= value) {
+            break;
+        }
+    } while (!opal_atomic_compare_exchange_strong_32 (addr, &old, value));
+
+    return old;
+}
+
+static inline int32_t opal_atomic_fetch_max_32 (opal_atomic_int32_t *addr, int32_t value)
+{
+    int32_t old = *addr;
+    do {
+        if (old >= value) {
+            break;
+        }
+    } while (!opal_atomic_compare_exchange_strong_32 (addr, &old, value));
+
+    return old;
+}
+
+static inline int64_t opal_atomic_fetch_min_64 (opal_atomic_int64_t *addr, int64_t value)
+{
+    int64_t old = *addr;
+    do {
+        if (old <= value) {
+            break;
+        }
+    } while (!opal_atomic_compare_exchange_strong_64 (addr, &old, value));
+
+    return old;
+}
+
+static inline int64_t opal_atomic_fetch_max_64 (opal_atomic_int64_t *addr, int64_t value)
+{
+    int64_t old = *addr;
+    do {
+        if (old >= value) {
+            break;
+        }
+    } while (!opal_atomic_compare_exchange_strong_64 (addr, &old, value));
+
+    return old;
+}
+
+static inline int32_t opal_atomic_min_fetch_32 (opal_atomic_int32_t *addr, int32_t value)
+{
+    int32_t old = opal_atomic_fetch_min_32 (addr, value);
+    return old <= value ? old : value;
+}
+
+static inline int32_t opal_atomic_max_fetch_32 (opal_atomic_int32_t *addr, int32_t value)
+{
+    int32_t old = opal_atomic_fetch_max_32 (addr, value);
+    return old >= value ? old : value;
+}
+
+static inline int64_t opal_atomic_min_fetch_64 (opal_atomic_int64_t *addr, int64_t value)
+{
+    int64_t old = opal_atomic_fetch_min_64 (addr, value);
+    return old <= value ? old : value;
+}
+
+static inline int64_t opal_atomic_max_fetch_64 (opal_atomic_int64_t *addr, int64_t value)
+{
+    int64_t old = opal_atomic_fetch_max_64 (addr, value);
+    return old >= value ? old : value;
+}
+
+#define OPAL_ATOMIC_LOCK_UNLOCKED false
+#define OPAL_ATOMIC_LOCK_LOCKED true
+
+#define OPAL_ATOMIC_LOCK_INIT ATOMIC_FLAG_INIT
+
+typedef atomic_flag opal_atomic_lock_t;
+
+/*
+ * Lock initialization function. It set the lock to UNLOCKED.
+ */
+static inline void opal_atomic_lock_init (opal_atomic_lock_t *lock, bool value)
+{
+    atomic_flag_clear (lock);
+}
+
+
+static inline int opal_atomic_trylock (opal_atomic_lock_t *lock)
+{
+    return (int) atomic_flag_test_and_set (lock);
+}
+
+
+static inline void opal_atomic_lock(opal_atomic_lock_t *lock)
+{
+    while (opal_atomic_trylock (lock)) {
+    }
+}
+
+
+static inline void opal_atomic_unlock (opal_atomic_lock_t *lock)
+{
+    atomic_flag_clear (lock);
+}
+
+
+#if OPAL_HAVE_C11_CSWAP_INT128
+
+/* the C11 atomic compare-exchange is lock free so use it */
+#define opal_atomic_compare_exchange_strong_128 atomic_compare_exchange_strong
+
+#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
+
+#elif OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128
+
+/* fall back on the __sync builtin if available since it will emit the expected instruction on x86_64 (cmpxchng16b) */
+__opal_attribute_always_inline__
+static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_int128_t *addr,
+                                                            opal_int128_t *oldval, opal_int128_t newval)
+{
+    opal_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval);
+    bool ret = prev == *oldval;
+    *oldval = prev;
+    return ret;
+}
+
+#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
+
+#else
+
+#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 0
+
+#endif
+
+#endif /* !defined(OPAL_ATOMIC_STDC_H) */

opal/include/opal/sys/sync_builtin/atomic.h

@@ -149,7 +149,7 @@ static inline int64_t opal_atomic_fetch_sub_64(opal_atomic_int64_t *addr, int64_
 #endif
 
 #if OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128
-static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_opal_int128_t *addr,
+static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_int128_t *addr,
                                                             opal_int128_t *oldval, opal_int128_t newval)
 {
     opal_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval);

opal/include/opal_stdatomic.h

@@ -14,6 +14,9 @@
 
 #include "opal_stdint.h"
 
+#if OPAL_ASSEMBLY_BUILTIN != OPAL_BUILTIN_C11
+
+typedef volatile int opal_atomic_int_t;
 typedef volatile long opal_atomic_long_t;
 
 typedef volatile int32_t opal_atomic_int32_t;
@@ -22,14 +25,42 @@ typedef volatile int64_t opal_atomic_int64_t;
 typedef volatile uint64_t opal_atomic_uint64_t;
 
 typedef volatile size_t opal_atomic_size_t;
+typedef volatile ssize_t opal_atomic_ssize_t;
 typedef volatile intptr_t opal_atomic_intptr_t;
 typedef volatile uintptr_t opal_atomic_uintptr_t;
 
+#else /* OPAL_HAVE_C__ATOMIC */
+
+#include <stdatomic.h>
+
+typedef atomic_int opal_atomic_int_t;
+typedef atomic_long opal_atomic_long_t;
+
+typedef _Atomic int32_t opal_atomic_int32_t;
+typedef _Atomic uint32_t opal_atomic_uint32_t;
+typedef _Atomic int64_t opal_atomic_int64_t;
+typedef _Atomic uint64_t opal_atomic_uint64_t;
+
+typedef _Atomic size_t opal_atomic_size_t;
+typedef _Atomic ssize_t opal_atomic_ssize_t;
+typedef _Atomic intptr_t opal_atomic_intptr_t;
+typedef _Atomic uintptr_t opal_atomic_uintptr_t;
+
+#endif /* OPAL_HAVE_C__ATOMIC */
+
 #if HAVE_OPAL_INT128_T
 
+/* do not use C11 atomics for __int128 if they are not lock free */
+#if OPAL_HAVE_C11_CSWAP_INT128
+
+typedef _Atomic opal_int128_t opal_atomic_int128_t;
+
+#else
+
 typedef volatile opal_int128_t opal_atomic_int128_t;
 
 #endif
 
+#endif
 
 #endif /* !defined(OPAL_STDATOMIC_H) */

opal/include/opal_stdint.h

@@ -28,6 +28,8 @@
 #ifndef OPAL_STDINT_H
 #define OPAL_STDINT_H 1
 
+#include "opal_config.h"
+
 /*
  * Include what we can and define what is missing.
  */

opal/threads/mutex_unix.h

@@ -76,14 +76,14 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_recursive_mutex_t);
         .m_lock_debug = 0,                                              \
         .m_lock_file = NULL,                                            \
         .m_lock_line = 0,                                               \
-        .m_lock_atomic = { .u = { .lock = OPAL_ATOMIC_LOCK_UNLOCKED } },     \
+        .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT,                         \
     }
 #else
 #define OPAL_MUTEX_STATIC_INIT                                          \
     {                                                                   \
         .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t),                    \
         .m_lock_pthread = PTHREAD_MUTEX_INITIALIZER,                    \
-        .m_lock_atomic = { .u = { .lock = OPAL_ATOMIC_LOCK_UNLOCKED } },     \
+        .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT,                         \
     }
 #endif
 
@@ -97,14 +97,14 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_recursive_mutex_t);
         .m_lock_debug = 0,                                              \
         .m_lock_file = NULL,                                            \
         .m_lock_line = 0,                                               \
-        .m_lock_atomic = { .u = { .lock = OPAL_ATOMIC_LOCK_UNLOCKED } },     \
+        .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT,                         \
     }
 #else
 #define OPAL_RECURSIVE_MUTEX_STATIC_INIT                                \
     {                                                                   \
         .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t),                    \
         .m_lock_pthread = OPAL_PTHREAD_RECURSIVE_MUTEX_INITIALIZER,     \
-        .m_lock_atomic = { .u = { .lock = OPAL_ATOMIC_LOCK_UNLOCKED } },     \
+        .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT,                         \
     }
 #endif
 

orte/runtime/orte_locks.c

@@ -25,12 +25,12 @@
 #include "orte/runtime/orte_locks.h"
 
 /* for everyone */
-opal_atomic_lock_t orte_finalize_lock = {{0}};
+opal_atomic_lock_t orte_finalize_lock = OPAL_ATOMIC_LOCK_INIT;
 
 /* for HNPs */
-opal_atomic_lock_t orte_abort_inprogress_lock = {{0}};
+opal_atomic_lock_t orte_abort_inprogress_lock = OPAL_ATOMIC_LOCK_INIT;
-opal_atomic_lock_t orte_jobs_complete_lock = {{0}};
+opal_atomic_lock_t orte_jobs_complete_lock = OPAL_ATOMIC_LOCK_INIT;
-opal_atomic_lock_t orte_quit_lock = {{0}};
+opal_atomic_lock_t orte_quit_lock = OPAL_ATOMIC_LOCK_INIT;
 
 int orte_locks_init(void)
 {