Add support for 128-bit compare and swap on x86_64 when available.

A 128-bit compare-and-swap will enable a better atomic lifo implementation
that uses the pointer + counter method to avoid ABA issues. This commit
adds configury to check for the instruction (cmpxchg16b) and adds an
implementation that uses the __int128 type available in C99.

config/opal_config_asm.m4

@@ -558,6 +558,25 @@ AC_DEFUN([OPAL_CHECK_SPARCV8PLUS],[
     unset sparc_result
 ])dnl
 
+dnl #################################################################
+dnl
+dnl OPAL_CHECK_CMPXCHG16B
+dnl
+dnl #################################################################
+AC_DEFUN([OPAL_CHECK_CMPXCHG16B],[
+    AC_MSG_CHECKING([if have x86_64 16-byte compare-and-exchange])
+    OPAL_VAR_SCOPE_PUSH([cmpxchg16b_result])
+    OPAL_TRY_ASSEMBLE([$opal_cv_asm_text
+            cmpxchg16b 0],
+        [AC_MSG_RESULT([yes])
+            cmpxchg16b_result=1],
+        [AC_MSG_RESULT([no])
+            cmpxchg16b_result=0])
+    AC_DEFINE_UNQUOTED([OPAL_HAVE_CMPXCHG16B], [$cmpxchg16b_result],
+        [Whether the processor supports the cmpxchg16b instruction])
+    OPAL_VAR_SCOPE_POP
+])dnl
+
 
 dnl #################################################################
 dnl
@@ -801,6 +820,7 @@ AC_DEFUN([OPAL_CONFIG_ASM],[
             fi
             OPAL_ASM_SUPPORT_64BIT=1
             OPAL_GCC_INLINE_ASSIGN='"xaddl %1,%0" : "=m"(ret), "+r"(negone) : "m"(ret)'
+            OPAL_CHECK_CMPXCHG16B
             ;;
 
         ia64-*)

configure.ac

@@ -366,6 +366,7 @@ AC_CHECK_TYPES(uint32_t)
 AC_CHECK_TYPES(int64_t)
 AC_CHECK_TYPES(uint64_t)
 AC_CHECK_TYPES(int128_t)
+AC_CHECK_TYPES(__int128)
 AC_CHECK_TYPES(uint128_t)
 AC_CHECK_TYPES(long long)
 

opal/include/opal/sys/amd64/atomic.h

@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -10,6 +11,8 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2007      Sun Microsystems, Inc.  All rights reserverd.
+ * Copyright (c) 2012-2014 Los Alamos National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -119,6 +122,31 @@ static inline int opal_atomic_cmpset_64( volatile int64_t *addr,
 #define opal_atomic_cmpset_acq_64 opal_atomic_cmpset_64
 #define opal_atomic_cmpset_rel_64 opal_atomic_cmpset_64
 
+#if OPAL_GCC_INLINE_ASSEMBLY && OPAL_HAVE_CMPXCHG16B && HAVE___INT128
+
+static inline __int128 opal_atomic_cmpset_128 (volatile __int128 *addr, __int128 oldval,
+                                               __int128 newval)
+{
+    unsigned char ret;
+
+    /* cmpxchg16b compares the value at the address with eax:edx (low:high). if the values are
+     * the same the contents of ebx:ecx are stores at the address. in all cases the value stored
+     * at the address is returned in eax:edx. */
+    __asm__ __volatile__ (SMPLOCK "cmpxchg16b (%%rsi)   \n\t"
+                                  "sete     %0      \n\t"
+                          : "=qm" (ret)
+                          : "S" (addr), "b" (((int64_t *)&newval)[0]), "c" (((int64_t *)&newval)[1]),
+                            "a" (((int64_t *)&oldval)[0]), "d" (((int64_t *)&oldval)[1])
+                          : "memory", "cc", "eax", "edx");
+
+   return ret;
+}
+
+#define OPAL_HAVE_ATOMIC_CMPSET_128 1
+
+#endif /* OPAL_GCC_INLINE_ASSEMBLY */
+
+
 #if OPAL_GCC_INLINE_ASSEMBLY
 
 #define OPAL_HAVE_ATOMIC_SWAP_32 1

opal/include/opal/sys/atomic.h

@@ -177,6 +177,9 @@ typedef struct opal_atomic_lock_t opal_atomic_lock_t;
 #ifndef OPAL_HAVE_ATOMIC_CMPSET_64
 #define OPAL_HAVE_ATOMIC_CMPSET_64 0
 #endif
+#ifndef OPAL_HAVE_ATOMIC_CMPSET_128
+#define OPAL_HAVE_ATOMIC_CMPSET_128 0
+#endif
 #endif /* DOXYGEN */
 
 /**********************************************************************

opal/include/opal/sys/atomic_impl.h

@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -10,6 +11,8 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2014 Los Alamos National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -266,9 +269,9 @@ static inline int opal_atomic_cmpset_rel_ptr(volatile void* addr,
 #if (OPAL_HAVE_ATOMIC_SWAP_32 || OPAL_HAVE_ATOMIC_SWAP_64)
 
 #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_SWAP_32
-#define opal_atomic_swap_ptr(addr, value) opal_atomic_swap_32((int32_t *) addr, value)
+#define opal_atomic_swap_ptr(addr, value) (void *) opal_atomic_swap_32((int32_t *) addr, (int32_t) value)
 #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_SWAP_64
-#define opal_atomic_swap_ptr(addr, value) opal_atomic_swap_64((int64_t *) addr, value)
+#define opal_atomic_swap_ptr(addr, value) (void *) opal_atomic_swap_64((int64_t *) addr, (int64_t) value)
 #endif
 
 #endif /* (OPAL_HAVE_ATOMIC_SWAP_32 || OPAL_HAVE_ATOMIC_SWAP_64) */