Merge pull request #297 from hjelmn/topic/cmpset128

Add support for 128-bit compare and swap on x86_64 when available.
2014-12-04 16:05:13 -06:00 · 2014-12-04 16:05:13 -06:00 · 077f4a2982
--- a/config/opal_config_asm.m4
+++ b/config/opal_config_asm.m4
@ -19,6 +19,37 @@ dnl $HEADER$
 dnl


+AC_DEFUN([OPAL_CHECK_SYNC_BUILTIN_CSWAP_INT128], [
+
+  OPAL_VAR_SCOPE_PUSH([sync_bool_compare_and_swap_128_result CFLAGS_save])
+
+  AC_MSG_CHECKING([for __sync builtin atomic compare-and-swap on 128-bit values])
+  AC_TRY_COMPILE([], [__int128 x = 0; __sync_bool_compare_and_swap (&x, 0, 1);],
+    [AC_MSG_RESULT([yes])
+     sync_bool_compare_and_swap_128_result=1],
+    [AC_MSG_RESULT([no])
+     sync_bool_compare_and_swap_128_result=0])
+
+  if test $sync_bool_compare_and_swap_128_result = 0 ; then
+      CFLAGS_save=$CFLAGS
+      CFLAGS="$CFLAGS -mcx16"
+
+      AC_MSG_CHECKING([for __sync builtin atomic compare-and-swap on 128-bit values with -mcx16 flag])
+      AC_TRY_COMPILE([], [__int128 x = 0; __sync_bool_compare_and_swap (&x, 0, 1);],
+        [AC_MSG_RESULT([yes])
+         sync_bool_compare_and_swap_128_result=1
+	 CFLAGS_save="$CFLAGS"],
+        [AC_MSG_RESULT([no])])
+
+      CFLAGS=$CFLAGS_save
+  fi
+
+  AC_DEFINE_UNQUOTED([OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128], [$sync_bool_compare_and_swap_128_result],
+	[Whether the __sync builtin atomic compare and swap supports 128-bit values])
+
+  OPAL_VAR_SCOPE_POP
+])
+
 AC_DEFUN([OPAL_CHECK_SYNC_BUILTINS], [
  AC_MSG_CHECKING([for __sync builtin atomics])

@ -558,6 +589,24 @@ AC_DEFUN([OPAL_CHECK_SPARCV8PLUS],[
    unset sparc_result
 ])dnl

+dnl #################################################################
+dnl
+dnl OPAL_CHECK_CMPXCHG16B
+dnl
+dnl #################################################################
+AC_DEFUN([OPAL_CHECK_CMPXCHG16B],[
+    AC_MSG_CHECKING([if have x86_64 16-byte compare-and-exchange])
+    OPAL_VAR_SCOPE_PUSH([cmpxchg16b_result])
+    OPAL_TRY_ASSEMBLE([$opal_cv_asm_text
+            cmpxchg16b 0],
+        [AC_MSG_RESULT([yes])
+            cmpxchg16b_result=1],
+        [AC_MSG_RESULT([no])
+            cmpxchg16b_result=0])
+    AC_DEFINE_UNQUOTED([OPAL_HAVE_CMPXCHG16B], [$cmpxchg16b_result],
+        [Whether the processor supports the cmpxchg16b instruction])
+    OPAL_VAR_SCOPE_POP
+])dnl

 dnl #################################################################
 dnl
@ -770,6 +819,7 @@ AC_DEFUN([OPAL_CONFIG_ASM],[
         [AC_MSG_ERROR([__sync builtin atomics requested but not found.])])
       AC_DEFINE([OPAL_C_GCC_INLINE_ASSEMBLY], [1],
         [Whether C compiler supports GCC style inline assembly])
+       OPAL_CHECK_SYNC_BUILTIN_CSWAP_INT128
    elif test "$enable_osx_builtin_atomics" = "yes" ; then
 	   AC_CHECK_HEADER([libkern/OSAtomic.h],[opal_cv_asm_builtin="BUILTIN_OSX"],
 	    [AC_MSG_ERROR([OSX builtin atomics requested but not found.])])
@ -801,6 +851,7 @@ AC_DEFUN([OPAL_CONFIG_ASM],[
            fi
            OPAL_ASM_SUPPORT_64BIT=1
            OPAL_GCC_INLINE_ASSIGN='"xaddl %1,%0" : "=m"(ret), "+r"(negone) : "m"(ret)'
+            OPAL_CHECK_CMPXCHG16B
            ;;

        ia64-*)
--- a/configure.ac
+++ b/configure.ac
@ -366,6 +366,7 @@ AC_CHECK_TYPES(uint32_t)
 AC_CHECK_TYPES(int64_t)
 AC_CHECK_TYPES(uint64_t)
 AC_CHECK_TYPES(int128_t)
+AC_CHECK_TYPES(__int128)
 AC_CHECK_TYPES(uint128_t)
 AC_CHECK_TYPES(long long)

--- a/opal/include/opal/sys/amd64/atomic.h
+++ b/opal/include/opal/sys/amd64/atomic.h
@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
 * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
 *                         University Research and Technology
@ -10,6 +11,8 @@
 * Copyright (c) 2004-2005 The Regents of the University of California.
 *                         All rights reserved.
 * Copyright (c) 2007      Sun Microsystems, Inc.  All rights reserverd.
+ * Copyright (c) 2012-2014 Los Alamos National Security, LLC. All rights
+ *                         reserved.
 * $COPYRIGHT$
 * 
 * Additional copyrights may follow
@ -119,6 +122,31 @@ static inline int opal_atomic_cmpset_64( volatile int64_t *addr,
 #define opal_atomic_cmpset_acq_64 opal_atomic_cmpset_64
 #define opal_atomic_cmpset_rel_64 opal_atomic_cmpset_64

+#if OPAL_GCC_INLINE_ASSEMBLY && OPAL_HAVE_CMPXCHG16B && HAVE_OPAL_INT128_T
+
+static inline int opal_atomic_cmpset_128 (volatile opal_int128_t *addr, opal_int128_t oldval,
+                                          opal_int128_t newval)
+{
+    unsigned char ret;
+
+    /* cmpxchg16b compares the value at the address with eax:edx (low:high). if the values are
+     * the same the contents of ebx:ecx are stores at the address. in all cases the value stored
+     * at the address is returned in eax:edx. */
+    __asm__ __volatile__ (SMPLOCK "cmpxchg16b (%%rsi)   \n\t"
+                                  "sete     %0      \n\t"
+                          : "=qm" (ret)
+                          : "S" (addr), "b" (((int64_t *)&newval)[0]), "c" (((int64_t *)&newval)[1]),
+                            "a" (((int64_t *)&oldval)[0]), "d" (((int64_t *)&oldval)[1])
+                          : "memory", "cc", "eax", "edx");
+
+    return (int) ret;
+}
+
+#define OPAL_HAVE_ATOMIC_CMPSET_128 1
+
+#endif /* OPAL_GCC_INLINE_ASSEMBLY */
+
+
 #if OPAL_GCC_INLINE_ASSEMBLY

 #define OPAL_HAVE_ATOMIC_SWAP_32 1
--- a/opal/include/opal/sys/atomic.h
+++ b/opal/include/opal/sys/atomic.h
@ -49,10 +49,7 @@
 #include "opal_config.h"

 #include "opal/sys/architecture.h"
-
-#ifdef HAVE_SYS_TYPES_H
-#include <sys/types.h>
-#endif
+#include "opal_stdint.h"

 /* do some quick #define cleanup in cases where we are doing
   testing... */
@ -177,6 +174,9 @@ typedef struct opal_atomic_lock_t opal_atomic_lock_t;
 #ifndef OPAL_HAVE_ATOMIC_CMPSET_64
 #define OPAL_HAVE_ATOMIC_CMPSET_64 0
 #endif
+#ifndef OPAL_HAVE_ATOMIC_CMPSET_128
+#define OPAL_HAVE_ATOMIC_CMPSET_128 0
+#endif
 #endif /* DOXYGEN */

 /**********************************************************************
--- a/opal/include/opal/sys/atomic_impl.h
+++ b/opal/include/opal/sys/atomic_impl.h
@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
 * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
 *                         University Research and Technology
@ -10,6 +11,8 @@
 * Copyright (c) 2004-2005 The Regents of the University of California.
 *                         All rights reserved.
 * Copyright (c) 2010-2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2012-2014 Los Alamos National Security, LLC. All rights
+ *                         reserved.
 * $COPYRIGHT$
 * 
 * Additional copyrights may follow
@ -266,9 +269,9 @@ static inline int opal_atomic_cmpset_rel_ptr(volatile void* addr,
 #if (OPAL_HAVE_ATOMIC_SWAP_32 || OPAL_HAVE_ATOMIC_SWAP_64)

 #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_SWAP_32
-#define opal_atomic_swap_ptr(addr, value) opal_atomic_swap_32((int32_t *) addr, value)
+#define opal_atomic_swap_ptr(addr, value) (void *) opal_atomic_swap_32((int32_t *) addr, (int32_t) value)
 #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_SWAP_64
-#define opal_atomic_swap_ptr(addr, value) opal_atomic_swap_64((int64_t *) addr, value)
+#define opal_atomic_swap_ptr(addr, value) (void *) opal_atomic_swap_64((int64_t *) addr, (int64_t) value)
 #endif

 #endif /* (OPAL_HAVE_ATOMIC_SWAP_32 || OPAL_HAVE_ATOMIC_SWAP_64) */
--- a/opal/include/opal/sys/sync_builtin/atomic.h
+++ b/opal/include/opal/sys/sync_builtin/atomic.h
@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
 * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
 *                         University Research and Technology
@ -10,6 +11,8 @@
 * Copyright (c) 2004-2005 The Regents of the University of California.
 *                         All rights reserved.
 * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
+ * Copyright (c) 2014      Los Alamos National Security, LLC. All rights
+ *                         reserved.
 * $COPYRIGHT$
 * 
 * Additional copyrights may follow
@ -106,6 +109,17 @@ static inline int opal_atomic_cmpset_64( volatile int64_t *addr,
    return __sync_bool_compare_and_swap(addr, oldval, newval);
 }

+#if OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128
+static inline int opal_atomic_cmpset_128 (volatile opal_int128_t *addr,
+                                          opal_int128_t oldval, opal_int128_t newval)
+{
+    return __sync_bool_compare_and_swap(addr, oldval, newval);
+}
+
+#define OPAL_HAVE_ATOMIC_CMPSET_128 1
+
+#endif
+
 #define OPAL_HAVE_ATOMIC_MATH_64 1
 #define OPAL_HAVE_ATOMIC_ADD_64 1
 static inline int64_t opal_atomic_add_64(volatile int64_t *addr, int64_t delta)
--- a/opal/include/opal_stdint.h
+++ b/opal/include/opal_stdint.h
@ -134,6 +134,33 @@ typedef unsigned long long uint64_t;

 #endif

+/* 128-bit */
+
+#ifdef HAVE_INT128_T
+
+typedef int128_t opal_int128_t;
+typedef uint128_t opal_uint128_t;
+
+#define HAVE_OPAL_INT128_T 1
+
+#elif HAVE___INT128
+
+/* suppress warning about __int128 type */
+#pragma GCC diagnostic ignored "-Wpedantic"
+typedef __int128 opal_int128_t;
+
+/* suppress warning about __int128 type */
+#pragma GCC diagnostic ignored "-Wpedantic"
+typedef unsigned __int128 opal_uint128_t;
+
+#define HAVE_OPAL_INT128_T 1
+
+#else
+
+#define HAVE_OPAL_INT128_T 0
+
+#endif
+
 /* Pointers */

 #if SIZEOF_VOID_P == SIZEOF_INT