diff --git a/config/opal_config_asm.m4 b/config/opal_config_asm.m4 index 5ec73d913f..41672d65d7 100644 --- a/config/opal_config_asm.m4 +++ b/config/opal_config_asm.m4 @@ -19,6 +19,37 @@ dnl $HEADER$ dnl +AC_DEFUN([OPAL_CHECK_SYNC_BUILTIN_CSWAP_INT128], [ + + OPAL_VAR_SCOPE_PUSH([sync_bool_compare_and_swap_128_result CFLAGS_save]) + + AC_MSG_CHECKING([for __sync builtin atomic compare-and-swap on 128-bit values]) + AC_TRY_COMPILE([], [__int128 x = 0; __sync_bool_compare_and_swap (&x, 0, 1);], + [AC_MSG_RESULT([yes]) + sync_bool_compare_and_swap_128_result=1], + [AC_MSG_RESULT([no]) + sync_bool_compare_and_swap_128_result=0]) + + if test $sync_bool_compare_and_swap_128_result = 0 ; then + CFLAGS_save=$CFLAGS + CFLAGS="$CFLAGS -mcx16" + + AC_MSG_CHECKING([for __sync builtin atomic compare-and-swap on 128-bit values with -mcx16 flag]) + AC_TRY_COMPILE([], [__int128 x = 0; __sync_bool_compare_and_swap (&x, 0, 1);], + [AC_MSG_RESULT([yes]) + sync_bool_compare_and_swap_128_result=1 + CFLAGS_save="$CFLAGS"], + [AC_MSG_RESULT([no])]) + + CFLAGS=$CFLAGS_save + fi + + AC_DEFINE_UNQUOTED([OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128], [$sync_bool_compare_and_swap_128_result], + [Whether the __sync builtin atomic compare and swap supports 128-bit values]) + + OPAL_VAR_SCOPE_POP +]) + AC_DEFUN([OPAL_CHECK_SYNC_BUILTINS], [ AC_MSG_CHECKING([for __sync builtin atomics]) @@ -558,6 +589,24 @@ AC_DEFUN([OPAL_CHECK_SPARCV8PLUS],[ unset sparc_result ])dnl +dnl ################################################################# +dnl +dnl OPAL_CHECK_CMPXCHG16B +dnl +dnl ################################################################# +AC_DEFUN([OPAL_CHECK_CMPXCHG16B],[ + AC_MSG_CHECKING([if have x86_64 16-byte compare-and-exchange]) + OPAL_VAR_SCOPE_PUSH([cmpxchg16b_result]) + OPAL_TRY_ASSEMBLE([$opal_cv_asm_text + cmpxchg16b 0], + [AC_MSG_RESULT([yes]) + cmpxchg16b_result=1], + [AC_MSG_RESULT([no]) + cmpxchg16b_result=0]) + AC_DEFINE_UNQUOTED([OPAL_HAVE_CMPXCHG16B], [$cmpxchg16b_result], + [Whether the processor supports the cmpxchg16b instruction]) + OPAL_VAR_SCOPE_POP +])dnl dnl ################################################################# dnl @@ -770,6 +819,7 @@ AC_DEFUN([OPAL_CONFIG_ASM],[ [AC_MSG_ERROR([__sync builtin atomics requested but not found.])]) AC_DEFINE([OPAL_C_GCC_INLINE_ASSEMBLY], [1], [Whether C compiler supports GCC style inline assembly]) + OPAL_CHECK_SYNC_BUILTIN_CSWAP_INT128 elif test "$enable_osx_builtin_atomics" = "yes" ; then AC_CHECK_HEADER([libkern/OSAtomic.h],[opal_cv_asm_builtin="BUILTIN_OSX"], [AC_MSG_ERROR([OSX builtin atomics requested but not found.])]) @@ -801,6 +851,7 @@ AC_DEFUN([OPAL_CONFIG_ASM],[ fi OPAL_ASM_SUPPORT_64BIT=1 OPAL_GCC_INLINE_ASSIGN='"xaddl %1,%0" : "=m"(ret), "+r"(negone) : "m"(ret)' + OPAL_CHECK_CMPXCHG16B ;; ia64-*) diff --git a/configure.ac b/configure.ac index 850e94f100..99d51e7b27 100644 --- a/configure.ac +++ b/configure.ac @@ -366,6 +366,7 @@ AC_CHECK_TYPES(uint32_t) AC_CHECK_TYPES(int64_t) AC_CHECK_TYPES(uint64_t) AC_CHECK_TYPES(int128_t) +AC_CHECK_TYPES(__int128) AC_CHECK_TYPES(uint128_t) AC_CHECK_TYPES(long long) diff --git a/opal/include/opal/sys/amd64/atomic.h b/opal/include/opal/sys/amd64/atomic.h index ddb6213c29..83bd76723a 100644 --- a/opal/include/opal/sys/amd64/atomic.h +++ b/opal/include/opal/sys/amd64/atomic.h @@ -1,3 +1,4 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology @@ -10,6 +11,8 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007 Sun Microsystems, Inc. All rights reserverd. + * Copyright (c) 2012-2014 Los Alamos National Security, LLC. All rights + * reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -119,6 +122,31 @@ static inline int opal_atomic_cmpset_64( volatile int64_t *addr, #define opal_atomic_cmpset_acq_64 opal_atomic_cmpset_64 #define opal_atomic_cmpset_rel_64 opal_atomic_cmpset_64 +#if OPAL_GCC_INLINE_ASSEMBLY && OPAL_HAVE_CMPXCHG16B && HAVE_OPAL_INT128_T + +static inline int opal_atomic_cmpset_128 (volatile opal_int128_t *addr, opal_int128_t oldval, + opal_int128_t newval) +{ + unsigned char ret; + + /* cmpxchg16b compares the value at the address with eax:edx (low:high). if the values are + * the same the contents of ebx:ecx are stores at the address. in all cases the value stored + * at the address is returned in eax:edx. */ + __asm__ __volatile__ (SMPLOCK "cmpxchg16b (%%rsi) \n\t" + "sete %0 \n\t" + : "=qm" (ret) + : "S" (addr), "b" (((int64_t *)&newval)[0]), "c" (((int64_t *)&newval)[1]), + "a" (((int64_t *)&oldval)[0]), "d" (((int64_t *)&oldval)[1]) + : "memory", "cc", "eax", "edx"); + + return (int) ret; +} + +#define OPAL_HAVE_ATOMIC_CMPSET_128 1 + +#endif /* OPAL_GCC_INLINE_ASSEMBLY */ + + #if OPAL_GCC_INLINE_ASSEMBLY #define OPAL_HAVE_ATOMIC_SWAP_32 1 diff --git a/opal/include/opal/sys/atomic.h b/opal/include/opal/sys/atomic.h index 220ded71a3..cae28a49b1 100644 --- a/opal/include/opal/sys/atomic.h +++ b/opal/include/opal/sys/atomic.h @@ -49,10 +49,7 @@ #include "opal_config.h" #include "opal/sys/architecture.h" - -#ifdef HAVE_SYS_TYPES_H -#include -#endif +#include "opal_stdint.h" /* do some quick #define cleanup in cases where we are doing testing... */ @@ -177,6 +174,9 @@ typedef struct opal_atomic_lock_t opal_atomic_lock_t; #ifndef OPAL_HAVE_ATOMIC_CMPSET_64 #define OPAL_HAVE_ATOMIC_CMPSET_64 0 #endif +#ifndef OPAL_HAVE_ATOMIC_CMPSET_128 +#define OPAL_HAVE_ATOMIC_CMPSET_128 0 +#endif #endif /* DOXYGEN */ /********************************************************************** diff --git a/opal/include/opal/sys/atomic_impl.h b/opal/include/opal/sys/atomic_impl.h index 8b70eeae3f..496fecdcc4 100644 --- a/opal/include/opal/sys/atomic_impl.h +++ b/opal/include/opal/sys/atomic_impl.h @@ -1,3 +1,4 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology @@ -10,6 +11,8 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2010-2014 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2012-2014 Los Alamos National Security, LLC. All rights + * reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -266,9 +269,9 @@ static inline int opal_atomic_cmpset_rel_ptr(volatile void* addr, #if (OPAL_HAVE_ATOMIC_SWAP_32 || OPAL_HAVE_ATOMIC_SWAP_64) #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_SWAP_32 -#define opal_atomic_swap_ptr(addr, value) opal_atomic_swap_32((int32_t *) addr, value) +#define opal_atomic_swap_ptr(addr, value) (void *) opal_atomic_swap_32((int32_t *) addr, (int32_t) value) #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_SWAP_64 -#define opal_atomic_swap_ptr(addr, value) opal_atomic_swap_64((int64_t *) addr, value) +#define opal_atomic_swap_ptr(addr, value) (void *) opal_atomic_swap_64((int64_t *) addr, (int64_t) value) #endif #endif /* (OPAL_HAVE_ATOMIC_SWAP_32 || OPAL_HAVE_ATOMIC_SWAP_64) */ diff --git a/opal/include/opal/sys/sync_builtin/atomic.h b/opal/include/opal/sys/sync_builtin/atomic.h index cc8991a646..d526a16008 100644 --- a/opal/include/opal/sys/sync_builtin/atomic.h +++ b/opal/include/opal/sys/sync_builtin/atomic.h @@ -1,3 +1,4 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology @@ -10,6 +11,8 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011 Sandia National Laboratories. All rights reserved. + * Copyright (c) 2014 Los Alamos National Security, LLC. All rights + * reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -106,6 +109,17 @@ static inline int opal_atomic_cmpset_64( volatile int64_t *addr, return __sync_bool_compare_and_swap(addr, oldval, newval); } +#if OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128 +static inline int opal_atomic_cmpset_128 (volatile opal_int128_t *addr, + opal_int128_t oldval, opal_int128_t newval) +{ + return __sync_bool_compare_and_swap(addr, oldval, newval); +} + +#define OPAL_HAVE_ATOMIC_CMPSET_128 1 + +#endif + #define OPAL_HAVE_ATOMIC_MATH_64 1 #define OPAL_HAVE_ATOMIC_ADD_64 1 static inline int64_t opal_atomic_add_64(volatile int64_t *addr, int64_t delta) diff --git a/opal/include/opal_stdint.h b/opal/include/opal_stdint.h index be5f799d73..713478a1d6 100644 --- a/opal/include/opal_stdint.h +++ b/opal/include/opal_stdint.h @@ -134,6 +134,33 @@ typedef unsigned long long uint64_t; #endif +/* 128-bit */ + +#ifdef HAVE_INT128_T + +typedef int128_t opal_int128_t; +typedef uint128_t opal_uint128_t; + +#define HAVE_OPAL_INT128_T 1 + +#elif HAVE___INT128 + +/* suppress warning about __int128 type */ +#pragma GCC diagnostic ignored "-Wpedantic" +typedef __int128 opal_int128_t; + +/* suppress warning about __int128 type */ +#pragma GCC diagnostic ignored "-Wpedantic" +typedef unsigned __int128 opal_uint128_t; + +#define HAVE_OPAL_INT128_T 1 + +#else + +#define HAVE_OPAL_INT128_T 0 + +#endif + /* Pointers */ #if SIZEOF_VOID_P == SIZEOF_INT