atomic: add support for __atomic builtins (#1735)
* atomic: add support for __atomic builtins This commit adds support for the gcc __atomic builtins. The __sync builtins are deprecated and have been replaced by these atomics. In addition, the new atomics support atomic exchange which was not supported by __sync. Signed-off-by: Nathan Hjelm <hjelmn@lanl.gov> * atomic: add support for transactional memory This commit adds support for using transactional memory when using opal atomic locks. This feature is enabled if the __HLE__ feature is available and the gcc builtin atomics are in use. Signed-off-by: Nathan Hjelm <hjelmn@lanl.gov>
Этот коммит содержится в:
Nathan Hjelm
bosilca
родитель
b85a5e62ab
Коммит
f33bbfd381
@ -116,6 +116,84 @@ __sync_add_and_fetch(&tmp, 1);],
|
||||
])
|
||||
|
||||
|
||||
AC_DEFUN([OPAL_CHECK_GCC_BUILTIN_CSWAP_INT128], [
|
||||
|
||||
OPAL_VAR_SCOPE_PUSH([atomic_compare_exchange_n_128_result CFLAGS_save])
|
||||
|
||||
AC_ARG_ENABLE([cross-cmpset128],[AC_HELP_STRING([--enable-cross-cmpset128],
|
||||
[enable the use of the __sync builtin atomic compare-and-swap 128 when cross compiling])])
|
||||
|
||||
atomic_compare_exchange_n_128_result=0
|
||||
|
||||
if test ! "$enable_cross_cmpset128" = "yes" ; then
|
||||
AC_MSG_CHECKING([for processor support of __atomic builtin atomic compare-and-swap on 128-bit values])
|
||||
|
||||
AC_RUN_IFELSE([AC_LANG_PROGRAM([], [__int128 x = 0, y = 0; __atomic_compare_exchange_n (&x, &y, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);])],
|
||||
[AC_MSG_RESULT([yes])
|
||||
atomic_compare_exchange_n_128_result=1],
|
||||
[AC_MSG_RESULT([no])],
|
||||
[AC_MSG_RESULT([no (cross compiling)])])
|
||||
|
||||
if test $atomic_compare_exchange_n_128_result = 0 ; then
|
||||
CFLAGS_save=$CFLAGS
|
||||
CFLAGS="$CFLAGS -mcx16"
|
||||
|
||||
AC_MSG_CHECKING([for __atomic builtin atomic compare-and-swap on 128-bit values with -mcx16 flag])
|
||||
AC_RUN_IFELSE([AC_LANG_PROGRAM([], [__int128 x = 0, y = 0; __atomic_compare_exchange_n (&x, &y, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);])],
|
||||
[AC_MSG_RESULT([yes])
|
||||
atomic_compare_exchange_n_128_result=1
|
||||
CFLAGS_save="$CFLAGS"],
|
||||
[AC_MSG_RESULT([no])],
|
||||
[AC_MSG_RESULT([no (cross compiling)])])
|
||||
|
||||
CFLAGS=$CFLAGS_save
|
||||
fi
|
||||
else
|
||||
AC_MSG_CHECKING([for compiler support of __atomic builtin atomic compare-and-swap on 128-bit values])
|
||||
|
||||
# Check if the compiler supports the __atomic builtin
|
||||
AC_TRY_LINK([], [__int128 x = 0; __atomic_bool_compare_and_swap (&x, 0, 1);],
|
||||
[AC_MSG_RESULT([yes])
|
||||
atomic_compare_exchange_n_128_result=1],
|
||||
[AC_MSG_RESULT([no])])
|
||||
|
||||
if test $atomic_compare_exchange_n_128_result = 0 ; then
|
||||
CFLAGS_save=$CFLAGS
|
||||
CFLAGS="$CFLAGS -mcx16"
|
||||
|
||||
AC_MSG_CHECKING([for __atomic builtin atomic compare-and-swap on 128-bit values with -mcx16 flag])
|
||||
AC_TRY_LINK([], [__int128 x = 0; __atomic_bool_compare_and_swap (&x, 0, 1);],
|
||||
[AC_MSG_RESULT([yes])
|
||||
atomic_compare_exchange_n_128_result=1
|
||||
CFLAGS_save="$CFLAGS"],
|
||||
[AC_MSG_RESULT([no])])
|
||||
|
||||
CFLAGS=$CFLAGS_save
|
||||
fi
|
||||
fi
|
||||
|
||||
AC_DEFINE_UNQUOTED([OPAL_HAVE_GCC_BUILTIN_CSWAP_INT128], [$atomic_compare_exchange_n_128_result],
|
||||
[Whether the __atomic builtin atomic compare and swap supports 128-bit values])
|
||||
|
||||
OPAL_VAR_SCOPE_POP
|
||||
])
|
||||
|
||||
AC_DEFUN([OPAL_CHECK_GCC_ATOMIC_BUILTINS], [
|
||||
AC_MSG_CHECKING([for __atomic builtin atomics])
|
||||
|
||||
AC_TRY_LINK([long tmp, old = 0;], [__atomic_thread_fence(__ATOMIC_SEQ_CST);
|
||||
__atomic_compare_exchange_n(&tmp, &old, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
|
||||
__atomic_add_fetch(&tmp, 1, __ATOMIC_RELAXED);],
|
||||
[AC_MSG_RESULT([yes])
|
||||
$1],
|
||||
[AC_MSG_RESULT([no])
|
||||
$2])
|
||||
|
||||
# Check for 128-bit support
|
||||
OPAL_CHECK_GCC_BUILTIN_CSWAP_INT128
|
||||
])
|
||||
|
||||
|
||||
dnl #################################################################
|
||||
dnl
|
||||
dnl OPAL_CHECK_ASM_TEXT
|
||||
@ -894,6 +972,9 @@ AC_DEFUN([OPAL_CONFIG_ASM],[
|
||||
[], [enable_osx_builtin_atomics="yes"])
|
||||
|
||||
opal_cv_asm_builtin="BUILTIN_NO"
|
||||
if test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes" ; then
|
||||
OPAL_CHECK_GCC_ATOMIC_BUILTINS([opal_cv_asm_builtin="BUILTIN_GCC"], [])
|
||||
fi
|
||||
if test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes" ; then
|
||||
OPAL_CHECK_SYNC_BUILTINS([opal_cv_asm_builtin="BUILTIN_SYNC"], [])
|
||||
fi
|
||||
@ -1032,7 +1113,7 @@ AC_MSG_ERROR([Can not continue.])
|
||||
opal_cv_asm_builtin="BUILTIN_NO"
|
||||
fi
|
||||
|
||||
if test "$opal_cv_asm_builtin" = "BUILTIN_SYNC" ; then
|
||||
if test "$opal_cv_asm_builtin" = "BUILTIN_SYNC" || test "$opal_cv_asm_builtin" = "BUILTIN_GCC" ; then
|
||||
AC_DEFINE([OPAL_C_GCC_INLINE_ASSEMBLY], [1],
|
||||
[Whether C compiler supports GCC style inline assembly])
|
||||
else
|
||||
|
||||
@ -36,3 +36,4 @@ include opal/sys/osx/Makefile.am
|
||||
include opal/sys/powerpc/Makefile.am
|
||||
include opal/sys/sparcv9/Makefile.am
|
||||
include opal/sys/sync_builtin/Makefile.am
|
||||
include opal/sys/gcc_builtin/Makefile.am
|
||||
|
||||
@ -39,7 +39,8 @@
|
||||
#define OPAL_ARM 0100
|
||||
#define OPAL_BUILTIN_SYNC 0200
|
||||
#define OPAL_BUILTIN_OSX 0201
|
||||
#define OPAL_BUILTIN_NO 0202
|
||||
#define OPAL_BUILTIN_GCC 0202
|
||||
#define OPAL_BUILTIN_NO 0203
|
||||
|
||||
/* Formats */
|
||||
#define OPAL_DEFAULT 1000 /* standard for given architecture */
|
||||
|
||||
@ -131,6 +131,14 @@ typedef struct opal_atomic_lock_t opal_atomic_lock_t;
|
||||
#define OPAL_HAVE_INLINE_ATOMIC_SWAP_64 1
|
||||
#endif
|
||||
|
||||
/**
|
||||
* Enumeration of lock states
|
||||
*/
|
||||
enum {
|
||||
OPAL_ATOMIC_UNLOCKED = 0,
|
||||
OPAL_ATOMIC_LOCKED = 1
|
||||
};
|
||||
|
||||
/**********************************************************************
|
||||
*
|
||||
* Load the appropriate architecture files and set some reasonable
|
||||
@ -141,6 +149,8 @@ typedef struct opal_atomic_lock_t opal_atomic_lock_t;
|
||||
/* don't include system-level gorp when generating doxygen files */
|
||||
#elif OPAL_ASSEMBLY_BUILTIN == OPAL_BUILTIN_SYNC
|
||||
#include "opal/sys/sync_builtin/atomic.h"
|
||||
#elif OPAL_ASSEMBLY_BUILTIN == OPAL_BUILTIN_GCC
|
||||
#include "opal/sys/gcc_builtin/atomic.h"
|
||||
#elif OPAL_ASSEMBLY_BUILTIN == OPAL_BUILTIN_OSX
|
||||
#include "opal/sys/osx/atomic.h"
|
||||
#elif OPAL_ASSEMBLY_ARCH == OPAL_AMD64
|
||||
@ -264,15 +274,6 @@ void opal_atomic_wmb(void);
|
||||
|
||||
#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_SPINLOCKS || (OPAL_HAVE_ATOMIC_CMPSET_32 || OPAL_HAVE_ATOMIC_CMPSET_64)
|
||||
|
||||
/**
|
||||
* Enumeration of lock states
|
||||
*/
|
||||
enum {
|
||||
OPAL_ATOMIC_UNLOCKED = 0,
|
||||
OPAL_ATOMIC_LOCKED = 1
|
||||
};
|
||||
|
||||
|
||||
/**
|
||||
* Initialize a lock to value
|
||||
*
|
||||
|
||||
25
opal/include/opal/sys/gcc_builtin/Makefile.am
Обычный файл
25
opal/include/opal/sys/gcc_builtin/Makefile.am
Обычный файл
@ -0,0 +1,25 @@
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2011 Sandia National Laboratories. All rights reserved.
|
||||
# Copyright (c) 2016 Los Alamos National Security, LLC. All rights
|
||||
# reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
# This makefile.am does not stand on its own - it is included from opal/include/Makefile.am
|
||||
|
||||
headers += \
|
||||
opal/sys/gcc_builtin/atomic.h
|
||||
202
opal/include/opal/sys/gcc_builtin/atomic.h
Обычный файл
202
opal/include/opal/sys/gcc_builtin/atomic.h
Обычный файл
@ -0,0 +1,202 @@
|
||||
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2013 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2011 Sandia National Laboratories. All rights reserved.
|
||||
* Copyright (c) 2014-2016 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#ifndef OPAL_SYS_ARCH_ATOMIC_H
|
||||
#define OPAL_SYS_ARCH_ATOMIC_H 1
|
||||
|
||||
#include <stdbool.h>
|
||||
|
||||
/**********************************************************************
|
||||
*
|
||||
* Memory Barriers
|
||||
*
|
||||
*********************************************************************/
|
||||
#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_MATH_32 1
|
||||
#define OPAL_HAVE_ATOMIC_CMPSET_32 1
|
||||
#define OPAL_HAVE_ATOMIC_ADD_32 1
|
||||
#define OPAL_HAVE_ATOMIC_SUB_32 1
|
||||
#define OPAL_HAVE_ATOMIC_SWAP_32 1
|
||||
#define OPAL_HAVE_ATOMIC_MATH_64 1
|
||||
#define OPAL_HAVE_ATOMIC_CMPSET_64 1
|
||||
#define OPAL_HAVE_ATOMIC_ADD_64 1
|
||||
#define OPAL_HAVE_ATOMIC_SUB_64 1
|
||||
#define OPAL_HAVE_ATOMIC_SWAP_64 1
|
||||
|
||||
|
||||
static inline void opal_atomic_mb(void)
|
||||
{
|
||||
__atomic_thread_fence (__ATOMIC_SEQ_CST);
|
||||
}
|
||||
|
||||
static inline void opal_atomic_rmb(void)
|
||||
{
|
||||
__atomic_thread_fence (__ATOMIC_ACQUIRE);
|
||||
}
|
||||
|
||||
static inline void opal_atomic_wmb(void)
|
||||
{
|
||||
__atomic_thread_fence (__ATOMIC_RELEASE);
|
||||
}
|
||||
|
||||
#define MB() opal_atomic_mb()
|
||||
|
||||
/**********************************************************************
|
||||
*
|
||||
* Atomic math operations
|
||||
*
|
||||
*********************************************************************/
|
||||
|
||||
static inline int opal_atomic_cmpset_acq_32( volatile int32_t *addr,
|
||||
int32_t oldval, int32_t newval)
|
||||
{
|
||||
return __atomic_compare_exchange_n (addr, &oldval, newval, false,
|
||||
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
|
||||
static inline int opal_atomic_cmpset_rel_32( volatile int32_t *addr,
|
||||
int32_t oldval, int32_t newval)
|
||||
{
|
||||
return __atomic_compare_exchange_n (addr, &oldval, newval, false,
|
||||
__ATOMIC_RELEASE, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline int opal_atomic_cmpset_32( volatile int32_t *addr,
|
||||
int32_t oldval, int32_t newval)
|
||||
{
|
||||
return __atomic_compare_exchange_n (addr, &oldval, newval, false,
|
||||
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline int32_t opal_atomic_swap_32 (volatile int32_t *addr, int32_t newval)
|
||||
{
|
||||
int32_t oldval;
|
||||
__atomic_exchange (addr, &newval, &oldval, __ATOMIC_RELAXED);
|
||||
return oldval;
|
||||
}
|
||||
|
||||
static inline int32_t opal_atomic_add_32(volatile int32_t *addr, int32_t delta)
|
||||
{
|
||||
return __atomic_add_fetch (addr, delta, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline int32_t opal_atomic_sub_32(volatile int32_t *addr, int32_t delta)
|
||||
{
|
||||
return __atomic_sub_fetch (addr, delta, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline int opal_atomic_cmpset_acq_64( volatile int64_t *addr,
|
||||
int64_t oldval, int64_t newval)
|
||||
{
|
||||
return __atomic_compare_exchange_n (addr, &oldval, newval, false,
|
||||
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline int opal_atomic_cmpset_rel_64( volatile int64_t *addr,
|
||||
int64_t oldval, int64_t newval)
|
||||
{
|
||||
return __atomic_compare_exchange_n (addr, &oldval, newval, false,
|
||||
__ATOMIC_RELEASE, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
|
||||
static inline int opal_atomic_cmpset_64( volatile int64_t *addr,
|
||||
int64_t oldval, int64_t newval)
|
||||
{
|
||||
return __atomic_compare_exchange_n (addr, &oldval, newval, false,
|
||||
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline int64_t opal_atomic_swap_64 (volatile int64_t *addr, int64_t newval)
|
||||
{
|
||||
int64_t oldval;
|
||||
__atomic_exchange (addr, &newval, &oldval, __ATOMIC_RELAXED);
|
||||
return oldval;
|
||||
}
|
||||
|
||||
static inline int64_t opal_atomic_add_64(volatile int64_t *addr, int64_t delta)
|
||||
{
|
||||
return __atomic_add_fetch (addr, delta, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline int64_t opal_atomic_sub_64(volatile int64_t *addr, int64_t delta)
|
||||
{
|
||||
return __atomic_sub_fetch (addr, delta, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
#if OPAL_HAVE_GCC_BUILTIN_CSWAP_INT128
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_CMPSET_128 1
|
||||
|
||||
static inline int opal_atomic_cmpset_128 (volatile opal_int128_t *addr,
|
||||
opal_int128_t oldval, opal_int128_t newval)
|
||||
{
|
||||
return __atomic_compare_exchange_n (addr, &oldval, newval, false,
|
||||
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#if defined(__HLE__)
|
||||
|
||||
#include <immintrin.h>
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_SPINLOCKS 1
|
||||
|
||||
static inline void opal_atomic_init (opal_atomic_lock_t* lock, int32_t value)
|
||||
{
|
||||
lock->u.lock = value;
|
||||
}
|
||||
|
||||
static inline int opal_atomic_trylock(opal_atomic_lock_t *lock)
|
||||
{
|
||||
int ret = __atomic_exchange_n (&lock->u.lock, OPAL_ATOMIC_LOCKED,
|
||||
__ATOMIC_ACQUIRE | __ATOMIC_HLE_ACQUIRE);
|
||||
if (OPAL_ATOMIC_LOCKED == ret) {
|
||||
/* abort the transaction */
|
||||
_mm_pause ();
|
||||
return 1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static inline void opal_atomic_lock (opal_atomic_lock_t *lock)
|
||||
{
|
||||
while (OPAL_ATOMIC_LOCKED == __atomic_exchange_n (&lock->u.lock, OPAL_ATOMIC_LOCKED,
|
||||
__ATOMIC_ACQUIRE | __ATOMIC_HLE_ACQUIRE)) {
|
||||
/* abort the transaction */
|
||||
_mm_pause ();
|
||||
}
|
||||
}
|
||||
|
||||
static inline void opal_atomic_unlock (opal_atomic_lock_t *lock)
|
||||
{
|
||||
__atomic_store_n (&lock->u.lock, OPAL_ATOMIC_UNLOCKED,
|
||||
__ATOMIC_RELEASE | __ATOMIC_HLE_RELEASE);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#endif /* ! OPAL_SYS_ARCH_ATOMIC_H */
|
||||
Загрузка…
x
Ссылка в новой задаче
Block a user