opal/atomic: always use C11 atomics if available
This commit disables the use of both the builtin and hand-written atomics if proper C11 atomic support is detected. This is the first step towards requiring the availability of C11 atomics for the C compiler used to build Open MPI. Signed-off-by: Nathan Hjelm <hjelmn@lanl.gov>
Этот коммит содержится в:
Nathan Hjelm
родитель
000f9eed4d
Коммит
fe6528b0d5
@ -13,7 +13,7 @@ dnl Copyright (c) 2008-2018 Cisco Systems, Inc. All rights reserved.
|
||||
dnl Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved.
|
||||
dnl Copyright (c) 2015-2017 Research Organization for Information Science
|
||||
dnl and Technology (RIST). All rights reserved.
|
||||
dnl Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights
|
||||
dnl Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights
|
||||
dnl reserved.
|
||||
dnl Copyright (c) 2017 Amazon.com, Inc. or its affiliates. All Rights
|
||||
dnl reserved.
|
||||
@ -122,6 +122,58 @@ int main(int argc, char** argv)
|
||||
}
|
||||
]])
|
||||
|
||||
dnl This is a C test to see if 128-bit __atomic_compare_exchange_n()
|
||||
dnl actually works (e.g., it compiles and links successfully on
|
||||
dnl ARM64+clang, but returns incorrect answers as of August 2018).
|
||||
AC_DEFUN([OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_TEST_SOURCE],[[
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdlib.h>
|
||||
#include <stdatomic.h>
|
||||
|
||||
typedef union {
|
||||
uint64_t fake@<:@2@:>@;
|
||||
_Atomic __int128 real;
|
||||
} ompi128;
|
||||
|
||||
static void test1(void)
|
||||
{
|
||||
// As of Aug 2018, we could not figure out a way to assign 128-bit
|
||||
// constants -- the compilers would not accept it. So use a fake
|
||||
// union to assign 2 uin64_t's to make a single __int128.
|
||||
ompi128 ptr = { .fake = { 0xFFEEDDCCBBAA0099, 0x8877665544332211 }};
|
||||
ompi128 expected = { .fake = { 0x11EEDDCCBBAA0099, 0x88776655443322FF }};
|
||||
ompi128 desired = { .fake = { 0x1122DDCCBBAA0099, 0x887766554433EEFF }};
|
||||
bool r = atomic_compare_exchange_strong (&ptr.real, &expected.real,
|
||||
desired.real, true,
|
||||
atomic_relaxed, atomic_relaxed);
|
||||
if ( !(r == false && ptr.real == expected.real)) {
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
static void test2(void)
|
||||
{
|
||||
ompi128 ptr = { .fake = { 0xFFEEDDCCBBAA0099, 0x8877665544332211 }};
|
||||
ompi128 expected = ptr;
|
||||
ompi128 desired = { .fake = { 0x1122DDCCBBAA0099, 0x887766554433EEFF }};
|
||||
bool r = atomic_compare_exchange_strong (&ptr.real, &expected.real,
|
||||
desired.real, true,
|
||||
atomic_relaxed, atomic_relaxed);
|
||||
if (!(r == true && ptr.real == desired.real)) {
|
||||
exit(2);
|
||||
}
|
||||
}
|
||||
|
||||
vvvvvvvvvvvvvvvvvvvv
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
test1();
|
||||
test2();
|
||||
return 0;
|
||||
}
|
||||
]])
|
||||
|
||||
dnl ------------------------------------------------------------------
|
||||
|
||||
dnl
|
||||
@ -329,6 +381,71 @@ __atomic_add_fetch(&tmp64, 1, __ATOMIC_RELAXED);],
|
||||
OPAL_CHECK_GCC_BUILTIN_CSWAP_INT128
|
||||
])
|
||||
|
||||
AC_DEFUN([OPAL_CHECK_C11_CSWAP_INT128], [
|
||||
OPAL_VAR_SCOPE_PUSH([atomic_compare_exchange_result atomic_compare_exchange_CFLAGS_save atomic_compare_exchange_LIBS_save])
|
||||
|
||||
atomic_compare_exchange_CFLAGS_save=$CFLAGS
|
||||
atomic_compare_exchange_LIBS_save=$LIBS
|
||||
|
||||
# Do we have C11 atomics on 128-bit integers?
|
||||
# Use a special macro because we need to check with a few different
|
||||
# CFLAGS/LIBS.
|
||||
OPAL_ASM_CHECK_ATOMIC_FUNC([atomic_compare_exchange_strong_16],
|
||||
[AC_LANG_SOURCE(OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_TEST_SOURCE)],
|
||||
[atomic_compare_exchange_result=1],
|
||||
[atomic_compare_exchange_result=0])
|
||||
|
||||
# If we have it and it works, check to make sure it is always lock
|
||||
# free.
|
||||
AS_IF([test $atomic_compare_exchange_result -eq 1],
|
||||
[AC_MSG_CHECKING([if C11 __int128 atomic compare-and-swap is always lock-free])
|
||||
AC_RUN_IFELSE([AC_LANG_PROGRAM([#include <stdatomic.h>], [_Atomic __int128_t x; if (!atomic_is_lock_free(&x)) { return 1; }])],
|
||||
[AC_MSG_RESULT([yes])],
|
||||
[atomic_compare_exchange_result=0
|
||||
# If this test fails, need to reset CFLAGS/LIBS (the
|
||||
# above tests atomically set CFLAGS/LIBS or not; this
|
||||
# test is running after the fact, so we have to undo
|
||||
# the side-effects of setting CFLAGS/LIBS if the above
|
||||
# tests passed).
|
||||
CFLAGS=$atomic_compare_exchange_CFLAGS_save
|
||||
LIBS=$atomic_compare_exchange_LIBS_save
|
||||
AC_MSG_RESULT([no])],
|
||||
[AC_MSG_RESULT([cannot test -- assume yes (cross compiling)])])
|
||||
])
|
||||
|
||||
AC_DEFINE_UNQUOTED([OPAL_HAVE_C11_CSWAP_INT128],
|
||||
[$atomic_compare_exchange_result],
|
||||
[Whether C11 atomic compare swap is both supported and lock-free on 128-bit values])
|
||||
|
||||
dnl If we could not find decent support for 128-bits atomic let's
|
||||
dnl try the GCC _sync
|
||||
AS_IF([test $atomic_compare_exchange_result -eq 0],
|
||||
[OPAL_CHECK_SYNC_BUILTIN_CSWAP_INT128])
|
||||
|
||||
OPAL_VAR_SCOPE_POP
|
||||
])
|
||||
|
||||
AC_DEFUN([OPAL_CHECK_GCC_ATOMIC_BUILTINS], [
|
||||
AC_MSG_CHECKING([for __atomic builtin atomics])
|
||||
|
||||
AC_TRY_LINK([
|
||||
#include <stdint.h>
|
||||
uint32_t tmp, old = 0;
|
||||
uint64_t tmp64, old64 = 0;], [
|
||||
__atomic_thread_fence(__ATOMIC_SEQ_CST);
|
||||
__atomic_compare_exchange_n(&tmp, &old, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
|
||||
__atomic_add_fetch(&tmp, 1, __ATOMIC_RELAXED);
|
||||
__atomic_compare_exchange_n(&tmp64, &old64, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
|
||||
__atomic_add_fetch(&tmp64, 1, __ATOMIC_RELAXED);],
|
||||
[AC_MSG_RESULT([yes])
|
||||
$1],
|
||||
[AC_MSG_RESULT([no])
|
||||
$2])
|
||||
|
||||
# Check for 128-bit support
|
||||
OPAL_CHECK_GCC_BUILTIN_CSWAP_INT128
|
||||
])
|
||||
|
||||
|
||||
dnl #################################################################
|
||||
dnl
|
||||
@ -1020,17 +1137,27 @@ AC_DEFUN([OPAL_CONFIG_ASM],[
|
||||
AC_REQUIRE([OPAL_SETUP_CC])
|
||||
AC_REQUIRE([AM_PROG_AS])
|
||||
|
||||
AC_ARG_ENABLE([c11-atomics],[AC_HELP_STRING([--enable-c11-atomics],
|
||||
[Enable use of C11 atomics if available (default: enabled)])])
|
||||
|
||||
AC_ARG_ENABLE([builtin-atomics],
|
||||
[AC_HELP_STRING([--enable-builtin-atomics],
|
||||
[Enable use of __sync builtin atomics (default: enabled)])])
|
||||
[Enable use of __sync builtin atomics (default: disabled)])])
|
||||
|
||||
opal_cv_asm_builtin="BUILTIN_NO"
|
||||
AS_IF([test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" != "no"],
|
||||
[OPAL_CHECK_GCC_ATOMIC_BUILTINS([opal_cv_asm_builtin="BUILTIN_GCC"], [])])
|
||||
AS_IF([test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" != "no"],
|
||||
[OPAL_CHECK_SYNC_BUILTINS([opal_cv_asm_builtin="BUILTIN_SYNC"], [])])
|
||||
AS_IF([test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes"],
|
||||
[AC_MSG_ERROR([__sync builtin atomics requested but not found.])])
|
||||
OPAL_CHECK_C11_CSWAP_INT128
|
||||
|
||||
if test "x$enable_c11_atomics" != "xno" && test "$opal_cv_c11_supported" = "yes" ; then
|
||||
opal_cv_asm_builtin="BUILTIN_C11"
|
||||
OPAL_CHECK_C11_CSWAP_INT128
|
||||
else
|
||||
opal_cv_asm_builtin="BUILTIN_NO"
|
||||
AS_IF([test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes"],
|
||||
[OPAL_CHECK_GCC_ATOMIC_BUILTINS([opal_cv_asm_builtin="BUILTIN_GCC"], [])])
|
||||
AS_IF([test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes"],
|
||||
[OPAL_CHECK_SYNC_BUILTINS([opal_cv_asm_builtin="BUILTIN_SYNC"], [])])
|
||||
AS_IF([test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes"],
|
||||
[AC_MSG_ERROR([__sync builtin atomics requested but not found.])])
|
||||
fi
|
||||
|
||||
OPAL_CHECK_ASM_PROC
|
||||
OPAL_CHECK_ASM_TEXT
|
||||
|
||||
@ -155,11 +155,9 @@ static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo)
|
||||
* update the head */
|
||||
|
||||
/* wait for next pointer to be updated by push */
|
||||
while (ghost == item->opal_list_next) {
|
||||
do {
|
||||
opal_atomic_rmb ();
|
||||
}
|
||||
|
||||
opal_atomic_rmb ();
|
||||
} while (ghost == item->opal_list_next);
|
||||
|
||||
/* update the head with the real next value. note that no other thread
|
||||
* will be attempting to update the head until after it has been updated
|
||||
|
||||
@ -55,7 +55,7 @@ int opal_class_init_epoch = 1;
|
||||
/*
|
||||
* Local variables
|
||||
*/
|
||||
static opal_atomic_lock_t class_lock = { { OPAL_ATOMIC_LOCK_UNLOCKED } };
|
||||
static opal_atomic_lock_t class_lock = OPAL_ATOMIC_LOCK_INIT;
|
||||
static void** classes = NULL;
|
||||
static int num_classes = 0;
|
||||
static int max_classes = 0;
|
||||
|
||||
@ -11,7 +11,7 @@
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
|
||||
# Copyright (c) 2011 Sandia National Laboratories. All rights reserved.
|
||||
# Copyright (c) 2016 Los Alamos National Security, LLC. All rights
|
||||
# Copyright (c) 2016-2018 Los Alamos National Security, LLC. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2017 Research Organization for Information Science
|
||||
# and Technology (RIST). All rights reserved.
|
||||
@ -27,6 +27,7 @@
|
||||
headers += \
|
||||
opal/sys/architecture.h \
|
||||
opal/sys/atomic.h \
|
||||
opal/sys/atomic_stdc.h \
|
||||
opal/sys/atomic_impl.h \
|
||||
opal/sys/timer.h \
|
||||
opal/sys/cma.h
|
||||
|
||||
@ -47,6 +47,7 @@
|
||||
#define OPAL_BUILTIN_SYNC 0200
|
||||
#define OPAL_BUILTIN_GCC 0202
|
||||
#define OPAL_BUILTIN_NO 0203
|
||||
#define OPAL_BUILTIN_C11 0204
|
||||
|
||||
/* Formats */
|
||||
#define OPAL_DEFAULT 1000 /* standard for given architecture */
|
||||
|
||||
@ -58,6 +58,12 @@
|
||||
#include "opal/sys/architecture.h"
|
||||
#include "opal_stdatomic.h"
|
||||
|
||||
#if OPAL_ASSEMBLY_BUILTIN == OPAL_BUILTIN_C11
|
||||
|
||||
#include "atomic_stdc.h"
|
||||
|
||||
#else /* !OPAL_C_HAVE__ATOMIC */
|
||||
|
||||
/* do some quick #define cleanup in cases where we are doing
|
||||
testing... */
|
||||
#ifdef OPAL_DISABLE_INLINE_ASM
|
||||
@ -147,6 +153,8 @@ enum {
|
||||
OPAL_ATOMIC_LOCK_LOCKED = 1
|
||||
};
|
||||
|
||||
#define OPAL_ATOMIC_LOCK_INIT {.u = {.lock = OPAL_ATOMIC_LOCK_UNLOCKED}}
|
||||
|
||||
/**********************************************************************
|
||||
*
|
||||
* Load the appropriate architecture files and set some reasonable
|
||||
@ -643,6 +651,8 @@ static inline intptr_t opal_atomic_fetch_sub_ptr( opal_atomic_intptr_t* addr, vo
|
||||
*/
|
||||
#include "opal/sys/atomic_impl.h"
|
||||
|
||||
#endif /* !OPAL_C_HAVE__ATOMIC */
|
||||
|
||||
END_C_DECLS
|
||||
|
||||
#endif /* OPAL_SYS_ATOMIC_H */
|
||||
|
||||
261
opal/include/opal/sys/atomic_stdc.h
Обычный файл
261
opal/include/opal/sys/atomic_stdc.h
Обычный файл
@ -0,0 +1,261 @@
|
||||
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
||||
/*
|
||||
* Copyright (c) 2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
/* This file provides shims between the opal atomics interface and the C11 atomics interface. It
|
||||
* is intended as the first step in moving to using C11 atomics across the entire codebase. Once
|
||||
* all officially supported compilers offer C11 atomic (GCC 4.9.0+, icc 2018+, pgi, xlc, etc) then
|
||||
* this shim will go away and the codebase will be updated to use C11's atomic support
|
||||
* directly.
|
||||
* This shim contains some functions already present in atomic_impl.h because we do not include
|
||||
* atomic_impl.h when using C11 atomics. It would require alot of #ifdefs to avoid duplicate
|
||||
* definitions to be worthwhile. */
|
||||
|
||||
#if !defined(OPAL_ATOMIC_STDC_H)
|
||||
#define OPAL_ATOMIC_STDC_H
|
||||
|
||||
#include <stdatomic.h>
|
||||
#include <stdint.h>
|
||||
#include "opal/include/opal_stdint.h"
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
|
||||
#define OPAL_HAVE_ATOMIC_SWAP_32 1
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_MATH_32 1
|
||||
#define OPAL_HAVE_ATOMIC_ADD_32 1
|
||||
#define OPAL_HAVE_ATOMIC_AND_32 1
|
||||
#define OPAL_HAVE_ATOMIC_OR_32 1
|
||||
#define OPAL_HAVE_ATOMIC_XOR_32 1
|
||||
#define OPAL_HAVE_ATOMIC_SUB_32 1
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
|
||||
#define OPAL_HAVE_ATOMIC_SWAP_64 1
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_MATH_64 1
|
||||
#define OPAL_HAVE_ATOMIC_ADD_64 1
|
||||
#define OPAL_HAVE_ATOMIC_AND_64 1
|
||||
#define OPAL_HAVE_ATOMIC_OR_64 1
|
||||
#define OPAL_HAVE_ATOMIC_XOR_64 1
|
||||
#define OPAL_HAVE_ATOMIC_SUB_64 1
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_LLSC_32 0
|
||||
#define OPAL_HAVE_ATOMIC_LLSC_64 0
|
||||
#define OPAL_HAVE_ATOMIC_LLSC_PTR 0
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_MIN_32 1
|
||||
#define OPAL_HAVE_ATOMIC_MAX_32 1
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_MIN_64 1
|
||||
#define OPAL_HAVE_ATOMIC_MAX_64 1
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_SPINLOCKS 1
|
||||
|
||||
static inline void opal_atomic_mb (void)
|
||||
{
|
||||
atomic_thread_fence (memory_order_seq_cst);
|
||||
}
|
||||
|
||||
static inline void opal_atomic_wmb (void)
|
||||
{
|
||||
atomic_thread_fence (memory_order_release);
|
||||
}
|
||||
|
||||
static inline void opal_atomic_rmb (void)
|
||||
{
|
||||
atomic_thread_fence (memory_order_acquire);
|
||||
}
|
||||
|
||||
#define opal_atomic_compare_exchange_strong_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_relaxed, memory_order_relaxed)
|
||||
#define opal_atomic_compare_exchange_strong_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_relaxed, memory_order_relaxed)
|
||||
#define opal_atomic_compare_exchange_strong_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_relaxed, memory_order_relaxed)
|
||||
#define opal_atomic_compare_exchange_strong_acq_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed)
|
||||
#define opal_atomic_compare_exchange_strong_acq_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed)
|
||||
#define opal_atomic_compare_exchange_strong_acq_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed)
|
||||
|
||||
#define opal_atomic_compare_exchange_strong_rel_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed)
|
||||
#define opal_atomic_compare_exchange_strong_rel_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed)
|
||||
#define opal_atomic_compare_exchange_strong_rel_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed)
|
||||
|
||||
#define opal_atomic_compare_exchange_strong(addr, oldval, newval) atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_relaxed, memory_order_relaxed)
|
||||
#define opal_atomic_compare_exchange_strong_acq(addr, oldval, newval) atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_acquire, memory_order_relaxed)
|
||||
#define opal_atomic_compare_exchange_strong_rel(addr, oldval, newval) atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_release, memory_order_relaxed)
|
||||
|
||||
#define opal_atomic_swap_32(addr, value) atomic_exchange_explicit (addr, value, memory_order_relaxed)
|
||||
#define opal_atomic_swap_64(addr, value) atomic_exchange_explicit (addr, value, memory_order_relaxed)
|
||||
#define opal_atomic_swap_ptr(addr, value) atomic_exchange_explicit (addr, value, memory_order_relaxed)
|
||||
|
||||
#define OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(op, bits, type, operator) \
|
||||
static inline type opal_atomic_fetch_ ## op ##_## bits (opal_atomic_ ## type *addr, type value) \
|
||||
{ \
|
||||
return atomic_fetch_ ## op ## _explicit (addr, value, memory_order_relaxed); \
|
||||
} \
|
||||
\
|
||||
static inline type opal_atomic_## op ## _fetch_ ## bits (opal_atomic_ ## type *addr, type value) \
|
||||
{ \
|
||||
return atomic_fetch_ ## op ## _explicit (addr, value, memory_order_relaxed) operator value; \
|
||||
}
|
||||
|
||||
OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, 32, int32_t, +)
|
||||
OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, 64, int64_t, +)
|
||||
OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, size_t, size_t, +)
|
||||
|
||||
OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, 32, int32_t, -)
|
||||
OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, 64, int64_t, -)
|
||||
OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, size_t, size_t, -)
|
||||
|
||||
OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(or, 32, int32_t, |)
|
||||
OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(or, 64, int64_t, |)
|
||||
|
||||
OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(xor, 32, int32_t, ^)
|
||||
OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(xor, 64, int64_t, ^)
|
||||
|
||||
OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(and, 32, int32_t, &)
|
||||
OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(and, 64, int64_t, &)
|
||||
|
||||
#define opal_atomic_add(addr, value) (void) atomic_fetch_add_explicit (addr, value, memory_order_relaxed)
|
||||
|
||||
static inline int32_t opal_atomic_fetch_min_32 (opal_atomic_int32_t *addr, int32_t value)
|
||||
{
|
||||
int32_t old = *addr;
|
||||
do {
|
||||
if (old <= value) {
|
||||
break;
|
||||
}
|
||||
} while (!opal_atomic_compare_exchange_strong_32 (addr, &old, value));
|
||||
|
||||
return old;
|
||||
}
|
||||
|
||||
static inline int32_t opal_atomic_fetch_max_32 (opal_atomic_int32_t *addr, int32_t value)
|
||||
{
|
||||
int32_t old = *addr;
|
||||
do {
|
||||
if (old >= value) {
|
||||
break;
|
||||
}
|
||||
} while (!opal_atomic_compare_exchange_strong_32 (addr, &old, value));
|
||||
|
||||
return old;
|
||||
}
|
||||
|
||||
static inline int64_t opal_atomic_fetch_min_64 (opal_atomic_int64_t *addr, int64_t value)
|
||||
{
|
||||
int64_t old = *addr;
|
||||
do {
|
||||
if (old <= value) {
|
||||
break;
|
||||
}
|
||||
} while (!opal_atomic_compare_exchange_strong_64 (addr, &old, value));
|
||||
|
||||
return old;
|
||||
}
|
||||
|
||||
static inline int64_t opal_atomic_fetch_max_64 (opal_atomic_int64_t *addr, int64_t value)
|
||||
{
|
||||
int64_t old = *addr;
|
||||
do {
|
||||
if (old >= value) {
|
||||
break;
|
||||
}
|
||||
} while (!opal_atomic_compare_exchange_strong_64 (addr, &old, value));
|
||||
|
||||
return old;
|
||||
}
|
||||
|
||||
static inline int32_t opal_atomic_min_fetch_32 (opal_atomic_int32_t *addr, int32_t value)
|
||||
{
|
||||
int32_t old = opal_atomic_fetch_min_32 (addr, value);
|
||||
return old <= value ? old : value;
|
||||
}
|
||||
|
||||
static inline int32_t opal_atomic_max_fetch_32 (opal_atomic_int32_t *addr, int32_t value)
|
||||
{
|
||||
int32_t old = opal_atomic_fetch_max_32 (addr, value);
|
||||
return old >= value ? old : value;
|
||||
}
|
||||
|
||||
static inline int64_t opal_atomic_min_fetch_64 (opal_atomic_int64_t *addr, int64_t value)
|
||||
{
|
||||
int64_t old = opal_atomic_fetch_min_64 (addr, value);
|
||||
return old <= value ? old : value;
|
||||
}
|
||||
|
||||
static inline int64_t opal_atomic_max_fetch_64 (opal_atomic_int64_t *addr, int64_t value)
|
||||
{
|
||||
int64_t old = opal_atomic_fetch_max_64 (addr, value);
|
||||
return old >= value ? old : value;
|
||||
}
|
||||
|
||||
#define OPAL_ATOMIC_LOCK_UNLOCKED false
|
||||
#define OPAL_ATOMIC_LOCK_LOCKED true
|
||||
|
||||
#define OPAL_ATOMIC_LOCK_INIT ATOMIC_FLAG_INIT
|
||||
|
||||
typedef atomic_flag opal_atomic_lock_t;
|
||||
|
||||
/*
|
||||
* Lock initialization function. It set the lock to UNLOCKED.
|
||||
*/
|
||||
static inline void opal_atomic_lock_init (opal_atomic_lock_t *lock, bool value)
|
||||
{
|
||||
atomic_flag_clear (lock);
|
||||
}
|
||||
|
||||
|
||||
static inline int opal_atomic_trylock (opal_atomic_lock_t *lock)
|
||||
{
|
||||
return (int) atomic_flag_test_and_set (lock);
|
||||
}
|
||||
|
||||
|
||||
static inline void opal_atomic_lock(opal_atomic_lock_t *lock)
|
||||
{
|
||||
while (opal_atomic_trylock (lock)) {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static inline void opal_atomic_unlock (opal_atomic_lock_t *lock)
|
||||
{
|
||||
atomic_flag_clear (lock);
|
||||
}
|
||||
|
||||
|
||||
#if OPAL_HAVE_C11_CSWAP_INT128
|
||||
|
||||
/* the C11 atomic compare-exchange is lock free so use it */
|
||||
#define opal_atomic_compare_exchange_strong_128 atomic_compare_exchange_strong
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
|
||||
|
||||
#elif OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128
|
||||
|
||||
/* fall back on the __sync builtin if available since it will emit the expected instruction on x86_64 (cmpxchng16b) */
|
||||
__opal_attribute_always_inline__
|
||||
static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_int128_t *addr,
|
||||
opal_int128_t *oldval, opal_int128_t newval)
|
||||
{
|
||||
opal_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval);
|
||||
bool ret = prev == *oldval;
|
||||
*oldval = prev;
|
||||
return ret;
|
||||
}
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
|
||||
|
||||
#else
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 0
|
||||
|
||||
#endif
|
||||
|
||||
#endif /* !defined(OPAL_ATOMIC_STDC_H) */
|
||||
@ -149,7 +149,7 @@ static inline int64_t opal_atomic_fetch_sub_64(opal_atomic_int64_t *addr, int64_
|
||||
#endif
|
||||
|
||||
#if OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128
|
||||
static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_opal_int128_t *addr,
|
||||
static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_int128_t *addr,
|
||||
opal_int128_t *oldval, opal_int128_t newval)
|
||||
{
|
||||
opal_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval);
|
||||
|
||||
@ -14,6 +14,9 @@
|
||||
|
||||
#include "opal_stdint.h"
|
||||
|
||||
#if OPAL_ASSEMBLY_BUILTIN != OPAL_BUILTIN_C11
|
||||
|
||||
typedef volatile int opal_atomic_int_t;
|
||||
typedef volatile long opal_atomic_long_t;
|
||||
|
||||
typedef volatile int32_t opal_atomic_int32_t;
|
||||
@ -22,14 +25,42 @@ typedef volatile int64_t opal_atomic_int64_t;
|
||||
typedef volatile uint64_t opal_atomic_uint64_t;
|
||||
|
||||
typedef volatile size_t opal_atomic_size_t;
|
||||
typedef volatile ssize_t opal_atomic_ssize_t;
|
||||
typedef volatile intptr_t opal_atomic_intptr_t;
|
||||
typedef volatile uintptr_t opal_atomic_uintptr_t;
|
||||
|
||||
#else /* OPAL_HAVE_C__ATOMIC */
|
||||
|
||||
#include <stdatomic.h>
|
||||
|
||||
typedef atomic_int opal_atomic_int_t;
|
||||
typedef atomic_long opal_atomic_long_t;
|
||||
|
||||
typedef _Atomic int32_t opal_atomic_int32_t;
|
||||
typedef _Atomic uint32_t opal_atomic_uint32_t;
|
||||
typedef _Atomic int64_t opal_atomic_int64_t;
|
||||
typedef _Atomic uint64_t opal_atomic_uint64_t;
|
||||
|
||||
typedef _Atomic size_t opal_atomic_size_t;
|
||||
typedef _Atomic ssize_t opal_atomic_ssize_t;
|
||||
typedef _Atomic intptr_t opal_atomic_intptr_t;
|
||||
typedef _Atomic uintptr_t opal_atomic_uintptr_t;
|
||||
|
||||
#endif /* OPAL_HAVE_C__ATOMIC */
|
||||
|
||||
#if HAVE_OPAL_INT128_T
|
||||
|
||||
/* do not use C11 atomics for __int128 if they are not lock free */
|
||||
#if OPAL_HAVE_C11_CSWAP_INT128
|
||||
|
||||
typedef _Atomic opal_int128_t opal_atomic_int128_t;
|
||||
|
||||
#else
|
||||
|
||||
typedef volatile opal_int128_t opal_atomic_int128_t;
|
||||
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
#endif /* !defined(OPAL_STDATOMIC_H) */
|
||||
|
||||
@ -28,6 +28,8 @@
|
||||
#ifndef OPAL_STDINT_H
|
||||
#define OPAL_STDINT_H 1
|
||||
|
||||
#include "opal_config.h"
|
||||
|
||||
/*
|
||||
* Include what we can and define what is missing.
|
||||
*/
|
||||
|
||||
@ -76,14 +76,14 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_recursive_mutex_t);
|
||||
.m_lock_debug = 0, \
|
||||
.m_lock_file = NULL, \
|
||||
.m_lock_line = 0, \
|
||||
.m_lock_atomic = { .u = { .lock = OPAL_ATOMIC_LOCK_UNLOCKED } }, \
|
||||
.m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \
|
||||
}
|
||||
#else
|
||||
#define OPAL_MUTEX_STATIC_INIT \
|
||||
{ \
|
||||
.super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), \
|
||||
.m_lock_pthread = PTHREAD_MUTEX_INITIALIZER, \
|
||||
.m_lock_atomic = { .u = { .lock = OPAL_ATOMIC_LOCK_UNLOCKED } }, \
|
||||
.m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \
|
||||
}
|
||||
#endif
|
||||
|
||||
@ -97,14 +97,14 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_recursive_mutex_t);
|
||||
.m_lock_debug = 0, \
|
||||
.m_lock_file = NULL, \
|
||||
.m_lock_line = 0, \
|
||||
.m_lock_atomic = { .u = { .lock = OPAL_ATOMIC_LOCK_UNLOCKED } }, \
|
||||
.m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \
|
||||
}
|
||||
#else
|
||||
#define OPAL_RECURSIVE_MUTEX_STATIC_INIT \
|
||||
{ \
|
||||
.super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), \
|
||||
.m_lock_pthread = OPAL_PTHREAD_RECURSIVE_MUTEX_INITIALIZER, \
|
||||
.m_lock_atomic = { .u = { .lock = OPAL_ATOMIC_LOCK_UNLOCKED } }, \
|
||||
.m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
@ -25,12 +25,12 @@
|
||||
#include "orte/runtime/orte_locks.h"
|
||||
|
||||
/* for everyone */
|
||||
opal_atomic_lock_t orte_finalize_lock = {{0}};
|
||||
opal_atomic_lock_t orte_finalize_lock = OPAL_ATOMIC_LOCK_INIT;
|
||||
|
||||
/* for HNPs */
|
||||
opal_atomic_lock_t orte_abort_inprogress_lock = {{0}};
|
||||
opal_atomic_lock_t orte_jobs_complete_lock = {{0}};
|
||||
opal_atomic_lock_t orte_quit_lock = {{0}};
|
||||
opal_atomic_lock_t orte_abort_inprogress_lock = OPAL_ATOMIC_LOCK_INIT;
|
||||
opal_atomic_lock_t orte_jobs_complete_lock = OPAL_ATOMIC_LOCK_INIT;
|
||||
opal_atomic_lock_t orte_quit_lock = OPAL_ATOMIC_LOCK_INIT;
|
||||
|
||||
int orte_locks_init(void)
|
||||
{
|
||||
|
||||
Загрузка…
x
Ссылка в новой задаче
Block a user