Update PMIx atomics
Signed-off-by: Ralph Castain <rhc@open-mpi.org>
Этот коммит содержится в:
Ralph Castain
родитель
795140e590
Коммит
7981818b84
@ -30,7 +30,7 @@ greek=
|
||||
# command, or with the date (if "git describe" fails) in the form of
|
||||
# "date<date>".
|
||||
|
||||
repo_rev=git32969ba
|
||||
repo_rev=git88fa9a9
|
||||
|
||||
# If tarball_version is not empty, it is used as the version string in
|
||||
# the tarball filename, regardless of all other versions listed in
|
||||
@ -44,7 +44,7 @@ tarball_version=
|
||||
|
||||
# The date when this release was created
|
||||
|
||||
date="Jun 16, 2018"
|
||||
date="Jun 17, 2018"
|
||||
|
||||
# The shared library version of each of PMIx's public libraries.
|
||||
# These versions are maintained in accordance with the "Library
|
||||
|
||||
@ -16,7 +16,7 @@ dnl Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights
|
||||
dnl reserved.
|
||||
dnl Copyright (c) 2015 Research Organization for Information Science
|
||||
dnl and Technology (RIST). All rights reserved.
|
||||
dnl Copyright (c) 2015-2017 Intel, Inc. All rights reserved.
|
||||
dnl Copyright (c) 2018 Intel, Inc. All rights reserved.
|
||||
dnl $COPYRIGHT$
|
||||
dnl
|
||||
dnl Additional copyrights may follow
|
||||
@ -24,19 +24,145 @@ dnl
|
||||
dnl $HEADER$
|
||||
dnl
|
||||
|
||||
AC_DEFUN([PMIX_CC_HELPER],[
|
||||
PMIX_VAR_SCOPE_PUSH([pmix_prog_cc_c11_helper_tmp])
|
||||
AC_MSG_CHECKING([$1])
|
||||
|
||||
pmix_prog_cc_c11_helper_tmp=0
|
||||
|
||||
AC_LINK_IFELSE([AC_LANG_PROGRAM([$3],[$4])],[
|
||||
$2=yes
|
||||
pmix_prog_cc_c11_helper_tmp=1], [$2=no])
|
||||
|
||||
AC_DEFINE_UNQUOTED([$5], [$pmix_prog_cc_c11_helper_tmp], [$6])
|
||||
|
||||
AC_MSG_RESULT([$$2])
|
||||
PMIX_VAR_SCOPE_POP
|
||||
])
|
||||
|
||||
|
||||
AC_DEFUN([PMIX_PROG_CC_C11_HELPER],[
|
||||
PMIX_VAR_SCOPE_PUSH([pmix_prog_cc_c11_helper_CFLAGS_save pmix_prog_cc_c11_helper__Thread_local_available pmix_prog_cc_c11_helper_atomic_var_available pmix_prog_cc_c11_helper__Atomic_available pmix_prog_cc_c11_helper__static_assert_available pmix_prog_cc_c11_helper__Generic_available])
|
||||
|
||||
pmix_prog_cc_c11_helper_CFLAGS_save=$CFLAGS
|
||||
CFLAGS="$CFLAGS $1"
|
||||
|
||||
PMIX_CC_HELPER([if $CC $1 supports C11 _Thread_local], [pmix_prog_cc_c11_helper__Thread_local_available],
|
||||
[],[[static _Thread_local int foo = 1;++foo;]], [PMIX_C_HAVE__THREAD_LOCAL],
|
||||
[Whether C compiler supports __Thread_local])
|
||||
|
||||
PMIX_CC_HELPER([if $CC $1 supports C11 atomic variables], [pmix_prog_cc_c11_helper_atomic_var_available],
|
||||
[[#include <stdatomic.h>]], [[static atomic_long foo = 1;++foo;]], [PMIX_C_HAVE_ATOMIC_CONV_VAR],
|
||||
[Whether C compiler support atomic convenience variables in stdatomic.h])
|
||||
|
||||
PMIX_CC_HELPER([if $CC $1 supports C11 _Atomic keyword], [pmix_prog_cc_c11_helper__Atomic_available],
|
||||
[[#include <stdatomic.h>]],[[static _Atomic long foo = 1;++foo;]], [PMIX_C_HAVE__ATOMIC],
|
||||
[Whether C compiler supports __Atomic keyword])
|
||||
|
||||
PMIX_CC_HELPER([if $CC $1 supports C11 _Generic keyword], [pmix_prog_cc_c11_helper__Generic_available],
|
||||
[[#define FOO(x) (_Generic (x, int: 1))]], [[static int x, y; y = FOO(x);]], [PMIX_C_HAVE__GENERIC],
|
||||
[Whether C compiler supports __Generic keyword])
|
||||
|
||||
PMIX_CC_HELPER([if $CC $1 supports C11 _Static_assert], [pmix_prog_cc_c11_helper__static_assert_available],
|
||||
[[#include <stdint.h>]],[[_Static_assert(sizeof(int64_t) == 8, "WTH");]], [PMIX_C_HAVE__STATIC_ASSERT],
|
||||
[Whether C compiler support _Static_assert keyword])
|
||||
|
||||
dnl At this time Open MPI only needs thread local and the atomic convenience types for C11 support. These
|
||||
dnl will likely be required in the future.
|
||||
AS_IF([test "x$pmix_prog_cc_c11_helper__Thread_local_available" = "xyes" && test "x$pmix_prog_cc_c11_helper_atomic_var_available" = "xyes"],
|
||||
[$2], [$3])
|
||||
|
||||
CFLAGS=$pmix_prog_cc_c11_helper_CFLAGS_save
|
||||
|
||||
PMIX_VAR_SCOPE_POP
|
||||
])
|
||||
|
||||
AC_DEFUN([PMIX_PROG_CC_C11],[
|
||||
PMIX_VAR_SCOPE_PUSH([pmix_prog_cc_c11_flags])
|
||||
if test -z "$pmix_cv_c11_supported" ; then
|
||||
pmix_cv_c11_supported=no
|
||||
pmix_cv_c11_flag_required=yes
|
||||
|
||||
AC_MSG_CHECKING([if $CC requires a flag for C11])
|
||||
|
||||
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
|
||||
#if __STDC_VERSION__ < 201112L
|
||||
#error "Without any CLI flags, this compiler does not support C11"
|
||||
#endif
|
||||
]],[])],
|
||||
[pmix_cv_c11_flag_required=no])
|
||||
|
||||
AC_MSG_RESULT([$pmix_cv_c11_flag_required])
|
||||
|
||||
if test "x$pmix_cv_c11_flag_required" = "xno" ; then
|
||||
AC_MSG_NOTICE([verifying $CC supports C11 without a flag])
|
||||
PMIX_PROG_CC_C11_HELPER([], [], [pmix_cv_c11_flag_required=yes])
|
||||
fi
|
||||
|
||||
if test "x$pmix_cv_c11_flag_required" = "xyes" ; then
|
||||
pmix_prog_cc_c11_flags="-std=gnu11 -std=c11 -c11"
|
||||
|
||||
AC_MSG_NOTICE([checking if $CC supports C11 with a flag])
|
||||
pmix_cv_c11_flag=
|
||||
for flag in $(echo $pmix_prog_cc_c11_flags | tr ' ' '\n') ; do
|
||||
PMIX_PROG_CC_C11_HELPER([$flag],[pmix_cv_c11_flag=$flag],[])
|
||||
if test "x$pmix_cv_c11_flag" != "x" ; then
|
||||
CFLAGS="$CFLAGS $pmix_cv_c11_flag"
|
||||
AC_MSG_NOTICE([using $flag to enable C11 support])
|
||||
pmix_cv_c11_supported=yes
|
||||
break
|
||||
fi
|
||||
done
|
||||
else
|
||||
AC_MSG_NOTICE([no flag required for C11 support])
|
||||
pmix_cv_c11_supported=yes
|
||||
fi
|
||||
fi
|
||||
|
||||
PMIX_VAR_SCOPE_POP
|
||||
])
|
||||
|
||||
|
||||
# PMIX_SETUP_CC()
|
||||
# ---------------
|
||||
# Do everything required to setup the C compiler. Safe to AC_REQUIRE
|
||||
# this macro.
|
||||
AC_DEFUN([PMIX_SETUP_CC],[
|
||||
# AM_PROG_CC_C_O AC_REQUIREs AC_PROG_CC, so we have to be a little
|
||||
# careful about ordering here, and AC_REQUIRE these things so that
|
||||
# they get stamped out in the right order.
|
||||
|
||||
# We require a C99 compliant compiler
|
||||
# The result of AC_PROG_CC_C99 is stored in ac_cv_prog_cc_c99
|
||||
if test "x$ac_cv_prog_cc_c99" = xno ; then
|
||||
AC_MSG_WARN([PMIx requires a C99 compiler])
|
||||
AC_MSG_ERROR([Aborting.])
|
||||
AC_REQUIRE([_PMIX_START_SETUP_CC])
|
||||
AC_REQUIRE([_PMIX_PROG_CC])
|
||||
AC_REQUIRE([AM_PROG_CC_C_O])
|
||||
|
||||
PMIX_PROG_CC_C11
|
||||
|
||||
if test $pmix_cv_c11_supported = no ; then
|
||||
# It is not currently an error if C11 support is not available. Uncomment the
|
||||
# following lines and update the warning when we require a C11 compiler.
|
||||
# AC_MSG_WARNING([Open MPI requires a C11 (or newer) compiler])
|
||||
# AC_MSG_ERROR([Aborting.])
|
||||
# From Open MPI 1.7 on we require a C99 compiant compiler
|
||||
AC_PROG_CC_C99
|
||||
# The result of AC_PROG_CC_C99 is stored in ac_cv_prog_cc_c99
|
||||
if test "x$ac_cv_prog_cc_c99" = xno ; then
|
||||
AC_MSG_WARN([Open MPI requires a C99 (or newer) compiler. C11 is recommended.])
|
||||
AC_MSG_ERROR([Aborting.])
|
||||
fi
|
||||
|
||||
# Get the correct result for C11 support flags now that the compiler flags have
|
||||
# changed
|
||||
PMIX_PROG_CC_C11_HELPER([],[],[])
|
||||
fi
|
||||
|
||||
# Check if compiler support __thread
|
||||
PMIX_VAR_SCOPE_PUSH([pmix_prog_cc__thread_available])
|
||||
PMIX_CC_HELPER([if $CC $1 supports __thread], [pmix_prog_cc__thread_available],
|
||||
[],[[static __thread int foo = 1;++foo;]], [PMIX_C_HAVE___THREAD],
|
||||
[Whether C compiler supports __thread])
|
||||
PMIX_VAR_SCOPE_POP
|
||||
|
||||
PMIX_C_COMPILER_VENDOR([pmix_c_vendor])
|
||||
|
||||
# Check for standard headers, needed here because needed before
|
||||
@ -59,6 +185,48 @@ AC_DEFUN([PMIX_SETUP_CC],[
|
||||
#endif])
|
||||
AC_DEFINE([_GNU_SOURCE])])
|
||||
|
||||
# Do we want code coverage
|
||||
if test "$WANT_COVERAGE" = "1"; then
|
||||
if test "$pmix_c_vendor" = "gnu" ; then
|
||||
# For compilers > gcc-4.x, use --coverage for
|
||||
# compiling and linking to circumvent trouble with
|
||||
# libgcov.
|
||||
CFLAGS_orig="$CFLAGS"
|
||||
LDFLAGS_orig="$LDFLAGS"
|
||||
|
||||
CFLAGS="$CFLAGS_orig --coverage"
|
||||
LDFLAGS="$LDFLAGS_orig --coverage"
|
||||
PMIX_COVERAGE_FLAGS=
|
||||
|
||||
AC_CACHE_CHECK([if $CC supports --coverage],
|
||||
[pmix_cv_cc_coverage],
|
||||
[AC_TRY_COMPILE([], [],
|
||||
[pmix_cv_cc_coverage="yes"],
|
||||
[pmix_cv_cc_coverage="no"])])
|
||||
|
||||
if test "$pmix_cv_cc_coverage" = "yes" ; then
|
||||
PMIX_COVERAGE_FLAGS="--coverage"
|
||||
CLEANFILES="*.gcno ${CLEANFILES}"
|
||||
CONFIG_CLEAN_FILES="*.gcda *.gcov ${CONFIG_CLEAN_FILES}"
|
||||
else
|
||||
PMIX_COVERAGE_FLAGS="-ftest-coverage -fprofile-arcs"
|
||||
CLEANFILES="*.bb *.bbg ${CLEANFILES}"
|
||||
CONFIG_CLEAN_FILES="*.da *.*.gcov ${CONFIG_CLEAN_FILES}"
|
||||
fi
|
||||
CFLAGS="$CFLAGS_orig $PMIX_COVERAGE_FLAGS"
|
||||
LDFLAGS="$LDFLAGS_orig $PMIX_COVERAGE_FLAGS"
|
||||
|
||||
PMIX_FLAGS_UNIQ(CFLAGS)
|
||||
PMIX_FLAGS_UNIQ(LDFLAGS)
|
||||
AC_MSG_WARN([$PMIX_COVERAGE_FLAGS has been added to CFLAGS (--enable-coverage)])
|
||||
|
||||
WANT_DEBUG=1
|
||||
else
|
||||
AC_MSG_WARN([Code coverage functionality is currently available only with GCC])
|
||||
AC_MSG_ERROR([Configure: Cannot continue])
|
||||
fi
|
||||
fi
|
||||
|
||||
# Do we want debugging?
|
||||
if test "$WANT_DEBUG" = "1" && test "$enable_debug_symbols" != "no" ; then
|
||||
CFLAGS="$CFLAGS -g"
|
||||
@ -288,21 +456,13 @@ AC_DEFUN([PMIX_SETUP_CC],[
|
||||
PMIX_ENSURE_CONTAINS_OPTFLAGS(["$CFLAGS"])
|
||||
AC_MSG_RESULT([$co_result])
|
||||
CFLAGS="$co_result"
|
||||
|
||||
##################################
|
||||
# C compiler characteristics
|
||||
##################################
|
||||
# Does the compiler support "ident"-like constructs?
|
||||
PMIX_CHECK_IDENT([CC], [CFLAGS], [c], [C])
|
||||
|
||||
])
|
||||
|
||||
|
||||
AC_DEFUN([_PMIX_START_SETUP_CC],[
|
||||
pmix_show_subtitle "C compiler and preprocessor"
|
||||
|
||||
AC_REQUIRE([AC_PROG_CC])
|
||||
# $%@#!@#% AIX!! This has to be called before anything invokes the C
|
||||
# $%@#!@#% AIX!! This has to be called before anything invokes the C
|
||||
# compiler.
|
||||
dnl AC_AIX
|
||||
])
|
||||
@ -314,10 +474,10 @@ AC_DEFUN([_PMIX_PROG_CC],[
|
||||
#
|
||||
PMIX_VAR_SCOPE_PUSH([pmix_cflags_save dummy pmix_cc_arvgv0])
|
||||
pmix_cflags_save="$CFLAGS"
|
||||
AC_PROG_CC_C99
|
||||
AC_PROG_CC
|
||||
BASECC="`basename $CC`"
|
||||
CFLAGS="$pmix_cflags_save"
|
||||
AC_DEFINE_UNQUOTED(PMIX_CC, "$CC", [PMIx underlying C compiler])
|
||||
AC_DEFINE_UNQUOTED(PMIX_CC, "$CC", [OMPI underlying C compiler])
|
||||
set dummy $CC
|
||||
pmix_cc_argv0=[$]2
|
||||
PMIX_WHICH([$pmix_cc_argv0], [PMIX_CC_ABSOLUTE])
|
||||
|
||||
@ -10,7 +10,7 @@
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2011 Sandia National Laboratories. All rights reserved.
|
||||
* Copyright (c) 2014-2017 Intel, Inc. All rights reserved.
|
||||
* Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
|
||||
* Copyright (c) 2016 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2017 Research Organization for Information Science
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
@ -11,7 +12,9 @@
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2010 IBM Corporation. All rights reserved.
|
||||
* Copyright (c) 2010 ARM ltd. All rights reserved.
|
||||
* Copyright (c) 2017 Intel, Inc. All rights reserved.
|
||||
* Copyright (c) 2017 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2018 Intel, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -105,12 +108,12 @@ void pmix_atomic_isync(void)
|
||||
|
||||
#if (PMIX_GCC_INLINE_ASSEMBLY && (PMIX_ASM_ARM_VERSION >= 6))
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_CMPSET_32 1
|
||||
#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
|
||||
#define PMIX_HAVE_ATOMIC_MATH_32 1
|
||||
static inline int pmix_atomic_cmpset_32(volatile int32_t *addr,
|
||||
int32_t oldval, int32_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
int32_t ret, tmp;
|
||||
int32_t prev, tmp;
|
||||
bool ret;
|
||||
|
||||
__asm__ __volatile__ (
|
||||
"1: ldrex %0, [%2] \n"
|
||||
@ -121,11 +124,13 @@ static inline int pmix_atomic_cmpset_32(volatile int32_t *addr,
|
||||
" bne 1b \n"
|
||||
"2: \n"
|
||||
|
||||
: "=&r" (ret), "=&r" (tmp)
|
||||
: "r" (addr), "r" (oldval), "r" (newval)
|
||||
: "=&r" (prev), "=&r" (tmp)
|
||||
: "r" (addr), "r" (*oldval), "r" (newval)
|
||||
: "cc", "memory");
|
||||
|
||||
return (ret == oldval);
|
||||
ret = (prev == *oldval);
|
||||
*oldval = prev;
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* these two functions aren't inlined in the non-gcc case because then
|
||||
@ -133,51 +138,50 @@ static inline int pmix_atomic_cmpset_32(volatile int32_t *addr,
|
||||
atomic_?mb can be inlined). Instead, we "inline" them by hand in
|
||||
the assembly, meaning there is one function call overhead instead
|
||||
of two */
|
||||
static inline int pmix_atomic_cmpset_acq_32(volatile int32_t *addr,
|
||||
int32_t oldval, int32_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
int rc;
|
||||
bool rc;
|
||||
|
||||
rc = pmix_atomic_cmpset_32(addr, oldval, newval);
|
||||
rc = pmix_atomic_compare_exchange_strong_32 (addr, oldval, newval);
|
||||
pmix_atomic_rmb();
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
static inline int pmix_atomic_cmpset_rel_32(volatile int32_t *addr,
|
||||
int32_t oldval, int32_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
pmix_atomic_wmb();
|
||||
return pmix_atomic_cmpset_32(addr, oldval, newval);
|
||||
return pmix_atomic_compare_exchange_strong_32 (addr, oldval, newval);
|
||||
}
|
||||
|
||||
#if (PMIX_ASM_SUPPORT_64BIT == 1)
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_CMPSET_64 1
|
||||
static inline int pmix_atomic_cmpset_64(volatile int64_t *addr,
|
||||
int64_t oldval, int64_t newval)
|
||||
#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
|
||||
static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
int64_t ret;
|
||||
int tmp;
|
||||
int64_t prev;
|
||||
int tmp;
|
||||
bool ret;
|
||||
|
||||
__asm__ __volatile__ (
|
||||
"1: ldrexd %0, %H0, [%2] \n"
|
||||
" cmp %0, %3 \n"
|
||||
" it eq \n"
|
||||
" cmpeq %H0, %H3 \n"
|
||||
" bne 2f \n"
|
||||
" strexd %1, %4, %H4, [%2] \n"
|
||||
" cmp %1, #0 \n"
|
||||
" bne 1b \n"
|
||||
"2: \n"
|
||||
|
||||
__asm__ __volatile__ (
|
||||
"1: ldrexd %0, %H0, [%2] \n"
|
||||
" cmp %0, %3 \n"
|
||||
" it eq \n"
|
||||
" cmpeq %H0, %H3 \n"
|
||||
" bne 2f \n"
|
||||
" strexd %1, %4, %H4, [%2] \n"
|
||||
" cmp %1, #0 \n"
|
||||
" bne 1b \n"
|
||||
"2: \n"
|
||||
: "=&r" (prev), "=&r" (tmp)
|
||||
: "r" (addr), "r" (*oldval), "r" (newval)
|
||||
: "cc", "memory");
|
||||
|
||||
: "=&r" (ret), "=&r" (tmp)
|
||||
: "r" (addr), "r" (oldval), "r" (newval)
|
||||
: "cc", "memory");
|
||||
|
||||
return (ret == oldval);
|
||||
ret = (prev == *oldval);
|
||||
*oldval = prev;
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* these two functions aren't inlined in the non-gcc case because then
|
||||
@ -185,91 +189,65 @@ static inline int pmix_atomic_cmpset_64(volatile int64_t *addr,
|
||||
atomic_?mb can be inlined). Instead, we "inline" them by hand in
|
||||
the assembly, meaning there is one function call overhead instead
|
||||
of two */
|
||||
static inline int pmix_atomic_cmpset_acq_64(volatile int64_t *addr,
|
||||
int64_t oldval, int64_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
int rc;
|
||||
bool rc;
|
||||
|
||||
rc = pmix_atomic_cmpset_64(addr, oldval, newval);
|
||||
rc = pmix_atomic_compare_exchange_strong_64 (addr, oldval, newval);
|
||||
pmix_atomic_rmb();
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
static inline int pmix_atomic_cmpset_rel_64(volatile int64_t *addr,
|
||||
int64_t oldval, int64_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
pmix_atomic_wmb();
|
||||
return pmix_atomic_cmpset_64(addr, oldval, newval);
|
||||
return pmix_atomic_compare_exchange_strong_64 (addr, oldval, newval);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_ADD_32 1
|
||||
static inline int32_t pmix_atomic_add_32(volatile int32_t* v, int inc)
|
||||
static inline int32_t pmix_atomic_fetch_add_32(volatile int32_t* v, int inc)
|
||||
{
|
||||
int32_t t;
|
||||
int tmp;
|
||||
int32_t t, old;
|
||||
int tmp;
|
||||
|
||||
__asm__ __volatile__(
|
||||
"1: ldrex %0, [%2] \n"
|
||||
" add %0, %0, %3 \n"
|
||||
" strex %1, %0, [%2] \n"
|
||||
" cmp %1, #0 \n"
|
||||
__asm__ __volatile__(
|
||||
"1: ldrex %1, [%3] \n"
|
||||
" add %0, %1, %4 \n"
|
||||
" strex %2, %0, [%3] \n"
|
||||
" cmp %2, #0 \n"
|
||||
" bne 1b \n"
|
||||
|
||||
: "=&r" (t), "=&r" (tmp)
|
||||
: "=&r" (t), "=&r" (old), "=&r" (tmp)
|
||||
: "r" (v), "r" (inc)
|
||||
: "cc", "memory");
|
||||
|
||||
|
||||
return t;
|
||||
return old;
|
||||
}
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_SUB_32 1
|
||||
static inline int32_t pmix_atomic_sub_32(volatile int32_t* v, int dec)
|
||||
static inline int32_t pmix_atomic_fetch_sub_32(volatile int32_t* v, int dec)
|
||||
{
|
||||
int32_t t;
|
||||
int tmp;
|
||||
int32_t t, old;
|
||||
int tmp;
|
||||
|
||||
__asm__ __volatile__(
|
||||
"1: ldrex %0, [%2] \n"
|
||||
" sub %0, %0, %3 \n"
|
||||
" strex %1, %0, [%2] \n"
|
||||
" cmp %1, #0 \n"
|
||||
__asm__ __volatile__(
|
||||
"1: ldrex %1, [%3] \n"
|
||||
" sub %0, %1, %4 \n"
|
||||
" strex %2, %0, [%3] \n"
|
||||
" cmp %2, #0 \n"
|
||||
" bne 1b \n"
|
||||
|
||||
: "=&r" (t), "=&r" (tmp)
|
||||
: "=&r" (t), "=&r" (old), "=&r" (tmp)
|
||||
: "r" (v), "r" (dec)
|
||||
: "cc", "memory");
|
||||
|
||||
return t;
|
||||
}
|
||||
|
||||
#else /* PMIX_ASM_ARM_VERSION <=5 or no GCC inline assembly */
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_CMPSET_32 1
|
||||
#define __kuser_cmpxchg (*((int (*)(int, int, volatile int*))(0xffff0fc0)))
|
||||
static inline int pmix_atomic_cmpset_32(volatile int32_t *addr,
|
||||
int32_t oldval, int32_t newval)
|
||||
{
|
||||
return !(__kuser_cmpxchg(oldval, newval, addr));
|
||||
}
|
||||
|
||||
static inline int pmix_atomic_cmpset_acq_32(volatile int32_t *addr,
|
||||
int32_t oldval, int32_t newval)
|
||||
{
|
||||
/* kernel function includes all necessary memory barriers */
|
||||
return pmix_atomic_cmpset_32(addr, oldval, newval);
|
||||
}
|
||||
|
||||
static inline int pmix_atomic_cmpset_rel_32(volatile int32_t *addr,
|
||||
int32_t oldval, int32_t newval)
|
||||
{
|
||||
/* kernel function includes all necessary memory barriers */
|
||||
return pmix_atomic_cmpset_32(addr, oldval, newval);
|
||||
return t;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
@ -12,9 +12,9 @@
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2010 IBM Corporation. All rights reserved.
|
||||
* Copyright (c) 2010 ARM ltd. All rights reserved.
|
||||
* Copyright (c) 2016 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2017 Intel, Inc. All rights reserved.
|
||||
* Copyright (c) 2018 Intel, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -30,15 +30,21 @@
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_MEM_BARRIER 1
|
||||
#define PMIX_HAVE_ATOMIC_LLSC_32 1
|
||||
#define PMIX_HAVE_ATOMIC_CMPSET_32 1
|
||||
#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
|
||||
#define PMIX_HAVE_ATOMIC_SWAP_32 1
|
||||
#define PMIX_HAVE_ATOMIC_MATH_32 1
|
||||
#define PMIX_HAVE_ATOMIC_CMPSET_64 1
|
||||
#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
|
||||
#define PMIX_HAVE_ATOMIC_SWAP_64 1
|
||||
#define PMIX_HAVE_ATOMIC_LLSC_64 1
|
||||
#define PMIX_HAVE_ATOMIC_ADD_32 1
|
||||
#define PMIX_HAVE_ATOMIC_AND_32 1
|
||||
#define PMIX_HAVE_ATOMIC_OR_32 1
|
||||
#define PMIX_HAVE_ATOMIC_XOR_32 1
|
||||
#define PMIX_HAVE_ATOMIC_SUB_32 1
|
||||
#define PMIX_HAVE_ATOMIC_ADD_64 1
|
||||
#define PMIX_HAVE_ATOMIC_AND_64 1
|
||||
#define PMIX_HAVE_ATOMIC_OR_64 1
|
||||
#define PMIX_HAVE_ATOMIC_XOR_64 1
|
||||
#define PMIX_HAVE_ATOMIC_SUB_64 1
|
||||
|
||||
#define PMIXMB() __asm__ __volatile__ ("dmb sy" : : : "memory")
|
||||
@ -77,10 +83,10 @@ static inline void pmix_atomic_isync (void)
|
||||
*
|
||||
*********************************************************************/
|
||||
|
||||
static inline int pmix_atomic_cmpset_32(volatile int32_t *addr,
|
||||
int32_t oldval, int32_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
int32_t ret, tmp;
|
||||
int32_t prev, tmp;
|
||||
bool ret;
|
||||
|
||||
__asm__ __volatile__ ("1: ldaxr %w0, [%2] \n"
|
||||
" cmp %w0, %w3 \n"
|
||||
@ -88,11 +94,13 @@ static inline int pmix_atomic_cmpset_32(volatile int32_t *addr,
|
||||
" stxr %w1, %w4, [%2] \n"
|
||||
" cbnz %w1, 1b \n"
|
||||
"2: \n"
|
||||
: "=&r" (ret), "=&r" (tmp)
|
||||
: "r" (addr), "r" (oldval), "r" (newval)
|
||||
: "=&r" (prev), "=&r" (tmp)
|
||||
: "r" (addr), "r" (*oldval), "r" (newval)
|
||||
: "cc", "memory");
|
||||
|
||||
return (ret == oldval);
|
||||
ret = (prev == *oldval);
|
||||
*oldval = prev;
|
||||
return ret;
|
||||
}
|
||||
|
||||
static inline int32_t pmix_atomic_swap_32(volatile int32_t *addr, int32_t newval)
|
||||
@ -114,10 +122,10 @@ static inline int32_t pmix_atomic_swap_32(volatile int32_t *addr, int32_t newval
|
||||
atomic_?mb can be inlined). Instead, we "inline" them by hand in
|
||||
the assembly, meaning there is one function call overhead instead
|
||||
of two */
|
||||
static inline int pmix_atomic_cmpset_acq_32(volatile int32_t *addr,
|
||||
int32_t oldval, int32_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
int32_t ret, tmp;
|
||||
int32_t prev, tmp;
|
||||
bool ret;
|
||||
|
||||
__asm__ __volatile__ ("1: ldaxr %w0, [%2] \n"
|
||||
" cmp %w0, %w3 \n"
|
||||
@ -125,18 +133,20 @@ static inline int pmix_atomic_cmpset_acq_32(volatile int32_t *addr,
|
||||
" stxr %w1, %w4, [%2] \n"
|
||||
" cbnz %w1, 1b \n"
|
||||
"2: \n"
|
||||
: "=&r" (ret), "=&r" (tmp)
|
||||
: "r" (addr), "r" (oldval), "r" (newval)
|
||||
: "=&r" (prev), "=&r" (tmp)
|
||||
: "r" (addr), "r" (*oldval), "r" (newval)
|
||||
: "cc", "memory");
|
||||
|
||||
return (ret == oldval);
|
||||
ret = (prev == *oldval);
|
||||
*oldval = prev;
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
static inline int pmix_atomic_cmpset_rel_32(volatile int32_t *addr,
|
||||
int32_t oldval, int32_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
int32_t ret, tmp;
|
||||
int32_t prev, tmp;
|
||||
bool ret;
|
||||
|
||||
__asm__ __volatile__ ("1: ldxr %w0, [%2] \n"
|
||||
" cmp %w0, %w3 \n"
|
||||
@ -144,41 +154,46 @@ static inline int pmix_atomic_cmpset_rel_32(volatile int32_t *addr,
|
||||
" stlxr %w1, %w4, [%2] \n"
|
||||
" cbnz %w1, 1b \n"
|
||||
"2: \n"
|
||||
: "=&r" (ret), "=&r" (tmp)
|
||||
: "r" (addr), "r" (oldval), "r" (newval)
|
||||
: "=&r" (prev), "=&r" (tmp)
|
||||
: "r" (addr), "r" (*oldval), "r" (newval)
|
||||
: "cc", "memory");
|
||||
|
||||
return (ret == oldval);
|
||||
}
|
||||
|
||||
static inline int32_t pmix_atomic_ll_32 (volatile int32_t *addr)
|
||||
{
|
||||
int32_t ret;
|
||||
|
||||
__asm__ __volatile__ ("ldaxr %w0, [%1] \n"
|
||||
: "=&r" (ret)
|
||||
: "r" (addr));
|
||||
|
||||
ret = (prev == *oldval);
|
||||
*oldval = prev;
|
||||
return ret;
|
||||
}
|
||||
|
||||
static inline int pmix_atomic_sc_32 (volatile int32_t *addr, int32_t newval)
|
||||
#define pmix_atomic_ll_32(addr, ret) \
|
||||
do { \
|
||||
volatile int32_t *_addr = (addr); \
|
||||
int32_t _ret; \
|
||||
\
|
||||
__asm__ __volatile__ ("ldaxr %w0, [%1] \n" \
|
||||
: "=&r" (_ret) \
|
||||
: "r" (_addr)); \
|
||||
\
|
||||
ret = (typeof(ret)) _ret; \
|
||||
} while (0)
|
||||
|
||||
#define pmix_atomic_sc_32(addr, newval, ret) \
|
||||
do { \
|
||||
volatile int32_t *_addr = (addr); \
|
||||
int32_t _newval = (int32_t) newval; \
|
||||
int _ret; \
|
||||
\
|
||||
__asm__ __volatile__ ("stlxr %w0, %w2, [%1] \n" \
|
||||
: "=&r" (_ret) \
|
||||
: "r" (_addr), "r" (_newval) \
|
||||
: "cc", "memory"); \
|
||||
\
|
||||
ret = (_ret == 0); \
|
||||
} while (0)
|
||||
|
||||
static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
int ret;
|
||||
|
||||
__asm__ __volatile__ ("stlxr %w0, %w2, [%1] \n"
|
||||
: "=&r" (ret)
|
||||
: "r" (addr), "r" (newval)
|
||||
: "cc", "memory");
|
||||
|
||||
return ret == 0;
|
||||
}
|
||||
|
||||
static inline int pmix_atomic_cmpset_64(volatile int64_t *addr,
|
||||
int64_t oldval, int64_t newval)
|
||||
{
|
||||
int64_t ret;
|
||||
int64_t prev;
|
||||
int tmp;
|
||||
bool ret;
|
||||
|
||||
__asm__ __volatile__ ("1: ldaxr %0, [%2] \n"
|
||||
" cmp %0, %3 \n"
|
||||
@ -186,11 +201,13 @@ static inline int pmix_atomic_cmpset_64(volatile int64_t *addr,
|
||||
" stxr %w1, %4, [%2] \n"
|
||||
" cbnz %w1, 1b \n"
|
||||
"2: \n"
|
||||
: "=&r" (ret), "=&r" (tmp)
|
||||
: "r" (addr), "r" (oldval), "r" (newval)
|
||||
: "=&r" (prev), "=&r" (tmp)
|
||||
: "r" (addr), "r" (*oldval), "r" (newval)
|
||||
: "cc", "memory");
|
||||
|
||||
return (ret == oldval);
|
||||
ret = (prev == *oldval);
|
||||
*oldval = prev;
|
||||
return ret;
|
||||
}
|
||||
|
||||
static inline int64_t pmix_atomic_swap_64 (volatile int64_t *addr, int64_t newval)
|
||||
@ -213,11 +230,11 @@ static inline int64_t pmix_atomic_swap_64 (volatile int64_t *addr, int64_t newva
|
||||
atomic_?mb can be inlined). Instead, we "inline" them by hand in
|
||||
the assembly, meaning there is one function call overhead instead
|
||||
of two */
|
||||
static inline int pmix_atomic_cmpset_acq_64(volatile int64_t *addr,
|
||||
int64_t oldval, int64_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
int64_t ret;
|
||||
int64_t prev;
|
||||
int tmp;
|
||||
bool ret;
|
||||
|
||||
__asm__ __volatile__ ("1: ldaxr %0, [%2] \n"
|
||||
" cmp %0, %3 \n"
|
||||
@ -225,19 +242,21 @@ static inline int pmix_atomic_cmpset_acq_64(volatile int64_t *addr,
|
||||
" stxr %w1, %4, [%2] \n"
|
||||
" cbnz %w1, 1b \n"
|
||||
"2: \n"
|
||||
: "=&r" (ret), "=&r" (tmp)
|
||||
: "r" (addr), "r" (oldval), "r" (newval)
|
||||
: "=&r" (prev), "=&r" (tmp)
|
||||
: "r" (addr), "r" (*oldval), "r" (newval)
|
||||
: "cc", "memory");
|
||||
|
||||
return (ret == oldval);
|
||||
ret = (prev == *oldval);
|
||||
*oldval = prev;
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
static inline int pmix_atomic_cmpset_rel_64(volatile int64_t *addr,
|
||||
int64_t oldval, int64_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
int64_t ret;
|
||||
int64_t prev;
|
||||
int tmp;
|
||||
bool ret;
|
||||
|
||||
__asm__ __volatile__ ("1: ldxr %0, [%2] \n"
|
||||
" cmp %0, %3 \n"
|
||||
@ -245,56 +264,67 @@ static inline int pmix_atomic_cmpset_rel_64(volatile int64_t *addr,
|
||||
" stlxr %w1, %4, [%2] \n"
|
||||
" cbnz %w1, 1b \n"
|
||||
"2: \n"
|
||||
: "=&r" (ret), "=&r" (tmp)
|
||||
: "r" (addr), "r" (oldval), "r" (newval)
|
||||
: "=&r" (prev), "=&r" (tmp)
|
||||
: "r" (addr), "r" (*oldval), "r" (newval)
|
||||
: "cc", "memory");
|
||||
|
||||
return (ret == oldval);
|
||||
}
|
||||
|
||||
static inline int64_t pmix_atomic_ll_64 (volatile int64_t *addr)
|
||||
{
|
||||
int64_t ret;
|
||||
|
||||
__asm__ __volatile__ ("ldaxr %0, [%1] \n"
|
||||
: "=&r" (ret)
|
||||
: "r" (addr));
|
||||
|
||||
ret = (prev == *oldval);
|
||||
*oldval = prev;
|
||||
return ret;
|
||||
}
|
||||
|
||||
static inline int pmix_atomic_sc_64 (volatile int64_t *addr, int64_t newval)
|
||||
{
|
||||
int ret;
|
||||
#define pmix_atomic_ll_64(addr, ret) \
|
||||
do { \
|
||||
volatile int64_t *_addr = (addr); \
|
||||
int64_t _ret; \
|
||||
\
|
||||
__asm__ __volatile__ ("ldaxr %0, [%1] \n" \
|
||||
: "=&r" (_ret) \
|
||||
: "r" (_addr)); \
|
||||
\
|
||||
ret = (typeof(ret)) _ret; \
|
||||
} while (0)
|
||||
|
||||
__asm__ __volatile__ ("stlxr %w0, %2, [%1] \n"
|
||||
: "=&r" (ret)
|
||||
: "r" (addr), "r" (newval)
|
||||
: "cc", "memory");
|
||||
|
||||
return ret == 0;
|
||||
}
|
||||
#define pmix_atomic_sc_64(addr, newval, ret) \
|
||||
do { \
|
||||
volatile int64_t *_addr = (addr); \
|
||||
int64_t _newval = (int64_t) newval; \
|
||||
int _ret; \
|
||||
\
|
||||
__asm__ __volatile__ ("stlxr %w0, %2, [%1] \n" \
|
||||
: "=&r" (_ret) \
|
||||
: "r" (_addr), "r" (_newval) \
|
||||
: "cc", "memory"); \
|
||||
\
|
||||
ret = (_ret == 0); \
|
||||
} while (0)
|
||||
|
||||
#define PMIX_ASM_MAKE_ATOMIC(type, bits, name, inst, reg) \
|
||||
static inline type pmix_atomic_ ## name ## _ ## bits (volatile type *addr, type value) \
|
||||
static inline type pmix_atomic_fetch_ ## name ## _ ## bits (volatile type *addr, type value) \
|
||||
{ \
|
||||
type newval; \
|
||||
type newval, old; \
|
||||
int32_t tmp; \
|
||||
\
|
||||
__asm__ __volatile__("1: ldxr %" reg "0, [%2] \n" \
|
||||
" " inst " %" reg "0, %" reg "0, %" reg "3 \n" \
|
||||
" stxr %w1, %" reg "0, [%2] \n" \
|
||||
" cbnz %w1, 1b \n" \
|
||||
: "=&r" (newval), "=&r" (tmp) \
|
||||
__asm__ __volatile__("1: ldxr %" reg "1, [%3] \n" \
|
||||
" " inst " %" reg "0, %" reg "1, %" reg "4 \n" \
|
||||
" stxr %w2, %" reg "0, [%3] \n" \
|
||||
" cbnz %w2, 1b \n" \
|
||||
: "=&r" (newval), "=&r" (old), "=&r" (tmp) \
|
||||
: "r" (addr), "r" (value) \
|
||||
: "cc", "memory"); \
|
||||
\
|
||||
return newval; \
|
||||
return old; \
|
||||
}
|
||||
|
||||
PMIX_ASM_MAKE_ATOMIC(int32_t, 32, add, "add", "w")
|
||||
PMIX_ASM_MAKE_ATOMIC(int32_t, 32, and, "and", "w")
|
||||
PMIX_ASM_MAKE_ATOMIC(int32_t, 32, or, "orr", "w")
|
||||
PMIX_ASM_MAKE_ATOMIC(int32_t, 32, xor, "eor", "w")
|
||||
PMIX_ASM_MAKE_ATOMIC(int32_t, 32, sub, "sub", "w")
|
||||
PMIX_ASM_MAKE_ATOMIC(int64_t, 64, add, "add", "")
|
||||
PMIX_ASM_MAKE_ATOMIC(int64_t, 64, and, "and", "")
|
||||
PMIX_ASM_MAKE_ATOMIC(int64_t, 64, or, "orr", "")
|
||||
PMIX_ASM_MAKE_ATOMIC(int64_t, 64, xor, "eor", "")
|
||||
PMIX_ASM_MAKE_ATOMIC(int64_t, 64, sub, "sub", "")
|
||||
|
||||
#endif /* PMIX_GCC_INLINE_ASSEMBLY */
|
||||
|
||||
@ -12,11 +12,11 @@
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2007 Sun Microsystems, Inc. All rights reserved.
|
||||
* Copyright (c) 2011 Sandia National Laboratories. All rights reserved.
|
||||
* Copyright (c) 2011-2015 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2011-2017 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2017 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* Copyright (c) 2017 Intel, Inc. All rights reserved.
|
||||
* Copyright (c) 2018 Intel, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -41,11 +41,11 @@
|
||||
*
|
||||
* - \c PMIX_HAVE_ATOMIC_MEM_BARRIER atomic memory barriers
|
||||
* - \c PMIX_HAVE_ATOMIC_SPINLOCKS atomic spinlocks
|
||||
* - \c PMIX_HAVE_ATOMIC_MATH_32 if 32 bit add/sub/cmpset can be done "atomicly"
|
||||
* - \c PMIX_HAVE_ATOMIC_MATH_64 if 64 bit add/sub/cmpset can be done "atomicly"
|
||||
* - \c PMIX_HAVE_ATOMIC_MATH_32 if 32 bit add/sub/compare-exchange can be done "atomicly"
|
||||
* - \c PMIX_HAVE_ATOMIC_MATH_64 if 64 bit add/sub/compare-exchange can be done "atomicly"
|
||||
*
|
||||
* Note that for the Atomic math, atomic add/sub may be implemented as
|
||||
* C code using pmix_atomic_cmpset. The appearance of atomic
|
||||
* C code using pmix_atomic_compare_exchange. The appearance of atomic
|
||||
* operation will be upheld in these cases.
|
||||
*/
|
||||
|
||||
@ -54,6 +54,8 @@
|
||||
|
||||
#include "pmix_config.h"
|
||||
|
||||
#include <stdbool.h>
|
||||
|
||||
#include "src/atomics/sys/architecture.h"
|
||||
#include "src/include/pmix_stdint.h"
|
||||
|
||||
@ -106,21 +108,33 @@ typedef struct pmix_atomic_lock_t pmix_atomic_lock_t;
|
||||
*********************************************************************/
|
||||
#if !PMIX_GCC_INLINE_ASSEMBLY
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_MEM_BARRIER 0
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_CMPSET_32 0
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_CMPSET_64 0
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32 0
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64 0
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_ADD_32 0
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_AND_32 0
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_OR_32 0
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_XOR_32 0
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_SUB_32 0
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_ADD_64 0
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_AND_64 0
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_OR_64 0
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_XOR_64 0
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_SUB_64 0
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_SWAP_32 0
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_SWAP_64 0
|
||||
#else
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_MEM_BARRIER 1
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_CMPSET_32 1
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_CMPSET_64 1
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32 1
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64 1
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_ADD_32 1
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_AND_32 1
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_OR_32 1
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_XOR_32 1
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_SUB_32 1
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_ADD_64 1
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_AND_64 1
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_OR_64 1
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_XOR_64 1
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_SUB_64 1
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_SWAP_32 1
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_SWAP_64 1
|
||||
@ -130,8 +144,8 @@ typedef struct pmix_atomic_lock_t pmix_atomic_lock_t;
|
||||
* Enumeration of lock states
|
||||
*/
|
||||
enum {
|
||||
PMIX_ATOMIC_UNLOCKED = 0,
|
||||
PMIX_ATOMIC_LOCKED = 1
|
||||
PMIX_ATOMIC_LOCK_UNLOCKED = 0,
|
||||
PMIX_ATOMIC_LOCK_LOCKED = 1
|
||||
};
|
||||
|
||||
/**********************************************************************
|
||||
@ -174,14 +188,14 @@ enum {
|
||||
/* compare and set operations can't really be emulated from software,
|
||||
so if these defines aren't already set, they should be set to 0
|
||||
now */
|
||||
#ifndef PMIX_HAVE_ATOMIC_CMPSET_32
|
||||
#define PMIX_HAVE_ATOMIC_CMPSET_32 0
|
||||
#ifndef PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32
|
||||
#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 0
|
||||
#endif
|
||||
#ifndef PMIX_HAVE_ATOMIC_CMPSET_64
|
||||
#define PMIX_HAVE_ATOMIC_CMPSET_64 0
|
||||
#ifndef PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64
|
||||
#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64 0
|
||||
#endif
|
||||
#ifndef PMIX_HAVE_ATOMIC_CMPSET_128
|
||||
#define PMIX_HAVE_ATOMIC_CMPSET_128 0
|
||||
#ifndef PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_128
|
||||
#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_128 0
|
||||
#endif
|
||||
#ifndef PMIX_HAVE_ATOMIC_LLSC_32
|
||||
#define PMIX_HAVE_ATOMIC_LLSC_32 0
|
||||
@ -257,7 +271,7 @@ void pmix_atomic_wmb(void);
|
||||
|
||||
/**********************************************************************
|
||||
*
|
||||
* Atomic spinlocks - always inlined, if have atomic cmpset
|
||||
* Atomic spinlocks - always inlined, if have atomic compare-and-swap
|
||||
*
|
||||
*********************************************************************/
|
||||
|
||||
@ -267,7 +281,7 @@ void pmix_atomic_wmb(void);
|
||||
#define PMIX_HAVE_ATOMIC_SPINLOCKS 0
|
||||
#endif
|
||||
|
||||
#if defined(DOXYGEN) || PMIX_HAVE_ATOMIC_SPINLOCKS || (PMIX_HAVE_ATOMIC_CMPSET_32 || PMIX_HAVE_ATOMIC_CMPSET_64)
|
||||
#if defined(DOXYGEN) || PMIX_HAVE_ATOMIC_SPINLOCKS || (PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
|
||||
|
||||
/**
|
||||
* Initialize a lock to value
|
||||
@ -278,7 +292,7 @@ void pmix_atomic_wmb(void);
|
||||
#if PMIX_HAVE_ATOMIC_SPINLOCKS == 0
|
||||
static inline
|
||||
#endif
|
||||
void pmix_atomic_init(pmix_atomic_lock_t* lock, int32_t value);
|
||||
void pmix_atomic_lock_init(pmix_atomic_lock_t* lock, int32_t value);
|
||||
|
||||
|
||||
/**
|
||||
@ -317,7 +331,7 @@ void pmix_atomic_unlock(pmix_atomic_lock_t *lock);
|
||||
|
||||
#if PMIX_HAVE_ATOMIC_SPINLOCKS == 0
|
||||
#undef PMIX_HAVE_ATOMIC_SPINLOCKS
|
||||
#define PMIX_HAVE_ATOMIC_SPINLOCKS (PMIX_HAVE_ATOMIC_CMPSET_32 || PMIX_HAVE_ATOMIC_CMPSET_64)
|
||||
#define PMIX_HAVE_ATOMIC_SPINLOCKS (PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
|
||||
#define PMIX_NEED_INLINE_ATOMIC_SPINLOCKS 1
|
||||
#endif
|
||||
|
||||
@ -334,48 +348,48 @@ void pmix_atomic_unlock(pmix_atomic_lock_t *lock);
|
||||
#endif
|
||||
#if defined(DOXYGEN) || PMIX_HAVE_ATOMIC_CMPSET_32
|
||||
|
||||
#if PMIX_HAVE_INLINE_ATOMIC_CMPSET_32
|
||||
#if PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32
|
||||
static inline
|
||||
#endif
|
||||
int pmix_atomic_cmpset_32(volatile int32_t *addr, int32_t oldval,
|
||||
int32_t newval);
|
||||
bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval,
|
||||
int32_t newval);
|
||||
|
||||
#if PMIX_HAVE_INLINE_ATOMIC_CMPSET_32
|
||||
#if PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32
|
||||
static inline
|
||||
#endif
|
||||
int pmix_atomic_cmpset_acq_32(volatile int32_t *addr, int32_t oldval,
|
||||
int32_t newval);
|
||||
bool pmix_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval,
|
||||
int32_t newval);
|
||||
|
||||
#if PMIX_HAVE_INLINE_ATOMIC_CMPSET_32
|
||||
#if PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32
|
||||
static inline
|
||||
#endif
|
||||
int pmix_atomic_cmpset_rel_32(volatile int32_t *addr, int32_t oldval,
|
||||
int32_t newval);
|
||||
bool pmix_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval,
|
||||
int32_t newval);
|
||||
#endif
|
||||
|
||||
|
||||
#if !defined(PMIX_HAVE_ATOMIC_CMPSET_64) && !defined(DOXYGEN)
|
||||
#define PMIX_HAVE_ATOMIC_CMPSET_64 0
|
||||
#if !defined(PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64) && !defined(DOXYGEN)
|
||||
#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64 0
|
||||
#endif
|
||||
#if defined(DOXYGEN) || PMIX_HAVE_ATOMIC_CMPSET_64
|
||||
#if defined(DOXYGEN) || PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64
|
||||
|
||||
#if PMIX_HAVE_INLINE_ATOMIC_CMPSET_64
|
||||
#if PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64
|
||||
static inline
|
||||
#endif
|
||||
int pmix_atomic_cmpset_64(volatile int64_t *addr, int64_t oldval,
|
||||
int64_t newval);
|
||||
bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval,
|
||||
int64_t newval);
|
||||
|
||||
#if PMIX_HAVE_INLINE_ATOMIC_CMPSET_64
|
||||
#if PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64
|
||||
static inline
|
||||
#endif
|
||||
int pmix_atomic_cmpset_acq_64(volatile int64_t *addr, int64_t oldval,
|
||||
int64_t newval);
|
||||
bool pmix_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval,
|
||||
int64_t newval);
|
||||
|
||||
#if PMIX_HAVE_INLINE_ATOMIC_CMPSET_64
|
||||
#if PMIX_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64
|
||||
static inline
|
||||
#endif
|
||||
int pmix_atomic_cmpset_rel_64(volatile int64_t *addr, int64_t oldval,
|
||||
int64_t newval);
|
||||
bool pmix_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval,
|
||||
int64_t newval);
|
||||
|
||||
#endif
|
||||
|
||||
@ -384,30 +398,29 @@ int pmix_atomic_cmpset_rel_64(volatile int64_t *addr, int64_t oldval,
|
||||
#define PMIX_HAVE_ATOMIC_MATH_32 0
|
||||
#endif
|
||||
|
||||
#if defined(DOXYGEN) || PMIX_HAVE_ATOMIC_MATH_32 || PMIX_HAVE_ATOMIC_CMPSET_32
|
||||
#if defined(DOXYGEN) || PMIX_HAVE_ATOMIC_MATH_32 || PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32
|
||||
|
||||
/* PMIX_HAVE_INLINE_ATOMIC_*_32 will be 1 if <arch>/atomic.h provides
|
||||
a static inline version of it (in assembly). If we have to fall
|
||||
back on cmpset 32, that too will be inline. */
|
||||
#if PMIX_HAVE_INLINE_ATOMIC_ADD_32 || (!defined(PMIX_HAVE_ATOMIC_ADD_32) && PMIX_HAVE_ATOMIC_CMPSET_32)
|
||||
static inline
|
||||
#endif
|
||||
int32_t pmix_atomic_add_32(volatile int32_t *addr, int delta);
|
||||
|
||||
/* PMIX_HAVE_INLINE_ATOMIC_*_32 will be 1 if <arch>/atomic.h provides
|
||||
a static inline version of it (in assembly). If we have to fall
|
||||
back to cmpset 32, that too will be inline. */
|
||||
#if PMIX_HAVE_INLINE_ATOMIC_SUB_32 || (!defined(PMIX_HAVE_ATOMIC_ADD_32) && PMIX_HAVE_ATOMIC_CMPSET_32)
|
||||
static inline
|
||||
#endif
|
||||
int32_t pmix_atomic_sub_32(volatile int32_t *addr, int delta);
|
||||
static inline int32_t pmix_atomic_add_fetch_32(volatile int32_t *addr, int delta);
|
||||
static inline int32_t pmix_atomic_fetch_add_32(volatile int32_t *addr, int delta);
|
||||
static inline int32_t pmix_atomic_and_fetch_32(volatile int32_t *addr, int32_t value);
|
||||
static inline int32_t pmix_atomic_fetch_and_32(volatile int32_t *addr, int32_t value);
|
||||
static inline int32_t pmix_atomic_or_fetch_32(volatile int32_t *addr, int32_t value);
|
||||
static inline int32_t pmix_atomic_fetch_or_32(volatile int32_t *addr, int32_t value);
|
||||
static inline int32_t pmix_atomic_xor_fetch_32(volatile int32_t *addr, int32_t value);
|
||||
static inline int32_t pmix_atomic_fetch_xor_32(volatile int32_t *addr, int32_t value);
|
||||
static inline int32_t pmix_atomic_sub_fetch_32(volatile int32_t *addr, int delta);
|
||||
static inline int32_t pmix_atomic_fetch_sub_32(volatile int32_t *addr, int delta);
|
||||
static inline int32_t pmix_atomic_min_fetch_32 (volatile int32_t *addr, int32_t value);
|
||||
static inline int32_t pmix_atomic_fetch_min_32 (volatile int32_t *addr, int32_t value);
|
||||
static inline int32_t pmix_atomic_max_fetch_32 (volatile int32_t *addr, int32_t value);
|
||||
static inline int32_t pmix_atomic_fetch_max_32 (volatile int32_t *addr, int32_t value);
|
||||
|
||||
#endif /* PMIX_HAVE_ATOMIC_MATH_32 */
|
||||
|
||||
#if ! PMIX_HAVE_ATOMIC_MATH_32
|
||||
/* fix up the value of pmix_have_atomic_math_32 to allow for C versions */
|
||||
#undef PMIX_HAVE_ATOMIC_MATH_32
|
||||
#define PMIX_HAVE_ATOMIC_MATH_32 PMIX_HAVE_ATOMIC_CMPSET_32
|
||||
#define PMIX_HAVE_ATOMIC_MATH_32 PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32
|
||||
#endif
|
||||
|
||||
#ifndef PMIX_HAVE_ATOMIC_MATH_64
|
||||
@ -415,30 +428,28 @@ int32_t pmix_atomic_sub_32(volatile int32_t *addr, int delta);
|
||||
#define PMIX_HAVE_ATOMIC_MATH_64 0
|
||||
#endif
|
||||
|
||||
#if defined(DOXYGEN) || PMIX_HAVE_ATOMIC_MATH_64 || PMIX_HAVE_ATOMIC_CMPSET_64
|
||||
#if defined(DOXYGEN) || PMIX_HAVE_ATOMIC_MATH_64 || PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64
|
||||
|
||||
/* PMIX_HAVE_INLINE_ATOMIC_*_64 will be 1 if <arch>/atomic.h provides
|
||||
a static inline version of it (in assembly). If we have to fall
|
||||
back to cmpset 64, that too will be inline */
|
||||
#if PMIX_HAVE_INLINE_ATOMIC_ADD_64 || (!defined(PMIX_HAVE_ATOMIC_ADD_64) && PMIX_HAVE_ATOMIC_CMPSET_64)
|
||||
static inline
|
||||
#endif
|
||||
int64_t pmix_atomic_add_64(volatile int64_t *addr, int64_t delta);
|
||||
static inline int64_t pmix_atomic_add_fetch_64(volatile int64_t *addr, int64_t delta);
|
||||
static inline int64_t pmix_atomic_fetch_add_64(volatile int64_t *addr, int64_t delta);
|
||||
static inline int64_t pmix_atomic_and_fetch_64(volatile int64_t *addr, int64_t value);
|
||||
static inline int64_t pmix_atomic_fetch_and_64(volatile int64_t *addr, int64_t value);
|
||||
static inline int64_t pmix_atomic_or_fetch_64(volatile int64_t *addr, int64_t value);
|
||||
static inline int64_t pmix_atomic_fetch_or_64(volatile int64_t *addr, int64_t value);
|
||||
static inline int64_t pmix_atomic_fetch_xor_64(volatile int64_t *addr, int64_t value);
|
||||
static inline int64_t pmix_atomic_sub_fetch_64(volatile int64_t *addr, int64_t delta);
|
||||
static inline int64_t pmix_atomic_fetch_sub_64(volatile int64_t *addr, int64_t delta);
|
||||
static inline int64_t pmix_atomic_min_fetch_64 (volatile int64_t *addr, int64_t value);
|
||||
static inline int64_t pmix_atomic_fetch_min_64 (volatile int64_t *addr, int64_t value);
|
||||
static inline int64_t pmix_atomic_max_fetch_64 (volatile int64_t *addr, int64_t value);
|
||||
static inline int64_t pmix_atomic_fetch_max_64 (volatile int64_t *addr, int64_t value);
|
||||
|
||||
/* PMIX_HAVE_INLINE_ATOMIC_*_64 will be 1 if <arch>/atomic.h provides
|
||||
a static inline version of it (in assembly). If we have to fall
|
||||
back to cmpset 64, that too will be inline */
|
||||
#if PMIX_HAVE_INLINE_ATOMIC_SUB_64 || (!defined(PMIX_HAVE_ATOMIC_ADD_64) && PMIX_HAVE_ATOMIC_CMPSET_64)
|
||||
static inline
|
||||
#endif
|
||||
int64_t pmix_atomic_sub_64(volatile int64_t *addr, int64_t delta);
|
||||
|
||||
#endif /* PMIX_HAVE_ATOMIC_MATH_32 */
|
||||
#endif /* PMIX_HAVE_ATOMIC_MATH_64 */
|
||||
|
||||
#if ! PMIX_HAVE_ATOMIC_MATH_64
|
||||
/* fix up the value of pmix_have_atomic_math_64 to allow for C versions */
|
||||
#undef PMIX_HAVE_ATOMIC_MATH_64
|
||||
#define PMIX_HAVE_ATOMIC_MATH_64 PMIX_HAVE_ATOMIC_CMPSET_64
|
||||
#define PMIX_HAVE_ATOMIC_MATH_64 PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64
|
||||
#endif
|
||||
|
||||
/* provide a size_t add/subtract. When in debug mode, make it an
|
||||
@ -448,114 +459,141 @@ int64_t pmix_atomic_sub_64(volatile int64_t *addr, int64_t delta);
|
||||
*/
|
||||
#if defined(DOXYGEN) || PMIX_ENABLE_DEBUG
|
||||
static inline size_t
|
||||
pmix_atomic_add_size_t(volatile size_t *addr, int delta)
|
||||
pmix_atomic_add_fetch_size_t(volatile size_t *addr, size_t delta)
|
||||
{
|
||||
#if SIZEOF_SIZE_T == 4
|
||||
return (size_t) pmix_atomic_add_32((int32_t*) addr, delta);
|
||||
return (size_t) pmix_atomic_add_fetch_32((int32_t*) addr, delta);
|
||||
#elif SIZEOF_SIZE_T == 8
|
||||
return (size_t) pmix_atomic_add_64((int64_t*) addr, delta);
|
||||
return (size_t) pmix_atomic_add_fetch_64((int64_t*) addr, delta);
|
||||
#else
|
||||
#error "Unknown size_t size"
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline size_t
|
||||
pmix_atomic_sub_size_t(volatile size_t *addr, int delta)
|
||||
pmix_atomic_fetch_add_size_t(volatile size_t *addr, size_t delta)
|
||||
{
|
||||
#if SIZEOF_SIZE_T == 4
|
||||
return (size_t) pmix_atomic_sub_32((int32_t*) addr, delta);
|
||||
return (size_t) pmix_atomic_fetch_add_32((int32_t*) addr, delta);
|
||||
#elif SIZEOF_SIZE_T == 8
|
||||
return (size_t) pmix_atomic_sub_64((int64_t*) addr, delta);
|
||||
return (size_t) pmix_atomic_fetch_add_64((int64_t*) addr, delta);
|
||||
#else
|
||||
#error "Unknown size_t size"
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline size_t
|
||||
pmix_atomic_sub_fetch_size_t(volatile size_t *addr, size_t delta)
|
||||
{
|
||||
#if SIZEOF_SIZE_T == 4
|
||||
return (size_t) pmix_atomic_sub_fetch_32((int32_t*) addr, delta);
|
||||
#elif SIZEOF_SIZE_T == 8
|
||||
return (size_t) pmix_atomic_sub_fetch_64((int64_t*) addr, delta);
|
||||
#else
|
||||
#error "Unknown size_t size"
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline size_t
|
||||
pmix_atomic_fetch_sub_size_t(volatile size_t *addr, size_t delta)
|
||||
{
|
||||
#if SIZEOF_SIZE_T == 4
|
||||
return (size_t) pmix_atomic_fetch_sub_32((int32_t*) addr, delta);
|
||||
#elif SIZEOF_SIZE_T == 8
|
||||
return (size_t) pmix_atomic_fetch_sub_64((int64_t*) addr, delta);
|
||||
#else
|
||||
#error "Unknown size_t size"
|
||||
#endif
|
||||
}
|
||||
|
||||
#else
|
||||
#if SIZEOF_SIZE_T == 4
|
||||
#define pmix_atomic_add_size_t(addr, delta) ((size_t) pmix_atomic_add_32((int32_t*) addr, delta))
|
||||
#define pmix_atomic_sub_size_t(addr, delta) ((size_t) pmix_atomic_sub_32((int32_t*) addr, delta))
|
||||
#elif SIZEOF_SIZE_T ==8
|
||||
#define pmix_atomic_add_size_t(addr, delta) ((size_t) pmix_atomic_add_64((int64_t*) addr, delta))
|
||||
#define pmix_atomic_sub_size_t(addr, delta) ((size_t) pmix_atomic_sub_64((int64_t*) addr, delta))
|
||||
#define pmix_atomic_add_fetch_size_t(addr, delta) ((size_t) pmix_atomic_add_fetch_32((volatile int32_t *) addr, delta))
|
||||
#define pmix_atomic_fetch_add_size_t(addr, delta) ((size_t) pmix_atomic_fetch_add_32((volatile int32_t *) addr, delta))
|
||||
#define pmix_atomic_sub_fetch_size_t(addr, delta) ((size_t) pmix_atomic_sub_fetch_32((volatile int32_t *) addr, delta))
|
||||
#define pmix_atomic_fetch_sub_size_t(addr, delta) ((size_t) pmix_atomic_fetch_sub_32((volatile int32_t *) addr, delta))
|
||||
#elif SIZEOF_SIZE_T == 8
|
||||
#define pmix_atomic_add_fetch_size_t(addr, delta) ((size_t) pmix_atomic_add_fetch_64((volatile int64_t *) addr, delta))
|
||||
#define pmix_atomic_fetch_add_size_t(addr, delta) ((size_t) pmix_atomic_fetch_add_64((volatile int64_t *) addr, delta))
|
||||
#define pmix_atomic_sub_fetch_size_t(addr, delta) ((size_t) pmix_atomic_sub_fetch_64((volatile int64_t *) addr, delta))
|
||||
#define pmix_atomic_fetch_sub_size_t(addr, delta) ((size_t) pmix_atomic_fetch_sub_64((volatile int64_t *) addr, delta))
|
||||
#else
|
||||
#error "Unknown size_t size"
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#if defined(DOXYGEN) || (PMIX_HAVE_ATOMIC_CMPSET_32 || PMIX_HAVE_ATOMIC_CMPSET_64)
|
||||
#if defined(DOXYGEN) || (PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
|
||||
/* these are always done with inline functions, so always mark as
|
||||
static inline */
|
||||
static inline int pmix_atomic_cmpset_xx(volatile void* addr, int64_t oldval,
|
||||
int64_t newval, size_t length);
|
||||
static inline int pmix_atomic_cmpset_acq_xx(volatile void* addr,
|
||||
int64_t oldval, int64_t newval,
|
||||
size_t length);
|
||||
static inline int pmix_atomic_cmpset_rel_xx(volatile void* addr,
|
||||
int64_t oldval, int64_t newval,
|
||||
size_t length);
|
||||
|
||||
static inline int pmix_atomic_cmpset_ptr(volatile void* addr,
|
||||
void* oldval,
|
||||
void* newval);
|
||||
static inline int pmix_atomic_cmpset_acq_ptr(volatile void* addr,
|
||||
void* oldval,
|
||||
void* newval);
|
||||
static inline int pmix_atomic_cmpset_rel_ptr(volatile void* addr,
|
||||
void* oldval,
|
||||
void* newval);
|
||||
static inline bool pmix_atomic_compare_exchange_strong_xx (volatile void *addr, void *oldval,
|
||||
int64_t newval, size_t length);
|
||||
static inline bool pmix_atomic_compare_exchange_strong_acq_xx (volatile void *addr, void *oldval,
|
||||
int64_t newval, size_t length);
|
||||
static inline bool pmix_atomic_compare_exchange_strong_rel_xx (volatile void *addr, void *oldval,
|
||||
int64_t newval, size_t length);
|
||||
|
||||
|
||||
static inline bool pmix_atomic_compare_exchange_strong_ptr (volatile void* addr, void *oldval,
|
||||
void *newval);
|
||||
static inline bool pmix_atomic_compare_exchange_strong_acq_ptr (volatile void* addr, void *oldval,
|
||||
void *newval);
|
||||
static inline bool pmix_atomic_compare_exchange_strong_rel_ptr (volatile void* addr, void *oldval,
|
||||
void *newval);
|
||||
|
||||
/**
|
||||
* Atomic compare and set of pointer with relaxed semantics. This
|
||||
* Atomic compare and set of generic type with relaxed semantics. This
|
||||
* macro detect at compile time the type of the first argument and
|
||||
* choose the correct function to be called.
|
||||
*
|
||||
* \note This macro should only be used for integer types.
|
||||
*
|
||||
* @param addr Address of <TYPE>.
|
||||
* @param oldval Comparison value <TYPE>.
|
||||
* @param oldval Comparison value address of <TYPE>.
|
||||
* @param newval New value to set if comparision is true <TYPE>.
|
||||
*
|
||||
* See pmix_atomic_cmpset_* for pseudo-code.
|
||||
* See pmix_atomic_compare_exchange_* for pseudo-code.
|
||||
*/
|
||||
#define pmix_atomic_cmpset( ADDR, OLDVAL, NEWVAL ) \
|
||||
pmix_atomic_cmpset_xx( (volatile void*)(ADDR), (intptr_t)(OLDVAL), \
|
||||
(intptr_t)(NEWVAL), sizeof(*(ADDR)) )
|
||||
#define pmix_atomic_compare_exchange_strong( ADDR, OLDVAL, NEWVAL ) \
|
||||
pmix_atomic_compare_exchange_strong_xx( (volatile void*)(ADDR), (void *)(OLDVAL), \
|
||||
(intptr_t)(NEWVAL), sizeof(*(ADDR)) )
|
||||
|
||||
/**
|
||||
* Atomic compare and set of pointer with acquire semantics. This
|
||||
* macro detect at compile time the type of the first argument
|
||||
* and choose the correct function to be called.
|
||||
* Atomic compare and set of generic type with acquire semantics. This
|
||||
* macro detect at compile time the type of the first argument and
|
||||
* choose the correct function to be called.
|
||||
*
|
||||
* \note This macro should only be used for integer types.
|
||||
*
|
||||
* @param addr Address of <TYPE>.
|
||||
* @param oldval Comparison value <TYPE>.
|
||||
* @param oldval Comparison value address of <TYPE>.
|
||||
* @param newval New value to set if comparision is true <TYPE>.
|
||||
*
|
||||
* See pmix_atomic_cmpset_acq_* for pseudo-code.
|
||||
* See pmix_atomic_compare_exchange_acq_* for pseudo-code.
|
||||
*/
|
||||
#define pmix_atomic_cmpset_acq( ADDR, OLDVAL, NEWVAL ) \
|
||||
pmix_atomic_cmpset_acq_xx( (volatile void*)(ADDR), (int64_t)(OLDVAL), \
|
||||
(int64_t)(NEWVAL), sizeof(*(ADDR)) )
|
||||
|
||||
#define pmix_atomic_compare_exchange_strong_acq( ADDR, OLDVAL, NEWVAL ) \
|
||||
pmix_atomic_compare_exchange_strong_acq_xx( (volatile void*)(ADDR), (void *)(OLDVAL), \
|
||||
(intptr_t)(NEWVAL), sizeof(*(ADDR)) )
|
||||
|
||||
/**
|
||||
* Atomic compare and set of pointer with release semantics. This
|
||||
* macro detect at compile time the type of the first argument
|
||||
* and choose the correct function to b
|
||||
* Atomic compare and set of generic type with release semantics. This
|
||||
* macro detect at compile time the type of the first argument and
|
||||
* choose the correct function to be called.
|
||||
*
|
||||
* \note This macro should only be used for integer types.
|
||||
*
|
||||
* @param addr Address of <TYPE>.
|
||||
* @param oldval Comparison value <TYPE>.
|
||||
* @param oldval Comparison value address of <TYPE>.
|
||||
* @param newval New value to set if comparision is true <TYPE>.
|
||||
*
|
||||
* See pmix_atomic_cmpsetrel_* for pseudo-code.
|
||||
* See pmix_atomic_compare_exchange_rel_* for pseudo-code.
|
||||
*/
|
||||
#define pmix_atomic_cmpset_rel( ADDR, OLDVAL, NEWVAL ) \
|
||||
pmix_atomic_cmpset_rel_xx( (volatile void*)(ADDR), (int64_t)(OLDVAL), \
|
||||
(int64_t)(NEWVAL), sizeof(*(ADDR)) )
|
||||
#define pmix_atomic_compare_exchange_strong_rel( ADDR, OLDVAL, NEWVAL ) \
|
||||
pmix_atomic_compare_exchange_strong_rel_xx( (volatile void*)(ADDR), (void *)(OLDVAL), \
|
||||
(intptr_t)(NEWVAL), sizeof(*(ADDR)) )
|
||||
|
||||
#endif /* (PMIX_HAVE_ATOMIC_CMPSET_32 || PMIX_HAVE_ATOMIC_CMPSET_64) */
|
||||
|
||||
#endif /* (PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64) */
|
||||
|
||||
#if defined(DOXYGEN) || (PMIX_HAVE_ATOMIC_MATH_32 || PMIX_HAVE_ATOMIC_MATH_64)
|
||||
|
||||
@ -563,15 +601,11 @@ static inline void pmix_atomic_add_xx(volatile void* addr,
|
||||
int32_t value, size_t length);
|
||||
static inline void pmix_atomic_sub_xx(volatile void* addr,
|
||||
int32_t value, size_t length);
|
||||
#if SIZEOF_VOID_P == 4 && PMIX_HAVE_ATOMIC_CMPSET_32
|
||||
static inline int32_t pmix_atomic_add_ptr( volatile void* addr, void* delta );
|
||||
static inline int32_t pmix_atomic_sub_ptr( volatile void* addr, void* delta );
|
||||
#elif SIZEOF_VOID_P == 8 && PMIX_HAVE_ATOMIC_CMPSET_64
|
||||
static inline int64_t pmix_atomic_add_ptr( volatile void* addr, void* delta );
|
||||
static inline int64_t pmix_atomic_sub_ptr( volatile void* addr, void* delta );
|
||||
#else
|
||||
#error Atomic arithmetic on pointers not supported
|
||||
#endif
|
||||
|
||||
static inline intptr_t pmix_atomic_add_fetch_ptr( volatile void* addr, void* delta );
|
||||
static inline intptr_t pmix_atomic_fetch_add_ptr( volatile void* addr, void* delta );
|
||||
static inline intptr_t pmix_atomic_sub_fetch_ptr( volatile void* addr, void* delta );
|
||||
static inline intptr_t pmix_atomic_fetch_sub_ptr( volatile void* addr, void* delta );
|
||||
|
||||
/**
|
||||
* Atomically increment the content depending on the type. This
|
||||
|
||||
@ -11,8 +11,9 @@
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2010-2014 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2012-2015 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2018 Intel, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -34,20 +35,63 @@
|
||||
*
|
||||
* Some architectures do not provide support for the 64 bits
|
||||
* atomic operations. Until we find a better solution let's just
|
||||
* undefine all those functions if there is no 64 bit cmpset
|
||||
* undefine all those functions if there is no 64 bit compare-exchange
|
||||
*
|
||||
*********************************************************************/
|
||||
#if PMIX_HAVE_ATOMIC_CMPSET_32
|
||||
#if PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32
|
||||
|
||||
#if !defined(PMIX_HAVE_ATOMIC_MIN_32)
|
||||
static inline int32_t pmix_atomic_fetch_min_32 (volatile int32_t *addr, int32_t value)
|
||||
{
|
||||
int32_t old = *addr;
|
||||
do {
|
||||
if (old <= value) {
|
||||
break;
|
||||
}
|
||||
} while (!pmix_atomic_compare_exchange_strong_32 (addr, &old, value));
|
||||
|
||||
return old;
|
||||
}
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_MIN_32 1
|
||||
|
||||
#endif /* PMIX_HAVE_ATOMIC_MIN_32 */
|
||||
|
||||
#if !defined(PMIX_HAVE_ATOMIC_MAX_32)
|
||||
static inline int32_t pmix_atomic_fetch_max_32 (volatile int32_t *addr, int32_t value)
|
||||
{
|
||||
int32_t old = *addr;
|
||||
do {
|
||||
if (old >= value) {
|
||||
break;
|
||||
}
|
||||
} while (!pmix_atomic_compare_exchange_strong_32 (addr, &old, value));
|
||||
|
||||
return old;
|
||||
}
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_MAX_32 1
|
||||
#endif /* PMIX_HAVE_ATOMIC_MAX_32 */
|
||||
|
||||
#define PMIX_ATOMIC_DEFINE_CMPXCG_OP(type, bits, operation, name) \
|
||||
static inline type pmix_atomic_fetch_ ## name ## _ ## bits (volatile type *addr, type value) \
|
||||
{ \
|
||||
type oldval; \
|
||||
do { \
|
||||
oldval = *addr; \
|
||||
} while (!pmix_atomic_compare_exchange_strong_ ## bits (addr, &oldval, oldval operation value)); \
|
||||
\
|
||||
return oldval; \
|
||||
}
|
||||
|
||||
#if !defined(PMIX_HAVE_ATOMIC_SWAP_32)
|
||||
#define PMIX_HAVE_ATOMIC_SWAP_32 1
|
||||
static inline int32_t pmix_atomic_swap_32(volatile int32_t *addr,
|
||||
int32_t newval)
|
||||
{
|
||||
int32_t old;
|
||||
int32_t old = *addr;
|
||||
do {
|
||||
old = *addr;
|
||||
} while (0 == pmix_atomic_cmpset_32(addr, old, newval));
|
||||
} while (!pmix_atomic_compare_exchange_strong_32 (addr, &old, newval));
|
||||
|
||||
return old;
|
||||
}
|
||||
@ -55,78 +99,124 @@ static inline int32_t pmix_atomic_swap_32(volatile int32_t *addr,
|
||||
|
||||
#if !defined(PMIX_HAVE_ATOMIC_ADD_32)
|
||||
#define PMIX_HAVE_ATOMIC_ADD_32 1
|
||||
static inline int32_t
|
||||
pmix_atomic_add_32(volatile int32_t *addr, int delta)
|
||||
{
|
||||
int32_t oldval;
|
||||
|
||||
do {
|
||||
oldval = *addr;
|
||||
} while (0 == pmix_atomic_cmpset_32(addr, oldval, oldval + delta));
|
||||
return (oldval + delta);
|
||||
}
|
||||
PMIX_ATOMIC_DEFINE_CMPXCG_OP(int32_t, 32, +, add)
|
||||
|
||||
#endif /* PMIX_HAVE_ATOMIC_ADD_32 */
|
||||
|
||||
#if !defined(PMIX_HAVE_ATOMIC_AND_32)
|
||||
#define PMIX_HAVE_ATOMIC_AND_32 1
|
||||
|
||||
PMIX_ATOMIC_DEFINE_CMPXCG_OP(int32_t, 32, &, and)
|
||||
|
||||
#endif /* PMIX_HAVE_ATOMIC_AND_32 */
|
||||
|
||||
#if !defined(PMIX_HAVE_ATOMIC_OR_32)
|
||||
#define PMIX_HAVE_ATOMIC_OR_32 1
|
||||
|
||||
PMIX_ATOMIC_DEFINE_CMPXCG_OP(int32_t, 32, |, or)
|
||||
|
||||
#endif /* PMIX_HAVE_ATOMIC_OR_32 */
|
||||
|
||||
#if !defined(PMIX_HAVE_ATOMIC_XOR_32)
|
||||
#define PMIX_HAVE_ATOMIC_XOR_32 1
|
||||
|
||||
PMIX_ATOMIC_DEFINE_CMPXCG_OP(int32_t, 32, ^, xor)
|
||||
|
||||
#endif /* PMIX_HAVE_ATOMIC_XOR_32 */
|
||||
|
||||
|
||||
#if !defined(PMIX_HAVE_ATOMIC_SUB_32)
|
||||
#define PMIX_HAVE_ATOMIC_SUB_32 1
|
||||
static inline int32_t
|
||||
pmix_atomic_sub_32(volatile int32_t *addr, int delta)
|
||||
{
|
||||
int32_t oldval;
|
||||
|
||||
do {
|
||||
oldval = *addr;
|
||||
} while (0 == pmix_atomic_cmpset_32(addr, oldval, oldval - delta));
|
||||
return (oldval - delta);
|
||||
}
|
||||
PMIX_ATOMIC_DEFINE_CMPXCG_OP(int32_t, 32, -, sub)
|
||||
|
||||
#endif /* PMIX_HAVE_ATOMIC_SUB_32 */
|
||||
|
||||
#endif /* PMIX_HAVE_ATOMIC_CMPSET_32 */
|
||||
#endif /* PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 */
|
||||
|
||||
|
||||
#if PMIX_HAVE_ATOMIC_CMPSET_64
|
||||
#if PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64
|
||||
|
||||
#if !defined(PMIX_HAVE_ATOMIC_MIN_64)
|
||||
static inline int64_t pmix_atomic_fetch_min_64 (volatile int64_t *addr, int64_t value)
|
||||
{
|
||||
int64_t old = *addr;
|
||||
do {
|
||||
if (old <= value) {
|
||||
break;
|
||||
}
|
||||
} while (!pmix_atomic_compare_exchange_strong_64 (addr, &old, value));
|
||||
|
||||
return old;
|
||||
}
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_MIN_64 1
|
||||
|
||||
#endif /* PMIX_HAVE_ATOMIC_MIN_64 */
|
||||
|
||||
#if !defined(PMIX_HAVE_ATOMIC_MAX_64)
|
||||
static inline int64_t pmix_atomic_fetch_max_64 (volatile int64_t *addr, int64_t value)
|
||||
{
|
||||
int64_t old = *addr;
|
||||
do {
|
||||
if (old >= value) {
|
||||
break;
|
||||
}
|
||||
} while (!pmix_atomic_compare_exchange_strong_64 (addr, &old, value));
|
||||
|
||||
return old;
|
||||
}
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_MAX_64 1
|
||||
#endif /* PMIX_HAVE_ATOMIC_MAX_64 */
|
||||
|
||||
#if !defined(PMIX_HAVE_ATOMIC_SWAP_64)
|
||||
#define PMIX_HAVE_ATOMIC_SWAP_64 1
|
||||
static inline int64_t pmix_atomic_swap_64(volatile int64_t *addr,
|
||||
int64_t newval)
|
||||
{
|
||||
int64_t old;
|
||||
int64_t old = *addr;
|
||||
do {
|
||||
old = *addr;
|
||||
} while (0 == pmix_atomic_cmpset_64(addr, old, newval));
|
||||
} while (!pmix_atomic_compare_exchange_strong_64 (addr, &old, newval));
|
||||
|
||||
return old;
|
||||
}
|
||||
#endif /* PMIX_HAVE_ATOMIC_SWAP_32 */
|
||||
#endif /* PMIX_HAVE_ATOMIC_SWAP_64 */
|
||||
|
||||
#if !defined(PMIX_HAVE_ATOMIC_ADD_64)
|
||||
#define PMIX_HAVE_ATOMIC_ADD_64 1
|
||||
static inline int64_t
|
||||
pmix_atomic_add_64(volatile int64_t *addr, int64_t delta)
|
||||
{
|
||||
int64_t oldval;
|
||||
|
||||
do {
|
||||
oldval = *addr;
|
||||
} while (0 == pmix_atomic_cmpset_64(addr, oldval, oldval + delta));
|
||||
return (oldval + delta);
|
||||
}
|
||||
PMIX_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, +, add)
|
||||
|
||||
#endif /* PMIX_HAVE_ATOMIC_ADD_64 */
|
||||
|
||||
#if !defined(PMIX_HAVE_ATOMIC_AND_64)
|
||||
#define PMIX_HAVE_ATOMIC_AND_64 1
|
||||
|
||||
PMIX_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, &, and)
|
||||
|
||||
#endif /* PMIX_HAVE_ATOMIC_AND_64 */
|
||||
|
||||
#if !defined(PMIX_HAVE_ATOMIC_OR_64)
|
||||
#define PMIX_HAVE_ATOMIC_OR_64 1
|
||||
|
||||
PMIX_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, |, or)
|
||||
|
||||
#endif /* PMIX_HAVE_ATOMIC_OR_64 */
|
||||
|
||||
#if !defined(PMIX_HAVE_ATOMIC_XOR_64)
|
||||
#define PMIX_HAVE_ATOMIC_XOR_64 1
|
||||
|
||||
PMIX_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, ^, xor)
|
||||
|
||||
#endif /* PMIX_HAVE_ATOMIC_XOR_64 */
|
||||
|
||||
#if !defined(PMIX_HAVE_ATOMIC_SUB_64)
|
||||
#define PMIX_HAVE_ATOMIC_SUB_64 1
|
||||
static inline int64_t
|
||||
pmix_atomic_sub_64(volatile int64_t *addr, int64_t delta)
|
||||
{
|
||||
int64_t oldval;
|
||||
|
||||
do {
|
||||
oldval = *addr;
|
||||
} while (0 == pmix_atomic_cmpset_64(addr, oldval, oldval - delta));
|
||||
return (oldval - delta);
|
||||
}
|
||||
PMIX_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, -, sub)
|
||||
|
||||
#endif /* PMIX_HAVE_ATOMIC_SUB_64 */
|
||||
|
||||
#else
|
||||
@ -139,130 +229,71 @@ pmix_atomic_sub_64(volatile int64_t *addr, int64_t delta)
|
||||
#define PMIX_HAVE_ATOMIC_SUB_64 0
|
||||
#endif
|
||||
|
||||
#endif /* PMIX_HAVE_ATOMIC_CMPSET_64 */
|
||||
#endif /* PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64 */
|
||||
|
||||
#if (PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64)
|
||||
|
||||
#if (PMIX_HAVE_ATOMIC_CMPSET_32 || PMIX_HAVE_ATOMIC_CMPSET_64)
|
||||
|
||||
static inline int
|
||||
pmix_atomic_cmpset_xx(volatile void* addr, int64_t oldval,
|
||||
int64_t newval, size_t length)
|
||||
{
|
||||
switch( length ) {
|
||||
#if PMIX_HAVE_ATOMIC_CMPSET_32
|
||||
case 4:
|
||||
return pmix_atomic_cmpset_32( (volatile int32_t*)addr,
|
||||
(int32_t)oldval, (int32_t)newval );
|
||||
#endif /* PMIX_HAVE_ATOMIC_CMPSET_32 */
|
||||
|
||||
#if PMIX_HAVE_ATOMIC_CMPSET_64
|
||||
case 8:
|
||||
return pmix_atomic_cmpset_64( (volatile int64_t*)addr,
|
||||
(int64_t)oldval, (int64_t)newval );
|
||||
#endif /* PMIX_HAVE_ATOMIC_CMPSET_64 */
|
||||
}
|
||||
abort();
|
||||
/* This should never happen, so deliberately abort (hopefully
|
||||
leaving a corefile for analysis) */
|
||||
}
|
||||
|
||||
|
||||
static inline int
|
||||
pmix_atomic_cmpset_acq_xx(volatile void* addr, int64_t oldval,
|
||||
int64_t newval, size_t length)
|
||||
{
|
||||
switch( length ) {
|
||||
#if PMIX_HAVE_ATOMIC_CMPSET_32
|
||||
case 4:
|
||||
return pmix_atomic_cmpset_acq_32( (volatile int32_t*)addr,
|
||||
(int32_t)oldval, (int32_t)newval );
|
||||
#endif /* PMIX_HAVE_ATOMIC_CMPSET_32 */
|
||||
|
||||
#if PMIX_HAVE_ATOMIC_CMPSET_64
|
||||
case 8:
|
||||
return pmix_atomic_cmpset_acq_64( (volatile int64_t*)addr,
|
||||
(int64_t)oldval, (int64_t)newval );
|
||||
#endif /* PMIX_HAVE_ATOMIC_CMPSET_64 */
|
||||
}
|
||||
/* This should never happen, so deliberately abort (hopefully
|
||||
leaving a corefile for analysis) */
|
||||
abort();
|
||||
}
|
||||
|
||||
|
||||
static inline int
|
||||
pmix_atomic_cmpset_rel_xx(volatile void* addr, int64_t oldval,
|
||||
int64_t newval, size_t length)
|
||||
{
|
||||
switch( length ) {
|
||||
#if PMIX_HAVE_ATOMIC_CMPSET_32
|
||||
case 4:
|
||||
return pmix_atomic_cmpset_rel_32( (volatile int32_t*)addr,
|
||||
(int32_t)oldval, (int32_t)newval );
|
||||
#endif /* PMIX_HAVE_ATOMIC_CMPSET_32 */
|
||||
|
||||
#if PMIX_HAVE_ATOMIC_CMPSET_64
|
||||
case 8:
|
||||
return pmix_atomic_cmpset_rel_64( (volatile int64_t*)addr,
|
||||
(int64_t)oldval, (int64_t)newval );
|
||||
#endif /* PMIX_HAVE_ATOMIC_CMPSET_64 */
|
||||
}
|
||||
/* This should never happen, so deliberately abort (hopefully
|
||||
leaving a corefile for analysis) */
|
||||
abort();
|
||||
}
|
||||
|
||||
|
||||
static inline int
|
||||
pmix_atomic_cmpset_ptr(volatile void* addr,
|
||||
void* oldval,
|
||||
void* newval)
|
||||
{
|
||||
#if SIZEOF_VOID_P == 4 && PMIX_HAVE_ATOMIC_CMPSET_32
|
||||
return pmix_atomic_cmpset_32((int32_t*) addr, (unsigned long) oldval,
|
||||
(unsigned long) newval);
|
||||
#elif SIZEOF_VOID_P == 8 && PMIX_HAVE_ATOMIC_CMPSET_64
|
||||
return pmix_atomic_cmpset_64((int64_t*) addr, (unsigned long) oldval,
|
||||
(unsigned long) newval);
|
||||
#if PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 && PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64
|
||||
#define PMIX_ATOMIC_DEFINE_CMPXCG_XX(semantics) \
|
||||
static inline bool \
|
||||
pmix_atomic_compare_exchange_strong ## semantics ## xx (volatile void* addr, void *oldval, \
|
||||
int64_t newval, const size_t length) \
|
||||
{ \
|
||||
switch (length) { \
|
||||
case 4: \
|
||||
return pmix_atomic_compare_exchange_strong_32 ((volatile int32_t *) addr, \
|
||||
(int32_t *) oldval, (int32_t) newval); \
|
||||
case 8: \
|
||||
return pmix_atomic_compare_exchange_strong_64 ((volatile int64_t *) addr, \
|
||||
(int64_t *) oldval, (int64_t) newval); \
|
||||
} \
|
||||
abort(); \
|
||||
}
|
||||
#elif PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32
|
||||
#define PMIX_ATOMIC_DEFINE_CMPXCG_XX(semantics) \
|
||||
static inline bool \
|
||||
pmix_atomic_compare_exchange_strong ## semantics ## xx (volatile void* addr, void *oldval, \
|
||||
int64_t newval, const size_t length) \
|
||||
{ \
|
||||
switch (length) { \
|
||||
case 4: \
|
||||
return pmix_atomic_compare_exchange_strong_32 ((volatile int32_t *) addr, \
|
||||
(int32_t *) oldval, (int32_t) newval); \
|
||||
} \
|
||||
abort(); \
|
||||
}
|
||||
#else
|
||||
abort();
|
||||
#error "Platform does not have required atomic compare-and-swap functionality"
|
||||
#endif
|
||||
}
|
||||
|
||||
PMIX_ATOMIC_DEFINE_CMPXCG_XX(_)
|
||||
PMIX_ATOMIC_DEFINE_CMPXCG_XX(_acq_)
|
||||
PMIX_ATOMIC_DEFINE_CMPXCG_XX(_rel_)
|
||||
|
||||
static inline int
|
||||
pmix_atomic_cmpset_acq_ptr(volatile void* addr,
|
||||
void* oldval,
|
||||
void* newval)
|
||||
{
|
||||
#if SIZEOF_VOID_P == 4 && PMIX_HAVE_ATOMIC_CMPSET_32
|
||||
return pmix_atomic_cmpset_acq_32((int32_t*) addr, (unsigned long) oldval,
|
||||
(unsigned long) newval);
|
||||
#elif SIZEOF_VOID_P == 8 && PMIX_HAVE_ATOMIC_CMPSET_64
|
||||
return pmix_atomic_cmpset_acq_64((int64_t*) addr, (unsigned long) oldval,
|
||||
(unsigned long) newval);
|
||||
#if SIZEOF_VOID_P == 4 && PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32
|
||||
#define PMIX_ATOMIC_DEFINE_CMPXCG_PTR_XX(semantics) \
|
||||
static inline bool \
|
||||
pmix_atomic_compare_exchange_strong ## semantics ## ptr (volatile void* addr, void *oldval, void *newval) \
|
||||
{ \
|
||||
return pmix_atomic_compare_exchange_strong_32 ((volatile int32_t *) addr, (int32_t *) oldval, (int32_t) newval); \
|
||||
}
|
||||
#elif SIZEOF_VOID_P == 8 && PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64
|
||||
#define PMIX_ATOMIC_DEFINE_CMPXCG_PTR_XX(semantics) \
|
||||
static inline bool \
|
||||
pmix_atomic_compare_exchange_strong ## semantics ## ptr (volatile void* addr, void *oldval, void *newval) \
|
||||
{ \
|
||||
return pmix_atomic_compare_exchange_strong_64 ((volatile int64_t *) addr, (int64_t *) oldval, (int64_t) newval); \
|
||||
}
|
||||
#else
|
||||
abort();
|
||||
#error "Can not define pmix_atomic_compare_exchange_strong_ptr with existing atomics"
|
||||
#endif
|
||||
}
|
||||
|
||||
PMIX_ATOMIC_DEFINE_CMPXCG_PTR_XX(_)
|
||||
PMIX_ATOMIC_DEFINE_CMPXCG_PTR_XX(_acq_)
|
||||
PMIX_ATOMIC_DEFINE_CMPXCG_PTR_XX(_rel_)
|
||||
|
||||
static inline int pmix_atomic_cmpset_rel_ptr(volatile void* addr,
|
||||
void* oldval,
|
||||
void* newval)
|
||||
{
|
||||
#if SIZEOF_VOID_P == 4 && PMIX_HAVE_ATOMIC_CMPSET_32
|
||||
return pmix_atomic_cmpset_rel_32((int32_t*) addr, (unsigned long) oldval,
|
||||
(unsigned long) newval);
|
||||
#elif SIZEOF_VOID_P == 8 && PMIX_HAVE_ATOMIC_CMPSET_64
|
||||
return pmix_atomic_cmpset_rel_64((int64_t*) addr, (unsigned long) oldval,
|
||||
(unsigned long) newval);
|
||||
#else
|
||||
abort();
|
||||
#endif
|
||||
}
|
||||
#endif /* (PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 || PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64) */
|
||||
|
||||
#endif /* (PMIX_HAVE_ATOMIC_CMPSET_32 || PMIX_HAVE_ATOMIC_CMPSET_64) */
|
||||
|
||||
#if (PMIX_HAVE_ATOMIC_SWAP_32 || PMIX_HAVE_ATOMIC_SWAP_64)
|
||||
|
||||
@ -278,15 +309,15 @@ static inline int pmix_atomic_cmpset_rel_ptr(volatile void* addr,
|
||||
|
||||
#if SIZEOF_VOID_P == 4 && PMIX_HAVE_ATOMIC_LLSC_32
|
||||
|
||||
#define pmix_atomic_ll_ptr(addr) (void *) pmix_atomic_ll_32((int32_t *) addr)
|
||||
#define pmix_atomic_sc_ptr(addr, newval) pmix_atomic_sc_32((int32_t *) addr, (int32_t) newval)
|
||||
#define pmix_atomic_ll_ptr(addr, ret) pmix_atomic_ll_32((volatile int32_t *) (addr), ret)
|
||||
#define pmix_atomic_sc_ptr(addr, value, ret) pmix_atomic_sc_32((volatile int32_t *) (addr), (intptr_t) (value), ret)
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_LLSC_PTR 1
|
||||
|
||||
#elif SIZEOF_VOID_P == 8 && PMIX_HAVE_ATOMIC_LLSC_64
|
||||
|
||||
#define pmix_atomic_ll_ptr(addr) (void *) pmix_atomic_ll_64((int64_t *) addr)
|
||||
#define pmix_atomic_sc_ptr(addr, newval) pmix_atomic_sc_64((int64_t *) addr, (int64_t) newval)
|
||||
#define pmix_atomic_ll_ptr(addr, ret) pmix_atomic_ll_64((volatile int64_t *) (addr), ret)
|
||||
#define pmix_atomic_sc_ptr(addr, value, ret) pmix_atomic_sc_64((volatile int64_t *) (addr), (intptr_t) (value), ret)
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_LLSC_PTR 1
|
||||
|
||||
@ -300,20 +331,19 @@ static inline int pmix_atomic_cmpset_rel_ptr(volatile void* addr,
|
||||
|
||||
#if PMIX_HAVE_ATOMIC_MATH_32 || PMIX_HAVE_ATOMIC_MATH_64
|
||||
|
||||
|
||||
static inline void
|
||||
pmix_atomic_add_xx(volatile void* addr, int32_t value, size_t length)
|
||||
pmix_atomic_add_xx(volatile void* addr, int32_t value, size_t length)
|
||||
{
|
||||
switch( length ) {
|
||||
#if PMIX_HAVE_ATOMIC_ADD_32
|
||||
case 4:
|
||||
pmix_atomic_add_32( (volatile int32_t*)addr, (int32_t)value );
|
||||
(void) pmix_atomic_fetch_add_32( (volatile int32_t*)addr, (int32_t)value );
|
||||
break;
|
||||
#endif /* PMIX_HAVE_ATOMIC_CMPSET_32 */
|
||||
#endif /* PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 */
|
||||
|
||||
#if PMIX_HAVE_ATOMIC_ADD_64
|
||||
case 8:
|
||||
pmix_atomic_add_64( (volatile int64_t*)addr, (int64_t)value );
|
||||
(void) pmix_atomic_fetch_add_64( (volatile int64_t*)addr, (int64_t)value );
|
||||
break;
|
||||
#endif /* PMIX_HAVE_ATOMIC_ADD_64 */
|
||||
default:
|
||||
@ -330,13 +360,13 @@ pmix_atomic_sub_xx(volatile void* addr, int32_t value, size_t length)
|
||||
switch( length ) {
|
||||
#if PMIX_HAVE_ATOMIC_SUB_32
|
||||
case 4:
|
||||
pmix_atomic_sub_32( (volatile int32_t*)addr, (int32_t)value );
|
||||
(void) pmix_atomic_fetch_sub_32( (volatile int32_t*)addr, (int32_t)value );
|
||||
break;
|
||||
#endif /* PMIX_HAVE_ATOMIC_SUB_32 */
|
||||
|
||||
#if PMIX_HAVE_ATOMIC_SUB_64
|
||||
case 8:
|
||||
pmix_atomic_sub_64( (volatile int64_t*)addr, (int64_t)value );
|
||||
(void) pmix_atomic_fetch_sub_64( (volatile int64_t*)addr, (int64_t)value );
|
||||
break;
|
||||
#endif /* PMIX_HAVE_ATOMIC_SUB_64 */
|
||||
default:
|
||||
@ -346,47 +376,102 @@ pmix_atomic_sub_xx(volatile void* addr, int32_t value, size_t length)
|
||||
}
|
||||
}
|
||||
|
||||
#if SIZEOF_VOID_P == 4 && PMIX_HAVE_ATOMIC_ADD_32
|
||||
static inline int32_t pmix_atomic_add_ptr( volatile void* addr,
|
||||
void* delta )
|
||||
#define PMIX_ATOMIC_DEFINE_OP_FETCH(op, operation, type, ptr_type, suffix) \
|
||||
static inline type pmix_atomic_ ## op ## _fetch_ ## suffix (volatile ptr_type *addr, type value) \
|
||||
{ \
|
||||
return pmix_atomic_fetch_ ## op ## _ ## suffix (addr, value) operation value; \
|
||||
}
|
||||
|
||||
PMIX_ATOMIC_DEFINE_OP_FETCH(add, +, int32_t, int32_t, 32)
|
||||
PMIX_ATOMIC_DEFINE_OP_FETCH(and, &, int32_t, int32_t, 32)
|
||||
PMIX_ATOMIC_DEFINE_OP_FETCH(or, |, int32_t, int32_t, 32)
|
||||
PMIX_ATOMIC_DEFINE_OP_FETCH(xor, ^, int32_t, int32_t, 32)
|
||||
PMIX_ATOMIC_DEFINE_OP_FETCH(sub, -, int32_t, int32_t, 32)
|
||||
|
||||
static inline int32_t pmix_atomic_min_fetch_32 (volatile int32_t *addr, int32_t value)
|
||||
{
|
||||
return pmix_atomic_add_32((int32_t*) addr, (unsigned long) delta);
|
||||
int32_t old = pmix_atomic_fetch_min_32 (addr, value);
|
||||
return old <= value ? old : value;
|
||||
}
|
||||
#elif SIZEOF_VOID_P == 8 && PMIX_HAVE_ATOMIC_ADD_64
|
||||
static inline int64_t pmix_atomic_add_ptr( volatile void* addr,
|
||||
void* delta )
|
||||
|
||||
static inline int32_t pmix_atomic_max_fetch_32 (volatile int32_t *addr, int32_t value)
|
||||
{
|
||||
return pmix_atomic_add_64((int64_t*) addr, (unsigned long) delta);
|
||||
int32_t old = pmix_atomic_fetch_max_32 (addr, value);
|
||||
return old >= value ? old : value;
|
||||
}
|
||||
#else
|
||||
static inline int32_t pmix_atomic_add_ptr( volatile void* addr,
|
||||
void* delta )
|
||||
|
||||
#if PMIX_HAVE_ATOMIC_MATH_64
|
||||
PMIX_ATOMIC_DEFINE_OP_FETCH(add, +, int64_t, int64_t, 64)
|
||||
PMIX_ATOMIC_DEFINE_OP_FETCH(and, &, int64_t, int64_t, 64)
|
||||
PMIX_ATOMIC_DEFINE_OP_FETCH(or, |, int64_t, int64_t, 64)
|
||||
PMIX_ATOMIC_DEFINE_OP_FETCH(xor, ^, int64_t, int64_t, 64)
|
||||
PMIX_ATOMIC_DEFINE_OP_FETCH(sub, -, int64_t, int64_t, 64)
|
||||
|
||||
static inline int64_t pmix_atomic_min_fetch_64 (volatile int64_t *addr, int64_t value)
|
||||
{
|
||||
abort();
|
||||
return 0;
|
||||
int64_t old = pmix_atomic_fetch_min_64 (addr, value);
|
||||
return old <= value ? old : value;
|
||||
}
|
||||
|
||||
static inline int64_t pmix_atomic_max_fetch_64 (volatile int64_t *addr, int64_t value)
|
||||
{
|
||||
int64_t old = pmix_atomic_fetch_max_64 (addr, value);
|
||||
return old >= value ? old : value;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#if SIZEOF_VOID_P == 4 && PMIX_HAVE_ATOMIC_SUB_32
|
||||
static inline int32_t pmix_atomic_sub_ptr( volatile void* addr,
|
||||
static inline intptr_t pmix_atomic_fetch_add_ptr( volatile void* addr,
|
||||
void* delta )
|
||||
{
|
||||
return pmix_atomic_sub_32((int32_t*) addr, (unsigned long) delta);
|
||||
}
|
||||
#elif SIZEOF_VOID_P == 8 && PMIX_HAVE_ATOMIC_SUB_32
|
||||
static inline int64_t pmix_atomic_sub_ptr( volatile void* addr,
|
||||
void* delta )
|
||||
{
|
||||
return pmix_atomic_sub_64((int64_t*) addr, (unsigned long) delta);
|
||||
}
|
||||
#if SIZEOF_VOID_P == 4 && PMIX_HAVE_ATOMIC_ADD_32
|
||||
return pmix_atomic_fetch_add_32((int32_t*) addr, (unsigned long) delta);
|
||||
#elif SIZEOF_VOID_P == 8 && PMIX_HAVE_ATOMIC_ADD_64
|
||||
return pmix_atomic_fetch_add_64((int64_t*) addr, (unsigned long) delta);
|
||||
#else
|
||||
static inline int32_t pmix_atomic_sub_ptr( volatile void* addr,
|
||||
abort ();
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline intptr_t pmix_atomic_add_fetch_ptr( volatile void* addr,
|
||||
void* delta )
|
||||
{
|
||||
#if SIZEOF_VOID_P == 4 && PMIX_HAVE_ATOMIC_ADD_32
|
||||
return pmix_atomic_add_fetch_32((int32_t*) addr, (unsigned long) delta);
|
||||
#elif SIZEOF_VOID_P == 8 && PMIX_HAVE_ATOMIC_ADD_64
|
||||
return pmix_atomic_add_fetch_64((int64_t*) addr, (unsigned long) delta);
|
||||
#else
|
||||
abort ();
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline intptr_t pmix_atomic_fetch_sub_ptr( volatile void* addr,
|
||||
void* delta )
|
||||
{
|
||||
#if SIZEOF_VOID_P == 4 && PMIX_HAVE_ATOMIC_SUB_32
|
||||
return pmix_atomic_fetch_sub_32((int32_t*) addr, (unsigned long) delta);
|
||||
#elif SIZEOF_VOID_P == 8 && PMIX_HAVE_ATOMIC_SUB_32
|
||||
return pmix_atomic_fetch_sub_64((int64_t*) addr, (unsigned long) delta);
|
||||
#else
|
||||
abort();
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline intptr_t pmix_atomic_sub_fetch_ptr( volatile void* addr,
|
||||
void* delta )
|
||||
{
|
||||
#if SIZEOF_VOID_P == 4 && PMIX_HAVE_ATOMIC_SUB_32
|
||||
return pmix_atomic_sub_fetch_32((int32_t*) addr, (unsigned long) delta);
|
||||
#elif SIZEOF_VOID_P == 8 && PMIX_HAVE_ATOMIC_SUB_32
|
||||
return pmix_atomic_sub_fetch_64((int64_t*) addr, (unsigned long) delta);
|
||||
#else
|
||||
abort();
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
#endif /* PMIX_HAVE_ATOMIC_MATH_32 || PMIX_HAVE_ATOMIC_MATH_64 */
|
||||
|
||||
@ -401,7 +486,7 @@ static inline int32_t pmix_atomic_sub_ptr( volatile void* addr,
|
||||
* Lock initialization function. It set the lock to UNLOCKED.
|
||||
*/
|
||||
static inline void
|
||||
pmix_atomic_init( pmix_atomic_lock_t* lock, int32_t value )
|
||||
pmix_atomic_lock_init( pmix_atomic_lock_t* lock, int32_t value )
|
||||
{
|
||||
lock->u.lock = value;
|
||||
}
|
||||
@ -410,21 +495,20 @@ pmix_atomic_init( pmix_atomic_lock_t* lock, int32_t value )
|
||||
static inline int
|
||||
pmix_atomic_trylock(pmix_atomic_lock_t *lock)
|
||||
{
|
||||
int ret = pmix_atomic_cmpset_acq_32( &(lock->u.lock),
|
||||
PMIX_ATOMIC_UNLOCKED, PMIX_ATOMIC_LOCKED);
|
||||
return (ret == 0) ? 1 : 0;
|
||||
int32_t unlocked = PMIX_ATOMIC_LOCK_UNLOCKED;
|
||||
bool ret = pmix_atomic_compare_exchange_strong_32 (&lock->u.lock, &unlocked, PMIX_ATOMIC_LOCK_LOCKED);
|
||||
return (ret == false) ? 1 : 0;
|
||||
}
|
||||
|
||||
|
||||
static inline void
|
||||
pmix_atomic_lock(pmix_atomic_lock_t *lock)
|
||||
{
|
||||
while( !pmix_atomic_cmpset_acq_32( &(lock->u.lock),
|
||||
PMIX_ATOMIC_UNLOCKED, PMIX_ATOMIC_LOCKED) ) {
|
||||
while (lock->u.lock == PMIX_ATOMIC_LOCKED) {
|
||||
/* spin */ ;
|
||||
}
|
||||
}
|
||||
while (pmix_atomic_trylock (lock)) {
|
||||
while (lock->u.lock == PMIX_ATOMIC_LOCK_LOCKED) {
|
||||
/* spin */ ;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -432,7 +516,7 @@ static inline void
|
||||
pmix_atomic_unlock(pmix_atomic_lock_t *lock)
|
||||
{
|
||||
pmix_atomic_wmb();
|
||||
lock->u.lock=PMIX_ATOMIC_UNLOCKED;
|
||||
lock->u.lock=PMIX_ATOMIC_LOCK_UNLOCKED;
|
||||
}
|
||||
|
||||
#endif /* PMIX_HAVE_ATOMIC_SPINLOCKS */
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
* reserved.
|
||||
* Copyright (c) 2017 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* Copyright (c) 2017 Intel, Inc. All rights reserved.
|
||||
* Copyright (c) 2018 Intel, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*/
|
||||
|
||||
|
||||
@ -11,11 +11,11 @@
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2011 Sandia National Laboratories. All rights reserved.
|
||||
* Copyright (c) 2014-2016 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2016 Research Organization for Information Science
|
||||
* Copyright (c) 2016-2017 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* Copyright (c) 2017 Intel, Inc. All rights reserved.
|
||||
* Copyright (c) 2018 Intel, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -26,8 +26,6 @@
|
||||
#ifndef PMIX_SYS_ARCH_ATOMIC_H
|
||||
#define PMIX_SYS_ARCH_ATOMIC_H 1
|
||||
|
||||
#include <stdbool.h>
|
||||
|
||||
/**********************************************************************
|
||||
*
|
||||
* Memory Barriers
|
||||
@ -36,13 +34,19 @@
|
||||
#define PMIX_HAVE_ATOMIC_MEM_BARRIER 1
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_MATH_32 1
|
||||
#define PMIX_HAVE_ATOMIC_CMPSET_32 1
|
||||
#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
|
||||
#define PMIX_HAVE_ATOMIC_ADD_32 1
|
||||
#define PMIX_HAVE_ATOMIC_AND_32 1
|
||||
#define PMIX_HAVE_ATOMIC_OR_32 1
|
||||
#define PMIX_HAVE_ATOMIC_XOR_32 1
|
||||
#define PMIX_HAVE_ATOMIC_SUB_32 1
|
||||
#define PMIX_HAVE_ATOMIC_SWAP_32 1
|
||||
#define PMIX_HAVE_ATOMIC_MATH_64 1
|
||||
#define PMIX_HAVE_ATOMIC_CMPSET_64 1
|
||||
#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
|
||||
#define PMIX_HAVE_ATOMIC_ADD_64 1
|
||||
#define PMIX_HAVE_ATOMIC_AND_64 1
|
||||
#define PMIX_HAVE_ATOMIC_OR_64 1
|
||||
#define PMIX_HAVE_ATOMIC_XOR_64 1
|
||||
#define PMIX_HAVE_ATOMIC_SUB_64 1
|
||||
#define PMIX_HAVE_ATOMIC_SWAP_64 1
|
||||
|
||||
@ -63,8 +67,6 @@ static inline void pmix_atomic_wmb(void)
|
||||
}
|
||||
|
||||
#define PMIXMB() pmix_atomic_mb()
|
||||
#define PMIXRMB() pmix_atomic_rmb()
|
||||
#define PMIXWMB() pmix_atomic_wmb()
|
||||
|
||||
/**********************************************************************
|
||||
*
|
||||
@ -80,26 +82,20 @@ static inline void pmix_atomic_wmb(void)
|
||||
#pragma error_messages(off, E_ARG_INCOMPATIBLE_WITH_ARG_L)
|
||||
#endif
|
||||
|
||||
static inline int pmix_atomic_cmpset_acq_32( volatile int32_t *addr,
|
||||
int32_t oldval, int32_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
return __atomic_compare_exchange_n (addr, &oldval, newval, false,
|
||||
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
|
||||
return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
|
||||
static inline int pmix_atomic_cmpset_rel_32( volatile int32_t *addr,
|
||||
int32_t oldval, int32_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
return __atomic_compare_exchange_n (addr, &oldval, newval, false,
|
||||
__ATOMIC_RELEASE, __ATOMIC_RELAXED);
|
||||
return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline int pmix_atomic_cmpset_32( volatile int32_t *addr,
|
||||
int32_t oldval, int32_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
return __atomic_compare_exchange_n (addr, &oldval, newval, false,
|
||||
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
|
||||
return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline int32_t pmix_atomic_swap_32 (volatile int32_t *addr, int32_t newval)
|
||||
@ -109,36 +105,45 @@ static inline int32_t pmix_atomic_swap_32 (volatile int32_t *addr, int32_t newva
|
||||
return oldval;
|
||||
}
|
||||
|
||||
static inline int32_t pmix_atomic_add_32(volatile int32_t *addr, int32_t delta)
|
||||
static inline int32_t pmix_atomic_fetch_add_32(volatile int32_t *addr, int32_t delta)
|
||||
{
|
||||
return __atomic_add_fetch (addr, delta, __ATOMIC_RELAXED);
|
||||
return __atomic_fetch_add (addr, delta, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline int32_t pmix_atomic_sub_32(volatile int32_t *addr, int32_t delta)
|
||||
static inline int32_t pmix_atomic_fetch_and_32(volatile int32_t *addr, int32_t value)
|
||||
{
|
||||
return __atomic_sub_fetch (addr, delta, __ATOMIC_RELAXED);
|
||||
return __atomic_fetch_and (addr, value, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline int pmix_atomic_cmpset_acq_64( volatile int64_t *addr,
|
||||
int64_t oldval, int64_t newval)
|
||||
static inline int32_t pmix_atomic_fetch_or_32(volatile int32_t *addr, int32_t value)
|
||||
{
|
||||
return __atomic_compare_exchange_n (addr, &oldval, newval, false,
|
||||
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
|
||||
return __atomic_fetch_or (addr, value, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline int pmix_atomic_cmpset_rel_64( volatile int64_t *addr,
|
||||
int64_t oldval, int64_t newval)
|
||||
static inline int32_t pmix_atomic_fetch_xor_32(volatile int32_t *addr, int32_t value)
|
||||
{
|
||||
return __atomic_compare_exchange_n (addr, &oldval, newval, false,
|
||||
__ATOMIC_RELEASE, __ATOMIC_RELAXED);
|
||||
return __atomic_fetch_xor (addr, value, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline int32_t pmix_atomic_fetch_sub_32(volatile int32_t *addr, int32_t delta)
|
||||
{
|
||||
return __atomic_fetch_sub (addr, delta, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline bool pmix_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline bool pmix_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
|
||||
static inline int pmix_atomic_cmpset_64( volatile int64_t *addr,
|
||||
int64_t oldval, int64_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
return __atomic_compare_exchange_n (addr, &oldval, newval, false,
|
||||
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
|
||||
return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline int64_t pmix_atomic_swap_64 (volatile int64_t *addr, int64_t newval)
|
||||
@ -148,37 +153,55 @@ static inline int64_t pmix_atomic_swap_64 (volatile int64_t *addr, int64_t newva
|
||||
return oldval;
|
||||
}
|
||||
|
||||
static inline int64_t pmix_atomic_add_64(volatile int64_t *addr, int64_t delta)
|
||||
static inline int64_t pmix_atomic_fetch_add_64(volatile int64_t *addr, int64_t delta)
|
||||
{
|
||||
return __atomic_add_fetch (addr, delta, __ATOMIC_RELAXED);
|
||||
return __atomic_fetch_add (addr, delta, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline int64_t pmix_atomic_sub_64(volatile int64_t *addr, int64_t delta)
|
||||
static inline int64_t pmix_atomic_fetch_and_64(volatile int64_t *addr, int64_t value)
|
||||
{
|
||||
return __atomic_sub_fetch (addr, delta, __ATOMIC_RELAXED);
|
||||
return __atomic_fetch_and (addr, value, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline int64_t pmix_atomic_fetch_or_64(volatile int64_t *addr, int64_t value)
|
||||
{
|
||||
return __atomic_fetch_or (addr, value, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline int64_t pmix_atomic_fetch_xor_64(volatile int64_t *addr, int64_t value)
|
||||
{
|
||||
return __atomic_fetch_xor (addr, value, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline int64_t pmix_atomic_fetch_sub_64(volatile int64_t *addr, int64_t delta)
|
||||
{
|
||||
return __atomic_fetch_sub (addr, delta, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
#if PMIX_HAVE_GCC_BUILTIN_CSWAP_INT128
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_CMPSET_128 1
|
||||
#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
|
||||
|
||||
static inline int pmix_atomic_cmpset_128 (volatile pmix_int128_t *addr,
|
||||
pmix_int128_t oldval, pmix_int128_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_128 (volatile pmix_int128_t *addr,
|
||||
pmix_int128_t *oldval, pmix_int128_t newval)
|
||||
{
|
||||
return __atomic_compare_exchange_n (addr, &oldval, newval, false,
|
||||
return __atomic_compare_exchange_n (addr, oldval, newval, false,
|
||||
__ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
#elif defined(PMIX_HAVE_SYNC_BUILTIN_CSWAP_INT128) && PMIX_HAVE_SYNC_BUILTIN_CSWAP_INT128
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_CMPSET_128 1
|
||||
#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
|
||||
|
||||
/* __atomic version is not lock-free so use legacy __sync version */
|
||||
|
||||
static inline int pmix_atomic_cmpset_128 (volatile pmix_int128_t *addr,
|
||||
pmix_int128_t oldval, pmix_int128_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_128 (volatile pmix_int128_t *addr,
|
||||
pmix_int128_t *oldval, pmix_int128_t newval)
|
||||
{
|
||||
return __sync_bool_compare_and_swap (addr, oldval, newval);
|
||||
pmix_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval);
|
||||
bool ret = prev == *oldval;
|
||||
*oldval = prev;
|
||||
return ret;
|
||||
}
|
||||
|
||||
#endif
|
||||
@ -189,16 +212,16 @@ static inline int pmix_atomic_cmpset_128 (volatile pmix_int128_t *addr,
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_SPINLOCKS 1
|
||||
|
||||
static inline void pmix_atomic_init (pmix_atomic_lock_t* lock, int32_t value)
|
||||
static inline void pmix_atomic_lock_init (pmix_atomic_lock_t* lock, int32_t value)
|
||||
{
|
||||
lock->u.lock = value;
|
||||
}
|
||||
|
||||
static inline int pmix_atomic_trylock(pmix_atomic_lock_t *lock)
|
||||
{
|
||||
int ret = __atomic_exchange_n (&lock->u.lock, PMIX_ATOMIC_LOCKED,
|
||||
int ret = __atomic_exchange_n (&lock->u.lock, PMIX_ATOMIC_LOCK_LOCKED,
|
||||
__ATOMIC_ACQUIRE | __ATOMIC_HLE_ACQUIRE);
|
||||
if (PMIX_ATOMIC_LOCKED == ret) {
|
||||
if (PMIX_ATOMIC_LOCK_LOCKED == ret) {
|
||||
/* abort the transaction */
|
||||
_mm_pause ();
|
||||
return 1;
|
||||
@ -209,7 +232,7 @@ static inline int pmix_atomic_trylock(pmix_atomic_lock_t *lock)
|
||||
|
||||
static inline void pmix_atomic_lock (pmix_atomic_lock_t *lock)
|
||||
{
|
||||
while (PMIX_ATOMIC_LOCKED == __atomic_exchange_n (&lock->u.lock, PMIX_ATOMIC_LOCKED,
|
||||
while (PMIX_ATOMIC_LOCK_LOCKED == __atomic_exchange_n (&lock->u.lock, PMIX_ATOMIC_LOCK_LOCKED,
|
||||
__ATOMIC_ACQUIRE | __ATOMIC_HLE_ACQUIRE)) {
|
||||
/* abort the transaction */
|
||||
_mm_pause ();
|
||||
@ -218,7 +241,7 @@ static inline void pmix_atomic_lock (pmix_atomic_lock_t *lock)
|
||||
|
||||
static inline void pmix_atomic_unlock (pmix_atomic_lock_t *lock)
|
||||
{
|
||||
__atomic_store_n (&lock->u.lock, PMIX_ATOMIC_UNLOCKED,
|
||||
__atomic_store_n (&lock->u.lock, PMIX_ATOMIC_LOCK_UNLOCKED,
|
||||
__ATOMIC_RELEASE | __ATOMIC_HLE_RELEASE);
|
||||
}
|
||||
|
||||
|
||||
@ -13,9 +13,9 @@
|
||||
* Copyright (c) 2007-2010 Oracle and/or its affiliates. All rights reserved.
|
||||
* Copyright (c) 2015 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2017 Intel, Inc. All rights reserved.
|
||||
* Copyright (c) 2018 Intel, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -30,7 +30,7 @@
|
||||
* On ia32, we use cmpxchg.
|
||||
*/
|
||||
|
||||
#define PMIXSMPLOCK "lock; "
|
||||
#define SMPLOCK "lock; "
|
||||
#define PMIXMB() __asm__ __volatile__("": : :"memory")
|
||||
|
||||
|
||||
@ -41,17 +41,12 @@
|
||||
*********************************************************************/
|
||||
#define PMIX_HAVE_ATOMIC_MEM_BARRIER 1
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_CMPSET_32 1
|
||||
#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_MATH_32 1
|
||||
#define PMIX_HAVE_ATOMIC_ADD_32 1
|
||||
#define PMIX_HAVE_ATOMIC_SUB_32 1
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_CMPSET_64 1
|
||||
|
||||
#undef PMIX_HAVE_INLINE_ATOMIC_CMPSET_64
|
||||
#define PMIX_HAVE_INLINE_ATOMIC_CMPSET_64 0
|
||||
|
||||
/**********************************************************************
|
||||
*
|
||||
* Memory Barriers
|
||||
@ -90,73 +85,23 @@ static inline void pmix_atomic_isync(void)
|
||||
*********************************************************************/
|
||||
#if PMIX_GCC_INLINE_ASSEMBLY
|
||||
|
||||
static inline int pmix_atomic_cmpset_32(volatile int32_t *addr,
|
||||
int32_t oldval,
|
||||
int32_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
unsigned char ret;
|
||||
__asm__ __volatile__ (
|
||||
PMIXSMPLOCK "cmpxchgl %3,%2 \n\t"
|
||||
SMPLOCK "cmpxchgl %3,%2 \n\t"
|
||||
"sete %0 \n\t"
|
||||
: "=qm" (ret), "+a" (oldval), "+m" (*addr)
|
||||
: "=qm" (ret), "+a" (*oldval), "+m" (*addr)
|
||||
: "q"(newval)
|
||||
: "memory", "cc");
|
||||
|
||||
return (int)ret;
|
||||
return (bool) ret;
|
||||
}
|
||||
|
||||
#endif /* PMIX_GCC_INLINE_ASSEMBLY */
|
||||
|
||||
#define pmix_atomic_cmpset_acq_32 pmix_atomic_cmpset_32
|
||||
#define pmix_atomic_cmpset_rel_32 pmix_atomic_cmpset_32
|
||||
|
||||
#if PMIX_GCC_INLINE_ASSEMBLY
|
||||
|
||||
#if 0
|
||||
|
||||
/* some versions of GCC won't let you use ebx period (even though they
|
||||
should be able to save / restore for the life of the inline
|
||||
assembly). For the beta, just use the non-inline version */
|
||||
|
||||
#ifndef ll_low /* GLIBC provides these somewhere, so protect */
|
||||
#define ll_low(x) *(((unsigned int*)&(x))+0)
|
||||
#define ll_high(x) *(((unsigned int*)&(x))+1)
|
||||
#endif
|
||||
|
||||
/* On Linux the EBX register is used by the shared libraries
|
||||
* to keep the global offset. In same time this register is
|
||||
* required by the cmpxchg8b instruction (as an input parameter).
|
||||
* This conflict force us to save the EBX before the cmpxchg8b
|
||||
* and to restore it afterward.
|
||||
*/
|
||||
static inline int pmix_atomic_cmpset_64(volatile int64_t *addr,
|
||||
int64_t oldval,
|
||||
int64_t newval)
|
||||
{
|
||||
/*
|
||||
* Compare EDX:EAX with m64. If equal, set ZF and load ECX:EBX into
|
||||
* m64. Else, clear ZF and load m64 into EDX:EAX.
|
||||
*/
|
||||
unsigned char ret;
|
||||
|
||||
__asm__ __volatile__(
|
||||
"push %%ebx \n\t"
|
||||
"movl %4, %%ebx \n\t"
|
||||
SMPLOCK "cmpxchg8b (%1) \n\t"
|
||||
"sete %0 \n\t"
|
||||
"pop %%ebx \n\t"
|
||||
: "=qm"(ret)
|
||||
: "D"(addr), "a"(ll_low(oldval)), "d"(ll_high(oldval)),
|
||||
"r"(ll_low(newval)), "c"(ll_high(newval))
|
||||
: "cc", "memory", "ebx");
|
||||
return (int) ret;
|
||||
}
|
||||
#endif /* if 0 */
|
||||
|
||||
#endif /* PMIX_GCC_INLINE_ASSEMBLY */
|
||||
|
||||
#define pmix_atomic_cmpset_acq_64 pmix_atomic_cmpset_64
|
||||
#define pmix_atomic_cmpset_rel_64 pmix_atomic_cmpset_64
|
||||
#define pmix_atomic_compare_exchange_strong_acq_32 pmix_atomic_compare_exchange_strong_32
|
||||
#define pmix_atomic_compare_exchange_strong_rel_32 pmix_atomic_compare_exchange_strong_32
|
||||
|
||||
#if PMIX_GCC_INLINE_ASSEMBLY
|
||||
|
||||
@ -186,16 +131,16 @@ static inline int32_t pmix_atomic_swap_32( volatile int32_t *addr,
|
||||
*
|
||||
* Atomically adds @i to @v.
|
||||
*/
|
||||
static inline int32_t pmix_atomic_add_32(volatile int32_t* v, int i)
|
||||
static inline int32_t pmix_atomic_fetch_add_32(volatile int32_t* v, int i)
|
||||
{
|
||||
int ret = i;
|
||||
__asm__ __volatile__(
|
||||
PMIXSMPLOCK "xaddl %1,%0"
|
||||
SMPLOCK "xaddl %1,%0"
|
||||
:"+m" (*v), "+r" (ret)
|
||||
:
|
||||
:"memory", "cc"
|
||||
);
|
||||
return (ret+i);
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
@ -206,16 +151,16 @@ static inline int32_t pmix_atomic_add_32(volatile int32_t* v, int i)
|
||||
*
|
||||
* Atomically subtracts @i from @v.
|
||||
*/
|
||||
static inline int32_t pmix_atomic_sub_32(volatile int32_t* v, int i)
|
||||
static inline int32_t pmix_atomic_fetch_sub_32(volatile int32_t* v, int i)
|
||||
{
|
||||
int ret = -i;
|
||||
__asm__ __volatile__(
|
||||
PMIXSMPLOCK "xaddl %1,%0"
|
||||
SMPLOCK "xaddl %1,%0"
|
||||
:"+m" (*v), "+r" (ret)
|
||||
:
|
||||
:"memory", "cc"
|
||||
);
|
||||
return (ret-i);
|
||||
return ret;
|
||||
}
|
||||
|
||||
#endif /* PMIX_GCC_INLINE_ASSEMBLY */
|
||||
|
||||
@ -11,9 +11,9 @@
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2010-2017 IBM Corporation. All rights reserved.
|
||||
* Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2017 Intel, Inc. All rights reserved.
|
||||
* Copyright (c) 2018 Intel, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -31,7 +31,7 @@
|
||||
#define PMIXMB() __asm__ __volatile__ ("sync" : : : "memory")
|
||||
#define PMIXRMB() __asm__ __volatile__ ("lwsync" : : : "memory")
|
||||
#define PMIXWMB() __asm__ __volatile__ ("lwsync" : : : "memory")
|
||||
#define PMIXISYNC() __asm__ __volatile__ ("isync" : : : "memory")
|
||||
#define ISYNC() __asm__ __volatile__ ("isync" : : : "memory")
|
||||
|
||||
|
||||
/**********************************************************************
|
||||
@ -41,21 +41,27 @@
|
||||
*********************************************************************/
|
||||
#define PMIX_HAVE_ATOMIC_MEM_BARRIER 1
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_CMPSET_32 1
|
||||
#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
|
||||
#define PMIX_HAVE_ATOMIC_SWAP_32 1
|
||||
#define PMIX_HAVE_ATOMIC_LLSC_32 1
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_MATH_32 1
|
||||
#define PMIX_HAVE_ATOMIC_ADD_32 1
|
||||
#define PMIX_HAVE_ATOMIC_AND_32 1
|
||||
#define PMIX_HAVE_ATOMIC_OR_32 1
|
||||
#define PMIX_HAVE_ATOMIC_XOR_32 1
|
||||
#define PMIX_HAVE_ATOMIC_SUB_32 1
|
||||
|
||||
|
||||
#if (PMIX_ASSEMBLY_ARCH == PMIX_POWERPC64) || PMIX_ASM_SUPPORT_64BIT
|
||||
#define PMIX_HAVE_ATOMIC_CMPSET_64 1
|
||||
#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
|
||||
#define PMIX_HAVE_ATOMIC_SWAP_64 1
|
||||
#define PMIX_HAVE_ATOMIC_LLSC_64 1
|
||||
#define PMIX_HAVE_ATOMIC_MATH_64 1
|
||||
#define PMIX_HAVE_ATOMIC_ADD_64 1
|
||||
#define PMIX_HAVE_ATOMIC_AND_64 1
|
||||
#define PMIX_HAVE_ATOMIC_OR_64 1
|
||||
#define PMIX_HAVE_ATOMIC_XOR_64 1
|
||||
#define PMIX_HAVE_ATOMIC_SUB_64 1
|
||||
#endif
|
||||
|
||||
@ -90,7 +96,7 @@ void pmix_atomic_wmb(void)
|
||||
static inline
|
||||
void pmix_atomic_isync(void)
|
||||
{
|
||||
PMIXISYNC();
|
||||
ISYNC();
|
||||
}
|
||||
|
||||
#elif PMIX_XLC_INLINE_ASSEMBLY /* end PMIX_GCC_INLINE_ASSEMBLY */
|
||||
@ -139,74 +145,77 @@ void pmix_atomic_isync(void)
|
||||
#define PMIX_ASM_VALUE64(x) x
|
||||
#endif
|
||||
|
||||
|
||||
static inline int pmix_atomic_cmpset_32(volatile int32_t *addr,
|
||||
int32_t oldval, int32_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
int32_t ret;
|
||||
int32_t prev;
|
||||
bool ret;
|
||||
|
||||
__asm__ __volatile__ (
|
||||
"1: lwarx %0, 0, %2 \n\t"
|
||||
" cmpw 0, %0, %3 \n\t"
|
||||
" bne- 2f \n\t"
|
||||
" stwcx. %4, 0, %2 \n\t"
|
||||
" bne- 1b \n\t"
|
||||
"2:"
|
||||
: "=&r" (ret), "=m" (*addr)
|
||||
: "r" PMIX_ASM_ADDR(addr), "r" (oldval), "r" (newval), "m" (*addr)
|
||||
: "cc", "memory");
|
||||
|
||||
return (ret == oldval);
|
||||
}
|
||||
|
||||
static inline int32_t pmix_atomic_ll_32 (volatile int32_t *addr)
|
||||
{
|
||||
int32_t ret;
|
||||
|
||||
__asm__ __volatile__ ("lwarx %0, 0, %1 \n\t"
|
||||
: "=&r" (ret)
|
||||
: "r" (addr)
|
||||
);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static inline int pmix_atomic_sc_32 (volatile int32_t *addr, int32_t newval)
|
||||
{
|
||||
int32_t ret, foo;
|
||||
|
||||
__asm__ __volatile__ (" stwcx. %4, 0, %3 \n\t"
|
||||
" li %0,0 \n\t"
|
||||
" bne- 1f \n\t"
|
||||
" ori %0,%0,1 \n\t"
|
||||
"1:"
|
||||
: "=r" (ret), "=m" (*addr), "=r" (foo)
|
||||
: "r" (addr), "r" (newval)
|
||||
__asm__ __volatile__ (
|
||||
"1: lwarx %0, 0, %2 \n\t"
|
||||
" cmpw 0, %0, %3 \n\t"
|
||||
" bne- 2f \n\t"
|
||||
" stwcx. %4, 0, %2 \n\t"
|
||||
" bne- 1b \n\t"
|
||||
"2:"
|
||||
: "=&r" (prev), "=m" (*addr)
|
||||
: "r" PMIX_ASM_ADDR(addr), "r" (*oldval), "r" (newval), "m" (*addr)
|
||||
: "cc", "memory");
|
||||
|
||||
ret = (prev == *oldval);
|
||||
*oldval = prev;
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* NTH: the LL/SC support is done through macros due to issues with non-optimized builds. The reason
|
||||
* is that even with an always_inline attribute the compiler may still emit instructions to store then
|
||||
* load the arguments to/from the stack. This sequence may cause the ll reservation to be cancelled. */
|
||||
#define pmix_atomic_ll_32(addr, ret) \
|
||||
do { \
|
||||
volatile int32_t *_addr = (addr); \
|
||||
int32_t _ret; \
|
||||
__asm__ __volatile__ ("lwarx %0, 0, %1 \n\t" \
|
||||
: "=&r" (_ret) \
|
||||
: "r" (_addr) \
|
||||
); \
|
||||
ret = (typeof(ret)) _ret; \
|
||||
} while (0)
|
||||
|
||||
#define pmix_atomic_sc_32(addr, value, ret) \
|
||||
do { \
|
||||
volatile int32_t *_addr = (addr); \
|
||||
int32_t _ret, _foo, _newval = (int32_t) value; \
|
||||
\
|
||||
__asm__ __volatile__ (" stwcx. %4, 0, %3 \n\t" \
|
||||
" li %0,0 \n\t" \
|
||||
" bne- 1f \n\t" \
|
||||
" ori %0,%0,1 \n\t" \
|
||||
"1:" \
|
||||
: "=r" (_ret), "=m" (*_addr), "=r" (_foo) \
|
||||
: "r" (_addr), "r" (_newval) \
|
||||
: "cc", "memory"); \
|
||||
ret = _ret; \
|
||||
} while (0)
|
||||
|
||||
/* these two functions aren't inlined in the non-gcc case because then
|
||||
there would be two function calls (since neither cmpset_32 nor
|
||||
atomic_?mb can be inlined). Instead, we "inline" them by hand in
|
||||
the assembly, meaning there is one function call overhead instead
|
||||
of two */
|
||||
static inline int pmix_atomic_cmpset_acq_32(volatile int32_t *addr,
|
||||
int32_t oldval, int32_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
int rc;
|
||||
bool rc;
|
||||
|
||||
rc = pmix_atomic_cmpset_32(addr, oldval, newval);
|
||||
rc = pmix_atomic_compare_exchange_strong_32 (addr, oldval, newval);
|
||||
pmix_atomic_rmb();
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
static inline int pmix_atomic_cmpset_rel_32(volatile int32_t *addr,
|
||||
int32_t oldval, int32_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
pmix_atomic_wmb();
|
||||
return pmix_atomic_cmpset_32(addr, oldval, newval);
|
||||
return pmix_atomic_compare_exchange_strong_32 (addr, oldval, newval);
|
||||
}
|
||||
|
||||
static inline int32_t pmix_atomic_swap_32(volatile int32_t *addr, int32_t newval)
|
||||
@ -230,106 +239,77 @@ static inline int32_t pmix_atomic_swap_32(volatile int32_t *addr, int32_t newval
|
||||
|
||||
#if PMIX_GCC_INLINE_ASSEMBLY
|
||||
|
||||
static inline int64_t pmix_atomic_add_64 (volatile int64_t* v, int64_t inc)
|
||||
{
|
||||
int64_t t;
|
||||
|
||||
__asm__ __volatile__("1: ldarx %0, 0, %3 \n\t"
|
||||
" add %0, %2, %0 \n\t"
|
||||
" stdcx. %0, 0, %3 \n\t"
|
||||
" bne- 1b \n\t"
|
||||
: "=&r" (t), "=m" (*v)
|
||||
: "r" (PMIX_ASM_VALUE64(inc)), "r" PMIX_ASM_ADDR(v), "m" (*v)
|
||||
: "cc");
|
||||
|
||||
return t;
|
||||
#define PMIX_ATOMIC_POWERPC_DEFINE_ATOMIC_64(type, instr) \
|
||||
static inline int64_t pmix_atomic_fetch_ ## type ## _64(volatile int64_t* v, int64_t val) \
|
||||
{ \
|
||||
int64_t t, old; \
|
||||
\
|
||||
__asm__ __volatile__( \
|
||||
"1: ldarx %1, 0, %4 \n\t" \
|
||||
" " #instr " %0, %3, %1 \n\t" \
|
||||
" stdcx. %0, 0, %4 \n\t" \
|
||||
" bne- 1b \n\t" \
|
||||
: "=&r" (t), "=&r" (old), "=m" (*v) \
|
||||
: "r" (PMIX_ASM_VALUE64(val)), "r" PMIX_ASM_ADDR(v), "m" (*v) \
|
||||
: "cc"); \
|
||||
\
|
||||
return old; \
|
||||
}
|
||||
|
||||
PMIX_ATOMIC_POWERPC_DEFINE_ATOMIC_64(add, add)
|
||||
PMIX_ATOMIC_POWERPC_DEFINE_ATOMIC_64(and, and)
|
||||
PMIX_ATOMIC_POWERPC_DEFINE_ATOMIC_64(or, or)
|
||||
PMIX_ATOMIC_POWERPC_DEFINE_ATOMIC_64(xor, xor)
|
||||
PMIX_ATOMIC_POWERPC_DEFINE_ATOMIC_64(sub, subf)
|
||||
|
||||
static inline int64_t pmix_atomic_sub_64 (volatile int64_t* v, int64_t dec)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
int64_t t;
|
||||
int64_t prev;
|
||||
bool ret;
|
||||
|
||||
__asm__ __volatile__(
|
||||
"1: ldarx %0,0,%3 \n\t"
|
||||
" subf %0,%2,%0 \n\t"
|
||||
" stdcx. %0,0,%3 \n\t"
|
||||
" bne- 1b \n\t"
|
||||
: "=&r" (t), "=m" (*v)
|
||||
: "r" (PMIX_ASM_VALUE64(dec)), "r" PMIX_ASM_ADDR(v), "m" (*v)
|
||||
: "cc");
|
||||
|
||||
return t;
|
||||
}
|
||||
|
||||
static inline int pmix_atomic_cmpset_64(volatile int64_t *addr,
|
||||
int64_t oldval, int64_t newval)
|
||||
{
|
||||
int64_t ret;
|
||||
|
||||
__asm__ __volatile__ (
|
||||
"1: ldarx %0, 0, %2 \n\t"
|
||||
" cmpd 0, %0, %3 \n\t"
|
||||
" bne- 2f \n\t"
|
||||
" stdcx. %4, 0, %2 \n\t"
|
||||
" bne- 1b \n\t"
|
||||
"2:"
|
||||
: "=&r" (ret), "=m" (*addr)
|
||||
: "r" (addr), "r" (PMIX_ASM_VALUE64(oldval)), "r" (PMIX_ASM_VALUE64(newval)), "m" (*addr)
|
||||
: "cc", "memory");
|
||||
|
||||
return (ret == oldval);
|
||||
}
|
||||
|
||||
static inline int64_t pmix_atomic_ll_64(volatile int64_t *addr)
|
||||
{
|
||||
int64_t ret;
|
||||
|
||||
__asm__ __volatile__ ("ldarx %0, 0, %1 \n\t"
|
||||
: "=&r" (ret)
|
||||
: "r" (addr)
|
||||
);
|
||||
return ret;
|
||||
}
|
||||
|
||||
static inline int pmix_atomic_sc_64(volatile int64_t *addr, int64_t newval)
|
||||
{
|
||||
int32_t ret;
|
||||
|
||||
__asm__ __volatile__ (" stdcx. %2, 0, %1 \n\t"
|
||||
" li %0,0 \n\t"
|
||||
" bne- 1f \n\t"
|
||||
" ori %0,%0,1 \n\t"
|
||||
"1:"
|
||||
: "=r" (ret)
|
||||
: "r" (addr), "r" (PMIX_ASM_VALUE64(newval))
|
||||
__asm__ __volatile__ (
|
||||
"1: ldarx %0, 0, %2 \n\t"
|
||||
" cmpd 0, %0, %3 \n\t"
|
||||
" bne- 2f \n\t"
|
||||
" stdcx. %4, 0, %2 \n\t"
|
||||
" bne- 1b \n\t"
|
||||
"2:"
|
||||
: "=&r" (prev), "=m" (*addr)
|
||||
: "r" (addr), "r" (PMIX_ASM_VALUE64(*oldval)), "r" (PMIX_ASM_VALUE64(newval)), "m" (*addr)
|
||||
: "cc", "memory");
|
||||
|
||||
ret = (prev == *oldval);
|
||||
*oldval = prev;
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* these two functions aren't inlined in the non-gcc case because then
|
||||
there would be two function calls (since neither cmpset_64 nor
|
||||
atomic_?mb can be inlined). Instead, we "inline" them by hand in
|
||||
the assembly, meaning there is one function call overhead instead
|
||||
of two */
|
||||
static inline int pmix_atomic_cmpset_acq_64(volatile int64_t *addr,
|
||||
int64_t oldval, int64_t newval)
|
||||
{
|
||||
int rc;
|
||||
#define pmix_atomic_ll_64(addr, ret) \
|
||||
do { \
|
||||
volatile int64_t *_addr = (addr); \
|
||||
int64_t _ret; \
|
||||
__asm__ __volatile__ ("ldarx %0, 0, %1 \n\t" \
|
||||
: "=&r" (_ret) \
|
||||
: "r" (_addr) \
|
||||
); \
|
||||
ret = (typeof(ret)) _ret; \
|
||||
} while (0)
|
||||
|
||||
rc = pmix_atomic_cmpset_64(addr, oldval, newval);
|
||||
pmix_atomic_rmb();
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
static inline int pmix_atomic_cmpset_rel_64(volatile int64_t *addr,
|
||||
int64_t oldval, int64_t newval)
|
||||
{
|
||||
pmix_atomic_wmb();
|
||||
return pmix_atomic_cmpset_64(addr, oldval, newval);
|
||||
}
|
||||
#define pmix_atomic_sc_64(addr, value, ret) \
|
||||
do { \
|
||||
volatile int64_t *_addr = (addr); \
|
||||
int64_t _foo, _newval = (int64_t) value; \
|
||||
int32_t _ret; \
|
||||
\
|
||||
__asm__ __volatile__ (" stdcx. %2, 0, %1 \n\t" \
|
||||
" li %0,0 \n\t" \
|
||||
" bne- 1f \n\t" \
|
||||
" ori %0,%0,1 \n\t" \
|
||||
"1:" \
|
||||
: "=r" (_ret) \
|
||||
: "r" (_addr), "r" (PMIX_ASM_VALUE64(_newval)) \
|
||||
: "cc", "memory"); \
|
||||
ret = _ret; \
|
||||
} while (0)
|
||||
|
||||
static inline int64_t pmix_atomic_swap_64(volatile int64_t *addr, int64_t newval)
|
||||
{
|
||||
@ -356,9 +336,9 @@ static inline int64_t pmix_atomic_swap_64(volatile int64_t *addr, int64_t newval
|
||||
|
||||
#if PMIX_GCC_INLINE_ASSEMBLY
|
||||
|
||||
static inline int pmix_atomic_cmpset_64(volatile int64_t *addr,
|
||||
int64_t oldval, int64_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
int64_t prev;
|
||||
int ret;
|
||||
|
||||
/*
|
||||
@ -373,90 +353,77 @@ static inline int pmix_atomic_cmpset_64(volatile int64_t *addr,
|
||||
* is very similar to the pure 64 bit version.
|
||||
*/
|
||||
__asm__ __volatile__ (
|
||||
"ld r4,%2 \n\t"
|
||||
"ld r5,%3 \n\t"
|
||||
"1: ldarx r9, 0, %1 \n\t"
|
||||
" cmpd 0, r9, r4 \n\t"
|
||||
"ld r4,%3 \n\t"
|
||||
"ld r5,%4 \n\t"
|
||||
"1: ldarx %1, 0, %2 \n\t"
|
||||
" cmpd 0, %1, r4 \n\t"
|
||||
" bne- 2f \n\t"
|
||||
" stdcx. r5, 0, %1 \n\t"
|
||||
" stdcx. r5, 0, %2 \n\t"
|
||||
" bne- 1b \n\t"
|
||||
"2: \n\t"
|
||||
"xor r5,r4,r9 \n\t"
|
||||
"xor r5,r4,%1 \n\t"
|
||||
"subfic r9,r5,0 \n\t"
|
||||
"adde %0,r9,r5 \n\t"
|
||||
: "=&r" (ret)
|
||||
: "=&r" (ret), "+r" (prev)
|
||||
: "r"PMIX_ASM_ADDR(addr),
|
||||
"m"(oldval), "m"(newval)
|
||||
"m"(*oldval), "m"(newval)
|
||||
: "r4", "r5", "r9", "cc", "memory");
|
||||
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* these two functions aren't inlined in the non-gcc case because then
|
||||
there would be two function calls (since neither cmpset_64 nor
|
||||
atomic_?mb can be inlined). Instead, we "inline" them by hand in
|
||||
the assembly, meaning there is one function call overhead instead
|
||||
of two */
|
||||
static inline int pmix_atomic_cmpset_acq_64(volatile int64_t *addr,
|
||||
int64_t oldval, int64_t newval)
|
||||
{
|
||||
int rc;
|
||||
|
||||
rc = pmix_atomic_cmpset_64(addr, oldval, newval);
|
||||
pmix_atomic_rmb();
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
static inline int pmix_atomic_cmpset_rel_64(volatile int64_t *addr,
|
||||
int64_t oldval, int64_t newval)
|
||||
{
|
||||
pmix_atomic_wmb();
|
||||
return pmix_atomic_cmpset_64(addr, oldval, newval);
|
||||
*oldval = prev;
|
||||
return (bool) ret;
|
||||
}
|
||||
|
||||
#endif /* PMIX_GCC_INLINE_ASSEMBLY */
|
||||
|
||||
#endif /* PMIX_ASM_SUPPORT_64BIT */
|
||||
|
||||
|
||||
#if PMIX_GCC_INLINE_ASSEMBLY
|
||||
|
||||
static inline int32_t pmix_atomic_add_32(volatile int32_t* v, int inc)
|
||||
/* these two functions aren't inlined in the non-gcc case because then
|
||||
there would be two function calls (since neither cmpset_64 nor
|
||||
atomic_?mb can be inlined). Instead, we "inline" them by hand in
|
||||
the assembly, meaning there is one function call overhead instead
|
||||
of two */
|
||||
static inline bool pmix_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
int32_t t;
|
||||
bool rc;
|
||||
|
||||
__asm__ __volatile__(
|
||||
"1: lwarx %0, 0, %3 \n\t"
|
||||
" add %0, %2, %0 \n\t"
|
||||
" stwcx. %0, 0, %3 \n\t"
|
||||
" bne- 1b \n\t"
|
||||
: "=&r" (t), "=m" (*v)
|
||||
: "r" (inc), "r" PMIX_ASM_ADDR(v), "m" (*v)
|
||||
: "cc");
|
||||
rc = pmix_atomic_compare_exchange_strong_64 (addr, oldval, newval);
|
||||
pmix_atomic_rmb();
|
||||
|
||||
return t;
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
static inline int32_t pmix_atomic_sub_32(volatile int32_t* v, int dec)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
int32_t t;
|
||||
|
||||
__asm__ __volatile__(
|
||||
"1: lwarx %0,0,%3 \n\t"
|
||||
" subf %0,%2,%0 \n\t"
|
||||
" stwcx. %0,0,%3 \n\t"
|
||||
" bne- 1b \n\t"
|
||||
: "=&r" (t), "=m" (*v)
|
||||
: "r" (dec), "r" PMIX_ASM_ADDR(v), "m" (*v)
|
||||
: "cc");
|
||||
|
||||
return t;
|
||||
pmix_atomic_wmb();
|
||||
return pmix_atomic_compare_exchange_strong_64 (addr, oldval, newval);
|
||||
}
|
||||
|
||||
|
||||
#define PMIX_ATOMIC_POWERPC_DEFINE_ATOMIC_32(type, instr) \
|
||||
static inline int32_t pmix_atomic_fetch_ ## type ## _32(volatile int32_t* v, int val) \
|
||||
{ \
|
||||
int32_t t, old; \
|
||||
\
|
||||
__asm__ __volatile__( \
|
||||
"1: lwarx %1, 0, %4 \n\t" \
|
||||
" " #instr " %0, %3, %1 \n\t" \
|
||||
" stwcx. %0, 0, %4 \n\t" \
|
||||
" bne- 1b \n\t" \
|
||||
: "=&r" (t), "=&r" (old), "=m" (*v) \
|
||||
: "r" (val), "r" PMIX_ASM_ADDR(v), "m" (*v) \
|
||||
: "cc"); \
|
||||
\
|
||||
return old; \
|
||||
}
|
||||
|
||||
PMIX_ATOMIC_POWERPC_DEFINE_ATOMIC_32(add, add)
|
||||
PMIX_ATOMIC_POWERPC_DEFINE_ATOMIC_32(and, and)
|
||||
PMIX_ATOMIC_POWERPC_DEFINE_ATOMIC_32(or, or)
|
||||
PMIX_ATOMIC_POWERPC_DEFINE_ATOMIC_32(xor, xor)
|
||||
PMIX_ATOMIC_POWERPC_DEFINE_ATOMIC_32(sub, subf)
|
||||
|
||||
#endif /* PMIX_GCC_INLINE_ASSEMBLY */
|
||||
|
||||
#endif /* ! PMIX_SYS_ARCH_ATOMIC_H */
|
||||
|
||||
@ -1,3 +1,4 @@
|
||||
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
@ -12,7 +13,9 @@
|
||||
* Copyright (c) 2007 Sun Microsystems, Inc. All rights reserverd.
|
||||
* Copyright (c) 2016 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* Copyright (c) 2017 Intel, Inc. All rights reserved.
|
||||
* Copyright (c) 2017 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2018 Intel, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -27,9 +30,9 @@
|
||||
* On sparc v9, use casa and casxa (compare and swap) instructions.
|
||||
*/
|
||||
|
||||
#define PMIXASI_P "0x80"
|
||||
#define ASI_P "0x80"
|
||||
|
||||
#define PMIXMEMBAR(type) __asm__ __volatile__ ("membar " type : : : "memory")
|
||||
#define MEMBAR(type) __asm__ __volatile__ ("membar " type : : : "memory")
|
||||
|
||||
|
||||
/**********************************************************************
|
||||
@ -39,9 +42,9 @@
|
||||
*********************************************************************/
|
||||
#define PMIX_HAVE_ATOMIC_MEM_BARRIER 1
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_CMPSET_32 1
|
||||
#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_CMPSET_64 1
|
||||
#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
|
||||
|
||||
|
||||
/**********************************************************************
|
||||
@ -53,19 +56,19 @@
|
||||
|
||||
static inline void pmix_atomic_mb(void)
|
||||
{
|
||||
PMIXMEMBAR("#LoadLoad | #LoadStore | #StoreStore | #StoreLoad");
|
||||
MEMBAR("#LoadLoad | #LoadStore | #StoreStore | #StoreLoad");
|
||||
}
|
||||
|
||||
|
||||
static inline void pmix_atomic_rmb(void)
|
||||
{
|
||||
PMIXMEMBAR("#LoadLoad");
|
||||
MEMBAR("#LoadLoad");
|
||||
}
|
||||
|
||||
|
||||
static inline void pmix_atomic_wmb(void)
|
||||
{
|
||||
PMIXMEMBAR("#StoreStore");
|
||||
MEMBAR("#StoreStore");
|
||||
}
|
||||
|
||||
static inline void pmix_atomic_isync(void)
|
||||
@ -83,50 +86,49 @@ static inline void pmix_atomic_isync(void)
|
||||
*********************************************************************/
|
||||
#if PMIX_GCC_INLINE_ASSEMBLY
|
||||
|
||||
static inline int pmix_atomic_cmpset_32( volatile int32_t *addr,
|
||||
int32_t oldval, int32_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
/* casa [reg(rs1)] %asi, reg(rs2), reg(rd)
|
||||
*
|
||||
* if (*(reg(rs1)) == reg(rs2) )
|
||||
* swap reg(rd), *(reg(rs1))
|
||||
* else
|
||||
* reg(rd) = *(reg(rs1))
|
||||
*/
|
||||
/* casa [reg(rs1)] %asi, reg(rs2), reg(rd)
|
||||
*
|
||||
* if (*(reg(rs1)) == reg(rs2) )
|
||||
* swap reg(rd), *(reg(rs1))
|
||||
* else
|
||||
* reg(rd) = *(reg(rs1))
|
||||
*/
|
||||
|
||||
int32_t ret = newval;
|
||||
int32_t prev = newval;
|
||||
bool ret;
|
||||
|
||||
__asm__ __volatile__("casa [%1] " PMIXASI_P ", %2, %0"
|
||||
: "+r" (ret)
|
||||
: "r" (addr), "r" (oldval));
|
||||
return (ret == oldval);
|
||||
__asm__ __volatile__("casa [%1] " ASI_P ", %2, %0"
|
||||
: "+r" (prev)
|
||||
: "r" (addr), "r" (*oldval));
|
||||
ret = (prev == *oldval);
|
||||
*oldval = prev;
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
static inline int pmix_atomic_cmpset_acq_32( volatile int32_t *addr,
|
||||
int32_t oldval, int32_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
int rc;
|
||||
bool rc;
|
||||
|
||||
rc = pmix_atomic_cmpset_32(addr, oldval, newval);
|
||||
pmix_atomic_rmb();
|
||||
rc = pmix_atomic_compare_exchange_strong_32 (addr, oldval, newval);
|
||||
pmix_atomic_rmb();
|
||||
|
||||
return rc;
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
static inline int pmix_atomic_cmpset_rel_32( volatile int32_t *addr,
|
||||
int32_t oldval, int32_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
pmix_atomic_wmb();
|
||||
return pmix_atomic_cmpset_32(addr, oldval, newval);
|
||||
pmix_atomic_wmb();
|
||||
return pmix_atomic_compare_exchange_strong_32 (addr, oldval, newval);
|
||||
}
|
||||
|
||||
|
||||
#if PMIX_ASSEMBLY_ARCH == PMIX_SPARCV9_64
|
||||
|
||||
static inline int pmix_atomic_cmpset_64( volatile int64_t *addr,
|
||||
int64_t oldval, int64_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
/* casa [reg(rs1)] %asi, reg(rs2), reg(rd)
|
||||
*
|
||||
@ -135,18 +137,20 @@ static inline int pmix_atomic_cmpset_64( volatile int64_t *addr,
|
||||
* else
|
||||
* reg(rd) = *(reg(rs1))
|
||||
*/
|
||||
int64_t ret = newval;
|
||||
int64_t prev = newval;
|
||||
bool ret;
|
||||
|
||||
__asm__ __volatile__("casxa [%1] " PMIXASI_P ", %2, %0"
|
||||
: "+r" (ret)
|
||||
: "r" (addr), "r" (oldval));
|
||||
return (ret == oldval);
|
||||
__asm__ __volatile__("casxa [%1] " ASI_P ", %2, %0"
|
||||
: "+r" (prev)
|
||||
: "r" (addr), "r" (*oldval));
|
||||
ret = (prev == *oldval);
|
||||
*oldval = prev;
|
||||
return ret;
|
||||
}
|
||||
|
||||
#else /* PMIX_ASSEMBLY_ARCH == PMIX_SPARCV9_64 */
|
||||
|
||||
static inline int pmix_atomic_cmpset_64( volatile int64_t *addr,
|
||||
int64_t oldval, int64_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
/* casa [reg(rs1)] %asi, reg(rs2), reg(rd)
|
||||
*
|
||||
@ -156,40 +160,41 @@ static inline int pmix_atomic_cmpset_64( volatile int64_t *addr,
|
||||
* reg(rd) = *(reg(rs1))
|
||||
*
|
||||
*/
|
||||
long long ret = newval;
|
||||
int64_t prev = newval;
|
||||
bool ret;
|
||||
|
||||
__asm__ __volatile__(
|
||||
"ldx %0, %%g1 \n\t" /* g1 = ret */
|
||||
"ldx %2, %%g2 \n\t" /* g2 = oldval */
|
||||
"casxa [%1] " PMIXASI_P ", %%g2, %%g1 \n\t"
|
||||
"casxa [%1] " ASI_P ", %%g2, %%g1 \n\t"
|
||||
"stx %%g1, %0 \n"
|
||||
: "+m"(ret)
|
||||
: "r"(addr), "m"(oldval)
|
||||
: "+m"(prev)
|
||||
: "r"(addr), "m"(*oldval)
|
||||
: "%g1", "%g2"
|
||||
);
|
||||
|
||||
return (ret == oldval);
|
||||
ret = (prev == *oldval);
|
||||
*oldval = prev;
|
||||
return ret;
|
||||
}
|
||||
|
||||
#endif /* PMIX_ASSEMBLY_ARCH == PMIX_SPARCV9_64 */
|
||||
|
||||
static inline int pmix_atomic_cmpset_acq_64( volatile int64_t *addr,
|
||||
int64_t oldval, int64_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
int rc;
|
||||
bool rc;
|
||||
|
||||
rc = pmix_atomic_cmpset_64(addr, oldval, newval);
|
||||
pmix_atomic_rmb();
|
||||
rc = pmix_atomic_compare_exchange_strong_64 (addr, oldval, newval);
|
||||
pmix_atomic_rmb();
|
||||
|
||||
return rc;
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
static inline int pmix_atomic_cmpset_rel_64( volatile int64_t *addr,
|
||||
int64_t oldval, int64_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
pmix_atomic_wmb();
|
||||
return pmix_atomic_cmpset_64(addr, oldval, newval);
|
||||
pmix_atomic_wmb();
|
||||
return pmix_atomic_compare_exchange_strong_64 (addr, oldval, newval);
|
||||
}
|
||||
|
||||
#endif /* PMIX_GCC_INLINE_ASSEMBLY */
|
||||
|
||||
@ -11,9 +11,11 @@
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2011 Sandia National Laboratories. All rights reserved.
|
||||
* Copyright (c) 2014-2016 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2017 Intel, Inc. All rights reserved.
|
||||
* Copyright (c) 2017 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* Copyright (c) 2018 Intel, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -54,83 +56,110 @@ static inline void pmix_atomic_wmb(void)
|
||||
*
|
||||
*********************************************************************/
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_CMPSET_32 1
|
||||
static inline int pmix_atomic_cmpset_acq_32( volatile int32_t *addr,
|
||||
int32_t oldval, int32_t newval)
|
||||
#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
|
||||
|
||||
static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
return __sync_bool_compare_and_swap(addr, oldval, newval);
|
||||
int32_t prev = __sync_val_compare_and_swap (addr, *oldval, newval);
|
||||
bool ret = prev == *oldval;
|
||||
*oldval = prev;
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
static inline int pmix_atomic_cmpset_rel_32( volatile int32_t *addr,
|
||||
int32_t oldval, int32_t newval)
|
||||
{
|
||||
return __sync_bool_compare_and_swap(addr, oldval, newval);}
|
||||
|
||||
static inline int pmix_atomic_cmpset_32( volatile int32_t *addr,
|
||||
int32_t oldval, int32_t newval)
|
||||
{
|
||||
return __sync_bool_compare_and_swap(addr, oldval, newval);
|
||||
}
|
||||
#define pmix_atomic_compare_exchange_strong_acq_32 pmix_atomic_compare_exchange_strong_32
|
||||
#define pmix_atomic_compare_exchange_strong_rel_32 pmix_atomic_compare_exchange_strong_32
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_MATH_32 1
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_ADD_32 1
|
||||
static inline int32_t pmix_atomic_add_32(volatile int32_t *addr, int32_t delta)
|
||||
static inline int32_t pmix_atomic_fetch_add_32(volatile int32_t *addr, int32_t delta)
|
||||
{
|
||||
return __sync_add_and_fetch(addr, delta);
|
||||
return __sync_fetch_and_add(addr, delta);
|
||||
}
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_AND_32 1
|
||||
static inline int32_t pmix_atomic_fetch_and_32(volatile int32_t *addr, int32_t value)
|
||||
{
|
||||
return __sync_fetch_and_and(addr, value);
|
||||
}
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_OR_32 1
|
||||
static inline int32_t pmix_atomic_fetch_or_32(volatile int32_t *addr, int32_t value)
|
||||
{
|
||||
return __sync_fetch_and_or(addr, value);
|
||||
}
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_XOR_32 1
|
||||
static inline int32_t pmix_atomic_fetch_xor_32(volatile int32_t *addr, int32_t value)
|
||||
{
|
||||
return __sync_fetch_and_xor(addr, value);
|
||||
}
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_SUB_32 1
|
||||
static inline int32_t pmix_atomic_sub_32(volatile int32_t *addr, int32_t delta)
|
||||
static inline int32_t pmix_atomic_fetch_sub_32(volatile int32_t *addr, int32_t delta)
|
||||
{
|
||||
return __sync_sub_and_fetch(addr, delta);
|
||||
return __sync_fetch_and_sub(addr, delta);
|
||||
}
|
||||
|
||||
#if PMIX_ASM_SYNC_HAVE_64BIT
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_CMPSET_64 1
|
||||
static inline int pmix_atomic_cmpset_acq_64( volatile int64_t *addr,
|
||||
int64_t oldval, int64_t newval)
|
||||
#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
|
||||
|
||||
static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
return __sync_bool_compare_and_swap(addr, oldval, newval);
|
||||
int64_t prev = __sync_val_compare_and_swap (addr, *oldval, newval);
|
||||
bool ret = prev == *oldval;
|
||||
*oldval = prev;
|
||||
return ret;
|
||||
}
|
||||
|
||||
static inline int pmix_atomic_cmpset_rel_64( volatile int64_t *addr,
|
||||
int64_t oldval, int64_t newval)
|
||||
{
|
||||
return __sync_bool_compare_and_swap(addr, oldval, newval);}
|
||||
|
||||
|
||||
static inline int pmix_atomic_cmpset_64( volatile int64_t *addr,
|
||||
int64_t oldval, int64_t newval)
|
||||
{
|
||||
return __sync_bool_compare_and_swap(addr, oldval, newval);
|
||||
}
|
||||
#define pmix_atomic_compare_exchange_strong_acq_64 pmix_atomic_compare_exchange_strong_64
|
||||
#define pmix_atomic_compare_exchange_strong_rel_64 pmix_atomic_compare_exchange_strong_64
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_MATH_64 1
|
||||
#define PMIX_HAVE_ATOMIC_ADD_64 1
|
||||
static inline int64_t pmix_atomic_add_64(volatile int64_t *addr, int64_t delta)
|
||||
static inline int64_t pmix_atomic_fetch_add_64(volatile int64_t *addr, int64_t delta)
|
||||
{
|
||||
return __sync_add_and_fetch(addr, delta);
|
||||
return __sync_fetch_and_add(addr, delta);
|
||||
}
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_AND_64 1
|
||||
static inline int64_t pmix_atomic_fetch_and_64(volatile int64_t *addr, int64_t value)
|
||||
{
|
||||
return __sync_fetch_and_and(addr, value);
|
||||
}
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_OR_64 1
|
||||
static inline int64_t pmix_atomic_fetch_or_64(volatile int64_t *addr, int64_t value)
|
||||
{
|
||||
return __sync_fetch_and_or(addr, value);
|
||||
}
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_XOR_64 1
|
||||
static inline int64_t pmix_atomic_fetch_xor_64(volatile int64_t *addr, int64_t value)
|
||||
{
|
||||
return __sync_fetch_and_xor(addr, value);
|
||||
}
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_SUB_64 1
|
||||
static inline int64_t pmix_atomic_sub_64(volatile int64_t *addr, int64_t delta)
|
||||
static inline int64_t pmix_atomic_fetch_sub_64(volatile int64_t *addr, int64_t delta)
|
||||
{
|
||||
return __sync_sub_and_fetch(addr, delta);
|
||||
return __sync_fetch_and_sub(addr, delta);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
#if PMIX_HAVE_SYNC_BUILTIN_CSWAP_INT128
|
||||
static inline int pmix_atomic_cmpset_128 (volatile pmix_int128_t *addr,
|
||||
pmix_int128_t oldval, pmix_int128_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_128 (volatile pmix_int128_t *addr,
|
||||
pmix_int128_t *oldval, pmix_int128_t newval)
|
||||
{
|
||||
return __sync_bool_compare_and_swap(addr, oldval, newval);
|
||||
pmix_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval);
|
||||
bool ret = prev == *oldval;
|
||||
*oldval = prev;
|
||||
return ret;
|
||||
}
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_CMPSET_128 1
|
||||
#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
|
||||
|
||||
#endif
|
||||
|
||||
|
||||
@ -11,9 +11,9 @@
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2016 Broadcom Limited. All rights reserved.
|
||||
* Copyright (c) 2016 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2017 Intel, Inc. All rights reserved.
|
||||
* Copyright (c) 2018 Intel, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -67,7 +67,7 @@
|
||||
BEGIN_C_DECLS
|
||||
|
||||
/* If you update this list, you probably also want to update
|
||||
src/mca/timer/linux/configure.m4. Or not. */
|
||||
pmix/mca/timer/linux/configure.m4. Or not. */
|
||||
|
||||
#if defined(DOXYGEN)
|
||||
/* don't include system-level gorp when generating doxygen files */
|
||||
|
||||
@ -11,11 +11,11 @@
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2007 Sun Microsystems, Inc. All rights reserverd.
|
||||
* Copyright (c) 2012-2014 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2016-2017 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* Copyright (c) 2017 Intel, Inc. All rights reserved.
|
||||
* Copyright (c) 2018 Intel, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -30,7 +30,7 @@
|
||||
*/
|
||||
|
||||
|
||||
#define PMIXSMPLOCK "lock; "
|
||||
#define SMPLOCK "lock; "
|
||||
#define PMIXMB() __asm__ __volatile__("": : :"memory")
|
||||
|
||||
|
||||
@ -41,9 +41,9 @@
|
||||
*********************************************************************/
|
||||
#define PMIX_HAVE_ATOMIC_MEM_BARRIER 1
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_CMPSET_32 1
|
||||
#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_CMPSET_64 1
|
||||
#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
|
||||
|
||||
/**********************************************************************
|
||||
*
|
||||
@ -83,68 +83,64 @@ static inline void pmix_atomic_isync(void)
|
||||
*********************************************************************/
|
||||
#if PMIX_GCC_INLINE_ASSEMBLY
|
||||
|
||||
static inline int pmix_atomic_cmpset_32( volatile int32_t *addr,
|
||||
int32_t oldval, int32_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
unsigned char ret;
|
||||
__asm__ __volatile__ (
|
||||
PMIXSMPLOCK "cmpxchgl %3,%2 \n\t"
|
||||
SMPLOCK "cmpxchgl %3,%2 \n\t"
|
||||
"sete %0 \n\t"
|
||||
: "=qm" (ret), "+a" (oldval), "+m" (*addr)
|
||||
: "=qm" (ret), "+a" (*oldval), "+m" (*addr)
|
||||
: "q"(newval)
|
||||
: "memory", "cc");
|
||||
|
||||
return (int)ret;
|
||||
return (bool) ret;
|
||||
}
|
||||
|
||||
#endif /* PMIX_GCC_INLINE_ASSEMBLY */
|
||||
|
||||
#define pmix_atomic_cmpset_acq_32 pmix_atomic_cmpset_32
|
||||
#define pmix_atomic_cmpset_rel_32 pmix_atomic_cmpset_32
|
||||
#define pmix_atomic_compare_exchange_strong_acq_32 pmix_atomic_compare_exchange_strong_32
|
||||
#define pmix_atomic_compare_exchange_strong_rel_32 pmix_atomic_compare_exchange_strong_32
|
||||
|
||||
#if PMIX_GCC_INLINE_ASSEMBLY
|
||||
|
||||
static inline int pmix_atomic_cmpset_64( volatile int64_t *addr,
|
||||
int64_t oldval, int64_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
unsigned char ret;
|
||||
__asm__ __volatile__ (
|
||||
PMIXSMPLOCK "cmpxchgq %3,%2 \n\t"
|
||||
SMPLOCK "cmpxchgq %3,%2 \n\t"
|
||||
"sete %0 \n\t"
|
||||
: "=qm" (ret), "+a" (oldval), "+m" (*((volatile long*)addr))
|
||||
: "=qm" (ret), "+a" (*oldval), "+m" (*((volatile long*)addr))
|
||||
: "q"(newval)
|
||||
: "memory", "cc"
|
||||
);
|
||||
|
||||
return (int)ret;
|
||||
return (bool) ret;
|
||||
}
|
||||
|
||||
#endif /* PMIX_GCC_INLINE_ASSEMBLY */
|
||||
|
||||
#define pmix_atomic_cmpset_acq_64 pmix_atomic_cmpset_64
|
||||
#define pmix_atomic_cmpset_rel_64 pmix_atomic_cmpset_64
|
||||
#define pmix_atomic_compare_exchange_strong_acq_64 pmix_atomic_compare_exchange_strong_64
|
||||
#define pmix_atomic_compare_exchange_strong_rel_64 pmix_atomic_compare_exchange_strong_64
|
||||
|
||||
#if PMIX_GCC_INLINE_ASSEMBLY && PMIX_HAVE_CMPXCHG16B && HAVE_PMIX_INT128_T
|
||||
|
||||
static inline int pmix_atomic_cmpset_128 (volatile pmix_int128_t *addr, pmix_int128_t oldval,
|
||||
pmix_int128_t newval)
|
||||
static inline bool pmix_atomic_compare_exchange_strong_128 (volatile pmix_int128_t *addr, pmix_int128_t *oldval, pmix_int128_t newval)
|
||||
{
|
||||
unsigned char ret;
|
||||
|
||||
/* cmpxchg16b compares the value at the address with eax:edx (low:high). if the values are
|
||||
* the same the contents of ebx:ecx are stores at the address. in all cases the value stored
|
||||
* at the address is returned in eax:edx. */
|
||||
__asm__ __volatile__ (PMIXSMPLOCK "cmpxchg16b (%%rsi) \n\t"
|
||||
__asm__ __volatile__ (SMPLOCK "cmpxchg16b (%%rsi) \n\t"
|
||||
"sete %0 \n\t"
|
||||
: "=qm" (ret)
|
||||
: "S" (addr), "b" (((int64_t *)&newval)[0]), "c" (((int64_t *)&newval)[1]),
|
||||
"a" (((int64_t *)&oldval)[0]), "d" (((int64_t *)&oldval)[1])
|
||||
: "=qm" (ret), "+a" (((int64_t *)oldval)[0]), "+d" (((int64_t *)oldval)[1])
|
||||
: "S" (addr), "b" (((int64_t *)&newval)[0]), "c" (((int64_t *)&newval)[1])
|
||||
: "memory", "cc");
|
||||
|
||||
return (int) ret;
|
||||
return (bool) ret;
|
||||
}
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_CMPSET_128 1
|
||||
#define PMIX_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
|
||||
|
||||
#endif /* PMIX_GCC_INLINE_ASSEMBLY */
|
||||
|
||||
@ -201,16 +197,16 @@ static inline int64_t pmix_atomic_swap_64( volatile int64_t *addr,
|
||||
*
|
||||
* Atomically adds @i to @v.
|
||||
*/
|
||||
static inline int32_t pmix_atomic_add_32(volatile int32_t* v, int i)
|
||||
static inline int32_t pmix_atomic_fetch_add_32(volatile int32_t* v, int i)
|
||||
{
|
||||
int ret = i;
|
||||
__asm__ __volatile__(
|
||||
PMIXSMPLOCK "xaddl %1,%0"
|
||||
SMPLOCK "xaddl %1,%0"
|
||||
:"+m" (*v), "+r" (ret)
|
||||
:
|
||||
:"memory", "cc"
|
||||
);
|
||||
return (ret+i);
|
||||
return ret;
|
||||
}
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_ADD_64 1
|
||||
@ -222,16 +218,16 @@ static inline int32_t pmix_atomic_add_32(volatile int32_t* v, int i)
|
||||
*
|
||||
* Atomically adds @i to @v.
|
||||
*/
|
||||
static inline int64_t pmix_atomic_add_64(volatile int64_t* v, int64_t i)
|
||||
static inline int64_t pmix_atomic_fetch_add_64(volatile int64_t* v, int64_t i)
|
||||
{
|
||||
int64_t ret = i;
|
||||
__asm__ __volatile__(
|
||||
PMIXSMPLOCK "xaddq %1,%0"
|
||||
SMPLOCK "xaddq %1,%0"
|
||||
:"+m" (*v), "+r" (ret)
|
||||
:
|
||||
:"memory", "cc"
|
||||
);
|
||||
return (ret+i);
|
||||
return ret;
|
||||
}
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_SUB_32 1
|
||||
@ -243,16 +239,16 @@ static inline int64_t pmix_atomic_add_64(volatile int64_t* v, int64_t i)
|
||||
*
|
||||
* Atomically subtracts @i from @v.
|
||||
*/
|
||||
static inline int32_t pmix_atomic_sub_32(volatile int32_t* v, int i)
|
||||
static inline int32_t pmix_atomic_fetch_sub_32(volatile int32_t* v, int i)
|
||||
{
|
||||
int ret = -i;
|
||||
__asm__ __volatile__(
|
||||
PMIXSMPLOCK "xaddl %1,%0"
|
||||
SMPLOCK "xaddl %1,%0"
|
||||
:"+m" (*v), "+r" (ret)
|
||||
:
|
||||
:"memory", "cc"
|
||||
);
|
||||
return (ret-i);
|
||||
return ret;
|
||||
}
|
||||
|
||||
#define PMIX_HAVE_ATOMIC_SUB_64 1
|
||||
@ -264,16 +260,16 @@ static inline int32_t pmix_atomic_sub_32(volatile int32_t* v, int i)
|
||||
*
|
||||
* Atomically subtracts @i from @v.
|
||||
*/
|
||||
static inline int64_t pmix_atomic_sub_64(volatile int64_t* v, int64_t i)
|
||||
static inline int64_t pmix_atomic_fetch_sub_64(volatile int64_t* v, int64_t i)
|
||||
{
|
||||
int64_t ret = -i;
|
||||
__asm__ __volatile__(
|
||||
PMIXSMPLOCK "xaddq %1,%0"
|
||||
SMPLOCK "xaddq %1,%0"
|
||||
:"+m" (*v), "+r" (ret)
|
||||
:
|
||||
:"memory", "cc"
|
||||
);
|
||||
return (ret-i);
|
||||
return ret;
|
||||
}
|
||||
|
||||
#endif /* PMIX_GCC_INLINE_ASSEMBLY */
|
||||
|
||||
@ -12,7 +12,7 @@
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2016 Los Alamos National Security, LLC. ALl rights
|
||||
* reserved.
|
||||
* Copyright (c) 2017 Intel, Inc. All rights reserved.
|
||||
* Copyright (c) 2018 Intel, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
|
||||
@ -11,7 +11,7 @@
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2013-2017 Intel, Inc. All rights reserved.
|
||||
* Copyright (c) 2013-2018 Intel, Inc. All rights reserved.
|
||||
* Copyright (c) 2016 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
@ -498,7 +498,7 @@ static inline pmix_object_t *pmix_obj_new(pmix_class_t * cls)
|
||||
static inline int pmix_obj_update(pmix_object_t *object, int inc) __pmix_attribute_always_inline__;
|
||||
static inline int pmix_obj_update(pmix_object_t *object, int inc)
|
||||
{
|
||||
return PMIX_THREAD_ADD32(&object->obj_reference_count, inc);
|
||||
return PMIX_THREAD_ADD_FETCH32(&object->obj_reference_count, inc);
|
||||
}
|
||||
|
||||
END_C_DECLS
|
||||
|
||||
@ -52,11 +52,6 @@
|
||||
#include PMIX_EVENT_HEADER
|
||||
#include PMIX_EVENT2_THREAD_HEADER
|
||||
|
||||
#if PMIX_CC_USE_PRAGMA_IDENT
|
||||
#pragma ident PMIX_VERSION
|
||||
#elif PMIX_CC_USE_IDENT
|
||||
#ident PMIX_VERSION
|
||||
#endif
|
||||
static const char pmix_version_string[] = PMIX_VERSION;
|
||||
|
||||
|
||||
|
||||
@ -60,11 +60,6 @@
|
||||
#include "src/runtime/pmix_rte.h"
|
||||
#include "src/runtime/pmix_progress_threads.h"
|
||||
|
||||
#if PMIX_CC_USE_PRAGMA_IDENT
|
||||
#pragma ident PMIX_IDENT_STRING
|
||||
#elif PMIX_CC_USE_IDENT
|
||||
#ident PMIX_IDENT_STRING
|
||||
#endif
|
||||
const char pmix_version_string[] = PMIX_IDENT_STRING;
|
||||
|
||||
PMIX_EXPORT int pmix_initialized = 0;
|
||||
|
||||
@ -14,7 +14,7 @@
|
||||
* reserved.
|
||||
* Copyright (c) 2015 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* Copyright (c) 2017 Intel, Inc. All rights reserved.
|
||||
* Copyright (c) 2017-2018 Intel, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -53,7 +53,7 @@ static void pmix_mutex_construct(pmix_mutex_t *m)
|
||||
#endif /* PMIX_ENABLE_DEBUG */
|
||||
|
||||
#if PMIX_HAVE_ATOMIC_SPINLOCKS
|
||||
pmix_atomic_init( &m->m_lock_atomic, PMIX_ATOMIC_UNLOCKED );
|
||||
pmix_atomic_lock_init( &m->m_lock_atomic, PMIX_ATOMIC_LOCK_UNLOCKED );
|
||||
#endif
|
||||
}
|
||||
|
||||
@ -84,7 +84,7 @@ static void pmix_recursive_mutex_construct(pmix_recursive_mutex_t *m)
|
||||
pthread_mutexattr_destroy(&attr);
|
||||
|
||||
#if PMIX_HAVE_ATOMIC_SPINLOCKS
|
||||
pmix_atomic_init( &m->m_lock_atomic, PMIX_ATOMIC_UNLOCKED );
|
||||
pmix_atomic_lock_init( &m->m_lock_atomic, PMIX_ATOMIC_LOCK_UNLOCKED );
|
||||
#endif
|
||||
}
|
||||
|
||||
|
||||
@ -14,7 +14,7 @@
|
||||
* reserved.
|
||||
* Copyright (c) 2015-2016 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* Copyright (c) 2017 Intel, Inc. All rights reserved.
|
||||
* Copyright (c) 2017-2018 Intel, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -77,14 +77,14 @@ PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_recursive_mutex_t);
|
||||
.m_lock_debug = 0, \
|
||||
.m_lock_file = NULL, \
|
||||
.m_lock_line = 0, \
|
||||
.m_lock_atomic = { .u = { .lock = PMIX_ATOMIC_UNLOCKED } }, \
|
||||
.m_lock_atomic = { .u = { .lock = PMIX_ATOMIC_LOCK_UNLOCKED } },\
|
||||
}
|
||||
#else
|
||||
#define PMIX_MUTEX_STATIC_INIT \
|
||||
{ \
|
||||
.super = PMIX_OBJ_STATIC_INIT(pmix_mutex_t), \
|
||||
.m_lock_pthread = PTHREAD_MUTEX_INITIALIZER, \
|
||||
.m_lock_atomic = { .u = { .lock = PMIX_ATOMIC_UNLOCKED } }, \
|
||||
.m_lock_atomic = { .u = { .lock = PMIX_ATOMIC_LOCK_UNLOCKED } },\
|
||||
}
|
||||
#endif
|
||||
|
||||
@ -98,14 +98,14 @@ PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_recursive_mutex_t);
|
||||
.m_lock_debug = 0, \
|
||||
.m_lock_file = NULL, \
|
||||
.m_lock_line = 0, \
|
||||
.m_lock_atomic = { .u = { .lock = PMIX_ATOMIC_UNLOCKED } }, \
|
||||
.m_lock_atomic = { .u = { .lock = PMIX_ATOMIC_LOCK_UNLOCKED } },\
|
||||
}
|
||||
#else
|
||||
#define PMIX_RECURSIVE_MUTEX_STATIC_INIT \
|
||||
{ \
|
||||
.super = PMIX_OBJ_STATIC_INIT(pmix_mutex_t), \
|
||||
.m_lock_pthread = PMIX_PTHREAD_RECURSIVE_MUTEX_INITIALIZER, \
|
||||
.m_lock_atomic = { .u = { .lock = PMIX_ATOMIC_UNLOCKED } }, \
|
||||
.m_lock_atomic = { .u = { .lock = PMIX_ATOMIC_LOCK_UNLOCKED } },\
|
||||
}
|
||||
#endif
|
||||
|
||||
|
||||
@ -13,9 +13,9 @@
|
||||
* Copyright (c) 2007-2014 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2014-2016 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2017 Intel, Inc. All rights reserved.
|
||||
* Copyright (c) 2018 Intel, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -31,26 +31,27 @@
|
||||
#include "src/atomics/sys/atomic.h"
|
||||
#include "src/include/prefetch.h"
|
||||
|
||||
|
||||
/**
|
||||
* Use an atomic operation for increment/decrement
|
||||
* Use an atomic operation for increment/decrement if pmix_using_threads()
|
||||
* indicates that threads are in use by the application or library.
|
||||
*/
|
||||
|
||||
#define PMIX_THREAD_DEFINE_ATOMIC_ADD(type, suffix) \
|
||||
static inline type pmix_thread_add_ ## suffix (volatile type *addr, type delta) \
|
||||
{ \
|
||||
return pmix_atomic_add_ ## suffix (addr, delta); \
|
||||
#define PMIX_THREAD_DEFINE_ATOMIC_OP(type, name, operator, suffix) \
|
||||
static inline type pmix_thread_ ## name ## _fetch_ ## suffix (volatile type *addr, type delta) \
|
||||
{ \
|
||||
return pmix_atomic_ ## name ## _fetch_ ## suffix (addr, delta); \
|
||||
} \
|
||||
\
|
||||
static inline type pmix_thread_fetch_ ## name ## _ ## suffix (volatile type *addr, type delta) \
|
||||
{ \
|
||||
return pmix_atomic_fetch_ ## name ## _ ## suffix (addr, delta); \
|
||||
}
|
||||
|
||||
#define PMIX_THREAD_DEFINE_ATOMIC_SUB(type, suffix) \
|
||||
static inline type pmix_thread_sub_ ## suffix (volatile type *addr, type delta) \
|
||||
#define PMIX_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(type, addr_type, suffix) \
|
||||
static inline bool pmix_thread_compare_exchange_strong_ ## suffix (volatile addr_type *addr, type *compare, type value) \
|
||||
{ \
|
||||
return pmix_atomic_sub_ ## suffix (addr, delta); \
|
||||
}
|
||||
|
||||
#define PMIX_THREAD_DEFINE_ATOMIC_CMPSET(type, addr_type, suffix) \
|
||||
static inline bool pmix_thread_cmpset_bool_ ## suffix (volatile addr_type *addr, type compare, type value) \
|
||||
{ \
|
||||
return pmix_atomic_cmpset_ ## suffix ((volatile type *) addr, compare, value); \
|
||||
return pmix_atomic_compare_exchange_strong_ ## suffix ((volatile type *) addr, compare, value); \
|
||||
}
|
||||
|
||||
#define PMIX_THREAD_DEFINE_ATOMIC_SWAP(type, addr_type, suffix) \
|
||||
@ -59,28 +60,60 @@ static inline type pmix_thread_swap_ ## suffix (volatile addr_type *ptr, type ne
|
||||
return pmix_atomic_swap_ ## suffix ((volatile type *) ptr, newvalue); \
|
||||
}
|
||||
|
||||
PMIX_THREAD_DEFINE_ATOMIC_ADD(int32_t, 32)
|
||||
PMIX_THREAD_DEFINE_ATOMIC_ADD(size_t, size_t)
|
||||
PMIX_THREAD_DEFINE_ATOMIC_SUB(size_t, size_t)
|
||||
PMIX_THREAD_DEFINE_ATOMIC_CMPSET(int32_t, int32_t, 32)
|
||||
PMIX_THREAD_DEFINE_ATOMIC_CMPSET(void *, intptr_t, ptr)
|
||||
PMIX_THREAD_DEFINE_ATOMIC_OP(int32_t, add, +, 32)
|
||||
PMIX_THREAD_DEFINE_ATOMIC_OP(size_t, add, +, size_t)
|
||||
PMIX_THREAD_DEFINE_ATOMIC_OP(int32_t, and, &, 32)
|
||||
PMIX_THREAD_DEFINE_ATOMIC_OP(int32_t, or, |, 32)
|
||||
PMIX_THREAD_DEFINE_ATOMIC_OP(int32_t, xor, ^, 32)
|
||||
PMIX_THREAD_DEFINE_ATOMIC_OP(int32_t, sub, -, 32)
|
||||
PMIX_THREAD_DEFINE_ATOMIC_OP(size_t, sub, -, size_t)
|
||||
|
||||
PMIX_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(int32_t, int32_t, 32)
|
||||
PMIX_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(void *, intptr_t, ptr)
|
||||
PMIX_THREAD_DEFINE_ATOMIC_SWAP(int32_t, int32_t, 32)
|
||||
PMIX_THREAD_DEFINE_ATOMIC_SWAP(void *, intptr_t, ptr)
|
||||
|
||||
#define PMIX_THREAD_ADD32 pmix_thread_add_32
|
||||
#define PMIX_ATOMIC_ADD32 pmix_thread_add_32
|
||||
#define PMIX_THREAD_ADD_FETCH32 pmix_thread_add_fetch_32
|
||||
#define PMIX_ATOMIC_ADD_FETCH32 pmix_thread_add_fetch_32
|
||||
|
||||
#define PMIX_THREAD_ADD_SIZE_T pmix_thread_add_size_t
|
||||
#define PMIX_ATOMIC_ADD_SIZE_T pmix_thread_add_size_t
|
||||
#define PMIX_THREAD_AND_FETCH32 pmix_thread_and_fetch_32
|
||||
#define PMIX_ATOMIC_AND_FETCH32 pmix_thread_and_fetch_32
|
||||
|
||||
#define PMIX_THREAD_SUB_SIZE_T pmix_thread_sub_size_t
|
||||
#define PMIX_ATOMIC_SUB_SIZE_T pmix_thread_sub_size_t
|
||||
#define PMIX_THREAD_OR_FETCH32 pmix_thread_or_fetch_32
|
||||
#define PMIX_ATOMIC_OR_FETCH32 pmix_thread_or_fetch_32
|
||||
|
||||
#define PMIX_THREAD_CMPSET_32 pmix_thread_cmpset_bool_32
|
||||
#define PMIX_ATOMIC_CMPSET_32 pmix_thread_cmpset_bool_32
|
||||
#define PMIX_THREAD_XOR_FETCH32 pmix_thread_xor_fetch_32
|
||||
#define PMIX_ATOMIC_XOR_FETCH32 pmix_thread_xor_fetch_32
|
||||
|
||||
#define PMIX_THREAD_CMPSET_PTR(x, y, z) pmix_thread_cmpset_bool_ptr ((volatile intptr_t *) x, (void *) y, (void *) z)
|
||||
#define PMIX_ATOMIC_CMPSET_PTR PMIX_THREAD_CMPSET_PTR
|
||||
#define PMIX_THREAD_ADD_FETCH_SIZE_T pmix_thread_add_fetch_size_t
|
||||
#define PMIX_ATOMIC_ADD_FETCH_SIZE_T pmix_thread_add_fetch_size_t
|
||||
|
||||
#define PMIX_THREAD_SUB_FETCH_SIZE_T pmix_thread_sub_fetch_size_t
|
||||
#define PMIX_ATOMIC_SUB_FETCH_SIZE_T pmix_thread_sub_fetch_size_t
|
||||
|
||||
#define PMIX_THREAD_FETCH_ADD32 pmix_thread_fetch_add_32
|
||||
#define PMIX_ATOMIC_FETCH_ADD32 pmix_thread_fetch_add_32
|
||||
|
||||
#define PMIX_THREAD_FETCH_AND32 pmix_thread_fetch_and_32
|
||||
#define PMIX_ATOMIC_FETCH_AND32 pmix_thread_fetch_and_32
|
||||
|
||||
#define PMIX_THREAD_FETCH_OR32 pmix_thread_fetch_or_32
|
||||
#define PMIX_ATOMIC_FETCH_OR32 pmix_thread_fetch_or_32
|
||||
|
||||
#define PMIX_THREAD_FETCH_XOR32 pmix_thread_fetch_xor_32
|
||||
#define PMIX_ATOMIC_FETCH_XOR32 pmix_thread_fetch_xor_32
|
||||
|
||||
#define PMIX_THREAD_FETCH_ADD_SIZE_T pmix_thread_fetch_add_size_t
|
||||
#define PMIX_ATOMIC_FETCH_ADD_SIZE_T pmix_thread_fetch_add_size_t
|
||||
|
||||
#define PMIX_THREAD_FETCH_SUB_SIZE_T pmix_thread_fetch_sub_size_t
|
||||
#define PMIX_ATOMIC_FETCH_SUB_SIZE_T pmix_thread_fetch_sub_size_t
|
||||
|
||||
#define PMIX_THREAD_COMPARE_EXCHANGE_STRONG_32 pmix_thread_compare_exchange_strong_32
|
||||
#define PMIX_ATOMIC_COMPARE_EXCHANGE_STRONG_32 pmix_thread_compare_exchange_strong_32
|
||||
|
||||
#define PMIX_THREAD_COMPARE_EXCHANGE_STRONG_PTR(x, y, z) pmix_thread_compare_exchange_strong_ptr ((volatile intptr_t *) x, (void *) y, (void *) z)
|
||||
#define PMIX_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR PMIX_THREAD_COMPARE_EXCHANGE_STRONG_PTR
|
||||
|
||||
#define PMIX_THREAD_SWAP_32 pmix_thread_swap_32
|
||||
#define PMIX_ATOMIC_SWAP_32 pmix_thread_swap_32
|
||||
@ -91,19 +124,58 @@ PMIX_THREAD_DEFINE_ATOMIC_SWAP(void *, intptr_t, ptr)
|
||||
/* define 64-bit macros is 64-bit atomic math is available */
|
||||
#if PMIX_HAVE_ATOMIC_MATH_64
|
||||
|
||||
PMIX_THREAD_DEFINE_ATOMIC_ADD(int64_t, 64)
|
||||
PMIX_THREAD_DEFINE_ATOMIC_CMPSET(int64_t, int64_t, 64)
|
||||
PMIX_THREAD_DEFINE_ATOMIC_OP(int64_t, add, +, 64)
|
||||
PMIX_THREAD_DEFINE_ATOMIC_OP(int64_t, and, &, 64)
|
||||
PMIX_THREAD_DEFINE_ATOMIC_OP(int64_t, or, |, 64)
|
||||
PMIX_THREAD_DEFINE_ATOMIC_OP(int64_t, xor, ^, 64)
|
||||
PMIX_THREAD_DEFINE_ATOMIC_OP(int64_t, sub, -, 64)
|
||||
PMIX_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(int64_t, int64_t, 64)
|
||||
PMIX_THREAD_DEFINE_ATOMIC_SWAP(int64_t, int64_t, 64)
|
||||
|
||||
#define PMIX_THREAD_ADD64 pmix_thread_add_64
|
||||
#define PMIX_ATOMIC_ADD64 pmix_thread_add_64
|
||||
#define PMIX_THREAD_ADD_FETCH64 pmix_thread_add_fetch_64
|
||||
#define PMIX_ATOMIC_ADD_FETCH64 pmix_thread_add_fetch_64
|
||||
|
||||
#define PMIX_THREAD_CMPSET_64 pmix_thread_cmpset_bool_64
|
||||
#define PMIX_ATOMIC_CMPSET_64 pmix_thread_cmpset_bool_64
|
||||
#define PMIX_THREAD_AND_FETCH64 pmix_thread_and_fetch_64
|
||||
#define PMIX_ATOMIC_AND_FETCH64 pmix_thread_and_fetch_64
|
||||
|
||||
#define PMIX_THREAD_OR_FETCH64 pmix_thread_or_fetch_64
|
||||
#define PMIX_ATOMIC_OR_FETCH64 pmix_thread_or_fetch_64
|
||||
|
||||
#define PMIX_THREAD_XOR_FETCH64 pmix_thread_xor_fetch_64
|
||||
#define PMIX_ATOMIC_XOR_FETCH64 pmix_thread_xor_fetch_64
|
||||
|
||||
#define PMIX_THREAD_FETCH_ADD64 pmix_thread_fetch_add_64
|
||||
#define PMIX_ATOMIC_FETCH_ADD64 pmix_thread_fetch_add_64
|
||||
|
||||
#define PMIX_THREAD_FETCH_AND64 pmix_thread_fetch_and_64
|
||||
#define PMIX_ATOMIC_FETCH_AND64 pmix_thread_fetch_and_64
|
||||
|
||||
#define PMIX_THREAD_FETCH_OR64 pmix_thread_fetch_or_64
|
||||
#define PMIX_ATOMIC_FETCH_OR64 pmix_thread_fetch_or_64
|
||||
|
||||
#define PMIX_THREAD_FETCH_XOR64 pmix_thread_fetch_xor_64
|
||||
#define PMIX_ATOMIC_FETCH_XOR64 pmix_thread_fetch_xor_64
|
||||
|
||||
#define PMIX_THREAD_COMPARE_EXCHANGE_STRONG_64 pmix_thread_compare_exchange_strong_64
|
||||
#define PMIX_ATOMIC_COMPARE_EXCHANGE_STRONG_64 pmix_thread_compare_exchange_strong_64
|
||||
|
||||
#define PMIX_THREAD_SWAP_64 pmix_thread_swap_64
|
||||
#define PMIX_ATOMIC_SWAP_64 pmix_thread_swap_64
|
||||
|
||||
#endif
|
||||
|
||||
/* thread local storage */
|
||||
#if PMIX_C_HAVE__THREAD_LOCAL
|
||||
#define pmix_thread_local _Thread_local
|
||||
#define PMIX_HAVE_THREAD_LOCAL 1
|
||||
|
||||
#elif PMIX_C_HAVE___THREAD /* PMIX_C_HAVE__THREAD_LOCAL */
|
||||
#define pmix_thread_local __thread
|
||||
#define PMIX_HAVE_THREAD_LOCAL 1
|
||||
#endif /* PMIX_C_HAVE___THREAD */
|
||||
|
||||
#if !defined(PMIX_HAVE_THREAD_LOCAL)
|
||||
#define PMIX_HAVE_THREAD_LOCAL 0
|
||||
#endif /* !defined(PMIX_HAVE_THREAD_LOCAL) */
|
||||
|
||||
#endif /* !defined(PMIX_THREAD_USAGE_H) */
|
||||
|
||||
@ -8,7 +8,7 @@
|
||||
* Copyright (c) 2016 Mellanox Technologies. All rights reserved.
|
||||
* Copyright (c) 2016 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* Copyright (c) 2017 Intel, Inc. All rights reserved.
|
||||
* Copyright (c) 2017-2018 Intel, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
@ -103,7 +103,7 @@ static inline void pmix_wait_sync_update(pmix_wait_sync_t *sync,
|
||||
int updates, int status)
|
||||
{
|
||||
if( PMIX_LIKELY(PMIX_SUCCESS == status) ) {
|
||||
if( 0 != (PMIX_THREAD_ADD32(&sync->count, -updates)) ) {
|
||||
if( 0 != (PMIX_THREAD_ADD_FETCH32(&sync->count, -updates)) ) {
|
||||
return;
|
||||
}
|
||||
} else {
|
||||
|
||||
@ -52,12 +52,6 @@
|
||||
#include PMIX_EVENT_HEADER
|
||||
#include PMIX_EVENT2_THREAD_HEADER
|
||||
|
||||
#if PMIX_CC_USE_PRAGMA_IDENT
|
||||
#pragma ident PMIX_VERSION
|
||||
#elif PMIX_CC_USE_IDENT
|
||||
#ident PMIX_VERSION
|
||||
#endif
|
||||
|
||||
#include "src/class/pmix_list.h"
|
||||
#include "src/util/argv.h"
|
||||
#include "src/util/error.h"
|
||||
|
||||
Загрузка…
x
Ссылка в новой задаче
Block a user