Merge pull request #5445 from hjelmn/asm_type
Update opal to use C11 atomics if available
Этот коммит содержится в:
Nathan Hjelm
GitHub
Коммит
1071d72130
@ -13,7 +13,7 @@ dnl Copyright (c) 2008-2018 Cisco Systems, Inc. All rights reserved.
|
||||
dnl Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved.
|
||||
dnl Copyright (c) 2015-2017 Research Organization for Information Science
|
||||
dnl and Technology (RIST). All rights reserved.
|
||||
dnl Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights
|
||||
dnl Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights
|
||||
dnl reserved.
|
||||
dnl Copyright (c) 2017 Amazon.com, Inc. or its affiliates. All Rights
|
||||
dnl reserved.
|
||||
@ -122,6 +122,58 @@ int main(int argc, char** argv)
|
||||
}
|
||||
]])
|
||||
|
||||
dnl This is a C test to see if 128-bit __atomic_compare_exchange_n()
|
||||
dnl actually works (e.g., it compiles and links successfully on
|
||||
dnl ARM64+clang, but returns incorrect answers as of August 2018).
|
||||
AC_DEFUN([OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_TEST_SOURCE],[[
|
||||
#include <stdint.h>
|
||||
#include <stdbool.h>
|
||||
#include <stdlib.h>
|
||||
#include <stdatomic.h>
|
||||
|
||||
typedef union {
|
||||
uint64_t fake@<:@2@:>@;
|
||||
_Atomic __int128 real;
|
||||
} ompi128;
|
||||
|
||||
static void test1(void)
|
||||
{
|
||||
// As of Aug 2018, we could not figure out a way to assign 128-bit
|
||||
// constants -- the compilers would not accept it. So use a fake
|
||||
// union to assign 2 uin64_t's to make a single __int128.
|
||||
ompi128 ptr = { .fake = { 0xFFEEDDCCBBAA0099, 0x8877665544332211 }};
|
||||
ompi128 expected = { .fake = { 0x11EEDDCCBBAA0099, 0x88776655443322FF }};
|
||||
ompi128 desired = { .fake = { 0x1122DDCCBBAA0099, 0x887766554433EEFF }};
|
||||
bool r = atomic_compare_exchange_strong (&ptr.real, &expected.real,
|
||||
desired.real, true,
|
||||
atomic_relaxed, atomic_relaxed);
|
||||
if ( !(r == false && ptr.real == expected.real)) {
|
||||
exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
static void test2(void)
|
||||
{
|
||||
ompi128 ptr = { .fake = { 0xFFEEDDCCBBAA0099, 0x8877665544332211 }};
|
||||
ompi128 expected = ptr;
|
||||
ompi128 desired = { .fake = { 0x1122DDCCBBAA0099, 0x887766554433EEFF }};
|
||||
bool r = atomic_compare_exchange_strong (&ptr.real, &expected.real,
|
||||
desired.real, true,
|
||||
atomic_relaxed, atomic_relaxed);
|
||||
if (!(r == true && ptr.real == desired.real)) {
|
||||
exit(2);
|
||||
}
|
||||
}
|
||||
|
||||
vvvvvvvvvvvvvvvvvvvv
|
||||
int main(int argc, char** argv)
|
||||
{
|
||||
test1();
|
||||
test2();
|
||||
return 0;
|
||||
}
|
||||
]])
|
||||
|
||||
dnl ------------------------------------------------------------------
|
||||
|
||||
dnl
|
||||
@ -329,6 +381,71 @@ __atomic_add_fetch(&tmp64, 1, __ATOMIC_RELAXED);],
|
||||
OPAL_CHECK_GCC_BUILTIN_CSWAP_INT128
|
||||
])
|
||||
|
||||
AC_DEFUN([OPAL_CHECK_C11_CSWAP_INT128], [
|
||||
OPAL_VAR_SCOPE_PUSH([atomic_compare_exchange_result atomic_compare_exchange_CFLAGS_save atomic_compare_exchange_LIBS_save])
|
||||
|
||||
atomic_compare_exchange_CFLAGS_save=$CFLAGS
|
||||
atomic_compare_exchange_LIBS_save=$LIBS
|
||||
|
||||
# Do we have C11 atomics on 128-bit integers?
|
||||
# Use a special macro because we need to check with a few different
|
||||
# CFLAGS/LIBS.
|
||||
OPAL_ASM_CHECK_ATOMIC_FUNC([atomic_compare_exchange_strong_16],
|
||||
[AC_LANG_SOURCE(OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_TEST_SOURCE)],
|
||||
[atomic_compare_exchange_result=1],
|
||||
[atomic_compare_exchange_result=0])
|
||||
|
||||
# If we have it and it works, check to make sure it is always lock
|
||||
# free.
|
||||
AS_IF([test $atomic_compare_exchange_result -eq 1],
|
||||
[AC_MSG_CHECKING([if C11 __int128 atomic compare-and-swap is always lock-free])
|
||||
AC_RUN_IFELSE([AC_LANG_PROGRAM([#include <stdatomic.h>], [_Atomic __int128_t x; if (!atomic_is_lock_free(&x)) { return 1; }])],
|
||||
[AC_MSG_RESULT([yes])],
|
||||
[atomic_compare_exchange_result=0
|
||||
# If this test fails, need to reset CFLAGS/LIBS (the
|
||||
# above tests atomically set CFLAGS/LIBS or not; this
|
||||
# test is running after the fact, so we have to undo
|
||||
# the side-effects of setting CFLAGS/LIBS if the above
|
||||
# tests passed).
|
||||
CFLAGS=$atomic_compare_exchange_CFLAGS_save
|
||||
LIBS=$atomic_compare_exchange_LIBS_save
|
||||
AC_MSG_RESULT([no])],
|
||||
[AC_MSG_RESULT([cannot test -- assume yes (cross compiling)])])
|
||||
])
|
||||
|
||||
AC_DEFINE_UNQUOTED([OPAL_HAVE_C11_CSWAP_INT128],
|
||||
[$atomic_compare_exchange_result],
|
||||
[Whether C11 atomic compare swap is both supported and lock-free on 128-bit values])
|
||||
|
||||
dnl If we could not find decent support for 128-bits atomic let's
|
||||
dnl try the GCC _sync
|
||||
AS_IF([test $atomic_compare_exchange_result -eq 0],
|
||||
[OPAL_CHECK_SYNC_BUILTIN_CSWAP_INT128])
|
||||
|
||||
OPAL_VAR_SCOPE_POP
|
||||
])
|
||||
|
||||
AC_DEFUN([OPAL_CHECK_GCC_ATOMIC_BUILTINS], [
|
||||
AC_MSG_CHECKING([for __atomic builtin atomics])
|
||||
|
||||
AC_TRY_LINK([
|
||||
#include <stdint.h>
|
||||
uint32_t tmp, old = 0;
|
||||
uint64_t tmp64, old64 = 0;], [
|
||||
__atomic_thread_fence(__ATOMIC_SEQ_CST);
|
||||
__atomic_compare_exchange_n(&tmp, &old, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
|
||||
__atomic_add_fetch(&tmp, 1, __ATOMIC_RELAXED);
|
||||
__atomic_compare_exchange_n(&tmp64, &old64, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED);
|
||||
__atomic_add_fetch(&tmp64, 1, __ATOMIC_RELAXED);],
|
||||
[AC_MSG_RESULT([yes])
|
||||
$1],
|
||||
[AC_MSG_RESULT([no])
|
||||
$2])
|
||||
|
||||
# Check for 128-bit support
|
||||
OPAL_CHECK_GCC_BUILTIN_CSWAP_INT128
|
||||
])
|
||||
|
||||
|
||||
dnl #################################################################
|
||||
dnl
|
||||
@ -1020,17 +1137,27 @@ AC_DEFUN([OPAL_CONFIG_ASM],[
|
||||
AC_REQUIRE([OPAL_SETUP_CC])
|
||||
AC_REQUIRE([AM_PROG_AS])
|
||||
|
||||
AC_ARG_ENABLE([c11-atomics],[AC_HELP_STRING([--enable-c11-atomics],
|
||||
[Enable use of C11 atomics if available (default: enabled)])])
|
||||
|
||||
AC_ARG_ENABLE([builtin-atomics],
|
||||
[AC_HELP_STRING([--enable-builtin-atomics],
|
||||
[Enable use of __sync builtin atomics (default: enabled)])])
|
||||
[Enable use of __sync builtin atomics (default: disabled)])])
|
||||
|
||||
opal_cv_asm_builtin="BUILTIN_NO"
|
||||
AS_IF([test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" != "no"],
|
||||
[OPAL_CHECK_GCC_ATOMIC_BUILTINS([opal_cv_asm_builtin="BUILTIN_GCC"], [])])
|
||||
AS_IF([test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" != "no"],
|
||||
[OPAL_CHECK_SYNC_BUILTINS([opal_cv_asm_builtin="BUILTIN_SYNC"], [])])
|
||||
AS_IF([test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes"],
|
||||
[AC_MSG_ERROR([__sync builtin atomics requested but not found.])])
|
||||
OPAL_CHECK_C11_CSWAP_INT128
|
||||
|
||||
if test "x$enable_c11_atomics" != "xno" && test "$opal_cv_c11_supported" = "yes" ; then
|
||||
opal_cv_asm_builtin="BUILTIN_C11"
|
||||
OPAL_CHECK_C11_CSWAP_INT128
|
||||
else
|
||||
opal_cv_asm_builtin="BUILTIN_NO"
|
||||
AS_IF([test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes"],
|
||||
[OPAL_CHECK_GCC_ATOMIC_BUILTINS([opal_cv_asm_builtin="BUILTIN_GCC"], [])])
|
||||
AS_IF([test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes"],
|
||||
[OPAL_CHECK_SYNC_BUILTINS([opal_cv_asm_builtin="BUILTIN_SYNC"], [])])
|
||||
AS_IF([test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes"],
|
||||
[AC_MSG_ERROR([__sync builtin atomics requested but not found.])])
|
||||
fi
|
||||
|
||||
OPAL_CHECK_ASM_PROC
|
||||
OPAL_CHECK_ASM_TEXT
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
||||
/*
|
||||
* Copyright (c) 2013-2016 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2013-2018 Los Alamos National Security, LLC. All rights
|
||||
* reseved.
|
||||
* Copyright (c) 2015 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
@ -99,7 +99,7 @@ int ompi_comm_request_schedule_append (ompi_comm_request_t *request, ompi_comm_r
|
||||
static int ompi_comm_request_progress (void)
|
||||
{
|
||||
ompi_comm_request_t *request, *next;
|
||||
static int32_t progressing = 0;
|
||||
static opal_atomic_int32_t progressing = 0;
|
||||
|
||||
/* don't allow re-entry */
|
||||
if (opal_atomic_swap_32 (&progressing, 1)) {
|
||||
|
||||
@ -75,7 +75,7 @@ struct ompi_datatype_t {
|
||||
struct opal_hash_table_t *d_keyhash; /**< Attribute fields */
|
||||
|
||||
void* args; /**< Data description for the user */
|
||||
void* packed_description; /**< Packed description of the datatype */
|
||||
opal_atomic_intptr_t packed_description; /**< Packed description of the datatype */
|
||||
uint64_t pml_data; /**< PML-specific information */
|
||||
/* --- cacheline 6 boundary (384 bytes) --- */
|
||||
char name[MPI_MAX_OBJECT_NAME];/**< Externally visible name */
|
||||
|
||||
@ -45,7 +45,7 @@ __ompi_datatype_create_from_args( int32_t* i, ptrdiff_t * a,
|
||||
ompi_datatype_t** d, int32_t type );
|
||||
|
||||
typedef struct __dt_args {
|
||||
int32_t ref_count;
|
||||
opal_atomic_int32_t ref_count;
|
||||
int32_t create_type;
|
||||
size_t total_pack_size;
|
||||
int32_t ci;
|
||||
@ -104,7 +104,7 @@ typedef struct __dt_args {
|
||||
pArgs->total_pack_size = (4 + (IC) + (DC)) * sizeof(int) + \
|
||||
(AC) * sizeof(ptrdiff_t); \
|
||||
(PDATA)->args = (void*)pArgs; \
|
||||
(PDATA)->packed_description = NULL; \
|
||||
(PDATA)->packed_description = 0; \
|
||||
} while(0)
|
||||
|
||||
|
||||
@ -483,12 +483,12 @@ int ompi_datatype_get_pack_description( ompi_datatype_t* datatype,
|
||||
{
|
||||
ompi_datatype_args_t* args = (ompi_datatype_args_t*)datatype->args;
|
||||
int next_index = OMPI_DATATYPE_MAX_PREDEFINED;
|
||||
void *packed_description = datatype->packed_description;
|
||||
void *packed_description = (void *) datatype->packed_description;
|
||||
void* recursive_buffer;
|
||||
|
||||
if (NULL == packed_description) {
|
||||
void *_tmp_ptr = NULL;
|
||||
if (opal_atomic_compare_exchange_strong_ptr (&datatype->packed_description, (void *) &_tmp_ptr, (void *) 1)) {
|
||||
if (opal_atomic_compare_exchange_strong_ptr (&datatype->packed_description, (intptr_t *) &_tmp_ptr, 1)) {
|
||||
if( ompi_datatype_is_predefined(datatype) ) {
|
||||
packed_description = malloc(2 * sizeof(int));
|
||||
} else if( NULL == args ) {
|
||||
@ -510,10 +510,10 @@ int ompi_datatype_get_pack_description( ompi_datatype_t* datatype,
|
||||
}
|
||||
|
||||
opal_atomic_wmb ();
|
||||
datatype->packed_description = packed_description;
|
||||
datatype->packed_description = (intptr_t) packed_description;
|
||||
} else {
|
||||
/* another thread beat us to it */
|
||||
packed_description = datatype->packed_description;
|
||||
packed_description = (void *) datatype->packed_description;
|
||||
}
|
||||
}
|
||||
|
||||
@ -521,11 +521,11 @@ int ompi_datatype_get_pack_description( ompi_datatype_t* datatype,
|
||||
struct timespec interval = {.tv_sec = 0, .tv_nsec = 1000};
|
||||
|
||||
/* wait until the packed description is updated */
|
||||
while ((void *) 1 == datatype->packed_description) {
|
||||
while (1 == datatype->packed_description) {
|
||||
nanosleep (&interval, NULL);
|
||||
}
|
||||
|
||||
packed_description = datatype->packed_description;
|
||||
packed_description = (void *) datatype->packed_description;
|
||||
}
|
||||
|
||||
*packed_buffer = (const void *) packed_description;
|
||||
@ -534,7 +534,7 @@ int ompi_datatype_get_pack_description( ompi_datatype_t* datatype,
|
||||
|
||||
size_t ompi_datatype_pack_description_length( ompi_datatype_t* datatype )
|
||||
{
|
||||
void *packed_description = datatype->packed_description;
|
||||
void *packed_description = (void *) datatype->packed_description;
|
||||
|
||||
if( ompi_datatype_is_predefined(datatype) ) {
|
||||
return 2 * sizeof(int);
|
||||
|
||||
@ -36,7 +36,7 @@ static void __ompi_datatype_allocate( ompi_datatype_t* datatype )
|
||||
datatype->id = -1;
|
||||
datatype->d_keyhash = NULL;
|
||||
datatype->name[0] = '\0';
|
||||
datatype->packed_description = NULL;
|
||||
datatype->packed_description = 0;
|
||||
datatype->pml_data = 0;
|
||||
}
|
||||
|
||||
@ -46,10 +46,10 @@ static void __ompi_datatype_release(ompi_datatype_t * datatype)
|
||||
ompi_datatype_release_args( datatype );
|
||||
datatype->args = NULL;
|
||||
}
|
||||
if( NULL != datatype->packed_description ) {
|
||||
free( datatype->packed_description );
|
||||
datatype->packed_description = NULL;
|
||||
}
|
||||
|
||||
free ((void *) datatype->packed_description );
|
||||
datatype->packed_description = 0;
|
||||
|
||||
if( datatype->d_f_to_c_index >= 0 ) {
|
||||
opal_pointer_array_set_item( &ompi_datatype_f_to_c_table, datatype->d_f_to_c_index, NULL );
|
||||
datatype->d_f_to_c_index = -1;
|
||||
|
||||
@ -406,7 +406,7 @@ extern const ompi_datatype_t* ompi_datatype_basicDatatypes[OMPI_DATATYPE_MPI_MAX
|
||||
.d_f_to_c_index = -1, \
|
||||
.d_keyhash = NULL, \
|
||||
.args = NULL, \
|
||||
.packed_description = NULL, \
|
||||
.packed_description = 0, \
|
||||
.name = "MPI_" # NAME
|
||||
|
||||
#define OMPI_DATATYPE_INITIALIZER_UNAVAILABLE(FLAGS) \
|
||||
|
||||
@ -383,7 +383,7 @@ opal_pointer_array_t ompi_datatype_f_to_c_table = {{0}};
|
||||
(PDST)->super.desc = (PSRC)->super.desc; \
|
||||
(PDST)->super.opt_desc = (PSRC)->super.opt_desc; \
|
||||
(PDST)->packed_description = (PSRC)->packed_description; \
|
||||
(PSRC)->packed_description = NULL; \
|
||||
(PSRC)->packed_description = 0; \
|
||||
/* transfer the ptypes */ \
|
||||
(PDST)->super.ptypes = (PSRC)->super.ptypes; \
|
||||
(PSRC)->super.ptypes = NULL; \
|
||||
|
||||
@ -356,7 +356,8 @@ static inline struct ompi_proc_t *ompi_group_dense_lookup (ompi_group_t *group,
|
||||
ompi_proc_t *real_proc =
|
||||
(ompi_proc_t *) ompi_proc_for_name (ompi_proc_sentinel_to_name ((uintptr_t) proc));
|
||||
|
||||
if (opal_atomic_compare_exchange_strong_ptr (group->grp_proc_pointers + peer_id, &proc, real_proc)) {
|
||||
if (opal_atomic_compare_exchange_strong_ptr ((opal_atomic_intptr_t *)(group->grp_proc_pointers + peer_id),
|
||||
(intptr_t *) &proc, (intptr_t) real_proc)) {
|
||||
OBJ_RETAIN(real_proc);
|
||||
}
|
||||
|
||||
|
||||
@ -75,7 +75,7 @@ struct ompi_coll_libnbc_component_t {
|
||||
mca_coll_base_component_2_0_0_t super;
|
||||
opal_free_list_t requests;
|
||||
opal_list_t active_requests;
|
||||
int32_t active_comms;
|
||||
opal_atomic_int32_t active_comms;
|
||||
opal_mutex_t lock; /* protect access to the active_requests list */
|
||||
};
|
||||
typedef struct ompi_coll_libnbc_component_t ompi_coll_libnbc_component_t;
|
||||
|
||||
@ -36,7 +36,7 @@ struct mca_coll_monitoring_module_t {
|
||||
mca_coll_base_module_t super;
|
||||
mca_coll_base_comm_coll_t real;
|
||||
mca_monitoring_coll_data_t*data;
|
||||
int32_t is_initialized;
|
||||
opal_atomic_int32_t is_initialized;
|
||||
};
|
||||
typedef struct mca_coll_monitoring_module_t mca_coll_monitoring_module_t;
|
||||
OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_coll_monitoring_module_t);
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
||||
/*
|
||||
* Copyright (c) 2013-2015 Sandia National Laboratories. All rights reserved.
|
||||
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2015 Bull SAS. All rights reserved.
|
||||
* Copyright (c) 2015 Research Organization for Information Science
|
||||
@ -91,7 +92,7 @@ typedef struct ompi_coll_portals4_tree_t {
|
||||
|
||||
struct mca_coll_portals4_module_t {
|
||||
mca_coll_base_module_t super;
|
||||
size_t coll_count;
|
||||
opal_atomic_size_t coll_count;
|
||||
|
||||
/* record handlers dedicated to fallback if offloaded operations are not supported */
|
||||
mca_coll_base_module_reduce_fn_t previous_reduce;
|
||||
|
||||
@ -114,7 +114,7 @@ BEGIN_C_DECLS
|
||||
typedef struct mca_coll_sm_in_use_flag_t {
|
||||
/** Number of processes currently using this set of
|
||||
segments */
|
||||
volatile uint32_t mcsiuf_num_procs_using;
|
||||
opal_atomic_uint32_t mcsiuf_num_procs_using;
|
||||
/** Must match data->mcb_count */
|
||||
volatile uint32_t mcsiuf_operation_count;
|
||||
} mca_coll_sm_in_use_flag_t;
|
||||
@ -152,7 +152,7 @@ BEGIN_C_DECLS
|
||||
/** Pointer to my parent's barrier control pages (will be NULL
|
||||
for communicator rank 0; odd index pages are "in", even
|
||||
index pages are "out") */
|
||||
uint32_t *mcb_barrier_control_parent;
|
||||
opal_atomic_uint32_t *mcb_barrier_control_parent;
|
||||
|
||||
/** Pointers to my childrens' barrier control pages (they're
|
||||
contiguous in memory, so we only point to the base -- the
|
||||
|
||||
@ -56,7 +56,8 @@ int mca_coll_sm_barrier_intra(struct ompi_communicator_t *comm,
|
||||
int rank, buffer_set;
|
||||
mca_coll_sm_comm_t *data;
|
||||
uint32_t i, num_children;
|
||||
volatile uint32_t *me_in, *me_out, *parent, *children = NULL;
|
||||
volatile uint32_t *me_in, *me_out, *children = NULL;
|
||||
opal_atomic_uint32_t *parent;
|
||||
int uint_control_size;
|
||||
mca_coll_sm_module_t *sm_module = (mca_coll_sm_module_t*) module;
|
||||
|
||||
|
||||
@ -372,7 +372,7 @@ int ompi_coll_sm_lazy_enable(mca_coll_base_module_t *module,
|
||||
data->mcb_barrier_control_me = (uint32_t*)
|
||||
(base + (rank * control_size * num_barrier_buffers * 2));
|
||||
if (data->mcb_tree[rank].mcstn_parent) {
|
||||
data->mcb_barrier_control_parent = (uint32_t*)
|
||||
data->mcb_barrier_control_parent = (opal_atomic_uint32_t*)
|
||||
(base +
|
||||
(data->mcb_tree[rank].mcstn_parent->mcstn_id * control_size *
|
||||
num_barrier_buffers * 2));
|
||||
|
||||
@ -7,7 +7,7 @@
|
||||
* Copyright (c) 2015 Bull SAS. All rights reserved.
|
||||
* Copyright (c) 2016-2017 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* Copyright (c) 2017 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2017-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
@ -34,7 +34,7 @@
|
||||
|
||||
/*** Monitoring specific variables ***/
|
||||
/* Keep tracks of how many components are currently using the common part */
|
||||
static int32_t mca_common_monitoring_hold = 0;
|
||||
static opal_atomic_int32_t mca_common_monitoring_hold = 0;
|
||||
/* Output parameters */
|
||||
int mca_common_monitoring_output_stream_id = -1;
|
||||
static opal_output_stream_t mca_common_monitoring_output_stream_obj = {
|
||||
@ -61,18 +61,18 @@ static char* mca_common_monitoring_initial_filename = "";
|
||||
static char* mca_common_monitoring_current_filename = NULL;
|
||||
|
||||
/* array for stroring monitoring data*/
|
||||
static size_t* pml_data = NULL;
|
||||
static size_t* pml_count = NULL;
|
||||
static size_t* filtered_pml_data = NULL;
|
||||
static size_t* filtered_pml_count = NULL;
|
||||
static size_t* osc_data_s = NULL;
|
||||
static size_t* osc_count_s = NULL;
|
||||
static size_t* osc_data_r = NULL;
|
||||
static size_t* osc_count_r = NULL;
|
||||
static size_t* coll_data = NULL;
|
||||
static size_t* coll_count = NULL;
|
||||
static opal_atomic_size_t* pml_data = NULL;
|
||||
static opal_atomic_size_t* pml_count = NULL;
|
||||
static opal_atomic_size_t* filtered_pml_data = NULL;
|
||||
static opal_atomic_size_t* filtered_pml_count = NULL;
|
||||
static opal_atomic_size_t* osc_data_s = NULL;
|
||||
static opal_atomic_size_t* osc_count_s = NULL;
|
||||
static opal_atomic_size_t* osc_data_r = NULL;
|
||||
static opal_atomic_size_t* osc_count_r = NULL;
|
||||
static opal_atomic_size_t* coll_data = NULL;
|
||||
static opal_atomic_size_t* coll_count = NULL;
|
||||
|
||||
static size_t* size_histogram = NULL;
|
||||
static opal_atomic_size_t* size_histogram = NULL;
|
||||
static const int max_size_histogram = 66;
|
||||
static double log10_2 = 0.;
|
||||
|
||||
@ -241,7 +241,7 @@ void mca_common_monitoring_finalize( void )
|
||||
opal_output_close(mca_common_monitoring_output_stream_id);
|
||||
free(mca_common_monitoring_output_stream_obj.lds_prefix);
|
||||
/* Free internal data structure */
|
||||
free(pml_data); /* a single allocation */
|
||||
free((void *) pml_data); /* a single allocation */
|
||||
opal_hash_table_remove_all( common_monitoring_translation_ht );
|
||||
OBJ_RELEASE(common_monitoring_translation_ht);
|
||||
mca_common_monitoring_coll_finalize();
|
||||
@ -446,7 +446,7 @@ int mca_common_monitoring_add_procs(struct ompi_proc_t **procs,
|
||||
|
||||
if( NULL == pml_data ) {
|
||||
int array_size = (10 + max_size_histogram) * nprocs_world;
|
||||
pml_data = (size_t*)calloc(array_size, sizeof(size_t));
|
||||
pml_data = (opal_atomic_size_t*)calloc(array_size, sizeof(size_t));
|
||||
pml_count = pml_data + nprocs_world;
|
||||
filtered_pml_data = pml_count + nprocs_world;
|
||||
filtered_pml_count = filtered_pml_data + nprocs_world;
|
||||
@ -493,7 +493,7 @@ int mca_common_monitoring_add_procs(struct ompi_proc_t **procs,
|
||||
static void mca_common_monitoring_reset( void )
|
||||
{
|
||||
int array_size = (10 + max_size_histogram) * nprocs_world;
|
||||
memset(pml_data, 0, array_size * sizeof(size_t));
|
||||
memset((void *) pml_data, 0, array_size * sizeof(size_t));
|
||||
mca_common_monitoring_coll_reset();
|
||||
}
|
||||
|
||||
|
||||
@ -30,12 +30,12 @@ struct mca_monitoring_coll_data_t {
|
||||
int world_rank;
|
||||
int is_released;
|
||||
ompi_communicator_t*p_comm;
|
||||
size_t o2a_count;
|
||||
size_t o2a_size;
|
||||
size_t a2o_count;
|
||||
size_t a2o_size;
|
||||
size_t a2a_count;
|
||||
size_t a2a_size;
|
||||
opal_atomic_size_t o2a_count;
|
||||
opal_atomic_size_t o2a_size;
|
||||
opal_atomic_size_t a2o_count;
|
||||
opal_atomic_size_t a2o_size;
|
||||
opal_atomic_size_t a2a_count;
|
||||
opal_atomic_size_t a2a_size;
|
||||
};
|
||||
|
||||
/* Collectives operation monitoring */
|
||||
|
||||
@ -34,7 +34,7 @@ static opal_mutex_t mca_common_ompio_cuda_mutex; /* lock for thread saf
|
||||
static mca_allocator_base_component_t* mca_common_ompio_allocator_component=NULL;
|
||||
static mca_allocator_base_module_t* mca_common_ompio_allocator=NULL;
|
||||
|
||||
static int32_t mca_common_ompio_cuda_init = 0;
|
||||
static opal_atomic_int32_t mca_common_ompio_cuda_init = 0;
|
||||
static int32_t mca_common_ompio_pagesize=4096;
|
||||
static void* mca_common_ompio_cuda_alloc_seg ( void *ctx, size_t *size );
|
||||
static void mca_common_ompio_cuda_free_seg ( void *ctx, void *buf );
|
||||
|
||||
@ -115,12 +115,12 @@ struct mca_mtl_portals4_module_t {
|
||||
opal_mutex_t short_block_mutex;
|
||||
|
||||
/** number of send-side operations started */
|
||||
uint64_t opcount;
|
||||
opal_atomic_uint64_t opcount;
|
||||
|
||||
#if OPAL_ENABLE_DEBUG
|
||||
/** number of receive-side operations started. Used only for
|
||||
debugging */
|
||||
uint64_t recv_opcount;
|
||||
opal_atomic_uint64_t recv_opcount;
|
||||
#endif
|
||||
|
||||
#if OMPI_MTL_PORTALS4_FLOW_CONTROL
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
||||
/*
|
||||
* Copyright (c) 2012 Sandia National Laboratories. All rights reserved.
|
||||
* Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
|
||||
@ -36,7 +36,7 @@ OBJ_CLASS_DECLARATION(ompi_mtl_portals4_pending_request_t);
|
||||
struct ompi_mtl_portals4_flowctl_t {
|
||||
int32_t flowctl_active;
|
||||
|
||||
int32_t send_slots;
|
||||
opal_atomic_int32_t send_slots;
|
||||
int32_t max_send_slots;
|
||||
opal_list_t pending_sends;
|
||||
opal_free_list_t pending_fl;
|
||||
@ -46,7 +46,7 @@ struct ompi_mtl_portals4_flowctl_t {
|
||||
|
||||
/** Flow control epoch counter. Triggered events should be
|
||||
based on epoch counter. */
|
||||
int64_t epoch_counter;
|
||||
opal_atomic_int64_t epoch_counter;
|
||||
|
||||
/** Flow control trigger CT. Only has meaning at root. */
|
||||
ptl_handle_ct_t trigger_ct_h;
|
||||
|
||||
@ -54,8 +54,8 @@ struct ompi_mtl_portals4_isend_request_t {
|
||||
struct ompi_mtl_portals4_pending_request_t *pending;
|
||||
#endif
|
||||
ptl_size_t length;
|
||||
int32_t pending_get;
|
||||
uint32_t event_count;
|
||||
opal_atomic_int32_t pending_get;
|
||||
opal_atomic_uint32_t event_count;
|
||||
};
|
||||
typedef struct ompi_mtl_portals4_isend_request_t ompi_mtl_portals4_isend_request_t;
|
||||
|
||||
@ -76,7 +76,7 @@ struct ompi_mtl_portals4_recv_request_t {
|
||||
void *delivery_ptr;
|
||||
size_t delivery_len;
|
||||
volatile bool req_started;
|
||||
int32_t pending_reply;
|
||||
opal_atomic_int32_t pending_reply;
|
||||
#if OPAL_ENABLE_DEBUG
|
||||
uint64_t opcount;
|
||||
ptl_hdr_data_t hdr_data;
|
||||
|
||||
@ -50,7 +50,7 @@
|
||||
OSC_MONITORING_SET_TEMPLATE_FCT_NAME(template) (ompi_osc_base_module_t*module) \
|
||||
{ \
|
||||
/* Define the ompi_osc_monitoring_module_## template ##_init_done variable */ \
|
||||
static int32_t init_done = 0; \
|
||||
opal_atomic_int32_t init_done = 0; \
|
||||
/* Define and set the ompi_osc_monitoring_## template \
|
||||
* ##_template variable. The functions recorded here are \
|
||||
* linked to the original functions of the original \
|
||||
|
||||
@ -95,7 +95,7 @@ struct ompi_osc_portals4_module_t {
|
||||
ptl_handle_md_t req_md_h; /* memory descriptor with event completion used by this window */
|
||||
ptl_handle_me_t data_me_h; /* data match list entry (MB are CID | OSC_PORTALS4_MB_DATA) */
|
||||
ptl_handle_me_t control_me_h; /* match list entry for control data (node_state_t). Match bits are (CID | OSC_PORTALS4_MB_CONTROL). */
|
||||
int64_t opcount;
|
||||
opal_atomic_int64_t opcount;
|
||||
ptl_match_bits_t match_bits; /* match bits for module. Same as cid for comm in most cases. */
|
||||
|
||||
ptl_iovec_t *origin_iovec_list; /* list of memory segments that compose the noncontiguous region */
|
||||
|
||||
@ -189,7 +189,7 @@ number_of_fragments(ptl_size_t length, ptl_size_t maxlength)
|
||||
|
||||
/* put in segments no larger than segment_length */
|
||||
static int
|
||||
segmentedPut(int64_t *opcount,
|
||||
segmentedPut(opal_atomic_int64_t *opcount,
|
||||
ptl_handle_md_t md_h,
|
||||
ptl_size_t origin_offset,
|
||||
ptl_size_t put_length,
|
||||
@ -236,7 +236,7 @@ segmentedPut(int64_t *opcount,
|
||||
|
||||
/* get in segments no larger than segment_length */
|
||||
static int
|
||||
segmentedGet(int64_t *opcount,
|
||||
segmentedGet(opal_atomic_int64_t *opcount,
|
||||
ptl_handle_md_t md_h,
|
||||
ptl_size_t origin_offset,
|
||||
ptl_size_t get_length,
|
||||
@ -280,7 +280,7 @@ segmentedGet(int64_t *opcount,
|
||||
|
||||
/* atomic op in segments no larger than segment_length */
|
||||
static int
|
||||
segmentedAtomic(int64_t *opcount,
|
||||
segmentedAtomic(opal_atomic_int64_t *opcount,
|
||||
ptl_handle_md_t md_h,
|
||||
ptl_size_t origin_offset,
|
||||
ptl_size_t length,
|
||||
@ -329,7 +329,7 @@ segmentedAtomic(int64_t *opcount,
|
||||
|
||||
/* atomic op in segments no larger than segment_length */
|
||||
static int
|
||||
segmentedFetchAtomic(int64_t *opcount,
|
||||
segmentedFetchAtomic(opal_atomic_int64_t *opcount,
|
||||
ptl_handle_md_t result_md_h,
|
||||
ptl_size_t result_offset,
|
||||
ptl_handle_md_t origin_md_h,
|
||||
@ -381,7 +381,7 @@ segmentedFetchAtomic(int64_t *opcount,
|
||||
|
||||
/* swap in segments no larger than segment_length */
|
||||
static int
|
||||
segmentedSwap(int64_t *opcount,
|
||||
segmentedSwap(opal_atomic_int64_t *opcount,
|
||||
ptl_handle_md_t result_md_h,
|
||||
ptl_size_t result_offset,
|
||||
ptl_handle_md_t origin_md_h,
|
||||
@ -1187,7 +1187,7 @@ fetch_atomic_to_iovec(ompi_osc_portals4_module_t *module,
|
||||
|
||||
/* put in the largest chunks possible given the noncontiguous restriction */
|
||||
static int
|
||||
put_to_noncontig(int64_t *opcount,
|
||||
put_to_noncontig(opal_atomic_int64_t *opcount,
|
||||
ptl_handle_md_t md_h,
|
||||
const void *origin_address,
|
||||
int origin_count,
|
||||
@ -1521,7 +1521,7 @@ atomic_to_noncontig(ompi_osc_portals4_module_t *module,
|
||||
|
||||
/* get from a noncontiguous remote to an (non)contiguous local */
|
||||
static int
|
||||
get_from_noncontig(int64_t *opcount,
|
||||
get_from_noncontig(opal_atomic_int64_t *opcount,
|
||||
ptl_handle_md_t md_h,
|
||||
const void *origin_address,
|
||||
int origin_count,
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
||||
/*
|
||||
* Copyright (c) 2011-2013 Sandia National Laboratories. All rights reserved.
|
||||
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
@ -18,7 +18,7 @@
|
||||
struct ompi_osc_portals4_request_t {
|
||||
ompi_request_t super;
|
||||
int32_t ops_expected;
|
||||
volatile int32_t ops_committed;
|
||||
opal_atomic_int32_t ops_committed;
|
||||
};
|
||||
typedef struct ompi_osc_portals4_request_t ompi_osc_portals4_request_t;
|
||||
|
||||
|
||||
@ -8,7 +8,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2007-2017 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2007-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved.
|
||||
@ -110,7 +110,7 @@ struct ompi_osc_pt2pt_peer_t {
|
||||
int rank;
|
||||
|
||||
/** pointer to the current send fragment for each outgoing target */
|
||||
struct ompi_osc_pt2pt_frag_t *active_frag;
|
||||
opal_atomic_intptr_t active_frag;
|
||||
|
||||
/** lock for this peer */
|
||||
opal_mutex_t lock;
|
||||
@ -119,10 +119,10 @@ struct ompi_osc_pt2pt_peer_t {
|
||||
opal_list_t queued_frags;
|
||||
|
||||
/** number of fragments incomming (negative - expected, positive - unsynchronized) */
|
||||
volatile int32_t passive_incoming_frag_count;
|
||||
opal_atomic_int32_t passive_incoming_frag_count;
|
||||
|
||||
/** peer flags */
|
||||
volatile int32_t flags;
|
||||
opal_atomic_int32_t flags;
|
||||
};
|
||||
typedef struct ompi_osc_pt2pt_peer_t ompi_osc_pt2pt_peer_t;
|
||||
|
||||
@ -208,16 +208,16 @@ struct ompi_osc_pt2pt_module_t {
|
||||
|
||||
/** Nmber of communication fragments started for this epoch, by
|
||||
peer. Not in peer data to make fence more manageable. */
|
||||
uint32_t *epoch_outgoing_frag_count;
|
||||
opal_atomic_uint32_t *epoch_outgoing_frag_count;
|
||||
|
||||
/** cyclic counter for a unique tage for long messages. */
|
||||
volatile uint32_t tag_counter;
|
||||
opal_atomic_uint32_t tag_counter;
|
||||
|
||||
/** number of outgoing fragments still to be completed */
|
||||
volatile int32_t outgoing_frag_count;
|
||||
opal_atomic_int32_t outgoing_frag_count;
|
||||
|
||||
/** number of incoming fragments */
|
||||
volatile int32_t active_incoming_frag_count;
|
||||
opal_atomic_int32_t active_incoming_frag_count;
|
||||
|
||||
/** Number of targets locked/being locked */
|
||||
unsigned int passive_target_access_epoch;
|
||||
@ -230,13 +230,13 @@ struct ompi_osc_pt2pt_module_t {
|
||||
|
||||
/** Number of "count" messages from the remote complete group
|
||||
we've received */
|
||||
volatile int32_t num_complete_msgs;
|
||||
opal_atomic_int32_t num_complete_msgs;
|
||||
|
||||
/* ********************* LOCK data ************************ */
|
||||
|
||||
/** Status of the local window lock. One of 0 (unlocked),
|
||||
MPI_LOCK_EXCLUSIVE, or MPI_LOCK_SHARED. */
|
||||
int32_t lock_status;
|
||||
opal_atomic_int32_t lock_status;
|
||||
|
||||
/** lock for locks_pending list */
|
||||
opal_mutex_t locks_pending_lock;
|
||||
@ -526,7 +526,7 @@ static inline void mark_incoming_completion (ompi_osc_pt2pt_module_t *module, in
|
||||
OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
|
||||
"mark_incoming_completion marking passive incoming complete. module %p, source = %d, count = %d",
|
||||
(void *) module, source, (int) peer->passive_incoming_frag_count + 1));
|
||||
new_value = OPAL_THREAD_ADD_FETCH32((int32_t *) &peer->passive_incoming_frag_count, 1);
|
||||
new_value = OPAL_THREAD_ADD_FETCH32((opal_atomic_int32_t *) &peer->passive_incoming_frag_count, 1);
|
||||
if (0 == new_value) {
|
||||
OPAL_THREAD_LOCK(&module->lock);
|
||||
opal_condition_broadcast(&module->cond);
|
||||
@ -550,7 +550,7 @@ static inline void mark_incoming_completion (ompi_osc_pt2pt_module_t *module, in
|
||||
*/
|
||||
static inline void mark_outgoing_completion (ompi_osc_pt2pt_module_t *module)
|
||||
{
|
||||
int32_t new_value = OPAL_THREAD_ADD_FETCH32((int32_t *) &module->outgoing_frag_count, 1);
|
||||
int32_t new_value = OPAL_THREAD_ADD_FETCH32((opal_atomic_int32_t *) &module->outgoing_frag_count, 1);
|
||||
OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
|
||||
"mark_outgoing_completion: outgoing_frag_count = %d", new_value));
|
||||
if (new_value >= 0) {
|
||||
@ -574,12 +574,12 @@ static inline void mark_outgoing_completion (ompi_osc_pt2pt_module_t *module)
|
||||
*/
|
||||
static inline void ompi_osc_signal_outgoing (ompi_osc_pt2pt_module_t *module, int target, int count)
|
||||
{
|
||||
OPAL_THREAD_ADD_FETCH32((int32_t *) &module->outgoing_frag_count, -count);
|
||||
OPAL_THREAD_ADD_FETCH32((opal_atomic_int32_t *) &module->outgoing_frag_count, -count);
|
||||
if (MPI_PROC_NULL != target) {
|
||||
OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
|
||||
"ompi_osc_signal_outgoing_passive: target = %d, count = %d, total = %d", target,
|
||||
count, module->epoch_outgoing_frag_count[target] + count));
|
||||
OPAL_THREAD_ADD_FETCH32((int32_t *) (module->epoch_outgoing_frag_count + target), count);
|
||||
OPAL_THREAD_ADD_FETCH32((opal_atomic_int32_t *) (module->epoch_outgoing_frag_count + target), count);
|
||||
}
|
||||
}
|
||||
|
||||
@ -717,7 +717,7 @@ static inline int get_tag(ompi_osc_pt2pt_module_t *module)
|
||||
/* the LSB of the tag is used be the receiver to determine if the
|
||||
message is a passive or active target (ie, where to mark
|
||||
completion). */
|
||||
int32_t tmp = OPAL_THREAD_ADD_FETCH32((volatile int32_t *) &module->tag_counter, 4);
|
||||
int32_t tmp = OPAL_THREAD_ADD_FETCH32((opal_atomic_int32_t *) &module->tag_counter, 4);
|
||||
return (tmp & OSC_PT2PT_FRAG_MASK) | !!(module->passive_target_access_epoch);
|
||||
}
|
||||
|
||||
|
||||
@ -8,7 +8,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2007-2016 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2007-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2010-2016 IBM Corporation. All rights reserved.
|
||||
* Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved.
|
||||
@ -166,17 +166,16 @@ int ompi_osc_pt2pt_fence(int assert, ompi_win_t *win)
|
||||
"osc pt2pt: fence done sending"));
|
||||
|
||||
/* find out how much data everyone is going to send us. */
|
||||
ret = module->comm->c_coll->coll_reduce_scatter_block (module->epoch_outgoing_frag_count,
|
||||
&incoming_reqs, 1, MPI_UINT32_T,
|
||||
MPI_SUM, module->comm,
|
||||
module->comm->c_coll->coll_reduce_scatter_block_module);
|
||||
ret = module->comm->c_coll->coll_reduce_scatter_block ((void *) module->epoch_outgoing_frag_count,
|
||||
&incoming_reqs, 1, MPI_UINT32_T,
|
||||
MPI_SUM, module->comm,
|
||||
module->comm->c_coll->coll_reduce_scatter_block_module);
|
||||
if (OMPI_SUCCESS != ret) {
|
||||
return ret;
|
||||
}
|
||||
|
||||
OPAL_THREAD_LOCK(&module->lock);
|
||||
bzero(module->epoch_outgoing_frag_count,
|
||||
sizeof(uint32_t) * ompi_comm_size(module->comm));
|
||||
bzero ((void *) module->epoch_outgoing_frag_count, sizeof(uint32_t) * ompi_comm_size(module->comm));
|
||||
|
||||
OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
|
||||
"osc pt2pt: fence expects %d requests",
|
||||
@ -366,8 +365,11 @@ int ompi_osc_pt2pt_complete (ompi_win_t *win)
|
||||
|
||||
/* XXX -- TODO -- since fragment are always delivered in order we do not need to count anything but long
|
||||
* requests. once that is done this can be removed. */
|
||||
if (peer->active_frag && (peer->active_frag->remain_len < sizeof (complete_req))) {
|
||||
++complete_req.frag_count;
|
||||
if (peer->active_frag) {
|
||||
ompi_osc_pt2pt_frag_t *active_frag = (ompi_osc_pt2pt_frag_t *) peer->active_frag;
|
||||
if (active_frag->remain_len < sizeof (complete_req)) {
|
||||
++complete_req.frag_count;
|
||||
}
|
||||
}
|
||||
|
||||
OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output,
|
||||
|
||||
@ -501,7 +501,7 @@ static void ompi_osc_pt2pt_peer_construct (ompi_osc_pt2pt_peer_t *peer)
|
||||
{
|
||||
OBJ_CONSTRUCT(&peer->queued_frags, opal_list_t);
|
||||
OBJ_CONSTRUCT(&peer->lock, opal_mutex_t);
|
||||
peer->active_frag = NULL;
|
||||
peer->active_frag = 0;
|
||||
peer->passive_incoming_frag_count = 0;
|
||||
peer->flags = 0;
|
||||
}
|
||||
|
||||
@ -8,7 +8,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2007-2017 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2007-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2009-2011 Oracle and/or its affiliates. All rights reserved.
|
||||
* Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved.
|
||||
@ -56,7 +56,7 @@ struct osc_pt2pt_accumulate_data_t {
|
||||
int peer;
|
||||
ompi_datatype_t *datatype;
|
||||
ompi_op_t *op;
|
||||
int request_count;
|
||||
opal_atomic_int32_t request_count;
|
||||
};
|
||||
typedef struct osc_pt2pt_accumulate_data_t osc_pt2pt_accumulate_data_t;
|
||||
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
||||
/*
|
||||
* Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved.
|
||||
* Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2015 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
@ -65,7 +65,7 @@ int ompi_osc_pt2pt_frag_start (ompi_osc_pt2pt_module_t *module,
|
||||
ompi_osc_pt2pt_peer_t *peer = ompi_osc_pt2pt_peer_lookup (module, frag->target);
|
||||
int ret;
|
||||
|
||||
assert(0 == frag->pending && peer->active_frag != frag);
|
||||
assert(0 == frag->pending && peer->active_frag != (intptr_t) frag);
|
||||
|
||||
/* we need to signal now that a frag is outgoing to ensure the count sent
|
||||
* with the unlock message is correct */
|
||||
@ -93,7 +93,7 @@ int ompi_osc_pt2pt_frag_start (ompi_osc_pt2pt_module_t *module,
|
||||
|
||||
static int ompi_osc_pt2pt_flush_active_frag (ompi_osc_pt2pt_module_t *module, ompi_osc_pt2pt_peer_t *peer)
|
||||
{
|
||||
ompi_osc_pt2pt_frag_t *active_frag = peer->active_frag;
|
||||
ompi_osc_pt2pt_frag_t *active_frag = (ompi_osc_pt2pt_frag_t *) peer->active_frag;
|
||||
int ret = OMPI_SUCCESS;
|
||||
|
||||
if (NULL == active_frag) {
|
||||
@ -105,7 +105,7 @@ static int ompi_osc_pt2pt_flush_active_frag (ompi_osc_pt2pt_module_t *module, om
|
||||
"osc pt2pt: flushing active fragment to target %d. pending: %d",
|
||||
active_frag->target, active_frag->pending));
|
||||
|
||||
if (opal_atomic_compare_exchange_strong_ptr (&peer->active_frag, &active_frag, NULL)) {
|
||||
if (opal_atomic_compare_exchange_strong_ptr (&peer->active_frag, (intptr_t *) &active_frag, 0)) {
|
||||
if (0 != OPAL_THREAD_ADD_FETCH32(&active_frag->pending, -1)) {
|
||||
/* communication going on while synchronizing; this is an rma usage bug */
|
||||
return OMPI_ERR_RMA_SYNC;
|
||||
|
||||
@ -33,7 +33,7 @@ struct ompi_osc_pt2pt_frag_t {
|
||||
char *top;
|
||||
|
||||
/* Number of operations which have started writing into the frag, but not yet completed doing so */
|
||||
volatile int32_t pending;
|
||||
opal_atomic_int32_t pending;
|
||||
int32_t pending_long_sends;
|
||||
ompi_osc_pt2pt_frag_header_t *header;
|
||||
ompi_osc_pt2pt_module_t *module;
|
||||
@ -66,8 +66,8 @@ static inline ompi_osc_pt2pt_frag_t *ompi_osc_pt2pt_frag_alloc_non_buffered (omp
|
||||
ompi_osc_pt2pt_frag_t *curr;
|
||||
|
||||
/* to ensure ordering flush the buffer on the peer */
|
||||
curr = peer->active_frag;
|
||||
if (NULL != curr && opal_atomic_compare_exchange_strong_ptr (&peer->active_frag, &curr, NULL)) {
|
||||
curr = (ompi_osc_pt2pt_frag_t *) peer->active_frag;
|
||||
if (NULL != curr && opal_atomic_compare_exchange_strong_ptr (&peer->active_frag, (intptr_t *) &curr, 0)) {
|
||||
/* If there's something pending, the pending finish will
|
||||
start the buffer. Otherwise, we need to start it now. */
|
||||
int ret = ompi_osc_pt2pt_frag_finish (module, curr);
|
||||
@ -131,7 +131,7 @@ static inline int _ompi_osc_pt2pt_frag_alloc (ompi_osc_pt2pt_module_t *module, i
|
||||
|
||||
OPAL_THREAD_LOCK(&module->lock);
|
||||
if (buffered) {
|
||||
curr = peer->active_frag;
|
||||
curr = (ompi_osc_pt2pt_frag_t *) peer->active_frag;
|
||||
if (NULL == curr || curr->remain_len < request_len || (long_send && curr->pending_long_sends == 32)) {
|
||||
curr = ompi_osc_pt2pt_frag_alloc_non_buffered (module, peer, request_len);
|
||||
if (OPAL_UNLIKELY(NULL == curr)) {
|
||||
@ -140,7 +140,7 @@ static inline int _ompi_osc_pt2pt_frag_alloc (ompi_osc_pt2pt_module_t *module, i
|
||||
}
|
||||
|
||||
curr->pending_long_sends = long_send;
|
||||
peer->active_frag = curr;
|
||||
peer->active_frag = (uintptr_t) curr;
|
||||
} else {
|
||||
OPAL_THREAD_ADD_FETCH32(&curr->header->num_ops, 1);
|
||||
curr->pending_long_sends += long_send;
|
||||
|
||||
@ -8,7 +8,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2007-2015 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2007-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved.
|
||||
@ -180,7 +180,7 @@ typedef struct ompi_osc_pt2pt_header_flush_ack_t ompi_osc_pt2pt_header_flush_ack
|
||||
struct ompi_osc_pt2pt_frag_header_t {
|
||||
ompi_osc_pt2pt_header_base_t base;
|
||||
uint32_t source; /* rank in window of source process */
|
||||
int32_t num_ops; /* number of operations in this buffer */
|
||||
opal_atomic_int32_t num_ops; /* number of operations in this buffer */
|
||||
uint32_t pad; /* ensure the fragment header is a multiple of 8 bytes */
|
||||
};
|
||||
typedef struct ompi_osc_pt2pt_frag_header_t ompi_osc_pt2pt_frag_header_t;
|
||||
|
||||
@ -8,7 +8,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2007-2016 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2007-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved.
|
||||
* Copyright (c) 2015 Research Organization for Information Science
|
||||
@ -104,13 +104,13 @@ int ompi_osc_pt2pt_free(ompi_win_t *win)
|
||||
free (module->recv_frags);
|
||||
}
|
||||
|
||||
if (NULL != module->epoch_outgoing_frag_count) free(module->epoch_outgoing_frag_count);
|
||||
free ((void *) module->epoch_outgoing_frag_count);
|
||||
|
||||
if (NULL != module->comm) {
|
||||
ompi_comm_free(&module->comm);
|
||||
}
|
||||
|
||||
if (NULL != module->free_after) free(module->free_after);
|
||||
free ((void *) module->free_after);
|
||||
|
||||
free (module);
|
||||
|
||||
|
||||
@ -8,7 +8,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2007-2017 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2007-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2010-2016 IBM Corporation. All rights reserved.
|
||||
* Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved.
|
||||
@ -157,7 +157,7 @@ int ompi_osc_pt2pt_lock_remote (ompi_osc_pt2pt_module_t *module, int target, omp
|
||||
|
||||
static inline int ompi_osc_pt2pt_unlock_remote (ompi_osc_pt2pt_module_t *module, int target, ompi_osc_pt2pt_sync_t *lock)
|
||||
{
|
||||
int32_t frag_count = opal_atomic_swap_32 ((int32_t *) module->epoch_outgoing_frag_count + target, -1);
|
||||
int32_t frag_count = opal_atomic_swap_32 ((opal_atomic_int32_t *) module->epoch_outgoing_frag_count + target, -1);
|
||||
ompi_osc_pt2pt_peer_t *peer = ompi_osc_pt2pt_peer_lookup (module, target);
|
||||
int lock_type = lock->sync.lock.type;
|
||||
ompi_osc_pt2pt_header_unlock_t unlock_req;
|
||||
@ -178,10 +178,13 @@ static inline int ompi_osc_pt2pt_unlock_remote (ompi_osc_pt2pt_module_t *module,
|
||||
unlock_req.lock_ptr = (uint64_t) (uintptr_t) lock;
|
||||
OSC_PT2PT_HTON(&unlock_req, module, target);
|
||||
|
||||
if (peer->active_frag && peer->active_frag->remain_len < sizeof (unlock_req)) {
|
||||
/* the peer should expect one more packet */
|
||||
++unlock_req.frag_count;
|
||||
--module->epoch_outgoing_frag_count[target];
|
||||
if (peer->active_frag) {
|
||||
ompi_osc_pt2pt_frag_t *active_frag = (ompi_osc_pt2pt_frag_t *) peer->active_frag;
|
||||
if (active_frag->remain_len < sizeof (unlock_req)) {
|
||||
/* the peer should expect one more packet */
|
||||
++unlock_req.frag_count;
|
||||
--module->epoch_outgoing_frag_count[target];
|
||||
}
|
||||
}
|
||||
|
||||
OPAL_OUTPUT_VERBOSE((25, ompi_osc_base_framework.framework_output,
|
||||
@ -204,7 +207,7 @@ static inline int ompi_osc_pt2pt_flush_remote (ompi_osc_pt2pt_module_t *module,
|
||||
{
|
||||
ompi_osc_pt2pt_peer_t *peer = ompi_osc_pt2pt_peer_lookup (module, target);
|
||||
ompi_osc_pt2pt_header_flush_t flush_req;
|
||||
int32_t frag_count = opal_atomic_swap_32 ((int32_t *) module->epoch_outgoing_frag_count + target, -1);
|
||||
int32_t frag_count = opal_atomic_swap_32 ((opal_atomic_int32_t *) module->epoch_outgoing_frag_count + target, -1);
|
||||
int ret;
|
||||
|
||||
(void) OPAL_THREAD_ADD_FETCH32(&lock->sync_expected, 1);
|
||||
@ -218,10 +221,13 @@ static inline int ompi_osc_pt2pt_flush_remote (ompi_osc_pt2pt_module_t *module,
|
||||
|
||||
/* XXX -- TODO -- since fragment are always delivered in order we do not need to count anything but long
|
||||
* requests. once that is done this can be removed. */
|
||||
if (peer->active_frag && (peer->active_frag->remain_len < sizeof (flush_req))) {
|
||||
/* the peer should expect one more packet */
|
||||
++flush_req.frag_count;
|
||||
--module->epoch_outgoing_frag_count[target];
|
||||
if (peer->active_frag) {
|
||||
ompi_osc_pt2pt_frag_t *active_frag = (ompi_osc_pt2pt_frag_t *) peer->active_frag;
|
||||
if (active_frag->remain_len < sizeof (flush_req)) {
|
||||
/* the peer should expect one more packet */
|
||||
++flush_req.frag_count;
|
||||
--module->epoch_outgoing_frag_count[target];
|
||||
}
|
||||
}
|
||||
|
||||
OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "flushing to target %d, frag_count: %d",
|
||||
|
||||
@ -28,7 +28,7 @@ struct ompi_osc_pt2pt_request_t {
|
||||
int origin_count;
|
||||
struct ompi_datatype_t *origin_dt;
|
||||
ompi_osc_pt2pt_module_t* module;
|
||||
int32_t outstanding_requests;
|
||||
opal_atomic_int32_t outstanding_requests;
|
||||
bool internal;
|
||||
};
|
||||
typedef struct ompi_osc_pt2pt_request_t ompi_osc_pt2pt_request_t;
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
||||
/*
|
||||
* Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
@ -74,7 +74,7 @@ struct ompi_osc_pt2pt_sync_t {
|
||||
int num_peers;
|
||||
|
||||
/** number of synchronization messages expected */
|
||||
volatile int32_t sync_expected;
|
||||
opal_atomic_int32_t sync_expected;
|
||||
|
||||
/** eager sends are active to all peers in this access epoch */
|
||||
volatile bool eager_send_active;
|
||||
|
||||
@ -265,7 +265,7 @@ struct ompi_osc_rdma_module_t {
|
||||
unsigned long get_retry_count;
|
||||
|
||||
/** outstanding atomic operations */
|
||||
volatile int32_t pending_ops;
|
||||
opal_atomic_int32_t pending_ops;
|
||||
};
|
||||
typedef struct ompi_osc_rdma_module_t ompi_osc_rdma_module_t;
|
||||
OMPI_MODULE_DECLSPEC extern ompi_osc_rdma_component_t mca_osc_rdma_component;
|
||||
|
||||
@ -259,7 +259,7 @@ static int ompi_osc_rdma_post_peer (ompi_osc_rdma_module_t *module, ompi_osc_rdm
|
||||
return ret;
|
||||
}
|
||||
} else {
|
||||
post_index = ompi_osc_rdma_counter_add ((osc_rdma_counter_t *) (intptr_t) target, 1) - 1;
|
||||
post_index = ompi_osc_rdma_counter_add ((osc_rdma_atomic_counter_t *) (intptr_t) target, 1) - 1;
|
||||
}
|
||||
|
||||
post_index &= OMPI_OSC_RDMA_POST_PEER_MAX - 1;
|
||||
@ -279,7 +279,7 @@ static int ompi_osc_rdma_post_peer (ompi_osc_rdma_module_t *module, ompi_osc_rdm
|
||||
return ret;
|
||||
}
|
||||
} else {
|
||||
result = !ompi_osc_rdma_lock_compare_exchange ((osc_rdma_counter_t *) target, &_tmp_value,
|
||||
result = !ompi_osc_rdma_lock_compare_exchange ((osc_rdma_atomic_counter_t *) target, &_tmp_value,
|
||||
1 + (osc_rdma_counter_t) my_rank);
|
||||
}
|
||||
|
||||
@ -491,7 +491,7 @@ int ompi_osc_rdma_complete_atomic (ompi_win_t *win)
|
||||
ret = ompi_osc_rdma_lock_btl_op (module, peer, target, MCA_BTL_ATOMIC_ADD, 1, true);
|
||||
assert (OMPI_SUCCESS == ret);
|
||||
} else {
|
||||
(void) ompi_osc_rdma_counter_add ((osc_rdma_counter_t *) target, 1);
|
||||
(void) ompi_osc_rdma_counter_add ((osc_rdma_atomic_counter_t *) target, 1);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -72,7 +72,7 @@ static inline int ompi_osc_rdma_frag_alloc (ompi_osc_rdma_module_t *module, size
|
||||
}
|
||||
}
|
||||
|
||||
if (!opal_atomic_compare_exchange_strong_ptr (&module->rdma_frag, &(void *){NULL}, curr)) {
|
||||
if (!opal_atomic_compare_exchange_strong_ptr ((opal_atomic_intptr_t *) &module->rdma_frag, &(intptr_t){0}, (intptr_t) curr)) {
|
||||
ompi_osc_rdma_deregister (module, curr->handle);
|
||||
curr->handle = NULL;
|
||||
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
||||
/*
|
||||
* Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
@ -15,13 +15,13 @@
|
||||
#include "osc_rdma_types.h"
|
||||
#include "osc_rdma_frag.h"
|
||||
|
||||
static inline int ompi_osc_rdma_trylock_local (volatile ompi_osc_rdma_lock_t *lock)
|
||||
static inline int ompi_osc_rdma_trylock_local (ompi_osc_rdma_atomic_lock_t *lock)
|
||||
{
|
||||
ompi_osc_rdma_lock_t _tmp_value = 0;
|
||||
return !ompi_osc_rdma_lock_compare_exchange (lock, &_tmp_value, OMPI_OSC_RDMA_LOCK_EXCLUSIVE);
|
||||
}
|
||||
|
||||
static inline void ompi_osc_rdma_unlock_local (volatile ompi_osc_rdma_lock_t *lock)
|
||||
static inline void ompi_osc_rdma_unlock_local (ompi_osc_rdma_atomic_lock_t *lock)
|
||||
{
|
||||
(void) ompi_osc_rdma_lock_add (lock, -OMPI_OSC_RDMA_LOCK_EXCLUSIVE);
|
||||
}
|
||||
@ -266,7 +266,7 @@ static inline int ompi_osc_rdma_lock_release_shared (ompi_osc_rdma_module_t *mod
|
||||
return ompi_osc_rdma_lock_btl_op (module, peer, lock, MCA_BTL_ATOMIC_ADD, value, false);
|
||||
}
|
||||
|
||||
(void) ompi_osc_rdma_lock_add ((volatile ompi_osc_rdma_lock_t *) lock, value);
|
||||
(void) ompi_osc_rdma_lock_add ((ompi_osc_rdma_atomic_lock_t *) lock, value);
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
@ -320,14 +320,14 @@ static inline int ompi_osc_rdma_lock_acquire_shared (ompi_osc_rdma_module_t *mod
|
||||
} while (1);
|
||||
} else {
|
||||
do {
|
||||
lock_state = ompi_osc_rdma_lock_add ((volatile ompi_osc_rdma_lock_t *) lock, value);
|
||||
lock_state = ompi_osc_rdma_lock_add ((ompi_osc_rdma_atomic_lock_t *) lock, value);
|
||||
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_DEBUG, "local shared lock incremented. old value 0x%lx",
|
||||
(unsigned long) lock_state);
|
||||
if (!(lock_state & check)) {
|
||||
break;
|
||||
}
|
||||
|
||||
(void) ompi_osc_rdma_lock_add ((volatile ompi_osc_rdma_lock_t *) lock, -value);
|
||||
(void) ompi_osc_rdma_lock_add ((ompi_osc_rdma_atomic_lock_t *) lock, -value);
|
||||
ompi_osc_rdma_progress (module);
|
||||
} while (1);
|
||||
}
|
||||
@ -378,7 +378,7 @@ static inline int ompi_osc_rdma_lock_try_acquire_exclusive (ompi_osc_rdma_module
|
||||
return lock_state != 0;
|
||||
}
|
||||
|
||||
return ompi_osc_rdma_trylock_local ((int64_t *)(intptr_t) lock);
|
||||
return ompi_osc_rdma_trylock_local ((ompi_osc_rdma_atomic_lock_t *)(intptr_t) lock);
|
||||
}
|
||||
|
||||
/**
|
||||
@ -431,7 +431,7 @@ static inline int ompi_osc_rdma_lock_release_exclusive (ompi_osc_rdma_module_t *
|
||||
abort ();
|
||||
}
|
||||
} else {
|
||||
ompi_osc_rdma_unlock_local ((volatile ompi_osc_rdma_lock_t *)(intptr_t) lock);
|
||||
ompi_osc_rdma_unlock_local ((ompi_osc_rdma_atomic_lock_t *)(intptr_t) lock);
|
||||
}
|
||||
|
||||
OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_DEBUG, "exclusive lock released");
|
||||
|
||||
@ -43,7 +43,7 @@ struct ompi_osc_rdma_peer_t {
|
||||
int rank;
|
||||
|
||||
/** peer flags */
|
||||
volatile int32_t flags;
|
||||
opal_atomic_int32_t flags;
|
||||
};
|
||||
typedef struct ompi_osc_rdma_peer_t ompi_osc_rdma_peer_t;
|
||||
|
||||
|
||||
@ -40,7 +40,7 @@ struct ompi_osc_rdma_request_t {
|
||||
void *origin_addr;
|
||||
|
||||
ompi_osc_rdma_module_t *module;
|
||||
volatile int32_t outstanding_requests;
|
||||
opal_atomic_int32_t outstanding_requests;
|
||||
bool internal;
|
||||
|
||||
ptrdiff_t offset;
|
||||
|
||||
@ -34,7 +34,7 @@ typedef enum ompi_osc_rdma_sync_type_t ompi_osc_rdma_sync_type_t;
|
||||
struct ompi_osc_rdma_module_t;
|
||||
|
||||
struct ompi_osc_rdma_sync_aligned_counter_t {
|
||||
volatile osc_rdma_counter_t counter;
|
||||
osc_rdma_atomic_counter_t counter;
|
||||
/* pad out to next cache line */
|
||||
uint64_t padding[7];
|
||||
};
|
||||
|
||||
@ -24,6 +24,7 @@ struct ompi_osc_rdma_peer_t;
|
||||
typedef int64_t osc_rdma_base_t;
|
||||
typedef int64_t osc_rdma_size_t;
|
||||
typedef int64_t osc_rdma_counter_t;
|
||||
typedef opal_atomic_int64_t osc_rdma_atomic_counter_t;
|
||||
|
||||
#define ompi_osc_rdma_counter_add opal_atomic_add_fetch_64
|
||||
|
||||
@ -32,6 +33,7 @@ typedef int64_t osc_rdma_counter_t;
|
||||
typedef int32_t osc_rdma_base_t;
|
||||
typedef int32_t osc_rdma_size_t;
|
||||
typedef int32_t osc_rdma_counter_t;
|
||||
typedef opal_atomic_int32_t osc_rdma_atomic_counter_t;
|
||||
|
||||
#define ompi_osc_rdma_counter_add opal_atomic_add_fetch_32
|
||||
|
||||
@ -42,8 +44,9 @@ typedef int32_t osc_rdma_counter_t;
|
||||
#define OMPI_OSC_RDMA_LOCK_EXCLUSIVE 0x8000000000000000l
|
||||
|
||||
typedef int64_t ompi_osc_rdma_lock_t;
|
||||
typedef opal_atomic_int64_t ompi_osc_rdma_atomic_lock_t;
|
||||
|
||||
static inline int64_t ompi_osc_rdma_lock_add (volatile int64_t *p, int64_t value)
|
||||
static inline int64_t ompi_osc_rdma_lock_add (opal_atomic_int64_t *p, int64_t value)
|
||||
{
|
||||
int64_t new;
|
||||
|
||||
@ -54,7 +57,7 @@ static inline int64_t ompi_osc_rdma_lock_add (volatile int64_t *p, int64_t value
|
||||
return new;
|
||||
}
|
||||
|
||||
static inline int ompi_osc_rdma_lock_compare_exchange (volatile int64_t *p, int64_t *comp, int64_t value)
|
||||
static inline int ompi_osc_rdma_lock_compare_exchange (opal_atomic_int64_t *p, int64_t *comp, int64_t value)
|
||||
{
|
||||
int ret;
|
||||
|
||||
@ -70,8 +73,9 @@ static inline int ompi_osc_rdma_lock_compare_exchange (volatile int64_t *p, int6
|
||||
#define OMPI_OSC_RDMA_LOCK_EXCLUSIVE 0x80000000l
|
||||
|
||||
typedef int32_t ompi_osc_rdma_lock_t;
|
||||
typedef opal_atomic_int32_t ompi_osc_rdma_atomic_lock_t;
|
||||
|
||||
static inline int32_t ompi_osc_rdma_lock_add (volatile int32_t *p, int32_t value)
|
||||
static inline int32_t ompi_osc_rdma_lock_add (opal_atomic_int32_t *p, int32_t value)
|
||||
{
|
||||
int32_t new;
|
||||
|
||||
@ -83,7 +87,7 @@ static inline int32_t ompi_osc_rdma_lock_add (volatile int32_t *p, int32_t value
|
||||
return new;
|
||||
}
|
||||
|
||||
static inline int ompi_osc_rdma_lock_compare_exchange (volatile int32_t *p, int32_t *comp, int32_t value)
|
||||
static inline int ompi_osc_rdma_lock_compare_exchange (opal_atomic_int32_t *p, int32_t *comp, int32_t value)
|
||||
{
|
||||
int ret;
|
||||
|
||||
@ -157,7 +161,7 @@ struct ompi_osc_rdma_state_t {
|
||||
/** post buffers */
|
||||
osc_rdma_counter_t post_peers[OMPI_OSC_RDMA_POST_PEER_MAX];
|
||||
/** counter for number of post messages received */
|
||||
osc_rdma_counter_t num_post_msgs;
|
||||
osc_rdma_atomic_counter_t num_post_msgs;
|
||||
/** counter for number of complete messages received */
|
||||
osc_rdma_counter_t num_complete_msgs;
|
||||
/** lock for the region state to ensure consistency */
|
||||
@ -195,11 +199,11 @@ struct ompi_osc_rdma_frag_t {
|
||||
opal_free_list_item_t super;
|
||||
|
||||
/* Number of operations which have started writing into the frag, but not yet completed doing so */
|
||||
volatile int32_t pending;
|
||||
opal_atomic_int32_t pending;
|
||||
#if OPAL_HAVE_ATOMIC_MATH_64
|
||||
volatile int64_t curr_index;
|
||||
opal_atomic_int64_t curr_index;
|
||||
#else
|
||||
volatile int32_t curr_index;
|
||||
opal_atomic_int32_t curr_index;
|
||||
#endif
|
||||
|
||||
struct ompi_osc_rdma_module_t *module;
|
||||
|
||||
@ -21,12 +21,14 @@
|
||||
#if OPAL_HAVE_ATOMIC_MATH_64
|
||||
|
||||
typedef uint64_t osc_sm_post_type_t;
|
||||
typedef opal_atomic_uint64_t osc_sm_post_atomic_type_t;
|
||||
#define OSC_SM_POST_BITS 6
|
||||
#define OSC_SM_POST_MASK 0x3f
|
||||
|
||||
#else
|
||||
|
||||
typedef uint32_t osc_sm_post_type_t;
|
||||
typedef opal_atomic_uint32_t osc_sm_post_atomic_type_t;
|
||||
#define OSC_SM_POST_BITS 5
|
||||
#define OSC_SM_POST_MASK 0x1f
|
||||
|
||||
@ -53,7 +55,7 @@ struct ompi_osc_sm_lock_t {
|
||||
typedef struct ompi_osc_sm_lock_t ompi_osc_sm_lock_t;
|
||||
|
||||
struct ompi_osc_sm_node_state_t {
|
||||
int32_t complete_count;
|
||||
opal_atomic_int32_t complete_count;
|
||||
ompi_osc_sm_lock_t lock;
|
||||
opal_atomic_lock_t accumulate_lock;
|
||||
};
|
||||
@ -98,7 +100,7 @@ struct ompi_osc_sm_module_t {
|
||||
ompi_osc_sm_node_state_t *my_node_state;
|
||||
ompi_osc_sm_node_state_t *node_states;
|
||||
|
||||
osc_sm_post_type_t ** volatile posts;
|
||||
osc_sm_post_atomic_type_t **posts;
|
||||
|
||||
opal_mutex_t lock;
|
||||
};
|
||||
|
||||
@ -162,9 +162,9 @@ ompi_osc_sm_start(struct ompi_group_t *group,
|
||||
opal_atomic_rmb ();
|
||||
|
||||
#if OPAL_HAVE_ATOMIC_MATH_64
|
||||
(void) opal_atomic_fetch_xor_64 ((volatile int64_t *) module->posts[my_rank] + rank_byte, rank_bit);
|
||||
(void) opal_atomic_fetch_xor_64 ((opal_atomic_int64_t *) module->posts[my_rank] + rank_byte, rank_bit);
|
||||
#else
|
||||
(void) opal_atomic_fetch_xor_32 ((volatile int32_t *) module->posts[my_rank] + rank_byte, rank_bit);
|
||||
(void) opal_atomic_fetch_xor_32 ((opal_atomic_int32_t *) module->posts[my_rank] + rank_byte, rank_bit);
|
||||
#endif
|
||||
}
|
||||
|
||||
@ -188,7 +188,7 @@ ompi_osc_sm_complete(struct ompi_win_t *win)
|
||||
opal_atomic_mb();
|
||||
|
||||
group = module->start_group;
|
||||
if (NULL == group || !OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&module->start_group, &group, NULL)) {
|
||||
if (NULL == group || !OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR((opal_atomic_intptr_t *) &module->start_group, (opal_atomic_intptr_t *) &group, 0)) {
|
||||
return OMPI_ERR_RMA_SYNC;
|
||||
}
|
||||
|
||||
@ -247,7 +247,11 @@ ompi_osc_sm_post(struct ompi_group_t *group,
|
||||
|
||||
gsize = ompi_group_size(module->post_group);
|
||||
for (int i = 0 ; i < gsize ; ++i) {
|
||||
opal_atomic_add ((volatile osc_sm_post_type_t *) module->posts[ranks[i]] + my_byte, my_bit);
|
||||
#if OPAL_HAVE_ATOMIC_MATH_64
|
||||
(void) opal_atomic_fetch_add_64 ((opal_atomic_int64_t *) module->posts[ranks[i]] + my_byte, my_bit);
|
||||
#else
|
||||
(void) opal_atomic_fetch_add_32 ((opal_atomic_int32_t *) module->posts[ranks[i]] + my_byte, my_bit);
|
||||
#endif
|
||||
}
|
||||
|
||||
opal_atomic_wmb ();
|
||||
|
||||
@ -236,7 +236,7 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit
|
||||
if (NULL == module->node_states) return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
|
||||
module->posts = calloc (1, sizeof(module->posts[0]) + sizeof (module->posts[0][0]));
|
||||
if (NULL == module->posts) return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
|
||||
module->posts[0] = (osc_sm_post_type_t *) (module->posts + 1);
|
||||
module->posts[0] = (osc_sm_post_atomic_type_t *) (module->posts + 1);
|
||||
} else {
|
||||
unsigned long total, *rbuf;
|
||||
int i, flag;
|
||||
@ -328,7 +328,7 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit
|
||||
if (NULL == module->posts) return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
|
||||
|
||||
/* set module->posts[0] first to ensure 64-bit alignment */
|
||||
module->posts[0] = (osc_sm_post_type_t *) (module->segment_base);
|
||||
module->posts[0] = (osc_sm_post_atomic_type_t *) (module->segment_base);
|
||||
module->global_state = (ompi_osc_sm_global_state_t *) (module->posts[0] + comm_size * post_size);
|
||||
module->node_states = (ompi_osc_sm_node_state_t *) (module->global_state + 1);
|
||||
|
||||
|
||||
@ -28,7 +28,7 @@ lk_fetch_add32(ompi_osc_sm_module_t *module,
|
||||
{
|
||||
/* opal_atomic_add_fetch_32 is an add then fetch so delta needs to be subtracted out to get the
|
||||
* old value */
|
||||
return opal_atomic_add_fetch_32((int32_t*) ((char*) &module->node_states[target].lock + offset),
|
||||
return opal_atomic_add_fetch_32((opal_atomic_int32_t *) ((char*) &module->node_states[target].lock + offset),
|
||||
delta) - delta;
|
||||
}
|
||||
|
||||
@ -39,7 +39,7 @@ lk_add32(ompi_osc_sm_module_t *module,
|
||||
size_t offset,
|
||||
uint32_t delta)
|
||||
{
|
||||
opal_atomic_add_fetch_32((int32_t*) ((char*) &module->node_states[target].lock + offset),
|
||||
opal_atomic_add_fetch_32((opal_atomic_int32_t *) ((char*) &module->node_states[target].lock + offset),
|
||||
delta);
|
||||
}
|
||||
|
||||
|
||||
@ -51,7 +51,7 @@ static size_t mca_pml_bsend_size; /* adjusted size of user buffe
|
||||
static size_t mca_pml_bsend_count; /* number of outstanding requests */
|
||||
static size_t mca_pml_bsend_pagesz; /* mmap page size */
|
||||
static int mca_pml_bsend_pagebits; /* number of bits in pagesz */
|
||||
static int32_t mca_pml_bsend_init = 0;
|
||||
static opal_atomic_int32_t mca_pml_bsend_init = 0;
|
||||
|
||||
/* defined in pml_base_open.c */
|
||||
extern char *ompi_pml_base_bsend_allocator_name;
|
||||
|
||||
@ -44,7 +44,7 @@ struct mca_pml_ob1_comm_proc_t {
|
||||
opal_object_t super;
|
||||
struct ompi_proc_t* ompi_proc;
|
||||
uint16_t expected_sequence; /**< send message sequence number - receiver side */
|
||||
volatile int32_t send_sequence; /**< send side sequence number */
|
||||
opal_atomic_int32_t send_sequence; /**< send side sequence number */
|
||||
struct mca_pml_ob1_recv_frag_t* frags_cant_match; /**< out-of-order fragment queues */
|
||||
#if !MCA_PML_OB1_CUSTOM_MATCH
|
||||
opal_list_t specific_receives; /**< queues of unmatched specific receives */
|
||||
|
||||
@ -53,7 +53,7 @@ static inline int mca_pml_ob1_process_pending_cuda_async_copies(void)
|
||||
}
|
||||
#endif /* OPAL_CUDA_SUPPORT */
|
||||
|
||||
static int mca_pml_ob1_progress_needed = 0;
|
||||
static opal_atomic_int32_t mca_pml_ob1_progress_needed = 0;
|
||||
int mca_pml_ob1_enable_progress(int32_t count)
|
||||
{
|
||||
int32_t progress_count = OPAL_ATOMIC_ADD_FETCH32(&mca_pml_ob1_progress_needed, count);
|
||||
|
||||
@ -41,9 +41,9 @@ BEGIN_C_DECLS
|
||||
struct mca_pml_ob1_recv_request_t {
|
||||
mca_pml_base_recv_request_t req_recv;
|
||||
opal_ptr_t remote_req_send;
|
||||
int32_t req_lock;
|
||||
int32_t req_pipeline_depth;
|
||||
size_t req_bytes_received; /**< amount of data transferred into the user buffer */
|
||||
opal_atomic_int32_t req_lock;
|
||||
opal_atomic_int32_t req_pipeline_depth;
|
||||
opal_atomic_size_t req_bytes_received; /**< amount of data transferred into the user buffer */
|
||||
size_t req_bytes_expected; /**< local size of the data as suggested by the user */
|
||||
size_t req_rdma_offset;
|
||||
size_t req_send_offset;
|
||||
|
||||
@ -46,11 +46,11 @@ struct mca_pml_ob1_send_request_t {
|
||||
mca_pml_base_send_request_t req_send;
|
||||
mca_bml_base_endpoint_t* req_endpoint;
|
||||
opal_ptr_t req_recv;
|
||||
int32_t req_state;
|
||||
int32_t req_lock;
|
||||
opal_atomic_int32_t req_state;
|
||||
opal_atomic_int32_t req_lock;
|
||||
bool req_throttle_sends;
|
||||
int32_t req_pipeline_depth;
|
||||
size_t req_bytes_delivered;
|
||||
opal_atomic_int32_t req_pipeline_depth;
|
||||
opal_atomic_size_t req_bytes_delivered;
|
||||
uint32_t req_rdma_cnt;
|
||||
mca_pml_ob1_send_pending_t req_pending;
|
||||
opal_mutex_t req_send_range_lock;
|
||||
|
||||
@ -28,7 +28,7 @@
|
||||
|
||||
static bool requests_initialized = false;
|
||||
static opal_list_t requests;
|
||||
static int32_t active_requests = 0;
|
||||
static opal_atomic_int32_t active_requests = 0;
|
||||
static bool in_progress = false;
|
||||
static opal_mutex_t lock;
|
||||
|
||||
|
||||
@ -51,7 +51,7 @@ struct ompi_predefined_datatype_t;
|
||||
/** Mutex to protect all the _init and _finalize variables */
|
||||
OMPI_DECLSPEC extern opal_mutex_t ompi_mpi_bootstrap_mutex;
|
||||
/** Did MPI start to initialize? */
|
||||
OMPI_DECLSPEC extern volatile int32_t ompi_mpi_state;
|
||||
OMPI_DECLSPEC extern opal_atomic_int32_t ompi_mpi_state;
|
||||
/** Has the RTE been initialized? */
|
||||
OMPI_DECLSPEC extern volatile bool ompi_rte_initialized;
|
||||
|
||||
|
||||
@ -130,7 +130,7 @@ const char ompi_version_string[] = OMPI_IDENT_STRING;
|
||||
* Global variables and symbols for the MPI layer
|
||||
*/
|
||||
|
||||
volatile int32_t ompi_mpi_state = OMPI_MPI_STATE_NOT_INITIALIZED;
|
||||
opal_atomic_int32_t ompi_mpi_state = OMPI_MPI_STATE_NOT_INITIALIZED;
|
||||
volatile bool ompi_rte_initialized = false;
|
||||
|
||||
bool ompi_mpi_thread_multiple = false;
|
||||
|
||||
@ -456,7 +456,7 @@ void ompi_spc_timer_stop(unsigned int event_id, opal_timer_t *cycles)
|
||||
/* This is denoted unlikely because the counters will often be turned off. */
|
||||
if( OPAL_UNLIKELY(IS_SPC_BIT_SET(ompi_spc_attached_event, event_id)) ) {
|
||||
*cycles = opal_timer_base_get_cycles() - *cycles;
|
||||
OPAL_THREAD_ADD_FETCH_SIZE_T(&ompi_spc_events[event_id].value, (ompi_spc_value_t)*cycles);
|
||||
OPAL_THREAD_ADD_FETCH_SIZE_T(&ompi_spc_events[event_id].value, (size_t) *cycles);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -166,7 +166,7 @@ typedef enum ompi_spc_counters {
|
||||
/* There is currently no support for atomics on long long values so we will default to
|
||||
* size_t for now until support for such atomics is implemented.
|
||||
*/
|
||||
typedef size_t ompi_spc_value_t;
|
||||
typedef opal_atomic_size_t ompi_spc_value_t;
|
||||
|
||||
/* A structure for storing the event data */
|
||||
typedef struct ompi_spc_s{
|
||||
|
||||
@ -10,7 +10,7 @@
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2014 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
@ -32,10 +32,10 @@ static void opal_fifo_construct (opal_fifo_t *fifo)
|
||||
fifo->opal_fifo_ghost.item_free = 0;
|
||||
|
||||
fifo->opal_fifo_head.data.counter = 0;
|
||||
fifo->opal_fifo_head.data.item = &fifo->opal_fifo_ghost;
|
||||
fifo->opal_fifo_head.data.item = (intptr_t) &fifo->opal_fifo_ghost;
|
||||
|
||||
fifo->opal_fifo_tail.data.counter = 0;
|
||||
fifo->opal_fifo_tail.data.item = &fifo->opal_fifo_ghost;
|
||||
fifo->opal_fifo_tail.data.item = (intptr_t) &fifo->opal_fifo_ghost;
|
||||
}
|
||||
|
||||
OBJ_CLASS_INSTANCE(opal_fifo_t, opal_object_t, opal_fifo_construct, NULL);
|
||||
|
||||
@ -100,13 +100,13 @@ static inline opal_list_item_t *opal_fifo_push_atomic (opal_fifo_t *fifo,
|
||||
|
||||
opal_atomic_wmb ();
|
||||
|
||||
if (ghost == tail.data.item) {
|
||||
if ((intptr_t) ghost == tail.data.item) {
|
||||
/* update the head */
|
||||
opal_counted_pointer_t head = {.value = fifo->opal_fifo_head.value};
|
||||
opal_update_counted_pointer (&fifo->opal_fifo_head, &head, item);
|
||||
} else {
|
||||
/* update previous item */
|
||||
tail.data.item->opal_list_next = item;
|
||||
((opal_list_item_t *) tail.data.item)->opal_list_next = item;
|
||||
}
|
||||
|
||||
return (opal_list_item_t *) tail.data.item;
|
||||
@ -129,12 +129,12 @@ static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo)
|
||||
item = (opal_list_item_t *) head.data.item;
|
||||
next = (opal_list_item_t *) item->opal_list_next;
|
||||
|
||||
if (ghost == tail.data.item && ghost == item) {
|
||||
if ((intptr_t) ghost == tail.data.item && ghost == item) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* the head or next pointer are in an inconsistent state. keep looping. */
|
||||
if (tail.data.item != item && ghost != tail.data.item && ghost == next) {
|
||||
if (tail.data.item != (intptr_t) item && (intptr_t) ghost != tail.data.item && ghost == next) {
|
||||
opal_read_counted_pointer (&fifo->opal_fifo_head, &head);
|
||||
continue;
|
||||
}
|
||||
@ -155,17 +155,15 @@ static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo)
|
||||
* update the head */
|
||||
|
||||
/* wait for next pointer to be updated by push */
|
||||
while (ghost == item->opal_list_next) {
|
||||
do {
|
||||
opal_atomic_rmb ();
|
||||
}
|
||||
|
||||
opal_atomic_rmb ();
|
||||
} while (ghost == item->opal_list_next);
|
||||
|
||||
/* update the head with the real next value. note that no other thread
|
||||
* will be attempting to update the head until after it has been updated
|
||||
* with the next pointer. push will not see an empty list and other pop
|
||||
* operations will loop until the head is consistent. */
|
||||
fifo->opal_fifo_head.data.item = (opal_list_item_t *) item->opal_list_next;
|
||||
fifo->opal_fifo_head.data.item = (intptr_t) item->opal_list_next;
|
||||
opal_atomic_wmb ();
|
||||
}
|
||||
}
|
||||
@ -191,13 +189,13 @@ static inline opal_list_item_t *opal_fifo_push_atomic (opal_fifo_t *fifo,
|
||||
opal_atomic_wmb ();
|
||||
|
||||
/* try to get the tail */
|
||||
tail_item = opal_atomic_swap_ptr (&fifo->opal_fifo_tail.data.item, item);
|
||||
tail_item = (opal_list_item_t *) opal_atomic_swap_ptr (&fifo->opal_fifo_tail.data.item, (intptr_t) item);
|
||||
|
||||
opal_atomic_wmb ();
|
||||
|
||||
if (ghost == tail_item) {
|
||||
/* update the head */
|
||||
fifo->opal_fifo_head.data.item = item;
|
||||
fifo->opal_fifo_head.data.item = (intptr_t) item;
|
||||
} else {
|
||||
/* update previous item */
|
||||
tail_item->opal_list_next = item;
|
||||
@ -230,7 +228,7 @@ static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo)
|
||||
|
||||
opal_atomic_ll_ptr(&fifo->opal_fifo_head.data.item, item);
|
||||
if (ghost == item) {
|
||||
if (ghost == fifo->opal_fifo_tail.data.item) {
|
||||
if ((intptr_t) ghost == fifo->opal_fifo_tail.data.item) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
@ -248,7 +246,7 @@ static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo)
|
||||
|
||||
/* protect against ABA issues by "locking" the head */
|
||||
do {
|
||||
if (!opal_atomic_swap_32 ((volatile int32_t *) &fifo->opal_fifo_head.data.counter, 1)) {
|
||||
if (!opal_atomic_swap_32 ((opal_atomic_int32_t *) &fifo->opal_fifo_head.data.counter, 1)) {
|
||||
break;
|
||||
}
|
||||
|
||||
@ -270,12 +268,12 @@ static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo)
|
||||
if (ghost == next) {
|
||||
void *tmp = item;
|
||||
|
||||
if (!opal_atomic_compare_exchange_strong_ptr (&fifo->opal_fifo_tail.data.item, &tmp, (void *) ghost)) {
|
||||
if (!opal_atomic_compare_exchange_strong_ptr (&fifo->opal_fifo_tail.data.item, (intptr_t *) &tmp, (intptr_t) ghost)) {
|
||||
do {
|
||||
opal_atomic_rmb ();
|
||||
} while (ghost == item->opal_list_next);
|
||||
|
||||
fifo->opal_fifo_head.data.item = (opal_list_item_t *) item->opal_list_next;
|
||||
fifo->opal_fifo_head.data.item = (intptr_t) item->opal_list_next;
|
||||
}
|
||||
}
|
||||
|
||||
@ -299,9 +297,9 @@ static inline opal_list_item_t *opal_fifo_push_st (opal_fifo_t *fifo,
|
||||
|
||||
item->opal_list_next = &fifo->opal_fifo_ghost;
|
||||
|
||||
fifo->opal_fifo_tail.data.item = item;
|
||||
fifo->opal_fifo_tail.data.item = (intptr_t) item;
|
||||
if (&fifo->opal_fifo_ghost == opal_fifo_head (fifo)) {
|
||||
fifo->opal_fifo_head.data.item = item;
|
||||
fifo->opal_fifo_head.data.item = (intptr_t) item;
|
||||
} else {
|
||||
prev->opal_list_next = item;
|
||||
}
|
||||
@ -317,9 +315,9 @@ static inline opal_list_item_t *opal_fifo_pop_st (opal_fifo_t *fifo)
|
||||
return NULL;
|
||||
}
|
||||
|
||||
fifo->opal_fifo_head.data.item = (opal_list_item_t *) item->opal_list_next;
|
||||
fifo->opal_fifo_head.data.item = (intptr_t) item->opal_list_next;
|
||||
if (&fifo->opal_fifo_ghost == opal_fifo_head (fifo)) {
|
||||
fifo->opal_fifo_tail.data.item = &fifo->opal_fifo_ghost;
|
||||
fifo->opal_fifo_tail.data.item = (intptr_t) &fifo->opal_fifo_ghost;
|
||||
}
|
||||
|
||||
item->opal_list_next = NULL;
|
||||
|
||||
@ -130,7 +130,7 @@ static opal_interval_tree_token_t opal_interval_tree_reader_get_token (opal_inte
|
||||
}
|
||||
}
|
||||
|
||||
while (!OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_32((volatile int32_t *) &tree->reader_epochs[token],
|
||||
while (!OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_32((opal_atomic_int32_t *) &tree->reader_epochs[token],
|
||||
&(int32_t) {UINT_MAX}, tree->epoch));
|
||||
|
||||
return token;
|
||||
|
||||
@ -83,11 +83,11 @@ struct opal_interval_tree_t {
|
||||
opal_free_list_t free_list; /**< the free list to get the memory from */
|
||||
opal_list_t gc_list; /**< list of nodes that need to be released */
|
||||
uint32_t epoch; /**< current update epoch */
|
||||
volatile size_t tree_size; /**< the current size of the tree */
|
||||
volatile int32_t lock; /**< update lock */
|
||||
volatile int32_t reader_count; /**< current highest reader slot to check */
|
||||
opal_atomic_size_t tree_size; /**< the current size of the tree */
|
||||
opal_atomic_int32_t lock; /**< update lock */
|
||||
opal_atomic_int32_t reader_count; /**< current highest reader slot to check */
|
||||
volatile uint32_t reader_id; /**< next reader slot to check */
|
||||
volatile uint32_t reader_epochs[OPAL_INTERVAL_TREE_MAX_READERS];
|
||||
opal_atomic_uint32_t reader_epochs[OPAL_INTERVAL_TREE_MAX_READERS];
|
||||
};
|
||||
typedef struct opal_interval_tree_t opal_interval_tree_t;
|
||||
|
||||
|
||||
@ -26,7 +26,7 @@ static void opal_lifo_construct (opal_lifo_t *lifo)
|
||||
{
|
||||
OBJ_CONSTRUCT(&lifo->opal_lifo_ghost, opal_list_item_t);
|
||||
lifo->opal_lifo_ghost.opal_list_next = &lifo->opal_lifo_ghost;
|
||||
lifo->opal_lifo_head.data.item = &lifo->opal_lifo_ghost;
|
||||
lifo->opal_lifo_head.data.item = (intptr_t) &lifo->opal_lifo_ghost;
|
||||
lifo->opal_lifo_head.data.counter = 0;
|
||||
}
|
||||
|
||||
|
||||
@ -48,11 +48,12 @@ union opal_counted_pointer_t {
|
||||
/** update counter used when cmpset_128 is available */
|
||||
uint64_t counter;
|
||||
/** list item pointer */
|
||||
volatile opal_list_item_t * volatile item;
|
||||
volatile opal_atomic_intptr_t item;
|
||||
} data;
|
||||
#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 && HAVE_OPAL_INT128_T
|
||||
/** used for atomics when there is a cmpset that can operate on
|
||||
* two 64-bit values */
|
||||
opal_atomic_int128_t atomic_value;
|
||||
opal_int128_t value;
|
||||
#endif
|
||||
};
|
||||
@ -65,17 +66,17 @@ typedef union opal_counted_pointer_t opal_counted_pointer_t;
|
||||
* to allow the upper level to detect if this element is the first one in the
|
||||
* list (if the list was empty before this operation).
|
||||
*/
|
||||
static inline bool opal_update_counted_pointer (volatile opal_counted_pointer_t *addr, opal_counted_pointer_t *old,
|
||||
static inline bool opal_update_counted_pointer (volatile opal_counted_pointer_t * volatile addr, opal_counted_pointer_t *old,
|
||||
opal_list_item_t *item)
|
||||
{
|
||||
opal_counted_pointer_t new_p;
|
||||
new_p.data.item = item;
|
||||
new_p.data.item = (intptr_t) item;
|
||||
new_p.data.counter = old->data.counter + 1;
|
||||
return opal_atomic_compare_exchange_strong_128 (&addr->value, &old->value, new_p.value);
|
||||
return opal_atomic_compare_exchange_strong_128 (&addr->atomic_value, &old->value, new_p.value);
|
||||
}
|
||||
|
||||
__opal_attribute_always_inline__
|
||||
static inline void opal_read_counted_pointer (volatile opal_counted_pointer_t *addr, opal_counted_pointer_t *value)
|
||||
static inline void opal_read_counted_pointer (volatile opal_counted_pointer_t * volatile addr, opal_counted_pointer_t *value)
|
||||
{
|
||||
/* most platforms do not read the value atomically so make sure we read the counted pointer in a specific order */
|
||||
value->data.counter = addr->data.counter;
|
||||
@ -151,7 +152,7 @@ static inline opal_list_item_t *opal_lifo_push_atomic (opal_lifo_t *lifo,
|
||||
opal_atomic_wmb ();
|
||||
|
||||
/* to protect against ABA issues it is sufficient to only update the counter in pop */
|
||||
if (opal_atomic_compare_exchange_strong_ptr (&lifo->opal_lifo_head.data.item, &next, item)) {
|
||||
if (opal_atomic_compare_exchange_strong_ptr (&lifo->opal_lifo_head.data.item, (intptr_t *) &next, (intptr_t) item)) {
|
||||
return next;
|
||||
}
|
||||
/* DO some kind of pause to release the bus */
|
||||
@ -200,7 +201,7 @@ static inline opal_list_item_t *opal_lifo_push_atomic (opal_lifo_t *lifo,
|
||||
do {
|
||||
item->opal_list_next = next;
|
||||
opal_atomic_wmb();
|
||||
if (opal_atomic_compare_exchange_strong_ptr (&lifo->opal_lifo_head.data.item, &next, item)) {
|
||||
if (opal_atomic_compare_exchange_strong_ptr (&lifo->opal_lifo_head.data.item, (intptr_t *) &next, (intptr_t) item)) {
|
||||
opal_atomic_wmb ();
|
||||
/* now safe to pop this item */
|
||||
item->item_free = 0;
|
||||
@ -254,7 +255,7 @@ static inline opal_list_item_t *opal_lifo_pop_atomic (opal_lifo_t* lifo)
|
||||
|
||||
while ((item=(opal_list_item_t *)lifo->opal_lifo_head.data.item) != ghost) {
|
||||
/* ensure it is safe to pop the head */
|
||||
if (opal_atomic_swap_32((volatile int32_t *) &item->item_free, 1)) {
|
||||
if (opal_atomic_swap_32((opal_atomic_int32_t *) &item->item_free, 1)) {
|
||||
continue;
|
||||
}
|
||||
|
||||
@ -262,8 +263,8 @@ static inline opal_list_item_t *opal_lifo_pop_atomic (opal_lifo_t* lifo)
|
||||
|
||||
head = item;
|
||||
/* try to swap out the head pointer */
|
||||
if (opal_atomic_compare_exchange_strong_ptr (&lifo->opal_lifo_head.data.item, &head,
|
||||
(void *) item->opal_list_next)) {
|
||||
if (opal_atomic_compare_exchange_strong_ptr (&lifo->opal_lifo_head.data.item, (intptr_t *) &head,
|
||||
(intptr_t) item->opal_list_next)) {
|
||||
break;
|
||||
}
|
||||
|
||||
@ -294,7 +295,7 @@ static inline opal_list_item_t *opal_lifo_push_st (opal_lifo_t *lifo,
|
||||
{
|
||||
item->opal_list_next = (opal_list_item_t *) lifo->opal_lifo_head.data.item;
|
||||
item->item_free = 0;
|
||||
lifo->opal_lifo_head.data.item = item;
|
||||
lifo->opal_lifo_head.data.item = (intptr_t) item;
|
||||
return (opal_list_item_t *) item->opal_list_next;
|
||||
}
|
||||
|
||||
@ -302,7 +303,7 @@ static inline opal_list_item_t *opal_lifo_pop_st (opal_lifo_t *lifo)
|
||||
{
|
||||
opal_list_item_t *item;
|
||||
item = (opal_list_item_t *) lifo->opal_lifo_head.data.item;
|
||||
lifo->opal_lifo_head.data.item = (opal_list_item_t *) item->opal_list_next;
|
||||
lifo->opal_lifo_head.data.item = (intptr_t) item->opal_list_next;
|
||||
if (item == &lifo->opal_lifo_ghost) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
@ -111,7 +111,7 @@ struct opal_list_item_t
|
||||
|
||||
#if OPAL_ENABLE_DEBUG
|
||||
/** Atomic reference count for debugging */
|
||||
volatile int32_t opal_list_item_refcount;
|
||||
opal_atomic_int32_t opal_list_item_refcount;
|
||||
/** The list this item belong to */
|
||||
volatile struct opal_list_t* opal_list_item_belong_to;
|
||||
#endif
|
||||
@ -654,7 +654,7 @@ static inline opal_list_item_t *opal_list_remove_first(opal_list_t *list)
|
||||
Caller now owns the item and should release the item
|
||||
when caller is done with it.
|
||||
*/
|
||||
volatile opal_list_item_t *item;
|
||||
opal_list_item_t *item;
|
||||
if ( 0 == list->opal_list_length ) {
|
||||
return (opal_list_item_t *)NULL;
|
||||
}
|
||||
@ -669,7 +669,7 @@ static inline opal_list_item_t *opal_list_remove_first(opal_list_t *list)
|
||||
list->opal_list_length--;
|
||||
|
||||
/* get pointer to first element on the list */
|
||||
item = list->opal_list_sentinel.opal_list_next;
|
||||
item = (opal_list_item_t *) list->opal_list_sentinel.opal_list_next;
|
||||
|
||||
/* reset previous pointer of next item on the list */
|
||||
item->opal_list_next->opal_list_prev = item->opal_list_prev;
|
||||
@ -686,11 +686,11 @@ static inline opal_list_item_t *opal_list_remove_first(opal_list_t *list)
|
||||
/* Spot check: ensure that the item we're returning is now on no
|
||||
lists */
|
||||
|
||||
OPAL_THREAD_ADD_FETCH32( &(item->opal_list_item_refcount), -1 );
|
||||
OPAL_THREAD_ADD_FETCH32( &item->opal_list_item_refcount, -1 );
|
||||
assert(0 == item->opal_list_item_refcount);
|
||||
#endif
|
||||
|
||||
return (opal_list_item_t *) item;
|
||||
return item;
|
||||
}
|
||||
|
||||
|
||||
@ -716,7 +716,7 @@ static inline opal_list_item_t *opal_list_remove_last(opal_list_t *list)
|
||||
Caller now owns the item and should release the item
|
||||
when caller is done with it.
|
||||
*/
|
||||
volatile opal_list_item_t *item;
|
||||
opal_list_item_t *item;
|
||||
if ( 0 == list->opal_list_length ) {
|
||||
return (opal_list_item_t *)NULL;
|
||||
}
|
||||
@ -731,7 +731,7 @@ static inline opal_list_item_t *opal_list_remove_last(opal_list_t *list)
|
||||
list->opal_list_length--;
|
||||
|
||||
/* get item */
|
||||
item = list->opal_list_sentinel.opal_list_prev;
|
||||
item = (opal_list_item_t *) list->opal_list_sentinel.opal_list_prev;
|
||||
|
||||
/* reset previous pointer on next to last pointer */
|
||||
item->opal_list_prev->opal_list_next = item->opal_list_next;
|
||||
@ -746,12 +746,12 @@ static inline opal_list_item_t *opal_list_remove_last(opal_list_t *list)
|
||||
/* Spot check: ensure that the item we're returning is now on no
|
||||
lists */
|
||||
|
||||
OPAL_THREAD_ADD_FETCH32( &(item->opal_list_item_refcount), -1 );
|
||||
OPAL_THREAD_ADD_FETCH32(&item->opal_list_item_refcount, -1 );
|
||||
assert(0 == item->opal_list_item_refcount);
|
||||
item->opal_list_item_belong_to = NULL;
|
||||
#endif
|
||||
|
||||
return (opal_list_item_t *) item;
|
||||
return item;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@ -55,7 +55,7 @@ int opal_class_init_epoch = 1;
|
||||
/*
|
||||
* Local variables
|
||||
*/
|
||||
static opal_atomic_lock_t class_lock = { { OPAL_ATOMIC_LOCK_UNLOCKED } };
|
||||
static opal_atomic_lock_t class_lock = OPAL_ATOMIC_LOCK_INIT;
|
||||
static void** classes = NULL;
|
||||
static int num_classes = 0;
|
||||
static int max_classes = 0;
|
||||
|
||||
@ -198,7 +198,7 @@ struct opal_object_t {
|
||||
uint64_t obj_magic_id;
|
||||
#endif
|
||||
opal_class_t *obj_class; /**< class descriptor */
|
||||
volatile int32_t obj_reference_count; /**< reference count */
|
||||
opal_atomic_int32_t obj_reference_count; /**< reference count */
|
||||
#if OPAL_ENABLE_DEBUG
|
||||
const char* cls_init_file_name; /**< In debug mode store the file where the object get contructed */
|
||||
int cls_init_lineno; /**< In debug mode store the line number where the object get contructed */
|
||||
|
||||
@ -120,7 +120,7 @@ typedef struct opal_tree_item_t
|
||||
|
||||
#if OPAL_ENABLE_DEBUG
|
||||
/** Atomic reference count for debugging */
|
||||
int32_t opal_tree_item_refcount;
|
||||
opal_atomic_int32_t opal_tree_item_refcount;
|
||||
/** The tree this item belong to */
|
||||
struct opal_tree_t* opal_tree_item_belong_to;
|
||||
#endif
|
||||
|
||||
@ -10,6 +10,8 @@
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2010-2011 Cisco Systems, Inc. All rights reserved.
|
||||
# Copyright (c) 2018 Los Alamos National Security, LLC. All rights
|
||||
# reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
@ -20,7 +22,8 @@
|
||||
headers = \
|
||||
opal_config_top.h \
|
||||
opal_config_bottom.h \
|
||||
opal_stdint.h
|
||||
opal_stdint.h \
|
||||
opal_stdatomic.h
|
||||
|
||||
nodist_headers = opal_config.h
|
||||
|
||||
|
||||
@ -11,7 +11,7 @@
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
|
||||
# Copyright (c) 2011 Sandia National Laboratories. All rights reserved.
|
||||
# Copyright (c) 2016 Los Alamos National Security, LLC. All rights
|
||||
# Copyright (c) 2016-2018 Los Alamos National Security, LLC. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2017 Research Organization for Information Science
|
||||
# and Technology (RIST). All rights reserved.
|
||||
@ -27,6 +27,7 @@
|
||||
headers += \
|
||||
opal/sys/architecture.h \
|
||||
opal/sys/atomic.h \
|
||||
opal/sys/atomic_stdc.h \
|
||||
opal/sys/atomic_impl.h \
|
||||
opal/sys/timer.h \
|
||||
opal/sys/cma.h
|
||||
|
||||
@ -47,6 +47,7 @@
|
||||
#define OPAL_BUILTIN_SYNC 0200
|
||||
#define OPAL_BUILTIN_GCC 0202
|
||||
#define OPAL_BUILTIN_NO 0203
|
||||
#define OPAL_BUILTIN_C11 0204
|
||||
|
||||
/* Formats */
|
||||
#define OPAL_DEFAULT 1000 /* standard for given architecture */
|
||||
|
||||
@ -12,7 +12,7 @@
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2010 IBM Corporation. All rights reserved.
|
||||
* Copyright (c) 2010 ARM ltd. All rights reserved.
|
||||
* Copyright (c) 2017 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2017-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
@ -109,7 +109,7 @@ void opal_atomic_isync(void)
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
|
||||
#define OPAL_HAVE_ATOMIC_MATH_32 1
|
||||
static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
int32_t prev, tmp;
|
||||
bool ret;
|
||||
@ -137,7 +137,7 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add
|
||||
atomic_?mb can be inlined). Instead, we "inline" them by hand in
|
||||
the assembly, meaning there is one function call overhead instead
|
||||
of two */
|
||||
static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
bool rc;
|
||||
|
||||
@ -148,7 +148,7 @@ static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t
|
||||
}
|
||||
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
opal_atomic_wmb();
|
||||
return opal_atomic_compare_exchange_strong_32 (addr, oldval, newval);
|
||||
@ -157,7 +157,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t
|
||||
#if (OPAL_ASM_SUPPORT_64BIT == 1)
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
|
||||
static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
int64_t prev;
|
||||
int tmp;
|
||||
@ -188,7 +188,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add
|
||||
atomic_?mb can be inlined). Instead, we "inline" them by hand in
|
||||
the assembly, meaning there is one function call overhead instead
|
||||
of two */
|
||||
static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
bool rc;
|
||||
|
||||
@ -199,7 +199,7 @@ static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t
|
||||
}
|
||||
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
opal_atomic_wmb();
|
||||
return opal_atomic_compare_exchange_strong_64 (addr, oldval, newval);
|
||||
@ -209,7 +209,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t
|
||||
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_ADD_32 1
|
||||
static inline int32_t opal_atomic_fetch_add_32(volatile int32_t* v, int inc)
|
||||
static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t* v, int inc)
|
||||
{
|
||||
int32_t t, old;
|
||||
int tmp;
|
||||
@ -230,7 +230,7 @@ static inline int32_t opal_atomic_fetch_add_32(volatile int32_t* v, int inc)
|
||||
}
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_SUB_32 1
|
||||
static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t* v, int dec)
|
||||
static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t* v, int dec)
|
||||
{
|
||||
int32_t t, old;
|
||||
int tmp;
|
||||
|
||||
@ -12,7 +12,7 @@
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2010 IBM Corporation. All rights reserved.
|
||||
* Copyright (c) 2010 ARM ltd. All rights reserved.
|
||||
* Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2016-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
@ -82,7 +82,7 @@ static inline void opal_atomic_isync (void)
|
||||
*
|
||||
*********************************************************************/
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
int32_t prev, tmp;
|
||||
bool ret;
|
||||
@ -102,7 +102,7 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add
|
||||
return ret;
|
||||
}
|
||||
|
||||
static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, int32_t newval)
|
||||
static inline int32_t opal_atomic_swap_32(opal_atomic_int32_t *addr, int32_t newval)
|
||||
{
|
||||
int32_t ret, tmp;
|
||||
|
||||
@ -121,7 +121,7 @@ static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, int32_t newval
|
||||
atomic_?mb can be inlined). Instead, we "inline" them by hand in
|
||||
the assembly, meaning there is one function call overhead instead
|
||||
of two */
|
||||
static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
int32_t prev, tmp;
|
||||
bool ret;
|
||||
@ -142,7 +142,7 @@ static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t
|
||||
}
|
||||
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
int32_t prev, tmp;
|
||||
bool ret;
|
||||
@ -164,7 +164,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t
|
||||
|
||||
#define opal_atomic_ll_32(addr, ret) \
|
||||
do { \
|
||||
volatile int32_t *_addr = (addr); \
|
||||
opal_atomic_int32_t *_addr = (addr); \
|
||||
int32_t _ret; \
|
||||
\
|
||||
__asm__ __volatile__ ("ldaxr %w0, [%1] \n" \
|
||||
@ -176,7 +176,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t
|
||||
|
||||
#define opal_atomic_sc_32(addr, newval, ret) \
|
||||
do { \
|
||||
volatile int32_t *_addr = (addr); \
|
||||
opal_atomic_int32_t *_addr = (addr); \
|
||||
int32_t _newval = (int32_t) newval; \
|
||||
int _ret; \
|
||||
\
|
||||
@ -188,7 +188,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t
|
||||
ret = (_ret == 0); \
|
||||
} while (0)
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
int64_t prev;
|
||||
int tmp;
|
||||
@ -209,7 +209,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add
|
||||
return ret;
|
||||
}
|
||||
|
||||
static inline int64_t opal_atomic_swap_64 (volatile int64_t *addr, int64_t newval)
|
||||
static inline int64_t opal_atomic_swap_64 (opal_atomic_int64_t *addr, int64_t newval)
|
||||
{
|
||||
int64_t ret;
|
||||
int tmp;
|
||||
@ -229,7 +229,7 @@ static inline int64_t opal_atomic_swap_64 (volatile int64_t *addr, int64_t newva
|
||||
atomic_?mb can be inlined). Instead, we "inline" them by hand in
|
||||
the assembly, meaning there is one function call overhead instead
|
||||
of two */
|
||||
static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
int64_t prev;
|
||||
int tmp;
|
||||
@ -251,7 +251,7 @@ static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t
|
||||
}
|
||||
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
int64_t prev;
|
||||
int tmp;
|
||||
@ -274,7 +274,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t
|
||||
|
||||
#define opal_atomic_ll_64(addr, ret) \
|
||||
do { \
|
||||
volatile int64_t *_addr = (addr); \
|
||||
opal_atomic_int64_t *_addr = (addr); \
|
||||
int64_t _ret; \
|
||||
\
|
||||
__asm__ __volatile__ ("ldaxr %0, [%1] \n" \
|
||||
@ -286,7 +286,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t
|
||||
|
||||
#define opal_atomic_sc_64(addr, newval, ret) \
|
||||
do { \
|
||||
volatile int64_t *_addr = (addr); \
|
||||
opal_atomic_int64_t *_addr = (addr); \
|
||||
int64_t _newval = (int64_t) newval; \
|
||||
int _ret; \
|
||||
\
|
||||
@ -299,7 +299,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t
|
||||
} while (0)
|
||||
|
||||
#define OPAL_ASM_MAKE_ATOMIC(type, bits, name, inst, reg) \
|
||||
static inline type opal_atomic_fetch_ ## name ## _ ## bits (volatile type *addr, type value) \
|
||||
static inline type opal_atomic_fetch_ ## name ## _ ## bits (opal_atomic_ ## type *addr, type value) \
|
||||
{ \
|
||||
type newval, old; \
|
||||
int32_t tmp; \
|
||||
|
||||
@ -56,7 +56,13 @@
|
||||
#include <stdbool.h>
|
||||
|
||||
#include "opal/sys/architecture.h"
|
||||
#include "opal_stdint.h"
|
||||
#include "opal_stdatomic.h"
|
||||
|
||||
#if OPAL_ASSEMBLY_BUILTIN == OPAL_BUILTIN_C11
|
||||
|
||||
#include "atomic_stdc.h"
|
||||
|
||||
#else /* !OPAL_C_HAVE__ATOMIC */
|
||||
|
||||
/* do some quick #define cleanup in cases where we are doing
|
||||
testing... */
|
||||
@ -92,7 +98,7 @@ BEGIN_C_DECLS
|
||||
*/
|
||||
struct opal_atomic_lock_t {
|
||||
union {
|
||||
volatile int32_t lock; /**< The lock address (an integer) */
|
||||
opal_atomic_int32_t lock; /**< The lock address (an integer) */
|
||||
volatile unsigned char sparc_lock; /**< The lock address on sparc */
|
||||
char padding[sizeof(int)]; /**< Array for optional padding */
|
||||
} u;
|
||||
@ -147,6 +153,8 @@ enum {
|
||||
OPAL_ATOMIC_LOCK_LOCKED = 1
|
||||
};
|
||||
|
||||
#define OPAL_ATOMIC_LOCK_INIT {.u = {.lock = OPAL_ATOMIC_LOCK_UNLOCKED}}
|
||||
|
||||
/**********************************************************************
|
||||
*
|
||||
* Load the appropriate architecture files and set some reasonable
|
||||
@ -350,19 +358,19 @@ void opal_atomic_unlock(opal_atomic_lock_t *lock);
|
||||
#if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32
|
||||
static inline
|
||||
#endif
|
||||
bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval,
|
||||
bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval,
|
||||
int32_t newval);
|
||||
|
||||
#if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32
|
||||
static inline
|
||||
#endif
|
||||
bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval,
|
||||
bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval,
|
||||
int32_t newval);
|
||||
|
||||
#if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32
|
||||
static inline
|
||||
#endif
|
||||
bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval,
|
||||
bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval,
|
||||
int32_t newval);
|
||||
#endif
|
||||
|
||||
@ -375,19 +383,19 @@ bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t
|
||||
#if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64
|
||||
static inline
|
||||
#endif
|
||||
bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval,
|
||||
bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval,
|
||||
int64_t newval);
|
||||
|
||||
#if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64
|
||||
static inline
|
||||
#endif
|
||||
bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval,
|
||||
bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval,
|
||||
int64_t newval);
|
||||
|
||||
#if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64
|
||||
static inline
|
||||
#endif
|
||||
bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval,
|
||||
bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval,
|
||||
int64_t newval);
|
||||
|
||||
#endif
|
||||
@ -399,20 +407,20 @@ bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t
|
||||
|
||||
#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
|
||||
|
||||
static inline int32_t opal_atomic_add_fetch_32(volatile int32_t *addr, int delta);
|
||||
static inline int32_t opal_atomic_fetch_add_32(volatile int32_t *addr, int delta);
|
||||
static inline int32_t opal_atomic_and_fetch_32(volatile int32_t *addr, int32_t value);
|
||||
static inline int32_t opal_atomic_fetch_and_32(volatile int32_t *addr, int32_t value);
|
||||
static inline int32_t opal_atomic_or_fetch_32(volatile int32_t *addr, int32_t value);
|
||||
static inline int32_t opal_atomic_fetch_or_32(volatile int32_t *addr, int32_t value);
|
||||
static inline int32_t opal_atomic_xor_fetch_32(volatile int32_t *addr, int32_t value);
|
||||
static inline int32_t opal_atomic_fetch_xor_32(volatile int32_t *addr, int32_t value);
|
||||
static inline int32_t opal_atomic_sub_fetch_32(volatile int32_t *addr, int delta);
|
||||
static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t *addr, int delta);
|
||||
static inline int32_t opal_atomic_min_fetch_32 (volatile int32_t *addr, int32_t value);
|
||||
static inline int32_t opal_atomic_fetch_min_32 (volatile int32_t *addr, int32_t value);
|
||||
static inline int32_t opal_atomic_max_fetch_32 (volatile int32_t *addr, int32_t value);
|
||||
static inline int32_t opal_atomic_fetch_max_32 (volatile int32_t *addr, int32_t value);
|
||||
static inline int32_t opal_atomic_add_fetch_32(opal_atomic_int32_t *addr, int delta);
|
||||
static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t *addr, int delta);
|
||||
static inline int32_t opal_atomic_and_fetch_32(opal_atomic_int32_t *addr, int32_t value);
|
||||
static inline int32_t opal_atomic_fetch_and_32(opal_atomic_int32_t *addr, int32_t value);
|
||||
static inline int32_t opal_atomic_or_fetch_32(opal_atomic_int32_t *addr, int32_t value);
|
||||
static inline int32_t opal_atomic_fetch_or_32(opal_atomic_int32_t *addr, int32_t value);
|
||||
static inline int32_t opal_atomic_xor_fetch_32(opal_atomic_int32_t *addr, int32_t value);
|
||||
static inline int32_t opal_atomic_fetch_xor_32(opal_atomic_int32_t *addr, int32_t value);
|
||||
static inline int32_t opal_atomic_sub_fetch_32(opal_atomic_int32_t *addr, int delta);
|
||||
static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t *addr, int delta);
|
||||
static inline int32_t opal_atomic_min_fetch_32 (opal_atomic_int32_t *addr, int32_t value);
|
||||
static inline int32_t opal_atomic_fetch_min_32 (opal_atomic_int32_t *addr, int32_t value);
|
||||
static inline int32_t opal_atomic_max_fetch_32 (opal_atomic_int32_t *addr, int32_t value);
|
||||
static inline int32_t opal_atomic_fetch_max_32 (opal_atomic_int32_t *addr, int32_t value);
|
||||
|
||||
#endif /* OPAL_HAVE_ATOMIC_MATH_32 */
|
||||
|
||||
@ -429,19 +437,19 @@ static inline int32_t opal_atomic_fetch_max_32 (volatile int32_t *addr, int32_t
|
||||
|
||||
#if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MATH_64 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
|
||||
|
||||
static inline int64_t opal_atomic_add_fetch_64(volatile int64_t *addr, int64_t delta);
|
||||
static inline int64_t opal_atomic_fetch_add_64(volatile int64_t *addr, int64_t delta);
|
||||
static inline int64_t opal_atomic_and_fetch_64(volatile int64_t *addr, int64_t value);
|
||||
static inline int64_t opal_atomic_fetch_and_64(volatile int64_t *addr, int64_t value);
|
||||
static inline int64_t opal_atomic_or_fetch_64(volatile int64_t *addr, int64_t value);
|
||||
static inline int64_t opal_atomic_fetch_or_64(volatile int64_t *addr, int64_t value);
|
||||
static inline int64_t opal_atomic_fetch_xor_64(volatile int64_t *addr, int64_t value);
|
||||
static inline int64_t opal_atomic_sub_fetch_64(volatile int64_t *addr, int64_t delta);
|
||||
static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t *addr, int64_t delta);
|
||||
static inline int64_t opal_atomic_min_fetch_64 (volatile int64_t *addr, int64_t value);
|
||||
static inline int64_t opal_atomic_fetch_min_64 (volatile int64_t *addr, int64_t value);
|
||||
static inline int64_t opal_atomic_max_fetch_64 (volatile int64_t *addr, int64_t value);
|
||||
static inline int64_t opal_atomic_fetch_max_64 (volatile int64_t *addr, int64_t value);
|
||||
static inline int64_t opal_atomic_add_fetch_64(opal_atomic_int64_t *addr, int64_t delta);
|
||||
static inline int64_t opal_atomic_fetch_add_64(opal_atomic_int64_t *addr, int64_t delta);
|
||||
static inline int64_t opal_atomic_and_fetch_64(opal_atomic_int64_t *addr, int64_t value);
|
||||
static inline int64_t opal_atomic_fetch_and_64(opal_atomic_int64_t *addr, int64_t value);
|
||||
static inline int64_t opal_atomic_or_fetch_64(opal_atomic_int64_t *addr, int64_t value);
|
||||
static inline int64_t opal_atomic_fetch_or_64(opal_atomic_int64_t *addr, int64_t value);
|
||||
static inline int64_t opal_atomic_fetch_xor_64(opal_atomic_int64_t *addr, int64_t value);
|
||||
static inline int64_t opal_atomic_sub_fetch_64(opal_atomic_int64_t *addr, int64_t delta);
|
||||
static inline int64_t opal_atomic_fetch_sub_64(opal_atomic_int64_t *addr, int64_t delta);
|
||||
static inline int64_t opal_atomic_min_fetch_64 (opal_atomic_int64_t *addr, int64_t value);
|
||||
static inline int64_t opal_atomic_fetch_min_64 (opal_atomic_int64_t *addr, int64_t value);
|
||||
static inline int64_t opal_atomic_max_fetch_64 (opal_atomic_int64_t *addr, int64_t value);
|
||||
static inline int64_t opal_atomic_fetch_max_64 (opal_atomic_int64_t *addr, int64_t value);
|
||||
|
||||
#endif /* OPAL_HAVE_ATOMIC_MATH_64 */
|
||||
|
||||
@ -458,7 +466,7 @@ static inline int64_t opal_atomic_fetch_max_64 (volatile int64_t *addr, int64_t
|
||||
*/
|
||||
#if defined(DOXYGEN) || OPAL_ENABLE_DEBUG
|
||||
static inline size_t
|
||||
opal_atomic_add_fetch_size_t(volatile size_t *addr, size_t delta)
|
||||
opal_atomic_add_fetch_size_t(opal_atomic_size_t *addr, size_t delta)
|
||||
{
|
||||
#if SIZEOF_SIZE_T == 4
|
||||
return (size_t) opal_atomic_add_fetch_32((int32_t*) addr, delta);
|
||||
@ -470,7 +478,7 @@ opal_atomic_add_fetch_size_t(volatile size_t *addr, size_t delta)
|
||||
}
|
||||
|
||||
static inline size_t
|
||||
opal_atomic_fetch_add_size_t(volatile size_t *addr, size_t delta)
|
||||
opal_atomic_fetch_add_size_t(opal_atomic_size_t *addr, size_t delta)
|
||||
{
|
||||
#if SIZEOF_SIZE_T == 4
|
||||
return (size_t) opal_atomic_fetch_add_32((int32_t*) addr, delta);
|
||||
@ -482,7 +490,7 @@ opal_atomic_fetch_add_size_t(volatile size_t *addr, size_t delta)
|
||||
}
|
||||
|
||||
static inline size_t
|
||||
opal_atomic_sub_fetch_size_t(volatile size_t *addr, size_t delta)
|
||||
opal_atomic_sub_fetch_size_t(opal_atomic_size_t *addr, size_t delta)
|
||||
{
|
||||
#if SIZEOF_SIZE_T == 4
|
||||
return (size_t) opal_atomic_sub_fetch_32((int32_t*) addr, delta);
|
||||
@ -494,7 +502,7 @@ opal_atomic_sub_fetch_size_t(volatile size_t *addr, size_t delta)
|
||||
}
|
||||
|
||||
static inline size_t
|
||||
opal_atomic_fetch_sub_size_t(volatile size_t *addr, size_t delta)
|
||||
opal_atomic_fetch_sub_size_t(opal_atomic_size_t *addr, size_t delta)
|
||||
{
|
||||
#if SIZEOF_SIZE_T == 4
|
||||
return (size_t) opal_atomic_fetch_sub_32((int32_t*) addr, delta);
|
||||
@ -507,15 +515,15 @@ opal_atomic_fetch_sub_size_t(volatile size_t *addr, size_t delta)
|
||||
|
||||
#else
|
||||
#if SIZEOF_SIZE_T == 4
|
||||
#define opal_atomic_add_fetch_size_t(addr, delta) ((size_t) opal_atomic_add_fetch_32((volatile int32_t *) addr, delta))
|
||||
#define opal_atomic_fetch_add_size_t(addr, delta) ((size_t) opal_atomic_fetch_add_32((volatile int32_t *) addr, delta))
|
||||
#define opal_atomic_sub_fetch_size_t(addr, delta) ((size_t) opal_atomic_sub_fetch_32((volatile int32_t *) addr, delta))
|
||||
#define opal_atomic_fetch_sub_size_t(addr, delta) ((size_t) opal_atomic_fetch_sub_32((volatile int32_t *) addr, delta))
|
||||
#define opal_atomic_add_fetch_size_t(addr, delta) ((size_t) opal_atomic_add_fetch_32((opal_atomic_int32_t *) addr, delta))
|
||||
#define opal_atomic_fetch_add_size_t(addr, delta) ((size_t) opal_atomic_fetch_add_32((opal_atomic_int32_t *) addr, delta))
|
||||
#define opal_atomic_sub_fetch_size_t(addr, delta) ((size_t) opal_atomic_sub_fetch_32((opal_atomic_int32_t *) addr, delta))
|
||||
#define opal_atomic_fetch_sub_size_t(addr, delta) ((size_t) opal_atomic_fetch_sub_32((opal_atomic_int32_t *) addr, delta))
|
||||
#elif SIZEOF_SIZE_T == 8
|
||||
#define opal_atomic_add_fetch_size_t(addr, delta) ((size_t) opal_atomic_add_fetch_64((volatile int64_t *) addr, delta))
|
||||
#define opal_atomic_fetch_add_size_t(addr, delta) ((size_t) opal_atomic_fetch_add_64((volatile int64_t *) addr, delta))
|
||||
#define opal_atomic_sub_fetch_size_t(addr, delta) ((size_t) opal_atomic_sub_fetch_64((volatile int64_t *) addr, delta))
|
||||
#define opal_atomic_fetch_sub_size_t(addr, delta) ((size_t) opal_atomic_fetch_sub_64((volatile int64_t *) addr, delta))
|
||||
#define opal_atomic_add_fetch_size_t(addr, delta) ((size_t) opal_atomic_add_fetch_64((opal_atomic_int64_t *) addr, delta))
|
||||
#define opal_atomic_fetch_add_size_t(addr, delta) ((size_t) opal_atomic_fetch_add_64((opal_atomic_int64_t *) addr, delta))
|
||||
#define opal_atomic_sub_fetch_size_t(addr, delta) ((size_t) opal_atomic_sub_fetch_64((opal_atomic_int64_t *) addr, delta))
|
||||
#define opal_atomic_fetch_sub_size_t(addr, delta) ((size_t) opal_atomic_fetch_sub_64((opal_atomic_int64_t *) addr, delta))
|
||||
#else
|
||||
#error "Unknown size_t size"
|
||||
#endif
|
||||
@ -525,20 +533,20 @@ opal_atomic_fetch_sub_size_t(volatile size_t *addr, size_t delta)
|
||||
/* these are always done with inline functions, so always mark as
|
||||
static inline */
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_xx (volatile void *addr, void *oldval,
|
||||
static inline bool opal_atomic_compare_exchange_strong_xx (opal_atomic_intptr_t *addr, intptr_t *oldval,
|
||||
int64_t newval, size_t length);
|
||||
static inline bool opal_atomic_compare_exchange_strong_acq_xx (volatile void *addr, void *oldval,
|
||||
static inline bool opal_atomic_compare_exchange_strong_acq_xx (opal_atomic_intptr_t *addr, intptr_t *oldval,
|
||||
int64_t newval, size_t length);
|
||||
static inline bool opal_atomic_compare_exchange_strong_rel_xx (volatile void *addr, void *oldval,
|
||||
static inline bool opal_atomic_compare_exchange_strong_rel_xx (opal_atomic_intptr_t *addr, intptr_t *oldval,
|
||||
int64_t newval, size_t length);
|
||||
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_ptr (volatile void* addr, void *oldval,
|
||||
void *newval);
|
||||
static inline bool opal_atomic_compare_exchange_strong_acq_ptr (volatile void* addr, void *oldval,
|
||||
void *newval);
|
||||
static inline bool opal_atomic_compare_exchange_strong_rel_ptr (volatile void* addr, void *oldval,
|
||||
void *newval);
|
||||
static inline bool opal_atomic_compare_exchange_strong_ptr (opal_atomic_intptr_t* addr, intptr_t *oldval,
|
||||
intptr_t newval);
|
||||
static inline bool opal_atomic_compare_exchange_strong_acq_ptr (opal_atomic_intptr_t* addr, intptr_t *oldval,
|
||||
intptr_t newval);
|
||||
static inline bool opal_atomic_compare_exchange_strong_rel_ptr (opal_atomic_intptr_t* addr, intptr_t *oldval,
|
||||
intptr_t newval);
|
||||
|
||||
/**
|
||||
* Atomic compare and set of generic type with relaxed semantics. This
|
||||
@ -554,7 +562,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_ptr (volatile void* a
|
||||
* See opal_atomic_compare_exchange_* for pseudo-code.
|
||||
*/
|
||||
#define opal_atomic_compare_exchange_strong( ADDR, OLDVAL, NEWVAL ) \
|
||||
opal_atomic_compare_exchange_strong_xx( (volatile void*)(ADDR), (void *)(OLDVAL), \
|
||||
opal_atomic_compare_exchange_strong_xx( (opal_atomic_intptr_t*)(ADDR), (intptr_t *)(OLDVAL), \
|
||||
(intptr_t)(NEWVAL), sizeof(*(ADDR)) )
|
||||
|
||||
/**
|
||||
@ -571,7 +579,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_ptr (volatile void* a
|
||||
* See opal_atomic_compare_exchange_acq_* for pseudo-code.
|
||||
*/
|
||||
#define opal_atomic_compare_exchange_strong_acq( ADDR, OLDVAL, NEWVAL ) \
|
||||
opal_atomic_compare_exchange_strong_acq_xx( (volatile void*)(ADDR), (void *)(OLDVAL), \
|
||||
opal_atomic_compare_exchange_strong_acq_xx( (opal_atomic_intptr_t*)(ADDR), (intptr_t *)(OLDVAL), \
|
||||
(intptr_t)(NEWVAL), sizeof(*(ADDR)) )
|
||||
|
||||
/**
|
||||
@ -588,7 +596,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_ptr (volatile void* a
|
||||
* See opal_atomic_compare_exchange_rel_* for pseudo-code.
|
||||
*/
|
||||
#define opal_atomic_compare_exchange_strong_rel( ADDR, OLDVAL, NEWVAL ) \
|
||||
opal_atomic_compare_exchange_strong_rel_xx( (volatile void*)(ADDR), (void *)(OLDVAL), \
|
||||
opal_atomic_compare_exchange_strong_rel_xx( (opal_atomic_intptr_t*)(ADDR), (intptr_t *)(OLDVAL), \
|
||||
(intptr_t)(NEWVAL), sizeof(*(ADDR)) )
|
||||
|
||||
|
||||
@ -596,15 +604,15 @@ static inline bool opal_atomic_compare_exchange_strong_rel_ptr (volatile void* a
|
||||
|
||||
#if defined(DOXYGEN) || (OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64)
|
||||
|
||||
static inline void opal_atomic_add_xx(volatile void* addr,
|
||||
static inline void opal_atomic_add_xx(opal_atomic_intptr_t* addr,
|
||||
int32_t value, size_t length);
|
||||
static inline void opal_atomic_sub_xx(volatile void* addr,
|
||||
static inline void opal_atomic_sub_xx(opal_atomic_intptr_t* addr,
|
||||
int32_t value, size_t length);
|
||||
|
||||
static inline intptr_t opal_atomic_add_fetch_ptr( volatile void* addr, void* delta );
|
||||
static inline intptr_t opal_atomic_fetch_add_ptr( volatile void* addr, void* delta );
|
||||
static inline intptr_t opal_atomic_sub_fetch_ptr( volatile void* addr, void* delta );
|
||||
static inline intptr_t opal_atomic_fetch_sub_ptr( volatile void* addr, void* delta );
|
||||
static inline intptr_t opal_atomic_add_fetch_ptr( opal_atomic_intptr_t* addr, void* delta );
|
||||
static inline intptr_t opal_atomic_fetch_add_ptr( opal_atomic_intptr_t* addr, void* delta );
|
||||
static inline intptr_t opal_atomic_sub_fetch_ptr( opal_atomic_intptr_t* addr, void* delta );
|
||||
static inline intptr_t opal_atomic_fetch_sub_ptr( opal_atomic_intptr_t* addr, void* delta );
|
||||
|
||||
/**
|
||||
* Atomically increment the content depending on the type. This
|
||||
@ -617,7 +625,7 @@ static inline intptr_t opal_atomic_fetch_sub_ptr( volatile void* addr, void* del
|
||||
* @param delta Value to add (converted to <TYPE>).
|
||||
*/
|
||||
#define opal_atomic_add( ADDR, VALUE ) \
|
||||
opal_atomic_add_xx( (volatile void*)(ADDR), (int32_t)(VALUE), \
|
||||
opal_atomic_add_xx( (opal_atomic_intptr_t*)(ADDR), (int32_t)(VALUE), \
|
||||
sizeof(*(ADDR)) )
|
||||
|
||||
/**
|
||||
@ -631,7 +639,7 @@ static inline intptr_t opal_atomic_fetch_sub_ptr( volatile void* addr, void* del
|
||||
* @param delta Value to substract (converted to <TYPE>).
|
||||
*/
|
||||
#define opal_atomic_sub( ADDR, VALUE ) \
|
||||
opal_atomic_sub_xx( (volatile void*)(ADDR), (int32_t)(VALUE), \
|
||||
opal_atomic_sub_xx( (opal_atomic_intptr_t*)(ADDR), (int32_t)(VALUE), \
|
||||
sizeof(*(ADDR)) )
|
||||
|
||||
#endif /* OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64 */
|
||||
@ -643,6 +651,8 @@ static inline intptr_t opal_atomic_fetch_sub_ptr( volatile void* addr, void* del
|
||||
*/
|
||||
#include "opal/sys/atomic_impl.h"
|
||||
|
||||
#endif /* !OPAL_C_HAVE__ATOMIC */
|
||||
|
||||
END_C_DECLS
|
||||
|
||||
#endif /* OPAL_SYS_ATOMIC_H */
|
||||
|
||||
@ -11,7 +11,7 @@
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2010-2014 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
@ -40,7 +40,7 @@
|
||||
#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
|
||||
|
||||
#if !defined(OPAL_HAVE_ATOMIC_MIN_32)
|
||||
static inline int32_t opal_atomic_fetch_min_32 (volatile int32_t *addr, int32_t value)
|
||||
static inline int32_t opal_atomic_fetch_min_32 (opal_atomic_int32_t *addr, int32_t value)
|
||||
{
|
||||
int32_t old = *addr;
|
||||
do {
|
||||
@ -57,7 +57,7 @@ static inline int32_t opal_atomic_fetch_min_32 (volatile int32_t *addr, int32_t
|
||||
#endif /* OPAL_HAVE_ATOMIC_MIN_32 */
|
||||
|
||||
#if !defined(OPAL_HAVE_ATOMIC_MAX_32)
|
||||
static inline int32_t opal_atomic_fetch_max_32 (volatile int32_t *addr, int32_t value)
|
||||
static inline int32_t opal_atomic_fetch_max_32 (opal_atomic_int32_t *addr, int32_t value)
|
||||
{
|
||||
int32_t old = *addr;
|
||||
do {
|
||||
@ -73,7 +73,7 @@ static inline int32_t opal_atomic_fetch_max_32 (volatile int32_t *addr, int32_t
|
||||
#endif /* OPAL_HAVE_ATOMIC_MAX_32 */
|
||||
|
||||
#define OPAL_ATOMIC_DEFINE_CMPXCG_OP(type, bits, operation, name) \
|
||||
static inline type opal_atomic_fetch_ ## name ## _ ## bits (volatile type *addr, type value) \
|
||||
static inline type opal_atomic_fetch_ ## name ## _ ## bits (opal_atomic_ ## type *addr, type value) \
|
||||
{ \
|
||||
type oldval; \
|
||||
do { \
|
||||
@ -85,7 +85,7 @@ static inline int32_t opal_atomic_fetch_max_32 (volatile int32_t *addr, int32_t
|
||||
|
||||
#if !defined(OPAL_HAVE_ATOMIC_SWAP_32)
|
||||
#define OPAL_HAVE_ATOMIC_SWAP_32 1
|
||||
static inline int32_t opal_atomic_swap_32(volatile int32_t *addr,
|
||||
static inline int32_t opal_atomic_swap_32(opal_atomic_int32_t *addr,
|
||||
int32_t newval)
|
||||
{
|
||||
int32_t old = *addr;
|
||||
@ -138,7 +138,7 @@ OPAL_ATOMIC_DEFINE_CMPXCG_OP(int32_t, 32, -, sub)
|
||||
#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
|
||||
|
||||
#if !defined(OPAL_HAVE_ATOMIC_MIN_64)
|
||||
static inline int64_t opal_atomic_fetch_min_64 (volatile int64_t *addr, int64_t value)
|
||||
static inline int64_t opal_atomic_fetch_min_64 (opal_atomic_int64_t *addr, int64_t value)
|
||||
{
|
||||
int64_t old = *addr;
|
||||
do {
|
||||
@ -155,7 +155,7 @@ static inline int64_t opal_atomic_fetch_min_64 (volatile int64_t *addr, int64_t
|
||||
#endif /* OPAL_HAVE_ATOMIC_MIN_64 */
|
||||
|
||||
#if !defined(OPAL_HAVE_ATOMIC_MAX_64)
|
||||
static inline int64_t opal_atomic_fetch_max_64 (volatile int64_t *addr, int64_t value)
|
||||
static inline int64_t opal_atomic_fetch_max_64 (opal_atomic_int64_t *addr, int64_t value)
|
||||
{
|
||||
int64_t old = *addr;
|
||||
do {
|
||||
@ -172,7 +172,7 @@ static inline int64_t opal_atomic_fetch_max_64 (volatile int64_t *addr, int64_t
|
||||
|
||||
#if !defined(OPAL_HAVE_ATOMIC_SWAP_64)
|
||||
#define OPAL_HAVE_ATOMIC_SWAP_64 1
|
||||
static inline int64_t opal_atomic_swap_64(volatile int64_t *addr,
|
||||
static inline int64_t opal_atomic_swap_64(opal_atomic_int64_t *addr,
|
||||
int64_t newval)
|
||||
{
|
||||
int64_t old = *addr;
|
||||
@ -235,15 +235,15 @@ OPAL_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, -, sub)
|
||||
#if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
|
||||
#define OPAL_ATOMIC_DEFINE_CMPXCG_XX(semantics) \
|
||||
static inline bool \
|
||||
opal_atomic_compare_exchange_strong ## semantics ## xx (volatile void* addr, void *oldval, \
|
||||
opal_atomic_compare_exchange_strong ## semantics ## xx (opal_atomic_intptr_t* addr, intptr_t *oldval, \
|
||||
int64_t newval, const size_t length) \
|
||||
{ \
|
||||
switch (length) { \
|
||||
case 4: \
|
||||
return opal_atomic_compare_exchange_strong_32 ((volatile int32_t *) addr, \
|
||||
return opal_atomic_compare_exchange_strong_32 ((opal_atomic_int32_t *) addr, \
|
||||
(int32_t *) oldval, (int32_t) newval); \
|
||||
case 8: \
|
||||
return opal_atomic_compare_exchange_strong_64 ((volatile int64_t *) addr, \
|
||||
return opal_atomic_compare_exchange_strong_64 ((opal_atomic_int64_t *) addr, \
|
||||
(int64_t *) oldval, (int64_t) newval); \
|
||||
} \
|
||||
abort(); \
|
||||
@ -251,12 +251,12 @@ OPAL_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, -, sub)
|
||||
#elif OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
|
||||
#define OPAL_ATOMIC_DEFINE_CMPXCG_XX(semantics) \
|
||||
static inline bool \
|
||||
opal_atomic_compare_exchange_strong ## semantics ## xx (volatile void* addr, void *oldval, \
|
||||
opal_atomic_compare_exchange_strong ## semantics ## xx (opal_atomic_intptr_t* addr, intptr_t *oldval, \
|
||||
int64_t newval, const size_t length) \
|
||||
{ \
|
||||
switch (length) { \
|
||||
case 4: \
|
||||
return opal_atomic_compare_exchange_strong_32 ((volatile int32_t *) addr, \
|
||||
return opal_atomic_compare_exchange_strong_32 ((opal_atomic_int32_t *) addr, \
|
||||
(int32_t *) oldval, (int32_t) newval); \
|
||||
} \
|
||||
abort(); \
|
||||
@ -272,16 +272,16 @@ OPAL_ATOMIC_DEFINE_CMPXCG_XX(_rel_)
|
||||
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32
|
||||
#define OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(semantics) \
|
||||
static inline bool \
|
||||
opal_atomic_compare_exchange_strong ## semantics ## ptr (volatile void* addr, void *oldval, void *newval) \
|
||||
opal_atomic_compare_exchange_strong ## semantics ## ptr (opal_atomic_intptr_t* addr, intptr_t *oldval, intptr_t newval) \
|
||||
{ \
|
||||
return opal_atomic_compare_exchange_strong_32 ((volatile int32_t *) addr, (int32_t *) oldval, (int32_t) newval); \
|
||||
return opal_atomic_compare_exchange_strong_32 ((opal_atomic_int32_t *) addr, (int32_t *) oldval, (int32_t) newval); \
|
||||
}
|
||||
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64
|
||||
#define OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(semantics) \
|
||||
static inline bool \
|
||||
opal_atomic_compare_exchange_strong ## semantics ## ptr (volatile void* addr, void *oldval, void *newval) \
|
||||
opal_atomic_compare_exchange_strong ## semantics ## ptr (opal_atomic_intptr_t* addr, intptr_t *oldval, intptr_t newval) \
|
||||
{ \
|
||||
return opal_atomic_compare_exchange_strong_64 ((volatile int64_t *) addr, (int64_t *) oldval, (int64_t) newval); \
|
||||
return opal_atomic_compare_exchange_strong_64 ((opal_atomic_int64_t *) addr, (int64_t *) oldval, (int64_t) newval); \
|
||||
}
|
||||
#else
|
||||
#error "Can not define opal_atomic_compare_exchange_strong_ptr with existing atomics"
|
||||
@ -297,9 +297,9 @@ OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(_rel_)
|
||||
#if (OPAL_HAVE_ATOMIC_SWAP_32 || OPAL_HAVE_ATOMIC_SWAP_64)
|
||||
|
||||
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_SWAP_32
|
||||
#define opal_atomic_swap_ptr(addr, value) (void *) opal_atomic_swap_32((int32_t *) addr, (int32_t) value)
|
||||
#define opal_atomic_swap_ptr(addr, value) (intptr_t) opal_atomic_swap_32((opal_atomic_int32_t *) addr, (int32_t) value)
|
||||
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_SWAP_64
|
||||
#define opal_atomic_swap_ptr(addr, value) (void *) opal_atomic_swap_64((int64_t *) addr, (int64_t) value)
|
||||
#define opal_atomic_swap_ptr(addr, value) (intptr_t) opal_atomic_swap_64((opal_atomic_int64_t *) addr, (int64_t) value)
|
||||
#endif
|
||||
|
||||
#endif /* (OPAL_HAVE_ATOMIC_SWAP_32 || OPAL_HAVE_ATOMIC_SWAP_64) */
|
||||
@ -308,15 +308,15 @@ OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(_rel_)
|
||||
|
||||
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_LLSC_32
|
||||
|
||||
#define opal_atomic_ll_ptr(addr, ret) opal_atomic_ll_32((volatile int32_t *) (addr), ret)
|
||||
#define opal_atomic_sc_ptr(addr, value, ret) opal_atomic_sc_32((volatile int32_t *) (addr), (intptr_t) (value), ret)
|
||||
#define opal_atomic_ll_ptr(addr, ret) opal_atomic_ll_32((opal_atomic_int32_t *) (addr), ret)
|
||||
#define opal_atomic_sc_ptr(addr, value, ret) opal_atomic_sc_32((opal_atomic_int32_t *) (addr), (intptr_t) (value), ret)
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_LLSC_PTR 1
|
||||
|
||||
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_LLSC_64
|
||||
|
||||
#define opal_atomic_ll_ptr(addr, ret) opal_atomic_ll_64((volatile int64_t *) (addr), ret)
|
||||
#define opal_atomic_sc_ptr(addr, value, ret) opal_atomic_sc_64((volatile int64_t *) (addr), (intptr_t) (value), ret)
|
||||
#define opal_atomic_ll_ptr(addr, ret) opal_atomic_ll_64((opal_atomic_int64_t *) (addr), ret)
|
||||
#define opal_atomic_sc_ptr(addr, value, ret) opal_atomic_sc_64((opal_atomic_int64_t *) (addr), (intptr_t) (value), ret)
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_LLSC_PTR 1
|
||||
|
||||
@ -331,18 +331,18 @@ OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(_rel_)
|
||||
#if OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64
|
||||
|
||||
static inline void
|
||||
opal_atomic_add_xx(volatile void* addr, int32_t value, size_t length)
|
||||
opal_atomic_add_xx(opal_atomic_intptr_t* addr, int32_t value, size_t length)
|
||||
{
|
||||
switch( length ) {
|
||||
#if OPAL_HAVE_ATOMIC_ADD_32
|
||||
case 4:
|
||||
(void) opal_atomic_fetch_add_32( (volatile int32_t*)addr, (int32_t)value );
|
||||
(void) opal_atomic_fetch_add_32( (opal_atomic_int32_t*)addr, (int32_t)value );
|
||||
break;
|
||||
#endif /* OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 */
|
||||
|
||||
#if OPAL_HAVE_ATOMIC_ADD_64
|
||||
case 8:
|
||||
(void) opal_atomic_fetch_add_64( (volatile int64_t*)addr, (int64_t)value );
|
||||
(void) opal_atomic_fetch_add_64( (opal_atomic_int64_t*)addr, (int64_t)value );
|
||||
break;
|
||||
#endif /* OPAL_HAVE_ATOMIC_ADD_64 */
|
||||
default:
|
||||
@ -354,18 +354,18 @@ static inline void
|
||||
|
||||
|
||||
static inline void
|
||||
opal_atomic_sub_xx(volatile void* addr, int32_t value, size_t length)
|
||||
opal_atomic_sub_xx(opal_atomic_intptr_t* addr, int32_t value, size_t length)
|
||||
{
|
||||
switch( length ) {
|
||||
#if OPAL_HAVE_ATOMIC_SUB_32
|
||||
case 4:
|
||||
(void) opal_atomic_fetch_sub_32( (volatile int32_t*)addr, (int32_t)value );
|
||||
(void) opal_atomic_fetch_sub_32( (opal_atomic_int32_t*)addr, (int32_t)value );
|
||||
break;
|
||||
#endif /* OPAL_HAVE_ATOMIC_SUB_32 */
|
||||
|
||||
#if OPAL_HAVE_ATOMIC_SUB_64
|
||||
case 8:
|
||||
(void) opal_atomic_fetch_sub_64( (volatile int64_t*)addr, (int64_t)value );
|
||||
(void) opal_atomic_fetch_sub_64( (opal_atomic_int64_t*)addr, (int64_t)value );
|
||||
break;
|
||||
#endif /* OPAL_HAVE_ATOMIC_SUB_64 */
|
||||
default:
|
||||
@ -376,7 +376,7 @@ opal_atomic_sub_xx(volatile void* addr, int32_t value, size_t length)
|
||||
}
|
||||
|
||||
#define OPAL_ATOMIC_DEFINE_OP_FETCH(op, operation, type, ptr_type, suffix) \
|
||||
static inline type opal_atomic_ ## op ## _fetch_ ## suffix (volatile ptr_type *addr, type value) \
|
||||
static inline type opal_atomic_ ## op ## _fetch_ ## suffix (opal_atomic_ ## ptr_type *addr, type value) \
|
||||
{ \
|
||||
return opal_atomic_fetch_ ## op ## _ ## suffix (addr, value) operation value; \
|
||||
}
|
||||
@ -387,13 +387,13 @@ OPAL_ATOMIC_DEFINE_OP_FETCH(or, |, int32_t, int32_t, 32)
|
||||
OPAL_ATOMIC_DEFINE_OP_FETCH(xor, ^, int32_t, int32_t, 32)
|
||||
OPAL_ATOMIC_DEFINE_OP_FETCH(sub, -, int32_t, int32_t, 32)
|
||||
|
||||
static inline int32_t opal_atomic_min_fetch_32 (volatile int32_t *addr, int32_t value)
|
||||
static inline int32_t opal_atomic_min_fetch_32 (opal_atomic_int32_t *addr, int32_t value)
|
||||
{
|
||||
int32_t old = opal_atomic_fetch_min_32 (addr, value);
|
||||
return old <= value ? old : value;
|
||||
}
|
||||
|
||||
static inline int32_t opal_atomic_max_fetch_32 (volatile int32_t *addr, int32_t value)
|
||||
static inline int32_t opal_atomic_max_fetch_32 (opal_atomic_int32_t *addr, int32_t value)
|
||||
{
|
||||
int32_t old = opal_atomic_fetch_max_32 (addr, value);
|
||||
return old >= value ? old : value;
|
||||
@ -406,13 +406,13 @@ OPAL_ATOMIC_DEFINE_OP_FETCH(or, |, int64_t, int64_t, 64)
|
||||
OPAL_ATOMIC_DEFINE_OP_FETCH(xor, ^, int64_t, int64_t, 64)
|
||||
OPAL_ATOMIC_DEFINE_OP_FETCH(sub, -, int64_t, int64_t, 64)
|
||||
|
||||
static inline int64_t opal_atomic_min_fetch_64 (volatile int64_t *addr, int64_t value)
|
||||
static inline int64_t opal_atomic_min_fetch_64 (opal_atomic_int64_t *addr, int64_t value)
|
||||
{
|
||||
int64_t old = opal_atomic_fetch_min_64 (addr, value);
|
||||
return old <= value ? old : value;
|
||||
}
|
||||
|
||||
static inline int64_t opal_atomic_max_fetch_64 (volatile int64_t *addr, int64_t value)
|
||||
static inline int64_t opal_atomic_max_fetch_64 (opal_atomic_int64_t *addr, int64_t value)
|
||||
{
|
||||
int64_t old = opal_atomic_fetch_max_64 (addr, value);
|
||||
return old >= value ? old : value;
|
||||
@ -420,52 +420,52 @@ static inline int64_t opal_atomic_max_fetch_64 (volatile int64_t *addr, int64_t
|
||||
|
||||
#endif
|
||||
|
||||
static inline intptr_t opal_atomic_fetch_add_ptr( volatile void* addr,
|
||||
static inline intptr_t opal_atomic_fetch_add_ptr( opal_atomic_intptr_t* addr,
|
||||
void* delta )
|
||||
{
|
||||
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_ADD_32
|
||||
return opal_atomic_fetch_add_32((int32_t*) addr, (unsigned long) delta);
|
||||
return opal_atomic_fetch_add_32((opal_atomic_int32_t*) addr, (unsigned long) delta);
|
||||
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_ADD_64
|
||||
return opal_atomic_fetch_add_64((int64_t*) addr, (unsigned long) delta);
|
||||
return opal_atomic_fetch_add_64((opal_atomic_int64_t*) addr, (unsigned long) delta);
|
||||
#else
|
||||
abort ();
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline intptr_t opal_atomic_add_fetch_ptr( volatile void* addr,
|
||||
static inline intptr_t opal_atomic_add_fetch_ptr( opal_atomic_intptr_t* addr,
|
||||
void* delta )
|
||||
{
|
||||
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_ADD_32
|
||||
return opal_atomic_add_fetch_32((int32_t*) addr, (unsigned long) delta);
|
||||
return opal_atomic_add_fetch_32((opal_atomic_int32_t*) addr, (unsigned long) delta);
|
||||
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_ADD_64
|
||||
return opal_atomic_add_fetch_64((int64_t*) addr, (unsigned long) delta);
|
||||
return opal_atomic_add_fetch_64((opal_atomic_int64_t*) addr, (unsigned long) delta);
|
||||
#else
|
||||
abort ();
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline intptr_t opal_atomic_fetch_sub_ptr( volatile void* addr,
|
||||
static inline intptr_t opal_atomic_fetch_sub_ptr( opal_atomic_intptr_t* addr,
|
||||
void* delta )
|
||||
{
|
||||
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_SUB_32
|
||||
return opal_atomic_fetch_sub_32((int32_t*) addr, (unsigned long) delta);
|
||||
return opal_atomic_fetch_sub_32((opal_atomic_int32_t*) addr, (unsigned long) delta);
|
||||
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_SUB_32
|
||||
return opal_atomic_fetch_sub_64((int64_t*) addr, (unsigned long) delta);
|
||||
return opal_atomic_fetch_sub_64((opal_atomic_int64_t*) addr, (unsigned long) delta);
|
||||
#else
|
||||
abort();
|
||||
return 0;
|
||||
#endif
|
||||
}
|
||||
|
||||
static inline intptr_t opal_atomic_sub_fetch_ptr( volatile void* addr,
|
||||
static inline intptr_t opal_atomic_sub_fetch_ptr( opal_atomic_intptr_t* addr,
|
||||
void* delta )
|
||||
{
|
||||
#if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_SUB_32
|
||||
return opal_atomic_sub_fetch_32((int32_t*) addr, (unsigned long) delta);
|
||||
return opal_atomic_sub_fetch_32((opal_atomic_int32_t*) addr, (unsigned long) delta);
|
||||
#elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_SUB_32
|
||||
return opal_atomic_sub_fetch_64((int64_t*) addr, (unsigned long) delta);
|
||||
return opal_atomic_sub_fetch_64((opal_atomic_int64_t*) addr, (unsigned long) delta);
|
||||
#else
|
||||
abort();
|
||||
return 0;
|
||||
|
||||
261
opal/include/opal/sys/atomic_stdc.h
Обычный файл
261
opal/include/opal/sys/atomic_stdc.h
Обычный файл
@ -0,0 +1,261 @@
|
||||
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
||||
/*
|
||||
* Copyright (c) 2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
/* This file provides shims between the opal atomics interface and the C11 atomics interface. It
|
||||
* is intended as the first step in moving to using C11 atomics across the entire codebase. Once
|
||||
* all officially supported compilers offer C11 atomic (GCC 4.9.0+, icc 2018+, pgi, xlc, etc) then
|
||||
* this shim will go away and the codebase will be updated to use C11's atomic support
|
||||
* directly.
|
||||
* This shim contains some functions already present in atomic_impl.h because we do not include
|
||||
* atomic_impl.h when using C11 atomics. It would require alot of #ifdefs to avoid duplicate
|
||||
* definitions to be worthwhile. */
|
||||
|
||||
#if !defined(OPAL_ATOMIC_STDC_H)
|
||||
#define OPAL_ATOMIC_STDC_H
|
||||
|
||||
#include <stdatomic.h>
|
||||
#include <stdint.h>
|
||||
#include "opal/include/opal_stdint.h"
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
|
||||
#define OPAL_HAVE_ATOMIC_SWAP_32 1
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_MATH_32 1
|
||||
#define OPAL_HAVE_ATOMIC_ADD_32 1
|
||||
#define OPAL_HAVE_ATOMIC_AND_32 1
|
||||
#define OPAL_HAVE_ATOMIC_OR_32 1
|
||||
#define OPAL_HAVE_ATOMIC_XOR_32 1
|
||||
#define OPAL_HAVE_ATOMIC_SUB_32 1
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
|
||||
#define OPAL_HAVE_ATOMIC_SWAP_64 1
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_MATH_64 1
|
||||
#define OPAL_HAVE_ATOMIC_ADD_64 1
|
||||
#define OPAL_HAVE_ATOMIC_AND_64 1
|
||||
#define OPAL_HAVE_ATOMIC_OR_64 1
|
||||
#define OPAL_HAVE_ATOMIC_XOR_64 1
|
||||
#define OPAL_HAVE_ATOMIC_SUB_64 1
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_LLSC_32 0
|
||||
#define OPAL_HAVE_ATOMIC_LLSC_64 0
|
||||
#define OPAL_HAVE_ATOMIC_LLSC_PTR 0
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_MIN_32 1
|
||||
#define OPAL_HAVE_ATOMIC_MAX_32 1
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_MIN_64 1
|
||||
#define OPAL_HAVE_ATOMIC_MAX_64 1
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_SPINLOCKS 1
|
||||
|
||||
static inline void opal_atomic_mb (void)
|
||||
{
|
||||
atomic_thread_fence (memory_order_seq_cst);
|
||||
}
|
||||
|
||||
static inline void opal_atomic_wmb (void)
|
||||
{
|
||||
atomic_thread_fence (memory_order_release);
|
||||
}
|
||||
|
||||
static inline void opal_atomic_rmb (void)
|
||||
{
|
||||
atomic_thread_fence (memory_order_acquire);
|
||||
}
|
||||
|
||||
#define opal_atomic_compare_exchange_strong_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_relaxed, memory_order_relaxed)
|
||||
#define opal_atomic_compare_exchange_strong_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_relaxed, memory_order_relaxed)
|
||||
#define opal_atomic_compare_exchange_strong_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_relaxed, memory_order_relaxed)
|
||||
#define opal_atomic_compare_exchange_strong_acq_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed)
|
||||
#define opal_atomic_compare_exchange_strong_acq_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed)
|
||||
#define opal_atomic_compare_exchange_strong_acq_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed)
|
||||
|
||||
#define opal_atomic_compare_exchange_strong_rel_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed)
|
||||
#define opal_atomic_compare_exchange_strong_rel_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed)
|
||||
#define opal_atomic_compare_exchange_strong_rel_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed)
|
||||
|
||||
#define opal_atomic_compare_exchange_strong(addr, oldval, newval) atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_relaxed, memory_order_relaxed)
|
||||
#define opal_atomic_compare_exchange_strong_acq(addr, oldval, newval) atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_acquire, memory_order_relaxed)
|
||||
#define opal_atomic_compare_exchange_strong_rel(addr, oldval, newval) atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_release, memory_order_relaxed)
|
||||
|
||||
#define opal_atomic_swap_32(addr, value) atomic_exchange_explicit (addr, value, memory_order_relaxed)
|
||||
#define opal_atomic_swap_64(addr, value) atomic_exchange_explicit (addr, value, memory_order_relaxed)
|
||||
#define opal_atomic_swap_ptr(addr, value) atomic_exchange_explicit (addr, value, memory_order_relaxed)
|
||||
|
||||
#define OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(op, bits, type, operator) \
|
||||
static inline type opal_atomic_fetch_ ## op ##_## bits (opal_atomic_ ## type *addr, type value) \
|
||||
{ \
|
||||
return atomic_fetch_ ## op ## _explicit (addr, value, memory_order_relaxed); \
|
||||
} \
|
||||
\
|
||||
static inline type opal_atomic_## op ## _fetch_ ## bits (opal_atomic_ ## type *addr, type value) \
|
||||
{ \
|
||||
return atomic_fetch_ ## op ## _explicit (addr, value, memory_order_relaxed) operator value; \
|
||||
}
|
||||
|
||||
OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, 32, int32_t, +)
|
||||
OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, 64, int64_t, +)
|
||||
OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, size_t, size_t, +)
|
||||
|
||||
OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, 32, int32_t, -)
|
||||
OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, 64, int64_t, -)
|
||||
OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, size_t, size_t, -)
|
||||
|
||||
OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(or, 32, int32_t, |)
|
||||
OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(or, 64, int64_t, |)
|
||||
|
||||
OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(xor, 32, int32_t, ^)
|
||||
OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(xor, 64, int64_t, ^)
|
||||
|
||||
OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(and, 32, int32_t, &)
|
||||
OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(and, 64, int64_t, &)
|
||||
|
||||
#define opal_atomic_add(addr, value) (void) atomic_fetch_add_explicit (addr, value, memory_order_relaxed)
|
||||
|
||||
static inline int32_t opal_atomic_fetch_min_32 (opal_atomic_int32_t *addr, int32_t value)
|
||||
{
|
||||
int32_t old = *addr;
|
||||
do {
|
||||
if (old <= value) {
|
||||
break;
|
||||
}
|
||||
} while (!opal_atomic_compare_exchange_strong_32 (addr, &old, value));
|
||||
|
||||
return old;
|
||||
}
|
||||
|
||||
static inline int32_t opal_atomic_fetch_max_32 (opal_atomic_int32_t *addr, int32_t value)
|
||||
{
|
||||
int32_t old = *addr;
|
||||
do {
|
||||
if (old >= value) {
|
||||
break;
|
||||
}
|
||||
} while (!opal_atomic_compare_exchange_strong_32 (addr, &old, value));
|
||||
|
||||
return old;
|
||||
}
|
||||
|
||||
static inline int64_t opal_atomic_fetch_min_64 (opal_atomic_int64_t *addr, int64_t value)
|
||||
{
|
||||
int64_t old = *addr;
|
||||
do {
|
||||
if (old <= value) {
|
||||
break;
|
||||
}
|
||||
} while (!opal_atomic_compare_exchange_strong_64 (addr, &old, value));
|
||||
|
||||
return old;
|
||||
}
|
||||
|
||||
static inline int64_t opal_atomic_fetch_max_64 (opal_atomic_int64_t *addr, int64_t value)
|
||||
{
|
||||
int64_t old = *addr;
|
||||
do {
|
||||
if (old >= value) {
|
||||
break;
|
||||
}
|
||||
} while (!opal_atomic_compare_exchange_strong_64 (addr, &old, value));
|
||||
|
||||
return old;
|
||||
}
|
||||
|
||||
static inline int32_t opal_atomic_min_fetch_32 (opal_atomic_int32_t *addr, int32_t value)
|
||||
{
|
||||
int32_t old = opal_atomic_fetch_min_32 (addr, value);
|
||||
return old <= value ? old : value;
|
||||
}
|
||||
|
||||
static inline int32_t opal_atomic_max_fetch_32 (opal_atomic_int32_t *addr, int32_t value)
|
||||
{
|
||||
int32_t old = opal_atomic_fetch_max_32 (addr, value);
|
||||
return old >= value ? old : value;
|
||||
}
|
||||
|
||||
static inline int64_t opal_atomic_min_fetch_64 (opal_atomic_int64_t *addr, int64_t value)
|
||||
{
|
||||
int64_t old = opal_atomic_fetch_min_64 (addr, value);
|
||||
return old <= value ? old : value;
|
||||
}
|
||||
|
||||
static inline int64_t opal_atomic_max_fetch_64 (opal_atomic_int64_t *addr, int64_t value)
|
||||
{
|
||||
int64_t old = opal_atomic_fetch_max_64 (addr, value);
|
||||
return old >= value ? old : value;
|
||||
}
|
||||
|
||||
#define OPAL_ATOMIC_LOCK_UNLOCKED false
|
||||
#define OPAL_ATOMIC_LOCK_LOCKED true
|
||||
|
||||
#define OPAL_ATOMIC_LOCK_INIT ATOMIC_FLAG_INIT
|
||||
|
||||
typedef atomic_flag opal_atomic_lock_t;
|
||||
|
||||
/*
|
||||
* Lock initialization function. It set the lock to UNLOCKED.
|
||||
*/
|
||||
static inline void opal_atomic_lock_init (opal_atomic_lock_t *lock, bool value)
|
||||
{
|
||||
atomic_flag_clear (lock);
|
||||
}
|
||||
|
||||
|
||||
static inline int opal_atomic_trylock (opal_atomic_lock_t *lock)
|
||||
{
|
||||
return (int) atomic_flag_test_and_set (lock);
|
||||
}
|
||||
|
||||
|
||||
static inline void opal_atomic_lock(opal_atomic_lock_t *lock)
|
||||
{
|
||||
while (opal_atomic_trylock (lock)) {
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
static inline void opal_atomic_unlock (opal_atomic_lock_t *lock)
|
||||
{
|
||||
atomic_flag_clear (lock);
|
||||
}
|
||||
|
||||
|
||||
#if OPAL_HAVE_C11_CSWAP_INT128
|
||||
|
||||
/* the C11 atomic compare-exchange is lock free so use it */
|
||||
#define opal_atomic_compare_exchange_strong_128 atomic_compare_exchange_strong
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
|
||||
|
||||
#elif OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128
|
||||
|
||||
/* fall back on the __sync builtin if available since it will emit the expected instruction on x86_64 (cmpxchng16b) */
|
||||
__opal_attribute_always_inline__
|
||||
static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_int128_t *addr,
|
||||
opal_int128_t *oldval, opal_int128_t newval)
|
||||
{
|
||||
opal_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval);
|
||||
bool ret = prev == *oldval;
|
||||
*oldval = prev;
|
||||
return ret;
|
||||
}
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
|
||||
|
||||
#else
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 0
|
||||
|
||||
#endif
|
||||
|
||||
#endif /* !defined(OPAL_ATOMIC_STDC_H) */
|
||||
@ -11,7 +11,7 @@
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2011 Sandia National Laboratories. All rights reserved.
|
||||
* Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2016-2017 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
@ -81,98 +81,98 @@ static inline void opal_atomic_wmb(void)
|
||||
#pragma error_messages(off, E_ARG_INCOMPATIBLE_WITH_ARG_L)
|
||||
#endif
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline int32_t opal_atomic_swap_32 (volatile int32_t *addr, int32_t newval)
|
||||
static inline int32_t opal_atomic_swap_32 (opal_atomic_int32_t *addr, int32_t newval)
|
||||
{
|
||||
int32_t oldval;
|
||||
__atomic_exchange (addr, &newval, &oldval, __ATOMIC_RELAXED);
|
||||
return oldval;
|
||||
}
|
||||
|
||||
static inline int32_t opal_atomic_fetch_add_32(volatile int32_t *addr, int32_t delta)
|
||||
static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t *addr, int32_t delta)
|
||||
{
|
||||
return __atomic_fetch_add (addr, delta, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline int32_t opal_atomic_fetch_and_32(volatile int32_t *addr, int32_t value)
|
||||
static inline int32_t opal_atomic_fetch_and_32(opal_atomic_int32_t *addr, int32_t value)
|
||||
{
|
||||
return __atomic_fetch_and (addr, value, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline int32_t opal_atomic_fetch_or_32(volatile int32_t *addr, int32_t value)
|
||||
static inline int32_t opal_atomic_fetch_or_32(opal_atomic_int32_t *addr, int32_t value)
|
||||
{
|
||||
return __atomic_fetch_or (addr, value, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline int32_t opal_atomic_fetch_xor_32(volatile int32_t *addr, int32_t value)
|
||||
static inline int32_t opal_atomic_fetch_xor_32(opal_atomic_int32_t *addr, int32_t value)
|
||||
{
|
||||
return __atomic_fetch_xor (addr, value, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t *addr, int32_t delta)
|
||||
static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t *addr, int32_t delta)
|
||||
{
|
||||
return __atomic_fetch_sub (addr, delta, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline int64_t opal_atomic_swap_64 (volatile int64_t *addr, int64_t newval)
|
||||
static inline int64_t opal_atomic_swap_64 (opal_atomic_int64_t *addr, int64_t newval)
|
||||
{
|
||||
int64_t oldval;
|
||||
__atomic_exchange (addr, &newval, &oldval, __ATOMIC_RELAXED);
|
||||
return oldval;
|
||||
}
|
||||
|
||||
static inline int64_t opal_atomic_fetch_add_64(volatile int64_t *addr, int64_t delta)
|
||||
static inline int64_t opal_atomic_fetch_add_64(opal_atomic_int64_t *addr, int64_t delta)
|
||||
{
|
||||
return __atomic_fetch_add (addr, delta, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline int64_t opal_atomic_fetch_and_64(volatile int64_t *addr, int64_t value)
|
||||
static inline int64_t opal_atomic_fetch_and_64(opal_atomic_int64_t *addr, int64_t value)
|
||||
{
|
||||
return __atomic_fetch_and (addr, value, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline int64_t opal_atomic_fetch_or_64(volatile int64_t *addr, int64_t value)
|
||||
static inline int64_t opal_atomic_fetch_or_64(opal_atomic_int64_t *addr, int64_t value)
|
||||
{
|
||||
return __atomic_fetch_or (addr, value, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline int64_t opal_atomic_fetch_xor_64(volatile int64_t *addr, int64_t value)
|
||||
static inline int64_t opal_atomic_fetch_xor_64(opal_atomic_int64_t *addr, int64_t value)
|
||||
{
|
||||
return __atomic_fetch_xor (addr, value, __ATOMIC_RELAXED);
|
||||
}
|
||||
|
||||
static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t *addr, int64_t delta)
|
||||
static inline int64_t opal_atomic_fetch_sub_64(opal_atomic_int64_t *addr, int64_t delta)
|
||||
{
|
||||
return __atomic_fetch_sub (addr, delta, __ATOMIC_RELAXED);
|
||||
}
|
||||
@ -181,7 +181,7 @@ static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t *addr, int64_t d
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128_t *addr,
|
||||
static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_int128_t *addr,
|
||||
opal_int128_t *oldval, opal_int128_t newval)
|
||||
{
|
||||
return __atomic_compare_exchange_n (addr, oldval, newval, false,
|
||||
@ -194,7 +194,7 @@ static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128
|
||||
|
||||
/* __atomic version is not lock-free so use legacy __sync version */
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128_t *addr,
|
||||
static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_opal_int128_t *addr,
|
||||
opal_int128_t *oldval, opal_int128_t newval)
|
||||
{
|
||||
opal_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval);
|
||||
|
||||
@ -13,7 +13,7 @@
|
||||
* Copyright (c) 2007-2010 Oracle and/or its affiliates. All rights reserved.
|
||||
* Copyright (c) 2015 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
@ -84,7 +84,7 @@ static inline void opal_atomic_isync(void)
|
||||
*********************************************************************/
|
||||
#if OPAL_GCC_INLINE_ASSEMBLY
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
unsigned char ret;
|
||||
__asm__ __volatile__ (
|
||||
@ -106,7 +106,7 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_SWAP_32 1
|
||||
|
||||
static inline int32_t opal_atomic_swap_32( volatile int32_t *addr,
|
||||
static inline int32_t opal_atomic_swap_32( opal_atomic_int32_t *addr,
|
||||
int32_t newval)
|
||||
{
|
||||
int32_t oldval;
|
||||
@ -130,7 +130,7 @@ static inline int32_t opal_atomic_swap_32( volatile int32_t *addr,
|
||||
*
|
||||
* Atomically adds @i to @v.
|
||||
*/
|
||||
static inline int32_t opal_atomic_fetch_add_32(volatile int32_t* v, int i)
|
||||
static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t* v, int i)
|
||||
{
|
||||
int ret = i;
|
||||
__asm__ __volatile__(
|
||||
@ -150,7 +150,7 @@ static inline int32_t opal_atomic_fetch_add_32(volatile int32_t* v, int i)
|
||||
*
|
||||
* Atomically subtracts @i from @v.
|
||||
*/
|
||||
static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t* v, int i)
|
||||
static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t* v, int i)
|
||||
{
|
||||
int ret = -i;
|
||||
__asm__ __volatile__(
|
||||
|
||||
@ -144,7 +144,7 @@ void opal_atomic_isync(void)
|
||||
#define OPAL_ASM_VALUE64(x) x
|
||||
#endif
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
int32_t prev;
|
||||
bool ret;
|
||||
@ -170,7 +170,7 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add
|
||||
* load the arguments to/from the stack. This sequence may cause the ll reservation to be cancelled. */
|
||||
#define opal_atomic_ll_32(addr, ret) \
|
||||
do { \
|
||||
volatile int32_t *_addr = (addr); \
|
||||
opal_atomic_int32_t *_addr = (addr); \
|
||||
int32_t _ret; \
|
||||
__asm__ __volatile__ ("lwarx %0, 0, %1 \n\t" \
|
||||
: "=&r" (_ret) \
|
||||
@ -181,7 +181,7 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add
|
||||
|
||||
#define opal_atomic_sc_32(addr, value, ret) \
|
||||
do { \
|
||||
volatile int32_t *_addr = (addr); \
|
||||
opal_atomic_int32_t *_addr = (addr); \
|
||||
int32_t _ret, _foo, _newval = (int32_t) value; \
|
||||
\
|
||||
__asm__ __volatile__ (" stwcx. %4, 0, %3 \n\t" \
|
||||
@ -200,7 +200,7 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add
|
||||
atomic_?mb can be inlined). Instead, we "inline" them by hand in
|
||||
the assembly, meaning there is one function call overhead instead
|
||||
of two */
|
||||
static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
bool rc;
|
||||
|
||||
@ -211,13 +211,13 @@ static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t
|
||||
}
|
||||
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
opal_atomic_wmb();
|
||||
return opal_atomic_compare_exchange_strong_32 (addr, oldval, newval);
|
||||
}
|
||||
|
||||
static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, int32_t newval)
|
||||
static inline int32_t opal_atomic_swap_32(opal_atomic_int32_t *addr, int32_t newval)
|
||||
{
|
||||
int32_t ret;
|
||||
|
||||
@ -239,7 +239,7 @@ static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, int32_t newval
|
||||
#if OPAL_GCC_INLINE_ASSEMBLY
|
||||
|
||||
#define OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(type, instr) \
|
||||
static inline int64_t opal_atomic_fetch_ ## type ## _64(volatile int64_t* v, int64_t val) \
|
||||
static inline int64_t opal_atomic_fetch_ ## type ## _64(opal_atomic_int64_t* v, int64_t val) \
|
||||
{ \
|
||||
int64_t t, old; \
|
||||
\
|
||||
@ -261,7 +261,7 @@ OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(or, or)
|
||||
OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(xor, xor)
|
||||
OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(sub, subf)
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
int64_t prev;
|
||||
bool ret;
|
||||
@ -284,7 +284,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add
|
||||
|
||||
#define opal_atomic_ll_64(addr, ret) \
|
||||
do { \
|
||||
volatile int64_t *_addr = (addr); \
|
||||
opal_atomic_int64_t *_addr = (addr); \
|
||||
int64_t _ret; \
|
||||
__asm__ __volatile__ ("ldarx %0, 0, %1 \n\t" \
|
||||
: "=&r" (_ret) \
|
||||
@ -295,7 +295,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add
|
||||
|
||||
#define opal_atomic_sc_64(addr, value, ret) \
|
||||
do { \
|
||||
volatile int64_t *_addr = (addr); \
|
||||
opal_atomic_int64_t *_addr = (addr); \
|
||||
int64_t _foo, _newval = (int64_t) value; \
|
||||
int32_t _ret; \
|
||||
\
|
||||
@ -310,7 +310,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add
|
||||
ret = _ret; \
|
||||
} while (0)
|
||||
|
||||
static inline int64_t opal_atomic_swap_64(volatile int64_t *addr, int64_t newval)
|
||||
static inline int64_t opal_atomic_swap_64(opal_atomic_int64_t *addr, int64_t newval)
|
||||
{
|
||||
int64_t ret;
|
||||
|
||||
@ -335,7 +335,7 @@ static inline int64_t opal_atomic_swap_64(volatile int64_t *addr, int64_t newval
|
||||
|
||||
#if OPAL_GCC_INLINE_ASSEMBLY
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
int64_t prev;
|
||||
int ret;
|
||||
@ -382,7 +382,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add
|
||||
atomic_?mb can be inlined). Instead, we "inline" them by hand in
|
||||
the assembly, meaning there is one function call overhead instead
|
||||
of two */
|
||||
static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
bool rc;
|
||||
|
||||
@ -393,7 +393,7 @@ static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t
|
||||
}
|
||||
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
opal_atomic_wmb();
|
||||
return opal_atomic_compare_exchange_strong_64 (addr, oldval, newval);
|
||||
@ -401,7 +401,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t
|
||||
|
||||
|
||||
#define OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_32(type, instr) \
|
||||
static inline int32_t opal_atomic_fetch_ ## type ## _32(volatile int32_t* v, int val) \
|
||||
static inline int32_t opal_atomic_fetch_ ## type ## _32(opal_atomic_int32_t* v, int val) \
|
||||
{ \
|
||||
int32_t t, old; \
|
||||
\
|
||||
|
||||
@ -13,7 +13,7 @@
|
||||
* Copyright (c) 2007 Sun Microsystems, Inc. All rights reserverd.
|
||||
* Copyright (c) 2016 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
* Copyright (c) 2017 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2017-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
@ -85,7 +85,7 @@ static inline void opal_atomic_isync(void)
|
||||
*********************************************************************/
|
||||
#if OPAL_GCC_INLINE_ASSEMBLY
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
/* casa [reg(rs1)] %asi, reg(rs2), reg(rd)
|
||||
*
|
||||
@ -107,7 +107,7 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add
|
||||
}
|
||||
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
bool rc;
|
||||
|
||||
@ -118,7 +118,7 @@ static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t
|
||||
}
|
||||
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
opal_atomic_wmb();
|
||||
return opal_atomic_compare_exchange_strong_32 (addr, oldval, newval);
|
||||
@ -127,7 +127,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t
|
||||
|
||||
#if OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
/* casa [reg(rs1)] %asi, reg(rs2), reg(rd)
|
||||
*
|
||||
@ -149,7 +149,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add
|
||||
|
||||
#else /* OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64 */
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
/* casa [reg(rs1)] %asi, reg(rs2), reg(rd)
|
||||
*
|
||||
@ -179,7 +179,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add
|
||||
|
||||
#endif /* OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64 */
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
bool rc;
|
||||
|
||||
@ -190,7 +190,7 @@ static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t
|
||||
}
|
||||
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
opal_atomic_wmb();
|
||||
return opal_atomic_compare_exchange_strong_64 (addr, oldval, newval);
|
||||
|
||||
@ -11,7 +11,7 @@
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2011 Sandia National Laboratories. All rights reserved.
|
||||
* Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2017 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
@ -57,7 +57,7 @@ static inline void opal_atomic_wmb(void)
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
int32_t prev = __sync_val_compare_and_swap (addr, *oldval, newval);
|
||||
bool ret = prev == *oldval;
|
||||
@ -71,31 +71,31 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add
|
||||
#define OPAL_HAVE_ATOMIC_MATH_32 1
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_ADD_32 1
|
||||
static inline int32_t opal_atomic_fetch_add_32(volatile int32_t *addr, int32_t delta)
|
||||
static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t *addr, int32_t delta)
|
||||
{
|
||||
return __sync_fetch_and_add(addr, delta);
|
||||
}
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_AND_32 1
|
||||
static inline int32_t opal_atomic_fetch_and_32(volatile int32_t *addr, int32_t value)
|
||||
static inline int32_t opal_atomic_fetch_and_32(opal_atomic_int32_t *addr, int32_t value)
|
||||
{
|
||||
return __sync_fetch_and_and(addr, value);
|
||||
}
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_OR_32 1
|
||||
static inline int32_t opal_atomic_fetch_or_32(volatile int32_t *addr, int32_t value)
|
||||
static inline int32_t opal_atomic_fetch_or_32(opal_atomic_int32_t *addr, int32_t value)
|
||||
{
|
||||
return __sync_fetch_and_or(addr, value);
|
||||
}
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_XOR_32 1
|
||||
static inline int32_t opal_atomic_fetch_xor_32(volatile int32_t *addr, int32_t value)
|
||||
static inline int32_t opal_atomic_fetch_xor_32(opal_atomic_int32_t *addr, int32_t value)
|
||||
{
|
||||
return __sync_fetch_and_xor(addr, value);
|
||||
}
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_SUB_32 1
|
||||
static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t *addr, int32_t delta)
|
||||
static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t *addr, int32_t delta)
|
||||
{
|
||||
return __sync_fetch_and_sub(addr, delta);
|
||||
}
|
||||
@ -104,7 +104,7 @@ static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t *addr, int32_t d
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
int64_t prev = __sync_val_compare_and_swap (addr, *oldval, newval);
|
||||
bool ret = prev == *oldval;
|
||||
@ -117,31 +117,31 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_MATH_64 1
|
||||
#define OPAL_HAVE_ATOMIC_ADD_64 1
|
||||
static inline int64_t opal_atomic_fetch_add_64(volatile int64_t *addr, int64_t delta)
|
||||
static inline int64_t opal_atomic_fetch_add_64(opal_atomic_int64_t *addr, int64_t delta)
|
||||
{
|
||||
return __sync_fetch_and_add(addr, delta);
|
||||
}
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_AND_64 1
|
||||
static inline int64_t opal_atomic_fetch_and_64(volatile int64_t *addr, int64_t value)
|
||||
static inline int64_t opal_atomic_fetch_and_64(opal_atomic_int64_t *addr, int64_t value)
|
||||
{
|
||||
return __sync_fetch_and_and(addr, value);
|
||||
}
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_OR_64 1
|
||||
static inline int64_t opal_atomic_fetch_or_64(volatile int64_t *addr, int64_t value)
|
||||
static inline int64_t opal_atomic_fetch_or_64(opal_atomic_int64_t *addr, int64_t value)
|
||||
{
|
||||
return __sync_fetch_and_or(addr, value);
|
||||
}
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_XOR_64 1
|
||||
static inline int64_t opal_atomic_fetch_xor_64(volatile int64_t *addr, int64_t value)
|
||||
static inline int64_t opal_atomic_fetch_xor_64(opal_atomic_int64_t *addr, int64_t value)
|
||||
{
|
||||
return __sync_fetch_and_xor(addr, value);
|
||||
}
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_SUB_64 1
|
||||
static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t *addr, int64_t delta)
|
||||
static inline int64_t opal_atomic_fetch_sub_64(opal_atomic_int64_t *addr, int64_t delta)
|
||||
{
|
||||
return __sync_fetch_and_sub(addr, delta);
|
||||
}
|
||||
@ -149,7 +149,7 @@ static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t *addr, int64_t d
|
||||
#endif
|
||||
|
||||
#if OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128
|
||||
static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128_t *addr,
|
||||
static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_int128_t *addr,
|
||||
opal_int128_t *oldval, opal_int128_t newval)
|
||||
{
|
||||
opal_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval);
|
||||
|
||||
@ -11,7 +11,7 @@
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2007 Sun Microsystems, Inc. All rights reserverd.
|
||||
* Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2016-2017 Research Organization for Information Science
|
||||
* and Technology (RIST). All rights reserved.
|
||||
@ -82,7 +82,7 @@ static inline void opal_atomic_isync(void)
|
||||
*********************************************************************/
|
||||
#if OPAL_GCC_INLINE_ASSEMBLY
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval)
|
||||
{
|
||||
unsigned char ret;
|
||||
__asm__ __volatile__ (
|
||||
@ -102,13 +102,13 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add
|
||||
|
||||
#if OPAL_GCC_INLINE_ASSEMBLY
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval)
|
||||
{
|
||||
unsigned char ret;
|
||||
__asm__ __volatile__ (
|
||||
SMPLOCK "cmpxchgq %3,%2 \n\t"
|
||||
"sete %0 \n\t"
|
||||
: "=qm" (ret), "+a" (*oldval), "+m" (*((volatile long*)addr))
|
||||
: "=qm" (ret), "+a" (*oldval), "+m" (*((opal_atomic_long_t *)addr))
|
||||
: "q"(newval)
|
||||
: "memory", "cc"
|
||||
);
|
||||
@ -123,7 +123,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add
|
||||
|
||||
#if OPAL_GCC_INLINE_ASSEMBLY && OPAL_HAVE_CMPXCHG16B && HAVE_OPAL_INT128_T
|
||||
|
||||
static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128_t *addr, opal_int128_t *oldval, opal_int128_t newval)
|
||||
static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_int128_t *addr, opal_int128_t *oldval, opal_int128_t newval)
|
||||
{
|
||||
unsigned char ret;
|
||||
|
||||
@ -150,7 +150,7 @@ static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128
|
||||
|
||||
#define OPAL_HAVE_ATOMIC_SWAP_64 1
|
||||
|
||||
static inline int32_t opal_atomic_swap_32( volatile int32_t *addr,
|
||||
static inline int32_t opal_atomic_swap_32( opal_atomic_int32_t *addr,
|
||||
int32_t newval)
|
||||
{
|
||||
int32_t oldval;
|
||||
@ -166,7 +166,7 @@ static inline int32_t opal_atomic_swap_32( volatile int32_t *addr,
|
||||
|
||||
#if OPAL_GCC_INLINE_ASSEMBLY
|
||||
|
||||
static inline int64_t opal_atomic_swap_64( volatile int64_t *addr,
|
||||
static inline int64_t opal_atomic_swap_64( opal_atomic_int64_t *addr,
|
||||
int64_t newval)
|
||||
{
|
||||
int64_t oldval;
|
||||
@ -196,7 +196,7 @@ static inline int64_t opal_atomic_swap_64( volatile int64_t *addr,
|
||||
*
|
||||
* Atomically adds @i to @v.
|
||||
*/
|
||||
static inline int32_t opal_atomic_fetch_add_32(volatile int32_t* v, int i)
|
||||
static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t* v, int i)
|
||||
{
|
||||
int ret = i;
|
||||
__asm__ __volatile__(
|
||||
@ -217,7 +217,7 @@ static inline int32_t opal_atomic_fetch_add_32(volatile int32_t* v, int i)
|
||||
*
|
||||
* Atomically adds @i to @v.
|
||||
*/
|
||||
static inline int64_t opal_atomic_fetch_add_64(volatile int64_t* v, int64_t i)
|
||||
static inline int64_t opal_atomic_fetch_add_64(opal_atomic_int64_t* v, int64_t i)
|
||||
{
|
||||
int64_t ret = i;
|
||||
__asm__ __volatile__(
|
||||
@ -238,7 +238,7 @@ static inline int64_t opal_atomic_fetch_add_64(volatile int64_t* v, int64_t i)
|
||||
*
|
||||
* Atomically subtracts @i from @v.
|
||||
*/
|
||||
static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t* v, int i)
|
||||
static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t* v, int i)
|
||||
{
|
||||
int ret = -i;
|
||||
__asm__ __volatile__(
|
||||
@ -259,7 +259,7 @@ static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t* v, int i)
|
||||
*
|
||||
* Atomically subtracts @i from @v.
|
||||
*/
|
||||
static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t* v, int64_t i)
|
||||
static inline int64_t opal_atomic_fetch_sub_64(opal_atomic_int64_t* v, int64_t i)
|
||||
{
|
||||
int64_t ret = -i;
|
||||
__asm__ __volatile__(
|
||||
|
||||
66
opal/include/opal_stdatomic.h
Обычный файл
66
opal/include/opal_stdatomic.h
Обычный файл
@ -0,0 +1,66 @@
|
||||
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
||||
/*
|
||||
* Copyright (c) 2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#if !defined(OPAL_STDATOMIC_H)
|
||||
#define OPAL_STDATOMIC_H
|
||||
|
||||
#include "opal_stdint.h"
|
||||
|
||||
#if OPAL_ASSEMBLY_BUILTIN != OPAL_BUILTIN_C11
|
||||
|
||||
typedef volatile int opal_atomic_int_t;
|
||||
typedef volatile long opal_atomic_long_t;
|
||||
|
||||
typedef volatile int32_t opal_atomic_int32_t;
|
||||
typedef volatile uint32_t opal_atomic_uint32_t;
|
||||
typedef volatile int64_t opal_atomic_int64_t;
|
||||
typedef volatile uint64_t opal_atomic_uint64_t;
|
||||
|
||||
typedef volatile size_t opal_atomic_size_t;
|
||||
typedef volatile ssize_t opal_atomic_ssize_t;
|
||||
typedef volatile intptr_t opal_atomic_intptr_t;
|
||||
typedef volatile uintptr_t opal_atomic_uintptr_t;
|
||||
|
||||
#else /* OPAL_HAVE_C__ATOMIC */
|
||||
|
||||
#include <stdatomic.h>
|
||||
|
||||
typedef atomic_int opal_atomic_int_t;
|
||||
typedef atomic_long opal_atomic_long_t;
|
||||
|
||||
typedef _Atomic int32_t opal_atomic_int32_t;
|
||||
typedef _Atomic uint32_t opal_atomic_uint32_t;
|
||||
typedef _Atomic int64_t opal_atomic_int64_t;
|
||||
typedef _Atomic uint64_t opal_atomic_uint64_t;
|
||||
|
||||
typedef _Atomic size_t opal_atomic_size_t;
|
||||
typedef _Atomic ssize_t opal_atomic_ssize_t;
|
||||
typedef _Atomic intptr_t opal_atomic_intptr_t;
|
||||
typedef _Atomic uintptr_t opal_atomic_uintptr_t;
|
||||
|
||||
#endif /* OPAL_HAVE_C__ATOMIC */
|
||||
|
||||
#if HAVE_OPAL_INT128_T
|
||||
|
||||
/* do not use C11 atomics for __int128 if they are not lock free */
|
||||
#if OPAL_HAVE_C11_CSWAP_INT128
|
||||
|
||||
typedef _Atomic opal_int128_t opal_atomic_int128_t;
|
||||
|
||||
#else
|
||||
|
||||
typedef volatile opal_int128_t opal_atomic_int128_t;
|
||||
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
#endif /* !defined(OPAL_STDATOMIC_H) */
|
||||
@ -28,6 +28,8 @@
|
||||
#ifndef OPAL_STDINT_H
|
||||
#define OPAL_STDINT_H 1
|
||||
|
||||
#include "opal_config.h"
|
||||
|
||||
/*
|
||||
* Include what we can and define what is missing.
|
||||
*/
|
||||
|
||||
@ -123,8 +123,8 @@ struct mca_btl_ofi_module_t {
|
||||
bool use_virt_addr;
|
||||
bool is_scalable_ep;
|
||||
|
||||
int64_t outstanding_rdma;
|
||||
int64_t outstanding_send;
|
||||
opal_atomic_int64_t outstanding_rdma;
|
||||
opal_atomic_int64_t outstanding_send;
|
||||
|
||||
/** linked list of BTL endpoints. this list is never searched so
|
||||
* there is no need for a complicated structure here at this time*/
|
||||
|
||||
@ -237,7 +237,7 @@ struct mca_btl_openib_component_t {
|
||||
int apm_lmc;
|
||||
int apm_ports;
|
||||
unsigned int buffer_alignment; /**< Preferred communication buffer alignment in Bytes (must be power of two) */
|
||||
int32_t error_counter; /**< Counts number on error events that we got on all devices */
|
||||
opal_atomic_int32_t error_counter; /**< Counts number on error events that we got on all devices */
|
||||
opal_event_base_t *async_evbase; /**< Async event base */
|
||||
bool use_async_event_thread; /**< Use the async event handler */
|
||||
mca_btl_openib_srq_manager_t srq_manager; /**< Hash table for all BTL SRQs */
|
||||
@ -404,8 +404,8 @@ typedef struct mca_btl_openib_device_t {
|
||||
#endif
|
||||
int xrc_fd;
|
||||
#endif
|
||||
int32_t non_eager_rdma_endpoints;
|
||||
int32_t eager_rdma_buffers_count;
|
||||
opal_atomic_int32_t non_eager_rdma_endpoints;
|
||||
opal_atomic_int32_t eager_rdma_buffers_count;
|
||||
struct mca_btl_base_endpoint_t **eager_rdma_buffers;
|
||||
/**< frags for control massages */
|
||||
opal_free_list_t send_free_control;
|
||||
@ -428,8 +428,8 @@ struct mca_btl_openib_module_pp_qp_t {
|
||||
|
||||
struct mca_btl_openib_module_srq_qp_t {
|
||||
struct ibv_srq *srq;
|
||||
int32_t rd_posted;
|
||||
int32_t sd_credits; /* the max number of outstanding sends on a QP when using SRQ */
|
||||
opal_atomic_int32_t rd_posted;
|
||||
opal_atomic_int32_t sd_credits; /* the max number of outstanding sends on a QP when using SRQ */
|
||||
/* i.e. the number of frags that can be outstanding (down counter) */
|
||||
opal_list_t pending_frags[2]; /**< list of high/low prio frags */
|
||||
/** The number of receive buffers that can be post in the current time.
|
||||
@ -487,12 +487,12 @@ struct mca_btl_openib_module_t {
|
||||
int apm_port; /**< Alternative port that may be used for APM */
|
||||
uint8_t src_path_bits; /**< offset from base lid (for LMC) */
|
||||
|
||||
int32_t num_peers;
|
||||
opal_atomic_int32_t num_peers;
|
||||
|
||||
opal_mutex_t ib_lock; /**< module level lock */
|
||||
|
||||
size_t eager_rdma_frag_size; /**< length of eager frag */
|
||||
volatile int32_t eager_rdma_channels; /**< number of open RDMA channels */
|
||||
opal_atomic_int32_t eager_rdma_channels; /**< number of open RDMA channels */
|
||||
|
||||
mca_btl_base_module_error_cb_fn_t error_cb; /**< error handler */
|
||||
|
||||
|
||||
@ -4,7 +4,7 @@
|
||||
* Copyright (c) 2007-2013 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2006-2007 Voltaire All rights reserved.
|
||||
* Copyright (c) 2009-2010 Oracle and/or its affiliates. All rights reserved
|
||||
* Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2013-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2014 Intel, Inc. All rights reserved.
|
||||
* Copyright (c) 2014 Bull SAS. All rights reserved.
|
||||
@ -38,7 +38,7 @@
|
||||
|
||||
static opal_list_t ignore_qp_err_list;
|
||||
static opal_mutex_t ignore_qp_err_list_lock;
|
||||
static int32_t btl_openib_async_device_count = 0;
|
||||
static opal_atomic_int32_t btl_openib_async_device_count = 0;
|
||||
|
||||
struct mca_btl_openib_async_poll {
|
||||
int active_poll_size;
|
||||
|
||||
@ -1,7 +1,7 @@
|
||||
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
||||
/*
|
||||
* Copyright (c) 2006-2007 Voltaire All rights reserved.
|
||||
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
@ -25,7 +25,7 @@ struct mca_btl_openib_eager_rdma_local_t {
|
||||
mca_btl_openib_reg_t *reg;
|
||||
uint16_t head; /**< RDMA buffer to poll */
|
||||
uint16_t tail; /**< Needed for credit managment */
|
||||
int32_t credits; /**< number of RDMA credits */
|
||||
opal_atomic_int32_t credits; /**< number of RDMA credits */
|
||||
int32_t rd_win;
|
||||
#if OPAL_ENABLE_DEBUG
|
||||
uint32_t seq;
|
||||
@ -38,8 +38,8 @@ typedef struct mca_btl_openib_eager_rdma_local_t mca_btl_openib_eager_rdma_local
|
||||
struct mca_btl_openib_eager_rdma_remote_t {
|
||||
opal_ptr_t base; /**< address of remote buffer */
|
||||
uint32_t rkey; /**< RKey for accessing remote buffer */
|
||||
int32_t head; /**< RDMA buffer to post to */
|
||||
int32_t tokens; /**< number of rdam tokens */
|
||||
opal_atomic_int32_t head; /**< RDMA buffer to post to */
|
||||
opal_atomic_int32_t tokens; /**< number of rdma tokens */
|
||||
#if OPAL_ENABLE_DEBUG
|
||||
uint32_t seq;
|
||||
#endif
|
||||
@ -108,7 +108,7 @@ typedef struct mca_btl_openib_eager_rdma_remote_t mca_btl_openib_eager_rdma_remo
|
||||
|
||||
#define MCA_BTL_OPENIB_RDMA_MOVE_INDEX(HEAD, OLD_HEAD) \
|
||||
do { \
|
||||
(OLD_HEAD) = (OPAL_THREAD_ADD_FETCH32(&(HEAD), 1) - 1) % mca_btl_openib_component.eager_rdma_num; \
|
||||
(OLD_HEAD) = (OPAL_THREAD_ADD_FETCH32((opal_atomic_int32_t *) &(HEAD), 1) - 1) % mca_btl_openib_component.eager_rdma_num; \
|
||||
} while(0)
|
||||
|
||||
#endif
|
||||
|
||||
@ -11,7 +11,7 @@
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2006-2013 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2006-2017 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2006-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2006-2007 Voltaire All rights reserved.
|
||||
* Copyright (c) 2006-2009 Mellanox Technologies, Inc. All rights reserved.
|
||||
@ -378,7 +378,7 @@ static void mca_btl_openib_endpoint_destruct(mca_btl_base_endpoint_t* endpoint)
|
||||
* was not in "connect" or "bad" flow (failed to allocate memory)
|
||||
* and changed the pointer back to NULL
|
||||
*/
|
||||
if(!opal_atomic_compare_exchange_strong_ptr(&endpoint->eager_rdma_local.base.pval, (void *) &_tmp_ptr, (void *) 1)) {
|
||||
if(!opal_atomic_compare_exchange_strong_ptr((opal_atomic_intptr_t *) &endpoint->eager_rdma_local.base.pval, (intptr_t *) &_tmp_ptr, 1)) {
|
||||
if (NULL != endpoint->eager_rdma_local.reg) {
|
||||
endpoint->endpoint_btl->device->rcache->rcache_deregister (endpoint->endpoint_btl->device->rcache,
|
||||
&endpoint->eager_rdma_local.reg->base);
|
||||
@ -899,8 +899,7 @@ void mca_btl_openib_endpoint_connect_eager_rdma(
|
||||
|
||||
/* Set local rdma pointer to 1 temporarily so other threads will not try
|
||||
* to enter the function */
|
||||
if(!opal_atomic_compare_exchange_strong_ptr (&endpoint->eager_rdma_local.base.pval, (void *) &_tmp_ptr,
|
||||
(void *) 1)) {
|
||||
if(!opal_atomic_compare_exchange_strong_ptr ((opal_atomic_intptr_t *) &endpoint->eager_rdma_local.base.pval, (intptr_t *) &_tmp_ptr, 1)) {
|
||||
return;
|
||||
}
|
||||
|
||||
@ -990,7 +989,7 @@ void mca_btl_openib_endpoint_connect_eager_rdma(
|
||||
do {
|
||||
_tmp_ptr = NULL;
|
||||
p = &device->eager_rdma_buffers[device->eager_rdma_buffers_count];
|
||||
} while(!opal_atomic_compare_exchange_strong_ptr (p, (void *) &_tmp_ptr, endpoint));
|
||||
} while(!opal_atomic_compare_exchange_strong_ptr ((opal_atomic_intptr_t *) p, (intptr_t *) &_tmp_ptr, (intptr_t) endpoint));
|
||||
|
||||
OPAL_THREAD_ADD_FETCH32(&openib_btl->eager_rdma_channels, 1);
|
||||
/* from this point progress function starts to poll new buffer */
|
||||
|
||||
@ -11,7 +11,7 @@
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2007-2009 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2006-2017 Los Alamos National Security, LLC. All rights
|
||||
* Copyright (c) 2006-2018 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2006-2007 Voltaire All rights reserved.
|
||||
* Copyright (c) 2007-2009 Mellanox Technologies. All rights reserved.
|
||||
@ -117,17 +117,17 @@ typedef struct mca_btl_openib_rem_info_t {
|
||||
* Agggregates all per peer qp info for an endpoint
|
||||
*/
|
||||
typedef struct mca_btl_openib_endpoint_pp_qp_t {
|
||||
int32_t sd_credits; /**< this rank's view of the credits
|
||||
opal_atomic_int32_t sd_credits; /**< this rank's view of the credits
|
||||
* available for sending:
|
||||
* this is the credits granted by the
|
||||
* remote peer which has some relation to the
|
||||
* number of receive buffers posted remotely
|
||||
*/
|
||||
int32_t rd_posted; /**< number of descriptors posted to the nic*/
|
||||
int32_t rd_credits; /**< number of credits to return to peer */
|
||||
int32_t cm_received; /**< Credit messages received */
|
||||
int32_t cm_return; /**< how may credits to return */
|
||||
int32_t cm_sent; /**< Outstanding number of credit messages */
|
||||
opal_atomic_int32_t rd_posted; /**< number of descriptors posted to the nic*/
|
||||
opal_atomic_int32_t rd_credits; /**< number of credits to return to peer */
|
||||
opal_atomic_int32_t cm_received; /**< Credit messages received */
|
||||
opal_atomic_int32_t cm_return; /**< how may credits to return */
|
||||
opal_atomic_int32_t cm_sent; /**< Outstanding number of credit messages */
|
||||
} mca_btl_openib_endpoint_pp_qp_t;
|
||||
|
||||
|
||||
@ -141,8 +141,8 @@ typedef struct mca_btl_openib_endpoint_srq_qp_t {
|
||||
typedef struct mca_btl_openib_qp_t {
|
||||
struct ibv_qp *lcl_qp;
|
||||
uint32_t lcl_psn;
|
||||
volatile int32_t sd_wqe; /**< number of available send wqe entries */
|
||||
int32_t sd_wqe_inflight;
|
||||
opal_atomic_int32_t sd_wqe; /**< number of available send wqe entries */
|
||||
opal_atomic_int32_t sd_wqe_inflight;
|
||||
int wqe_count;
|
||||
int users;
|
||||
opal_mutex_t lock;
|
||||
@ -154,7 +154,7 @@ typedef struct mca_btl_openib_endpoint_qp_t {
|
||||
available */
|
||||
opal_list_t no_wqe_pending_frags[2]; /**< put fragments here if there is no wqe
|
||||
available */
|
||||
int32_t rd_credit_send_lock; /**< Lock credit send fragment */
|
||||
opal_atomic_int32_t rd_credit_send_lock; /**< Lock credit send fragment */
|
||||
mca_btl_openib_send_control_frag_t *credit_frag;
|
||||
size_t ib_inline_max; /**< max size of inline send*/
|
||||
union {
|
||||
@ -226,7 +226,7 @@ struct mca_btl_base_endpoint_t {
|
||||
opal_list_t pending_put_frags;
|
||||
|
||||
/** number of available get tokens */
|
||||
int32_t get_tokens;
|
||||
opal_atomic_int32_t get_tokens;
|
||||
|
||||
/** subnet id of this endpoint*/
|
||||
uint64_t subnet_id;
|
||||
@ -235,7 +235,7 @@ struct mca_btl_base_endpoint_t {
|
||||
struct ib_address_t *ib_addr;
|
||||
|
||||
/** number of eager received */
|
||||
int32_t eager_recv_count;
|
||||
opal_atomic_int32_t eager_recv_count;
|
||||
/** info about remote RDMA buffer */
|
||||
mca_btl_openib_eager_rdma_remote_t eager_rdma_remote;
|
||||
/** info about local RDMA buffer */
|
||||
|
||||
@ -92,7 +92,7 @@ struct mca_btl_portals4_module_t {
|
||||
|
||||
/* number of processes we're actively connected to. Needed to
|
||||
know when to do activation / shutdown */
|
||||
int32_t portals_num_procs;
|
||||
opal_atomic_int32_t portals_num_procs;
|
||||
|
||||
/* number of the interface (btl) */
|
||||
uint32_t interface_num;
|
||||
@ -111,11 +111,11 @@ struct mca_btl_portals4_module_t {
|
||||
ptl_handle_eq_t recv_eq_h;
|
||||
|
||||
/* number outstanding sends and local rdma */
|
||||
volatile int32_t portals_outstanding_ops;
|
||||
opal_atomic_int32_t portals_outstanding_ops;
|
||||
int32_t portals_max_outstanding_ops;
|
||||
|
||||
/* key to use for next rdma operation */
|
||||
volatile int64_t portals_rdma_key;
|
||||
opal_atomic_int64_t portals_rdma_key;
|
||||
|
||||
/* our portals network interface */
|
||||
ptl_handle_ni_t portals_ni_h;
|
||||
|
||||
@ -159,8 +159,8 @@ struct mca_btl_smcuda_component_t {
|
||||
struct mca_btl_base_endpoint_t **sm_peers;
|
||||
|
||||
opal_free_list_t pending_send_fl;
|
||||
int num_outstanding_frags; /**< number of fragments sent but not yet returned to free list */
|
||||
int num_pending_sends; /**< total number on all of my pending-send queues */
|
||||
opal_atomic_int32_t num_outstanding_frags; /**< number of fragments sent but not yet returned to free list */
|
||||
opal_atomic_int32_t num_pending_sends; /**< total number on all of my pending-send queues */
|
||||
int mem_node;
|
||||
int num_mem_nodes;
|
||||
|
||||
|
||||
@ -59,7 +59,7 @@ static inline void mca_btl_uct_context_unlock (mca_btl_uct_device_context_t *con
|
||||
|
||||
static inline int mca_btl_uct_get_context_index (void)
|
||||
{
|
||||
static volatile uint32_t next_uct_index = 0;
|
||||
static opal_atomic_uint32_t next_uct_index = 0;
|
||||
int context_id;
|
||||
|
||||
#if OPAL_C_HAVE__THREAD_LOCAL
|
||||
@ -68,7 +68,7 @@ static inline int mca_btl_uct_get_context_index (void)
|
||||
|
||||
context_id = uct_index;
|
||||
if (OPAL_UNLIKELY(-1 == context_id)) {
|
||||
context_id = uct_index = opal_atomic_fetch_add_32 ((volatile int32_t *) &next_uct_index, 1) %
|
||||
context_id = uct_index = opal_atomic_fetch_add_32 ((opal_atomic_int32_t *) &next_uct_index, 1) %
|
||||
mca_btl_uct_component.num_contexts_per_module;
|
||||
}
|
||||
} else {
|
||||
@ -92,7 +92,7 @@ mca_btl_uct_module_get_tl_context_specific (mca_btl_uct_module_t *module, mca_bt
|
||||
mca_btl_uct_device_context_t *new_context;
|
||||
|
||||
new_context = mca_btl_uct_context_create (module, tl, context_id);
|
||||
if (!opal_atomic_compare_exchange_strong_ptr (&tl->uct_dev_contexts[context_id], &context, new_context)) {
|
||||
if (!opal_atomic_compare_exchange_strong_ptr ((opal_atomic_intptr_t *) &tl->uct_dev_contexts[context_id], &context, new_context)) {
|
||||
mca_btl_uct_context_destroy (new_context);
|
||||
} else {
|
||||
context = new_context;
|
||||
|
||||
@ -94,7 +94,7 @@ typedef struct mca_btl_uct_conn_req_t mca_btl_uct_conn_req_t;
|
||||
*/
|
||||
struct mca_btl_uct_tl_endpoint_t {
|
||||
/** current flags (connected, requested, etc) */
|
||||
volatile int32_t flags;
|
||||
opal_atomic_int32_t flags;
|
||||
|
||||
/** UCT endpoint handle */
|
||||
uct_ep_h uct_ep;
|
||||
|
||||
@ -114,7 +114,7 @@ struct mca_btl_ugni_device_t {
|
||||
int dev_index;
|
||||
|
||||
/** number of SMSG connections */
|
||||
volatile int32_t smsg_connections;
|
||||
opal_atomic_int32_t smsg_connections;
|
||||
|
||||
/** boolean indicating that the device was recently flushed */
|
||||
volatile bool flushed;
|
||||
@ -168,7 +168,7 @@ typedef struct mca_btl_ugni_module_t {
|
||||
gni_ep_handle_t wildcard_ep;
|
||||
struct mca_btl_base_endpoint_t *local_ep;
|
||||
|
||||
volatile int32_t active_datagrams;
|
||||
opal_atomic_int32_t active_datagrams;
|
||||
opal_event_t connection_event;
|
||||
|
||||
struct mca_btl_ugni_endpoint_attr_t wc_remote_attr, wc_local_attr;
|
||||
@ -188,13 +188,13 @@ typedef struct mca_btl_ugni_module_t {
|
||||
opal_pointer_array_t pending_smsg_frags_bb;
|
||||
|
||||
int32_t reg_max;
|
||||
volatile int32_t reg_count;
|
||||
opal_atomic_int32_t reg_count;
|
||||
|
||||
/* used to calculate the fraction of registered memory resources
|
||||
* this rank should be limited too */
|
||||
int nlocal_procs;
|
||||
|
||||
volatile int32_t active_rdma_count;
|
||||
opal_atomic_int32_t active_rdma_count;
|
||||
|
||||
mca_rcache_base_module_t *rcache;
|
||||
} mca_btl_ugni_module_t;
|
||||
|
||||
@ -70,7 +70,7 @@ typedef struct mca_btl_base_endpoint_t {
|
||||
bool dg_posted;
|
||||
|
||||
/** protect against re-entry to SMSG */
|
||||
int32_t smsg_progressing;
|
||||
opal_atomic_int32_t smsg_progressing;
|
||||
|
||||
int index;
|
||||
} mca_btl_base_endpoint_t;
|
||||
|
||||
@ -68,7 +68,7 @@ struct mca_btl_ugni_base_frag_t;
|
||||
|
||||
typedef struct mca_btl_ugni_base_frag_t {
|
||||
mca_btl_base_descriptor_t base;
|
||||
volatile int32_t ref_cnt;
|
||||
opal_atomic_int32_t ref_cnt;
|
||||
uint32_t msg_id;
|
||||
uint16_t hdr_size;
|
||||
uint16_t flags;
|
||||
|
||||
@ -715,7 +715,7 @@ static void mca_btl_vader_progress_endpoints (void)
|
||||
|
||||
static int mca_btl_vader_component_progress (void)
|
||||
{
|
||||
static int32_t lock = 0;
|
||||
static opal_atomic_int32_t lock = 0;
|
||||
int count = 0;
|
||||
|
||||
if (opal_using_threads()) {
|
||||
|
||||
@ -62,7 +62,7 @@ typedef struct mca_btl_base_endpoint_t {
|
||||
|
||||
int32_t peer_smp_rank; /**< my peer's SMP process rank. Used for accessing
|
||||
* SMP specfic data structures. */
|
||||
volatile size_t send_count; /**< number of fragments sent to this peer */
|
||||
opal_atomic_size_t send_count; /**< number of fragments sent to this peer */
|
||||
char *segment_base; /**< start of the peer's segment (in the address space
|
||||
* of this process) */
|
||||
|
||||
|
||||
@ -30,26 +30,25 @@
|
||||
#include "btl_vader_endpoint.h"
|
||||
#include "btl_vader_frag.h"
|
||||
|
||||
#define vader_item_compare_exchange(x, y, z) opal_atomic_compare_exchange_strong_ptr ((volatile void **) (x), (void **) (y), (void *) (z))
|
||||
#define vader_item_compare_exchange(x, y, z) opal_atomic_compare_exchange_strong_ptr ((opal_atomic_intptr_t *) (x), (intptr_t *) (y), (intptr_t) (z))
|
||||
|
||||
#if SIZEOF_VOID_P == 8
|
||||
#define vader_item_swap(x, y) opal_atomic_swap_64((volatile int64_t *)(x), (int64_t)(y))
|
||||
#define vader_item_swap(x, y) opal_atomic_swap_64((opal_atomic_int64_t *)(x), (int64_t)(y))
|
||||
|
||||
#define MCA_BTL_VADER_OFFSET_MASK 0xffffffffll
|
||||
#define MCA_BTL_VADER_OFFSET_BITS 32
|
||||
#define MCA_BTL_VADER_BITNESS 64
|
||||
|
||||
typedef int64_t fifo_value_t;
|
||||
#else
|
||||
#define vader_item_swap(x, y) opal_atomic_swap_32((volatile int32_t *)(x), (int32_t)(y))
|
||||
#define vader_item_swap(x, y) opal_atomic_swap_32((opal_atomic_int32_t *)(x), (int32_t)(y))
|
||||
|
||||
#define MCA_BTL_VADER_OFFSET_MASK 0x00ffffffl
|
||||
#define MCA_BTL_VADER_OFFSET_BITS 24
|
||||
#define MCA_BTL_VADER_BITNESS 32
|
||||
|
||||
typedef int32_t fifo_value_t;
|
||||
#endif
|
||||
|
||||
typedef opal_atomic_intptr_t atomic_fifo_value_t;
|
||||
typedef intptr_t fifo_value_t;
|
||||
|
||||
#define VADER_FIFO_FREE ((fifo_value_t)-2)
|
||||
|
||||
/*
|
||||
@ -68,9 +67,9 @@
|
||||
|
||||
/* lock free fifo */
|
||||
typedef struct vader_fifo_t {
|
||||
volatile fifo_value_t fifo_head;
|
||||
volatile fifo_value_t fifo_tail;
|
||||
volatile int32_t fbox_available;
|
||||
atomic_fifo_value_t fifo_head;
|
||||
atomic_fifo_value_t fifo_tail;
|
||||
opal_atomic_int32_t fbox_available;
|
||||
} vader_fifo_t;
|
||||
|
||||
/* large enough to ensure the fifo is on its own cache line */
|
||||
|
||||
Некоторые файлы не были показаны из-за слишком большого количества измененных файлов Показать больше
Загрузка…
x
Ссылка в новой задаче
Block a user