diff --git a/config/opal_config_asm.m4 b/config/opal_config_asm.m4 index 65675d16b5..9602a53c36 100644 --- a/config/opal_config_asm.m4 +++ b/config/opal_config_asm.m4 @@ -13,7 +13,7 @@ dnl Copyright (c) 2008-2018 Cisco Systems, Inc. All rights reserved. dnl Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. dnl Copyright (c) 2015-2017 Research Organization for Information Science dnl and Technology (RIST). All rights reserved. -dnl Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights +dnl Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights dnl reserved. dnl Copyright (c) 2017 Amazon.com, Inc. or its affiliates. All Rights dnl reserved. @@ -122,6 +122,58 @@ int main(int argc, char** argv) } ]]) +dnl This is a C test to see if 128-bit __atomic_compare_exchange_n() +dnl actually works (e.g., it compiles and links successfully on +dnl ARM64+clang, but returns incorrect answers as of August 2018). +AC_DEFUN([OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_TEST_SOURCE],[[ +#include +#include +#include +#include + +typedef union { + uint64_t fake@<:@2@:>@; + _Atomic __int128 real; +} ompi128; + +static void test1(void) +{ + // As of Aug 2018, we could not figure out a way to assign 128-bit + // constants -- the compilers would not accept it. So use a fake + // union to assign 2 uin64_t's to make a single __int128. + ompi128 ptr = { .fake = { 0xFFEEDDCCBBAA0099, 0x8877665544332211 }}; + ompi128 expected = { .fake = { 0x11EEDDCCBBAA0099, 0x88776655443322FF }}; + ompi128 desired = { .fake = { 0x1122DDCCBBAA0099, 0x887766554433EEFF }}; + bool r = atomic_compare_exchange_strong (&ptr.real, &expected.real, + desired.real, true, + atomic_relaxed, atomic_relaxed); + if ( !(r == false && ptr.real == expected.real)) { + exit(1); + } +} + +static void test2(void) +{ + ompi128 ptr = { .fake = { 0xFFEEDDCCBBAA0099, 0x8877665544332211 }}; + ompi128 expected = ptr; + ompi128 desired = { .fake = { 0x1122DDCCBBAA0099, 0x887766554433EEFF }}; + bool r = atomic_compare_exchange_strong (&ptr.real, &expected.real, + desired.real, true, + atomic_relaxed, atomic_relaxed); + if (!(r == true && ptr.real == desired.real)) { + exit(2); + } +} + +vvvvvvvvvvvvvvvvvvvv +int main(int argc, char** argv) +{ + test1(); + test2(); + return 0; +} +]]) + dnl ------------------------------------------------------------------ dnl @@ -329,6 +381,71 @@ __atomic_add_fetch(&tmp64, 1, __ATOMIC_RELAXED);], OPAL_CHECK_GCC_BUILTIN_CSWAP_INT128 ]) +AC_DEFUN([OPAL_CHECK_C11_CSWAP_INT128], [ + OPAL_VAR_SCOPE_PUSH([atomic_compare_exchange_result atomic_compare_exchange_CFLAGS_save atomic_compare_exchange_LIBS_save]) + + atomic_compare_exchange_CFLAGS_save=$CFLAGS + atomic_compare_exchange_LIBS_save=$LIBS + + # Do we have C11 atomics on 128-bit integers? + # Use a special macro because we need to check with a few different + # CFLAGS/LIBS. + OPAL_ASM_CHECK_ATOMIC_FUNC([atomic_compare_exchange_strong_16], + [AC_LANG_SOURCE(OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_TEST_SOURCE)], + [atomic_compare_exchange_result=1], + [atomic_compare_exchange_result=0]) + + # If we have it and it works, check to make sure it is always lock + # free. + AS_IF([test $atomic_compare_exchange_result -eq 1], + [AC_MSG_CHECKING([if C11 __int128 atomic compare-and-swap is always lock-free]) + AC_RUN_IFELSE([AC_LANG_PROGRAM([#include ], [_Atomic __int128_t x; if (!atomic_is_lock_free(&x)) { return 1; }])], + [AC_MSG_RESULT([yes])], + [atomic_compare_exchange_result=0 + # If this test fails, need to reset CFLAGS/LIBS (the + # above tests atomically set CFLAGS/LIBS or not; this + # test is running after the fact, so we have to undo + # the side-effects of setting CFLAGS/LIBS if the above + # tests passed). + CFLAGS=$atomic_compare_exchange_CFLAGS_save + LIBS=$atomic_compare_exchange_LIBS_save + AC_MSG_RESULT([no])], + [AC_MSG_RESULT([cannot test -- assume yes (cross compiling)])]) + ]) + + AC_DEFINE_UNQUOTED([OPAL_HAVE_C11_CSWAP_INT128], + [$atomic_compare_exchange_result], + [Whether C11 atomic compare swap is both supported and lock-free on 128-bit values]) + + dnl If we could not find decent support for 128-bits atomic let's + dnl try the GCC _sync + AS_IF([test $atomic_compare_exchange_result -eq 0], + [OPAL_CHECK_SYNC_BUILTIN_CSWAP_INT128]) + + OPAL_VAR_SCOPE_POP +]) + +AC_DEFUN([OPAL_CHECK_GCC_ATOMIC_BUILTINS], [ + AC_MSG_CHECKING([for __atomic builtin atomics]) + + AC_TRY_LINK([ +#include +uint32_t tmp, old = 0; +uint64_t tmp64, old64 = 0;], [ +__atomic_thread_fence(__ATOMIC_SEQ_CST); +__atomic_compare_exchange_n(&tmp, &old, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED); +__atomic_add_fetch(&tmp, 1, __ATOMIC_RELAXED); +__atomic_compare_exchange_n(&tmp64, &old64, 1, 0, __ATOMIC_RELAXED, __ATOMIC_RELAXED); +__atomic_add_fetch(&tmp64, 1, __ATOMIC_RELAXED);], + [AC_MSG_RESULT([yes]) + $1], + [AC_MSG_RESULT([no]) + $2]) + + # Check for 128-bit support + OPAL_CHECK_GCC_BUILTIN_CSWAP_INT128 +]) + dnl ################################################################# dnl @@ -1020,17 +1137,27 @@ AC_DEFUN([OPAL_CONFIG_ASM],[ AC_REQUIRE([OPAL_SETUP_CC]) AC_REQUIRE([AM_PROG_AS]) + AC_ARG_ENABLE([c11-atomics],[AC_HELP_STRING([--enable-c11-atomics], + [Enable use of C11 atomics if available (default: enabled)])]) + AC_ARG_ENABLE([builtin-atomics], [AC_HELP_STRING([--enable-builtin-atomics], - [Enable use of __sync builtin atomics (default: enabled)])]) + [Enable use of __sync builtin atomics (default: disabled)])]) - opal_cv_asm_builtin="BUILTIN_NO" - AS_IF([test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" != "no"], - [OPAL_CHECK_GCC_ATOMIC_BUILTINS([opal_cv_asm_builtin="BUILTIN_GCC"], [])]) - AS_IF([test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" != "no"], - [OPAL_CHECK_SYNC_BUILTINS([opal_cv_asm_builtin="BUILTIN_SYNC"], [])]) - AS_IF([test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes"], - [AC_MSG_ERROR([__sync builtin atomics requested but not found.])]) + OPAL_CHECK_C11_CSWAP_INT128 + + if test "x$enable_c11_atomics" != "xno" && test "$opal_cv_c11_supported" = "yes" ; then + opal_cv_asm_builtin="BUILTIN_C11" + OPAL_CHECK_C11_CSWAP_INT128 + else + opal_cv_asm_builtin="BUILTIN_NO" + AS_IF([test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes"], + [OPAL_CHECK_GCC_ATOMIC_BUILTINS([opal_cv_asm_builtin="BUILTIN_GCC"], [])]) + AS_IF([test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes"], + [OPAL_CHECK_SYNC_BUILTINS([opal_cv_asm_builtin="BUILTIN_SYNC"], [])]) + AS_IF([test "$opal_cv_asm_builtin" = "BUILTIN_NO" && test "$enable_builtin_atomics" = "yes"], + [AC_MSG_ERROR([__sync builtin atomics requested but not found.])]) + fi OPAL_CHECK_ASM_PROC OPAL_CHECK_ASM_TEXT diff --git a/ompi/communicator/comm_request.c b/ompi/communicator/comm_request.c index 272fc33600..934cbc75df 100644 --- a/ompi/communicator/comm_request.c +++ b/ompi/communicator/comm_request.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2013-2016 Los Alamos National Security, LLC. All rights + * Copyright (c) 2013-2018 Los Alamos National Security, LLC. All rights * reseved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -99,7 +99,7 @@ int ompi_comm_request_schedule_append (ompi_comm_request_t *request, ompi_comm_r static int ompi_comm_request_progress (void) { ompi_comm_request_t *request, *next; - static int32_t progressing = 0; + static opal_atomic_int32_t progressing = 0; /* don't allow re-entry */ if (opal_atomic_swap_32 (&progressing, 1)) { diff --git a/ompi/datatype/ompi_datatype.h b/ompi/datatype/ompi_datatype.h index 8b48bc3097..d9df1f1ae4 100644 --- a/ompi/datatype/ompi_datatype.h +++ b/ompi/datatype/ompi_datatype.h @@ -75,7 +75,7 @@ struct ompi_datatype_t { struct opal_hash_table_t *d_keyhash; /**< Attribute fields */ void* args; /**< Data description for the user */ - void* packed_description; /**< Packed description of the datatype */ + opal_atomic_intptr_t packed_description; /**< Packed description of the datatype */ uint64_t pml_data; /**< PML-specific information */ /* --- cacheline 6 boundary (384 bytes) --- */ char name[MPI_MAX_OBJECT_NAME];/**< Externally visible name */ diff --git a/ompi/datatype/ompi_datatype_args.c b/ompi/datatype/ompi_datatype_args.c index 737d3e5182..2e8af79215 100644 --- a/ompi/datatype/ompi_datatype_args.c +++ b/ompi/datatype/ompi_datatype_args.c @@ -45,7 +45,7 @@ __ompi_datatype_create_from_args( int32_t* i, ptrdiff_t * a, ompi_datatype_t** d, int32_t type ); typedef struct __dt_args { - int32_t ref_count; + opal_atomic_int32_t ref_count; int32_t create_type; size_t total_pack_size; int32_t ci; @@ -104,7 +104,7 @@ typedef struct __dt_args { pArgs->total_pack_size = (4 + (IC) + (DC)) * sizeof(int) + \ (AC) * sizeof(ptrdiff_t); \ (PDATA)->args = (void*)pArgs; \ - (PDATA)->packed_description = NULL; \ + (PDATA)->packed_description = 0; \ } while(0) @@ -483,12 +483,12 @@ int ompi_datatype_get_pack_description( ompi_datatype_t* datatype, { ompi_datatype_args_t* args = (ompi_datatype_args_t*)datatype->args; int next_index = OMPI_DATATYPE_MAX_PREDEFINED; - void *packed_description = datatype->packed_description; + void *packed_description = (void *) datatype->packed_description; void* recursive_buffer; if (NULL == packed_description) { void *_tmp_ptr = NULL; - if (opal_atomic_compare_exchange_strong_ptr (&datatype->packed_description, (void *) &_tmp_ptr, (void *) 1)) { + if (opal_atomic_compare_exchange_strong_ptr (&datatype->packed_description, (intptr_t *) &_tmp_ptr, 1)) { if( ompi_datatype_is_predefined(datatype) ) { packed_description = malloc(2 * sizeof(int)); } else if( NULL == args ) { @@ -510,10 +510,10 @@ int ompi_datatype_get_pack_description( ompi_datatype_t* datatype, } opal_atomic_wmb (); - datatype->packed_description = packed_description; + datatype->packed_description = (intptr_t) packed_description; } else { /* another thread beat us to it */ - packed_description = datatype->packed_description; + packed_description = (void *) datatype->packed_description; } } @@ -521,11 +521,11 @@ int ompi_datatype_get_pack_description( ompi_datatype_t* datatype, struct timespec interval = {.tv_sec = 0, .tv_nsec = 1000}; /* wait until the packed description is updated */ - while ((void *) 1 == datatype->packed_description) { + while (1 == datatype->packed_description) { nanosleep (&interval, NULL); } - packed_description = datatype->packed_description; + packed_description = (void *) datatype->packed_description; } *packed_buffer = (const void *) packed_description; @@ -534,7 +534,7 @@ int ompi_datatype_get_pack_description( ompi_datatype_t* datatype, size_t ompi_datatype_pack_description_length( ompi_datatype_t* datatype ) { - void *packed_description = datatype->packed_description; + void *packed_description = (void *) datatype->packed_description; if( ompi_datatype_is_predefined(datatype) ) { return 2 * sizeof(int); diff --git a/ompi/datatype/ompi_datatype_create.c b/ompi/datatype/ompi_datatype_create.c index 69ec1b2c6c..aa0760c2c5 100644 --- a/ompi/datatype/ompi_datatype_create.c +++ b/ompi/datatype/ompi_datatype_create.c @@ -36,7 +36,7 @@ static void __ompi_datatype_allocate( ompi_datatype_t* datatype ) datatype->id = -1; datatype->d_keyhash = NULL; datatype->name[0] = '\0'; - datatype->packed_description = NULL; + datatype->packed_description = 0; datatype->pml_data = 0; } @@ -46,10 +46,10 @@ static void __ompi_datatype_release(ompi_datatype_t * datatype) ompi_datatype_release_args( datatype ); datatype->args = NULL; } - if( NULL != datatype->packed_description ) { - free( datatype->packed_description ); - datatype->packed_description = NULL; - } + + free ((void *) datatype->packed_description ); + datatype->packed_description = 0; + if( datatype->d_f_to_c_index >= 0 ) { opal_pointer_array_set_item( &ompi_datatype_f_to_c_table, datatype->d_f_to_c_index, NULL ); datatype->d_f_to_c_index = -1; diff --git a/ompi/datatype/ompi_datatype_internal.h b/ompi/datatype/ompi_datatype_internal.h index 0cbfb25a95..7de5863b2f 100644 --- a/ompi/datatype/ompi_datatype_internal.h +++ b/ompi/datatype/ompi_datatype_internal.h @@ -406,7 +406,7 @@ extern const ompi_datatype_t* ompi_datatype_basicDatatypes[OMPI_DATATYPE_MPI_MAX .d_f_to_c_index = -1, \ .d_keyhash = NULL, \ .args = NULL, \ - .packed_description = NULL, \ + .packed_description = 0, \ .name = "MPI_" # NAME #define OMPI_DATATYPE_INITIALIZER_UNAVAILABLE(FLAGS) \ diff --git a/ompi/datatype/ompi_datatype_module.c b/ompi/datatype/ompi_datatype_module.c index 3ee09173cd..ef4398f576 100644 --- a/ompi/datatype/ompi_datatype_module.c +++ b/ompi/datatype/ompi_datatype_module.c @@ -383,7 +383,7 @@ opal_pointer_array_t ompi_datatype_f_to_c_table = {{0}}; (PDST)->super.desc = (PSRC)->super.desc; \ (PDST)->super.opt_desc = (PSRC)->super.opt_desc; \ (PDST)->packed_description = (PSRC)->packed_description; \ - (PSRC)->packed_description = NULL; \ + (PSRC)->packed_description = 0; \ /* transfer the ptypes */ \ (PDST)->super.ptypes = (PSRC)->super.ptypes; \ (PSRC)->super.ptypes = NULL; \ diff --git a/ompi/group/group.h b/ompi/group/group.h index 30664f8a4e..661666246e 100644 --- a/ompi/group/group.h +++ b/ompi/group/group.h @@ -356,7 +356,8 @@ static inline struct ompi_proc_t *ompi_group_dense_lookup (ompi_group_t *group, ompi_proc_t *real_proc = (ompi_proc_t *) ompi_proc_for_name (ompi_proc_sentinel_to_name ((uintptr_t) proc)); - if (opal_atomic_compare_exchange_strong_ptr (group->grp_proc_pointers + peer_id, &proc, real_proc)) { + if (opal_atomic_compare_exchange_strong_ptr ((opal_atomic_intptr_t *)(group->grp_proc_pointers + peer_id), + (intptr_t *) &proc, (intptr_t) real_proc)) { OBJ_RETAIN(real_proc); } diff --git a/ompi/mca/coll/libnbc/coll_libnbc.h b/ompi/mca/coll/libnbc/coll_libnbc.h index 967a779425..56cb2d75df 100644 --- a/ompi/mca/coll/libnbc/coll_libnbc.h +++ b/ompi/mca/coll/libnbc/coll_libnbc.h @@ -75,7 +75,7 @@ struct ompi_coll_libnbc_component_t { mca_coll_base_component_2_0_0_t super; opal_free_list_t requests; opal_list_t active_requests; - int32_t active_comms; + opal_atomic_int32_t active_comms; opal_mutex_t lock; /* protect access to the active_requests list */ }; typedef struct ompi_coll_libnbc_component_t ompi_coll_libnbc_component_t; diff --git a/ompi/mca/coll/monitoring/coll_monitoring.h b/ompi/mca/coll/monitoring/coll_monitoring.h index 8046c98208..1fc5cca20d 100644 --- a/ompi/mca/coll/monitoring/coll_monitoring.h +++ b/ompi/mca/coll/monitoring/coll_monitoring.h @@ -36,7 +36,7 @@ struct mca_coll_monitoring_module_t { mca_coll_base_module_t super; mca_coll_base_comm_coll_t real; mca_monitoring_coll_data_t*data; - int32_t is_initialized; + opal_atomic_int32_t is_initialized; }; typedef struct mca_coll_monitoring_module_t mca_coll_monitoring_module_t; OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_coll_monitoring_module_t); diff --git a/ompi/mca/coll/portals4/coll_portals4.h b/ompi/mca/coll/portals4/coll_portals4.h index 175c951e81..ce01084f32 100644 --- a/ompi/mca/coll/portals4/coll_portals4.h +++ b/ompi/mca/coll/portals4/coll_portals4.h @@ -1,6 +1,7 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2013-2015 Sandia National Laboratories. All rights reserved. - * Copyright (c) 2015 Los Alamos National Security, LLC. All rights + * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2015 Bull SAS. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science @@ -91,7 +92,7 @@ typedef struct ompi_coll_portals4_tree_t { struct mca_coll_portals4_module_t { mca_coll_base_module_t super; - size_t coll_count; + opal_atomic_size_t coll_count; /* record handlers dedicated to fallback if offloaded operations are not supported */ mca_coll_base_module_reduce_fn_t previous_reduce; diff --git a/ompi/mca/coll/sm/coll_sm.h b/ompi/mca/coll/sm/coll_sm.h index b2da6ede42..e40ca73c2f 100644 --- a/ompi/mca/coll/sm/coll_sm.h +++ b/ompi/mca/coll/sm/coll_sm.h @@ -114,7 +114,7 @@ BEGIN_C_DECLS typedef struct mca_coll_sm_in_use_flag_t { /** Number of processes currently using this set of segments */ - volatile uint32_t mcsiuf_num_procs_using; + opal_atomic_uint32_t mcsiuf_num_procs_using; /** Must match data->mcb_count */ volatile uint32_t mcsiuf_operation_count; } mca_coll_sm_in_use_flag_t; @@ -152,7 +152,7 @@ BEGIN_C_DECLS /** Pointer to my parent's barrier control pages (will be NULL for communicator rank 0; odd index pages are "in", even index pages are "out") */ - uint32_t *mcb_barrier_control_parent; + opal_atomic_uint32_t *mcb_barrier_control_parent; /** Pointers to my childrens' barrier control pages (they're contiguous in memory, so we only point to the base -- the diff --git a/ompi/mca/coll/sm/coll_sm_barrier.c b/ompi/mca/coll/sm/coll_sm_barrier.c index 2722bbf09f..e8e0fb5b78 100644 --- a/ompi/mca/coll/sm/coll_sm_barrier.c +++ b/ompi/mca/coll/sm/coll_sm_barrier.c @@ -56,7 +56,8 @@ int mca_coll_sm_barrier_intra(struct ompi_communicator_t *comm, int rank, buffer_set; mca_coll_sm_comm_t *data; uint32_t i, num_children; - volatile uint32_t *me_in, *me_out, *parent, *children = NULL; + volatile uint32_t *me_in, *me_out, *children = NULL; + opal_atomic_uint32_t *parent; int uint_control_size; mca_coll_sm_module_t *sm_module = (mca_coll_sm_module_t*) module; diff --git a/ompi/mca/coll/sm/coll_sm_module.c b/ompi/mca/coll/sm/coll_sm_module.c index 8922a70eaf..d1804a0061 100644 --- a/ompi/mca/coll/sm/coll_sm_module.c +++ b/ompi/mca/coll/sm/coll_sm_module.c @@ -372,7 +372,7 @@ int ompi_coll_sm_lazy_enable(mca_coll_base_module_t *module, data->mcb_barrier_control_me = (uint32_t*) (base + (rank * control_size * num_barrier_buffers * 2)); if (data->mcb_tree[rank].mcstn_parent) { - data->mcb_barrier_control_parent = (uint32_t*) + data->mcb_barrier_control_parent = (opal_atomic_uint32_t*) (base + (data->mcb_tree[rank].mcstn_parent->mcstn_id * control_size * num_barrier_buffers * 2)); diff --git a/ompi/mca/common/monitoring/common_monitoring.c b/ompi/mca/common/monitoring/common_monitoring.c index e521ca5641..8598155156 100644 --- a/ompi/mca/common/monitoring/common_monitoring.c +++ b/ompi/mca/common/monitoring/common_monitoring.c @@ -7,7 +7,7 @@ * Copyright (c) 2015 Bull SAS. All rights reserved. * Copyright (c) 2016-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2017-2018 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -34,7 +34,7 @@ /*** Monitoring specific variables ***/ /* Keep tracks of how many components are currently using the common part */ -static int32_t mca_common_monitoring_hold = 0; +static opal_atomic_int32_t mca_common_monitoring_hold = 0; /* Output parameters */ int mca_common_monitoring_output_stream_id = -1; static opal_output_stream_t mca_common_monitoring_output_stream_obj = { @@ -61,18 +61,18 @@ static char* mca_common_monitoring_initial_filename = ""; static char* mca_common_monitoring_current_filename = NULL; /* array for stroring monitoring data*/ -static size_t* pml_data = NULL; -static size_t* pml_count = NULL; -static size_t* filtered_pml_data = NULL; -static size_t* filtered_pml_count = NULL; -static size_t* osc_data_s = NULL; -static size_t* osc_count_s = NULL; -static size_t* osc_data_r = NULL; -static size_t* osc_count_r = NULL; -static size_t* coll_data = NULL; -static size_t* coll_count = NULL; +static opal_atomic_size_t* pml_data = NULL; +static opal_atomic_size_t* pml_count = NULL; +static opal_atomic_size_t* filtered_pml_data = NULL; +static opal_atomic_size_t* filtered_pml_count = NULL; +static opal_atomic_size_t* osc_data_s = NULL; +static opal_atomic_size_t* osc_count_s = NULL; +static opal_atomic_size_t* osc_data_r = NULL; +static opal_atomic_size_t* osc_count_r = NULL; +static opal_atomic_size_t* coll_data = NULL; +static opal_atomic_size_t* coll_count = NULL; -static size_t* size_histogram = NULL; +static opal_atomic_size_t* size_histogram = NULL; static const int max_size_histogram = 66; static double log10_2 = 0.; @@ -241,7 +241,7 @@ void mca_common_monitoring_finalize( void ) opal_output_close(mca_common_monitoring_output_stream_id); free(mca_common_monitoring_output_stream_obj.lds_prefix); /* Free internal data structure */ - free(pml_data); /* a single allocation */ + free((void *) pml_data); /* a single allocation */ opal_hash_table_remove_all( common_monitoring_translation_ht ); OBJ_RELEASE(common_monitoring_translation_ht); mca_common_monitoring_coll_finalize(); @@ -446,7 +446,7 @@ int mca_common_monitoring_add_procs(struct ompi_proc_t **procs, if( NULL == pml_data ) { int array_size = (10 + max_size_histogram) * nprocs_world; - pml_data = (size_t*)calloc(array_size, sizeof(size_t)); + pml_data = (opal_atomic_size_t*)calloc(array_size, sizeof(size_t)); pml_count = pml_data + nprocs_world; filtered_pml_data = pml_count + nprocs_world; filtered_pml_count = filtered_pml_data + nprocs_world; @@ -493,7 +493,7 @@ int mca_common_monitoring_add_procs(struct ompi_proc_t **procs, static void mca_common_monitoring_reset( void ) { int array_size = (10 + max_size_histogram) * nprocs_world; - memset(pml_data, 0, array_size * sizeof(size_t)); + memset((void *) pml_data, 0, array_size * sizeof(size_t)); mca_common_monitoring_coll_reset(); } diff --git a/ompi/mca/common/monitoring/common_monitoring_coll.c b/ompi/mca/common/monitoring/common_monitoring_coll.c index 571f48070e..4fce29028e 100644 --- a/ompi/mca/common/monitoring/common_monitoring_coll.c +++ b/ompi/mca/common/monitoring/common_monitoring_coll.c @@ -30,12 +30,12 @@ struct mca_monitoring_coll_data_t { int world_rank; int is_released; ompi_communicator_t*p_comm; - size_t o2a_count; - size_t o2a_size; - size_t a2o_count; - size_t a2o_size; - size_t a2a_count; - size_t a2a_size; + opal_atomic_size_t o2a_count; + opal_atomic_size_t o2a_size; + opal_atomic_size_t a2o_count; + opal_atomic_size_t a2o_size; + opal_atomic_size_t a2a_count; + opal_atomic_size_t a2a_size; }; /* Collectives operation monitoring */ diff --git a/ompi/mca/common/ompio/common_ompio_cuda.c b/ompi/mca/common/ompio/common_ompio_cuda.c index e6ace6fe4f..d2ad66a413 100644 --- a/ompi/mca/common/ompio/common_ompio_cuda.c +++ b/ompi/mca/common/ompio/common_ompio_cuda.c @@ -34,7 +34,7 @@ static opal_mutex_t mca_common_ompio_cuda_mutex; /* lock for thread saf static mca_allocator_base_component_t* mca_common_ompio_allocator_component=NULL; static mca_allocator_base_module_t* mca_common_ompio_allocator=NULL; -static int32_t mca_common_ompio_cuda_init = 0; +static opal_atomic_int32_t mca_common_ompio_cuda_init = 0; static int32_t mca_common_ompio_pagesize=4096; static void* mca_common_ompio_cuda_alloc_seg ( void *ctx, size_t *size ); static void mca_common_ompio_cuda_free_seg ( void *ctx, void *buf ); diff --git a/ompi/mca/mtl/portals4/mtl_portals4.h b/ompi/mca/mtl/portals4/mtl_portals4.h index 52b21b9354..70db436b91 100644 --- a/ompi/mca/mtl/portals4/mtl_portals4.h +++ b/ompi/mca/mtl/portals4/mtl_portals4.h @@ -115,12 +115,12 @@ struct mca_mtl_portals4_module_t { opal_mutex_t short_block_mutex; /** number of send-side operations started */ - uint64_t opcount; + opal_atomic_uint64_t opcount; #if OPAL_ENABLE_DEBUG /** number of receive-side operations started. Used only for debugging */ - uint64_t recv_opcount; + opal_atomic_uint64_t recv_opcount; #endif #if OMPI_MTL_PORTALS4_FLOW_CONTROL diff --git a/ompi/mca/mtl/portals4/mtl_portals4_flowctl.c b/ompi/mca/mtl/portals4/mtl_portals4_flowctl.c index 19d3b600b3..f690630d7e 100644 --- a/ompi/mca/mtl/portals4/mtl_portals4_flowctl.c +++ b/ompi/mca/mtl/portals4/mtl_portals4_flowctl.c @@ -1,7 +1,7 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2012 Sandia National Laboratories. All rights reserved. - * Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * diff --git a/ompi/mca/mtl/portals4/mtl_portals4_flowctl.h b/ompi/mca/mtl/portals4/mtl_portals4_flowctl.h index 7cc634b669..ee1fbd9a76 100644 --- a/ompi/mca/mtl/portals4/mtl_portals4_flowctl.h +++ b/ompi/mca/mtl/portals4/mtl_portals4_flowctl.h @@ -36,7 +36,7 @@ OBJ_CLASS_DECLARATION(ompi_mtl_portals4_pending_request_t); struct ompi_mtl_portals4_flowctl_t { int32_t flowctl_active; - int32_t send_slots; + opal_atomic_int32_t send_slots; int32_t max_send_slots; opal_list_t pending_sends; opal_free_list_t pending_fl; @@ -46,7 +46,7 @@ struct ompi_mtl_portals4_flowctl_t { /** Flow control epoch counter. Triggered events should be based on epoch counter. */ - int64_t epoch_counter; + opal_atomic_int64_t epoch_counter; /** Flow control trigger CT. Only has meaning at root. */ ptl_handle_ct_t trigger_ct_h; diff --git a/ompi/mca/mtl/portals4/mtl_portals4_request.h b/ompi/mca/mtl/portals4/mtl_portals4_request.h index c7e3c31e47..ede07b4e50 100644 --- a/ompi/mca/mtl/portals4/mtl_portals4_request.h +++ b/ompi/mca/mtl/portals4/mtl_portals4_request.h @@ -54,8 +54,8 @@ struct ompi_mtl_portals4_isend_request_t { struct ompi_mtl_portals4_pending_request_t *pending; #endif ptl_size_t length; - int32_t pending_get; - uint32_t event_count; + opal_atomic_int32_t pending_get; + opal_atomic_uint32_t event_count; }; typedef struct ompi_mtl_portals4_isend_request_t ompi_mtl_portals4_isend_request_t; @@ -76,7 +76,7 @@ struct ompi_mtl_portals4_recv_request_t { void *delivery_ptr; size_t delivery_len; volatile bool req_started; - int32_t pending_reply; + opal_atomic_int32_t pending_reply; #if OPAL_ENABLE_DEBUG uint64_t opcount; ptl_hdr_data_t hdr_data; diff --git a/ompi/mca/osc/monitoring/osc_monitoring_module.h b/ompi/mca/osc/monitoring/osc_monitoring_module.h index 9c7dfb12e7..e03f81290a 100644 --- a/ompi/mca/osc/monitoring/osc_monitoring_module.h +++ b/ompi/mca/osc/monitoring/osc_monitoring_module.h @@ -50,7 +50,7 @@ OSC_MONITORING_SET_TEMPLATE_FCT_NAME(template) (ompi_osc_base_module_t*module) \ { \ /* Define the ompi_osc_monitoring_module_## template ##_init_done variable */ \ - static int32_t init_done = 0; \ + opal_atomic_int32_t init_done = 0; \ /* Define and set the ompi_osc_monitoring_## template \ * ##_template variable. The functions recorded here are \ * linked to the original functions of the original \ diff --git a/ompi/mca/osc/portals4/osc_portals4.h b/ompi/mca/osc/portals4/osc_portals4.h index 19a3b6e5c5..6b58554489 100644 --- a/ompi/mca/osc/portals4/osc_portals4.h +++ b/ompi/mca/osc/portals4/osc_portals4.h @@ -95,7 +95,7 @@ struct ompi_osc_portals4_module_t { ptl_handle_md_t req_md_h; /* memory descriptor with event completion used by this window */ ptl_handle_me_t data_me_h; /* data match list entry (MB are CID | OSC_PORTALS4_MB_DATA) */ ptl_handle_me_t control_me_h; /* match list entry for control data (node_state_t). Match bits are (CID | OSC_PORTALS4_MB_CONTROL). */ - int64_t opcount; + opal_atomic_int64_t opcount; ptl_match_bits_t match_bits; /* match bits for module. Same as cid for comm in most cases. */ ptl_iovec_t *origin_iovec_list; /* list of memory segments that compose the noncontiguous region */ diff --git a/ompi/mca/osc/portals4/osc_portals4_comm.c b/ompi/mca/osc/portals4/osc_portals4_comm.c index b125f2aee5..07c69699c9 100644 --- a/ompi/mca/osc/portals4/osc_portals4_comm.c +++ b/ompi/mca/osc/portals4/osc_portals4_comm.c @@ -189,7 +189,7 @@ number_of_fragments(ptl_size_t length, ptl_size_t maxlength) /* put in segments no larger than segment_length */ static int -segmentedPut(int64_t *opcount, +segmentedPut(opal_atomic_int64_t *opcount, ptl_handle_md_t md_h, ptl_size_t origin_offset, ptl_size_t put_length, @@ -236,7 +236,7 @@ segmentedPut(int64_t *opcount, /* get in segments no larger than segment_length */ static int -segmentedGet(int64_t *opcount, +segmentedGet(opal_atomic_int64_t *opcount, ptl_handle_md_t md_h, ptl_size_t origin_offset, ptl_size_t get_length, @@ -280,7 +280,7 @@ segmentedGet(int64_t *opcount, /* atomic op in segments no larger than segment_length */ static int -segmentedAtomic(int64_t *opcount, +segmentedAtomic(opal_atomic_int64_t *opcount, ptl_handle_md_t md_h, ptl_size_t origin_offset, ptl_size_t length, @@ -329,7 +329,7 @@ segmentedAtomic(int64_t *opcount, /* atomic op in segments no larger than segment_length */ static int -segmentedFetchAtomic(int64_t *opcount, +segmentedFetchAtomic(opal_atomic_int64_t *opcount, ptl_handle_md_t result_md_h, ptl_size_t result_offset, ptl_handle_md_t origin_md_h, @@ -381,7 +381,7 @@ segmentedFetchAtomic(int64_t *opcount, /* swap in segments no larger than segment_length */ static int -segmentedSwap(int64_t *opcount, +segmentedSwap(opal_atomic_int64_t *opcount, ptl_handle_md_t result_md_h, ptl_size_t result_offset, ptl_handle_md_t origin_md_h, @@ -1187,7 +1187,7 @@ fetch_atomic_to_iovec(ompi_osc_portals4_module_t *module, /* put in the largest chunks possible given the noncontiguous restriction */ static int -put_to_noncontig(int64_t *opcount, +put_to_noncontig(opal_atomic_int64_t *opcount, ptl_handle_md_t md_h, const void *origin_address, int origin_count, @@ -1521,7 +1521,7 @@ atomic_to_noncontig(ompi_osc_portals4_module_t *module, /* get from a noncontiguous remote to an (non)contiguous local */ static int -get_from_noncontig(int64_t *opcount, +get_from_noncontig(opal_atomic_int64_t *opcount, ptl_handle_md_t md_h, const void *origin_address, int origin_count, diff --git a/ompi/mca/osc/portals4/osc_portals4_request.h b/ompi/mca/osc/portals4/osc_portals4_request.h index ae1be6f44d..42e38566ee 100644 --- a/ompi/mca/osc/portals4/osc_portals4_request.h +++ b/ompi/mca/osc/portals4/osc_portals4_request.h @@ -1,7 +1,7 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2011-2013 Sandia National Laboratories. All rights reserved. - * Copyright (c) 2015 Los Alamos National Security, LLC. All rights + * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -18,7 +18,7 @@ struct ompi_osc_portals4_request_t { ompi_request_t super; int32_t ops_expected; - volatile int32_t ops_committed; + opal_atomic_int32_t ops_committed; }; typedef struct ompi_osc_portals4_request_t ompi_osc_portals4_request_t; diff --git a/ompi/mca/osc/pt2pt/osc_pt2pt.h b/ompi/mca/osc/pt2pt/osc_pt2pt.h index 4b1a423ded..85bad73846 100644 --- a/ompi/mca/osc/pt2pt/osc_pt2pt.h +++ b/ompi/mca/osc/pt2pt/osc_pt2pt.h @@ -8,7 +8,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2007-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2007-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved. @@ -110,7 +110,7 @@ struct ompi_osc_pt2pt_peer_t { int rank; /** pointer to the current send fragment for each outgoing target */ - struct ompi_osc_pt2pt_frag_t *active_frag; + opal_atomic_intptr_t active_frag; /** lock for this peer */ opal_mutex_t lock; @@ -119,10 +119,10 @@ struct ompi_osc_pt2pt_peer_t { opal_list_t queued_frags; /** number of fragments incomming (negative - expected, positive - unsynchronized) */ - volatile int32_t passive_incoming_frag_count; + opal_atomic_int32_t passive_incoming_frag_count; /** peer flags */ - volatile int32_t flags; + opal_atomic_int32_t flags; }; typedef struct ompi_osc_pt2pt_peer_t ompi_osc_pt2pt_peer_t; @@ -208,16 +208,16 @@ struct ompi_osc_pt2pt_module_t { /** Nmber of communication fragments started for this epoch, by peer. Not in peer data to make fence more manageable. */ - uint32_t *epoch_outgoing_frag_count; + opal_atomic_uint32_t *epoch_outgoing_frag_count; /** cyclic counter for a unique tage for long messages. */ - volatile uint32_t tag_counter; + opal_atomic_uint32_t tag_counter; /** number of outgoing fragments still to be completed */ - volatile int32_t outgoing_frag_count; + opal_atomic_int32_t outgoing_frag_count; /** number of incoming fragments */ - volatile int32_t active_incoming_frag_count; + opal_atomic_int32_t active_incoming_frag_count; /** Number of targets locked/being locked */ unsigned int passive_target_access_epoch; @@ -230,13 +230,13 @@ struct ompi_osc_pt2pt_module_t { /** Number of "count" messages from the remote complete group we've received */ - volatile int32_t num_complete_msgs; + opal_atomic_int32_t num_complete_msgs; /* ********************* LOCK data ************************ */ /** Status of the local window lock. One of 0 (unlocked), MPI_LOCK_EXCLUSIVE, or MPI_LOCK_SHARED. */ - int32_t lock_status; + opal_atomic_int32_t lock_status; /** lock for locks_pending list */ opal_mutex_t locks_pending_lock; @@ -526,7 +526,7 @@ static inline void mark_incoming_completion (ompi_osc_pt2pt_module_t *module, in OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "mark_incoming_completion marking passive incoming complete. module %p, source = %d, count = %d", (void *) module, source, (int) peer->passive_incoming_frag_count + 1)); - new_value = OPAL_THREAD_ADD_FETCH32((int32_t *) &peer->passive_incoming_frag_count, 1); + new_value = OPAL_THREAD_ADD_FETCH32((opal_atomic_int32_t *) &peer->passive_incoming_frag_count, 1); if (0 == new_value) { OPAL_THREAD_LOCK(&module->lock); opal_condition_broadcast(&module->cond); @@ -550,7 +550,7 @@ static inline void mark_incoming_completion (ompi_osc_pt2pt_module_t *module, in */ static inline void mark_outgoing_completion (ompi_osc_pt2pt_module_t *module) { - int32_t new_value = OPAL_THREAD_ADD_FETCH32((int32_t *) &module->outgoing_frag_count, 1); + int32_t new_value = OPAL_THREAD_ADD_FETCH32((opal_atomic_int32_t *) &module->outgoing_frag_count, 1); OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "mark_outgoing_completion: outgoing_frag_count = %d", new_value)); if (new_value >= 0) { @@ -574,12 +574,12 @@ static inline void mark_outgoing_completion (ompi_osc_pt2pt_module_t *module) */ static inline void ompi_osc_signal_outgoing (ompi_osc_pt2pt_module_t *module, int target, int count) { - OPAL_THREAD_ADD_FETCH32((int32_t *) &module->outgoing_frag_count, -count); + OPAL_THREAD_ADD_FETCH32((opal_atomic_int32_t *) &module->outgoing_frag_count, -count); if (MPI_PROC_NULL != target) { OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "ompi_osc_signal_outgoing_passive: target = %d, count = %d, total = %d", target, count, module->epoch_outgoing_frag_count[target] + count)); - OPAL_THREAD_ADD_FETCH32((int32_t *) (module->epoch_outgoing_frag_count + target), count); + OPAL_THREAD_ADD_FETCH32((opal_atomic_int32_t *) (module->epoch_outgoing_frag_count + target), count); } } @@ -717,7 +717,7 @@ static inline int get_tag(ompi_osc_pt2pt_module_t *module) /* the LSB of the tag is used be the receiver to determine if the message is a passive or active target (ie, where to mark completion). */ - int32_t tmp = OPAL_THREAD_ADD_FETCH32((volatile int32_t *) &module->tag_counter, 4); + int32_t tmp = OPAL_THREAD_ADD_FETCH32((opal_atomic_int32_t *) &module->tag_counter, 4); return (tmp & OSC_PT2PT_FRAG_MASK) | !!(module->passive_target_access_epoch); } diff --git a/ompi/mca/osc/pt2pt/osc_pt2pt_active_target.c b/ompi/mca/osc/pt2pt/osc_pt2pt_active_target.c index 33df9440a6..59831842bc 100644 --- a/ompi/mca/osc/pt2pt/osc_pt2pt_active_target.c +++ b/ompi/mca/osc/pt2pt/osc_pt2pt_active_target.c @@ -8,7 +8,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2007-2016 Los Alamos National Security, LLC. All rights + * Copyright (c) 2007-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2010-2016 IBM Corporation. All rights reserved. * Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved. @@ -166,17 +166,16 @@ int ompi_osc_pt2pt_fence(int assert, ompi_win_t *win) "osc pt2pt: fence done sending")); /* find out how much data everyone is going to send us. */ - ret = module->comm->c_coll->coll_reduce_scatter_block (module->epoch_outgoing_frag_count, - &incoming_reqs, 1, MPI_UINT32_T, - MPI_SUM, module->comm, - module->comm->c_coll->coll_reduce_scatter_block_module); + ret = module->comm->c_coll->coll_reduce_scatter_block ((void *) module->epoch_outgoing_frag_count, + &incoming_reqs, 1, MPI_UINT32_T, + MPI_SUM, module->comm, + module->comm->c_coll->coll_reduce_scatter_block_module); if (OMPI_SUCCESS != ret) { return ret; } OPAL_THREAD_LOCK(&module->lock); - bzero(module->epoch_outgoing_frag_count, - sizeof(uint32_t) * ompi_comm_size(module->comm)); + bzero ((void *) module->epoch_outgoing_frag_count, sizeof(uint32_t) * ompi_comm_size(module->comm)); OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "osc pt2pt: fence expects %d requests", @@ -366,8 +365,11 @@ int ompi_osc_pt2pt_complete (ompi_win_t *win) /* XXX -- TODO -- since fragment are always delivered in order we do not need to count anything but long * requests. once that is done this can be removed. */ - if (peer->active_frag && (peer->active_frag->remain_len < sizeof (complete_req))) { - ++complete_req.frag_count; + if (peer->active_frag) { + ompi_osc_pt2pt_frag_t *active_frag = (ompi_osc_pt2pt_frag_t *) peer->active_frag; + if (active_frag->remain_len < sizeof (complete_req)) { + ++complete_req.frag_count; + } } OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, diff --git a/ompi/mca/osc/pt2pt/osc_pt2pt_component.c b/ompi/mca/osc/pt2pt/osc_pt2pt_component.c index 6838f78a56..0fbcd2af32 100644 --- a/ompi/mca/osc/pt2pt/osc_pt2pt_component.c +++ b/ompi/mca/osc/pt2pt/osc_pt2pt_component.c @@ -501,7 +501,7 @@ static void ompi_osc_pt2pt_peer_construct (ompi_osc_pt2pt_peer_t *peer) { OBJ_CONSTRUCT(&peer->queued_frags, opal_list_t); OBJ_CONSTRUCT(&peer->lock, opal_mutex_t); - peer->active_frag = NULL; + peer->active_frag = 0; peer->passive_incoming_frag_count = 0; peer->flags = 0; } diff --git a/ompi/mca/osc/pt2pt/osc_pt2pt_data_move.c b/ompi/mca/osc/pt2pt/osc_pt2pt_data_move.c index 1342c01a69..d09e5311fb 100644 --- a/ompi/mca/osc/pt2pt/osc_pt2pt_data_move.c +++ b/ompi/mca/osc/pt2pt/osc_pt2pt_data_move.c @@ -8,7 +8,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2007-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2007-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2009-2011 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved. @@ -56,7 +56,7 @@ struct osc_pt2pt_accumulate_data_t { int peer; ompi_datatype_t *datatype; ompi_op_t *op; - int request_count; + opal_atomic_int32_t request_count; }; typedef struct osc_pt2pt_accumulate_data_t osc_pt2pt_accumulate_data_t; diff --git a/ompi/mca/osc/pt2pt/osc_pt2pt_frag.c b/ompi/mca/osc/pt2pt/osc_pt2pt_frag.c index c14afeb257..317eff94bc 100644 --- a/ompi/mca/osc/pt2pt/osc_pt2pt_frag.c +++ b/ompi/mca/osc/pt2pt/osc_pt2pt_frag.c @@ -1,7 +1,7 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved. - * Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -65,7 +65,7 @@ int ompi_osc_pt2pt_frag_start (ompi_osc_pt2pt_module_t *module, ompi_osc_pt2pt_peer_t *peer = ompi_osc_pt2pt_peer_lookup (module, frag->target); int ret; - assert(0 == frag->pending && peer->active_frag != frag); + assert(0 == frag->pending && peer->active_frag != (intptr_t) frag); /* we need to signal now that a frag is outgoing to ensure the count sent * with the unlock message is correct */ @@ -93,7 +93,7 @@ int ompi_osc_pt2pt_frag_start (ompi_osc_pt2pt_module_t *module, static int ompi_osc_pt2pt_flush_active_frag (ompi_osc_pt2pt_module_t *module, ompi_osc_pt2pt_peer_t *peer) { - ompi_osc_pt2pt_frag_t *active_frag = peer->active_frag; + ompi_osc_pt2pt_frag_t *active_frag = (ompi_osc_pt2pt_frag_t *) peer->active_frag; int ret = OMPI_SUCCESS; if (NULL == active_frag) { @@ -105,7 +105,7 @@ static int ompi_osc_pt2pt_flush_active_frag (ompi_osc_pt2pt_module_t *module, om "osc pt2pt: flushing active fragment to target %d. pending: %d", active_frag->target, active_frag->pending)); - if (opal_atomic_compare_exchange_strong_ptr (&peer->active_frag, &active_frag, NULL)) { + if (opal_atomic_compare_exchange_strong_ptr (&peer->active_frag, (intptr_t *) &active_frag, 0)) { if (0 != OPAL_THREAD_ADD_FETCH32(&active_frag->pending, -1)) { /* communication going on while synchronizing; this is an rma usage bug */ return OMPI_ERR_RMA_SYNC; diff --git a/ompi/mca/osc/pt2pt/osc_pt2pt_frag.h b/ompi/mca/osc/pt2pt/osc_pt2pt_frag.h index 4ed38930d5..4ef7679f72 100644 --- a/ompi/mca/osc/pt2pt/osc_pt2pt_frag.h +++ b/ompi/mca/osc/pt2pt/osc_pt2pt_frag.h @@ -33,7 +33,7 @@ struct ompi_osc_pt2pt_frag_t { char *top; /* Number of operations which have started writing into the frag, but not yet completed doing so */ - volatile int32_t pending; + opal_atomic_int32_t pending; int32_t pending_long_sends; ompi_osc_pt2pt_frag_header_t *header; ompi_osc_pt2pt_module_t *module; @@ -66,8 +66,8 @@ static inline ompi_osc_pt2pt_frag_t *ompi_osc_pt2pt_frag_alloc_non_buffered (omp ompi_osc_pt2pt_frag_t *curr; /* to ensure ordering flush the buffer on the peer */ - curr = peer->active_frag; - if (NULL != curr && opal_atomic_compare_exchange_strong_ptr (&peer->active_frag, &curr, NULL)) { + curr = (ompi_osc_pt2pt_frag_t *) peer->active_frag; + if (NULL != curr && opal_atomic_compare_exchange_strong_ptr (&peer->active_frag, (intptr_t *) &curr, 0)) { /* If there's something pending, the pending finish will start the buffer. Otherwise, we need to start it now. */ int ret = ompi_osc_pt2pt_frag_finish (module, curr); @@ -131,7 +131,7 @@ static inline int _ompi_osc_pt2pt_frag_alloc (ompi_osc_pt2pt_module_t *module, i OPAL_THREAD_LOCK(&module->lock); if (buffered) { - curr = peer->active_frag; + curr = (ompi_osc_pt2pt_frag_t *) peer->active_frag; if (NULL == curr || curr->remain_len < request_len || (long_send && curr->pending_long_sends == 32)) { curr = ompi_osc_pt2pt_frag_alloc_non_buffered (module, peer, request_len); if (OPAL_UNLIKELY(NULL == curr)) { @@ -140,7 +140,7 @@ static inline int _ompi_osc_pt2pt_frag_alloc (ompi_osc_pt2pt_module_t *module, i } curr->pending_long_sends = long_send; - peer->active_frag = curr; + peer->active_frag = (uintptr_t) curr; } else { OPAL_THREAD_ADD_FETCH32(&curr->header->num_ops, 1); curr->pending_long_sends += long_send; diff --git a/ompi/mca/osc/pt2pt/osc_pt2pt_header.h b/ompi/mca/osc/pt2pt/osc_pt2pt_header.h index f979d9bf61..cec5f50120 100644 --- a/ompi/mca/osc/pt2pt/osc_pt2pt_header.h +++ b/ompi/mca/osc/pt2pt/osc_pt2pt_header.h @@ -8,7 +8,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2007-2015 Los Alamos National Security, LLC. All rights + * Copyright (c) 2007-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. @@ -180,7 +180,7 @@ typedef struct ompi_osc_pt2pt_header_flush_ack_t ompi_osc_pt2pt_header_flush_ack struct ompi_osc_pt2pt_frag_header_t { ompi_osc_pt2pt_header_base_t base; uint32_t source; /* rank in window of source process */ - int32_t num_ops; /* number of operations in this buffer */ + opal_atomic_int32_t num_ops; /* number of operations in this buffer */ uint32_t pad; /* ensure the fragment header is a multiple of 8 bytes */ }; typedef struct ompi_osc_pt2pt_frag_header_t ompi_osc_pt2pt_frag_header_t; diff --git a/ompi/mca/osc/pt2pt/osc_pt2pt_module.c b/ompi/mca/osc/pt2pt/osc_pt2pt_module.c index e637174b90..b13a83f14b 100644 --- a/ompi/mca/osc/pt2pt/osc_pt2pt_module.c +++ b/ompi/mca/osc/pt2pt/osc_pt2pt_module.c @@ -8,7 +8,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2007-2016 Los Alamos National Security, LLC. All rights + * Copyright (c) 2007-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science @@ -104,13 +104,13 @@ int ompi_osc_pt2pt_free(ompi_win_t *win) free (module->recv_frags); } - if (NULL != module->epoch_outgoing_frag_count) free(module->epoch_outgoing_frag_count); + free ((void *) module->epoch_outgoing_frag_count); if (NULL != module->comm) { ompi_comm_free(&module->comm); } - if (NULL != module->free_after) free(module->free_after); + free ((void *) module->free_after); free (module); diff --git a/ompi/mca/osc/pt2pt/osc_pt2pt_passive_target.c b/ompi/mca/osc/pt2pt/osc_pt2pt_passive_target.c index 091757511f..59b585337c 100644 --- a/ompi/mca/osc/pt2pt/osc_pt2pt_passive_target.c +++ b/ompi/mca/osc/pt2pt/osc_pt2pt_passive_target.c @@ -8,7 +8,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2007-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2007-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2010-2016 IBM Corporation. All rights reserved. * Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved. @@ -157,7 +157,7 @@ int ompi_osc_pt2pt_lock_remote (ompi_osc_pt2pt_module_t *module, int target, omp static inline int ompi_osc_pt2pt_unlock_remote (ompi_osc_pt2pt_module_t *module, int target, ompi_osc_pt2pt_sync_t *lock) { - int32_t frag_count = opal_atomic_swap_32 ((int32_t *) module->epoch_outgoing_frag_count + target, -1); + int32_t frag_count = opal_atomic_swap_32 ((opal_atomic_int32_t *) module->epoch_outgoing_frag_count + target, -1); ompi_osc_pt2pt_peer_t *peer = ompi_osc_pt2pt_peer_lookup (module, target); int lock_type = lock->sync.lock.type; ompi_osc_pt2pt_header_unlock_t unlock_req; @@ -178,10 +178,13 @@ static inline int ompi_osc_pt2pt_unlock_remote (ompi_osc_pt2pt_module_t *module, unlock_req.lock_ptr = (uint64_t) (uintptr_t) lock; OSC_PT2PT_HTON(&unlock_req, module, target); - if (peer->active_frag && peer->active_frag->remain_len < sizeof (unlock_req)) { - /* the peer should expect one more packet */ - ++unlock_req.frag_count; - --module->epoch_outgoing_frag_count[target]; + if (peer->active_frag) { + ompi_osc_pt2pt_frag_t *active_frag = (ompi_osc_pt2pt_frag_t *) peer->active_frag; + if (active_frag->remain_len < sizeof (unlock_req)) { + /* the peer should expect one more packet */ + ++unlock_req.frag_count; + --module->epoch_outgoing_frag_count[target]; + } } OPAL_OUTPUT_VERBOSE((25, ompi_osc_base_framework.framework_output, @@ -204,7 +207,7 @@ static inline int ompi_osc_pt2pt_flush_remote (ompi_osc_pt2pt_module_t *module, { ompi_osc_pt2pt_peer_t *peer = ompi_osc_pt2pt_peer_lookup (module, target); ompi_osc_pt2pt_header_flush_t flush_req; - int32_t frag_count = opal_atomic_swap_32 ((int32_t *) module->epoch_outgoing_frag_count + target, -1); + int32_t frag_count = opal_atomic_swap_32 ((opal_atomic_int32_t *) module->epoch_outgoing_frag_count + target, -1); int ret; (void) OPAL_THREAD_ADD_FETCH32(&lock->sync_expected, 1); @@ -218,10 +221,13 @@ static inline int ompi_osc_pt2pt_flush_remote (ompi_osc_pt2pt_module_t *module, /* XXX -- TODO -- since fragment are always delivered in order we do not need to count anything but long * requests. once that is done this can be removed. */ - if (peer->active_frag && (peer->active_frag->remain_len < sizeof (flush_req))) { - /* the peer should expect one more packet */ - ++flush_req.frag_count; - --module->epoch_outgoing_frag_count[target]; + if (peer->active_frag) { + ompi_osc_pt2pt_frag_t *active_frag = (ompi_osc_pt2pt_frag_t *) peer->active_frag; + if (active_frag->remain_len < sizeof (flush_req)) { + /* the peer should expect one more packet */ + ++flush_req.frag_count; + --module->epoch_outgoing_frag_count[target]; + } } OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "flushing to target %d, frag_count: %d", diff --git a/ompi/mca/osc/pt2pt/osc_pt2pt_request.h b/ompi/mca/osc/pt2pt/osc_pt2pt_request.h index dee5c86892..09446de1b3 100644 --- a/ompi/mca/osc/pt2pt/osc_pt2pt_request.h +++ b/ompi/mca/osc/pt2pt/osc_pt2pt_request.h @@ -28,7 +28,7 @@ struct ompi_osc_pt2pt_request_t { int origin_count; struct ompi_datatype_t *origin_dt; ompi_osc_pt2pt_module_t* module; - int32_t outstanding_requests; + opal_atomic_int32_t outstanding_requests; bool internal; }; typedef struct ompi_osc_pt2pt_request_t ompi_osc_pt2pt_request_t; diff --git a/ompi/mca/osc/pt2pt/osc_pt2pt_sync.h b/ompi/mca/osc/pt2pt/osc_pt2pt_sync.h index fe359bf6cf..cf5da61bf8 100644 --- a/ompi/mca/osc/pt2pt/osc_pt2pt_sync.h +++ b/ompi/mca/osc/pt2pt/osc_pt2pt_sync.h @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights + * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -74,7 +74,7 @@ struct ompi_osc_pt2pt_sync_t { int num_peers; /** number of synchronization messages expected */ - volatile int32_t sync_expected; + opal_atomic_int32_t sync_expected; /** eager sends are active to all peers in this access epoch */ volatile bool eager_send_active; diff --git a/ompi/mca/osc/rdma/osc_rdma.h b/ompi/mca/osc/rdma/osc_rdma.h index b3743f261e..6f1232c532 100644 --- a/ompi/mca/osc/rdma/osc_rdma.h +++ b/ompi/mca/osc/rdma/osc_rdma.h @@ -265,7 +265,7 @@ struct ompi_osc_rdma_module_t { unsigned long get_retry_count; /** outstanding atomic operations */ - volatile int32_t pending_ops; + opal_atomic_int32_t pending_ops; }; typedef struct ompi_osc_rdma_module_t ompi_osc_rdma_module_t; OMPI_MODULE_DECLSPEC extern ompi_osc_rdma_component_t mca_osc_rdma_component; diff --git a/ompi/mca/osc/rdma/osc_rdma_active_target.c b/ompi/mca/osc/rdma/osc_rdma_active_target.c index f677394da0..80c31fc00b 100644 --- a/ompi/mca/osc/rdma/osc_rdma_active_target.c +++ b/ompi/mca/osc/rdma/osc_rdma_active_target.c @@ -259,7 +259,7 @@ static int ompi_osc_rdma_post_peer (ompi_osc_rdma_module_t *module, ompi_osc_rdm return ret; } } else { - post_index = ompi_osc_rdma_counter_add ((osc_rdma_counter_t *) (intptr_t) target, 1) - 1; + post_index = ompi_osc_rdma_counter_add ((osc_rdma_atomic_counter_t *) (intptr_t) target, 1) - 1; } post_index &= OMPI_OSC_RDMA_POST_PEER_MAX - 1; @@ -279,7 +279,7 @@ static int ompi_osc_rdma_post_peer (ompi_osc_rdma_module_t *module, ompi_osc_rdm return ret; } } else { - result = !ompi_osc_rdma_lock_compare_exchange ((osc_rdma_counter_t *) target, &_tmp_value, + result = !ompi_osc_rdma_lock_compare_exchange ((osc_rdma_atomic_counter_t *) target, &_tmp_value, 1 + (osc_rdma_counter_t) my_rank); } @@ -491,7 +491,7 @@ int ompi_osc_rdma_complete_atomic (ompi_win_t *win) ret = ompi_osc_rdma_lock_btl_op (module, peer, target, MCA_BTL_ATOMIC_ADD, 1, true); assert (OMPI_SUCCESS == ret); } else { - (void) ompi_osc_rdma_counter_add ((osc_rdma_counter_t *) target, 1); + (void) ompi_osc_rdma_counter_add ((osc_rdma_atomic_counter_t *) target, 1); } } diff --git a/ompi/mca/osc/rdma/osc_rdma_frag.h b/ompi/mca/osc/rdma/osc_rdma_frag.h index beecce93be..f174f48e7f 100644 --- a/ompi/mca/osc/rdma/osc_rdma_frag.h +++ b/ompi/mca/osc/rdma/osc_rdma_frag.h @@ -72,7 +72,7 @@ static inline int ompi_osc_rdma_frag_alloc (ompi_osc_rdma_module_t *module, size } } - if (!opal_atomic_compare_exchange_strong_ptr (&module->rdma_frag, &(void *){NULL}, curr)) { + if (!opal_atomic_compare_exchange_strong_ptr ((opal_atomic_intptr_t *) &module->rdma_frag, &(intptr_t){0}, (intptr_t) curr)) { ompi_osc_rdma_deregister (module, curr->handle); curr->handle = NULL; diff --git a/ompi/mca/osc/rdma/osc_rdma_lock.h b/ompi/mca/osc/rdma/osc_rdma_lock.h index 7af4d703f6..f90341b94a 100644 --- a/ompi/mca/osc/rdma/osc_rdma_lock.h +++ b/ompi/mca/osc/rdma/osc_rdma_lock.h @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -15,13 +15,13 @@ #include "osc_rdma_types.h" #include "osc_rdma_frag.h" -static inline int ompi_osc_rdma_trylock_local (volatile ompi_osc_rdma_lock_t *lock) +static inline int ompi_osc_rdma_trylock_local (ompi_osc_rdma_atomic_lock_t *lock) { ompi_osc_rdma_lock_t _tmp_value = 0; return !ompi_osc_rdma_lock_compare_exchange (lock, &_tmp_value, OMPI_OSC_RDMA_LOCK_EXCLUSIVE); } -static inline void ompi_osc_rdma_unlock_local (volatile ompi_osc_rdma_lock_t *lock) +static inline void ompi_osc_rdma_unlock_local (ompi_osc_rdma_atomic_lock_t *lock) { (void) ompi_osc_rdma_lock_add (lock, -OMPI_OSC_RDMA_LOCK_EXCLUSIVE); } @@ -266,7 +266,7 @@ static inline int ompi_osc_rdma_lock_release_shared (ompi_osc_rdma_module_t *mod return ompi_osc_rdma_lock_btl_op (module, peer, lock, MCA_BTL_ATOMIC_ADD, value, false); } - (void) ompi_osc_rdma_lock_add ((volatile ompi_osc_rdma_lock_t *) lock, value); + (void) ompi_osc_rdma_lock_add ((ompi_osc_rdma_atomic_lock_t *) lock, value); return OMPI_SUCCESS; } @@ -320,14 +320,14 @@ static inline int ompi_osc_rdma_lock_acquire_shared (ompi_osc_rdma_module_t *mod } while (1); } else { do { - lock_state = ompi_osc_rdma_lock_add ((volatile ompi_osc_rdma_lock_t *) lock, value); + lock_state = ompi_osc_rdma_lock_add ((ompi_osc_rdma_atomic_lock_t *) lock, value); OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_DEBUG, "local shared lock incremented. old value 0x%lx", (unsigned long) lock_state); if (!(lock_state & check)) { break; } - (void) ompi_osc_rdma_lock_add ((volatile ompi_osc_rdma_lock_t *) lock, -value); + (void) ompi_osc_rdma_lock_add ((ompi_osc_rdma_atomic_lock_t *) lock, -value); ompi_osc_rdma_progress (module); } while (1); } @@ -378,7 +378,7 @@ static inline int ompi_osc_rdma_lock_try_acquire_exclusive (ompi_osc_rdma_module return lock_state != 0; } - return ompi_osc_rdma_trylock_local ((int64_t *)(intptr_t) lock); + return ompi_osc_rdma_trylock_local ((ompi_osc_rdma_atomic_lock_t *)(intptr_t) lock); } /** @@ -431,7 +431,7 @@ static inline int ompi_osc_rdma_lock_release_exclusive (ompi_osc_rdma_module_t * abort (); } } else { - ompi_osc_rdma_unlock_local ((volatile ompi_osc_rdma_lock_t *)(intptr_t) lock); + ompi_osc_rdma_unlock_local ((ompi_osc_rdma_atomic_lock_t *)(intptr_t) lock); } OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_DEBUG, "exclusive lock released"); diff --git a/ompi/mca/osc/rdma/osc_rdma_peer.h b/ompi/mca/osc/rdma/osc_rdma_peer.h index a0db4c4a7f..087731aa46 100644 --- a/ompi/mca/osc/rdma/osc_rdma_peer.h +++ b/ompi/mca/osc/rdma/osc_rdma_peer.h @@ -43,7 +43,7 @@ struct ompi_osc_rdma_peer_t { int rank; /** peer flags */ - volatile int32_t flags; + opal_atomic_int32_t flags; }; typedef struct ompi_osc_rdma_peer_t ompi_osc_rdma_peer_t; diff --git a/ompi/mca/osc/rdma/osc_rdma_request.h b/ompi/mca/osc/rdma/osc_rdma_request.h index ad052e172c..089a5d8eb8 100644 --- a/ompi/mca/osc/rdma/osc_rdma_request.h +++ b/ompi/mca/osc/rdma/osc_rdma_request.h @@ -40,7 +40,7 @@ struct ompi_osc_rdma_request_t { void *origin_addr; ompi_osc_rdma_module_t *module; - volatile int32_t outstanding_requests; + opal_atomic_int32_t outstanding_requests; bool internal; ptrdiff_t offset; diff --git a/ompi/mca/osc/rdma/osc_rdma_sync.h b/ompi/mca/osc/rdma/osc_rdma_sync.h index 202bf79265..ce0daa5db9 100644 --- a/ompi/mca/osc/rdma/osc_rdma_sync.h +++ b/ompi/mca/osc/rdma/osc_rdma_sync.h @@ -34,7 +34,7 @@ typedef enum ompi_osc_rdma_sync_type_t ompi_osc_rdma_sync_type_t; struct ompi_osc_rdma_module_t; struct ompi_osc_rdma_sync_aligned_counter_t { - volatile osc_rdma_counter_t counter; + osc_rdma_atomic_counter_t counter; /* pad out to next cache line */ uint64_t padding[7]; }; diff --git a/ompi/mca/osc/rdma/osc_rdma_types.h b/ompi/mca/osc/rdma/osc_rdma_types.h index 4acb40154d..25fcc1dc4e 100644 --- a/ompi/mca/osc/rdma/osc_rdma_types.h +++ b/ompi/mca/osc/rdma/osc_rdma_types.h @@ -24,6 +24,7 @@ struct ompi_osc_rdma_peer_t; typedef int64_t osc_rdma_base_t; typedef int64_t osc_rdma_size_t; typedef int64_t osc_rdma_counter_t; +typedef opal_atomic_int64_t osc_rdma_atomic_counter_t; #define ompi_osc_rdma_counter_add opal_atomic_add_fetch_64 @@ -32,6 +33,7 @@ typedef int64_t osc_rdma_counter_t; typedef int32_t osc_rdma_base_t; typedef int32_t osc_rdma_size_t; typedef int32_t osc_rdma_counter_t; +typedef opal_atomic_int32_t osc_rdma_atomic_counter_t; #define ompi_osc_rdma_counter_add opal_atomic_add_fetch_32 @@ -42,8 +44,9 @@ typedef int32_t osc_rdma_counter_t; #define OMPI_OSC_RDMA_LOCK_EXCLUSIVE 0x8000000000000000l typedef int64_t ompi_osc_rdma_lock_t; +typedef opal_atomic_int64_t ompi_osc_rdma_atomic_lock_t; -static inline int64_t ompi_osc_rdma_lock_add (volatile int64_t *p, int64_t value) +static inline int64_t ompi_osc_rdma_lock_add (opal_atomic_int64_t *p, int64_t value) { int64_t new; @@ -54,7 +57,7 @@ static inline int64_t ompi_osc_rdma_lock_add (volatile int64_t *p, int64_t value return new; } -static inline int ompi_osc_rdma_lock_compare_exchange (volatile int64_t *p, int64_t *comp, int64_t value) +static inline int ompi_osc_rdma_lock_compare_exchange (opal_atomic_int64_t *p, int64_t *comp, int64_t value) { int ret; @@ -70,8 +73,9 @@ static inline int ompi_osc_rdma_lock_compare_exchange (volatile int64_t *p, int6 #define OMPI_OSC_RDMA_LOCK_EXCLUSIVE 0x80000000l typedef int32_t ompi_osc_rdma_lock_t; +typedef opal_atomic_int32_t ompi_osc_rdma_atomic_lock_t; -static inline int32_t ompi_osc_rdma_lock_add (volatile int32_t *p, int32_t value) +static inline int32_t ompi_osc_rdma_lock_add (opal_atomic_int32_t *p, int32_t value) { int32_t new; @@ -83,7 +87,7 @@ static inline int32_t ompi_osc_rdma_lock_add (volatile int32_t *p, int32_t value return new; } -static inline int ompi_osc_rdma_lock_compare_exchange (volatile int32_t *p, int32_t *comp, int32_t value) +static inline int ompi_osc_rdma_lock_compare_exchange (opal_atomic_int32_t *p, int32_t *comp, int32_t value) { int ret; @@ -157,7 +161,7 @@ struct ompi_osc_rdma_state_t { /** post buffers */ osc_rdma_counter_t post_peers[OMPI_OSC_RDMA_POST_PEER_MAX]; /** counter for number of post messages received */ - osc_rdma_counter_t num_post_msgs; + osc_rdma_atomic_counter_t num_post_msgs; /** counter for number of complete messages received */ osc_rdma_counter_t num_complete_msgs; /** lock for the region state to ensure consistency */ @@ -195,11 +199,11 @@ struct ompi_osc_rdma_frag_t { opal_free_list_item_t super; /* Number of operations which have started writing into the frag, but not yet completed doing so */ - volatile int32_t pending; + opal_atomic_int32_t pending; #if OPAL_HAVE_ATOMIC_MATH_64 - volatile int64_t curr_index; + opal_atomic_int64_t curr_index; #else - volatile int32_t curr_index; + opal_atomic_int32_t curr_index; #endif struct ompi_osc_rdma_module_t *module; diff --git a/ompi/mca/osc/sm/osc_sm.h b/ompi/mca/osc/sm/osc_sm.h index f0917b54cb..01e9698205 100644 --- a/ompi/mca/osc/sm/osc_sm.h +++ b/ompi/mca/osc/sm/osc_sm.h @@ -21,12 +21,14 @@ #if OPAL_HAVE_ATOMIC_MATH_64 typedef uint64_t osc_sm_post_type_t; +typedef opal_atomic_uint64_t osc_sm_post_atomic_type_t; #define OSC_SM_POST_BITS 6 #define OSC_SM_POST_MASK 0x3f #else typedef uint32_t osc_sm_post_type_t; +typedef opal_atomic_uint32_t osc_sm_post_atomic_type_t; #define OSC_SM_POST_BITS 5 #define OSC_SM_POST_MASK 0x1f @@ -53,7 +55,7 @@ struct ompi_osc_sm_lock_t { typedef struct ompi_osc_sm_lock_t ompi_osc_sm_lock_t; struct ompi_osc_sm_node_state_t { - int32_t complete_count; + opal_atomic_int32_t complete_count; ompi_osc_sm_lock_t lock; opal_atomic_lock_t accumulate_lock; }; @@ -98,7 +100,7 @@ struct ompi_osc_sm_module_t { ompi_osc_sm_node_state_t *my_node_state; ompi_osc_sm_node_state_t *node_states; - osc_sm_post_type_t ** volatile posts; + osc_sm_post_atomic_type_t **posts; opal_mutex_t lock; }; diff --git a/ompi/mca/osc/sm/osc_sm_active_target.c b/ompi/mca/osc/sm/osc_sm_active_target.c index ab0f73f87c..237d2b8407 100644 --- a/ompi/mca/osc/sm/osc_sm_active_target.c +++ b/ompi/mca/osc/sm/osc_sm_active_target.c @@ -162,9 +162,9 @@ ompi_osc_sm_start(struct ompi_group_t *group, opal_atomic_rmb (); #if OPAL_HAVE_ATOMIC_MATH_64 - (void) opal_atomic_fetch_xor_64 ((volatile int64_t *) module->posts[my_rank] + rank_byte, rank_bit); + (void) opal_atomic_fetch_xor_64 ((opal_atomic_int64_t *) module->posts[my_rank] + rank_byte, rank_bit); #else - (void) opal_atomic_fetch_xor_32 ((volatile int32_t *) module->posts[my_rank] + rank_byte, rank_bit); + (void) opal_atomic_fetch_xor_32 ((opal_atomic_int32_t *) module->posts[my_rank] + rank_byte, rank_bit); #endif } @@ -188,7 +188,7 @@ ompi_osc_sm_complete(struct ompi_win_t *win) opal_atomic_mb(); group = module->start_group; - if (NULL == group || !OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&module->start_group, &group, NULL)) { + if (NULL == group || !OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR((opal_atomic_intptr_t *) &module->start_group, (opal_atomic_intptr_t *) &group, 0)) { return OMPI_ERR_RMA_SYNC; } @@ -247,7 +247,11 @@ ompi_osc_sm_post(struct ompi_group_t *group, gsize = ompi_group_size(module->post_group); for (int i = 0 ; i < gsize ; ++i) { - opal_atomic_add ((volatile osc_sm_post_type_t *) module->posts[ranks[i]] + my_byte, my_bit); +#if OPAL_HAVE_ATOMIC_MATH_64 + (void) opal_atomic_fetch_add_64 ((opal_atomic_int64_t *) module->posts[ranks[i]] + my_byte, my_bit); +#else + (void) opal_atomic_fetch_add_32 ((opal_atomic_int32_t *) module->posts[ranks[i]] + my_byte, my_bit); +#endif } opal_atomic_wmb (); diff --git a/ompi/mca/osc/sm/osc_sm_component.c b/ompi/mca/osc/sm/osc_sm_component.c index de891e71a1..1006291584 100644 --- a/ompi/mca/osc/sm/osc_sm_component.c +++ b/ompi/mca/osc/sm/osc_sm_component.c @@ -236,7 +236,7 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit if (NULL == module->node_states) return OMPI_ERR_TEMP_OUT_OF_RESOURCE; module->posts = calloc (1, sizeof(module->posts[0]) + sizeof (module->posts[0][0])); if (NULL == module->posts) return OMPI_ERR_TEMP_OUT_OF_RESOURCE; - module->posts[0] = (osc_sm_post_type_t *) (module->posts + 1); + module->posts[0] = (osc_sm_post_atomic_type_t *) (module->posts + 1); } else { unsigned long total, *rbuf; int i, flag; @@ -328,7 +328,7 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit if (NULL == module->posts) return OMPI_ERR_TEMP_OUT_OF_RESOURCE; /* set module->posts[0] first to ensure 64-bit alignment */ - module->posts[0] = (osc_sm_post_type_t *) (module->segment_base); + module->posts[0] = (osc_sm_post_atomic_type_t *) (module->segment_base); module->global_state = (ompi_osc_sm_global_state_t *) (module->posts[0] + comm_size * post_size); module->node_states = (ompi_osc_sm_node_state_t *) (module->global_state + 1); diff --git a/ompi/mca/osc/sm/osc_sm_passive_target.c b/ompi/mca/osc/sm/osc_sm_passive_target.c index a3388b776a..8c27561e54 100644 --- a/ompi/mca/osc/sm/osc_sm_passive_target.c +++ b/ompi/mca/osc/sm/osc_sm_passive_target.c @@ -28,7 +28,7 @@ lk_fetch_add32(ompi_osc_sm_module_t *module, { /* opal_atomic_add_fetch_32 is an add then fetch so delta needs to be subtracted out to get the * old value */ - return opal_atomic_add_fetch_32((int32_t*) ((char*) &module->node_states[target].lock + offset), + return opal_atomic_add_fetch_32((opal_atomic_int32_t *) ((char*) &module->node_states[target].lock + offset), delta) - delta; } @@ -39,7 +39,7 @@ lk_add32(ompi_osc_sm_module_t *module, size_t offset, uint32_t delta) { - opal_atomic_add_fetch_32((int32_t*) ((char*) &module->node_states[target].lock + offset), + opal_atomic_add_fetch_32((opal_atomic_int32_t *) ((char*) &module->node_states[target].lock + offset), delta); } diff --git a/ompi/mca/pml/base/pml_base_bsend.c b/ompi/mca/pml/base/pml_base_bsend.c index ef6be82599..4da15522b8 100644 --- a/ompi/mca/pml/base/pml_base_bsend.c +++ b/ompi/mca/pml/base/pml_base_bsend.c @@ -51,7 +51,7 @@ static size_t mca_pml_bsend_size; /* adjusted size of user buffe static size_t mca_pml_bsend_count; /* number of outstanding requests */ static size_t mca_pml_bsend_pagesz; /* mmap page size */ static int mca_pml_bsend_pagebits; /* number of bits in pagesz */ -static int32_t mca_pml_bsend_init = 0; +static opal_atomic_int32_t mca_pml_bsend_init = 0; /* defined in pml_base_open.c */ extern char *ompi_pml_base_bsend_allocator_name; diff --git a/ompi/mca/pml/ob1/pml_ob1_comm.h b/ompi/mca/pml/ob1/pml_ob1_comm.h index 6366649678..780c9a44fc 100644 --- a/ompi/mca/pml/ob1/pml_ob1_comm.h +++ b/ompi/mca/pml/ob1/pml_ob1_comm.h @@ -44,7 +44,7 @@ struct mca_pml_ob1_comm_proc_t { opal_object_t super; struct ompi_proc_t* ompi_proc; uint16_t expected_sequence; /**< send message sequence number - receiver side */ - volatile int32_t send_sequence; /**< send side sequence number */ + opal_atomic_int32_t send_sequence; /**< send side sequence number */ struct mca_pml_ob1_recv_frag_t* frags_cant_match; /**< out-of-order fragment queues */ #if !MCA_PML_OB1_CUSTOM_MATCH opal_list_t specific_receives; /**< queues of unmatched specific receives */ diff --git a/ompi/mca/pml/ob1/pml_ob1_progress.c b/ompi/mca/pml/ob1/pml_ob1_progress.c index e1f84e796b..b17f2ee0a6 100644 --- a/ompi/mca/pml/ob1/pml_ob1_progress.c +++ b/ompi/mca/pml/ob1/pml_ob1_progress.c @@ -53,7 +53,7 @@ static inline int mca_pml_ob1_process_pending_cuda_async_copies(void) } #endif /* OPAL_CUDA_SUPPORT */ -static int mca_pml_ob1_progress_needed = 0; +static opal_atomic_int32_t mca_pml_ob1_progress_needed = 0; int mca_pml_ob1_enable_progress(int32_t count) { int32_t progress_count = OPAL_ATOMIC_ADD_FETCH32(&mca_pml_ob1_progress_needed, count); diff --git a/ompi/mca/pml/ob1/pml_ob1_recvreq.h b/ompi/mca/pml/ob1/pml_ob1_recvreq.h index bc37a00abf..2bfe294f59 100644 --- a/ompi/mca/pml/ob1/pml_ob1_recvreq.h +++ b/ompi/mca/pml/ob1/pml_ob1_recvreq.h @@ -41,9 +41,9 @@ BEGIN_C_DECLS struct mca_pml_ob1_recv_request_t { mca_pml_base_recv_request_t req_recv; opal_ptr_t remote_req_send; - int32_t req_lock; - int32_t req_pipeline_depth; - size_t req_bytes_received; /**< amount of data transferred into the user buffer */ + opal_atomic_int32_t req_lock; + opal_atomic_int32_t req_pipeline_depth; + opal_atomic_size_t req_bytes_received; /**< amount of data transferred into the user buffer */ size_t req_bytes_expected; /**< local size of the data as suggested by the user */ size_t req_rdma_offset; size_t req_send_offset; diff --git a/ompi/mca/pml/ob1/pml_ob1_sendreq.h b/ompi/mca/pml/ob1/pml_ob1_sendreq.h index 587740dcda..787f48b554 100644 --- a/ompi/mca/pml/ob1/pml_ob1_sendreq.h +++ b/ompi/mca/pml/ob1/pml_ob1_sendreq.h @@ -46,11 +46,11 @@ struct mca_pml_ob1_send_request_t { mca_pml_base_send_request_t req_send; mca_bml_base_endpoint_t* req_endpoint; opal_ptr_t req_recv; - int32_t req_state; - int32_t req_lock; + opal_atomic_int32_t req_state; + opal_atomic_int32_t req_lock; bool req_throttle_sends; - int32_t req_pipeline_depth; - size_t req_bytes_delivered; + opal_atomic_int32_t req_pipeline_depth; + opal_atomic_size_t req_bytes_delivered; uint32_t req_rdma_cnt; mca_pml_ob1_send_pending_t req_pending; opal_mutex_t req_send_range_lock; diff --git a/ompi/request/grequestx.c b/ompi/request/grequestx.c index acd688eacf..3edde17ee3 100644 --- a/ompi/request/grequestx.c +++ b/ompi/request/grequestx.c @@ -28,7 +28,7 @@ static bool requests_initialized = false; static opal_list_t requests; -static int32_t active_requests = 0; +static opal_atomic_int32_t active_requests = 0; static bool in_progress = false; static opal_mutex_t lock; diff --git a/ompi/runtime/mpiruntime.h b/ompi/runtime/mpiruntime.h index 32f1c13878..a57173158b 100644 --- a/ompi/runtime/mpiruntime.h +++ b/ompi/runtime/mpiruntime.h @@ -51,7 +51,7 @@ struct ompi_predefined_datatype_t; /** Mutex to protect all the _init and _finalize variables */ OMPI_DECLSPEC extern opal_mutex_t ompi_mpi_bootstrap_mutex; /** Did MPI start to initialize? */ -OMPI_DECLSPEC extern volatile int32_t ompi_mpi_state; +OMPI_DECLSPEC extern opal_atomic_int32_t ompi_mpi_state; /** Has the RTE been initialized? */ OMPI_DECLSPEC extern volatile bool ompi_rte_initialized; diff --git a/ompi/runtime/ompi_mpi_init.c b/ompi/runtime/ompi_mpi_init.c index 17c0e1f665..de71a33042 100644 --- a/ompi/runtime/ompi_mpi_init.c +++ b/ompi/runtime/ompi_mpi_init.c @@ -130,7 +130,7 @@ const char ompi_version_string[] = OMPI_IDENT_STRING; * Global variables and symbols for the MPI layer */ -volatile int32_t ompi_mpi_state = OMPI_MPI_STATE_NOT_INITIALIZED; +opal_atomic_int32_t ompi_mpi_state = OMPI_MPI_STATE_NOT_INITIALIZED; volatile bool ompi_rte_initialized = false; bool ompi_mpi_thread_multiple = false; diff --git a/ompi/runtime/ompi_spc.c b/ompi/runtime/ompi_spc.c index caee2cda6f..99190a8cc1 100644 --- a/ompi/runtime/ompi_spc.c +++ b/ompi/runtime/ompi_spc.c @@ -456,7 +456,7 @@ void ompi_spc_timer_stop(unsigned int event_id, opal_timer_t *cycles) /* This is denoted unlikely because the counters will often be turned off. */ if( OPAL_UNLIKELY(IS_SPC_BIT_SET(ompi_spc_attached_event, event_id)) ) { *cycles = opal_timer_base_get_cycles() - *cycles; - OPAL_THREAD_ADD_FETCH_SIZE_T(&ompi_spc_events[event_id].value, (ompi_spc_value_t)*cycles); + OPAL_THREAD_ADD_FETCH_SIZE_T(&ompi_spc_events[event_id].value, (size_t) *cycles); } } diff --git a/ompi/runtime/ompi_spc.h b/ompi/runtime/ompi_spc.h index 54851de038..5d040511c3 100644 --- a/ompi/runtime/ompi_spc.h +++ b/ompi/runtime/ompi_spc.h @@ -166,7 +166,7 @@ typedef enum ompi_spc_counters { /* There is currently no support for atomics on long long values so we will default to * size_t for now until support for such atomics is implemented. */ -typedef size_t ompi_spc_value_t; +typedef opal_atomic_size_t ompi_spc_value_t; /* A structure for storing the event data */ typedef struct ompi_spc_s{ diff --git a/opal/class/opal_fifo.c b/opal/class/opal_fifo.c index 6e23e13457..238801d492 100644 --- a/opal/class/opal_fifo.c +++ b/opal/class/opal_fifo.c @@ -10,7 +10,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2014 Los Alamos National Security, LLC. All rights + * Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -32,10 +32,10 @@ static void opal_fifo_construct (opal_fifo_t *fifo) fifo->opal_fifo_ghost.item_free = 0; fifo->opal_fifo_head.data.counter = 0; - fifo->opal_fifo_head.data.item = &fifo->opal_fifo_ghost; + fifo->opal_fifo_head.data.item = (intptr_t) &fifo->opal_fifo_ghost; fifo->opal_fifo_tail.data.counter = 0; - fifo->opal_fifo_tail.data.item = &fifo->opal_fifo_ghost; + fifo->opal_fifo_tail.data.item = (intptr_t) &fifo->opal_fifo_ghost; } OBJ_CLASS_INSTANCE(opal_fifo_t, opal_object_t, opal_fifo_construct, NULL); diff --git a/opal/class/opal_fifo.h b/opal/class/opal_fifo.h index 556688bc76..cd3f8320a3 100644 --- a/opal/class/opal_fifo.h +++ b/opal/class/opal_fifo.h @@ -100,13 +100,13 @@ static inline opal_list_item_t *opal_fifo_push_atomic (opal_fifo_t *fifo, opal_atomic_wmb (); - if (ghost == tail.data.item) { + if ((intptr_t) ghost == tail.data.item) { /* update the head */ opal_counted_pointer_t head = {.value = fifo->opal_fifo_head.value}; opal_update_counted_pointer (&fifo->opal_fifo_head, &head, item); } else { /* update previous item */ - tail.data.item->opal_list_next = item; + ((opal_list_item_t *) tail.data.item)->opal_list_next = item; } return (opal_list_item_t *) tail.data.item; @@ -129,12 +129,12 @@ static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo) item = (opal_list_item_t *) head.data.item; next = (opal_list_item_t *) item->opal_list_next; - if (ghost == tail.data.item && ghost == item) { + if ((intptr_t) ghost == tail.data.item && ghost == item) { return NULL; } /* the head or next pointer are in an inconsistent state. keep looping. */ - if (tail.data.item != item && ghost != tail.data.item && ghost == next) { + if (tail.data.item != (intptr_t) item && (intptr_t) ghost != tail.data.item && ghost == next) { opal_read_counted_pointer (&fifo->opal_fifo_head, &head); continue; } @@ -155,17 +155,15 @@ static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo) * update the head */ /* wait for next pointer to be updated by push */ - while (ghost == item->opal_list_next) { + do { opal_atomic_rmb (); - } - - opal_atomic_rmb (); + } while (ghost == item->opal_list_next); /* update the head with the real next value. note that no other thread * will be attempting to update the head until after it has been updated * with the next pointer. push will not see an empty list and other pop * operations will loop until the head is consistent. */ - fifo->opal_fifo_head.data.item = (opal_list_item_t *) item->opal_list_next; + fifo->opal_fifo_head.data.item = (intptr_t) item->opal_list_next; opal_atomic_wmb (); } } @@ -191,13 +189,13 @@ static inline opal_list_item_t *opal_fifo_push_atomic (opal_fifo_t *fifo, opal_atomic_wmb (); /* try to get the tail */ - tail_item = opal_atomic_swap_ptr (&fifo->opal_fifo_tail.data.item, item); + tail_item = (opal_list_item_t *) opal_atomic_swap_ptr (&fifo->opal_fifo_tail.data.item, (intptr_t) item); opal_atomic_wmb (); if (ghost == tail_item) { /* update the head */ - fifo->opal_fifo_head.data.item = item; + fifo->opal_fifo_head.data.item = (intptr_t) item; } else { /* update previous item */ tail_item->opal_list_next = item; @@ -230,7 +228,7 @@ static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo) opal_atomic_ll_ptr(&fifo->opal_fifo_head.data.item, item); if (ghost == item) { - if (ghost == fifo->opal_fifo_tail.data.item) { + if ((intptr_t) ghost == fifo->opal_fifo_tail.data.item) { return NULL; } @@ -248,7 +246,7 @@ static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo) /* protect against ABA issues by "locking" the head */ do { - if (!opal_atomic_swap_32 ((volatile int32_t *) &fifo->opal_fifo_head.data.counter, 1)) { + if (!opal_atomic_swap_32 ((opal_atomic_int32_t *) &fifo->opal_fifo_head.data.counter, 1)) { break; } @@ -270,12 +268,12 @@ static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo) if (ghost == next) { void *tmp = item; - if (!opal_atomic_compare_exchange_strong_ptr (&fifo->opal_fifo_tail.data.item, &tmp, (void *) ghost)) { + if (!opal_atomic_compare_exchange_strong_ptr (&fifo->opal_fifo_tail.data.item, (intptr_t *) &tmp, (intptr_t) ghost)) { do { opal_atomic_rmb (); } while (ghost == item->opal_list_next); - fifo->opal_fifo_head.data.item = (opal_list_item_t *) item->opal_list_next; + fifo->opal_fifo_head.data.item = (intptr_t) item->opal_list_next; } } @@ -299,9 +297,9 @@ static inline opal_list_item_t *opal_fifo_push_st (opal_fifo_t *fifo, item->opal_list_next = &fifo->opal_fifo_ghost; - fifo->opal_fifo_tail.data.item = item; + fifo->opal_fifo_tail.data.item = (intptr_t) item; if (&fifo->opal_fifo_ghost == opal_fifo_head (fifo)) { - fifo->opal_fifo_head.data.item = item; + fifo->opal_fifo_head.data.item = (intptr_t) item; } else { prev->opal_list_next = item; } @@ -317,9 +315,9 @@ static inline opal_list_item_t *opal_fifo_pop_st (opal_fifo_t *fifo) return NULL; } - fifo->opal_fifo_head.data.item = (opal_list_item_t *) item->opal_list_next; + fifo->opal_fifo_head.data.item = (intptr_t) item->opal_list_next; if (&fifo->opal_fifo_ghost == opal_fifo_head (fifo)) { - fifo->opal_fifo_tail.data.item = &fifo->opal_fifo_ghost; + fifo->opal_fifo_tail.data.item = (intptr_t) &fifo->opal_fifo_ghost; } item->opal_list_next = NULL; diff --git a/opal/class/opal_interval_tree.c b/opal/class/opal_interval_tree.c index e8ccda2024..a20ef980df 100644 --- a/opal/class/opal_interval_tree.c +++ b/opal/class/opal_interval_tree.c @@ -130,7 +130,7 @@ static opal_interval_tree_token_t opal_interval_tree_reader_get_token (opal_inte } } - while (!OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_32((volatile int32_t *) &tree->reader_epochs[token], + while (!OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_32((opal_atomic_int32_t *) &tree->reader_epochs[token], &(int32_t) {UINT_MAX}, tree->epoch)); return token; diff --git a/opal/class/opal_interval_tree.h b/opal/class/opal_interval_tree.h index cbb36eb1cc..68f6fe4cef 100644 --- a/opal/class/opal_interval_tree.h +++ b/opal/class/opal_interval_tree.h @@ -83,11 +83,11 @@ struct opal_interval_tree_t { opal_free_list_t free_list; /**< the free list to get the memory from */ opal_list_t gc_list; /**< list of nodes that need to be released */ uint32_t epoch; /**< current update epoch */ - volatile size_t tree_size; /**< the current size of the tree */ - volatile int32_t lock; /**< update lock */ - volatile int32_t reader_count; /**< current highest reader slot to check */ + opal_atomic_size_t tree_size; /**< the current size of the tree */ + opal_atomic_int32_t lock; /**< update lock */ + opal_atomic_int32_t reader_count; /**< current highest reader slot to check */ volatile uint32_t reader_id; /**< next reader slot to check */ - volatile uint32_t reader_epochs[OPAL_INTERVAL_TREE_MAX_READERS]; + opal_atomic_uint32_t reader_epochs[OPAL_INTERVAL_TREE_MAX_READERS]; }; typedef struct opal_interval_tree_t opal_interval_tree_t; diff --git a/opal/class/opal_lifo.c b/opal/class/opal_lifo.c index 1c403fd64c..c2fc30c51c 100644 --- a/opal/class/opal_lifo.c +++ b/opal/class/opal_lifo.c @@ -26,7 +26,7 @@ static void opal_lifo_construct (opal_lifo_t *lifo) { OBJ_CONSTRUCT(&lifo->opal_lifo_ghost, opal_list_item_t); lifo->opal_lifo_ghost.opal_list_next = &lifo->opal_lifo_ghost; - lifo->opal_lifo_head.data.item = &lifo->opal_lifo_ghost; + lifo->opal_lifo_head.data.item = (intptr_t) &lifo->opal_lifo_ghost; lifo->opal_lifo_head.data.counter = 0; } diff --git a/opal/class/opal_lifo.h b/opal/class/opal_lifo.h index 945932683b..4d07ef6ccd 100644 --- a/opal/class/opal_lifo.h +++ b/opal/class/opal_lifo.h @@ -48,11 +48,12 @@ union opal_counted_pointer_t { /** update counter used when cmpset_128 is available */ uint64_t counter; /** list item pointer */ - volatile opal_list_item_t * volatile item; + volatile opal_atomic_intptr_t item; } data; #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 && HAVE_OPAL_INT128_T /** used for atomics when there is a cmpset that can operate on * two 64-bit values */ + opal_atomic_int128_t atomic_value; opal_int128_t value; #endif }; @@ -65,17 +66,17 @@ typedef union opal_counted_pointer_t opal_counted_pointer_t; * to allow the upper level to detect if this element is the first one in the * list (if the list was empty before this operation). */ -static inline bool opal_update_counted_pointer (volatile opal_counted_pointer_t *addr, opal_counted_pointer_t *old, +static inline bool opal_update_counted_pointer (volatile opal_counted_pointer_t * volatile addr, opal_counted_pointer_t *old, opal_list_item_t *item) { opal_counted_pointer_t new_p; - new_p.data.item = item; + new_p.data.item = (intptr_t) item; new_p.data.counter = old->data.counter + 1; - return opal_atomic_compare_exchange_strong_128 (&addr->value, &old->value, new_p.value); + return opal_atomic_compare_exchange_strong_128 (&addr->atomic_value, &old->value, new_p.value); } __opal_attribute_always_inline__ -static inline void opal_read_counted_pointer (volatile opal_counted_pointer_t *addr, opal_counted_pointer_t *value) +static inline void opal_read_counted_pointer (volatile opal_counted_pointer_t * volatile addr, opal_counted_pointer_t *value) { /* most platforms do not read the value atomically so make sure we read the counted pointer in a specific order */ value->data.counter = addr->data.counter; @@ -151,7 +152,7 @@ static inline opal_list_item_t *opal_lifo_push_atomic (opal_lifo_t *lifo, opal_atomic_wmb (); /* to protect against ABA issues it is sufficient to only update the counter in pop */ - if (opal_atomic_compare_exchange_strong_ptr (&lifo->opal_lifo_head.data.item, &next, item)) { + if (opal_atomic_compare_exchange_strong_ptr (&lifo->opal_lifo_head.data.item, (intptr_t *) &next, (intptr_t) item)) { return next; } /* DO some kind of pause to release the bus */ @@ -200,7 +201,7 @@ static inline opal_list_item_t *opal_lifo_push_atomic (opal_lifo_t *lifo, do { item->opal_list_next = next; opal_atomic_wmb(); - if (opal_atomic_compare_exchange_strong_ptr (&lifo->opal_lifo_head.data.item, &next, item)) { + if (opal_atomic_compare_exchange_strong_ptr (&lifo->opal_lifo_head.data.item, (intptr_t *) &next, (intptr_t) item)) { opal_atomic_wmb (); /* now safe to pop this item */ item->item_free = 0; @@ -254,7 +255,7 @@ static inline opal_list_item_t *opal_lifo_pop_atomic (opal_lifo_t* lifo) while ((item=(opal_list_item_t *)lifo->opal_lifo_head.data.item) != ghost) { /* ensure it is safe to pop the head */ - if (opal_atomic_swap_32((volatile int32_t *) &item->item_free, 1)) { + if (opal_atomic_swap_32((opal_atomic_int32_t *) &item->item_free, 1)) { continue; } @@ -262,8 +263,8 @@ static inline opal_list_item_t *opal_lifo_pop_atomic (opal_lifo_t* lifo) head = item; /* try to swap out the head pointer */ - if (opal_atomic_compare_exchange_strong_ptr (&lifo->opal_lifo_head.data.item, &head, - (void *) item->opal_list_next)) { + if (opal_atomic_compare_exchange_strong_ptr (&lifo->opal_lifo_head.data.item, (intptr_t *) &head, + (intptr_t) item->opal_list_next)) { break; } @@ -294,7 +295,7 @@ static inline opal_list_item_t *opal_lifo_push_st (opal_lifo_t *lifo, { item->opal_list_next = (opal_list_item_t *) lifo->opal_lifo_head.data.item; item->item_free = 0; - lifo->opal_lifo_head.data.item = item; + lifo->opal_lifo_head.data.item = (intptr_t) item; return (opal_list_item_t *) item->opal_list_next; } @@ -302,7 +303,7 @@ static inline opal_list_item_t *opal_lifo_pop_st (opal_lifo_t *lifo) { opal_list_item_t *item; item = (opal_list_item_t *) lifo->opal_lifo_head.data.item; - lifo->opal_lifo_head.data.item = (opal_list_item_t *) item->opal_list_next; + lifo->opal_lifo_head.data.item = (intptr_t) item->opal_list_next; if (item == &lifo->opal_lifo_ghost) { return NULL; } diff --git a/opal/class/opal_list.h b/opal/class/opal_list.h index 5edd6730d5..f9f1f00ab2 100644 --- a/opal/class/opal_list.h +++ b/opal/class/opal_list.h @@ -111,7 +111,7 @@ struct opal_list_item_t #if OPAL_ENABLE_DEBUG /** Atomic reference count for debugging */ - volatile int32_t opal_list_item_refcount; + opal_atomic_int32_t opal_list_item_refcount; /** The list this item belong to */ volatile struct opal_list_t* opal_list_item_belong_to; #endif @@ -654,7 +654,7 @@ static inline opal_list_item_t *opal_list_remove_first(opal_list_t *list) Caller now owns the item and should release the item when caller is done with it. */ - volatile opal_list_item_t *item; + opal_list_item_t *item; if ( 0 == list->opal_list_length ) { return (opal_list_item_t *)NULL; } @@ -669,7 +669,7 @@ static inline opal_list_item_t *opal_list_remove_first(opal_list_t *list) list->opal_list_length--; /* get pointer to first element on the list */ - item = list->opal_list_sentinel.opal_list_next; + item = (opal_list_item_t *) list->opal_list_sentinel.opal_list_next; /* reset previous pointer of next item on the list */ item->opal_list_next->opal_list_prev = item->opal_list_prev; @@ -686,11 +686,11 @@ static inline opal_list_item_t *opal_list_remove_first(opal_list_t *list) /* Spot check: ensure that the item we're returning is now on no lists */ - OPAL_THREAD_ADD_FETCH32( &(item->opal_list_item_refcount), -1 ); + OPAL_THREAD_ADD_FETCH32( &item->opal_list_item_refcount, -1 ); assert(0 == item->opal_list_item_refcount); #endif - return (opal_list_item_t *) item; + return item; } @@ -716,7 +716,7 @@ static inline opal_list_item_t *opal_list_remove_last(opal_list_t *list) Caller now owns the item and should release the item when caller is done with it. */ - volatile opal_list_item_t *item; + opal_list_item_t *item; if ( 0 == list->opal_list_length ) { return (opal_list_item_t *)NULL; } @@ -731,7 +731,7 @@ static inline opal_list_item_t *opal_list_remove_last(opal_list_t *list) list->opal_list_length--; /* get item */ - item = list->opal_list_sentinel.opal_list_prev; + item = (opal_list_item_t *) list->opal_list_sentinel.opal_list_prev; /* reset previous pointer on next to last pointer */ item->opal_list_prev->opal_list_next = item->opal_list_next; @@ -746,12 +746,12 @@ static inline opal_list_item_t *opal_list_remove_last(opal_list_t *list) /* Spot check: ensure that the item we're returning is now on no lists */ - OPAL_THREAD_ADD_FETCH32( &(item->opal_list_item_refcount), -1 ); + OPAL_THREAD_ADD_FETCH32(&item->opal_list_item_refcount, -1 ); assert(0 == item->opal_list_item_refcount); item->opal_list_item_belong_to = NULL; #endif - return (opal_list_item_t *) item; + return item; } /** diff --git a/opal/class/opal_object.c b/opal/class/opal_object.c index cd09d647f6..64fef5712f 100644 --- a/opal/class/opal_object.c +++ b/opal/class/opal_object.c @@ -55,7 +55,7 @@ int opal_class_init_epoch = 1; /* * Local variables */ -static opal_atomic_lock_t class_lock = { { OPAL_ATOMIC_LOCK_UNLOCKED } }; +static opal_atomic_lock_t class_lock = OPAL_ATOMIC_LOCK_INIT; static void** classes = NULL; static int num_classes = 0; static int max_classes = 0; diff --git a/opal/class/opal_object.h b/opal/class/opal_object.h index 4e2da95c20..4d3380f0da 100644 --- a/opal/class/opal_object.h +++ b/opal/class/opal_object.h @@ -198,7 +198,7 @@ struct opal_object_t { uint64_t obj_magic_id; #endif opal_class_t *obj_class; /**< class descriptor */ - volatile int32_t obj_reference_count; /**< reference count */ + opal_atomic_int32_t obj_reference_count; /**< reference count */ #if OPAL_ENABLE_DEBUG const char* cls_init_file_name; /**< In debug mode store the file where the object get contructed */ int cls_init_lineno; /**< In debug mode store the line number where the object get contructed */ diff --git a/opal/class/opal_tree.h b/opal/class/opal_tree.h index e1a42a3355..e73d0a48d9 100644 --- a/opal/class/opal_tree.h +++ b/opal/class/opal_tree.h @@ -120,7 +120,7 @@ typedef struct opal_tree_item_t #if OPAL_ENABLE_DEBUG /** Atomic reference count for debugging */ - int32_t opal_tree_item_refcount; + opal_atomic_int32_t opal_tree_item_refcount; /** The tree this item belong to */ struct opal_tree_t* opal_tree_item_belong_to; #endif diff --git a/opal/include/Makefile.am b/opal/include/Makefile.am index 464faafd0a..0a1c195d65 100644 --- a/opal/include/Makefile.am +++ b/opal/include/Makefile.am @@ -10,6 +10,8 @@ # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2010-2011 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2018 Los Alamos National Security, LLC. All rights +# reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -20,7 +22,8 @@ headers = \ opal_config_top.h \ opal_config_bottom.h \ - opal_stdint.h + opal_stdint.h \ + opal_stdatomic.h nodist_headers = opal_config.h diff --git a/opal/include/opal/sys/Makefile.am b/opal/include/opal/sys/Makefile.am index 9387ed6da1..cb5c471ca5 100644 --- a/opal/include/opal/sys/Makefile.am +++ b/opal/include/opal/sys/Makefile.am @@ -11,7 +11,7 @@ # All rights reserved. # Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2011 Sandia National Laboratories. All rights reserved. -# Copyright (c) 2016 Los Alamos National Security, LLC. All rights +# Copyright (c) 2016-2018 Los Alamos National Security, LLC. All rights # reserved. # Copyright (c) 2017 Research Organization for Information Science # and Technology (RIST). All rights reserved. @@ -27,6 +27,7 @@ headers += \ opal/sys/architecture.h \ opal/sys/atomic.h \ + opal/sys/atomic_stdc.h \ opal/sys/atomic_impl.h \ opal/sys/timer.h \ opal/sys/cma.h diff --git a/opal/include/opal/sys/architecture.h b/opal/include/opal/sys/architecture.h index ee9aa96901..2e61f0d795 100644 --- a/opal/include/opal/sys/architecture.h +++ b/opal/include/opal/sys/architecture.h @@ -47,6 +47,7 @@ #define OPAL_BUILTIN_SYNC 0200 #define OPAL_BUILTIN_GCC 0202 #define OPAL_BUILTIN_NO 0203 +#define OPAL_BUILTIN_C11 0204 /* Formats */ #define OPAL_DEFAULT 1000 /* standard for given architecture */ diff --git a/opal/include/opal/sys/arm/atomic.h b/opal/include/opal/sys/arm/atomic.h index 6d4db3ad7a..ff2c5bab35 100644 --- a/opal/include/opal/sys/arm/atomic.h +++ b/opal/include/opal/sys/arm/atomic.h @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2010 IBM Corporation. All rights reserved. * Copyright (c) 2010 ARM ltd. All rights reserved. - * Copyright (c) 2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2017-2018 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -109,7 +109,7 @@ void opal_atomic_isync(void) #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1 #define OPAL_HAVE_ATOMIC_MATH_32 1 -static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { int32_t prev, tmp; bool ret; @@ -137,7 +137,7 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { bool rc; @@ -148,7 +148,7 @@ static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t } -static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { opal_atomic_wmb(); return opal_atomic_compare_exchange_strong_32 (addr, oldval, newval); @@ -157,7 +157,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t #if (OPAL_ASM_SUPPORT_64BIT == 1) #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1 -static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { int64_t prev; int tmp; @@ -188,7 +188,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { bool rc; @@ -199,7 +199,7 @@ static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t } -static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { opal_atomic_wmb(); return opal_atomic_compare_exchange_strong_64 (addr, oldval, newval); @@ -209,7 +209,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t #define OPAL_HAVE_ATOMIC_ADD_32 1 -static inline int32_t opal_atomic_fetch_add_32(volatile int32_t* v, int inc) +static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t* v, int inc) { int32_t t, old; int tmp; @@ -230,7 +230,7 @@ static inline int32_t opal_atomic_fetch_add_32(volatile int32_t* v, int inc) } #define OPAL_HAVE_ATOMIC_SUB_32 1 -static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t* v, int dec) +static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t* v, int dec) { int32_t t, old; int tmp; diff --git a/opal/include/opal/sys/arm64/atomic.h b/opal/include/opal/sys/arm64/atomic.h index 55a4628813..76409623b1 100644 --- a/opal/include/opal/sys/arm64/atomic.h +++ b/opal/include/opal/sys/arm64/atomic.h @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2010 IBM Corporation. All rights reserved. * Copyright (c) 2010 ARM ltd. All rights reserved. - * Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2016-2018 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -82,7 +82,7 @@ static inline void opal_atomic_isync (void) * *********************************************************************/ -static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { int32_t prev, tmp; bool ret; @@ -102,7 +102,7 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add return ret; } -static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, int32_t newval) +static inline int32_t opal_atomic_swap_32(opal_atomic_int32_t *addr, int32_t newval) { int32_t ret, tmp; @@ -121,7 +121,7 @@ static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, int32_t newval atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { int32_t prev, tmp; bool ret; @@ -142,7 +142,7 @@ static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t } -static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { int32_t prev, tmp; bool ret; @@ -164,7 +164,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t #define opal_atomic_ll_32(addr, ret) \ do { \ - volatile int32_t *_addr = (addr); \ + opal_atomic_int32_t *_addr = (addr); \ int32_t _ret; \ \ __asm__ __volatile__ ("ldaxr %w0, [%1] \n" \ @@ -176,7 +176,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t #define opal_atomic_sc_32(addr, newval, ret) \ do { \ - volatile int32_t *_addr = (addr); \ + opal_atomic_int32_t *_addr = (addr); \ int32_t _newval = (int32_t) newval; \ int _ret; \ \ @@ -188,7 +188,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t ret = (_ret == 0); \ } while (0) -static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { int64_t prev; int tmp; @@ -209,7 +209,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add return ret; } -static inline int64_t opal_atomic_swap_64 (volatile int64_t *addr, int64_t newval) +static inline int64_t opal_atomic_swap_64 (opal_atomic_int64_t *addr, int64_t newval) { int64_t ret; int tmp; @@ -229,7 +229,7 @@ static inline int64_t opal_atomic_swap_64 (volatile int64_t *addr, int64_t newva atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { int64_t prev; int tmp; @@ -251,7 +251,7 @@ static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t } -static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { int64_t prev; int tmp; @@ -274,7 +274,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t #define opal_atomic_ll_64(addr, ret) \ do { \ - volatile int64_t *_addr = (addr); \ + opal_atomic_int64_t *_addr = (addr); \ int64_t _ret; \ \ __asm__ __volatile__ ("ldaxr %0, [%1] \n" \ @@ -286,7 +286,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t #define opal_atomic_sc_64(addr, newval, ret) \ do { \ - volatile int64_t *_addr = (addr); \ + opal_atomic_int64_t *_addr = (addr); \ int64_t _newval = (int64_t) newval; \ int _ret; \ \ @@ -299,7 +299,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t } while (0) #define OPAL_ASM_MAKE_ATOMIC(type, bits, name, inst, reg) \ - static inline type opal_atomic_fetch_ ## name ## _ ## bits (volatile type *addr, type value) \ + static inline type opal_atomic_fetch_ ## name ## _ ## bits (opal_atomic_ ## type *addr, type value) \ { \ type newval, old; \ int32_t tmp; \ diff --git a/opal/include/opal/sys/atomic.h b/opal/include/opal/sys/atomic.h index 3b165f00e0..cf0115622e 100644 --- a/opal/include/opal/sys/atomic.h +++ b/opal/include/opal/sys/atomic.h @@ -56,7 +56,13 @@ #include #include "opal/sys/architecture.h" -#include "opal_stdint.h" +#include "opal_stdatomic.h" + +#if OPAL_ASSEMBLY_BUILTIN == OPAL_BUILTIN_C11 + +#include "atomic_stdc.h" + +#else /* !OPAL_C_HAVE__ATOMIC */ /* do some quick #define cleanup in cases where we are doing testing... */ @@ -92,7 +98,7 @@ BEGIN_C_DECLS */ struct opal_atomic_lock_t { union { - volatile int32_t lock; /**< The lock address (an integer) */ + opal_atomic_int32_t lock; /**< The lock address (an integer) */ volatile unsigned char sparc_lock; /**< The lock address on sparc */ char padding[sizeof(int)]; /**< Array for optional padding */ } u; @@ -147,6 +153,8 @@ enum { OPAL_ATOMIC_LOCK_LOCKED = 1 }; +#define OPAL_ATOMIC_LOCK_INIT {.u = {.lock = OPAL_ATOMIC_LOCK_UNLOCKED}} + /********************************************************************** * * Load the appropriate architecture files and set some reasonable @@ -350,19 +358,19 @@ void opal_atomic_unlock(opal_atomic_lock_t *lock); #if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32 static inline #endif -bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, +bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval); #if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32 static inline #endif -bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, +bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval); #if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32 static inline #endif -bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, +bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval); #endif @@ -375,19 +383,19 @@ bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t #if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64 static inline #endif -bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, +bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval); #if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64 static inline #endif -bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, +bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval); #if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64 static inline #endif -bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, +bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval); #endif @@ -399,20 +407,20 @@ bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t #if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 -static inline int32_t opal_atomic_add_fetch_32(volatile int32_t *addr, int delta); -static inline int32_t opal_atomic_fetch_add_32(volatile int32_t *addr, int delta); -static inline int32_t opal_atomic_and_fetch_32(volatile int32_t *addr, int32_t value); -static inline int32_t opal_atomic_fetch_and_32(volatile int32_t *addr, int32_t value); -static inline int32_t opal_atomic_or_fetch_32(volatile int32_t *addr, int32_t value); -static inline int32_t opal_atomic_fetch_or_32(volatile int32_t *addr, int32_t value); -static inline int32_t opal_atomic_xor_fetch_32(volatile int32_t *addr, int32_t value); -static inline int32_t opal_atomic_fetch_xor_32(volatile int32_t *addr, int32_t value); -static inline int32_t opal_atomic_sub_fetch_32(volatile int32_t *addr, int delta); -static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t *addr, int delta); -static inline int32_t opal_atomic_min_fetch_32 (volatile int32_t *addr, int32_t value); -static inline int32_t opal_atomic_fetch_min_32 (volatile int32_t *addr, int32_t value); -static inline int32_t opal_atomic_max_fetch_32 (volatile int32_t *addr, int32_t value); -static inline int32_t opal_atomic_fetch_max_32 (volatile int32_t *addr, int32_t value); +static inline int32_t opal_atomic_add_fetch_32(opal_atomic_int32_t *addr, int delta); +static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t *addr, int delta); +static inline int32_t opal_atomic_and_fetch_32(opal_atomic_int32_t *addr, int32_t value); +static inline int32_t opal_atomic_fetch_and_32(opal_atomic_int32_t *addr, int32_t value); +static inline int32_t opal_atomic_or_fetch_32(opal_atomic_int32_t *addr, int32_t value); +static inline int32_t opal_atomic_fetch_or_32(opal_atomic_int32_t *addr, int32_t value); +static inline int32_t opal_atomic_xor_fetch_32(opal_atomic_int32_t *addr, int32_t value); +static inline int32_t opal_atomic_fetch_xor_32(opal_atomic_int32_t *addr, int32_t value); +static inline int32_t opal_atomic_sub_fetch_32(opal_atomic_int32_t *addr, int delta); +static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t *addr, int delta); +static inline int32_t opal_atomic_min_fetch_32 (opal_atomic_int32_t *addr, int32_t value); +static inline int32_t opal_atomic_fetch_min_32 (opal_atomic_int32_t *addr, int32_t value); +static inline int32_t opal_atomic_max_fetch_32 (opal_atomic_int32_t *addr, int32_t value); +static inline int32_t opal_atomic_fetch_max_32 (opal_atomic_int32_t *addr, int32_t value); #endif /* OPAL_HAVE_ATOMIC_MATH_32 */ @@ -429,19 +437,19 @@ static inline int32_t opal_atomic_fetch_max_32 (volatile int32_t *addr, int32_t #if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MATH_64 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 -static inline int64_t opal_atomic_add_fetch_64(volatile int64_t *addr, int64_t delta); -static inline int64_t opal_atomic_fetch_add_64(volatile int64_t *addr, int64_t delta); -static inline int64_t opal_atomic_and_fetch_64(volatile int64_t *addr, int64_t value); -static inline int64_t opal_atomic_fetch_and_64(volatile int64_t *addr, int64_t value); -static inline int64_t opal_atomic_or_fetch_64(volatile int64_t *addr, int64_t value); -static inline int64_t opal_atomic_fetch_or_64(volatile int64_t *addr, int64_t value); -static inline int64_t opal_atomic_fetch_xor_64(volatile int64_t *addr, int64_t value); -static inline int64_t opal_atomic_sub_fetch_64(volatile int64_t *addr, int64_t delta); -static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t *addr, int64_t delta); -static inline int64_t opal_atomic_min_fetch_64 (volatile int64_t *addr, int64_t value); -static inline int64_t opal_atomic_fetch_min_64 (volatile int64_t *addr, int64_t value); -static inline int64_t opal_atomic_max_fetch_64 (volatile int64_t *addr, int64_t value); -static inline int64_t opal_atomic_fetch_max_64 (volatile int64_t *addr, int64_t value); +static inline int64_t opal_atomic_add_fetch_64(opal_atomic_int64_t *addr, int64_t delta); +static inline int64_t opal_atomic_fetch_add_64(opal_atomic_int64_t *addr, int64_t delta); +static inline int64_t opal_atomic_and_fetch_64(opal_atomic_int64_t *addr, int64_t value); +static inline int64_t opal_atomic_fetch_and_64(opal_atomic_int64_t *addr, int64_t value); +static inline int64_t opal_atomic_or_fetch_64(opal_atomic_int64_t *addr, int64_t value); +static inline int64_t opal_atomic_fetch_or_64(opal_atomic_int64_t *addr, int64_t value); +static inline int64_t opal_atomic_fetch_xor_64(opal_atomic_int64_t *addr, int64_t value); +static inline int64_t opal_atomic_sub_fetch_64(opal_atomic_int64_t *addr, int64_t delta); +static inline int64_t opal_atomic_fetch_sub_64(opal_atomic_int64_t *addr, int64_t delta); +static inline int64_t opal_atomic_min_fetch_64 (opal_atomic_int64_t *addr, int64_t value); +static inline int64_t opal_atomic_fetch_min_64 (opal_atomic_int64_t *addr, int64_t value); +static inline int64_t opal_atomic_max_fetch_64 (opal_atomic_int64_t *addr, int64_t value); +static inline int64_t opal_atomic_fetch_max_64 (opal_atomic_int64_t *addr, int64_t value); #endif /* OPAL_HAVE_ATOMIC_MATH_64 */ @@ -458,7 +466,7 @@ static inline int64_t opal_atomic_fetch_max_64 (volatile int64_t *addr, int64_t */ #if defined(DOXYGEN) || OPAL_ENABLE_DEBUG static inline size_t -opal_atomic_add_fetch_size_t(volatile size_t *addr, size_t delta) +opal_atomic_add_fetch_size_t(opal_atomic_size_t *addr, size_t delta) { #if SIZEOF_SIZE_T == 4 return (size_t) opal_atomic_add_fetch_32((int32_t*) addr, delta); @@ -470,7 +478,7 @@ opal_atomic_add_fetch_size_t(volatile size_t *addr, size_t delta) } static inline size_t -opal_atomic_fetch_add_size_t(volatile size_t *addr, size_t delta) +opal_atomic_fetch_add_size_t(opal_atomic_size_t *addr, size_t delta) { #if SIZEOF_SIZE_T == 4 return (size_t) opal_atomic_fetch_add_32((int32_t*) addr, delta); @@ -482,7 +490,7 @@ opal_atomic_fetch_add_size_t(volatile size_t *addr, size_t delta) } static inline size_t -opal_atomic_sub_fetch_size_t(volatile size_t *addr, size_t delta) +opal_atomic_sub_fetch_size_t(opal_atomic_size_t *addr, size_t delta) { #if SIZEOF_SIZE_T == 4 return (size_t) opal_atomic_sub_fetch_32((int32_t*) addr, delta); @@ -494,7 +502,7 @@ opal_atomic_sub_fetch_size_t(volatile size_t *addr, size_t delta) } static inline size_t -opal_atomic_fetch_sub_size_t(volatile size_t *addr, size_t delta) +opal_atomic_fetch_sub_size_t(opal_atomic_size_t *addr, size_t delta) { #if SIZEOF_SIZE_T == 4 return (size_t) opal_atomic_fetch_sub_32((int32_t*) addr, delta); @@ -507,15 +515,15 @@ opal_atomic_fetch_sub_size_t(volatile size_t *addr, size_t delta) #else #if SIZEOF_SIZE_T == 4 -#define opal_atomic_add_fetch_size_t(addr, delta) ((size_t) opal_atomic_add_fetch_32((volatile int32_t *) addr, delta)) -#define opal_atomic_fetch_add_size_t(addr, delta) ((size_t) opal_atomic_fetch_add_32((volatile int32_t *) addr, delta)) -#define opal_atomic_sub_fetch_size_t(addr, delta) ((size_t) opal_atomic_sub_fetch_32((volatile int32_t *) addr, delta)) -#define opal_atomic_fetch_sub_size_t(addr, delta) ((size_t) opal_atomic_fetch_sub_32((volatile int32_t *) addr, delta)) +#define opal_atomic_add_fetch_size_t(addr, delta) ((size_t) opal_atomic_add_fetch_32((opal_atomic_int32_t *) addr, delta)) +#define opal_atomic_fetch_add_size_t(addr, delta) ((size_t) opal_atomic_fetch_add_32((opal_atomic_int32_t *) addr, delta)) +#define opal_atomic_sub_fetch_size_t(addr, delta) ((size_t) opal_atomic_sub_fetch_32((opal_atomic_int32_t *) addr, delta)) +#define opal_atomic_fetch_sub_size_t(addr, delta) ((size_t) opal_atomic_fetch_sub_32((opal_atomic_int32_t *) addr, delta)) #elif SIZEOF_SIZE_T == 8 -#define opal_atomic_add_fetch_size_t(addr, delta) ((size_t) opal_atomic_add_fetch_64((volatile int64_t *) addr, delta)) -#define opal_atomic_fetch_add_size_t(addr, delta) ((size_t) opal_atomic_fetch_add_64((volatile int64_t *) addr, delta)) -#define opal_atomic_sub_fetch_size_t(addr, delta) ((size_t) opal_atomic_sub_fetch_64((volatile int64_t *) addr, delta)) -#define opal_atomic_fetch_sub_size_t(addr, delta) ((size_t) opal_atomic_fetch_sub_64((volatile int64_t *) addr, delta)) +#define opal_atomic_add_fetch_size_t(addr, delta) ((size_t) opal_atomic_add_fetch_64((opal_atomic_int64_t *) addr, delta)) +#define opal_atomic_fetch_add_size_t(addr, delta) ((size_t) opal_atomic_fetch_add_64((opal_atomic_int64_t *) addr, delta)) +#define opal_atomic_sub_fetch_size_t(addr, delta) ((size_t) opal_atomic_sub_fetch_64((opal_atomic_int64_t *) addr, delta)) +#define opal_atomic_fetch_sub_size_t(addr, delta) ((size_t) opal_atomic_fetch_sub_64((opal_atomic_int64_t *) addr, delta)) #else #error "Unknown size_t size" #endif @@ -525,20 +533,20 @@ opal_atomic_fetch_sub_size_t(volatile size_t *addr, size_t delta) /* these are always done with inline functions, so always mark as static inline */ -static inline bool opal_atomic_compare_exchange_strong_xx (volatile void *addr, void *oldval, +static inline bool opal_atomic_compare_exchange_strong_xx (opal_atomic_intptr_t *addr, intptr_t *oldval, int64_t newval, size_t length); -static inline bool opal_atomic_compare_exchange_strong_acq_xx (volatile void *addr, void *oldval, +static inline bool opal_atomic_compare_exchange_strong_acq_xx (opal_atomic_intptr_t *addr, intptr_t *oldval, int64_t newval, size_t length); -static inline bool opal_atomic_compare_exchange_strong_rel_xx (volatile void *addr, void *oldval, +static inline bool opal_atomic_compare_exchange_strong_rel_xx (opal_atomic_intptr_t *addr, intptr_t *oldval, int64_t newval, size_t length); -static inline bool opal_atomic_compare_exchange_strong_ptr (volatile void* addr, void *oldval, - void *newval); -static inline bool opal_atomic_compare_exchange_strong_acq_ptr (volatile void* addr, void *oldval, - void *newval); -static inline bool opal_atomic_compare_exchange_strong_rel_ptr (volatile void* addr, void *oldval, - void *newval); +static inline bool opal_atomic_compare_exchange_strong_ptr (opal_atomic_intptr_t* addr, intptr_t *oldval, + intptr_t newval); +static inline bool opal_atomic_compare_exchange_strong_acq_ptr (opal_atomic_intptr_t* addr, intptr_t *oldval, + intptr_t newval); +static inline bool opal_atomic_compare_exchange_strong_rel_ptr (opal_atomic_intptr_t* addr, intptr_t *oldval, + intptr_t newval); /** * Atomic compare and set of generic type with relaxed semantics. This @@ -554,7 +562,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_ptr (volatile void* a * See opal_atomic_compare_exchange_* for pseudo-code. */ #define opal_atomic_compare_exchange_strong( ADDR, OLDVAL, NEWVAL ) \ - opal_atomic_compare_exchange_strong_xx( (volatile void*)(ADDR), (void *)(OLDVAL), \ + opal_atomic_compare_exchange_strong_xx( (opal_atomic_intptr_t*)(ADDR), (intptr_t *)(OLDVAL), \ (intptr_t)(NEWVAL), sizeof(*(ADDR)) ) /** @@ -571,7 +579,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_ptr (volatile void* a * See opal_atomic_compare_exchange_acq_* for pseudo-code. */ #define opal_atomic_compare_exchange_strong_acq( ADDR, OLDVAL, NEWVAL ) \ - opal_atomic_compare_exchange_strong_acq_xx( (volatile void*)(ADDR), (void *)(OLDVAL), \ + opal_atomic_compare_exchange_strong_acq_xx( (opal_atomic_intptr_t*)(ADDR), (intptr_t *)(OLDVAL), \ (intptr_t)(NEWVAL), sizeof(*(ADDR)) ) /** @@ -588,7 +596,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_ptr (volatile void* a * See opal_atomic_compare_exchange_rel_* for pseudo-code. */ #define opal_atomic_compare_exchange_strong_rel( ADDR, OLDVAL, NEWVAL ) \ - opal_atomic_compare_exchange_strong_rel_xx( (volatile void*)(ADDR), (void *)(OLDVAL), \ + opal_atomic_compare_exchange_strong_rel_xx( (opal_atomic_intptr_t*)(ADDR), (intptr_t *)(OLDVAL), \ (intptr_t)(NEWVAL), sizeof(*(ADDR)) ) @@ -596,15 +604,15 @@ static inline bool opal_atomic_compare_exchange_strong_rel_ptr (volatile void* a #if defined(DOXYGEN) || (OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64) -static inline void opal_atomic_add_xx(volatile void* addr, +static inline void opal_atomic_add_xx(opal_atomic_intptr_t* addr, int32_t value, size_t length); -static inline void opal_atomic_sub_xx(volatile void* addr, +static inline void opal_atomic_sub_xx(opal_atomic_intptr_t* addr, int32_t value, size_t length); -static inline intptr_t opal_atomic_add_fetch_ptr( volatile void* addr, void* delta ); -static inline intptr_t opal_atomic_fetch_add_ptr( volatile void* addr, void* delta ); -static inline intptr_t opal_atomic_sub_fetch_ptr( volatile void* addr, void* delta ); -static inline intptr_t opal_atomic_fetch_sub_ptr( volatile void* addr, void* delta ); +static inline intptr_t opal_atomic_add_fetch_ptr( opal_atomic_intptr_t* addr, void* delta ); +static inline intptr_t opal_atomic_fetch_add_ptr( opal_atomic_intptr_t* addr, void* delta ); +static inline intptr_t opal_atomic_sub_fetch_ptr( opal_atomic_intptr_t* addr, void* delta ); +static inline intptr_t opal_atomic_fetch_sub_ptr( opal_atomic_intptr_t* addr, void* delta ); /** * Atomically increment the content depending on the type. This @@ -617,7 +625,7 @@ static inline intptr_t opal_atomic_fetch_sub_ptr( volatile void* addr, void* del * @param delta Value to add (converted to ). */ #define opal_atomic_add( ADDR, VALUE ) \ - opal_atomic_add_xx( (volatile void*)(ADDR), (int32_t)(VALUE), \ + opal_atomic_add_xx( (opal_atomic_intptr_t*)(ADDR), (int32_t)(VALUE), \ sizeof(*(ADDR)) ) /** @@ -631,7 +639,7 @@ static inline intptr_t opal_atomic_fetch_sub_ptr( volatile void* addr, void* del * @param delta Value to substract (converted to ). */ #define opal_atomic_sub( ADDR, VALUE ) \ - opal_atomic_sub_xx( (volatile void*)(ADDR), (int32_t)(VALUE), \ + opal_atomic_sub_xx( (opal_atomic_intptr_t*)(ADDR), (int32_t)(VALUE), \ sizeof(*(ADDR)) ) #endif /* OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64 */ @@ -643,6 +651,8 @@ static inline intptr_t opal_atomic_fetch_sub_ptr( volatile void* addr, void* del */ #include "opal/sys/atomic_impl.h" +#endif /* !OPAL_C_HAVE__ATOMIC */ + END_C_DECLS #endif /* OPAL_SYS_ATOMIC_H */ diff --git a/opal/include/opal/sys/atomic_impl.h b/opal/include/opal/sys/atomic_impl.h index 027b771162..051492296d 100644 --- a/opal/include/opal/sys/atomic_impl.h +++ b/opal/include/opal/sys/atomic_impl.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2010-2014 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -40,7 +40,7 @@ #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 #if !defined(OPAL_HAVE_ATOMIC_MIN_32) -static inline int32_t opal_atomic_fetch_min_32 (volatile int32_t *addr, int32_t value) +static inline int32_t opal_atomic_fetch_min_32 (opal_atomic_int32_t *addr, int32_t value) { int32_t old = *addr; do { @@ -57,7 +57,7 @@ static inline int32_t opal_atomic_fetch_min_32 (volatile int32_t *addr, int32_t #endif /* OPAL_HAVE_ATOMIC_MIN_32 */ #if !defined(OPAL_HAVE_ATOMIC_MAX_32) -static inline int32_t opal_atomic_fetch_max_32 (volatile int32_t *addr, int32_t value) +static inline int32_t opal_atomic_fetch_max_32 (opal_atomic_int32_t *addr, int32_t value) { int32_t old = *addr; do { @@ -73,7 +73,7 @@ static inline int32_t opal_atomic_fetch_max_32 (volatile int32_t *addr, int32_t #endif /* OPAL_HAVE_ATOMIC_MAX_32 */ #define OPAL_ATOMIC_DEFINE_CMPXCG_OP(type, bits, operation, name) \ - static inline type opal_atomic_fetch_ ## name ## _ ## bits (volatile type *addr, type value) \ + static inline type opal_atomic_fetch_ ## name ## _ ## bits (opal_atomic_ ## type *addr, type value) \ { \ type oldval; \ do { \ @@ -85,7 +85,7 @@ static inline int32_t opal_atomic_fetch_max_32 (volatile int32_t *addr, int32_t #if !defined(OPAL_HAVE_ATOMIC_SWAP_32) #define OPAL_HAVE_ATOMIC_SWAP_32 1 -static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, +static inline int32_t opal_atomic_swap_32(opal_atomic_int32_t *addr, int32_t newval) { int32_t old = *addr; @@ -138,7 +138,7 @@ OPAL_ATOMIC_DEFINE_CMPXCG_OP(int32_t, 32, -, sub) #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 #if !defined(OPAL_HAVE_ATOMIC_MIN_64) -static inline int64_t opal_atomic_fetch_min_64 (volatile int64_t *addr, int64_t value) +static inline int64_t opal_atomic_fetch_min_64 (opal_atomic_int64_t *addr, int64_t value) { int64_t old = *addr; do { @@ -155,7 +155,7 @@ static inline int64_t opal_atomic_fetch_min_64 (volatile int64_t *addr, int64_t #endif /* OPAL_HAVE_ATOMIC_MIN_64 */ #if !defined(OPAL_HAVE_ATOMIC_MAX_64) -static inline int64_t opal_atomic_fetch_max_64 (volatile int64_t *addr, int64_t value) +static inline int64_t opal_atomic_fetch_max_64 (opal_atomic_int64_t *addr, int64_t value) { int64_t old = *addr; do { @@ -172,7 +172,7 @@ static inline int64_t opal_atomic_fetch_max_64 (volatile int64_t *addr, int64_t #if !defined(OPAL_HAVE_ATOMIC_SWAP_64) #define OPAL_HAVE_ATOMIC_SWAP_64 1 -static inline int64_t opal_atomic_swap_64(volatile int64_t *addr, +static inline int64_t opal_atomic_swap_64(opal_atomic_int64_t *addr, int64_t newval) { int64_t old = *addr; @@ -235,15 +235,15 @@ OPAL_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, -, sub) #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 #define OPAL_ATOMIC_DEFINE_CMPXCG_XX(semantics) \ static inline bool \ - opal_atomic_compare_exchange_strong ## semantics ## xx (volatile void* addr, void *oldval, \ + opal_atomic_compare_exchange_strong ## semantics ## xx (opal_atomic_intptr_t* addr, intptr_t *oldval, \ int64_t newval, const size_t length) \ { \ switch (length) { \ case 4: \ - return opal_atomic_compare_exchange_strong_32 ((volatile int32_t *) addr, \ + return opal_atomic_compare_exchange_strong_32 ((opal_atomic_int32_t *) addr, \ (int32_t *) oldval, (int32_t) newval); \ case 8: \ - return opal_atomic_compare_exchange_strong_64 ((volatile int64_t *) addr, \ + return opal_atomic_compare_exchange_strong_64 ((opal_atomic_int64_t *) addr, \ (int64_t *) oldval, (int64_t) newval); \ } \ abort(); \ @@ -251,12 +251,12 @@ OPAL_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, -, sub) #elif OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 #define OPAL_ATOMIC_DEFINE_CMPXCG_XX(semantics) \ static inline bool \ - opal_atomic_compare_exchange_strong ## semantics ## xx (volatile void* addr, void *oldval, \ + opal_atomic_compare_exchange_strong ## semantics ## xx (opal_atomic_intptr_t* addr, intptr_t *oldval, \ int64_t newval, const size_t length) \ { \ switch (length) { \ case 4: \ - return opal_atomic_compare_exchange_strong_32 ((volatile int32_t *) addr, \ + return opal_atomic_compare_exchange_strong_32 ((opal_atomic_int32_t *) addr, \ (int32_t *) oldval, (int32_t) newval); \ } \ abort(); \ @@ -272,16 +272,16 @@ OPAL_ATOMIC_DEFINE_CMPXCG_XX(_rel_) #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 #define OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(semantics) \ static inline bool \ - opal_atomic_compare_exchange_strong ## semantics ## ptr (volatile void* addr, void *oldval, void *newval) \ + opal_atomic_compare_exchange_strong ## semantics ## ptr (opal_atomic_intptr_t* addr, intptr_t *oldval, intptr_t newval) \ { \ - return opal_atomic_compare_exchange_strong_32 ((volatile int32_t *) addr, (int32_t *) oldval, (int32_t) newval); \ + return opal_atomic_compare_exchange_strong_32 ((opal_atomic_int32_t *) addr, (int32_t *) oldval, (int32_t) newval); \ } #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 #define OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(semantics) \ static inline bool \ - opal_atomic_compare_exchange_strong ## semantics ## ptr (volatile void* addr, void *oldval, void *newval) \ + opal_atomic_compare_exchange_strong ## semantics ## ptr (opal_atomic_intptr_t* addr, intptr_t *oldval, intptr_t newval) \ { \ - return opal_atomic_compare_exchange_strong_64 ((volatile int64_t *) addr, (int64_t *) oldval, (int64_t) newval); \ + return opal_atomic_compare_exchange_strong_64 ((opal_atomic_int64_t *) addr, (int64_t *) oldval, (int64_t) newval); \ } #else #error "Can not define opal_atomic_compare_exchange_strong_ptr with existing atomics" @@ -297,9 +297,9 @@ OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(_rel_) #if (OPAL_HAVE_ATOMIC_SWAP_32 || OPAL_HAVE_ATOMIC_SWAP_64) #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_SWAP_32 -#define opal_atomic_swap_ptr(addr, value) (void *) opal_atomic_swap_32((int32_t *) addr, (int32_t) value) +#define opal_atomic_swap_ptr(addr, value) (intptr_t) opal_atomic_swap_32((opal_atomic_int32_t *) addr, (int32_t) value) #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_SWAP_64 -#define opal_atomic_swap_ptr(addr, value) (void *) opal_atomic_swap_64((int64_t *) addr, (int64_t) value) +#define opal_atomic_swap_ptr(addr, value) (intptr_t) opal_atomic_swap_64((opal_atomic_int64_t *) addr, (int64_t) value) #endif #endif /* (OPAL_HAVE_ATOMIC_SWAP_32 || OPAL_HAVE_ATOMIC_SWAP_64) */ @@ -308,15 +308,15 @@ OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(_rel_) #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_LLSC_32 -#define opal_atomic_ll_ptr(addr, ret) opal_atomic_ll_32((volatile int32_t *) (addr), ret) -#define opal_atomic_sc_ptr(addr, value, ret) opal_atomic_sc_32((volatile int32_t *) (addr), (intptr_t) (value), ret) +#define opal_atomic_ll_ptr(addr, ret) opal_atomic_ll_32((opal_atomic_int32_t *) (addr), ret) +#define opal_atomic_sc_ptr(addr, value, ret) opal_atomic_sc_32((opal_atomic_int32_t *) (addr), (intptr_t) (value), ret) #define OPAL_HAVE_ATOMIC_LLSC_PTR 1 #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_LLSC_64 -#define opal_atomic_ll_ptr(addr, ret) opal_atomic_ll_64((volatile int64_t *) (addr), ret) -#define opal_atomic_sc_ptr(addr, value, ret) opal_atomic_sc_64((volatile int64_t *) (addr), (intptr_t) (value), ret) +#define opal_atomic_ll_ptr(addr, ret) opal_atomic_ll_64((opal_atomic_int64_t *) (addr), ret) +#define opal_atomic_sc_ptr(addr, value, ret) opal_atomic_sc_64((opal_atomic_int64_t *) (addr), (intptr_t) (value), ret) #define OPAL_HAVE_ATOMIC_LLSC_PTR 1 @@ -331,18 +331,18 @@ OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(_rel_) #if OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64 static inline void - opal_atomic_add_xx(volatile void* addr, int32_t value, size_t length) + opal_atomic_add_xx(opal_atomic_intptr_t* addr, int32_t value, size_t length) { switch( length ) { #if OPAL_HAVE_ATOMIC_ADD_32 case 4: - (void) opal_atomic_fetch_add_32( (volatile int32_t*)addr, (int32_t)value ); + (void) opal_atomic_fetch_add_32( (opal_atomic_int32_t*)addr, (int32_t)value ); break; #endif /* OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 */ #if OPAL_HAVE_ATOMIC_ADD_64 case 8: - (void) opal_atomic_fetch_add_64( (volatile int64_t*)addr, (int64_t)value ); + (void) opal_atomic_fetch_add_64( (opal_atomic_int64_t*)addr, (int64_t)value ); break; #endif /* OPAL_HAVE_ATOMIC_ADD_64 */ default: @@ -354,18 +354,18 @@ static inline void static inline void -opal_atomic_sub_xx(volatile void* addr, int32_t value, size_t length) +opal_atomic_sub_xx(opal_atomic_intptr_t* addr, int32_t value, size_t length) { switch( length ) { #if OPAL_HAVE_ATOMIC_SUB_32 case 4: - (void) opal_atomic_fetch_sub_32( (volatile int32_t*)addr, (int32_t)value ); + (void) opal_atomic_fetch_sub_32( (opal_atomic_int32_t*)addr, (int32_t)value ); break; #endif /* OPAL_HAVE_ATOMIC_SUB_32 */ #if OPAL_HAVE_ATOMIC_SUB_64 case 8: - (void) opal_atomic_fetch_sub_64( (volatile int64_t*)addr, (int64_t)value ); + (void) opal_atomic_fetch_sub_64( (opal_atomic_int64_t*)addr, (int64_t)value ); break; #endif /* OPAL_HAVE_ATOMIC_SUB_64 */ default: @@ -376,7 +376,7 @@ opal_atomic_sub_xx(volatile void* addr, int32_t value, size_t length) } #define OPAL_ATOMIC_DEFINE_OP_FETCH(op, operation, type, ptr_type, suffix) \ - static inline type opal_atomic_ ## op ## _fetch_ ## suffix (volatile ptr_type *addr, type value) \ + static inline type opal_atomic_ ## op ## _fetch_ ## suffix (opal_atomic_ ## ptr_type *addr, type value) \ { \ return opal_atomic_fetch_ ## op ## _ ## suffix (addr, value) operation value; \ } @@ -387,13 +387,13 @@ OPAL_ATOMIC_DEFINE_OP_FETCH(or, |, int32_t, int32_t, 32) OPAL_ATOMIC_DEFINE_OP_FETCH(xor, ^, int32_t, int32_t, 32) OPAL_ATOMIC_DEFINE_OP_FETCH(sub, -, int32_t, int32_t, 32) -static inline int32_t opal_atomic_min_fetch_32 (volatile int32_t *addr, int32_t value) +static inline int32_t opal_atomic_min_fetch_32 (opal_atomic_int32_t *addr, int32_t value) { int32_t old = opal_atomic_fetch_min_32 (addr, value); return old <= value ? old : value; } -static inline int32_t opal_atomic_max_fetch_32 (volatile int32_t *addr, int32_t value) +static inline int32_t opal_atomic_max_fetch_32 (opal_atomic_int32_t *addr, int32_t value) { int32_t old = opal_atomic_fetch_max_32 (addr, value); return old >= value ? old : value; @@ -406,13 +406,13 @@ OPAL_ATOMIC_DEFINE_OP_FETCH(or, |, int64_t, int64_t, 64) OPAL_ATOMIC_DEFINE_OP_FETCH(xor, ^, int64_t, int64_t, 64) OPAL_ATOMIC_DEFINE_OP_FETCH(sub, -, int64_t, int64_t, 64) -static inline int64_t opal_atomic_min_fetch_64 (volatile int64_t *addr, int64_t value) +static inline int64_t opal_atomic_min_fetch_64 (opal_atomic_int64_t *addr, int64_t value) { int64_t old = opal_atomic_fetch_min_64 (addr, value); return old <= value ? old : value; } -static inline int64_t opal_atomic_max_fetch_64 (volatile int64_t *addr, int64_t value) +static inline int64_t opal_atomic_max_fetch_64 (opal_atomic_int64_t *addr, int64_t value) { int64_t old = opal_atomic_fetch_max_64 (addr, value); return old >= value ? old : value; @@ -420,52 +420,52 @@ static inline int64_t opal_atomic_max_fetch_64 (volatile int64_t *addr, int64_t #endif -static inline intptr_t opal_atomic_fetch_add_ptr( volatile void* addr, +static inline intptr_t opal_atomic_fetch_add_ptr( opal_atomic_intptr_t* addr, void* delta ) { #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_ADD_32 - return opal_atomic_fetch_add_32((int32_t*) addr, (unsigned long) delta); + return opal_atomic_fetch_add_32((opal_atomic_int32_t*) addr, (unsigned long) delta); #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_ADD_64 - return opal_atomic_fetch_add_64((int64_t*) addr, (unsigned long) delta); + return opal_atomic_fetch_add_64((opal_atomic_int64_t*) addr, (unsigned long) delta); #else abort (); return 0; #endif } -static inline intptr_t opal_atomic_add_fetch_ptr( volatile void* addr, +static inline intptr_t opal_atomic_add_fetch_ptr( opal_atomic_intptr_t* addr, void* delta ) { #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_ADD_32 - return opal_atomic_add_fetch_32((int32_t*) addr, (unsigned long) delta); + return opal_atomic_add_fetch_32((opal_atomic_int32_t*) addr, (unsigned long) delta); #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_ADD_64 - return opal_atomic_add_fetch_64((int64_t*) addr, (unsigned long) delta); + return opal_atomic_add_fetch_64((opal_atomic_int64_t*) addr, (unsigned long) delta); #else abort (); return 0; #endif } -static inline intptr_t opal_atomic_fetch_sub_ptr( volatile void* addr, +static inline intptr_t opal_atomic_fetch_sub_ptr( opal_atomic_intptr_t* addr, void* delta ) { #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_SUB_32 - return opal_atomic_fetch_sub_32((int32_t*) addr, (unsigned long) delta); + return opal_atomic_fetch_sub_32((opal_atomic_int32_t*) addr, (unsigned long) delta); #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_SUB_32 - return opal_atomic_fetch_sub_64((int64_t*) addr, (unsigned long) delta); + return opal_atomic_fetch_sub_64((opal_atomic_int64_t*) addr, (unsigned long) delta); #else abort(); return 0; #endif } -static inline intptr_t opal_atomic_sub_fetch_ptr( volatile void* addr, +static inline intptr_t opal_atomic_sub_fetch_ptr( opal_atomic_intptr_t* addr, void* delta ) { #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_SUB_32 - return opal_atomic_sub_fetch_32((int32_t*) addr, (unsigned long) delta); + return opal_atomic_sub_fetch_32((opal_atomic_int32_t*) addr, (unsigned long) delta); #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_SUB_32 - return opal_atomic_sub_fetch_64((int64_t*) addr, (unsigned long) delta); + return opal_atomic_sub_fetch_64((opal_atomic_int64_t*) addr, (unsigned long) delta); #else abort(); return 0; diff --git a/opal/include/opal/sys/atomic_stdc.h b/opal/include/opal/sys/atomic_stdc.h new file mode 100644 index 0000000000..d545ed8f16 --- /dev/null +++ b/opal/include/opal/sys/atomic_stdc.h @@ -0,0 +1,261 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2018 Los Alamos National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +/* This file provides shims between the opal atomics interface and the C11 atomics interface. It + * is intended as the first step in moving to using C11 atomics across the entire codebase. Once + * all officially supported compilers offer C11 atomic (GCC 4.9.0+, icc 2018+, pgi, xlc, etc) then + * this shim will go away and the codebase will be updated to use C11's atomic support + * directly. + * This shim contains some functions already present in atomic_impl.h because we do not include + * atomic_impl.h when using C11 atomics. It would require alot of #ifdefs to avoid duplicate + * definitions to be worthwhile. */ + +#if !defined(OPAL_ATOMIC_STDC_H) +#define OPAL_ATOMIC_STDC_H + +#include +#include +#include "opal/include/opal_stdint.h" + +#define OPAL_HAVE_ATOMIC_MEM_BARRIER 1 + +#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1 +#define OPAL_HAVE_ATOMIC_SWAP_32 1 + +#define OPAL_HAVE_ATOMIC_MATH_32 1 +#define OPAL_HAVE_ATOMIC_ADD_32 1 +#define OPAL_HAVE_ATOMIC_AND_32 1 +#define OPAL_HAVE_ATOMIC_OR_32 1 +#define OPAL_HAVE_ATOMIC_XOR_32 1 +#define OPAL_HAVE_ATOMIC_SUB_32 1 + +#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1 +#define OPAL_HAVE_ATOMIC_SWAP_64 1 + +#define OPAL_HAVE_ATOMIC_MATH_64 1 +#define OPAL_HAVE_ATOMIC_ADD_64 1 +#define OPAL_HAVE_ATOMIC_AND_64 1 +#define OPAL_HAVE_ATOMIC_OR_64 1 +#define OPAL_HAVE_ATOMIC_XOR_64 1 +#define OPAL_HAVE_ATOMIC_SUB_64 1 + +#define OPAL_HAVE_ATOMIC_LLSC_32 0 +#define OPAL_HAVE_ATOMIC_LLSC_64 0 +#define OPAL_HAVE_ATOMIC_LLSC_PTR 0 + +#define OPAL_HAVE_ATOMIC_MIN_32 1 +#define OPAL_HAVE_ATOMIC_MAX_32 1 + +#define OPAL_HAVE_ATOMIC_MIN_64 1 +#define OPAL_HAVE_ATOMIC_MAX_64 1 + +#define OPAL_HAVE_ATOMIC_SPINLOCKS 1 + +static inline void opal_atomic_mb (void) +{ + atomic_thread_fence (memory_order_seq_cst); +} + +static inline void opal_atomic_wmb (void) +{ + atomic_thread_fence (memory_order_release); +} + +static inline void opal_atomic_rmb (void) +{ + atomic_thread_fence (memory_order_acquire); +} + +#define opal_atomic_compare_exchange_strong_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_relaxed, memory_order_relaxed) +#define opal_atomic_compare_exchange_strong_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_relaxed, memory_order_relaxed) +#define opal_atomic_compare_exchange_strong_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_relaxed, memory_order_relaxed) +#define opal_atomic_compare_exchange_strong_acq_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed) +#define opal_atomic_compare_exchange_strong_acq_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed) +#define opal_atomic_compare_exchange_strong_acq_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_acquire, memory_order_relaxed) + +#define opal_atomic_compare_exchange_strong_rel_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed) +#define opal_atomic_compare_exchange_strong_rel_64(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed) +#define opal_atomic_compare_exchange_strong_rel_ptr(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_release, memory_order_relaxed) + +#define opal_atomic_compare_exchange_strong(addr, oldval, newval) atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_relaxed, memory_order_relaxed) +#define opal_atomic_compare_exchange_strong_acq(addr, oldval, newval) atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_acquire, memory_order_relaxed) +#define opal_atomic_compare_exchange_strong_rel(addr, oldval, newval) atomic_compare_exchange_strong_explicit (addr, oldval, newval, memory_order_release, memory_order_relaxed) + +#define opal_atomic_swap_32(addr, value) atomic_exchange_explicit (addr, value, memory_order_relaxed) +#define opal_atomic_swap_64(addr, value) atomic_exchange_explicit (addr, value, memory_order_relaxed) +#define opal_atomic_swap_ptr(addr, value) atomic_exchange_explicit (addr, value, memory_order_relaxed) + +#define OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(op, bits, type, operator) \ + static inline type opal_atomic_fetch_ ## op ##_## bits (opal_atomic_ ## type *addr, type value) \ + { \ + return atomic_fetch_ ## op ## _explicit (addr, value, memory_order_relaxed); \ + } \ + \ + static inline type opal_atomic_## op ## _fetch_ ## bits (opal_atomic_ ## type *addr, type value) \ + { \ + return atomic_fetch_ ## op ## _explicit (addr, value, memory_order_relaxed) operator value; \ + } + +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, 32, int32_t, +) +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, 64, int64_t, +) +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(add, size_t, size_t, +) + +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, 32, int32_t, -) +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, 64, int64_t, -) +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(sub, size_t, size_t, -) + +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(or, 32, int32_t, |) +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(or, 64, int64_t, |) + +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(xor, 32, int32_t, ^) +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(xor, 64, int64_t, ^) + +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(and, 32, int32_t, &) +OPAL_ATOMIC_STDC_DEFINE_FETCH_OP(and, 64, int64_t, &) + +#define opal_atomic_add(addr, value) (void) atomic_fetch_add_explicit (addr, value, memory_order_relaxed) + +static inline int32_t opal_atomic_fetch_min_32 (opal_atomic_int32_t *addr, int32_t value) +{ + int32_t old = *addr; + do { + if (old <= value) { + break; + } + } while (!opal_atomic_compare_exchange_strong_32 (addr, &old, value)); + + return old; +} + +static inline int32_t opal_atomic_fetch_max_32 (opal_atomic_int32_t *addr, int32_t value) +{ + int32_t old = *addr; + do { + if (old >= value) { + break; + } + } while (!opal_atomic_compare_exchange_strong_32 (addr, &old, value)); + + return old; +} + +static inline int64_t opal_atomic_fetch_min_64 (opal_atomic_int64_t *addr, int64_t value) +{ + int64_t old = *addr; + do { + if (old <= value) { + break; + } + } while (!opal_atomic_compare_exchange_strong_64 (addr, &old, value)); + + return old; +} + +static inline int64_t opal_atomic_fetch_max_64 (opal_atomic_int64_t *addr, int64_t value) +{ + int64_t old = *addr; + do { + if (old >= value) { + break; + } + } while (!opal_atomic_compare_exchange_strong_64 (addr, &old, value)); + + return old; +} + +static inline int32_t opal_atomic_min_fetch_32 (opal_atomic_int32_t *addr, int32_t value) +{ + int32_t old = opal_atomic_fetch_min_32 (addr, value); + return old <= value ? old : value; +} + +static inline int32_t opal_atomic_max_fetch_32 (opal_atomic_int32_t *addr, int32_t value) +{ + int32_t old = opal_atomic_fetch_max_32 (addr, value); + return old >= value ? old : value; +} + +static inline int64_t opal_atomic_min_fetch_64 (opal_atomic_int64_t *addr, int64_t value) +{ + int64_t old = opal_atomic_fetch_min_64 (addr, value); + return old <= value ? old : value; +} + +static inline int64_t opal_atomic_max_fetch_64 (opal_atomic_int64_t *addr, int64_t value) +{ + int64_t old = opal_atomic_fetch_max_64 (addr, value); + return old >= value ? old : value; +} + +#define OPAL_ATOMIC_LOCK_UNLOCKED false +#define OPAL_ATOMIC_LOCK_LOCKED true + +#define OPAL_ATOMIC_LOCK_INIT ATOMIC_FLAG_INIT + +typedef atomic_flag opal_atomic_lock_t; + +/* + * Lock initialization function. It set the lock to UNLOCKED. + */ +static inline void opal_atomic_lock_init (opal_atomic_lock_t *lock, bool value) +{ + atomic_flag_clear (lock); +} + + +static inline int opal_atomic_trylock (opal_atomic_lock_t *lock) +{ + return (int) atomic_flag_test_and_set (lock); +} + + +static inline void opal_atomic_lock(opal_atomic_lock_t *lock) +{ + while (opal_atomic_trylock (lock)) { + } +} + + +static inline void opal_atomic_unlock (opal_atomic_lock_t *lock) +{ + atomic_flag_clear (lock); +} + + +#if OPAL_HAVE_C11_CSWAP_INT128 + +/* the C11 atomic compare-exchange is lock free so use it */ +#define opal_atomic_compare_exchange_strong_128 atomic_compare_exchange_strong + +#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1 + +#elif OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128 + +/* fall back on the __sync builtin if available since it will emit the expected instruction on x86_64 (cmpxchng16b) */ +__opal_attribute_always_inline__ +static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_int128_t *addr, + opal_int128_t *oldval, opal_int128_t newval) +{ + opal_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval); + bool ret = prev == *oldval; + *oldval = prev; + return ret; +} + +#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1 + +#else + +#define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 0 + +#endif + +#endif /* !defined(OPAL_ATOMIC_STDC_H) */ diff --git a/opal/include/opal/sys/gcc_builtin/atomic.h b/opal/include/opal/sys/gcc_builtin/atomic.h index c6ef6eb9c3..516f5fb10b 100644 --- a/opal/include/opal/sys/gcc_builtin/atomic.h +++ b/opal/include/opal/sys/gcc_builtin/atomic.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011 Sandia National Laboratories. All rights reserved. - * Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2016-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -81,98 +81,98 @@ static inline void opal_atomic_wmb(void) #pragma error_messages(off, E_ARG_INCOMPATIBLE_WITH_ARG_L) #endif -static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); } -static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED); } -static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); } -static inline int32_t opal_atomic_swap_32 (volatile int32_t *addr, int32_t newval) +static inline int32_t opal_atomic_swap_32 (opal_atomic_int32_t *addr, int32_t newval) { int32_t oldval; __atomic_exchange (addr, &newval, &oldval, __ATOMIC_RELAXED); return oldval; } -static inline int32_t opal_atomic_fetch_add_32(volatile int32_t *addr, int32_t delta) +static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t *addr, int32_t delta) { return __atomic_fetch_add (addr, delta, __ATOMIC_RELAXED); } -static inline int32_t opal_atomic_fetch_and_32(volatile int32_t *addr, int32_t value) +static inline int32_t opal_atomic_fetch_and_32(opal_atomic_int32_t *addr, int32_t value) { return __atomic_fetch_and (addr, value, __ATOMIC_RELAXED); } -static inline int32_t opal_atomic_fetch_or_32(volatile int32_t *addr, int32_t value) +static inline int32_t opal_atomic_fetch_or_32(opal_atomic_int32_t *addr, int32_t value) { return __atomic_fetch_or (addr, value, __ATOMIC_RELAXED); } -static inline int32_t opal_atomic_fetch_xor_32(volatile int32_t *addr, int32_t value) +static inline int32_t opal_atomic_fetch_xor_32(opal_atomic_int32_t *addr, int32_t value) { return __atomic_fetch_xor (addr, value, __ATOMIC_RELAXED); } -static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t *addr, int32_t delta) +static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t *addr, int32_t delta) { return __atomic_fetch_sub (addr, delta, __ATOMIC_RELAXED); } -static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); } -static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED); } -static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); } -static inline int64_t opal_atomic_swap_64 (volatile int64_t *addr, int64_t newval) +static inline int64_t opal_atomic_swap_64 (opal_atomic_int64_t *addr, int64_t newval) { int64_t oldval; __atomic_exchange (addr, &newval, &oldval, __ATOMIC_RELAXED); return oldval; } -static inline int64_t opal_atomic_fetch_add_64(volatile int64_t *addr, int64_t delta) +static inline int64_t opal_atomic_fetch_add_64(opal_atomic_int64_t *addr, int64_t delta) { return __atomic_fetch_add (addr, delta, __ATOMIC_RELAXED); } -static inline int64_t opal_atomic_fetch_and_64(volatile int64_t *addr, int64_t value) +static inline int64_t opal_atomic_fetch_and_64(opal_atomic_int64_t *addr, int64_t value) { return __atomic_fetch_and (addr, value, __ATOMIC_RELAXED); } -static inline int64_t opal_atomic_fetch_or_64(volatile int64_t *addr, int64_t value) +static inline int64_t opal_atomic_fetch_or_64(opal_atomic_int64_t *addr, int64_t value) { return __atomic_fetch_or (addr, value, __ATOMIC_RELAXED); } -static inline int64_t opal_atomic_fetch_xor_64(volatile int64_t *addr, int64_t value) +static inline int64_t opal_atomic_fetch_xor_64(opal_atomic_int64_t *addr, int64_t value) { return __atomic_fetch_xor (addr, value, __ATOMIC_RELAXED); } -static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t *addr, int64_t delta) +static inline int64_t opal_atomic_fetch_sub_64(opal_atomic_int64_t *addr, int64_t delta) { return __atomic_fetch_sub (addr, delta, __ATOMIC_RELAXED); } @@ -181,7 +181,7 @@ static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t *addr, int64_t d #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1 -static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128_t *addr, +static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_int128_t *addr, opal_int128_t *oldval, opal_int128_t newval) { return __atomic_compare_exchange_n (addr, oldval, newval, false, @@ -194,7 +194,7 @@ static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128 /* __atomic version is not lock-free so use legacy __sync version */ -static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128_t *addr, +static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_opal_int128_t *addr, opal_int128_t *oldval, opal_int128_t newval) { opal_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval); diff --git a/opal/include/opal/sys/ia32/atomic.h b/opal/include/opal/sys/ia32/atomic.h index bb863dec14..eac18b4b00 100644 --- a/opal/include/opal/sys/ia32/atomic.h +++ b/opal/include/opal/sys/ia32/atomic.h @@ -13,7 +13,7 @@ * Copyright (c) 2007-2010 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -84,7 +84,7 @@ static inline void opal_atomic_isync(void) *********************************************************************/ #if OPAL_GCC_INLINE_ASSEMBLY -static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { unsigned char ret; __asm__ __volatile__ ( @@ -106,7 +106,7 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add #define OPAL_HAVE_ATOMIC_SWAP_32 1 -static inline int32_t opal_atomic_swap_32( volatile int32_t *addr, +static inline int32_t opal_atomic_swap_32( opal_atomic_int32_t *addr, int32_t newval) { int32_t oldval; @@ -130,7 +130,7 @@ static inline int32_t opal_atomic_swap_32( volatile int32_t *addr, * * Atomically adds @i to @v. */ -static inline int32_t opal_atomic_fetch_add_32(volatile int32_t* v, int i) +static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t* v, int i) { int ret = i; __asm__ __volatile__( @@ -150,7 +150,7 @@ static inline int32_t opal_atomic_fetch_add_32(volatile int32_t* v, int i) * * Atomically subtracts @i from @v. */ -static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t* v, int i) +static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t* v, int i) { int ret = -i; __asm__ __volatile__( diff --git a/opal/include/opal/sys/powerpc/atomic.h b/opal/include/opal/sys/powerpc/atomic.h index da7414e5a0..90312d7543 100644 --- a/opal/include/opal/sys/powerpc/atomic.h +++ b/opal/include/opal/sys/powerpc/atomic.h @@ -144,7 +144,7 @@ void opal_atomic_isync(void) #define OPAL_ASM_VALUE64(x) x #endif -static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { int32_t prev; bool ret; @@ -170,7 +170,7 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add * load the arguments to/from the stack. This sequence may cause the ll reservation to be cancelled. */ #define opal_atomic_ll_32(addr, ret) \ do { \ - volatile int32_t *_addr = (addr); \ + opal_atomic_int32_t *_addr = (addr); \ int32_t _ret; \ __asm__ __volatile__ ("lwarx %0, 0, %1 \n\t" \ : "=&r" (_ret) \ @@ -181,7 +181,7 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add #define opal_atomic_sc_32(addr, value, ret) \ do { \ - volatile int32_t *_addr = (addr); \ + opal_atomic_int32_t *_addr = (addr); \ int32_t _ret, _foo, _newval = (int32_t) value; \ \ __asm__ __volatile__ (" stwcx. %4, 0, %3 \n\t" \ @@ -200,7 +200,7 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { bool rc; @@ -211,13 +211,13 @@ static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t } -static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { opal_atomic_wmb(); return opal_atomic_compare_exchange_strong_32 (addr, oldval, newval); } -static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, int32_t newval) +static inline int32_t opal_atomic_swap_32(opal_atomic_int32_t *addr, int32_t newval) { int32_t ret; @@ -239,7 +239,7 @@ static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, int32_t newval #if OPAL_GCC_INLINE_ASSEMBLY #define OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(type, instr) \ -static inline int64_t opal_atomic_fetch_ ## type ## _64(volatile int64_t* v, int64_t val) \ +static inline int64_t opal_atomic_fetch_ ## type ## _64(opal_atomic_int64_t* v, int64_t val) \ { \ int64_t t, old; \ \ @@ -261,7 +261,7 @@ OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(or, or) OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(xor, xor) OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(sub, subf) -static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { int64_t prev; bool ret; @@ -284,7 +284,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add #define opal_atomic_ll_64(addr, ret) \ do { \ - volatile int64_t *_addr = (addr); \ + opal_atomic_int64_t *_addr = (addr); \ int64_t _ret; \ __asm__ __volatile__ ("ldarx %0, 0, %1 \n\t" \ : "=&r" (_ret) \ @@ -295,7 +295,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add #define opal_atomic_sc_64(addr, value, ret) \ do { \ - volatile int64_t *_addr = (addr); \ + opal_atomic_int64_t *_addr = (addr); \ int64_t _foo, _newval = (int64_t) value; \ int32_t _ret; \ \ @@ -310,7 +310,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add ret = _ret; \ } while (0) -static inline int64_t opal_atomic_swap_64(volatile int64_t *addr, int64_t newval) +static inline int64_t opal_atomic_swap_64(opal_atomic_int64_t *addr, int64_t newval) { int64_t ret; @@ -335,7 +335,7 @@ static inline int64_t opal_atomic_swap_64(volatile int64_t *addr, int64_t newval #if OPAL_GCC_INLINE_ASSEMBLY -static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { int64_t prev; int ret; @@ -382,7 +382,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { bool rc; @@ -393,7 +393,7 @@ static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t } -static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { opal_atomic_wmb(); return opal_atomic_compare_exchange_strong_64 (addr, oldval, newval); @@ -401,7 +401,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t #define OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_32(type, instr) \ -static inline int32_t opal_atomic_fetch_ ## type ## _32(volatile int32_t* v, int val) \ +static inline int32_t opal_atomic_fetch_ ## type ## _32(opal_atomic_int32_t* v, int val) \ { \ int32_t t, old; \ \ diff --git a/opal/include/opal/sys/sparcv9/atomic.h b/opal/include/opal/sys/sparcv9/atomic.h index c79e32b1eb..e91d03cd92 100644 --- a/opal/include/opal/sys/sparcv9/atomic.h +++ b/opal/include/opal/sys/sparcv9/atomic.h @@ -13,7 +13,7 @@ * Copyright (c) 2007 Sun Microsystems, Inc. All rights reserverd. * Copyright (c) 2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2017-2018 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -85,7 +85,7 @@ static inline void opal_atomic_isync(void) *********************************************************************/ #if OPAL_GCC_INLINE_ASSEMBLY -static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { /* casa [reg(rs1)] %asi, reg(rs2), reg(rd) * @@ -107,7 +107,7 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add } -static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { bool rc; @@ -118,7 +118,7 @@ static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t } -static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { opal_atomic_wmb(); return opal_atomic_compare_exchange_strong_32 (addr, oldval, newval); @@ -127,7 +127,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t #if OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64 -static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { /* casa [reg(rs1)] %asi, reg(rs2), reg(rd) * @@ -149,7 +149,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add #else /* OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64 */ -static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { /* casa [reg(rs1)] %asi, reg(rs2), reg(rd) * @@ -179,7 +179,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add #endif /* OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64 */ -static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { bool rc; @@ -190,7 +190,7 @@ static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t } -static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { opal_atomic_wmb(); return opal_atomic_compare_exchange_strong_64 (addr, oldval, newval); diff --git a/opal/include/opal/sys/sync_builtin/atomic.h b/opal/include/opal/sys/sync_builtin/atomic.h index 11decc2f0a..6a9c5386ed 100644 --- a/opal/include/opal/sys/sync_builtin/atomic.h +++ b/opal/include/opal/sys/sync_builtin/atomic.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011 Sandia National Laboratories. All rights reserved. - * Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -57,7 +57,7 @@ static inline void opal_atomic_wmb(void) #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1 -static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { int32_t prev = __sync_val_compare_and_swap (addr, *oldval, newval); bool ret = prev == *oldval; @@ -71,31 +71,31 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add #define OPAL_HAVE_ATOMIC_MATH_32 1 #define OPAL_HAVE_ATOMIC_ADD_32 1 -static inline int32_t opal_atomic_fetch_add_32(volatile int32_t *addr, int32_t delta) +static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t *addr, int32_t delta) { return __sync_fetch_and_add(addr, delta); } #define OPAL_HAVE_ATOMIC_AND_32 1 -static inline int32_t opal_atomic_fetch_and_32(volatile int32_t *addr, int32_t value) +static inline int32_t opal_atomic_fetch_and_32(opal_atomic_int32_t *addr, int32_t value) { return __sync_fetch_and_and(addr, value); } #define OPAL_HAVE_ATOMIC_OR_32 1 -static inline int32_t opal_atomic_fetch_or_32(volatile int32_t *addr, int32_t value) +static inline int32_t opal_atomic_fetch_or_32(opal_atomic_int32_t *addr, int32_t value) { return __sync_fetch_and_or(addr, value); } #define OPAL_HAVE_ATOMIC_XOR_32 1 -static inline int32_t opal_atomic_fetch_xor_32(volatile int32_t *addr, int32_t value) +static inline int32_t opal_atomic_fetch_xor_32(opal_atomic_int32_t *addr, int32_t value) { return __sync_fetch_and_xor(addr, value); } #define OPAL_HAVE_ATOMIC_SUB_32 1 -static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t *addr, int32_t delta) +static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t *addr, int32_t delta) { return __sync_fetch_and_sub(addr, delta); } @@ -104,7 +104,7 @@ static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t *addr, int32_t d #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1 -static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { int64_t prev = __sync_val_compare_and_swap (addr, *oldval, newval); bool ret = prev == *oldval; @@ -117,31 +117,31 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add #define OPAL_HAVE_ATOMIC_MATH_64 1 #define OPAL_HAVE_ATOMIC_ADD_64 1 -static inline int64_t opal_atomic_fetch_add_64(volatile int64_t *addr, int64_t delta) +static inline int64_t opal_atomic_fetch_add_64(opal_atomic_int64_t *addr, int64_t delta) { return __sync_fetch_and_add(addr, delta); } #define OPAL_HAVE_ATOMIC_AND_64 1 -static inline int64_t opal_atomic_fetch_and_64(volatile int64_t *addr, int64_t value) +static inline int64_t opal_atomic_fetch_and_64(opal_atomic_int64_t *addr, int64_t value) { return __sync_fetch_and_and(addr, value); } #define OPAL_HAVE_ATOMIC_OR_64 1 -static inline int64_t opal_atomic_fetch_or_64(volatile int64_t *addr, int64_t value) +static inline int64_t opal_atomic_fetch_or_64(opal_atomic_int64_t *addr, int64_t value) { return __sync_fetch_and_or(addr, value); } #define OPAL_HAVE_ATOMIC_XOR_64 1 -static inline int64_t opal_atomic_fetch_xor_64(volatile int64_t *addr, int64_t value) +static inline int64_t opal_atomic_fetch_xor_64(opal_atomic_int64_t *addr, int64_t value) { return __sync_fetch_and_xor(addr, value); } #define OPAL_HAVE_ATOMIC_SUB_64 1 -static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t *addr, int64_t delta) +static inline int64_t opal_atomic_fetch_sub_64(opal_atomic_int64_t *addr, int64_t delta) { return __sync_fetch_and_sub(addr, delta); } @@ -149,7 +149,7 @@ static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t *addr, int64_t d #endif #if OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128 -static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128_t *addr, +static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_int128_t *addr, opal_int128_t *oldval, opal_int128_t newval) { opal_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval); diff --git a/opal/include/opal/sys/x86_64/atomic.h b/opal/include/opal/sys/x86_64/atomic.h index 49d740de38..8853eb5c2b 100644 --- a/opal/include/opal/sys/x86_64/atomic.h +++ b/opal/include/opal/sys/x86_64/atomic.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007 Sun Microsystems, Inc. All rights reserverd. - * Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2016-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -82,7 +82,7 @@ static inline void opal_atomic_isync(void) *********************************************************************/ #if OPAL_GCC_INLINE_ASSEMBLY -static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { unsigned char ret; __asm__ __volatile__ ( @@ -102,13 +102,13 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add #if OPAL_GCC_INLINE_ASSEMBLY -static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { unsigned char ret; __asm__ __volatile__ ( SMPLOCK "cmpxchgq %3,%2 \n\t" "sete %0 \n\t" - : "=qm" (ret), "+a" (*oldval), "+m" (*((volatile long*)addr)) + : "=qm" (ret), "+a" (*oldval), "+m" (*((opal_atomic_long_t *)addr)) : "q"(newval) : "memory", "cc" ); @@ -123,7 +123,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add #if OPAL_GCC_INLINE_ASSEMBLY && OPAL_HAVE_CMPXCHG16B && HAVE_OPAL_INT128_T -static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128_t *addr, opal_int128_t *oldval, opal_int128_t newval) +static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_int128_t *addr, opal_int128_t *oldval, opal_int128_t newval) { unsigned char ret; @@ -150,7 +150,7 @@ static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128 #define OPAL_HAVE_ATOMIC_SWAP_64 1 -static inline int32_t opal_atomic_swap_32( volatile int32_t *addr, +static inline int32_t opal_atomic_swap_32( opal_atomic_int32_t *addr, int32_t newval) { int32_t oldval; @@ -166,7 +166,7 @@ static inline int32_t opal_atomic_swap_32( volatile int32_t *addr, #if OPAL_GCC_INLINE_ASSEMBLY -static inline int64_t opal_atomic_swap_64( volatile int64_t *addr, +static inline int64_t opal_atomic_swap_64( opal_atomic_int64_t *addr, int64_t newval) { int64_t oldval; @@ -196,7 +196,7 @@ static inline int64_t opal_atomic_swap_64( volatile int64_t *addr, * * Atomically adds @i to @v. */ -static inline int32_t opal_atomic_fetch_add_32(volatile int32_t* v, int i) +static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t* v, int i) { int ret = i; __asm__ __volatile__( @@ -217,7 +217,7 @@ static inline int32_t opal_atomic_fetch_add_32(volatile int32_t* v, int i) * * Atomically adds @i to @v. */ -static inline int64_t opal_atomic_fetch_add_64(volatile int64_t* v, int64_t i) +static inline int64_t opal_atomic_fetch_add_64(opal_atomic_int64_t* v, int64_t i) { int64_t ret = i; __asm__ __volatile__( @@ -238,7 +238,7 @@ static inline int64_t opal_atomic_fetch_add_64(volatile int64_t* v, int64_t i) * * Atomically subtracts @i from @v. */ -static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t* v, int i) +static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t* v, int i) { int ret = -i; __asm__ __volatile__( @@ -259,7 +259,7 @@ static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t* v, int i) * * Atomically subtracts @i from @v. */ -static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t* v, int64_t i) +static inline int64_t opal_atomic_fetch_sub_64(opal_atomic_int64_t* v, int64_t i) { int64_t ret = -i; __asm__ __volatile__( diff --git a/opal/include/opal_stdatomic.h b/opal/include/opal_stdatomic.h new file mode 100644 index 0000000000..854413c87f --- /dev/null +++ b/opal/include/opal_stdatomic.h @@ -0,0 +1,66 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2018 Los Alamos National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#if !defined(OPAL_STDATOMIC_H) +#define OPAL_STDATOMIC_H + +#include "opal_stdint.h" + +#if OPAL_ASSEMBLY_BUILTIN != OPAL_BUILTIN_C11 + +typedef volatile int opal_atomic_int_t; +typedef volatile long opal_atomic_long_t; + +typedef volatile int32_t opal_atomic_int32_t; +typedef volatile uint32_t opal_atomic_uint32_t; +typedef volatile int64_t opal_atomic_int64_t; +typedef volatile uint64_t opal_atomic_uint64_t; + +typedef volatile size_t opal_atomic_size_t; +typedef volatile ssize_t opal_atomic_ssize_t; +typedef volatile intptr_t opal_atomic_intptr_t; +typedef volatile uintptr_t opal_atomic_uintptr_t; + +#else /* OPAL_HAVE_C__ATOMIC */ + +#include + +typedef atomic_int opal_atomic_int_t; +typedef atomic_long opal_atomic_long_t; + +typedef _Atomic int32_t opal_atomic_int32_t; +typedef _Atomic uint32_t opal_atomic_uint32_t; +typedef _Atomic int64_t opal_atomic_int64_t; +typedef _Atomic uint64_t opal_atomic_uint64_t; + +typedef _Atomic size_t opal_atomic_size_t; +typedef _Atomic ssize_t opal_atomic_ssize_t; +typedef _Atomic intptr_t opal_atomic_intptr_t; +typedef _Atomic uintptr_t opal_atomic_uintptr_t; + +#endif /* OPAL_HAVE_C__ATOMIC */ + +#if HAVE_OPAL_INT128_T + +/* do not use C11 atomics for __int128 if they are not lock free */ +#if OPAL_HAVE_C11_CSWAP_INT128 + +typedef _Atomic opal_int128_t opal_atomic_int128_t; + +#else + +typedef volatile opal_int128_t opal_atomic_int128_t; + +#endif + +#endif + +#endif /* !defined(OPAL_STDATOMIC_H) */ diff --git a/opal/include/opal_stdint.h b/opal/include/opal_stdint.h index 4089cb55a9..efbd72c645 100644 --- a/opal/include/opal_stdint.h +++ b/opal/include/opal_stdint.h @@ -28,6 +28,8 @@ #ifndef OPAL_STDINT_H #define OPAL_STDINT_H 1 +#include "opal_config.h" + /* * Include what we can and define what is missing. */ diff --git a/opal/mca/btl/ofi/btl_ofi.h b/opal/mca/btl/ofi/btl_ofi.h index a96bbf0b56..f8e44d93c7 100644 --- a/opal/mca/btl/ofi/btl_ofi.h +++ b/opal/mca/btl/ofi/btl_ofi.h @@ -123,8 +123,8 @@ struct mca_btl_ofi_module_t { bool use_virt_addr; bool is_scalable_ep; - int64_t outstanding_rdma; - int64_t outstanding_send; + opal_atomic_int64_t outstanding_rdma; + opal_atomic_int64_t outstanding_send; /** linked list of BTL endpoints. this list is never searched so * there is no need for a complicated structure here at this time*/ diff --git a/opal/mca/btl/openib/btl_openib.h b/opal/mca/btl/openib/btl_openib.h index 6b4dd0466b..bb657f0608 100644 --- a/opal/mca/btl/openib/btl_openib.h +++ b/opal/mca/btl/openib/btl_openib.h @@ -237,7 +237,7 @@ struct mca_btl_openib_component_t { int apm_lmc; int apm_ports; unsigned int buffer_alignment; /**< Preferred communication buffer alignment in Bytes (must be power of two) */ - int32_t error_counter; /**< Counts number on error events that we got on all devices */ + opal_atomic_int32_t error_counter; /**< Counts number on error events that we got on all devices */ opal_event_base_t *async_evbase; /**< Async event base */ bool use_async_event_thread; /**< Use the async event handler */ mca_btl_openib_srq_manager_t srq_manager; /**< Hash table for all BTL SRQs */ @@ -404,8 +404,8 @@ typedef struct mca_btl_openib_device_t { #endif int xrc_fd; #endif - int32_t non_eager_rdma_endpoints; - int32_t eager_rdma_buffers_count; + opal_atomic_int32_t non_eager_rdma_endpoints; + opal_atomic_int32_t eager_rdma_buffers_count; struct mca_btl_base_endpoint_t **eager_rdma_buffers; /**< frags for control massages */ opal_free_list_t send_free_control; @@ -428,8 +428,8 @@ struct mca_btl_openib_module_pp_qp_t { struct mca_btl_openib_module_srq_qp_t { struct ibv_srq *srq; - int32_t rd_posted; - int32_t sd_credits; /* the max number of outstanding sends on a QP when using SRQ */ + opal_atomic_int32_t rd_posted; + opal_atomic_int32_t sd_credits; /* the max number of outstanding sends on a QP when using SRQ */ /* i.e. the number of frags that can be outstanding (down counter) */ opal_list_t pending_frags[2]; /**< list of high/low prio frags */ /** The number of receive buffers that can be post in the current time. @@ -487,12 +487,12 @@ struct mca_btl_openib_module_t { int apm_port; /**< Alternative port that may be used for APM */ uint8_t src_path_bits; /**< offset from base lid (for LMC) */ - int32_t num_peers; + opal_atomic_int32_t num_peers; opal_mutex_t ib_lock; /**< module level lock */ size_t eager_rdma_frag_size; /**< length of eager frag */ - volatile int32_t eager_rdma_channels; /**< number of open RDMA channels */ + opal_atomic_int32_t eager_rdma_channels; /**< number of open RDMA channels */ mca_btl_base_module_error_cb_fn_t error_cb; /**< error handler */ diff --git a/opal/mca/btl/openib/btl_openib_async.c b/opal/mca/btl/openib/btl_openib_async.c index 5c52f9566b..3957bae2a9 100644 --- a/opal/mca/btl/openib/btl_openib_async.c +++ b/opal/mca/btl/openib/btl_openib_async.c @@ -4,7 +4,7 @@ * Copyright (c) 2007-2013 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2006-2007 Voltaire All rights reserved. * Copyright (c) 2009-2010 Oracle and/or its affiliates. All rights reserved - * Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights + * Copyright (c) 2013-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2014 Intel, Inc. All rights reserved. * Copyright (c) 2014 Bull SAS. All rights reserved. @@ -38,7 +38,7 @@ static opal_list_t ignore_qp_err_list; static opal_mutex_t ignore_qp_err_list_lock; -static int32_t btl_openib_async_device_count = 0; +static opal_atomic_int32_t btl_openib_async_device_count = 0; struct mca_btl_openib_async_poll { int active_poll_size; diff --git a/opal/mca/btl/openib/btl_openib_eager_rdma.h b/opal/mca/btl/openib/btl_openib_eager_rdma.h index 5acb038177..808178a457 100644 --- a/opal/mca/btl/openib/btl_openib_eager_rdma.h +++ b/opal/mca/btl/openib/btl_openib_eager_rdma.h @@ -1,7 +1,7 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2006-2007 Voltaire All rights reserved. - * Copyright (c) 2015 Los Alamos National Security, LLC. All rights + * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -25,7 +25,7 @@ struct mca_btl_openib_eager_rdma_local_t { mca_btl_openib_reg_t *reg; uint16_t head; /**< RDMA buffer to poll */ uint16_t tail; /**< Needed for credit managment */ - int32_t credits; /**< number of RDMA credits */ + opal_atomic_int32_t credits; /**< number of RDMA credits */ int32_t rd_win; #if OPAL_ENABLE_DEBUG uint32_t seq; @@ -38,8 +38,8 @@ typedef struct mca_btl_openib_eager_rdma_local_t mca_btl_openib_eager_rdma_local struct mca_btl_openib_eager_rdma_remote_t { opal_ptr_t base; /**< address of remote buffer */ uint32_t rkey; /**< RKey for accessing remote buffer */ - int32_t head; /**< RDMA buffer to post to */ - int32_t tokens; /**< number of rdam tokens */ + opal_atomic_int32_t head; /**< RDMA buffer to post to */ + opal_atomic_int32_t tokens; /**< number of rdma tokens */ #if OPAL_ENABLE_DEBUG uint32_t seq; #endif @@ -108,7 +108,7 @@ typedef struct mca_btl_openib_eager_rdma_remote_t mca_btl_openib_eager_rdma_remo #define MCA_BTL_OPENIB_RDMA_MOVE_INDEX(HEAD, OLD_HEAD) \ do { \ - (OLD_HEAD) = (OPAL_THREAD_ADD_FETCH32(&(HEAD), 1) - 1) % mca_btl_openib_component.eager_rdma_num; \ + (OLD_HEAD) = (OPAL_THREAD_ADD_FETCH32((opal_atomic_int32_t *) &(HEAD), 1) - 1) % mca_btl_openib_component.eager_rdma_num; \ } while(0) #endif diff --git a/opal/mca/btl/openib/btl_openib_endpoint.c b/opal/mca/btl/openib/btl_openib_endpoint.c index be01664b1c..e1f9a32681 100644 --- a/opal/mca/btl/openib/btl_openib_endpoint.c +++ b/opal/mca/btl/openib/btl_openib_endpoint.c @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006-2013 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2006-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2006-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2006-2007 Voltaire All rights reserved. * Copyright (c) 2006-2009 Mellanox Technologies, Inc. All rights reserved. @@ -378,7 +378,7 @@ static void mca_btl_openib_endpoint_destruct(mca_btl_base_endpoint_t* endpoint) * was not in "connect" or "bad" flow (failed to allocate memory) * and changed the pointer back to NULL */ - if(!opal_atomic_compare_exchange_strong_ptr(&endpoint->eager_rdma_local.base.pval, (void *) &_tmp_ptr, (void *) 1)) { + if(!opal_atomic_compare_exchange_strong_ptr((opal_atomic_intptr_t *) &endpoint->eager_rdma_local.base.pval, (intptr_t *) &_tmp_ptr, 1)) { if (NULL != endpoint->eager_rdma_local.reg) { endpoint->endpoint_btl->device->rcache->rcache_deregister (endpoint->endpoint_btl->device->rcache, &endpoint->eager_rdma_local.reg->base); @@ -899,8 +899,7 @@ void mca_btl_openib_endpoint_connect_eager_rdma( /* Set local rdma pointer to 1 temporarily so other threads will not try * to enter the function */ - if(!opal_atomic_compare_exchange_strong_ptr (&endpoint->eager_rdma_local.base.pval, (void *) &_tmp_ptr, - (void *) 1)) { + if(!opal_atomic_compare_exchange_strong_ptr ((opal_atomic_intptr_t *) &endpoint->eager_rdma_local.base.pval, (intptr_t *) &_tmp_ptr, 1)) { return; } @@ -990,7 +989,7 @@ void mca_btl_openib_endpoint_connect_eager_rdma( do { _tmp_ptr = NULL; p = &device->eager_rdma_buffers[device->eager_rdma_buffers_count]; - } while(!opal_atomic_compare_exchange_strong_ptr (p, (void *) &_tmp_ptr, endpoint)); + } while(!opal_atomic_compare_exchange_strong_ptr ((opal_atomic_intptr_t *) p, (intptr_t *) &_tmp_ptr, (intptr_t) endpoint)); OPAL_THREAD_ADD_FETCH32(&openib_btl->eager_rdma_channels, 1); /* from this point progress function starts to poll new buffer */ diff --git a/opal/mca/btl/openib/btl_openib_endpoint.h b/opal/mca/btl/openib/btl_openib_endpoint.h index 89c42c595e..b3901c5665 100644 --- a/opal/mca/btl/openib/btl_openib_endpoint.h +++ b/opal/mca/btl/openib/btl_openib_endpoint.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2009 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2006-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2006-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2006-2007 Voltaire All rights reserved. * Copyright (c) 2007-2009 Mellanox Technologies. All rights reserved. @@ -117,17 +117,17 @@ typedef struct mca_btl_openib_rem_info_t { * Agggregates all per peer qp info for an endpoint */ typedef struct mca_btl_openib_endpoint_pp_qp_t { - int32_t sd_credits; /**< this rank's view of the credits + opal_atomic_int32_t sd_credits; /**< this rank's view of the credits * available for sending: * this is the credits granted by the * remote peer which has some relation to the * number of receive buffers posted remotely */ - int32_t rd_posted; /**< number of descriptors posted to the nic*/ - int32_t rd_credits; /**< number of credits to return to peer */ - int32_t cm_received; /**< Credit messages received */ - int32_t cm_return; /**< how may credits to return */ - int32_t cm_sent; /**< Outstanding number of credit messages */ + opal_atomic_int32_t rd_posted; /**< number of descriptors posted to the nic*/ + opal_atomic_int32_t rd_credits; /**< number of credits to return to peer */ + opal_atomic_int32_t cm_received; /**< Credit messages received */ + opal_atomic_int32_t cm_return; /**< how may credits to return */ + opal_atomic_int32_t cm_sent; /**< Outstanding number of credit messages */ } mca_btl_openib_endpoint_pp_qp_t; @@ -141,8 +141,8 @@ typedef struct mca_btl_openib_endpoint_srq_qp_t { typedef struct mca_btl_openib_qp_t { struct ibv_qp *lcl_qp; uint32_t lcl_psn; - volatile int32_t sd_wqe; /**< number of available send wqe entries */ - int32_t sd_wqe_inflight; + opal_atomic_int32_t sd_wqe; /**< number of available send wqe entries */ + opal_atomic_int32_t sd_wqe_inflight; int wqe_count; int users; opal_mutex_t lock; @@ -154,7 +154,7 @@ typedef struct mca_btl_openib_endpoint_qp_t { available */ opal_list_t no_wqe_pending_frags[2]; /**< put fragments here if there is no wqe available */ - int32_t rd_credit_send_lock; /**< Lock credit send fragment */ + opal_atomic_int32_t rd_credit_send_lock; /**< Lock credit send fragment */ mca_btl_openib_send_control_frag_t *credit_frag; size_t ib_inline_max; /**< max size of inline send*/ union { @@ -226,7 +226,7 @@ struct mca_btl_base_endpoint_t { opal_list_t pending_put_frags; /** number of available get tokens */ - int32_t get_tokens; + opal_atomic_int32_t get_tokens; /** subnet id of this endpoint*/ uint64_t subnet_id; @@ -235,7 +235,7 @@ struct mca_btl_base_endpoint_t { struct ib_address_t *ib_addr; /** number of eager received */ - int32_t eager_recv_count; + opal_atomic_int32_t eager_recv_count; /** info about remote RDMA buffer */ mca_btl_openib_eager_rdma_remote_t eager_rdma_remote; /** info about local RDMA buffer */ diff --git a/opal/mca/btl/portals4/btl_portals4.h b/opal/mca/btl/portals4/btl_portals4.h index 92c294b80a..41e7da7a74 100644 --- a/opal/mca/btl/portals4/btl_portals4.h +++ b/opal/mca/btl/portals4/btl_portals4.h @@ -92,7 +92,7 @@ struct mca_btl_portals4_module_t { /* number of processes we're actively connected to. Needed to know when to do activation / shutdown */ - int32_t portals_num_procs; + opal_atomic_int32_t portals_num_procs; /* number of the interface (btl) */ uint32_t interface_num; @@ -111,11 +111,11 @@ struct mca_btl_portals4_module_t { ptl_handle_eq_t recv_eq_h; /* number outstanding sends and local rdma */ - volatile int32_t portals_outstanding_ops; + opal_atomic_int32_t portals_outstanding_ops; int32_t portals_max_outstanding_ops; /* key to use for next rdma operation */ - volatile int64_t portals_rdma_key; + opal_atomic_int64_t portals_rdma_key; /* our portals network interface */ ptl_handle_ni_t portals_ni_h; diff --git a/opal/mca/btl/smcuda/btl_smcuda.h b/opal/mca/btl/smcuda/btl_smcuda.h index 2fe7df377d..caff2622e3 100644 --- a/opal/mca/btl/smcuda/btl_smcuda.h +++ b/opal/mca/btl/smcuda/btl_smcuda.h @@ -159,8 +159,8 @@ struct mca_btl_smcuda_component_t { struct mca_btl_base_endpoint_t **sm_peers; opal_free_list_t pending_send_fl; - int num_outstanding_frags; /**< number of fragments sent but not yet returned to free list */ - int num_pending_sends; /**< total number on all of my pending-send queues */ + opal_atomic_int32_t num_outstanding_frags; /**< number of fragments sent but not yet returned to free list */ + opal_atomic_int32_t num_pending_sends; /**< total number on all of my pending-send queues */ int mem_node; int num_mem_nodes; diff --git a/opal/mca/btl/uct/btl_uct_device_context.h b/opal/mca/btl/uct/btl_uct_device_context.h index ccb4f3be71..c2d1fac2f3 100644 --- a/opal/mca/btl/uct/btl_uct_device_context.h +++ b/opal/mca/btl/uct/btl_uct_device_context.h @@ -59,7 +59,7 @@ static inline void mca_btl_uct_context_unlock (mca_btl_uct_device_context_t *con static inline int mca_btl_uct_get_context_index (void) { - static volatile uint32_t next_uct_index = 0; + static opal_atomic_uint32_t next_uct_index = 0; int context_id; #if OPAL_C_HAVE__THREAD_LOCAL @@ -68,7 +68,7 @@ static inline int mca_btl_uct_get_context_index (void) context_id = uct_index; if (OPAL_UNLIKELY(-1 == context_id)) { - context_id = uct_index = opal_atomic_fetch_add_32 ((volatile int32_t *) &next_uct_index, 1) % + context_id = uct_index = opal_atomic_fetch_add_32 ((opal_atomic_int32_t *) &next_uct_index, 1) % mca_btl_uct_component.num_contexts_per_module; } } else { @@ -92,7 +92,7 @@ mca_btl_uct_module_get_tl_context_specific (mca_btl_uct_module_t *module, mca_bt mca_btl_uct_device_context_t *new_context; new_context = mca_btl_uct_context_create (module, tl, context_id); - if (!opal_atomic_compare_exchange_strong_ptr (&tl->uct_dev_contexts[context_id], &context, new_context)) { + if (!opal_atomic_compare_exchange_strong_ptr ((opal_atomic_intptr_t *) &tl->uct_dev_contexts[context_id], &context, new_context)) { mca_btl_uct_context_destroy (new_context); } else { context = new_context; diff --git a/opal/mca/btl/uct/btl_uct_types.h b/opal/mca/btl/uct/btl_uct_types.h index f7731d9e44..86619f28e1 100644 --- a/opal/mca/btl/uct/btl_uct_types.h +++ b/opal/mca/btl/uct/btl_uct_types.h @@ -94,7 +94,7 @@ typedef struct mca_btl_uct_conn_req_t mca_btl_uct_conn_req_t; */ struct mca_btl_uct_tl_endpoint_t { /** current flags (connected, requested, etc) */ - volatile int32_t flags; + opal_atomic_int32_t flags; /** UCT endpoint handle */ uct_ep_h uct_ep; diff --git a/opal/mca/btl/ugni/btl_ugni.h b/opal/mca/btl/ugni/btl_ugni.h index 367391a9ed..c56dc32326 100644 --- a/opal/mca/btl/ugni/btl_ugni.h +++ b/opal/mca/btl/ugni/btl_ugni.h @@ -114,7 +114,7 @@ struct mca_btl_ugni_device_t { int dev_index; /** number of SMSG connections */ - volatile int32_t smsg_connections; + opal_atomic_int32_t smsg_connections; /** boolean indicating that the device was recently flushed */ volatile bool flushed; @@ -168,7 +168,7 @@ typedef struct mca_btl_ugni_module_t { gni_ep_handle_t wildcard_ep; struct mca_btl_base_endpoint_t *local_ep; - volatile int32_t active_datagrams; + opal_atomic_int32_t active_datagrams; opal_event_t connection_event; struct mca_btl_ugni_endpoint_attr_t wc_remote_attr, wc_local_attr; @@ -188,13 +188,13 @@ typedef struct mca_btl_ugni_module_t { opal_pointer_array_t pending_smsg_frags_bb; int32_t reg_max; - volatile int32_t reg_count; + opal_atomic_int32_t reg_count; /* used to calculate the fraction of registered memory resources * this rank should be limited too */ int nlocal_procs; - volatile int32_t active_rdma_count; + opal_atomic_int32_t active_rdma_count; mca_rcache_base_module_t *rcache; } mca_btl_ugni_module_t; diff --git a/opal/mca/btl/ugni/btl_ugni_endpoint.h b/opal/mca/btl/ugni/btl_ugni_endpoint.h index 3ece68d671..71bfcca901 100644 --- a/opal/mca/btl/ugni/btl_ugni_endpoint.h +++ b/opal/mca/btl/ugni/btl_ugni_endpoint.h @@ -70,7 +70,7 @@ typedef struct mca_btl_base_endpoint_t { bool dg_posted; /** protect against re-entry to SMSG */ - int32_t smsg_progressing; + opal_atomic_int32_t smsg_progressing; int index; } mca_btl_base_endpoint_t; diff --git a/opal/mca/btl/ugni/btl_ugni_frag.h b/opal/mca/btl/ugni/btl_ugni_frag.h index 7b4a6a8d0d..c8e91568c9 100644 --- a/opal/mca/btl/ugni/btl_ugni_frag.h +++ b/opal/mca/btl/ugni/btl_ugni_frag.h @@ -68,7 +68,7 @@ struct mca_btl_ugni_base_frag_t; typedef struct mca_btl_ugni_base_frag_t { mca_btl_base_descriptor_t base; - volatile int32_t ref_cnt; + opal_atomic_int32_t ref_cnt; uint32_t msg_id; uint16_t hdr_size; uint16_t flags; diff --git a/opal/mca/btl/vader/btl_vader_component.c b/opal/mca/btl/vader/btl_vader_component.c index 135506a1e0..115bceb081 100644 --- a/opal/mca/btl/vader/btl_vader_component.c +++ b/opal/mca/btl/vader/btl_vader_component.c @@ -715,7 +715,7 @@ static void mca_btl_vader_progress_endpoints (void) static int mca_btl_vader_component_progress (void) { - static int32_t lock = 0; + static opal_atomic_int32_t lock = 0; int count = 0; if (opal_using_threads()) { diff --git a/opal/mca/btl/vader/btl_vader_endpoint.h b/opal/mca/btl/vader/btl_vader_endpoint.h index d3a39e08f2..9e26335222 100644 --- a/opal/mca/btl/vader/btl_vader_endpoint.h +++ b/opal/mca/btl/vader/btl_vader_endpoint.h @@ -62,7 +62,7 @@ typedef struct mca_btl_base_endpoint_t { int32_t peer_smp_rank; /**< my peer's SMP process rank. Used for accessing * SMP specfic data structures. */ - volatile size_t send_count; /**< number of fragments sent to this peer */ + opal_atomic_size_t send_count; /**< number of fragments sent to this peer */ char *segment_base; /**< start of the peer's segment (in the address space * of this process) */ diff --git a/opal/mca/btl/vader/btl_vader_fifo.h b/opal/mca/btl/vader/btl_vader_fifo.h index 0dc70bc8a1..1a56b52e97 100644 --- a/opal/mca/btl/vader/btl_vader_fifo.h +++ b/opal/mca/btl/vader/btl_vader_fifo.h @@ -30,26 +30,25 @@ #include "btl_vader_endpoint.h" #include "btl_vader_frag.h" -#define vader_item_compare_exchange(x, y, z) opal_atomic_compare_exchange_strong_ptr ((volatile void **) (x), (void **) (y), (void *) (z)) +#define vader_item_compare_exchange(x, y, z) opal_atomic_compare_exchange_strong_ptr ((opal_atomic_intptr_t *) (x), (intptr_t *) (y), (intptr_t) (z)) #if SIZEOF_VOID_P == 8 - #define vader_item_swap(x, y) opal_atomic_swap_64((volatile int64_t *)(x), (int64_t)(y)) + #define vader_item_swap(x, y) opal_atomic_swap_64((opal_atomic_int64_t *)(x), (int64_t)(y)) #define MCA_BTL_VADER_OFFSET_MASK 0xffffffffll #define MCA_BTL_VADER_OFFSET_BITS 32 #define MCA_BTL_VADER_BITNESS 64 - - typedef int64_t fifo_value_t; #else - #define vader_item_swap(x, y) opal_atomic_swap_32((volatile int32_t *)(x), (int32_t)(y)) + #define vader_item_swap(x, y) opal_atomic_swap_32((opal_atomic_int32_t *)(x), (int32_t)(y)) #define MCA_BTL_VADER_OFFSET_MASK 0x00ffffffl #define MCA_BTL_VADER_OFFSET_BITS 24 #define MCA_BTL_VADER_BITNESS 32 - - typedef int32_t fifo_value_t; #endif +typedef opal_atomic_intptr_t atomic_fifo_value_t; +typedef intptr_t fifo_value_t; + #define VADER_FIFO_FREE ((fifo_value_t)-2) /* @@ -68,9 +67,9 @@ /* lock free fifo */ typedef struct vader_fifo_t { - volatile fifo_value_t fifo_head; - volatile fifo_value_t fifo_tail; - volatile int32_t fbox_available; + atomic_fifo_value_t fifo_head; + atomic_fifo_value_t fifo_tail; + opal_atomic_int32_t fbox_available; } vader_fifo_t; /* large enough to ensure the fifo is on its own cache line */ diff --git a/opal/mca/btl/vader/btl_vader_sc_emu.c b/opal/mca/btl/vader/btl_vader_sc_emu.c index 4f0b289ea6..8e13f0444b 100644 --- a/opal/mca/btl/vader/btl_vader_sc_emu.c +++ b/opal/mca/btl/vader/btl_vader_sc_emu.c @@ -13,7 +13,7 @@ #include "btl_vader_frag.h" #if OPAL_HAVE_ATOMIC_MATH_64 -static void mca_btl_vader_sc_emu_atomic_64 (int64_t *operand, volatile int64_t *addr, mca_btl_base_atomic_op_t op) +static void mca_btl_vader_sc_emu_atomic_64 (int64_t *operand, opal_atomic_int64_t *addr, mca_btl_base_atomic_op_t op) { int64_t result = 0; @@ -52,7 +52,7 @@ static void mca_btl_vader_sc_emu_atomic_64 (int64_t *operand, volatile int64_t * #endif #if OPAL_HAVE_ATOMIC_MATH_32 -static void mca_btl_vader_sc_emu_atomic_32 (int32_t *operand, volatile int32_t *addr, mca_btl_base_atomic_op_t op) +static void mca_btl_vader_sc_emu_atomic_32 (int32_t *operand, opal_atomic_int32_t *addr, mca_btl_base_atomic_op_t op) { int32_t result = 0; @@ -123,10 +123,10 @@ static void mca_btl_vader_sc_emu_rdma (mca_btl_base_module_t *btl, mca_btl_base_ #if OPAL_HAVE_ATOMIC_MATH_64 case MCA_BTL_VADER_OP_CSWAP: if (!(hdr->flags & MCA_BTL_ATOMIC_FLAG_32BIT)) { - opal_atomic_compare_exchange_strong_64 ((volatile int64_t *) hdr->addr, &hdr->operand[0], hdr->operand[1]); + opal_atomic_compare_exchange_strong_64 ((opal_atomic_int64_t *) hdr->addr, &hdr->operand[0], hdr->operand[1]); #if OPAL_HAVE_ATOMIC_MATH_32 } else { - opal_atomic_compare_exchange_strong_32 ((volatile int32_t *) hdr->addr, (int32_t *) &hdr->operand[0], + opal_atomic_compare_exchange_strong_32 ((opal_atomic_int32_t *) hdr->addr, (int32_t *) &hdr->operand[0], (int32_t) hdr->operand[1]); #else } else { diff --git a/opal/mca/common/sm/common_sm.h b/opal/mca/common/sm/common_sm.h index 819b82f6ee..7c4456f045 100644 --- a/opal/mca/common/sm/common_sm.h +++ b/opal/mca/common/sm/common_sm.h @@ -1,3 +1,4 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology @@ -10,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2010-2012 Los Alamos National Security, LLC. + * Copyright (c) 2010-2018 Los Alamos National Security, LLC. * All rights reserved. * $COPYRIGHT$ * @@ -42,13 +43,13 @@ typedef struct mca_common_sm_seg_header_t { /* lock to control atomic access */ opal_atomic_lock_t seg_lock; /* indicates whether or not the segment is ready for use */ - volatile int32_t seg_inited; + opal_atomic_int32_t seg_inited; /* number of local processes that are attached to the shared memory segment. * this is primarily used as a way of determining whether or not it is safe * to unlink the shared memory backing store. for example, once seg_att * is equal to the number of local processes, then we can safely unlink. */ - volatile size_t seg_num_procs_inited; + opal_atomic_size_t seg_num_procs_inited; /* offset to next available memory location available for allocation */ size_t seg_offset; /* total size of the segment */ diff --git a/opal/mca/mpool/hugepage/mpool_hugepage.h b/opal/mca/mpool/hugepage/mpool_hugepage.h index 9b76df62e3..8ac5373281 100644 --- a/opal/mca/mpool/hugepage/mpool_hugepage.h +++ b/opal/mca/mpool/hugepage/mpool_hugepage.h @@ -46,7 +46,7 @@ struct mca_mpool_hugepage_component_t { opal_list_t huge_pages; mca_mpool_hugepage_module_t *modules; int module_count; - unsigned long bytes_allocated; + opal_atomic_size_t bytes_allocated; }; typedef struct mca_mpool_hugepage_component_t mca_mpool_hugepage_component_t; @@ -62,7 +62,7 @@ struct mca_mpool_hugepage_hugepage_t { /** path for mmapped files */ char *path; /** counter to help ensure unique file names for mmaped files */ - volatile int32_t count; + opal_atomic_int32_t count; /** some platforms allow allocation of hugepages through mmap flags */ int mmap_flags; }; diff --git a/opal/mca/mpool/hugepage/mpool_hugepage_component.c b/opal/mca/mpool/hugepage/mpool_hugepage_component.c index 02320b9f91..0a6f4f6b38 100644 --- a/opal/mca/mpool/hugepage/mpool_hugepage_component.c +++ b/opal/mca/mpool/hugepage/mpool_hugepage_component.c @@ -135,7 +135,7 @@ static int mca_mpool_hugepage_register(void) "hugepage component", OPAL_INFO_LVL_3, MCA_BASE_PVAR_CLASS_SIZE, MCA_BASE_VAR_TYPE_UNSIGNED_LONG, NULL, MCA_BASE_VAR_BIND_NO_OBJECT, MCA_BASE_PVAR_FLAG_READONLY | MCA_BASE_PVAR_FLAG_CONTINUOUS, - NULL, NULL, NULL, &mca_mpool_hugepage_component.bytes_allocated); + NULL, NULL, NULL, (void *) &mca_mpool_hugepage_component.bytes_allocated); return OPAL_SUCCESS; } diff --git a/opal/mca/mpool/hugepage/mpool_hugepage_module.c b/opal/mca/mpool/hugepage/mpool_hugepage_module.c index 89a8b7eb6d..e174614b61 100644 --- a/opal/mca/mpool/hugepage/mpool_hugepage_module.c +++ b/opal/mca/mpool/hugepage/mpool_hugepage_module.c @@ -183,7 +183,7 @@ void *mca_mpool_hugepage_seg_alloc (void *ctx, size_t *sizep) opal_mutex_lock (&hugepage_module->lock); opal_rb_tree_insert (&hugepage_module->allocation_tree, base, (void *) (intptr_t) size); - opal_atomic_add (&mca_mpool_hugepage_component.bytes_allocated, (int64_t) size); + (void) opal_atomic_fetch_add_size_t (&mca_mpool_hugepage_component.bytes_allocated, size); opal_mutex_unlock (&hugepage_module->lock); OPAL_OUTPUT_VERBOSE((MCA_BASE_VERBOSE_TRACE, opal_mpool_base_framework.framework_verbose, @@ -207,7 +207,7 @@ void mca_mpool_hugepage_seg_free (void *ctx, void *addr) OPAL_OUTPUT_VERBOSE((MCA_BASE_VERBOSE_TRACE, opal_mpool_base_framework.framework_verbose, "freeing segment %p of size %lu bytes", addr, size)); munmap (addr, size); - opal_atomic_add (&mca_mpool_hugepage_component.bytes_allocated, -(int64_t) size); + (void) opal_atomic_fetch_add_size_t (&mca_mpool_hugepage_component.bytes_allocated, -size); } opal_mutex_unlock (&hugepage_module->lock); diff --git a/opal/mca/rcache/grdma/rcache_grdma_module.c b/opal/mca/rcache/grdma/rcache_grdma_module.c index 4e64b5df14..6b564327dd 100644 --- a/opal/mca/rcache/grdma/rcache_grdma_module.c +++ b/opal/mca/rcache/grdma/rcache_grdma_module.c @@ -215,7 +215,7 @@ static inline void mca_rcache_grdma_add_to_lru (mca_rcache_grdma_module_t *rcach opal_atomic_wmb (); /* mark this registration as being in the LRU */ - opal_atomic_fetch_or_32 ((volatile int32_t *) &grdma_reg->flags, MCA_RCACHE_GRDMA_REG_FLAG_IN_LRU); + opal_atomic_fetch_or_32 ((opal_atomic_int32_t *) &grdma_reg->flags, MCA_RCACHE_GRDMA_REG_FLAG_IN_LRU); opal_mutex_unlock (&rcache_grdma->cache->vma_module->vma_lock); } @@ -263,7 +263,7 @@ static int mca_rcache_grdma_check_cached (mca_rcache_base_registration_t *grdma_ } /* This segment fits fully within an existing segment. */ - (void) opal_atomic_fetch_add_32 ((volatile int32_t *) &rcache_grdma->stat_cache_hit, 1); + (void) opal_atomic_fetch_add_32 ((opal_atomic_int32_t *) &rcache_grdma->stat_cache_hit, 1); OPAL_OUTPUT_VERBOSE((MCA_BASE_VERBOSE_TRACE, opal_rcache_base_framework.framework_output, "returning existing registration %p. references %d", (void *) grdma_reg, ref_cnt)); return 1; @@ -321,7 +321,7 @@ static int mca_rcache_grdma_register (mca_rcache_base_module_t *rcache, void *ad /* get updated access flags */ access_flags = find_args.access_flags; - OPAL_THREAD_ADD_FETCH32((volatile int32_t *) &rcache_grdma->stat_cache_miss, 1); + OPAL_THREAD_ADD_FETCH32((opal_atomic_int32_t *) &rcache_grdma->stat_cache_miss, 1); } item = opal_free_list_get_mt (&rcache_grdma->reg_list); @@ -447,7 +447,7 @@ typedef struct gc_add_args_t gc_add_args_t; static int mca_rcache_grdma_add_to_gc (mca_rcache_base_registration_t *grdma_reg) { mca_rcache_grdma_module_t *rcache_grdma = (mca_rcache_grdma_module_t *) grdma_reg->rcache; - uint32_t flags = opal_atomic_fetch_or_32 ((volatile int32_t *) &grdma_reg->flags, MCA_RCACHE_FLAGS_INVALID); + uint32_t flags = opal_atomic_fetch_or_32 ((opal_atomic_int32_t *) &grdma_reg->flags, MCA_RCACHE_FLAGS_INVALID); if ((flags & MCA_RCACHE_FLAGS_INVALID) || 0 != grdma_reg->ref_count) { /* nothing to do */ diff --git a/opal/mca/rcache/rcache.h b/opal/mca/rcache/rcache.h index fa808dabac..f95fa1a597 100644 --- a/opal/mca/rcache/rcache.h +++ b/opal/mca/rcache/rcache.h @@ -92,9 +92,9 @@ struct mca_rcache_base_registration_t { /** artifact of old mpool/rcache architecture. used by cuda code */ unsigned char *alloc_base; /** number of outstanding references */ - volatile int32_t ref_count; + opal_atomic_int32_t ref_count; /** registration flags */ - volatile uint32_t flags; + opal_atomic_uint32_t flags; /** internal rcache context */ void *rcache_context; #if OPAL_CUDA_GDR_SUPPORT diff --git a/opal/runtime/opal_progress.c b/opal/runtime/opal_progress.c index 8c88a32c67..d7ce887ef2 100644 --- a/opal/runtime/opal_progress.c +++ b/opal/runtime/opal_progress.c @@ -73,13 +73,13 @@ static opal_timer_t event_progress_last_time = 0; static opal_timer_t event_progress_delta = 0; #else /* current count down until we tick the event library */ -static int32_t event_progress_counter = 0; +static opal_atomic_int32_t event_progress_counter = 0; /* reset value for counter when it hits 0 */ static int32_t event_progress_delta = 0; #endif /* users of the event library from MPI cause the tick rate to be every time */ -static int32_t num_event_users = 0; +static opal_atomic_int32_t num_event_users = 0; #if OPAL_ENABLE_DEBUG static int debug_output = -1; @@ -410,7 +410,7 @@ static int _opal_progress_register (opal_progress_callback_t cb, volatile opal_p opal_atomic_wmb (); /* swap out callback array */ - old = opal_atomic_swap_ptr (cbs, tmp); + old = (opal_progress_callback_t *) opal_atomic_swap_ptr ((opal_atomic_intptr_t *) cbs, (intptr_t) tmp); opal_atomic_wmb (); @@ -471,7 +471,7 @@ static int _opal_progress_unregister (opal_progress_callback_t cb, volatile opal for (size_t i = (size_t) ret ; i < *callback_array_len - 1 ; ++i) { /* copy callbacks atomically since another thread may be in * opal_progress(). */ - (void) opal_atomic_swap_ptr (callback_array + i, callback_array[i+1]); + (void) opal_atomic_swap_ptr ((opal_atomic_intptr_t *) (callback_array + i), (intptr_t) callback_array[i+1]); } callback_array[*callback_array_len] = fake_cb; diff --git a/opal/threads/mutex_unix.h b/opal/threads/mutex_unix.h index 1b748369c2..1cafdedd4e 100644 --- a/opal/threads/mutex_unix.h +++ b/opal/threads/mutex_unix.h @@ -10,7 +10,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2007-2015 Los Alamos National Security, LLC. All rights + * Copyright (c) 2007-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2015-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -76,14 +76,14 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_recursive_mutex_t); .m_lock_debug = 0, \ .m_lock_file = NULL, \ .m_lock_line = 0, \ - .m_lock_atomic = { .u = { .lock = OPAL_ATOMIC_LOCK_UNLOCKED } }, \ + .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \ } #else #define OPAL_MUTEX_STATIC_INIT \ { \ .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), \ .m_lock_pthread = PTHREAD_MUTEX_INITIALIZER, \ - .m_lock_atomic = { .u = { .lock = OPAL_ATOMIC_LOCK_UNLOCKED } }, \ + .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \ } #endif @@ -97,14 +97,14 @@ OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_recursive_mutex_t); .m_lock_debug = 0, \ .m_lock_file = NULL, \ .m_lock_line = 0, \ - .m_lock_atomic = { .u = { .lock = OPAL_ATOMIC_LOCK_UNLOCKED } }, \ + .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \ } #else #define OPAL_RECURSIVE_MUTEX_STATIC_INIT \ { \ .super = OPAL_OBJ_STATIC_INIT(opal_mutex_t), \ .m_lock_pthread = OPAL_PTHREAD_RECURSIVE_MUTEX_INITIALIZER, \ - .m_lock_atomic = { .u = { .lock = OPAL_ATOMIC_LOCK_UNLOCKED } }, \ + .m_lock_atomic = OPAL_ATOMIC_LOCK_INIT, \ } #endif diff --git a/opal/threads/thread_usage.h b/opal/threads/thread_usage.h index 178c8ceaab..8ce65362c1 100644 --- a/opal/threads/thread_usage.h +++ b/opal/threads/thread_usage.h @@ -13,7 +13,7 @@ * Copyright (c) 2007-2014 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2014-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -94,7 +94,7 @@ static inline bool opal_set_using_threads(bool have) */ #define OPAL_THREAD_DEFINE_ATOMIC_OP(type, name, operator, suffix) \ -static inline type opal_thread_ ## name ## _fetch_ ## suffix (volatile type *addr, type delta) \ +static inline type opal_thread_ ## name ## _fetch_ ## suffix (opal_atomic_ ## type *addr, type delta) \ { \ if (OPAL_UNLIKELY(opal_using_threads())) { \ return opal_atomic_ ## name ## _fetch_ ## suffix (addr, delta); \ @@ -104,7 +104,7 @@ static inline type opal_thread_ ## name ## _fetch_ ## suffix (volatile type *add return *addr; \ } \ \ -static inline type opal_thread_fetch_ ## name ## _ ## suffix (volatile type *addr, type delta) \ +static inline type opal_thread_fetch_ ## name ## _ ## suffix (opal_atomic_ ## type *addr, type delta) \ { \ if (OPAL_UNLIKELY(opal_using_threads())) { \ return opal_atomic_fetch_ ## name ## _ ## suffix (addr, delta); \ @@ -116,10 +116,10 @@ static inline type opal_thread_fetch_ ## name ## _ ## suffix (volatile type *add } #define OPAL_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(type, addr_type, suffix) \ -static inline bool opal_thread_compare_exchange_strong_ ## suffix (volatile addr_type *addr, type *compare, type value) \ +static inline bool opal_thread_compare_exchange_strong_ ## suffix (opal_atomic_ ## addr_type *addr, type *compare, type value) \ { \ if (OPAL_UNLIKELY(opal_using_threads())) { \ - return opal_atomic_compare_exchange_strong_ ## suffix ((volatile type *) addr, compare, value); \ + return opal_atomic_compare_exchange_strong_ ## suffix (addr, (addr_type *) compare, (addr_type) value); \ } \ \ if ((type) *addr == *compare) { \ @@ -133,10 +133,10 @@ static inline bool opal_thread_compare_exchange_strong_ ## suffix (volatile addr } #define OPAL_THREAD_DEFINE_ATOMIC_SWAP(type, addr_type, suffix) \ -static inline type opal_thread_swap_ ## suffix (volatile addr_type *ptr, type newvalue) \ +static inline type opal_thread_swap_ ## suffix (opal_atomic_ ## addr_type *ptr, type newvalue) \ { \ if (opal_using_threads ()) { \ - return opal_atomic_swap_ ## suffix ((volatile type *) ptr, newvalue); \ + return (type) opal_atomic_swap_ ## suffix (ptr, (addr_type) newvalue); \ } \ \ type old = ((type *) ptr)[0]; \ @@ -154,9 +154,9 @@ OPAL_THREAD_DEFINE_ATOMIC_OP(int32_t, sub, -, 32) OPAL_THREAD_DEFINE_ATOMIC_OP(size_t, sub, -, size_t) OPAL_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(int32_t, int32_t, 32) -OPAL_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(void *, intptr_t, ptr) +OPAL_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(intptr_t, intptr_t, ptr) OPAL_THREAD_DEFINE_ATOMIC_SWAP(int32_t, int32_t, 32) -OPAL_THREAD_DEFINE_ATOMIC_SWAP(void *, intptr_t, ptr) +OPAL_THREAD_DEFINE_ATOMIC_SWAP(intptr_t, intptr_t, ptr) #define OPAL_THREAD_ADD_FETCH32 opal_thread_add_fetch_32 #define OPAL_ATOMIC_ADD_FETCH32 opal_thread_add_fetch_32 @@ -197,13 +197,13 @@ OPAL_THREAD_DEFINE_ATOMIC_SWAP(void *, intptr_t, ptr) #define OPAL_THREAD_COMPARE_EXCHANGE_STRONG_32 opal_thread_compare_exchange_strong_32 #define OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_32 opal_thread_compare_exchange_strong_32 -#define OPAL_THREAD_COMPARE_EXCHANGE_STRONG_PTR(x, y, z) opal_thread_compare_exchange_strong_ptr ((volatile intptr_t *) x, (void *) y, (void *) z) +#define OPAL_THREAD_COMPARE_EXCHANGE_STRONG_PTR(x, y, z) opal_thread_compare_exchange_strong_ptr ((opal_atomic_intptr_t *) x, (intptr_t *) y, (intptr_t) z) #define OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR OPAL_THREAD_COMPARE_EXCHANGE_STRONG_PTR #define OPAL_THREAD_SWAP_32 opal_thread_swap_32 #define OPAL_ATOMIC_SWAP_32 opal_thread_swap_32 -#define OPAL_THREAD_SWAP_PTR(x, y) opal_thread_swap_ptr ((volatile intptr_t *) x, (void *) y) +#define OPAL_THREAD_SWAP_PTR(x, y) opal_thread_swap_ptr ((opal_atomic_intptr_t *) x, (intptr_t) y) #define OPAL_ATOMIC_SWAP_PTR OPAL_THREAD_SWAP_PTR /* define 64-bit macros is 64-bit atomic math is available */ diff --git a/opal/threads/wait_sync.h b/opal/threads/wait_sync.h index 1e59467035..cb095c0b1b 100644 --- a/opal/threads/wait_sync.h +++ b/opal/threads/wait_sync.h @@ -26,7 +26,7 @@ BEGIN_C_DECLS typedef struct ompi_wait_sync_t { - int32_t count; + opal_atomic_int32_t count; int32_t status; pthread_cond_t condition; pthread_mutex_t lock; diff --git a/orte/runtime/orte_locks.c b/orte/runtime/orte_locks.c index 9cf31a4889..386bc472b8 100644 --- a/orte/runtime/orte_locks.c +++ b/orte/runtime/orte_locks.c @@ -25,12 +25,12 @@ #include "orte/runtime/orte_locks.h" /* for everyone */ -opal_atomic_lock_t orte_finalize_lock = {{0}}; +opal_atomic_lock_t orte_finalize_lock = OPAL_ATOMIC_LOCK_INIT; /* for HNPs */ -opal_atomic_lock_t orte_abort_inprogress_lock = {{0}}; -opal_atomic_lock_t orte_jobs_complete_lock = {{0}}; -opal_atomic_lock_t orte_quit_lock = {{0}}; +opal_atomic_lock_t orte_abort_inprogress_lock = OPAL_ATOMIC_LOCK_INIT; +opal_atomic_lock_t orte_jobs_complete_lock = OPAL_ATOMIC_LOCK_INIT; +opal_atomic_lock_t orte_quit_lock = OPAL_ATOMIC_LOCK_INIT; int orte_locks_init(void) { diff --git a/test/asm/atomic_cmpset.c b/test/asm/atomic_cmpset.c index 4a06847703..9fe964125a 100644 --- a/test/asm/atomic_cmpset.c +++ b/test/asm/atomic_cmpset.c @@ -45,33 +45,33 @@ int nreps = 100; int nthreads = 2; int enable_verbose = 0; -volatile int32_t vol32 = 0; -int32_t val32 = 0; +opal_atomic_int32_t vol32 = 0; +opal_atomic_int32_t val32 = 0; int32_t old32 = 0; int32_t new32 = 0; #if OPAL_HAVE_ATOMIC_MATH_64 -volatile int64_t vol64 = 0; -int64_t val64 = 0; +opal_atomic_int64_t vol64 = 0; +opal_atomic_int64_t val64 = 0; int64_t old64 = 0; int64_t new64 = 0; #endif #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 -volatile opal_int128_t vol128; -opal_int128_t val128; +opal_atomic_int128_t vol128; +opal_atomic_int128_t val128; opal_int128_t old128; opal_int128_t new128; #endif -volatile int volint = 0; -int valint = 0; +opal_atomic_int_t volint = 0; +opal_atomic_int_t valint = 0; int oldint = 0; int newint = 0; -volatile void *volptr = NULL; -void *oldptr = NULL; -void *newptr = NULL; +opal_atomic_intptr_t volptr = 0; +intptr_t oldptr = 0; +intptr_t newptr = 0; static void *thread_main(void *arg) @@ -235,39 +235,39 @@ int main(int argc, char *argv[]) /* -- cmpset ptr tests -- */ - volptr = (void *) 42, oldptr = (void *) 42, newptr = (void *) 50; + volptr = 42, oldptr = 42, newptr = 50; assert(opal_atomic_compare_exchange_strong_ptr (&volptr, &oldptr, newptr) == true); opal_atomic_rmb(); assert(volptr == newptr); - assert(oldptr == (void *) 42); + assert(oldptr == 42); - volptr = (void *) 42, oldptr = (void *) 420, newptr = (void *) 50; + volptr = 42, oldptr = 420, newptr = 50; assert(opal_atomic_compare_exchange_strong_ptr (&volptr, &oldptr, newptr) == false); opal_atomic_rmb(); - assert(volptr == (void *) 42); - assert(oldptr == (void *) 42); + assert(volptr == 42); + assert(oldptr == 42); - volptr = (void *) 42, oldptr = (void *) 42, newptr = (void *) 50; + volptr = 42, oldptr = 42, newptr = 50; assert(opal_atomic_compare_exchange_strong_acq_ptr (&volptr, &oldptr, newptr) == true); assert(volptr == newptr); - assert(oldptr == (void *) 42); + assert(oldptr == 42); - volptr = (void *) 42, oldptr = (void *) 420, newptr = (void *) 50; + volptr = 42, oldptr = 420, newptr = 50; assert(opal_atomic_compare_exchange_strong_acq_ptr (&volptr, &oldptr, newptr) == false); - assert(volptr == (void *) 42); - assert(oldptr == (void *) 42); + assert(volptr == 42); + assert(oldptr == 42); - volptr = (void *) 42, oldptr = (void *) 42, newptr = (void *) 50; + volptr = 42, oldptr = 42, newptr = 50; assert(opal_atomic_compare_exchange_strong_rel_ptr (&volptr, &oldptr, newptr) == true); opal_atomic_rmb(); assert(volptr == newptr); - assert(oldptr == (void *) 42); + assert(oldptr == 42); - volptr = (void *) 42, oldptr = (void *) 420, newptr = (void *) 50; + volptr = 42, oldptr = 420, newptr = 50; assert(opal_atomic_compare_exchange_strong_rel_ptr (&volptr, &oldptr, newptr) == false); opal_atomic_rmb(); - assert(volptr == (void *) 42); - assert(oldptr == (void *) 42); + assert(volptr == 42); + assert(oldptr == 42); /* -- add_32 tests -- */ diff --git a/test/asm/atomic_math.c b/test/asm/atomic_math.c index 54f771cc26..8629ad9780 100644 --- a/test/asm/atomic_math.c +++ b/test/asm/atomic_math.c @@ -32,11 +32,11 @@ #define TEST_REPS 500 -int32_t val32 = 0; +opal_atomic_int32_t val32 = 0; #if OPAL_HAVE_ATOMIC_MATH_64 -int64_t val64 = 0; +opal_atomic_int64_t val64 = 0; #endif -int valint = 0; +opal_atomic_int_t valint = 0; static void* atomic_math_test(void* arg) { diff --git a/test/threads/opal_thread.c b/test/threads/opal_thread.c index 1c6480d19c..e795190c5a 100644 --- a/test/threads/opal_thread.c +++ b/test/threads/opal_thread.c @@ -21,7 +21,7 @@ /* Only have the body of this test if we have thread support */ -static volatile int count = 0; +static opal_atomic_int_t count = 0; static void* thr1_run(opal_object_t* obj)