From 000f9eed4d15d8c192154237218de14f18f4cebd Mon Sep 17 00:00:00 2001 From: Nathan Hjelm Date: Wed, 11 Jul 2018 13:34:03 -0600 Subject: [PATCH] opal: add types for atomic variables This commit updates the entire codebase to use specific opal types for all atomic variables. This is a change from the prior atomic support which required the use of the volatile keyword. This is the first step towards implementing support for C11 atomics as that interface requires the use of types declared with the _Atomic keyword. Signed-off-by: Nathan Hjelm --- ompi/communicator/comm_request.c | 4 +- ompi/datatype/ompi_datatype.h | 2 +- ompi/datatype/ompi_datatype_args.c | 18 +-- ompi/datatype/ompi_datatype_create.c | 10 +- ompi/datatype/ompi_datatype_internal.h | 2 +- ompi/datatype/ompi_datatype_module.c | 2 +- ompi/group/group.h | 3 +- ompi/mca/coll/libnbc/coll_libnbc.h | 2 +- ompi/mca/coll/monitoring/coll_monitoring.h | 2 +- ompi/mca/coll/portals4/coll_portals4.h | 5 +- ompi/mca/coll/sm/coll_sm.h | 4 +- ompi/mca/coll/sm/coll_sm_barrier.c | 3 +- ompi/mca/coll/sm/coll_sm_module.c | 2 +- .../mca/common/monitoring/common_monitoring.c | 32 ++--- .../monitoring/common_monitoring_coll.c | 12 +- ompi/mca/common/ompio/common_ompio_cuda.c | 2 +- ompi/mca/mtl/portals4/mtl_portals4.h | 4 +- ompi/mca/mtl/portals4/mtl_portals4_flowctl.c | 2 +- ompi/mca/mtl/portals4/mtl_portals4_flowctl.h | 4 +- ompi/mca/mtl/portals4/mtl_portals4_request.h | 6 +- .../osc/monitoring/osc_monitoring_module.h | 2 +- ompi/mca/osc/portals4/osc_portals4.h | 2 +- ompi/mca/osc/portals4/osc_portals4_comm.c | 14 +- ompi/mca/osc/portals4/osc_portals4_request.h | 4 +- ompi/mca/osc/pt2pt/osc_pt2pt.h | 30 ++-- ompi/mca/osc/pt2pt/osc_pt2pt_active_target.c | 20 +-- ompi/mca/osc/pt2pt/osc_pt2pt_component.c | 2 +- ompi/mca/osc/pt2pt/osc_pt2pt_data_move.c | 4 +- ompi/mca/osc/pt2pt/osc_pt2pt_frag.c | 8 +- ompi/mca/osc/pt2pt/osc_pt2pt_frag.h | 10 +- ompi/mca/osc/pt2pt/osc_pt2pt_header.h | 4 +- ompi/mca/osc/pt2pt/osc_pt2pt_module.c | 6 +- ompi/mca/osc/pt2pt/osc_pt2pt_passive_target.c | 28 ++-- ompi/mca/osc/pt2pt/osc_pt2pt_request.h | 2 +- ompi/mca/osc/pt2pt/osc_pt2pt_sync.h | 4 +- ompi/mca/osc/rdma/osc_rdma.h | 2 +- ompi/mca/osc/rdma/osc_rdma_active_target.c | 6 +- ompi/mca/osc/rdma/osc_rdma_frag.h | 2 +- ompi/mca/osc/rdma/osc_rdma_lock.h | 16 +-- ompi/mca/osc/rdma/osc_rdma_peer.h | 2 +- ompi/mca/osc/rdma/osc_rdma_request.h | 2 +- ompi/mca/osc/rdma/osc_rdma_sync.h | 2 +- ompi/mca/osc/rdma/osc_rdma_types.h | 20 +-- ompi/mca/osc/sm/osc_sm.h | 6 +- ompi/mca/osc/sm/osc_sm_active_target.c | 12 +- ompi/mca/osc/sm/osc_sm_component.c | 4 +- ompi/mca/osc/sm/osc_sm_passive_target.c | 4 +- ompi/mca/pml/base/pml_base_bsend.c | 2 +- ompi/mca/pml/ob1/pml_ob1_comm.h | 2 +- ompi/mca/pml/ob1/pml_ob1_progress.c | 2 +- ompi/mca/pml/ob1/pml_ob1_recvreq.h | 6 +- ompi/mca/pml/ob1/pml_ob1_sendreq.h | 8 +- ompi/request/grequestx.c | 2 +- ompi/runtime/mpiruntime.h | 2 +- ompi/runtime/ompi_mpi_init.c | 2 +- ompi/runtime/ompi_spc.c | 2 +- ompi/runtime/ompi_spc.h | 2 +- opal/class/opal_fifo.c | 6 +- opal/class/opal_fifo.h | 30 ++-- opal/class/opal_interval_tree.c | 2 +- opal/class/opal_interval_tree.h | 8 +- opal/class/opal_lifo.c | 2 +- opal/class/opal_lifo.h | 25 ++-- opal/class/opal_list.h | 18 +-- opal/class/opal_object.h | 2 +- opal/class/opal_tree.h | 2 +- opal/include/Makefile.am | 5 +- opal/include/opal/sys/arm/atomic.h | 18 +-- opal/include/opal/sys/arm64/atomic.h | 28 ++-- opal/include/opal/sys/atomic.h | 134 +++++++++--------- opal/include/opal/sys/atomic_impl.h | 92 ++++++------ opal/include/opal/sys/gcc_builtin/atomic.h | 42 +++--- opal/include/opal/sys/ia32/atomic.h | 10 +- opal/include/opal/sys/powerpc/atomic.h | 30 ++-- opal/include/opal/sys/sparcv9/atomic.h | 16 +-- opal/include/opal/sys/sync_builtin/atomic.h | 28 ++-- opal/include/opal/sys/x86_64/atomic.h | 22 +-- opal/include/opal_stdatomic.h | 35 +++++ opal/mca/btl/ofi/btl_ofi.h | 4 +- opal/mca/btl/openib/btl_openib.h | 14 +- opal/mca/btl/openib/btl_openib_async.c | 4 +- opal/mca/btl/openib/btl_openib_eager_rdma.h | 10 +- opal/mca/btl/openib/btl_openib_endpoint.c | 9 +- opal/mca/btl/openib/btl_openib_endpoint.h | 24 ++-- opal/mca/btl/portals4/btl_portals4.h | 6 +- opal/mca/btl/smcuda/btl_smcuda.h | 4 +- opal/mca/btl/uct/btl_uct_device_context.h | 6 +- opal/mca/btl/uct/btl_uct_types.h | 2 +- opal/mca/btl/ugni/btl_ugni.h | 8 +- opal/mca/btl/ugni/btl_ugni_endpoint.h | 2 +- opal/mca/btl/ugni/btl_ugni_frag.h | 2 +- opal/mca/btl/vader/btl_vader_component.c | 2 +- opal/mca/btl/vader/btl_vader_endpoint.h | 2 +- opal/mca/btl/vader/btl_vader_fifo.h | 19 ++- opal/mca/btl/vader/btl_vader_sc_emu.c | 8 +- opal/mca/common/sm/common_sm.h | 7 +- opal/mca/mpool/hugepage/mpool_hugepage.h | 4 +- .../mpool/hugepage/mpool_hugepage_component.c | 2 +- .../mpool/hugepage/mpool_hugepage_module.c | 4 +- opal/mca/rcache/grdma/rcache_grdma_module.c | 8 +- opal/mca/rcache/rcache.h | 4 +- opal/runtime/opal_progress.c | 8 +- opal/threads/mutex_unix.h | 2 +- opal/threads/thread_usage.h | 22 +-- opal/threads/wait_sync.h | 2 +- test/asm/atomic_cmpset.c | 52 +++---- test/asm/atomic_math.c | 6 +- test/threads/opal_thread.c | 2 +- 108 files changed, 616 insertions(+), 557 deletions(-) create mode 100644 opal/include/opal_stdatomic.h diff --git a/ompi/communicator/comm_request.c b/ompi/communicator/comm_request.c index 272fc33600..934cbc75df 100644 --- a/ompi/communicator/comm_request.c +++ b/ompi/communicator/comm_request.c @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2013-2016 Los Alamos National Security, LLC. All rights + * Copyright (c) 2013-2018 Los Alamos National Security, LLC. All rights * reseved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -99,7 +99,7 @@ int ompi_comm_request_schedule_append (ompi_comm_request_t *request, ompi_comm_r static int ompi_comm_request_progress (void) { ompi_comm_request_t *request, *next; - static int32_t progressing = 0; + static opal_atomic_int32_t progressing = 0; /* don't allow re-entry */ if (opal_atomic_swap_32 (&progressing, 1)) { diff --git a/ompi/datatype/ompi_datatype.h b/ompi/datatype/ompi_datatype.h index 8b48bc3097..d9df1f1ae4 100644 --- a/ompi/datatype/ompi_datatype.h +++ b/ompi/datatype/ompi_datatype.h @@ -75,7 +75,7 @@ struct ompi_datatype_t { struct opal_hash_table_t *d_keyhash; /**< Attribute fields */ void* args; /**< Data description for the user */ - void* packed_description; /**< Packed description of the datatype */ + opal_atomic_intptr_t packed_description; /**< Packed description of the datatype */ uint64_t pml_data; /**< PML-specific information */ /* --- cacheline 6 boundary (384 bytes) --- */ char name[MPI_MAX_OBJECT_NAME];/**< Externally visible name */ diff --git a/ompi/datatype/ompi_datatype_args.c b/ompi/datatype/ompi_datatype_args.c index 737d3e5182..2e8af79215 100644 --- a/ompi/datatype/ompi_datatype_args.c +++ b/ompi/datatype/ompi_datatype_args.c @@ -45,7 +45,7 @@ __ompi_datatype_create_from_args( int32_t* i, ptrdiff_t * a, ompi_datatype_t** d, int32_t type ); typedef struct __dt_args { - int32_t ref_count; + opal_atomic_int32_t ref_count; int32_t create_type; size_t total_pack_size; int32_t ci; @@ -104,7 +104,7 @@ typedef struct __dt_args { pArgs->total_pack_size = (4 + (IC) + (DC)) * sizeof(int) + \ (AC) * sizeof(ptrdiff_t); \ (PDATA)->args = (void*)pArgs; \ - (PDATA)->packed_description = NULL; \ + (PDATA)->packed_description = 0; \ } while(0) @@ -483,12 +483,12 @@ int ompi_datatype_get_pack_description( ompi_datatype_t* datatype, { ompi_datatype_args_t* args = (ompi_datatype_args_t*)datatype->args; int next_index = OMPI_DATATYPE_MAX_PREDEFINED; - void *packed_description = datatype->packed_description; + void *packed_description = (void *) datatype->packed_description; void* recursive_buffer; if (NULL == packed_description) { void *_tmp_ptr = NULL; - if (opal_atomic_compare_exchange_strong_ptr (&datatype->packed_description, (void *) &_tmp_ptr, (void *) 1)) { + if (opal_atomic_compare_exchange_strong_ptr (&datatype->packed_description, (intptr_t *) &_tmp_ptr, 1)) { if( ompi_datatype_is_predefined(datatype) ) { packed_description = malloc(2 * sizeof(int)); } else if( NULL == args ) { @@ -510,10 +510,10 @@ int ompi_datatype_get_pack_description( ompi_datatype_t* datatype, } opal_atomic_wmb (); - datatype->packed_description = packed_description; + datatype->packed_description = (intptr_t) packed_description; } else { /* another thread beat us to it */ - packed_description = datatype->packed_description; + packed_description = (void *) datatype->packed_description; } } @@ -521,11 +521,11 @@ int ompi_datatype_get_pack_description( ompi_datatype_t* datatype, struct timespec interval = {.tv_sec = 0, .tv_nsec = 1000}; /* wait until the packed description is updated */ - while ((void *) 1 == datatype->packed_description) { + while (1 == datatype->packed_description) { nanosleep (&interval, NULL); } - packed_description = datatype->packed_description; + packed_description = (void *) datatype->packed_description; } *packed_buffer = (const void *) packed_description; @@ -534,7 +534,7 @@ int ompi_datatype_get_pack_description( ompi_datatype_t* datatype, size_t ompi_datatype_pack_description_length( ompi_datatype_t* datatype ) { - void *packed_description = datatype->packed_description; + void *packed_description = (void *) datatype->packed_description; if( ompi_datatype_is_predefined(datatype) ) { return 2 * sizeof(int); diff --git a/ompi/datatype/ompi_datatype_create.c b/ompi/datatype/ompi_datatype_create.c index 69ec1b2c6c..aa0760c2c5 100644 --- a/ompi/datatype/ompi_datatype_create.c +++ b/ompi/datatype/ompi_datatype_create.c @@ -36,7 +36,7 @@ static void __ompi_datatype_allocate( ompi_datatype_t* datatype ) datatype->id = -1; datatype->d_keyhash = NULL; datatype->name[0] = '\0'; - datatype->packed_description = NULL; + datatype->packed_description = 0; datatype->pml_data = 0; } @@ -46,10 +46,10 @@ static void __ompi_datatype_release(ompi_datatype_t * datatype) ompi_datatype_release_args( datatype ); datatype->args = NULL; } - if( NULL != datatype->packed_description ) { - free( datatype->packed_description ); - datatype->packed_description = NULL; - } + + free ((void *) datatype->packed_description ); + datatype->packed_description = 0; + if( datatype->d_f_to_c_index >= 0 ) { opal_pointer_array_set_item( &ompi_datatype_f_to_c_table, datatype->d_f_to_c_index, NULL ); datatype->d_f_to_c_index = -1; diff --git a/ompi/datatype/ompi_datatype_internal.h b/ompi/datatype/ompi_datatype_internal.h index 0cbfb25a95..7de5863b2f 100644 --- a/ompi/datatype/ompi_datatype_internal.h +++ b/ompi/datatype/ompi_datatype_internal.h @@ -406,7 +406,7 @@ extern const ompi_datatype_t* ompi_datatype_basicDatatypes[OMPI_DATATYPE_MPI_MAX .d_f_to_c_index = -1, \ .d_keyhash = NULL, \ .args = NULL, \ - .packed_description = NULL, \ + .packed_description = 0, \ .name = "MPI_" # NAME #define OMPI_DATATYPE_INITIALIZER_UNAVAILABLE(FLAGS) \ diff --git a/ompi/datatype/ompi_datatype_module.c b/ompi/datatype/ompi_datatype_module.c index 3ee09173cd..ef4398f576 100644 --- a/ompi/datatype/ompi_datatype_module.c +++ b/ompi/datatype/ompi_datatype_module.c @@ -383,7 +383,7 @@ opal_pointer_array_t ompi_datatype_f_to_c_table = {{0}}; (PDST)->super.desc = (PSRC)->super.desc; \ (PDST)->super.opt_desc = (PSRC)->super.opt_desc; \ (PDST)->packed_description = (PSRC)->packed_description; \ - (PSRC)->packed_description = NULL; \ + (PSRC)->packed_description = 0; \ /* transfer the ptypes */ \ (PDST)->super.ptypes = (PSRC)->super.ptypes; \ (PSRC)->super.ptypes = NULL; \ diff --git a/ompi/group/group.h b/ompi/group/group.h index 30664f8a4e..661666246e 100644 --- a/ompi/group/group.h +++ b/ompi/group/group.h @@ -356,7 +356,8 @@ static inline struct ompi_proc_t *ompi_group_dense_lookup (ompi_group_t *group, ompi_proc_t *real_proc = (ompi_proc_t *) ompi_proc_for_name (ompi_proc_sentinel_to_name ((uintptr_t) proc)); - if (opal_atomic_compare_exchange_strong_ptr (group->grp_proc_pointers + peer_id, &proc, real_proc)) { + if (opal_atomic_compare_exchange_strong_ptr ((opal_atomic_intptr_t *)(group->grp_proc_pointers + peer_id), + (intptr_t *) &proc, (intptr_t) real_proc)) { OBJ_RETAIN(real_proc); } diff --git a/ompi/mca/coll/libnbc/coll_libnbc.h b/ompi/mca/coll/libnbc/coll_libnbc.h index 967a779425..56cb2d75df 100644 --- a/ompi/mca/coll/libnbc/coll_libnbc.h +++ b/ompi/mca/coll/libnbc/coll_libnbc.h @@ -75,7 +75,7 @@ struct ompi_coll_libnbc_component_t { mca_coll_base_component_2_0_0_t super; opal_free_list_t requests; opal_list_t active_requests; - int32_t active_comms; + opal_atomic_int32_t active_comms; opal_mutex_t lock; /* protect access to the active_requests list */ }; typedef struct ompi_coll_libnbc_component_t ompi_coll_libnbc_component_t; diff --git a/ompi/mca/coll/monitoring/coll_monitoring.h b/ompi/mca/coll/monitoring/coll_monitoring.h index 8046c98208..1fc5cca20d 100644 --- a/ompi/mca/coll/monitoring/coll_monitoring.h +++ b/ompi/mca/coll/monitoring/coll_monitoring.h @@ -36,7 +36,7 @@ struct mca_coll_monitoring_module_t { mca_coll_base_module_t super; mca_coll_base_comm_coll_t real; mca_monitoring_coll_data_t*data; - int32_t is_initialized; + opal_atomic_int32_t is_initialized; }; typedef struct mca_coll_monitoring_module_t mca_coll_monitoring_module_t; OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_coll_monitoring_module_t); diff --git a/ompi/mca/coll/portals4/coll_portals4.h b/ompi/mca/coll/portals4/coll_portals4.h index 175c951e81..ce01084f32 100644 --- a/ompi/mca/coll/portals4/coll_portals4.h +++ b/ompi/mca/coll/portals4/coll_portals4.h @@ -1,6 +1,7 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2013-2015 Sandia National Laboratories. All rights reserved. - * Copyright (c) 2015 Los Alamos National Security, LLC. All rights + * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2015 Bull SAS. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science @@ -91,7 +92,7 @@ typedef struct ompi_coll_portals4_tree_t { struct mca_coll_portals4_module_t { mca_coll_base_module_t super; - size_t coll_count; + opal_atomic_size_t coll_count; /* record handlers dedicated to fallback if offloaded operations are not supported */ mca_coll_base_module_reduce_fn_t previous_reduce; diff --git a/ompi/mca/coll/sm/coll_sm.h b/ompi/mca/coll/sm/coll_sm.h index b2da6ede42..e40ca73c2f 100644 --- a/ompi/mca/coll/sm/coll_sm.h +++ b/ompi/mca/coll/sm/coll_sm.h @@ -114,7 +114,7 @@ BEGIN_C_DECLS typedef struct mca_coll_sm_in_use_flag_t { /** Number of processes currently using this set of segments */ - volatile uint32_t mcsiuf_num_procs_using; + opal_atomic_uint32_t mcsiuf_num_procs_using; /** Must match data->mcb_count */ volatile uint32_t mcsiuf_operation_count; } mca_coll_sm_in_use_flag_t; @@ -152,7 +152,7 @@ BEGIN_C_DECLS /** Pointer to my parent's barrier control pages (will be NULL for communicator rank 0; odd index pages are "in", even index pages are "out") */ - uint32_t *mcb_barrier_control_parent; + opal_atomic_uint32_t *mcb_barrier_control_parent; /** Pointers to my childrens' barrier control pages (they're contiguous in memory, so we only point to the base -- the diff --git a/ompi/mca/coll/sm/coll_sm_barrier.c b/ompi/mca/coll/sm/coll_sm_barrier.c index 2722bbf09f..e8e0fb5b78 100644 --- a/ompi/mca/coll/sm/coll_sm_barrier.c +++ b/ompi/mca/coll/sm/coll_sm_barrier.c @@ -56,7 +56,8 @@ int mca_coll_sm_barrier_intra(struct ompi_communicator_t *comm, int rank, buffer_set; mca_coll_sm_comm_t *data; uint32_t i, num_children; - volatile uint32_t *me_in, *me_out, *parent, *children = NULL; + volatile uint32_t *me_in, *me_out, *children = NULL; + opal_atomic_uint32_t *parent; int uint_control_size; mca_coll_sm_module_t *sm_module = (mca_coll_sm_module_t*) module; diff --git a/ompi/mca/coll/sm/coll_sm_module.c b/ompi/mca/coll/sm/coll_sm_module.c index 8922a70eaf..d1804a0061 100644 --- a/ompi/mca/coll/sm/coll_sm_module.c +++ b/ompi/mca/coll/sm/coll_sm_module.c @@ -372,7 +372,7 @@ int ompi_coll_sm_lazy_enable(mca_coll_base_module_t *module, data->mcb_barrier_control_me = (uint32_t*) (base + (rank * control_size * num_barrier_buffers * 2)); if (data->mcb_tree[rank].mcstn_parent) { - data->mcb_barrier_control_parent = (uint32_t*) + data->mcb_barrier_control_parent = (opal_atomic_uint32_t*) (base + (data->mcb_tree[rank].mcstn_parent->mcstn_id * control_size * num_barrier_buffers * 2)); diff --git a/ompi/mca/common/monitoring/common_monitoring.c b/ompi/mca/common/monitoring/common_monitoring.c index e521ca5641..8598155156 100644 --- a/ompi/mca/common/monitoring/common_monitoring.c +++ b/ompi/mca/common/monitoring/common_monitoring.c @@ -7,7 +7,7 @@ * Copyright (c) 2015 Bull SAS. All rights reserved. * Copyright (c) 2016-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2017-2018 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -34,7 +34,7 @@ /*** Monitoring specific variables ***/ /* Keep tracks of how many components are currently using the common part */ -static int32_t mca_common_monitoring_hold = 0; +static opal_atomic_int32_t mca_common_monitoring_hold = 0; /* Output parameters */ int mca_common_monitoring_output_stream_id = -1; static opal_output_stream_t mca_common_monitoring_output_stream_obj = { @@ -61,18 +61,18 @@ static char* mca_common_monitoring_initial_filename = ""; static char* mca_common_monitoring_current_filename = NULL; /* array for stroring monitoring data*/ -static size_t* pml_data = NULL; -static size_t* pml_count = NULL; -static size_t* filtered_pml_data = NULL; -static size_t* filtered_pml_count = NULL; -static size_t* osc_data_s = NULL; -static size_t* osc_count_s = NULL; -static size_t* osc_data_r = NULL; -static size_t* osc_count_r = NULL; -static size_t* coll_data = NULL; -static size_t* coll_count = NULL; +static opal_atomic_size_t* pml_data = NULL; +static opal_atomic_size_t* pml_count = NULL; +static opal_atomic_size_t* filtered_pml_data = NULL; +static opal_atomic_size_t* filtered_pml_count = NULL; +static opal_atomic_size_t* osc_data_s = NULL; +static opal_atomic_size_t* osc_count_s = NULL; +static opal_atomic_size_t* osc_data_r = NULL; +static opal_atomic_size_t* osc_count_r = NULL; +static opal_atomic_size_t* coll_data = NULL; +static opal_atomic_size_t* coll_count = NULL; -static size_t* size_histogram = NULL; +static opal_atomic_size_t* size_histogram = NULL; static const int max_size_histogram = 66; static double log10_2 = 0.; @@ -241,7 +241,7 @@ void mca_common_monitoring_finalize( void ) opal_output_close(mca_common_monitoring_output_stream_id); free(mca_common_monitoring_output_stream_obj.lds_prefix); /* Free internal data structure */ - free(pml_data); /* a single allocation */ + free((void *) pml_data); /* a single allocation */ opal_hash_table_remove_all( common_monitoring_translation_ht ); OBJ_RELEASE(common_monitoring_translation_ht); mca_common_monitoring_coll_finalize(); @@ -446,7 +446,7 @@ int mca_common_monitoring_add_procs(struct ompi_proc_t **procs, if( NULL == pml_data ) { int array_size = (10 + max_size_histogram) * nprocs_world; - pml_data = (size_t*)calloc(array_size, sizeof(size_t)); + pml_data = (opal_atomic_size_t*)calloc(array_size, sizeof(size_t)); pml_count = pml_data + nprocs_world; filtered_pml_data = pml_count + nprocs_world; filtered_pml_count = filtered_pml_data + nprocs_world; @@ -493,7 +493,7 @@ int mca_common_monitoring_add_procs(struct ompi_proc_t **procs, static void mca_common_monitoring_reset( void ) { int array_size = (10 + max_size_histogram) * nprocs_world; - memset(pml_data, 0, array_size * sizeof(size_t)); + memset((void *) pml_data, 0, array_size * sizeof(size_t)); mca_common_monitoring_coll_reset(); } diff --git a/ompi/mca/common/monitoring/common_monitoring_coll.c b/ompi/mca/common/monitoring/common_monitoring_coll.c index 571f48070e..4fce29028e 100644 --- a/ompi/mca/common/monitoring/common_monitoring_coll.c +++ b/ompi/mca/common/monitoring/common_monitoring_coll.c @@ -30,12 +30,12 @@ struct mca_monitoring_coll_data_t { int world_rank; int is_released; ompi_communicator_t*p_comm; - size_t o2a_count; - size_t o2a_size; - size_t a2o_count; - size_t a2o_size; - size_t a2a_count; - size_t a2a_size; + opal_atomic_size_t o2a_count; + opal_atomic_size_t o2a_size; + opal_atomic_size_t a2o_count; + opal_atomic_size_t a2o_size; + opal_atomic_size_t a2a_count; + opal_atomic_size_t a2a_size; }; /* Collectives operation monitoring */ diff --git a/ompi/mca/common/ompio/common_ompio_cuda.c b/ompi/mca/common/ompio/common_ompio_cuda.c index e6ace6fe4f..d2ad66a413 100644 --- a/ompi/mca/common/ompio/common_ompio_cuda.c +++ b/ompi/mca/common/ompio/common_ompio_cuda.c @@ -34,7 +34,7 @@ static opal_mutex_t mca_common_ompio_cuda_mutex; /* lock for thread saf static mca_allocator_base_component_t* mca_common_ompio_allocator_component=NULL; static mca_allocator_base_module_t* mca_common_ompio_allocator=NULL; -static int32_t mca_common_ompio_cuda_init = 0; +static opal_atomic_int32_t mca_common_ompio_cuda_init = 0; static int32_t mca_common_ompio_pagesize=4096; static void* mca_common_ompio_cuda_alloc_seg ( void *ctx, size_t *size ); static void mca_common_ompio_cuda_free_seg ( void *ctx, void *buf ); diff --git a/ompi/mca/mtl/portals4/mtl_portals4.h b/ompi/mca/mtl/portals4/mtl_portals4.h index 52b21b9354..70db436b91 100644 --- a/ompi/mca/mtl/portals4/mtl_portals4.h +++ b/ompi/mca/mtl/portals4/mtl_portals4.h @@ -115,12 +115,12 @@ struct mca_mtl_portals4_module_t { opal_mutex_t short_block_mutex; /** number of send-side operations started */ - uint64_t opcount; + opal_atomic_uint64_t opcount; #if OPAL_ENABLE_DEBUG /** number of receive-side operations started. Used only for debugging */ - uint64_t recv_opcount; + opal_atomic_uint64_t recv_opcount; #endif #if OMPI_MTL_PORTALS4_FLOW_CONTROL diff --git a/ompi/mca/mtl/portals4/mtl_portals4_flowctl.c b/ompi/mca/mtl/portals4/mtl_portals4_flowctl.c index 19d3b600b3..f690630d7e 100644 --- a/ompi/mca/mtl/portals4/mtl_portals4_flowctl.c +++ b/ompi/mca/mtl/portals4/mtl_portals4_flowctl.c @@ -1,7 +1,7 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2012 Sandia National Laboratories. All rights reserved. - * Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * diff --git a/ompi/mca/mtl/portals4/mtl_portals4_flowctl.h b/ompi/mca/mtl/portals4/mtl_portals4_flowctl.h index 7cc634b669..ee1fbd9a76 100644 --- a/ompi/mca/mtl/portals4/mtl_portals4_flowctl.h +++ b/ompi/mca/mtl/portals4/mtl_portals4_flowctl.h @@ -36,7 +36,7 @@ OBJ_CLASS_DECLARATION(ompi_mtl_portals4_pending_request_t); struct ompi_mtl_portals4_flowctl_t { int32_t flowctl_active; - int32_t send_slots; + opal_atomic_int32_t send_slots; int32_t max_send_slots; opal_list_t pending_sends; opal_free_list_t pending_fl; @@ -46,7 +46,7 @@ struct ompi_mtl_portals4_flowctl_t { /** Flow control epoch counter. Triggered events should be based on epoch counter. */ - int64_t epoch_counter; + opal_atomic_int64_t epoch_counter; /** Flow control trigger CT. Only has meaning at root. */ ptl_handle_ct_t trigger_ct_h; diff --git a/ompi/mca/mtl/portals4/mtl_portals4_request.h b/ompi/mca/mtl/portals4/mtl_portals4_request.h index c7e3c31e47..ede07b4e50 100644 --- a/ompi/mca/mtl/portals4/mtl_portals4_request.h +++ b/ompi/mca/mtl/portals4/mtl_portals4_request.h @@ -54,8 +54,8 @@ struct ompi_mtl_portals4_isend_request_t { struct ompi_mtl_portals4_pending_request_t *pending; #endif ptl_size_t length; - int32_t pending_get; - uint32_t event_count; + opal_atomic_int32_t pending_get; + opal_atomic_uint32_t event_count; }; typedef struct ompi_mtl_portals4_isend_request_t ompi_mtl_portals4_isend_request_t; @@ -76,7 +76,7 @@ struct ompi_mtl_portals4_recv_request_t { void *delivery_ptr; size_t delivery_len; volatile bool req_started; - int32_t pending_reply; + opal_atomic_int32_t pending_reply; #if OPAL_ENABLE_DEBUG uint64_t opcount; ptl_hdr_data_t hdr_data; diff --git a/ompi/mca/osc/monitoring/osc_monitoring_module.h b/ompi/mca/osc/monitoring/osc_monitoring_module.h index 9c7dfb12e7..e03f81290a 100644 --- a/ompi/mca/osc/monitoring/osc_monitoring_module.h +++ b/ompi/mca/osc/monitoring/osc_monitoring_module.h @@ -50,7 +50,7 @@ OSC_MONITORING_SET_TEMPLATE_FCT_NAME(template) (ompi_osc_base_module_t*module) \ { \ /* Define the ompi_osc_monitoring_module_## template ##_init_done variable */ \ - static int32_t init_done = 0; \ + opal_atomic_int32_t init_done = 0; \ /* Define and set the ompi_osc_monitoring_## template \ * ##_template variable. The functions recorded here are \ * linked to the original functions of the original \ diff --git a/ompi/mca/osc/portals4/osc_portals4.h b/ompi/mca/osc/portals4/osc_portals4.h index 19a3b6e5c5..6b58554489 100644 --- a/ompi/mca/osc/portals4/osc_portals4.h +++ b/ompi/mca/osc/portals4/osc_portals4.h @@ -95,7 +95,7 @@ struct ompi_osc_portals4_module_t { ptl_handle_md_t req_md_h; /* memory descriptor with event completion used by this window */ ptl_handle_me_t data_me_h; /* data match list entry (MB are CID | OSC_PORTALS4_MB_DATA) */ ptl_handle_me_t control_me_h; /* match list entry for control data (node_state_t). Match bits are (CID | OSC_PORTALS4_MB_CONTROL). */ - int64_t opcount; + opal_atomic_int64_t opcount; ptl_match_bits_t match_bits; /* match bits for module. Same as cid for comm in most cases. */ ptl_iovec_t *origin_iovec_list; /* list of memory segments that compose the noncontiguous region */ diff --git a/ompi/mca/osc/portals4/osc_portals4_comm.c b/ompi/mca/osc/portals4/osc_portals4_comm.c index b125f2aee5..07c69699c9 100644 --- a/ompi/mca/osc/portals4/osc_portals4_comm.c +++ b/ompi/mca/osc/portals4/osc_portals4_comm.c @@ -189,7 +189,7 @@ number_of_fragments(ptl_size_t length, ptl_size_t maxlength) /* put in segments no larger than segment_length */ static int -segmentedPut(int64_t *opcount, +segmentedPut(opal_atomic_int64_t *opcount, ptl_handle_md_t md_h, ptl_size_t origin_offset, ptl_size_t put_length, @@ -236,7 +236,7 @@ segmentedPut(int64_t *opcount, /* get in segments no larger than segment_length */ static int -segmentedGet(int64_t *opcount, +segmentedGet(opal_atomic_int64_t *opcount, ptl_handle_md_t md_h, ptl_size_t origin_offset, ptl_size_t get_length, @@ -280,7 +280,7 @@ segmentedGet(int64_t *opcount, /* atomic op in segments no larger than segment_length */ static int -segmentedAtomic(int64_t *opcount, +segmentedAtomic(opal_atomic_int64_t *opcount, ptl_handle_md_t md_h, ptl_size_t origin_offset, ptl_size_t length, @@ -329,7 +329,7 @@ segmentedAtomic(int64_t *opcount, /* atomic op in segments no larger than segment_length */ static int -segmentedFetchAtomic(int64_t *opcount, +segmentedFetchAtomic(opal_atomic_int64_t *opcount, ptl_handle_md_t result_md_h, ptl_size_t result_offset, ptl_handle_md_t origin_md_h, @@ -381,7 +381,7 @@ segmentedFetchAtomic(int64_t *opcount, /* swap in segments no larger than segment_length */ static int -segmentedSwap(int64_t *opcount, +segmentedSwap(opal_atomic_int64_t *opcount, ptl_handle_md_t result_md_h, ptl_size_t result_offset, ptl_handle_md_t origin_md_h, @@ -1187,7 +1187,7 @@ fetch_atomic_to_iovec(ompi_osc_portals4_module_t *module, /* put in the largest chunks possible given the noncontiguous restriction */ static int -put_to_noncontig(int64_t *opcount, +put_to_noncontig(opal_atomic_int64_t *opcount, ptl_handle_md_t md_h, const void *origin_address, int origin_count, @@ -1521,7 +1521,7 @@ atomic_to_noncontig(ompi_osc_portals4_module_t *module, /* get from a noncontiguous remote to an (non)contiguous local */ static int -get_from_noncontig(int64_t *opcount, +get_from_noncontig(opal_atomic_int64_t *opcount, ptl_handle_md_t md_h, const void *origin_address, int origin_count, diff --git a/ompi/mca/osc/portals4/osc_portals4_request.h b/ompi/mca/osc/portals4/osc_portals4_request.h index ae1be6f44d..42e38566ee 100644 --- a/ompi/mca/osc/portals4/osc_portals4_request.h +++ b/ompi/mca/osc/portals4/osc_portals4_request.h @@ -1,7 +1,7 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2011-2013 Sandia National Laboratories. All rights reserved. - * Copyright (c) 2015 Los Alamos National Security, LLC. All rights + * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -18,7 +18,7 @@ struct ompi_osc_portals4_request_t { ompi_request_t super; int32_t ops_expected; - volatile int32_t ops_committed; + opal_atomic_int32_t ops_committed; }; typedef struct ompi_osc_portals4_request_t ompi_osc_portals4_request_t; diff --git a/ompi/mca/osc/pt2pt/osc_pt2pt.h b/ompi/mca/osc/pt2pt/osc_pt2pt.h index 4b1a423ded..85bad73846 100644 --- a/ompi/mca/osc/pt2pt/osc_pt2pt.h +++ b/ompi/mca/osc/pt2pt/osc_pt2pt.h @@ -8,7 +8,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2007-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2007-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved. @@ -110,7 +110,7 @@ struct ompi_osc_pt2pt_peer_t { int rank; /** pointer to the current send fragment for each outgoing target */ - struct ompi_osc_pt2pt_frag_t *active_frag; + opal_atomic_intptr_t active_frag; /** lock for this peer */ opal_mutex_t lock; @@ -119,10 +119,10 @@ struct ompi_osc_pt2pt_peer_t { opal_list_t queued_frags; /** number of fragments incomming (negative - expected, positive - unsynchronized) */ - volatile int32_t passive_incoming_frag_count; + opal_atomic_int32_t passive_incoming_frag_count; /** peer flags */ - volatile int32_t flags; + opal_atomic_int32_t flags; }; typedef struct ompi_osc_pt2pt_peer_t ompi_osc_pt2pt_peer_t; @@ -208,16 +208,16 @@ struct ompi_osc_pt2pt_module_t { /** Nmber of communication fragments started for this epoch, by peer. Not in peer data to make fence more manageable. */ - uint32_t *epoch_outgoing_frag_count; + opal_atomic_uint32_t *epoch_outgoing_frag_count; /** cyclic counter for a unique tage for long messages. */ - volatile uint32_t tag_counter; + opal_atomic_uint32_t tag_counter; /** number of outgoing fragments still to be completed */ - volatile int32_t outgoing_frag_count; + opal_atomic_int32_t outgoing_frag_count; /** number of incoming fragments */ - volatile int32_t active_incoming_frag_count; + opal_atomic_int32_t active_incoming_frag_count; /** Number of targets locked/being locked */ unsigned int passive_target_access_epoch; @@ -230,13 +230,13 @@ struct ompi_osc_pt2pt_module_t { /** Number of "count" messages from the remote complete group we've received */ - volatile int32_t num_complete_msgs; + opal_atomic_int32_t num_complete_msgs; /* ********************* LOCK data ************************ */ /** Status of the local window lock. One of 0 (unlocked), MPI_LOCK_EXCLUSIVE, or MPI_LOCK_SHARED. */ - int32_t lock_status; + opal_atomic_int32_t lock_status; /** lock for locks_pending list */ opal_mutex_t locks_pending_lock; @@ -526,7 +526,7 @@ static inline void mark_incoming_completion (ompi_osc_pt2pt_module_t *module, in OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "mark_incoming_completion marking passive incoming complete. module %p, source = %d, count = %d", (void *) module, source, (int) peer->passive_incoming_frag_count + 1)); - new_value = OPAL_THREAD_ADD_FETCH32((int32_t *) &peer->passive_incoming_frag_count, 1); + new_value = OPAL_THREAD_ADD_FETCH32((opal_atomic_int32_t *) &peer->passive_incoming_frag_count, 1); if (0 == new_value) { OPAL_THREAD_LOCK(&module->lock); opal_condition_broadcast(&module->cond); @@ -550,7 +550,7 @@ static inline void mark_incoming_completion (ompi_osc_pt2pt_module_t *module, in */ static inline void mark_outgoing_completion (ompi_osc_pt2pt_module_t *module) { - int32_t new_value = OPAL_THREAD_ADD_FETCH32((int32_t *) &module->outgoing_frag_count, 1); + int32_t new_value = OPAL_THREAD_ADD_FETCH32((opal_atomic_int32_t *) &module->outgoing_frag_count, 1); OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "mark_outgoing_completion: outgoing_frag_count = %d", new_value)); if (new_value >= 0) { @@ -574,12 +574,12 @@ static inline void mark_outgoing_completion (ompi_osc_pt2pt_module_t *module) */ static inline void ompi_osc_signal_outgoing (ompi_osc_pt2pt_module_t *module, int target, int count) { - OPAL_THREAD_ADD_FETCH32((int32_t *) &module->outgoing_frag_count, -count); + OPAL_THREAD_ADD_FETCH32((opal_atomic_int32_t *) &module->outgoing_frag_count, -count); if (MPI_PROC_NULL != target) { OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "ompi_osc_signal_outgoing_passive: target = %d, count = %d, total = %d", target, count, module->epoch_outgoing_frag_count[target] + count)); - OPAL_THREAD_ADD_FETCH32((int32_t *) (module->epoch_outgoing_frag_count + target), count); + OPAL_THREAD_ADD_FETCH32((opal_atomic_int32_t *) (module->epoch_outgoing_frag_count + target), count); } } @@ -717,7 +717,7 @@ static inline int get_tag(ompi_osc_pt2pt_module_t *module) /* the LSB of the tag is used be the receiver to determine if the message is a passive or active target (ie, where to mark completion). */ - int32_t tmp = OPAL_THREAD_ADD_FETCH32((volatile int32_t *) &module->tag_counter, 4); + int32_t tmp = OPAL_THREAD_ADD_FETCH32((opal_atomic_int32_t *) &module->tag_counter, 4); return (tmp & OSC_PT2PT_FRAG_MASK) | !!(module->passive_target_access_epoch); } diff --git a/ompi/mca/osc/pt2pt/osc_pt2pt_active_target.c b/ompi/mca/osc/pt2pt/osc_pt2pt_active_target.c index 33df9440a6..59831842bc 100644 --- a/ompi/mca/osc/pt2pt/osc_pt2pt_active_target.c +++ b/ompi/mca/osc/pt2pt/osc_pt2pt_active_target.c @@ -8,7 +8,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2007-2016 Los Alamos National Security, LLC. All rights + * Copyright (c) 2007-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2010-2016 IBM Corporation. All rights reserved. * Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved. @@ -166,17 +166,16 @@ int ompi_osc_pt2pt_fence(int assert, ompi_win_t *win) "osc pt2pt: fence done sending")); /* find out how much data everyone is going to send us. */ - ret = module->comm->c_coll->coll_reduce_scatter_block (module->epoch_outgoing_frag_count, - &incoming_reqs, 1, MPI_UINT32_T, - MPI_SUM, module->comm, - module->comm->c_coll->coll_reduce_scatter_block_module); + ret = module->comm->c_coll->coll_reduce_scatter_block ((void *) module->epoch_outgoing_frag_count, + &incoming_reqs, 1, MPI_UINT32_T, + MPI_SUM, module->comm, + module->comm->c_coll->coll_reduce_scatter_block_module); if (OMPI_SUCCESS != ret) { return ret; } OPAL_THREAD_LOCK(&module->lock); - bzero(module->epoch_outgoing_frag_count, - sizeof(uint32_t) * ompi_comm_size(module->comm)); + bzero ((void *) module->epoch_outgoing_frag_count, sizeof(uint32_t) * ompi_comm_size(module->comm)); OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "osc pt2pt: fence expects %d requests", @@ -366,8 +365,11 @@ int ompi_osc_pt2pt_complete (ompi_win_t *win) /* XXX -- TODO -- since fragment are always delivered in order we do not need to count anything but long * requests. once that is done this can be removed. */ - if (peer->active_frag && (peer->active_frag->remain_len < sizeof (complete_req))) { - ++complete_req.frag_count; + if (peer->active_frag) { + ompi_osc_pt2pt_frag_t *active_frag = (ompi_osc_pt2pt_frag_t *) peer->active_frag; + if (active_frag->remain_len < sizeof (complete_req)) { + ++complete_req.frag_count; + } } OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, diff --git a/ompi/mca/osc/pt2pt/osc_pt2pt_component.c b/ompi/mca/osc/pt2pt/osc_pt2pt_component.c index 6838f78a56..0fbcd2af32 100644 --- a/ompi/mca/osc/pt2pt/osc_pt2pt_component.c +++ b/ompi/mca/osc/pt2pt/osc_pt2pt_component.c @@ -501,7 +501,7 @@ static void ompi_osc_pt2pt_peer_construct (ompi_osc_pt2pt_peer_t *peer) { OBJ_CONSTRUCT(&peer->queued_frags, opal_list_t); OBJ_CONSTRUCT(&peer->lock, opal_mutex_t); - peer->active_frag = NULL; + peer->active_frag = 0; peer->passive_incoming_frag_count = 0; peer->flags = 0; } diff --git a/ompi/mca/osc/pt2pt/osc_pt2pt_data_move.c b/ompi/mca/osc/pt2pt/osc_pt2pt_data_move.c index 1342c01a69..d09e5311fb 100644 --- a/ompi/mca/osc/pt2pt/osc_pt2pt_data_move.c +++ b/ompi/mca/osc/pt2pt/osc_pt2pt_data_move.c @@ -8,7 +8,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2007-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2007-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2009-2011 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved. @@ -56,7 +56,7 @@ struct osc_pt2pt_accumulate_data_t { int peer; ompi_datatype_t *datatype; ompi_op_t *op; - int request_count; + opal_atomic_int32_t request_count; }; typedef struct osc_pt2pt_accumulate_data_t osc_pt2pt_accumulate_data_t; diff --git a/ompi/mca/osc/pt2pt/osc_pt2pt_frag.c b/ompi/mca/osc/pt2pt/osc_pt2pt_frag.c index c14afeb257..317eff94bc 100644 --- a/ompi/mca/osc/pt2pt/osc_pt2pt_frag.c +++ b/ompi/mca/osc/pt2pt/osc_pt2pt_frag.c @@ -1,7 +1,7 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved. - * Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -65,7 +65,7 @@ int ompi_osc_pt2pt_frag_start (ompi_osc_pt2pt_module_t *module, ompi_osc_pt2pt_peer_t *peer = ompi_osc_pt2pt_peer_lookup (module, frag->target); int ret; - assert(0 == frag->pending && peer->active_frag != frag); + assert(0 == frag->pending && peer->active_frag != (intptr_t) frag); /* we need to signal now that a frag is outgoing to ensure the count sent * with the unlock message is correct */ @@ -93,7 +93,7 @@ int ompi_osc_pt2pt_frag_start (ompi_osc_pt2pt_module_t *module, static int ompi_osc_pt2pt_flush_active_frag (ompi_osc_pt2pt_module_t *module, ompi_osc_pt2pt_peer_t *peer) { - ompi_osc_pt2pt_frag_t *active_frag = peer->active_frag; + ompi_osc_pt2pt_frag_t *active_frag = (ompi_osc_pt2pt_frag_t *) peer->active_frag; int ret = OMPI_SUCCESS; if (NULL == active_frag) { @@ -105,7 +105,7 @@ static int ompi_osc_pt2pt_flush_active_frag (ompi_osc_pt2pt_module_t *module, om "osc pt2pt: flushing active fragment to target %d. pending: %d", active_frag->target, active_frag->pending)); - if (opal_atomic_compare_exchange_strong_ptr (&peer->active_frag, &active_frag, NULL)) { + if (opal_atomic_compare_exchange_strong_ptr (&peer->active_frag, (intptr_t *) &active_frag, 0)) { if (0 != OPAL_THREAD_ADD_FETCH32(&active_frag->pending, -1)) { /* communication going on while synchronizing; this is an rma usage bug */ return OMPI_ERR_RMA_SYNC; diff --git a/ompi/mca/osc/pt2pt/osc_pt2pt_frag.h b/ompi/mca/osc/pt2pt/osc_pt2pt_frag.h index 4ed38930d5..4ef7679f72 100644 --- a/ompi/mca/osc/pt2pt/osc_pt2pt_frag.h +++ b/ompi/mca/osc/pt2pt/osc_pt2pt_frag.h @@ -33,7 +33,7 @@ struct ompi_osc_pt2pt_frag_t { char *top; /* Number of operations which have started writing into the frag, but not yet completed doing so */ - volatile int32_t pending; + opal_atomic_int32_t pending; int32_t pending_long_sends; ompi_osc_pt2pt_frag_header_t *header; ompi_osc_pt2pt_module_t *module; @@ -66,8 +66,8 @@ static inline ompi_osc_pt2pt_frag_t *ompi_osc_pt2pt_frag_alloc_non_buffered (omp ompi_osc_pt2pt_frag_t *curr; /* to ensure ordering flush the buffer on the peer */ - curr = peer->active_frag; - if (NULL != curr && opal_atomic_compare_exchange_strong_ptr (&peer->active_frag, &curr, NULL)) { + curr = (ompi_osc_pt2pt_frag_t *) peer->active_frag; + if (NULL != curr && opal_atomic_compare_exchange_strong_ptr (&peer->active_frag, (intptr_t *) &curr, 0)) { /* If there's something pending, the pending finish will start the buffer. Otherwise, we need to start it now. */ int ret = ompi_osc_pt2pt_frag_finish (module, curr); @@ -131,7 +131,7 @@ static inline int _ompi_osc_pt2pt_frag_alloc (ompi_osc_pt2pt_module_t *module, i OPAL_THREAD_LOCK(&module->lock); if (buffered) { - curr = peer->active_frag; + curr = (ompi_osc_pt2pt_frag_t *) peer->active_frag; if (NULL == curr || curr->remain_len < request_len || (long_send && curr->pending_long_sends == 32)) { curr = ompi_osc_pt2pt_frag_alloc_non_buffered (module, peer, request_len); if (OPAL_UNLIKELY(NULL == curr)) { @@ -140,7 +140,7 @@ static inline int _ompi_osc_pt2pt_frag_alloc (ompi_osc_pt2pt_module_t *module, i } curr->pending_long_sends = long_send; - peer->active_frag = curr; + peer->active_frag = (uintptr_t) curr; } else { OPAL_THREAD_ADD_FETCH32(&curr->header->num_ops, 1); curr->pending_long_sends += long_send; diff --git a/ompi/mca/osc/pt2pt/osc_pt2pt_header.h b/ompi/mca/osc/pt2pt/osc_pt2pt_header.h index f979d9bf61..cec5f50120 100644 --- a/ompi/mca/osc/pt2pt/osc_pt2pt_header.h +++ b/ompi/mca/osc/pt2pt/osc_pt2pt_header.h @@ -8,7 +8,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2007-2015 Los Alamos National Security, LLC. All rights + * Copyright (c) 2007-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. @@ -180,7 +180,7 @@ typedef struct ompi_osc_pt2pt_header_flush_ack_t ompi_osc_pt2pt_header_flush_ack struct ompi_osc_pt2pt_frag_header_t { ompi_osc_pt2pt_header_base_t base; uint32_t source; /* rank in window of source process */ - int32_t num_ops; /* number of operations in this buffer */ + opal_atomic_int32_t num_ops; /* number of operations in this buffer */ uint32_t pad; /* ensure the fragment header is a multiple of 8 bytes */ }; typedef struct ompi_osc_pt2pt_frag_header_t ompi_osc_pt2pt_frag_header_t; diff --git a/ompi/mca/osc/pt2pt/osc_pt2pt_module.c b/ompi/mca/osc/pt2pt/osc_pt2pt_module.c index e637174b90..b13a83f14b 100644 --- a/ompi/mca/osc/pt2pt/osc_pt2pt_module.c +++ b/ompi/mca/osc/pt2pt/osc_pt2pt_module.c @@ -8,7 +8,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2007-2016 Los Alamos National Security, LLC. All rights + * Copyright (c) 2007-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science @@ -104,13 +104,13 @@ int ompi_osc_pt2pt_free(ompi_win_t *win) free (module->recv_frags); } - if (NULL != module->epoch_outgoing_frag_count) free(module->epoch_outgoing_frag_count); + free ((void *) module->epoch_outgoing_frag_count); if (NULL != module->comm) { ompi_comm_free(&module->comm); } - if (NULL != module->free_after) free(module->free_after); + free ((void *) module->free_after); free (module); diff --git a/ompi/mca/osc/pt2pt/osc_pt2pt_passive_target.c b/ompi/mca/osc/pt2pt/osc_pt2pt_passive_target.c index 091757511f..59b585337c 100644 --- a/ompi/mca/osc/pt2pt/osc_pt2pt_passive_target.c +++ b/ompi/mca/osc/pt2pt/osc_pt2pt_passive_target.c @@ -8,7 +8,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2007-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2007-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2010-2016 IBM Corporation. All rights reserved. * Copyright (c) 2012-2013 Sandia National Laboratories. All rights reserved. @@ -157,7 +157,7 @@ int ompi_osc_pt2pt_lock_remote (ompi_osc_pt2pt_module_t *module, int target, omp static inline int ompi_osc_pt2pt_unlock_remote (ompi_osc_pt2pt_module_t *module, int target, ompi_osc_pt2pt_sync_t *lock) { - int32_t frag_count = opal_atomic_swap_32 ((int32_t *) module->epoch_outgoing_frag_count + target, -1); + int32_t frag_count = opal_atomic_swap_32 ((opal_atomic_int32_t *) module->epoch_outgoing_frag_count + target, -1); ompi_osc_pt2pt_peer_t *peer = ompi_osc_pt2pt_peer_lookup (module, target); int lock_type = lock->sync.lock.type; ompi_osc_pt2pt_header_unlock_t unlock_req; @@ -178,10 +178,13 @@ static inline int ompi_osc_pt2pt_unlock_remote (ompi_osc_pt2pt_module_t *module, unlock_req.lock_ptr = (uint64_t) (uintptr_t) lock; OSC_PT2PT_HTON(&unlock_req, module, target); - if (peer->active_frag && peer->active_frag->remain_len < sizeof (unlock_req)) { - /* the peer should expect one more packet */ - ++unlock_req.frag_count; - --module->epoch_outgoing_frag_count[target]; + if (peer->active_frag) { + ompi_osc_pt2pt_frag_t *active_frag = (ompi_osc_pt2pt_frag_t *) peer->active_frag; + if (active_frag->remain_len < sizeof (unlock_req)) { + /* the peer should expect one more packet */ + ++unlock_req.frag_count; + --module->epoch_outgoing_frag_count[target]; + } } OPAL_OUTPUT_VERBOSE((25, ompi_osc_base_framework.framework_output, @@ -204,7 +207,7 @@ static inline int ompi_osc_pt2pt_flush_remote (ompi_osc_pt2pt_module_t *module, { ompi_osc_pt2pt_peer_t *peer = ompi_osc_pt2pt_peer_lookup (module, target); ompi_osc_pt2pt_header_flush_t flush_req; - int32_t frag_count = opal_atomic_swap_32 ((int32_t *) module->epoch_outgoing_frag_count + target, -1); + int32_t frag_count = opal_atomic_swap_32 ((opal_atomic_int32_t *) module->epoch_outgoing_frag_count + target, -1); int ret; (void) OPAL_THREAD_ADD_FETCH32(&lock->sync_expected, 1); @@ -218,10 +221,13 @@ static inline int ompi_osc_pt2pt_flush_remote (ompi_osc_pt2pt_module_t *module, /* XXX -- TODO -- since fragment are always delivered in order we do not need to count anything but long * requests. once that is done this can be removed. */ - if (peer->active_frag && (peer->active_frag->remain_len < sizeof (flush_req))) { - /* the peer should expect one more packet */ - ++flush_req.frag_count; - --module->epoch_outgoing_frag_count[target]; + if (peer->active_frag) { + ompi_osc_pt2pt_frag_t *active_frag = (ompi_osc_pt2pt_frag_t *) peer->active_frag; + if (active_frag->remain_len < sizeof (flush_req)) { + /* the peer should expect one more packet */ + ++flush_req.frag_count; + --module->epoch_outgoing_frag_count[target]; + } } OPAL_OUTPUT_VERBOSE((50, ompi_osc_base_framework.framework_output, "flushing to target %d, frag_count: %d", diff --git a/ompi/mca/osc/pt2pt/osc_pt2pt_request.h b/ompi/mca/osc/pt2pt/osc_pt2pt_request.h index dee5c86892..09446de1b3 100644 --- a/ompi/mca/osc/pt2pt/osc_pt2pt_request.h +++ b/ompi/mca/osc/pt2pt/osc_pt2pt_request.h @@ -28,7 +28,7 @@ struct ompi_osc_pt2pt_request_t { int origin_count; struct ompi_datatype_t *origin_dt; ompi_osc_pt2pt_module_t* module; - int32_t outstanding_requests; + opal_atomic_int32_t outstanding_requests; bool internal; }; typedef struct ompi_osc_pt2pt_request_t ompi_osc_pt2pt_request_t; diff --git a/ompi/mca/osc/pt2pt/osc_pt2pt_sync.h b/ompi/mca/osc/pt2pt/osc_pt2pt_sync.h index fe359bf6cf..cf5da61bf8 100644 --- a/ompi/mca/osc/pt2pt/osc_pt2pt_sync.h +++ b/ompi/mca/osc/pt2pt/osc_pt2pt_sync.h @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2015-2016 Los Alamos National Security, LLC. All rights + * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -74,7 +74,7 @@ struct ompi_osc_pt2pt_sync_t { int num_peers; /** number of synchronization messages expected */ - volatile int32_t sync_expected; + opal_atomic_int32_t sync_expected; /** eager sends are active to all peers in this access epoch */ volatile bool eager_send_active; diff --git a/ompi/mca/osc/rdma/osc_rdma.h b/ompi/mca/osc/rdma/osc_rdma.h index b3743f261e..6f1232c532 100644 --- a/ompi/mca/osc/rdma/osc_rdma.h +++ b/ompi/mca/osc/rdma/osc_rdma.h @@ -265,7 +265,7 @@ struct ompi_osc_rdma_module_t { unsigned long get_retry_count; /** outstanding atomic operations */ - volatile int32_t pending_ops; + opal_atomic_int32_t pending_ops; }; typedef struct ompi_osc_rdma_module_t ompi_osc_rdma_module_t; OMPI_MODULE_DECLSPEC extern ompi_osc_rdma_component_t mca_osc_rdma_component; diff --git a/ompi/mca/osc/rdma/osc_rdma_active_target.c b/ompi/mca/osc/rdma/osc_rdma_active_target.c index f677394da0..80c31fc00b 100644 --- a/ompi/mca/osc/rdma/osc_rdma_active_target.c +++ b/ompi/mca/osc/rdma/osc_rdma_active_target.c @@ -259,7 +259,7 @@ static int ompi_osc_rdma_post_peer (ompi_osc_rdma_module_t *module, ompi_osc_rdm return ret; } } else { - post_index = ompi_osc_rdma_counter_add ((osc_rdma_counter_t *) (intptr_t) target, 1) - 1; + post_index = ompi_osc_rdma_counter_add ((osc_rdma_atomic_counter_t *) (intptr_t) target, 1) - 1; } post_index &= OMPI_OSC_RDMA_POST_PEER_MAX - 1; @@ -279,7 +279,7 @@ static int ompi_osc_rdma_post_peer (ompi_osc_rdma_module_t *module, ompi_osc_rdm return ret; } } else { - result = !ompi_osc_rdma_lock_compare_exchange ((osc_rdma_counter_t *) target, &_tmp_value, + result = !ompi_osc_rdma_lock_compare_exchange ((osc_rdma_atomic_counter_t *) target, &_tmp_value, 1 + (osc_rdma_counter_t) my_rank); } @@ -491,7 +491,7 @@ int ompi_osc_rdma_complete_atomic (ompi_win_t *win) ret = ompi_osc_rdma_lock_btl_op (module, peer, target, MCA_BTL_ATOMIC_ADD, 1, true); assert (OMPI_SUCCESS == ret); } else { - (void) ompi_osc_rdma_counter_add ((osc_rdma_counter_t *) target, 1); + (void) ompi_osc_rdma_counter_add ((osc_rdma_atomic_counter_t *) target, 1); } } diff --git a/ompi/mca/osc/rdma/osc_rdma_frag.h b/ompi/mca/osc/rdma/osc_rdma_frag.h index beecce93be..f174f48e7f 100644 --- a/ompi/mca/osc/rdma/osc_rdma_frag.h +++ b/ompi/mca/osc/rdma/osc_rdma_frag.h @@ -72,7 +72,7 @@ static inline int ompi_osc_rdma_frag_alloc (ompi_osc_rdma_module_t *module, size } } - if (!opal_atomic_compare_exchange_strong_ptr (&module->rdma_frag, &(void *){NULL}, curr)) { + if (!opal_atomic_compare_exchange_strong_ptr ((opal_atomic_intptr_t *) &module->rdma_frag, &(intptr_t){0}, (intptr_t) curr)) { ompi_osc_rdma_deregister (module, curr->handle); curr->handle = NULL; diff --git a/ompi/mca/osc/rdma/osc_rdma_lock.h b/ompi/mca/osc/rdma/osc_rdma_lock.h index 7af4d703f6..f90341b94a 100644 --- a/ompi/mca/osc/rdma/osc_rdma_lock.h +++ b/ompi/mca/osc/rdma/osc_rdma_lock.h @@ -1,6 +1,6 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* - * Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -15,13 +15,13 @@ #include "osc_rdma_types.h" #include "osc_rdma_frag.h" -static inline int ompi_osc_rdma_trylock_local (volatile ompi_osc_rdma_lock_t *lock) +static inline int ompi_osc_rdma_trylock_local (ompi_osc_rdma_atomic_lock_t *lock) { ompi_osc_rdma_lock_t _tmp_value = 0; return !ompi_osc_rdma_lock_compare_exchange (lock, &_tmp_value, OMPI_OSC_RDMA_LOCK_EXCLUSIVE); } -static inline void ompi_osc_rdma_unlock_local (volatile ompi_osc_rdma_lock_t *lock) +static inline void ompi_osc_rdma_unlock_local (ompi_osc_rdma_atomic_lock_t *lock) { (void) ompi_osc_rdma_lock_add (lock, -OMPI_OSC_RDMA_LOCK_EXCLUSIVE); } @@ -266,7 +266,7 @@ static inline int ompi_osc_rdma_lock_release_shared (ompi_osc_rdma_module_t *mod return ompi_osc_rdma_lock_btl_op (module, peer, lock, MCA_BTL_ATOMIC_ADD, value, false); } - (void) ompi_osc_rdma_lock_add ((volatile ompi_osc_rdma_lock_t *) lock, value); + (void) ompi_osc_rdma_lock_add ((ompi_osc_rdma_atomic_lock_t *) lock, value); return OMPI_SUCCESS; } @@ -320,14 +320,14 @@ static inline int ompi_osc_rdma_lock_acquire_shared (ompi_osc_rdma_module_t *mod } while (1); } else { do { - lock_state = ompi_osc_rdma_lock_add ((volatile ompi_osc_rdma_lock_t *) lock, value); + lock_state = ompi_osc_rdma_lock_add ((ompi_osc_rdma_atomic_lock_t *) lock, value); OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_DEBUG, "local shared lock incremented. old value 0x%lx", (unsigned long) lock_state); if (!(lock_state & check)) { break; } - (void) ompi_osc_rdma_lock_add ((volatile ompi_osc_rdma_lock_t *) lock, -value); + (void) ompi_osc_rdma_lock_add ((ompi_osc_rdma_atomic_lock_t *) lock, -value); ompi_osc_rdma_progress (module); } while (1); } @@ -378,7 +378,7 @@ static inline int ompi_osc_rdma_lock_try_acquire_exclusive (ompi_osc_rdma_module return lock_state != 0; } - return ompi_osc_rdma_trylock_local ((int64_t *)(intptr_t) lock); + return ompi_osc_rdma_trylock_local ((ompi_osc_rdma_atomic_lock_t *)(intptr_t) lock); } /** @@ -431,7 +431,7 @@ static inline int ompi_osc_rdma_lock_release_exclusive (ompi_osc_rdma_module_t * abort (); } } else { - ompi_osc_rdma_unlock_local ((volatile ompi_osc_rdma_lock_t *)(intptr_t) lock); + ompi_osc_rdma_unlock_local ((ompi_osc_rdma_atomic_lock_t *)(intptr_t) lock); } OSC_RDMA_VERBOSE(MCA_BASE_VERBOSE_DEBUG, "exclusive lock released"); diff --git a/ompi/mca/osc/rdma/osc_rdma_peer.h b/ompi/mca/osc/rdma/osc_rdma_peer.h index a0db4c4a7f..087731aa46 100644 --- a/ompi/mca/osc/rdma/osc_rdma_peer.h +++ b/ompi/mca/osc/rdma/osc_rdma_peer.h @@ -43,7 +43,7 @@ struct ompi_osc_rdma_peer_t { int rank; /** peer flags */ - volatile int32_t flags; + opal_atomic_int32_t flags; }; typedef struct ompi_osc_rdma_peer_t ompi_osc_rdma_peer_t; diff --git a/ompi/mca/osc/rdma/osc_rdma_request.h b/ompi/mca/osc/rdma/osc_rdma_request.h index ad052e172c..089a5d8eb8 100644 --- a/ompi/mca/osc/rdma/osc_rdma_request.h +++ b/ompi/mca/osc/rdma/osc_rdma_request.h @@ -40,7 +40,7 @@ struct ompi_osc_rdma_request_t { void *origin_addr; ompi_osc_rdma_module_t *module; - volatile int32_t outstanding_requests; + opal_atomic_int32_t outstanding_requests; bool internal; ptrdiff_t offset; diff --git a/ompi/mca/osc/rdma/osc_rdma_sync.h b/ompi/mca/osc/rdma/osc_rdma_sync.h index 202bf79265..ce0daa5db9 100644 --- a/ompi/mca/osc/rdma/osc_rdma_sync.h +++ b/ompi/mca/osc/rdma/osc_rdma_sync.h @@ -34,7 +34,7 @@ typedef enum ompi_osc_rdma_sync_type_t ompi_osc_rdma_sync_type_t; struct ompi_osc_rdma_module_t; struct ompi_osc_rdma_sync_aligned_counter_t { - volatile osc_rdma_counter_t counter; + osc_rdma_atomic_counter_t counter; /* pad out to next cache line */ uint64_t padding[7]; }; diff --git a/ompi/mca/osc/rdma/osc_rdma_types.h b/ompi/mca/osc/rdma/osc_rdma_types.h index 4acb40154d..25fcc1dc4e 100644 --- a/ompi/mca/osc/rdma/osc_rdma_types.h +++ b/ompi/mca/osc/rdma/osc_rdma_types.h @@ -24,6 +24,7 @@ struct ompi_osc_rdma_peer_t; typedef int64_t osc_rdma_base_t; typedef int64_t osc_rdma_size_t; typedef int64_t osc_rdma_counter_t; +typedef opal_atomic_int64_t osc_rdma_atomic_counter_t; #define ompi_osc_rdma_counter_add opal_atomic_add_fetch_64 @@ -32,6 +33,7 @@ typedef int64_t osc_rdma_counter_t; typedef int32_t osc_rdma_base_t; typedef int32_t osc_rdma_size_t; typedef int32_t osc_rdma_counter_t; +typedef opal_atomic_int32_t osc_rdma_atomic_counter_t; #define ompi_osc_rdma_counter_add opal_atomic_add_fetch_32 @@ -42,8 +44,9 @@ typedef int32_t osc_rdma_counter_t; #define OMPI_OSC_RDMA_LOCK_EXCLUSIVE 0x8000000000000000l typedef int64_t ompi_osc_rdma_lock_t; +typedef opal_atomic_int64_t ompi_osc_rdma_atomic_lock_t; -static inline int64_t ompi_osc_rdma_lock_add (volatile int64_t *p, int64_t value) +static inline int64_t ompi_osc_rdma_lock_add (opal_atomic_int64_t *p, int64_t value) { int64_t new; @@ -54,7 +57,7 @@ static inline int64_t ompi_osc_rdma_lock_add (volatile int64_t *p, int64_t value return new; } -static inline int ompi_osc_rdma_lock_compare_exchange (volatile int64_t *p, int64_t *comp, int64_t value) +static inline int ompi_osc_rdma_lock_compare_exchange (opal_atomic_int64_t *p, int64_t *comp, int64_t value) { int ret; @@ -70,8 +73,9 @@ static inline int ompi_osc_rdma_lock_compare_exchange (volatile int64_t *p, int6 #define OMPI_OSC_RDMA_LOCK_EXCLUSIVE 0x80000000l typedef int32_t ompi_osc_rdma_lock_t; +typedef opal_atomic_int32_t ompi_osc_rdma_atomic_lock_t; -static inline int32_t ompi_osc_rdma_lock_add (volatile int32_t *p, int32_t value) +static inline int32_t ompi_osc_rdma_lock_add (opal_atomic_int32_t *p, int32_t value) { int32_t new; @@ -83,7 +87,7 @@ static inline int32_t ompi_osc_rdma_lock_add (volatile int32_t *p, int32_t value return new; } -static inline int ompi_osc_rdma_lock_compare_exchange (volatile int32_t *p, int32_t *comp, int32_t value) +static inline int ompi_osc_rdma_lock_compare_exchange (opal_atomic_int32_t *p, int32_t *comp, int32_t value) { int ret; @@ -157,7 +161,7 @@ struct ompi_osc_rdma_state_t { /** post buffers */ osc_rdma_counter_t post_peers[OMPI_OSC_RDMA_POST_PEER_MAX]; /** counter for number of post messages received */ - osc_rdma_counter_t num_post_msgs; + osc_rdma_atomic_counter_t num_post_msgs; /** counter for number of complete messages received */ osc_rdma_counter_t num_complete_msgs; /** lock for the region state to ensure consistency */ @@ -195,11 +199,11 @@ struct ompi_osc_rdma_frag_t { opal_free_list_item_t super; /* Number of operations which have started writing into the frag, but not yet completed doing so */ - volatile int32_t pending; + opal_atomic_int32_t pending; #if OPAL_HAVE_ATOMIC_MATH_64 - volatile int64_t curr_index; + opal_atomic_int64_t curr_index; #else - volatile int32_t curr_index; + opal_atomic_int32_t curr_index; #endif struct ompi_osc_rdma_module_t *module; diff --git a/ompi/mca/osc/sm/osc_sm.h b/ompi/mca/osc/sm/osc_sm.h index f0917b54cb..01e9698205 100644 --- a/ompi/mca/osc/sm/osc_sm.h +++ b/ompi/mca/osc/sm/osc_sm.h @@ -21,12 +21,14 @@ #if OPAL_HAVE_ATOMIC_MATH_64 typedef uint64_t osc_sm_post_type_t; +typedef opal_atomic_uint64_t osc_sm_post_atomic_type_t; #define OSC_SM_POST_BITS 6 #define OSC_SM_POST_MASK 0x3f #else typedef uint32_t osc_sm_post_type_t; +typedef opal_atomic_uint32_t osc_sm_post_atomic_type_t; #define OSC_SM_POST_BITS 5 #define OSC_SM_POST_MASK 0x1f @@ -53,7 +55,7 @@ struct ompi_osc_sm_lock_t { typedef struct ompi_osc_sm_lock_t ompi_osc_sm_lock_t; struct ompi_osc_sm_node_state_t { - int32_t complete_count; + opal_atomic_int32_t complete_count; ompi_osc_sm_lock_t lock; opal_atomic_lock_t accumulate_lock; }; @@ -98,7 +100,7 @@ struct ompi_osc_sm_module_t { ompi_osc_sm_node_state_t *my_node_state; ompi_osc_sm_node_state_t *node_states; - osc_sm_post_type_t ** volatile posts; + osc_sm_post_atomic_type_t **posts; opal_mutex_t lock; }; diff --git a/ompi/mca/osc/sm/osc_sm_active_target.c b/ompi/mca/osc/sm/osc_sm_active_target.c index ab0f73f87c..237d2b8407 100644 --- a/ompi/mca/osc/sm/osc_sm_active_target.c +++ b/ompi/mca/osc/sm/osc_sm_active_target.c @@ -162,9 +162,9 @@ ompi_osc_sm_start(struct ompi_group_t *group, opal_atomic_rmb (); #if OPAL_HAVE_ATOMIC_MATH_64 - (void) opal_atomic_fetch_xor_64 ((volatile int64_t *) module->posts[my_rank] + rank_byte, rank_bit); + (void) opal_atomic_fetch_xor_64 ((opal_atomic_int64_t *) module->posts[my_rank] + rank_byte, rank_bit); #else - (void) opal_atomic_fetch_xor_32 ((volatile int32_t *) module->posts[my_rank] + rank_byte, rank_bit); + (void) opal_atomic_fetch_xor_32 ((opal_atomic_int32_t *) module->posts[my_rank] + rank_byte, rank_bit); #endif } @@ -188,7 +188,7 @@ ompi_osc_sm_complete(struct ompi_win_t *win) opal_atomic_mb(); group = module->start_group; - if (NULL == group || !OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR(&module->start_group, &group, NULL)) { + if (NULL == group || !OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR((opal_atomic_intptr_t *) &module->start_group, (opal_atomic_intptr_t *) &group, 0)) { return OMPI_ERR_RMA_SYNC; } @@ -247,7 +247,11 @@ ompi_osc_sm_post(struct ompi_group_t *group, gsize = ompi_group_size(module->post_group); for (int i = 0 ; i < gsize ; ++i) { - opal_atomic_add ((volatile osc_sm_post_type_t *) module->posts[ranks[i]] + my_byte, my_bit); +#if OPAL_HAVE_ATOMIC_MATH_64 + (void) opal_atomic_fetch_add_64 ((opal_atomic_int64_t *) module->posts[ranks[i]] + my_byte, my_bit); +#else + (void) opal_atomic_fetch_add_32 ((opal_atomic_int32_t *) module->posts[ranks[i]] + my_byte, my_bit); +#endif } opal_atomic_wmb (); diff --git a/ompi/mca/osc/sm/osc_sm_component.c b/ompi/mca/osc/sm/osc_sm_component.c index de891e71a1..1006291584 100644 --- a/ompi/mca/osc/sm/osc_sm_component.c +++ b/ompi/mca/osc/sm/osc_sm_component.c @@ -236,7 +236,7 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit if (NULL == module->node_states) return OMPI_ERR_TEMP_OUT_OF_RESOURCE; module->posts = calloc (1, sizeof(module->posts[0]) + sizeof (module->posts[0][0])); if (NULL == module->posts) return OMPI_ERR_TEMP_OUT_OF_RESOURCE; - module->posts[0] = (osc_sm_post_type_t *) (module->posts + 1); + module->posts[0] = (osc_sm_post_atomic_type_t *) (module->posts + 1); } else { unsigned long total, *rbuf; int i, flag; @@ -328,7 +328,7 @@ component_select(struct ompi_win_t *win, void **base, size_t size, int disp_unit if (NULL == module->posts) return OMPI_ERR_TEMP_OUT_OF_RESOURCE; /* set module->posts[0] first to ensure 64-bit alignment */ - module->posts[0] = (osc_sm_post_type_t *) (module->segment_base); + module->posts[0] = (osc_sm_post_atomic_type_t *) (module->segment_base); module->global_state = (ompi_osc_sm_global_state_t *) (module->posts[0] + comm_size * post_size); module->node_states = (ompi_osc_sm_node_state_t *) (module->global_state + 1); diff --git a/ompi/mca/osc/sm/osc_sm_passive_target.c b/ompi/mca/osc/sm/osc_sm_passive_target.c index a3388b776a..8c27561e54 100644 --- a/ompi/mca/osc/sm/osc_sm_passive_target.c +++ b/ompi/mca/osc/sm/osc_sm_passive_target.c @@ -28,7 +28,7 @@ lk_fetch_add32(ompi_osc_sm_module_t *module, { /* opal_atomic_add_fetch_32 is an add then fetch so delta needs to be subtracted out to get the * old value */ - return opal_atomic_add_fetch_32((int32_t*) ((char*) &module->node_states[target].lock + offset), + return opal_atomic_add_fetch_32((opal_atomic_int32_t *) ((char*) &module->node_states[target].lock + offset), delta) - delta; } @@ -39,7 +39,7 @@ lk_add32(ompi_osc_sm_module_t *module, size_t offset, uint32_t delta) { - opal_atomic_add_fetch_32((int32_t*) ((char*) &module->node_states[target].lock + offset), + opal_atomic_add_fetch_32((opal_atomic_int32_t *) ((char*) &module->node_states[target].lock + offset), delta); } diff --git a/ompi/mca/pml/base/pml_base_bsend.c b/ompi/mca/pml/base/pml_base_bsend.c index ef6be82599..4da15522b8 100644 --- a/ompi/mca/pml/base/pml_base_bsend.c +++ b/ompi/mca/pml/base/pml_base_bsend.c @@ -51,7 +51,7 @@ static size_t mca_pml_bsend_size; /* adjusted size of user buffe static size_t mca_pml_bsend_count; /* number of outstanding requests */ static size_t mca_pml_bsend_pagesz; /* mmap page size */ static int mca_pml_bsend_pagebits; /* number of bits in pagesz */ -static int32_t mca_pml_bsend_init = 0; +static opal_atomic_int32_t mca_pml_bsend_init = 0; /* defined in pml_base_open.c */ extern char *ompi_pml_base_bsend_allocator_name; diff --git a/ompi/mca/pml/ob1/pml_ob1_comm.h b/ompi/mca/pml/ob1/pml_ob1_comm.h index 6366649678..780c9a44fc 100644 --- a/ompi/mca/pml/ob1/pml_ob1_comm.h +++ b/ompi/mca/pml/ob1/pml_ob1_comm.h @@ -44,7 +44,7 @@ struct mca_pml_ob1_comm_proc_t { opal_object_t super; struct ompi_proc_t* ompi_proc; uint16_t expected_sequence; /**< send message sequence number - receiver side */ - volatile int32_t send_sequence; /**< send side sequence number */ + opal_atomic_int32_t send_sequence; /**< send side sequence number */ struct mca_pml_ob1_recv_frag_t* frags_cant_match; /**< out-of-order fragment queues */ #if !MCA_PML_OB1_CUSTOM_MATCH opal_list_t specific_receives; /**< queues of unmatched specific receives */ diff --git a/ompi/mca/pml/ob1/pml_ob1_progress.c b/ompi/mca/pml/ob1/pml_ob1_progress.c index e1f84e796b..b17f2ee0a6 100644 --- a/ompi/mca/pml/ob1/pml_ob1_progress.c +++ b/ompi/mca/pml/ob1/pml_ob1_progress.c @@ -53,7 +53,7 @@ static inline int mca_pml_ob1_process_pending_cuda_async_copies(void) } #endif /* OPAL_CUDA_SUPPORT */ -static int mca_pml_ob1_progress_needed = 0; +static opal_atomic_int32_t mca_pml_ob1_progress_needed = 0; int mca_pml_ob1_enable_progress(int32_t count) { int32_t progress_count = OPAL_ATOMIC_ADD_FETCH32(&mca_pml_ob1_progress_needed, count); diff --git a/ompi/mca/pml/ob1/pml_ob1_recvreq.h b/ompi/mca/pml/ob1/pml_ob1_recvreq.h index bc37a00abf..2bfe294f59 100644 --- a/ompi/mca/pml/ob1/pml_ob1_recvreq.h +++ b/ompi/mca/pml/ob1/pml_ob1_recvreq.h @@ -41,9 +41,9 @@ BEGIN_C_DECLS struct mca_pml_ob1_recv_request_t { mca_pml_base_recv_request_t req_recv; opal_ptr_t remote_req_send; - int32_t req_lock; - int32_t req_pipeline_depth; - size_t req_bytes_received; /**< amount of data transferred into the user buffer */ + opal_atomic_int32_t req_lock; + opal_atomic_int32_t req_pipeline_depth; + opal_atomic_size_t req_bytes_received; /**< amount of data transferred into the user buffer */ size_t req_bytes_expected; /**< local size of the data as suggested by the user */ size_t req_rdma_offset; size_t req_send_offset; diff --git a/ompi/mca/pml/ob1/pml_ob1_sendreq.h b/ompi/mca/pml/ob1/pml_ob1_sendreq.h index 587740dcda..787f48b554 100644 --- a/ompi/mca/pml/ob1/pml_ob1_sendreq.h +++ b/ompi/mca/pml/ob1/pml_ob1_sendreq.h @@ -46,11 +46,11 @@ struct mca_pml_ob1_send_request_t { mca_pml_base_send_request_t req_send; mca_bml_base_endpoint_t* req_endpoint; opal_ptr_t req_recv; - int32_t req_state; - int32_t req_lock; + opal_atomic_int32_t req_state; + opal_atomic_int32_t req_lock; bool req_throttle_sends; - int32_t req_pipeline_depth; - size_t req_bytes_delivered; + opal_atomic_int32_t req_pipeline_depth; + opal_atomic_size_t req_bytes_delivered; uint32_t req_rdma_cnt; mca_pml_ob1_send_pending_t req_pending; opal_mutex_t req_send_range_lock; diff --git a/ompi/request/grequestx.c b/ompi/request/grequestx.c index acd688eacf..3edde17ee3 100644 --- a/ompi/request/grequestx.c +++ b/ompi/request/grequestx.c @@ -28,7 +28,7 @@ static bool requests_initialized = false; static opal_list_t requests; -static int32_t active_requests = 0; +static opal_atomic_int32_t active_requests = 0; static bool in_progress = false; static opal_mutex_t lock; diff --git a/ompi/runtime/mpiruntime.h b/ompi/runtime/mpiruntime.h index 32f1c13878..a57173158b 100644 --- a/ompi/runtime/mpiruntime.h +++ b/ompi/runtime/mpiruntime.h @@ -51,7 +51,7 @@ struct ompi_predefined_datatype_t; /** Mutex to protect all the _init and _finalize variables */ OMPI_DECLSPEC extern opal_mutex_t ompi_mpi_bootstrap_mutex; /** Did MPI start to initialize? */ -OMPI_DECLSPEC extern volatile int32_t ompi_mpi_state; +OMPI_DECLSPEC extern opal_atomic_int32_t ompi_mpi_state; /** Has the RTE been initialized? */ OMPI_DECLSPEC extern volatile bool ompi_rte_initialized; diff --git a/ompi/runtime/ompi_mpi_init.c b/ompi/runtime/ompi_mpi_init.c index b35c491b73..02c81b3704 100644 --- a/ompi/runtime/ompi_mpi_init.c +++ b/ompi/runtime/ompi_mpi_init.c @@ -130,7 +130,7 @@ const char ompi_version_string[] = OMPI_IDENT_STRING; * Global variables and symbols for the MPI layer */ -volatile int32_t ompi_mpi_state = OMPI_MPI_STATE_NOT_INITIALIZED; +opal_atomic_int32_t ompi_mpi_state = OMPI_MPI_STATE_NOT_INITIALIZED; volatile bool ompi_rte_initialized = false; bool ompi_mpi_thread_multiple = false; diff --git a/ompi/runtime/ompi_spc.c b/ompi/runtime/ompi_spc.c index caee2cda6f..99190a8cc1 100644 --- a/ompi/runtime/ompi_spc.c +++ b/ompi/runtime/ompi_spc.c @@ -456,7 +456,7 @@ void ompi_spc_timer_stop(unsigned int event_id, opal_timer_t *cycles) /* This is denoted unlikely because the counters will often be turned off. */ if( OPAL_UNLIKELY(IS_SPC_BIT_SET(ompi_spc_attached_event, event_id)) ) { *cycles = opal_timer_base_get_cycles() - *cycles; - OPAL_THREAD_ADD_FETCH_SIZE_T(&ompi_spc_events[event_id].value, (ompi_spc_value_t)*cycles); + OPAL_THREAD_ADD_FETCH_SIZE_T(&ompi_spc_events[event_id].value, (size_t) *cycles); } } diff --git a/ompi/runtime/ompi_spc.h b/ompi/runtime/ompi_spc.h index 54851de038..5d040511c3 100644 --- a/ompi/runtime/ompi_spc.h +++ b/ompi/runtime/ompi_spc.h @@ -166,7 +166,7 @@ typedef enum ompi_spc_counters { /* There is currently no support for atomics on long long values so we will default to * size_t for now until support for such atomics is implemented. */ -typedef size_t ompi_spc_value_t; +typedef opal_atomic_size_t ompi_spc_value_t; /* A structure for storing the event data */ typedef struct ompi_spc_s{ diff --git a/opal/class/opal_fifo.c b/opal/class/opal_fifo.c index 6e23e13457..238801d492 100644 --- a/opal/class/opal_fifo.c +++ b/opal/class/opal_fifo.c @@ -10,7 +10,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2014 Los Alamos National Security, LLC. All rights + * Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -32,10 +32,10 @@ static void opal_fifo_construct (opal_fifo_t *fifo) fifo->opal_fifo_ghost.item_free = 0; fifo->opal_fifo_head.data.counter = 0; - fifo->opal_fifo_head.data.item = &fifo->opal_fifo_ghost; + fifo->opal_fifo_head.data.item = (intptr_t) &fifo->opal_fifo_ghost; fifo->opal_fifo_tail.data.counter = 0; - fifo->opal_fifo_tail.data.item = &fifo->opal_fifo_ghost; + fifo->opal_fifo_tail.data.item = (intptr_t) &fifo->opal_fifo_ghost; } OBJ_CLASS_INSTANCE(opal_fifo_t, opal_object_t, opal_fifo_construct, NULL); diff --git a/opal/class/opal_fifo.h b/opal/class/opal_fifo.h index 556688bc76..2a0607ac34 100644 --- a/opal/class/opal_fifo.h +++ b/opal/class/opal_fifo.h @@ -100,13 +100,13 @@ static inline opal_list_item_t *opal_fifo_push_atomic (opal_fifo_t *fifo, opal_atomic_wmb (); - if (ghost == tail.data.item) { + if ((intptr_t) ghost == tail.data.item) { /* update the head */ opal_counted_pointer_t head = {.value = fifo->opal_fifo_head.value}; opal_update_counted_pointer (&fifo->opal_fifo_head, &head, item); } else { /* update previous item */ - tail.data.item->opal_list_next = item; + ((opal_list_item_t *) tail.data.item)->opal_list_next = item; } return (opal_list_item_t *) tail.data.item; @@ -129,12 +129,12 @@ static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo) item = (opal_list_item_t *) head.data.item; next = (opal_list_item_t *) item->opal_list_next; - if (ghost == tail.data.item && ghost == item) { + if ((intptr_t) ghost == tail.data.item && ghost == item) { return NULL; } /* the head or next pointer are in an inconsistent state. keep looping. */ - if (tail.data.item != item && ghost != tail.data.item && ghost == next) { + if (tail.data.item != (intptr_t) item && (intptr_t) ghost != tail.data.item && ghost == next) { opal_read_counted_pointer (&fifo->opal_fifo_head, &head); continue; } @@ -165,7 +165,7 @@ static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo) * will be attempting to update the head until after it has been updated * with the next pointer. push will not see an empty list and other pop * operations will loop until the head is consistent. */ - fifo->opal_fifo_head.data.item = (opal_list_item_t *) item->opal_list_next; + fifo->opal_fifo_head.data.item = (intptr_t) item->opal_list_next; opal_atomic_wmb (); } } @@ -191,13 +191,13 @@ static inline opal_list_item_t *opal_fifo_push_atomic (opal_fifo_t *fifo, opal_atomic_wmb (); /* try to get the tail */ - tail_item = opal_atomic_swap_ptr (&fifo->opal_fifo_tail.data.item, item); + tail_item = (opal_list_item_t *) opal_atomic_swap_ptr (&fifo->opal_fifo_tail.data.item, (intptr_t) item); opal_atomic_wmb (); if (ghost == tail_item) { /* update the head */ - fifo->opal_fifo_head.data.item = item; + fifo->opal_fifo_head.data.item = (intptr_t) item; } else { /* update previous item */ tail_item->opal_list_next = item; @@ -230,7 +230,7 @@ static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo) opal_atomic_ll_ptr(&fifo->opal_fifo_head.data.item, item); if (ghost == item) { - if (ghost == fifo->opal_fifo_tail.data.item) { + if ((intptr_t) ghost == fifo->opal_fifo_tail.data.item) { return NULL; } @@ -248,7 +248,7 @@ static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo) /* protect against ABA issues by "locking" the head */ do { - if (!opal_atomic_swap_32 ((volatile int32_t *) &fifo->opal_fifo_head.data.counter, 1)) { + if (!opal_atomic_swap_32 ((opal_atomic_int32_t *) &fifo->opal_fifo_head.data.counter, 1)) { break; } @@ -270,12 +270,12 @@ static inline opal_list_item_t *opal_fifo_pop_atomic (opal_fifo_t *fifo) if (ghost == next) { void *tmp = item; - if (!opal_atomic_compare_exchange_strong_ptr (&fifo->opal_fifo_tail.data.item, &tmp, (void *) ghost)) { + if (!opal_atomic_compare_exchange_strong_ptr (&fifo->opal_fifo_tail.data.item, (intptr_t *) &tmp, (intptr_t) ghost)) { do { opal_atomic_rmb (); } while (ghost == item->opal_list_next); - fifo->opal_fifo_head.data.item = (opal_list_item_t *) item->opal_list_next; + fifo->opal_fifo_head.data.item = (intptr_t) item->opal_list_next; } } @@ -299,9 +299,9 @@ static inline opal_list_item_t *opal_fifo_push_st (opal_fifo_t *fifo, item->opal_list_next = &fifo->opal_fifo_ghost; - fifo->opal_fifo_tail.data.item = item; + fifo->opal_fifo_tail.data.item = (intptr_t) item; if (&fifo->opal_fifo_ghost == opal_fifo_head (fifo)) { - fifo->opal_fifo_head.data.item = item; + fifo->opal_fifo_head.data.item = (intptr_t) item; } else { prev->opal_list_next = item; } @@ -317,9 +317,9 @@ static inline opal_list_item_t *opal_fifo_pop_st (opal_fifo_t *fifo) return NULL; } - fifo->opal_fifo_head.data.item = (opal_list_item_t *) item->opal_list_next; + fifo->opal_fifo_head.data.item = (intptr_t) item->opal_list_next; if (&fifo->opal_fifo_ghost == opal_fifo_head (fifo)) { - fifo->opal_fifo_tail.data.item = &fifo->opal_fifo_ghost; + fifo->opal_fifo_tail.data.item = (intptr_t) &fifo->opal_fifo_ghost; } item->opal_list_next = NULL; diff --git a/opal/class/opal_interval_tree.c b/opal/class/opal_interval_tree.c index e8ccda2024..a20ef980df 100644 --- a/opal/class/opal_interval_tree.c +++ b/opal/class/opal_interval_tree.c @@ -130,7 +130,7 @@ static opal_interval_tree_token_t opal_interval_tree_reader_get_token (opal_inte } } - while (!OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_32((volatile int32_t *) &tree->reader_epochs[token], + while (!OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_32((opal_atomic_int32_t *) &tree->reader_epochs[token], &(int32_t) {UINT_MAX}, tree->epoch)); return token; diff --git a/opal/class/opal_interval_tree.h b/opal/class/opal_interval_tree.h index cbb36eb1cc..68f6fe4cef 100644 --- a/opal/class/opal_interval_tree.h +++ b/opal/class/opal_interval_tree.h @@ -83,11 +83,11 @@ struct opal_interval_tree_t { opal_free_list_t free_list; /**< the free list to get the memory from */ opal_list_t gc_list; /**< list of nodes that need to be released */ uint32_t epoch; /**< current update epoch */ - volatile size_t tree_size; /**< the current size of the tree */ - volatile int32_t lock; /**< update lock */ - volatile int32_t reader_count; /**< current highest reader slot to check */ + opal_atomic_size_t tree_size; /**< the current size of the tree */ + opal_atomic_int32_t lock; /**< update lock */ + opal_atomic_int32_t reader_count; /**< current highest reader slot to check */ volatile uint32_t reader_id; /**< next reader slot to check */ - volatile uint32_t reader_epochs[OPAL_INTERVAL_TREE_MAX_READERS]; + opal_atomic_uint32_t reader_epochs[OPAL_INTERVAL_TREE_MAX_READERS]; }; typedef struct opal_interval_tree_t opal_interval_tree_t; diff --git a/opal/class/opal_lifo.c b/opal/class/opal_lifo.c index 1c403fd64c..c2fc30c51c 100644 --- a/opal/class/opal_lifo.c +++ b/opal/class/opal_lifo.c @@ -26,7 +26,7 @@ static void opal_lifo_construct (opal_lifo_t *lifo) { OBJ_CONSTRUCT(&lifo->opal_lifo_ghost, opal_list_item_t); lifo->opal_lifo_ghost.opal_list_next = &lifo->opal_lifo_ghost; - lifo->opal_lifo_head.data.item = &lifo->opal_lifo_ghost; + lifo->opal_lifo_head.data.item = (intptr_t) &lifo->opal_lifo_ghost; lifo->opal_lifo_head.data.counter = 0; } diff --git a/opal/class/opal_lifo.h b/opal/class/opal_lifo.h index 945932683b..4d07ef6ccd 100644 --- a/opal/class/opal_lifo.h +++ b/opal/class/opal_lifo.h @@ -48,11 +48,12 @@ union opal_counted_pointer_t { /** update counter used when cmpset_128 is available */ uint64_t counter; /** list item pointer */ - volatile opal_list_item_t * volatile item; + volatile opal_atomic_intptr_t item; } data; #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 && HAVE_OPAL_INT128_T /** used for atomics when there is a cmpset that can operate on * two 64-bit values */ + opal_atomic_int128_t atomic_value; opal_int128_t value; #endif }; @@ -65,17 +66,17 @@ typedef union opal_counted_pointer_t opal_counted_pointer_t; * to allow the upper level to detect if this element is the first one in the * list (if the list was empty before this operation). */ -static inline bool opal_update_counted_pointer (volatile opal_counted_pointer_t *addr, opal_counted_pointer_t *old, +static inline bool opal_update_counted_pointer (volatile opal_counted_pointer_t * volatile addr, opal_counted_pointer_t *old, opal_list_item_t *item) { opal_counted_pointer_t new_p; - new_p.data.item = item; + new_p.data.item = (intptr_t) item; new_p.data.counter = old->data.counter + 1; - return opal_atomic_compare_exchange_strong_128 (&addr->value, &old->value, new_p.value); + return opal_atomic_compare_exchange_strong_128 (&addr->atomic_value, &old->value, new_p.value); } __opal_attribute_always_inline__ -static inline void opal_read_counted_pointer (volatile opal_counted_pointer_t *addr, opal_counted_pointer_t *value) +static inline void opal_read_counted_pointer (volatile opal_counted_pointer_t * volatile addr, opal_counted_pointer_t *value) { /* most platforms do not read the value atomically so make sure we read the counted pointer in a specific order */ value->data.counter = addr->data.counter; @@ -151,7 +152,7 @@ static inline opal_list_item_t *opal_lifo_push_atomic (opal_lifo_t *lifo, opal_atomic_wmb (); /* to protect against ABA issues it is sufficient to only update the counter in pop */ - if (opal_atomic_compare_exchange_strong_ptr (&lifo->opal_lifo_head.data.item, &next, item)) { + if (opal_atomic_compare_exchange_strong_ptr (&lifo->opal_lifo_head.data.item, (intptr_t *) &next, (intptr_t) item)) { return next; } /* DO some kind of pause to release the bus */ @@ -200,7 +201,7 @@ static inline opal_list_item_t *opal_lifo_push_atomic (opal_lifo_t *lifo, do { item->opal_list_next = next; opal_atomic_wmb(); - if (opal_atomic_compare_exchange_strong_ptr (&lifo->opal_lifo_head.data.item, &next, item)) { + if (opal_atomic_compare_exchange_strong_ptr (&lifo->opal_lifo_head.data.item, (intptr_t *) &next, (intptr_t) item)) { opal_atomic_wmb (); /* now safe to pop this item */ item->item_free = 0; @@ -254,7 +255,7 @@ static inline opal_list_item_t *opal_lifo_pop_atomic (opal_lifo_t* lifo) while ((item=(opal_list_item_t *)lifo->opal_lifo_head.data.item) != ghost) { /* ensure it is safe to pop the head */ - if (opal_atomic_swap_32((volatile int32_t *) &item->item_free, 1)) { + if (opal_atomic_swap_32((opal_atomic_int32_t *) &item->item_free, 1)) { continue; } @@ -262,8 +263,8 @@ static inline opal_list_item_t *opal_lifo_pop_atomic (opal_lifo_t* lifo) head = item; /* try to swap out the head pointer */ - if (opal_atomic_compare_exchange_strong_ptr (&lifo->opal_lifo_head.data.item, &head, - (void *) item->opal_list_next)) { + if (opal_atomic_compare_exchange_strong_ptr (&lifo->opal_lifo_head.data.item, (intptr_t *) &head, + (intptr_t) item->opal_list_next)) { break; } @@ -294,7 +295,7 @@ static inline opal_list_item_t *opal_lifo_push_st (opal_lifo_t *lifo, { item->opal_list_next = (opal_list_item_t *) lifo->opal_lifo_head.data.item; item->item_free = 0; - lifo->opal_lifo_head.data.item = item; + lifo->opal_lifo_head.data.item = (intptr_t) item; return (opal_list_item_t *) item->opal_list_next; } @@ -302,7 +303,7 @@ static inline opal_list_item_t *opal_lifo_pop_st (opal_lifo_t *lifo) { opal_list_item_t *item; item = (opal_list_item_t *) lifo->opal_lifo_head.data.item; - lifo->opal_lifo_head.data.item = (opal_list_item_t *) item->opal_list_next; + lifo->opal_lifo_head.data.item = (intptr_t) item->opal_list_next; if (item == &lifo->opal_lifo_ghost) { return NULL; } diff --git a/opal/class/opal_list.h b/opal/class/opal_list.h index 5edd6730d5..f9f1f00ab2 100644 --- a/opal/class/opal_list.h +++ b/opal/class/opal_list.h @@ -111,7 +111,7 @@ struct opal_list_item_t #if OPAL_ENABLE_DEBUG /** Atomic reference count for debugging */ - volatile int32_t opal_list_item_refcount; + opal_atomic_int32_t opal_list_item_refcount; /** The list this item belong to */ volatile struct opal_list_t* opal_list_item_belong_to; #endif @@ -654,7 +654,7 @@ static inline opal_list_item_t *opal_list_remove_first(opal_list_t *list) Caller now owns the item and should release the item when caller is done with it. */ - volatile opal_list_item_t *item; + opal_list_item_t *item; if ( 0 == list->opal_list_length ) { return (opal_list_item_t *)NULL; } @@ -669,7 +669,7 @@ static inline opal_list_item_t *opal_list_remove_first(opal_list_t *list) list->opal_list_length--; /* get pointer to first element on the list */ - item = list->opal_list_sentinel.opal_list_next; + item = (opal_list_item_t *) list->opal_list_sentinel.opal_list_next; /* reset previous pointer of next item on the list */ item->opal_list_next->opal_list_prev = item->opal_list_prev; @@ -686,11 +686,11 @@ static inline opal_list_item_t *opal_list_remove_first(opal_list_t *list) /* Spot check: ensure that the item we're returning is now on no lists */ - OPAL_THREAD_ADD_FETCH32( &(item->opal_list_item_refcount), -1 ); + OPAL_THREAD_ADD_FETCH32( &item->opal_list_item_refcount, -1 ); assert(0 == item->opal_list_item_refcount); #endif - return (opal_list_item_t *) item; + return item; } @@ -716,7 +716,7 @@ static inline opal_list_item_t *opal_list_remove_last(opal_list_t *list) Caller now owns the item and should release the item when caller is done with it. */ - volatile opal_list_item_t *item; + opal_list_item_t *item; if ( 0 == list->opal_list_length ) { return (opal_list_item_t *)NULL; } @@ -731,7 +731,7 @@ static inline opal_list_item_t *opal_list_remove_last(opal_list_t *list) list->opal_list_length--; /* get item */ - item = list->opal_list_sentinel.opal_list_prev; + item = (opal_list_item_t *) list->opal_list_sentinel.opal_list_prev; /* reset previous pointer on next to last pointer */ item->opal_list_prev->opal_list_next = item->opal_list_next; @@ -746,12 +746,12 @@ static inline opal_list_item_t *opal_list_remove_last(opal_list_t *list) /* Spot check: ensure that the item we're returning is now on no lists */ - OPAL_THREAD_ADD_FETCH32( &(item->opal_list_item_refcount), -1 ); + OPAL_THREAD_ADD_FETCH32(&item->opal_list_item_refcount, -1 ); assert(0 == item->opal_list_item_refcount); item->opal_list_item_belong_to = NULL; #endif - return (opal_list_item_t *) item; + return item; } /** diff --git a/opal/class/opal_object.h b/opal/class/opal_object.h index 4e2da95c20..4d3380f0da 100644 --- a/opal/class/opal_object.h +++ b/opal/class/opal_object.h @@ -198,7 +198,7 @@ struct opal_object_t { uint64_t obj_magic_id; #endif opal_class_t *obj_class; /**< class descriptor */ - volatile int32_t obj_reference_count; /**< reference count */ + opal_atomic_int32_t obj_reference_count; /**< reference count */ #if OPAL_ENABLE_DEBUG const char* cls_init_file_name; /**< In debug mode store the file where the object get contructed */ int cls_init_lineno; /**< In debug mode store the line number where the object get contructed */ diff --git a/opal/class/opal_tree.h b/opal/class/opal_tree.h index e1a42a3355..e73d0a48d9 100644 --- a/opal/class/opal_tree.h +++ b/opal/class/opal_tree.h @@ -120,7 +120,7 @@ typedef struct opal_tree_item_t #if OPAL_ENABLE_DEBUG /** Atomic reference count for debugging */ - int32_t opal_tree_item_refcount; + opal_atomic_int32_t opal_tree_item_refcount; /** The tree this item belong to */ struct opal_tree_t* opal_tree_item_belong_to; #endif diff --git a/opal/include/Makefile.am b/opal/include/Makefile.am index 464faafd0a..0a1c195d65 100644 --- a/opal/include/Makefile.am +++ b/opal/include/Makefile.am @@ -10,6 +10,8 @@ # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2010-2011 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2018 Los Alamos National Security, LLC. All rights +# reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -20,7 +22,8 @@ headers = \ opal_config_top.h \ opal_config_bottom.h \ - opal_stdint.h + opal_stdint.h \ + opal_stdatomic.h nodist_headers = opal_config.h diff --git a/opal/include/opal/sys/arm/atomic.h b/opal/include/opal/sys/arm/atomic.h index 6d4db3ad7a..ff2c5bab35 100644 --- a/opal/include/opal/sys/arm/atomic.h +++ b/opal/include/opal/sys/arm/atomic.h @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2010 IBM Corporation. All rights reserved. * Copyright (c) 2010 ARM ltd. All rights reserved. - * Copyright (c) 2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2017-2018 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -109,7 +109,7 @@ void opal_atomic_isync(void) #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1 #define OPAL_HAVE_ATOMIC_MATH_32 1 -static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { int32_t prev, tmp; bool ret; @@ -137,7 +137,7 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { bool rc; @@ -148,7 +148,7 @@ static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t } -static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { opal_atomic_wmb(); return opal_atomic_compare_exchange_strong_32 (addr, oldval, newval); @@ -157,7 +157,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t #if (OPAL_ASM_SUPPORT_64BIT == 1) #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1 -static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { int64_t prev; int tmp; @@ -188,7 +188,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { bool rc; @@ -199,7 +199,7 @@ static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t } -static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { opal_atomic_wmb(); return opal_atomic_compare_exchange_strong_64 (addr, oldval, newval); @@ -209,7 +209,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t #define OPAL_HAVE_ATOMIC_ADD_32 1 -static inline int32_t opal_atomic_fetch_add_32(volatile int32_t* v, int inc) +static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t* v, int inc) { int32_t t, old; int tmp; @@ -230,7 +230,7 @@ static inline int32_t opal_atomic_fetch_add_32(volatile int32_t* v, int inc) } #define OPAL_HAVE_ATOMIC_SUB_32 1 -static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t* v, int dec) +static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t* v, int dec) { int32_t t, old; int tmp; diff --git a/opal/include/opal/sys/arm64/atomic.h b/opal/include/opal/sys/arm64/atomic.h index 55a4628813..76409623b1 100644 --- a/opal/include/opal/sys/arm64/atomic.h +++ b/opal/include/opal/sys/arm64/atomic.h @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2010 IBM Corporation. All rights reserved. * Copyright (c) 2010 ARM ltd. All rights reserved. - * Copyright (c) 2016-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2016-2018 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -82,7 +82,7 @@ static inline void opal_atomic_isync (void) * *********************************************************************/ -static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { int32_t prev, tmp; bool ret; @@ -102,7 +102,7 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add return ret; } -static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, int32_t newval) +static inline int32_t opal_atomic_swap_32(opal_atomic_int32_t *addr, int32_t newval) { int32_t ret, tmp; @@ -121,7 +121,7 @@ static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, int32_t newval atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { int32_t prev, tmp; bool ret; @@ -142,7 +142,7 @@ static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t } -static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { int32_t prev, tmp; bool ret; @@ -164,7 +164,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t #define opal_atomic_ll_32(addr, ret) \ do { \ - volatile int32_t *_addr = (addr); \ + opal_atomic_int32_t *_addr = (addr); \ int32_t _ret; \ \ __asm__ __volatile__ ("ldaxr %w0, [%1] \n" \ @@ -176,7 +176,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t #define opal_atomic_sc_32(addr, newval, ret) \ do { \ - volatile int32_t *_addr = (addr); \ + opal_atomic_int32_t *_addr = (addr); \ int32_t _newval = (int32_t) newval; \ int _ret; \ \ @@ -188,7 +188,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t ret = (_ret == 0); \ } while (0) -static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { int64_t prev; int tmp; @@ -209,7 +209,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add return ret; } -static inline int64_t opal_atomic_swap_64 (volatile int64_t *addr, int64_t newval) +static inline int64_t opal_atomic_swap_64 (opal_atomic_int64_t *addr, int64_t newval) { int64_t ret; int tmp; @@ -229,7 +229,7 @@ static inline int64_t opal_atomic_swap_64 (volatile int64_t *addr, int64_t newva atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { int64_t prev; int tmp; @@ -251,7 +251,7 @@ static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t } -static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { int64_t prev; int tmp; @@ -274,7 +274,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t #define opal_atomic_ll_64(addr, ret) \ do { \ - volatile int64_t *_addr = (addr); \ + opal_atomic_int64_t *_addr = (addr); \ int64_t _ret; \ \ __asm__ __volatile__ ("ldaxr %0, [%1] \n" \ @@ -286,7 +286,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t #define opal_atomic_sc_64(addr, newval, ret) \ do { \ - volatile int64_t *_addr = (addr); \ + opal_atomic_int64_t *_addr = (addr); \ int64_t _newval = (int64_t) newval; \ int _ret; \ \ @@ -299,7 +299,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t } while (0) #define OPAL_ASM_MAKE_ATOMIC(type, bits, name, inst, reg) \ - static inline type opal_atomic_fetch_ ## name ## _ ## bits (volatile type *addr, type value) \ + static inline type opal_atomic_fetch_ ## name ## _ ## bits (opal_atomic_ ## type *addr, type value) \ { \ type newval, old; \ int32_t tmp; \ diff --git a/opal/include/opal/sys/atomic.h b/opal/include/opal/sys/atomic.h index 3b165f00e0..49cb18ac17 100644 --- a/opal/include/opal/sys/atomic.h +++ b/opal/include/opal/sys/atomic.h @@ -56,7 +56,7 @@ #include #include "opal/sys/architecture.h" -#include "opal_stdint.h" +#include "opal_stdatomic.h" /* do some quick #define cleanup in cases where we are doing testing... */ @@ -92,7 +92,7 @@ BEGIN_C_DECLS */ struct opal_atomic_lock_t { union { - volatile int32_t lock; /**< The lock address (an integer) */ + opal_atomic_int32_t lock; /**< The lock address (an integer) */ volatile unsigned char sparc_lock; /**< The lock address on sparc */ char padding[sizeof(int)]; /**< Array for optional padding */ } u; @@ -350,19 +350,19 @@ void opal_atomic_unlock(opal_atomic_lock_t *lock); #if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32 static inline #endif -bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, +bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval); #if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32 static inline #endif -bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, +bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval); #if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_32 static inline #endif -bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, +bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval); #endif @@ -375,19 +375,19 @@ bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t #if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64 static inline #endif -bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, +bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval); #if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64 static inline #endif -bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, +bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval); #if OPAL_HAVE_INLINE_ATOMIC_COMPARE_EXCHANGE_64 static inline #endif -bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, +bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval); #endif @@ -399,20 +399,20 @@ bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t #if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 -static inline int32_t opal_atomic_add_fetch_32(volatile int32_t *addr, int delta); -static inline int32_t opal_atomic_fetch_add_32(volatile int32_t *addr, int delta); -static inline int32_t opal_atomic_and_fetch_32(volatile int32_t *addr, int32_t value); -static inline int32_t opal_atomic_fetch_and_32(volatile int32_t *addr, int32_t value); -static inline int32_t opal_atomic_or_fetch_32(volatile int32_t *addr, int32_t value); -static inline int32_t opal_atomic_fetch_or_32(volatile int32_t *addr, int32_t value); -static inline int32_t opal_atomic_xor_fetch_32(volatile int32_t *addr, int32_t value); -static inline int32_t opal_atomic_fetch_xor_32(volatile int32_t *addr, int32_t value); -static inline int32_t opal_atomic_sub_fetch_32(volatile int32_t *addr, int delta); -static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t *addr, int delta); -static inline int32_t opal_atomic_min_fetch_32 (volatile int32_t *addr, int32_t value); -static inline int32_t opal_atomic_fetch_min_32 (volatile int32_t *addr, int32_t value); -static inline int32_t opal_atomic_max_fetch_32 (volatile int32_t *addr, int32_t value); -static inline int32_t opal_atomic_fetch_max_32 (volatile int32_t *addr, int32_t value); +static inline int32_t opal_atomic_add_fetch_32(opal_atomic_int32_t *addr, int delta); +static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t *addr, int delta); +static inline int32_t opal_atomic_and_fetch_32(opal_atomic_int32_t *addr, int32_t value); +static inline int32_t opal_atomic_fetch_and_32(opal_atomic_int32_t *addr, int32_t value); +static inline int32_t opal_atomic_or_fetch_32(opal_atomic_int32_t *addr, int32_t value); +static inline int32_t opal_atomic_fetch_or_32(opal_atomic_int32_t *addr, int32_t value); +static inline int32_t opal_atomic_xor_fetch_32(opal_atomic_int32_t *addr, int32_t value); +static inline int32_t opal_atomic_fetch_xor_32(opal_atomic_int32_t *addr, int32_t value); +static inline int32_t opal_atomic_sub_fetch_32(opal_atomic_int32_t *addr, int delta); +static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t *addr, int delta); +static inline int32_t opal_atomic_min_fetch_32 (opal_atomic_int32_t *addr, int32_t value); +static inline int32_t opal_atomic_fetch_min_32 (opal_atomic_int32_t *addr, int32_t value); +static inline int32_t opal_atomic_max_fetch_32 (opal_atomic_int32_t *addr, int32_t value); +static inline int32_t opal_atomic_fetch_max_32 (opal_atomic_int32_t *addr, int32_t value); #endif /* OPAL_HAVE_ATOMIC_MATH_32 */ @@ -429,19 +429,19 @@ static inline int32_t opal_atomic_fetch_max_32 (volatile int32_t *addr, int32_t #if defined(DOXYGEN) || OPAL_HAVE_ATOMIC_MATH_64 || OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 -static inline int64_t opal_atomic_add_fetch_64(volatile int64_t *addr, int64_t delta); -static inline int64_t opal_atomic_fetch_add_64(volatile int64_t *addr, int64_t delta); -static inline int64_t opal_atomic_and_fetch_64(volatile int64_t *addr, int64_t value); -static inline int64_t opal_atomic_fetch_and_64(volatile int64_t *addr, int64_t value); -static inline int64_t opal_atomic_or_fetch_64(volatile int64_t *addr, int64_t value); -static inline int64_t opal_atomic_fetch_or_64(volatile int64_t *addr, int64_t value); -static inline int64_t opal_atomic_fetch_xor_64(volatile int64_t *addr, int64_t value); -static inline int64_t opal_atomic_sub_fetch_64(volatile int64_t *addr, int64_t delta); -static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t *addr, int64_t delta); -static inline int64_t opal_atomic_min_fetch_64 (volatile int64_t *addr, int64_t value); -static inline int64_t opal_atomic_fetch_min_64 (volatile int64_t *addr, int64_t value); -static inline int64_t opal_atomic_max_fetch_64 (volatile int64_t *addr, int64_t value); -static inline int64_t opal_atomic_fetch_max_64 (volatile int64_t *addr, int64_t value); +static inline int64_t opal_atomic_add_fetch_64(opal_atomic_int64_t *addr, int64_t delta); +static inline int64_t opal_atomic_fetch_add_64(opal_atomic_int64_t *addr, int64_t delta); +static inline int64_t opal_atomic_and_fetch_64(opal_atomic_int64_t *addr, int64_t value); +static inline int64_t opal_atomic_fetch_and_64(opal_atomic_int64_t *addr, int64_t value); +static inline int64_t opal_atomic_or_fetch_64(opal_atomic_int64_t *addr, int64_t value); +static inline int64_t opal_atomic_fetch_or_64(opal_atomic_int64_t *addr, int64_t value); +static inline int64_t opal_atomic_fetch_xor_64(opal_atomic_int64_t *addr, int64_t value); +static inline int64_t opal_atomic_sub_fetch_64(opal_atomic_int64_t *addr, int64_t delta); +static inline int64_t opal_atomic_fetch_sub_64(opal_atomic_int64_t *addr, int64_t delta); +static inline int64_t opal_atomic_min_fetch_64 (opal_atomic_int64_t *addr, int64_t value); +static inline int64_t opal_atomic_fetch_min_64 (opal_atomic_int64_t *addr, int64_t value); +static inline int64_t opal_atomic_max_fetch_64 (opal_atomic_int64_t *addr, int64_t value); +static inline int64_t opal_atomic_fetch_max_64 (opal_atomic_int64_t *addr, int64_t value); #endif /* OPAL_HAVE_ATOMIC_MATH_64 */ @@ -458,7 +458,7 @@ static inline int64_t opal_atomic_fetch_max_64 (volatile int64_t *addr, int64_t */ #if defined(DOXYGEN) || OPAL_ENABLE_DEBUG static inline size_t -opal_atomic_add_fetch_size_t(volatile size_t *addr, size_t delta) +opal_atomic_add_fetch_size_t(opal_atomic_size_t *addr, size_t delta) { #if SIZEOF_SIZE_T == 4 return (size_t) opal_atomic_add_fetch_32((int32_t*) addr, delta); @@ -470,7 +470,7 @@ opal_atomic_add_fetch_size_t(volatile size_t *addr, size_t delta) } static inline size_t -opal_atomic_fetch_add_size_t(volatile size_t *addr, size_t delta) +opal_atomic_fetch_add_size_t(opal_atomic_size_t *addr, size_t delta) { #if SIZEOF_SIZE_T == 4 return (size_t) opal_atomic_fetch_add_32((int32_t*) addr, delta); @@ -482,7 +482,7 @@ opal_atomic_fetch_add_size_t(volatile size_t *addr, size_t delta) } static inline size_t -opal_atomic_sub_fetch_size_t(volatile size_t *addr, size_t delta) +opal_atomic_sub_fetch_size_t(opal_atomic_size_t *addr, size_t delta) { #if SIZEOF_SIZE_T == 4 return (size_t) opal_atomic_sub_fetch_32((int32_t*) addr, delta); @@ -494,7 +494,7 @@ opal_atomic_sub_fetch_size_t(volatile size_t *addr, size_t delta) } static inline size_t -opal_atomic_fetch_sub_size_t(volatile size_t *addr, size_t delta) +opal_atomic_fetch_sub_size_t(opal_atomic_size_t *addr, size_t delta) { #if SIZEOF_SIZE_T == 4 return (size_t) opal_atomic_fetch_sub_32((int32_t*) addr, delta); @@ -507,15 +507,15 @@ opal_atomic_fetch_sub_size_t(volatile size_t *addr, size_t delta) #else #if SIZEOF_SIZE_T == 4 -#define opal_atomic_add_fetch_size_t(addr, delta) ((size_t) opal_atomic_add_fetch_32((volatile int32_t *) addr, delta)) -#define opal_atomic_fetch_add_size_t(addr, delta) ((size_t) opal_atomic_fetch_add_32((volatile int32_t *) addr, delta)) -#define opal_atomic_sub_fetch_size_t(addr, delta) ((size_t) opal_atomic_sub_fetch_32((volatile int32_t *) addr, delta)) -#define opal_atomic_fetch_sub_size_t(addr, delta) ((size_t) opal_atomic_fetch_sub_32((volatile int32_t *) addr, delta)) +#define opal_atomic_add_fetch_size_t(addr, delta) ((size_t) opal_atomic_add_fetch_32((opal_atomic_int32_t *) addr, delta)) +#define opal_atomic_fetch_add_size_t(addr, delta) ((size_t) opal_atomic_fetch_add_32((opal_atomic_int32_t *) addr, delta)) +#define opal_atomic_sub_fetch_size_t(addr, delta) ((size_t) opal_atomic_sub_fetch_32((opal_atomic_int32_t *) addr, delta)) +#define opal_atomic_fetch_sub_size_t(addr, delta) ((size_t) opal_atomic_fetch_sub_32((opal_atomic_int32_t *) addr, delta)) #elif SIZEOF_SIZE_T == 8 -#define opal_atomic_add_fetch_size_t(addr, delta) ((size_t) opal_atomic_add_fetch_64((volatile int64_t *) addr, delta)) -#define opal_atomic_fetch_add_size_t(addr, delta) ((size_t) opal_atomic_fetch_add_64((volatile int64_t *) addr, delta)) -#define opal_atomic_sub_fetch_size_t(addr, delta) ((size_t) opal_atomic_sub_fetch_64((volatile int64_t *) addr, delta)) -#define opal_atomic_fetch_sub_size_t(addr, delta) ((size_t) opal_atomic_fetch_sub_64((volatile int64_t *) addr, delta)) +#define opal_atomic_add_fetch_size_t(addr, delta) ((size_t) opal_atomic_add_fetch_64((opal_atomic_int64_t *) addr, delta)) +#define opal_atomic_fetch_add_size_t(addr, delta) ((size_t) opal_atomic_fetch_add_64((opal_atomic_int64_t *) addr, delta)) +#define opal_atomic_sub_fetch_size_t(addr, delta) ((size_t) opal_atomic_sub_fetch_64((opal_atomic_int64_t *) addr, delta)) +#define opal_atomic_fetch_sub_size_t(addr, delta) ((size_t) opal_atomic_fetch_sub_64((opal_atomic_int64_t *) addr, delta)) #else #error "Unknown size_t size" #endif @@ -525,20 +525,20 @@ opal_atomic_fetch_sub_size_t(volatile size_t *addr, size_t delta) /* these are always done with inline functions, so always mark as static inline */ -static inline bool opal_atomic_compare_exchange_strong_xx (volatile void *addr, void *oldval, +static inline bool opal_atomic_compare_exchange_strong_xx (opal_atomic_intptr_t *addr, intptr_t *oldval, int64_t newval, size_t length); -static inline bool opal_atomic_compare_exchange_strong_acq_xx (volatile void *addr, void *oldval, +static inline bool opal_atomic_compare_exchange_strong_acq_xx (opal_atomic_intptr_t *addr, intptr_t *oldval, int64_t newval, size_t length); -static inline bool opal_atomic_compare_exchange_strong_rel_xx (volatile void *addr, void *oldval, +static inline bool opal_atomic_compare_exchange_strong_rel_xx (opal_atomic_intptr_t *addr, intptr_t *oldval, int64_t newval, size_t length); -static inline bool opal_atomic_compare_exchange_strong_ptr (volatile void* addr, void *oldval, - void *newval); -static inline bool opal_atomic_compare_exchange_strong_acq_ptr (volatile void* addr, void *oldval, - void *newval); -static inline bool opal_atomic_compare_exchange_strong_rel_ptr (volatile void* addr, void *oldval, - void *newval); +static inline bool opal_atomic_compare_exchange_strong_ptr (opal_atomic_intptr_t* addr, intptr_t *oldval, + intptr_t newval); +static inline bool opal_atomic_compare_exchange_strong_acq_ptr (opal_atomic_intptr_t* addr, intptr_t *oldval, + intptr_t newval); +static inline bool opal_atomic_compare_exchange_strong_rel_ptr (opal_atomic_intptr_t* addr, intptr_t *oldval, + intptr_t newval); /** * Atomic compare and set of generic type with relaxed semantics. This @@ -554,7 +554,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_ptr (volatile void* a * See opal_atomic_compare_exchange_* for pseudo-code. */ #define opal_atomic_compare_exchange_strong( ADDR, OLDVAL, NEWVAL ) \ - opal_atomic_compare_exchange_strong_xx( (volatile void*)(ADDR), (void *)(OLDVAL), \ + opal_atomic_compare_exchange_strong_xx( (opal_atomic_intptr_t*)(ADDR), (intptr_t *)(OLDVAL), \ (intptr_t)(NEWVAL), sizeof(*(ADDR)) ) /** @@ -571,7 +571,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_ptr (volatile void* a * See opal_atomic_compare_exchange_acq_* for pseudo-code. */ #define opal_atomic_compare_exchange_strong_acq( ADDR, OLDVAL, NEWVAL ) \ - opal_atomic_compare_exchange_strong_acq_xx( (volatile void*)(ADDR), (void *)(OLDVAL), \ + opal_atomic_compare_exchange_strong_acq_xx( (opal_atomic_intptr_t*)(ADDR), (intptr_t *)(OLDVAL), \ (intptr_t)(NEWVAL), sizeof(*(ADDR)) ) /** @@ -588,7 +588,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_ptr (volatile void* a * See opal_atomic_compare_exchange_rel_* for pseudo-code. */ #define opal_atomic_compare_exchange_strong_rel( ADDR, OLDVAL, NEWVAL ) \ - opal_atomic_compare_exchange_strong_rel_xx( (volatile void*)(ADDR), (void *)(OLDVAL), \ + opal_atomic_compare_exchange_strong_rel_xx( (opal_atomic_intptr_t*)(ADDR), (intptr_t *)(OLDVAL), \ (intptr_t)(NEWVAL), sizeof(*(ADDR)) ) @@ -596,15 +596,15 @@ static inline bool opal_atomic_compare_exchange_strong_rel_ptr (volatile void* a #if defined(DOXYGEN) || (OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64) -static inline void opal_atomic_add_xx(volatile void* addr, +static inline void opal_atomic_add_xx(opal_atomic_intptr_t* addr, int32_t value, size_t length); -static inline void opal_atomic_sub_xx(volatile void* addr, +static inline void opal_atomic_sub_xx(opal_atomic_intptr_t* addr, int32_t value, size_t length); -static inline intptr_t opal_atomic_add_fetch_ptr( volatile void* addr, void* delta ); -static inline intptr_t opal_atomic_fetch_add_ptr( volatile void* addr, void* delta ); -static inline intptr_t opal_atomic_sub_fetch_ptr( volatile void* addr, void* delta ); -static inline intptr_t opal_atomic_fetch_sub_ptr( volatile void* addr, void* delta ); +static inline intptr_t opal_atomic_add_fetch_ptr( opal_atomic_intptr_t* addr, void* delta ); +static inline intptr_t opal_atomic_fetch_add_ptr( opal_atomic_intptr_t* addr, void* delta ); +static inline intptr_t opal_atomic_sub_fetch_ptr( opal_atomic_intptr_t* addr, void* delta ); +static inline intptr_t opal_atomic_fetch_sub_ptr( opal_atomic_intptr_t* addr, void* delta ); /** * Atomically increment the content depending on the type. This @@ -617,7 +617,7 @@ static inline intptr_t opal_atomic_fetch_sub_ptr( volatile void* addr, void* del * @param delta Value to add (converted to ). */ #define opal_atomic_add( ADDR, VALUE ) \ - opal_atomic_add_xx( (volatile void*)(ADDR), (int32_t)(VALUE), \ + opal_atomic_add_xx( (opal_atomic_intptr_t*)(ADDR), (int32_t)(VALUE), \ sizeof(*(ADDR)) ) /** @@ -631,7 +631,7 @@ static inline intptr_t opal_atomic_fetch_sub_ptr( volatile void* addr, void* del * @param delta Value to substract (converted to ). */ #define opal_atomic_sub( ADDR, VALUE ) \ - opal_atomic_sub_xx( (volatile void*)(ADDR), (int32_t)(VALUE), \ + opal_atomic_sub_xx( (opal_atomic_intptr_t*)(ADDR), (int32_t)(VALUE), \ sizeof(*(ADDR)) ) #endif /* OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64 */ diff --git a/opal/include/opal/sys/atomic_impl.h b/opal/include/opal/sys/atomic_impl.h index 027b771162..051492296d 100644 --- a/opal/include/opal/sys/atomic_impl.h +++ b/opal/include/opal/sys/atomic_impl.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2010-2014 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -40,7 +40,7 @@ #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 #if !defined(OPAL_HAVE_ATOMIC_MIN_32) -static inline int32_t opal_atomic_fetch_min_32 (volatile int32_t *addr, int32_t value) +static inline int32_t opal_atomic_fetch_min_32 (opal_atomic_int32_t *addr, int32_t value) { int32_t old = *addr; do { @@ -57,7 +57,7 @@ static inline int32_t opal_atomic_fetch_min_32 (volatile int32_t *addr, int32_t #endif /* OPAL_HAVE_ATOMIC_MIN_32 */ #if !defined(OPAL_HAVE_ATOMIC_MAX_32) -static inline int32_t opal_atomic_fetch_max_32 (volatile int32_t *addr, int32_t value) +static inline int32_t opal_atomic_fetch_max_32 (opal_atomic_int32_t *addr, int32_t value) { int32_t old = *addr; do { @@ -73,7 +73,7 @@ static inline int32_t opal_atomic_fetch_max_32 (volatile int32_t *addr, int32_t #endif /* OPAL_HAVE_ATOMIC_MAX_32 */ #define OPAL_ATOMIC_DEFINE_CMPXCG_OP(type, bits, operation, name) \ - static inline type opal_atomic_fetch_ ## name ## _ ## bits (volatile type *addr, type value) \ + static inline type opal_atomic_fetch_ ## name ## _ ## bits (opal_atomic_ ## type *addr, type value) \ { \ type oldval; \ do { \ @@ -85,7 +85,7 @@ static inline int32_t opal_atomic_fetch_max_32 (volatile int32_t *addr, int32_t #if !defined(OPAL_HAVE_ATOMIC_SWAP_32) #define OPAL_HAVE_ATOMIC_SWAP_32 1 -static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, +static inline int32_t opal_atomic_swap_32(opal_atomic_int32_t *addr, int32_t newval) { int32_t old = *addr; @@ -138,7 +138,7 @@ OPAL_ATOMIC_DEFINE_CMPXCG_OP(int32_t, 32, -, sub) #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 #if !defined(OPAL_HAVE_ATOMIC_MIN_64) -static inline int64_t opal_atomic_fetch_min_64 (volatile int64_t *addr, int64_t value) +static inline int64_t opal_atomic_fetch_min_64 (opal_atomic_int64_t *addr, int64_t value) { int64_t old = *addr; do { @@ -155,7 +155,7 @@ static inline int64_t opal_atomic_fetch_min_64 (volatile int64_t *addr, int64_t #endif /* OPAL_HAVE_ATOMIC_MIN_64 */ #if !defined(OPAL_HAVE_ATOMIC_MAX_64) -static inline int64_t opal_atomic_fetch_max_64 (volatile int64_t *addr, int64_t value) +static inline int64_t opal_atomic_fetch_max_64 (opal_atomic_int64_t *addr, int64_t value) { int64_t old = *addr; do { @@ -172,7 +172,7 @@ static inline int64_t opal_atomic_fetch_max_64 (volatile int64_t *addr, int64_t #if !defined(OPAL_HAVE_ATOMIC_SWAP_64) #define OPAL_HAVE_ATOMIC_SWAP_64 1 -static inline int64_t opal_atomic_swap_64(volatile int64_t *addr, +static inline int64_t opal_atomic_swap_64(opal_atomic_int64_t *addr, int64_t newval) { int64_t old = *addr; @@ -235,15 +235,15 @@ OPAL_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, -, sub) #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 #define OPAL_ATOMIC_DEFINE_CMPXCG_XX(semantics) \ static inline bool \ - opal_atomic_compare_exchange_strong ## semantics ## xx (volatile void* addr, void *oldval, \ + opal_atomic_compare_exchange_strong ## semantics ## xx (opal_atomic_intptr_t* addr, intptr_t *oldval, \ int64_t newval, const size_t length) \ { \ switch (length) { \ case 4: \ - return opal_atomic_compare_exchange_strong_32 ((volatile int32_t *) addr, \ + return opal_atomic_compare_exchange_strong_32 ((opal_atomic_int32_t *) addr, \ (int32_t *) oldval, (int32_t) newval); \ case 8: \ - return opal_atomic_compare_exchange_strong_64 ((volatile int64_t *) addr, \ + return opal_atomic_compare_exchange_strong_64 ((opal_atomic_int64_t *) addr, \ (int64_t *) oldval, (int64_t) newval); \ } \ abort(); \ @@ -251,12 +251,12 @@ OPAL_ATOMIC_DEFINE_CMPXCG_OP(int64_t, 64, -, sub) #elif OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 #define OPAL_ATOMIC_DEFINE_CMPXCG_XX(semantics) \ static inline bool \ - opal_atomic_compare_exchange_strong ## semantics ## xx (volatile void* addr, void *oldval, \ + opal_atomic_compare_exchange_strong ## semantics ## xx (opal_atomic_intptr_t* addr, intptr_t *oldval, \ int64_t newval, const size_t length) \ { \ switch (length) { \ case 4: \ - return opal_atomic_compare_exchange_strong_32 ((volatile int32_t *) addr, \ + return opal_atomic_compare_exchange_strong_32 ((opal_atomic_int32_t *) addr, \ (int32_t *) oldval, (int32_t) newval); \ } \ abort(); \ @@ -272,16 +272,16 @@ OPAL_ATOMIC_DEFINE_CMPXCG_XX(_rel_) #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 #define OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(semantics) \ static inline bool \ - opal_atomic_compare_exchange_strong ## semantics ## ptr (volatile void* addr, void *oldval, void *newval) \ + opal_atomic_compare_exchange_strong ## semantics ## ptr (opal_atomic_intptr_t* addr, intptr_t *oldval, intptr_t newval) \ { \ - return opal_atomic_compare_exchange_strong_32 ((volatile int32_t *) addr, (int32_t *) oldval, (int32_t) newval); \ + return opal_atomic_compare_exchange_strong_32 ((opal_atomic_int32_t *) addr, (int32_t *) oldval, (int32_t) newval); \ } #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 #define OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(semantics) \ static inline bool \ - opal_atomic_compare_exchange_strong ## semantics ## ptr (volatile void* addr, void *oldval, void *newval) \ + opal_atomic_compare_exchange_strong ## semantics ## ptr (opal_atomic_intptr_t* addr, intptr_t *oldval, intptr_t newval) \ { \ - return opal_atomic_compare_exchange_strong_64 ((volatile int64_t *) addr, (int64_t *) oldval, (int64_t) newval); \ + return opal_atomic_compare_exchange_strong_64 ((opal_atomic_int64_t *) addr, (int64_t *) oldval, (int64_t) newval); \ } #else #error "Can not define opal_atomic_compare_exchange_strong_ptr with existing atomics" @@ -297,9 +297,9 @@ OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(_rel_) #if (OPAL_HAVE_ATOMIC_SWAP_32 || OPAL_HAVE_ATOMIC_SWAP_64) #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_SWAP_32 -#define opal_atomic_swap_ptr(addr, value) (void *) opal_atomic_swap_32((int32_t *) addr, (int32_t) value) +#define opal_atomic_swap_ptr(addr, value) (intptr_t) opal_atomic_swap_32((opal_atomic_int32_t *) addr, (int32_t) value) #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_SWAP_64 -#define opal_atomic_swap_ptr(addr, value) (void *) opal_atomic_swap_64((int64_t *) addr, (int64_t) value) +#define opal_atomic_swap_ptr(addr, value) (intptr_t) opal_atomic_swap_64((opal_atomic_int64_t *) addr, (int64_t) value) #endif #endif /* (OPAL_HAVE_ATOMIC_SWAP_32 || OPAL_HAVE_ATOMIC_SWAP_64) */ @@ -308,15 +308,15 @@ OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(_rel_) #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_LLSC_32 -#define opal_atomic_ll_ptr(addr, ret) opal_atomic_ll_32((volatile int32_t *) (addr), ret) -#define opal_atomic_sc_ptr(addr, value, ret) opal_atomic_sc_32((volatile int32_t *) (addr), (intptr_t) (value), ret) +#define opal_atomic_ll_ptr(addr, ret) opal_atomic_ll_32((opal_atomic_int32_t *) (addr), ret) +#define opal_atomic_sc_ptr(addr, value, ret) opal_atomic_sc_32((opal_atomic_int32_t *) (addr), (intptr_t) (value), ret) #define OPAL_HAVE_ATOMIC_LLSC_PTR 1 #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_LLSC_64 -#define opal_atomic_ll_ptr(addr, ret) opal_atomic_ll_64((volatile int64_t *) (addr), ret) -#define opal_atomic_sc_ptr(addr, value, ret) opal_atomic_sc_64((volatile int64_t *) (addr), (intptr_t) (value), ret) +#define opal_atomic_ll_ptr(addr, ret) opal_atomic_ll_64((opal_atomic_int64_t *) (addr), ret) +#define opal_atomic_sc_ptr(addr, value, ret) opal_atomic_sc_64((opal_atomic_int64_t *) (addr), (intptr_t) (value), ret) #define OPAL_HAVE_ATOMIC_LLSC_PTR 1 @@ -331,18 +331,18 @@ OPAL_ATOMIC_DEFINE_CMPXCG_PTR_XX(_rel_) #if OPAL_HAVE_ATOMIC_MATH_32 || OPAL_HAVE_ATOMIC_MATH_64 static inline void - opal_atomic_add_xx(volatile void* addr, int32_t value, size_t length) + opal_atomic_add_xx(opal_atomic_intptr_t* addr, int32_t value, size_t length) { switch( length ) { #if OPAL_HAVE_ATOMIC_ADD_32 case 4: - (void) opal_atomic_fetch_add_32( (volatile int32_t*)addr, (int32_t)value ); + (void) opal_atomic_fetch_add_32( (opal_atomic_int32_t*)addr, (int32_t)value ); break; #endif /* OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 */ #if OPAL_HAVE_ATOMIC_ADD_64 case 8: - (void) opal_atomic_fetch_add_64( (volatile int64_t*)addr, (int64_t)value ); + (void) opal_atomic_fetch_add_64( (opal_atomic_int64_t*)addr, (int64_t)value ); break; #endif /* OPAL_HAVE_ATOMIC_ADD_64 */ default: @@ -354,18 +354,18 @@ static inline void static inline void -opal_atomic_sub_xx(volatile void* addr, int32_t value, size_t length) +opal_atomic_sub_xx(opal_atomic_intptr_t* addr, int32_t value, size_t length) { switch( length ) { #if OPAL_HAVE_ATOMIC_SUB_32 case 4: - (void) opal_atomic_fetch_sub_32( (volatile int32_t*)addr, (int32_t)value ); + (void) opal_atomic_fetch_sub_32( (opal_atomic_int32_t*)addr, (int32_t)value ); break; #endif /* OPAL_HAVE_ATOMIC_SUB_32 */ #if OPAL_HAVE_ATOMIC_SUB_64 case 8: - (void) opal_atomic_fetch_sub_64( (volatile int64_t*)addr, (int64_t)value ); + (void) opal_atomic_fetch_sub_64( (opal_atomic_int64_t*)addr, (int64_t)value ); break; #endif /* OPAL_HAVE_ATOMIC_SUB_64 */ default: @@ -376,7 +376,7 @@ opal_atomic_sub_xx(volatile void* addr, int32_t value, size_t length) } #define OPAL_ATOMIC_DEFINE_OP_FETCH(op, operation, type, ptr_type, suffix) \ - static inline type opal_atomic_ ## op ## _fetch_ ## suffix (volatile ptr_type *addr, type value) \ + static inline type opal_atomic_ ## op ## _fetch_ ## suffix (opal_atomic_ ## ptr_type *addr, type value) \ { \ return opal_atomic_fetch_ ## op ## _ ## suffix (addr, value) operation value; \ } @@ -387,13 +387,13 @@ OPAL_ATOMIC_DEFINE_OP_FETCH(or, |, int32_t, int32_t, 32) OPAL_ATOMIC_DEFINE_OP_FETCH(xor, ^, int32_t, int32_t, 32) OPAL_ATOMIC_DEFINE_OP_FETCH(sub, -, int32_t, int32_t, 32) -static inline int32_t opal_atomic_min_fetch_32 (volatile int32_t *addr, int32_t value) +static inline int32_t opal_atomic_min_fetch_32 (opal_atomic_int32_t *addr, int32_t value) { int32_t old = opal_atomic_fetch_min_32 (addr, value); return old <= value ? old : value; } -static inline int32_t opal_atomic_max_fetch_32 (volatile int32_t *addr, int32_t value) +static inline int32_t opal_atomic_max_fetch_32 (opal_atomic_int32_t *addr, int32_t value) { int32_t old = opal_atomic_fetch_max_32 (addr, value); return old >= value ? old : value; @@ -406,13 +406,13 @@ OPAL_ATOMIC_DEFINE_OP_FETCH(or, |, int64_t, int64_t, 64) OPAL_ATOMIC_DEFINE_OP_FETCH(xor, ^, int64_t, int64_t, 64) OPAL_ATOMIC_DEFINE_OP_FETCH(sub, -, int64_t, int64_t, 64) -static inline int64_t opal_atomic_min_fetch_64 (volatile int64_t *addr, int64_t value) +static inline int64_t opal_atomic_min_fetch_64 (opal_atomic_int64_t *addr, int64_t value) { int64_t old = opal_atomic_fetch_min_64 (addr, value); return old <= value ? old : value; } -static inline int64_t opal_atomic_max_fetch_64 (volatile int64_t *addr, int64_t value) +static inline int64_t opal_atomic_max_fetch_64 (opal_atomic_int64_t *addr, int64_t value) { int64_t old = opal_atomic_fetch_max_64 (addr, value); return old >= value ? old : value; @@ -420,52 +420,52 @@ static inline int64_t opal_atomic_max_fetch_64 (volatile int64_t *addr, int64_t #endif -static inline intptr_t opal_atomic_fetch_add_ptr( volatile void* addr, +static inline intptr_t opal_atomic_fetch_add_ptr( opal_atomic_intptr_t* addr, void* delta ) { #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_ADD_32 - return opal_atomic_fetch_add_32((int32_t*) addr, (unsigned long) delta); + return opal_atomic_fetch_add_32((opal_atomic_int32_t*) addr, (unsigned long) delta); #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_ADD_64 - return opal_atomic_fetch_add_64((int64_t*) addr, (unsigned long) delta); + return opal_atomic_fetch_add_64((opal_atomic_int64_t*) addr, (unsigned long) delta); #else abort (); return 0; #endif } -static inline intptr_t opal_atomic_add_fetch_ptr( volatile void* addr, +static inline intptr_t opal_atomic_add_fetch_ptr( opal_atomic_intptr_t* addr, void* delta ) { #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_ADD_32 - return opal_atomic_add_fetch_32((int32_t*) addr, (unsigned long) delta); + return opal_atomic_add_fetch_32((opal_atomic_int32_t*) addr, (unsigned long) delta); #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_ADD_64 - return opal_atomic_add_fetch_64((int64_t*) addr, (unsigned long) delta); + return opal_atomic_add_fetch_64((opal_atomic_int64_t*) addr, (unsigned long) delta); #else abort (); return 0; #endif } -static inline intptr_t opal_atomic_fetch_sub_ptr( volatile void* addr, +static inline intptr_t opal_atomic_fetch_sub_ptr( opal_atomic_intptr_t* addr, void* delta ) { #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_SUB_32 - return opal_atomic_fetch_sub_32((int32_t*) addr, (unsigned long) delta); + return opal_atomic_fetch_sub_32((opal_atomic_int32_t*) addr, (unsigned long) delta); #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_SUB_32 - return opal_atomic_fetch_sub_64((int64_t*) addr, (unsigned long) delta); + return opal_atomic_fetch_sub_64((opal_atomic_int64_t*) addr, (unsigned long) delta); #else abort(); return 0; #endif } -static inline intptr_t opal_atomic_sub_fetch_ptr( volatile void* addr, +static inline intptr_t opal_atomic_sub_fetch_ptr( opal_atomic_intptr_t* addr, void* delta ) { #if SIZEOF_VOID_P == 4 && OPAL_HAVE_ATOMIC_SUB_32 - return opal_atomic_sub_fetch_32((int32_t*) addr, (unsigned long) delta); + return opal_atomic_sub_fetch_32((opal_atomic_int32_t*) addr, (unsigned long) delta); #elif SIZEOF_VOID_P == 8 && OPAL_HAVE_ATOMIC_SUB_32 - return opal_atomic_sub_fetch_64((int64_t*) addr, (unsigned long) delta); + return opal_atomic_sub_fetch_64((opal_atomic_int64_t*) addr, (unsigned long) delta); #else abort(); return 0; diff --git a/opal/include/opal/sys/gcc_builtin/atomic.h b/opal/include/opal/sys/gcc_builtin/atomic.h index c6ef6eb9c3..516f5fb10b 100644 --- a/opal/include/opal/sys/gcc_builtin/atomic.h +++ b/opal/include/opal/sys/gcc_builtin/atomic.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011 Sandia National Laboratories. All rights reserved. - * Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2016-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -81,98 +81,98 @@ static inline void opal_atomic_wmb(void) #pragma error_messages(off, E_ARG_INCOMPATIBLE_WITH_ARG_L) #endif -static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); } -static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED); } -static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); } -static inline int32_t opal_atomic_swap_32 (volatile int32_t *addr, int32_t newval) +static inline int32_t opal_atomic_swap_32 (opal_atomic_int32_t *addr, int32_t newval) { int32_t oldval; __atomic_exchange (addr, &newval, &oldval, __ATOMIC_RELAXED); return oldval; } -static inline int32_t opal_atomic_fetch_add_32(volatile int32_t *addr, int32_t delta) +static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t *addr, int32_t delta) { return __atomic_fetch_add (addr, delta, __ATOMIC_RELAXED); } -static inline int32_t opal_atomic_fetch_and_32(volatile int32_t *addr, int32_t value) +static inline int32_t opal_atomic_fetch_and_32(opal_atomic_int32_t *addr, int32_t value) { return __atomic_fetch_and (addr, value, __ATOMIC_RELAXED); } -static inline int32_t opal_atomic_fetch_or_32(volatile int32_t *addr, int32_t value) +static inline int32_t opal_atomic_fetch_or_32(opal_atomic_int32_t *addr, int32_t value) { return __atomic_fetch_or (addr, value, __ATOMIC_RELAXED); } -static inline int32_t opal_atomic_fetch_xor_32(volatile int32_t *addr, int32_t value) +static inline int32_t opal_atomic_fetch_xor_32(opal_atomic_int32_t *addr, int32_t value) { return __atomic_fetch_xor (addr, value, __ATOMIC_RELAXED); } -static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t *addr, int32_t delta) +static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t *addr, int32_t delta) { return __atomic_fetch_sub (addr, delta, __ATOMIC_RELAXED); } -static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); } -static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_RELEASE, __ATOMIC_RELAXED); } -static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { return __atomic_compare_exchange_n (addr, oldval, newval, false, __ATOMIC_ACQUIRE, __ATOMIC_RELAXED); } -static inline int64_t opal_atomic_swap_64 (volatile int64_t *addr, int64_t newval) +static inline int64_t opal_atomic_swap_64 (opal_atomic_int64_t *addr, int64_t newval) { int64_t oldval; __atomic_exchange (addr, &newval, &oldval, __ATOMIC_RELAXED); return oldval; } -static inline int64_t opal_atomic_fetch_add_64(volatile int64_t *addr, int64_t delta) +static inline int64_t opal_atomic_fetch_add_64(opal_atomic_int64_t *addr, int64_t delta) { return __atomic_fetch_add (addr, delta, __ATOMIC_RELAXED); } -static inline int64_t opal_atomic_fetch_and_64(volatile int64_t *addr, int64_t value) +static inline int64_t opal_atomic_fetch_and_64(opal_atomic_int64_t *addr, int64_t value) { return __atomic_fetch_and (addr, value, __ATOMIC_RELAXED); } -static inline int64_t opal_atomic_fetch_or_64(volatile int64_t *addr, int64_t value) +static inline int64_t opal_atomic_fetch_or_64(opal_atomic_int64_t *addr, int64_t value) { return __atomic_fetch_or (addr, value, __ATOMIC_RELAXED); } -static inline int64_t opal_atomic_fetch_xor_64(volatile int64_t *addr, int64_t value) +static inline int64_t opal_atomic_fetch_xor_64(opal_atomic_int64_t *addr, int64_t value) { return __atomic_fetch_xor (addr, value, __ATOMIC_RELAXED); } -static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t *addr, int64_t delta) +static inline int64_t opal_atomic_fetch_sub_64(opal_atomic_int64_t *addr, int64_t delta) { return __atomic_fetch_sub (addr, delta, __ATOMIC_RELAXED); } @@ -181,7 +181,7 @@ static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t *addr, int64_t d #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 1 -static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128_t *addr, +static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_int128_t *addr, opal_int128_t *oldval, opal_int128_t newval) { return __atomic_compare_exchange_n (addr, oldval, newval, false, @@ -194,7 +194,7 @@ static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128 /* __atomic version is not lock-free so use legacy __sync version */ -static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128_t *addr, +static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_opal_int128_t *addr, opal_int128_t *oldval, opal_int128_t newval) { opal_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval); diff --git a/opal/include/opal/sys/ia32/atomic.h b/opal/include/opal/sys/ia32/atomic.h index bb863dec14..eac18b4b00 100644 --- a/opal/include/opal/sys/ia32/atomic.h +++ b/opal/include/opal/sys/ia32/atomic.h @@ -13,7 +13,7 @@ * Copyright (c) 2007-2010 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -84,7 +84,7 @@ static inline void opal_atomic_isync(void) *********************************************************************/ #if OPAL_GCC_INLINE_ASSEMBLY -static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { unsigned char ret; __asm__ __volatile__ ( @@ -106,7 +106,7 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add #define OPAL_HAVE_ATOMIC_SWAP_32 1 -static inline int32_t opal_atomic_swap_32( volatile int32_t *addr, +static inline int32_t opal_atomic_swap_32( opal_atomic_int32_t *addr, int32_t newval) { int32_t oldval; @@ -130,7 +130,7 @@ static inline int32_t opal_atomic_swap_32( volatile int32_t *addr, * * Atomically adds @i to @v. */ -static inline int32_t opal_atomic_fetch_add_32(volatile int32_t* v, int i) +static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t* v, int i) { int ret = i; __asm__ __volatile__( @@ -150,7 +150,7 @@ static inline int32_t opal_atomic_fetch_add_32(volatile int32_t* v, int i) * * Atomically subtracts @i from @v. */ -static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t* v, int i) +static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t* v, int i) { int ret = -i; __asm__ __volatile__( diff --git a/opal/include/opal/sys/powerpc/atomic.h b/opal/include/opal/sys/powerpc/atomic.h index da7414e5a0..90312d7543 100644 --- a/opal/include/opal/sys/powerpc/atomic.h +++ b/opal/include/opal/sys/powerpc/atomic.h @@ -144,7 +144,7 @@ void opal_atomic_isync(void) #define OPAL_ASM_VALUE64(x) x #endif -static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { int32_t prev; bool ret; @@ -170,7 +170,7 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add * load the arguments to/from the stack. This sequence may cause the ll reservation to be cancelled. */ #define opal_atomic_ll_32(addr, ret) \ do { \ - volatile int32_t *_addr = (addr); \ + opal_atomic_int32_t *_addr = (addr); \ int32_t _ret; \ __asm__ __volatile__ ("lwarx %0, 0, %1 \n\t" \ : "=&r" (_ret) \ @@ -181,7 +181,7 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add #define opal_atomic_sc_32(addr, value, ret) \ do { \ - volatile int32_t *_addr = (addr); \ + opal_atomic_int32_t *_addr = (addr); \ int32_t _ret, _foo, _newval = (int32_t) value; \ \ __asm__ __volatile__ (" stwcx. %4, 0, %3 \n\t" \ @@ -200,7 +200,7 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { bool rc; @@ -211,13 +211,13 @@ static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t } -static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { opal_atomic_wmb(); return opal_atomic_compare_exchange_strong_32 (addr, oldval, newval); } -static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, int32_t newval) +static inline int32_t opal_atomic_swap_32(opal_atomic_int32_t *addr, int32_t newval) { int32_t ret; @@ -239,7 +239,7 @@ static inline int32_t opal_atomic_swap_32(volatile int32_t *addr, int32_t newval #if OPAL_GCC_INLINE_ASSEMBLY #define OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(type, instr) \ -static inline int64_t opal_atomic_fetch_ ## type ## _64(volatile int64_t* v, int64_t val) \ +static inline int64_t opal_atomic_fetch_ ## type ## _64(opal_atomic_int64_t* v, int64_t val) \ { \ int64_t t, old; \ \ @@ -261,7 +261,7 @@ OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(or, or) OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(xor, xor) OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_64(sub, subf) -static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { int64_t prev; bool ret; @@ -284,7 +284,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add #define opal_atomic_ll_64(addr, ret) \ do { \ - volatile int64_t *_addr = (addr); \ + opal_atomic_int64_t *_addr = (addr); \ int64_t _ret; \ __asm__ __volatile__ ("ldarx %0, 0, %1 \n\t" \ : "=&r" (_ret) \ @@ -295,7 +295,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add #define opal_atomic_sc_64(addr, value, ret) \ do { \ - volatile int64_t *_addr = (addr); \ + opal_atomic_int64_t *_addr = (addr); \ int64_t _foo, _newval = (int64_t) value; \ int32_t _ret; \ \ @@ -310,7 +310,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add ret = _ret; \ } while (0) -static inline int64_t opal_atomic_swap_64(volatile int64_t *addr, int64_t newval) +static inline int64_t opal_atomic_swap_64(opal_atomic_int64_t *addr, int64_t newval) { int64_t ret; @@ -335,7 +335,7 @@ static inline int64_t opal_atomic_swap_64(volatile int64_t *addr, int64_t newval #if OPAL_GCC_INLINE_ASSEMBLY -static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { int64_t prev; int ret; @@ -382,7 +382,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add atomic_?mb can be inlined). Instead, we "inline" them by hand in the assembly, meaning there is one function call overhead instead of two */ -static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { bool rc; @@ -393,7 +393,7 @@ static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t } -static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { opal_atomic_wmb(); return opal_atomic_compare_exchange_strong_64 (addr, oldval, newval); @@ -401,7 +401,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t #define OPAL_ATOMIC_POWERPC_DEFINE_ATOMIC_32(type, instr) \ -static inline int32_t opal_atomic_fetch_ ## type ## _32(volatile int32_t* v, int val) \ +static inline int32_t opal_atomic_fetch_ ## type ## _32(opal_atomic_int32_t* v, int val) \ { \ int32_t t, old; \ \ diff --git a/opal/include/opal/sys/sparcv9/atomic.h b/opal/include/opal/sys/sparcv9/atomic.h index c79e32b1eb..e91d03cd92 100644 --- a/opal/include/opal/sys/sparcv9/atomic.h +++ b/opal/include/opal/sys/sparcv9/atomic.h @@ -13,7 +13,7 @@ * Copyright (c) 2007 Sun Microsystems, Inc. All rights reserverd. * Copyright (c) 2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2017-2018 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -85,7 +85,7 @@ static inline void opal_atomic_isync(void) *********************************************************************/ #if OPAL_GCC_INLINE_ASSEMBLY -static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { /* casa [reg(rs1)] %asi, reg(rs2), reg(rd) * @@ -107,7 +107,7 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add } -static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_acq_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { bool rc; @@ -118,7 +118,7 @@ static inline bool opal_atomic_compare_exchange_strong_acq_32 (volatile int32_t } -static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_rel_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { opal_atomic_wmb(); return opal_atomic_compare_exchange_strong_32 (addr, oldval, newval); @@ -127,7 +127,7 @@ static inline bool opal_atomic_compare_exchange_strong_rel_32 (volatile int32_t #if OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64 -static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { /* casa [reg(rs1)] %asi, reg(rs2), reg(rd) * @@ -149,7 +149,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add #else /* OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64 */ -static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { /* casa [reg(rs1)] %asi, reg(rs2), reg(rd) * @@ -179,7 +179,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add #endif /* OPAL_ASSEMBLY_ARCH == OPAL_SPARCV9_64 */ -static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_acq_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { bool rc; @@ -190,7 +190,7 @@ static inline bool opal_atomic_compare_exchange_strong_acq_64 (volatile int64_t } -static inline bool opal_atomic_compare_exchange_strong_rel_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_rel_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { opal_atomic_wmb(); return opal_atomic_compare_exchange_strong_64 (addr, oldval, newval); diff --git a/opal/include/opal/sys/sync_builtin/atomic.h b/opal/include/opal/sys/sync_builtin/atomic.h index 11decc2f0a..4d5e3f5227 100644 --- a/opal/include/opal/sys/sync_builtin/atomic.h +++ b/opal/include/opal/sys/sync_builtin/atomic.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2011 Sandia National Laboratories. All rights reserved. - * Copyright (c) 2014-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2014-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -57,7 +57,7 @@ static inline void opal_atomic_wmb(void) #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_32 1 -static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { int32_t prev = __sync_val_compare_and_swap (addr, *oldval, newval); bool ret = prev == *oldval; @@ -71,31 +71,31 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add #define OPAL_HAVE_ATOMIC_MATH_32 1 #define OPAL_HAVE_ATOMIC_ADD_32 1 -static inline int32_t opal_atomic_fetch_add_32(volatile int32_t *addr, int32_t delta) +static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t *addr, int32_t delta) { return __sync_fetch_and_add(addr, delta); } #define OPAL_HAVE_ATOMIC_AND_32 1 -static inline int32_t opal_atomic_fetch_and_32(volatile int32_t *addr, int32_t value) +static inline int32_t opal_atomic_fetch_and_32(opal_atomic_int32_t *addr, int32_t value) { return __sync_fetch_and_and(addr, value); } #define OPAL_HAVE_ATOMIC_OR_32 1 -static inline int32_t opal_atomic_fetch_or_32(volatile int32_t *addr, int32_t value) +static inline int32_t opal_atomic_fetch_or_32(opal_atomic_int32_t *addr, int32_t value) { return __sync_fetch_and_or(addr, value); } #define OPAL_HAVE_ATOMIC_XOR_32 1 -static inline int32_t opal_atomic_fetch_xor_32(volatile int32_t *addr, int32_t value) +static inline int32_t opal_atomic_fetch_xor_32(opal_atomic_int32_t *addr, int32_t value) { return __sync_fetch_and_xor(addr, value); } #define OPAL_HAVE_ATOMIC_SUB_32 1 -static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t *addr, int32_t delta) +static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t *addr, int32_t delta) { return __sync_fetch_and_sub(addr, delta); } @@ -104,7 +104,7 @@ static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t *addr, int32_t d #define OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_64 1 -static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { int64_t prev = __sync_val_compare_and_swap (addr, *oldval, newval); bool ret = prev == *oldval; @@ -117,31 +117,31 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add #define OPAL_HAVE_ATOMIC_MATH_64 1 #define OPAL_HAVE_ATOMIC_ADD_64 1 -static inline int64_t opal_atomic_fetch_add_64(volatile int64_t *addr, int64_t delta) +static inline int64_t opal_atomic_fetch_add_64(opal_atomic_int64_t *addr, int64_t delta) { return __sync_fetch_and_add(addr, delta); } #define OPAL_HAVE_ATOMIC_AND_64 1 -static inline int64_t opal_atomic_fetch_and_64(volatile int64_t *addr, int64_t value) +static inline int64_t opal_atomic_fetch_and_64(opal_atomic_int64_t *addr, int64_t value) { return __sync_fetch_and_and(addr, value); } #define OPAL_HAVE_ATOMIC_OR_64 1 -static inline int64_t opal_atomic_fetch_or_64(volatile int64_t *addr, int64_t value) +static inline int64_t opal_atomic_fetch_or_64(opal_atomic_int64_t *addr, int64_t value) { return __sync_fetch_and_or(addr, value); } #define OPAL_HAVE_ATOMIC_XOR_64 1 -static inline int64_t opal_atomic_fetch_xor_64(volatile int64_t *addr, int64_t value) +static inline int64_t opal_atomic_fetch_xor_64(opal_atomic_int64_t *addr, int64_t value) { return __sync_fetch_and_xor(addr, value); } #define OPAL_HAVE_ATOMIC_SUB_64 1 -static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t *addr, int64_t delta) +static inline int64_t opal_atomic_fetch_sub_64(opal_atomic_int64_t *addr, int64_t delta) { return __sync_fetch_and_sub(addr, delta); } @@ -149,7 +149,7 @@ static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t *addr, int64_t d #endif #if OPAL_HAVE_SYNC_BUILTIN_CSWAP_INT128 -static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128_t *addr, +static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_opal_int128_t *addr, opal_int128_t *oldval, opal_int128_t newval) { opal_int128_t prev = __sync_val_compare_and_swap (addr, *oldval, newval); diff --git a/opal/include/opal/sys/x86_64/atomic.h b/opal/include/opal/sys/x86_64/atomic.h index 49d740de38..8853eb5c2b 100644 --- a/opal/include/opal/sys/x86_64/atomic.h +++ b/opal/include/opal/sys/x86_64/atomic.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007 Sun Microsystems, Inc. All rights reserverd. - * Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2012-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2016-2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -82,7 +82,7 @@ static inline void opal_atomic_isync(void) *********************************************************************/ #if OPAL_GCC_INLINE_ASSEMBLY -static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *addr, int32_t *oldval, int32_t newval) +static inline bool opal_atomic_compare_exchange_strong_32 (opal_atomic_int32_t *addr, int32_t *oldval, int32_t newval) { unsigned char ret; __asm__ __volatile__ ( @@ -102,13 +102,13 @@ static inline bool opal_atomic_compare_exchange_strong_32 (volatile int32_t *add #if OPAL_GCC_INLINE_ASSEMBLY -static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *addr, int64_t *oldval, int64_t newval) +static inline bool opal_atomic_compare_exchange_strong_64 (opal_atomic_int64_t *addr, int64_t *oldval, int64_t newval) { unsigned char ret; __asm__ __volatile__ ( SMPLOCK "cmpxchgq %3,%2 \n\t" "sete %0 \n\t" - : "=qm" (ret), "+a" (*oldval), "+m" (*((volatile long*)addr)) + : "=qm" (ret), "+a" (*oldval), "+m" (*((opal_atomic_long_t *)addr)) : "q"(newval) : "memory", "cc" ); @@ -123,7 +123,7 @@ static inline bool opal_atomic_compare_exchange_strong_64 (volatile int64_t *add #if OPAL_GCC_INLINE_ASSEMBLY && OPAL_HAVE_CMPXCHG16B && HAVE_OPAL_INT128_T -static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128_t *addr, opal_int128_t *oldval, opal_int128_t newval) +static inline bool opal_atomic_compare_exchange_strong_128 (opal_atomic_int128_t *addr, opal_int128_t *oldval, opal_int128_t newval) { unsigned char ret; @@ -150,7 +150,7 @@ static inline bool opal_atomic_compare_exchange_strong_128 (volatile opal_int128 #define OPAL_HAVE_ATOMIC_SWAP_64 1 -static inline int32_t opal_atomic_swap_32( volatile int32_t *addr, +static inline int32_t opal_atomic_swap_32( opal_atomic_int32_t *addr, int32_t newval) { int32_t oldval; @@ -166,7 +166,7 @@ static inline int32_t opal_atomic_swap_32( volatile int32_t *addr, #if OPAL_GCC_INLINE_ASSEMBLY -static inline int64_t opal_atomic_swap_64( volatile int64_t *addr, +static inline int64_t opal_atomic_swap_64( opal_atomic_int64_t *addr, int64_t newval) { int64_t oldval; @@ -196,7 +196,7 @@ static inline int64_t opal_atomic_swap_64( volatile int64_t *addr, * * Atomically adds @i to @v. */ -static inline int32_t opal_atomic_fetch_add_32(volatile int32_t* v, int i) +static inline int32_t opal_atomic_fetch_add_32(opal_atomic_int32_t* v, int i) { int ret = i; __asm__ __volatile__( @@ -217,7 +217,7 @@ static inline int32_t opal_atomic_fetch_add_32(volatile int32_t* v, int i) * * Atomically adds @i to @v. */ -static inline int64_t opal_atomic_fetch_add_64(volatile int64_t* v, int64_t i) +static inline int64_t opal_atomic_fetch_add_64(opal_atomic_int64_t* v, int64_t i) { int64_t ret = i; __asm__ __volatile__( @@ -238,7 +238,7 @@ static inline int64_t opal_atomic_fetch_add_64(volatile int64_t* v, int64_t i) * * Atomically subtracts @i from @v. */ -static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t* v, int i) +static inline int32_t opal_atomic_fetch_sub_32(opal_atomic_int32_t* v, int i) { int ret = -i; __asm__ __volatile__( @@ -259,7 +259,7 @@ static inline int32_t opal_atomic_fetch_sub_32(volatile int32_t* v, int i) * * Atomically subtracts @i from @v. */ -static inline int64_t opal_atomic_fetch_sub_64(volatile int64_t* v, int64_t i) +static inline int64_t opal_atomic_fetch_sub_64(opal_atomic_int64_t* v, int64_t i) { int64_t ret = -i; __asm__ __volatile__( diff --git a/opal/include/opal_stdatomic.h b/opal/include/opal_stdatomic.h new file mode 100644 index 0000000000..6617e63de8 --- /dev/null +++ b/opal/include/opal_stdatomic.h @@ -0,0 +1,35 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2018 Los Alamos National Security, LLC. All rights + * reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#if !defined(OPAL_STDATOMIC_H) +#define OPAL_STDATOMIC_H + +#include "opal_stdint.h" + +typedef volatile long opal_atomic_long_t; + +typedef volatile int32_t opal_atomic_int32_t; +typedef volatile uint32_t opal_atomic_uint32_t; +typedef volatile int64_t opal_atomic_int64_t; +typedef volatile uint64_t opal_atomic_uint64_t; + +typedef volatile size_t opal_atomic_size_t; +typedef volatile intptr_t opal_atomic_intptr_t; +typedef volatile uintptr_t opal_atomic_uintptr_t; + +#if HAVE_OPAL_INT128_T + +typedef volatile opal_int128_t opal_atomic_int128_t; + +#endif + + +#endif /* !defined(OPAL_STDATOMIC_H) */ diff --git a/opal/mca/btl/ofi/btl_ofi.h b/opal/mca/btl/ofi/btl_ofi.h index a96bbf0b56..f8e44d93c7 100644 --- a/opal/mca/btl/ofi/btl_ofi.h +++ b/opal/mca/btl/ofi/btl_ofi.h @@ -123,8 +123,8 @@ struct mca_btl_ofi_module_t { bool use_virt_addr; bool is_scalable_ep; - int64_t outstanding_rdma; - int64_t outstanding_send; + opal_atomic_int64_t outstanding_rdma; + opal_atomic_int64_t outstanding_send; /** linked list of BTL endpoints. this list is never searched so * there is no need for a complicated structure here at this time*/ diff --git a/opal/mca/btl/openib/btl_openib.h b/opal/mca/btl/openib/btl_openib.h index 6b4dd0466b..bb657f0608 100644 --- a/opal/mca/btl/openib/btl_openib.h +++ b/opal/mca/btl/openib/btl_openib.h @@ -237,7 +237,7 @@ struct mca_btl_openib_component_t { int apm_lmc; int apm_ports; unsigned int buffer_alignment; /**< Preferred communication buffer alignment in Bytes (must be power of two) */ - int32_t error_counter; /**< Counts number on error events that we got on all devices */ + opal_atomic_int32_t error_counter; /**< Counts number on error events that we got on all devices */ opal_event_base_t *async_evbase; /**< Async event base */ bool use_async_event_thread; /**< Use the async event handler */ mca_btl_openib_srq_manager_t srq_manager; /**< Hash table for all BTL SRQs */ @@ -404,8 +404,8 @@ typedef struct mca_btl_openib_device_t { #endif int xrc_fd; #endif - int32_t non_eager_rdma_endpoints; - int32_t eager_rdma_buffers_count; + opal_atomic_int32_t non_eager_rdma_endpoints; + opal_atomic_int32_t eager_rdma_buffers_count; struct mca_btl_base_endpoint_t **eager_rdma_buffers; /**< frags for control massages */ opal_free_list_t send_free_control; @@ -428,8 +428,8 @@ struct mca_btl_openib_module_pp_qp_t { struct mca_btl_openib_module_srq_qp_t { struct ibv_srq *srq; - int32_t rd_posted; - int32_t sd_credits; /* the max number of outstanding sends on a QP when using SRQ */ + opal_atomic_int32_t rd_posted; + opal_atomic_int32_t sd_credits; /* the max number of outstanding sends on a QP when using SRQ */ /* i.e. the number of frags that can be outstanding (down counter) */ opal_list_t pending_frags[2]; /**< list of high/low prio frags */ /** The number of receive buffers that can be post in the current time. @@ -487,12 +487,12 @@ struct mca_btl_openib_module_t { int apm_port; /**< Alternative port that may be used for APM */ uint8_t src_path_bits; /**< offset from base lid (for LMC) */ - int32_t num_peers; + opal_atomic_int32_t num_peers; opal_mutex_t ib_lock; /**< module level lock */ size_t eager_rdma_frag_size; /**< length of eager frag */ - volatile int32_t eager_rdma_channels; /**< number of open RDMA channels */ + opal_atomic_int32_t eager_rdma_channels; /**< number of open RDMA channels */ mca_btl_base_module_error_cb_fn_t error_cb; /**< error handler */ diff --git a/opal/mca/btl/openib/btl_openib_async.c b/opal/mca/btl/openib/btl_openib_async.c index 5c52f9566b..3957bae2a9 100644 --- a/opal/mca/btl/openib/btl_openib_async.c +++ b/opal/mca/btl/openib/btl_openib_async.c @@ -4,7 +4,7 @@ * Copyright (c) 2007-2013 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2006-2007 Voltaire All rights reserved. * Copyright (c) 2009-2010 Oracle and/or its affiliates. All rights reserved - * Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights + * Copyright (c) 2013-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2014 Intel, Inc. All rights reserved. * Copyright (c) 2014 Bull SAS. All rights reserved. @@ -38,7 +38,7 @@ static opal_list_t ignore_qp_err_list; static opal_mutex_t ignore_qp_err_list_lock; -static int32_t btl_openib_async_device_count = 0; +static opal_atomic_int32_t btl_openib_async_device_count = 0; struct mca_btl_openib_async_poll { int active_poll_size; diff --git a/opal/mca/btl/openib/btl_openib_eager_rdma.h b/opal/mca/btl/openib/btl_openib_eager_rdma.h index 5acb038177..808178a457 100644 --- a/opal/mca/btl/openib/btl_openib_eager_rdma.h +++ b/opal/mca/btl/openib/btl_openib_eager_rdma.h @@ -1,7 +1,7 @@ /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2006-2007 Voltaire All rights reserved. - * Copyright (c) 2015 Los Alamos National Security, LLC. All rights + * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -25,7 +25,7 @@ struct mca_btl_openib_eager_rdma_local_t { mca_btl_openib_reg_t *reg; uint16_t head; /**< RDMA buffer to poll */ uint16_t tail; /**< Needed for credit managment */ - int32_t credits; /**< number of RDMA credits */ + opal_atomic_int32_t credits; /**< number of RDMA credits */ int32_t rd_win; #if OPAL_ENABLE_DEBUG uint32_t seq; @@ -38,8 +38,8 @@ typedef struct mca_btl_openib_eager_rdma_local_t mca_btl_openib_eager_rdma_local struct mca_btl_openib_eager_rdma_remote_t { opal_ptr_t base; /**< address of remote buffer */ uint32_t rkey; /**< RKey for accessing remote buffer */ - int32_t head; /**< RDMA buffer to post to */ - int32_t tokens; /**< number of rdam tokens */ + opal_atomic_int32_t head; /**< RDMA buffer to post to */ + opal_atomic_int32_t tokens; /**< number of rdma tokens */ #if OPAL_ENABLE_DEBUG uint32_t seq; #endif @@ -108,7 +108,7 @@ typedef struct mca_btl_openib_eager_rdma_remote_t mca_btl_openib_eager_rdma_remo #define MCA_BTL_OPENIB_RDMA_MOVE_INDEX(HEAD, OLD_HEAD) \ do { \ - (OLD_HEAD) = (OPAL_THREAD_ADD_FETCH32(&(HEAD), 1) - 1) % mca_btl_openib_component.eager_rdma_num; \ + (OLD_HEAD) = (OPAL_THREAD_ADD_FETCH32((opal_atomic_int32_t *) &(HEAD), 1) - 1) % mca_btl_openib_component.eager_rdma_num; \ } while(0) #endif diff --git a/opal/mca/btl/openib/btl_openib_endpoint.c b/opal/mca/btl/openib/btl_openib_endpoint.c index be01664b1c..e1f9a32681 100644 --- a/opal/mca/btl/openib/btl_openib_endpoint.c +++ b/opal/mca/btl/openib/btl_openib_endpoint.c @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2006-2013 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2006-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2006-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2006-2007 Voltaire All rights reserved. * Copyright (c) 2006-2009 Mellanox Technologies, Inc. All rights reserved. @@ -378,7 +378,7 @@ static void mca_btl_openib_endpoint_destruct(mca_btl_base_endpoint_t* endpoint) * was not in "connect" or "bad" flow (failed to allocate memory) * and changed the pointer back to NULL */ - if(!opal_atomic_compare_exchange_strong_ptr(&endpoint->eager_rdma_local.base.pval, (void *) &_tmp_ptr, (void *) 1)) { + if(!opal_atomic_compare_exchange_strong_ptr((opal_atomic_intptr_t *) &endpoint->eager_rdma_local.base.pval, (intptr_t *) &_tmp_ptr, 1)) { if (NULL != endpoint->eager_rdma_local.reg) { endpoint->endpoint_btl->device->rcache->rcache_deregister (endpoint->endpoint_btl->device->rcache, &endpoint->eager_rdma_local.reg->base); @@ -899,8 +899,7 @@ void mca_btl_openib_endpoint_connect_eager_rdma( /* Set local rdma pointer to 1 temporarily so other threads will not try * to enter the function */ - if(!opal_atomic_compare_exchange_strong_ptr (&endpoint->eager_rdma_local.base.pval, (void *) &_tmp_ptr, - (void *) 1)) { + if(!opal_atomic_compare_exchange_strong_ptr ((opal_atomic_intptr_t *) &endpoint->eager_rdma_local.base.pval, (intptr_t *) &_tmp_ptr, 1)) { return; } @@ -990,7 +989,7 @@ void mca_btl_openib_endpoint_connect_eager_rdma( do { _tmp_ptr = NULL; p = &device->eager_rdma_buffers[device->eager_rdma_buffers_count]; - } while(!opal_atomic_compare_exchange_strong_ptr (p, (void *) &_tmp_ptr, endpoint)); + } while(!opal_atomic_compare_exchange_strong_ptr ((opal_atomic_intptr_t *) p, (intptr_t *) &_tmp_ptr, (intptr_t) endpoint)); OPAL_THREAD_ADD_FETCH32(&openib_btl->eager_rdma_channels, 1); /* from this point progress function starts to poll new buffer */ diff --git a/opal/mca/btl/openib/btl_openib_endpoint.h b/opal/mca/btl/openib/btl_openib_endpoint.h index 89c42c595e..b3901c5665 100644 --- a/opal/mca/btl/openib/btl_openib_endpoint.h +++ b/opal/mca/btl/openib/btl_openib_endpoint.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2009 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2006-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2006-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2006-2007 Voltaire All rights reserved. * Copyright (c) 2007-2009 Mellanox Technologies. All rights reserved. @@ -117,17 +117,17 @@ typedef struct mca_btl_openib_rem_info_t { * Agggregates all per peer qp info for an endpoint */ typedef struct mca_btl_openib_endpoint_pp_qp_t { - int32_t sd_credits; /**< this rank's view of the credits + opal_atomic_int32_t sd_credits; /**< this rank's view of the credits * available for sending: * this is the credits granted by the * remote peer which has some relation to the * number of receive buffers posted remotely */ - int32_t rd_posted; /**< number of descriptors posted to the nic*/ - int32_t rd_credits; /**< number of credits to return to peer */ - int32_t cm_received; /**< Credit messages received */ - int32_t cm_return; /**< how may credits to return */ - int32_t cm_sent; /**< Outstanding number of credit messages */ + opal_atomic_int32_t rd_posted; /**< number of descriptors posted to the nic*/ + opal_atomic_int32_t rd_credits; /**< number of credits to return to peer */ + opal_atomic_int32_t cm_received; /**< Credit messages received */ + opal_atomic_int32_t cm_return; /**< how may credits to return */ + opal_atomic_int32_t cm_sent; /**< Outstanding number of credit messages */ } mca_btl_openib_endpoint_pp_qp_t; @@ -141,8 +141,8 @@ typedef struct mca_btl_openib_endpoint_srq_qp_t { typedef struct mca_btl_openib_qp_t { struct ibv_qp *lcl_qp; uint32_t lcl_psn; - volatile int32_t sd_wqe; /**< number of available send wqe entries */ - int32_t sd_wqe_inflight; + opal_atomic_int32_t sd_wqe; /**< number of available send wqe entries */ + opal_atomic_int32_t sd_wqe_inflight; int wqe_count; int users; opal_mutex_t lock; @@ -154,7 +154,7 @@ typedef struct mca_btl_openib_endpoint_qp_t { available */ opal_list_t no_wqe_pending_frags[2]; /**< put fragments here if there is no wqe available */ - int32_t rd_credit_send_lock; /**< Lock credit send fragment */ + opal_atomic_int32_t rd_credit_send_lock; /**< Lock credit send fragment */ mca_btl_openib_send_control_frag_t *credit_frag; size_t ib_inline_max; /**< max size of inline send*/ union { @@ -226,7 +226,7 @@ struct mca_btl_base_endpoint_t { opal_list_t pending_put_frags; /** number of available get tokens */ - int32_t get_tokens; + opal_atomic_int32_t get_tokens; /** subnet id of this endpoint*/ uint64_t subnet_id; @@ -235,7 +235,7 @@ struct mca_btl_base_endpoint_t { struct ib_address_t *ib_addr; /** number of eager received */ - int32_t eager_recv_count; + opal_atomic_int32_t eager_recv_count; /** info about remote RDMA buffer */ mca_btl_openib_eager_rdma_remote_t eager_rdma_remote; /** info about local RDMA buffer */ diff --git a/opal/mca/btl/portals4/btl_portals4.h b/opal/mca/btl/portals4/btl_portals4.h index 92c294b80a..41e7da7a74 100644 --- a/opal/mca/btl/portals4/btl_portals4.h +++ b/opal/mca/btl/portals4/btl_portals4.h @@ -92,7 +92,7 @@ struct mca_btl_portals4_module_t { /* number of processes we're actively connected to. Needed to know when to do activation / shutdown */ - int32_t portals_num_procs; + opal_atomic_int32_t portals_num_procs; /* number of the interface (btl) */ uint32_t interface_num; @@ -111,11 +111,11 @@ struct mca_btl_portals4_module_t { ptl_handle_eq_t recv_eq_h; /* number outstanding sends and local rdma */ - volatile int32_t portals_outstanding_ops; + opal_atomic_int32_t portals_outstanding_ops; int32_t portals_max_outstanding_ops; /* key to use for next rdma operation */ - volatile int64_t portals_rdma_key; + opal_atomic_int64_t portals_rdma_key; /* our portals network interface */ ptl_handle_ni_t portals_ni_h; diff --git a/opal/mca/btl/smcuda/btl_smcuda.h b/opal/mca/btl/smcuda/btl_smcuda.h index 2fe7df377d..caff2622e3 100644 --- a/opal/mca/btl/smcuda/btl_smcuda.h +++ b/opal/mca/btl/smcuda/btl_smcuda.h @@ -159,8 +159,8 @@ struct mca_btl_smcuda_component_t { struct mca_btl_base_endpoint_t **sm_peers; opal_free_list_t pending_send_fl; - int num_outstanding_frags; /**< number of fragments sent but not yet returned to free list */ - int num_pending_sends; /**< total number on all of my pending-send queues */ + opal_atomic_int32_t num_outstanding_frags; /**< number of fragments sent but not yet returned to free list */ + opal_atomic_int32_t num_pending_sends; /**< total number on all of my pending-send queues */ int mem_node; int num_mem_nodes; diff --git a/opal/mca/btl/uct/btl_uct_device_context.h b/opal/mca/btl/uct/btl_uct_device_context.h index ccb4f3be71..c2d1fac2f3 100644 --- a/opal/mca/btl/uct/btl_uct_device_context.h +++ b/opal/mca/btl/uct/btl_uct_device_context.h @@ -59,7 +59,7 @@ static inline void mca_btl_uct_context_unlock (mca_btl_uct_device_context_t *con static inline int mca_btl_uct_get_context_index (void) { - static volatile uint32_t next_uct_index = 0; + static opal_atomic_uint32_t next_uct_index = 0; int context_id; #if OPAL_C_HAVE__THREAD_LOCAL @@ -68,7 +68,7 @@ static inline int mca_btl_uct_get_context_index (void) context_id = uct_index; if (OPAL_UNLIKELY(-1 == context_id)) { - context_id = uct_index = opal_atomic_fetch_add_32 ((volatile int32_t *) &next_uct_index, 1) % + context_id = uct_index = opal_atomic_fetch_add_32 ((opal_atomic_int32_t *) &next_uct_index, 1) % mca_btl_uct_component.num_contexts_per_module; } } else { @@ -92,7 +92,7 @@ mca_btl_uct_module_get_tl_context_specific (mca_btl_uct_module_t *module, mca_bt mca_btl_uct_device_context_t *new_context; new_context = mca_btl_uct_context_create (module, tl, context_id); - if (!opal_atomic_compare_exchange_strong_ptr (&tl->uct_dev_contexts[context_id], &context, new_context)) { + if (!opal_atomic_compare_exchange_strong_ptr ((opal_atomic_intptr_t *) &tl->uct_dev_contexts[context_id], &context, new_context)) { mca_btl_uct_context_destroy (new_context); } else { context = new_context; diff --git a/opal/mca/btl/uct/btl_uct_types.h b/opal/mca/btl/uct/btl_uct_types.h index f7731d9e44..86619f28e1 100644 --- a/opal/mca/btl/uct/btl_uct_types.h +++ b/opal/mca/btl/uct/btl_uct_types.h @@ -94,7 +94,7 @@ typedef struct mca_btl_uct_conn_req_t mca_btl_uct_conn_req_t; */ struct mca_btl_uct_tl_endpoint_t { /** current flags (connected, requested, etc) */ - volatile int32_t flags; + opal_atomic_int32_t flags; /** UCT endpoint handle */ uct_ep_h uct_ep; diff --git a/opal/mca/btl/ugni/btl_ugni.h b/opal/mca/btl/ugni/btl_ugni.h index 367391a9ed..c56dc32326 100644 --- a/opal/mca/btl/ugni/btl_ugni.h +++ b/opal/mca/btl/ugni/btl_ugni.h @@ -114,7 +114,7 @@ struct mca_btl_ugni_device_t { int dev_index; /** number of SMSG connections */ - volatile int32_t smsg_connections; + opal_atomic_int32_t smsg_connections; /** boolean indicating that the device was recently flushed */ volatile bool flushed; @@ -168,7 +168,7 @@ typedef struct mca_btl_ugni_module_t { gni_ep_handle_t wildcard_ep; struct mca_btl_base_endpoint_t *local_ep; - volatile int32_t active_datagrams; + opal_atomic_int32_t active_datagrams; opal_event_t connection_event; struct mca_btl_ugni_endpoint_attr_t wc_remote_attr, wc_local_attr; @@ -188,13 +188,13 @@ typedef struct mca_btl_ugni_module_t { opal_pointer_array_t pending_smsg_frags_bb; int32_t reg_max; - volatile int32_t reg_count; + opal_atomic_int32_t reg_count; /* used to calculate the fraction of registered memory resources * this rank should be limited too */ int nlocal_procs; - volatile int32_t active_rdma_count; + opal_atomic_int32_t active_rdma_count; mca_rcache_base_module_t *rcache; } mca_btl_ugni_module_t; diff --git a/opal/mca/btl/ugni/btl_ugni_endpoint.h b/opal/mca/btl/ugni/btl_ugni_endpoint.h index 3ece68d671..71bfcca901 100644 --- a/opal/mca/btl/ugni/btl_ugni_endpoint.h +++ b/opal/mca/btl/ugni/btl_ugni_endpoint.h @@ -70,7 +70,7 @@ typedef struct mca_btl_base_endpoint_t { bool dg_posted; /** protect against re-entry to SMSG */ - int32_t smsg_progressing; + opal_atomic_int32_t smsg_progressing; int index; } mca_btl_base_endpoint_t; diff --git a/opal/mca/btl/ugni/btl_ugni_frag.h b/opal/mca/btl/ugni/btl_ugni_frag.h index 7b4a6a8d0d..c8e91568c9 100644 --- a/opal/mca/btl/ugni/btl_ugni_frag.h +++ b/opal/mca/btl/ugni/btl_ugni_frag.h @@ -68,7 +68,7 @@ struct mca_btl_ugni_base_frag_t; typedef struct mca_btl_ugni_base_frag_t { mca_btl_base_descriptor_t base; - volatile int32_t ref_cnt; + opal_atomic_int32_t ref_cnt; uint32_t msg_id; uint16_t hdr_size; uint16_t flags; diff --git a/opal/mca/btl/vader/btl_vader_component.c b/opal/mca/btl/vader/btl_vader_component.c index 135506a1e0..115bceb081 100644 --- a/opal/mca/btl/vader/btl_vader_component.c +++ b/opal/mca/btl/vader/btl_vader_component.c @@ -715,7 +715,7 @@ static void mca_btl_vader_progress_endpoints (void) static int mca_btl_vader_component_progress (void) { - static int32_t lock = 0; + static opal_atomic_int32_t lock = 0; int count = 0; if (opal_using_threads()) { diff --git a/opal/mca/btl/vader/btl_vader_endpoint.h b/opal/mca/btl/vader/btl_vader_endpoint.h index d3a39e08f2..9e26335222 100644 --- a/opal/mca/btl/vader/btl_vader_endpoint.h +++ b/opal/mca/btl/vader/btl_vader_endpoint.h @@ -62,7 +62,7 @@ typedef struct mca_btl_base_endpoint_t { int32_t peer_smp_rank; /**< my peer's SMP process rank. Used for accessing * SMP specfic data structures. */ - volatile size_t send_count; /**< number of fragments sent to this peer */ + opal_atomic_size_t send_count; /**< number of fragments sent to this peer */ char *segment_base; /**< start of the peer's segment (in the address space * of this process) */ diff --git a/opal/mca/btl/vader/btl_vader_fifo.h b/opal/mca/btl/vader/btl_vader_fifo.h index 0dc70bc8a1..1a56b52e97 100644 --- a/opal/mca/btl/vader/btl_vader_fifo.h +++ b/opal/mca/btl/vader/btl_vader_fifo.h @@ -30,26 +30,25 @@ #include "btl_vader_endpoint.h" #include "btl_vader_frag.h" -#define vader_item_compare_exchange(x, y, z) opal_atomic_compare_exchange_strong_ptr ((volatile void **) (x), (void **) (y), (void *) (z)) +#define vader_item_compare_exchange(x, y, z) opal_atomic_compare_exchange_strong_ptr ((opal_atomic_intptr_t *) (x), (intptr_t *) (y), (intptr_t) (z)) #if SIZEOF_VOID_P == 8 - #define vader_item_swap(x, y) opal_atomic_swap_64((volatile int64_t *)(x), (int64_t)(y)) + #define vader_item_swap(x, y) opal_atomic_swap_64((opal_atomic_int64_t *)(x), (int64_t)(y)) #define MCA_BTL_VADER_OFFSET_MASK 0xffffffffll #define MCA_BTL_VADER_OFFSET_BITS 32 #define MCA_BTL_VADER_BITNESS 64 - - typedef int64_t fifo_value_t; #else - #define vader_item_swap(x, y) opal_atomic_swap_32((volatile int32_t *)(x), (int32_t)(y)) + #define vader_item_swap(x, y) opal_atomic_swap_32((opal_atomic_int32_t *)(x), (int32_t)(y)) #define MCA_BTL_VADER_OFFSET_MASK 0x00ffffffl #define MCA_BTL_VADER_OFFSET_BITS 24 #define MCA_BTL_VADER_BITNESS 32 - - typedef int32_t fifo_value_t; #endif +typedef opal_atomic_intptr_t atomic_fifo_value_t; +typedef intptr_t fifo_value_t; + #define VADER_FIFO_FREE ((fifo_value_t)-2) /* @@ -68,9 +67,9 @@ /* lock free fifo */ typedef struct vader_fifo_t { - volatile fifo_value_t fifo_head; - volatile fifo_value_t fifo_tail; - volatile int32_t fbox_available; + atomic_fifo_value_t fifo_head; + atomic_fifo_value_t fifo_tail; + opal_atomic_int32_t fbox_available; } vader_fifo_t; /* large enough to ensure the fifo is on its own cache line */ diff --git a/opal/mca/btl/vader/btl_vader_sc_emu.c b/opal/mca/btl/vader/btl_vader_sc_emu.c index 4f0b289ea6..8e13f0444b 100644 --- a/opal/mca/btl/vader/btl_vader_sc_emu.c +++ b/opal/mca/btl/vader/btl_vader_sc_emu.c @@ -13,7 +13,7 @@ #include "btl_vader_frag.h" #if OPAL_HAVE_ATOMIC_MATH_64 -static void mca_btl_vader_sc_emu_atomic_64 (int64_t *operand, volatile int64_t *addr, mca_btl_base_atomic_op_t op) +static void mca_btl_vader_sc_emu_atomic_64 (int64_t *operand, opal_atomic_int64_t *addr, mca_btl_base_atomic_op_t op) { int64_t result = 0; @@ -52,7 +52,7 @@ static void mca_btl_vader_sc_emu_atomic_64 (int64_t *operand, volatile int64_t * #endif #if OPAL_HAVE_ATOMIC_MATH_32 -static void mca_btl_vader_sc_emu_atomic_32 (int32_t *operand, volatile int32_t *addr, mca_btl_base_atomic_op_t op) +static void mca_btl_vader_sc_emu_atomic_32 (int32_t *operand, opal_atomic_int32_t *addr, mca_btl_base_atomic_op_t op) { int32_t result = 0; @@ -123,10 +123,10 @@ static void mca_btl_vader_sc_emu_rdma (mca_btl_base_module_t *btl, mca_btl_base_ #if OPAL_HAVE_ATOMIC_MATH_64 case MCA_BTL_VADER_OP_CSWAP: if (!(hdr->flags & MCA_BTL_ATOMIC_FLAG_32BIT)) { - opal_atomic_compare_exchange_strong_64 ((volatile int64_t *) hdr->addr, &hdr->operand[0], hdr->operand[1]); + opal_atomic_compare_exchange_strong_64 ((opal_atomic_int64_t *) hdr->addr, &hdr->operand[0], hdr->operand[1]); #if OPAL_HAVE_ATOMIC_MATH_32 } else { - opal_atomic_compare_exchange_strong_32 ((volatile int32_t *) hdr->addr, (int32_t *) &hdr->operand[0], + opal_atomic_compare_exchange_strong_32 ((opal_atomic_int32_t *) hdr->addr, (int32_t *) &hdr->operand[0], (int32_t) hdr->operand[1]); #else } else { diff --git a/opal/mca/common/sm/common_sm.h b/opal/mca/common/sm/common_sm.h index 819b82f6ee..7c4456f045 100644 --- a/opal/mca/common/sm/common_sm.h +++ b/opal/mca/common/sm/common_sm.h @@ -1,3 +1,4 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana * University Research and Technology @@ -10,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2009-2010 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2010-2012 Los Alamos National Security, LLC. + * Copyright (c) 2010-2018 Los Alamos National Security, LLC. * All rights reserved. * $COPYRIGHT$ * @@ -42,13 +43,13 @@ typedef struct mca_common_sm_seg_header_t { /* lock to control atomic access */ opal_atomic_lock_t seg_lock; /* indicates whether or not the segment is ready for use */ - volatile int32_t seg_inited; + opal_atomic_int32_t seg_inited; /* number of local processes that are attached to the shared memory segment. * this is primarily used as a way of determining whether or not it is safe * to unlink the shared memory backing store. for example, once seg_att * is equal to the number of local processes, then we can safely unlink. */ - volatile size_t seg_num_procs_inited; + opal_atomic_size_t seg_num_procs_inited; /* offset to next available memory location available for allocation */ size_t seg_offset; /* total size of the segment */ diff --git a/opal/mca/mpool/hugepage/mpool_hugepage.h b/opal/mca/mpool/hugepage/mpool_hugepage.h index 9b76df62e3..8ac5373281 100644 --- a/opal/mca/mpool/hugepage/mpool_hugepage.h +++ b/opal/mca/mpool/hugepage/mpool_hugepage.h @@ -46,7 +46,7 @@ struct mca_mpool_hugepage_component_t { opal_list_t huge_pages; mca_mpool_hugepage_module_t *modules; int module_count; - unsigned long bytes_allocated; + opal_atomic_size_t bytes_allocated; }; typedef struct mca_mpool_hugepage_component_t mca_mpool_hugepage_component_t; @@ -62,7 +62,7 @@ struct mca_mpool_hugepage_hugepage_t { /** path for mmapped files */ char *path; /** counter to help ensure unique file names for mmaped files */ - volatile int32_t count; + opal_atomic_int32_t count; /** some platforms allow allocation of hugepages through mmap flags */ int mmap_flags; }; diff --git a/opal/mca/mpool/hugepage/mpool_hugepage_component.c b/opal/mca/mpool/hugepage/mpool_hugepage_component.c index 02320b9f91..0a6f4f6b38 100644 --- a/opal/mca/mpool/hugepage/mpool_hugepage_component.c +++ b/opal/mca/mpool/hugepage/mpool_hugepage_component.c @@ -135,7 +135,7 @@ static int mca_mpool_hugepage_register(void) "hugepage component", OPAL_INFO_LVL_3, MCA_BASE_PVAR_CLASS_SIZE, MCA_BASE_VAR_TYPE_UNSIGNED_LONG, NULL, MCA_BASE_VAR_BIND_NO_OBJECT, MCA_BASE_PVAR_FLAG_READONLY | MCA_BASE_PVAR_FLAG_CONTINUOUS, - NULL, NULL, NULL, &mca_mpool_hugepage_component.bytes_allocated); + NULL, NULL, NULL, (void *) &mca_mpool_hugepage_component.bytes_allocated); return OPAL_SUCCESS; } diff --git a/opal/mca/mpool/hugepage/mpool_hugepage_module.c b/opal/mca/mpool/hugepage/mpool_hugepage_module.c index 89a8b7eb6d..e174614b61 100644 --- a/opal/mca/mpool/hugepage/mpool_hugepage_module.c +++ b/opal/mca/mpool/hugepage/mpool_hugepage_module.c @@ -183,7 +183,7 @@ void *mca_mpool_hugepage_seg_alloc (void *ctx, size_t *sizep) opal_mutex_lock (&hugepage_module->lock); opal_rb_tree_insert (&hugepage_module->allocation_tree, base, (void *) (intptr_t) size); - opal_atomic_add (&mca_mpool_hugepage_component.bytes_allocated, (int64_t) size); + (void) opal_atomic_fetch_add_size_t (&mca_mpool_hugepage_component.bytes_allocated, size); opal_mutex_unlock (&hugepage_module->lock); OPAL_OUTPUT_VERBOSE((MCA_BASE_VERBOSE_TRACE, opal_mpool_base_framework.framework_verbose, @@ -207,7 +207,7 @@ void mca_mpool_hugepage_seg_free (void *ctx, void *addr) OPAL_OUTPUT_VERBOSE((MCA_BASE_VERBOSE_TRACE, opal_mpool_base_framework.framework_verbose, "freeing segment %p of size %lu bytes", addr, size)); munmap (addr, size); - opal_atomic_add (&mca_mpool_hugepage_component.bytes_allocated, -(int64_t) size); + (void) opal_atomic_fetch_add_size_t (&mca_mpool_hugepage_component.bytes_allocated, -size); } opal_mutex_unlock (&hugepage_module->lock); diff --git a/opal/mca/rcache/grdma/rcache_grdma_module.c b/opal/mca/rcache/grdma/rcache_grdma_module.c index 4e64b5df14..6b564327dd 100644 --- a/opal/mca/rcache/grdma/rcache_grdma_module.c +++ b/opal/mca/rcache/grdma/rcache_grdma_module.c @@ -215,7 +215,7 @@ static inline void mca_rcache_grdma_add_to_lru (mca_rcache_grdma_module_t *rcach opal_atomic_wmb (); /* mark this registration as being in the LRU */ - opal_atomic_fetch_or_32 ((volatile int32_t *) &grdma_reg->flags, MCA_RCACHE_GRDMA_REG_FLAG_IN_LRU); + opal_atomic_fetch_or_32 ((opal_atomic_int32_t *) &grdma_reg->flags, MCA_RCACHE_GRDMA_REG_FLAG_IN_LRU); opal_mutex_unlock (&rcache_grdma->cache->vma_module->vma_lock); } @@ -263,7 +263,7 @@ static int mca_rcache_grdma_check_cached (mca_rcache_base_registration_t *grdma_ } /* This segment fits fully within an existing segment. */ - (void) opal_atomic_fetch_add_32 ((volatile int32_t *) &rcache_grdma->stat_cache_hit, 1); + (void) opal_atomic_fetch_add_32 ((opal_atomic_int32_t *) &rcache_grdma->stat_cache_hit, 1); OPAL_OUTPUT_VERBOSE((MCA_BASE_VERBOSE_TRACE, opal_rcache_base_framework.framework_output, "returning existing registration %p. references %d", (void *) grdma_reg, ref_cnt)); return 1; @@ -321,7 +321,7 @@ static int mca_rcache_grdma_register (mca_rcache_base_module_t *rcache, void *ad /* get updated access flags */ access_flags = find_args.access_flags; - OPAL_THREAD_ADD_FETCH32((volatile int32_t *) &rcache_grdma->stat_cache_miss, 1); + OPAL_THREAD_ADD_FETCH32((opal_atomic_int32_t *) &rcache_grdma->stat_cache_miss, 1); } item = opal_free_list_get_mt (&rcache_grdma->reg_list); @@ -447,7 +447,7 @@ typedef struct gc_add_args_t gc_add_args_t; static int mca_rcache_grdma_add_to_gc (mca_rcache_base_registration_t *grdma_reg) { mca_rcache_grdma_module_t *rcache_grdma = (mca_rcache_grdma_module_t *) grdma_reg->rcache; - uint32_t flags = opal_atomic_fetch_or_32 ((volatile int32_t *) &grdma_reg->flags, MCA_RCACHE_FLAGS_INVALID); + uint32_t flags = opal_atomic_fetch_or_32 ((opal_atomic_int32_t *) &grdma_reg->flags, MCA_RCACHE_FLAGS_INVALID); if ((flags & MCA_RCACHE_FLAGS_INVALID) || 0 != grdma_reg->ref_count) { /* nothing to do */ diff --git a/opal/mca/rcache/rcache.h b/opal/mca/rcache/rcache.h index fa808dabac..f95fa1a597 100644 --- a/opal/mca/rcache/rcache.h +++ b/opal/mca/rcache/rcache.h @@ -92,9 +92,9 @@ struct mca_rcache_base_registration_t { /** artifact of old mpool/rcache architecture. used by cuda code */ unsigned char *alloc_base; /** number of outstanding references */ - volatile int32_t ref_count; + opal_atomic_int32_t ref_count; /** registration flags */ - volatile uint32_t flags; + opal_atomic_uint32_t flags; /** internal rcache context */ void *rcache_context; #if OPAL_CUDA_GDR_SUPPORT diff --git a/opal/runtime/opal_progress.c b/opal/runtime/opal_progress.c index 8c88a32c67..d7ce887ef2 100644 --- a/opal/runtime/opal_progress.c +++ b/opal/runtime/opal_progress.c @@ -73,13 +73,13 @@ static opal_timer_t event_progress_last_time = 0; static opal_timer_t event_progress_delta = 0; #else /* current count down until we tick the event library */ -static int32_t event_progress_counter = 0; +static opal_atomic_int32_t event_progress_counter = 0; /* reset value for counter when it hits 0 */ static int32_t event_progress_delta = 0; #endif /* users of the event library from MPI cause the tick rate to be every time */ -static int32_t num_event_users = 0; +static opal_atomic_int32_t num_event_users = 0; #if OPAL_ENABLE_DEBUG static int debug_output = -1; @@ -410,7 +410,7 @@ static int _opal_progress_register (opal_progress_callback_t cb, volatile opal_p opal_atomic_wmb (); /* swap out callback array */ - old = opal_atomic_swap_ptr (cbs, tmp); + old = (opal_progress_callback_t *) opal_atomic_swap_ptr ((opal_atomic_intptr_t *) cbs, (intptr_t) tmp); opal_atomic_wmb (); @@ -471,7 +471,7 @@ static int _opal_progress_unregister (opal_progress_callback_t cb, volatile opal for (size_t i = (size_t) ret ; i < *callback_array_len - 1 ; ++i) { /* copy callbacks atomically since another thread may be in * opal_progress(). */ - (void) opal_atomic_swap_ptr (callback_array + i, callback_array[i+1]); + (void) opal_atomic_swap_ptr ((opal_atomic_intptr_t *) (callback_array + i), (intptr_t) callback_array[i+1]); } callback_array[*callback_array_len] = fake_cb; diff --git a/opal/threads/mutex_unix.h b/opal/threads/mutex_unix.h index 1b748369c2..a61623826c 100644 --- a/opal/threads/mutex_unix.h +++ b/opal/threads/mutex_unix.h @@ -10,7 +10,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2007-2015 Los Alamos National Security, LLC. All rights + * Copyright (c) 2007-2018 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2015-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. diff --git a/opal/threads/thread_usage.h b/opal/threads/thread_usage.h index 178c8ceaab..8ce65362c1 100644 --- a/opal/threads/thread_usage.h +++ b/opal/threads/thread_usage.h @@ -13,7 +13,7 @@ * Copyright (c) 2007-2014 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2014-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights + * Copyright (c) 2015-2018 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -94,7 +94,7 @@ static inline bool opal_set_using_threads(bool have) */ #define OPAL_THREAD_DEFINE_ATOMIC_OP(type, name, operator, suffix) \ -static inline type opal_thread_ ## name ## _fetch_ ## suffix (volatile type *addr, type delta) \ +static inline type opal_thread_ ## name ## _fetch_ ## suffix (opal_atomic_ ## type *addr, type delta) \ { \ if (OPAL_UNLIKELY(opal_using_threads())) { \ return opal_atomic_ ## name ## _fetch_ ## suffix (addr, delta); \ @@ -104,7 +104,7 @@ static inline type opal_thread_ ## name ## _fetch_ ## suffix (volatile type *add return *addr; \ } \ \ -static inline type opal_thread_fetch_ ## name ## _ ## suffix (volatile type *addr, type delta) \ +static inline type opal_thread_fetch_ ## name ## _ ## suffix (opal_atomic_ ## type *addr, type delta) \ { \ if (OPAL_UNLIKELY(opal_using_threads())) { \ return opal_atomic_fetch_ ## name ## _ ## suffix (addr, delta); \ @@ -116,10 +116,10 @@ static inline type opal_thread_fetch_ ## name ## _ ## suffix (volatile type *add } #define OPAL_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(type, addr_type, suffix) \ -static inline bool opal_thread_compare_exchange_strong_ ## suffix (volatile addr_type *addr, type *compare, type value) \ +static inline bool opal_thread_compare_exchange_strong_ ## suffix (opal_atomic_ ## addr_type *addr, type *compare, type value) \ { \ if (OPAL_UNLIKELY(opal_using_threads())) { \ - return opal_atomic_compare_exchange_strong_ ## suffix ((volatile type *) addr, compare, value); \ + return opal_atomic_compare_exchange_strong_ ## suffix (addr, (addr_type *) compare, (addr_type) value); \ } \ \ if ((type) *addr == *compare) { \ @@ -133,10 +133,10 @@ static inline bool opal_thread_compare_exchange_strong_ ## suffix (volatile addr } #define OPAL_THREAD_DEFINE_ATOMIC_SWAP(type, addr_type, suffix) \ -static inline type opal_thread_swap_ ## suffix (volatile addr_type *ptr, type newvalue) \ +static inline type opal_thread_swap_ ## suffix (opal_atomic_ ## addr_type *ptr, type newvalue) \ { \ if (opal_using_threads ()) { \ - return opal_atomic_swap_ ## suffix ((volatile type *) ptr, newvalue); \ + return (type) opal_atomic_swap_ ## suffix (ptr, (addr_type) newvalue); \ } \ \ type old = ((type *) ptr)[0]; \ @@ -154,9 +154,9 @@ OPAL_THREAD_DEFINE_ATOMIC_OP(int32_t, sub, -, 32) OPAL_THREAD_DEFINE_ATOMIC_OP(size_t, sub, -, size_t) OPAL_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(int32_t, int32_t, 32) -OPAL_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(void *, intptr_t, ptr) +OPAL_THREAD_DEFINE_ATOMIC_COMPARE_EXCHANGE(intptr_t, intptr_t, ptr) OPAL_THREAD_DEFINE_ATOMIC_SWAP(int32_t, int32_t, 32) -OPAL_THREAD_DEFINE_ATOMIC_SWAP(void *, intptr_t, ptr) +OPAL_THREAD_DEFINE_ATOMIC_SWAP(intptr_t, intptr_t, ptr) #define OPAL_THREAD_ADD_FETCH32 opal_thread_add_fetch_32 #define OPAL_ATOMIC_ADD_FETCH32 opal_thread_add_fetch_32 @@ -197,13 +197,13 @@ OPAL_THREAD_DEFINE_ATOMIC_SWAP(void *, intptr_t, ptr) #define OPAL_THREAD_COMPARE_EXCHANGE_STRONG_32 opal_thread_compare_exchange_strong_32 #define OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_32 opal_thread_compare_exchange_strong_32 -#define OPAL_THREAD_COMPARE_EXCHANGE_STRONG_PTR(x, y, z) opal_thread_compare_exchange_strong_ptr ((volatile intptr_t *) x, (void *) y, (void *) z) +#define OPAL_THREAD_COMPARE_EXCHANGE_STRONG_PTR(x, y, z) opal_thread_compare_exchange_strong_ptr ((opal_atomic_intptr_t *) x, (intptr_t *) y, (intptr_t) z) #define OPAL_ATOMIC_COMPARE_EXCHANGE_STRONG_PTR OPAL_THREAD_COMPARE_EXCHANGE_STRONG_PTR #define OPAL_THREAD_SWAP_32 opal_thread_swap_32 #define OPAL_ATOMIC_SWAP_32 opal_thread_swap_32 -#define OPAL_THREAD_SWAP_PTR(x, y) opal_thread_swap_ptr ((volatile intptr_t *) x, (void *) y) +#define OPAL_THREAD_SWAP_PTR(x, y) opal_thread_swap_ptr ((opal_atomic_intptr_t *) x, (intptr_t) y) #define OPAL_ATOMIC_SWAP_PTR OPAL_THREAD_SWAP_PTR /* define 64-bit macros is 64-bit atomic math is available */ diff --git a/opal/threads/wait_sync.h b/opal/threads/wait_sync.h index 1e59467035..cb095c0b1b 100644 --- a/opal/threads/wait_sync.h +++ b/opal/threads/wait_sync.h @@ -26,7 +26,7 @@ BEGIN_C_DECLS typedef struct ompi_wait_sync_t { - int32_t count; + opal_atomic_int32_t count; int32_t status; pthread_cond_t condition; pthread_mutex_t lock; diff --git a/test/asm/atomic_cmpset.c b/test/asm/atomic_cmpset.c index 4a06847703..9fe964125a 100644 --- a/test/asm/atomic_cmpset.c +++ b/test/asm/atomic_cmpset.c @@ -45,33 +45,33 @@ int nreps = 100; int nthreads = 2; int enable_verbose = 0; -volatile int32_t vol32 = 0; -int32_t val32 = 0; +opal_atomic_int32_t vol32 = 0; +opal_atomic_int32_t val32 = 0; int32_t old32 = 0; int32_t new32 = 0; #if OPAL_HAVE_ATOMIC_MATH_64 -volatile int64_t vol64 = 0; -int64_t val64 = 0; +opal_atomic_int64_t vol64 = 0; +opal_atomic_int64_t val64 = 0; int64_t old64 = 0; int64_t new64 = 0; #endif #if OPAL_HAVE_ATOMIC_COMPARE_EXCHANGE_128 -volatile opal_int128_t vol128; -opal_int128_t val128; +opal_atomic_int128_t vol128; +opal_atomic_int128_t val128; opal_int128_t old128; opal_int128_t new128; #endif -volatile int volint = 0; -int valint = 0; +opal_atomic_int_t volint = 0; +opal_atomic_int_t valint = 0; int oldint = 0; int newint = 0; -volatile void *volptr = NULL; -void *oldptr = NULL; -void *newptr = NULL; +opal_atomic_intptr_t volptr = 0; +intptr_t oldptr = 0; +intptr_t newptr = 0; static void *thread_main(void *arg) @@ -235,39 +235,39 @@ int main(int argc, char *argv[]) /* -- cmpset ptr tests -- */ - volptr = (void *) 42, oldptr = (void *) 42, newptr = (void *) 50; + volptr = 42, oldptr = 42, newptr = 50; assert(opal_atomic_compare_exchange_strong_ptr (&volptr, &oldptr, newptr) == true); opal_atomic_rmb(); assert(volptr == newptr); - assert(oldptr == (void *) 42); + assert(oldptr == 42); - volptr = (void *) 42, oldptr = (void *) 420, newptr = (void *) 50; + volptr = 42, oldptr = 420, newptr = 50; assert(opal_atomic_compare_exchange_strong_ptr (&volptr, &oldptr, newptr) == false); opal_atomic_rmb(); - assert(volptr == (void *) 42); - assert(oldptr == (void *) 42); + assert(volptr == 42); + assert(oldptr == 42); - volptr = (void *) 42, oldptr = (void *) 42, newptr = (void *) 50; + volptr = 42, oldptr = 42, newptr = 50; assert(opal_atomic_compare_exchange_strong_acq_ptr (&volptr, &oldptr, newptr) == true); assert(volptr == newptr); - assert(oldptr == (void *) 42); + assert(oldptr == 42); - volptr = (void *) 42, oldptr = (void *) 420, newptr = (void *) 50; + volptr = 42, oldptr = 420, newptr = 50; assert(opal_atomic_compare_exchange_strong_acq_ptr (&volptr, &oldptr, newptr) == false); - assert(volptr == (void *) 42); - assert(oldptr == (void *) 42); + assert(volptr == 42); + assert(oldptr == 42); - volptr = (void *) 42, oldptr = (void *) 42, newptr = (void *) 50; + volptr = 42, oldptr = 42, newptr = 50; assert(opal_atomic_compare_exchange_strong_rel_ptr (&volptr, &oldptr, newptr) == true); opal_atomic_rmb(); assert(volptr == newptr); - assert(oldptr == (void *) 42); + assert(oldptr == 42); - volptr = (void *) 42, oldptr = (void *) 420, newptr = (void *) 50; + volptr = 42, oldptr = 420, newptr = 50; assert(opal_atomic_compare_exchange_strong_rel_ptr (&volptr, &oldptr, newptr) == false); opal_atomic_rmb(); - assert(volptr == (void *) 42); - assert(oldptr == (void *) 42); + assert(volptr == 42); + assert(oldptr == 42); /* -- add_32 tests -- */ diff --git a/test/asm/atomic_math.c b/test/asm/atomic_math.c index 54f771cc26..8629ad9780 100644 --- a/test/asm/atomic_math.c +++ b/test/asm/atomic_math.c @@ -32,11 +32,11 @@ #define TEST_REPS 500 -int32_t val32 = 0; +opal_atomic_int32_t val32 = 0; #if OPAL_HAVE_ATOMIC_MATH_64 -int64_t val64 = 0; +opal_atomic_int64_t val64 = 0; #endif -int valint = 0; +opal_atomic_int_t valint = 0; static void* atomic_math_test(void* arg) { diff --git a/test/threads/opal_thread.c b/test/threads/opal_thread.c index 1c6480d19c..e795190c5a 100644 --- a/test/threads/opal_thread.c +++ b/test/threads/opal_thread.c @@ -21,7 +21,7 @@ /* Only have the body of this test if we have thread support */ -static volatile int count = 0; +static opal_atomic_int_t count = 0; static void* thr1_run(opal_object_t* obj)