diff --git a/ompi/mca/coll/base/coll_base_util.c b/ompi/mca/coll/base/coll_base_util.c index 6187098598..57fe14bad2 100644 --- a/ompi/mca/coll/base/coll_base_util.c +++ b/ompi/mca/coll/base/coll_base_util.c @@ -9,8 +9,8 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2014-2017 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -26,6 +26,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/mca/coll/base/coll_tags.h" #include "ompi/mca/coll/base/coll_base_functions.h" +#include "ompi/mca/topo/base/base.h" #include "ompi/mca/pml/pml.h" #include "coll_base_util.h" @@ -103,3 +104,187 @@ int ompi_rounddown(int num, int factor) num /= factor; return num * factor; /* floor(num / factor) * factor */ } + +static void release_objs_callback(struct ompi_coll_base_nbc_request_t *request) { + if (NULL != request->data.objs.objs[0]) { + OBJ_RELEASE(request->data.objs.objs[0]); + } + if (NULL != request->data.objs.objs[1]) { + OBJ_RELEASE(request->data.objs.objs[1]); + } +} + +static int complete_objs_callback(struct ompi_request_t *req) { + struct ompi_coll_base_nbc_request_t *request = (ompi_coll_base_nbc_request_t *)req; + int rc = OMPI_SUCCESS; + assert (NULL != request); + if (NULL != request->cb.req_complete_cb) { + rc = request->cb.req_complete_cb(request->req_complete_cb_data); + } + release_objs_callback(request); + return rc; +} + +static int free_objs_callback(struct ompi_request_t **rptr) { + struct ompi_coll_base_nbc_request_t *request = *(ompi_coll_base_nbc_request_t **)rptr; + int rc = OMPI_SUCCESS; + if (NULL != request->cb.req_free) { + rc = request->cb.req_free(rptr); + } + release_objs_callback(request); + return rc; +} + +int ompi_coll_base_retain_op( ompi_request_t *req, ompi_op_t *op, + ompi_datatype_t *type) { + ompi_coll_base_nbc_request_t *request = (ompi_coll_base_nbc_request_t *)req; + bool retain = false; + if (!ompi_op_is_intrinsic(op)) { + OBJ_RETAIN(op); + request->data.op.op = op; + retain = true; + } + if (!ompi_datatype_is_predefined(type)) { + OBJ_RETAIN(type); + request->data.op.datatype = type; + retain = true; + } + if (OPAL_UNLIKELY(retain)) { + /* We need to consider two cases : + * - non blocking collectives: + * the objects can be released when MPI_Wait() completes + * and we use the req_complete_cb callback + * - persistent non blocking collectives: + * the objects can only be released when the request is freed + * (e.g. MPI_Request_free() completes) and we use req_free callback + */ + if (req->req_persistent) { + request->cb.req_free = req->req_free; + req->req_free = free_objs_callback; + } else { + request->cb.req_complete_cb = req->req_complete_cb; + request->req_complete_cb_data = req->req_complete_cb_data; + req->req_complete_cb = complete_objs_callback; + req->req_complete_cb_data = request; + } + } + return OMPI_SUCCESS; +} + +int ompi_coll_base_retain_datatypes( ompi_request_t *req, ompi_datatype_t *stype, + ompi_datatype_t *rtype) { + ompi_coll_base_nbc_request_t *request = (ompi_coll_base_nbc_request_t *)req; + bool retain = false; + if (NULL != stype && !ompi_datatype_is_predefined(stype)) { + OBJ_RETAIN(stype); + request->data.types.stype = stype; + retain = true; + } + if (NULL != rtype && !ompi_datatype_is_predefined(rtype)) { + OBJ_RETAIN(rtype); + request->data.types.rtype = rtype; + retain = true; + } + if (OPAL_UNLIKELY(retain)) { + if (req->req_persistent) { + request->cb.req_free = req->req_free; + req->req_free = free_objs_callback; + } else { + request->cb.req_complete_cb = req->req_complete_cb; + request->req_complete_cb_data = req->req_complete_cb_data; + req->req_complete_cb = complete_objs_callback; + req->req_complete_cb_data = request; + } + } + return OMPI_SUCCESS; +} + +static void release_vecs_callback(ompi_coll_base_nbc_request_t *request) { + ompi_communicator_t *comm = request->super.req_mpi_object.comm; + int scount, rcount; + if (OMPI_COMM_IS_TOPO(comm)) { + (void)mca_topo_base_neighbor_count (comm, &rcount, &scount); + } else { + scount = rcount = OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm); + } + for (int i=0; idata.vecs.stypes && NULL != request->data.vecs.stypes[i]) { + OMPI_DATATYPE_RELEASE(request->data.vecs.stypes[i]); + } + } + for (int i=0; idata.vecs.rtypes && NULL != request->data.vecs.rtypes[i]) { + OMPI_DATATYPE_RELEASE(request->data.vecs.rtypes[i]); + } + } +} + +static int complete_vecs_callback(struct ompi_request_t *req) { + ompi_coll_base_nbc_request_t *request = (ompi_coll_base_nbc_request_t *)req; + int rc = OMPI_SUCCESS; + assert (NULL != request); + if (NULL != request->cb.req_complete_cb) { + rc = request->cb.req_complete_cb(request->req_complete_cb_data); + } + release_vecs_callback(request); + return rc; +} + +static int free_vecs_callback(struct ompi_request_t **rptr) { + struct ompi_coll_base_nbc_request_t *request = *(ompi_coll_base_nbc_request_t **)rptr; + int rc = OMPI_SUCCESS; + if (NULL != request->cb.req_free) { + rc = request->cb.req_free(rptr); + } + release_vecs_callback(request); + return rc; +} + +int ompi_coll_base_retain_datatypes_w( ompi_request_t *req, + ompi_datatype_t *stypes[], ompi_datatype_t *rtypes[]) { + ompi_coll_base_nbc_request_t *request = (ompi_coll_base_nbc_request_t *)req; + bool retain = false; + ompi_communicator_t *comm = request->super.req_mpi_object.comm; + int scount, rcount; + if (OMPI_COMM_IS_TOPO(comm)) { + (void)mca_topo_base_neighbor_count (comm, &rcount, &scount); + } else { + scount = rcount = OMPI_COMM_IS_INTER(comm)?ompi_comm_remote_size(comm):ompi_comm_size(comm); + } + + for (int i=0; idata.vecs.stypes = stypes; + request->data.vecs.rtypes = rtypes; + if (req->req_persistent) { + request->cb.req_free = req->req_free; + req->req_free = free_vecs_callback; + } else { + request->cb.req_complete_cb = req->req_complete_cb; + request->req_complete_cb_data = req->req_complete_cb_data; + req->req_complete_cb = complete_vecs_callback; + req->req_complete_cb_data = request; + } + } + return OMPI_SUCCESS; +} + +static void nbc_req_cons(ompi_coll_base_nbc_request_t *req) { + req->cb.req_complete_cb = NULL; + req->req_complete_cb_data = NULL; + req->data.objs.objs[0] = NULL; + req->data.objs.objs[1] = NULL; +} + +OBJ_CLASS_INSTANCE(ompi_coll_base_nbc_request_t, ompi_request_t, nbc_req_cons, NULL); diff --git a/ompi/mca/coll/base/coll_base_util.h b/ompi/mca/coll/base/coll_base_util.h index 8306b8fe83..a5b8016124 100644 --- a/ompi/mca/coll/base/coll_base_util.h +++ b/ompi/mca/coll/base/coll_base_util.h @@ -9,8 +9,8 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2014-2017 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -27,10 +27,41 @@ #include "ompi/mca/mca.h" #include "ompi/datatype/ompi_datatype.h" #include "ompi/request/request.h" +#include "ompi/op/op.h" #include "ompi/mca/pml/pml.h" BEGIN_C_DECLS +struct ompi_coll_base_nbc_request_t { + ompi_request_t super; + union { + ompi_request_complete_fn_t req_complete_cb; + ompi_request_free_fn_t req_free; + } cb; + void *req_complete_cb_data; + union { + struct { + ompi_op_t *op; + ompi_datatype_t *datatype; + } op; + struct { + ompi_datatype_t *stype; + ompi_datatype_t *rtype; + } types; + struct { + opal_object_t *objs[2]; + } objs; + struct { + ompi_datatype_t **stypes; + ompi_datatype_t **rtypes; + } vecs; + } data; +}; + +OMPI_DECLSPEC OBJ_CLASS_DECLARATION(ompi_coll_base_nbc_request_t); + +typedef struct ompi_coll_base_nbc_request_t ompi_coll_base_nbc_request_t; + /** * A MPI_like function doing a send and a receive simultaneously. * If one of the communications results in a zero-byte message the @@ -84,5 +115,17 @@ unsigned int ompi_mirror_perm(unsigned int x, int nbits); */ int ompi_rounddown(int num, int factor); +int ompi_coll_base_retain_op( ompi_request_t *request, + ompi_op_t *op, + ompi_datatype_t *type); + +int ompi_coll_base_retain_datatypes( ompi_request_t *request, + ompi_datatype_t *stype, + ompi_datatype_t *rtype); + +int ompi_coll_base_retain_datatypes_w( ompi_request_t *request, + ompi_datatype_t *stypes[], + ompi_datatype_t *rtypes[]); + END_C_DECLS #endif /* MCA_COLL_BASE_UTIL_EXPORT_H */ diff --git a/ompi/mca/coll/libnbc/coll_libnbc.h b/ompi/mca/coll/libnbc/coll_libnbc.h index 967a779425..17abf86f2a 100644 --- a/ompi/mca/coll/libnbc/coll_libnbc.h +++ b/ompi/mca/coll/libnbc/coll_libnbc.h @@ -13,8 +13,8 @@ * Copyright (c) 2008 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2014-2017 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2016-2017 IBM Corporation. All rights reserved. * Copyright (c) 2018 FUJITSU LIMITED. All rights reserved. * $COPYRIGHT$ @@ -28,7 +28,7 @@ #define MCA_COLL_LIBNBC_EXPORT_H #include "ompi/mca/coll/coll.h" -#include "ompi/request/request.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "opal/sys/atomic.h" BEGIN_C_DECLS @@ -114,7 +114,7 @@ typedef struct NBC_Schedule NBC_Schedule; OBJ_CLASS_DECLARATION(NBC_Schedule); struct ompi_coll_libnbc_request_t { - ompi_request_t super; + ompi_coll_base_nbc_request_t super; MPI_Comm comm; long row_offset; bool nbc_complete; /* status in libnbc level */ @@ -138,13 +138,13 @@ typedef ompi_coll_libnbc_request_t NBC_Handle; opal_free_list_item_t *item; \ item = opal_free_list_wait (&mca_coll_libnbc_component.requests); \ req = (ompi_coll_libnbc_request_t*) item; \ - OMPI_REQUEST_INIT(&req->super, persistent); \ - req->super.req_mpi_object.comm = comm; \ + OMPI_REQUEST_INIT(&req->super.super, persistent); \ + req->super.super.req_mpi_object.comm = comm; \ } while (0) #define OMPI_COLL_LIBNBC_REQUEST_RETURN(req) \ do { \ - OMPI_REQUEST_FINI(&(req)->super); \ + OMPI_REQUEST_FINI(&(req)->super.super); \ opal_free_list_return (&mca_coll_libnbc_component.requests, \ (opal_free_list_item_t*) (req)); \ } while (0) diff --git a/ompi/mca/coll/libnbc/coll_libnbc_component.c b/ompi/mca/coll/libnbc/coll_libnbc_component.c index bf4960d923..c5b1656385 100644 --- a/ompi/mca/coll/libnbc/coll_libnbc_component.c +++ b/ompi/mca/coll/libnbc/coll_libnbc_component.c @@ -13,8 +13,8 @@ * Copyright (c) 2008 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2016-2017 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2016-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * Copyright (c) 2017 Ian Bradley Morgan and Anthony Skjellum. All * rights reserved. @@ -328,21 +328,21 @@ ompi_coll_libnbc_progress(void) /* done, remove and complete */ OPAL_THREAD_LOCK(&mca_coll_libnbc_component.lock); opal_list_remove_item(&mca_coll_libnbc_component.active_requests, - &request->super.super.super); + &request->super.super.super.super); OPAL_THREAD_UNLOCK(&mca_coll_libnbc_component.lock); if( OMPI_SUCCESS == res || NBC_OK == res || NBC_SUCCESS == res ) { - request->super.req_status.MPI_ERROR = OMPI_SUCCESS; + request->super.super.req_status.MPI_ERROR = OMPI_SUCCESS; } else { - request->super.req_status.MPI_ERROR = res; + request->super.super.req_status.MPI_ERROR = res; } - if(request->super.req_persistent) { + if(request->super.super.req_persistent) { /* reset for the next communication */ request->row_offset = 0; } - if(!request->super.req_persistent || !REQUEST_COMPLETE(&request->super)) { - ompi_request_complete(&request->super, true); + if(!request->super.super.req_persistent || !REQUEST_COMPLETE(&request->super.super)) { + ompi_request_complete(&request->super.super, true); } } OPAL_THREAD_LOCK(&mca_coll_libnbc_component.lock); @@ -407,7 +407,7 @@ request_start(size_t count, ompi_request_t ** requests) NBC_DEBUG(5, "tmpbuf address=%p size=%u\n", handle->tmpbuf, sizeof(handle->tmpbuf)); NBC_DEBUG(5, "--------------------------------\n"); - handle->super.req_complete = REQUEST_PENDING; + handle->super.super.req_complete = REQUEST_PENDING; handle->nbc_complete = false; res = NBC_Start(handle); @@ -437,7 +437,7 @@ request_free(struct ompi_request_t **ompi_req) ompi_coll_libnbc_request_t *request = (ompi_coll_libnbc_request_t*) *ompi_req; - if( !REQUEST_COMPLETE(&request->super) ) { + if( !REQUEST_COMPLETE(&request->super.super) ) { return MPI_ERR_REQUEST; } @@ -451,11 +451,11 @@ request_free(struct ompi_request_t **ompi_req) static void request_construct(ompi_coll_libnbc_request_t *request) { - request->super.req_type = OMPI_REQUEST_COLL; - request->super.req_status._cancelled = 0; - request->super.req_start = request_start; - request->super.req_free = request_free; - request->super.req_cancel = request_cancel; + request->super.super.req_type = OMPI_REQUEST_COLL; + request->super.super.req_status._cancelled = 0; + request->super.super.req_start = request_start; + request->super.super.req_free = request_free; + request->super.super.req_cancel = request_cancel; } diff --git a/ompi/mca/coll/libnbc/nbc.c b/ompi/mca/coll/libnbc/nbc.c index 039e6d4c9b..171f5a37e9 100644 --- a/ompi/mca/coll/libnbc/nbc.c +++ b/ompi/mca/coll/libnbc/nbc.c @@ -3,15 +3,15 @@ * Copyright (c) 2006 The Trustees of Indiana University and Indiana * University Research and Technology * Corporation. All rights reserved. - * Copyright (c) 2013 The University of Tennessee and The University + * Copyright (c) 2013-2018 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. * Copyright (c) 2006 The Technical University of Chemnitz. All * rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * * Author(s): Torsten Hoefler * @@ -335,8 +335,14 @@ int NBC_Progress(NBC_Handle *handle) { while (handle->req_count) { ompi_request_t *subreq = handle->req_array[handle->req_count - 1]; if (REQUEST_COMPLETE(subreq)) { - ompi_request_free(&subreq); + if(OPAL_UNLIKELY( OMPI_SUCCESS != subreq->req_status.MPI_ERROR )) { + NBC_Error ("MPI Error in NBC subrequest %p : %d", subreq, subreq->req_status.MPI_ERROR); + /* copy the error code from the underlying request and let the + * round finish */ + handle->super.super.req_status.MPI_ERROR = subreq->req_status.MPI_ERROR; + } handle->req_count--; + ompi_request_free(&subreq); } else { flag = false; break; @@ -349,6 +355,26 @@ int NBC_Progress(NBC_Handle *handle) { /* a round is finished */ if (flag) { + /* reset handle for next round */ + if (NULL != handle->req_array) { + /* free request array */ + free (handle->req_array); + handle->req_array = NULL; + } + + handle->req_count = 0; + + /* previous round had an error */ + if (OPAL_UNLIKELY(OMPI_SUCCESS != handle->super.super.req_status.MPI_ERROR)) { + res = handle->super.super.req_status.MPI_ERROR; + NBC_Error("NBC_Progress: an error %d was found during schedule %p at row-offset %li - aborting the schedule\n", res, handle->schedule, handle->row_offset); + handle->nbc_complete = true; + if (!handle->super.super.req_persistent) { + NBC_Free(handle); + } + return res; + } + /* adjust delim to start of current round */ NBC_DEBUG(5, "NBC_Progress: going in schedule %p to row-offset: %li\n", handle->schedule, handle->row_offset); delim = handle->schedule->data + handle->row_offset; @@ -358,20 +384,12 @@ int NBC_Progress(NBC_Handle *handle) { /* adjust delim to end of current round -> delimiter */ delim = delim + size; - if (NULL != handle->req_array) { - /* free request array */ - free (handle->req_array); - handle->req_array = NULL; - } - - handle->req_count = 0; - if (*delim == 0) { /* this was the last round - we're done */ NBC_DEBUG(5, "NBC_Progress last round finished - we're done\n"); handle->nbc_complete = true; - if (!handle->super.req_persistent) { + if (!handle->super.super.req_persistent) { NBC_Free(handle); } @@ -637,14 +655,15 @@ int NBC_Start(NBC_Handle *handle) { } /* kick off first round */ - handle->super.req_state = OMPI_REQUEST_ACTIVE; + handle->super.super.req_state = OMPI_REQUEST_ACTIVE; + handle->super.super.req_status.MPI_ERROR = OMPI_SUCCESS; res = NBC_Start_round(handle); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { return res; } OPAL_THREAD_LOCK(&mca_coll_libnbc_component.lock); - opal_list_append(&mca_coll_libnbc_component.active_requests, &(handle->super.super.super)); + opal_list_append(&mca_coll_libnbc_component.active_requests, (opal_list_item_t *)handle); OPAL_THREAD_UNLOCK(&mca_coll_libnbc_component.lock); return OMPI_SUCCESS; diff --git a/ompi/mpi/c/iallgather.c b/ompi/mpi/c/iallgather.c index 7d2740b651..8e0abe3fe8 100644 --- a/ompi/mpi/c/iallgather.c +++ b/ompi/mpi/c/iallgather.c @@ -14,8 +14,8 @@ * Copyright (c) 2012 Oak Ridge National Laboratory. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -31,6 +31,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -102,6 +103,9 @@ int MPI_Iallgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, err = comm->c_coll->coll_iallgather(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, request, comm->c_coll->coll_iallgather_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes(*request, (MPI_IN_PLACE==sendbuf)?NULL:sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/iallgatherv.c b/ompi/mpi/c/iallgatherv.c index 0373a15b1d..e743cb9b06 100644 --- a/ompi/mpi/c/iallgatherv.c +++ b/ompi/mpi/c/iallgatherv.c @@ -14,8 +14,8 @@ * Copyright (c) 2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -31,6 +31,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -126,6 +127,9 @@ int MPI_Iallgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype, recvbuf, recvcounts, displs, recvtype, comm, request, comm->c_coll->coll_iallgatherv_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes(*request, (MPI_IN_PLACE==sendbuf)?NULL:sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/iallreduce.c b/ompi/mpi/c/iallreduce.c index d0ea511cf8..bfa968c55b 100644 --- a/ompi/mpi/c/iallreduce.c +++ b/ompi/mpi/c/iallreduce.c @@ -12,8 +12,8 @@ * All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * $COPYRIGHT$ * @@ -31,6 +31,7 @@ #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" #include "ompi/op/op.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -112,10 +113,11 @@ int MPI_Iallreduce(const void *sendbuf, void *recvbuf, int count, /* Invoke the coll component to perform the back-end operation */ - OBJ_RETAIN(op); err = comm->c_coll->coll_iallreduce(sendbuf, recvbuf, count, datatype, op, comm, request, comm->c_coll->coll_iallreduce_module); - OBJ_RELEASE(op); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_op(*request, op, datatype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/ialltoall.c b/ompi/mpi/c/ialltoall.c index 2d46b76f38..0637f29f39 100644 --- a/ompi/mpi/c/ialltoall.c +++ b/ompi/mpi/c/ialltoall.c @@ -14,8 +14,8 @@ * Copyright (c) 2012 Oak Ridge National Laboratory. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2014-2016 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -31,6 +31,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -101,5 +102,8 @@ int MPI_Ialltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype, err = comm->c_coll->coll_ialltoall(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, request, comm->c_coll->coll_ialltoall_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes(*request, (MPI_IN_PLACE==sendbuf)?NULL:sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/ialltoallv.c b/ompi/mpi/c/ialltoallv.c index 577b382894..cef857cdf7 100644 --- a/ompi/mpi/c/ialltoallv.c +++ b/ompi/mpi/c/ialltoallv.c @@ -13,8 +13,8 @@ * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2014-2016 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -30,6 +30,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -130,6 +131,9 @@ int MPI_Ialltoallv(const void *sendbuf, const int sendcounts[], const int sdispl err = comm->c_coll->coll_ialltoallv(sendbuf, sendcounts, sdispls, sendtype, recvbuf, recvcounts, rdispls, recvtype, comm, request, comm->c_coll->coll_ialltoallv_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes(*request, (MPI_IN_PLACE==sendbuf)?NULL:sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/ialltoallw.c b/ompi/mpi/c/ialltoallw.c index b7bc86eaa7..6dc4af8854 100644 --- a/ompi/mpi/c/ialltoallw.c +++ b/ompi/mpi/c/ialltoallw.c @@ -13,8 +13,8 @@ * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2014-2016 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -30,6 +30,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -127,6 +128,9 @@ int MPI_Ialltoallw(const void *sendbuf, const int sendcounts[], const int sdispl sendtypes, recvbuf, recvcounts, rdispls, recvtypes, comm, request, comm->c_coll->coll_ialltoallw_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes_w(*request, (MPI_IN_PLACE==sendbuf)?NULL:sendtypes, recvtypes); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/ibcast.c b/ompi/mpi/c/ibcast.c index 1f049b4c6d..2dcdbb9633 100644 --- a/ompi/mpi/c/ibcast.c +++ b/ompi/mpi/c/ibcast.c @@ -1,7 +1,7 @@ /* * Copyright (c) 2012 Oak Rigde National Laboratory. All rights reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2017-2018 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. @@ -19,6 +19,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -86,5 +87,13 @@ int MPI_Ibcast(void *buffer, int count, MPI_Datatype datatype, err = comm->c_coll->coll_ibcast(buffer, count, datatype, root, comm, request, comm->c_coll->coll_ibcast_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + if (!OMPI_COMM_IS_INTRA(comm)) { + if (MPI_PROC_NULL == root) { + datatype = NULL; + } + } + ompi_coll_base_retain_datatypes(*request, datatype, NULL); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/iexscan.c b/ompi/mpi/c/iexscan.c index 14cf23c590..4c56e08f1e 100644 --- a/ompi/mpi/c/iexscan.c +++ b/ompi/mpi/c/iexscan.c @@ -12,8 +12,8 @@ * All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -29,6 +29,7 @@ #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" #include "ompi/op/op.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -82,10 +83,11 @@ int MPI_Iexscan(const void *sendbuf, void *recvbuf, int count, /* Invoke the coll component to perform the back-end operation */ - OBJ_RETAIN(op); err = comm->c_coll->coll_iexscan(sendbuf, recvbuf, count, datatype, op, comm, request, comm->c_coll->coll_iexscan_module); - OBJ_RELEASE(op); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_op(*request, op, datatype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/igather.c b/ompi/mpi/c/igather.c index 3fcda7e806..c876daa7ec 100644 --- a/ompi/mpi/c/igather.c +++ b/ompi/mpi/c/igather.c @@ -15,8 +15,8 @@ * Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -31,6 +31,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -173,5 +174,24 @@ int MPI_Igather(const void *sendbuf, int sendcount, MPI_Datatype sendtype, err = comm->c_coll->coll_igather(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, request, comm->c_coll->coll_igather_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + if (OMPI_COMM_IS_INTRA(comm)) { + if (MPI_IN_PLACE == sendbuf) { + sendtype = NULL; + } else if (ompi_comm_rank(comm) != root) { + recvtype = NULL; + } + } else { + if (MPI_ROOT == root) { + sendtype = NULL; + } else if (MPI_PROC_NULL == root) { + sendtype = NULL; + recvtype = NULL; + } else { + recvtype = NULL; + } + } + ompi_coll_base_retain_datatypes(*request, sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/igatherv.c b/ompi/mpi/c/igatherv.c index e2deab3cc9..1d575dce4c 100644 --- a/ompi/mpi/c/igatherv.c +++ b/ompi/mpi/c/igatherv.c @@ -13,7 +13,7 @@ * Copyright (c) 2006-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015-2018 Research Organization for Information Science + * Copyright (c) 2015-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * @@ -29,6 +29,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -196,5 +197,24 @@ int MPI_Igatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype, err = comm->c_coll->coll_igatherv(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, root, comm, request, comm->c_coll->coll_igatherv_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + if (OMPI_COMM_IS_INTRA(comm)) { + if (MPI_IN_PLACE == sendbuf) { + sendtype = NULL; + } else if (ompi_comm_rank(comm) != root) { + recvtype = NULL; + } + } else { + if (MPI_ROOT == root) { + sendtype = NULL; + } else if (MPI_PROC_NULL == root) { + sendtype = NULL; + recvtype = NULL; + } else { + recvtype = NULL; + } + } + ompi_coll_base_retain_datatypes(*request, sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/ineighbor_allgather.c b/ompi/mpi/c/ineighbor_allgather.c index 2706ea44d4..cba5b5d4e3 100644 --- a/ompi/mpi/c/ineighbor_allgather.c +++ b/ompi/mpi/c/ineighbor_allgather.c @@ -14,8 +14,8 @@ * Copyright (c) 2012 Oak Rigde National Laboratory. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * $COPYRIGHT$ * @@ -32,6 +32,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/mca/topo/topo.h" #include "ompi/mca/topo/base/base.h" @@ -124,6 +125,9 @@ int MPI_Ineighbor_allgather(const void *sendbuf, int sendcount, MPI_Datatype sen err = comm->c_coll->coll_ineighbor_allgather(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, request, comm->c_coll->coll_ineighbor_allgather_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes(*request, sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/ineighbor_allgatherv.c b/ompi/mpi/c/ineighbor_allgatherv.c index 2f3c244064..58dedb6105 100644 --- a/ompi/mpi/c/ineighbor_allgatherv.c +++ b/ompi/mpi/c/ineighbor_allgatherv.c @@ -14,8 +14,8 @@ * Copyright (c) 2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * $COPYRIGHT$ * @@ -32,6 +32,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/mca/topo/topo.h" #include "ompi/mca/topo/base/base.h" @@ -147,6 +148,9 @@ int MPI_Ineighbor_allgatherv(const void *sendbuf, int sendcount, MPI_Datatype se recvbuf, (int *) recvcounts, (int *) displs, recvtype, comm, request, comm->c_coll->coll_ineighbor_allgatherv_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes(*request, sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/ineighbor_alltoall.c b/ompi/mpi/c/ineighbor_alltoall.c index b3d0846421..b03b7cc50f 100644 --- a/ompi/mpi/c/ineighbor_alltoall.c +++ b/ompi/mpi/c/ineighbor_alltoall.c @@ -14,8 +14,8 @@ * Copyright (c) 2012 Oak Ridge National Laboratory. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2014-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * $COPYRIGHT$ * @@ -32,6 +32,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/mca/topo/topo.h" #include "ompi/mca/topo/base/base.h" @@ -124,5 +125,8 @@ int MPI_Ineighbor_alltoall(const void *sendbuf, int sendcount, MPI_Datatype send err = comm->c_coll->coll_ineighbor_alltoall(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, request, comm->c_coll->coll_ineighbor_alltoall_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes(*request, sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/ineighbor_alltoallv.c b/ompi/mpi/c/ineighbor_alltoallv.c index 9645e15b05..a44d081e10 100644 --- a/ompi/mpi/c/ineighbor_alltoallv.c +++ b/ompi/mpi/c/ineighbor_alltoallv.c @@ -31,6 +31,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/mca/topo/topo.h" #include "ompi/mca/topo/base/base.h" @@ -147,6 +148,9 @@ int MPI_Ineighbor_alltoallv(const void *sendbuf, const int sendcounts[], const i err = comm->c_coll->coll_ineighbor_alltoallv(sendbuf, sendcounts, sdispls, sendtype, recvbuf, recvcounts, rdispls, recvtype, comm, request, comm->c_coll->coll_ineighbor_alltoallv_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes(*request, sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/ineighbor_alltoallw.c b/ompi/mpi/c/ineighbor_alltoallw.c index 150f28d717..efb4d24f5f 100644 --- a/ompi/mpi/c/ineighbor_alltoallw.c +++ b/ompi/mpi/c/ineighbor_alltoallw.c @@ -31,6 +31,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/mca/topo/topo.h" #include "ompi/mca/topo/base/base.h" @@ -147,6 +148,9 @@ int MPI_Ineighbor_alltoallw(const void *sendbuf, const int sendcounts[], const M err = comm->c_coll->coll_ineighbor_alltoallw(sendbuf, sendcounts, sdispls, sendtypes, recvbuf, recvcounts, rdispls, recvtypes, comm, request, comm->c_coll->coll_ineighbor_alltoallw_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes_w(*request, sendtypes, recvtypes); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/ireduce.c b/ompi/mpi/c/ireduce.c index 4794888782..be552250fc 100644 --- a/ompi/mpi/c/ireduce.c +++ b/ompi/mpi/c/ireduce.c @@ -13,8 +13,8 @@ * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * $COPYRIGHT$ * @@ -31,6 +31,7 @@ #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" #include "ompi/op/op.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -136,10 +137,11 @@ int MPI_Ireduce(const void *sendbuf, void *recvbuf, int count, OPAL_CR_ENTER_LIBRARY(); /* Invoke the coll component to perform the back-end operation */ - OBJ_RETAIN(op); err = comm->c_coll->coll_ireduce(sendbuf, recvbuf, count, datatype, op, root, comm, request, comm->c_coll->coll_ireduce_module); - OBJ_RELEASE(op); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_op(*request, op, datatype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/ireduce_scatter.c b/ompi/mpi/c/ireduce_scatter.c index 211b217971..56525fa19f 100644 --- a/ompi/mpi/c/ireduce_scatter.c +++ b/ompi/mpi/c/ireduce_scatter.c @@ -13,8 +13,8 @@ * Copyright (c) 2006-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * $COPYRIGHT$ * @@ -31,6 +31,7 @@ #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" #include "ompi/op/op.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -133,10 +134,11 @@ int MPI_Ireduce_scatter(const void *sendbuf, void *recvbuf, const int recvcounts /* Invoke the coll component to perform the back-end operation */ - OBJ_RETAIN(op); err = comm->c_coll->coll_ireduce_scatter(sendbuf, recvbuf, recvcounts, datatype, op, comm, request, comm->c_coll->coll_ireduce_scatter_module); - OBJ_RELEASE(op); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_op(*request, op, datatype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/ireduce_scatter_block.c b/ompi/mpi/c/ireduce_scatter_block.c index ded4abf223..ce43ab3cd4 100644 --- a/ompi/mpi/c/ireduce_scatter_block.c +++ b/ompi/mpi/c/ireduce_scatter_block.c @@ -14,8 +14,8 @@ * Copyright (c) 2012 Oak Ridge National Labs. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -31,6 +31,7 @@ #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" #include "ompi/op/op.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -100,10 +101,11 @@ int MPI_Ireduce_scatter_block(const void *sendbuf, void *recvbuf, int recvcount, /* Invoke the coll component to perform the back-end operation */ - OBJ_RETAIN(op); err = comm->c_coll->coll_ireduce_scatter_block(sendbuf, recvbuf, recvcount, datatype, op, comm, request, comm->c_coll->coll_ireduce_scatter_block_module); - OBJ_RELEASE(op); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_op(*request, op, datatype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/iscan.c b/ompi/mpi/c/iscan.c index 34502b8e36..cfae0ff409 100644 --- a/ompi/mpi/c/iscan.c +++ b/ompi/mpi/c/iscan.c @@ -13,8 +13,8 @@ * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -30,6 +30,7 @@ #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" #include "ompi/op/op.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -96,11 +97,12 @@ int MPI_Iscan(const void *sendbuf, void *recvbuf, int count, /* Call the coll component to actually perform the allgather */ - OBJ_RETAIN(op); err = comm->c_coll->coll_iscan(sendbuf, recvbuf, count, datatype, op, comm, request, comm->c_coll->coll_iscan_module); - OBJ_RELEASE(op); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_op(*request, op, datatype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/iscatter.c b/ompi/mpi/c/iscatter.c index 79a22d57a5..3357ad2115 100644 --- a/ompi/mpi/c/iscatter.c +++ b/ompi/mpi/c/iscatter.c @@ -15,8 +15,8 @@ * Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -31,6 +31,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -156,5 +157,24 @@ int MPI_Iscatter(const void *sendbuf, int sendcount, MPI_Datatype sendtype, err = comm->c_coll->coll_iscatter(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, request, comm->c_coll->coll_iscatter_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + if (OMPI_COMM_IS_INTRA(comm)) { + if (MPI_IN_PLACE == recvbuf) { + recvtype = NULL; + } else if (ompi_comm_rank(comm) != root) { + sendtype = NULL; + } + } else { + if (MPI_ROOT == root) { + recvtype = NULL; + } else if (MPI_PROC_NULL == root) { + sendtype = NULL; + recvtype = NULL; + } else { + sendtype = NULL; + } + } + ompi_coll_base_retain_datatypes(*request, sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpi/c/iscatterv.c b/ompi/mpi/c/iscatterv.c index 66ae9003ca..2d164662f4 100644 --- a/ompi/mpi/c/iscatterv.c +++ b/ompi/mpi/c/iscatterv.c @@ -13,8 +13,8 @@ * Copyright (c) 2006-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -29,6 +29,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -196,5 +197,24 @@ int MPI_Iscatterv(const void *sendbuf, const int sendcounts[], const int displs[ err = comm->c_coll->coll_iscatterv(sendbuf, sendcounts, displs, sendtype, recvbuf, recvcount, recvtype, root, comm, request, comm->c_coll->coll_iscatterv_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + if (OMPI_COMM_IS_INTRA(comm)) { + if (MPI_IN_PLACE == recvbuf) { + recvtype = NULL; + } else if (ompi_comm_rank(comm) != root) { + sendtype = NULL; + } + } else { + if (MPI_ROOT == root) { + recvtype = NULL; + } else if (MPI_PROC_NULL == root) { + sendtype = NULL; + recvtype = NULL; + } else { + sendtype = NULL; + } + } + ompi_coll_base_retain_datatypes(*request, sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpiext/pcollreq/c/allgather_init.c b/ompi/mpiext/pcollreq/c/allgather_init.c index 46a568bc65..4b699f91a1 100644 --- a/ompi/mpiext/pcollreq/c/allgather_init.c +++ b/ompi/mpiext/pcollreq/c/allgather_init.c @@ -14,8 +14,8 @@ * Copyright (c) 2012 Oak Ridge National Laboratory. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -31,6 +31,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h" #include "ompi/runtime/ompi_spc.h" @@ -103,6 +104,9 @@ int MPIX_Allgather_init(const void *sendbuf, int sendcount, MPI_Datatype sendtyp err = comm->c_coll->coll_allgather_init(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, info, request, comm->c_coll->coll_allgather_init_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes(*request, (MPI_IN_PLACE==sendbuf)?NULL:sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpiext/pcollreq/c/allgatherv_init.c b/ompi/mpiext/pcollreq/c/allgatherv_init.c index d4b3c7368a..2021ab9668 100644 --- a/ompi/mpiext/pcollreq/c/allgatherv_init.c +++ b/ompi/mpiext/pcollreq/c/allgatherv_init.c @@ -14,8 +14,8 @@ * Copyright (c) 2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -31,6 +31,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h" #include "ompi/runtime/ompi_spc.h" @@ -128,6 +129,9 @@ int MPIX_Allgatherv_init(const void *sendbuf, int sendcount, MPI_Datatype sendty recvbuf, recvcounts, displs, recvtype, comm, info, request, comm->c_coll->coll_allgatherv_init_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes(*request, (MPI_IN_PLACE==sendbuf)?NULL:sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpiext/pcollreq/c/allreduce_init.c b/ompi/mpiext/pcollreq/c/allreduce_init.c index dd224f1115..1213395f3e 100644 --- a/ompi/mpiext/pcollreq/c/allreduce_init.c +++ b/ompi/mpiext/pcollreq/c/allreduce_init.c @@ -12,8 +12,8 @@ * All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * Copyright (c) 2018 FUJITSU LIMITED. All rights reserved. * $COPYRIGHT$ @@ -32,6 +32,7 @@ #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" #include "ompi/op/op.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h" #include "ompi/runtime/ompi_spc.h" @@ -115,9 +116,10 @@ int MPIX_Allreduce_init(const void *sendbuf, void *recvbuf, int count, /* Invoke the coll component to perform the back-end operation */ - OBJ_RETAIN(op); err = comm->c_coll->coll_allreduce_init(sendbuf, recvbuf, count, datatype, op, comm, info, request, comm->c_coll->coll_allreduce_init_module); - OBJ_RELEASE(op); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_op(*request, op, datatype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpiext/pcollreq/c/alltoall_init.c b/ompi/mpiext/pcollreq/c/alltoall_init.c index b176f63c75..7cb3621647 100644 --- a/ompi/mpiext/pcollreq/c/alltoall_init.c +++ b/ompi/mpiext/pcollreq/c/alltoall_init.c @@ -14,8 +14,8 @@ * Copyright (c) 2012 Oak Ridge National Laboratory. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2014-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -31,6 +31,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h" #include "ompi/runtime/ompi_spc.h" @@ -102,5 +103,8 @@ int MPIX_Alltoall_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype err = comm->c_coll->coll_alltoall_init(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, info, request, comm->c_coll->coll_alltoall_init_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes(*request, (MPI_IN_PLACE==sendbuf)?NULL:sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpiext/pcollreq/c/alltoallv_init.c b/ompi/mpiext/pcollreq/c/alltoallv_init.c index 06d5922b2a..3d34536fb0 100644 --- a/ompi/mpiext/pcollreq/c/alltoallv_init.c +++ b/ompi/mpiext/pcollreq/c/alltoallv_init.c @@ -13,8 +13,8 @@ * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2014-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -30,6 +30,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h" #include "ompi/runtime/ompi_spc.h" @@ -131,6 +132,9 @@ int MPIX_Alltoallv_init(const void *sendbuf, const int sendcounts[], const int s err = comm->c_coll->coll_alltoallv_init(sendbuf, sendcounts, sdispls, sendtype, recvbuf, recvcounts, rdispls, recvtype, comm, info, request, comm->c_coll->coll_alltoallv_init_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes(*request, (MPI_IN_PLACE==sendbuf)?NULL:sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpiext/pcollreq/c/alltoallw_init.c b/ompi/mpiext/pcollreq/c/alltoallw_init.c index 405cc4c4f8..50902f1f63 100644 --- a/ompi/mpiext/pcollreq/c/alltoallw_init.c +++ b/ompi/mpiext/pcollreq/c/alltoallw_init.c @@ -13,8 +13,8 @@ * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2014-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -30,6 +30,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h" #include "ompi/runtime/ompi_spc.h" @@ -128,6 +129,9 @@ int MPIX_Alltoallw_init(const void *sendbuf, const int sendcounts[], const int s sendtypes, recvbuf, recvcounts, rdispls, recvtypes, comm, info, request, comm->c_coll->coll_alltoallw_init_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes_w(*request, (MPI_IN_PLACE==sendbuf)?NULL:sendtypes, recvtypes); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpiext/pcollreq/c/bcast_init.c b/ompi/mpiext/pcollreq/c/bcast_init.c index 6a2798a970..9cf71a7a67 100644 --- a/ompi/mpiext/pcollreq/c/bcast_init.c +++ b/ompi/mpiext/pcollreq/c/bcast_init.c @@ -1,7 +1,7 @@ /* * Copyright (c) 2012 Oak Rigde National Laboratory. All rights reserved. - * Copyright (c) 2015-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2017-2018 The University of Tennessee and The University * of Tennessee Research Foundation. All rights * reserved. @@ -19,6 +19,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h" #include "ompi/runtime/ompi_spc.h" @@ -87,5 +88,13 @@ int MPIX_Bcast_init(void *buffer, int count, MPI_Datatype datatype, err = comm->c_coll->coll_bcast_init(buffer, count, datatype, root, comm, info, request, comm->c_coll->coll_bcast_init_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + if (!OMPI_COMM_IS_INTRA(comm)) { + if (MPI_PROC_NULL == root) { + datatype = NULL; + } + } + ompi_coll_base_retain_datatypes(*request, datatype, NULL); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpiext/pcollreq/c/exscan_init.c b/ompi/mpiext/pcollreq/c/exscan_init.c index 23f155429c..f8e34ced68 100644 --- a/ompi/mpiext/pcollreq/c/exscan_init.c +++ b/ompi/mpiext/pcollreq/c/exscan_init.c @@ -12,8 +12,8 @@ * All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -29,6 +29,7 @@ #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" #include "ompi/op/op.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -84,10 +85,11 @@ int MPIX_Exscan_init(const void *sendbuf, void *recvbuf, int count, /* Invoke the coll component to perform the back-end operation */ - OBJ_RETAIN(op); err = comm->c_coll->coll_exscan_init(sendbuf, recvbuf, count, datatype, op, comm, info, request, comm->c_coll->coll_exscan_init_module); - OBJ_RELEASE(op); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_op(*request, op, datatype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpiext/pcollreq/c/gather_init.c b/ompi/mpiext/pcollreq/c/gather_init.c index f62dd9b54d..051a0eaa13 100644 --- a/ompi/mpiext/pcollreq/c/gather_init.c +++ b/ompi/mpiext/pcollreq/c/gather_init.c @@ -15,8 +15,8 @@ * Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -31,6 +31,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h" #include "ompi/runtime/ompi_spc.h" @@ -174,5 +175,24 @@ int MPIX_Gather_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype, err = comm->c_coll->coll_gather_init(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, info, request, comm->c_coll->coll_gather_init_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + if (OMPI_COMM_IS_INTRA(comm)) { + if (MPI_IN_PLACE == sendbuf) { + sendtype = NULL; + } else if (ompi_comm_rank(comm) != root) { + recvtype = NULL; + } + } else { + if (MPI_ROOT == root) { + sendtype = NULL; + } else if (MPI_PROC_NULL == root) { + sendtype = NULL; + recvtype = NULL; + } else { + recvtype = NULL; + } + } + ompi_coll_base_retain_datatypes(*request, sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpiext/pcollreq/c/gatherv_init.c b/ompi/mpiext/pcollreq/c/gatherv_init.c index fbbd346008..bd875a051c 100644 --- a/ompi/mpiext/pcollreq/c/gatherv_init.c +++ b/ompi/mpiext/pcollreq/c/gatherv_init.c @@ -13,8 +13,8 @@ * Copyright (c) 2006-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -29,6 +29,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h" #include "ompi/memchecker.h" #include "ompi/runtime/ompi_spc.h" @@ -199,5 +200,24 @@ int MPIX_Gatherv_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype, recvcounts, displs, recvtype, root, comm, info, request, comm->c_coll->coll_gatherv_init_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + if (OMPI_COMM_IS_INTRA(comm)) { + if (MPI_IN_PLACE == sendbuf) { + sendtype = NULL; + } else if (ompi_comm_rank(comm) != root) { + recvtype = NULL; + } + } else { + if (MPI_ROOT == root) { + sendtype = NULL; + } else if (MPI_PROC_NULL == root) { + sendtype = NULL; + recvtype = NULL; + } else { + recvtype = NULL; + } + } + ompi_coll_base_retain_datatypes(*request, sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpiext/pcollreq/c/neighbor_allgather_init.c b/ompi/mpiext/pcollreq/c/neighbor_allgather_init.c index 4494b507b7..cd3037d0bd 100644 --- a/ompi/mpiext/pcollreq/c/neighbor_allgather_init.c +++ b/ompi/mpiext/pcollreq/c/neighbor_allgather_init.c @@ -14,8 +14,8 @@ * Copyright (c) 2012 Oak Rigde National Laboratory. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * $COPYRIGHT$ * @@ -32,6 +32,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/mca/topo/topo.h" #include "ompi/mca/topo/base/base.h" @@ -125,6 +126,9 @@ int MPIX_Neighbor_allgather_init(const void *sendbuf, int sendcount, MPI_Datatyp err = comm->c_coll->coll_neighbor_allgather_init(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, comm, info, request, comm->c_coll->coll_neighbor_allgather_init_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes(*request, sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpiext/pcollreq/c/neighbor_allgatherv_init.c b/ompi/mpiext/pcollreq/c/neighbor_allgatherv_init.c index 66fa0487c5..3e53b84631 100644 --- a/ompi/mpiext/pcollreq/c/neighbor_allgatherv_init.c +++ b/ompi/mpiext/pcollreq/c/neighbor_allgatherv_init.c @@ -14,8 +14,8 @@ * Copyright (c) 2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * $COPYRIGHT$ * @@ -32,6 +32,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/mca/topo/topo.h" #include "ompi/mca/topo/base/base.h" @@ -149,6 +150,9 @@ int MPIX_Neighbor_allgatherv_init(const void *sendbuf, int sendcount, MPI_Dataty recvbuf, (int *) recvcounts, (int *) displs, recvtype, comm, info, request, comm->c_coll->coll_neighbor_allgatherv_init_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes(*request, sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpiext/pcollreq/c/neighbor_alltoall_init.c b/ompi/mpiext/pcollreq/c/neighbor_alltoall_init.c index c564ee7e9e..c2b0ac3c19 100644 --- a/ompi/mpiext/pcollreq/c/neighbor_alltoall_init.c +++ b/ompi/mpiext/pcollreq/c/neighbor_alltoall_init.c @@ -14,8 +14,8 @@ * Copyright (c) 2012 Oak Ridge National Laboratory. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2014-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * $COPYRIGHT$ * @@ -32,6 +32,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/mca/topo/topo.h" #include "ompi/mca/topo/base/base.h" @@ -126,5 +127,8 @@ int MPIX_Neighbor_alltoall_init(const void *sendbuf, int sendcount, MPI_Datatype recvbuf, recvcount, recvtype, comm, info, request, comm->c_coll->coll_neighbor_alltoall_init_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes(*request, sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpiext/pcollreq/c/neighbor_alltoallv_init.c b/ompi/mpiext/pcollreq/c/neighbor_alltoallv_init.c index 8d3503bf57..f86e256d81 100644 --- a/ompi/mpiext/pcollreq/c/neighbor_alltoallv_init.c +++ b/ompi/mpiext/pcollreq/c/neighbor_alltoallv_init.c @@ -13,8 +13,8 @@ * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2014-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * $COPYRIGHT$ * @@ -31,6 +31,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/mca/topo/topo.h" #include "ompi/mca/topo/base/base.h" @@ -149,6 +150,9 @@ int MPIX_Neighbor_alltoallv_init(const void *sendbuf, const int sendcounts[], co sendtype, recvbuf, recvcounts, rdispls, recvtype, comm, info, request, comm->c_coll->coll_neighbor_alltoallv_init_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes(*request, sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpiext/pcollreq/c/neighbor_alltoallw_init.c b/ompi/mpiext/pcollreq/c/neighbor_alltoallw_init.c index 68e2b2cad2..1143ccbb3c 100644 --- a/ompi/mpiext/pcollreq/c/neighbor_alltoallw_init.c +++ b/ompi/mpiext/pcollreq/c/neighbor_alltoallw_init.c @@ -13,8 +13,8 @@ * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2017 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2014-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2017 IBM Corporation. All rights reserved. * $COPYRIGHT$ * @@ -31,6 +31,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/mca/topo/topo.h" #include "ompi/mca/topo/base/base.h" @@ -149,6 +150,9 @@ int MPIX_Neighbor_alltoallw_init(const void *sendbuf, const int sendcounts[], co recvbuf, recvcounts, rdispls, recvtypes, comm, info, request, comm->c_coll->coll_neighbor_alltoallw_init_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_datatypes_w(*request, sendtypes, recvtypes); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpiext/pcollreq/c/reduce_init.c b/ompi/mpiext/pcollreq/c/reduce_init.c index 1e72877d50..d3b50747bf 100644 --- a/ompi/mpiext/pcollreq/c/reduce_init.c +++ b/ompi/mpiext/pcollreq/c/reduce_init.c @@ -13,8 +13,8 @@ * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * Copyright (c) 2018 FUJITSU LIMITED. All rights reserved. * $COPYRIGHT$ @@ -32,6 +32,7 @@ #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" #include "ompi/op/op.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h" #include "ompi/runtime/ompi_spc.h" @@ -139,10 +140,11 @@ int MPIX_Reduce_init(const void *sendbuf, void *recvbuf, int count, OPAL_CR_ENTER_LIBRARY(); /* Invoke the coll component to perform the back-end operation */ - OBJ_RETAIN(op); err = comm->c_coll->coll_reduce_init(sendbuf, recvbuf, count, datatype, op, root, comm, info, request, comm->c_coll->coll_reduce_init_module); - OBJ_RELEASE(op); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_op(*request, op, datatype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpiext/pcollreq/c/reduce_scatter_block_init.c b/ompi/mpiext/pcollreq/c/reduce_scatter_block_init.c index ef000ae6e1..c0b8c344e6 100644 --- a/ompi/mpiext/pcollreq/c/reduce_scatter_block_init.c +++ b/ompi/mpiext/pcollreq/c/reduce_scatter_block_init.c @@ -14,8 +14,8 @@ * Copyright (c) 2012 Oak Ridge National Labs. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -31,6 +31,7 @@ #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" #include "ompi/op/op.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h" #include "ompi/runtime/ompi_spc.h" @@ -101,10 +102,11 @@ int MPIX_Reduce_scatter_block_init(const void *sendbuf, void *recvbuf, int recvc /* Invoke the coll component to perform the back-end operation */ - OBJ_RETAIN(op); err = comm->c_coll->coll_reduce_scatter_block_init(sendbuf, recvbuf, recvcount, datatype, op, comm, info, request, comm->c_coll->coll_reduce_scatter_block_init_module); - OBJ_RELEASE(op); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_op(*request, op, datatype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpiext/pcollreq/c/reduce_scatter_init.c b/ompi/mpiext/pcollreq/c/reduce_scatter_init.c index b8c470f064..5bf5712e3e 100644 --- a/ompi/mpiext/pcollreq/c/reduce_scatter_init.c +++ b/ompi/mpiext/pcollreq/c/reduce_scatter_init.c @@ -13,8 +13,8 @@ * Copyright (c) 2006-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2016 IBM Corporation. All rights reserved. * Copyright (c) 2018 FUJITSU LIMITED. All rights reserved. * $COPYRIGHT$ @@ -32,6 +32,7 @@ #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" #include "ompi/op/op.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h" #include "ompi/runtime/ompi_spc.h" @@ -135,10 +136,11 @@ int MPIX_Reduce_scatter_init(const void *sendbuf, void *recvbuf, const int recvc /* Invoke the coll component to perform the back-end operation */ - OBJ_RETAIN(op); err = comm->c_coll->coll_reduce_scatter_init(sendbuf, recvbuf, recvcounts, datatype, op, comm, info, request, comm->c_coll->coll_reduce_scatter_init_module); - OBJ_RELEASE(op); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_op(*request, op, datatype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpiext/pcollreq/c/scan_init.c b/ompi/mpiext/pcollreq/c/scan_init.c index 8ff34dd5f5..35540c1a10 100644 --- a/ompi/mpiext/pcollreq/c/scan_init.c +++ b/ompi/mpiext/pcollreq/c/scan_init.c @@ -13,8 +13,8 @@ * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -30,6 +30,7 @@ #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" #include "ompi/op/op.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h" #include "ompi/runtime/ompi_spc.h" @@ -98,11 +99,12 @@ int MPIX_Scan_init(const void *sendbuf, void *recvbuf, int count, /* Call the coll component to actually perform the allgather */ - OBJ_RETAIN(op); err = comm->c_coll->coll_scan_init(sendbuf, recvbuf, count, datatype, op, comm, info, request, comm->c_coll->coll_scan_init_module); - OBJ_RELEASE(op); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + ompi_coll_base_retain_op(*request, op, datatype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpiext/pcollreq/c/scatter_init.c b/ompi/mpiext/pcollreq/c/scatter_init.c index 30ee31f88d..7ab7700c62 100644 --- a/ompi/mpiext/pcollreq/c/scatter_init.c +++ b/ompi/mpiext/pcollreq/c/scatter_init.c @@ -15,8 +15,8 @@ * Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -31,6 +31,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h" #include "ompi/runtime/ompi_spc.h" @@ -157,5 +158,24 @@ int MPIX_Scatter_init(const void *sendbuf, int sendcount, MPI_Datatype sendtype, err = comm->c_coll->coll_scatter_init(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, info, request, comm->c_coll->coll_scatter_init_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + if (OMPI_COMM_IS_INTRA(comm)) { + if (MPI_IN_PLACE == recvbuf) { + recvtype = NULL; + } else if (ompi_comm_rank(comm) != root) { + sendtype = NULL; + } + } else { + if (MPI_ROOT == root) { + recvtype = NULL; + } else if (MPI_PROC_NULL == root) { + sendtype = NULL; + recvtype = NULL; + } else { + sendtype = NULL; + } + } + ompi_coll_base_retain_datatypes(*request, sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); } diff --git a/ompi/mpiext/pcollreq/c/scatterv_init.c b/ompi/mpiext/pcollreq/c/scatterv_init.c index fef368caf7..d2d53c7fd9 100644 --- a/ompi/mpiext/pcollreq/c/scatterv_init.c +++ b/ompi/mpiext/pcollreq/c/scatterv_init.c @@ -13,8 +13,8 @@ * Copyright (c) 2006-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2015-2018 Research Organization for Information Science - * and Technology (RIST). All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -29,6 +29,7 @@ #include "ompi/communicator/communicator.h" #include "ompi/errhandler/errhandler.h" #include "ompi/datatype/ompi_datatype.h" +#include "ompi/mca/coll/base/coll_base_util.h" #include "ompi/memchecker.h" #include "ompi/mpiext/pcollreq/c/mpiext_pcollreq_c.h" #include "ompi/runtime/ompi_spc.h" @@ -197,5 +198,24 @@ int MPIX_Scatterv_init(const void *sendbuf, const int sendcounts[], const int di err = comm->c_coll->coll_scatterv_init(sendbuf, sendcounts, displs, sendtype, recvbuf, recvcount, recvtype, root, comm, info, request, comm->c_coll->coll_scatterv_init_module); + if (OPAL_LIKELY(OMPI_SUCCESS == err)) { + if (OMPI_COMM_IS_INTRA(comm)) { + if (MPI_IN_PLACE == recvbuf) { + recvtype = NULL; + } else if (ompi_comm_rank(comm) != root) { + sendtype = NULL; + } + } else { + if (MPI_ROOT == root) { + recvtype = NULL; + } else if (MPI_PROC_NULL == root) { + sendtype = NULL; + recvtype = NULL; + } else { + sendtype = NULL; + } + } + ompi_coll_base_retain_datatypes(*request, sendtype, recvtype); + } OMPI_ERRHANDLER_RETURN(err, comm, err, FUNC_NAME); }