From c753e9baff865a5c9c53b3a50e8e41d6c1b39f09 Mon Sep 17 00:00:00 2001 From: Gilles Gouaillardet Date: Tue, 12 Dec 2017 14:42:21 +0900 Subject: [PATCH] coll/libnbc: code refactoring prepare the upcoming persistent collectives by pre-factoring some code Signed-off-by: Gilles Gouaillardet fixup 808c3c62cd9475edd91ecde9d2d53b12e28b2c04 --- ompi/mca/coll/libnbc/nbc.c | 13 +- ompi/mca/coll/libnbc/nbc_iallgather.c | 52 +++++- ompi/mca/coll/libnbc/nbc_iallgatherv.c | 50 ++++- ompi/mca/coll/libnbc/nbc_iallreduce.c | 47 ++++- ompi/mca/coll/libnbc/nbc_ialltoall.c | 50 ++++- ompi/mca/coll/libnbc/nbc_ialltoallv.c | 58 +++++- ompi/mca/coll/libnbc/nbc_ialltoallw.c | 58 +++++- ompi/mca/coll/libnbc/nbc_ibarrier.c | 42 ++++- ompi/mca/coll/libnbc/nbc_ibcast.c | 48 ++++- ompi/mca/coll/libnbc/nbc_iexscan.c | 24 ++- ompi/mca/coll/libnbc/nbc_igather.c | 56 +++++- ompi/mca/coll/libnbc/nbc_igatherv.c | 56 +++++- .../mca/coll/libnbc/nbc_ineighbor_allgather.c | 175 +++++++++++++++++- .../coll/libnbc/nbc_ineighbor_allgatherv.c | 26 ++- ompi/mca/coll/libnbc/nbc_ineighbor_alltoall.c | 23 ++- .../mca/coll/libnbc/nbc_ineighbor_alltoallv.c | 26 ++- .../mca/coll/libnbc/nbc_ineighbor_alltoallw.c | 26 ++- ompi/mca/coll/libnbc/nbc_ireduce.c | 46 ++++- ompi/mca/coll/libnbc/nbc_ireduce_scatter.c | 45 ++++- .../coll/libnbc/nbc_ireduce_scatter_block.c | 46 ++++- ompi/mca/coll/libnbc/nbc_iscan.c | 23 ++- ompi/mca/coll/libnbc/nbc_iscatter.c | 52 +++++- ompi/mca/coll/libnbc/nbc_iscatterv.c | 52 +++++- 23 files changed, 962 insertions(+), 132 deletions(-) diff --git a/ompi/mca/coll/libnbc/nbc.c b/ompi/mca/coll/libnbc/nbc.c index 8ab54b7a57..662247976e 100644 --- a/ompi/mca/coll/libnbc/nbc.c +++ b/ompi/mca/coll/libnbc/nbc.c @@ -641,6 +641,10 @@ int NBC_Init_comm(MPI_Comm comm, NBC_Comminfo *comminfo) { int NBC_Start(NBC_Handle *handle) { int res; + /* bozo case */ + if ((ompi_request_t *)handle == &ompi_request_empty) { + return OMPI_SUCCESS; + } /* kick off first round */ res = NBC_Start_round(handle); if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { @@ -654,7 +658,6 @@ int NBC_Start(NBC_Handle *handle) { } int NBC_Schedule_request(NBC_Schedule *schedule, ompi_communicator_t *comm, ompi_coll_libnbc_module_t *module, ompi_request_t **request, void *tmpbuf) { - int res; int tmp_tag; bool need_register = false; ompi_coll_libnbc_request_t *handle; @@ -705,14 +708,8 @@ int NBC_Schedule_request(NBC_Schedule *schedule, ompi_communicator_t *comm, ompi handle->tmpbuf = tmpbuf; handle->schedule = schedule; - - res = NBC_Start (handle); - if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { - NBC_Return_handle (handle); - return res; - } - *request = (ompi_request_t *) handle; + return OMPI_SUCCESS; } diff --git a/ompi/mca/coll/libnbc/nbc_iallgather.c b/ompi/mca/coll/libnbc/nbc_iallgather.c index dd20b7a40f..0b2b721698 100644 --- a/ompi/mca/coll/libnbc/nbc_iallgather.c +++ b/ompi/mca/coll/libnbc/nbc_iallgather.c @@ -43,9 +43,9 @@ int NBC_Allgather_args_compare(NBC_Allgather_args *a, NBC_Allgather_args *b, voi * the algorithm uses p-1 rounds * each node sends the packet it received last round (or has in round 0) to it's right neighbor (modulo p) * each node receives from it's left (modulo p) neighbor */ -int ompi_coll_libnbc_iallgather(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount, - MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) +static int nbc_iallgather(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount, + MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int rank, p, res; MPI_Aint rcvext; @@ -163,9 +163,29 @@ int ompi_coll_libnbc_iallgather(const void* sendbuf, int sendcount, MPI_Datatype return OMPI_SUCCESS; } -int ompi_coll_libnbc_iallgather_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount, - MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) +int ompi_coll_libnbc_iallgather(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount, + MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) +{ + int res = nbc_iallgather(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, + comm, request, module); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + return res; + } + + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle (*(ompi_coll_libnbc_request_t **)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} + +static int nbc_iallgather_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount, + MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int res, rsize; MPI_Aint rcvext; @@ -219,3 +239,23 @@ int ompi_coll_libnbc_iallgather_inter(const void* sendbuf, int sendcount, MPI_Da return OMPI_SUCCESS; } + +int ompi_coll_libnbc_iallgather_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount, + MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_iallgather_inter(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, + comm, request, module); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + return res; + } + + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle (*(ompi_coll_libnbc_request_t **)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} + diff --git a/ompi/mca/coll/libnbc/nbc_iallgatherv.c b/ompi/mca/coll/libnbc/nbc_iallgatherv.c index ac711c6e87..9a93227650 100644 --- a/ompi/mca/coll/libnbc/nbc_iallgatherv.c +++ b/ompi/mca/coll/libnbc/nbc_iallgatherv.c @@ -33,9 +33,9 @@ * second round: * each node sends to node (rank+2)%p sendcount elements * each node receives from node (rank-2)%p recvcounts[(rank+2)%p] elements */ -int ompi_coll_libnbc_iallgatherv(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs, - MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) +static int nbc_iallgatherv(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs, + MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int rank, p, res, speer, rpeer; MPI_Aint rcvext; @@ -108,9 +108,28 @@ int ompi_coll_libnbc_iallgatherv(const void* sendbuf, int sendcount, MPI_Datatyp return OMPI_SUCCESS; } -int ompi_coll_libnbc_iallgatherv_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs, - MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) +int ompi_coll_libnbc_iallgatherv(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs, + MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_iallgatherv(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, + comm, request, module); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + return res; + } + + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} + +static int nbc_iallgatherv_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs, + MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int res, rsize; MPI_Aint rcvext; @@ -167,3 +186,22 @@ int ompi_coll_libnbc_iallgatherv_inter(const void* sendbuf, int sendcount, MPI_D return OMPI_SUCCESS; } + +int ompi_coll_libnbc_iallgatherv_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs, + MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_iallgatherv_inter(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, + comm, request, module); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + return res; + } + + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} diff --git a/ompi/mca/coll/libnbc/nbc_iallreduce.c b/ompi/mca/coll/libnbc/nbc_iallreduce.c index 1a1e17039c..ccf8e9b929 100644 --- a/ompi/mca/coll/libnbc/nbc_iallreduce.c +++ b/ompi/mca/coll/libnbc/nbc_iallreduce.c @@ -52,9 +52,9 @@ int NBC_Allreduce_args_compare(NBC_Allreduce_args *a, NBC_Allreduce_args *b, voi } #endif -int ompi_coll_libnbc_iallreduce(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op, - struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) +static int nbc_iallreduce(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op, + struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int rank, p, res; ptrdiff_t ext, lb; @@ -190,9 +190,27 @@ int ompi_coll_libnbc_iallreduce(const void* sendbuf, void* recvbuf, int count, M return OMPI_SUCCESS; } -int ompi_coll_libnbc_iallreduce_inter(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op, - struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) +int ompi_coll_libnbc_iallreduce(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op, + struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_iallreduce(sendbuf, recvbuf, count, datatype, op, comm, request, module); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + return res; + } + + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} + +static int nbc_iallreduce_inter(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op, + struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int rank, res, rsize; size_t size; @@ -254,6 +272,23 @@ int ompi_coll_libnbc_iallreduce_inter(const void* sendbuf, void* recvbuf, int co return OMPI_SUCCESS; } +int ompi_coll_libnbc_iallreduce_inter(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op, + struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_iallreduce_inter(sendbuf, recvbuf, count, datatype, op, comm, request, module); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + return res; + } + + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} /* binomial allreduce (binomial tree up and binomial bcast down) * working principle: diff --git a/ompi/mca/coll/libnbc/nbc_ialltoall.c b/ompi/mca/coll/libnbc/nbc_ialltoall.c index 77432194aa..00e600bf70 100644 --- a/ompi/mca/coll/libnbc/nbc_ialltoall.c +++ b/ompi/mca/coll/libnbc/nbc_ialltoall.c @@ -53,9 +53,9 @@ int NBC_Alltoall_args_compare(NBC_Alltoall_args *a, NBC_Alltoall_args *b, void * #endif /* simple linear MPI_Ialltoall the (simple) algorithm just sends to all nodes */ -int ompi_coll_libnbc_ialltoall(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount, - MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) +static int nbc_ialltoall(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount, + MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int rank, p, res, datasize; size_t a2asize, sndsize; @@ -274,9 +274,28 @@ int ompi_coll_libnbc_ialltoall(const void* sendbuf, int sendcount, MPI_Datatype return OMPI_SUCCESS; } -int ompi_coll_libnbc_ialltoall_inter (const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount, - MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) +int ompi_coll_libnbc_ialltoall(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount, + MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_ialltoall(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, + comm, request, module); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + return res; + } + + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} + +static int nbc_ialltoall_inter (const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount, + MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int res, rsize; MPI_Aint sndext, rcvext; @@ -339,6 +358,25 @@ int ompi_coll_libnbc_ialltoall_inter (const void* sendbuf, int sendcount, MPI_Da return OMPI_SUCCESS; } +int ompi_coll_libnbc_ialltoall_inter (const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount, + MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_ialltoall_inter(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, + comm, request, module); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + return res; + } + + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} + static inline int a2a_sched_pairwise(int rank, int p, MPI_Aint sndext, MPI_Aint rcvext, NBC_Schedule* schedule, const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount, MPI_Datatype recvtype, MPI_Comm comm) { diff --git a/ompi/mca/coll/libnbc/nbc_ialltoallv.c b/ompi/mca/coll/libnbc/nbc_ialltoallv.c index 61f9d1a419..31d39265d5 100644 --- a/ompi/mca/coll/libnbc/nbc_ialltoallv.c +++ b/ompi/mca/coll/libnbc/nbc_ialltoallv.c @@ -40,10 +40,10 @@ static inline int a2av_sched_inplace(int rank, int p, NBC_Schedule *schedule, * would not be sufficient ... we simply do not cache it */ /* simple linear Alltoallv */ -int ompi_coll_libnbc_ialltoallv(const void* sendbuf, const int *sendcounts, const int *sdispls, - MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls, - MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) +static int nbc_ialltoallv(const void* sendbuf, const int *sendcounts, const int *sdispls, + MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls, + MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int rank, p, res; MPI_Aint sndext, rcvext; @@ -137,11 +137,32 @@ int ompi_coll_libnbc_ialltoallv(const void* sendbuf, const int *sendcounts, cons return OMPI_SUCCESS; } +int ompi_coll_libnbc_ialltoallv(const void* sendbuf, const int *sendcounts, const int *sdispls, + MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls, + MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_ialltoallv(sendbuf, sendcounts, sdispls, sendtype, + recvbuf, recvcounts, rdispls, recvtype, + comm, request, module); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + return res; + } + + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} + /* simple linear Alltoallv */ -int ompi_coll_libnbc_ialltoallv_inter (const void* sendbuf, const int *sendcounts, const int *sdispls, - MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls, - MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) +static int nbc_ialltoallv_inter (const void* sendbuf, const int *sendcounts, const int *sdispls, + MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls, + MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int res, rsize; MPI_Aint sndext, rcvext; @@ -204,6 +225,27 @@ int ompi_coll_libnbc_ialltoallv_inter (const void* sendbuf, const int *sendcount return OMPI_SUCCESS; } +int ompi_coll_libnbc_ialltoallv_inter (const void* sendbuf, const int *sendcounts, const int *sdispls, + MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls, + MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_ialltoallv_inter(sendbuf, sendcounts, sdispls, sendtype, + recvbuf, recvcounts, rdispls, recvtype, + comm, request, module); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + return res; + } + + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} + __opal_attribute_unused__ static inline int a2av_sched_linear(int rank, int p, NBC_Schedule *schedule, const void *sendbuf, const int *sendcounts, const int *sdispls, diff --git a/ompi/mca/coll/libnbc/nbc_ialltoallw.c b/ompi/mca/coll/libnbc/nbc_ialltoallw.c index 164a250eaf..9ad2441d77 100644 --- a/ompi/mca/coll/libnbc/nbc_ialltoallw.c +++ b/ompi/mca/coll/libnbc/nbc_ialltoallw.c @@ -40,10 +40,10 @@ static inline int a2aw_sched_inplace(int rank, int p, NBC_Schedule *schedule, * would not be sufficient ... we simply do not cache it */ /* simple linear Alltoallw */ -int ompi_coll_libnbc_ialltoallw(const void* sendbuf, const int *sendcounts, const int *sdispls, - struct ompi_datatype_t * const *sendtypes, void* recvbuf, const int *recvcounts, const int *rdispls, - struct ompi_datatype_t * const *recvtypes, struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) +static int nbc_ialltoallw(const void* sendbuf, const int *sendcounts, const int *sdispls, + struct ompi_datatype_t * const *sendtypes, void* recvbuf, const int *recvcounts, const int *rdispls, + struct ompi_datatype_t * const *recvtypes, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int rank, p, res; NBC_Schedule *schedule; @@ -123,11 +123,32 @@ int ompi_coll_libnbc_ialltoallw(const void* sendbuf, const int *sendcounts, cons return OMPI_SUCCESS; } +int ompi_coll_libnbc_ialltoallw(const void* sendbuf, const int *sendcounts, const int *sdispls, + struct ompi_datatype_t * const *sendtypes, void* recvbuf, const int *recvcounts, const int *rdispls, + struct ompi_datatype_t * const *recvtypes, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_ialltoallw(sendbuf, sendcounts, sdispls, sendtypes, + recvbuf, recvcounts, rdispls, recvtypes, + comm, request, module); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + return res; + } + + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} + /* simple linear Alltoallw */ -int ompi_coll_libnbc_ialltoallw_inter (const void* sendbuf, const int *sendcounts, const int *sdispls, - struct ompi_datatype_t * const *sendtypes, void* recvbuf, const int *recvcounts, const int *rdispls, - struct ompi_datatype_t * const *recvtypes, struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) +static int nbc_ialltoallw_inter (const void* sendbuf, const int *sendcounts, const int *sdispls, + struct ompi_datatype_t * const *sendtypes, void* recvbuf, const int *recvcounts, const int *rdispls, + struct ompi_datatype_t * const *recvtypes, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int res, rsize; NBC_Schedule *schedule; @@ -177,6 +198,27 @@ int ompi_coll_libnbc_ialltoallw_inter (const void* sendbuf, const int *sendcount return OMPI_SUCCESS; } +int ompi_coll_libnbc_ialltoallw_inter(const void* sendbuf, const int *sendcounts, const int *sdispls, + struct ompi_datatype_t * const *sendtypes, void* recvbuf, const int *recvcounts, const int *rdispls, + struct ompi_datatype_t * const *recvtypes, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_ialltoallw_inter(sendbuf, sendcounts, sdispls, sendtypes, + recvbuf, recvcounts, rdispls, recvtypes, + comm, request, module); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + return res; + } + + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} + static inline int a2aw_sched_linear(int rank, int p, NBC_Schedule *schedule, const void *sendbuf, const int *sendcounts, const int *sdispls, struct ompi_datatype_t * const * sendtypes, diff --git a/ompi/mca/coll/libnbc/nbc_ibarrier.c b/ompi/mca/coll/libnbc/nbc_ibarrier.c index 686fc93e79..5d8d9ad096 100644 --- a/ompi/mca/coll/libnbc/nbc_ibarrier.c +++ b/ompi/mca/coll/libnbc/nbc_ibarrier.c @@ -21,8 +21,8 @@ #include "nbc_internal.h" /* Dissemination implementation of MPI_Ibarrier */ -int ompi_coll_libnbc_ibarrier(struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) +static int nbc_ibarrier(struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int rank, p, maxround, res, recvpeer, sendpeer; NBC_Schedule *schedule; @@ -98,8 +98,25 @@ int ompi_coll_libnbc_ibarrier(struct ompi_communicator_t *comm, ompi_request_t * return OMPI_SUCCESS; } -int ompi_coll_libnbc_ibarrier_inter(struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) +int ompi_coll_libnbc_ibarrier(struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_ibarrier(comm, request, module); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + return res; + } + + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} + +static int nbc_ibarrier_inter(struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int rank, res, rsize; NBC_Schedule *schedule; @@ -167,3 +184,20 @@ int ompi_coll_libnbc_ibarrier_inter(struct ompi_communicator_t *comm, ompi_reque } return OMPI_SUCCESS; } + +int ompi_coll_libnbc_ibarrier_inter(struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_ibarrier_inter(comm, request, module); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + return res; + } + + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} diff --git a/ompi/mca/coll/libnbc/nbc_ibcast.c b/ompi/mca/coll/libnbc/nbc_ibcast.c index 932341847d..ecc158ac40 100644 --- a/ompi/mca/coll/libnbc/nbc_ibcast.c +++ b/ompi/mca/coll/libnbc/nbc_ibcast.c @@ -44,9 +44,9 @@ int NBC_Bcast_args_compare(NBC_Bcast_args *a, NBC_Bcast_args *b, void *param) { } #endif -int ompi_coll_libnbc_ibcast(void *buffer, int count, MPI_Datatype datatype, int root, - struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) +static int nbc_ibcast(void *buffer, int count, MPI_Datatype datatype, int root, + struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int rank, p, res, segsize; size_t size; @@ -171,6 +171,24 @@ int ompi_coll_libnbc_ibcast(void *buffer, int count, MPI_Datatype datatype, int return OMPI_SUCCESS; } +int ompi_coll_libnbc_ibcast(void *buffer, int count, MPI_Datatype datatype, int root, + struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) +{ + int res = nbc_ibcast(buffer, count, datatype, root, comm, request, module); + if (OPAL_LIKELY(OMPI_SUCCESS != res)) { + return res; + } + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} + /* better binomial bcast * working principle: * - each node gets a virtual rank vrank @@ -323,9 +341,9 @@ static inline int bcast_sched_chain(int rank, int p, int root, NBC_Schedule *sch return OMPI_SUCCESS; } -int ompi_coll_libnbc_ibcast_inter(void *buffer, int count, MPI_Datatype datatype, int root, - struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) { +static int nbc_ibcast_inter(void *buffer, int count, MPI_Datatype datatype, int root, + struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int res; NBC_Schedule *schedule; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; @@ -374,3 +392,21 @@ int ompi_coll_libnbc_ibcast_inter(void *buffer, int count, MPI_Datatype datatype return OMPI_SUCCESS; } + +int ompi_coll_libnbc_ibcast_inter(void *buffer, int count, MPI_Datatype datatype, int root, + struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_ibcast_inter(buffer, count, datatype, root, comm, request, module); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + return res; + } + + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} diff --git a/ompi/mca/coll/libnbc/nbc_iexscan.c b/ompi/mca/coll/libnbc/nbc_iexscan.c index a9fb0fba14..3b1dc050b9 100644 --- a/ompi/mca/coll/libnbc/nbc_iexscan.c +++ b/ompi/mca/coll/libnbc/nbc_iexscan.c @@ -45,9 +45,9 @@ int NBC_Scan_args_compare(NBC_Scan_args *a, NBC_Scan_args *b, void *param) { * 3. all but rank p-1 do sends to it's right neigbor and exits * */ -int ompi_coll_libnbc_iexscan(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op, - struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) { +static int nbc_iexscan(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op, + struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int rank, p, res; ptrdiff_t gap, span; NBC_Schedule *schedule; @@ -191,3 +191,21 @@ int ompi_coll_libnbc_iexscan(const void* sendbuf, void* recvbuf, int count, MPI_ return OMPI_SUCCESS; } + +int ompi_coll_libnbc_iexscan(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op, + struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_iexscan(sendbuf, recvbuf, count, datatype, op, comm, request, module); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + return res; + } + + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} diff --git a/ompi/mca/coll/libnbc/nbc_igather.c b/ompi/mca/coll/libnbc/nbc_igather.c index bafb58517c..ae9061056d 100644 --- a/ompi/mca/coll/libnbc/nbc_igather.c +++ b/ompi/mca/coll/libnbc/nbc_igather.c @@ -43,10 +43,10 @@ int NBC_Gather_args_compare(NBC_Gather_args *a, NBC_Gather_args *b, void *param) } #endif -int ompi_coll_libnbc_igather(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, - int recvcount, MPI_Datatype recvtype, int root, - struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) { +static int nbc_igather(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, + int recvcount, MPI_Datatype recvtype, int root, + struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int rank, p, res; MPI_Aint rcvext = 0; NBC_Schedule *schedule; @@ -169,10 +169,30 @@ int ompi_coll_libnbc_igather(const void* sendbuf, int sendcount, MPI_Datatype se return OMPI_SUCCESS; } -int ompi_coll_libnbc_igather_inter (const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, - int recvcount, MPI_Datatype recvtype, int root, - struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) { +int ompi_coll_libnbc_igather(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, + int recvcount, MPI_Datatype recvtype, int root, + struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_igather(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, + comm, request, module); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + return res; + } + + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} + +static int nbc_igather_inter (const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, + int recvcount, MPI_Datatype recvtype, int root, + struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int res, rsize; MPI_Aint rcvext = 0; NBC_Schedule *schedule; @@ -228,3 +248,23 @@ int ompi_coll_libnbc_igather_inter (const void* sendbuf, int sendcount, MPI_Data return OMPI_SUCCESS; } + +int ompi_coll_libnbc_igather_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, + int recvcount, MPI_Datatype recvtype, int root, + struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_igather_inter(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, + comm, request, module); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + return res; + } + + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} diff --git a/ompi/mca/coll/libnbc/nbc_igatherv.c b/ompi/mca/coll/libnbc/nbc_igatherv.c index a15f800482..cb5625c055 100644 --- a/ompi/mca/coll/libnbc/nbc_igatherv.c +++ b/ompi/mca/coll/libnbc/nbc_igatherv.c @@ -28,10 +28,10 @@ * would not be sufficient ... we simply do not cache it */ -int ompi_coll_libnbc_igatherv(const void* sendbuf, int sendcount, MPI_Datatype sendtype, - void* recvbuf, const int *recvcounts, const int *displs, MPI_Datatype recvtype, - int root, struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) { +static int nbc_igatherv(const void* sendbuf, int sendcount, MPI_Datatype sendtype, + void* recvbuf, const int *recvcounts, const int *displs, MPI_Datatype recvtype, + int root, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int rank, p, res; MPI_Aint rcvext = 0; NBC_Schedule *schedule; @@ -104,10 +104,30 @@ int ompi_coll_libnbc_igatherv(const void* sendbuf, int sendcount, MPI_Datatype s return OMPI_SUCCESS; } -int ompi_coll_libnbc_igatherv_inter (const void* sendbuf, int sendcount, MPI_Datatype sendtype, - void* recvbuf, const int *recvcounts, const int *displs, MPI_Datatype recvtype, - int root, struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) { +int ompi_coll_libnbc_igatherv(const void* sendbuf, int sendcount, MPI_Datatype sendtype, + void* recvbuf, const int *recvcounts, const int *displs, MPI_Datatype recvtype, + int root, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_igatherv(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, root, + comm, request, module); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + return res; + } + + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} + +static int nbc_igatherv_inter (const void* sendbuf, int sendcount, MPI_Datatype sendtype, + void* recvbuf, const int *recvcounts, const int *displs, MPI_Datatype recvtype, + int root, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int res, rsize; MPI_Aint rcvext; NBC_Schedule *schedule; @@ -163,3 +183,23 @@ int ompi_coll_libnbc_igatherv_inter (const void* sendbuf, int sendcount, MPI_Dat return OMPI_SUCCESS; } + +int ompi_coll_libnbc_igatherv_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype, + void* recvbuf, const int *recvcounts, const int *displs, MPI_Datatype recvtype, + int root, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_igatherv_inter(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, root, + comm, request, module); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + return res; + } + + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} diff --git a/ompi/mca/coll/libnbc/nbc_ineighbor_allgather.c b/ompi/mca/coll/libnbc/nbc_ineighbor_allgather.c index 77fbf3978f..fce26db1ee 100644 --- a/ompi/mca/coll/libnbc/nbc_ineighbor_allgather.c +++ b/ompi/mca/coll/libnbc/nbc_ineighbor_allgather.c @@ -43,9 +43,9 @@ int NBC_Ineighbor_allgather_args_compare(NBC_Ineighbor_allgather_args *a, NBC_In #endif -int ompi_coll_libnbc_ineighbor_allgather(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf, - int rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm, - ompi_request_t ** request, struct mca_coll_base_module_2_2_0_t *module) { +static int nbc_ineighbor_allgather(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf, + int rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm, + ompi_request_t ** request, struct mca_coll_base_module_2_2_0_t *module) { int res, indegree, outdegree, *srcs, *dsts; MPI_Aint rcvext; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; @@ -160,3 +160,172 @@ int ompi_coll_libnbc_ineighbor_allgather(const void *sbuf, int scount, MPI_Datat return OMPI_SUCCESS; } + +int ompi_coll_libnbc_ineighbor_allgather(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf, + int rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm, + ompi_request_t ** request, struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_ineighbor_allgather(sbuf, scount, stype, rbuf, rcount, rtype, comm, request, module); + if (OPAL_LIKELY(OMPI_SUCCESS != res)) { + return res; + } + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} + +/* better binomial bcast + * working principle: + * - each node gets a virtual rank vrank + * - the 'root' node get vrank 0 + * - node 0 gets the vrank of the 'root' + * - all other ranks stay identical (they do not matter) + * + * Algorithm: + * - each node with vrank > 2^r and vrank < 2^r+1 receives from node + * vrank - 2^r (vrank=1 receives from 0, vrank 0 receives never) + * - each node sends each round r to node vrank + 2^r + * - a node stops to send if 2^r > commsize + */ +#define RANK2VRANK(rank, vrank, root) \ +{ \ + vrank = rank; \ + if (rank == 0) vrank = root; \ + if (rank == root) vrank = 0; \ +} +#define VRANK2RANK(rank, vrank, root) \ +{ \ + rank = vrank; \ + if (vrank == 0) rank = root; \ + if (vrank == root) rank = 0; \ +} +static inline int bcast_sched_binomial(int rank, int p, int root, NBC_Schedule *schedule, void *buffer, int count, MPI_Datatype datatype) { + int maxr, vrank, peer, res; + + maxr = (int)ceil((log((double)p)/LOG2)); + + RANK2VRANK(rank, vrank, root); + + /* receive from the right hosts */ + if (vrank != 0) { + for (int r = 0 ; r < maxr ; ++r) { + if ((vrank >= (1 << r)) && (vrank < (1 << (r + 1)))) { + VRANK2RANK(peer, vrank - (1 << r), root); + res = NBC_Sched_recv (buffer, false, count, datatype, peer, schedule, false); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + return res; + } + } + } + + res = NBC_Sched_barrier (schedule); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + return res; + } + } + + /* now send to the right hosts */ + for (int r = 0 ; r < maxr ; ++r) { + if (((vrank + (1 << r) < p) && (vrank < (1 << r))) || (vrank == 0)) { + VRANK2RANK(peer, vrank + (1 << r), root); + res = NBC_Sched_send (buffer, false, count, datatype, peer, schedule, false); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + return res; + } + } + } + + return OMPI_SUCCESS; +} + +/* simple linear MPI_Ibcast */ +static inline int bcast_sched_linear(int rank, int p, int root, NBC_Schedule *schedule, void *buffer, int count, MPI_Datatype datatype) { + int res; + + /* send to all others */ + if(rank == root) { + for (int peer = 0 ; peer < p ; ++peer) { + if (peer != root) { + /* send msg to peer */ + res = NBC_Sched_send (buffer, false, count, datatype, peer, schedule, false); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + return res; + } + } + } + } else { + /* recv msg from root */ + res = NBC_Sched_recv (buffer, false, count, datatype, root, schedule, false); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + return res; + } + } + + return OMPI_SUCCESS; +} + +/* simple chained MPI_Ibcast */ +static inline int bcast_sched_chain(int rank, int p, int root, NBC_Schedule *schedule, void *buffer, int count, MPI_Datatype datatype, int fragsize, size_t size) { + int res, vrank, rpeer, speer, numfrag, fragcount, thiscount; + MPI_Aint ext; + char *buf; + + RANK2VRANK(rank, vrank, root); + VRANK2RANK(rpeer, vrank-1, root); + VRANK2RANK(speer, vrank+1, root); + res = ompi_datatype_type_extent(datatype, &ext); + if (MPI_SUCCESS != res) { + NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res); + return res; + } + + if (count == 0) { + return OMPI_SUCCESS; + } + + numfrag = count * size/fragsize; + if ((count * size) % fragsize != 0) { + numfrag++; + } + + fragcount = count/numfrag; + + for (int fragnum = 0 ; fragnum < numfrag ; ++fragnum) { + buf = (char *) buffer + fragnum * fragcount * ext; + thiscount = fragcount; + if (fragnum == numfrag-1) { + /* last fragment may not be full */ + thiscount = count - fragcount * fragnum; + } + + /* root does not receive */ + if (vrank != 0) { + res = NBC_Sched_recv (buf, false, thiscount, datatype, rpeer, schedule, true); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + return res; + } + } + + /* last rank does not send */ + if (vrank != p-1) { + res = NBC_Sched_send (buf, false, thiscount, datatype, speer, schedule, false); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + return res; + } + + /* this barrier here seems awaward but isn't!!!! */ + if (vrank == 0) { + res = NBC_Sched_barrier (schedule); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + return res; + } + } + } + } + + return OMPI_SUCCESS; +} diff --git a/ompi/mca/coll/libnbc/nbc_ineighbor_allgatherv.c b/ompi/mca/coll/libnbc/nbc_ineighbor_allgatherv.c index d963fcc423..3725c5d0c6 100644 --- a/ompi/mca/coll/libnbc/nbc_ineighbor_allgatherv.c +++ b/ompi/mca/coll/libnbc/nbc_ineighbor_allgatherv.c @@ -43,10 +43,10 @@ int NBC_Ineighbor_allgatherv_args_compare(NBC_Ineighbor_allgatherv_args *a, NBC_ #endif -int ompi_coll_libnbc_ineighbor_allgatherv(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf, - const int *rcounts, const int *displs, MPI_Datatype rtype, - struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) { +static int nbc_ineighbor_allgatherv(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf, + const int *rcounts, const int *displs, MPI_Datatype rtype, + struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int res, indegree, outdegree, *srcs, *dsts; MPI_Aint rcvext; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; @@ -162,3 +162,21 @@ int ompi_coll_libnbc_ineighbor_allgatherv(const void *sbuf, int scount, MPI_Data return OMPI_SUCCESS; } + +int ompi_coll_libnbc_ineighbor_allgatherv(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf, + const int *rcounts, const int *displs, MPI_Datatype rtype, + struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_ineighbor_allgatherv(sbuf, scount, stype, rbuf, rcounts, displs, rtype, comm, request, module); + if (OPAL_LIKELY(OMPI_SUCCESS != res)) { + return res; + } + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} diff --git a/ompi/mca/coll/libnbc/nbc_ineighbor_alltoall.c b/ompi/mca/coll/libnbc/nbc_ineighbor_alltoall.c index d9ae492ee2..575d4cf316 100644 --- a/ompi/mca/coll/libnbc/nbc_ineighbor_alltoall.c +++ b/ompi/mca/coll/libnbc/nbc_ineighbor_alltoall.c @@ -40,9 +40,9 @@ int NBC_Ineighbor_alltoall_args_compare(NBC_Ineighbor_alltoall_args *a, NBC_Inei } #endif -int ompi_coll_libnbc_ineighbor_alltoall(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf, - int rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm, - ompi_request_t ** request, struct mca_coll_base_module_2_2_0_t *module) { +static int nbc_ineighbor_alltoall(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf, + int rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm, + ompi_request_t ** request, struct mca_coll_base_module_2_2_0_t *module) { int res, indegree, outdegree, *srcs, *dsts; MPI_Aint sndext, rcvext; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; @@ -164,3 +164,20 @@ int ompi_coll_libnbc_ineighbor_alltoall(const void *sbuf, int scount, MPI_Dataty return OMPI_SUCCESS; } + +int ompi_coll_libnbc_ineighbor_alltoall(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf, + int rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm, + ompi_request_t ** request, struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_ineighbor_alltoall(sbuf, scount, stype, rbuf, rcount, rtype, comm, request, module); + if (OPAL_LIKELY(OMPI_SUCCESS != res)) { + return res; + } + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} diff --git a/ompi/mca/coll/libnbc/nbc_ineighbor_alltoallv.c b/ompi/mca/coll/libnbc/nbc_ineighbor_alltoallv.c index 4caf50e010..a791eceab6 100644 --- a/ompi/mca/coll/libnbc/nbc_ineighbor_alltoallv.c +++ b/ompi/mca/coll/libnbc/nbc_ineighbor_alltoallv.c @@ -43,10 +43,10 @@ int NBC_Ineighbor_alltoallv_args_compare(NBC_Ineighbor_alltoallv_args *a, NBC_In #endif -int ompi_coll_libnbc_ineighbor_alltoallv(const void *sbuf, const int *scounts, const int *sdispls, MPI_Datatype stype, - void *rbuf, const int *rcounts, const int *rdispls, MPI_Datatype rtype, - struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) { +static int nbc_ineighbor_alltoallv(const void *sbuf, const int *scounts, const int *sdispls, MPI_Datatype stype, + void *rbuf, const int *rcounts, const int *rdispls, MPI_Datatype rtype, + struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int res, indegree, outdegree, *srcs, *dsts; MPI_Aint sndext, rcvext; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; @@ -169,3 +169,21 @@ int ompi_coll_libnbc_ineighbor_alltoallv(const void *sbuf, const int *scounts, c return OMPI_SUCCESS; } + +int ompi_coll_libnbc_ineighbor_alltoallv(const void *sbuf, const int *scounts, const int *sdispls, MPI_Datatype stype, + void *rbuf, const int *rcounts, const int *rdispls, MPI_Datatype rtype, + struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_ineighbor_alltoallv(sbuf, scounts, sdispls, stype, rbuf, rcounts, rdispls, rtype, comm, request, module); + if (OPAL_LIKELY(OMPI_SUCCESS != res)) { + return res; + } + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} diff --git a/ompi/mca/coll/libnbc/nbc_ineighbor_alltoallw.c b/ompi/mca/coll/libnbc/nbc_ineighbor_alltoallw.c index 10033010c6..580dfaef3e 100644 --- a/ompi/mca/coll/libnbc/nbc_ineighbor_alltoallw.c +++ b/ompi/mca/coll/libnbc/nbc_ineighbor_alltoallw.c @@ -42,10 +42,10 @@ int NBC_Ineighbor_alltoallw_args_compare(NBC_Ineighbor_alltoallw_args *a, NBC_In } #endif -int ompi_coll_libnbc_ineighbor_alltoallw(const void *sbuf, const int *scounts, const MPI_Aint *sdisps, struct ompi_datatype_t * const *stypes, - void *rbuf, const int *rcounts, const MPI_Aint *rdisps, struct ompi_datatype_t * const *rtypes, - struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) { +static int nbc_ineighbor_alltoallw(const void *sbuf, const int *scounts, const MPI_Aint *sdisps, struct ompi_datatype_t * const *stypes, + void *rbuf, const int *rcounts, const MPI_Aint *rdisps, struct ompi_datatype_t * const *rtypes, + struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int res, indegree, outdegree, *srcs, *dsts; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; NBC_Schedule *schedule; @@ -154,3 +154,21 @@ int ompi_coll_libnbc_ineighbor_alltoallw(const void *sbuf, const int *scounts, c return OMPI_SUCCESS; } + +int ompi_coll_libnbc_ineighbor_alltoallw(const void *sbuf, const int *scounts, const MPI_Aint *sdisps, struct ompi_datatype_t * const *stypes, + void *rbuf, const int *rcounts, const MPI_Aint *rdisps, struct ompi_datatype_t * const *rtypes, + struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_ineighbor_alltoallw(sbuf, scounts, sdisps, stypes, rbuf, rcounts, rdisps, rtypes, comm, request, module); + if (OPAL_LIKELY(OMPI_SUCCESS != res)) { + return res; + } + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} diff --git a/ompi/mca/coll/libnbc/nbc_ireduce.c b/ompi/mca/coll/libnbc/nbc_ireduce.c index b35801aeb2..cc02c85460 100644 --- a/ompi/mca/coll/libnbc/nbc_ireduce.c +++ b/ompi/mca/coll/libnbc/nbc_ireduce.c @@ -52,9 +52,9 @@ int NBC_Reduce_args_compare(NBC_Reduce_args *a, NBC_Reduce_args *b, void *param) #endif /* the non-blocking reduce */ -int ompi_coll_libnbc_ireduce(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, - MPI_Op op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) { +static int nbc_ireduce(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, + MPI_Op op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int rank, p, res, segsize; size_t size; MPI_Aint ext; @@ -203,9 +203,26 @@ int ompi_coll_libnbc_ireduce(const void* sendbuf, void* recvbuf, int count, MPI_ return OMPI_SUCCESS; } -int ompi_coll_libnbc_ireduce_inter(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, - MPI_Op op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) { +int ompi_coll_libnbc_ireduce(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, + MPI_Op op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_ireduce(sendbuf, recvbuf, count, datatype, op, root, comm, request, module); + if (OPAL_LIKELY(OMPI_SUCCESS != res)) { + return res; + } + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} + +static int nbc_ireduce_inter(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, + MPI_Op op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int rank, res, rsize; NBC_Schedule *schedule; ompi_coll_libnbc_module_t *libnbc_module = (ompi_coll_libnbc_module_t*) module; @@ -251,6 +268,23 @@ int ompi_coll_libnbc_ireduce_inter(const void* sendbuf, void* recvbuf, int count return OMPI_SUCCESS; } +int ompi_coll_libnbc_ireduce_inter(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, + MPI_Op op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_ireduce_inter(sendbuf, recvbuf, count, datatype, op, root, comm, request, module); + if (OPAL_LIKELY(OMPI_SUCCESS != res)) { + return res; + } + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} + /* binomial reduce * if op is not commutative, reduce on rank 0, and then send the result to root rank diff --git a/ompi/mca/coll/libnbc/nbc_ireduce_scatter.c b/ompi/mca/coll/libnbc/nbc_ireduce_scatter.c index 49edfeb7d3..ccfd8db9ea 100644 --- a/ompi/mca/coll/libnbc/nbc_ireduce_scatter.c +++ b/ompi/mca/coll/libnbc/nbc_ireduce_scatter.c @@ -41,9 +41,9 @@ * */ -int ompi_coll_libnbc_ireduce_scatter(const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype, - MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) { +static int nbc_ireduce_scatter(const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype, + MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int peer, rank, maxr, p, res, count; MPI_Aint ext; ptrdiff_t gap, span, span_align; @@ -203,9 +203,25 @@ int ompi_coll_libnbc_ireduce_scatter(const void* sendbuf, void* recvbuf, const i return OMPI_SUCCESS; } -int ompi_coll_libnbc_ireduce_scatter_inter (const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype, - MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) { +int ompi_coll_libnbc_ireduce_scatter (const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype, + MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_ireduce_scatter(sendbuf, recvbuf, recvcounts, datatype, op, comm, request, module); + if (OPAL_LIKELY(OMPI_SUCCESS != res)) { + return res; + } + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} +static int nbc_ireduce_scatter_inter (const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype, + MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int rank, res, count, lsize, rsize; MPI_Aint ext; ptrdiff_t gap, span, span_align; @@ -327,3 +343,20 @@ int ompi_coll_libnbc_ireduce_scatter_inter (const void* sendbuf, void* recvbuf, return OMPI_SUCCESS; } + +int ompi_coll_libnbc_ireduce_scatter_inter (const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype, + MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_ireduce_scatter_inter(sendbuf, recvbuf, recvcounts, datatype, op, comm, request, module); + if (OPAL_LIKELY(OMPI_SUCCESS != res)) { + return res; + } + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} diff --git a/ompi/mca/coll/libnbc/nbc_ireduce_scatter_block.c b/ompi/mca/coll/libnbc/nbc_ireduce_scatter_block.c index 5c1cedf7c2..b6f3dd1d0a 100644 --- a/ompi/mca/coll/libnbc/nbc_ireduce_scatter_block.c +++ b/ompi/mca/coll/libnbc/nbc_ireduce_scatter_block.c @@ -39,9 +39,9 @@ * */ -int ompi_coll_libnbc_ireduce_scatter_block(const void* sendbuf, void* recvbuf, int recvcount, MPI_Datatype datatype, - MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) { +static int nbc_ireduce_scatter_block(const void* sendbuf, void* recvbuf, int recvcount, MPI_Datatype datatype, + MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int peer, rank, maxr, p, res, count; MPI_Aint ext; ptrdiff_t gap, span; @@ -205,9 +205,26 @@ int ompi_coll_libnbc_ireduce_scatter_block(const void* sendbuf, void* recvbuf, i return OMPI_SUCCESS; } -int ompi_coll_libnbc_ireduce_scatter_block_inter(const void *sendbuf, void *recvbuf, int rcount, struct ompi_datatype_t *dtype, - struct ompi_op_t *op, struct ompi_communicator_t *comm, - ompi_request_t **request, struct mca_coll_base_module_2_2_0_t *module) { +int ompi_coll_libnbc_ireduce_scatter_block(const void* sendbuf, void* recvbuf, int recvcount, MPI_Datatype datatype, + MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_ireduce_scatter_block(sendbuf, recvbuf, recvcount, datatype, op, comm, request, module); + if (OPAL_LIKELY(OMPI_SUCCESS != res)) { + return res; + } + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} + +static int nbc_ireduce_scatter_block_inter(const void *sendbuf, void *recvbuf, int rcount, struct ompi_datatype_t *dtype, + struct ompi_op_t *op, struct ompi_communicator_t *comm, + ompi_request_t **request, struct mca_coll_base_module_2_2_0_t *module) { int rank, res, count, lsize, rsize; MPI_Aint ext; ptrdiff_t gap, span, span_align; @@ -325,3 +342,20 @@ int ompi_coll_libnbc_ireduce_scatter_block_inter(const void *sendbuf, void *recv return OMPI_SUCCESS; } + +int ompi_coll_libnbc_ireduce_scatter_block_inter(const void* sendbuf, void* recvbuf, int recvcount, MPI_Datatype datatype, + MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_ireduce_scatter_block_inter(sendbuf, recvbuf, recvcount, datatype, op, comm, request, module); + if (OPAL_LIKELY(OMPI_SUCCESS != res)) { + return res; + } + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} diff --git a/ompi/mca/coll/libnbc/nbc_iscan.c b/ompi/mca/coll/libnbc/nbc_iscan.c index 87333251a0..9b016f6d86 100644 --- a/ompi/mca/coll/libnbc/nbc_iscan.c +++ b/ompi/mca/coll/libnbc/nbc_iscan.c @@ -45,9 +45,9 @@ int NBC_Scan_args_compare(NBC_Scan_args *a, NBC_Scan_args *b, void *param) { * 3. all but rank p-1 do sends to it's right neighbor and exits * */ -int ompi_coll_libnbc_iscan(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op, - struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) { +static int nbc_iscan(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op, + struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int rank, p, res; ptrdiff_t gap, span; NBC_Schedule *schedule; @@ -168,3 +168,20 @@ int ompi_coll_libnbc_iscan(const void* sendbuf, void* recvbuf, int count, MPI_Da return OMPI_SUCCESS; } + +int ompi_coll_libnbc_iscan(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op, + struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_iscan(sendbuf, recvbuf, count, datatype, op, comm, request, module); + if (OPAL_LIKELY(OMPI_SUCCESS != res)) { + return res; + } + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} diff --git a/ompi/mca/coll/libnbc/nbc_iscatter.c b/ompi/mca/coll/libnbc/nbc_iscatter.c index 48b0917cdc..eb40ee49f9 100644 --- a/ompi/mca/coll/libnbc/nbc_iscatter.c +++ b/ompi/mca/coll/libnbc/nbc_iscatter.c @@ -44,10 +44,10 @@ int NBC_Scatter_args_compare(NBC_Scatter_args *a, NBC_Scatter_args *b, void *par #endif /* simple linear MPI_Iscatter */ -int ompi_coll_libnbc_iscatter (const void* sendbuf, int sendcount, MPI_Datatype sendtype, - void* recvbuf, int recvcount, MPI_Datatype recvtype, int root, - struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) { +static int nbc_iscatter (const void* sendbuf, int sendcount, MPI_Datatype sendtype, + void* recvbuf, int recvcount, MPI_Datatype recvtype, int root, + struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int rank, p, res; MPI_Aint sndext = 0; NBC_Schedule *schedule; @@ -166,10 +166,28 @@ int ompi_coll_libnbc_iscatter (const void* sendbuf, int sendcount, MPI_Datatype return OMPI_SUCCESS; } -int ompi_coll_libnbc_iscatter_inter (const void* sendbuf, int sendcount, MPI_Datatype sendtype, - void* recvbuf, int recvcount, MPI_Datatype recvtype, int root, - struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) { +int ompi_coll_libnbc_iscatter (const void* sendbuf, int sendcount, MPI_Datatype sendtype, + void* recvbuf, int recvcount, MPI_Datatype recvtype, int root, + struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_iscatter(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, request, module); + if (OPAL_LIKELY(OMPI_SUCCESS != res)) { + return res; + } + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} + +static int nbc_iscatter_inter (const void* sendbuf, int sendcount, MPI_Datatype sendtype, + void* recvbuf, int recvcount, MPI_Datatype recvtype, int root, + struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int res, rsize; MPI_Aint sndext; NBC_Schedule *schedule; @@ -225,3 +243,21 @@ int ompi_coll_libnbc_iscatter_inter (const void* sendbuf, int sendcount, MPI_Dat return OMPI_SUCCESS; } + +int ompi_coll_libnbc_iscatter_inter (const void* sendbuf, int sendcount, MPI_Datatype sendtype, + void* recvbuf, int recvcount, MPI_Datatype recvtype, int root, + struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_iscatter_inter(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, request, module); + if (OPAL_LIKELY(OMPI_SUCCESS != res)) { + return res; + } + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} diff --git a/ompi/mca/coll/libnbc/nbc_iscatterv.c b/ompi/mca/coll/libnbc/nbc_iscatterv.c index b16ef085c1..1b5a6a2519 100644 --- a/ompi/mca/coll/libnbc/nbc_iscatterv.c +++ b/ompi/mca/coll/libnbc/nbc_iscatterv.c @@ -27,10 +27,10 @@ * would not be sufficient ... we simply do not cache it */ /* simple linear MPI_Iscatterv */ -int ompi_coll_libnbc_iscatterv(const void* sendbuf, const int *sendcounts, const int *displs, MPI_Datatype sendtype, - void* recvbuf, int recvcount, MPI_Datatype recvtype, int root, - struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) { +static int nbc_iscatterv(const void* sendbuf, const int *sendcounts, const int *displs, MPI_Datatype sendtype, + void* recvbuf, int recvcount, MPI_Datatype recvtype, int root, + struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int rank, p, res; MPI_Aint sndext; NBC_Schedule *schedule; @@ -101,10 +101,28 @@ int ompi_coll_libnbc_iscatterv(const void* sendbuf, const int *sendcounts, const return OMPI_SUCCESS; } -int ompi_coll_libnbc_iscatterv_inter (const void* sendbuf, const int *sendcounts, const int *displs, MPI_Datatype sendtype, - void* recvbuf, int recvcount, MPI_Datatype recvtype, int root, - struct ompi_communicator_t *comm, ompi_request_t ** request, - struct mca_coll_base_module_2_2_0_t *module) { +int ompi_coll_libnbc_iscatterv(const void* sendbuf, const int *sendcounts, const int *displs, MPI_Datatype sendtype, + void* recvbuf, int recvcount, MPI_Datatype recvtype, int root, + struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_iscatterv(sendbuf, sendcounts, displs, sendtype, recvbuf, recvcount, recvtype, root, comm, request, module); + if (OPAL_LIKELY(OMPI_SUCCESS != res)) { + return res; + } + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +} + +static int nbc_iscatterv_inter (const void* sendbuf, const int *sendcounts, const int *displs, MPI_Datatype sendtype, + void* recvbuf, int recvcount, MPI_Datatype recvtype, int root, + struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { int res, rsize; MPI_Aint sndext; NBC_Schedule *schedule; @@ -159,3 +177,21 @@ int ompi_coll_libnbc_iscatterv_inter (const void* sendbuf, const int *sendcounts return OMPI_SUCCESS; } + +int ompi_coll_libnbc_iscatterv_inter(const void* sendbuf, const int *sendcounts, const int *displs, MPI_Datatype sendtype, + void* recvbuf, int recvcount, MPI_Datatype recvtype, int root, + struct ompi_communicator_t *comm, ompi_request_t ** request, + struct mca_coll_base_module_2_2_0_t *module) { + int res = nbc_iscatterv_inter(sendbuf, sendcounts, displs, sendtype, recvbuf, recvcount, recvtype, root, comm, request, module); + if (OPAL_LIKELY(OMPI_SUCCESS != res)) { + return res; + } + res = NBC_Start(*(ompi_coll_libnbc_request_t **)request); + if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) { + NBC_Return_handle ((ompi_coll_libnbc_request_t *)request); + *request = &ompi_request_null.request; + return res; + } + + return OMPI_SUCCESS; +}