coll/libnbc: code refactoring

prepare the upcoming persistent collectives by pre-factoring some code

Signed-off-by: Gilles Gouaillardet <gilles@rist.or.jp>

fixup 808c3c62cd9475edd91ecde9d2d53b12e28b2c04

ompi/mca/coll/libnbc/nbc.c

@@ -641,6 +641,10 @@ int  NBC_Init_comm(MPI_Comm comm, NBC_Comminfo *comminfo) {
 int NBC_Start(NBC_Handle *handle) {
   int res;
 
+  /* bozo case */
+  if ((ompi_request_t *)handle == &ompi_request_empty) {
+    return OMPI_SUCCESS;
+  }
   /* kick off first round */
   res = NBC_Start_round(handle);
   if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
@@ -654,7 +658,6 @@ int NBC_Start(NBC_Handle *handle) {
 }
 
 int NBC_Schedule_request(NBC_Schedule *schedule, ompi_communicator_t *comm, ompi_coll_libnbc_module_t *module, ompi_request_t **request, void *tmpbuf) {
-  int res;
   int tmp_tag;
   bool need_register = false;
   ompi_coll_libnbc_request_t *handle;
@@ -705,14 +708,8 @@ int NBC_Schedule_request(NBC_Schedule *schedule, ompi_communicator_t *comm, ompi
 
   handle->tmpbuf = tmpbuf;
   handle->schedule = schedule;
-
-  res = NBC_Start (handle);
-  if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
-    NBC_Return_handle (handle);
-    return res;
-  }
-
   *request = (ompi_request_t *) handle;
+
   return OMPI_SUCCESS;
 }
 

ompi/mca/coll/libnbc/nbc_iallgather.c

@@ -43,7 +43,7 @@ int NBC_Allgather_args_compare(NBC_Allgather_args *a, NBC_Allgather_args *b, voi
  * the algorithm uses p-1 rounds
  * each node sends the packet it received last round (or has in round 0) to it's right neighbor (modulo p)
  * each node receives from it's left (modulo p) neighbor */
-int ompi_coll_libnbc_iallgather(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+static int nbc_iallgather(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
                           MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
                           struct mca_coll_base_module_2_2_0_t *module)
 {
@@ -163,7 +163,27 @@ int ompi_coll_libnbc_iallgather(const void* sendbuf, int sendcount, MPI_Datatype
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_iallgather_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+int ompi_coll_libnbc_iallgather(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+                                MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
+                                struct mca_coll_base_module_2_2_0_t *module)
+{
+    int res = nbc_iallgather(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype,
+                             comm, request, module);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+  
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle (*(ompi_coll_libnbc_request_t **)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}
+
+static int nbc_iallgather_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
                                 MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                 struct mca_coll_base_module_2_2_0_t *module)
 {
@@ -219,3 +239,23 @@ int ompi_coll_libnbc_iallgather_inter(const void* sendbuf, int sendcount, MPI_Da
 
   return OMPI_SUCCESS;
 }
+
+int ompi_coll_libnbc_iallgather_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+				      MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
+				      struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_iallgather_inter(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype,
+                                   comm, request, module);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle (*(ompi_coll_libnbc_request_t **)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}
+    

ompi/mca/coll/libnbc/nbc_iallgatherv.c

@@ -33,7 +33,7 @@
  * second round:
  *   each node sends to node (rank+2)%p sendcount elements
  *   each node receives from node (rank-2)%p recvcounts[(rank+2)%p] elements */
-int ompi_coll_libnbc_iallgatherv(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs,
+static int nbc_iallgatherv(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs,
                            MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
                            struct mca_coll_base_module_2_2_0_t *module)
 {
@@ -108,7 +108,26 @@ int ompi_coll_libnbc_iallgatherv(const void* sendbuf, int sendcount, MPI_Datatyp
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_iallgatherv_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs,
+int ompi_coll_libnbc_iallgatherv(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs,
+                                 MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
+                                 struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_iallgatherv(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype,
+                              comm, request, module);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+  
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}
+
+static int nbc_iallgatherv_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs,
                                  MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                  struct mca_coll_base_module_2_2_0_t *module)
 {
@@ -167,3 +186,22 @@ int ompi_coll_libnbc_iallgatherv_inter(const void* sendbuf, int sendcount, MPI_D
 
   return OMPI_SUCCESS;
 }
+
+int ompi_coll_libnbc_iallgatherv_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *displs,
+                                       MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
+                                       struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_iallgatherv_inter(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype,
+                                    comm, request, module);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+  
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}

ompi/mca/coll/libnbc/nbc_iallreduce.c

@@ -52,7 +52,7 @@ int NBC_Allreduce_args_compare(NBC_Allreduce_args *a, NBC_Allreduce_args *b, voi
 }
 #endif
 
-int ompi_coll_libnbc_iallreduce(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
+static int nbc_iallreduce(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
                           struct ompi_communicator_t *comm, ompi_request_t ** request,
                           struct mca_coll_base_module_2_2_0_t *module)
 {
@@ -190,7 +190,25 @@ int ompi_coll_libnbc_iallreduce(const void* sendbuf, void* recvbuf, int count, M
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_iallreduce_inter(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
+int ompi_coll_libnbc_iallreduce(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
+                                struct ompi_communicator_t *comm, ompi_request_t ** request,
+                                struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_iallreduce(sendbuf, recvbuf, count, datatype, op, comm, request, module);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+  
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}
+
+static int nbc_iallreduce_inter(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
                                 struct ompi_communicator_t *comm, ompi_request_t ** request,
                                 struct mca_coll_base_module_2_2_0_t *module)
 {
@@ -254,6 +272,23 @@ int ompi_coll_libnbc_iallreduce_inter(const void* sendbuf, void* recvbuf, int co
   return OMPI_SUCCESS;
 }
 
+int ompi_coll_libnbc_iallreduce_inter(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
+                                      struct ompi_communicator_t *comm, ompi_request_t ** request,
+                                      struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_iallreduce_inter(sendbuf, recvbuf, count, datatype, op, comm, request, module);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+  
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}
 
 /* binomial allreduce (binomial tree up and binomial bcast down)
  * working principle:

ompi/mca/coll/libnbc/nbc_ialltoall.c

@@ -53,7 +53,7 @@ int NBC_Alltoall_args_compare(NBC_Alltoall_args *a, NBC_Alltoall_args *b, void *
 #endif
 
 /* simple linear MPI_Ialltoall the (simple) algorithm just sends to all nodes */
-int ompi_coll_libnbc_ialltoall(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+static int nbc_ialltoall(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
                          MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
                          struct mca_coll_base_module_2_2_0_t *module)
 {
@@ -274,7 +274,26 @@ int ompi_coll_libnbc_ialltoall(const void* sendbuf, int sendcount, MPI_Datatype
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_ialltoall_inter (const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+int ompi_coll_libnbc_ialltoall(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+                               MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
+                               struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_ialltoall(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype,
+                            comm, request, module);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+  
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}
+
+static int nbc_ialltoall_inter (const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
                                 MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                 struct mca_coll_base_module_2_2_0_t *module)
 {
@@ -339,6 +358,25 @@ int ompi_coll_libnbc_ialltoall_inter (const void* sendbuf, int sendcount, MPI_Da
   return OMPI_SUCCESS;
 }
 
+int ompi_coll_libnbc_ialltoall_inter (const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
+				      MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
+				      struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_ialltoall_inter(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype,
+                                  comm, request, module);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+  
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}
+
 static inline int a2a_sched_pairwise(int rank, int p, MPI_Aint sndext, MPI_Aint rcvext, NBC_Schedule* schedule,
                                      const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf, int recvcount,
                                      MPI_Datatype recvtype, MPI_Comm comm) {

ompi/mca/coll/libnbc/nbc_ialltoallv.c

@@ -40,7 +40,7 @@ static inline int a2av_sched_inplace(int rank, int p, NBC_Schedule *schedule,
  * would not be sufficient ... we simply do not cache it */
 
 /* simple linear Alltoallv */
-int ompi_coll_libnbc_ialltoallv(const void* sendbuf, const int *sendcounts, const int *sdispls,
+static int nbc_ialltoallv(const void* sendbuf, const int *sendcounts, const int *sdispls,
                           MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls,
                           MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
                           struct mca_coll_base_module_2_2_0_t *module)
@@ -137,8 +137,29 @@ int ompi_coll_libnbc_ialltoallv(const void* sendbuf, const int *sendcounts, cons
   return OMPI_SUCCESS;
 }
 
+int ompi_coll_libnbc_ialltoallv(const void* sendbuf, const int *sendcounts, const int *sdispls,
+                                MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls,
+                                MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
+                                struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_ialltoallv(sendbuf, sendcounts, sdispls, sendtype,
+                             recvbuf, recvcounts, rdispls, recvtype,
+                             comm, request, module);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+  
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}
+
 /* simple linear Alltoallv */
-int ompi_coll_libnbc_ialltoallv_inter (const void* sendbuf, const int *sendcounts, const int *sdispls,
+static int nbc_ialltoallv_inter (const void* sendbuf, const int *sendcounts, const int *sdispls,
                                  MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls,
                                  MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                  struct mca_coll_base_module_2_2_0_t *module)
@@ -204,6 +225,27 @@ int ompi_coll_libnbc_ialltoallv_inter (const void* sendbuf, const int *sendcount
   return OMPI_SUCCESS;
 }
 
+int ompi_coll_libnbc_ialltoallv_inter (const void* sendbuf, const int *sendcounts, const int *sdispls,
+				       MPI_Datatype sendtype, void* recvbuf, const int *recvcounts, const int *rdispls,
+				       MPI_Datatype recvtype, struct ompi_communicator_t *comm, ompi_request_t ** request,
+				       struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_ialltoallv_inter(sendbuf, sendcounts, sdispls, sendtype,
+                                   recvbuf, recvcounts, rdispls, recvtype,
+                                   comm, request, module);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+  
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}
+
 __opal_attribute_unused__
 static inline int a2av_sched_linear(int rank, int p, NBC_Schedule *schedule,
                                     const void *sendbuf, const int *sendcounts, const int *sdispls,

ompi/mca/coll/libnbc/nbc_ialltoallw.c

@@ -40,7 +40,7 @@ static inline int a2aw_sched_inplace(int rank, int p, NBC_Schedule *schedule,
  * would not be sufficient ... we simply do not cache it */
 
 /* simple linear Alltoallw */
-int ompi_coll_libnbc_ialltoallw(const void* sendbuf, const int *sendcounts, const int *sdispls,
+static int nbc_ialltoallw(const void* sendbuf, const int *sendcounts, const int *sdispls,
                           struct ompi_datatype_t * const *sendtypes, void* recvbuf, const int *recvcounts, const int *rdispls,
                           struct ompi_datatype_t * const *recvtypes, struct ompi_communicator_t *comm, ompi_request_t ** request,
                           struct mca_coll_base_module_2_2_0_t *module)
@@ -123,8 +123,29 @@ int ompi_coll_libnbc_ialltoallw(const void* sendbuf, const int *sendcounts, cons
   return OMPI_SUCCESS;
 } 
 
+int ompi_coll_libnbc_ialltoallw(const void* sendbuf, const int *sendcounts, const int *sdispls,
+                                struct ompi_datatype_t * const *sendtypes, void* recvbuf, const int *recvcounts, const int *rdispls,
+                                struct ompi_datatype_t * const *recvtypes, struct ompi_communicator_t *comm, ompi_request_t ** request,
+				struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_ialltoallw(sendbuf, sendcounts, sdispls, sendtypes,
+                             recvbuf, recvcounts, rdispls, recvtypes,
+                             comm, request, module);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+  
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}
+
 /* simple linear Alltoallw */
-int ompi_coll_libnbc_ialltoallw_inter (const void* sendbuf, const int *sendcounts, const int *sdispls,
+static int nbc_ialltoallw_inter (const void* sendbuf, const int *sendcounts, const int *sdispls,
                                  struct ompi_datatype_t * const *sendtypes, void* recvbuf, const int *recvcounts, const int *rdispls,
                                  struct ompi_datatype_t * const *recvtypes, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                  struct mca_coll_base_module_2_2_0_t *module)
@@ -177,6 +198,27 @@ int ompi_coll_libnbc_ialltoallw_inter (const void* sendbuf, const int *sendcount
   return OMPI_SUCCESS;
 }
 
+int ompi_coll_libnbc_ialltoallw_inter(const void* sendbuf, const int *sendcounts, const int *sdispls,
+                                      struct ompi_datatype_t * const *sendtypes, void* recvbuf, const int *recvcounts, const int *rdispls,
+                                      struct ompi_datatype_t * const *recvtypes, struct ompi_communicator_t *comm, ompi_request_t ** request,
+				      struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_ialltoallw_inter(sendbuf, sendcounts, sdispls, sendtypes,
+                                   recvbuf, recvcounts, rdispls, recvtypes,
+                                   comm, request, module);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+  
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}
+
 static inline int a2aw_sched_linear(int rank, int p, NBC_Schedule *schedule,
                                     const void *sendbuf, const int *sendcounts, const int *sdispls,
                                     struct ompi_datatype_t * const * sendtypes,

ompi/mca/coll/libnbc/nbc_ibarrier.c

@@ -21,7 +21,7 @@
 #include "nbc_internal.h"
 
 /* Dissemination implementation of MPI_Ibarrier */
-int ompi_coll_libnbc_ibarrier(struct ompi_communicator_t *comm, ompi_request_t ** request,
+static int nbc_ibarrier(struct ompi_communicator_t *comm, ompi_request_t ** request,
                         struct mca_coll_base_module_2_2_0_t *module)
 {
   int rank, p, maxround, res, recvpeer, sendpeer;
@@ -98,7 +98,24 @@ int ompi_coll_libnbc_ibarrier(struct ompi_communicator_t *comm, ompi_request_t *
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_ibarrier_inter(struct ompi_communicator_t *comm, ompi_request_t ** request,
+int ompi_coll_libnbc_ibarrier(struct ompi_communicator_t *comm, ompi_request_t ** request,
+                              struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_ibarrier(comm, request, module);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+  
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}
+
+static int nbc_ibarrier_inter(struct ompi_communicator_t *comm, ompi_request_t ** request,
                                                     struct mca_coll_base_module_2_2_0_t *module)
 {
   int rank, res, rsize;
@@ -167,3 +184,20 @@ int ompi_coll_libnbc_ibarrier_inter(struct ompi_communicator_t *comm, ompi_reque
   }
   return OMPI_SUCCESS;
 }
+
+int ompi_coll_libnbc_ibarrier_inter(struct ompi_communicator_t *comm, ompi_request_t ** request,
+                                    struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_ibarrier_inter(comm, request, module);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+  
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}

ompi/mca/coll/libnbc/nbc_ibcast.c

@@ -44,7 +44,7 @@ int NBC_Bcast_args_compare(NBC_Bcast_args *a, NBC_Bcast_args *b, void *param) {
 }
 #endif
 
-int ompi_coll_libnbc_ibcast(void *buffer, int count, MPI_Datatype datatype, int root,
+static int nbc_ibcast(void *buffer, int count, MPI_Datatype datatype, int root,
                       struct ompi_communicator_t *comm, ompi_request_t ** request,
                       struct mca_coll_base_module_2_2_0_t *module)
 {
@@ -171,6 +171,24 @@ int ompi_coll_libnbc_ibcast(void *buffer, int count, MPI_Datatype datatype, int
   return OMPI_SUCCESS;
 }
 
+int ompi_coll_libnbc_ibcast(void *buffer, int count, MPI_Datatype datatype, int root,
+                            struct ompi_communicator_t *comm, ompi_request_t ** request,
+                            struct mca_coll_base_module_2_2_0_t *module)
+{
+    int res = nbc_ibcast(buffer, count, datatype, root, comm, request, module);
+    if (OPAL_LIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}
+
 /* better binomial bcast
  * working principle:
  * - each node gets a virtual rank vrank
@@ -323,7 +341,7 @@ static inline int bcast_sched_chain(int rank, int p, int root, NBC_Schedule *sch
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_ibcast_inter(void *buffer, int count, MPI_Datatype datatype, int root,
+static int nbc_ibcast_inter(void *buffer, int count, MPI_Datatype datatype, int root,
                             struct ompi_communicator_t *comm, ompi_request_t ** request,
                             struct mca_coll_base_module_2_2_0_t *module) {
   int res;
@@ -374,3 +392,21 @@ int ompi_coll_libnbc_ibcast_inter(void *buffer, int count, MPI_Datatype datatype
 
   return OMPI_SUCCESS;
 }
+
+int ompi_coll_libnbc_ibcast_inter(void *buffer, int count, MPI_Datatype datatype, int root,
+                                  struct ompi_communicator_t *comm, ompi_request_t ** request,
+                                  struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_ibcast_inter(buffer, count, datatype, root, comm, request, module);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+  
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}

ompi/mca/coll/libnbc/nbc_iexscan.c

@@ -45,7 +45,7 @@ int NBC_Scan_args_compare(NBC_Scan_args *a, NBC_Scan_args *b, void *param) {
  * 3. all but rank p-1 do sends to it's right neigbor and exits
  *
  */
-int ompi_coll_libnbc_iexscan(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
+static int nbc_iexscan(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
                        struct ompi_communicator_t *comm, ompi_request_t ** request,
                        struct mca_coll_base_module_2_2_0_t *module) {
     int rank, p, res;
@@ -191,3 +191,21 @@ int ompi_coll_libnbc_iexscan(const void* sendbuf, void* recvbuf, int count, MPI_
 
     return OMPI_SUCCESS;
 }
+
+int ompi_coll_libnbc_iexscan(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
+                             struct ompi_communicator_t *comm, ompi_request_t ** request,
+                             struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_iexscan(sendbuf, recvbuf, count, datatype, op, comm, request, module);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+  
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}

ompi/mca/coll/libnbc/nbc_igather.c

@@ -43,7 +43,7 @@ int NBC_Gather_args_compare(NBC_Gather_args *a, NBC_Gather_args *b, void *param)
 }
 #endif
 
-int ompi_coll_libnbc_igather(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf,
+static int nbc_igather(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf,
                        int recvcount, MPI_Datatype recvtype, int root,
                        struct ompi_communicator_t *comm, ompi_request_t ** request,
                        struct mca_coll_base_module_2_2_0_t *module) {
@@ -169,7 +169,27 @@ int ompi_coll_libnbc_igather(const void* sendbuf, int sendcount, MPI_Datatype se
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_igather_inter (const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf,
+int ompi_coll_libnbc_igather(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf,
+                             int recvcount, MPI_Datatype recvtype, int root,
+                             struct ompi_communicator_t *comm, ompi_request_t ** request,
+                             struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_igather(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root,
+                          comm, request, module);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+  
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}
+
+static int nbc_igather_inter (const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf,
                               int recvcount, MPI_Datatype recvtype, int root,
                               struct ompi_communicator_t *comm, ompi_request_t ** request,
                               struct mca_coll_base_module_2_2_0_t *module) {
@@ -228,3 +248,23 @@ int ompi_coll_libnbc_igather_inter (const void* sendbuf, int sendcount, MPI_Data
 
     return OMPI_SUCCESS;
 }
+
+int ompi_coll_libnbc_igather_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype, void* recvbuf,
+                                   int recvcount, MPI_Datatype recvtype, int root,
+                                   struct ompi_communicator_t *comm, ompi_request_t ** request,
+                                   struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_igather_inter(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root,
+                                comm, request, module);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+  
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}

ompi/mca/coll/libnbc/nbc_igatherv.c

@@ -28,7 +28,7 @@
  * would not be sufficient ... we simply do not cache it */
 
 
-int ompi_coll_libnbc_igatherv(const void* sendbuf, int sendcount, MPI_Datatype sendtype,
+static int nbc_igatherv(const void* sendbuf, int sendcount, MPI_Datatype sendtype,
                         void* recvbuf, const int *recvcounts, const int *displs, MPI_Datatype recvtype,
                         int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
                         struct mca_coll_base_module_2_2_0_t *module) {
@@ -104,7 +104,27 @@ int ompi_coll_libnbc_igatherv(const void* sendbuf, int sendcount, MPI_Datatype s
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_igatherv_inter (const void* sendbuf, int sendcount, MPI_Datatype sendtype,
+int ompi_coll_libnbc_igatherv(const void* sendbuf, int sendcount, MPI_Datatype sendtype,
+                              void* recvbuf, const int *recvcounts, const int *displs, MPI_Datatype recvtype,
+                              int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
+                              struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_igatherv(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, root,
+                           comm, request, module);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+  
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}
+
+static int nbc_igatherv_inter (const void* sendbuf, int sendcount, MPI_Datatype sendtype,
                                void* recvbuf, const int *recvcounts, const int *displs, MPI_Datatype recvtype,
                                int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                struct mca_coll_base_module_2_2_0_t *module) {
@@ -163,3 +183,23 @@ int ompi_coll_libnbc_igatherv_inter (const void* sendbuf, int sendcount, MPI_Dat
 
   return OMPI_SUCCESS;
 }
+
+int ompi_coll_libnbc_igatherv_inter(const void* sendbuf, int sendcount, MPI_Datatype sendtype,
+                                    void* recvbuf, const int *recvcounts, const int *displs, MPI_Datatype recvtype,
+                                    int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
+                                    struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_igatherv_inter(sendbuf, sendcount, sendtype, recvbuf, recvcounts, displs, recvtype, root,
+                                 comm, request, module);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+  
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}

ompi/mca/coll/libnbc/nbc_ineighbor_allgather.c

@@ -43,7 +43,7 @@ int NBC_Ineighbor_allgather_args_compare(NBC_Ineighbor_allgather_args *a, NBC_In
 #endif
 
 
-int ompi_coll_libnbc_ineighbor_allgather(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
+static int nbc_ineighbor_allgather(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
                                    int rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm,
                                    ompi_request_t ** request, struct mca_coll_base_module_2_2_0_t *module) {
   int res, indegree, outdegree, *srcs, *dsts;
@@ -160,3 +160,172 @@ int ompi_coll_libnbc_ineighbor_allgather(const void *sbuf, int scount, MPI_Datat
 
   return OMPI_SUCCESS;
 }
+
+int ompi_coll_libnbc_ineighbor_allgather(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
+                                         int rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm,
+                                         ompi_request_t ** request, struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_ineighbor_allgather(sbuf, scount, stype, rbuf, rcount, rtype, comm, request, module);
+    if (OPAL_LIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}
+
+/* better binomial bcast
+ * working principle:
+ * - each node gets a virtual rank vrank
+ * - the 'root' node get vrank 0
+ * - node 0 gets the vrank of the 'root'
+ * - all other ranks stay identical (they do not matter)
+ *
+ * Algorithm:
+ * - each node with vrank > 2^r and vrank < 2^r+1 receives from node
+ *   vrank - 2^r (vrank=1 receives from 0, vrank 0 receives never)
+ * - each node sends each round r to node vrank + 2^r
+ * - a node stops to send if 2^r > commsize
+ */
+#define RANK2VRANK(rank, vrank, root) \
+{ \
+  vrank = rank; \
+  if (rank == 0) vrank = root; \
+  if (rank == root) vrank = 0; \
+}
+#define VRANK2RANK(rank, vrank, root) \
+{ \
+  rank = vrank; \
+  if (vrank == 0) rank = root; \
+  if (vrank == root) rank = 0; \
+}
+static inline int bcast_sched_binomial(int rank, int p, int root, NBC_Schedule *schedule, void *buffer, int count, MPI_Datatype datatype) {
+  int maxr, vrank, peer, res;
+
+  maxr = (int)ceil((log((double)p)/LOG2));
+
+  RANK2VRANK(rank, vrank, root);
+
+  /* receive from the right hosts  */
+  if (vrank != 0) {
+    for (int r = 0 ; r < maxr ; ++r) {
+      if ((vrank >= (1 << r)) && (vrank < (1 << (r + 1)))) {
+        VRANK2RANK(peer, vrank - (1 << r), root);
+        res = NBC_Sched_recv (buffer, false, count, datatype, peer, schedule, false);
+        if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+          return res;
+        }
+      }
+    }
+
+    res = NBC_Sched_barrier (schedule);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+      return res;
+    }
+  }
+
+  /* now send to the right hosts */
+  for (int r = 0 ; r < maxr ; ++r) {
+    if (((vrank + (1 << r) < p) && (vrank < (1 << r))) || (vrank == 0)) {
+      VRANK2RANK(peer, vrank + (1 << r), root);
+      res = NBC_Sched_send (buffer, false, count, datatype, peer, schedule, false);
+      if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        return res;
+      }
+    }
+  }
+
+  return OMPI_SUCCESS;
+}
+
+/* simple linear MPI_Ibcast */
+static inline int bcast_sched_linear(int rank, int p, int root, NBC_Schedule *schedule, void *buffer, int count, MPI_Datatype datatype) {
+  int res;
+
+  /* send to all others */
+  if(rank == root) {
+    for (int peer = 0 ; peer < p ; ++peer) {
+      if (peer != root) {
+        /* send msg to peer */
+        res = NBC_Sched_send (buffer, false, count, datatype, peer, schedule, false);
+        if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+          return res;
+        }
+      }
+    }
+  } else {
+    /* recv msg from root */
+    res = NBC_Sched_recv (buffer, false, count, datatype, root, schedule, false);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+      return res;
+    }
+  }
+
+  return OMPI_SUCCESS;
+}
+
+/* simple chained MPI_Ibcast */
+static inline int bcast_sched_chain(int rank, int p, int root, NBC_Schedule *schedule, void *buffer, int count, MPI_Datatype datatype, int fragsize, size_t size) {
+  int res, vrank, rpeer, speer, numfrag, fragcount, thiscount;
+  MPI_Aint ext;
+  char *buf;
+
+  RANK2VRANK(rank, vrank, root);
+  VRANK2RANK(rpeer, vrank-1, root);
+  VRANK2RANK(speer, vrank+1, root);
+  res = ompi_datatype_type_extent(datatype, &ext);
+  if (MPI_SUCCESS != res) {
+    NBC_Error("MPI Error in ompi_datatype_type_extent() (%i)", res);
+    return res;
+  }
+
+  if (count == 0) {
+    return OMPI_SUCCESS;
+  }
+
+  numfrag = count * size/fragsize;
+  if ((count * size) % fragsize != 0) {
+    numfrag++;
+  }
+
+  fragcount = count/numfrag;
+
+  for (int fragnum = 0 ; fragnum < numfrag ; ++fragnum) {
+    buf = (char *) buffer + fragnum * fragcount * ext;
+    thiscount = fragcount;
+    if (fragnum == numfrag-1) {
+      /* last fragment may not be full */
+      thiscount = count - fragcount * fragnum;
+    }
+
+    /* root does not receive */
+    if (vrank != 0) {
+      res = NBC_Sched_recv (buf, false, thiscount, datatype, rpeer, schedule, true);
+      if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        return res;
+      }
+    }
+
+    /* last rank does not send */
+    if (vrank != p-1) {
+      res = NBC_Sched_send (buf, false, thiscount, datatype, speer, schedule, false);
+      if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        return res;
+      }
+
+      /* this barrier here seems awaward but isn't!!!! */
+      if (vrank == 0)  {
+        res = NBC_Sched_barrier (schedule);
+        if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+          return res;
+        }
+      }
+    }
+  }
+
+  return OMPI_SUCCESS;
+}

ompi/mca/coll/libnbc/nbc_ineighbor_allgatherv.c

@@ -43,7 +43,7 @@ int NBC_Ineighbor_allgatherv_args_compare(NBC_Ineighbor_allgatherv_args *a, NBC_
 #endif
 
 
-int ompi_coll_libnbc_ineighbor_allgatherv(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
+static int nbc_ineighbor_allgatherv(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
                                     const int *rcounts, const int *displs, MPI_Datatype rtype,
                                     struct ompi_communicator_t *comm, ompi_request_t ** request,
                                     struct mca_coll_base_module_2_2_0_t *module) {
@@ -162,3 +162,21 @@ int ompi_coll_libnbc_ineighbor_allgatherv(const void *sbuf, int scount, MPI_Data
 
   return OMPI_SUCCESS;
 }
+
+int ompi_coll_libnbc_ineighbor_allgatherv(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
+					  const int *rcounts, const int *displs, MPI_Datatype rtype,
+					  struct ompi_communicator_t *comm, ompi_request_t ** request,
+					  struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_ineighbor_allgatherv(sbuf, scount, stype, rbuf, rcounts, displs, rtype, comm, request, module);
+    if (OPAL_LIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}

ompi/mca/coll/libnbc/nbc_ineighbor_alltoall.c

@@ -40,7 +40,7 @@ int NBC_Ineighbor_alltoall_args_compare(NBC_Ineighbor_alltoall_args *a, NBC_Inei
 }
 #endif
 
-int ompi_coll_libnbc_ineighbor_alltoall(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
+static int nbc_ineighbor_alltoall(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
                                   int rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm,
                                   ompi_request_t ** request, struct mca_coll_base_module_2_2_0_t *module) {
   int res, indegree, outdegree, *srcs, *dsts;
@@ -164,3 +164,20 @@ int ompi_coll_libnbc_ineighbor_alltoall(const void *sbuf, int scount, MPI_Dataty
 
   return OMPI_SUCCESS;
 }
+
+int ompi_coll_libnbc_ineighbor_alltoall(const void *sbuf, int scount, MPI_Datatype stype, void *rbuf,
+                                        int rcount, MPI_Datatype rtype, struct ompi_communicator_t *comm,
+                                        ompi_request_t ** request, struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_ineighbor_alltoall(sbuf, scount, stype, rbuf, rcount, rtype, comm, request, module);
+    if (OPAL_LIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}

ompi/mca/coll/libnbc/nbc_ineighbor_alltoallv.c

@@ -43,7 +43,7 @@ int NBC_Ineighbor_alltoallv_args_compare(NBC_Ineighbor_alltoallv_args *a, NBC_In
 #endif
 
 
-int ompi_coll_libnbc_ineighbor_alltoallv(const void *sbuf, const int *scounts, const int *sdispls, MPI_Datatype stype,
+static int nbc_ineighbor_alltoallv(const void *sbuf, const int *scounts, const int *sdispls, MPI_Datatype stype,
                                    void *rbuf, const int *rcounts, const int *rdispls, MPI_Datatype rtype,
                                    struct ompi_communicator_t *comm, ompi_request_t ** request,
                                    struct mca_coll_base_module_2_2_0_t *module) {
@@ -169,3 +169,21 @@ int ompi_coll_libnbc_ineighbor_alltoallv(const void *sbuf, const int *scounts, c
 
   return OMPI_SUCCESS;
 }
+
+int ompi_coll_libnbc_ineighbor_alltoallv(const void *sbuf, const int *scounts, const int *sdispls, MPI_Datatype stype,
+                                         void *rbuf, const int *rcounts, const int *rdispls, MPI_Datatype rtype,
+                                         struct ompi_communicator_t *comm, ompi_request_t ** request,
+                                         struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_ineighbor_alltoallv(sbuf, scounts, sdispls, stype, rbuf, rcounts, rdispls, rtype, comm, request, module);
+    if (OPAL_LIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}

ompi/mca/coll/libnbc/nbc_ineighbor_alltoallw.c

@@ -42,7 +42,7 @@ int NBC_Ineighbor_alltoallw_args_compare(NBC_Ineighbor_alltoallw_args *a, NBC_In
 }
 #endif
 
-int ompi_coll_libnbc_ineighbor_alltoallw(const void *sbuf, const int *scounts, const MPI_Aint *sdisps, struct ompi_datatype_t * const *stypes,
+static int nbc_ineighbor_alltoallw(const void *sbuf, const int *scounts, const MPI_Aint *sdisps, struct ompi_datatype_t * const *stypes,
                                    void *rbuf, const int *rcounts, const MPI_Aint *rdisps, struct ompi_datatype_t * const *rtypes,
                                    struct ompi_communicator_t *comm, ompi_request_t ** request,
                                    struct mca_coll_base_module_2_2_0_t *module) {
@@ -154,3 +154,21 @@ int ompi_coll_libnbc_ineighbor_alltoallw(const void *sbuf, const int *scounts, c
 
   return OMPI_SUCCESS;
 }
+
+int ompi_coll_libnbc_ineighbor_alltoallw(const void *sbuf, const int *scounts, const MPI_Aint *sdisps, struct ompi_datatype_t * const *stypes,
+                                         void *rbuf, const int *rcounts, const MPI_Aint *rdisps, struct ompi_datatype_t * const *rtypes,
+                                         struct ompi_communicator_t *comm, ompi_request_t ** request,
+                                         struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_ineighbor_alltoallw(sbuf, scounts, sdisps, stypes, rbuf, rcounts, rdisps, rtypes, comm, request, module);
+    if (OPAL_LIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}

ompi/mca/coll/libnbc/nbc_ireduce.c

@@ -52,7 +52,7 @@ int NBC_Reduce_args_compare(NBC_Reduce_args *a, NBC_Reduce_args *b, void *param)
 #endif
 
 /* the non-blocking reduce */
-int ompi_coll_libnbc_ireduce(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype,
+static int nbc_ireduce(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype,
                        MPI_Op op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
                        struct mca_coll_base_module_2_2_0_t *module) {
   int rank, p, res, segsize;
@@ -203,7 +203,24 @@ int ompi_coll_libnbc_ireduce(const void* sendbuf, void* recvbuf, int count, MPI_
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_ireduce_inter(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype,
+int ompi_coll_libnbc_ireduce(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype,
+                             MPI_Op op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
+                             struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_ireduce(sendbuf, recvbuf, count, datatype, op, root, comm, request, module);
+    if (OPAL_LIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}
+
+static int nbc_ireduce_inter(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype,
                              MPI_Op op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
                              struct mca_coll_base_module_2_2_0_t *module) {
   int rank, res, rsize;
@@ -251,6 +268,23 @@ int ompi_coll_libnbc_ireduce_inter(const void* sendbuf, void* recvbuf, int count
   return OMPI_SUCCESS;
 }
 
+int ompi_coll_libnbc_ireduce_inter(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype,
+                                   MPI_Op op, int root, struct ompi_communicator_t *comm, ompi_request_t ** request,
+                                   struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_ireduce_inter(sendbuf, recvbuf, count, datatype, op, root, comm, request, module);
+    if (OPAL_LIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}
+
 
 /* binomial reduce
  * if op is not commutative, reduce on rank 0, and then send the result to root rank

ompi/mca/coll/libnbc/nbc_ireduce_scatter.c

@@ -41,7 +41,7 @@
  *
  */
 
-int ompi_coll_libnbc_ireduce_scatter(const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype,
+static int nbc_ireduce_scatter(const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype,
                                MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                struct mca_coll_base_module_2_2_0_t *module) {
   int peer, rank, maxr, p, res, count;
@@ -203,7 +203,23 @@ int ompi_coll_libnbc_ireduce_scatter(const void* sendbuf, void* recvbuf, const i
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_ireduce_scatter_inter (const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype,
+int ompi_coll_libnbc_ireduce_scatter (const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype,
+                                      MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request,
+                                      struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_ireduce_scatter(sendbuf, recvbuf, recvcounts, datatype, op, comm, request, module);
+    if (OPAL_LIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}
+static int nbc_ireduce_scatter_inter (const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype,
                                       MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                       struct mca_coll_base_module_2_2_0_t *module) {
   int rank, res, count, lsize, rsize;
@@ -327,3 +343,20 @@ int ompi_coll_libnbc_ireduce_scatter_inter (const void* sendbuf, void* recvbuf,
 
   return OMPI_SUCCESS;
 }
+
+int ompi_coll_libnbc_ireduce_scatter_inter (const void* sendbuf, void* recvbuf, const int *recvcounts, MPI_Datatype datatype,
+                                            MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request,
+                                            struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_ireduce_scatter_inter(sendbuf, recvbuf, recvcounts, datatype, op, comm, request, module);
+    if (OPAL_LIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}

ompi/mca/coll/libnbc/nbc_ireduce_scatter_block.c

@@ -39,7 +39,7 @@
  *
  */
 
-int ompi_coll_libnbc_ireduce_scatter_block(const void* sendbuf, void* recvbuf, int recvcount, MPI_Datatype datatype,
+static int nbc_ireduce_scatter_block(const void* sendbuf, void* recvbuf, int recvcount, MPI_Datatype datatype,
                                      MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request,
                                      struct mca_coll_base_module_2_2_0_t *module) {
   int peer, rank, maxr, p, res, count;
@@ -205,7 +205,24 @@ int ompi_coll_libnbc_ireduce_scatter_block(const void* sendbuf, void* recvbuf, i
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_ireduce_scatter_block_inter(const void *sendbuf, void *recvbuf, int rcount, struct ompi_datatype_t *dtype,
+int ompi_coll_libnbc_ireduce_scatter_block(const void* sendbuf, void* recvbuf, int recvcount, MPI_Datatype datatype,
+                                           MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request,
+                                           struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_ireduce_scatter_block(sendbuf, recvbuf, recvcount, datatype, op, comm, request, module);
+    if (OPAL_LIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}
+
+static int nbc_ireduce_scatter_block_inter(const void *sendbuf, void *recvbuf, int rcount, struct ompi_datatype_t *dtype,
                                            struct ompi_op_t *op, struct ompi_communicator_t *comm,
                                            ompi_request_t **request, struct mca_coll_base_module_2_2_0_t *module) {
   int rank, res, count, lsize, rsize;
@@ -325,3 +342,20 @@ int ompi_coll_libnbc_ireduce_scatter_block_inter(const void *sendbuf, void *recv
 
   return OMPI_SUCCESS;
 }
+
+int ompi_coll_libnbc_ireduce_scatter_block_inter(const void* sendbuf, void* recvbuf, int recvcount, MPI_Datatype datatype,
+                                                 MPI_Op op, struct ompi_communicator_t *comm, ompi_request_t ** request,
+                                                 struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_ireduce_scatter_block_inter(sendbuf, recvbuf, recvcount, datatype, op, comm, request, module);
+    if (OPAL_LIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}

ompi/mca/coll/libnbc/nbc_iscan.c

@@ -45,7 +45,7 @@ int NBC_Scan_args_compare(NBC_Scan_args *a, NBC_Scan_args *b, void *param) {
  * 3. all but rank p-1 do sends to it's right neighbor and exits
  *
  */
-int ompi_coll_libnbc_iscan(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
+static int nbc_iscan(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
                      struct ompi_communicator_t *comm, ompi_request_t ** request,
                      struct mca_coll_base_module_2_2_0_t *module) {
   int rank, p, res;
@@ -168,3 +168,20 @@ int ompi_coll_libnbc_iscan(const void* sendbuf, void* recvbuf, int count, MPI_Da
 
   return OMPI_SUCCESS;
 }
+
+int ompi_coll_libnbc_iscan(const void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op,
+                           struct ompi_communicator_t *comm, ompi_request_t ** request,
+                           struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_iscan(sendbuf, recvbuf, count, datatype, op, comm, request, module);
+    if (OPAL_LIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}

ompi/mca/coll/libnbc/nbc_iscatter.c

@@ -44,7 +44,7 @@ int NBC_Scatter_args_compare(NBC_Scatter_args *a, NBC_Scatter_args *b, void *par
 #endif
 
 /* simple linear MPI_Iscatter */
-int ompi_coll_libnbc_iscatter (const void* sendbuf, int sendcount, MPI_Datatype sendtype,
+static int nbc_iscatter (const void* sendbuf, int sendcount, MPI_Datatype sendtype,
                          void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
                          struct ompi_communicator_t *comm, ompi_request_t ** request,
                          struct mca_coll_base_module_2_2_0_t *module) {
@@ -166,7 +166,25 @@ int ompi_coll_libnbc_iscatter (const void* sendbuf, int sendcount, MPI_Datatype
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_iscatter_inter (const void* sendbuf, int sendcount, MPI_Datatype sendtype,
+int ompi_coll_libnbc_iscatter (const void* sendbuf, int sendcount, MPI_Datatype sendtype,
+                               void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
+                               struct ompi_communicator_t *comm, ompi_request_t ** request,
+                               struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_iscatter(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, request, module);
+    if (OPAL_LIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}
+
+static int nbc_iscatter_inter (const void* sendbuf, int sendcount, MPI_Datatype sendtype,
                                void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
                                struct ompi_communicator_t *comm, ompi_request_t ** request,
                                struct mca_coll_base_module_2_2_0_t *module) {
@@ -225,3 +243,21 @@ int ompi_coll_libnbc_iscatter_inter (const void* sendbuf, int sendcount, MPI_Dat
 
     return OMPI_SUCCESS;
 }
+
+int ompi_coll_libnbc_iscatter_inter (const void* sendbuf, int sendcount, MPI_Datatype sendtype,
+                                     void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
+                                     struct ompi_communicator_t *comm, ompi_request_t ** request,
+                                     struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_iscatter_inter(sendbuf, sendcount, sendtype, recvbuf, recvcount, recvtype, root, comm, request, module);
+    if (OPAL_LIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}

ompi/mca/coll/libnbc/nbc_iscatterv.c

@@ -27,7 +27,7 @@
  * would not be sufficient ... we simply do not cache it */
 
 /* simple linear MPI_Iscatterv */
-int ompi_coll_libnbc_iscatterv(const void* sendbuf, const int *sendcounts, const int *displs, MPI_Datatype sendtype,
+static int nbc_iscatterv(const void* sendbuf, const int *sendcounts, const int *displs, MPI_Datatype sendtype,
                          void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
                          struct ompi_communicator_t *comm, ompi_request_t ** request,
                          struct mca_coll_base_module_2_2_0_t *module) {
@@ -101,7 +101,25 @@ int ompi_coll_libnbc_iscatterv(const void* sendbuf, const int *sendcounts, const
   return OMPI_SUCCESS;
 }
 
-int ompi_coll_libnbc_iscatterv_inter (const void* sendbuf, const int *sendcounts, const int *displs, MPI_Datatype sendtype,
+int ompi_coll_libnbc_iscatterv(const void* sendbuf, const int *sendcounts, const int *displs, MPI_Datatype sendtype,
+                               void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
+                               struct ompi_communicator_t *comm, ompi_request_t ** request,
+                               struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_iscatterv(sendbuf, sendcounts, displs, sendtype, recvbuf, recvcount, recvtype, root, comm, request, module);
+    if (OPAL_LIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}
+
+static int nbc_iscatterv_inter (const void* sendbuf, const int *sendcounts, const int *displs, MPI_Datatype sendtype,
                                 void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
                                 struct ompi_communicator_t *comm, ompi_request_t ** request,
                                 struct mca_coll_base_module_2_2_0_t *module) {
@@ -159,3 +177,21 @@ int ompi_coll_libnbc_iscatterv_inter (const void* sendbuf, const int *sendcounts
 
     return OMPI_SUCCESS;
 }
+
+int ompi_coll_libnbc_iscatterv_inter(const void* sendbuf, const int *sendcounts, const int *displs, MPI_Datatype sendtype,
+                                     void* recvbuf, int recvcount, MPI_Datatype recvtype, int root,
+                                     struct ompi_communicator_t *comm, ompi_request_t ** request,
+                                     struct mca_coll_base_module_2_2_0_t *module) {
+    int res = nbc_iscatterv_inter(sendbuf, sendcounts, displs, sendtype, recvbuf, recvcount, recvtype, root, comm, request, module);
+    if (OPAL_LIKELY(OMPI_SUCCESS != res)) {
+        return res;
+    }
+    res = NBC_Start(*(ompi_coll_libnbc_request_t **)request);
+    if (OPAL_UNLIKELY(OMPI_SUCCESS != res)) {
+        NBC_Return_handle ((ompi_coll_libnbc_request_t *)request);
+        *request = &ompi_request_null.request;
+        return res;
+    }
+
+    return OMPI_SUCCESS;
+}