Ibarrier works! :)

This commit was SVN r26680.

ompi/mca/coll/libnbc/coll_libnbc.h

@@ -25,20 +25,6 @@
 
 BEGIN_C_DECLS
 
-/* Globally exported variables */
-OMPI_MODULE_DECLSPEC extern const mca_coll_base_component_2_0_0_t mca_coll_libnbc_component;
-
-int ompi_coll_libnbc_ibarrier(struct ompi_communicator_t *comm, ompi_request_t ** request,
-                              struct mca_coll_base_module_2_0_0_t *module);
-
-
-struct ompi_coll_libnbc_module_t {
-    mca_coll_base_module_t super;
-};
-typedef struct ompi_coll_libnbc_module_t ompi_coll_libnbc_module_t;
-OBJ_CLASS_DECLARATION(ompi_coll_libnbc_module_t);
-
-
 /* Function return codes  */
 #define NBC_OK 0 /* everything went fine */
 #define NBC_SUCCESS 0 /* everything went fine (MPI compliant :) */
@@ -54,14 +40,20 @@ OBJ_CLASS_DECLARATION(ompi_coll_libnbc_module_t);
 /* number of implemented collective functions */
 #define NBC_NUM_COLL 19
 
-/* a schedule is basically a pointer to some memory location where the
- * schedule array resides */ 
-typedef void* NBC_Schedule;
+struct ompi_coll_libnbc_component_t {
+    mca_coll_base_component_2_0_0_t super;
+    ompi_free_list_t requests;
+    opal_list_t active_requests;
+    uint32_t active_comms;
+};
+typedef struct ompi_coll_libnbc_component_t ompi_coll_libnbc_component_t;
 
-/* used to hang off a communicator */
-typedef struct {
-  MPI_Comm mycomm; /* save the shadow communicator here */
-  int tag;
+/* Globally exported variables */
+OMPI_MODULE_DECLSPEC extern ompi_coll_libnbc_component_t mca_coll_libnbc_component;
+
+struct ompi_coll_libnbc_module_t {
+    mca_coll_base_module_t super;
+    int tag;
 #ifdef NBC_CACHE_SCHEDULE
   void *NBC_Dict[NBC_NUM_COLL]; /* this should point to a struct
                                       hb_tree, but since this is a
@@ -70,12 +62,19 @@ typedef struct {
                                       it ...*/
   int NBC_Dict_size[NBC_NUM_COLL];
 #endif
-} NBC_Comminfo;
+};
+typedef struct ompi_coll_libnbc_module_t ompi_coll_libnbc_module_t;
+OBJ_CLASS_DECLARATION(ompi_coll_libnbc_module_t);
+
+typedef ompi_coll_libnbc_module_t NBC_Comminfo;
+
+/* a schedule is basically a pointer to some memory location where the
+ * schedule array resides */ 
+typedef void* NBC_Schedule;
 
 struct ompi_coll_libnbc_request_t {
     ompi_request_t super;
     MPI_Comm comm;
-    MPI_Comm mycomm;
     long row_offset;
     int tag;
     volatile int req_count;
@@ -92,6 +91,31 @@ OBJ_CLASS_DECLARATION(ompi_coll_libnbc_request_t);
 
 typedef ompi_coll_libnbc_request_t NBC_Handle;
 
+
+#define OMPI_COLL_LIBNBC_REQUEST_ALLOC(comm, req, rc)                   \
+    do {                                                                \
+        ompi_free_list_item_t *item;                                    \
+        OMPI_FREE_LIST_WAIT(&mca_coll_libnbc_component.requests,        \
+                            item, rc);                                  \
+        req = (ompi_coll_libnbc_request_t*) item;                       \
+        OMPI_REQUEST_INIT(&req->super, false);                          \
+        req->super.req_mpi_object.comm = comm;                          \
+        req->super.req_complete = false;                                \
+        req->super.req_state = OMPI_REQUEST_ACTIVE;                     \
+    } while (0)
+
+#define OMPI_COLL_LIBNBC_REQUEST_RETURN(req)                            \
+    do {                                                                \
+        OMPI_REQUEST_FINI(&request->super);                             \
+        OMPI_FREE_LIST_RETURN(&mca_coll_libnbc_component.requests,      \
+                              (ompi_free_list_item_t*) req);            \
+    } while (0)
+
+int NBC_Init_comm(MPI_Comm comm, ompi_coll_libnbc_module_t *module);
+int NBC_Progress(NBC_Handle *handle);
+int ompi_coll_libnbc_ibarrier(struct ompi_communicator_t *comm, ompi_request_t ** request,
+                              struct mca_coll_base_module_2_0_0_t *module);
+
 END_C_DECLS
 
 #endif /* MCA_COLL_LIBNBC_EXPORT_H */

ompi/mca/coll/libnbc/coll_libnbc_component.c

@@ -35,55 +35,86 @@ const char *mca_coll_libnbc_component_version_string =
 static int libnbc_priority = 10;
 
 
+static int libnbc_open(void);
+static int libnbc_close(void);
 static int libnbc_register(void);
 static int libnbc_init_query(bool, bool);
 static mca_coll_base_module_t *libnbc_comm_query(struct ompi_communicator_t *, int *);
 static int libnbc_module_enable(mca_coll_base_module_t *, struct ompi_communicator_t *);
-
+static int libnbc_progress(void);
 
 /*
  * Instantiate the public struct with all of our public information
  * and pointers to our public functions in it
  */
 
-const mca_coll_base_component_2_0_0_t mca_coll_libnbc_component = {
-
-    /* First, the mca_component_t struct containing meta information
-     * about the component itself */
-
+ompi_coll_libnbc_component_t mca_coll_libnbc_component = {
     {
-     MCA_COLL_BASE_VERSION_2_0_0,
+        /* First, the mca_component_t struct containing meta information
+         * about the component itself */
+        {
+            MCA_COLL_BASE_VERSION_2_0_0,
 
-     /* Component name and version */
-     "libnbc",
-     OMPI_MAJOR_VERSION,
-     OMPI_MINOR_VERSION,
-     OMPI_RELEASE_VERSION,
+            /* Component name and version */
+            "libnbc",
+            OMPI_MAJOR_VERSION,
+            OMPI_MINOR_VERSION,
+            OMPI_RELEASE_VERSION,
 
-     /* Component open and close functions */
-     NULL,
-     NULL,
-     NULL,
-     libnbc_register
-    },
-    {
-        /* The component is checkpoint ready */
-        MCA_BASE_METADATA_PARAM_CHECKPOINT
-    },
+            /* Component open and close functions */
+            libnbc_open,
+            libnbc_close,
+            NULL,
+            libnbc_register
+        },
+        {
+            /* The component is checkpoint ready */
+            MCA_BASE_METADATA_PARAM_CHECKPOINT
+        },
 
-    /* Initialization / querying functions */
-
-    libnbc_init_query,
-    libnbc_comm_query
+        /* Initialization / querying functions */
+        libnbc_init_query,
+        libnbc_comm_query
+    }
 };
 
 
+static int
+libnbc_open(void)
+{
+    int ret;
+
+    OBJ_CONSTRUCT(&mca_coll_libnbc_component.requests, ompi_free_list_t);
+    ret = ompi_free_list_init(&mca_coll_libnbc_component.requests,
+                              sizeof(ompi_coll_libnbc_request_t),
+                              OBJ_CLASS(ompi_coll_libnbc_request_t),
+                              0,
+                              -1,
+                              8,
+                              NULL);
+    if (OMPI_SUCCESS != ret) return ret;
+
+    OBJ_CONSTRUCT(&mca_coll_libnbc_component.active_requests, opal_list_t);
+    mca_coll_libnbc_component.active_comms = 0;
+
+    return OMPI_SUCCESS;
+}
+
+static int
+libnbc_close(void)
+{
+    OBJ_DESTRUCT(&mca_coll_libnbc_component.requests);
+
+    return OMPI_SUCCESS;
+}
+
+
 static int
 libnbc_register(void)
 {
     /* Use a low priority, but allow other components to be lower */
 
-    mca_base_param_reg_int(&mca_coll_libnbc_component.collm_version,
+    mca_base_param_reg_int(&mca_coll_libnbc_component.super.collm_version,
                            "priority",
                            "Priority of the libnbc coll component",
                            false, false, libnbc_priority,
@@ -130,6 +161,11 @@ libnbc_comm_query(struct ompi_communicator_t *comm,
 
     module->super.ft_event = NULL;
 
+    if (OMPI_SUCCESS != NBC_Init_comm(comm, module)) {
+        OBJ_RELEASE(module);
+        return NULL;
+    }
+
     return &(module->super);
 }
 
@@ -142,18 +178,47 @@ libnbc_module_enable(mca_coll_base_module_t *module,
                      struct ompi_communicator_t *comm)
 {
     /* All done */
+    if (0 == mca_coll_libnbc_component.active_comms++) {
+        opal_progress_register(libnbc_progress);
+    }
     return OMPI_SUCCESS;
 }
 
 
+static int
+libnbc_progress(void)
+{
+    opal_list_item_t *item;
+
+    for (item = opal_list_get_first(&mca_coll_libnbc_component.active_requests) ;
+         item != opal_list_get_end(&mca_coll_libnbc_component.active_requests) ;
+         item = opal_list_get_next(item)) {
+        ompi_coll_libnbc_request_t* request = (ompi_coll_libnbc_request_t*) item;
+        if (NBC_OK == NBC_Progress(request)) {
+            request->super.req_status.MPI_ERROR = OMPI_SUCCESS;
+            /* done, remove */
+            item = opal_list_remove_item(&mca_coll_libnbc_component.active_requests,
+                                         &request->super.super.super);
+        }
+        item = opal_list_get_next(item);
+    }
+
+    return 0;
+}
+
+
 static void
 libnbc_module_construct(ompi_coll_libnbc_module_t *module)
 {
 }
 
+
 static void
 libnbc_module_destruct(ompi_coll_libnbc_module_t *module)
 {
+    if (0 == --mca_coll_libnbc_component.active_comms) {
+        opal_progress_unregister(libnbc_progress);
+    }
 }
 
 
@@ -161,3 +226,44 @@ OBJ_CLASS_INSTANCE(ompi_coll_libnbc_module_t,
                    mca_coll_base_module_t,
                    libnbc_module_construct,
                    libnbc_module_destruct);
+
+
+static int
+request_cancel(struct ompi_request_t *request, int complete)
+{
+    return MPI_ERR_REQUEST;
+}
+
+
+static int
+request_free(struct ompi_request_t **ompi_req)
+{
+    ompi_coll_libnbc_request_t *request = 
+        (ompi_coll_libnbc_request_t*) *ompi_req;
+
+    if (true != request->super.req_complete) {
+        return MPI_ERR_REQUEST;
+    }
+
+    OMPI_COLL_LIBNBC_REQUEST_RETURN(request);
+
+    *ompi_req = MPI_REQUEST_NULL;
+
+    return OMPI_SUCCESS;
+}
+
+
+static void
+request_construct(ompi_coll_libnbc_request_t *request)
+{
+    request->super.req_type = OMPI_REQUEST_COLL;
+    request->super.req_status._cancelled = 0;
+    request->super.req_free = request_free;
+    request->super.req_cancel = request_cancel;
+}
+
+
+OBJ_CLASS_INSTANCE(ompi_coll_libnbc_request_t, 
+                   ompi_request_t,
+                   request_construct,
+                   NULL);

ompi/mca/coll/libnbc/nbc.c

@@ -25,33 +25,6 @@ void NBC_Print_times(double div) {
 }
 #endif
 
-/* is NBC globally initialized */
-static char GNBC_Initialized=0;
-
-/* the keyval (global) */
-static int gkeyval=MPI_KEYVAL_INVALID; 
-
-static int NBC_Key_copy(MPI_Comm oldcomm, int keyval, void *extra_state, void *attribute_val_in, void *attribute_val_out, int *flag) {
-  /* delete the attribute in the new comm  - it will be created at the
-   * first usage */
-  *flag = 0;
-
-  return MPI_SUCCESS;
-}
-
-static int NBC_Key_delete(MPI_Comm comm, int keyval, void *attribute_val, void *extra_state) {
-  NBC_Comminfo *comminfo;
-
-  if(keyval == gkeyval) {
-    comminfo=(NBC_Comminfo*)attribute_val;
-    free((void*)comminfo);
-  } else {
-    printf("Got wrong keyval!(%i)\n", keyval); 
-  }
-
-  return MPI_SUCCESS;
-}
-
 /* allocates a new schedule array */
 int NBC_Sched_create(NBC_Schedule* schedule) {
   
@@ -299,6 +272,10 @@ static inline int NBC_Free(NBC_Handle* handle) {
     handle->tmpbuf = NULL;
   }
 
+  OPAL_THREAD_LOCK(&ompi_request_lock);
+  ompi_request_complete(&handle->super, true);
+  OPAL_THREAD_UNLOCK(&ompi_request_lock);
+
   return NBC_OK;
 }
 
@@ -416,10 +393,10 @@ static inline int NBC_Start_round(NBC_Handle *handle) {
 #ifdef HAVE_OMPI
         handle->req_array = (MPI_Request*)realloc((void*)handle->req_array, (handle->req_count)*sizeof(MPI_Request));
         NBC_CHECK_NULL(handle->req_array);
-        /*res = MCA_PML_CALL(isend_init(buf1, sendargs->count, sendargs->datatype, sendargs->dest, handle->tag, MCA_PML_BASE_SEND_STANDARD, handle->mycomm, handle->req_array+handle->req_count-1));
-        printf("MPI_Isend(%lu, %i, %lu, %i, %i, %lu) (%i)\n", (unsigned long)buf1, sendargs->count, (unsigned long)sendargs->datatype, sendargs->dest, handle->tag, (unsigned long)handle->mycomm, res);*/
-        res = MPI_Isend(buf1, sendargs->count, sendargs->datatype, sendargs->dest, handle->tag, handle->mycomm, handle->req_array+handle->req_count-1);
-        if(OMPI_SUCCESS != res) { printf("Error in MPI_Isend(%lu, %i, %lu, %i, %i, %lu) (%i)\n", (unsigned long)buf1, sendargs->count, (unsigned long)sendargs->datatype, sendargs->dest, handle->tag, (unsigned long)handle->mycomm, res); ret=res; goto error; }
+        /*res = MCA_PML_CALL(isend_init(buf1, sendargs->count, sendargs->datatype, sendargs->dest, handle->tag, MCA_PML_BASE_SEND_STANDARD, handle->comm, handle->req_array+handle->req_count-1));
+        printf("MPI_Isend(%lu, %i, %lu, %i, %i, %lu) (%i)\n", (unsigned long)buf1, sendargs->count, (unsigned long)sendargs->datatype, sendargs->dest, handle->tag, (unsigned long)handle->comm, res);*/
+        res = MPI_Isend(buf1, sendargs->count, sendargs->datatype, sendargs->dest, handle->tag, handle->comm, handle->req_array+handle->req_count-1);
+        if(OMPI_SUCCESS != res) { printf("Error in MPI_Isend(%lu, %i, %lu, %i, %i, %lu) (%i)\n", (unsigned long)buf1, sendargs->count, (unsigned long)sendargs->datatype, sendargs->dest, handle->tag, (unsigned long)handle->comm, res); ret=res; goto error; }
 #endif
 #ifdef NBC_TIMING
     Isend_time += MPI_Wtime();
@@ -444,10 +421,10 @@ static inline int NBC_Start_round(NBC_Handle *handle) {
 #ifdef HAVE_OMPI
         handle->req_array = (MPI_Request*)realloc((void*)handle->req_array, (handle->req_count)*sizeof(MPI_Request));
         NBC_CHECK_NULL(handle->req_array);
-        /*res = MCA_PML_CALL(irecv(buf1, recvargs->count, recvargs->datatype, recvargs->source, handle->tag, handle->mycomm, handle->req_array+handle->req_count-1)); 
-        printf("MPI_Irecv(%lu, %i, %lu, %i, %i, %lu) (%i)\n", (unsigned long)buf1, recvargs->count, (unsigned long)recvargs->datatype, recvargs->source, handle->tag, (unsigned long)handle->mycomm, res); */
-        res = MPI_Irecv(buf1, recvargs->count, recvargs->datatype, recvargs->source, handle->tag, handle->mycomm, handle->req_array+handle->req_count-1);
-        if(OMPI_SUCCESS != res) { printf("Error in MPI_Irecv(%lu, %i, %lu, %i, %i, %lu) (%i)\n", (unsigned long)buf1, recvargs->count, (unsigned long)recvargs->datatype, recvargs->source, handle->tag, (unsigned long)handle->mycomm, res); ret=res; goto error; }
+        /*res = MCA_PML_CALL(irecv(buf1, recvargs->count, recvargs->datatype, recvargs->source, handle->tag, handle->comm, handle->req_array+handle->req_count-1)); 
+        printf("MPI_Irecv(%lu, %i, %lu, %i, %i, %lu) (%i)\n", (unsigned long)buf1, recvargs->count, (unsigned long)recvargs->datatype, recvargs->source, handle->tag, (unsigned long)handle->comm, res); */
+        res = MPI_Irecv(buf1, recvargs->count, recvargs->datatype, recvargs->source, handle->tag, handle->comm, handle->req_array+handle->req_count-1);
+        if(OMPI_SUCCESS != res) { printf("Error in MPI_Irecv(%lu, %i, %lu, %i, %i, %lu) (%i)\n", (unsigned long)buf1, recvargs->count, (unsigned long)recvargs->datatype, recvargs->source, handle->tag, (unsigned long)handle->comm, res); ret=res; goto error; }
 #endif
 #ifdef NBC_TIMING
     Irecv_time += MPI_Wtime();
@@ -488,7 +465,7 @@ static inline int NBC_Start_round(NBC_Handle *handle) {
           buf2=(char*)handle->tmpbuf+(long)copyargs->tgt;
         else
           buf2=copyargs->tgt;
-        res = NBC_Copy(buf1, copyargs->srccount, copyargs->srctype, buf2, copyargs->tgtcount, copyargs->tgttype, handle->mycomm);
+        res = NBC_Copy(buf1, copyargs->srccount, copyargs->srctype, buf2, copyargs->tgtcount, copyargs->tgttype, handle->comm);
         if(res != NBC_OK) { printf("NBC_Copy() failed (code: %i)\n", res); ret=res; goto error; }
         break;
       case UNPACK:
@@ -505,7 +482,7 @@ static inline int NBC_Start_round(NBC_Handle *handle) {
           buf2=(char*)handle->tmpbuf+(long)unpackargs->outbuf;
         else
           buf2=unpackargs->outbuf;
-        res = NBC_Unpack(buf1, unpackargs->count, unpackargs->datatype, buf2, handle->mycomm);
+        res = NBC_Unpack(buf1, unpackargs->count, unpackargs->datatype, buf2, handle->comm);
         if(res != NBC_OK) { printf("NBC_Unpack() failed (code: %i)\n", res); ret=res; goto error; }
         break;
       default:
@@ -531,18 +508,14 @@ error:
   return ret;
 }
 
-static inline int NBC_Initialize(void) {
-  GNBC_Initialized = 1;
-
-  return NBC_OK;
-}
-
-int NBC_Init_handle(struct ompi_communicator_t *comm, ompi_coll_libnbc_request_t **request, ompi_coll_libnbc_module_t *module)
+int NBC_Init_handle(struct ompi_communicator_t *comm, ompi_coll_libnbc_request_t **request, ompi_coll_libnbc_module_t *comminfo)
 {
-  int res, flag;
-  NBC_Comminfo *comminfo;
+  int res;
+  ompi_coll_libnbc_request_t *handle;
 
-  NBC_Handle *handle = NULL;
+  OMPI_COLL_LIBNBC_REQUEST_ALLOC(comm, handle, res);
+  if (OMPI_SUCCESS != res) return res;
+  *request = handle;
 
   handle->tmpbuf = NULL;
   handle->req_count = 0;
@@ -553,27 +526,11 @@ int NBC_Init_handle(struct ompi_communicator_t *comm, ompi_coll_libnbc_request_t
   handle->row_offset = sizeof(int);
 
   /******************** Do the tag and shadow comm administration ...  ***************/
-  
-  /* otherwise we have to do the normal attribute stuff :-( */
-  /* keyval is not initialized yet, we have to init it */
-  if(MPI_KEYVAL_INVALID == gkeyval) {
-    res = MPI_Keyval_create(NBC_Key_copy, NBC_Key_delete, &(gkeyval), NULL); 
-    if((MPI_SUCCESS != res)) { printf("Error in MPI_Keyval_create() (%i)\n", res); return res; }
-  } 
 
-  res = MPI_Attr_get(comm, gkeyval, &comminfo, &flag);
-  if((MPI_SUCCESS != res)) { printf("Error in MPI_Attr_get() (%i)\n", res); return res; }
-
-  if (flag) {
-    /* we found it */
-    comminfo->tag++;
-  } else {
-    /* we have to create a new one */
-    comminfo = NBC_Init_comm(comm);
-    if(comminfo == NULL) { printf("Error in NBC_Init_comm() %i\n", res); return NBC_OOR; }
-  }
+  /* we found it */
+  comminfo->tag++;
   handle->tag=comminfo->tag;
-  handle->mycomm=comminfo->mycomm;
+  handle->comm=comm;
   /*printf("got comminfo: %lu tag: %i\n", comminfo, comminfo->tag);*/
 
   /* reset counter ... */ 
@@ -591,35 +548,25 @@ int NBC_Init_handle(struct ompi_communicator_t *comm, ompi_coll_libnbc_request_t
   return NBC_OK;
 }
 
-NBC_Comminfo* NBC_Init_comm(MPI_Comm comm) {
-  int res;
-  NBC_Comminfo *comminfo;
-
-  comminfo = (NBC_Comminfo*)malloc(sizeof(NBC_Comminfo));
-  if(comminfo == NULL) { printf("Error in malloc()\n"); return NULL; }
-
+int  NBC_Init_comm(MPI_Comm comm, NBC_Comminfo *comminfo) {
   /* set tag to 1 */
   comminfo->tag=1;
-  /* dup and save shadow communicator */
-  res = MPI_Comm_dup(comm, &(comminfo->mycomm));
-  if((MPI_SUCCESS != res)) { printf("Error in MPI_Comm_dup() (%i)\n", res); return NULL; }
-  NBC_DEBUG(1, "created a shadow communicator for %lu ... %lu\n", (unsigned long)comm, (unsigned long)comminfo->mycomm);
 
 #ifdef NBC_CACHE_SCHEDULE
 #if 0
   /* initialize the NBC_ALLTOALL SchedCache tree */
   comminfo->NBC_Dict[NBC_ALLTOALL] = hb_tree_new((dict_cmp_func)NBC_Alltoall_args_compare, NBC_SchedCache_args_delete_key_dummy, NBC_SchedCache_args_delete);
-  if(comminfo->NBC_Dict[NBC_ALLTOALL] == NULL) { printf("Error in hb_tree_new()\n"); return NULL; }
+  if(comminfo->NBC_Dict[NBC_ALLTOALL] == NULL) { printf("Error in hb_tree_new()\n"); return OMPI_ERROR;; }
   NBC_DEBUG(1, "added tree at address %lu\n", (unsigned long)comminfo->NBC_Dict[NBC_ALLTOALL]);
   comminfo->NBC_Dict_size[NBC_ALLTOALL] = 0;
   /* initialize the NBC_ALLGATHER SchedCache tree */
   comminfo->NBC_Dict[NBC_ALLGATHER] = hb_tree_new((dict_cmp_func)NBC_Allgather_args_compare, NBC_SchedCache_args_delete_key_dummy, NBC_SchedCache_args_delete);
-  if(comminfo->NBC_Dict[NBC_ALLGATHER] == NULL) { printf("Error in hb_tree_new()\n"); return NULL; }
+  if(comminfo->NBC_Dict[NBC_ALLGATHER] == NULL) { printf("Error in hb_tree_new()\n"); return OMPI_ERROR;; }
   NBC_DEBUG(1, "added tree at address %lu\n", (unsigned long)comminfo->NBC_Dict[NBC_ALLGATHER]);
   comminfo->NBC_Dict_size[NBC_ALLGATHER] = 0;
   /* initialize the NBC_ALLREDUCE SchedCache tree */
   comminfo->NBC_Dict[NBC_ALLREDUCE] = hb_tree_new((dict_cmp_func)NBC_Allreduce_args_compare, NBC_SchedCache_args_delete_key_dummy, NBC_SchedCache_args_delete);
-  if(comminfo->NBC_Dict[NBC_ALLREDUCE] == NULL) { printf("Error in hb_tree_new()\n"); return NULL; }
+  if(comminfo->NBC_Dict[NBC_ALLREDUCE] == NULL) { printf("Error in hb_tree_new()\n"); return OMPI_ERROR;; }
   NBC_DEBUG(1, "added tree at address %lu\n", (unsigned long)comminfo->NBC_Dict[NBC_ALLREDUCE]);
   comminfo->NBC_Dict_size[NBC_ALLREDUCE] = 0;
 #endif
@@ -629,47 +576,43 @@ NBC_Comminfo* NBC_Init_comm(MPI_Comm comm) {
 #if 0
   /* initialize the NBC_BCAST SchedCache tree */
   comminfo->NBC_Dict[NBC_BCAST] = hb_tree_new((dict_cmp_func)NBC_Bcast_args_compare, NBC_SchedCache_args_delete_key_dummy, NBC_SchedCache_args_delete);
-  if(comminfo->NBC_Dict[NBC_BCAST] == NULL) { printf("Error in hb_tree_new()\n"); return NULL; }
+  if(comminfo->NBC_Dict[NBC_BCAST] == NULL) { printf("Error in hb_tree_new()\n"); return OMPI_ERROR;; }
   NBC_DEBUG(1, "added tree at address %lu\n", (unsigned long)comminfo->NBC_Dict[NBC_BCAST]);
   comminfo->NBC_Dict_size[NBC_BCAST] = 0;
   /* initialize the NBC_GATHER SchedCache tree */
   comminfo->NBC_Dict[NBC_GATHER] = hb_tree_new((dict_cmp_func)NBC_Gather_args_compare, NBC_SchedCache_args_delete_key_dummy, NBC_SchedCache_args_delete);
-  if(comminfo->NBC_Dict[NBC_GATHER] == NULL) { printf("Error in hb_tree_new()\n"); return NULL; }
+  if(comminfo->NBC_Dict[NBC_GATHER] == NULL) { printf("Error in hb_tree_new()\n"); return OMPI_ERROR;; }
   NBC_DEBUG(1, "added tree at address %lu\n", (unsigned long)comminfo->NBC_Dict[NBC_GATHER]);
   comminfo->NBC_Dict_size[NBC_GATHER] = 0;
   /* initialize the NBC_REDUCE SchedCache tree */
   comminfo->NBC_Dict[NBC_REDUCE] = hb_tree_new((dict_cmp_func)NBC_Reduce_args_compare, NBC_SchedCache_args_delete_key_dummy, NBC_SchedCache_args_delete);
-  if(comminfo->NBC_Dict[NBC_REDUCE] == NULL) { printf("Error in hb_tree_new()\n"); return NULL; }
+  if(comminfo->NBC_Dict[NBC_REDUCE] == NULL) { printf("Error in hb_tree_new()\n"); return OMPI_ERROR;; }
   NBC_DEBUG(1, "added tree at address %lu\n", (unsigned long)comminfo->NBC_Dict[NBC_REDUCE]);
   comminfo->NBC_Dict_size[NBC_REDUCE] = 0;
   /* initialize the NBC_SCAN SchedCache tree */
   comminfo->NBC_Dict[NBC_SCAN] = hb_tree_new((dict_cmp_func)NBC_Scan_args_compare, NBC_SchedCache_args_delete_key_dummy, NBC_SchedCache_args_delete);
-  if(comminfo->NBC_Dict[NBC_SCAN] == NULL) { printf("Error in hb_tree_new()\n"); return NULL; }
+  if(comminfo->NBC_Dict[NBC_SCAN] == NULL) { printf("Error in hb_tree_new()\n"); return OMPI_ERROR;; }
   NBC_DEBUG(1, "added tree at address %lu\n", (unsigned long)comminfo->NBC_Dict[NBC_SCAN]);
   comminfo->NBC_Dict_size[NBC_SCAN] = 0;
   /* initialize the NBC_SCATTER SchedCache tree */
   comminfo->NBC_Dict[NBC_SCATTER] = hb_tree_new((dict_cmp_func)NBC_Scatter_args_compare, NBC_SchedCache_args_delete_key_dummy, NBC_SchedCache_args_delete);
-  if(comminfo->NBC_Dict[NBC_SCATTER] == NULL) { printf("Error in hb_tree_new()\n"); return NULL; }
+  if(comminfo->NBC_Dict[NBC_SCATTER] == NULL) { printf("Error in hb_tree_new()\n"); return OMPI_ERROR;; }
   NBC_DEBUG(1, "added tree at address %lu\n", (unsigned long)comminfo->NBC_Dict[NBC_SCATTER]);
   comminfo->NBC_Dict_size[NBC_SCATTER] = 0;
   /* initialize the NBC_ICART_SHIFT_XCHG SchedCache tree */
   comminfo->NBC_Dict[NBC_CART_SHIFT_XCHG] = hb_tree_new((dict_cmp_func)NBC_Icart_shift_xchg_args_compare, NBC_SchedCache_args_delete_key_dummy, NBC_SchedCache_args_delete);
-  if(comminfo->NBC_Dict[NBC_CART_SHIFT_XCHG] == NULL) { printf("Error in hb_tree_new()\n"); return NULL; }
+  if(comminfo->NBC_Dict[NBC_CART_SHIFT_XCHG] == NULL) { printf("Error in hb_tree_new()\n"); return OMPI_ERROR;; }
   NBC_DEBUG(1, "added tree at address %lu\n", (unsigned long)comminfo->NBC_Dict[NBC_CART_SHIFT_XCHG]);
   comminfo->NBC_Dict_size[NBC_CART_SHIFT_XCHG] = 0;
   /* initialize the NBC_INEIGHBOR_XCHG SchedCache tree */
   comminfo->NBC_Dict[NBC_NEIGHBOR_XCHG] = hb_tree_new((dict_cmp_func)NBC_Ineighbor_xchg_args_compare, NBC_SchedCache_args_delete_key_dummy, NBC_SchedCache_args_delete);
-  if(comminfo->NBC_Dict[NBC_NEIGHBOR_XCHG] == NULL) { printf("Error in hb_tree_new()\n"); return NULL; }
+  if(comminfo->NBC_Dict[NBC_NEIGHBOR_XCHG] == NULL) { printf("Error in hb_tree_new()\n"); return OMPI_ERROR;; }
   NBC_DEBUG(1, "added tree at address %lu\n", (unsigned long)comminfo->NBC_Dict[NBC_NEIGHBOR_XCHG]);
   comminfo->NBC_Dict_size[NBC_NEIGHBOR_XCHG] = 0;
 #endif
 #endif
 
-  /* put the new attribute to the comm */
-  res = MPI_Attr_put(comm, gkeyval, comminfo); 
-  if((MPI_SUCCESS != res)) { printf("Error in MPI_Attr_put() (%i)\n", res); return NULL; }
-
-  return comminfo;
+  return OMPI_SUCCESS;
 }
 
 int NBC_Start(NBC_Handle *handle, NBC_Schedule *schedule) {
@@ -681,26 +624,11 @@ int NBC_Start(NBC_Handle *handle, NBC_Schedule *schedule) {
   res = NBC_Start_round(handle);
   if((NBC_OK != res)) { printf("Error in NBC_Start_round() (%i)\n", res); return res; }
 
-  return NBC_OK;
-}
-
-int NBC_Wait(NBC_Handle *handle) {
-  /* poll */
-  while(NBC_OK != NBC_Progress(handle));
-
-  NBC_DEBUG(3, "finished request with tag %i\n", handle->tag);
-  
-  return NBC_OK;
-}
-
-int NBC_Test(NBC_Handle *handle, int *flag, MPI_Status *status) {
-  int ret = NBC_Progress(handle);
-  *flag = ret;
+  opal_list_append(&mca_coll_libnbc_component.active_requests, &(handle->super.super.super));
 
   return NBC_OK;
 }
 
-
 #ifdef NBC_CACHE_SCHEDULE
 void NBC_SchedCache_args_delete_key_dummy(void *k) {
     /* do nothing because the key and the data element are identical :-) 

ompi/mca/coll/libnbc/nbc.h

@@ -36,14 +36,6 @@ int NBC_Ireduce(void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype,
 int NBC_Iallreduce(void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, NBC_Handle* handle);
 int NBC_Iscan(void* sendbuf, void* recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, NBC_Handle* handle);
 int NBC_Ireduce_scatter(void* sendbuf, void* recvbuf, int *recvcounts, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm, NBC_Handle* handle);
-int NBC_Icart_shift_xchg(void *sbuf, int scount, MPI_Datatype stype, void *rbuf, int rcount, MPI_Datatype rtype, int direction, int disp, MPI_Comm comm, NBC_Handle* handle);
-int NBC_Ineighbor_xchg(void *sbuf, int scount, MPI_Datatype stype, void *rbuf, int rcount, MPI_Datatype rtype, MPI_Comm comm, NBC_Handle* handle);
-int NBC_Ineighbor_xchgv(void *sbuf, int *scounts, int *sdispls, MPI_Datatype stype, void *rbuf, int *rcounts, int *rdispls, MPI_Datatype rtype, MPI_Comm comm, NBC_Handle* handle);
-
-int NBC_Comm_neighbors(MPI_Comm comm, int maxindegree, int sources[], int sourceweights[], int maxoutdegree, int destinations[], int destweights[]);
-int NBC_Comm_neighbors_count(MPI_Comm comm, int *indegree, int *outdegree, int *weighted);
-int NBC_Wait(NBC_Handle *handle);
-int NBC_Test(NBC_Handle *handle, int *flag, MPI_Status *status);
 
 /* TODO: some hacks */
 int NBC_Operation(void *buf3, void *buf1, void *buf2, MPI_Op op, MPI_Datatype type, int count);

ompi/mca/coll/libnbc/nbc_internal.h

@@ -28,7 +28,7 @@
  * still in the cache and second, the tmpbuf used by the schedule must
  * be attached to the handle that uses this schedule !!!! 
  * I.E., THIS IS EXPERIMENTAL AND MIGHT NOT WORK */
-#define NBC_CACHE_SCHEDULE 
+/* #define NBC_CACHE_SCHEDULE  */
 #define NBC_SCHED_DICT_UPPER 1024 /* max. number of dict entries */
 #define NBC_SCHED_DICT_LOWER 512  /* nuber of dict entries after wipe, if SCHED_DICT_UPPER is reached */
 
@@ -288,12 +288,10 @@ void NBC_SchedCache_args_delete_key_dummy(void *k);
 #endif
 
 
-int NBC_Progress(NBC_Handle *handle);
 int NBC_Start(NBC_Handle *handle, NBC_Schedule *schedule);
 int NBC_Init_handle(struct ompi_communicator_t *comm, ompi_coll_libnbc_request_t **request, ompi_coll_libnbc_module_t *module);
 static inline int NBC_Type_intrinsic(MPI_Datatype type);
 static inline int NBC_Copy(void *src, int srccount, MPI_Datatype srctype, void *tgt, int tgtcount, MPI_Datatype tgttype, MPI_Comm comm);
-NBC_Comminfo* NBC_Init_comm(MPI_Comm comm);
 int NBC_Create_fortran_handle(int *fhandle, NBC_Handle **handle);
 
 /* some macros */

ompi/request/request_dbg.h

@@ -24,6 +24,7 @@ typedef enum {
     OMPI_REQUEST_IO,       /**< MPI-2 IO request */
     OMPI_REQUEST_GEN,      /**< MPI-2 generalized request */
     OMPI_REQUEST_WIN,      /**< MPI-2 one-sided request */
+    OMPI_REQUEST_COLL,     /**< MPI-3 non-blocking collectives request */
     OMPI_REQUEST_NULL,     /**< NULL request */
     OMPI_REQUEST_NOOP,     /**< A request that does nothing (e.g., to PROC_NULL) */
     OMPI_REQUEST_MAX       /**< Maximum request type */