Add support for MPI_Comm_dup_with_info, MPI_Comm_create_group, and

MPI_Comm_idup.

As part of this work I implemented a basic request scheduler in
ompi/comm/comm_request.c. This scheduler might be useful for more
than just communicator requests and could be moved to ompi/request
if there is a demand. Otherwise I will leave it where it is.

Added a non-blocking version of ompi_comm_set to support ompi_comm_idup.
The call makes a recursive call to comm_dup and a non-blocking version
was needed. To simplify the code the blocking version calls the nonblocking
version and waits on the resulting request if one exists.

cmr=v1.7.4:reviewer=jsquyres:ticket=trac:3796

This commit was SVN r29334.

The following Trac tickets were found above:
  Ticket 3796 --> https://svn.open-mpi.org/trac/ompi/ticket/3796

ompi/communicator/Makefile.am

@@ -10,6 +10,8 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
+# Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+#                         reserved.
 # $COPYRIGHT$
 # 
 # Additional copyrights may follow
@@ -20,9 +22,11 @@
 # This makefile.am does not stand on its own - it is included from ompi/Makefile.am
 
 headers += \
-        communicator/communicator.h
+        communicator/communicator.h \
+	communicator/comm_request.h
 
 libmpi_la_SOURCES += \
         communicator/comm_init.c \
 	communicator/comm.c \
-	communicator/comm_cid.c
+	communicator/comm_cid.c \
+	communicator/comm_request.c

ompi/communicator/comm.c

@@ -99,10 +99,41 @@ int ompi_comm_set ( ompi_communicator_t **ncomm,
                     bool copy_topocomponent, 
                     ompi_group_t *local_group, 
                     ompi_group_t *remote_group )
+{
+    ompi_request_t *req;
+    int rc;
+
+    rc = ompi_comm_set_nb (ncomm, oldcomm, local_size, local_ranks, remote_size, remote_ranks,
+                           attr, errh, copy_topocomponent, local_group, remote_group, &req);
+    if (OMPI_SUCCESS != rc) {
+        return rc;
+    }
+
+    if (NULL != req) {
+        ompi_request_wait (&req, MPI_STATUS_IGNORE);
+    }
+
+    return OMPI_SUCCESS;
+}
+
+int ompi_comm_set_nb ( ompi_communicator_t **ncomm,
+                       ompi_communicator_t *oldcomm,
+                       int local_size,
+                       int *local_ranks,
+                       int remote_size,
+                       int *remote_ranks,
+                       opal_hash_table_t *attr,
+                       ompi_errhandler_t *errh,
+                       bool copy_topocomponent,
+                       ompi_group_t *local_group,
+                       ompi_group_t *remote_group,
+                       ompi_request_t **req )
 {
     ompi_communicator_t *newcomm = NULL;
     int ret;
-    
+
+    *req = NULL;
+
     /* ompi_comm_allocate */
     newcomm = OBJ_NEW(ompi_communicator_t);
     /* fill in the inscribing hyper-cube dimensions */
@@ -134,9 +165,9 @@ int ompi_comm_set ( ompi_communicator_t **ncomm,
         }
         newcomm->c_flags |= OMPI_COMM_INTER;
         if ( OMPI_COMM_IS_INTRA(oldcomm) ) {
-            ompi_comm_dup(oldcomm, &newcomm->c_local_comm);
+            ompi_comm_idup(oldcomm, &newcomm->c_local_comm, req);
         } else {
-            ompi_comm_dup(oldcomm->c_local_comm, &newcomm->c_local_comm);
+            ompi_comm_idup(oldcomm->c_local_comm, &newcomm->c_local_comm, req);
         }
     } else { 
         newcomm->c_remote_group = newcomm->c_local_group;
@@ -845,6 +876,14 @@ ompi_comm_split_type(ompi_communicator_t *comm,
 /**********************************************************************/
 /**********************************************************************/
 int ompi_comm_dup ( ompi_communicator_t * comm, ompi_communicator_t **newcomm )
+{
+    return ompi_comm_dup_with_info (comm, NULL, newcomm);
+}
+
+/**********************************************************************/
+/**********************************************************************/
+/**********************************************************************/
+int ompi_comm_dup_with_info ( ompi_communicator_t * comm, ompi_info_t *info, ompi_communicator_t **newcomm )
 {
     ompi_communicator_t *newcomp = NULL;
     int rsize = 0, mode = OMPI_COMM_CID_INTRA, rc = OMPI_SUCCESS;
@@ -906,6 +945,202 @@ int ompi_comm_dup ( ompi_communicator_t * comm, ompi_communicator_t **newcomm )
     *newcomm = newcomp;
     return MPI_SUCCESS;
 }
+
+struct ompi_comm_idup_with_info_context {
+    ompi_communicator_t *comm;
+    ompi_communicator_t *newcomp;
+    ompi_communicator_t **newcomm;
+};
+
+static int ompi_comm_idup_with_info_activate (ompi_comm_request_t *request);
+static int ompi_comm_idup_with_info_finish (ompi_comm_request_t *request);
+static int ompi_comm_idup_getcid (ompi_comm_request_t *request);
+
+int ompi_comm_idup (ompi_communicator_t *comm, ompi_communicator_t **newcomm, ompi_request_t **req)
+{
+    return ompi_comm_idup_with_info (comm, NULL, newcomm, req);
+}
+
+int ompi_comm_idup_with_info (ompi_communicator_t *comm, ompi_info_t *info, ompi_communicator_t **newcomm, ompi_request_t **req)
+{
+    struct ompi_comm_idup_with_info_context *context;
+    ompi_comm_request_t *request;
+    ompi_request_t *subreq;
+    int rsize = 0, rc;
+
+    *newcomm = MPI_COMM_NULL;
+
+    request = ompi_comm_request_get ();
+    if (NULL == request) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    context = calloc (1, sizeof (*context));
+    if (NULL == context) {
+        ompi_comm_request_return (request);
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    context->comm    = comm;
+    context->newcomm = newcomm;
+
+    request->context = context;
+
+    rc =  ompi_comm_set_nb (&context->newcomp,                      /* new comm */
+                            comm,                                   /* old comm */
+                            comm->c_local_group->grp_proc_count,    /* local_size */
+                            NULL,                                   /* local_procs */
+                            rsize,                                  /* remote_size */
+                            NULL,                                   /* remote_procs */
+                            comm->c_keyhash,                        /* attrs */
+                            comm->error_handler,                    /* error handler */
+                            true,                                   /* copy the topo */
+                            comm->c_local_group,                    /* local group */
+                            comm->c_remote_group,                   /* remote group */
+                            &subreq);                               /* new subrequest */
+    if (NULL == context->newcomp) {
+        ompi_comm_request_return (request);
+        return rc;
+    }
+
+    ompi_comm_request_schedule_append (request, ompi_comm_idup_getcid, &subreq, subreq ? 1 : 0);
+
+    /* kick off the request */
+    ompi_comm_request_start (request);
+    *req = &request->super;
+
+    return OMPI_SUCCESS;
+}
+
+static int ompi_comm_idup_getcid (ompi_comm_request_t *request)
+{
+    struct ompi_comm_idup_with_info_context *context =
+        (struct ompi_comm_idup_with_info_context *) request->context;
+    ompi_request_t *subreq;
+    int rc, mode;
+
+    if (OMPI_COMM_IS_INTER(context->comm)){
+        mode  = OMPI_COMM_CID_INTER;
+    } else {
+        mode  = OMPI_COMM_CID_INTRA;
+    }
+
+    /* Determine context id. It is identical to f_2_c_handle */
+    rc = ompi_comm_nextcid_nb (context->newcomp,  /* new communicator */
+                               context->comm,     /* old comm */
+                               NULL,              /* bridge comm */
+                               mode,              /* mode */
+                               &subreq);          /* new subrequest */
+    if (OMPI_SUCCESS != rc) {
+        ompi_comm_request_return (request);
+        return rc;
+    }
+
+    ompi_comm_request_schedule_append (request, ompi_comm_idup_with_info_activate, &subreq, 1);
+
+    return OMPI_SUCCESS;
+}
+
+static int ompi_comm_idup_with_info_activate (ompi_comm_request_t *request)
+{
+    struct ompi_comm_idup_with_info_context *context =
+        (struct ompi_comm_idup_with_info_context *) request->context;
+    ompi_request_t *subreq;
+    int rc, mode;
+
+    if (OMPI_COMM_IS_INTER(context->comm)){
+        mode  = OMPI_COMM_CID_INTER;
+    } else {
+        mode  = OMPI_COMM_CID_INTRA;
+    }
+
+    /* Set name for debugging purposes */
+    snprintf(context->newcomp->c_name, MPI_MAX_OBJECT_NAME, "MPI COMMUNICATOR %d DUP FROM %d",
+             context->newcomp->c_contextid, context->comm->c_contextid );
+
+    /* activate communicator and init coll-module */
+    rc = ompi_comm_activate_nb (&context->newcomp, context->comm, NULL, mode, &subreq);
+    if ( OMPI_SUCCESS != rc ) {
+        return rc;
+    }
+
+    ompi_comm_request_schedule_append (request, ompi_comm_idup_with_info_finish, &subreq, 1);
+
+    return OMPI_SUCCESS;
+}
+
+static int ompi_comm_idup_with_info_finish (ompi_comm_request_t *request)
+{
+    struct ompi_comm_idup_with_info_context *context =
+        (struct ompi_comm_idup_with_info_context *) request->context;
+
+    *context->newcomm = context->newcomp;
+
+    /* done */
+    return MPI_SUCCESS;
+}
+
+/**********************************************************************/
+/**********************************************************************/
+/**********************************************************************/
+int ompi_comm_create_group (ompi_communicator_t *comm, ompi_group_t *group, int tag, ompi_communicator_t **newcomm)
+{
+    ompi_communicator_t *newcomp = NULL;
+    int mode = OMPI_COMM_CID_GROUP, rc = OMPI_SUCCESS;
+
+    *newcomm = MPI_COMM_NULL;
+
+    rc =  ompi_comm_set ( &newcomp,                               /* new comm */
+                          comm,                                   /* old comm */
+                          group->grp_proc_count,                  /* local_size */
+                          NULL,                                   /* local_procs*/
+                          0,                                      /* remote_size */
+                          NULL,                                   /* remote_procs */
+                          comm->c_keyhash,                        /* attrs */
+                          comm->error_handler,                    /* error handler */
+                          true,                                   /* copy the topo */
+                          group,                                  /* local group */
+                          NULL);                                  /* remote group */
+    if ( NULL == newcomp ) {
+        rc =  MPI_ERR_INTERN;
+        return rc;
+    }
+    if ( MPI_SUCCESS != rc) {
+        return rc;
+    }
+
+    /* Determine context id. It is identical to f_2_c_handle */
+    rc = ompi_comm_nextcid ( newcomp,  /* new communicator */
+                             comm,     /* old comm */
+                             newcomp,  /* bridge comm (used to pass the group into the group allreduce) */
+                             &tag,     /* user defined tag */
+                             NULL,     /* remote_leader */
+                             mode,     /* mode */
+                             -1 );     /* send_first */
+    if ( OMPI_SUCCESS != rc ) {
+        return rc;
+    }
+
+    /* Set name for debugging purposes */
+    snprintf(newcomp->c_name, MPI_MAX_OBJECT_NAME, "MPI COMMUNICATOR %d GROUP FROM %d",
+             newcomp->c_contextid, comm->c_contextid );
+
+    /* activate communicator and init coll-module */
+    rc = ompi_comm_activate( &newcomp, /* new communicator */
+                             comm,
+                             newcomp,
+                             &tag,
+                             NULL,
+                             mode,
+                             -1 );
+    if ( OMPI_SUCCESS != rc ) {
+        return rc;
+    }
+
+    *newcomm = newcomp;
+    return MPI_SUCCESS;
+}
+
 /**********************************************************************/
 /**********************************************************************/
 /**********************************************************************/

ompi/communicator/comm_request.c

@@ -0,0 +1,248 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reseved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "comm_request.h"
+
+opal_free_list_t ompi_comm_requests;
+opal_list_t ompi_comm_requests_active;
+opal_mutex_t ompi_comm_request_mutex;
+bool ompi_comm_request_progress_active;
+bool ompi_comm_request_initialized = false;
+
+typedef struct ompi_comm_request_item_t {
+    opal_list_item_t super;
+    ompi_comm_request_callback_fn_t callback;
+    ompi_request_t *subreqs[OMPI_COMM_REQUEST_MAX_SUBREQ];
+    int subreq_count;
+} ompi_comm_request_item_t;
+OBJ_CLASS_DECLARATION(ompi_comm_request_item_t);
+
+static int ompi_comm_request_progress (void);
+
+void ompi_comm_request_init (void)
+{
+    OBJ_CONSTRUCT(&ompi_comm_requests, opal_free_list_t);
+    (void) opal_free_list_init (&ompi_comm_requests, sizeof (ompi_comm_request_t),
+                                OBJ_CLASS(ompi_comm_request_t), 0, -1, 8);
+
+    OBJ_CONSTRUCT(&ompi_comm_requests_active, opal_list_t);
+    ompi_comm_request_progress_active = false;
+    OBJ_CONSTRUCT(&ompi_comm_request_mutex, opal_mutex_t);
+    ompi_comm_request_initialized = true;
+}
+
+void ompi_comm_request_fini (void)
+{
+    if (!ompi_comm_request_initialized) {
+        return;
+    }
+
+    ompi_comm_request_initialized = false;
+
+    opal_mutex_lock (&ompi_comm_request_mutex);
+    if (ompi_comm_request_progress_active) {
+        opal_progress_unregister (ompi_comm_request_progress);
+    }
+    opal_mutex_unlock (&ompi_comm_request_mutex);
+    OBJ_DESTRUCT(&ompi_comm_request_mutex);
+    OBJ_DESTRUCT(&ompi_comm_requests_active);
+    OBJ_DESTRUCT(&ompi_comm_requests);
+}
+
+
+int ompi_comm_request_schedule_append (ompi_comm_request_t *request, ompi_comm_request_callback_fn_t callback,
+                            ompi_request_t *subreqs[], int subreq_count)
+{
+    ompi_comm_request_item_t *request_item;
+    int i;
+
+    if (subreq_count > OMPI_COMM_REQUEST_MAX_SUBREQ) {
+        return OMPI_ERR_BAD_PARAM;
+    }
+
+    request_item = OBJ_NEW(ompi_comm_request_item_t);
+    if (NULL == request_item) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    request_item->callback = callback;
+
+    for (i = 0 ; i < subreq_count ; ++i) {
+        request_item->subreqs[i] = subreqs[i];
+    }
+
+    request_item->subreq_count = subreq_count;
+
+    opal_list_append (&request->schedule, &request_item->super);
+
+    return OMPI_SUCCESS;
+}
+
+static int ompi_comm_request_progress (void)
+{
+    ompi_comm_request_t *request, *next;
+
+    if (opal_mutex_trylock (&ompi_comm_request_mutex)) {
+        return 0;
+    }
+
+    OPAL_LIST_FOREACH_SAFE(request, next, &ompi_comm_requests_active, ompi_comm_request_t) {
+        int rc = OMPI_SUCCESS;
+
+        if (opal_list_get_size (&request->schedule)) {
+            ompi_comm_request_item_t *request_item = (ompi_comm_request_item_t *) opal_list_remove_first (&request->schedule);
+            int item_complete = true;
+
+            /* don't call ompi_request_test_all as it causes a recursive call into opal_progress */
+            while (request_item->subreq_count) {
+                ompi_request_t *subreq = request_item->subreqs[request_item->subreq_count-1];
+                if (true == subreq->req_complete) {
+                    ompi_request_free (&subreq);
+                    request_item->subreq_count--;
+                } else {
+                    item_complete = false;
+                    break;
+                }
+            }
+
+            if (item_complete) {
+                if (request_item->callback) {
+                    opal_mutex_unlock (&ompi_comm_request_mutex);
+                    rc = request_item->callback (request);
+                    opal_mutex_lock (&ompi_comm_request_mutex);
+                }
+                OBJ_RELEASE(request_item);
+            } else {
+                opal_list_prepend (&request->schedule, &request_item->super);
+            }
+        }
+
+        /* if the request schedule is empty then the request is complete */
+        if (0 == opal_list_get_size (&request->schedule)) {
+            opal_list_remove_item (&ompi_comm_requests_active, (opal_list_item_t *) request);
+            request->super.req_status.MPI_ERROR = (OMPI_SUCCESS == rc) ? MPI_SUCCESS : MPI_ERR_INTERN;
+            ompi_request_complete (&request->super, true);
+        }
+    }
+
+    if (0 == opal_list_get_size (&ompi_comm_requests_active)) {
+        /* no more active requests. disable this progress function */
+        ompi_comm_request_progress_active = false;
+        opal_progress_unregister (ompi_comm_request_progress);
+    }
+
+    opal_mutex_unlock (&ompi_comm_request_mutex);
+
+    return 1;
+}
+
+void ompi_comm_request_start (ompi_comm_request_t *request)
+{
+    opal_mutex_lock (&ompi_comm_request_mutex);
+    opal_list_append (&ompi_comm_requests_active, (opal_list_item_t *) request);
+
+    /* check if we need to start the communicator request progress function */
+    if (!ompi_comm_request_progress_active) {
+        opal_progress_register (ompi_comm_request_progress);
+        ompi_comm_request_progress_active = true;
+    }
+
+    opal_mutex_unlock (&ompi_comm_request_mutex);
+}
+
+static int ompi_comm_request_cancel (struct ompi_request_t *ompi_req, int complete)
+{
+    ompi_comm_request_t *tmp, *request = (ompi_comm_request_t *) ompi_req;
+    ompi_comm_request_item_t *item, *next;
+
+    opal_mutex_lock (&ompi_comm_request_mutex);
+
+    OPAL_LIST_FOREACH_SAFE(item, next, &request->schedule, ompi_comm_request_item_t) {
+        for (int i = 0 ; i < item->subreq_count ; ++i) {
+            ompi_request_cancel (item->subreqs[i]);
+        }
+
+        opal_list_remove_item (&request->schedule, &item->super);
+        OBJ_RELEASE(item);
+    }
+
+    /* remove the request for the list of active requests */
+    OPAL_LIST_FOREACH(tmp, &ompi_comm_requests_active, ompi_comm_request_t) {
+        if (tmp == request) {
+            opal_list_remove_item (&ompi_comm_requests_active, (opal_list_item_t *) request);
+            break;
+        }
+    }
+
+    opal_mutex_unlock (&ompi_comm_request_mutex);
+
+    return MPI_ERR_REQUEST;
+}
+
+static int ompi_comm_request_free (struct ompi_request_t **ompi_req)
+{
+    ompi_comm_request_t *request = (ompi_comm_request_t *) *ompi_req;
+
+    if (!(*ompi_req)->req_complete) {
+        return MPI_ERR_REQUEST;
+    }
+
+    (*ompi_req)->req_complete = false;
+    ompi_comm_request_return (request);
+
+    *ompi_req = MPI_REQUEST_NULL;
+
+    return OMPI_SUCCESS;
+}
+
+static void ompi_comm_request_construct (ompi_comm_request_t *request)
+{
+    request->context = NULL;
+
+    request->super.req_type = OMPI_REQUEST_COMM;
+    request->super.req_status._cancelled = 0;
+    request->super.req_free = ompi_comm_request_free;
+    request->super.req_cancel = ompi_comm_request_cancel;
+
+    OBJ_CONSTRUCT(&request->schedule, opal_list_t);
+}
+
+static void ompi_comm_request_destruct (ompi_comm_request_t *request)
+{
+    OBJ_DESTRUCT(&request->schedule);
+}
+OBJ_CLASS_INSTANCE(ompi_comm_request_t, ompi_request_t,
+                   ompi_comm_request_construct,
+                   ompi_comm_request_destruct);
+
+OBJ_CLASS_INSTANCE(ompi_comm_request_item_t, opal_list_item_t, NULL, NULL);
+
+ompi_comm_request_t *ompi_comm_request_get (void)
+{
+    opal_free_list_item_t *item;
+    int rc;
+
+    OPAL_FREE_LIST_GET(&ompi_comm_requests, item, rc);
+    (void) rc;
+
+    return (ompi_comm_request_t *) item;
+}
+
+void ompi_comm_request_return (ompi_comm_request_t *request)
+{
+    if (request->context) {
+        free (request->context);
+        request->context = NULL;
+    }
+
+    OPAL_FREE_LIST_RETURN(&ompi_comm_requests, (opal_free_list_item_t *) request);
+}
+

ompi/communicator/comm_request.h

@@ -0,0 +1,40 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reseved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#if !defined(OMPI_COMM_REQ_H)
+#define OMPI_COMM_REQ_H
+
+#include "opal/class/opal_list.h"
+#include "opal/class/opal_free_list.h"
+#include "ompi/request/request.h"
+
+/* increase this number if more subrequests are needed */
+#define OMPI_COMM_REQUEST_MAX_SUBREQ 2
+
+typedef struct ompi_comm_request_t {
+    ompi_request_t super;
+
+    void *context;
+    opal_list_t schedule;
+} ompi_comm_request_t;
+OBJ_CLASS_DECLARATION(ompi_comm_request_t);
+
+typedef int (*ompi_comm_request_callback_fn_t) (ompi_comm_request_t *);
+
+void ompi_comm_request_init (void);
+void ompi_comm_request_fini (void);
+int ompi_comm_request_schedule_append (ompi_comm_request_t *request, ompi_comm_request_callback_fn_t callback,
+                            ompi_request_t *subreqs[], int subreq_count);
+void ompi_comm_request_start (ompi_comm_request_t *request);
+ompi_comm_request_t *ompi_comm_request_get (void);
+void ompi_comm_request_return (ompi_comm_request_t *request);
+
+#endif /* OMPI_COMM_REQ_H */

ompi/communicator/communicator.h

@@ -31,6 +31,7 @@
 #include "opal/class/opal_object.h"
 #include "ompi/errhandler/errhandler.h"
 #include "opal/threads/mutex.h"
+#include "ompi/communicator/comm_request.h"
 
 #include "mpi.h"
 #include "ompi/group/group.h"
@@ -91,6 +92,7 @@ OMPI_DECLSPEC OBJ_CLASS_DECLARATION(ompi_communicator_t);
 #define OMPI_COMM_CID_INTER        0x00000040
 #define OMPI_COMM_CID_INTRA_BRIDGE 0x00000080
 #define OMPI_COMM_CID_INTRA_OOB    0x00000100
+#define OMPI_COMM_CID_GROUP        0x00000200
 
 /**
  * The block of CIDs allocated for MPI_COMM_WORLD
@@ -359,6 +361,13 @@ OMPI_DECLSPEC int ompi_comm_group (ompi_communicator_t *comm, ompi_group_t **gro
 int ompi_comm_create (ompi_communicator_t* comm, ompi_group_t *group,
                       ompi_communicator_t** newcomm);
 
+
+/**
+ * Non-collective create communicator based on a group
+ */
+int ompi_comm_create_group (ompi_communicator_t *comm, ompi_group_t *group, int tag,
+                            ompi_communicator_t **newcomm);
+
 /**
  * Take an almost complete communicator and reserve the CID as well
  * as activate it (initialize the collective and the topologies).
@@ -416,6 +425,37 @@ OMPI_DECLSPEC int ompi_comm_split_type(ompi_communicator_t *comm,
  * @param newcomm:   the new communicator or MPI_COMM_NULL if any error is detected.
  */
 OMPI_DECLSPEC int ompi_comm_dup (ompi_communicator_t *comm, ompi_communicator_t **newcomm);
+
+/**
+ * dup a communicator (non-blocking). Parameter are identical to the MPI-counterpart
+ * of the function. It has been extracted, since we need to be able
+ * to dup a communicator internally as well.
+ *
+ * @param comm:      input communicator
+ * @param newcomm:   the new communicator or MPI_COMM_NULL if any error is detected.
+ */
+OMPI_DECLSPEC int ompi_comm_idup (ompi_communicator_t *comm, ompi_communicator_t **newcomm, ompi_request_t **request);
+
+/**
+ * dup a communicator with info. Parameter are identical to the MPI-counterpart
+ * of the function. It has been extracted, since we need to be able
+ * to dup a communicator internally as well.
+ *
+ * @param comm:      input communicator
+ * @param newcomm:   the new communicator or MPI_COMM_NULL if any error is detected.
+ */
+OMPI_DECLSPEC int ompi_comm_dup_with_info (ompi_communicator_t *comm, ompi_info_t *info, ompi_communicator_t **newcomm);
+
+/**
+ * dup a communicator (non-blocking) with info.
+ * of the function. It has been extracted, since we need to be able
+ * to dup a communicator internally as well.
+ *
+ * @param comm:      input communicator
+ * @param newcomm:   the new communicator or MPI_COMM_NULL if any error is detected.
+ */
+OMPI_DECLSPEC int ompi_comm_idup_with_info (ompi_communicator_t *comm, ompi_info_t *info, ompi_communicator_t **newcomm, ompi_request_t **req);
+
 /**
  * compare two communicators.
  *
@@ -469,6 +509,21 @@ OMPI_DECLSPEC int ompi_comm_nextcid ( ompi_communicator_t* newcomm,
                                       int mode,
                                       int send_first);
 
+/**
+ * allocate new communicator ID (non-blocking)
+ * @param newcomm:    pointer to the new communicator
+ * @param oldcomm:    original comm
+ * @param bridgecomm: bridge comm for intercomm_create
+ * @param mode: combination of input
+ *              OMPI_COMM_CID_INTRA:        intra-comm
+ *              OMPI_COMM_CID_INTER:        inter-comm
+ * This routine has to be thread safe in the final version.
+ */
+OMPI_DECLSPEC int ompi_comm_nextcid_nb (ompi_communicator_t* newcomm,
+                                        ompi_communicator_t* comm,
+                                        ompi_communicator_t* bridgecomm,
+                                        int mode, ompi_request_t **req);
+
 /**
  * shut down the communicator infrastructure.
  */
@@ -489,6 +544,20 @@ OMPI_DECLSPEC int ompi_comm_set ( ompi_communicator_t** newcomm,
                                   bool copy_topocomponent,
                                   ompi_group_t *local_group,
                                   ompi_group_t *remote_group   );
+
+OMPI_DECLSPEC int ompi_comm_set_nb ( ompi_communicator_t **ncomm,
+                                     ompi_communicator_t *oldcomm,
+                                     int local_size,
+                                     int *local_ranks,
+                                     int remote_size,
+                                     int *remote_ranks,
+                                     opal_hash_table_t *attr,
+                                     ompi_errhandler_t *errh,
+                                     bool copy_topocomponent,
+                                     ompi_group_t *local_group,
+                                     ompi_group_t *remote_group,
+                                     ompi_request_t **req );
+
 /**
  * This is a short-hand routine used in intercomm_create.
  * The routine makes sure, that all processes have afterwards
@@ -525,6 +594,11 @@ OMPI_DECLSPEC int ompi_comm_activate ( ompi_communicator_t** newcomm,
                                        int mode,
                                        int send_first );
 
+OMPI_DECLSPEC int ompi_comm_activate_nb (ompi_communicator_t **newcomm,
+                                         ompi_communicator_t *comm,
+                                         ompi_communicator_t *bridgecomm,
+                                         int mode, ompi_request_t **req);
+
 /**
  * a simple function to dump the structure
  */

ompi/include/mpi.h.in

@@ -1240,10 +1240,13 @@ OMPI_DECLSPEC  int MPI_Comm_create_errhandler(MPI_Comm_errhandler_function *func
 OMPI_DECLSPEC  int MPI_Comm_create_keyval(MPI_Comm_copy_attr_function *comm_copy_attr_fn,
                                           MPI_Comm_delete_attr_function *comm_delete_attr_fn,
                                           int *comm_keyval, void *extra_state);
+OMPI_DECLSPEC  int MPI_Comm_create_group(MPI_Comm comm, MPI_Group group, int tag, MPI_Comm *newcomm);
 OMPI_DECLSPEC  int MPI_Comm_create(MPI_Comm comm, MPI_Group group, MPI_Comm *newcomm);
 OMPI_DECLSPEC  int MPI_Comm_delete_attr(MPI_Comm comm, int comm_keyval);
 OMPI_DECLSPEC  int MPI_Comm_disconnect(MPI_Comm *comm);
 OMPI_DECLSPEC  int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm);
+OMPI_DECLSPEC  int MPI_Comm_idup(MPI_Comm comm, MPI_Comm *newcomm, MPI_Request *request);
+OMPI_DECLSPEC  int MPI_Comm_dup_with_info(MPI_Comm comm, MPI_Info info, MPI_Comm *newcomm);
 OMPI_DECLSPEC  MPI_Comm MPI_Comm_f2c(MPI_Fint comm);
 OMPI_DECLSPEC  int MPI_Comm_free_keyval(int *comm_keyval);
 OMPI_DECLSPEC  int MPI_Comm_free(MPI_Comm *comm);
@@ -1909,10 +1912,13 @@ OMPI_DECLSPEC  int PMPI_Comm_create_errhandler(MPI_Comm_errhandler_function *fun
 OMPI_DECLSPEC  int PMPI_Comm_create_keyval(MPI_Comm_copy_attr_function *comm_copy_attr_fn,
                                            MPI_Comm_delete_attr_function *comm_delete_attr_fn,
                                            int *comm_keyval, void *extra_state);
+OMPI_DECLSPEC  int PMPI_Comm_create_group(MPI_Comm comm, MPI_Group group, int tag, MPI_Comm *newcomm);
 OMPI_DECLSPEC  int PMPI_Comm_create(MPI_Comm comm, MPI_Group group, MPI_Comm *newcomm);
 OMPI_DECLSPEC  int PMPI_Comm_delete_attr(MPI_Comm comm, int comm_keyval);
 OMPI_DECLSPEC  int PMPI_Comm_disconnect(MPI_Comm *comm);
 OMPI_DECLSPEC  int PMPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm);
+OMPI_DECLSPEC  int PMPI_Comm_idup(MPI_Comm comm, MPI_Comm *newcomm, MPI_Request *request);
+OMPI_DECLSPEC  int PMPI_Comm_dup_with_info(MPI_Comm comm, MPI_Info info, MPI_Comm *newcomm);
 OMPI_DECLSPEC  MPI_Comm PMPI_Comm_f2c(MPI_Fint comm);
 OMPI_DECLSPEC  int PMPI_Comm_free_keyval(int *comm_keyval);
 OMPI_DECLSPEC  int PMPI_Comm_free(MPI_Comm *comm);

ompi/mpi/c/Makefile.am

@@ -11,7 +11,9 @@
 #                         All rights reserved.
 # Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
-# Copyright (c) 2012      Oak Rigde National Laboratory. All rights reserved.
+# Copyright (c) 2012      Oak Ridge National Laboratory. All rights reserved.
+# Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+#                         reserved.
 # $COPYRIGHT$
 # 
 # Additional copyrights may follow
@@ -106,10 +108,13 @@ libmpi_c_mpi_la_SOURCES = \
         comm_connect.c \
         comm_create.c \
         comm_create_errhandler.c \
+        comm_create_group.c \
         comm_create_keyval.c \
         comm_delete_attr.c \
         comm_disconnect.c \
         comm_dup.c \
+        comm_dup_with_info.c \
+        comm_idup.c \
         comm_f2c.c \
         comm_free.c \
         comm_free_keyval.c \

ompi/mpi/c/comm_create_group.c

@@ -0,0 +1,80 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2008 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/memchecker.h"
+
+#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#pragma weak MPI_Comm_create_group = PMPI_Comm_create_group
+#endif
+
+#if OMPI_PROFILING_DEFINES
+#include "ompi/mpi/c/profile/defines.h"
+#endif
+
+static const char FUNC_NAME[] = "MPI_Comm_create_group";
+
+
+int MPI_Comm_create_group (MPI_Comm comm, MPI_Group group, int tag, MPI_Comm *newcomm) {
+    int rc;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_comm_invalid (comm))
+            return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_COMM,
+                                          FUNC_NAME);
+
+        if (tag < 0 || tag > mca_pml.pml_max_tag)
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_TAG,
+                                          FUNC_NAME);
+
+        if ( NULL == group )
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_GROUP,
+                                          FUNC_NAME);
+
+        if ( NULL == newcomm )
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+    }
+
+    if (MPI_GROUP_NULL == group || MPI_UNDEFINED == ompi_group_rank (group)) {
+        *newcomm = MPI_COMM_NULL;
+        return MPI_SUCCESS;
+    }
+
+    OPAL_CR_ENTER_LIBRARY();
+
+    rc = ompi_comm_create_group ((ompi_communicator_t *) comm, (ompi_group_t *) group,
+                                 tag, (ompi_communicator_t **) newcomm);
+    OMPI_ERRHANDLER_RETURN (rc, comm, rc, FUNC_NAME);
+}

ompi/mpi/c/comm_dup_with_info.c

@@ -0,0 +1,72 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2008 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2006-2008 University of Houston.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+
+#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#pragma weak MPI_Comm_dup_with_info = PMPI_Comm_dup_with_info
+#endif
+
+#if OMPI_PROFILING_DEFINES
+#include "ompi/mpi/c/profile/defines.h"
+#endif
+
+static const char FUNC_NAME[] = "MPI_Comm_dup_with_info";
+
+int MPI_Comm_dup_with_info(MPI_Comm comm, MPI_Info info, MPI_Comm *newcomm)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    /* argument checking */
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_comm_invalid (comm))
+            return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_COMM,
+                                          FUNC_NAME);
+        if (NULL == info || ompi_info_is_freed(info)) {
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_INFO,
+                                          FUNC_NAME);
+        }
+
+        if ( NULL == newcomm )
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+    }
+
+    OPAL_CR_ENTER_LIBRARY();
+
+    rc = ompi_comm_dup_with_info (comm, info, newcomm);
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
+

ompi/mpi/c/comm_idup.c

@@ -0,0 +1,68 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2008 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2006-2008 University of Houston.  All rights reserved.
+ * Copyright (c) 2013      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <stdio.h>
+
+#include "ompi/mpi/c/bindings.h"
+#include "ompi/runtime/params.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/errhandler/errhandler.h"
+#include "ompi/memchecker.h"
+
+#if OPAL_HAVE_WEAK_SYMBOLS && OMPI_PROFILING_DEFINES
+#pragma weak MPI_Comm_idup = PMPI_Comm_idup
+#endif
+
+#if OMPI_PROFILING_DEFINES
+#include "ompi/mpi/c/profile/defines.h"
+#endif
+
+static const char FUNC_NAME[] = "MPI_Comm_idup";
+
+int MPI_Comm_idup(MPI_Comm comm, MPI_Comm *newcomm, MPI_Request *request)
+{
+    int rc;
+
+    MEMCHECKER(
+        memchecker_comm(comm);
+    );
+
+    /* argument checking */
+    if ( MPI_PARAM_CHECK ) {
+        OMPI_ERR_INIT_FINALIZE(FUNC_NAME);
+
+        if (ompi_comm_invalid (comm))
+            return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_COMM,
+                                          FUNC_NAME);
+
+        if ( NULL == newcomm )
+            return OMPI_ERRHANDLER_INVOKE(comm, MPI_ERR_ARG,
+                                          FUNC_NAME);
+    }
+
+    OPAL_CR_ENTER_LIBRARY();
+
+    rc = ompi_comm_idup (comm, newcomm, request);
+    OMPI_ERRHANDLER_RETURN(rc, comm, rc, FUNC_NAME);
+}
+

ompi/mpi/c/profile/Makefile.am

@@ -12,7 +12,9 @@
 #                         All rights reserved.
 # Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
-# Copyright (c)      2012 Oak Rigde National Laboratory. All rights reserved.
+# Copyright (c)      2012 Oak Ridge National Laboratory. All rights reserved.
+# Copyright (c) 2013      Los Alamos National Security, LLC. All rights
+#                         reserved.
 # $COPYRIGHT$
 # 
 # Additional copyrights may follow
@@ -88,10 +90,13 @@ nodist_libmpi_c_pmpi_la_SOURCES = \
         pcomm_connect.c \
         pcomm_create.c \
         pcomm_create_errhandler.c \
+        pcomm_create_group.c \
         pcomm_create_keyval.c \
         pcomm_delete_attr.c \
         pcomm_disconnect.c \
         pcomm_dup.c \
+        pcomm_dup_with_info.c \
+        pcomm_idup.c \
         pcomm_f2c.c \
         pcomm_free.c \
         pcomm_free_keyval.c \

ompi/mpi/c/profile/defines.h

@@ -76,10 +76,13 @@
 #define MPI_Comm_connect PMPI_Comm_connect
 #define MPI_Comm_create_errhandler PMPI_Comm_create_errhandler
 #define MPI_Comm_create_keyval PMPI_Comm_create_keyval
+#define MPI_Comm_create_group PMPI_Comm_create_group
 #define MPI_Comm_create PMPI_Comm_create
 #define MPI_Comm_delete_attr PMPI_Comm_delete_attr
 #define MPI_Comm_disconnect PMPI_Comm_disconnect
 #define MPI_Comm_dup PMPI_Comm_dup
+#define MPI_Comm_dup_with_info PMPI_Comm_dup_with_info
+#define MPI_Comm_idup PMPI_Comm_idup
 #define MPI_Comm_f2c PMPI_Comm_f2c
 #define MPI_Comm_free_keyval PMPI_Comm_free_keyval
 #define MPI_Comm_free PMPI_Comm_free

ompi/request/request_dbg.h

@@ -27,6 +27,7 @@ typedef enum {
     OMPI_REQUEST_COLL,     /**< MPI-3 non-blocking collectives request */
     OMPI_REQUEST_NULL,     /**< NULL request */
     OMPI_REQUEST_NOOP,     /**< A request that does nothing (e.g., to PROC_NULL) */
+    OMPI_REQUEST_COMM,     /**< MPI-3 non-blocking communicator duplication */
     OMPI_REQUEST_MAX       /**< Maximum request type */
 } ompi_request_type_t;
 

ompi/runtime/ompi_mpi_finalize.c

@@ -283,6 +283,9 @@ int ompi_mpi_finalize(void)
         return ret;
     }
 
+    /* release resources held by comm requests */
+    ompi_comm_request_fini ();
+
     if (OMPI_SUCCESS != (ret = ompi_message_finalize())) {
         return ret;
     }

ompi/runtime/ompi_mpi_init.c

@@ -871,6 +871,9 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
 	goto error;
     }
 
+    /* Prepare communicator requests */
+    ompi_comm_request_init ();
+
     /* Init coll for the comms. This has to be after dpm_base_select, 
        (since dpm.mark_dyncomm is not set in the communicator creation
        function else), but before dpm.dyncom_init, since this function