Added new module for intercomm collectives. This will require an

autogen.

This commit was SVN r14149.

ompi/communicator/comm.c

@@ -9,6 +9,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2007      University of Houston. All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -101,12 +102,18 @@ int ompi_comm_set ( ompi_communicator_t *newcomm,
     ompi_set_group_rank(newcomm->c_local_group, my_gpointer);
     newcomm->c_my_rank = newcomm->c_local_group->grp_my_rank;
 
-    /* Set remote group, if applicable */
+    /* Set remote group and duplicate the local comm, if applicable */
     if ( 0 < remote_size) {        
         memcpy ( newcomm->c_remote_group->grp_proc_pointers, 
                  remote_procs, remote_size * sizeof(ompi_proc_t *));
         ompi_group_increment_proc_count(newcomm->c_remote_group);
         newcomm->c_flags |= OMPI_COMM_INTER;
+	if ( OMPI_COMM_IS_INTRA(oldcomm) ) {
+	    ompi_comm_dup(oldcomm, &newcomm->c_local_comm,1);
+	}
+	else {
+	    ompi_comm_dup(oldcomm->c_local_comm, &newcomm->c_local_comm,1);
+	}
     }
 
     /* Check how many different jobids are represented in this communicator.
@@ -323,6 +330,7 @@ int ompi_comm_create ( ompi_communicator_t *comm, ompi_group_t *group,
                               NULL,     /* remote_leader */
                               mode,     /* mode */
                               -1,       /* send first */
+			      0,        /* sync_flag */
                               NULL );   /* coll component */
                              
     if ( OMPI_SUCCESS != rc ) {
@@ -542,13 +550,14 @@ int ompi_comm_split ( ompi_communicator_t* comm, int color, int key,
 
     /* Activate the communicator and init coll-component */
     rc = ompi_comm_activate ( newcomp,  /* new communicator */ 
-                             comm,     /* old comm */
-                             NULL,     /* bridge comm */
-                             NULL,     /* local leader */
-                             NULL,     /* remote_leader */
-                             mode,     /* mode */
-                             -1,       /* send first */
-                             NULL );   /* coll component */
+			      comm,     /* old comm */
+			      NULL,     /* bridge comm */
+			      NULL,     /* local leader */
+			      NULL,     /* remote_leader */
+			      mode,     /* mode */
+			      -1,       /* send first */
+			      0,        /* sync_flag */
+			      NULL );   /* coll component */
                              
     if ( OMPI_SUCCESS != rc ) {
         goto exit;
@@ -588,7 +597,8 @@ int ompi_comm_split ( ompi_communicator_t* comm, int color, int key,
 /**********************************************************************/
 /**********************************************************************/
 /**********************************************************************/
-int ompi_comm_dup ( ompi_communicator_t * comm, ompi_communicator_t **newcomm)
+int ompi_comm_dup ( ompi_communicator_t * comm, ompi_communicator_t **newcomm, 
+		    int sync_flag)
 {
     ompi_communicator_t *comp=NULL;
     ompi_communicator_t *newcomp=NULL;
@@ -643,18 +653,38 @@ int ompi_comm_dup ( ompi_communicator_t * comm, ompi_communicator_t **newcomm)
     snprintf(newcomp->c_name, MPI_MAX_OBJECT_NAME, "MPI COMMUNICATOR %d DUP FROM %d", 
              newcomp->c_contextid, comm->c_contextid );
 
-    /* activate communicator and init coll-module */
-    rc = ompi_comm_activate (newcomp,  /* new communicator */ 
-                             comp,     /* old comm */
-                             NULL,     /* bridge comm */
-                             NULL,     /* local leader */
-                             NULL,     /* remote_leader */
-                             mode,     /* mode */
-                             -1,      /* send_first */
-                             (mca_base_component_t *) comp->c_coll_selected_component /* coll component */
-                             );
-    if ( MPI_SUCCESS != rc ) {
-        return rc;
+    if(0 == sync_flag) {
+	/* activate communicator and init coll-module */
+	rc = ompi_comm_activate (newcomp,  /* new communicator */ 
+				 comp,     /* old comm */
+				 NULL,     /* bridge comm */
+				 NULL,     /* local leader */
+				 NULL,     /* remote_leader */
+				 mode,     /* mode */
+				 -1,       /* send_first */
+				 0,        /* sync_flag */
+				 (mca_base_component_t *) comp->c_coll_selected_component /* coll component */
+	    );
+	if ( MPI_SUCCESS != rc ) {
+	    return rc;
+	}
+    }
+    else { 
+	/* activate communicator and init coll-module without synchronizing processes*/
+	rc = ompi_comm_activate (newcomp,  /* new communicator */ 
+				 comp,     /* old comm */
+				 NULL,     /* bridge comm */
+				 NULL,     /* local leader */
+				 NULL,     /* remote_leader */
+				 mode,     /* mode */
+				 -1,       /* send_first */
+				 1,        /* sync_flag */
+				 (mca_base_component_t *) comp->c_coll_selected_component /* coll component */
+	    );
+	if ( MPI_SUCCESS != rc ) {
+	    return rc;
+	}
+	
     }
 
     *newcomm = newcomp;
@@ -796,6 +826,9 @@ int ompi_comm_free ( ompi_communicator_t **comm )
           we delay releasing the attributes -- we need to release the
           attributes right away so that we can report the error right
           away. */
+    if ( OMPI_COMM_IS_INTER(*comm) ) {
+	ompi_comm_free (&(*comm)->c_local_comm);
+    }
 
     if (NULL != (*comm)->c_keyhash) {
         ret = ompi_attr_delete_all(COMM_ATTR, *comm, (*comm)->c_keyhash);
@@ -1315,6 +1348,7 @@ int ompi_topo_create (ompi_communicator_t *old_comm,
                                NULL,     /* remote_leader */
                                OMPI_COMM_CID_INTRA,   /* mode */
                                -1,       /* send first, doesn't matter */
+			       0,        /* sync_flag */
                                NULL );   /* coll component */
 
     if (OMPI_SUCCESS != ret) {

ompi/communicator/comm_cid.c

@@ -379,36 +379,38 @@ int ompi_comm_activate ( ompi_communicator_t* newcomm,
                          void* remote_leader,
                          int mode, 
                          int send_first, 
+			 int sync_flag,
                          mca_base_component_t *collcomponent )
 {
     int ok=0, gok=0;
     ompi_comm_cid_allredfct* allredfnct;
 
-    /* Step 1: the barrier, after which it is allowed to 
-     * send messages over the new communicator 
-     */
-    switch (mode) 
+    if (0 == sync_flag) {
+	/* Step 1: the barrier, after which it is allowed to 
+	 * send messages over the new communicator 
+	 */
+	switch (mode) 
         {
-        case OMPI_COMM_CID_INTRA: 
-            allredfnct=(ompi_comm_cid_allredfct*)ompi_comm_allreduce_intra;
-            break;
-        case OMPI_COMM_CID_INTER:
-            allredfnct=(ompi_comm_cid_allredfct*)ompi_comm_allreduce_inter;
-            break;
-        case OMPI_COMM_CID_INTRA_BRIDGE: 
-            allredfnct=(ompi_comm_cid_allredfct*)ompi_comm_allreduce_intra_bridge;
-            break;
-        case OMPI_COMM_CID_INTRA_OOB: 
-            allredfnct=(ompi_comm_cid_allredfct*)ompi_comm_allreduce_intra_oob;
-            break;
-        default: 
-            return MPI_UNDEFINED;
-            break;
+	    case OMPI_COMM_CID_INTRA: 
+		allredfnct=(ompi_comm_cid_allredfct*)ompi_comm_allreduce_intra;
+		break;
+	    case OMPI_COMM_CID_INTER:
+		allredfnct=(ompi_comm_cid_allredfct*)ompi_comm_allreduce_inter;
+		break;
+	    case OMPI_COMM_CID_INTRA_BRIDGE: 
+		allredfnct=(ompi_comm_cid_allredfct*)ompi_comm_allreduce_intra_bridge;
+		break;
+	    case OMPI_COMM_CID_INTRA_OOB: 
+		allredfnct=(ompi_comm_cid_allredfct*)ompi_comm_allreduce_intra_oob;
+		break;
+	    default: 
+		return MPI_UNDEFINED;
+		break;
         }
-
-    (allredfnct)(&ok, &gok, 1, MPI_MIN, comm, bridgecomm,
-                 local_leader, remote_leader, send_first );
-    
+	
+	(allredfnct)(&ok, &gok, 1, MPI_MIN, comm, bridgecomm,
+		     local_leader, remote_leader, send_first );
+    }
     /* Check to see if this process is in the new communicator.
 
        Specifically, this function is invoked by all proceses in the

ompi/communicator/comm_dyn.c

@@ -240,6 +240,7 @@ int ompi_comm_connect_accept ( ompi_communicator_t *comm, int root,
                               rport,                   /* remote leader */
                               OMPI_COMM_CID_INTRA_OOB, /* mode */
                               send_first,              /* send or recv first */
+			      0,                       /* sync_flag */
                               NULL );                  /* coll component */
     if ( OMPI_SUCCESS != rc ) {
         goto exit;

ompi/communicator/communicator.h

@@ -10,7 +10,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2006      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2006      University of Houston. All rights reserved.
+ * Copyright (c) 2006-2007 University of Houston. All rights reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -111,6 +111,10 @@ struct ompi_communicator_t {
     ompi_group_t        *c_local_group;
     ompi_group_t       *c_remote_group;
 
+    struct ompi_communicator_t *c_local_comm; /* a duplicate of the local 
+						 communicator in case the comm is 
+						 an inter-comm*/
+					 
     /* Attributes */
     struct opal_hash_table_t       *c_keyhash;
 
@@ -306,10 +310,13 @@ struct ompi_communicator_t {
      * of the function. It has been extracted, since we need to be able
      * to dup a communicator internally as well.
      *
-     * @param comm: input communicator
+     * @param comm:      input communicator
+     *        sync_flag: 0 if processes need to synchronize in activate
+     *                   1 if the do not (optimization to c_local_comm creation)
      *
      */
-    int ompi_comm_dup (ompi_communicator_t *comm, ompi_communicator_t **newcomm);
+    int ompi_comm_dup (ompi_communicator_t *comm, ompi_communicator_t **newcomm, 
+		       int sync_flag);
 
 
     /**
@@ -410,6 +417,7 @@ struct ompi_communicator_t {
                              void* remote_leader,
                              int mode,
                              int send_first,
+			     int sync_flag,
                              mca_base_component_t *collcomponent );
 
 

ompi/mca/coll/inter/Makefile.am

@@ -0,0 +1,58 @@
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if OMPI_BUILD_coll_inter_DSO
+component_noinst =
+component_install = mca_coll_inter.la
+else
+component_noinst = libmca_coll_inter.la
+component_install =
+endif
+
+mcacomponentdir = $(libdir)/openmpi
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_coll_inter_la_SOURCES = $(sources)
+mca_coll_inter_la_LDFLAGS = -module -avoid-version
+mca_coll_inter_la_LIBADD = \
+        $(top_ompi_builddir)/ompi/libmpi.la \
+        $(top_ompi_builddir)/orte/libopen-rte.la \
+        $(top_ompi_builddir)/opal/libopen-pal.la
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_coll_inter_la_SOURCES = $(sources)
+libmca_coll_inter_la_LDFLAGS = -module -avoid-version
+
+# Source files
+
+sources = \
+        coll_inter.h \
+        coll_inter.c \
+        coll_inter_allreduce.c \
+	coll_inter_allgather.c \
+	coll_inter_allgatherv.c \
+	coll_inter_gather.c \
+	coll_inter_gatherv.c \
+        coll_inter_scatter.c \
+	coll_inter_scatterv.c \
+        coll_inter_bcast.c \
+        coll_inter_component.c \
+        coll_inter_reduce.c

ompi/mca/coll/inter/coll_inter.c

@@ -0,0 +1,213 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 University of Houston. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "coll_inter.h"
+
+#include <stdio.h>
+
+#include "mpi.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/group/group.h"
+#include "ompi/proc/proc.h"
+#include "ompi/op/op.h"
+
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/coll/base/base.h"
+#include "ompi/mca/coll/base/coll_tags.h"
+
+#include "ompi/class/ompi_bitmap.h"
+#include "ompi/mca/bml/bml.h"
+#include "ompi/mca/bml/base/base.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/mca/btl/btl.h"
+
+
+static void mca_coll_inter_dump_struct ( struct mca_coll_base_comm_t *c);
+
+/*
+ * Linear set of collective algorithms
+ */
+static const mca_coll_base_module_1_0_0_t inter = {
+
+  /* Initialization / finalization functions */
+
+  mca_coll_inter_module_init,
+  mca_coll_inter_module_finalize,
+
+  /* Collective function pointers */
+  /* function pointers marked with NULL are not yet implemented
+     and will use the functions provided in the basic module */
+  mca_coll_inter_allgather_inter,
+  mca_coll_inter_allgatherv_inter,
+  mca_coll_inter_allreduce_inter, 
+  NULL, /* alltoall */
+  NULL, /* alltoallv */
+  NULL, /* alltoallw */
+  NULL, /* barrier */
+  mca_coll_inter_bcast_inter,
+  NULL,  /* exscan */
+  mca_coll_inter_gather_inter,  
+  mca_coll_inter_gatherv_inter,  
+  mca_coll_inter_reduce_inter,
+  NULL,  /* reduce_scatter */
+  NULL,  /* scan */
+  mca_coll_inter_scatter_inter,  
+  mca_coll_inter_scatterv_inter
+};
+
+
+static const mca_coll_base_module_1_0_0_t null_inter = {
+
+  /* Initialization / finalization functions */
+
+  mca_coll_inter_module_init,
+  mca_coll_inter_module_finalize,
+
+  /* Collective function pointers */
+  /* function pointers marked with NULL are not yet implemented
+     and will use the functions provided in the basic module */
+  NULL,
+  NULL,
+  NULL,
+  NULL,
+  NULL,
+  NULL,
+  NULL,
+  NULL,
+  NULL, 
+  NULL, 
+  NULL, 
+  NULL,
+  NULL, 
+  NULL,
+  NULL, 
+  NULL
+};
+
+
+
+/*
+ * Initial query function that is invoked during MPI_INIT, allowing
+ * this module to indicate what level of thread support it provides.
+ */
+int mca_coll_inter_init_query(bool allow_inter_user_threads,
+                             bool have_hidden_user_threads)
+{
+
+    /* Don't ask. All done */
+    return OMPI_SUCCESS;
+}
+
+
+/*
+ * Invoked when there's a new communicator that has been created.
+ * Look at the communicator and decide which set of functions and
+ * priority we want to return.
+ */
+const mca_coll_base_module_1_0_0_t *
+mca_coll_inter_comm_query(struct ompi_communicator_t *comm, int *priority,
+                             struct mca_coll_base_comm_t **data)
+{
+    int size, rsize;
+
+    /* This module only works for inter-communicators */
+    if ( !OMPI_COMM_IS_INTER(comm) ) {
+        return NULL;
+    }
+
+    /* Get the priority level */
+    *priority = mca_coll_inter_priority_param;
+    if ( 0 >= mca_coll_inter_priority_param ) {
+	return NULL;
+    }
+
+    size = ompi_comm_size(comm);
+    rsize = ompi_comm_remote_size(comm);
+    
+    if ( size < mca_coll_inter_crossover && rsize < mca_coll_inter_crossover) {
+	return NULL;
+    }
+ 
+    return &inter;
+}
+    
+
+/*
+ * Init module on the communicator
+ */
+const struct mca_coll_base_module_1_0_0_t *
+mca_coll_inter_module_init(struct ompi_communicator_t *comm)
+{
+    int size, rank;
+    struct mca_coll_base_comm_t *data=NULL;
+    
+    rank = ompi_comm_rank(comm);
+    size = ompi_comm_size(comm);
+    
+    data = (struct mca_coll_base_comm_t *) malloc ( sizeof(struct mca_coll_base_comm_t));
+    if ( NULL == data ) {
+	return NULL;
+    }
+    data->inter_comm     = comm;
+
+    comm->c_coll_selected_data=data;
+    
+    if ( mca_coll_inter_verbose_param ) {
+      mca_coll_inter_dump_struct (data);
+    }
+  
+    return &inter;
+}
+
+
+/*
+ * Finalize module on the communicator
+ */
+int mca_coll_inter_module_finalize(struct ompi_communicator_t *comm)
+{
+    struct mca_coll_base_comm_t *data=NULL;
+
+    data   = comm->c_coll_selected_data;
+    free ( data );
+    comm->c_coll_selected_data = NULL;
+
+    return OMPI_SUCCESS;
+}
+
+int mca_coll_inter_comm_unquery ( struct ompi_communicator_t *comm,
+				     struct mca_coll_base_comm_t *data )
+{
+    return OMPI_SUCCESS;
+}
+
+
+
+static void mca_coll_inter_dump_struct ( struct mca_coll_base_comm_t *c)
+{
+    int rank;
+
+    rank = ompi_comm_rank ( c->inter_comm );
+
+    printf("%d: Dump of inter-struct for  comm %s cid %u\n", 
+           rank, c->inter_comm->c_name, c->inter_comm->c_contextid);
+
+    
+    return;
+}

ompi/mca/coll/inter/coll_inter.h

@@ -0,0 +1,122 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 University of Houston. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#ifndef MCA_COLL_INTER_EXPORT_H
+#define MCA_COLL_INTER_EXPORT_H
+
+#define mca_coll_inter_crossover 1
+#include "ompi_config.h"
+
+#include "mpi.h"
+#include "opal/mca/mca.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/request/request.h"
+#include "ompi/communicator/communicator.h"
+
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+
+/*
+ * Globally exported variable
+ */
+
+OMPI_MODULE_DECLSPEC extern const mca_coll_base_component_1_0_0_t mca_coll_inter_component;
+extern int mca_coll_inter_priority_param;
+extern int mca_coll_inter_verbose_param;
+
+
+/*
+ * Data structure for attaching data to the communicator 
+ */
+
+/* Clarifying some terminology:
+ *  comm:    the input communicator, consisting of several lower level communicators.
+*/
+
+    struct mca_coll_base_comm_t {
+	struct ompi_communicator_t        *inter_comm; /* link back to the attached comm */ 
+    };
+
+
+/*
+ * coll API functions
+ */
+int mca_coll_inter_init_query(bool allow_inter_user_threads,
+				 bool have_hidden_threads);
+const struct mca_coll_base_module_1_0_0_t *
+mca_coll_inter_comm_query(struct ompi_communicator_t *comm, 
+			     int *priority, struct mca_coll_base_comm_t **data);
+int mca_coll_inter_comm_unquery(struct ompi_communicator_t *comm, 
+				   struct mca_coll_base_comm_t *data);
+
+const struct mca_coll_base_module_1_0_0_t *
+mca_coll_inter_module_init(struct ompi_communicator_t *comm);
+
+int mca_coll_inter_module_finalize(struct ompi_communicator_t *comm);
+
+int mca_coll_inter_allgather_inter(void *sbuf, int scount, 
+				   struct ompi_datatype_t *sdtype, 
+				   void *rbuf, int rcount, 
+				   struct ompi_datatype_t *rdtype, 
+				   struct ompi_communicator_t *comm);
+int mca_coll_inter_allgatherv_inter(void *sbuf, int scount,
+				    struct ompi_datatype_t *sdtype,
+				    void *rbuf, int *rcounts, int *disps,
+				    struct ompi_datatype_t *rdtype,
+				    struct ompi_communicator_t *comm);
+int mca_coll_inter_allreduce_inter(void *sbuf, void *rbuf, int count, 
+				   struct ompi_datatype_t *dtype, 
+				   struct ompi_op_t *op, 
+				   struct ompi_communicator_t *comm);
+int mca_coll_inter_bcast_inter(void *buff, int count, 
+			       struct ompi_datatype_t *datatype,
+			       int root, 
+			       struct ompi_communicator_t *comm);
+int mca_coll_inter_gather_inter(void *sbuf, int scount, 
+				struct ompi_datatype_t *sdtype, 
+				void *rbuf, int rcount, 
+				struct ompi_datatype_t *rdtype, 
+				int root, 
+				struct ompi_communicator_t *comm);
+int mca_coll_inter_gatherv_inter(void *sbuf, int scount, 
+				 struct ompi_datatype_t *sdtype, 
+				 void *rbuf, int *rcounts, int *disps, 
+				 struct ompi_datatype_t *rdtype, 
+				 int root, 
+				 struct ompi_communicator_t *comm);
+int mca_coll_inter_reduce_inter(void *sbuf, void* rbuf, int count, 
+				struct ompi_datatype_t *dtype, 
+				struct ompi_op_t *op, 
+				int root,
+				struct ompi_communicator_t *comm);
+int mca_coll_inter_scatter_inter(void *sbuf, int scount, 
+				 struct ompi_datatype_t *sdtype, void *rbuf, 
+				 int rcount, struct ompi_datatype_t *rdtype, 
+				 int root, struct ompi_communicator_t *comm);
+int mca_coll_inter_scatterv_inter(void *sbuf, int *scounts, int *disps, 
+				  struct ompi_datatype_t *sdtype, 
+				  void* rbuf, int rcount, 
+				  struct ompi_datatype_t *rdtype, int root, 
+				  struct ompi_communicator_t *comm);
+
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+#endif /* MCA_COLL_INTER_EXPORT_H */

ompi/mca/coll/inter/coll_inter_allgather.c

@@ -0,0 +1,109 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 University of Houston. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "coll_inter.h"
+
+#include <stdlib.h>
+
+#include "mpi.h"
+#include "ompi/constants.h"
+#include "ompi/datatype/datatype.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/pml/pml.h"
+#include "ompi/mca/coll/base/coll_tags.h"
+
+/*
+ *	allgather_inter
+ *
+ *	Function:	- allgather using other MPI collections
+ *	Accepts:	- same as MPI_Allgather()
+ *	Returns:	- MPI_SUCCESS or error code
+ */
+int
+mca_coll_inter_allgather_inter(void *sbuf, int scount,
+                               struct ompi_datatype_t *sdtype,
+                               void *rbuf, int rcount,
+                               struct ompi_datatype_t *rdtype,
+                               struct ompi_communicator_t *comm)
+{
+    int rank, root = 0, size, rsize, err;
+    char *ptmp = NULL;
+    ptrdiff_t slb, sextent, incr;
+    ompi_request_t *req[2];
+
+    rank = ompi_comm_rank(comm);
+    size = ompi_comm_size(comm->c_local_comm);
+    rsize = ompi_comm_remote_size(comm);
+
+    /* Perform the gather locally at the root */
+    err = ompi_ddt_get_extent(sdtype, &slb, &sextent);
+    if (OMPI_SUCCESS != err) {
+	return OMPI_ERROR;
+    }
+    
+    incr = sextent * scount;
+    ptmp = (char*)malloc(size * incr); 
+    if (NULL == ptmp) {
+	return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    err = comm->c_local_comm->c_coll.coll_gather(sbuf, scount, sdtype, 
+						 ptmp, scount, sdtype, 
+						 0, comm->c_local_comm);
+    if (OMPI_SUCCESS != err) {
+	goto exit;
+    }
+
+    if (rank == root) {
+	/* Do a send-recv between the two root procs. to avoid deadlock */
+        err = MCA_PML_CALL(irecv(rbuf, rcount*rsize, rdtype, 0,
+                                 MCA_COLL_BASE_TAG_ALLGATHER, comm,
+                                 &(req[0])));
+        if (OMPI_SUCCESS != err) {
+            goto exit;
+        }
+
+        err = MCA_PML_CALL(isend(ptmp, scount*size, sdtype, 0,
+                                 MCA_COLL_BASE_TAG_ALLGATHER,
+                                 MCA_PML_BASE_SEND_STANDARD,
+                                 comm, &(req[1])));
+        if (OMPI_SUCCESS != err) {
+            goto exit;
+        }
+
+        err = ompi_request_wait_all(2, req, MPI_STATUSES_IGNORE);
+        if (OMPI_SUCCESS != err) {
+            goto exit;
+        }
+    }
+    /* bcast the message to all the local processes */
+    err = comm->c_local_comm->c_coll.coll_bcast(rbuf, rcount*rsize, rdtype, 
+						root, comm->c_local_comm);
+    if (OMPI_SUCCESS != err) {
+            goto exit;
+    }
+
+  exit:
+    if (NULL != ptmp) {
+        free(ptmp);
+    }
+
+    return err;
+}

ompi/mca/coll/inter/coll_inter_allgatherv.c

@@ -0,0 +1,150 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 University of Houston. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "coll_inter.h"
+
+#include "mpi.h"
+#include "ompi/datatype/datatype.h"
+#include "ompi/constants.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/coll/base/coll_tags.h"
+#include "ompi/mca/pml/pml.h"
+
+
+/*
+ *	allgatherv_inter
+ *
+ *	Function:	- allgatherv using other MPI collectives
+ *	Accepts:	- same as MPI_Allgatherv()
+ *	Returns:	- MPI_SUCCESS or error code
+ */
+int
+mca_coll_inter_allgatherv_inter(void *sbuf, int scount,
+                                struct ompi_datatype_t *sdtype,
+                                void *rbuf, int *rcounts, int *disps,
+                                struct ompi_datatype_t *rdtype,
+                                struct ompi_communicator_t *comm)
+{
+    int i, rank, size, size_local, total=0, err;
+    int *count=NULL,*displace=NULL;
+    char *ptmp=NULL;
+    MPI_Aint incr;
+    MPI_Aint extent;
+    MPI_Aint lb;
+    ompi_datatype_t *ndtype;
+    ompi_request_t *req[2];
+
+    rank = ompi_comm_rank(comm);
+    size_local = ompi_comm_size(comm->c_local_comm);
+    size = ompi_comm_remote_size(comm);
+
+    if (0 == rank) {
+	count = (int *)malloc(sizeof(int) * size_local);
+	if (NULL == count) {
+	    return OMPI_ERR_OUT_OF_RESOURCE;
+	}
+	displace = (int *)malloc(sizeof(int) * size_local);
+	if (NULL == displace) {
+	    return OMPI_ERR_OUT_OF_RESOURCE;
+	}
+    }
+    /* Local gather to get the scount of each process */
+    err = comm->c_local_comm->c_coll.coll_gather(&scount, 1, MPI_INT, 
+						 count, 1, MPI_INT, 
+						 0, comm->c_local_comm);
+    if (OMPI_SUCCESS != err) {
+	return err;
+    }
+    if(0 == rank) {
+	displace[0] = 0;
+	for (i = 1; i < size_local; i++) {
+	    displace[i] = displace[i-1] + count[i-1];
+	}
+	/* Perform the gatherv locally with the first process as root */
+	err = ompi_ddt_get_extent(sdtype, &lb, &extent);
+	if (OMPI_SUCCESS != err) {
+	    return OMPI_ERROR;
+	}
+	incr = 0;
+	for (i = 0; i < size_local; i++) {
+	    incr = incr + extent*count[i];
+	}
+	ptmp = (char*)malloc(incr); 
+	if (NULL == ptmp) {
+	    return OMPI_ERR_OUT_OF_RESOURCE;
+	}
+    }
+    err = comm->c_local_comm->c_coll.coll_gatherv(sbuf, scount, sdtype, 
+						  ptmp, count, displace, 
+						  sdtype,0, comm->c_local_comm);
+    if (OMPI_SUCCESS != err) {
+	return err;
+    }
+    
+    ompi_ddt_create_indexed(size,rcounts,disps,rdtype,&ndtype);
+    ompi_ddt_commit(&ndtype);
+
+    if (0 == rank) { 
+	for (i = 0; i < size_local; i++) {
+	    total = total + count[i];
+	}
+	/* Exchange data between roots */
+	err = MCA_PML_CALL(irecv(rbuf, 1, ndtype, 0,
+                                 MCA_COLL_BASE_TAG_ALLGATHERV, comm,
+                                 &(req[0])));
+        if (OMPI_SUCCESS != err) {
+            return err;
+        }
+
+        err = MCA_PML_CALL(isend(ptmp, total, sdtype, 0,
+                                 MCA_COLL_BASE_TAG_ALLGATHERV,
+                                 MCA_PML_BASE_SEND_STANDARD,
+                                 comm, &(req[1])));
+        if (OMPI_SUCCESS != err) {
+            return err;
+        }
+
+        err = ompi_request_wait_all(2, req, MPI_STATUSES_IGNORE);
+        if (OMPI_SUCCESS != err) {
+            return err;
+	}
+    }
+    
+    /* bcast the message to all the local processes */
+    err = comm->c_local_comm->c_coll.coll_bcast(rbuf, 1, ndtype, 
+						0, comm->c_local_comm);
+    if (OMPI_SUCCESS != err) {
+            return err;
+    }
+
+    ompi_ddt_destroy(&ndtype);
+    if (NULL != ptmp) {
+	free(ptmp);
+    }
+    if (NULL != displace) {
+	free(displace);
+    }
+    if (NULL != count) {
+	free(count);
+    }
+
+    return err;
+
+}

ompi/mca/coll/inter/coll_inter_allreduce.c

@@ -0,0 +1,108 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 University of Houston. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "coll_inter.h"
+
+#include "mpi.h"
+#include "ompi/constants.h"
+#include "ompi/datatype/datatype.h"
+#include "ompi/op/op.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/coll/base/coll_tags.h"
+#include "ompi/mca/pml/pml.h"
+
+/*
+ *	allreduce_inter
+ *
+ *	Function:	- allreduce using other MPI collectives
+ *	Accepts:	- same as MPI_Allreduce()
+ *	Returns:	- MPI_SUCCESS or error code
+ */
+int
+mca_coll_inter_allreduce_inter(void *sbuf, void *rbuf, int count,
+                               struct ompi_datatype_t *dtype,
+                               struct ompi_op_t *op,
+                               struct ompi_communicator_t *comm)
+{
+    int err, rank, root = 0, rsize;
+    ptrdiff_t lb, extent;
+    char *tmpbuf = NULL, *pml_buffer = NULL;
+    ompi_request_t *req[2];
+
+    rank = ompi_comm_rank(comm);
+    rsize = ompi_comm_remote_size(comm);
+
+    
+    /* Perform the reduction locally */
+    err = ompi_ddt_get_extent(dtype, &lb, &extent);
+    if (OMPI_SUCCESS != err) {
+	return OMPI_ERROR;
+    }
+    
+    tmpbuf = (char *) malloc(count * extent);
+    if (NULL == tmpbuf) {
+	return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+    pml_buffer = tmpbuf - lb;
+   
+    err = comm->c_local_comm->c_coll.coll_reduce(sbuf, pml_buffer, count,
+						 dtype, op, root, 
+						 comm->c_local_comm);
+    if (OMPI_SUCCESS != err) {
+	goto exit;
+    }
+
+    if (rank == root) {
+	/* Do a send-recv between the two root procs. to avoid deadlock */
+        err = MCA_PML_CALL(irecv(rbuf, count, dtype, 0,
+                                 MCA_COLL_BASE_TAG_ALLREDUCE, comm,
+                                 &(req[0])));
+        if (OMPI_SUCCESS != err) {
+            goto exit;
+        }
+
+        err = MCA_PML_CALL(isend(pml_buffer, count, dtype, 0,
+                                 MCA_COLL_BASE_TAG_ALLREDUCE,
+                                 MCA_PML_BASE_SEND_STANDARD,
+                                 comm, &(req[1])));
+        if (OMPI_SUCCESS != err) {
+            goto exit;
+        }
+
+        err = ompi_request_wait_all(2, req, MPI_STATUSES_IGNORE);
+        if (OMPI_SUCCESS != err) {
+            goto exit;
+        }
+    }
+
+    /* bcast the message to all the local processes */
+    err = comm->c_local_comm->c_coll.coll_bcast(rbuf, count, dtype, 
+						root, comm->c_local_comm);
+    if (OMPI_SUCCESS != err) {
+            goto exit;
+    }
+
+  exit:
+    if (NULL != tmpbuf) {
+        free(tmpbuf);
+    }
+
+    return err;
+}

ompi/mca/coll/inter/coll_inter_bcast.c

@@ -0,0 +1,80 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 University of Houston. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "coll_inter.h"
+
+#include "mpi.h"
+#include "ompi/constants.h"
+#include "opal/util/output.h"
+#include "ompi/communicator/communicator.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/coll/base/coll_tags.h"
+#include "ompi/mca/pml/pml.h"
+
+
+/*
+ *	bcast_inter
+ *
+ *	Function:	- broadcast using the local_comm
+ *	Accepts:	- same arguments as MPI_Bcast()
+ *	Returns:	- MPI_SUCCESS or error code
+ */
+int
+mca_coll_inter_bcast_inter(void *buff, int count,
+                               struct ompi_datatype_t *datatype, int root,
+                               struct ompi_communicator_t *comm)
+{
+    int rsize;
+    int rank;
+    int err;
+
+    rsize = ompi_comm_remote_size(comm);
+    rank = ompi_comm_rank(comm);
+
+    if (MPI_PROC_NULL == root) {
+        /* do nothing */
+        err = OMPI_SUCCESS;
+    } else if (MPI_ROOT != root) {
+        /* Non-root, first process recieves the data and bcast to others */
+	if ( 0 == rank ) {
+	    err = MCA_PML_CALL(recv(buff, count, datatype, root,
+				    MCA_COLL_BASE_TAG_BCAST, comm,
+				    MPI_STATUS_IGNORE));
+	    if (OMPI_SUCCESS != err) {
+                return err;
+            }
+	}
+	err = comm->c_local_comm->c_coll.coll_bcast(buff, count, datatype, 0, 
+					      comm->c_local_comm);
+    } else {
+        /* root section, send to the first process of the remote group */
+	err = MCA_PML_CALL(send(buff, count, datatype, 0,
+				MCA_COLL_BASE_TAG_BCAST,
+				MCA_PML_BASE_SEND_STANDARD, 
+				comm));
+	if (OMPI_SUCCESS != err) {
+	    return err;
+	}
+    }
+
+    /* All done */
+    return err;
+}
+

ompi/mca/coll/inter/coll_inter_component.c

@@ -0,0 +1,112 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 University of Houston. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ *
+ * These symbols are in a file by themselves to provide nice linker
+ * semantics.  Since linkers generally pull in symbols by object
+ * files, keeping these symbols as the only symbols in this file
+ * prevents utility programs such as "ompi_info" from having to import
+ * entire components just to query their version and parameters.
+ */
+
+#include "ompi_config.h"
+#include "coll_inter.h"
+
+#include "mpi.h"
+#include "ompi/mca/coll/coll.h"
+
+/*
+ * Public string showing the coll ompi_inter component version number
+ */
+const char *mca_coll_inter_component_version_string =
+  "OMPI/MPI inter collective MCA component version " OMPI_VERSION;
+
+/*
+ * Global variable
+ */
+int mca_coll_inter_priority_param = 40;
+int mca_coll_inter_verbose_param = 0;
+
+
+/*
+ * Local function
+ */
+static int inter_open(void);
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+
+const mca_coll_base_component_1_0_0_t mca_coll_inter_component = {
+
+  /* First, the mca_component_t struct containing meta information
+     about the component itself */
+
+  {
+    /* Indicate that we are a coll v1.0.0 component (which also implies a
+       specific MCA version) */
+
+    MCA_COLL_BASE_VERSION_1_0_0,
+
+    /* Component name and version */
+
+    "inter",
+    OMPI_MAJOR_VERSION,
+    OMPI_MINOR_VERSION,
+    OMPI_RELEASE_VERSION,
+
+    /* Component open and close functions */
+
+    inter_open,
+    NULL
+  },
+
+  /* Next the MCA v1.0.0 component meta data */
+
+  {
+   /* Whether the component is checkpointable or not */
+   true
+  },
+
+  /* Initialization / querying functions */
+  mca_coll_inter_init_query,
+  mca_coll_inter_comm_query,
+  mca_coll_inter_comm_unquery
+};
+
+
+static int inter_open(void)
+{
+    
+
+    /* Use a high priority, but allow other components to be higher */
+    mca_base_param_reg_int(&mca_coll_inter_component.collm_version, 
+			   "priority",
+                           "Priority of the inter coll component",
+                           false, false, mca_coll_inter_priority_param,
+                           &mca_coll_inter_priority_param);
+
+
+    mca_base_param_reg_int(&mca_coll_inter_component.collm_version, 
+			   "verbose",
+                           "Turn verbose message of the inter coll component on/off",
+                           false, false, mca_coll_inter_verbose_param,
+                           &mca_coll_inter_verbose_param);
+
+    return OMPI_SUCCESS;
+}

ompi/mca/coll/inter/coll_inter_gather.c

@@ -0,0 +1,100 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 University of Houston. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "coll_inter.h"
+
+#include "mpi.h"
+#include "ompi/constants.h"
+#include "ompi/datatype/datatype.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/coll/base/coll_tags.h"
+#include "ompi/mca/pml/pml.h"
+
+
+/*
+ *	gather_inter
+ *
+ *	Function:	- basic gather operation
+ *	Accepts:	- same arguments as MPI_Gather()
+ *	Returns:	- MPI_SUCCESS or error code
+ */
+int
+mca_coll_inter_gather_inter(void *sbuf, int scount,
+                            struct ompi_datatype_t *sdtype,
+                            void *rbuf, int rcount,
+                            struct ompi_datatype_t *rdtype,
+                            int root, struct ompi_communicator_t *comm)
+{
+    int err;
+    int rank;
+    int size,size_local;
+    char *ptmp = NULL;
+    MPI_Aint incr;
+    MPI_Aint extent;
+    MPI_Aint lb;
+
+    size = ompi_comm_remote_size(comm);
+    rank = ompi_comm_rank(comm);
+
+    if (MPI_PROC_NULL == root) {
+        /* do nothing */
+        err = OMPI_SUCCESS;
+    } else if (MPI_ROOT != root) {
+	/* Perform the gather locally with the first process as root */
+	err = ompi_ddt_get_extent(sdtype, &lb, &extent);
+	if (OMPI_SUCCESS != err) {
+	    return OMPI_ERROR;
+	}
+	
+	incr = extent * scount;
+	size_local = ompi_comm_size(comm->c_local_comm);
+	ptmp = (char*)malloc(size_local * incr); 
+	if (NULL == ptmp) {
+            return OMPI_ERR_OUT_OF_RESOURCE;
+        }
+	
+	err = comm->c_local_comm->c_coll.coll_gather(sbuf, scount, sdtype, 
+						     ptmp, scount, sdtype, 
+						     0, comm->c_local_comm);
+	if (0 == rank) { 
+	    /* First process sends data to the root */
+	    err = MCA_PML_CALL(send(ptmp, scount*size_local, sdtype, root,
+				    MCA_COLL_BASE_TAG_GATHER,
+				    MCA_PML_BASE_SEND_STANDARD, comm));
+	    if (OMPI_SUCCESS != err) {
+                return err;
+            }
+	}
+	if (NULL != ptmp) {
+	    free(ptmp);
+	}
+    } else {
+        /* I am the root, loop receiving the data. */
+	err = MCA_PML_CALL(recv(rbuf, rcount*size, rdtype, 0,
+				MCA_COLL_BASE_TAG_GATHER,
+				comm, MPI_STATUS_IGNORE));
+	if (OMPI_SUCCESS != err) {
+	    return err;
+	}
+    }
+
+    /* All done */
+    return err;
+}

ompi/mca/coll/inter/coll_inter_gatherv.c

@@ -0,0 +1,141 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 University of Houston. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "coll_inter.h"
+
+#include "mpi.h"
+#include "ompi/constants.h"
+#include "ompi/datatype/datatype.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/coll/base/coll_tags.h"
+#include "ompi/mca/pml/pml.h"
+
+/*
+ *	gatherv_inter
+ *
+ *	Function:	- gatherv operation using a local gather on c_local_comm
+ *	Accepts:	- same arguments as MPI_Gatherv()
+ *	Returns:	- MPI_SUCCESS or error code
+ */
+int
+mca_coll_inter_gatherv_inter(void *sbuf, int scount,
+                             struct ompi_datatype_t *sdtype,
+                             void *rbuf, int *rcounts, int *disps,
+                             struct ompi_datatype_t *rdtype, int root,
+                             struct ompi_communicator_t *comm)
+{
+    
+    int i, rank, size, size_local, total=0, err;
+    int *count=NULL,*displace=NULL;
+    char *ptmp=NULL;
+    MPI_Aint incr;
+    MPI_Aint extent;
+    MPI_Aint lb;
+    ompi_datatype_t *ndtype;
+
+    size = ompi_comm_remote_size(comm);
+    rank = ompi_comm_rank(comm);
+    size_local = ompi_comm_size(comm);
+
+    if (MPI_PROC_NULL == root) {
+        /* do nothing */
+        err = OMPI_SUCCESS;
+    } else if (MPI_ROOT != root) {
+	if (0 == rank) {
+	    count = (int *)malloc(sizeof(int) * size_local);
+	    if (NULL == count) {
+		return OMPI_ERR_OUT_OF_RESOURCE;
+	    }
+	    displace = (int *)malloc(sizeof(int) * size_local);
+	    if (NULL == displace) {
+		return OMPI_ERR_OUT_OF_RESOURCE;
+	    }
+	}
+
+	err = comm->c_local_comm->c_coll.coll_gather(&scount, 1, MPI_INT, 
+						     count, 1, MPI_INT, 
+						     0, comm->c_local_comm);
+	if (OMPI_SUCCESS != err) {
+	    return err;
+	}
+	if(0 == rank) {
+	    displace[0] = 0;
+	    for (i = 1; i < size_local; i++) {
+		displace[i] = displace[i-1] + count[i-1];
+	    }
+	    /* Perform the gatherv locally with the first process as root */
+	    err = ompi_ddt_get_extent(sdtype, &lb, &extent);
+	    if (OMPI_SUCCESS != err) {
+		return OMPI_ERROR;
+	    }
+	    incr = 0;
+	    for (i = 0; i < size_local; i++) {
+		incr = incr + extent*count[i];
+	    }
+	    ptmp = (char*)malloc(incr); 
+	    if (NULL == ptmp) {
+		return OMPI_ERR_OUT_OF_RESOURCE;
+	    }
+	}
+	err = comm->c_local_comm->c_coll.coll_gatherv(sbuf, scount, sdtype, 
+						     ptmp, count, displace, 
+						     sdtype,0, comm->c_local_comm);
+	if (OMPI_SUCCESS != err) {
+	    return err;
+	}
+
+	if (0 == rank) { 
+	    for (i = 0; i < size_local; i++) {
+		total = total + count[i];
+	    }
+	    /* First process sends data to the root */
+	    err = MCA_PML_CALL(send(ptmp, total, sdtype, root,
+				    MCA_COLL_BASE_TAG_GATHERV,
+				    MCA_PML_BASE_SEND_STANDARD, comm));
+	    if (OMPI_SUCCESS != err) {
+                return err;
+            }
+	}
+	if (NULL != ptmp) {
+	    free(ptmp);
+	}
+	if (NULL != displace) {
+	    free(displace);
+	}
+	if (NULL != count) {
+	    free(count);
+	}
+    } else {
+        /* I am the root, loop receiving the data. */
+	ompi_ddt_create_indexed(size,rcounts,disps,rdtype,&ndtype);
+	ompi_ddt_commit(&ndtype);
+	
+	err = MCA_PML_CALL(recv(rbuf, 1, ndtype, 0,
+				MCA_COLL_BASE_TAG_GATHERV,
+				comm, MPI_STATUS_IGNORE));
+	if (OMPI_SUCCESS != err) {
+	    return err;
+	}
+	ompi_ddt_destroy(&ndtype);
+    }
+
+    /* All done */
+    return err;
+}

ompi/mca/coll/inter/coll_inter_reduce.c

@@ -0,0 +1,94 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 University of Houston. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "coll_inter.h"
+
+#include <stdio.h>
+
+#include "mpi.h"
+#include "ompi/constants.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/coll/base/coll_tags.h"
+#include "ompi/op/op.h"
+#include "ompi/mca/pml/pml.h"
+
+/*
+ *	reduce_inter
+ *
+ *	Function:	- reduction using the local_comm
+ *	Accepts:	- same as MPI_Reduce()
+ *	Returns:	- MPI_SUCCESS or error code
+ */
+int
+mca_coll_inter_reduce_inter(void *sbuf, void *rbuf, int count,
+                                struct ompi_datatype_t *dtype,
+                                struct ompi_op_t *op,
+                                int root, struct ompi_communicator_t *comm)
+{
+    int rank, err, size;
+    ptrdiff_t true_lb, true_extent, lb, extent;
+    char *free_buffer = NULL;
+    char *pml_buffer = NULL;
+
+    /* Initialize */
+    rank = ompi_comm_rank(comm);
+    size = ompi_comm_remote_size(comm);
+
+    if (MPI_PROC_NULL == root) {
+        /* do nothing */
+        err = OMPI_SUCCESS;
+    } else if (MPI_ROOT != root) {
+	/* Perform the reduce locally with the first process as root */
+	ompi_ddt_get_extent(dtype, &lb, &extent);
+	ompi_ddt_get_true_extent(dtype, &true_lb, &true_extent);
+
+	free_buffer = (char*)malloc(true_extent + (count - 1) * extent);
+	if (NULL == free_buffer) {
+	    return OMPI_ERR_OUT_OF_RESOURCE;
+	}
+	pml_buffer = free_buffer - lb;
+
+	err = comm->c_local_comm->c_coll.coll_reduce(sbuf, pml_buffer, count,
+						     dtype, op, 0, comm->c_local_comm);
+	if (0 == rank) {
+	    /* First process sends the result to the root */
+	    err = MCA_PML_CALL(send(pml_buffer, count, dtype, root,
+				    MCA_COLL_BASE_TAG_REDUCE,
+				    MCA_PML_BASE_SEND_STANDARD, comm));
+	    if (OMPI_SUCCESS != err) {
+                return err;
+            }
+	}
+	
+	if (NULL != free_buffer) {
+	    free(free_buffer);
+	}
+    } else {
+        /* Root receives the reduced message from the first process  */
+	err = MCA_PML_CALL(recv(rbuf, count, dtype, 0,
+				MCA_COLL_BASE_TAG_REDUCE, comm,
+				MPI_STATUS_IGNORE));
+	if (OMPI_SUCCESS != err) {
+	    return err;
+	}
+    }
+    /* All done */
+    return err;
+}

ompi/mca/coll/inter/coll_inter_scatter.c

@@ -0,0 +1,96 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 University of Houston. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "coll_inter.h"
+
+#include "mpi.h"
+#include "ompi/constants.h"
+#include "ompi/datatype/datatype.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/coll/base/coll_tags.h"
+#include "ompi/mca/pml/pml.h"
+
+/*
+ *	scatter_inter
+ *
+ *	Function:	- scatter operation
+ *	Accepts:	- same arguments as MPI_Scatter()
+ *	Returns:	- MPI_SUCCESS or error code
+ */
+int
+mca_coll_inter_scatter_inter(void *sbuf, int scount,
+                             struct ompi_datatype_t *sdtype,
+                             void *rbuf, int rcount,
+                             struct ompi_datatype_t *rdtype,
+                             int root, struct ompi_communicator_t *comm)
+{
+    int rank, size, size_local, err;
+    char *ptmp = NULL;
+    ptrdiff_t lb, incr;
+
+    /* Initialize */
+
+    rank = ompi_comm_rank(comm);
+    size = ompi_comm_remote_size(comm);
+
+    if (MPI_PROC_NULL == root) {
+        /* do nothing */
+        err = OMPI_SUCCESS;
+    } else if (MPI_ROOT != root) {
+        /* First process receives the data from root */
+	if(0 == rank) { 
+	    err = ompi_ddt_get_extent(sdtype, &lb, &incr);
+	    if (OMPI_SUCCESS != err) {
+		return OMPI_ERROR;
+	    }
+	    
+	    incr *= scount;
+	    size_local = ompi_comm_size(comm->c_local_comm);
+	    ptmp = (char*)malloc(size_local * incr);
+	    if (NULL == ptmp) {
+		return OMPI_ERR_OUT_OF_RESOURCE;
+	    }
+
+	    err = MCA_PML_CALL(recv(ptmp, rcount*size_local, rdtype, 
+				    root, MCA_COLL_BASE_TAG_SCATTER,
+				    comm, MPI_STATUS_IGNORE));
+	    if (OMPI_SUCCESS != err) {
+                return err;
+            }
+	}
+	/* Perform the scatter locally with the first process as root */
+	err = comm->c_local_comm->c_coll.coll_scatter(ptmp, rcount, rdtype, 
+						      rbuf, rcount, rdtype, 
+						      0, comm->c_local_comm);
+	if (NULL != ptmp) {
+	    free(ptmp);
+	}
+    } else {
+	/* Root sends data to the first process in the remote group */
+	err = MCA_PML_CALL(send(sbuf, scount*size, sdtype, 0,
+				MCA_COLL_BASE_TAG_SCATTER,
+				MCA_PML_BASE_SEND_STANDARD, comm));
+	if (OMPI_SUCCESS != err) {
+	    return err;
+	}
+    }
+
+    return err;
+}

ompi/mca/coll/inter/coll_inter_scatterv.c

@@ -0,0 +1,143 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2005 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 University of Houston. All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "coll_inter.h"
+
+#include "mpi.h"
+#include "ompi/constants.h"
+#include "ompi/datatype/datatype.h"
+#include "ompi/mca/coll/coll.h"
+#include "ompi/mca/coll/base/coll_tags.h"
+#include "ompi/mca/pml/pml.h"
+
+
+/*
+ *	scatterv_inter
+ *
+ *	Function:	- scatterv operation
+ *	Accepts:	- same arguments as MPI_Scatterv()
+ *	Returns:	- MPI_SUCCESS or error code
+ */
+int
+mca_coll_inter_scatterv_inter(void *sbuf, int *scounts,
+                              int *disps, struct ompi_datatype_t *sdtype,
+                              void *rbuf, int rcount,
+                              struct ompi_datatype_t *rdtype, int root,
+                              struct ompi_communicator_t *comm)
+{
+    int i, rank, size, err, total, size_local;
+    int *counts=NULL,*displace=NULL;
+    char *ptmp=NULL;
+    MPI_Aint incr;
+    MPI_Aint extent;
+    MPI_Aint lb;
+    ompi_datatype_t *ndtype;
+
+    /* Initialize */
+
+    rank = ompi_comm_rank(comm);
+    size = ompi_comm_remote_size(comm);
+    size_local = ompi_comm_size(comm);
+
+    if (MPI_PROC_NULL == root) {
+        /* do nothing */
+        err = OMPI_SUCCESS;
+    } else if (MPI_ROOT != root) {
+	if(0 == rank) {
+	    /* local root recieves the counts from the root */
+	    counts = (int *)malloc(sizeof(int) * size_local);
+	    err = MCA_PML_CALL(recv(counts, size_local, MPI_INT,
+				    root, MCA_COLL_BASE_TAG_SCATTERV,
+				    comm, MPI_STATUS_IGNORE));
+	    if (OMPI_SUCCESS != err) {
+		return err;
+	    }
+	    /* calculate the whole buffer size and recieve it from root */
+	    err = ompi_ddt_get_extent(rdtype, &lb, &extent);
+	    if (OMPI_SUCCESS != err) {
+		return OMPI_ERROR;
+	    }
+	    incr = 0;
+	    for (i = 0; i < size_local; i++) {
+		incr = incr + extent*counts[i];
+	    }
+	    ptmp = (char*)malloc(incr); 
+	    if (NULL == ptmp) {
+		return OMPI_ERR_OUT_OF_RESOURCE;
+	    }
+	    total = 0;
+	    for (i = 0; i < size_local; i++) {
+		total = total + counts[i];
+	    }
+	    err = MCA_PML_CALL(recv(ptmp, total, rdtype,
+				    root, MCA_COLL_BASE_TAG_SCATTERV,
+				    comm, MPI_STATUS_IGNORE));
+	    if (OMPI_SUCCESS != err) {
+		return err;
+	    }
+	    /* set the local displacement i.e. no displacements here */
+	    displace = (int *)malloc(sizeof(int) * size_local);
+	    displace[0] = 0;
+	    for (i = 1; i < size_local; i++) {
+		displace[i] = displace[i-1] + counts[i-1];
+	    }
+	}
+	/* perform the scatterv locally */
+	err = comm->c_local_comm->c_coll.coll_scatterv(ptmp, counts, displace, 
+						       rdtype, rbuf, rcount, 
+						       rdtype, 0, comm->c_local_comm);
+	if (OMPI_SUCCESS != err) {
+	    return err;
+	}
+
+	if (NULL != ptmp) {
+	    free(ptmp);
+	}
+	if (NULL != displace) {
+	    free(displace);
+	}
+	if (NULL != counts) {
+	    free(counts);
+	}
+
+    } else {
+	err = MCA_PML_CALL(send(scounts, size, MPI_INT, 0,
+				MCA_COLL_BASE_TAG_SCATTERV,
+				MCA_PML_BASE_SEND_STANDARD, comm));
+	if (OMPI_SUCCESS != err) {
+	    return err;
+	}
+
+	ompi_ddt_create_indexed(size,scounts,disps,sdtype,&ndtype);
+	ompi_ddt_commit(&ndtype);
+	
+	err = MCA_PML_CALL(send(sbuf, 1, ndtype, 0,
+				MCA_COLL_BASE_TAG_SCATTERV,
+				MCA_PML_BASE_SEND_STANDARD, comm));
+	if (OMPI_SUCCESS != err) {
+	    return err;
+	}
+	ompi_ddt_destroy(&ndtype);
+
+    }
+
+    /* All done */
+    return err;
+}

ompi/mca/coll/inter/configure.params

@@ -0,0 +1,24 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# Copyright (c) 2007      Los Alamos National Security, LLC.  All rights
+#                         reserved. 
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+# Specific to this module
+
+PARAM_CONFIG_FILES="Makefile"

ompi/mca/osc/pt2pt/osc_pt2pt_component.c

@@ -245,7 +245,7 @@ ompi_osc_pt2pt_component_select(ompi_win_t *win,
 
     module->p2p_win = win;
 
-    ret = ompi_comm_dup(comm, &(module->p2p_comm));
+    ret = ompi_comm_dup(comm, &(module->p2p_comm), 0);
     if (ret != OMPI_SUCCESS) {
         OBJ_DESTRUCT(&(module->p2p_acc_lock));
         OBJ_DESTRUCT(&(module->p2p_lock));

ompi/mca/osc/rdma/osc_rdma_component.c

@@ -258,7 +258,7 @@ ompi_osc_rdma_component_select(ompi_win_t *win,
 
     module->p2p_win = win;
 
-    ret = ompi_comm_dup(comm, &(module->p2p_comm));
+    ret = ompi_comm_dup(comm, &(module->p2p_comm), 0);
     if (ret != OMPI_SUCCESS) {
         OBJ_DESTRUCT(&(module->p2p_acc_lock));
         OBJ_DESTRUCT(&(module->p2p_lock));

ompi/mpi/c/comm_dup.c

@@ -51,6 +51,6 @@ int MPI_Comm_dup(MPI_Comm comm, MPI_Comm *newcomm)
     }
 
     
-    rc = ompi_comm_dup ( comm, newcomm);
+    rc = ompi_comm_dup ( comm, newcomm, 0);
     OMPI_ERRHANDLER_RETURN ( rc, comm, rc, FUNC_NAME);
 }

ompi/mpi/c/intercomm_create.c

@@ -174,6 +174,7 @@ int MPI_Intercomm_create(MPI_Comm local_comm, int local_leader,
                               &rleader,                    /* remote_leader */
                               OMPI_COMM_CID_INTRA_BRIDGE,  /* mode */
                               -1,                          /* send_first */
+			      0,                           /* sync_flag */
                               NULL );                      /* coll component */
 
     if ( MPI_SUCCESS != rc ) {

ompi/mpi/c/intercomm_merge.c

@@ -133,6 +133,7 @@ int MPI_Intercomm_merge(MPI_Comm intercomm, int high,
                               NULL,                 /* remote_leader */
                               OMPI_COMM_CID_INTER,  /* mode */
                               -1,                   /* send_first */
+			      0,                    /* sync_flag */
                               NULL );               /* coll module */
     if ( OMPI_SUCCESS != rc ) {
         goto exit;