Changing a lot of stuff in topo. This commit is just the intergration

of what has been done in tmp/anju-topo-work branch. For more detailed 
information on the commits, please see those logs

This commit was SVN r1787.

src/communicator/comm.c

@@ -15,6 +15,8 @@
 #include "mca/pml/pml.h"
 #include "mca/coll/coll.h"
 #include "mca/coll/base/base.h"
+#include "mca/topo/topo.h"
+#include "mca/topo/base/base.h"
 #include "mca/ns/base/base.h"
 
 #include "mca/pml/pml.h"
@@ -29,6 +31,16 @@
 */
 static int rankkeycompare(const void *, const void *);
 
+/**
+ * to fill the rest of the stuff for the communicator when either
+ * MPI_Cart_create or MPI_Graph_create is used 
+ */ 
+static int ompi_comm_fill_rest (ompi_communicator_t *comm,
+                                int num_procs,
+                                ompi_proc_t **proc_pointers,
+                                int my_rank,
+                                ompi_errhandler_t *errh,
+                                mca_base_module_t *coll_module);
 /*
 ** typedef for the allgather_intra required in comm_split.
 ** the reason for introducing this abstraction is, that
@@ -53,6 +65,7 @@ static int ompi_comm_allgather_emulate_intra (void* inbuf, int incount, MPI_Data
  * This is the function setting all elements of a communicator.
  * All other routines are just used to determine these elements.
  */   
+
 ompi_communicator_t * ompi_comm_set ( ompi_communicator_t* oldcomm,
                                       int local_size, 
                                       ompi_proc_t **local_procs,
@@ -100,12 +113,21 @@ ompi_communicator_t * ompi_comm_set ( ompi_communicator_t* oldcomm,
     /* Set Topology, if required */
     
     if ( NULL != topomodule ) {
+        /*
+         * This functions is never used o determine the topology 
+         * module. The topology module is determined only by the 
+         * ompi_cart_create and ompi_comm_create functions. Have 
+         * to see what ahppens during MPI_Comm_dup though. During 
+         * this the topology information has to be copied into the 
+         * new communicator which includes selecting a new topology 
+         * module and setting the information which is on that 
+         * communicator into this communicator. This probably is 
+         * another function in this file. 
+         */ 
         if (OMPI_COMM_IS_CART ( oldcomm ) )
             newcomm->c_flags |= OMPI_COMM_CART;
         if (OMPI_COMM_IS_GRAPH ( oldcomm ) ) 
             newcomm->c_flags |= OMPI_COMM_GRAPH;
-        
-        /*set the topo-module */
     }
 
     /* Copy attributes and call according copy functions, 
@@ -135,7 +157,6 @@ ompi_communicator_t * ompi_comm_set ( ompi_communicator_t* oldcomm,
 }
 
 
-
 /**********************************************************************/
 /**********************************************************************/
 /**********************************************************************/
@@ -836,3 +857,275 @@ static int rankkeycompare (const void *p, const void *q)
     return ( 0 );
 }
 
+
+/*************************************************************************************
+ * Counterpart of MPI_Cart/Graph_create. This will be called from the top level MPI. The
+ * condition for INTER communicator is already checked by the time this has been 
+ * invoked. This function should do somewhat the same things which ompi_comm_create
+ * does. It will however select a module for topology and then call the cart_create
+ * on that module so that it can re-arrange the proc structure as required (if the 
+ * reorder flag is true). It will then use this proc structure to create the communicator
+ * using ompi_comm_set.
+ */
+int ompi_topo_create (ompi_communicator_t *old_comm, 
+                      int ndims_or_nnodes,
+                      int *dims_or_index,
+                      int *periods_or_edges,
+                      bool reorder,
+                      ompi_communicator_t **comm_topo,
+                      int cart_or_graph){
+
+    ompi_communicator_t *new_comm;
+    int new_rank;
+    ompi_proc_t **topo_procs;
+    int num_procs;
+    int ret;
+
+    /* 
+     * Allocate a comm structure. This structure is used later to pass down to 
+     * topo_base_comm_select so that the actions structure pertaining to the 
+     * selected topology module can be used to re-aarange the procs.
+     */
+    *comm_topo = MPI_COMM_NULL; 
+    new_comm = OBJ_NEW (ompi_communicator_t);
+
+    /* allocate the data for the common good */
+    new_comm->c_topo_comm = malloc(sizeof(mca_topo_comm_t));
+
+    if (NULL == new_comm->c_topo_comm) {
+        OBJ_RELEASE(new_comm);
+        return OMPI_ERROR;
+    }
+
+    /* select the topology module on the communicator */
+
+    if (OMPI_SUCCESS != (ret = mca_topo_base_comm_select (new_comm, NULL))) {
+        free(new_comm->c_topo_comm); 
+        OBJ_RELEASE(new_comm);
+        return ret;
+    }
+
+    /* since the topo module has initialised, let us now initialise the 
+     * topo comm structure */
+#define FREE_COMMUNICATOR(new_comm) \
+    if (NULL != new_comm->c_topo_comm->mtc_dims_or_index) { \
+        free(new_comm->c_topo_comm->mtc_dims_or_index); \
+    } \
+    if (NULL != new_comm->c_topo_comm->mtc_periods_or_edges) { \
+        free(new_comm->c_topo_comm->mtc_periods_or_edges); \
+    } \
+    if (NULL != new_comm->c_topo_comm->mtc_coords) { \
+        free(new_comm->c_topo_comm->mtc_coords); \
+    } \
+    free(new_comm->c_topo_comm);  \
+    OBJ_RELEASE (new_comm); 
+
+    new_comm->c_flags |= cart_or_graph;
+
+    new_comm->c_topo_comm->mtc_ndims_or_nnodes = ndims_or_nnodes;
+    
+    new_comm->c_topo_comm->mtc_dims_or_index = NULL;
+    new_comm->c_topo_comm->mtc_periods_or_edges = NULL;
+    new_comm->c_topo_comm->mtc_reorder = reorder;
+
+    new_comm->c_topo_comm->mtc_coords = NULL;
+
+    new_comm->c_topo_comm->mtc_dims_or_index = malloc (sizeof(int) * ndims_or_nnodes);
+    if (NULL == new_comm->c_topo_comm->mtc_dims_or_index) {
+        FREE_COMMUNICATOR(new_comm);
+        return OMPI_ERROR;
+    }
+    memcpy (new_comm->c_topo_comm->mtc_dims_or_index,
+            dims_or_index, ndims_or_nnodes * sizeof(int));
+    
+    /* Now the topology module has been selected, let the module re-arrange 
+     * the proc ranks if need be. This is a down-call into the topo
+     * module and does not have anything to do with this level */
+
+    /* first, copy the proc structure from the previous communicator over to the
+     * new one. the topology module can then work on this and rearrange it as 
+     * it deems fit.
+     */
+    num_procs = old_comm->c_local_group->grp_proc_count;
+    topo_procs = (ompi_proc_t **)malloc (num_procs * sizeof(ompi_proc_t *));
+    memcpy (topo_procs, 
+            old_comm->c_local_group->grp_proc_pointers,
+            num_procs * sizeof(ompi_proc_t *));
+    new_rank = old_comm->c_local_group->grp_my_rank;
+
+    if (OMPI_COMM_CART == cart_or_graph) {
+
+        /* A cartesian system has been requested. Call the right function */
+
+        /* Note that we fill in the basic information, i.e, copy the information
+         * which was provided to us over into the structure. The base module 
+         * functions are free to change it as they deem fit */
+
+        new_comm->c_topo_comm->mtc_periods_or_edges = malloc (sizeof(int) * ndims_or_nnodes);
+        if (NULL == new_comm->c_topo_comm->mtc_periods_or_edges) {
+            FREE_COMMUNICATOR(new_comm);
+            return OMPI_ERROR;
+        }
+        memcpy (new_comm->c_topo_comm->mtc_periods_or_edges,
+                periods_or_edges, ndims_or_nnodes * sizeof(int));
+
+        new_comm->c_topo_comm->mtc_coords = malloc (sizeof(int) * ndims_or_nnodes);
+        if (NULL == new_comm->c_topo_comm->mtc_coords) {
+            FREE_COMMUNICATOR(new_comm);
+            return OMPI_ERROR;
+        }
+
+        if (OMPI_SUCCESS != 
+            (ret = new_comm->c_topo->topo_cart_create (new_comm->c_topo_comm,
+                                                       &num_procs,
+                                                       topo_procs,
+                                                       &new_rank,
+                                                       ndims_or_nnodes,
+                                                       dims_or_index,
+                                                       periods_or_edges,
+                                                       reorder))) {
+            return ret;
+        }
+
+    } else if (OMPI_COMM_GRAPH == cart_or_graph) {
+
+        /* A graph system has been requested. Call the right function */
+
+        /* Note that we fill in the basic information, i.e, copy the information
+         * which was provided to us over into the structure. The base module 
+         * functions are free to change it as they deem fit */
+
+        new_comm->c_topo_comm->mtc_periods_or_edges = 
+                malloc (sizeof(int) * dims_or_index[ndims_or_nnodes-1]);
+        if (NULL == new_comm->c_topo_comm->mtc_periods_or_edges) {
+            FREE_COMMUNICATOR(new_comm);
+            return OMPI_ERROR;
+        }
+        memcpy (new_comm->c_topo_comm->mtc_periods_or_edges,
+                periods_or_edges, dims_or_index[ndims_or_nnodes-1] * sizeof(int));
+
+        if (OMPI_SUCCESS != 
+            (ret = new_comm->c_topo->topo_graph_create (new_comm->c_topo_comm,
+                                                        &num_procs,
+                                                        topo_procs,
+                                                        &new_rank,
+                                                        ndims_or_nnodes,
+                                                        dims_or_index,
+                                                        periods_or_edges,
+                                                        reorder))) {
+            return ret;
+        }
+
+    }
+
+    /* if the returned rank is -1, then this process is not in the 
+     * new topology, so free everything we have allocated and return */
+
+    if (MPI_UNDEFINED == new_rank) {
+        FREE_COMMUNICATOR(new_comm);
+        return OMPI_SUCCESS; 
+    }
+
+
+    /* Determine context id. It is identical to f_2_c_handle */
+
+    ret = ompi_comm_nextcid ( new_comm,  /* new communicator */
+                              old_comm,     /* old comm */
+                              NULL,     /* bridge comm */
+                              NULL,     /* local leader */
+                              NULL,     /* remote_leader */
+                              OMPI_COMM_CID_INTRA);   /* mode */
+
+    if (OMPI_SUCCESS != ret) {
+        /* something wrong happened during setting the communicator */
+        FREE_COMMUNICATOR(new_comm);
+        return ret;
+    }
+
+    /* Now, the topology module has been selected and the group which has 
+     * the topology information has been created. All we need to do now is 
+     * to fill the rest of the information into the communicator. The following
+     * steps are not just similar to ompi_comm_set, but are actually the same */
+
+    ret = ompi_comm_fill_rest(new_comm,                /* the communicator */
+                              num_procs,               /* local size */
+                              topo_procs,              /* process structure */
+                              new_rank,                /* rank of the process */
+                              old_comm->error_handler, /* error handler */
+                              NULL);                   /*coll module */
+
+    if (OMPI_SUCCESS != ret) {
+        /* something wrong happened during setting the communicator */
+        FREE_COMMUNICATOR(new_comm);
+        return ret;
+    }
+    
+#undef FREE_COMMUNICATOR
+    /* finally, set the communicator to comm_cart */
+
+    *comm_topo = new_comm;
+
+    return OMPI_SUCCESS;
+}
+
+static int ompi_comm_fill_rest (ompi_communicator_t *comm,
+                                int num_procs,
+                                ompi_proc_t **proc_pointers,
+                                int my_rank,
+                                ompi_errhandler_t *errh,
+                                mca_base_module_t *coll_module) {
+
+    int ret;
+
+
+    /* allocate a group structure for the new communicator */
+    comm->c_local_group = ompi_group_allocate(num_procs);
+
+    /* free the malloced  proc pointers */
+    free(comm->c_local_group->grp_proc_pointers);
+    
+    /* set the group information */
+    comm->c_local_group->grp_proc_pointers = proc_pointers;
+
+    /* set the remote group to be the same as local group */
+    comm->c_remote_group = comm->c_local_group;
+
+    /* retain these proc pointers */
+    ompi_group_increment_proc_count(comm->c_local_group);
+        
+    /* set the rank information */
+    ompi_set_group_rank(comm->c_local_group, 
+            comm->c_local_group->grp_proc_pointers[my_rank]);
+    comm->c_my_rank = comm->c_local_group->grp_my_rank;
+
+    /* set the error handler */
+    comm->error_handler = errh;
+    OBJ_RETAIN (comm->error_handler);
+
+    /* set name for debugging purposes */
+    /* there is no cid at this stage ... make this right and make edgars
+     * code call this function and remove dupli cde
+     */  
+    snprintf (comm->c_name, MPI_MAX_OBJECT_NAME, "MPI_COMMUNICATOR %d",
+              comm->c_contextid);
+
+    /* determine the cube dimensions */
+    comm->c_cube_dim = ompi_cube_dim(comm->c_local_group->grp_proc_count);
+
+    /* copy attributes and call according copy functions */
+    ompi_attr_hash_init(&comm->c_keyhash);
+
+    /* initialize PML stuff on the communicator */
+    if (OMPI_SUCCESS != (ret = mca_pml.pml_add_comm(comm))) {
+        /* some error has happened */
+        return ret;
+    }
+
+    /* initialize the coll module */
+    if (OMPI_SUCCESS != (ret = mca_coll_base_comm_select (comm, NULL))) {
+        /* some error has happened */
+        return ret;
+    }
+
+    return OMPI_SUCCESS;
+}

src/communicator/comm_init.c

@@ -14,6 +14,8 @@
 #include "mca/pml/pml.h"
 #include "mca/coll/coll.h"
 #include "mca/coll/base/base.h"
+#include "mca/topo/topo.h"
+#include "mca/topo/base/base.h"
 #include "mca/ns/base/base.h"
 
 
@@ -202,6 +204,7 @@ static void ompi_comm_construct(ompi_communicator_t* comm)
     comm->error_handler  = NULL;
     comm->c_pml_comm     = NULL;
     comm->c_topo_comm    = NULL; 
+    comm->c_topo_module = NULL;
 
     comm->c_coll_selected_module = NULL;
     comm->c_coll_selected_data   = NULL;
@@ -219,12 +222,44 @@ static void ompi_comm_destruct(ompi_communicator_t* comm)
 
     /* Release topology information */
 
-    /* ...Anju add more here... */
+    mca_topo_base_comm_unselect(comm);
 
+    /*  Check if the communicator is a topology */
+    
+    if (OMPI_COMM_IS_CART(comm) || OMPI_COMM_IS_GRAPH(comm)) {
+
+        /* check and free individual things */
+        
+        if (NULL != comm->c_topo_comm) {
+
+            /* check for all pointers and free them */
+
+            if (NULL != comm->c_topo_comm->mtc_dims_or_index) {
+                free(comm->c_topo_comm->mtc_dims_or_index);
+            }
+        
+            if (NULL != comm->c_topo_comm->mtc_dims_or_index) {
+                free(comm->c_topo_comm->mtc_periods_or_edges);
+            }
+
+            if (NULL != comm->c_topo_comm->mtc_dims_or_index) {
+                free(comm->c_topo_comm->mtc_coords);
+            }
+
+            free(comm->c_topo_comm);
+        }
+
+    }
+
+    if (NULL != comm->c_local_group) {
         OBJ_RELEASE ( comm->c_local_group );
+    }
+    if (NULL != comm->c_remote_group) {
         OBJ_RELEASE ( comm->c_remote_group );
-    
+    }
+    if (NULL != comm->c_remote_group) {
         OBJ_RELEASE ( comm->error_handler );
+    }
 
     /* reset the ompi_comm_f_to_c_table entry */
     if ( NULL != ompi_pointer_array_get_item ( &ompi_mpi_communicators,
@@ -236,7 +271,3 @@ static void ompi_comm_destruct(ompi_communicator_t* comm)
 
     return;
 }
-
-
-
-

src/communicator/communicator.h

@@ -66,10 +66,13 @@ struct ompi_communicator_t {
 
     /* Hooks for topo module to hang things */
 
-    mca_topo_1_0_0_t c_topo; /**< structure of function pointers */
-    mca_topo_comm_t *c_topo_comm; /**<structure containing information
+    const mca_topo_1_0_0_t *c_topo; /**< structure of function pointers */
+
+    mca_topo_comm_t *c_topo_comm; /**<structure containing basic information
                                     *about the topology */
 
+    struct mca_topo_module_comm_t *c_topo_module; /**< component specific data */
+
     /* index in Fortran <-> C translation array */
 
     int c_f_to_c_index;
@@ -192,6 +195,18 @@ extern "C" {
     int ompi_comm_create ( ompi_communicator_t* comm, ompi_group_t *group, 
                           ompi_communicator_t** newcomm );
 
+
+    /**
+     * create a cartesian communicator
+     */
+    int ompi_topo_create (ompi_communicator_t *old_comm, 
+                          int ndims_or_nnodes,
+                          int *dims_or_index,
+                          int *periods_or_edges,
+                          bool reorder,
+                          ompi_communicator_t **comm_cart,
+                          int cart_or_graph);
+    
     /**
      * split a communicator based on color and key. Parameters
      * are identical to the MPI-counterpart of the function.

src/mca/Makefile.am

@@ -40,6 +40,7 @@ libmca_la_LIBADD = \
         oob/libmca_oob.la \
         pcm/libmca_pcm.la \
         pml/libmca_pml.la \
+        topo/libmca_topo.la \
         ptl/libmca_ptl.la
 libmca_la_DEPENDENCIES = $(libmca_la_LIBADD)
 

src/mca/base/mca_base_init_select_modules.c

@@ -11,6 +11,8 @@
 #include "mca/base/base.h"
 #include "mca/coll/coll.h"
 #include "mca/coll/base/base.h"
+#include "mca/topo/topo.h"
+#include "mca/topo/base/base.h"
 #include "mca/ptl/ptl.h"
 #include "mca/ptl/base/base.h"
 #include "mca/pml/pml.h"
@@ -65,6 +67,14 @@ int mca_base_init_select_modules(int requested,
   allow_multi_user_threads &= user_threads;
   have_hidden_threads |= hidden_threads;
 
+
+  if (OMPI_SUCCESS != mca_topo_base_find_available(&user_threads, 
+                                                   &hidden_threads)) {
+    return OMPI_ERROR;
+  }
+  allow_multi_user_threads &= user_threads;
+  have_hidden_threads |= hidden_threads;
+
   /* Now that we have a final list of all available modules, do the
      selection.  pml is already selected. */
 

src/mpi/c/cart_coords.c

@@ -48,7 +48,7 @@ int MPI_Cart_coords(MPI_Comm comm, int rank, int maxdims, int *coords) {
     }
 
     /* get the function pointer on this communicator */
-    func = comm->c_topo.topo_cart_coords;
+    func = comm->c_topo->topo_cart_coords;
     if (NULL == func) {
         return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_OTHER, 
                                      "MPI_Cart_coords");

src/mpi/c/cart_create.c

@@ -22,7 +22,7 @@ int MPI_Cart_create(MPI_Comm old_comm, int ndims, int *dims,
                     int *periods, int reorder, MPI_Comm *comm_cart) {
 
     int err;
-    mca_topo_base_cart_create_fn_t func;
+    bool re_order = false;
 
     /* check the arguments */
     if (MPI_PARAM_CHECK) {
@@ -42,17 +42,47 @@ int MPI_Cart_create(MPI_Comm old_comm, int ndims, int *dims,
             return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_ARG,
                                           "MPI_Cart_create");
         }
-    }
-    /* get the function pointer to do the right thing */
-    func = old_comm->c_topo.topo_cart_create;
-    if (NULL == func) {
-        return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_OTHER, 
+        if (0 > reorder || 1 < reorder) {
+            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_ARG,
                                           "MPI_Cart_create");
         }
 
-    /* call the function */
-    if ( MPI_SUCCESS != 
-            (err = func(old_comm, ndims, dims, periods, reorder, comm_cart))) {
+        /* check if the number of processes on the grid are corrct */
+        {
+           int i;
+           int *p = dims;
+           int count_nodes = 1;
+           int parent_procs = ompi_comm_size(old_comm);
+
+           for (i=0; i < ndims; i++) {
+               count_nodes *= *p;
+               p++;
+           }
+
+           if (parent_procs < count_nodes) {
+               return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_ARG,
+                                              "MPI_Cart_create");
+           }
+        }
+    }
+
+    /* everything seems to be alright with the communicator, we can go 
+     * ahead and select a topology module for this purpose and create 
+     * the new cartesian communicator
+     */
+
+    re_order = (1 == reorder)? true :false;
+
+    err = ompi_topo_create (old_comm,
+                            ndims,
+                            dims,
+                            periods,
+                            re_order,
+                            comm_cart,
+                            OMPI_COMM_CART);
+
+    /* check the error status */
+    if (MPI_SUCCESS != err) {
         return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, err, "MPI_Cart_create");
     }
     

src/mpi/c/cart_get.c

@@ -41,7 +41,7 @@ int MPI_Cart_get(MPI_Comm comm, int maxdims, int *dims,
         }
     }
     /* get the function pointer to do the right thing */
-    func = comm->c_topo.topo_cart_get;
+    func = comm->c_topo->topo_cart_get;
     if (NULL == func) {
         return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_OTHER, 
                                      "MPI_Cart_get");

src/mpi/c/cart_map.c

@@ -44,7 +44,7 @@ int MPI_Cart_map(MPI_Comm comm, int ndims, int *dims,
     }
 
     /* get the function pointer on this communicator */
-    func = comm->c_topo.topo_cart_map;
+    func = comm->c_topo->topo_cart_map;
     if (NULL == func) {
         return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_OTHER, 
                                      "MPI_Cart_map");

src/mpi/c/cart_rank.c

@@ -44,7 +44,7 @@ int MPI_Cart_rank(MPI_Comm comm, int *coords, int *rank) {
     }
 
     /* get the function pointer on this communicator */
-    func = comm->c_topo.topo_cart_rank;
+    func = comm->c_topo->topo_cart_rank;
     if (NULL == func) {
         return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_OTHER, 
                                      "MPI_Cart_rank");

src/mpi/c/cart_shift.c

@@ -48,7 +48,7 @@ int MPI_Cart_shift(MPI_Comm comm, int direction, int disp,
     }
 
     /* get the function pointer on this communicator */
-    func = comm->c_topo.topo_cart_shift;
+    func = comm->c_topo->topo_cart_shift;
     if (NULL == func) {
         return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_OTHER, 
                                      "MPI_Cart_shift");

src/mpi/c/cart_sub.c

@@ -44,7 +44,7 @@ int MPI_Cart_sub(MPI_Comm comm, int *remain_dims, MPI_Comm *new_comm) {
     }
 
     /* get the function pointer on this communicator */
-    func = comm->c_topo.topo_cart_sub;
+    func = comm->c_topo->topo_cart_sub;
     if (NULL == func) {
         return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_OTHER, 
                                      "MPI_Cart_sub");

src/mpi/c/graph_create.c

@@ -18,19 +18,19 @@
 #include "mpi/c/profile/defines.h"
 #endif
 
-int MPI_Graph_create(MPI_Comm comm_old, int nnodes, int *index,
+int MPI_Graph_create(MPI_Comm old_comm, int nnodes, int *index,
                      int *edges, int reorder, MPI_Comm *comm_graph) {
 
     int err;
-    mca_topo_base_graph_create_fn_t func;
+    bool re_order = false;
 
     /* check the arguments */
     if (MPI_PARAM_CHECK) {
-        if (MPI_COMM_NULL == comm_old) {
+        if (MPI_COMM_NULL == old_comm) {
             return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_COMM,
                                           "MPI_Graph_create");
         }
-        if (OMPI_COMM_IS_INTER(comm_old)) {
+        if (OMPI_COMM_IS_INTER(old_comm)) {
             return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_COMM,
                                           "MPI_Graph_create");
         }
@@ -38,17 +38,36 @@ int MPI_Graph_create(MPI_Comm comm_old, int nnodes, int *index,
             return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_ARG,
                                           "MPI_Graph_create");
         }
-    }
-    /* get the function pointer to do the right thing */
-    func = comm_old->c_topo.topo_graph_create;
-    if (NULL == func) {
-        return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_OTHER, 
+
+        if (nnodes > ompi_comm_size(old_comm)) {
+            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_ARG,
                                           "MPI_Graph_create");
         }
 
-    /* call the function */
-    if ( MPI_SUCCESS != 
-            (err = func(comm_old, nnodes, index, edges, reorder, comm_graph))) {
+        if (0 > reorder || 1 < reorder) {
+            return OMPI_ERRHANDLER_INVOKE (MPI_COMM_WORLD, MPI_ERR_ARG,
+                                          "MPI_Graph_create: boo");
+        }
+    }
+
+    /* 
+     * everything seems to be alright with the communicator, we can go 
+     * ahead and select a topology module for this purpose and create 
+     * the new graph communicator
+     */
+
+    re_order = (1 == reorder) ? true:false;
+
+    err = ompi_topo_create ((struct ompi_communicator_t *)old_comm,
+                            nnodes,
+                            index,
+                            edges,
+                            re_order,
+                            (struct ompi_communicator_t **)comm_graph,
+                            OMPI_COMM_GRAPH);
+
+    /* check the error status */
+    if (MPI_SUCCESS != err) {
         return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, err, "MPI_Graph_create");
     }
     

src/mpi/c/graph_get.c

@@ -43,7 +43,7 @@ int MPI_Graph_get(MPI_Comm comm, int maxindex, int maxedges,
         }
     }
     /* get the function pointer to do the right thing */
-    func = comm->c_topo.topo_graph_get;
+    func = comm->c_topo->topo_graph_get;
     if (NULL == func) {
         return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_OTHER, 
                                      "MPI_Graph_get");

src/mpi/c/graph_map.c

@@ -39,7 +39,7 @@ int MPI_Graph_map(MPI_Comm comm, int nnodes, int *index, int *edges,
         }
     }
     /* map the function pointer to do the right thing */
-    func = comm->c_topo.topo_graph_map;
+    func = comm->c_topo->topo_graph_map;
     if (NULL == func) {
         return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_OTHER, 
                                      "MPI_Graph_map");

src/mpi/c/graph_neighbors.c

@@ -48,7 +48,7 @@ int MPI_Graph_neighbors(MPI_Comm comm, int rank, int maxneighbors,
         }
     }
     /* neighbors the function pointer to do the right thing */
-    func = comm->c_topo.topo_graph_neighbors;
+    func = comm->c_topo->topo_graph_neighbors;
     if (NULL == func) {
         return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_OTHER, 
                                      "MPI_Graph_neighbors");

src/mpi/c/graph_neighbors_count.c

@@ -46,7 +46,7 @@ int MPI_Graph_neighbors_count(MPI_Comm comm, int rank, int *nneighbors) {
         }
     }
     /* get the function pointer to do the right thing */
-    func = comm->c_topo.topo_graph_neighbors_count;
+    func = comm->c_topo->topo_graph_neighbors_count;
     if (NULL == func) {
         return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_OTHER, 
                                      "MPI_Graph_neighbors_count");

src/mpi/c/graphdims_get.c

@@ -42,7 +42,7 @@ int MPI_Graphdims_get(MPI_Comm comm, int *nnodes, int *nedges) {
         }
     }
     /* get the function pointer to do the right thing */
-    func = comm->c_topo.topo_graphdims_get;
+    func = comm->c_topo->topo_graphdims_get;
     if (NULL == func) {
         return OMPI_ERRHANDLER_INVOKE(MPI_COMM_WORLD, MPI_ERR_OTHER, 
                                      "MPI_Graphdims_get");

src/runtime/ompi_mpi_init.c

@@ -27,6 +27,8 @@
 #include "mca/pml/base/base.h"
 #include "mca/coll/coll.h"
 #include "mca/coll/base/base.h"
+#include "mca/topo/topo.h"
+#include "mca/topo/base/base.h"
 
 
 /*
@@ -129,6 +131,10 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided)
         printf("show_help: ompi_mpi_init failed in mca_coll_base_init\n");
         return ret;
     }
+    if (OMPI_SUCCESS != (ret = mca_topo_base_open())) {
+        /* JMS show_help */
+        return ret;
+    }
 
     /* Select which pml, ptl, and coll modules to use, and determine the
          final thread level */