diff --git a/src/communicator/communicator.h b/src/communicator/communicator.h index 17e5f23ced..f377a82075 100644 --- a/src/communicator/communicator.h +++ b/src/communicator/communicator.h @@ -12,6 +12,7 @@ #include "mpi.h" #include "group/group.h" #include "mca/coll/coll.h" +#include "mca/topo/topo.h" #include "lfc/lam_hash_table.h" #include "attribute/attribute.h" @@ -43,16 +44,10 @@ struct lam_communicator_t { /* Attributes */ lam_hash_table_t *c_keyhash; - /* Topology information */ - int c_cube_dim; /**< Inscribing cube dimension */ - int c_topo_type; /**< Topology type */ - int c_topo_nprocs; /**< Number of processes */ - int c_topo_ndims; /**< Number of cart dimensions */ - int c_topo_nedges; /**< Graph edges */ - int *c_topo_dims; /**< Cart dimensions */ - int *c_topo_coords; /**< Cart coordinates */ - int *c_topo_index; /**< Graph indices */ - int *c_topo_edges; /**< Graph edges */ + /* Hooks for topo module to hang things */ + mca_topo_1_0_0_t c_topo; /**< structure of function pointers */ + mca_topo_comm_t *c_topo_comm; /** C translation array */ diff --git a/src/mca/Makefile.am b/src/mca/Makefile.am index c952e55e37..7d84877d15 100644 --- a/src/mca/Makefile.am +++ b/src/mca/Makefile.am @@ -14,6 +14,7 @@ SUBDIRS = \ pcm \ pml \ ptl \ + topo \ registry # Source code files diff --git a/src/mca/topo/base/base.h b/src/mca/topo/base/base.h index 6c9020a63e..3ec777c953 100644 --- a/src/mca/topo/base/base.h +++ b/src/mca/topo/base/base.h @@ -6,8 +6,9 @@ #define MCA_TOPO_BASE_H #include "lam_config.h" -#include "mpi.h" +#include "mpi.h" +#include "lfc/lam_list.h" #include "mca/topo/topo.h" /* @@ -18,13 +19,13 @@ extern "C" { #endif int mca_topo_base_open(void); int mca_topo_base_close(void); - int mca_topo_base_select(lam_list_t *selected, + int mca_topo_base_select(mca_topo_t *selected, bool *allow_multi_user_threads, bool *have_hidden_threads); - int mca_topo_base_init_comm (lam_communicator_t *comm); - int mca_topo_base_get_param (lam_communicator_t *comm, int keyval); + int mca_topo_base_init_comm (MPI_Comm comm); + int mca_topo_base_get_param (MPI_Comm comm, int keyval); const mca_topo_1_0_0_t * mca_topo_unity_query(int *priority, @@ -36,64 +37,64 @@ extern "C" { * for graph_map() and cart_map() for their topology modules. * But they can implement these glue functions if they want. */ - int topo_base_cart_coords (lam_communicator_t *comm, + int topo_base_cart_coords (MPI_Comm comm, int rank, int maxdims, int *coords); - int topo_base_cart_create (lam_communicator_t *old_comm, + int topo_base_cart_create (MPI_Comm old_comm, int ndims, int *dims, int *periods, int reorder, - lam_communicator_t *comm_cart); + MPI_Comm *comm_cart); - int topo_base_cartdim_get (lam_communicator_t *comm, + int topo_base_cartdim_get (MPI_Comm comm, int *ndims); - int topo_base_cart_get (lam_communicator_t *comm, + int topo_base_cart_get (MPI_Comm comm, int maxdims, int *dims, int *periods, int *coords); - int topo_base_cart_rank (lam_communicator_t *comm, + int topo_base_cart_rank (MPI_Comm comm, int *coords, int *rank); - int topo_base_cart_shift (lam_communicator_t *comm, + int topo_base_cart_shift (MPI_Comm comm, int direction, int disp, int *rank_source, int *rank_dest); - int topo_base_cart_sub (lam_communicator_t *comm, + int topo_base_cart_sub (MPI_Comm comm, int *remain_dims, - lam_communicator_t **new_comm); + MPI_Comm *new_comm); - int topo_base_graph_create (lam_communicator_t *comm_old, + int topo_base_graph_create (MPI_Comm comm_old, int nnodes, int *index, int *edges, int reorder, - lam_communicator_t **comm_graph); + MPI_Comm *comm_graph); - int topo_base_graph_dims_get (lam_communicator_t *comm, + int topo_base_graph_dims_get (MPI_Comm comm, int *nodes, int *nedges); - int topo_base_graph_get (lam_communicator_t *comm, + int topo_base_graph_get (MPI_Comm comm, int maxindex, int maxedges, int *index, int *edges); - int topo_base_graph_neighbors (lam_communicator_t *comm, + int topo_base_graph_neighbors (MPI_Comm comm, int rank, int maxneighbors, int *neighbors); - int topo_base_graph_neighbors_count (lam_communicator_t *comm, + int topo_base_graph_neighbors_count (MPI_Comm comm, int rank, int *nneighbors); diff --git a/src/mca/topo/base/topo_base_cart_coords.c b/src/mca/topo/base/topo_base_cart_coords.c index 71f39e4e7e..acbf9169e5 100644 --- a/src/mca/topo/base/topo_base_cart_coords.c +++ b/src/mca/topo/base/topo_base_cart_coords.c @@ -3,6 +3,8 @@ */ #include "mca/topo/base/base.h" +#include "communicator/communicator.h" +#include "mca/topo/topo.h" /* * function - determines process coords in cartesian topology given @@ -22,7 +24,7 @@ * @retval MPI_ERR_ARG */ -int topo_base_cart_coords (lam_communicator_t *comm, +int topo_base_cart_coords (MPI_Comm comm, int rank, int maxdims, int *coords){ @@ -34,9 +36,9 @@ int topo_base_cart_coords (lam_communicator_t *comm, /* * loop computing the co-ordinates */ - d = comm->c_topo_dims; - remprocs = comm->c_topo_nprocs; - for (i = 0; (i < comm->c_topo_ndims) && (i < maxdims); ++i, ++d) { + d = comm->c_topo_comm->mtc_dims; + remprocs = comm->c_topo_comm->mtc_nprocs; + for (i = 0; (i < comm->c_topo_comm->mtc_ndims) && (i < maxdims); ++i, ++d) { dim = (*d > 0) ? *d : -(*d); remprocs /= dim; *coords++ = rank / remprocs; diff --git a/src/mca/topo/base/topo_base_cart_create.c b/src/mca/topo/base/topo_base_cart_create.c index 062f2c6445..3b624f5180 100644 --- a/src/mca/topo/base/topo_base_cart_create.c +++ b/src/mca/topo/base/topo_base_cart_create.c @@ -3,6 +3,8 @@ */ #include "mca/topo/base/base.h" +#include "communicator/communicator.h" +#include "mca/topo/topo.h" /* * function - makes a new communicator to which topology information @@ -22,17 +24,19 @@ * @retval MPI_SUCCESS */ -int topo_base_cart_create (lam_communicator_t *old_comm, +int topo_base_cart_create (MPI_Comm old_comm, int ndims, int *dims, int *periods, int reorder, - lam_communicator_t *comm_cart){ - lam_group_t *newcomm; - lam_group_t *newgroup; + MPI_Comm *comm_cart){ + MPI_Comm newcomm; +#if 0 + MPI_Group newgroup; + int rank; +#endif int size; int nprocs; - int rank; int err; int range[1][3]; int i; @@ -51,7 +55,9 @@ int topo_base_cart_create (lam_communicator_t *old_comm, /* * Create the group for the new communicator. */ +#if 0 err = lam_comm_size (comm, &size); +#endif if (err != MPI_SUCCESS) { return err; } @@ -61,12 +67,16 @@ int topo_base_cart_create (lam_communicator_t *old_comm, } if (nprocs == size) { +#if 0 err = lam_comm_group (comm, &newgroup); +#endif } else { range[0][0] = 0; range[0][1] = nprocs - 1; range[0][2] = 1; +#if 0 err = lam_group_range_incl (comm->c_group, 1, range, &newgroup); +#endif } if (err != MPI_SUCCESS) { @@ -75,9 +85,13 @@ int topo_base_cart_create (lam_communicator_t *old_comm, /* * Create the new communicator. */ +#if 0 err = lam_comm_create (comm, newgroup, comm_cart); +#endif if (err != MPI_SUCCESS) { +#if 0 lam_group_free (&newgroup); +#endif return err; } /* @@ -85,16 +99,16 @@ int topo_base_cart_create (lam_communicator_t *old_comm, */ newcomm = *comm_cart; if (newcomm != MPI_COMM_NULL) { - newcomm->c_topo_type = MPI_CART; - newcomm->c_topo_nprocs = nprocs; - newcomm->c_topo_ndims = ndims; - newcomm->c_topo_dims = (int *) + newcomm->c_topo_comm->mtc_type = MPI_CART; + newcomm->c_topo_comm->mtc_nprocs = nprocs; + newcomm->c_topo_comm->mtc_ndims = ndims; + newcomm->c_topo_comm->mtc_dims = (int *) malloc((unsigned) 2 * ndims * sizeof(int)); - if (newcomm->c_topo_dims == 0) { + if (newcomm->c_topo_comm->mtc_dims == 0) { return MPI_ERR_OTHER; } - newcomm->c_topo_coords = newcomm->c_topo_dims + ndims; - for (i = 0, p = newcomm->c_topo_dims; i < ndims; ++i, ++p) { + newcomm->c_topo_comm->mtc_coords = newcomm->c_topo_comm->mtc_dims + ndims; + for (i = 0, p = newcomm->c_topo_comm->mtc_dims; i < ndims; ++i, ++p) { *p = (*periods) ? -(*dims) : *dims; ++dims; ++periods; @@ -102,19 +116,25 @@ int topo_base_cart_create (lam_communicator_t *old_comm, /* * Compute the caller's coordinates. */ +#if 0 err = lam_comm_rank (newcomm, &rank); +#endif if (err != MPI_SUCCESS) { return err; } - err = lam_cart_coors (newcomm, rank, - ndims, newcomm->c_topo_coords); +#if 0 + err = lam_cart_coords (newcomm, rank, + ndims, newcomm->c_topo_comm->mtc_coords); +#endif if (err != MPI_SUCCESS) { return err; } } +#if 0 err = lam_group_free (&newgroup); +#endif if (err != MPI_SUCCESS) { return err; } diff --git a/src/mca/topo/base/topo_base_cart_get.c b/src/mca/topo/base/topo_base_cart_get.c index 49b34a1af1..501707c7d6 100644 --- a/src/mca/topo/base/topo_base_cart_get.c +++ b/src/mca/topo/base/topo_base_cart_get.c @@ -3,6 +3,8 @@ */ #include "mca/topo/base/base.h" +#include "communicator/communicator.h" +#include "mca/topo/topo.h" /* * function - retrieves Cartesian topology information associated with a @@ -19,7 +21,7 @@ * * @retval MPI_SUCCESS */ -int topo_base_cart_get (lam_communicator_t *comm, +int topo_base_cart_get (MPI_Comm comm, int maxdims, int *dims, int *periods, @@ -28,9 +30,9 @@ int topo_base_cart_get (lam_communicator_t *comm, int *d; int *c; - d = comm->c_topo_dims; - c = comm->c_topo_coords; - for (i = 0; (i < comm->c_topo_ndims) && (i < maxdims); ++i) { + d = comm->c_topo_comm->mtc_dims; + c = comm->c_topo_comm->mtc_coords; + for (i = 0; (i < comm->c_topo_comm->mtc_ndims) && (i < maxdims); ++i) { if (*d > 0) { *dims++ = *d++; *periods++ = 0; diff --git a/src/mca/topo/base/topo_base_cart_rank.c b/src/mca/topo/base/topo_base_cart_rank.c index 3c970a174c..192d18927f 100644 --- a/src/mca/topo/base/topo_base_cart_rank.c +++ b/src/mca/topo/base/topo_base_cart_rank.c @@ -3,6 +3,8 @@ */ #include "mca/topo/base/base.h" +#include "communicator/communicator.h" +#include "mca/topo/topo.h" /* * function - Determines process rank in communicator given Cartesian @@ -19,7 +21,7 @@ * @retval MPI_ERR_ARG */ -int topo_base_cart_rank (lam_communicator_t *comm, +int topo_base_cart_rank (MPI_Comm comm, int *coords, int *rank){ int prank; @@ -35,8 +37,8 @@ int topo_base_cart_rank (lam_communicator_t *comm, */ factor = 1; prank = 0; - i = comm->c_topo_ndims - 1; - d = comm->c_topo_dims + i; + i = comm->c_topo_comm->mtc_ndims - 1; + d = comm->c_topo_comm->mtc_dims + i; c = coords + i; for (; i >= 0; --i, --c, --d) { diff --git a/src/mca/topo/base/topo_base_cart_shift.c b/src/mca/topo/base/topo_base_cart_shift.c index c865f869d6..69f8bf2db0 100644 --- a/src/mca/topo/base/topo_base_cart_shift.c +++ b/src/mca/topo/base/topo_base_cart_shift.c @@ -3,6 +3,8 @@ */ #include "mca/topo/base/base.h" +#include "communicator/communicator.h" +#include "mca/topo/topo.h" /* * function - Returns the shifted source and destination ranks, given a @@ -23,7 +25,7 @@ * @retval MPI_ERR_COMM * @retval MPI_ERR_ARG */ -int topo_base_cart_shift (lam_communicator_t *comm, +int topo_base_cart_shift (MPI_Comm comm, int direction, int disp, int *rank_source, @@ -40,7 +42,9 @@ int topo_base_cart_shift (lam_communicator_t *comm, /* * Handle the trivial case. */ - ord = comm->c_group->g_myrank; +#if 0 + ord = lam_comm_rank(comm); +#endif if (disp == 0) { *rank_dest = *rank_source = ord; return MPI_SUCCESS; @@ -48,9 +52,9 @@ int topo_base_cart_shift (lam_communicator_t *comm, /* * Compute the rank factor and ordinate. */ - factor = comm->c_topo_nprocs; - p = comm->c_topo_dims; - for (i = 0; (i < comm->c_topo_ndims) && (i <= direction); ++i, ++p) { + factor = comm->c_topo_comm->mtc_nprocs; + p = comm->c_topo_comm->mtc_dims; + for (i = 0; (i < comm->c_topo_comm->mtc_ndims) && (i <= direction); ++i, ++p) { if ((thisdirection = *p) > 0) { thisperiod = 0; } else { @@ -75,14 +79,20 @@ int topo_base_cart_shift (lam_communicator_t *comm, } else { destord %= thisdirection; if (destord < 0) destord += thisdirection; - *rank_dest = comm->c_group->g_myrank + ((destord - ord) * factor); +#if 0 + *rank_dest = lam_comm_rank(comm); +#endif + *rank_dest += ((destord - ord) * factor); } if ( ((srcord < 0) || (srcord >= thisdirection)) && (!thisperiod) ) { *rank_source = MPI_PROC_NULL; } else { srcord %= thisdirection; if (srcord < 0) srcord += thisdirection; - *rank_source = comm->c_group->g_myrank + ((srcord - ord) * factor); +#if 0 + *rank_dest = lam_comm_rank(comm); +#endif + *rank_dest += ((srcord - ord) * factor); } return MPI_SUCCESS; diff --git a/src/mca/topo/base/topo_base_cart_sub.c b/src/mca/topo/base/topo_base_cart_sub.c index cd67f0c5c8..b39535ec06 100644 --- a/src/mca/topo/base/topo_base_cart_sub.c +++ b/src/mca/topo/base/topo_base_cart_sub.c @@ -3,6 +3,8 @@ */ #include "mca/topo/base/base.h" +#include "communicator/communicator.h" +#include "mca/topo/topo.h" /* * function - partitions a communicator into subgroups which @@ -19,16 +21,18 @@ * @retval MPI_ERR_TOPOLOGY * @retval MPI_ERR_COMM */ -int topo_base_cart_sub (lam_communicator_t *comm, +int topo_base_cart_sub (MPI_Comm comm, int *remain_dims, - lam_communicator_t **new_comm){ - lam_communicator_t *newcomm; + MPI_Comm *new_comm){ + MPI_Comm newcomm; int errcode; int colour; int key; int colfactor; int keyfactor; +#if 0 int rank; +#endif int ndim; int dim; int allfalse; @@ -46,9 +50,9 @@ int topo_base_cart_sub (lam_communicator_t *comm, ndim = 0; allfalse = 0; - i = comm->c_topo_ndims - 1; - d = comm->c_topo_dims + i; - c = comm->c_topo_coords + i; + i = comm->c_topo_comm->mtc_ndims - 1; + d = comm->c_topo_comm->mtc_dims + i; + c = comm->c_topo_comm->mtc_coords + i; r = remain_dims + i; for (; i >= 0; --i, --d, --c, --r) { @@ -68,14 +72,18 @@ int topo_base_cart_sub (lam_communicator_t *comm, * have a communicator unless you're in it). */ if (ndim == 0) { +#if 0 lam_comm_rank (comm, &colour); +#endif ndim = 1; allfalse = 1; } /* * Split the communicator. */ +#if 0 errcode = lam_comm_split (comm, colour, key, new_comm); +#endif if (errcode != MPI_SUCCESS) { return errcode; } @@ -84,36 +92,40 @@ int topo_base_cart_sub (lam_communicator_t *comm, */ newcomm = *new_comm; if (newcomm != MPI_COMM_NULL) { - newcomm->c_topo_type = MPI_CART; - newcomm->c_topo_nprocs = keyfactor; - newcomm->c_topo_ndims = ndim; - newcomm->c_topo_dims = (int *) + newcomm->c_topo_comm->mtc_type = MPI_CART; + newcomm->c_topo_comm->mtc_nprocs = keyfactor; + newcomm->c_topo_comm->mtc_ndims = ndim; + newcomm->c_topo_comm->mtc_dims = (int *) malloc((unsigned) 2 * ndim * sizeof(int)); - if (newcomm->c_topo_dims == 0) { + if (newcomm->c_topo_comm->mtc_dims == 0) { return MPI_ERR_OTHER; } - newcomm->c_topo_coords = newcomm->c_topo_dims + ndim; + newcomm->c_topo_comm->mtc_coords = newcomm->c_topo_comm->mtc_dims + ndim; if (!allfalse) { - p = newcomm->c_topo_dims; - d = comm->c_topo_dims; + p = newcomm->c_topo_comm->mtc_dims; + d = comm->c_topo_comm->mtc_dims; r = remain_dims; - for (i = 0; i < comm->c_topo_ndims; ++i, ++d, ++r) { + for (i = 0; i < comm->c_topo_comm->mtc_ndims; ++i, ++d, ++r) { if (*r) { *p++ = *d; } } } else { - newcomm->c_topo_dims[0] = 1; + newcomm->c_topo_comm->mtc_dims[0] = 1; } /* * Compute the caller's coordinates. */ +#if 0 errcode = lam_comm_rank (newcomm, &rank); +#endif if (errcode != MPI_SUCCESS) { return errcode; } +#if 0 errcode = lam_cart_coords (newcomm, rank, - ndim, newcomm->c_topo_coords); + ndim, newcomm->c_topo_comm->mtc_coords); +#endif if (errcode != MPI_SUCCESS) { return errcode; } diff --git a/src/mca/topo/base/topo_base_cartdim_get.c b/src/mca/topo/base/topo_base_cartdim_get.c index c798b4a0a4..c0c4dde35e 100644 --- a/src/mca/topo/base/topo_base_cartdim_get.c +++ b/src/mca/topo/base/topo_base_cartdim_get.c @@ -3,6 +3,8 @@ */ #include "mca/topo/base/base.h" +#include "communicator/communicator.h" +#include "mca/topo/topo.h" /* * function - retrieves Cartesian topology information associated with a @@ -14,10 +16,10 @@ * @retval MPI_SUCCESS * @retval MPI_ERR_COMM */ -int topo_base_cartdim_get (lam_communicator_t *comm, +int topo_base_cartdim_get (MPI_Comm comm, int *ndims){ - *ndims = comm->c_topo_ndims; + *ndims = comm->c_topo_comm->mtc_ndims; return MPI_SUCCESS; } diff --git a/src/mca/topo/base/topo_base_close.c b/src/mca/topo/base/topo_base_close.c index b62dd2ebf9..c22557114d 100644 --- a/src/mca/topo/base/topo_base_close.c +++ b/src/mca/topo/base/topo_base_close.c @@ -5,11 +5,11 @@ #include -#include "constants.h" +#include "include/constants.h" #include "mca/mca.h" #include "mca/base/base.h" -#include "mca/coll/coll.h" -#include "mca/coll/base/base.h" +#include "mca/topo/topo.h" +#include "mca/topo/base/base.h" int mca_topo_base_close(void) { extern lam_list_t mca_topo_base_modules_available; diff --git a/src/mca/topo/base/topo_base_graph_create.c b/src/mca/topo/base/topo_base_graph_create.c index bd6e98e334..2b0fb3e767 100644 --- a/src/mca/topo/base/topo_base_graph_create.c +++ b/src/mca/topo/base/topo_base_graph_create.c @@ -3,6 +3,8 @@ */ #include "mca/topo/base/base.h" +#include "communicator/communicator.h" +#include "mca/topo/topo.h" /* * @@ -20,14 +22,15 @@ * @retval MPI_ERR_OUT_OF_RESOURCE */ -int topo_base_graph_create (lam_communicator_t *comm_old, +int topo_base_graph_create (MPI_Comm comm_old, int nnodes, int *index, int *edges, int reorder, - lam_communicator_t **comm_graph) { - - lam_group_t *newgroup; + MPI_Comm *comm_graph) { +#if 0 + MPI_Group newgroup; +#endif int nedges; int size; int err; @@ -43,7 +46,7 @@ int topo_base_graph_create (lam_communicator_t *comm_old, topo = (int *) malloc((unsigned) (nnodes + nedges) * sizeof(int)); if (topo == 0) { printf ("Out of resources\n"); - return MPI_ERR_OUT_OF_RESOURCE; + return MPI_ERR_SYSRESOURCE; } for (i = 0, p = topo; i < nnodes; ++i, ++p) { @@ -60,7 +63,9 @@ int topo_base_graph_create (lam_communicator_t *comm_old, /* * Create the group for the new communicator. */ +#if 0 err = lam_comm_size (comm_old, &size); +#endif if (err != MPI_SUCCESS) { free((char *) topo); return err; @@ -72,12 +77,16 @@ int topo_base_graph_create (lam_communicator_t *comm_old, } if (nnodes == size) { +#if 0 err = lam_comm_group (comm_old, &newgroup); +#endif } else { range[0][0] = 0; range[0][1] = nnodes - 1; range[0][2] = 1; +#if 0 err = lam_group_range_incl(comm_old->c_group, 1, range, &newgroup); +#endif } if (err != MPI_SUCCESS) { free((char *) topo); @@ -86,24 +95,30 @@ int topo_base_graph_create (lam_communicator_t *comm_old, /* * Create the new communicator. */ +#if 0 err = lam_comm_create (comm_old, newgroup, comm_graph); +#endif if (err != MPI_SUCCESS) { free((char *) topo); +#if 0 lam_group_free (&newgroup); +#endif return err; } /* * Set the communicator topology information. */ if (*comm_graph != MPI_COMM_NULL) { - (*comm_graph)->c_topo_type = MPI_GRAPH; - (*comm_graph)->c_topo_nprocs = nnodes; - (*comm_graph)->c_topo_nedges = nedges; - (*comm_graph)->c_topo_index = topo; - (*comm_graph)->c_topo_edges = topo + nnodes; + (*comm_graph)->c_topo_comm->mtc_type = MPI_GRAPH; + (*comm_graph)->c_topo_comm->mtc_nprocs = nnodes; + (*comm_graph)->c_topo_comm->mtc_nedges = nedges; + (*comm_graph)->c_topo_comm->mtc_index = topo; + (*comm_graph)->c_topo_comm->mtc_edges = topo + nnodes; } +#if 0 err = lam_group_free (&newgroup); +#endif if (err != MPI_SUCCESS) { return err; } diff --git a/src/mca/topo/base/topo_base_graph_dims_get.c b/src/mca/topo/base/topo_base_graph_dims_get.c index 6febf8a6cd..c3bcc9bc51 100644 --- a/src/mca/topo/base/topo_base_graph_dims_get.c +++ b/src/mca/topo/base/topo_base_graph_dims_get.c @@ -3,6 +3,8 @@ */ #include "mca/topo/base/base.h" +#include "communicator/communicator.h" +#include "mca/topo/topo.h" /* * function - Retrieves graph topology information associated with a @@ -17,12 +19,12 @@ * @retval MPI_ERR_COMM * @retval MPI_ERR_ARG */ -int topo_base_graph_dims_get (lam_communicator_t *comm, +int topo_base_graph_dims_get (MPI_Comm comm, int *nodes, int *nedges){ - *nodes = comm->c_topo_nprocs; - *nedges = comm->c_topo_nedges; + *nodes = comm->c_topo_comm->mtc_nprocs; + *nedges = comm->c_topo_comm->mtc_nedges; return MPI_SUCCESS; } diff --git a/src/mca/topo/base/topo_base_graph_get.c b/src/mca/topo/base/topo_base_graph_get.c index 439e5782ce..b0256c564d 100644 --- a/src/mca/topo/base/topo_base_graph_get.c +++ b/src/mca/topo/base/topo_base_graph_get.c @@ -3,6 +3,8 @@ */ #include "mca/topo/base/base.h" +#include "communicator/communicator.h" +#include "mca/topo/topo.h" /* * function - retrieves graph topology information associated with a @@ -17,7 +19,7 @@ * @retval MPI_SUCCESS */ -int topo_base_graph_get (lam_communicator_t *comm, +int topo_base_graph_get (MPI_Comm comm, int maxindex, int maxedges, int *index, @@ -28,13 +30,13 @@ int topo_base_graph_get (lam_communicator_t *comm, /* * Fill the nodes and edges arrays. */ - p = comm->c_topo_index; - for (i = 0; (i < comm->c_topo_nprocs) && (i < maxindex); ++i, ++p) { - *nodes++ = *p; + p = comm->c_topo_comm->mtc_index; + for (i = 0; (i < comm->c_topo_comm->mtc_nprocs) && (i < maxindex); ++i, ++p) { + *index++ = *p; } - p = comm->c_topo_edges; - for (i = 0; (i < comm->c_topo_nedges) && (i < maxedges); ++i, ++p) { + p = comm->c_topo_comm->mtc_edges; + for (i = 0; (i < comm->c_topo_comm->mtc_nedges) && (i < maxedges); ++i, ++p) { *edges++ = *p; } diff --git a/src/mca/topo/base/topo_base_graph_neighbors_count.c b/src/mca/topo/base/topo_base_graph_neighbors_count.c index 4f1b5182d1..9defe93d22 100644 --- a/src/mca/topo/base/topo_base_graph_neighbors_count.c +++ b/src/mca/topo/base/topo_base_graph_neighbors_count.c @@ -3,6 +3,8 @@ */ #include "mca/topo/base/base.h" +#include "communicator/communicator.h" +#include "mca/topo/topo.h" /* * function - returns the number of neighbors of a node @@ -15,12 +17,12 @@ * @retval MPI_SUCCESS */ -int topo_base_graph_neighbors_count (lam_communicator_t *comm, +int topo_base_graph_neighbors_count (MPI_Comm comm, int rank, int *nneighbors){ - *nneighbors = comm->c_topo_index[rank]; + *nneighbors = comm->c_topo_comm->mtc_index[rank]; if (rank > 0) { - *nneighbors -= comm->c_topo_index[rank - 1]; + *nneighbors -= comm->c_topo_comm->mtc_index[rank - 1]; } return MPI_SUCCESS; diff --git a/src/mca/topo/base/topo_base_select.c b/src/mca/topo/base/topo_base_select.c index 7e3620830c..89e958c38b 100644 --- a/src/mca/topo/base/topo_base_select.c +++ b/src/mca/topo/base/topo_base_select.c @@ -4,13 +4,13 @@ #include "lam_config.h" -#include "mca/mca.h" #include "lfc/lam_list.h" #include "runtime/runtime.h" #include "mca/mca.h" #include "mca/base/base.h" #include "mca/topo/topo.h" #include "mca/topo/base/base.h" +#include "util/output.h" /* @@ -46,18 +46,21 @@ typedef struct opened_module_t opened_module_t; */ int mca_topo_base_select (mca_topo_t *selected, - bool *allow_multi_user_thread, + bool *allow_multi_user_threads, bool *have_hidden_threads) { int priority; int best_priority; bool user_threads; bool hidden_threads; + bool best_user_threads; + bool best_hidden_threads; lam_list_item_t *item; mca_base_module_list_item_t *mli; mca_topo_base_module_t *module; mca_topo_base_module_t *best_module; mca_topo_t *actions; + lam_list_t opened; opened_module_t *om; /* @@ -98,10 +101,10 @@ int mca_topo_base_select (mca_topo_t *selected, /* * query did not return any action which can be used */ - lam_output_verbose(10, mca_topo_base_module, + lam_output_verbose(10, mca_topo_base_output, "select: query returned failure"); } else { - lam_output_verbose(10, mca_topo_base_module, + lam_output_verbose(10, mca_topo_base_output, "select: query returned priority &d", priority); /* @@ -170,7 +173,7 @@ int mca_topo_base_select (mca_topo_t *selected, * structure we need to fill it in with the base structure * function pointers. This is yet to be done */ - if (NULL != om->om_actions->init) { + if (NULL != om->om_actions->topo_init) { /* * commenting this out for now since I m * not sure of the calling conventions diff --git a/src/mca/topo/topo.h b/src/mca/topo/topo.h index 93e8090044..71460d693d 100644 --- a/src/mca/topo/topo.h +++ b/src/mca/topo/topo.h @@ -10,7 +10,6 @@ #include "mpi.h" #include "mca/mca.h" #include "mca/base/base.h" -#include "communicator/communicator.h" /* * ****************************************************************** @@ -50,7 +49,7 @@ * *********** These functions go in the module struct ********** * **************** module struct ******************************* */ -typedef const struct mca_topo_1_0_0_t * +typedef struct mca_topo_1_0_0_t * (*mca_topo_base_comm_query_1_0_0_fn_t)(int *priority, bool *allow_multi_user_threads, bool *have_hidden_threads); @@ -75,6 +74,30 @@ typedef mca_topo_base_module_1_0_0_t mca_topo_base_module_t; * *********************** module struct ends here ****************** * ****************************************************************** */ +/* + * ****************************************************************** + * *********************** information structure ****************** + * ****************************************************************** + */ +struct mca_topo_comm_1_0_0_t { + int mtc_cube_dim; /**< Inscribing cube dimension */ + int mtc_type; /**< Topology type */ + int mtc_nprocs; /**< Number of processes */ + int mtc_ndims; /**< Number of cart dimensions */ + int mtc_nedges; /**< Graph edges */ + int *mtc_dims; /**< Cart dimensions */ + int *mtc_coords; /**< Cart coordinates */ + int *mtc_index; /**< Graph indices */ + int *mtc_edges; /**< Graph edges */ +}; +typedef struct mca_topo_comm_1_0_0_t mca_topo_comm_1_0_0_t; +typedef mca_topo_comm_1_0_0_t mca_topo_comm_t; + +/* + * ****************************************************************** + * *********************** information structure ****************** + * ****************************************************************** + */ /* * *********************************************************************** @@ -84,89 +107,89 @@ typedef mca_topo_base_module_1_0_0_t mca_topo_base_module_t; * *********************************************************************** */ -typedef int (*mca_topo_base_init_1_0_0_fn_t) - (lam_commmunicator_t *comm, - mca_topo_1_0_0_t **new_topo); +typedef int (*mca_topo_base_init_1_0_0_fn_t) (void); +/* (MPI_Comm comm, + mca_topo_1_0_0_t **new_topo); */ typedef int (*mca_topo_base_cart_coords_fn_t) - (lam_communicator_t* comm, + (MPI_Comm comm, int rank, int maxdims, int *coords); typedef int (*mca_topo_base_cart_create_fn_t) - (lam_communicator_t* old_comm, + (MPI_Comm old_comm, int ndims, int *dims, int *periods, int redorder, - lam_communicator_t** comm_cart); + MPI_Comm* comm_cart); typedef int (*mca_topo_base_cart_get_fn_t) - (lam_communicator_t* comm, + (MPI_Comm comm, int maxdims, int *dims, int *periods, int *coords); typedef int (*mca_topo_base_cartdim_get_fn_t) - (lam_communicator_t *comm, + (MPI_Comm comm, int *ndims); typedef int (*mca_topo_base_cart_map_fn_t) - (lam_communicator_t* comm, + (MPI_Comm comm, int ndims, int *dims, int *periods, int *newrank); typedef int (*mca_topo_base_cart_rank_fn_t) - (lam_communicator_t* comm, + (MPI_Comm comm, int *coords, int *rank); typedef int (*mca_topo_base_cart_shift_fn_t) - (lam_communicator_t* comm, + (MPI_Comm comm, int direction, int disp, int *rank_source, int *rank_dest); typedef int (*mca_topo_base_cart_sub_fn_t) - (lam_communicator_t* comm, + (MPI_Comm comm, int *remain_dims, - lam_communicator_t** new_comm); + MPI_Comm* new_comm); typedef int (*mca_topo_base_graph_create_fn_t) - (lam_communicator_t* comm_old, + (MPI_Comm comm_old, int nnodes, int *index, int *edges, int reorder, - lam_communicator_t** comm_graph); + MPI_Comm* comm_graph); typedef int (*mca_topo_base_graph_get_fn_t) - (lam_communicator_t* comm, + (MPI_Comm comm, int maxindex, int maxedges, int *index, int *edges); typedef int (*mca_topo_base_graph_map_fn_t) - (lam_communicator_t* comm, + (MPI_Comm comm, int nnodes, int *index, int *edges, int *newrank); typedef int (*mca_topo_base_graph_neighbors_fn_t) - (lam_communicator_t* comm, + (MPI_Comm comm, int rank, int maxneighbors, int *neighbors); typedef int (*mca_topo_base_graph_neighbors_count_fn_t) - (lam_communicator_t* comm, + (MPI_Comm comm, int rank, int *nneighbors); @@ -223,7 +246,7 @@ typedef mca_topo_1_0_0_t mca_topo_t; extern "C" { #endif const mca_topo_1_0_0_t * - mca_topo_unity_comm_query(int *priority + mca_topo_unity_comm_query(int *priority, bool *allow_multi_user_threads, bool *have_hidden_threads); #if defined(c_plusplus) || defined(__cplusplus)