first cut on the framework, which detects the two layer (sm/non-sm) hierarchy.

It compiles, is however still far, far away from being usable.

This commit was SVN r4527.

src/mca/coll/hierarch/coll_hierarch.c

@@ -23,6 +23,23 @@
 #include "mca/coll/base/base.h"
 #include "coll_hierarch.h"
 
+/**
+ * NOTE NOTE NOTE NOTE:
+ * this is a preliminary version dealing just with sm/non-sm layers.
+ * It's main purpose is to understand the information and data flow
+ * better, and for developing a first cut of the required interfaces.
+ *
+ * EG, Stuttgart, Feb. 24 2005
+ */
+
+#include "mca/ptl/ptl.h"
+#include "mca/pml/teg/src/pml_teg_proc.h"
+#include "mca/pml/teg/src/pml_teg_ptl.h"
+
+/* local functions and data */
+static void mca_coll_hierarch_checkfor_component (struct ompi_communicator_t *comm,
+						  char *component_name, int *key,
+						  int *done );
 
 /*
  * Linear set of collective algorithms
@@ -56,35 +73,6 @@ static const mca_coll_base_module_1_0_0_t intra = {
 };
 
 
-/*
- * Linear set of collective algorithms for intercommunicators
- */
-static const mca_coll_base_module_1_0_0_t inter = {
-
-  /* Initialization / finalization functions */
-
-  mca_coll_hierarch_module_init,
-  mca_coll_hierarch_module_finalize,
-
-  /* Collective function pointers */
-  /* No inter-communicator functions are provided at the moment */
-  NULL, 
-  NULL, 
-  NULL, 
-  NULL, 
-  NULL, 
-  NULL, 
-  NULL, 
-  NULL, 
-  NULL, 
-  NULL, 
-  NULL, 
-  NULL, 
-  NULL, 
-  NULL,
-  NULL, 
-  NULL
-};
 
 
 /*
@@ -109,14 +97,68 @@ int mca_coll_hierarch_init_query(bool *allow_hierarch_user_threads,
  * priority we want to return.
  */
 const mca_coll_base_module_1_0_0_t *
-mca_coll_hierarch_comm_query(struct ompi_communicator_t *comm, int *priority)
+mca_coll_hierarch_comm_query(struct ompi_communicator_t *comm, int *priority,
+			     struct mca_coll_base_comm_t **data)
 {
+    int size;
+    int color, ncount, maxncount;
+    int *colorarr=NULL;
+    
+
+    /* Get the priority level attached to this module */
     if (OMPI_SUCCESS != mca_base_param_lookup_int(mca_coll_hierarch_priority_param, 
 						  priority)) {
 	return NULL;
     }
     
-  return OMPI_COMM_IS_INTER(comm) ? &inter : &intra;
+    /* This module only works for intra-communicators at the moment */
+    if ( OMPI_COMM_IS_INTER(comm) ) {
+	*priority = 0;
+	return NULL;
+    }
+
+    /* Check now, whether all process in this communicator can talk with
+       sm or not. If yes, then there is no need for the hierarchical
+       module */
+    size = ompi_comm_size(comm);
+    colorarr = (int *) malloc ( sizeof(int) * size );
+    if ( NULL == colorarr ) {
+	*priority = 0;
+	return NULL;
+    }
+
+    mca_coll_hierarch_checkfor_component ( comm, "sm", &color, &ncount);
+ 
+#define KNOW_HOW_TO_CALL_ALLGATHER
+#ifdef KNOW_HOW_TO_CALL_ALLGATHER
+    comm->c_coll_basic_module->coll_allreduce (&ncount, &maxncount, 1, MPI_INT, 
+					       MPI_MAX, comm );
+
+    comm->c_coll_basic_module->coll_allgather (&color, 1, MPI_INT, 
+					       colorarr, 1, MPI_INT, comm );
+
+#else
+    maxncount = ncount;
+#endif
+    if ( 1 == maxncount ) {
+	/* 
+	 * this means, no process has a partner to which it can talk with 'sm',
+	 * no need for the hierarchical component 
+	 */
+	*priority = 0;
+	return NULL;
+    }
+    else if ( maxncount == size ) {
+	/* 
+	 * everybody can talk to every other process with sm, 
+	 * no need for the hierarchical module 
+	 */
+	*priority = 0;
+	return NULL;
+    }
+
+    *data = (struct mca_coll_base_comm_t *) colorarr; 
+    return &intra;
 }
 
 
@@ -126,13 +168,102 @@ mca_coll_hierarch_comm_query(struct ompi_communicator_t *comm, int *priority)
 const struct mca_coll_base_module_1_0_0_t *
 mca_coll_hierarch_module_init(struct ompi_communicator_t *comm)
 {
-    mca_base_param_lookup_int(mca_coll_hierarch_verbose_param,
+    int color, ncount;
-                              &mca_coll_hierarch_verbose);
+    int *colorarr=NULL, *llr=NULL;
-    if (mca_coll_hierarch_verbose > 0) {
+    int size, rank, ret=OMPI_SUCCESS;
-        printf("Hello!  This is the \"hierarch\" coll component.  I'll be your coll component\ntoday.  Please tip your waitresses well.\n");
+    int i, j, c;
+    int found;
+
+    struct ompi_communicator_t *llcomm=NULL;
+    struct mca_coll_base_comm_t *data=NULL;
+    struct mca_coll_base_module_1_0_0_t *hier_mod=NULL;
+
+    rank = ompi_comm_rank(comm);
+    size = ompi_comm_size(comm);
+    mca_coll_hierarch_checkfor_component ( comm, "sm", &color, &ncount);
+    
+    /* Generate the subcommunicator based on the color returned by
+       the previous function. */
+    ret = ompi_comm_split ( comm, color, rank, &llcomm, 0 );
+    if ( OMPI_SUCCESS != ret ) {
+	goto exit;
     }
 
-    return OMPI_COMM_IS_INTER(comm) ? &inter : &intra;
+    /* store the low-level communicator and a certain number of requests
+       on the communicator */
+    data = calloc ( 1, sizeof(struct mca_coll_base_comm_t));
+    if ( NULL == data ) {
+	goto exit;
+    }
+        
+    data->hier_llcomm   = llcomm;
+    data->hier_num_reqs = 2 * size;
+    data->hier_reqs     = (ompi_request_t **) malloc (sizeof(ompi_request_t)*2*size);
+    if ( NULL == data->hier_reqs ) {
+	goto exit;
+    }
+
+    /* determine how many local leader there are and who they are */
+    colorarr = (int *) comm->c_coll_selected_data;
+    llr      = (int *) calloc (1, sizeof(int) * size);
+    if (NULL == llr ) {
+	goto exit;
+    }
+
+    for (c=0, i=0; i<size; i++ ){
+	found=0;
+	for (j=0; j<c ; j++) {
+	    if ( colorarr[i] == llr[j] ) {
+		found=0;
+		break;
+	    }
+	}
+	if ( !found ) {
+	    if ( MPI_UNDEFINED == colorarr[i] ) {
+		llr[c] = i;
+	    }
+	    else {
+		llr[c] = colorarr[i];
+	    }
+	    if ( llr[c] == color ) {
+		data->hier_my_lleader = c;
+	    }
+	    c++;
+	}
+
+    }
+    
+    data->hier_num_lleaders = c-1;
+    data->hier_lleaders = (int *) malloc ( sizeof(int) * data->hier_num_lleaders);
+    if ( NULL == data->hier_lleaders ) {
+	goto exit;
+    }
+
+    memcpy ( data->hier_lleaders, llr, data->hier_num_lleaders * sizeof(int));
+    comm->c_coll_selected_data = (struct mca_coll_base_comm_t *)data;
+    hier_mod = &intra;
+
+ exit:
+    if ( NULL != llr ) {
+	free (llr);
+    }
+    if ( NULL != colorarr ) {
+	free ( colorarr ) ;
+    }
+    if ( OMPI_SUCCESS != ret ) {
+	ompi_comm_free ( &llcomm );
+	if ( NULL != data ) {
+	    if ( NULL != data->hier_reqs ) {
+		free ( data->hier_reqs);
+	    }
+	    if ( NULL != data->hier_lleaders ) {
+		free ( data->hier_lleaders);
+	    }
+	    free ( data );
+	}
+    }
+
+    return hier_mod;
 }
 
 
@@ -141,5 +272,136 @@ mca_coll_hierarch_module_init(struct ompi_communicator_t *comm)
  */
 int mca_coll_hierarch_module_finalize(struct ompi_communicator_t *comm)
 {
+    struct ompi_communicator_t *llcomm=NULL;
+    struct mca_coll_base_comm_t *data=NULL;
+
+    data   = comm->c_coll_selected_data;
+    llcomm = data->hier_llcomm;
+
+    ompi_comm_free (&llcomm);
+    free ( data );
+    
+    comm->c_coll_selected_data = NULL;
     return OMPI_SUCCESS;
 }
+
+int mca_coll_hierarch_comm_unquery ( struct ompi_communicator_t *comm,
+				     struct mca_coll_base_comm_t *data )
+{
+    free (data);
+    return OMPI_SUCCESS;
+}
+
+/**********************************************************************/
+/**********************************************************************/
+/**********************************************************************/
+/* This function checks how many processes are using the component
+   'component_name' for communication and returns this count in 
+   'ncount'. Furthermore it returns a 'key', which can be used to split
+   the communicator into subgroups, such that the new communicators
+   will definitly have all processes communicate with this component.
+*/
+static void 
+mca_coll_hierarch_checkfor_component ( struct ompi_communicator_t *comm,
+				       char *component_name, 
+				       int *key,
+				       int *ncount )
+{
+    mca_pml_proc_t *proc=NULL;
+    mca_ptl_proc_t *ptl_proc=NULL;
+    mca_ptl_base_module_t *ptl_module=NULL;
+    mca_ptl_base_component_t *ptr=NULL;
+
+    int i, j, size, listsize;
+
+    int counter=0;
+    int firstproc=999999;
+    int myrank = -1;
+
+    size = ompi_comm_size ( comm );
+    for ( i=0; i<size; i++ ) {
+	proc = mca_pml_teg_proc_lookup_remote (comm, i);
+
+#ifdef TRY_NEXT_INSTEAD OF FIRST
+	ptl_proc=mca_ptl_array_get_next(&proc->proc_ptl_next);
+	listsize = mca_ptl_array_get_size(&proc->proc_ptl_next);
+#else
+	ptl_proc=mca_ptl_array_get_next(&proc->proc_ptl_first);
+	listsize = mca_ptl_array_get_size(&proc->proc_ptl_first);
+#endif
+	for ( j=0; j<listsize; j++) {
+	    ptl_module = ptl_proc->ptl;
+	    ptr = ptl_module->ptl_component;
+
+	    /* sanity check */
+	    if ( strcmp(ptr->ptlm_version.mca_type_name,"ptl") ) {
+		printf("Oops, got the wrong component! type_name = %s\n",
+		       ptr->ptlm_version.mca_type_name );
+	    }
+	    
+	    /* check for myself.
+	       ATTENTION: this relies on having the self-ptl-component loaded
+	       at this case. Need something better! 
+	    */
+	    if ( !strcmp (ptr->ptlm_version.mca_component_name, "self")) {
+		counter++;
+		myrank = i;
+		continue;
+	    }
+
+		
+	    /* check for the required component */
+	    if (! strcmp (ptr->ptlm_version.mca_component_name, component_name)){
+#ifdef VERBOSE
+		printf("found component %s for rank %d \n", component_name, i );
+#endif
+		counter++;
+		
+		if (i<firstproc ) {
+		    firstproc = i;
+		}
+	    }
+	}
+    }
+    
+    /* Here we might introduce later on an allreduce step to determine,
+       whether we agree on the result or not  */
+
+    *ncount = counter; /* true */
+    /* Print the final result */
+    if ( counter == 1 ) {
+	/* this is the section indicating, that we are not 
+	   using this component */
+	if ( myrank == -1 ) {
+#ifdef VERBOSE
+	    printf("something really weird has happened!\n");
+#endif
+	}
+	else {
+#ifdef VERBOSE
+	    printf("component  %s is not used to talk to anyone in this comm\n",
+		   component_name );
+#endif
+	    firstproc = MPI_UNDEFINED;
+	}
+    }
+    else {
+	if ( myrank < firstproc ) {
+	    firstproc = myrank;
+	}
+#ifdef VERBOSE
+	if ( counter == size ) {
+	    printf("I can talk to all processes in this comm using %s key=%d\n",
+		   component_name, firstproc );
+	}
+	else {
+	    printf(" I can talk to %d processes in this comm using %s key=%d\n",
+		   counter, component_name, firstproc );
+	}
+#endif
+    }
+
+    *key = firstproc;
+    return;
+}
+

src/mca/coll/hierarch/coll_hierarch.h

@@ -36,6 +36,19 @@ extern int mca_coll_hierarch_priority_param;
 extern int mca_coll_hierarch_verbose_param;
 extern int mca_coll_hierarch_verbose;
 
+/*
+ * Data structure for attaching data to the communicator 
+ */
+    struct mca_coll_base_comm_t {
+	struct ompi_communicator_t *hier_llcomm; /* low level communicator */
+	int                   hier_num_lleaders; /* number of local leaders */
+	int                      *hier_lleaders; /* list of local leaders */
+	int                     hier_my_lleader; /* pos. of my lleader in hier_lleaders */
+	int                       hier_num_reqs; /* num. of requests */
+	ompi_request_t              **hier_reqs; /* list of requests */
+    };
+
+
 
 /*
  * coll API functions
@@ -48,8 +61,9 @@ extern int mca_coll_hierarch_verbose;
 				     bool *have_hidden_threads);
     const struct mca_coll_base_module_1_0_0_t *
     mca_coll_hierarch_comm_query(struct ompi_communicator_t *comm, 
-				 int *priority);
+				 int *priority, struct mca_coll_base_comm_t **data);
-    int mca_coll_hierarch_comm_unquery(struct ompi_communicator_t *comm);
+    int mca_coll_hierarch_comm_unquery(struct ompi_communicator_t *comm, 
+				       struct mca_coll_base_comm_t *data);
     
     const struct mca_coll_base_module_1_0_0_t *
             mca_coll_hierarch_module_init(struct ompi_communicator_t *comm);
@@ -60,45 +74,21 @@ extern int mca_coll_hierarch_verbose;
 					  void *rbuf, int rcount, 
 					  struct ompi_datatype_t *rdtype, 
 					  struct ompi_communicator_t *comm);
-    int mca_coll_hierarch_allgather_inter(void *sbuf, int scount, 
-					  struct ompi_datatype_t *sdtype, 
-					  void *rbuf, int rcount, 
-					  struct ompi_datatype_t *rdtype, 
-					  struct ompi_communicator_t *comm);
-
     int mca_coll_hierarch_allgatherv_intra(void *sbuf, int scount, 
 					   struct ompi_datatype_t *sdtype, 
 					   void * rbuf, int *rcounts, 
 					   int *disps, 
 					   struct ompi_datatype_t *rdtype, 
 					   struct ompi_communicator_t *comm);
-    int mca_coll_hierarch_allgatherv_inter(void *sbuf, int scount, 
-					   struct ompi_datatype_t *sdtype, 
-					   void * rbuf, int *rcounts, 
-					   int *disps, 
-					   struct ompi_datatype_t *rdtype, 
-					   struct ompi_communicator_t *comm);
-    
     int mca_coll_hierarch_allreduce_intra(void *sbuf, void *rbuf, int count, 
 					  struct ompi_datatype_t *dtype, 
 					  struct ompi_op_t *op, 
 					  struct ompi_communicator_t *comm);
-    int mca_coll_hierarch_allreduce_inter(void *sbuf, void *rbuf, int count, 
-					  struct ompi_datatype_t *dtype, 
-					  struct ompi_op_t *op, 
-					  struct ompi_communicator_t *comm);
-    
     int mca_coll_hierarch_alltoall_intra(void *sbuf, int scount, 
 					 struct ompi_datatype_t *sdtype, 
 					 void* rbuf, int rcount, 
 					 struct ompi_datatype_t *rdtype, 
 					 struct ompi_communicator_t *comm);
-    int mca_coll_hierarch_alltoall_inter(void *sbuf, int scount, 
-					 struct ompi_datatype_t *sdtype, 
-					 void* rbuf, int rcount, 
-					 struct ompi_datatype_t *rdtype, 
-					 struct ompi_communicator_t *comm);
-    
     int mca_coll_hierarch_alltoallv_intra(void *sbuf, int *scounts, 
 					  int *sdisps, 
 					  struct ompi_datatype_t *sdtype, 
@@ -106,14 +96,6 @@ extern int mca_coll_hierarch_verbose;
 					  int *rdisps, 
 					  struct ompi_datatype_t *rdtype, 
 					  struct ompi_communicator_t *comm);
-    int mca_coll_hierarch_alltoallv_inter(void *sbuf, int *scounts, 
-					  int *sdisps, 
-					  struct ompi_datatype_t *sdtype, 
-					  void *rbuf, int *rcounts, 
-					  int *rdisps, 
-					  struct ompi_datatype_t *rdtype, 
-					  struct ompi_communicator_t *comm);
-    
     int mca_coll_hierarch_alltoallw_intra(void *sbuf, int *scounts, 
 					  int *sdisps, 
 					  struct ompi_datatype_t **sdtypes, 
@@ -121,113 +103,50 @@ extern int mca_coll_hierarch_verbose;
 					  int *rdisps, 
 					  struct ompi_datatype_t **rdtypes, 
 					  struct ompi_communicator_t *comm);
-    int mca_coll_hierarch_alltoallw_inter(void *sbuf, int *scounts, 
-					  int *sdisps, 
-					  struct ompi_datatype_t **sdtypes, 
-					  void *rbuf, int *rcounts, 
-					  int *rdisps, 
-					  struct ompi_datatype_t **rdtypes, 
-					  struct ompi_communicator_t *comm);
-    
     int mca_coll_hierarch_barrier_intra(struct ompi_communicator_t *comm);
-    int mca_coll_hierarch_barrier_inter(struct ompi_communicator_t *comm);
-    
     int mca_coll_hierarch_bcast_intra(void *buff, int count, 
 				      struct ompi_datatype_t *datatype,
 				      int root, 
 				      struct ompi_communicator_t *comm);
-    int mca_coll_hierarch_bcast_inter(void *buff, int count, 
-				      struct ompi_datatype_t *datatype, 
-				      int root, 
-				      struct ompi_communicator_t *comm);
-    
     int mca_coll_hierarch_exscan_intra(void *sbuf, void *rbuf, int count, 
 				       struct ompi_datatype_t *dtype, 
 				       struct ompi_op_t *op, 
 				       struct ompi_communicator_t *comm);
-    int mca_coll_hierarch_exscan_inter(void *sbuf, void *rbuf, int count, 
-				       struct ompi_datatype_t *dtype, 
-				       struct ompi_op_t *op, 
-				       struct ompi_communicator_t *comm);
-    
     int mca_coll_hierarch_gather_intra(void *sbuf, int scount, 
 				       struct ompi_datatype_t *sdtype, 
 				       void *rbuf, int rcount, 
 				       struct ompi_datatype_t *rdtype, 
 				       int root, 
 				       struct ompi_communicator_t *comm);
-    int mca_coll_hierarch_gather_inter(void *sbuf, int scount, 
-				       struct ompi_datatype_t *sdtype, 
-				       void *rbuf, int rcount, 
-				       struct ompi_datatype_t *rdtype, 
-				       int root, 
-				       struct ompi_communicator_t *comm);
-    
     int mca_coll_hierarch_gatherv_intra(void *sbuf, int scount, 
 					struct ompi_datatype_t *sdtype, 
 					void *rbuf, int *rcounts, int *disps, 
 					struct ompi_datatype_t *rdtype, 
 					int root, 
 					struct ompi_communicator_t *comm);
-    int mca_coll_hierarch_gatherv_inter(void *sbuf, int scount, 
-					struct ompi_datatype_t *sdtype, 
-					void *rbuf, int *rcounts, int *disps, 
-					struct ompi_datatype_t *rdtype, 
-					int root, 
-					struct ompi_communicator_t *comm);
-    
     int mca_coll_hierarch_reduce_intra(void *sbuf, void* rbuf, int count, 
 				       struct ompi_datatype_t *dtype, 
 				       struct ompi_op_t *op, 
 				       int root,
 				       struct ompi_communicator_t *comm);
-    int mca_coll_hierarch_reduce_inter(void *sbuf, void* rbuf, int count, 
-				       struct ompi_datatype_t *dtype,
-				       struct ompi_op_t *op, 
-				       int root,
-				       struct ompi_communicator_t *comm);
-    
     int mca_coll_hierarch_reduce_scatter_intra(void *sbuf, void *rbuf, 
 					       int *rcounts, 
 					       struct ompi_datatype_t *dtype, 
 					       struct ompi_op_t *op, 
 					       struct ompi_communicator_t *comm);
-    int mca_coll_hierarch_reduce_scatter_inter(void *sbuf, void *rbuf, 
-					       int *rcounts, 
-					       struct ompi_datatype_t *dtype, 
-					       struct ompi_op_t *op, 
-					       struct ompi_communicator_t *comm);
-    
     int mca_coll_hierarch_scan_intra(void *sbuf, void *rbuf, int count, 
 				     struct ompi_datatype_t *dtype, 
 				     struct ompi_op_t *op, 
 				     struct ompi_communicator_t *comm);
-    int mca_coll_hierarch_scan_inter(void *sbuf, void *rbuf, int count, 
-				     struct ompi_datatype_t *dtype, 
-				     struct ompi_op_t *op, 
-				     struct ompi_communicator_t *comm);
-    
     int mca_coll_hierarch_scatter_intra(void *sbuf, int scount, 
 					struct ompi_datatype_t *sdtype, void *rbuf, 
 					int rcount, struct ompi_datatype_t *rdtype, 
 					int root, struct ompi_communicator_t *comm);
-    int mca_coll_hierarch_scatter_inter(void *sbuf, int scount, 
-					struct ompi_datatype_t *sdtype, void *rbuf, 
-					int rcount, struct ompi_datatype_t *rdtype, 
-					int root, struct ompi_communicator_t *comm);
-    
     int mca_coll_hierarch_scatterv_intra(void *sbuf, int *scounts, int *disps, 
 					 struct ompi_datatype_t *sdtype, 
 					 void* rbuf, int rcount, 
 					 struct ompi_datatype_t *rdtype, int root, 
 					 struct ompi_communicator_t *comm);
-    int mca_coll_hierarch_scatterv_inter(void *sbuf, int *scounts, int *disps, 
-					 struct ompi_datatype_t *sdtype, 
-					 void* rbuf, int rcount, 
-					 struct ompi_datatype_t *rdtype, int root, 
-					 struct ompi_communicator_t *comm);
-    
-    
 
 #if defined(c_plusplus) || defined(__cplusplus)
 }

src/mca/coll/hierarch/coll_hierarch_component.c

@@ -85,7 +85,7 @@ const mca_coll_base_component_1_0_0_t mca_coll_hierarch_component = {
 
   mca_coll_hierarch_init_query,
   mca_coll_hierarch_comm_query,
-  NULL
+  mca_coll_hierarch_comm_unquery
 };