switching the root and creating (if necessary) the new local leader sub-communicators seems to work as well. Thoroughly tested with bcast, not yet that exhaustivly tested for the reduction.

This commit was SVN r7773.

ompi/mca/coll/hierarch/coll_hierarch.c

@@ -164,7 +164,7 @@ mca_coll_hierarch_comm_query(struct ompi_communicator_t *comm, int *priority,
 						  &ignore_sm)) {
         return NULL;
     }
-    if (OMPI_SUCCESS != mca_base_param_lookup_int(mca_coll_hierarch_verbose, 
+    if (OMPI_SUCCESS != mca_base_param_lookup_int(mca_coll_hierarch_verbose_param, 
 						  &mca_coll_hier_verbose)) {
         return NULL;
     }
@@ -360,7 +360,9 @@ mca_coll_hierarch_module_init(struct ompi_communicator_t *comm)
     OBJ_CONSTRUCT(&(data->hier_llead), ompi_pointer_array_t);
     ompi_pointer_array_add ( &(data->hier_llead), llead);
     
-    mca_coll_hierarch_dump_struct (data);
+    if ( mca_coll_hier_verbose ) {
+      mca_coll_hierarch_dump_struct (data);
+    }
     
  exit:
     if ( NULL != llr ) {
@@ -578,6 +580,7 @@ struct ompi_communicator_t*  mca_coll_hierarch_get_llcomm (int root,
 	if ( OMPI_SUCCESS != rc ) {
 	    return NULL;
 	}
+	llead->llcomm = llcomm;
 
 	/* Store the new element on the data struct */
 	ompi_pointer_array_add ( &(data->hier_llead), llead);

ompi/mca/coll/hierarch/coll_hierarch.h

@@ -35,7 +35,6 @@ extern "C" {
 extern const mca_coll_base_component_1_0_0_t mca_coll_hierarch_component;
 extern int mca_coll_hierarch_priority_param;
 extern int mca_coll_hierarch_verbose_param;
-extern int mca_coll_hierarch_verbose;
 extern int mca_coll_hierarch_use_rdma_param;
 extern int mca_coll_hierarch_ignore_sm_param;
 

ompi/mca/coll/hierarch/coll_hierarch_bcast.c

@@ -51,17 +51,18 @@ int mca_coll_hierarch_bcast_intra(void *buff,
     data   = comm->c_coll_selected_data;
     lcomm  = data->hier_lcomm;
 
-    if ( mca_coll_hier_verbose ) {
+    if ( mca_coll_hier_verbose ) { 
       printf("%s:%d: executing hierarchical bcast with cnt=%d and root=%d\n",
 	     comm->c_name, rank, count, root );
-    }
+    } 
 
-    /* This function returns the local leader communicator
-       which *always* contains the root of this operation.
-       This might involve creating a new communicator. This is 
-       also the reason, that *every* process in comm has to call 
-       this function
-    */
+    /* 
+     * This function returns the local leader communicator
+     * which *always* contains the root of this operation.
+     * This might involve creating a new communicator. This is 
+     * also the reason, that *every* process in comm has to call 
+     * this function
+     */
     llcomm = mca_coll_hierarch_get_llcomm ( root, data, &llroot, &lroot);
 
     /* Bcast on the upper level among the local leaders */
@@ -73,9 +74,8 @@ int mca_coll_hierarch_bcast_intra(void *buff,
     }
 
     /* once the local leaders got the data from the root, they can distribute
-       it to the processes in their local, low-leve communicator.
-    */
-
+     * it to the processes in their local, low-leve communicator.
+     */
     if ( MPI_COMM_NULL != lcomm ) {
 	ret = lcomm->c_coll.coll_bcast(buff, count, datatype, lroot, lcomm );
     }

ompi/mca/coll/hierarch/coll_hierarch_component.c

@@ -37,7 +37,6 @@ const char *mca_coll_hierarch_component_version_string =
  */
 int mca_coll_hierarch_priority_param = -1;
 int mca_coll_hierarch_verbose_param = -1;
-int mca_coll_hierarch_verbose = 0;
 int mca_coll_hierarch_use_rdma_param=-1;   
 int mca_coll_hierarch_ignore_sm_param=-1;   
 
@@ -95,13 +94,11 @@ static int hierarch_open(void)
     /* Use a high priority, but allow other components to be higher */
     
     mca_coll_hierarch_priority_param = 
-        mca_base_param_register_int("coll", "hierarch", "priority", NULL, 50);
+      mca_base_param_register_int("coll", "hierarch", "priority", NULL, 50);
     mca_coll_hierarch_verbose_param = 
-        mca_base_param_register_int("coll", "hierarch", "verbose", NULL, 
-                                    mca_coll_hierarch_verbose);
+      mca_base_param_register_int("coll", "hierarch", "verbose", NULL, 0);
     mca_coll_hierarch_use_rdma_param = 
         mca_base_param_register_int("coll", "hierarch", "use_rdma", NULL, 0);
-
     mca_coll_hierarch_ignore_sm_param = 
         mca_base_param_register_int("coll", "hierarch", "ignore_sm", NULL, 0);
 

ompi/mca/coll/hierarch/coll_hierarch_reduce.c

@@ -58,12 +58,10 @@ int mca_coll_hierarch_reduce_intra(void *sbuf, void *rbuf, int count,
 	     comm->c_name, rank, count, root );
     }
 
-
     llcomm = mca_coll_hierarch_get_llcomm ( root, data, &llroot, &lroot);
 
     if ( MPI_COMM_NULL != lcomm ) {
-	
-	lrank = ompi_comm_rank (lcomm);
+      lrank = ompi_comm_rank (lcomm);
 	if ( lrank == lroot ) {
 	    ompi_ddt_get_extent(dtype, &lb, &extent);
 	    ompi_ddt_get_true_extent(dtype, &true_lb, &true_extent);