reduce has now been tested quite extensively for all (predefined) operations

and for all root nodes and passed all tests.
First cut on barrier (which from my perspective does not make sense from the
performance point of view) and on allreduce (which might make sense),

This commit was SVN r7774.

ompi/mca/coll/hierarch/Makefile.am

@@ -45,6 +45,7 @@ libmca_coll_hierarch_la_LDFLAGS = -module -avoid-version
 sources = \
         coll_hierarch.h \
         coll_hierarch.c \
+        coll_hierarch_allreduce.c \
         coll_hierarch_barrier.c \
         coll_hierarch_bcast.c \
         coll_hierarch_component.c \

ompi/mca/coll/hierarch/coll_hierarch.c

@@ -69,22 +69,22 @@ static const mca_coll_base_module_1_0_0_t intra = {
   /* Collective function pointers */
   /* function pointers marked with NULL are not yet implemented
      and will use the functions provided in the basic module */
-  NULL,
-  NULL,
-  NULL,
-  NULL,
-  NULL,
-  NULL,
+  NULL, /* allgather */
+  NULL, /* allgatherv */
+  mca_coll_hierarch_allreduce_intra, 
+  NULL, /* alltoall */
+  NULL, /* alltoallv */
+  NULL, /* alltoallw */
   mca_coll_hierarch_barrier_intra,
   mca_coll_hierarch_bcast_intra,
-  NULL, 
-  NULL, 
-  NULL, 
+  NULL,  /* exscan */
+  NULL,  /* gather */
+  NULL,  /* gatherv */
   mca_coll_hierarch_reduce_intra,
-  NULL, 
-  NULL,
-  NULL, 
-  NULL
+  NULL,  /* reduce_scatter */
+  NULL,  /* scan */
+  NULL,  /* scatter */
+  NULL   /* scatterv */
 };
 
 

ompi/mca/coll/hierarch/coll_hierarch.h

@@ -37,6 +37,7 @@ extern int mca_coll_hierarch_priority_param;
 extern int mca_coll_hierarch_verbose_param;
 extern int mca_coll_hierarch_use_rdma_param;
 extern int mca_coll_hierarch_ignore_sm_param;
+extern int mca_coll_hier_verbose;
 
 
 #define HIER_DEFAULT_NUM_LLEAD 5

ompi/mca/coll/hierarch/coll_hierarch_allreduce.c

@@ -0,0 +1,105 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "coll_hierarch.h"
+
+#include <stdio.h>
+
+#include "mpi.h"
+#include "ompi/include/constants.h"
+#include "opal/util/output.h"
+#include "communicator/communicator.h"
+#include "datatype/datatype.h"
+#include "mca/coll/coll.h"
+
+
+/*
+ *	reduce_intra
+ *
+ *	Function:	- reduction using two level hierarchy algorithm
+ *	Accepts:	- same as MPI_Reduce()
+ *	Returns:	- MPI_SUCCESS or error code
+ */
+int mca_coll_hierarch_allreduce_intra(void *sbuf, void *rbuf, int count,
+				      struct ompi_datatype_t *dtype, 
+				      struct ompi_op_t *op,
+				      struct ompi_communicator_t *comm)
+{
+    struct mca_coll_base_comm_t *data=NULL;
+    struct ompi_communicator_t *llcomm=NULL;
+    struct ompi_communicator_t *lcomm=NULL;
+    int rank, lrank;
+    int lroot, llroot;
+    long extent, true_extent, lb, true_lb;
+    char *tmpbuf=NULL, *tbuf=NULL;
+    int ret;
+    int root=0;
+
+    rank   = ompi_comm_rank ( comm );
+    data   = comm->c_coll_selected_data;
+    lcomm  = data->hier_lcomm;
+
+    if ( mca_coll_hier_verbose ) {
+      printf("%s:%d: executing hierarchical allreduce with cnt=%d \n",
+	     comm->c_name, rank, count );
+    }
+
+    llcomm = mca_coll_hierarch_get_llcomm ( root, data, &llroot, &lroot);
+
+    if ( MPI_COMM_NULL != lcomm ) {
+      lrank = ompi_comm_rank (lcomm);
+	if ( lrank == lroot ) {
+	    ompi_ddt_get_extent(dtype, &lb, &extent);
+	    ompi_ddt_get_true_extent(dtype, &true_lb, &true_extent);
+
+	    tbuf = malloc(true_extent + (count - 1) * extent);
+	    if (NULL == tbuf) {
+		return OMPI_ERR_OUT_OF_RESOURCE;
+	    }
+	    tmpbuf = tbuf - lb;
+	}
+
+	if ( MPI_IN_PLACE != sbuf ) {
+	    ret = lcomm->c_coll.coll_reduce (sbuf, tmpbuf, count, dtype, 
+					     op, lroot, lcomm);
+	}
+	else {
+            ret = lcomm->c_coll.coll_reduce (rbuf, tmpbuf, count, dtype, 
+					     op, lroot, lcomm);
+	}
+	if ( OMPI_SUCCESS != ret ) {
+	    goto exit;
+	}
+    }
+
+    if ( MPI_UNDEFINED != llroot ) {
+	ret = llcomm->c_coll.coll_allreduce (tmpbuf, rbuf, count, dtype,
+					     op, llcomm);
+    }
+
+    if ( MPI_COMM_NULL != lcomm ) {
+	ret = lcomm->c_coll.coll_bcast(rbuf, count, dtype, lroot, lcomm );
+    }
+
+
+ exit:
+    if ( NULL != tmpbuf ) {
+	free ( tmpbuf );
+    }
+
+    return ret;
+}

ompi/mca/coll/hierarch/coll_hierarch_barrier.c

@@ -14,28 +14,66 @@
  * $HEADER$
  */
 
+
 #include "ompi_config.h"
 #include "coll_hierarch.h"
 
 #include "mpi.h"
 #include "ompi/include/constants.h"
 #include "opal/util/output.h"
+#include "communicator/communicator.h"
 #include "mca/coll/coll.h"
-#include "mca/coll/base/base.h"
-#include "coll_hierarch.h"
 
 
 /*
  *	barrier_intra
  *
- *	Function:	- barrier using O(N) algorithm
+ *	Function:	- barrier using hierarchical algorithm
  *	Accepts:	- same as MPI_Barrier()
  *	Returns:	- MPI_SUCCESS or error code
  */
 int mca_coll_hierarch_barrier_intra(struct ompi_communicator_t *comm)
 {
-  opal_output_verbose(10, mca_coll_base_output, "In hierarch barrier_intra");
-  return MPI_SUCCESS;
+    struct mca_coll_base_comm_t *data=NULL;
+    struct ompi_communicator_t *llcomm=NULL;
+    struct ompi_communicator_t *lcomm=NULL;
+    int root=0;
+    int lroot, llroot;
+    int rank, ret;
+    
+    rank   = ompi_comm_rank ( comm );
+    data   = comm->c_coll_selected_data;
+    lcomm  = data->hier_lcomm;
+
+    if ( mca_coll_hier_verbose ) { 
+      printf("%s:%d: executing hierarchical barrier\n", comm->c_name, rank );
+    } 
+
+    llcomm = mca_coll_hierarch_get_llcomm ( root, data, &llroot, &lroot);
+
+    /* 
+     * Barrier consists of three steps:
+     * - barrier on the low-level communicators
+     * - barrier among the local leaders
+     * - barrier on the low-level communicators. This step is 
+     *   necessary to avoid that any non local leaders exit too early.
+     */
+    if ( MPI_COMM_NULL != lcomm ) {
+	ret = lcomm->c_coll.coll_barrier ( lcomm );
+        if ( OMPI_SUCCESS != ret ) {
+           return ret;
+	}
+    }
+
+    if ( MPI_UNDEFINED != llroot ) {
+	ret = llcomm->c_coll.coll_barrier ( llcomm );
+    }
+
+    if ( MPI_COMM_NULL != lcomm ) {
+	ret = lcomm->c_coll.coll_barrier ( lcomm );
+    }
+
+    return  ret;
 }
 
 

ompi/mca/coll/hierarch/coll_hierarch_bcast.c

@@ -31,8 +31,6 @@
  *	Returns:	- MPI_SUCCESS or error code
  */
 
-extern int mca_coll_hier_verbose;
-
 
 int mca_coll_hierarch_bcast_intra(void *buff, 
 				  int count,

ompi/mca/coll/hierarch/coll_hierarch_reduce.c

@@ -26,7 +26,6 @@
 #include "datatype/datatype.h"
 #include "mca/coll/coll.h"
 
-extern int mca_coll_hier_verbose;
 
 /*
  *	reduce_intra