Modifications that are necessary in order to meet latest libhcoll API.

cmr:v1.7.3:reviewer=jladd

This commit was SVN r29202.

ompi/mca/coll/hcoll/coll_hcoll_component.c

@@ -68,7 +68,7 @@ mca_coll_hcoll_component_t mca_coll_hcoll_component = {
 int mca_coll_hcoll_get_lib(void)
 {
 
-    memset(&mca_coll_hcoll_component.hcoll_ops, 
+    memset(&mca_coll_hcoll_component.hcoll_ops,
             0, sizeof(mca_coll_hcoll_component.hcoll_ops));
 
     return OMPI_SUCCESS;
@@ -115,12 +115,12 @@ static int reg_string(const char* param_name,
             NULL, 0, 0, OPAL_INFO_LVL_9,
             MCA_BASE_VAR_SCOPE_READONLY, storage);
     if (NULL != deprecated_param_name) {
-        (void) mca_base_var_register_synonym(index, 
+        (void) mca_base_var_register_synonym(index,
                 "ompi", "coll", "hcoll", deprecated_param_name,
                 MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
     }
 
-    if (0 != (flags & REGSTR_EMPTY_OK) && 
+    if (0 != (flags & REGSTR_EMPTY_OK) &&
             (NULL == *storage || 0 == strlen(*storage))) {
         opal_output(0, "Bad parameter value for parameter \"%s\"",
                 param_name);
@@ -132,8 +132,8 @@ static int reg_string(const char* param_name,
 
 
 
-/* 
- * Utility routine for integer parameter registration 
+/*
+ * Utility routine for integer parameter registration
  */
 static int reg_int(const char* param_name,
         const char* deprecated_param_name,
@@ -149,7 +149,7 @@ static int reg_int(const char* param_name,
             NULL, 0, 0,OPAL_INFO_LVL_9,
             MCA_BASE_VAR_SCOPE_READONLY, storage);
     if (NULL != deprecated_param_name) {
-        (void) mca_base_var_register_synonym(index, 
+        (void) mca_base_var_register_synonym(index,
                 "ompi", "coll", "hcoll", deprecated_param_name,
                 MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
     }
@@ -174,9 +174,9 @@ static int hcoll_register(void)
 {
 
     int ret, tmp;
-    
+
     ret = OMPI_SUCCESS;
-    
+
 #define CHECK(expr) do {                    \
             tmp = (expr);                       \
             if (OMPI_SUCCESS != tmp) ret = tmp; \
@@ -221,20 +221,13 @@ static int hcoll_open(void)
 
     hcoll_rte_fns_setup();
 
-    opal_progress_register(hcoll_progress_fn);
-    rc = hcoll_init();
-
-    if (HCOLL_SUCCESS != rc){
-        opal_progress_unregister(hcoll_progress_fn);
-        HCOL_VERBOSE(1,"Hcol library init failed");
-        return OMPI_ERROR;
-    }
     return OMPI_SUCCESS;
 }
 
 static int hcoll_close(void)
 {
     int rc;
+    HCOL_VERBOSE(5,"HCOLL FINALIZE");
     rc = hcoll_finalize();
     opal_progress_unregister(hcoll_progress_fn);
     if (HCOLL_SUCCESS != rc){

ompi/mca/coll/hcoll/coll_hcoll_module.c

@@ -56,7 +56,8 @@ static void mca_coll_hcoll_module_destruct(mca_coll_hcoll_module_t *hcoll_module
     OBJ_RELEASE(hcoll_module->previous_alltoallv_module);
     OBJ_RELEASE(hcoll_module->previous_alltoallw_module);
     OBJ_RELEASE(hcoll_module->previous_reduce_scatter_module);
-    hcoll_destroy_context(hcoll_module->hcoll_context);
+    hcoll_destroy_context(hcoll_module->hcoll_context,
+                          (rte_grp_handle_t) hcoll_module->comm);
     mca_coll_hcoll_module_clear(hcoll_module);
 }
 
@@ -106,7 +107,7 @@ static int mca_coll_hcoll_module_enable(mca_coll_base_module_t *module,
 
     hcoll_set_runtime_tag_offset(-100,mca_pml.pml_max_tag);
 
-    hcoll_module->hcoll_context = 
+    hcoll_module->hcoll_context =
             hcoll_create_context((rte_grp_handle_t)comm);
     if (NULL == hcoll_module->hcoll_context){
         HCOL_VERBOSE(1,"hcoll_create_context returned NULL");
@@ -153,7 +154,6 @@ static int mca_coll_hcoll_module_enable(mca_coll_base_module_t *module,
         sleep(1);
     }
 #endif
-    hcoll_module->super.coll_barrier(comm,module);
     return OMPI_SUCCESS;
 }
 
@@ -171,7 +171,7 @@ mca_coll_hcoll_comm_query(struct ompi_communicator_t *comm, int *priority)
 {
     mca_coll_base_module_t *module;
     mca_coll_hcoll_module_t *hcoll_module;
-
+    static bool libhcoll_initialized = false;
     *priority = 0;
     module = NULL;
 
@@ -179,6 +179,24 @@ mca_coll_hcoll_comm_query(struct ompi_communicator_t *comm, int *priority)
         goto exit;
     }
 
+    if (!libhcoll_initialized)
+    {
+        /* libhcoll should be initialized here since current implmentation of
+           mxm bcol in libhcoll needs world_group fully functional during init
+
+           world_group, i.e. ompi_comm_world, is not ready at hcoll component open
+           call */
+        opal_progress_register(hcoll_progress_fn);
+        int rc = hcoll_init();
+
+        if (HCOLL_SUCCESS != rc){
+            mca_coll_hcoll_component.hcoll_enable = 0;
+            opal_progress_unregister(hcoll_progress_fn);
+            HCOL_VERBOSE(0,"Hcol library init failed");
+            return NULL;
+        }
+        libhcoll_initialized = true;
+    }
     hcoll_module = OBJ_NEW(mca_coll_hcoll_module_t);
     if (!hcoll_module){
         goto exit;

ompi/mca/coll/hcoll/coll_hcoll_rte.c

@@ -92,6 +92,9 @@ static void* get_coll_handle(void);
 static int coll_handle_test(void* handle);
 static void coll_handle_free(void *handle);
 static void coll_handle_complete(void *handle);
+static int group_id(rte_grp_handle_t group);
+
+static int world_rank(rte_grp_handle_t grp_h, rte_ec_handle_t ec);
 /* Module Constructors */
 
 static void init_module_fns(void){
@@ -110,6 +113,8 @@ static void init_module_fns(void){
     hcoll_rte_functions.rte_coll_handle_test_fn = coll_handle_test;
     hcoll_rte_functions.rte_coll_handle_free_fn = coll_handle_free;
     hcoll_rte_functions.rte_coll_handle_complete_fn = coll_handle_complete;
+    hcoll_rte_functions.rte_group_id_fn = group_id;
+    hcoll_rte_functions.rte_world_rank_fn = world_rank;
 }
 
 
@@ -165,7 +170,7 @@ static int recv_nb(struct dte_data_representation_t data,
 #if RTE_DEBUG
     assert(ec_h.group == grp_h);
 #endif
-    if (! ec_h.handle) {
+    if (NULL == ec_h.handle && -1 != ec_h.rank) {
         fprintf(stderr,"***Error in hcolrte_rml_recv_nb: wrong null argument: "
                 "ec_h.handle = %p, ec_h.rank = %d\n",ec_h.handle,ec_h.rank);
         return 1;
@@ -381,6 +386,9 @@ static uint32_t jobid(void){
     return ORTE_PROC_MY_NAME->jobid;
 }
 
+static int group_id(rte_grp_handle_t group){
+    return ((ompi_communicator_t *)group)->c_contextid;
+}
 
 static void* get_coll_handle(void)
 {
@@ -393,6 +401,8 @@ static void* get_coll_handle(void)
     }
     ompi_req = (ompi_request_t *)item;
     OMPI_REQUEST_INIT(ompi_req,false);
+    ompi_req->req_complete_cb = NULL;
+    ompi_req->req_status.MPI_ERROR = MPI_SUCCESS;
     return (void *)ompi_req;
 }
 
@@ -415,3 +425,7 @@ static void coll_handle_complete(void *handle)
 }
 
 
+static int world_rank(rte_grp_handle_t grp_h, rte_ec_handle_t ec){
+    ompi_proc_t *proc = (ompi_proc_t *)ec.handle;
+    return proc->proc_name.vpid;
+}