Continue bringing comm_spawn back online. Ensure all RM frameworks post their HNP receives. Fix the rmgr proxy component.

Still need some work on the proxy component, and on job termination for persistent daemon case.

This commit was SVN r11928.

orte/mca/pls/base/pls_base_close.c

@@ -23,15 +23,29 @@
 #include "orte/orte_constants.h"
 #include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
-#include "orte/mca/pls/base/base.h"
 #include "opal/util/output.h"
 
+#include "orte/util/proc_info.h"
+#include "orte/mca/errmgr/errmgr.h"
+
+#include "orte/mca/pls/base/pls_private.h"
+#include "orte/mca/pls/base/base.h"
 
 int orte_pls_base_finalize(void)
 {
+    int rc;
+    
     /* Finalize the selected module */
     orte_pls.finalize();
 
+    /* if we are an HNP, then stop our receive */
+    if (orte_process_info.seed) {
+        if (ORTE_SUCCESS != (rc = orte_pls_base_comm_stop())) {
+            ORTE_ERROR_LOG(rc);
+            return rc;
+        }
+    }
+    
     return ORTE_SUCCESS;
 }
 

orte/mca/pls/base/pls_base_select.c

@@ -24,6 +24,11 @@
 #include "opal/util/show_help.h"
 #include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
+
+#include "orte/util/proc_info.h"
+#include "orte/mca/errmgr/errmgr.h"
+
+#include "orte/mca/pls/base/pls_private.h"
 #include "orte/mca/pls/base/base.h"
 
 
@@ -39,6 +44,7 @@ int orte_pls_base_select(void)
     orte_pls_base_component_t *component, *best_component = NULL;
     orte_pls_base_module_t *module, *best_module = NULL;
     int priority, best_priority = -1;
+    int rc;
 
     /* Query all the opened components and see if they want to run */
 
@@ -101,5 +107,13 @@ int orte_pls_base_select(void)
     orte_pls_base.selected_component = *best_component;
     orte_pls_base.selected = true;
 
+    /* if we are an HNP, then start our receive */
+    if (orte_process_info.seed) {
+        if (ORTE_SUCCESS != (rc = orte_pls_base_comm_start())) {
+            ORTE_ERROR_LOG(rc);
+            return rc;
+        }
+    }
+    
     return ORTE_SUCCESS;
 }

orte/mca/pls/gridengine/pls_gridengine_component.c

@@ -141,8 +141,6 @@ orte_pls_gridengine_component_init - initialize component, check if we can run o
 */
 orte_pls_base_module_t *orte_pls_gridengine_component_init(int *priority)
 {
-    int rc;
-    
     /* if we are not an HNP, then don't select us */
     if (!orte_process_info.seed) {
         return NULL;
@@ -153,11 +151,6 @@ orte_pls_base_module_t *orte_pls_gridengine_component_init(int *priority)
         opal_output_verbose(10, orte_pls_base.pls_output,
             "pls:gridengine: available for selection");
 
-        /* ensure the receive gets posted */
-        if (ORTE_SUCCESS != (rc = orte_pls_base_comm_start())) {
-            ORTE_ERROR_LOG(rc);
-        }
-        
         *priority = mca_pls_gridengine_component.priority;
         return &orte_pls_gridengine_module;
     }

orte/mca/pls/rsh/pls_rsh_component.c

@@ -179,7 +179,6 @@ orte_pls_base_module_t *orte_pls_rsh_component_init(int *priority)
 {
     char *bname;
     size_t i;
-    int rc;
 
     /* if we are not an HNP, then don't select us */
     if (!orte_process_info.seed) {
@@ -230,11 +229,6 @@ orte_pls_base_module_t *orte_pls_rsh_component_init(int *priority)
     }
     *priority = mca_pls_rsh_component.priority;
     
-    /* ensure the receive gets posted */
-    if (ORTE_SUCCESS != (rc = orte_pls_base_comm_start())) {
-        ORTE_ERROR_LOG(rc);
-    }
-    
     return &orte_pls_rsh_module;
 }
 

orte/mca/pls/slurm/pls_slurm_component.c

@@ -143,11 +143,6 @@ static orte_pls_base_module_t *pls_slurm_init(int *priority)
             opal_output(0, "pls:slurm: available for selection");
         }
 
-        /* ensure the receive gets posted */
-        if (ORTE_SUCCESS != (rc = orte_pls_base_comm_start())) {
-            ORTE_ERROR_LOG(rc);
-        }
-        
         return &orte_pls_slurm_module;
     }
 

orte/mca/ras/base/ras_base_close.c

@@ -24,12 +24,18 @@
 
 #include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
+
+#include "orte/util/proc_info.h"
+#include "orte/mca/errmgr/errmgr.h"
+
+#include "orte/mca/ras/base/ras_private.h"
 #include "orte/mca/ras/base/base.h"
 
 
 int orte_ras_base_finalize(void)
 {
     opal_list_item_t* item;
+    int rc;
 
     if (orte_ras_base.ras_available_valid) {
         /* Finalize all available modules */
@@ -40,6 +46,13 @@ int orte_ras_base_finalize(void)
             OBJ_RELEASE(cmp);
         }
         OBJ_DESTRUCT(&orte_ras_base.ras_available);
+
+        /* if we are an HNP, stop the receive */
+        if (orte_process_info.seed) {
+            if (ORTE_SUCCESS != (rc = orte_ras_base_comm_stop())) {
+                ORTE_ERROR_LOG(rc);
+            }
+        }
     }
 
     return ORTE_SUCCESS;

orte/mca/ras/base/ras_base_find_available.c

@@ -24,6 +24,11 @@
 #include "opal/mca/base/base.h"
 #include "opal/util/output.h"
 
+#include "orte/util/proc_info.h"
+
+#include "orte/mca/errmgr/errmgr.h"
+
+#include "orte/mca/ras/base/ras_private.h"
 #include "orte/mca/ras/base/base.h"
 
 
@@ -53,7 +58,7 @@ int orte_ras_base_find_available(void)
     mca_base_component_list_item_t *cli;
     orte_ras_base_component_t *component;
     orte_ras_base_module_t *module;
-    int priority;
+    int priority, rc;
     orte_ras_base_cmp_t *cmp;
 
     orte_ras_base.ras_available_valid = false;
@@ -100,6 +105,14 @@ int orte_ras_base_find_available(void)
 
         /* Sort the resulting available list in priority order */
         opal_list_sort(&orte_ras_base.ras_available, compare);
+
+        /* if we are an HNP, start the receive */
+        if (orte_process_info.seed) {
+            if (ORTE_SUCCESS  != (rc = orte_ras_base_comm_start())) {
+                ORTE_ERROR_LOG(rc);
+                return rc;
+            }
+        }
     }
     
     return ORTE_SUCCESS;

orte/mca/ras/base/ras_base_receive.c

@@ -62,6 +62,22 @@ int orte_ras_base_comm_start(void)
 }
 
 
+int orte_ras_base_comm_stop(void)
+{
+    int rc;
+
+    if (!recv_issued) {
+        return ORTE_SUCCESS;
+    }
+
+    if (ORTE_SUCCESS != (rc = orte_rml.recv_cancel(ORTE_RML_NAME_ANY, ORTE_RML_TAG_RAS))) {
+        ORTE_ERROR_LOG(rc);
+    }
+    recv_issued = false;
+
+    return rc;
+}
+
 
 /*
  * handle message from proxies

orte/mca/ras/base/ras_private.h

@@ -116,7 +116,7 @@ int orte_ras_base_node_segment_empty(bool *empty);
  * oob interface
  */
 int orte_ras_base_comm_start(void);
-
+int orte_ras_base_comm_stop(void);
 void orte_ras_base_recv(int status, orte_process_name_t* sender,
                         orte_buffer_t* buffer, orte_rml_tag_t tag, void* cbdata);
 

orte/mca/ras/dash_host/ras_dash_host_module.c

@@ -192,7 +192,7 @@ cleanup:
     for (i = 0; i < num_context; i++) {
         OBJ_RELEASE(context[i]);
     }
-    free(context);
+    if (NULL != context) free(context);
     return rc;
 }
 

orte/mca/rds/base/rds_base_close.c

@@ -24,12 +24,18 @@
 
 #include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
+
+#include "orte/util/proc_info.h"
+#include "orte/mca/errmgr/errmgr.h"
+
+#include "orte/mca/rds/base/rds_private.h"
 #include "orte/mca/rds/base/base.h"
 
 
 int orte_rds_base_finalize(void)
 {
     opal_list_item_t* item;
+    int rc;
 
     /* if we are using the "null" component, then do nothing */
     if (orte_rds_base.no_op_selected) {
@@ -42,6 +48,16 @@ int orte_rds_base_finalize(void)
         selected->component->rds_fini();
         OBJ_RELEASE(selected);
     }
+
+    /* if we are an HNP, then cancel our receive */
+    if (orte_process_info.seed) {
+        if (ORTE_SUCCESS != (rc = orte_rds_base_comm_stop())) {
+            ORTE_ERROR_LOG(rc);
+            return NULL;
+        }
+    }
+
+    
     return ORTE_SUCCESS;
 }
     

orte/mca/rds/base/rds_base_receive.c

@@ -62,6 +62,23 @@ int orte_rds_base_comm_start(void)
 }
 
 
+int orte_rds_base_comm_stop(void)
+{
+    int rc;
+
+    if (!recv_issued) {
+        return ORTE_SUCCESS;
+    }
+
+    if (ORTE_SUCCESS != (rc = orte_rml.recv_cancel(ORTE_RML_NAME_ANY, ORTE_RML_TAG_RDS))) {
+        ORTE_ERROR_LOG(rc);
+    }
+    recv_issued = false;
+
+    return rc;
+}
+
+
 
 /*
  * handle message from proxies

orte/mca/rds/base/rds_base_select.c

@@ -24,6 +24,11 @@
 #include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
 #include "opal/util/output.h"
+
+#include "orte/util/proc_info.h"
+#include "orte/mca/errmgr/errmgr.h"
+
+#include "orte/mca/rds/base/rds_private.h"
 #include "orte/mca/rds/base/base.h"
 
 
@@ -40,6 +45,7 @@ int orte_rds_base_select(void)
     mca_base_component_list_item_t *cli;
     orte_rds_base_component_t *component;
     orte_rds_base_module_t *module = NULL;
+    int rc;
 
     /* if we are using the "null" component, then do nothing */
     if (orte_rds_base.no_op_selected) {
@@ -76,6 +82,14 @@ int orte_rds_base_select(void)
         return ORTE_ERROR;
     }
 
+    /* if we are an HNP, issue non-blocking receive for call_back function */
+    if (orte_process_info.seed) {
+        if (ORTE_SUCCESS != (rc = orte_rds_base_comm_start())) {
+            ORTE_ERROR_LOG(rc);
+            return rc;
+        }
+    }
+    
     return ORTE_SUCCESS;
 }
 

orte/mca/rds/base/rds_private.h

@@ -52,6 +52,7 @@ ORTE_DECLSPEC int orte_rds_base_query(void);
  * oob interface
  */
 int orte_rds_base_comm_start(void);
+int orte_rds_base_comm_stop(void);
 
 void orte_rds_base_recv(int status, orte_process_name_t* sender,
                         orte_buffer_t* buffer, orte_rml_tag_t tag, void* cbdata);

orte/mca/rds/hostfile/rds_hostfile_component.c

@@ -97,19 +97,11 @@ static int orte_rds_hostfile_open(void)
 
 static orte_rds_base_module_t *orte_rds_hostfile_init(void)
 {
-    int rc;
-    
     /* if we are NOT an HNP, then don't select us */
     if (!orte_process_info.seed) {
         return NULL;
     }
 
-    /* issue non-blocking receive for call_back function */
-    if (ORTE_SUCCESS != (rc = orte_rds_base_comm_start())) {
-        ORTE_ERROR_LOG(rc);
-        return NULL;
-    }
-    
     return &orte_rds_hostfile_module;
 }
 

orte/mca/rds/resfile/rds_resfile_component.c

@@ -103,8 +103,6 @@ static int orte_rds_resfile_open(void)
 
 static orte_rds_base_module_t *orte_rds_resfile_init(void)
 {
-    int rc;
-    
     /* if we are not an HNP, then don't select us */
     if (!orte_process_info.seed) {
         return NULL;
@@ -113,12 +111,6 @@ static orte_rds_base_module_t *orte_rds_resfile_init(void)
     /* if we are an HNP, then volunteer */
     OBJ_DESTRUCT(&mca_rds_resfile_component.lock);
 
-    /* issue non-blocking receive for call_back function */
-    if (ORTE_SUCCESS != (rc = orte_rds_base_comm_start())) {
-        ORTE_ERROR_LOG(rc);
-        return NULL;
-    }
-
     return &orte_rds_resfile_module;
 }
 

orte/mca/rmaps/base/rmaps_base_close.c

@@ -25,6 +25,9 @@
 #include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
 
+#include "orte/util/proc_info.h"
+#include "orte/mca/errmgr/errmgr.h"
+
 #include "orte/mca/rmaps/base/rmaps_private.h"
 #include "orte/mca/rmaps/base/base.h"
 
@@ -32,6 +35,7 @@
 int orte_rmaps_base_finalize(void)
 {
     opal_list_item_t* item;
+    int rc;
 
     if (!orte_rmaps_base.no_op_selected) {
         /* Finalize all available modules */
@@ -49,6 +53,14 @@ int orte_rmaps_base_finalize(void)
         }
     }
 
+    /* if we are an HNP, then cancel our receive */
+    if (orte_process_info.seed) {
+        if (ORTE_SUCCESS != (rc = orte_rmaps_base_comm_stop())) {
+            return rc;
+        }
+    }
+
+    
     return ORTE_SUCCESS;
 }
 

orte/mca/rmaps/base/rmaps_base_find_avail.c

@@ -25,6 +25,9 @@
 #include "opal/mca/mca.h"
 #include "opal/mca/base/base.h"
 
+#include "orte/util/proc_info.h"
+#include "orte/mca/errmgr/errmgr.h"
+
 #include "orte/mca/rmaps/base/rmaps_private.h"
 #include "orte/mca/rmaps/base/base.h"
 
@@ -51,7 +54,7 @@ int orte_rmaps_base_find_available(void)
     orte_rmaps_base_component_t *component;
     orte_rmaps_base_module_t *module;
     orte_rmaps_base_cmp_t *cmp;
-    int priority;
+    int priority, rc;
     
     /* construct the list to hold any available components */
     OBJ_CONSTRUCT(&orte_rmaps_base.rmaps_available, opal_list_t);
@@ -96,6 +99,13 @@ int orte_rmaps_base_find_available(void)
     /* Sort the resulting available list in priority order */
     opal_list_sort(&orte_rmaps_base.rmaps_available, compare);
 
+    /* if we are an HNP, start the receive function */
+    if (orte_process_info.seed) {
+        if (ORTE_SUCCESS != (rc = orte_rmaps_base_comm_start())) {
+            return rc;
+        }
+    }
+
     /* all done */
     return ORTE_SUCCESS;
 }

orte/mca/rmaps/round_robin/rmaps_rr_component.c

@@ -102,18 +102,11 @@ static int orte_rmaps_round_robin_open(void)
 static orte_rmaps_base_module_t* 
 orte_rmaps_round_robin_init(int *priority)
 {    
-    int rc;
-    
     /* if I am NOT an HNP, then don't consider me! */
     if (!orte_process_info.seed) {
         return NULL;
     }
     
-    /* start the receive function */
-    if (ORTE_SUCCESS != (rc = orte_rmaps_base_comm_start())) {
-        return NULL;
-    }
-    
     *priority = mca_rmaps_round_robin_component.priority;
     return &orte_rmaps_round_robin_module;
 }

orte/mca/rmgr/base/rmgr_base_open.c

@@ -115,8 +115,11 @@ static void orte_app_context_destructor(orte_app_context_t* app_context)
                 OBJ_RELEASE(app_context->map_data[i]);
             }
         }
+        if (NULL != app_context->map_data) {
             free(app_context->map_data);
         }
+    }
+    
     if (NULL != app_context->prefix_dir) {
         free(app_context->prefix_dir);
     }

orte/mca/rmgr/base/rmgr_base_receive.c

@@ -178,6 +178,8 @@ void orte_rmgr_base_recv(int status, orte_process_name_t* sender,
             }
                 
             /* process the request */
+            /* init the job to be INVALID so we setup the job */
+            job = ORTE_JOBID_INVALID;
             if (ORTE_SUCCESS != (rc = orte_rmgr.spawn_job(context, num_context, &job,
                                                           0, NULL, NULL, ORTE_PROC_STATE_NONE))) {
                 ORTE_ERROR_LOG(rc);

orte/mca/rmgr/urm/rmgr_urm.c

@@ -300,11 +300,18 @@ static int orte_rmgr_urm_spawn_job(
     /*
      * Initialize job segment and allocate resources
      */ /* JJH Insert C/N mapping stuff here */
+     
+    /* If the jobid = ORTE_JOBID_INVALID, then we need to
+     * get one assigned to us. Otherwise, we are entering
+     * with a valid jobid, so no need to get one
+     */
+    if (ORTE_JOBID_INVALID == *jobid) { /* setup the job */
         if (ORTE_SUCCESS !=
             (rc = orte_rmgr_urm_setup_job(app_context,num_context,jobid))) {
             ORTE_ERROR_LOG(rc);
             return rc;
         }
+    }
 
     if (ORTE_SUCCESS != (rc = orte_ras.allocate_job(*jobid))) {
         ORTE_ERROR_LOG(rc);

orte/tools/orterun/orterun.c

@@ -84,7 +84,7 @@ static struct opal_event int_handler;
 static struct opal_event sigusr1_handler;
 static struct opal_event sigusr2_handler;
 #endif  /* __WINDOWS__ */
-static orte_jobid_t jobid = ORTE_JOBID_MAX;
+static orte_jobid_t jobid = ORTE_JOBID_INVALID;
 static orte_pointer_array_t *apps_pa;
 static bool wait_for_job_completion = true;
 static char *orterun_basename = NULL;
@@ -755,10 +755,10 @@ static void abort_signal_callback(int fd, short flags, void *arg)
     /* terminate the job - this will also wakeup orterun so
      * it can kill all the orteds
      */
-    if (jobid != ORTE_JOBID_MAX) {
+    if (jobid != ORTE_JOBID_INVALID) {
         ret = orte_pls.terminate_job(jobid);
         if (ORTE_SUCCESS != ret) {
-            jobid = ORTE_JOBID_MAX;
+            jobid = ORTE_JOBID_INVALID;
         }
     }