Add a set of default errmgr components that support solely the default "everything dies on error" behavior. Set their priority to be selected by default, but provide params to adjust those priorities to allow other component selection.

This commit was SVN r25139.

orte/mca/errmgr/app/errmgr_app_component.c

@@ -59,7 +59,7 @@ orte_errmgr_base_component_t mca_errmgr_app_component =
     /* opal_output handler */
     -1,
     /* Default priority */
-    0
+    5
 };
 
 static int errmgr_app_open(void) 
@@ -78,7 +78,7 @@ static int errmgr_app_component_query(mca_base_module_t **module, int *priority)
         /* keep our priority low so that other modules are higher
          * and will run before us
          */
-        *priority = 0;
+        *priority = 5;
         *module = (mca_base_module_t *)&orte_errmgr_app_module;
         return ORTE_SUCCESS;        
     }

orte/mca/errmgr/default_app/.windows

@@ -0,0 +1,12 @@
+#
+# Copyright (c) 2008-2010 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+# Specific to this module
+mca_link_libraries=libopen-rte

orte/mca/errmgr/default_app/Makefile.am

@@ -0,0 +1,36 @@
+#
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved. 
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+EXTRA_DIST = .windows
+
+sources = \
+        errmgr_default_app.h \
+        errmgr_default_app_component.c \
+        errmgr_default_app.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_orte_errmgr_default_app_DSO
+component_noinst =
+component_install = mca_errmgr_default_app.la
+else
+component_noinst = libmca_errmgr_default_app.la
+component_install =
+endif
+
+mcacomponentdir = $(pkglibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_errmgr_default_app_la_SOURCES = $(sources)
+mca_errmgr_default_app_la_LDFLAGS = -module -avoid-version
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_errmgr_default_app_la_SOURCES =$(sources)
+libmca_errmgr_default_app_la_LDFLAGS = -module -avoid-version

orte/mca/errmgr/default_app/errmgr_default_app.c

@@ -0,0 +1,137 @@
+/*
+ * Copyright (c) 2009-2011 The Trustees of Indiana University.
+ *                         All rights reserved.
+ *
+ * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved. 
+ *
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ *    
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "orte_config.h"
+
+#include <sys/types.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif  /* HAVE_UNISTD_H */
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+
+#include "opal/util/output.h"
+#include "opal/dss/dss.h"
+
+#include "orte/util/error_strings.h"
+#include "orte/util/name_fns.h"
+#include "orte/runtime/orte_globals.h"
+#include "orte/mca/rml/rml.h"
+#include "orte/mca/routed/routed.h"
+#include "orte/mca/odls/odls_types.h"
+
+#include "orte/mca/errmgr/base/base.h"
+#include "orte/mca/errmgr/base/errmgr_private.h"
+#include "errmgr_default_app.h"
+
+/*
+ * Module functions: Global
+ */
+static int init(void);
+static int finalize(void);
+
+static int update_state(orte_jobid_t job,
+                        orte_job_state_t jobstate,
+                        orte_process_name_t *proc_name,
+                        orte_proc_state_t state,
+                        pid_t pid,
+                        orte_exit_code_t exit_code);
+
+static int abort_peers(orte_process_name_t *procs,
+                       orte_std_cntr_t num_procs);
+
+/******************
+ * HNP module
+ ******************/
+orte_errmgr_base_module_t orte_errmgr_default_app_module = {
+    init,
+    finalize,
+    orte_errmgr_base_log,
+    orte_errmgr_base_abort,
+    abort_peers,
+    update_state,
+    NULL,
+    NULL,
+    NULL,
+    orte_errmgr_base_register_migration_warning,
+    NULL
+};
+
+/************************
+ * API Definitions
+ ************************/
+static int init(void)
+{
+    return ORTE_SUCCESS;
+}
+
+static int finalize(void)
+{
+    return ORTE_SUCCESS;
+}
+
+static int update_state(orte_jobid_t job,
+                        orte_job_state_t jobstate,
+                        orte_process_name_t *proc,
+                        orte_proc_state_t state,
+                        pid_t pid,
+                        orte_exit_code_t exit_code)
+{
+    orte_ns_cmp_bitmask_t mask;
+
+    OPAL_OUTPUT_VERBOSE((1, orte_errmgr_base.output,
+                         "%s errmgr:default_app: job %s reported state %s"
+                         " for proc %s state %s exit_code %d",
+                         ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
+                         ORTE_JOBID_PRINT(job),
+                         orte_job_state_to_str(jobstate),
+                         (NULL == proc) ? "NULL" : ORTE_NAME_PRINT(proc),
+                         orte_proc_state_to_str(state), exit_code));
+    
+    /*
+     * if orte is trying to shutdown, just let it
+     */
+    if (orte_finalizing) {
+        return ORTE_SUCCESS;
+    }
+
+    if (ORTE_PROC_STATE_COMM_FAILED == state) {
+        mask = ORTE_NS_CMP_ALL;
+        /* if it is our own connection, ignore it */
+        if (OPAL_EQUAL == orte_util_compare_name_fields(mask, ORTE_PROC_MY_NAME, proc)) {
+            return ORTE_SUCCESS;
+        }
+        /* see is this was a lifeline */
+        if (ORTE_SUCCESS != orte_routed.route_lost(proc)) {
+            return ORTE_ERR_UNRECOVERABLE;
+        }
+    }
+    return ORTE_SUCCESS;
+}
+
+static int abort_peers(orte_process_name_t *procs, orte_std_cntr_t num_procs)
+{
+    /* just abort */
+    if (0 < opal_output_get_verbosity(orte_errmgr_base.output)) {
+        orte_errmgr_base_abort(ORTE_ERROR_DEFAULT_EXIT_CODE, "%s called abort_peers",
+                               ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
+    } else {
+        orte_errmgr_base_abort(ORTE_ERROR_DEFAULT_EXIT_CODE, NULL);
+    }
+    return ORTE_SUCCESS;
+}

orte/mca/errmgr/default_app/errmgr_default_app.h

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2010      Cisco Systems, Inc. All rights reserved.
+ *
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+/**
+ * @file
+ * 
+ */
+
+#ifndef MCA_ERRMGR_default_app_EXPORT_H
+#define MCA_ERRMGR_default_app_EXPORT_H
+
+#include "orte_config.h"
+
+#include "orte/mca/errmgr/errmgr.h"
+
+BEGIN_C_DECLS
+
+/*
+ * Local Component structures
+ */
+
+ORTE_MODULE_DECLSPEC extern orte_errmgr_base_component_t mca_errmgr_default_app_component;
+
+ORTE_DECLSPEC extern orte_errmgr_base_module_t orte_errmgr_default_app_module;
+
+END_C_DECLS
+
+#endif /* MCA_ERRMGR_app_EXPORT_H */

orte/mca/errmgr/default_app/errmgr_default_app_component.c

@@ -0,0 +1,90 @@
+/*
+ * Copyright (c) 2010      Cisco Systems, Inc. All rights reserved.
+ *
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "orte_config.h"
+#include "opal/util/output.h"
+
+#include "orte/mca/errmgr/errmgr.h"
+#include "orte/mca/errmgr/base/base.h"
+#include "errmgr_default_app.h"
+
+/*
+ * Public string for version number
+ */
+const char *orte_errmgr_default_app_component_version_string = 
+    "ORTE ERRMGR default_app MCA component version " ORTE_VERSION;
+
+/*
+ * Local functionality
+ */
+static int errmgr_default_app_open(void);
+static int errmgr_default_app_close(void);
+static int errmgr_default_app_component_query(mca_base_module_t **module, int *priority);
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointer to our public functions in it
+ */
+orte_errmgr_base_component_t mca_errmgr_default_app_component =
+{
+    /* Handle the general mca_component_t struct containing 
+     *  meta information about the component
+     */
+    {
+        ORTE_ERRMGR_BASE_VERSION_3_0_0,
+        /* Component name and version */
+        "default_app",
+        ORTE_MAJOR_VERSION,
+        ORTE_MINOR_VERSION,
+        ORTE_RELEASE_VERSION,
+        
+        /* Component open and close functions */
+        errmgr_default_app_open,
+        errmgr_default_app_close,
+        errmgr_default_app_component_query
+    },
+    {
+        /* The component is checkpoint ready */
+        MCA_BASE_METADATA_PARAM_CHECKPOINT
+    },
+};
+
+static int my_priority;
+
+static int errmgr_default_app_open(void) 
+{
+    mca_base_component_t *c = &mca_errmgr_default_app_component.base_version;
+
+    mca_base_param_reg_int(c, "priority",
+                           "Priority of the default_app errmgr component",
+                           false, false, 1000,
+                           &my_priority);
+
+    return ORTE_SUCCESS;
+}
+
+static int errmgr_default_app_close(void)
+{
+    return ORTE_SUCCESS;
+}
+
+static int errmgr_default_app_component_query(mca_base_module_t **module, int *priority)
+{
+    if (ORTE_PROC_IS_APP) {
+        /* set our priority high as we are the default for apps */
+        *priority = my_priority;
+        *module = (mca_base_module_t *)&orte_errmgr_default_app_module;
+        return ORTE_SUCCESS;        
+    }
+    
+    *priority = -1;
+    *module = NULL;
+    return ORTE_ERROR;
+}

orte/mca/errmgr/default_hnp/.windows

@@ -0,0 +1,12 @@
+#
+# Copyright (c) 2008-2010 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+# Specific to this module
+mca_link_libraries=libopen-rte

orte/mca/errmgr/default_hnp/Makefile.am

@@ -0,0 +1,36 @@
+#
+# Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved. 
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+EXTRA_DIST = .windows
+
+sources = \
+        errmgr_default_hnp.h \
+        errmgr_default_hnp_component.c \
+        errmgr_default_hnp.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_orte_errmgr_default_hnp_DSO
+component_noinst =
+component_install = mca_errmgr_default_hnp.la
+else
+component_noinst = libmca_errmgr_default_hnp.la
+component_install =
+endif
+
+mcacomponentdir = $(pkglibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_errmgr_default_hnp_la_SOURCES = $(sources)
+mca_errmgr_default_hnp_la_LDFLAGS = -module -avoid-version
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_errmgr_default_hnp_la_SOURCES =$(sources)
+libmca_errmgr_default_hnp_la_LDFLAGS = -module -avoid-version

orte/mca/errmgr/default_hnp/errmgr_default_hnp.h

@@ -0,0 +1,38 @@
+/*
+ * Copyright (c) 2010      Cisco Systems, Inc. All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ *
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+/**
+ * @file
+ * 
+ */
+
+#ifndef MCA_ERRMGR_default_hnp_EXPORT_H
+#define MCA_ERRMGR_default_hnp_EXPORT_H
+
+#include "orte_config.h"
+
+#include "orte/mca/errmgr/errmgr.h"
+
+BEGIN_C_DECLS
+
+/*
+ * Local Component structures
+ */
+
+ORTE_MODULE_DECLSPEC extern orte_errmgr_base_component_t mca_errmgr_default_hnp_component;
+
+ORTE_DECLSPEC extern orte_errmgr_base_module_t orte_errmgr_default_hnp_module;
+
+END_C_DECLS
+
+#endif /* MCA_ERRMGR_default_hnp_EXPORT_H */

orte/mca/errmgr/default_hnp/errmgr_default_hnp_component.c

@@ -0,0 +1,90 @@
+/*
+ * Copyright (c) 2010      Cisco Systems, Inc. All rights reserved.
+ *
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "orte_config.h"
+#include "opal/util/output.h"
+
+#include "orte/mca/errmgr/errmgr.h"
+#include "orte/mca/errmgr/base/base.h"
+#include "orte/mca/errmgr/base/errmgr_private.h"
+#include "errmgr_default_hnp.h"
+
+/*
+ * Public string for version number
+ */
+const char *orte_errmgr_default_hnp_component_version_string = 
+    "ORTE ERRMGR default_hnp MCA component version " ORTE_VERSION;
+
+/*
+ * Local functionality
+ */
+static int default_hnp_open(void);
+static int default_hnp_close(void);
+static int default_hnp_component_query(mca_base_module_t **module, int *priority);
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointer to our public functions in it
+ */
+orte_errmgr_base_component_t mca_errmgr_default_hnp_component = {
+    /* Handle the general mca_component_t struct containing 
+     *  meta information about the component default_hnp
+     */
+    {
+        ORTE_ERRMGR_BASE_VERSION_3_0_0,
+        /* Component name and version */
+        "default_hnp",
+        ORTE_MAJOR_VERSION,
+        ORTE_MINOR_VERSION,
+        ORTE_RELEASE_VERSION,
+        
+        /* Component open and close functions */
+        default_hnp_open,
+        default_hnp_close,
+        default_hnp_component_query
+    },
+    {
+        /* The component is checkpoint ready */
+        MCA_BASE_METADATA_PARAM_CHECKPOINT
+    },
+};
+
+static int my_priority;
+
+static int default_hnp_open(void) 
+{
+    mca_base_component_t *c = &mca_errmgr_default_hnp_component.base_version;
+
+    mca_base_param_reg_int(c, "priority",
+                           "Priority of the default_hnp errmgr component",
+                           false, false, 1000,
+                           &my_priority);
+
+    return ORTE_SUCCESS;
+}
+
+static int default_hnp_close(void)
+{
+    return ORTE_SUCCESS;
+}
+
+static int default_hnp_component_query(mca_base_module_t **module, int *priority)
+{
+    if( ORTE_PROC_IS_HNP ) {
+        /* we are the default HNP component */
+        *priority = my_priority;
+        *module = (mca_base_module_t *)&orte_errmgr_default_hnp_module;
+        return ORTE_SUCCESS;
+    }
+
+    *module = NULL;
+    *priority = -1;
+    return ORTE_ERROR;
+}

orte/mca/errmgr/default_orted/.windows

@@ -0,0 +1,12 @@
+#
+# Copyright (c) 2008-2010 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+# Specific to this module
+mca_link_libraries=libopen-rte

orte/mca/errmgr/default_orted/Makefile.am

@@ -0,0 +1,36 @@
+#
+# Copyright (c) 2010-2011 Cisco Systems, Inc.  All rights reserved. 
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+EXTRA_DIST = .windows
+
+sources = \
+        errmgr_default_orted.h \
+        errmgr_default_orted_component.c \
+        errmgr_default_orted.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_orte_errmgr_default_orted_DSO
+component_noinst =
+component_install = mca_errmgr_default_orted.la
+else
+component_noinst = libmca_errmgr_default_orted.la
+component_install =
+endif
+
+mcacomponentdir = $(pkglibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_errmgr_default_orted_la_SOURCES = $(sources)
+mca_errmgr_default_orted_la_LDFLAGS = -module -avoid-version
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_errmgr_default_orted_la_SOURCES =$(sources)
+libmca_errmgr_default_orted_la_LDFLAGS = -module -avoid-version

orte/mca/errmgr/default_orted/errmgr_default_orted.c

@@ -0,0 +1,845 @@
+/*
+ * Copyright (c) 2009-2010 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2010      Cisco Systems, Inc.  All rights reserved. 
+ * Copyright (c) 2010-2011 Oak Ridge National Labs.  All rights reserved.
+ * Copyright (c) 2004-2011 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "orte_config.h"
+
+#include <sys/types.h>
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif  /* HAVE_UNISTD_H */
+#ifdef HAVE_STRING_H
+#include <string.h>
+#endif
+
+#include "opal/util/output.h"
+#include "opal/util/opal_sos.h"
+#include "opal/dss/dss.h"
+
+#include "orte/util/error_strings.h"
+#include "orte/util/name_fns.h"
+#include "orte/util/proc_info.h"
+#include "orte/util/session_dir.h"
+#include "orte/util/show_help.h"
+#include "orte/util/nidmap.h"
+#include "orte/runtime/orte_globals.h"
+#include "orte/runtime/data_type_support/orte_dt_support.h"
+#include "orte/mca/rml/rml.h"
+#include "orte/mca/odls/odls.h"
+#include "orte/mca/odls/base/base.h"
+#include "orte/mca/odls/base/odls_private.h"
+#include "orte/mca/plm/plm_types.h"
+#include "orte/mca/routed/routed.h"
+#include "orte/mca/sensor/sensor.h"
+#include "orte/mca/ess/ess.h"
+#include "orte/runtime/orte_quit.h"
+#include "orte/runtime/orte_globals.h"
+
+#include "orte/mca/errmgr/errmgr.h"
+#include "orte/mca/errmgr/base/base.h"
+#include "orte/mca/errmgr/base/errmgr_private.h"
+
+#include "errmgr_default_orted.h"
+
+/* Local functions */
+static bool any_live_children(orte_jobid_t job);
+static int pack_state_update(opal_buffer_t *alert, orte_odls_job_t *jobdat);
+static int pack_state_for_proc(opal_buffer_t *alert, orte_odls_child_t *child);
+static bool all_children_registered(orte_jobid_t job);
+static int pack_child_contact_info(orte_jobid_t job, opal_buffer_t *buf);
+static void failed_start(orte_odls_job_t *jobdat, orte_exit_code_t exit_code);
+static void update_local_children(orte_odls_job_t *jobdat,
+                                  orte_job_state_t jobstate,
+                                  orte_proc_state_t state);
+static void killprocs(orte_jobid_t job, orte_vpid_t vpid);
+
+/*
+ * Module functions: Global
+ */
+static int init(void);
+static int finalize(void);
+
+static int predicted_fault(opal_list_t *proc_list,
+                           opal_list_t *node_list,
+                           opal_list_t *suggested_map);
+
+static int update_state(orte_jobid_t job,
+                        orte_job_state_t jobstate,
+                        orte_process_name_t *proc,
+                        orte_proc_state_t state,
+                        pid_t pid,
+                        orte_exit_code_t exit_code);
+
+static int suggest_map_targets(orte_proc_t *proc,
+                               orte_node_t *oldnode,
+                               opal_list_t *node_list);
+
+static int ft_event(int state);
+
+
+/******************
+ * default_orted module
+ ******************/
+orte_errmgr_base_module_t orte_errmgr_default_orted_module = {
+    init,
+    finalize,
+    orte_errmgr_base_log,
+    orte_errmgr_base_abort,
+    orte_errmgr_base_abort_peers,
+    update_state,
+    predicted_fault,
+    suggest_map_targets,
+    ft_event,
+    orte_errmgr_base_register_migration_warning,
+    NULL
+};
+
+/************************
+ * API Definitions
+ ************************/
+static int init(void)
+{
+    return ORTE_SUCCESS;
+}
+
+static int finalize(void)
+{
+    return ORTE_SUCCESS;
+}
+
+static void cbfunc(int status, orte_process_name_t* sender,
+                   opal_buffer_t *buffer, orte_rml_tag_t tag,
+                   void* cbdata)
+{
+    OBJ_RELEASE(buffer);
+}
+
+static int update_state(orte_jobid_t job,
+                        orte_job_state_t jobstate,
+                        orte_process_name_t *proc,
+                        orte_proc_state_t state,
+                        pid_t pid,
+                        orte_exit_code_t exit_code)
+{
+    opal_list_item_t *item, *next;
+    orte_odls_job_t *jobdat = NULL;
+    orte_odls_child_t *child;
+    opal_buffer_t *alert;
+    orte_plm_cmd_flag_t cmd;
+    int rc=ORTE_SUCCESS;
+    orte_vpid_t null=ORTE_VPID_INVALID;
+    orte_ns_cmp_bitmask_t mask;
+
+    /*
+     * if orte is trying to shutdown, just let it
+     */
+    if (orte_finalizing) {
+        return ORTE_SUCCESS;
+    }
+
+    OPAL_OUTPUT_VERBOSE((10, orte_errmgr_base.output,
+                "errmgr:default_orted:update_state() %s) "
+                "------- %s state updated for process %s",
+                ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
+                ((NULL == proc) ? "App. Process" : 
+                 (proc->jobid == ORTE_PROC_MY_HNP->jobid ? "Daemon" : "App. Process")),
+                (NULL == proc) ? "NULL" : ORTE_NAME_PRINT(proc)));
+
+    /* if this is a heartbeat failure, let the HNP handle it */
+    if (ORTE_JOB_STATE_HEARTBEAT_FAILED == jobstate ||
+        ORTE_PROC_STATE_HEARTBEAT_FAILED == state) {
+        return ORTE_SUCCESS;
+    }
+
+    /***   UPDATE COMMAND FOR A JOB   ***/
+    if (NULL == proc) {
+        /* this is an update for an entire job */
+        if (ORTE_JOBID_INVALID == job) {
+            /* whatever happened, we don't know what job
+             * it happened to
+             */
+            orte_show_help("help-orte-errmgr.txt", "errmgr:unknown-job-error",
+                           true, orte_job_state_to_str(jobstate));
+            alert = OBJ_NEW(opal_buffer_t);
+            /* pack update state command */
+            cmd = ORTE_PLM_UPDATE_PROC_STATE;
+            if (ORTE_SUCCESS != (rc = opal_dss.pack(alert, &cmd, 1, ORTE_PLM_CMD))) {
+                ORTE_ERROR_LOG(rc);
+                return rc;
+            }
+            /* pack the "invalid" jobid */
+            if (ORTE_SUCCESS != (rc = opal_dss.pack(alert, &job, 1, ORTE_JOBID))) {
+                ORTE_ERROR_LOG(rc);
+                return rc;
+            }
+            if (0 > (rc = orte_rml.send_buffer_nb(ORTE_PROC_MY_HNP, alert, ORTE_RML_TAG_PLM, 0, cbfunc, NULL))) {
+                ORTE_ERROR_LOG(rc);
+            } else {
+                rc = ORTE_SUCCESS;
+            }
+            return rc;
+        }
+
+        /* lookup the local jobdat for this job */
+        jobdat = NULL;
+        for (item = opal_list_get_first(&orte_local_jobdata);
+             item != opal_list_get_end(&orte_local_jobdata);
+             item = opal_list_get_next(item)) {
+            jobdat = (orte_odls_job_t*)item;
+
+            /* is this the specified job? */
+            if (jobdat->jobid == job) {
+                break;
+            }
+        }
+        if (NULL == jobdat) {
+            return ORTE_ERR_NOT_FOUND;
+        }
+
+        switch (jobstate) {
+        case ORTE_JOB_STATE_FAILED_TO_START:
+            failed_start(jobdat, exit_code);
+            break;
+        case ORTE_JOB_STATE_RUNNING:
+            /* update all local child states */
+            update_local_children(jobdat, jobstate, ORTE_PROC_STATE_RUNNING);
+            break;
+        case ORTE_JOB_STATE_SENSOR_BOUND_EXCEEDED:
+            /* update all procs in job */
+            update_local_children(jobdat, jobstate, ORTE_PROC_STATE_SENSOR_BOUND_EXCEEDED);
+            /* order all local procs for this job to be killed */
+            killprocs(jobdat->jobid, ORTE_VPID_WILDCARD);
+        case ORTE_JOB_STATE_COMM_FAILED:
+            /* kill all local procs */
+            killprocs(ORTE_JOBID_WILDCARD, ORTE_VPID_WILDCARD);
+            /* tell the caller we can't recover */
+            return ORTE_ERR_UNRECOVERABLE;
+            break;
+        case ORTE_JOB_STATE_HEARTBEAT_FAILED:
+            /* let the HNP handle this */
+            return ORTE_SUCCESS;
+            break;
+
+        default:
+            break;
+        }
+        alert = OBJ_NEW(opal_buffer_t);
+        /* pack update state command */
+        cmd = ORTE_PLM_UPDATE_PROC_STATE;
+        if (ORTE_SUCCESS != (rc = opal_dss.pack(alert, &cmd, 1, ORTE_PLM_CMD))) {
+            ORTE_ERROR_LOG(rc);
+            goto FINAL_CLEANUP;
+        }
+        /* pack the job info */
+        if (ORTE_SUCCESS != (rc = pack_state_update(alert, jobdat))) {
+            ORTE_ERROR_LOG(rc);
+        }
+        /* send it */
+        if (0 > (rc = orte_rml.send_buffer_nb(ORTE_PROC_MY_HNP, alert, ORTE_RML_TAG_PLM, 0, cbfunc, NULL))) {
+            ORTE_ERROR_LOG(rc);
+        } else {
+            rc = ORTE_SUCCESS;
+        }
+        return rc;
+    }
+
+    /* if this was a failed comm, then see if it was to our
+     * lifeline
+     */
+    if (ORTE_PROC_STATE_COMM_FAILED == state) {
+        /* if it is our own connection, ignore it */
+        if (OPAL_EQUAL == orte_util_compare_name_fields(ORTE_NS_CMP_ALL, ORTE_PROC_MY_NAME, proc)) {
+            return ORTE_SUCCESS;
+        }
+        /* was it a daemon? */
+        if (proc->jobid != ORTE_PROC_MY_NAME->jobid) {
+            /* nope - ignore */
+            return ORTE_SUCCESS;
+        }
+        /* see if this was a lifeline */
+        if (ORTE_SUCCESS != orte_routed.route_lost(proc)) {
+            /* kill our children */
+            killprocs(ORTE_JOBID_WILDCARD, ORTE_VPID_WILDCARD);
+            /* terminate - our routed children will see
+             * us leave and automatically die
+             */
+            orte_quit();
+        }
+        /* if not, then indicate we can continue */
+        return ORTE_SUCCESS;
+    }
+
+    /* lookup the local jobdat for this job */
+    jobdat = NULL;
+    for (item = opal_list_get_first(&orte_local_jobdata);
+         item != opal_list_get_end(&orte_local_jobdata);
+         item = opal_list_get_next(item)) {
+        jobdat = (orte_odls_job_t*)item;
+
+        /* is this the specified job? */
+        if (jobdat->jobid == proc->jobid) {
+            break;
+        }
+    }
+    if (NULL == jobdat) {
+        /* must already be complete */
+        return ORTE_SUCCESS;
+    }
+
+    /* if there are no local procs for this job, we can
+     * ignore this call
+     */
+    if (0 == jobdat->num_local_procs) {
+        return ORTE_SUCCESS;
+    }
+
+    OPAL_OUTPUT_VERBOSE((5, orte_errmgr_base.output,
+                         "%s errmgr:default_orted got state %s for proc %s pid %d",
+                         ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
+                         orte_proc_state_to_str(state),
+                         ORTE_NAME_PRINT(proc), pid));
+ 
+    /***  UPDATE COMMAND FOR A SPECIFIC PROCESS ***/
+    if (ORTE_PROC_STATE_SENSOR_BOUND_EXCEEDED == state) {
+        /* find this proc in the local children */
+        for (item = opal_list_get_first(&orte_local_children);
+             item != opal_list_get_end(&orte_local_children);
+             item = opal_list_get_next(item)) {
+            child = (orte_odls_child_t*)item;
+            mask = ORTE_NS_CMP_ALL;
+            if (OPAL_EQUAL == orte_util_compare_name_fields(mask, child->name, proc)) {
+                if (ORTE_PROC_STATE_UNTERMINATED > child->state) {
+                    child->state = state;
+                    child->exit_code = exit_code;
+                    /* Decrement the number of local procs */
+                    jobdat->num_local_procs--;
+                    /* kill this proc */
+                    killprocs(proc->jobid, proc->vpid);
+                }
+                return ORTE_SUCCESS;
+            }
+        }
+    }
+
+    if (ORTE_PROC_STATE_TERM_NON_ZERO == state) {
+        if (!orte_abort_non_zero_exit) {
+            /* treat this as normal termination */
+            goto REPORT_STATE;
+        }
+    }
+
+    if (ORTE_PROC_STATE_TERMINATED < state) {
+        /* if the job hasn't completed and the state is abnormally
+         * terminated, then we need to alert the HNP right away
+         */
+        alert = OBJ_NEW(opal_buffer_t);
+        /* pack update state command */
+        cmd = ORTE_PLM_UPDATE_PROC_STATE;
+        if (ORTE_SUCCESS != (rc = opal_dss.pack(alert, &cmd, 1, ORTE_PLM_CMD))) {
+            ORTE_ERROR_LOG(rc);
+            goto FINAL_CLEANUP;
+        }
+        /* pack only the data for this proc - have to start with the jobid
+         * so the receiver can unpack it correctly
+         */
+        if (ORTE_SUCCESS != (rc = opal_dss.pack(alert, &proc->jobid, 1, ORTE_JOBID))) {
+            ORTE_ERROR_LOG(rc);
+            return rc;
+        }
+
+        /* find this proc in the local children */
+        for (item = opal_list_get_first(&orte_local_children);
+             item != opal_list_get_end(&orte_local_children);
+             item = opal_list_get_next(item)) {
+            child = (orte_odls_child_t*)item;
+            mask = ORTE_NS_CMP_ALL;
+            if (OPAL_EQUAL == orte_util_compare_name_fields(mask, child->name, proc)) {
+                if (ORTE_PROC_STATE_UNTERMINATED > child->state) {
+                    child->state = state;
+                    child->exit_code = exit_code;
+                }
+                /* now pack the child's info */
+                if (ORTE_SUCCESS != (rc = pack_state_for_proc(alert, child))) {
+                    ORTE_ERROR_LOG(rc);
+                    return rc;
+                }
+                /* remove the child from our local list as it is no longer alive */
+                opal_list_remove_item(&orte_local_children, &child->super);
+                /* Decrement the number of local procs */
+                jobdat->num_local_procs--;
+
+                OPAL_OUTPUT_VERBOSE((5, orte_errmgr_base.output,
+                                     "%s errmgr:default_orted reporting proc %s aborted to HNP (local procs = %d)",
+                                     ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
+                                     ORTE_NAME_PRINT(child->name),
+                                     jobdat->num_local_procs));
+                
+                /* release the child object */
+                OBJ_RELEASE(child);
+                /* done with loop */
+                break;
+            }
+        }
+
+        /* send it */
+    if (0 > (rc = orte_rml.send_buffer_nb(ORTE_PROC_MY_HNP, alert, ORTE_RML_TAG_PLM, 0, cbfunc, NULL))) {
+            ORTE_ERROR_LOG(rc);
+        } else {
+            rc = ORTE_SUCCESS;
+        }
+        return rc;
+    }
+
+ REPORT_STATE:
+    /* find this proc in the local children so we can update its state */
+    for (item = opal_list_get_first(&orte_local_children);
+         item != opal_list_get_end(&orte_local_children);
+         item = opal_list_get_next(item)) {
+        child = (orte_odls_child_t*)item;
+        mask = ORTE_NS_CMP_ALL;
+        if (OPAL_EQUAL == orte_util_compare_name_fields(mask, child->name, proc)) {
+            if (ORTE_PROC_STATE_UNTERMINATED > child->state) {
+                child->state = state;
+                if (0 < pid) {
+                    child->pid = pid;
+                }
+                child->exit_code = exit_code;
+            }
+            /* done with loop */
+            break;
+        }
+    }
+
+    if (ORTE_PROC_STATE_REGISTERED == state) {
+        /* see if everyone in this job has registered */
+        if (all_children_registered(proc->jobid)) {
+            /* once everyone registers, send their contact info to
+             * the HNP so it is available to debuggers and anyone
+             * else that needs it
+             */
+
+            OPAL_OUTPUT_VERBOSE((5, orte_errmgr_base.output,
+                                 "%s errmgr:default_orted: sending contact info to HNP",
+                                 ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
+            
+            alert = OBJ_NEW(opal_buffer_t);
+            /* pack init routes command */
+            cmd = ORTE_PLM_INIT_ROUTES_CMD;
+            if (ORTE_SUCCESS != (rc = opal_dss.pack(alert, &cmd, 1, ORTE_PLM_CMD))) {
+                ORTE_ERROR_LOG(rc);
+                goto FINAL_CLEANUP;
+            }
+            /* pack the jobid */
+            if (ORTE_SUCCESS != (rc = opal_dss.pack(alert, &proc->jobid, 1, ORTE_JOBID))) {
+                ORTE_ERROR_LOG(rc);
+                goto FINAL_CLEANUP;
+            }
+            /* pack all the local child vpids and epochs */
+            for (item = opal_list_get_first(&orte_local_children);
+                 item != opal_list_get_end(&orte_local_children);
+                 item = opal_list_get_next(item)) {
+                child = (orte_odls_child_t*)item;
+                if (child->name->jobid == proc->jobid) {
+                    if (ORTE_SUCCESS != (rc = opal_dss.pack(alert, &child->name->vpid, 1, ORTE_VPID))) {
+                        ORTE_ERROR_LOG(rc);
+                        goto FINAL_CLEANUP;
+                    }
+                }
+            }
+            /* pack an invalid marker */
+            if (ORTE_SUCCESS != (rc = opal_dss.pack(alert, &null, 1, ORTE_VPID))) {
+                ORTE_ERROR_LOG(rc);
+                goto FINAL_CLEANUP;
+            }
+            /* add in contact info for all procs in the job */
+            if (ORTE_SUCCESS != (rc = pack_child_contact_info(proc->jobid, alert))) {
+                ORTE_ERROR_LOG(rc);
+                OBJ_DESTRUCT(&alert);
+                return rc;
+            }
+            /* send it */
+            if (0 > (rc = orte_rml.send_buffer_nb(ORTE_PROC_MY_HNP, alert, ORTE_RML_TAG_PLM, 0, cbfunc, NULL))) {
+                ORTE_ERROR_LOG(rc);
+            } else {
+                rc = ORTE_SUCCESS;
+            }
+        }        
+        return rc;
+    }
+
+    /* only other state is terminated - see if anyone is left alive */
+    if (!any_live_children(proc->jobid)) {
+        /* lookup the local jobdat for this job */
+        jobdat = NULL;
+        for (item = opal_list_get_first(&orte_local_jobdata);
+             item != opal_list_get_end(&orte_local_jobdata);
+             item = opal_list_get_next(item)) {
+            jobdat = (orte_odls_job_t*)item;
+
+            /* is this the specified job? */
+            if (jobdat->jobid == proc->jobid) {
+                break;
+            }
+        }
+        if (NULL == jobdat) {
+            /* race condition - may not have been formed yet */
+            return ORTE_SUCCESS;
+        }
+
+        alert = OBJ_NEW(opal_buffer_t);
+        /* pack update state command */
+        cmd = ORTE_PLM_UPDATE_PROC_STATE;
+        if (ORTE_SUCCESS != (rc = opal_dss.pack(alert, &cmd, 1, ORTE_PLM_CMD))) {
+            ORTE_ERROR_LOG(rc);
+            goto FINAL_CLEANUP;
+        }
+        /* pack the data for the job */
+        if (ORTE_SUCCESS != (rc = pack_state_update(alert, jobdat))) {
+            ORTE_ERROR_LOG(rc);
+        }
+
+FINAL_CLEANUP:
+        OPAL_OUTPUT_VERBOSE((5, orte_errmgr_base.output,
+                             "%s errmgr:default_orted reporting all procs in %s terminated",
+                             ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
+                             ORTE_JOBID_PRINT(jobdat->jobid)));
+        
+        /* remove all of this job's children from the global list - do not lock
+         * the thread as we are already locked
+         */
+        for (item = opal_list_get_first(&orte_local_children);
+             item != opal_list_get_end(&orte_local_children);
+             item = next) {
+            child = (orte_odls_child_t*)item;
+            next = opal_list_get_next(item);
+
+            if (jobdat->jobid == child->name->jobid) {
+                opal_list_remove_item(&orte_local_children, &child->super);
+                OBJ_RELEASE(child);
+            }
+        }
+
+        /* ensure the job's local session directory tree is removed */
+        orte_session_dir_cleanup(jobdat->jobid);
+
+        /* remove this job from our local job data since it is complete */
+        opal_list_remove_item(&orte_local_jobdata, &jobdat->super);
+        OBJ_RELEASE(jobdat);
+
+        /* send it */
+        if (0 > (rc = orte_rml.send_buffer_nb(ORTE_PROC_MY_HNP, alert, ORTE_RML_TAG_PLM, 0, cbfunc, NULL))) {
+            ORTE_ERROR_LOG(rc);
+        } else {
+            rc = ORTE_SUCCESS;
+        }
+
+        /* indicate that the job is complete */
+        return rc;
+    }
+    return ORTE_SUCCESS;
+}
+
+static int predicted_fault(opal_list_t *proc_list,
+                           opal_list_t *node_list,
+                           opal_list_t *suggested_map)
+{
+    return ORTE_ERR_NOT_IMPLEMENTED;
+}
+
+static int suggest_map_targets(orte_proc_t *proc,
+                               orte_node_t *oldnode,
+                               opal_list_t *node_list)
+{
+    return ORTE_ERR_NOT_IMPLEMENTED;
+}
+
+static int ft_event(int state)
+{
+    return ORTE_SUCCESS;
+}
+
+
+/*****************
+ * Local Functions
+ *****************/
+static bool any_live_children(orte_jobid_t job)
+{
+    opal_list_item_t *item;
+    orte_odls_child_t *child;
+
+    /* the thread is locked elsewhere - don't try to do it again here */
+
+    for (item = opal_list_get_first(&orte_local_children);
+         item != opal_list_get_end(&orte_local_children);
+         item = opal_list_get_next(item)) {
+        child = (orte_odls_child_t*)item;
+
+        /* is this child part of the specified job? */
+        if ((job == child->name->jobid || ORTE_JOBID_WILDCARD == job) &&
+            child->alive) {
+            return true;
+        }
+    }
+
+    /* if we get here, then nobody is left alive from that job */
+    return false;
+
+}
+
+static int pack_state_for_proc(opal_buffer_t *alert, orte_odls_child_t *child)
+{
+    int rc;
+
+    /* pack the child's vpid */
+    if (ORTE_SUCCESS != (rc = opal_dss.pack(alert, &(child->name->vpid), 1, ORTE_VPID))) {
+        ORTE_ERROR_LOG(rc);
+        return rc;
+    }
+    /* pack the pid */
+    if (ORTE_SUCCESS != (rc = opal_dss.pack(alert, &child->pid, 1, OPAL_PID))) {
+        ORTE_ERROR_LOG(rc);
+        return rc;
+    }
+    /* if we are timing things, pack the time the proc was launched */
+    if (orte_timing) {
+        int64_t tmp;
+        tmp = child->starttime.tv_sec;
+        if (ORTE_SUCCESS != (rc = opal_dss.pack(alert, &tmp, 1, OPAL_INT64))) {
+            ORTE_ERROR_LOG(rc);
+            return rc;
+        }
+        tmp = child->starttime.tv_usec;
+        if (ORTE_SUCCESS != (rc = opal_dss.pack(alert, &tmp, 1, OPAL_INT64))) {
+            ORTE_ERROR_LOG(rc);
+            return rc;
+        }
+    }
+    /* pack its state */
+    if (ORTE_SUCCESS != (rc = opal_dss.pack(alert, &child->state, 1, ORTE_PROC_STATE))) {
+        ORTE_ERROR_LOG(rc);
+        return rc;
+    }
+    /* pack its exit code */
+    if (ORTE_SUCCESS != (rc = opal_dss.pack(alert, &child->exit_code, 1, ORTE_EXIT_CODE))) {
+        ORTE_ERROR_LOG(rc);
+        return rc;
+    }
+
+    return ORTE_SUCCESS;
+}
+
+static int pack_state_update(opal_buffer_t *alert, orte_odls_job_t *jobdat)
+{
+    int rc;
+    opal_list_item_t *item, *next;
+    orte_odls_child_t *child;
+    orte_vpid_t null=ORTE_VPID_INVALID;
+
+    /* pack the jobid */
+    if (ORTE_SUCCESS != (rc = opal_dss.pack(alert, &jobdat->jobid, 1, ORTE_JOBID))) {
+        ORTE_ERROR_LOG(rc);
+        return rc;
+    }
+    /* if we are timing things, pack the time the launch msg for this job was recvd */
+    if (orte_timing) {
+        int64_t tmp;
+        tmp = jobdat->launch_msg_recvd.tv_sec;
+        if (ORTE_SUCCESS != (rc = opal_dss.pack(alert, &tmp, 1, OPAL_INT64))) {
+            ORTE_ERROR_LOG(rc);
+            return rc;
+        }
+        tmp = jobdat->launch_msg_recvd.tv_usec;
+        if (ORTE_SUCCESS != (rc = opal_dss.pack(alert, &tmp, 1, OPAL_INT64))) {
+            ORTE_ERROR_LOG(rc);
+            return rc;
+        }
+    }
+    for (item = opal_list_get_first(&orte_local_children);
+            item != opal_list_get_end(&orte_local_children);
+            item = next) {
+        child = (orte_odls_child_t*)item;
+        next = opal_list_get_next(item);
+        /* if this child is part of the job... */
+        if (child->name->jobid == jobdat->jobid) {
+            if (ORTE_SUCCESS != (rc = pack_state_for_proc(alert, child))) {
+                ORTE_ERROR_LOG(rc);
+                return rc;
+            }
+        }
+    }
+    /* flag that this job is complete so the receiver can know */
+    if (ORTE_SUCCESS != (rc = opal_dss.pack(alert, &null, 1, ORTE_VPID))) {
+        ORTE_ERROR_LOG(rc);
+        return rc;
+    }
+
+    return ORTE_SUCCESS;
+}
+
+static bool all_children_registered(orte_jobid_t job)
+{
+    opal_list_item_t *item;
+    orte_odls_child_t *child;
+
+    /* the thread is locked elsewhere - don't try to do it again here */
+
+    for (item = opal_list_get_first(&orte_local_children);
+         item != opal_list_get_end(&orte_local_children);
+         item = opal_list_get_next(item)) {
+        child = (orte_odls_child_t*)item;
+
+        /* is this child part of the specified job? */
+        if (OPAL_EQUAL == opal_dss.compare(&child->name->jobid, &job, ORTE_JOBID)) {
+            /* if this child has terminated, we consider it as having
+             * registered for the purposes of this function. If it never
+             * did register, then we will send a NULL rml_uri back to
+             * the HNP, which will then know that the proc did not register.
+             * If other procs did register, then the HNP can declare an
+             * abnormal termination
+             */
+            if (ORTE_PROC_STATE_UNTERMINATED < child->state) {
+                /* this proc has terminated somehow - consider it
+                 * as registered for now
+                 */
+                continue;
+            }
+            /* if this child is *not* registered yet, return false */
+            if (!child->init_recvd) {
+                return false;
+            }
+            /* if this child has registered a finalize, return false */
+            if (child->fini_recvd) {
+                return false;
+            }
+        }
+    }
+
+    /* if we get here, then everyone in the job is currently registered */
+    return true;
+
+}
+
+static int pack_child_contact_info(orte_jobid_t job, opal_buffer_t *buf)
+{
+    opal_list_item_t *item;
+    orte_odls_child_t *child;
+    int rc;
+
+    /* the thread is locked elsewhere - don't try to do it again here */
+
+    for (item = opal_list_get_first(&orte_local_children);
+         item != opal_list_get_end(&orte_local_children);
+         item = opal_list_get_next(item)) {
+        child = (orte_odls_child_t*)item;
+
+        /* is this child part of the specified job? */
+        if (OPAL_EQUAL == opal_dss.compare(&child->name->jobid, &job, ORTE_JOBID)) {
+            /* pack the child's vpid - must be done in case rml_uri is NULL */
+            if (ORTE_SUCCESS != (rc = opal_dss.pack(buf, &(child->name->vpid), 1, ORTE_VPID))) {
+                ORTE_ERROR_LOG(rc);
+                return rc;
+            }            
+            /* pack the contact info */
+            if (ORTE_SUCCESS != (rc = opal_dss.pack(buf, &child->rml_uri, 1, OPAL_STRING))) {
+                ORTE_ERROR_LOG(rc);
+                return rc;
+            }
+        }
+    }
+
+    return ORTE_SUCCESS;
+
+}
+
+static void failed_start(orte_odls_job_t *jobdat, orte_exit_code_t exit_code)
+{
+    opal_list_item_t *item;
+    orte_odls_child_t *child;
+
+    /* set the state */
+    jobdat->state = ORTE_JOB_STATE_FAILED_TO_START;
+
+    for (item = opal_list_get_first(&orte_local_children);
+         item != opal_list_get_end(&orte_local_children);
+         item = opal_list_get_next(item)) {
+        child = (orte_odls_child_t*)item;
+        if (child->name->jobid == jobdat->jobid) {
+            if (ORTE_PROC_STATE_LAUNCHED > child->state ||
+                ORTE_PROC_STATE_FAILED_TO_START == child->state) {
+                /* this proc never launched - flag that the iof
+                 * is complete or else we will hang waiting for
+                 * pipes to close that were never opened
+                 */
+                child->iof_complete = true;
+                /* ditto for waitpid */
+                child->waitpid_recvd = true;
+            }
+        }
+    }
+    OPAL_OUTPUT_VERBOSE((1, orte_errmgr_base.output,
+                         "%s errmgr:hnp: job %s reported incomplete start",
+                         ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
+                         ORTE_JOBID_PRINT(jobdat->jobid)));
+    return;
+}
+
+static void update_local_children(orte_odls_job_t *jobdat, orte_job_state_t jobstate, orte_proc_state_t state)
+{
+    opal_list_item_t *item;
+    orte_odls_child_t *child;
+
+    /* update job state */
+    jobdat->state = jobstate;
+    /* update children */
+    for (item = opal_list_get_first(&orte_local_children);
+         item != opal_list_get_end(&orte_local_children);
+         item = opal_list_get_next(item)) {
+        child = (orte_odls_child_t*)item;
+        if (jobdat->jobid == child->name->jobid) {
+            child->state = state;
+        }
+    }
+}
+
+static void killprocs(orte_jobid_t job, orte_vpid_t vpid)
+{
+    opal_pointer_array_t cmd;
+    orte_proc_t proc;
+    int rc;
+
+    /* stop local sensors for this job */
+    if (ORTE_VPID_WILDCARD == vpid) {
+        orte_sensor.stop(job);
+    }
+
+    if (ORTE_JOBID_WILDCARD == job 
+        && ORTE_VPID_WILDCARD == vpid) {
+        if (ORTE_SUCCESS != (rc = orte_odls.kill_local_procs(NULL))) {
+            ORTE_ERROR_LOG(rc);
+        }
+        return;
+    }
+
+    OBJ_CONSTRUCT(&cmd, opal_pointer_array_t);
+    OBJ_CONSTRUCT(&proc, orte_proc_t);
+    proc.name.jobid = job;
+    proc.name.vpid = vpid;
+    ORTE_EPOCH_SET(proc.name.epoch,orte_ess.proc_get_epoch(&(proc.name)));
+    opal_pointer_array_add(&cmd, &proc);
+    if (ORTE_SUCCESS != (rc = orte_odls.kill_local_procs(&cmd))) {
+        ORTE_ERROR_LOG(rc);
+    }
+    OBJ_DESTRUCT(&cmd);
+    OBJ_DESTRUCT(&proc);
+}

orte/mca/errmgr/default_orted/errmgr_default_orted.h

@@ -0,0 +1,35 @@
+/*
+ * Copyright (c) 2010      Cisco Systems, Inc. All rights reserved.
+ *
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+/**
+ * @file
+ * 
+ */
+
+#ifndef MCA_ERRMGR_default_orted_EXPORT_H
+#define MCA_ERRMGR_default_orted_EXPORT_H
+
+#include "orte_config.h"
+
+#include "orte/mca/errmgr/errmgr.h"
+
+BEGIN_C_DECLS
+
+/*
+ * Local Component structures
+ */
+
+ORTE_MODULE_DECLSPEC extern orte_errmgr_base_component_t mca_errmgr_default_orted_component;
+
+ORTE_DECLSPEC extern orte_errmgr_base_module_t orte_errmgr_default_orted_module;
+
+END_C_DECLS
+
+#endif /* MCA_ERRMGR_default_orted_EXPORT_H */

orte/mca/errmgr/default_orted/errmgr_default_orted_component.c

@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2010      Cisco Systems, Inc. All rights reserved.
+ *
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "orte_config.h"
+#include "opal/util/output.h"
+
+#include "orte/mca/errmgr/errmgr.h"
+#include "orte/mca/errmgr/base/base.h"
+#include "errmgr_default_orted.h"
+
+/*
+ * Public string for version number
+ */
+const char *orte_errmgr_default_orted_component_version_string = 
+    "ORTE ERRMGR default_orted MCA component version " ORTE_VERSION;
+
+/*
+ * Local functionality
+ */
+static int errmgr_default_orted_open(void);
+static int errmgr_default_orted_close(void);
+static int errmgr_default_orted_component_query(mca_base_module_t **module, int *priority);
+
+/*
+ * Instantiate the public struct with all of our public information
+ * and pointer to our public functions in it
+ */
+orte_errmgr_base_component_t mca_errmgr_default_orted_component =
+{
+    /* Handle the general mca_component_t struct containing 
+     *  meta information about the component itdefault_orted
+     */
+    {
+        ORTE_ERRMGR_BASE_VERSION_3_0_0,
+        /* Component name and version */
+        "default_orted",
+        ORTE_MAJOR_VERSION,
+        ORTE_MINOR_VERSION,
+        ORTE_RELEASE_VERSION,
+        
+        /* Component open and close functions */
+        errmgr_default_orted_open,
+        errmgr_default_orted_close,
+        errmgr_default_orted_component_query
+    },
+    {
+        /* The component is checkpoint ready */
+        MCA_BASE_METADATA_PARAM_CHECKPOINT
+    }
+};
+
+static int my_priority;
+
+static int errmgr_default_orted_open(void) 
+{
+    mca_base_component_t *c = &mca_errmgr_default_orted_component.base_version;
+
+    mca_base_param_reg_int(c, "priority",
+                           "Priority of the default_orted errmgr component",
+                           false, false, 1000,
+                           &my_priority);
+
+    return ORTE_SUCCESS;
+}
+
+static int errmgr_default_orted_close(void)
+{
+    return ORTE_SUCCESS;
+}
+
+static int errmgr_default_orted_component_query(mca_base_module_t **module, int *priority)
+{
+    if (ORTE_PROC_IS_DAEMON) {
+        /* we are the default component for daemons */
+        *priority = my_priority;
+        *module = (mca_base_module_t *)&orte_errmgr_default_orted_module;
+        return ORTE_SUCCESS;        
+    }
+    
+    *priority = -1;
+    *module = NULL;
+    return ORTE_ERROR;
+}
+

orte/mca/errmgr/hnp/errmgr_hnp_component.c

@@ -61,7 +61,7 @@ orte_errmgr_hnp_component_t mca_errmgr_hnp_component = {
         /* opal_output handler */
         -1,
         /* Default priority */
-        0
+        5
     }
 };
 

orte/mca/errmgr/orted/errmgr_orted_component.c

@@ -72,7 +72,7 @@ static int errmgr_orted_component_query(mca_base_module_t **module, int *priorit
         /* keep our priority low so that other modules are higher
          * and will run before us
          */
-        *priority = 0;
+        *priority = 5;
         *module = (mca_base_module_t *)&orte_errmgr_orted_module;
         return ORTE_SUCCESS;        
     }