efbb5c9b7c
* add hnp and orted modules to the errmgr framework. The HNP module contains much of the code that was in the errmgr base since that code could only be executed by the HNP anyway. * update the odls to report process states directly into the active errmgr module, thus removing the need to send messages looped back into the odls cmd processor. Let the active errmgr module decide what to do at various states. * remove the code to track application state progress from the plm_base_launch_support.c code. Update the plm modules to call the errmgr directly when a launch fails. * update the plm_base_receive.c code to call the errmgr with state updates from remote daemons * update the routed modules to reflect that process state is updated in the errmgr * ensure that the orted's open the errmgr and select their appropriate module * add new pretty-print utilities to print process and job state. Move the pretty-print of time info to a globally-accessible place * define a global orte_comm function to send messages from orted's to the HNP so that others can overlay the standard RML methods, if desired. * update the orterun help output to reflect that the "term w/o sync" error message can result from three, not two, scenarios This commit was SVN r23023.
289 строки
9.2 KiB
C
289 строки
9.2 KiB
C
/*
|
|
* Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
|
|
* University Research and Technology
|
|
* Corporation. All rights reserved.
|
|
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
|
* of Tennessee Research Foundation. All rights
|
|
* reserved.
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
* University of Stuttgart. All rights reserved.
|
|
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
* All rights reserved.
|
|
* Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
|
|
|
|
#include "orte_config.h"
|
|
#include "orte/constants.h"
|
|
|
|
#ifdef HAVE_UNISTD_H
|
|
#include <unistd.h>
|
|
#endif
|
|
#include <stdlib.h>
|
|
#include <stdarg.h>
|
|
|
|
#include "opal/util/trace.h"
|
|
#include "opal/util/output.h"
|
|
|
|
#include "orte/util/name_fns.h"
|
|
#include "orte/util/session_dir.h"
|
|
#include "orte/mca/ess/ess.h"
|
|
#include "orte/mca/odls/odls.h"
|
|
#include "orte/mca/routed/routed.h"
|
|
#include "orte/runtime/orte_globals.h"
|
|
|
|
#include "orte/mca/errmgr/errmgr.h"
|
|
#include "orte/mca/errmgr/base/base.h"
|
|
#include "orte/mca/errmgr/base/errmgr_private.h"
|
|
|
|
/*
|
|
* Public interfaces
|
|
*/
|
|
void orte_errmgr_base_log(int error_code, char *filename, int line)
|
|
{
|
|
OPAL_TRACE(1);
|
|
|
|
if (ORTE_ERR_SILENT == error_code) {
|
|
/* if the error is silent, say nothing */
|
|
return;
|
|
}
|
|
|
|
opal_output(0, "%s ORTE_ERROR_LOG: %s in file %s at line %d",
|
|
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
|
|
ORTE_ERROR_NAME(error_code), filename, line);
|
|
}
|
|
|
|
int orte_errmgr_base_update_state(orte_jobid_t job,
|
|
orte_job_state_t jobstate,
|
|
orte_process_name_t *name,
|
|
orte_proc_state_t state,
|
|
orte_exit_code_t exit_code)
|
|
{
|
|
int rc;
|
|
int i;
|
|
orte_errmgr_stack_state_t stack_state;
|
|
orte_errmgr_base_module_t *module;
|
|
|
|
if( ORTE_PROC_IS_APP ) {
|
|
return ORTE_SUCCESS;
|
|
}
|
|
|
|
if( !orte_errmgr_base_shutting_down ) {
|
|
OPAL_OUTPUT_VERBOSE((10, orte_errmgr_base_output,
|
|
"errmgr:base:update_state() %s) "
|
|
"------- %s state updated for process %s",
|
|
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
|
|
(NULL == name) ? "App. Process" : (name->jobid == ORTE_PROC_MY_HNP->jobid ? "Daemon" : "App. Process"),
|
|
(NULL == name) ? "NULL" : ORTE_NAME_PRINT(name)));
|
|
}
|
|
|
|
stack_state = ORTE_ERRMGR_STACK_STATE_NONE;
|
|
stack_state |= ORTE_ERRMGR_STACK_STATE_JOB_ABORT;
|
|
|
|
/********************************
|
|
* Call the active modules
|
|
********************************/
|
|
for (i = 0; i < orte_errmgr_base_modules.size; ++i) {
|
|
module = (orte_errmgr_base_module_t*)opal_pointer_array_get_item(&orte_errmgr_base_modules, i);
|
|
if( NULL == module ) {
|
|
continue;
|
|
}
|
|
if( NULL != module->update_state ) {
|
|
rc = module->update_state(job, jobstate, name, state, exit_code, &stack_state);
|
|
if (ORTE_SUCCESS != rc || ORTE_ERRMGR_STACK_STATE_COMPLETE & stack_state) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
return ORTE_SUCCESS;
|
|
}
|
|
|
|
int orte_errmgr_base_abort(int error_code, char *fmt, ...)
|
|
{
|
|
va_list arglist;
|
|
|
|
/* If there was a message, output it */
|
|
va_start(arglist, fmt);
|
|
if( NULL != fmt ) {
|
|
char* buffer = NULL;
|
|
vasprintf( &buffer, fmt, arglist );
|
|
opal_output( 0, "%s", buffer );
|
|
free( buffer );
|
|
}
|
|
va_end(arglist);
|
|
|
|
/* if I am a daemon or the HNP... */
|
|
if (ORTE_PROC_IS_HNP || ORTE_PROC_IS_DAEMON) {
|
|
/* whack my local procs */
|
|
orte_odls.kill_local_procs(NULL);
|
|
/* whack any session directories */
|
|
orte_session_dir_cleanup(ORTE_JOBID_WILDCARD);
|
|
} else {
|
|
/* cleanup my session directory */
|
|
orte_session_dir_finalize(ORTE_PROC_MY_NAME);
|
|
}
|
|
|
|
/* abnormal exit */
|
|
orte_ess.abort(error_code, false);
|
|
|
|
return ORTE_SUCCESS;
|
|
}
|
|
|
|
int orte_errmgr_base_predicted_fault(char ***proc_list,
|
|
char ***node_list,
|
|
char ***suggested_nodes)
|
|
{
|
|
orte_errmgr_base_module_t *module = NULL;
|
|
int i, rc;
|
|
orte_errmgr_stack_state_t stack_state = ORTE_ERRMGR_STACK_STATE_NONE;
|
|
|
|
OPAL_OUTPUT_VERBOSE((10, orte_errmgr_base_output,
|
|
"errmgr:base:predicted_fault() %s) "
|
|
"------- Notifying components... (%3d active components)",
|
|
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
|
|
orte_errmgr_base_modules.size));
|
|
|
|
for(i = 0; i < orte_errmgr_base_modules.size; ++i) {
|
|
module = (orte_errmgr_base_module_t*)opal_pointer_array_get_item(&orte_errmgr_base_modules, i);
|
|
if( NULL == module ) {
|
|
continue;
|
|
}
|
|
if( NULL != module->predicted_fault ) {
|
|
rc = module->predicted_fault(proc_list, node_list, suggested_nodes, &stack_state);
|
|
if (ORTE_SUCCESS != rc || ORTE_ERRMGR_STACK_STATE_COMPLETE & stack_state) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
return ORTE_SUCCESS;
|
|
}
|
|
|
|
int orte_errmgr_base_suggest_map_targets(orte_proc_t *proc,
|
|
orte_node_t *oldnode,
|
|
opal_list_t *node_list)
|
|
{
|
|
orte_errmgr_base_module_t *module = NULL;
|
|
int i, rc;
|
|
orte_errmgr_stack_state_t stack_state = ORTE_ERRMGR_STACK_STATE_NONE;
|
|
|
|
/*
|
|
* If the user did not ask for recovery, then do not process recovery events
|
|
*/
|
|
if( !orte_errmgr_base_enable_recovery ) {
|
|
OPAL_OUTPUT_VERBOSE((10, orte_errmgr_base_output,
|
|
"errmgr:base:suggest_map_targets() %s) "
|
|
"------- Recovery currently disabled! Skipping...",
|
|
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME) ));
|
|
return ORTE_SUCCESS;
|
|
}
|
|
|
|
OPAL_OUTPUT_VERBOSE((10, orte_errmgr_base_output,
|
|
"errmgr:base:suggest_map_targets() %s) "
|
|
"------- Notifying components... (%3d active components)",
|
|
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
|
|
orte_errmgr_base_modules.size));
|
|
|
|
for(i = 0; i < orte_errmgr_base_modules.size; ++i) {
|
|
module = (orte_errmgr_base_module_t*)opal_pointer_array_get_item(&orte_errmgr_base_modules, i);
|
|
if( NULL == module ) {
|
|
continue;
|
|
}
|
|
if( NULL != module->suggest_map_targets ) {
|
|
rc = module->suggest_map_targets(proc, oldnode, node_list, &stack_state);
|
|
if (ORTE_SUCCESS != rc || ORTE_ERRMGR_STACK_STATE_COMPLETE & stack_state) {
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
return ORTE_SUCCESS;
|
|
}
|
|
|
|
int orte_errmgr_base_ft_event(int state)
|
|
{
|
|
orte_errmgr_base_module_t *module = NULL;
|
|
int i;
|
|
|
|
OPAL_OUTPUT_VERBOSE((10, orte_errmgr_base_output,
|
|
"errmgr:base:ft_event() %s) "
|
|
"------- Notifying components... (%3d active components)",
|
|
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
|
|
orte_errmgr_base_modules.size));
|
|
|
|
for(i = 0; i < orte_errmgr_base_modules.size; ++i) {
|
|
module = (orte_errmgr_base_module_t*)opal_pointer_array_get_item(&orte_errmgr_base_modules, i);
|
|
if( NULL == module ) {
|
|
continue;
|
|
}
|
|
if( NULL != module->ft_event ) {
|
|
module->ft_event(state);
|
|
}
|
|
}
|
|
|
|
return ORTE_SUCCESS;
|
|
}
|
|
|
|
void orte_errmgr_base_update_runtime(orte_job_t *jdata,
|
|
orte_process_name_t *proc,
|
|
orte_proc_state_t state,
|
|
orte_errmgr_stack_state_t *stack_state)
|
|
{
|
|
orte_proc_t *loc_proc;
|
|
int32_t i;
|
|
|
|
/* has this already been done */
|
|
if (ORTE_ERRMGR_STACK_STATE_UPDATED & *stack_state) {
|
|
return;
|
|
}
|
|
*stack_state |= ORTE_ERRMGR_STACK_STATE_UPDATED;
|
|
|
|
/*
|
|
* orterun is trying to shutdown, so just let it
|
|
*/
|
|
if (orte_errmgr_base_shutting_down) {
|
|
return;
|
|
}
|
|
|
|
/*
|
|
* orte_errmgr_base_incomplete_start() will pass a NULL since all processes
|
|
* are effected by this fault.
|
|
* JJH: Since we do not handle the recovery from such errors yet, just
|
|
* skip processing, and go to the abort sequence.
|
|
*/
|
|
if (NULL == proc) {
|
|
return;
|
|
}
|
|
|
|
/*
|
|
* Remove the route to this process
|
|
*/
|
|
orte_routed.delete_route(proc);
|
|
|
|
/*
|
|
* Set the process state in the job data structure
|
|
*/
|
|
loc_proc = NULL;
|
|
for (i = 0; i < jdata->procs->size; ++i) {
|
|
if (NULL == (loc_proc = (orte_proc_t*)opal_pointer_array_get_item(jdata->procs, i))) {
|
|
continue;
|
|
}
|
|
|
|
if (loc_proc->name.vpid != proc->vpid) {
|
|
continue;
|
|
}
|
|
|
|
loc_proc->state = state;
|
|
if (ORTE_PROC_STATE_UNTERMINATED < state) {
|
|
jdata->num_terminated++;
|
|
}
|
|
break;
|
|
}
|
|
}
|