ab5ea61100
1. add a "cancel_operation" API to the pls components that allows orterun to demand that an orted operation (e.g., terminate_job) be immediately cancelled and abandoned. 2. changes the pls orted commands from blocking to non-blocking. This allows us to interrupt those operations should an orted be non-responsive. The change also adds an orte_abort_timeout that limits how long orterun will automatically wait for the orteds to respond - if the terminate command, for example, doesn't see orted response within that time, then we printout an appropriate error message and just give up. 3. modifies orterun to allow multiple ctrl-c's to simply abort the program even if the orteds have not responded 4. does some cleanup on the orte-level mca params so that their implementation looks a lot more like that of ompi - makes it easier to maintain. This change also includes the definition of an orte_abort_timeout struct and associated MCA param (can't have too many!) so you can set the time after which orterun gives up on waiting for orteds to respond This needs more testing before migrating to 1.2. This commit was SVN r13304.
254 строки
8.1 KiB
C
254 строки
8.1 KiB
C
/*
|
|
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
|
* University Research and Technology
|
|
* Corporation. All rights reserved.
|
|
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
|
* of Tennessee Research Foundation. All rights
|
|
* reserved.
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
* University of Stuttgart. All rights reserved.
|
|
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
* All rights reserved.
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
|
|
|
|
#include "orte_config.h"
|
|
#include "orte/orte_constants.h"
|
|
|
|
#include <stdlib.h>
|
|
#include <stdarg.h>
|
|
|
|
#include "opal/class/opal_list.h"
|
|
#include "opal/util/trace.h"
|
|
#include "opal/util/output.h"
|
|
|
|
#include "orte/runtime/runtime.h"
|
|
#include "orte/runtime/params.h"
|
|
#include "orte/mca/ns/ns_types.h"
|
|
#include "orte/mca/gpr/gpr.h"
|
|
#include "orte/mca/pls/pls.h"
|
|
#include "orte/mca/smr/smr.h"
|
|
#include "orte/mca/schema/schema.h"
|
|
#include "orte/dss/dss.h"
|
|
#include "orte/mca/rmgr/rmgr.h"
|
|
|
|
#include "orte/mca/errmgr/base/base.h"
|
|
#include "orte/mca/errmgr/hnp/errmgr_hnp.h"
|
|
|
|
/*
|
|
* This function gets called when the someone updates a process
|
|
* state to indicate it has aborted. That action results in
|
|
* the firing of a registry trigger that passes a minimal
|
|
* data message here. The only part of that message we need
|
|
* is the segment name so we can extract the jobid from it
|
|
*
|
|
* Various components will follow their own strategy for dealing with
|
|
* this situation. For this component, we simply kill the job.
|
|
*/
|
|
int orte_errmgr_hnp_proc_aborted(orte_gpr_notify_message_t *msg)
|
|
{
|
|
orte_jobid_t job;
|
|
orte_vpid_t start, range;
|
|
orte_std_cntr_t num;
|
|
char *segment;
|
|
char *tokens[] = {
|
|
ORTE_JOB_GLOBALS,
|
|
NULL
|
|
};
|
|
orte_data_value_t dval = ORTE_DATA_VALUE_EMPTY;
|
|
opal_list_t attrs;
|
|
opal_list_item_t *item;
|
|
int rc;
|
|
|
|
OPAL_TRACE(1);
|
|
|
|
opal_output(orte_errmgr_base_output, "errmgr:hnp: proc abort has been detected");
|
|
|
|
/* This trigger is named, so we can extract the jobid
|
|
* directly from the trigger name
|
|
*/
|
|
if (ORTE_SUCCESS != (rc = orte_schema.extract_jobid_from_std_trigger_name(&job, msg->target))) {
|
|
ORTE_ERROR_LOG(rc);
|
|
return rc;
|
|
}
|
|
|
|
/* set the job state */
|
|
if (ORTE_SUCCESS != (rc = orte_smr.set_job_state(job, ORTE_JOB_STATE_ABORTED))) {
|
|
ORTE_ERROR_LOG(rc);
|
|
return rc;
|
|
}
|
|
|
|
/* tell the pls to terminate the job AND ALL ITS DESCENDANTS */
|
|
OBJ_CONSTRUCT(&attrs, opal_list_t);
|
|
orte_rmgr.add_attribute(&attrs, ORTE_NS_INCLUDE_DESCENDANTS, ORTE_UNDEF, NULL, ORTE_RMGR_ATTR_OVERRIDE);
|
|
if (ORTE_SUCCESS != (rc = orte_pls.terminate_job(job, &orte_abort_timeout, &attrs))) {
|
|
ORTE_ERROR_LOG(rc);
|
|
}
|
|
while (NULL != (item = opal_list_remove_first(&attrs))) OBJ_RELEASE(item);
|
|
OBJ_DESTRUCT(&attrs);
|
|
|
|
/* orterun will only wakeup when all procs report terminated. The terminate_job
|
|
* function *should* have done that - however, it is possible during abnormal
|
|
* startup that it will fail to happen. If we get here, we force the issue by
|
|
* deliberately causing the TERMINATE trigger to fire
|
|
*/
|
|
if (ORTE_SUCCESS != (rc = orte_rmgr.get_vpid_range(job, &start, &range))) {
|
|
ORTE_ERROR_LOG(rc);
|
|
return rc;
|
|
}
|
|
if (ORTE_SUCCESS != (rc = orte_schema.get_job_segment_name(&segment, job))) {
|
|
ORTE_ERROR_LOG(rc);
|
|
return rc;
|
|
}
|
|
num = range;
|
|
if (ORTE_SUCCESS != (rc = orte_dss.set(&dval, (void*)&num, ORTE_STD_CNTR))) {
|
|
ORTE_ERROR_LOG(rc);
|
|
return rc;
|
|
}
|
|
if (ORTE_SUCCESS != (rc = orte_gpr.put_1(ORTE_GPR_OVERWRITE | ORTE_GPR_TOKENS_AND | ORTE_GPR_KEYS_OR,
|
|
segment, tokens, ORTE_PROC_NUM_TERMINATED, &dval))) {
|
|
ORTE_ERROR_LOG(rc);
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
/*
|
|
* This function gets called when someone updates a process
|
|
* state to indicate it failed to start. That action results in
|
|
* the firing of a registry trigger that passes a minimal
|
|
* data message here. The only part of that message we need
|
|
* is the segment name so we can extract the jobid from it
|
|
*
|
|
* Various components will follow their own strategy for dealing with
|
|
* this situation. For this component, we simply kill the job.
|
|
*/
|
|
int orte_errmgr_hnp_incomplete_start(orte_gpr_notify_message_t *msg)
|
|
{
|
|
orte_jobid_t job;
|
|
int rc;
|
|
|
|
OPAL_TRACE(1);
|
|
|
|
/* This trigger is named, so we can extract the jobid
|
|
* directly from the trigger name
|
|
*/
|
|
if (ORTE_SUCCESS != (rc = orte_schema.extract_jobid_from_std_trigger_name(&job, msg->target))) {
|
|
ORTE_ERROR_LOG(rc);
|
|
return rc;
|
|
}
|
|
|
|
/* set the job state */
|
|
if (ORTE_SUCCESS != (rc = orte_smr.set_job_state(job, ORTE_JOB_STATE_FAILED_TO_START))) {
|
|
ORTE_ERROR_LOG(rc);
|
|
return rc;
|
|
}
|
|
|
|
/* tell the pls to terminate the job - just kill this job, not any descendants since
|
|
* the job is just trying to start
|
|
*/
|
|
if (ORTE_SUCCESS != (rc = orte_pls.terminate_job(job, &orte_abort_timeout, NULL))) {
|
|
ORTE_ERROR_LOG(rc);
|
|
}
|
|
|
|
return rc;
|
|
}
|
|
|
|
/*
|
|
* This function gets called when the HNP itself detects an internal error!
|
|
* Ideally, we would find some way to tell all the active jobs to die before
|
|
* we depart ourselves. Unfortunately, at this time, we aren't sure we can do
|
|
* this - later, we'll add some more intelligence by, for example, checking
|
|
* the error code to see if it's something that would allow us to alert
|
|
* the remote orteds.
|
|
*
|
|
* For now, we'll just depart!
|
|
*/
|
|
void orte_errmgr_hnp_error_detected(int error_code, char *fmt, ...)
|
|
{
|
|
va_list arglist;
|
|
|
|
/* If there was a message, output it */
|
|
|
|
va_start(arglist, fmt);
|
|
if( NULL != fmt ) {
|
|
char* buffer = NULL;
|
|
vasprintf( &buffer, fmt, arglist );
|
|
opal_output( 0, buffer );
|
|
free( buffer );
|
|
}
|
|
va_end(arglist);
|
|
|
|
/* abnormal exit */
|
|
orte_abort(error_code, false);
|
|
}
|
|
|
|
/*
|
|
* This function gets called when the HNP desperately needs to just die.
|
|
* Nothing can be done by definition here - this function ONLY gets
|
|
* called as an absolute last resort
|
|
*/
|
|
void orte_errmgr_hnp_abort(void)
|
|
{
|
|
OPAL_TRACE(1);
|
|
|
|
/* abnormal exit */
|
|
orte_abort(-1, false);
|
|
}
|
|
|
|
/*
|
|
* This function gets called when a process wants to request that the HNP
|
|
* abort some set of processes for it. Since this component IS for the HNP,
|
|
* that means we need to actually execute this request! Call upon the PLS
|
|
* as needed to execute the abort requests
|
|
*/
|
|
int orte_errmgr_hnp_abort_procs_request(orte_process_name_t *procs, orte_std_cntr_t nprocs)
|
|
{
|
|
int rc;
|
|
|
|
OPAL_TRACE(1);
|
|
|
|
rc = ORTE_SUCCESS;
|
|
return rc;
|
|
}
|
|
|
|
/*
|
|
* Register the HNP's errmgr functions to be called when the job encounters
|
|
* certain pre-identified problem states.
|
|
*
|
|
* NOTE: It is imperative that ONLY the HNP perform this registration!
|
|
*/
|
|
int orte_errmgr_hnp_register_job(orte_jobid_t job)
|
|
{
|
|
/* we need to setup two counters and their corresponding triggers - one
|
|
* to alert us when something fails to launch, and another for when
|
|
* someone aborts
|
|
*/
|
|
int rc;
|
|
|
|
OPAL_TRACE(1);
|
|
|
|
/* define the ABORT trigger to fire when any process aborts */
|
|
if (ORTE_SUCCESS != (rc = orte_smr.define_alert_monitor(job, ORTE_NUM_ABORTED_TRIGGER,
|
|
ORTE_PROC_NUM_ABORTED, 0, 1, true,
|
|
orte_errmgr_hnp_proc_aborted, NULL))) {
|
|
ORTE_ERROR_LOG(rc);
|
|
return rc;
|
|
}
|
|
|
|
/* define the FAILED_LAUNCH trigger to fire when the launch fails */
|
|
if (ORTE_SUCCESS != (rc = orte_smr.define_alert_monitor(job, ORTE_FAILED_TO_START_TRIGGER,
|
|
ORTE_PROC_NUM_FAILED_START, 0, 1, true,
|
|
orte_errmgr_hnp_incomplete_start, NULL))) {
|
|
ORTE_ERROR_LOG(rc);
|
|
return rc;
|
|
}
|
|
|
|
return ORTE_SUCCESS;
|
|
}
|