b6196e8a39
This commit introduces the necessary logic to avoid that conflict. If a PLS component can identify that a daemon has failed, then we will set a flag indicating that fact. The xcast system will subsequently check that flag and, if it is set, will send all messages direct to the recipient. In the case of "kill local procs" and "terminate", the messages will go directly to each orted, thus bypassing any orted that has failed. In addition, the xcast system will -not- wait for the messages to complete, but will return immediately (i.e., operate in non-blocking mode). Orterun will wait (via an event timer) for a period of time based on the number of daemons in the system to allow the messages to attempt to be delivered - at the end of that time, orterun will simply exit, alerting the user to the problem and -strongly- recommending they run orte-clean. I could only test this on slurm for the case where all daemons unexpectedly died - srun apparently only executes its waitpid callback when all launched functions terminate. I have asked that Jeff integrate this capability into the OOB as he is working on it so that we execute it whenever a socket to an orted is unexpectedly closed. Meantime, the functionality will rarely get called, but at least the logic is available for anyone whose environment can support it. This commit was SVN r16451.
69 строки
2.0 KiB
C
69 строки
2.0 KiB
C
/*
|
|
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
|
* University Research and Technology
|
|
* Corporation. All rights reserved.
|
|
* Copyright (c) 2004-2006 The University of Tennessee and The University
|
|
* of Tennessee Research Foundation. All rights
|
|
* reserved.
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
* University of Stuttgart. All rights reserved.
|
|
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
* All rights reserved.
|
|
* Copyright (c) 2007 Sun Microsystems, Inc. All rights reserved.
|
|
* Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
|
|
/**
|
|
* @file
|
|
*
|
|
* Global params for OpenRTE
|
|
*/
|
|
#ifndef ORTE_RUNTIME_PARAM_H
|
|
#define ORTE_RUNTIME_PARAM_H
|
|
|
|
#include "orte_config.h"
|
|
|
|
#ifdef HAVE_SYS_TIME_H
|
|
#include <sys/time.h>
|
|
#endif
|
|
|
|
#include "opal/threads/mutex.h"
|
|
#include "opal/threads/condition.h"
|
|
|
|
#if defined(c_plusplus) || defined(__cplusplus)
|
|
extern "C" {
|
|
#endif
|
|
|
|
/* globals used by RTE - instanced in orte_params.c */
|
|
|
|
ORTE_DECLSPEC extern bool orte_debug_flag, orte_reuse_daemons, orte_timing;
|
|
ORTE_DECLSPEC extern bool orte_debug_daemons_flag, orte_debug_daemons_file_flag;
|
|
ORTE_DECLSPEC extern bool orte_infrastructure, orted_spin_flag, orte_no_daemonize_flag;
|
|
|
|
ORTE_DECLSPEC extern bool orte_daemon_died;
|
|
|
|
ORTE_DECLSPEC extern struct timeval orte_abort_timeout;
|
|
|
|
ORTE_DECLSPEC extern char **orte_launch_environ;
|
|
ORTE_DECLSPEC extern char **orted_cmd_line;
|
|
ORTE_DECLSPEC extern opal_mutex_t orted_comm_mutex;
|
|
ORTE_DECLSPEC extern opal_condition_t orted_comm_cond;
|
|
ORTE_DECLSPEC extern bool orte_orterun;
|
|
ORTE_DECLSPEC extern bool orted_comm_exit_cond;
|
|
|
|
/**
|
|
* Whether ORTE is initialized or not
|
|
*/
|
|
ORTE_DECLSPEC extern bool orte_initialized;
|
|
|
|
#if defined(c_plusplus) || defined(__cplusplus)
|
|
}
|
|
#endif
|
|
|
|
#endif /* ORTE_RUNTIME_PARAM_H */
|