7bee71aa59
Add a new function to opal_progress that tells us our recursion depth to support that solution. Yes, I know this sounds picky, but good ol' Jeff managed to make it happen by driving his cluster near to death... Also ensure that we declare "failed" for the daemon job when daemons fail instead of the application job. This is important so that orte knows that it cannot use xcast to tell daemons to "exit", nor should it expect all daemons to respond. Otherwise, it is possible to hang. After lots of testing, decide to default (again) to slurm detecting failed orteds. This proved necessary to avoid rather annoying hangs that were difficult to recover from. There are conditions where slurm will fail to launch all daemons (slurm folks are working on it), and yet again, good ol' Jeff managed to find both of them. Thanks you Jeff! :-/ This commit was SVN r18611.
234 строки
7.0 KiB
C
234 строки
7.0 KiB
C
/*
|
|
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
|
* University Research and Technology
|
|
* Corporation. All rights reserved.
|
|
* Copyright (c) 2004-2006 The University of Tennessee and The University
|
|
* of Tennessee Research Foundation. All rights
|
|
* reserved.
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
* University of Stuttgart. All rights reserved.
|
|
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
* All rights reserved.
|
|
* Copyright (c) 2006 Los Alamos National Security, LLC. All rights
|
|
* reserved.
|
|
*
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
|
|
/**
|
|
* @file
|
|
*
|
|
* Progress engine for Open MPI
|
|
*/
|
|
|
|
#ifndef OPAL_RUNTIME_OPAL_PROGRESS_H
|
|
#define OPAL_RUNTIME_OPAL_PROGRESS_H
|
|
|
|
#if defined(c_plusplus) || defined(__cplusplus)
|
|
extern "C" {
|
|
#endif
|
|
|
|
#include "opal/threads/mutex.h"
|
|
|
|
/**
|
|
* Initialize the progress engine
|
|
*
|
|
* Initialize the progress engine, including constructing the
|
|
* proper locks and allocating space for the progress registration
|
|
* functions. At this point, any function in the progress engine
|
|
* interface may be called.
|
|
*/
|
|
OPAL_DECLSPEC int opal_progress_init(void);
|
|
|
|
|
|
/**
|
|
* Shut down the progress engine
|
|
*
|
|
* Shut down the progress engine. This includes deregistering all
|
|
* registered callbacks and freeing all resources. After finalize
|
|
* returns, no calls into the progress interface are allowed.
|
|
*/
|
|
OPAL_DECLSPEC int opal_progress_finalize(void);
|
|
|
|
|
|
/**
|
|
* Progress all pending events
|
|
*
|
|
* Progress all pending events. All registered event handlers will be
|
|
* called every call into opal_progress(). The event library will be
|
|
* called if opal_progress_event_users is greater than 0 (adjustments
|
|
* can be made by calling opal_progress_event_users_add() and
|
|
* opal_progress_event_users_delete()) or the time since the last call
|
|
* into the event library is greater than the progress tick rate (by
|
|
* default, 10ms).
|
|
*/
|
|
OPAL_DECLSPEC void opal_progress(void);
|
|
|
|
|
|
/**
|
|
* Control how the event library is called
|
|
*
|
|
* Adjust the flags argument used to call opal_event_loop() from
|
|
* opal_progress(). The default argument is OPAL_EVLOOP_ONELOOP,
|
|
* meaning that the call to opal_event_loop() will block pending
|
|
* events, but may block for a period of time.
|
|
*
|
|
* @param flags One of the valid vlags argument to
|
|
* opal_event_loop().
|
|
* @return Previous value of flags used to call
|
|
* opal_event_loop().
|
|
*/
|
|
OPAL_DECLSPEC int opal_progress_set_event_flag(int flags);
|
|
|
|
|
|
/**
|
|
* Increase the number of users of the event library
|
|
*
|
|
* Increase the number of users of the event library. This count is
|
|
* used by opal_progress to determine if opal_event_loop() should be
|
|
* called every call to opal_progress() or only after a time has
|
|
* elapsed since the last call (by default, 10ms). The count defaults
|
|
* to 0, meaning that opal_progress_event_users_increment() must be
|
|
* called at least once for the event loop to be called on every entry
|
|
* to opal_progress().
|
|
*
|
|
*/
|
|
OPAL_DECLSPEC void opal_progress_event_users_increment(void);
|
|
|
|
|
|
/**
|
|
* Decrease the number of users of the event library
|
|
*
|
|
* Decrease the number of users of the event library. This count is
|
|
* used by opal_progress to determine if opal_event_loop() should be
|
|
* called every call to opal_progress() or only after a time has
|
|
* elapsed since the last call (by default, 10ms).
|
|
*/
|
|
OPAL_DECLSPEC void opal_progress_event_users_decrement(void);
|
|
|
|
|
|
/**
|
|
* Set whether opal_progress() should yield when idle
|
|
*
|
|
* Set whether opal_progress() should yield the processor (either by
|
|
* sched_yield() or SwitchToThread()) if no events were progressed
|
|
* during the progress loop. The return value of the callback
|
|
* functions is used to determine whether or not yielding is required.
|
|
* By default, the event loop will yield when the progress function is
|
|
* idle.
|
|
*
|
|
* @param yieldopt Whether to yield when idle.
|
|
* @return Previous value of the yield_when_idle option.
|
|
*/
|
|
OPAL_DECLSPEC bool opal_progress_set_yield_when_idle(bool yieldopt);
|
|
|
|
|
|
/**
|
|
* Set time between calls into the event library
|
|
*
|
|
* Set time between calls into the event library when there are no
|
|
* users of the event library (set by
|
|
* opal_progress_event_users_increment() and
|
|
* opal_progress_event_users_decrement()).
|
|
*
|
|
* @param polltime Time (in microseconds) between calls to the event
|
|
* library
|
|
*/
|
|
OPAL_DECLSPEC void opal_progress_set_event_poll_rate(int microseconds);
|
|
|
|
|
|
/**
|
|
* Progress callback function typedef
|
|
*
|
|
* Prototype for the a progress function callback. Progress function
|
|
* callbacks can be registered with opal_progress_register() and
|
|
* deregistered with opal_progress_deregister(). It should be noted
|
|
* that either registering or deregistering a function callback is an
|
|
* extraordinarily expensive operation and should not be used for
|
|
* potentially short callback lifetimes.
|
|
*
|
|
* @return Number of events progressed during the callback
|
|
*/
|
|
typedef int (*opal_progress_callback_t)(void);
|
|
|
|
|
|
/**
|
|
* Register an event to be progressed
|
|
*
|
|
* Register an event to be progressed during calls to opal_progress().
|
|
* Please read the note in opal_progress_callback_t.
|
|
*/
|
|
OPAL_DECLSPEC int opal_progress_register(opal_progress_callback_t cb);
|
|
|
|
|
|
/**
|
|
* Deregister previously registered event
|
|
*
|
|
* Deregister an event to be progressed during calls to opal_progress().
|
|
* Please read the note in opal_progress_callback_t.
|
|
*/
|
|
OPAL_DECLSPEC int opal_progress_unregister(opal_progress_callback_t cb);
|
|
|
|
|
|
OPAL_DECLSPEC extern volatile int32_t opal_progress_thread_count;
|
|
OPAL_DECLSPEC extern int opal_progress_spin_count;
|
|
|
|
static inline bool opal_progress_threads(void)
|
|
{
|
|
return (opal_progress_thread_count > 0);
|
|
}
|
|
|
|
|
|
/**
|
|
* Progress until flag is true or poll iterations completed
|
|
*/
|
|
static inline bool opal_progress_spin(volatile bool* complete)
|
|
{
|
|
int32_t c;
|
|
OPAL_THREAD_ADD32(&opal_progress_thread_count,1);
|
|
for (c = 0; c < opal_progress_spin_count; c++) {
|
|
if (true == *complete) {
|
|
OPAL_THREAD_ADD32(&opal_progress_thread_count,-1);
|
|
return true;
|
|
}
|
|
opal_progress();
|
|
}
|
|
OPAL_THREAD_ADD32(&opal_progress_thread_count,-1);
|
|
return false;
|
|
}
|
|
|
|
|
|
/**
|
|
* \internal
|
|
* Don't use this variable; use the opal_progress_recursion_depth()
|
|
* function.
|
|
*/
|
|
OPAL_DECLSPEC extern
|
|
#if OMPI_HAVE_THREAD_SUPPORT
|
|
volatile
|
|
#endif
|
|
uint32_t opal_progress_recursion_depth_counter;
|
|
|
|
/**
|
|
* Return the current level of recursion -- 0 means that we are not
|
|
* under an opal_progress() call at all. 1 means that you're in the
|
|
* top-level opal_progress() function (i.e., not deep in recursion).
|
|
* Higher values mean that you're that many levels deep in recursion.
|
|
*/
|
|
static inline uint32_t opal_progress_recursion_depth(void)
|
|
{
|
|
return opal_progress_recursion_depth_counter;
|
|
}
|
|
|
|
|
|
#if defined(c_plusplus) || defined(__cplusplus)
|
|
}
|
|
#endif
|
|
|
|
#endif
|
|
|