fceabb2498
This is a fairly intrusive change, but outside of the moving of opal/event to opal/mca/event, the only changes involved (a) changing all calls to opal_event functions to reflect the new framework instead, and (b) ensuring that all opal_event_t objects are properly constructed since they are now true opal_objects. Note: Shiqing has just returned from vacation and has not yet had a chance to complete the Windows integration. Thus, this commit almost certainly breaks Windows support on the trunk. However, I want this to have a chance to soak for as long as possible before I become less available a week from today (going to be at a class for 5 days, and thus will only be sparingly available) so we can find and fix any problems. Biggest change is moving the libevent code from opal/event to a new opal/mca/event framework. This was done to make it much easier to update libevent in the future. New versions can be inserted as a new component and tested in parallel with the current version until validated, then we can remove the earlier version if we so choose. This is a statically built framework ala installdirs, so only one component will build at a time. There is no selection logic - the sole compiled component simply loads its function pointers into the opal_event struct. I have gone thru the code base and converted all the libevent calls I could find. However, I cannot compile nor test every environment. It is therefore quite likely that errors remain in the system. Please keep an eye open for two things: 1. compile-time errors: these will be obvious as calls to the old functions (e.g., opal_evtimer_new) must be replaced by the new framework APIs (e.g., opal_event.evtimer_new) 2. run-time errors: these will likely show up as segfaults due to missing constructors on opal_event_t objects. It appears that it became a typical practice for people to "init" an opal_event_t by simply using memset to zero it out. This will no longer work - you must either OBJ_NEW or OBJ_CONSTRUCT an opal_event_t. I tried to catch these cases, but may have missed some. Believe me, you'll know when you hit it. There is also the issue of the new libevent "no recursion" behavior. As I described on a recent email, we will have to discuss this and figure out what, if anything, we need to do. This commit was SVN r23925.
413 строки
14 KiB
C
413 строки
14 KiB
C
/*
|
|
* Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
|
|
* University Research and Technology
|
|
* Corporation. All rights reserved.
|
|
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
|
* of Tennessee Research Foundation. All rights
|
|
* reserved.
|
|
* Copyright (c) 2004-2007 High Performance Computing Center Stuttgart,
|
|
* University of Stuttgart. All rights reserved.
|
|
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
* All rights reserved.
|
|
* Copyright (c) 2008 Cisco Systems, Inc. All rights reserved.
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
|
|
/**
|
|
* @file
|
|
*
|
|
* Checkpoint functionality for Open MPI
|
|
*/
|
|
|
|
#include "opal_config.h"
|
|
#include "opal/mca/crs/crs.h"
|
|
#include "opal/mca/event/event.h"
|
|
#include "opal/util/output.h"
|
|
#include "opal/prefetch.h"
|
|
|
|
#ifndef OPAL_CR_H
|
|
#define OPAL_CR_H
|
|
|
|
|
|
BEGIN_C_DECLS
|
|
|
|
/*
|
|
* Some defines shared with opal-[checkpoint|restart] commands
|
|
*/
|
|
#define OPAL_CR_DONE ((char) 0)
|
|
#define OPAL_CR_ACK ((char) 1)
|
|
#define OPAL_CR_CHECKPOINT ((char) 2)
|
|
#define OPAL_CR_NAMED_PROG_R ("opal_cr_prog_read")
|
|
#define OPAL_CR_NAMED_PROG_W ("opal_cr_prog_write")
|
|
#define OPAL_CR_BASE_ENV_NAME ("opal_cr_restart-env")
|
|
|
|
/*
|
|
* Possible responses to a checkpoint request from opal-checkpoint
|
|
*/
|
|
enum opal_cr_ckpt_cmd_state_t {
|
|
OPAL_CHECKPOINT_CMD_START, /* Checkpoint is starting on this request */
|
|
OPAL_CHECKPOINT_CMD_IN_PROGRESS, /* Checkpoint is currently running */
|
|
OPAL_CHECKPOINT_CMD_NULL, /* Checkpoint cannot be started because it is not supported */
|
|
OPAL_CHECKPOINT_CMD_ERROR, /* An error occurred such that the checkpoint cannot be completed */
|
|
/* State of the checkpoint operation */
|
|
OPAL_CR_STATUS_NONE, /* No checkpoint in progress */
|
|
OPAL_CR_STATUS_REQUESTED, /* Checkpoint has been requested */
|
|
OPAL_CR_STATUS_RUNNING, /* Checkpoint is currently running */
|
|
OPAL_CR_STATUS_TERM, /* Checkpoint is running and will terminate process upon completion */
|
|
/* State of the continue operation */
|
|
OPAL_CR_STATUS_CONTINUE,
|
|
/* State of the restart operation */
|
|
OPAL_CR_STATUS_RESTART_PRE,
|
|
OPAL_CR_STATUS_RESTART_POST
|
|
};
|
|
typedef enum opal_cr_ckpt_cmd_state_t opal_cr_ckpt_cmd_state_t;
|
|
|
|
/* An output handle to be used by the cr runtime
|
|
* functionality as an argument to opal_output() */
|
|
OPAL_DECLSPEC extern int opal_cr_output;
|
|
|
|
/* Directory containing the named pipes for communication
|
|
* with the opal-checkpoint tool */
|
|
OPAL_DECLSPEC extern char * opal_cr_pipe_dir;
|
|
|
|
/* Signal that opal-checkpoint uses to contact the
|
|
* application process */
|
|
OPAL_DECLSPEC extern int opal_cr_entry_point_signal;
|
|
|
|
/* If Checkpointing is enabled in this application */
|
|
OPAL_DECLSPEC extern bool opal_cr_is_enabled;
|
|
|
|
/* If the application running is a tool
|
|
* (e.g., opal-checkpoint, orted, ...) */
|
|
OPAL_DECLSPEC extern bool opal_cr_is_tool;
|
|
|
|
/* If a checkpoint has been requested */
|
|
OPAL_DECLSPEC extern int opal_cr_checkpoint_request;
|
|
|
|
/* The current state of a checkpoint operation */
|
|
OPAL_DECLSPEC extern int opal_cr_checkpointing_state;
|
|
|
|
#if OPAL_ENABLE_CRDEBUG == 1
|
|
/* Whether or not C/R Debugging is enabled for this process */
|
|
OPAL_DECLSPEC extern int MPIR_debug_with_checkpoint;
|
|
|
|
/*
|
|
* Set/clear the current thread id for the checkpointing thread
|
|
*/
|
|
OPAL_DECLSPEC int opal_cr_debug_set_current_ckpt_thread_self(void);
|
|
OPAL_DECLSPEC int opal_cr_debug_clear_current_ckpt_thread(void);
|
|
|
|
/*
|
|
* This MPI Debugger function needs to be accessed here and have a specific
|
|
* name. Thus we are breaking the traditional naming conventions to provide this functionality.
|
|
*/
|
|
OPAL_DECLSPEC int MPIR_checkpoint_debugger_detach(void);
|
|
|
|
/**
|
|
* A tight loop to wait for debugger to release this process from the
|
|
* breakpoint.
|
|
*/
|
|
OPAL_DECLSPEC void *MPIR_checkpoint_debugger_breakpoint(void);
|
|
|
|
/**
|
|
* A function for the debugger or CRS to force all threads into
|
|
*/
|
|
OPAL_DECLSPEC void *MPIR_checkpoint_debugger_waitpoint(void);
|
|
|
|
/**
|
|
* A signal handler to force all threads to wait when debugger detaches
|
|
*/
|
|
OPAL_DECLSPEC void MPIR_checkpoint_debugger_signal_handler(int signo);
|
|
#endif
|
|
|
|
/*
|
|
* Refresh environment variables after a restart
|
|
*/
|
|
OPAL_DECLSPEC int opal_cr_refresh_environ(int prev_pid);
|
|
|
|
/*
|
|
* If this is an application that doesn't want to have
|
|
* a notification callback installed, set this to false.
|
|
* To see the effect, this must be called before opal_cr_init().
|
|
* Default: Enabled
|
|
*/
|
|
OPAL_DECLSPEC int opal_cr_set_enabled(bool);
|
|
|
|
/**
|
|
* Initialize the notification and coordination
|
|
* elements.
|
|
*/
|
|
OPAL_DECLSPEC int opal_cr_init(void);
|
|
|
|
/**
|
|
* Finalize the notification and coordination
|
|
* elements.
|
|
*/
|
|
OPAL_DECLSPEC int opal_cr_finalize(void);
|
|
|
|
/*************************************************
|
|
* Check to see if a checkpoint has been requested
|
|
*
|
|
* When the checkpoint thread is disabled:
|
|
* This will be checked whenever the MPI Library
|
|
* is entered by the application. It will stop
|
|
* the application for the duration of the entire
|
|
* checkpoint.
|
|
* When the checkpoint thread is enabled:
|
|
* The request is handled in the thread parallel
|
|
* with the execution of the program regardless
|
|
* of where the program is in exection.
|
|
* The problem with this method is that it
|
|
* requires the support of progress threads
|
|
* which is currently not working properly :/
|
|
*
|
|
*************************************************/
|
|
OPAL_DECLSPEC void opal_cr_test_if_checkpoint_ready(void);
|
|
|
|
/* If the checkpoint operation should be stalled to
|
|
* wait for another sevice to complete before
|
|
* continuing with the checkpoint */
|
|
OPAL_DECLSPEC extern bool opal_cr_stall_check;
|
|
OPAL_DECLSPEC extern bool opal_cr_currently_stalled;
|
|
|
|
#if OPAL_ENABLE_FT_THREAD == 1
|
|
/* Some thread functions */
|
|
OPAL_DECLSPEC void opal_cr_thread_init_library(void);
|
|
OPAL_DECLSPEC void opal_cr_thread_finalize_library(void);
|
|
OPAL_DECLSPEC void opal_cr_thread_abort_library(void);
|
|
OPAL_DECLSPEC void opal_cr_thread_enter_library(void);
|
|
OPAL_DECLSPEC void opal_cr_thread_exit_library(void);
|
|
OPAL_DECLSPEC void opal_cr_thread_noop_progress(void);
|
|
#endif /* OPAL_ENABLE_FT_THREAD == 1 */
|
|
|
|
/*
|
|
* If not using FT then make the #defines noops
|
|
*/
|
|
#if OPAL_ENABLE_FT == 0 || OPAL_ENABLE_FT_CR == 0
|
|
#define OPAL_CR_TEST_CHECKPOINT_READY() ;
|
|
#define OPAL_CR_TEST_CHECKPOINT_READY_STALL() ;
|
|
#define OPAL_CR_INIT_LIBRARY() ;
|
|
#define OPAL_CR_FINALIZE_LIBRARY() ;
|
|
#define OPAL_CR_ABORT_LIBRARY() ;
|
|
#define OPAL_CR_ENTER_LIBRARY() ;
|
|
#define OPAL_CR_EXIT_LIBRARY() ;
|
|
#define OPAL_CR_NOOP_PROGRESS() ;
|
|
#endif /* #if OPAL_ENABLE_FT == 0 || OPAL_ENABLE_FT_CR == 0 */
|
|
|
|
/*
|
|
* If using FT
|
|
*/
|
|
#if OPAL_ENABLE_FT_CR == 1
|
|
#define OPAL_CR_TEST_CHECKPOINT_READY() \
|
|
{ \
|
|
if(OPAL_UNLIKELY(opal_cr_is_enabled) ) { \
|
|
opal_cr_test_if_checkpoint_ready(); \
|
|
} \
|
|
}
|
|
|
|
#define OPAL_CR_TEST_CHECKPOINT_READY_STALL() \
|
|
{ \
|
|
if(OPAL_UNLIKELY(opal_cr_is_enabled && !opal_cr_stall_check)) { \
|
|
opal_cr_test_if_checkpoint_ready(); \
|
|
} \
|
|
}
|
|
|
|
/* If *not* using FT thread */
|
|
#if OPAL_ENABLE_FT_THREAD == 0
|
|
#define OPAL_CR_INIT_LIBRARY() OPAL_CR_TEST_CHECKPOINT_READY();
|
|
#define OPAL_CR_FINALIZE_LIBRARY() OPAL_CR_TEST_CHECKPOINT_READY();
|
|
#define OPAL_CR_ABORT_LIBRARY() OPAL_CR_TEST_CHECKPOINT_READY();
|
|
#define OPAL_CR_ENTER_LIBRARY() OPAL_CR_TEST_CHECKPOINT_READY();
|
|
#define OPAL_CR_EXIT_LIBRARY() OPAL_CR_TEST_CHECKPOINT_READY();
|
|
#define OPAL_CR_NOOP_PROGRESS() OPAL_CR_TEST_CHECKPOINT_READY();
|
|
#endif /* OPAL_ENABLE_FT_THREAD == 0 */
|
|
|
|
/* If using FT thread */
|
|
#if OPAL_ENABLE_FT_THREAD == 1
|
|
#define OPAL_CR_INIT_LIBRARY() \
|
|
{ \
|
|
opal_cr_thread_init_library(); \
|
|
}
|
|
#define OPAL_CR_FINALIZE_LIBRARY() \
|
|
{ \
|
|
opal_cr_thread_finalize_library(); \
|
|
}
|
|
#define OPAL_CR_ABORT_LIBRARY() \
|
|
{ \
|
|
opal_cr_thread_abort_library(); \
|
|
}
|
|
#define OPAL_CR_ENTER_LIBRARY() \
|
|
{ \
|
|
opal_cr_thread_enter_library(); \
|
|
}
|
|
#define OPAL_CR_EXIT_LIBRARY() \
|
|
{ \
|
|
opal_cr_thread_exit_library(); \
|
|
}
|
|
#define OPAL_CR_NOOP_PROGRESS() \
|
|
{ \
|
|
opal_cr_thread_noop_progress(); \
|
|
}
|
|
#endif /* OPAL_ENABLE_FT_THREAD == 1 */
|
|
|
|
#endif /* OPAL_ENABLE_FT_CR == 1 */
|
|
|
|
/*******************************
|
|
* Notification Routines
|
|
*******************************/
|
|
/*******************************
|
|
* Notification Routines
|
|
*******************************/
|
|
/**
|
|
* A function to respond to the async checkpoint request
|
|
* this is useful when figuring out who should respond
|
|
* when stalling.
|
|
*/
|
|
typedef int (*opal_cr_notify_callback_fn_t) (opal_cr_ckpt_cmd_state_t);
|
|
|
|
OPAL_DECLSPEC int opal_cr_reg_notify_callback
|
|
(opal_cr_notify_callback_fn_t new_func,
|
|
opal_cr_notify_callback_fn_t *prev_func);
|
|
|
|
/**
|
|
* Function to go through the INC
|
|
* - Call Registered INC_Coord(CHECKPOINT)
|
|
* - Call the CRS.checkpoint()
|
|
* - Call Registered INC_Coord(state)
|
|
*/
|
|
OPAL_DECLSPEC int opal_cr_inc_core(pid_t pid,
|
|
opal_crs_base_snapshot_t *snapshot,
|
|
opal_crs_base_ckpt_options_t *options,
|
|
int *state);
|
|
|
|
OPAL_DECLSPEC int opal_cr_inc_core_prep(void);
|
|
OPAL_DECLSPEC int opal_cr_inc_core_ckpt(pid_t pid,
|
|
opal_crs_base_snapshot_t *snapshot,
|
|
opal_crs_base_ckpt_options_t *options,
|
|
int *state);
|
|
OPAL_DECLSPEC int opal_cr_inc_core_recover(int state);
|
|
|
|
|
|
/*******************************
|
|
* User Coordination Routines
|
|
*******************************/
|
|
typedef enum {
|
|
OMPI_CR_INC_PRE_CRS_PRE_MPI = 0,
|
|
OMPI_CR_INC_PRE_CRS_POST_MPI = 1,
|
|
OMPI_CR_INC_CRS_PRE_CKPT = 2,
|
|
OMPI_CR_INC_CRS_POST_CKPT = 3,
|
|
OMPI_CR_INC_POST_CRS_PRE_MPI = 4,
|
|
OMPI_CR_INC_POST_CRS_POST_MPI = 5,
|
|
OMPI_CR_INC_MAX = 6
|
|
} opal_cr_user_inc_callback_event_t;
|
|
|
|
typedef enum {
|
|
OMPI_CR_INC_STATE_PREPARE = 0,
|
|
OMPI_CR_INC_STATE_CONTINUE = 1,
|
|
OMPI_CR_INC_STATE_RESTART = 2,
|
|
OMPI_CR_INC_STATE_ERROR = 3
|
|
} opal_cr_user_inc_callback_state_t;
|
|
|
|
/**
|
|
* User coordination callback routine
|
|
*/
|
|
typedef int (*opal_cr_user_inc_callback_fn_t)(opal_cr_user_inc_callback_event_t event,
|
|
opal_cr_user_inc_callback_state_t state);
|
|
|
|
OPAL_DECLSPEC int opal_cr_user_inc_register_callback
|
|
(opal_cr_user_inc_callback_event_t event,
|
|
opal_cr_user_inc_callback_fn_t function,
|
|
opal_cr_user_inc_callback_fn_t *prev_function);
|
|
|
|
OPAL_DECLSPEC int trigger_user_inc_callback(opal_cr_user_inc_callback_event_t event,
|
|
opal_cr_user_inc_callback_state_t state);
|
|
|
|
|
|
/*******************************
|
|
* Coordination Routines
|
|
*******************************/
|
|
/**
|
|
* Coordination callback routine signature
|
|
*/
|
|
typedef int (*opal_cr_coord_callback_fn_t) (int);
|
|
|
|
/**
|
|
* Register a checkpoint coodination routine
|
|
* for a higher level.
|
|
*/
|
|
OPAL_DECLSPEC int opal_cr_reg_coord_callback
|
|
(opal_cr_coord_callback_fn_t new_func,
|
|
opal_cr_coord_callback_fn_t *prev_func);
|
|
|
|
/**
|
|
* OPAL Checkpoint Coordination Routine
|
|
*/
|
|
OPAL_DECLSPEC int opal_cr_coord(int state);
|
|
|
|
/**
|
|
* Checkpoint life-cycle timing
|
|
*/
|
|
OPAL_DECLSPEC void opal_cr_set_time(int idx);
|
|
OPAL_DECLSPEC void opal_cr_display_all_timers(void);
|
|
OPAL_DECLSPEC void opal_cr_clear_timers(void);
|
|
|
|
OPAL_DECLSPEC extern bool opal_cr_timing_enabled;
|
|
OPAL_DECLSPEC extern bool opal_cr_timing_barrier_enabled;
|
|
OPAL_DECLSPEC extern int opal_cr_timing_my_rank;
|
|
OPAL_DECLSPEC extern int opal_cr_timing_target_rank;
|
|
|
|
|
|
#define OPAL_CR_TIMER_ENTRY0 0
|
|
#define OPAL_CR_TIMER_ENTRY1 1
|
|
#define OPAL_CR_TIMER_ENTRY2 2
|
|
#define OPAL_CR_TIMER_CRCPBR0 3
|
|
#define OPAL_CR_TIMER_CRCP0 4
|
|
#define OPAL_CR_TIMER_CRCPBR1 5
|
|
#define OPAL_CR_TIMER_P2P0 6
|
|
#define OPAL_CR_TIMER_P2P1 7
|
|
#define OPAL_CR_TIMER_P2PBR0 8
|
|
#define OPAL_CR_TIMER_CORE0 9
|
|
#define OPAL_CR_TIMER_CORE1 10
|
|
#define OPAL_CR_TIMER_COREBR0 11
|
|
#define OPAL_CR_TIMER_P2P2 12
|
|
#define OPAL_CR_TIMER_P2PBR1 13
|
|
#define OPAL_CR_TIMER_P2P3 14
|
|
#define OPAL_CR_TIMER_P2PBR2 15
|
|
#define OPAL_CR_TIMER_CRCP1 16
|
|
#define OPAL_CR_TIMER_COREBR1 17
|
|
#define OPAL_CR_TIMER_CORE2 18
|
|
#define OPAL_CR_TIMER_ENTRY3 19
|
|
#define OPAL_CR_TIMER_ENTRY4 20
|
|
#define OPAL_CR_TIMER_MAX 21
|
|
|
|
|
|
#define OPAL_CR_CLEAR_TIMERS() \
|
|
{ \
|
|
if(OPAL_UNLIKELY(opal_cr_timing_enabled > 0)) { \
|
|
opal_cr_clear_timers(); \
|
|
} \
|
|
}
|
|
|
|
#define OPAL_CR_SET_TIMER(idx) \
|
|
{ \
|
|
if(OPAL_UNLIKELY(opal_cr_timing_enabled > 0)) { \
|
|
opal_cr_set_time(idx); \
|
|
} \
|
|
}
|
|
|
|
#define OPAL_CR_DISPLAY_ALL_TIMERS() \
|
|
{ \
|
|
if(OPAL_UNLIKELY(opal_cr_timing_enabled > 0)) { \
|
|
opal_cr_display_all_timers(); \
|
|
} \
|
|
}
|
|
|
|
END_C_DECLS
|
|
|
|
#endif /* OPAL_CR_H */
|
|
|