5406fdfb80
This commit looks larger than it really is since it includes a fair amount of code cleanup.
The SIGSTOP/SIGCONT+checkpointing work uses some of the functionality in r20391. Basic use case below (note that the checkpoint generated is useable as usual if the stopped application is terminated).
{{{
shell 1) mpirun -np 2 -am ft-enable-cr my-app
... running ...
shell 2) ompi-checkpoint --stop -v MPIRUN_PID
[localhost:001300] [ 0.00 / 0.20] Requested - ...
[localhost:001300] [ 0.00 / 0.20] Pending - ...
[localhost:001300] [ 0.01 / 0.21] Running - ...
[localhost:001300] [ 1.01 / 1.22] Stopped - ompi_global_snapshot_1234.ckpt
Snapshot Ref.: 0 ompi_global_snapshot_1234.ckpt
shell 2) killall -CONT mpirun
... Application Continues execution in shell 1 ...
}}}
Other items in this commit are mostly cleanup that has been sitting off-trunk for too long:
* Add a new {{{opal_crs_base_ckpt_options_t}}} type that encapsulates the various options that could be passed to the CRS. Currently only TERM and STOP, but this makes adding others ''much'' easier.
* Eliminate ORTE_SNAPC_CKPT_STATE_PENDING_TERM, since it served a redundant purpose with the new options type.
* Lay some basic ground work for some future features.
This commit was SVN r21995.
The following SVN revision numbers were found above:
r20391 --> open-mpi/ompi@0704b98668
324 строки
10 KiB
C
324 строки
10 KiB
C
/*
|
|
* Copyright (c) 2004-2009 The Trustees of Indiana University and Indiana
|
|
* University Research and Technology
|
|
* Corporation. All rights reserved.
|
|
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
|
* of Tennessee Research Foundation. All rights
|
|
* reserved.
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
* University of Stuttgart. All rights reserved.
|
|
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
* All rights reserved.
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
/**
|
|
* @file
|
|
*
|
|
* Snapshot Coordination (SNAPC) Interface
|
|
*
|
|
* Terminology:
|
|
* ------------
|
|
* Global Snapshot Coordinator:
|
|
* - HNP(s) coordination function.
|
|
* Local Snapshot Coordinator
|
|
* - VHNP(s) [e.g., orted] coordination function
|
|
* Application Snapshot Coordinator
|
|
* - Application level coordinaton function
|
|
* Local Snapshot
|
|
* - Snapshot generated by a single process in the parallel job
|
|
* Local Snapshot Reference
|
|
* - A generic reference to the physical Local Snapshot
|
|
* Global Snapshot
|
|
* - Snapshot generated for the entire parallel job
|
|
* Global Snapshot Reference
|
|
* - A generic reference to the physical Global Snapshot
|
|
*
|
|
* General Description:
|
|
* ---------------------
|
|
* This framework is tasked with:
|
|
* - Initiating the checkpoint in the system
|
|
* - Physically moving the local snapshot files to a location
|
|
* Initially this location, is the node on which the Head Node Process (HNP)
|
|
* is running, but later this will be a replicated checkpoint server or
|
|
* the like.
|
|
* - Generating a 'global snapshot handle' that the user can use to restart
|
|
* the parallel job.
|
|
*
|
|
* Each component will have 3 teirs of behavior that must behave in concert:
|
|
* - Global Snapshot Coordinator
|
|
* This is the HNPs tasks. Mostly distributing the notification of the
|
|
* checkpoint, and then compiling the physical and virtual nature of the
|
|
* global snapshot handle.
|
|
* - Local Snapshot Coordinator
|
|
* This is the VHNPs (or orted, if available) tasks. This will involve
|
|
* working with the Global Snapshot Coordinator to route the physical
|
|
* and virtual 'local snapshot's from the application to the desired
|
|
* location. This process must also notify the Global Snapshot Coordinator
|
|
* when it's set of processes have completed the checkpoint.
|
|
* - Application Snapshot Coordinator
|
|
* This is the application level coordinator. This is very light, just
|
|
* a subscription to be triggered when it needs to checkpoint, and then,
|
|
* once finished with the checkpoint, notify the Local Snapshot Coordinator
|
|
* that it is complete.
|
|
* If there is no orted (so no bootproxy), then the application assumes the
|
|
* responsibility of the Local Snapshot Coordinator as well.
|
|
*
|
|
*/
|
|
|
|
#ifndef MCA_SNAPC_H
|
|
#define MCA_SNAPC_H
|
|
|
|
#include "orte_config.h"
|
|
#include "orte/constants.h"
|
|
#include "orte/types.h"
|
|
|
|
#include "opal/mca/mca.h"
|
|
#include "opal/mca/base/base.h"
|
|
#include "opal/mca/crs/crs.h"
|
|
#include "opal/mca/crs/base/base.h"
|
|
|
|
#include "opal/class/opal_object.h"
|
|
#include "opal/util/output.h"
|
|
|
|
BEGIN_C_DECLS
|
|
|
|
/**
|
|
* States that a process can be in while checkpointing
|
|
*/
|
|
/* Reached an error */
|
|
#define ORTE_SNAPC_CKPT_STATE_ERROR 0
|
|
|
|
/* Doing no checkpoint -- Quiet state */
|
|
#define ORTE_SNAPC_CKPT_STATE_NONE 1
|
|
/* There has been a request for a checkpoint from one of the applications */
|
|
#define ORTE_SNAPC_CKPT_STATE_REQUEST 2
|
|
/* There is a Pending checkpoint for this process */
|
|
#define ORTE_SNAPC_CKPT_STATE_PENDING 3
|
|
/* Running the checkpoint */
|
|
#define ORTE_SNAPC_CKPT_STATE_RUNNING 4
|
|
/* All Processes have been stopped */
|
|
#define ORTE_SNAPC_CKPT_STATE_STOPPED 5
|
|
/* Finished the checkpoint locally */
|
|
#define ORTE_SNAPC_CKPT_STATE_FINISHED_LOCAL 6
|
|
/* File Transfer in progress */
|
|
#define ORTE_SNAPC_CKPT_STATE_FILE_XFER 7
|
|
/* Finished the checkpoint */
|
|
#define ORTE_SNAPC_CKPT_STATE_FINISHED 8
|
|
/* Unable to checkpoint this job */
|
|
#define ORTE_SNAPC_CKPT_STATE_NO_CKPT 9
|
|
#define ORTE_SNAPC_CKPT_MAX 10
|
|
|
|
/**
|
|
* Definition of a orte local snapshot.
|
|
* Similar to the opal_crs_base_snapshot_t except that it
|
|
* contains process contact information.
|
|
*/
|
|
struct orte_snapc_base_local_snapshot_1_0_0_t {
|
|
/** List super object */
|
|
opal_list_item_t super;
|
|
|
|
/** ORTE Process name */
|
|
orte_process_name_t process_name;
|
|
|
|
/** State of the checkpoint */
|
|
int state;
|
|
|
|
/** Unique name of the local snapshot */
|
|
char * reference_name;
|
|
|
|
/** Local location of the local snapshot Absolute path */
|
|
char * local_location;
|
|
|
|
/** Remote location of the local snapshot Absolute path */
|
|
char * remote_location;
|
|
|
|
/** CRS agent */
|
|
char * opal_crs;
|
|
|
|
};
|
|
typedef struct orte_snapc_base_local_snapshot_1_0_0_t orte_snapc_base_local_snapshot_1_0_0_t;
|
|
typedef struct orte_snapc_base_local_snapshot_1_0_0_t orte_snapc_base_local_snapshot_t;
|
|
|
|
ORTE_DECLSPEC OBJ_CLASS_DECLARATION(orte_snapc_base_local_snapshot_t);
|
|
|
|
/**
|
|
* Definition of the global snapshot.
|
|
* Each component is assumed to have extened this definition
|
|
* in the same way they extern the orte_snapc_base_compoinent_t below.
|
|
*/
|
|
struct orte_snapc_base_global_snapshot_1_0_0_t {
|
|
/** This is an object, so must have super */
|
|
opal_list_item_t super;
|
|
|
|
/** A list of orte_snapc_base_snapshot_t's */
|
|
opal_list_t local_snapshots;
|
|
|
|
/** Unique name of the global snapshot */
|
|
char * reference_name;
|
|
|
|
/** Location of the global snapshot Absolute path */
|
|
char * local_location;
|
|
|
|
/** Sequence Number */
|
|
int seq_num;
|
|
|
|
/** Start Timestamp */
|
|
char * start_time;
|
|
|
|
/** End Timestamp */
|
|
char * end_time;
|
|
};
|
|
typedef struct orte_snapc_base_global_snapshot_1_0_0_t orte_snapc_base_global_snapshot_1_0_0_t;
|
|
typedef struct orte_snapc_base_global_snapshot_1_0_0_t orte_snapc_base_global_snapshot_t;
|
|
|
|
ORTE_DECLSPEC OBJ_CLASS_DECLARATION(orte_snapc_base_global_snapshot_t);
|
|
|
|
struct orte_snapc_base_quiesce_1_0_0_t {
|
|
/** Parent is an object type */
|
|
opal_object_t super;
|
|
|
|
/** Current epoch */
|
|
int epoch;
|
|
/** Requested CRS */
|
|
char * crs_name;
|
|
/** Handle for reference */
|
|
char * handle;
|
|
/** snapshot list */
|
|
orte_snapc_base_global_snapshot_t *snapshot;
|
|
|
|
/** Target Directory */
|
|
char * target_dir;
|
|
/** Command Line */
|
|
char * cmdline;
|
|
/** State of operation if checkpointing */
|
|
opal_crs_state_type_t cr_state;
|
|
/** Checkpointing? */
|
|
bool checkpointing;
|
|
/** Restarting? */
|
|
bool restarting;
|
|
};
|
|
typedef struct orte_snapc_base_quiesce_1_0_0_t orte_snapc_base_quiesce_1_0_0_t;
|
|
typedef struct orte_snapc_base_quiesce_1_0_0_t orte_snapc_base_quiesce_t;
|
|
|
|
ORTE_DECLSPEC OBJ_CLASS_DECLARATION(orte_snapc_base_quiesce_t);
|
|
|
|
/**
|
|
* Module initialization function.
|
|
* Returns ORTE_SUCCESS
|
|
*/
|
|
typedef int (*orte_snapc_base_module_init_fn_t)
|
|
(bool seed, bool app);
|
|
|
|
/**
|
|
* Module finalization function.
|
|
* Returns ORTE_SUCCESS
|
|
*/
|
|
typedef int (*orte_snapc_base_module_finalize_fn_t)
|
|
(void);
|
|
|
|
/**
|
|
* Setup the necessary structures for this job
|
|
* Returns ORTE_SUCCESS
|
|
*/
|
|
typedef int (*orte_snapc_base_setup_job_fn_t)
|
|
(orte_jobid_t jobid);
|
|
|
|
/**
|
|
* Setup the necessary structures for this job
|
|
* Returns ORTE_SUCCESS
|
|
*/
|
|
typedef int (*orte_snapc_base_release_job_fn_t)
|
|
(orte_jobid_t jobid);
|
|
|
|
|
|
/**
|
|
* Handle fault tolerance updates
|
|
*
|
|
* @param[in] state Fault tolerance state update
|
|
*
|
|
* @retval ORTE_SUCCESS The operation completed successfully
|
|
* @retval ORTE_ERROR An unspecifed error occurred
|
|
*/
|
|
typedef int (*orte_snapc_base_ft_event_fn_t)(int state);
|
|
|
|
/**
|
|
* Start a checkpoint originating from an internal source.
|
|
*
|
|
* This really only makes sense to call from an application, but in the future
|
|
* we may allow the checkpoint operation to use this function from the local
|
|
* coordinator.
|
|
*
|
|
* @param[out] epoch Epoch number to associate with this checkpoint operation
|
|
* Returns ORTE_SUCCESS
|
|
*/
|
|
typedef int (*orte_snapc_base_start_checkpoint_fn_t)
|
|
(orte_snapc_base_quiesce_t *datum);
|
|
|
|
/**
|
|
* Signal end of checkpoint epoch originating from an internal source.
|
|
*
|
|
* @param[in] epoch Epoch number to associate with this checkpoint operation
|
|
* Returns ORTE_SUCCESS
|
|
*/
|
|
typedef int (*orte_snapc_base_end_checkpoint_fn_t)
|
|
(orte_snapc_base_quiesce_t *datum);
|
|
|
|
/**
|
|
* Structure for SNAPC components.
|
|
*/
|
|
struct orte_snapc_base_component_2_0_0_t {
|
|
/** MCA base component */
|
|
mca_base_component_t base_version;
|
|
/** MCA base data */
|
|
mca_base_component_data_t base_data;
|
|
|
|
/** Verbosity Level */
|
|
int verbose;
|
|
/** Output Handle for opal_output */
|
|
int output_handle;
|
|
/** Default Priority */
|
|
int priority;
|
|
};
|
|
typedef struct orte_snapc_base_component_2_0_0_t orte_snapc_base_component_2_0_0_t;
|
|
typedef struct orte_snapc_base_component_2_0_0_t orte_snapc_base_component_t;
|
|
|
|
/**
|
|
* Structure for SNAPC modules
|
|
*/
|
|
struct orte_snapc_base_module_1_0_0_t {
|
|
/** Initialization Function */
|
|
orte_snapc_base_module_init_fn_t snapc_init;
|
|
/** Finalization Function */
|
|
orte_snapc_base_module_finalize_fn_t snapc_finalize;
|
|
/** Setup structures for a job */
|
|
orte_snapc_base_setup_job_fn_t setup_job;
|
|
/** Release job */
|
|
orte_snapc_base_release_job_fn_t release_job;
|
|
/** Handle any FT Notifications */
|
|
orte_snapc_base_ft_event_fn_t ft_event;
|
|
/** Handle internal request for checkpoint */
|
|
orte_snapc_base_start_checkpoint_fn_t start_ckpt;
|
|
orte_snapc_base_end_checkpoint_fn_t end_ckpt;
|
|
};
|
|
typedef struct orte_snapc_base_module_1_0_0_t orte_snapc_base_module_1_0_0_t;
|
|
typedef struct orte_snapc_base_module_1_0_0_t orte_snapc_base_module_t;
|
|
|
|
ORTE_DECLSPEC extern orte_snapc_base_module_t orte_snapc;
|
|
ORTE_DECLSPEC extern orte_snapc_base_component_t orte_snapc_base_selected_component;
|
|
|
|
/**
|
|
* Macro for use in components that are of type SNAPC
|
|
*/
|
|
#define ORTE_SNAPC_BASE_VERSION_2_0_0 \
|
|
MCA_BASE_VERSION_2_0_0, \
|
|
"snapc", 2, 0, 0
|
|
|
|
END_C_DECLS
|
|
|
|
#endif /* ORTE_SNAPC_H */
|
|
|