1
1
openmpi/orte/mca/sstore/sstore.h

405 строки
12 KiB
C
Исходник Обычный вид История

A number of C/R enhancements per RFC below: http://www.open-mpi.org/community/lists/devel/2010/07/8240.php Documentation: http://osl.iu.edu/research/ft/ Major Changes: -------------- * Added C/R-enabled Debugging support. Enabled with the --enable-crdebug flag. See the following website for more information: http://osl.iu.edu/research/ft/crdebug/ * Added Stable Storage (SStore) framework for checkpoint storage * 'central' component does a direct to central storage save * 'stage' component stages checkpoints to central storage while the application continues execution. * 'stage' supports offline compression of checkpoints before moving (sstore_stage_compress) * 'stage' supports local caching of checkpoints to improve automatic recovery (sstore_stage_caching) * Added Compression (compress) framework to support * Add two new ErrMgr recovery policies * {{{crmig}}} C/R Process Migration * {{{autor}}} C/R Automatic Recovery * Added the {{{ompi-migrate}}} command line tool to support the {{{crmig}}} ErrMgr component * Added CR MPI Ext functions (enable them with {{{--enable-mpi-ext=cr}}} configure option) * {{{OMPI_CR_Checkpoint}}} (Fixes trac:2342) * {{{OMPI_CR_Restart}}} * {{{OMPI_CR_Migrate}}} (may need some more work for mapping rules) * {{{OMPI_CR_INC_register_callback}}} (Fixes trac:2192) * {{{OMPI_CR_Quiesce_start}}} * {{{OMPI_CR_Quiesce_checkpoint}}} * {{{OMPI_CR_Quiesce_end}}} * {{{OMPI_CR_self_register_checkpoint_callback}}} * {{{OMPI_CR_self_register_restart_callback}}} * {{{OMPI_CR_self_register_continue_callback}}} * The ErrMgr predicted_fault() interface has been changed to take an opal_list_t of ErrMgr defined types. This will allow us to better support a wider range of fault prediction services in the future. * Add a progress meter to: * FileM rsh (filem_rsh_process_meter) * SnapC full (snapc_full_progress_meter) * SStore stage (sstore_stage_progress_meter) * Added 2 new command line options to ompi-restart * --showme : Display the full command line that would have been exec'ed. * --mpirun_opts : Command line options to pass directly to mpirun. (Fixes trac:2413) * Deprecated some MCA params: * crs_base_snapshot_dir deprecated, use sstore_stage_local_snapshot_dir * snapc_base_global_snapshot_dir deprecated, use sstore_base_global_snapshot_dir * snapc_base_global_shared deprecated, use sstore_stage_global_is_shared * snapc_base_store_in_place deprecated, replaced with different components of SStore * snapc_base_global_snapshot_ref deprecated, use sstore_base_global_snapshot_ref * snapc_base_establish_global_snapshot_dir deprecated, never well supported * snapc_full_skip_filem deprecated, use sstore_stage_skip_filem Minor Changes: -------------- * Fixes trac:1924 : {{{ompi-restart}}} now recognizes path prefixed checkpoint handles and does the right thing. * Fixes trac:2097 : {{{ompi-info}}} should now report all available CRS components * Fixes trac:2161 : Manual checkpoint movement. A user can 'mv' a checkpoint directory from the original location to another and still restart from it. * Fixes trac:2208 : Honor various TMPDIR varaibles instead of forcing {{{/tmp}}} * Move {{{ompi_cr_continue_like_restart}}} to {{{orte_cr_continue_like_restart}}} to be more flexible in where this should be set. * opal_crs_base_metadata_write* functions have been moved to SStore to support a wider range of metadata handling functionality. * Cleanup the CRS framework and components to work with the SStore framework. * Cleanup the SnapC framework and components to work with the SStore framework (cleans up these code paths considerably). * Add 'quiesce' hook to CRCP for a future enhancement. * We now require a BLCR version that supports {{{cr_request_file()}}} or {{{cr_request_checkpoint()}}} in order to make the code more maintainable. Note that {{{cr_request_file}}} has been deprecated since 0.7.0, so we prefer to use {{{cr_request_checkpoint()}}}. * Add optional application level INC callbacks (registered through the CR MPI Ext interface). * Increase the {{{opal_cr_thread_sleep_wait}}} parameter to 1000 microseconds to make the C/R thread less aggressive. * {{{opal-restart}}} now looks for cache directories before falling back on stable storage when asked. * {{{opal-restart}}} also support local decompression before restarting * {{{orte-checkpoint}}} now uses the SStore framework to work with the metadata * {{{orte-restart}}} now uses the SStore framework to work with the metadata * Remove the {{{orte-restart}}} preload option. This was removed since the user only needs to select the 'stage' component in order to support this functionality. * Since the '-am' parameter is saved in the metadata, {{{ompi-restart}}} no longer hard codes {{{-am ft-enable-cr}}}. * Fix {{{hnp}}} ErrMgr so that if a previous component in the stack has 'fixed' the problem, then it should be skipped. * Make sure to decrement the number of 'num_local_procs' in the orted when one goes away. * odls now checks the SStore framework to see if it needs to load any checkpoint files before launching (to support 'stage'). This separates the SStore logic from the --preload-[binary|files] options. * Add unique IDs to the named pipes established between the orted and the app in SnapC. This is to better support migration and automatic recovery activities. * Improve the checks for 'already checkpointing' error path. * A a recovery output timer, to show how long it takes to restart a job * Do a better job of cleaning up the old session directory on restart. * Add a local module to the autor and crmig ErrMgr components. These small modules prevent the 'orted' component from attempting a local recovery (Which does not work for MPI apps at the moment) * Add a fix for bounding the checkpointable region between MPI_Init and MPI_Finalize. This commit was SVN r23587. The following Trac tickets were found above: Ticket 1924 --> https://svn.open-mpi.org/trac/ompi/ticket/1924 Ticket 2097 --> https://svn.open-mpi.org/trac/ompi/ticket/2097 Ticket 2161 --> https://svn.open-mpi.org/trac/ompi/ticket/2161 Ticket 2192 --> https://svn.open-mpi.org/trac/ompi/ticket/2192 Ticket 2208 --> https://svn.open-mpi.org/trac/ompi/ticket/2208 Ticket 2342 --> https://svn.open-mpi.org/trac/ompi/ticket/2342 Ticket 2413 --> https://svn.open-mpi.org/trac/ompi/ticket/2413
2010-08-11 00:51:11 +04:00
/*
* Copyright (c) 2010 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*
* Distributed Stable Storage (SStore) Interface
*
*/
#ifndef MCA_SSTORE_H
#define MCA_SSTORE_H
#include "orte_config.h"
#include "orte/constants.h"
#include "orte/types.h"
#include "orte/runtime/orte_globals.h"
#include "opal/mca/mca.h"
#include "opal/mca/base/base.h"
#include "opal/class/opal_object.h"
BEGIN_C_DECLS
/**
* Keys accepted as metadata
*/
typedef uint32_t orte_sstore_base_key_t;
/** CRS Component */
#define SSTORE_METADATA_LOCAL_CRS_COMP 0
/** Compress Component */
#define SSTORE_METADATA_LOCAL_COMPRESS_COMP 1
/** Compress Component Postfix */
#define SSTORE_METADATA_LOCAL_COMPRESS_POSTFIX 2
/** Process PID */
#define SSTORE_METADATA_LOCAL_PID 3
/** Checkpoint Context File */
#define SSTORE_METADATA_LOCAL_CONTEXT 4
/** Directory to make on restart */
#define SSTORE_METADATA_LOCAL_MKDIR 5
/** File to touch on restart */
#define SSTORE_METADATA_LOCAL_TOUCH 6
/** Local snapshot reference (e.g., opal_snapshot_0.ckpt) */
#define SSTORE_METADATA_LOCAL_SNAP_REF 7
/** Local snapshot reference format string (e.g., opal_snapshot_%d.ckpt) passed vpid */
#define SSTORE_METADATA_LOCAL_SNAP_REF_FMT 8
/** Local snapshot directory (Full Path excluding reference) */
#define SSTORE_METADATA_LOCAL_SNAP_LOC 9
/** Local snapshot reference directory (Full Path) */
#define SSTORE_METADATA_LOCAL_SNAP_REF_LOC_FMT 10
/** Local snapshot metadata file (Full Path) */
#define SSTORE_METADATA_LOCAL_SNAP_META 11
/** Global snapshot reference (e.g., ompi_global_snapshot_1234.ckpt) */
#define SSTORE_METADATA_GLOBAL_SNAP_REF 12
/** Global snapshot location (Relative Path from base) */
#define SSTORE_METADATA_GLOBAL_SNAP_LOC 13
/** Global snapshot location (Full path) */
#define SSTORE_METADATA_GLOBAL_SNAP_LOC_ABS 14
/** Global snapshot metadata file (Full path) */
#define SSTORE_METADATA_GLOBAL_SNAP_META 15
/** Global snapshot sequence number */
#define SSTORE_METADATA_GLOBAL_SNAP_SEQ 16
/** AMCA Parameter to be preserved for ompi-restart */
#define SSTORE_METADATA_GLOBAL_AMCA_PARAM 17
/** Total number of sequence numbers for this snapshot */
#define SSTORE_METADATA_GLOBAL_SNAP_NUM_SEQ 18
/** Comma separated list of all sequence numbers for this snapshot */
#define SSTORE_METADATA_GLOBAL_SNAP_ALL_SEQ 19
/** Access the current default base directory (Full Path) */
#define SSTORE_METADATA_BASE_LOC 20
/** The local process is skipping the checkpoint
* Usually this is because there is a migration, and it is not participating
*/
#define SSTORE_METADATA_LOCAL_SKIP_CKPT 21
/** A Migration checkpoint does not necessarily contain all of the processes
* in the job, so it is not a checkpoint that can be restarted from normally.
* Therefore, it needs to be marked specially. */
#define SSTORE_METADATA_GLOBAL_MIGRATING 22
/** */
#define SSTORE_METADATA_MAX 23
/**
* Storage handle
*/
#define ORTE_SSTORE_HANDLE OPAL_UINT32
typedef uint32_t orte_sstore_base_handle_t;
ORTE_DECLSPEC extern orte_sstore_base_handle_t orte_sstore_handle_current;
ORTE_DECLSPEC extern orte_sstore_base_handle_t orte_sstore_handle_last_stable;
#define ORTE_SSTORE_HANDLE_INVALID 0
/**
* Local and Global snapshot information structure
* Primarily used by orte-restart as an abstract way to handle metadata
*/
struct orte_sstore_base_local_snapshot_info_1_0_0_t {
/** List super object */
opal_list_item_t super;
/** Stable Storage Handle */
orte_sstore_base_handle_t ss_handle;
/** ORTE Process name */
orte_process_name_t process_name;
/** CRS Component */
char *crs_comp;
/** Compress Component */
char *compress_comp;
/** Compress Component Postfix */
char *compress_postfix;
/** Start/End Timestamps */
char *start_time;
char *end_time;
};
typedef struct orte_sstore_base_local_snapshot_info_1_0_0_t orte_sstore_base_local_snapshot_info_1_0_0_t;
typedef struct orte_sstore_base_local_snapshot_info_1_0_0_t orte_sstore_base_local_snapshot_info_t;
ORTE_DECLSPEC OBJ_CLASS_DECLARATION(orte_sstore_base_local_snapshot_info_t);
struct orte_sstore_base_global_snapshot_info_1_0_0_t {
/** List super object */
opal_list_item_t super;
/** A list of orte_sstore_base_local_snapshot_info_t's */
opal_list_t local_snapshots;
/** Stable Storage Handle */
orte_sstore_base_handle_t ss_handle;
/** Start Timestamp */
char * start_time;
/** End Timestamp */
char * end_time;
/** Sequence number */
int seq_num;
/** Reference */
char *reference;
/** AMCA parameter used */
char *amca_param;
/** Internal use only: Cache some information on the structure */
int num_seqs;
char ** all_seqs;
char *basedir;
char *metadata_filename;
};
typedef struct orte_sstore_base_global_snapshot_info_1_0_0_t orte_sstore_base_global_snapshot_info_1_0_0_t;
typedef struct orte_sstore_base_global_snapshot_info_1_0_0_t orte_sstore_base_global_snapshot_info_t;
ORTE_DECLSPEC OBJ_CLASS_DECLARATION(orte_sstore_base_global_snapshot_info_t);
/**
* Module initialization function.
* Returns ORTE_SUCCESS
*/
typedef int (*orte_sstore_base_module_init_fn_t)
(void);
/**
* Module finalization function.
* Returns ORTE_SUCCESS
*/
typedef int (*orte_sstore_base_module_finalize_fn_t)
(void);
/**
* Request a checkpoint storage handle from stable storage
*
* @param handle Checkpoint storage handle
* @param key Key to use as an identifier
* @param value Value of the key specified
*
* @return ORTE_SUCCESS on success
* @return ORTE_ERROR on failure
*/
typedef int (*orte_sstore_base_request_checkpoint_handle_fn_t)
(orte_sstore_base_handle_t *handle, int seq, orte_jobid_t jobid);
/**
* Request a restart storage handle from stable storage
* This function will fail if the key cannot be matched.
* If multiple matches exist, it will return the latest one.
* If they key is NULL, then the latest entry will be used.
*
* @param handle Restart storage handle
*
* @return ORTE_SUCCESS on success
* @return ORTE_ERROR on failure
*/
typedef int (*orte_sstore_base_request_restart_handle_fn_t)
(orte_sstore_base_handle_t *handle,
char *basedir, char *ref, int seq,
orte_sstore_base_global_snapshot_info_t *snapshot);
/**
* Request snapshot info from a given handle.
* If they key is NULL, then the latest entry will be used.
*
* @param handle Restart storage handle
*
* @return ORTE_SUCCESS on success
* @return ORTE_ERROR on failure
*/
typedef int (*orte_sstore_base_request_global_snapshot_data_fn_t)
(orte_sstore_base_handle_t *handle,
orte_sstore_base_global_snapshot_info_t *snapshot);
/**
* Register access to a handle.
*
* @param handle Storage handle
*
* @return ORTE_SUCCESS on success
* @return ORTE_ERROR on failure
*/
typedef int (*orte_sstore_base_register_handle_fn_t)
(orte_sstore_base_handle_t handle);
/**
* Get attribute on the storage handle
*
* @param handle Storage handle
* @param key Key to access
* @param value Value of the key. NULL if not avaialble
*
* @return ORTE_SUCCESS on success
* @return ORTE_ERROR on failure
*/
typedef int (*orte_sstore_base_get_attribute_fn_t)
(orte_sstore_base_handle_t handle, orte_sstore_base_key_t key, char **value);
/**
* Set attribute on the storage handle
*
* @param handle Storage handle
* @param key Key to set
* @param value Value of the key.
*
* @return ORTE_SUCCESS on success
* @return ORTE_ERROR on failure
*/
typedef int (*orte_sstore_base_set_attribute_fn_t)
(orte_sstore_base_handle_t handle, orte_sstore_base_key_t key, char *value);
/**
* Synchronize the handle
*
* @param handle Storage handle
*
* @return ORTE_SUCCESS on success
* @return ORTE_ERROR on failure
*/
typedef int (*orte_sstore_base_sync_fn_t)
(orte_sstore_base_handle_t handle);
/**
* Remove data associated with the handle
*
* @param handle Storage handle
*
* @return ORTE_SUCCESS on success
* @return ORTE_ERROR on failure
*/
typedef int (*orte_sstore_base_remove_fn_t)
(orte_sstore_base_handle_t handle);
/**
* Pack a handle into a buffer
* Only called between the HNP and ORTED (or Global and Local SnapC coordinators)
*
* @param peer Peer to which this is being sent (or NULL if to all peers)
* @param buffer Buffer to pack the data into
* @param handle Storage handle
*
* @return ORTE_SUCCESS on success
* @return ORTE_ERROR on failure
*/
typedef int (*orte_sstore_base_pack_fn_t)
(orte_process_name_t* peer, opal_buffer_t* buffer, orte_sstore_base_handle_t handle);
/**
* Unack a handle from a buffer
* Only called between the HNP and ORTED (or Global and Local SnapC coordinators)
*
* @param peer Peer from which this was received
* @param buffer Buffer to unpack the data
* @param handle Storage handle
*
* @return ORTE_SUCCESS on success
* @return ORTE_ERROR on failure
*/
typedef int (*orte_sstore_base_unpack_fn_t)
(orte_process_name_t* peer, opal_buffer_t* buffer, orte_sstore_base_handle_t *handle);
/**
* Fetch application context dependencies before local launch
*
* @param app Application context
*
* @return ORTE_SUCCESS on success
* @return ORTE_ERROR on failure
*/
typedef int (*orte_sstore_base_fetch_app_deps_fn_t)
(orte_app_context_t *app);
/**
* Wait for all application context dependencies to be fetched
*
* @return ORTE_SUCCESS on success
* @return ORTE_ERROR on failure
*/
typedef int (*orte_sstore_base_wait_all_deps_fn_t)
(void);
/**
* Structure for SSTORE components.
*/
struct orte_sstore_base_component_2_0_0_t {
/** MCA base component */
mca_base_component_t base_version;
/** MCA base data */
mca_base_component_data_t base_data;
/** Verbosity Level */
int verbose;
/** Output Handle for opal_output */
int output_handle;
/** Default Priority */
int priority;
};
typedef struct orte_sstore_base_component_2_0_0_t orte_sstore_base_component_2_0_0_t;
typedef struct orte_sstore_base_component_2_0_0_t orte_sstore_base_component_t;
/**
* Structure for SSTORE modules
*/
struct orte_sstore_base_module_1_0_0_t {
/** Initialization Function */
orte_sstore_base_module_init_fn_t sstore_init;
/** Finalization Function */
orte_sstore_base_module_finalize_fn_t sstore_finalize;
/** Request handle */
orte_sstore_base_request_checkpoint_handle_fn_t request_checkpoint_handle;
orte_sstore_base_request_restart_handle_fn_t request_restart_handle;
orte_sstore_base_request_global_snapshot_data_fn_t request_global_snapshot_data;
orte_sstore_base_register_handle_fn_t register_handle;
/** Get/Set Attributes */
orte_sstore_base_get_attribute_fn_t get_attr;
orte_sstore_base_set_attribute_fn_t set_attr;
/** Sync */
orte_sstore_base_sync_fn_t sync;
/** Remove */
orte_sstore_base_remove_fn_t remove;
/** Pack/Unpack Handle */
orte_sstore_base_pack_fn_t pack_handle;
orte_sstore_base_unpack_fn_t unpack_handle;
/** Launch Helpers */
orte_sstore_base_fetch_app_deps_fn_t fetch_app_deps;
orte_sstore_base_wait_all_deps_fn_t wait_all_deps;
};
typedef struct orte_sstore_base_module_1_0_0_t orte_sstore_base_module_1_0_0_t;
typedef struct orte_sstore_base_module_1_0_0_t orte_sstore_base_module_t;
ORTE_DECLSPEC extern orte_sstore_base_module_t orte_sstore;
/**
* Macro for use in components that are of type SSTORE
*/
#define ORTE_SSTORE_BASE_VERSION_2_0_0 \
MCA_BASE_VERSION_2_0_0, \
"sstore", 2, 0, 0
END_C_DECLS
#endif /* ORTE_SSTORE_H */