2006-09-14 21:29:51 +00:00
|
|
|
/*
|
2007-09-27 14:37:04 +00:00
|
|
|
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
|
2006-09-14 21:29:51 +00:00
|
|
|
* University Research and Technology
|
|
|
|
* Corporation. All rights reserved.
|
|
|
|
* Copyright (c) 2004-2006 The University of Tennessee and The University
|
|
|
|
* of Tennessee Research Foundation. All rights
|
|
|
|
* reserved.
|
|
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
|
|
* University of Stuttgart. All rights reserved.
|
|
|
|
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
|
|
* All rights reserved.
|
|
|
|
* $COPYRIGHT$
|
|
|
|
*
|
|
|
|
* Additional copyrights may follow
|
|
|
|
*
|
|
|
|
* $HEADER$
|
|
|
|
*/
|
|
|
|
/** @file:
|
|
|
|
*/
|
|
|
|
|
|
|
|
#ifndef MCA_ODLS_PRIVATE_H
|
|
|
|
#define MCA_ODLS_PRIVATE_H
|
|
|
|
|
|
|
|
/*
|
|
|
|
* includes
|
|
|
|
*/
|
|
|
|
#include "orte_config.h"
|
2008-02-28 01:57:57 +00:00
|
|
|
#include "orte/types.h"
|
2006-09-14 21:29:51 +00:00
|
|
|
|
|
|
|
#include "opal/class/opal_list.h"
|
2008-04-14 18:26:08 +00:00
|
|
|
#include "opal/class/opal_pointer_array.h"
|
Repair the MPI-2 dynamic operations. This includes:
1. repair of the linear and direct routed modules
2. repair of the ompi/pubsub/orte module to correctly init routes to the ompi-server, and correctly handle failure to correctly parse the provided ompi-server URI
3. modification of orterun to accept both "file" and "FILE" for designating where the ompi-server URI is to be found - purely a convenience feature
4. resolution of a message ordering problem during the connect/accept handshake that allowed the "send-first" proc to attempt to send to the "recv-first" proc before the HNP had actually updated its routes.
Let this be a further reminder to all - message ordering is NOT guaranteed in the OOB
5. Repair the ompi/dpm/orte module to correctly init routes during connect/accept.
Reminder to all: messages sent to procs in another job family (i.e., started by a different mpirun) are ALWAYS routed through the respective HNPs. As per the comments in orte/routed, this is REQUIRED to maintain connect/accept (where only the root proc on each side is capable of init'ing the routes), allow communication between mpirun's using different routing modules, and to minimize connections on tools such as ompi-server. It is all taken care of "under the covers" by the OOB to ensure that a route back to the sender is maintained, even when the different mpirun's are using different routed modules.
6. corrections in the orte/odls to ensure proper identification of daemons participating in a dynamic launch
7. corrections in build/nidmap to support update of an existing nidmap during dynamic launch
8. corrected implementation of the update_arch function in the ESS, along with consolidation of a number of ESS operations into base functions for easier maintenance. The ability to support info from multiple jobs was added, although we don't currently do so - this will come later to support further fault recovery strategies
9. minor updates to several functions to remove unnecessary and/or no longer used variables and envar's, add some debugging output, etc.
10. addition of a new macro ORTE_PROC_IS_DAEMON that resolves to true if the provided proc is a daemon
There is still more cleanup to be done for efficiency, but this at least works.
Tested on single-node Mac, multi-node SLURM via odin. Tests included connect/accept, publish/lookup/unpublish, comm_spawn, comm_spawn_multiple, and singleton comm_spawn.
Fixes ticket #1256
This commit was SVN r18804.
2008-07-03 17:53:37 +00:00
|
|
|
#include "opal/class/opal_value_array.h"
|
2007-10-10 15:02:10 +00:00
|
|
|
#include "opal/threads/mutex.h"
|
|
|
|
#include "opal/threads/condition.h"
|
2008-02-28 01:57:57 +00:00
|
|
|
#include "opal/dss/dss_types.h"
|
2008-05-02 12:00:08 +00:00
|
|
|
|
|
|
|
#include "orte/mca/grpcomm/grpcomm_types.h"
|
2008-02-28 01:57:57 +00:00
|
|
|
#include "orte/mca/plm/plm_types.h"
|
2007-10-10 15:02:10 +00:00
|
|
|
#include "orte/mca/rmaps/rmaps_types.h"
|
|
|
|
#include "orte/mca/rml/rml_types.h"
|
2008-02-28 01:57:57 +00:00
|
|
|
#include "orte/runtime/orte_globals.h"
|
2006-09-14 21:29:51 +00:00
|
|
|
|
|
|
|
#include "orte/mca/odls/odls_types.h"
|
|
|
|
|
2008-02-28 01:57:57 +00:00
|
|
|
BEGIN_C_DECLS
|
2006-09-14 21:29:51 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* General ODLS types
|
|
|
|
*/
|
2007-07-14 15:14:07 +00:00
|
|
|
|
2008-04-09 22:10:53 +00:00
|
|
|
typedef struct {
|
2006-09-14 21:29:51 +00:00
|
|
|
/** Verbose/debug output stream */
|
|
|
|
int output;
|
|
|
|
/** Time to allow process to forcibly die */
|
|
|
|
int timeout_before_sigkill;
|
2007-10-10 15:02:10 +00:00
|
|
|
/* mutex */
|
|
|
|
opal_mutex_t mutex;
|
|
|
|
/* condition variable */
|
|
|
|
opal_condition_t cond;
|
2008-04-30 19:49:53 +00:00
|
|
|
/* byte object to store daemon map for later xmit to procs */
|
|
|
|
opal_byte_object_t *dmap;
|
2008-08-13 17:47:24 +00:00
|
|
|
/* any co-spawned debugger daemon */
|
|
|
|
orte_odls_job_t *debugger;
|
|
|
|
/* debugger launched */
|
|
|
|
bool debugger_launched;
|
2009-01-30 22:47:30 +00:00
|
|
|
/* list of ranks to be displayed on separate xterms */
|
|
|
|
opal_list_t xterm_ranks;
|
|
|
|
/* the xterm cmd to be used */
|
|
|
|
char **xtermcmd;
|
2006-09-14 21:29:51 +00:00
|
|
|
} orte_odls_globals_t;
|
|
|
|
|
2006-10-06 07:08:17 +00:00
|
|
|
ORTE_DECLSPEC extern orte_odls_globals_t orte_odls_globals;
|
2008-04-14 18:26:08 +00:00
|
|
|
|
|
|
|
|
2007-10-10 15:02:10 +00:00
|
|
|
/*
|
|
|
|
* Default functions that are common to most environments - can
|
|
|
|
* be overridden by specific environments if they need something
|
|
|
|
* different (e.g., bproc)
|
|
|
|
*/
|
2007-10-11 00:02:49 +00:00
|
|
|
ORTE_DECLSPEC int
|
2008-02-28 01:57:57 +00:00
|
|
|
orte_odls_base_default_get_add_procs_data(opal_buffer_t *data,
|
|
|
|
orte_jobid_t job);
|
2007-10-11 00:02:49 +00:00
|
|
|
|
2008-12-10 19:18:36 +00:00
|
|
|
ORTE_DECLSPEC int
|
|
|
|
orte_odls_base_default_update_daemon_info(opal_buffer_t *data);
|
|
|
|
|
2007-10-11 00:02:49 +00:00
|
|
|
ORTE_DECLSPEC int
|
2008-02-28 01:57:57 +00:00
|
|
|
orte_odls_base_default_construct_child_list(opal_buffer_t *data,
|
2008-04-30 19:49:53 +00:00
|
|
|
orte_jobid_t *job);
|
2007-10-10 15:02:10 +00:00
|
|
|
|
|
|
|
/* define a function that will fork a local proc */
|
|
|
|
typedef int (*orte_odls_base_fork_local_proc_fn_t)(orte_app_context_t *context,
|
|
|
|
orte_odls_child_t *child,
|
2008-08-13 17:47:24 +00:00
|
|
|
char **environ_copy,
|
Roll in the revamped IOF subsystem. Per the devel mailing list email, this is a complete rewrite of the iof framework designed to simplify the code for maintainability, and to support features we had planned to do, but were too difficult to implement in the old code. Specifically, the new code:
1. completely and cleanly separates responsibilities between the HNP, orted, and tool components.
2. removes all wireup messaging during launch and shutdown.
3. maintains flow control for stdin to avoid large-scale consumption of memory by orteds when large input files are forwarded. This is done using an xon/xoff protocol.
4. enables specification of stdin recipients on the mpirun cmd line. Allowed options include rank, "all", or "none". Default is rank 0.
5. creates a new MPI_Info key "ompi_stdin_target" that supports the above options for child jobs. Default is "none".
6. adds a new tool "orte-iof" that can connect to a running mpirun and display the output. Cmd line options allow selection of any combination of stdout, stderr, and stddiag. Default is stdout.
7. adds a new mpirun and orte-iof cmd line option "tag-output" that will tag each line of output with process name and stream ident. For example, "[1,0]<stdout>this is output"
This is not intended for the 1.3 release as it is a major change requiring considerable soak time.
This commit was SVN r19767.
2008-10-18 00:00:49 +00:00
|
|
|
orte_job_controls_t controls,
|
|
|
|
orte_vpid_t stdin_target);
|
2007-10-10 15:02:10 +00:00
|
|
|
|
2007-10-11 00:02:49 +00:00
|
|
|
ORTE_DECLSPEC int
|
2008-02-28 01:57:57 +00:00
|
|
|
orte_odls_base_default_launch_local(orte_jobid_t job,
|
2007-10-11 00:02:49 +00:00
|
|
|
orte_odls_base_fork_local_proc_fn_t fork_local);
|
2007-10-10 15:02:10 +00:00
|
|
|
|
2007-10-11 00:02:49 +00:00
|
|
|
ORTE_DECLSPEC int
|
2008-02-28 01:57:57 +00:00
|
|
|
orte_odls_base_default_deliver_message(orte_jobid_t job, opal_buffer_t *buffer, orte_rml_tag_t tag);
|
2007-10-10 15:02:10 +00:00
|
|
|
|
2007-10-11 00:02:49 +00:00
|
|
|
ORTE_DECLSPEC void odls_base_default_wait_local_proc(pid_t pid, int status, void* cbdata);
|
2007-10-10 15:02:10 +00:00
|
|
|
|
|
|
|
/* define a function type to signal a local proc */
|
|
|
|
typedef int (*orte_odls_base_signal_local_fn_t)(pid_t pid, int signum);
|
|
|
|
|
2007-10-11 00:02:49 +00:00
|
|
|
ORTE_DECLSPEC int
|
|
|
|
orte_odls_base_default_signal_local_procs(const orte_process_name_t *proc, int32_t signal,
|
|
|
|
orte_odls_base_signal_local_fn_t signal_local);
|
2007-10-10 15:02:10 +00:00
|
|
|
|
|
|
|
/* define a function type for killing a local proc */
|
|
|
|
typedef int (*orte_odls_base_kill_local_fn_t)(pid_t pid, int signum);
|
|
|
|
|
|
|
|
/* define a function type to detect that a child died */
|
|
|
|
typedef bool (*orte_odls_base_child_died_fn_t)(pid_t pid, unsigned int timeout, int *exit_status);
|
|
|
|
|
2007-10-11 00:02:49 +00:00
|
|
|
ORTE_DECLSPEC int
|
|
|
|
orte_odls_base_default_kill_local_procs(orte_jobid_t job, bool set_state,
|
|
|
|
orte_odls_base_kill_local_fn_t kill_local,
|
|
|
|
orte_odls_base_child_died_fn_t child_died);
|
2007-10-10 15:02:10 +00:00
|
|
|
|
2008-04-30 19:49:53 +00:00
|
|
|
ORTE_DECLSPEC int orte_odls_base_default_require_sync(orte_process_name_t *proc,
|
|
|
|
opal_buffer_t *buffer,
|
|
|
|
bool drop_nidmap);
|
2006-09-14 21:29:51 +00:00
|
|
|
|
2007-09-27 13:13:29 +00:00
|
|
|
/*
|
|
|
|
* Preload binary/files functions
|
|
|
|
*/
|
|
|
|
ORTE_DECLSPEC int orte_odls_base_preload_files_app_context(orte_app_context_t* context);
|
|
|
|
|
2008-12-22 20:23:05 +00:00
|
|
|
/*
|
|
|
|
* Obtain process stats on a child proc
|
|
|
|
*/
|
|
|
|
ORTE_DECLSPEC int orte_odls_base_get_proc_stats(opal_buffer_t *answer, orte_process_name_t *proc);
|
|
|
|
|
2008-02-28 01:57:57 +00:00
|
|
|
END_C_DECLS
|
|
|
|
|
2006-09-14 21:29:51 +00:00
|
|
|
#endif
|