2007-07-20 05:34:02 +04:00
|
|
|
/*
|
|
|
|
|
* Copyright (c) 2007 Los Alamos National Security, LLC.
|
|
|
|
|
* All rights reserved.
|
2008-05-06 22:08:45 +04:00
|
|
|
* Copyright (c) 2004-2008 The Trustees of Indiana University.
|
|
|
|
|
* All rights reserved.
|
2011-06-24 00:38:02 +04:00
|
|
|
* Copyright (c) 2004-2011 The University of Tennessee and The University
|
|
|
|
|
* of Tennessee Research Foundation. All rights
|
|
|
|
|
* reserved.
|
2014-11-14 22:44:24 +03:00
|
|
|
* Copyright (c) 2014 Intel, Inc. All rights reserved.
|
2007-07-20 05:34:02 +04:00
|
|
|
* $COPYRIGHT$
|
|
|
|
|
*
|
|
|
|
|
* Additional copyrights may follow
|
|
|
|
|
*
|
|
|
|
|
* $HEADER$
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* @file
|
|
|
|
|
*
|
|
|
|
|
* Routing table for the RML
|
|
|
|
|
*
|
|
|
|
|
* A flexible routing infrastructure for the RML. Provides "next hop"
|
|
|
|
|
* service. Only deals with orte_process_name_ts.
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
#ifndef ORTE_MCA_ROUTED_ROUTED_H_
|
|
|
|
|
#define ORTE_MCA_ROUTED_ROUTED_H_
|
|
|
|
|
|
2009-03-04 18:35:54 +03:00
|
|
|
#include "orte_config.h"
|
|
|
|
|
|
2007-07-20 05:34:02 +04:00
|
|
|
#ifdef HAVE_UNISTD_H
|
|
|
|
|
#include <unistd.h>
|
|
|
|
|
#endif
|
|
|
|
|
|
2008-02-28 04:57:57 +03:00
|
|
|
#include "orte/types.h"
|
2007-07-20 05:34:02 +04:00
|
|
|
#include "opal/mca/mca.h"
|
2008-02-28 04:57:57 +03:00
|
|
|
|
|
|
|
|
#include "opal/dss/dss_types.h"
|
2007-07-20 05:34:02 +04:00
|
|
|
|
|
|
|
|
#include "opal/mca/crs/crs.h"
|
|
|
|
|
#include "opal/mca/crs/base/base.h"
|
|
|
|
|
|
2009-01-27 22:13:56 +03:00
|
|
|
#include "orte/mca/routed/routed_types.h"
|
|
|
|
|
|
2007-07-20 05:34:02 +04:00
|
|
|
BEGIN_C_DECLS
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/* ******************************************************************** */
|
|
|
|
|
|
|
|
|
|
|
2008-02-28 04:57:57 +03:00
|
|
|
struct opal_buffer_t;
|
2007-07-20 05:34:02 +04:00
|
|
|
struct orte_rml_module_t;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/* ******************************************************************** */
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* routed component interface
|
|
|
|
|
*
|
|
|
|
|
* Component interface for the routed framework. A public instance of
|
|
|
|
|
* this structure, called mca_routed_[component name]_component, must
|
|
|
|
|
* exist in any routed component.
|
|
|
|
|
*/
|
2008-07-29 02:40:57 +04:00
|
|
|
struct orte_routed_component_2_0_0_t {
|
2007-07-20 05:34:02 +04:00
|
|
|
/* Base component description */
|
2008-05-06 22:08:45 +04:00
|
|
|
mca_base_component_t base_version;
|
2007-07-20 05:34:02 +04:00
|
|
|
/* Base component data block */
|
2008-07-29 02:40:57 +04:00
|
|
|
mca_base_component_data_t base_data;
|
2007-07-20 05:34:02 +04:00
|
|
|
};
|
|
|
|
|
/** Convienence typedef */
|
2008-07-29 02:40:57 +04:00
|
|
|
typedef struct orte_routed_component_2_0_0_t orte_routed_component_t;
|
2007-07-20 05:34:02 +04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
/* ******************************************************************** */
|
2007-10-16 20:09:41 +04:00
|
|
|
/**
|
|
|
|
|
* Initialize the routed module
|
|
|
|
|
*
|
|
|
|
|
* Do whatever needs to be done to initialize the selected module
|
|
|
|
|
*
|
|
|
|
|
* @retval ORTE_SUCCESS Success
|
|
|
|
|
* @retval ORTE_ERROR Error code from whatever was encountered
|
|
|
|
|
*/
|
|
|
|
|
typedef int (*orte_routed_module_init_fn_t)(void);
|
2007-07-20 05:34:02 +04:00
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* Finalize the routed module
|
|
|
|
|
*
|
|
|
|
|
* Finalize the routed module, ending cleaning up all resources
|
|
|
|
|
* associated with the module. After the finalize function is called,
|
|
|
|
|
* all interface functions (and the module structure itself) are not
|
|
|
|
|
* available for use.
|
|
|
|
|
*
|
|
|
|
|
* @note Whether or not the finalize function returns successfully,
|
|
|
|
|
* the module should not be used once this function is called.
|
|
|
|
|
*
|
|
|
|
|
* @retval ORTE_SUCCESS Success
|
|
|
|
|
* @retval ORTE_ERROR An unspecified error occurred
|
|
|
|
|
*/
|
|
|
|
|
typedef int (*orte_routed_module_finalize_fn_t)(void);
|
|
|
|
|
|
|
|
|
|
|
2008-07-22 01:37:09 +04:00
|
|
|
/*
|
|
|
|
|
* Delete route
|
|
|
|
|
*
|
|
|
|
|
* Delete the route to the specified proc from the routing table. Note
|
|
|
|
|
* that wildcards are supported to remove routes from, for example, all
|
|
|
|
|
* procs in a given job
|
|
|
|
|
*/
|
|
|
|
|
typedef int (*orte_routed_module_delete_route_fn_t)(orte_process_name_t *proc);
|
|
|
|
|
|
2007-07-20 05:34:02 +04:00
|
|
|
/**
|
|
|
|
|
* Update route table with new information
|
|
|
|
|
*
|
|
|
|
|
* Update routing table with a new entry. If an existing exact match
|
|
|
|
|
* for the entry exists, it will be replaced with the current
|
|
|
|
|
* information. If the entry is new, it will be inserted behind all
|
|
|
|
|
* entries of similar "mask". So a wildcard cellid entry will be
|
|
|
|
|
* inserted after any fully-specified entries and any other wildcard
|
|
|
|
|
* cellid entries, but before any wildcard cellid and jobid entries.
|
|
|
|
|
*
|
|
|
|
|
* @retval ORTE_SUCCESS Success
|
|
|
|
|
* @retval ORTE_ERR_NOT_SUPPORTED The updated is not supported. This
|
|
|
|
|
* is likely due to using partially-specified
|
|
|
|
|
* names with a component that does not support
|
|
|
|
|
* such functionality
|
|
|
|
|
* @retval ORTE_ERROR An unspecified error occurred
|
|
|
|
|
*/
|
|
|
|
|
typedef int (*orte_routed_module_update_route_fn_t)(orte_process_name_t *target,
|
|
|
|
|
orte_process_name_t *route);
|
|
|
|
|
|
2008-03-26 04:00:24 +03:00
|
|
|
/**
|
|
|
|
|
* Get the next hop towards the target
|
|
|
|
|
*
|
|
|
|
|
* Obtain the next process on the route to the target. ORTE's routing system
|
|
|
|
|
* works one hop at-a-time, so this function doesn't return the entire path
|
|
|
|
|
* to the target - it only returns the next hop. This could be the target itself,
|
|
|
|
|
* or it could be an intermediate relay. By design, we -never- use application
|
|
|
|
|
* procs as relays, so any relay will be an orted.
|
|
|
|
|
*/
|
2007-07-20 05:34:02 +04:00
|
|
|
typedef orte_process_name_t (*orte_routed_module_get_route_fn_t)(orte_process_name_t *target);
|
|
|
|
|
|
2008-03-26 04:00:24 +03:00
|
|
|
/**
|
|
|
|
|
* Initialize the routing table
|
|
|
|
|
*
|
|
|
|
|
* Initialize the routing table for the specified job. This can be rather complex
|
|
|
|
|
* and depends entirely upon both the selected module AND whether the function
|
|
|
|
|
* is being called by the HNP, an orted, a tool, or an application proc. To
|
|
|
|
|
* understand what is happening, you really need to look at the specific module.
|
|
|
|
|
*
|
|
|
|
|
* Regardless, at the end of the function, the routes to any other process in the
|
|
|
|
|
* specified job -must- be defined (even if it is direct)
|
|
|
|
|
*/
|
2008-02-28 04:57:57 +03:00
|
|
|
typedef int (*orte_routed_module_init_routes_fn_t)(orte_jobid_t job, opal_buffer_t *ndat);
|
These changes were mostly captured in a prior RFC (except for #2 below) and are aimed specifically at improving startup performance and setting up the remaining modifications described in that RFC.
The commit has been tested for C/R and Cray operations, and on Odin (SLURM, rsh) and RoadRunner (TM). I tried to update all environments, but obviously could not test them. I know that Windows needs some work, and have highlighted what is know to be needed in the odls process component.
This represents a lot of work by Brian, Tim P, Josh, and myself, with much advice from Jeff and others. For posterity, I have appended a copy of the email describing the work that was done:
As we have repeatedly noted, the modex operation in MPI_Init is the single greatest consumer of time during startup. To-date, we have executed that operation as an ORTE stage gate that held the process until a startup message containing all required modex (and OOB contact info - see #3 below) info could be sent to it. Each process would send its data to the HNP's registry, which assembled and sent the message when all processes had reported in.
In addition, ORTE had taken responsibility for monitoring process status as it progressed through a series of "stage gates". The process reported its status at each gate, and ORTE would then send a "release" message once all procs had reported in.
The incoming changes revamp these procedures in three ways:
1. eliminating the ORTE stage gate system and cleanly delineating responsibility between the OMPI and ORTE layers for MPI init/finalize. The modex stage gate (STG1) has been replaced by a collective operation in the modex itself that performs an allgather on the required modex info. The allgather is implemented using the orte_grpcomm framework since the BTL's are not active at that point. At the moment, the grpcomm framework only has a "basic" component analogous to OMPI's "basic" coll framework - I would recommend that the MPI team create additional, more advanced components to improve performance of this step.
The other stage gates have been replaced by orte_grpcomm barrier functions. We tried to use MPI barriers instead (since the BTL's are active at that point), but - as we discussed on the telecon - these are not currently true barriers so the job would hang when we fell through while messages were still in process. Note that the grpcomm barrier doesn't actually resolve that problem, but Brian has pointed out that we are unlikely to ever see it violated. Again, you might want to spend a little time on an advanced barrier algorithm as the one in "basic" is very simplistic.
Summarizing this change: ORTE no longer tracks process state nor has direct responsibility for synchronizing jobs. This is now done via collective operations within the MPI layer, albeit using ORTE collective communication services. I -strongly- urge the MPI team to implement advanced collective algorithms to improve the performance of this critical procedure.
2. reducing the volume of data exchanged during modex. Data in the modex consisted of the process name, the name of the node where that process is located (expressed as a string), plus a string representation of all contact info. The nodename was required in order for the modex to determine if the process was local or not - in addition, some people like to have it to print pretty error messages when a connection failed.
The size of this data has been reduced in three ways:
(a) reducing the size of the process name itself. The process name consisted of two 32-bit fields for the jobid and vpid. This is far larger than any current system, or system likely to exist in the near future, can support. Accordingly, the default size of these fields has been reduced to 16-bits, which means you can have 32k procs in each of 32k jobs. Since the daemons must have a vpid, and we require one daemon/node, this also restricts the default configuration to 32k nodes.
To support any future "mega-clusters", a configuration option --enable-jumbo-apps has been added. This option increases the jobid and vpid field sizes to 32-bits. Someday, if necessary, someone can add yet another option to increase them to 64-bits, I suppose.
(b) replacing the string nodename with an integer nodeid. Since we have one daemon/node, the nodeid corresponds to the local daemon's vpid. This replaces an often lengthy string with only 2 (or at most 4) bytes, a substantial reduction.
(c) when the mca param requesting that nodenames be sent to support pretty error messages, a second mca param is now used to request FQDN - otherwise, the domain name is stripped (by default) from the message to save space. If someone wants to combine those into a single param somehow (perhaps with an argument?), they are welcome to do so - I didn't want to alter what people are already using.
While these may seem like small savings, they actually amount to a significant impact when aggregated across the entire modex operation. Since every proc must receive the modex data regardless of the collective used to send it, just reducing the size of the process name removes nearly 400MBytes of communication from a 32k proc job (admittedly, much of this comm may occur in parallel). So it does add up pretty quickly.
3. routing RML messages to reduce connections. The default messaging system remains point-to-point - i.e., each proc opens a socket to every proc it communicates with and sends its messages directly. A new option uses the orteds as routers - i.e., each proc only opens a single socket to its local orted. All messages are sent from the proc to the orted, which forwards the message to the orted on the node where the intended recipient proc is located - that orted then forwards the message to its local proc (the recipient). This greatly reduces the connection storm we have encountered during startup.
It also has the benefit of removing the sharing of every proc's OOB contact with every other proc. The orted routing tables are populated during launch since every orted gets a map of where every proc is being placed. Each proc, therefore, only needs to know the contact info for its local daemon, which is passed in via the environment when the proc is fork/exec'd by the daemon. This alone removes ~50 bytes/process of communication that was in the current STG1 startup message - so for our 32k proc job, this saves us roughly 32k*50 = 1.6MBytes sent to 32k procs = 51GBytes of messaging.
Note that you can use the new routing method by specifying -mca routed tree - if you so desire. This mode will become the default at some point in the future.
There are a few minor additional changes in the commit that I'll just note in passing:
* propagation of command line mca params to the orteds - fixes ticket #1073. See note there for details.
* requiring of "finalize" prior to "exit" for MPI procs - fixes ticket #1144. See note there for details.
* cleanup of some stale header files
This commit was SVN r16364.
2007-10-05 23:48:23 +04:00
|
|
|
|
2008-03-26 04:00:24 +03:00
|
|
|
/**
|
|
|
|
|
* Report a route as "lost"
|
|
|
|
|
*
|
|
|
|
|
* Report that an existing connection has been lost, therefore potentially
|
|
|
|
|
* "breaking" a route in the routing table. It is critical that broken
|
|
|
|
|
* connections be reported so that the selected routing module has the
|
|
|
|
|
* option of dealing with it. This could consist of nothing more than
|
|
|
|
|
* removing that route from the routing table, or could - in the case
|
|
|
|
|
* of a "lifeline" connection - result in abort of the process.
|
|
|
|
|
*/
|
|
|
|
|
typedef int (*orte_routed_module_route_lost_fn_t)(const orte_process_name_t *route);
|
|
|
|
|
|
2008-05-05 06:59:09 +04:00
|
|
|
/*
|
|
|
|
|
* Is this route defined?
|
|
|
|
|
*
|
|
|
|
|
* Check to see if a route to the specified target has been defined. The
|
|
|
|
|
* function returns "true" if it has, and "false" if no route to the
|
|
|
|
|
* target was previously defined.
|
|
|
|
|
*
|
|
|
|
|
* This is needed because routed modules will return their "wildcard"
|
|
|
|
|
* route if we request a route to a target that they don't know about.
|
|
|
|
|
* In some cases, though, we truly -do- need to know if a route was
|
|
|
|
|
* specifically defined.
|
|
|
|
|
*/
|
|
|
|
|
typedef bool (*orte_routed_module_route_is_defined_fn_t)(const orte_process_name_t *target);
|
|
|
|
|
|
2008-03-26 04:00:24 +03:00
|
|
|
/**
|
2008-10-03 00:08:27 +04:00
|
|
|
* Get wireup data for daemons
|
2008-03-26 04:00:24 +03:00
|
|
|
*
|
2008-10-03 00:08:27 +04:00
|
|
|
* Add whatever routing data
|
2010-07-20 08:22:45 +04:00
|
|
|
* this module requires to allow inter-process messaging.
|
2008-03-26 04:00:24 +03:00
|
|
|
*/
|
2008-10-03 00:08:27 +04:00
|
|
|
typedef int (*orte_routed_module_get_wireup_info_fn_t)(opal_buffer_t *buf);
|
2008-03-26 04:00:24 +03:00
|
|
|
|
2008-05-05 06:59:09 +04:00
|
|
|
/*
|
2012-04-06 18:23:13 +04:00
|
|
|
* Update the module's routing plan
|
2008-05-05 06:59:09 +04:00
|
|
|
*
|
2012-04-06 18:23:13 +04:00
|
|
|
* Called only by a daemon and the HNP, this function creates a plan
|
|
|
|
|
* for routing messages within ORTE, especially for routing collectives
|
|
|
|
|
* used during wireup
|
2008-05-05 06:59:09 +04:00
|
|
|
*/
|
2012-04-06 18:23:13 +04:00
|
|
|
typedef void (*orte_routed_module_update_routing_plan_fn_t)(void);
|
2008-05-05 06:59:09 +04:00
|
|
|
|
|
|
|
|
/*
|
Per the PMIx RFC:
WHAT: Merge the PMIx branch into the devel repo, creating a new
OPAL “lmix” framework to abstract PMI support for all RTEs.
Replace the ORTE daemon-level collectives with a new PMIx
server and update the ORTE grpcomm framework to support
server-to-server collectives
WHY: We’ve had problems dealing with variations in PMI implementations,
and need to extend the existing PMI definitions to meet exascale
requirements.
WHEN: Mon, Aug 25
WHERE: https://github.com/rhc54/ompi-svn-mirror.git
Several community members have been working on a refactoring of the current PMI support within OMPI. Although the APIs are common, Slurm and Cray implement a different range of capabilities, and package them differently. For example, Cray provides an integrated PMI-1/2 library, while Slurm separates the two and requires the user to specify the one to be used at runtime. In addition, several bugs in the Slurm implementations have caused problems requiring extra coding.
All this has led to a slew of #if’s in the PMI code and bugs when the corner-case logic for one implementation accidentally traps the other. Extending this support to other implementations would have increased this complexity to an unacceptable level.
Accordingly, we have:
* created a new OPAL “pmix” framework to abstract the PMI support, with separate components for Cray, Slurm PMI-1, and Slurm PMI-2 implementations.
* Replaced the current ORTE grpcomm daemon-based collective operation with an integrated PMIx server, and updated the grpcomm APIs to provide more flexible, multi-algorithm support for collective operations. At this time, only the xcast and allgather operations are supported.
* Replaced the current global collective id with a signature based on the names of the participating procs. The allows an unlimited number of collectives to be executed by any group of processes, subject to the requirement that only one collective can be active at a time for a unique combination of procs. Note that a proc can be involved in any number of simultaneous collectives - it is the specific combination of procs that is subject to the constraint
* removed the prior OMPI/OPAL modex code
* added new macros for executing modex send/recv to simplify use of the new APIs. The send macros allow the caller to specify whether or not the BTL supports async modex operations - if so, then the non-blocking “fence” operation is used, if the active PMIx component supports it. Otherwise, the default is a full blocking modex exchange as we currently perform.
* retained the current flag that directs us to use a blocking fence operation, but only to retrieve data upon demand
This commit was SVN r32570.
2014-08-21 22:56:47 +04:00
|
|
|
* Get the routing list for an xcast collective
|
2008-05-05 06:59:09 +04:00
|
|
|
*
|
Per the PMIx RFC:
WHAT: Merge the PMIx branch into the devel repo, creating a new
OPAL “lmix” framework to abstract PMI support for all RTEs.
Replace the ORTE daemon-level collectives with a new PMIx
server and update the ORTE grpcomm framework to support
server-to-server collectives
WHY: We’ve had problems dealing with variations in PMI implementations,
and need to extend the existing PMI definitions to meet exascale
requirements.
WHEN: Mon, Aug 25
WHERE: https://github.com/rhc54/ompi-svn-mirror.git
Several community members have been working on a refactoring of the current PMI support within OMPI. Although the APIs are common, Slurm and Cray implement a different range of capabilities, and package them differently. For example, Cray provides an integrated PMI-1/2 library, while Slurm separates the two and requires the user to specify the one to be used at runtime. In addition, several bugs in the Slurm implementations have caused problems requiring extra coding.
All this has led to a slew of #if’s in the PMI code and bugs when the corner-case logic for one implementation accidentally traps the other. Extending this support to other implementations would have increased this complexity to an unacceptable level.
Accordingly, we have:
* created a new OPAL “pmix” framework to abstract the PMI support, with separate components for Cray, Slurm PMI-1, and Slurm PMI-2 implementations.
* Replaced the current ORTE grpcomm daemon-based collective operation with an integrated PMIx server, and updated the grpcomm APIs to provide more flexible, multi-algorithm support for collective operations. At this time, only the xcast and allgather operations are supported.
* Replaced the current global collective id with a signature based on the names of the participating procs. The allows an unlimited number of collectives to be executed by any group of processes, subject to the requirement that only one collective can be active at a time for a unique combination of procs. Note that a proc can be involved in any number of simultaneous collectives - it is the specific combination of procs that is subject to the constraint
* removed the prior OMPI/OPAL modex code
* added new macros for executing modex send/recv to simplify use of the new APIs. The send macros allow the caller to specify whether or not the BTL supports async modex operations - if so, then the non-blocking “fence” operation is used, if the active PMIx component supports it. Otherwise, the default is a full blocking modex exchange as we currently perform.
* retained the current flag that directs us to use a blocking fence operation, but only to retrieve data upon demand
This commit was SVN r32570.
2014-08-21 22:56:47 +04:00
|
|
|
* Fills the target list with orte_namelist_t so that
|
|
|
|
|
* the grpcomm framework will know who to send xcast to
|
2012-04-06 18:23:13 +04:00
|
|
|
* next
|
2008-10-03 00:08:27 +04:00
|
|
|
*/
|
Per the PMIx RFC:
WHAT: Merge the PMIx branch into the devel repo, creating a new
OPAL “lmix” framework to abstract PMI support for all RTEs.
Replace the ORTE daemon-level collectives with a new PMIx
server and update the ORTE grpcomm framework to support
server-to-server collectives
WHY: We’ve had problems dealing with variations in PMI implementations,
and need to extend the existing PMI definitions to meet exascale
requirements.
WHEN: Mon, Aug 25
WHERE: https://github.com/rhc54/ompi-svn-mirror.git
Several community members have been working on a refactoring of the current PMI support within OMPI. Although the APIs are common, Slurm and Cray implement a different range of capabilities, and package them differently. For example, Cray provides an integrated PMI-1/2 library, while Slurm separates the two and requires the user to specify the one to be used at runtime. In addition, several bugs in the Slurm implementations have caused problems requiring extra coding.
All this has led to a slew of #if’s in the PMI code and bugs when the corner-case logic for one implementation accidentally traps the other. Extending this support to other implementations would have increased this complexity to an unacceptable level.
Accordingly, we have:
* created a new OPAL “pmix” framework to abstract the PMI support, with separate components for Cray, Slurm PMI-1, and Slurm PMI-2 implementations.
* Replaced the current ORTE grpcomm daemon-based collective operation with an integrated PMIx server, and updated the grpcomm APIs to provide more flexible, multi-algorithm support for collective operations. At this time, only the xcast and allgather operations are supported.
* Replaced the current global collective id with a signature based on the names of the participating procs. The allows an unlimited number of collectives to be executed by any group of processes, subject to the requirement that only one collective can be active at a time for a unique combination of procs. Note that a proc can be involved in any number of simultaneous collectives - it is the specific combination of procs that is subject to the constraint
* removed the prior OMPI/OPAL modex code
* added new macros for executing modex send/recv to simplify use of the new APIs. The send macros allow the caller to specify whether or not the BTL supports async modex operations - if so, then the non-blocking “fence” operation is used, if the active PMIx component supports it. Otherwise, the default is a full blocking modex exchange as we currently perform.
* retained the current flag that directs us to use a blocking fence operation, but only to retrieve data upon demand
This commit was SVN r32570.
2014-08-21 22:56:47 +04:00
|
|
|
typedef void (*orte_routed_module_get_routing_list_fn_t)(opal_list_t *coll);
|
2007-07-20 05:34:02 +04:00
|
|
|
|
2008-12-21 02:23:11 +03:00
|
|
|
/*
|
|
|
|
|
* Set lifeline process
|
|
|
|
|
*
|
|
|
|
|
* Defines the lifeline to be the specified process. Should contact to
|
|
|
|
|
* that process be lost, the errmgr will be called, possibly resulting
|
|
|
|
|
* in termination of the process and job.
|
|
|
|
|
*/
|
|
|
|
|
typedef int (*orte_routed_module_set_lifeline_fn_t)(orte_process_name_t *proc);
|
|
|
|
|
|
2010-07-18 01:03:27 +04:00
|
|
|
/*
|
|
|
|
|
* Get the number of routes supported by this process
|
|
|
|
|
*
|
|
|
|
|
* Returns the size of the routing tree using an O(1) function
|
|
|
|
|
*/
|
|
|
|
|
typedef size_t (*orte_routed_module_num_routes_fn_t)(void);
|
|
|
|
|
|
2008-04-23 04:17:12 +04:00
|
|
|
/**
|
|
|
|
|
* Handle fault tolerance updates
|
|
|
|
|
*
|
|
|
|
|
* @param[in] state Fault tolerance state update
|
|
|
|
|
*
|
|
|
|
|
* @retval ORTE_SUCCESS The operation completed successfully
|
|
|
|
|
* @retval ORTE_ERROR An unspecifed error occurred
|
|
|
|
|
*/
|
|
|
|
|
typedef int (*orte_routed_module_ft_event_fn_t)(int state);
|
|
|
|
|
|
2007-07-20 05:34:02 +04:00
|
|
|
/* ******************************************************************** */
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/**
|
|
|
|
|
* routed module interface
|
|
|
|
|
*
|
|
|
|
|
* Module interface to the routed communication system. A global
|
|
|
|
|
* instance of this module, orte_routed, provices an interface into the
|
|
|
|
|
* active routed interface.
|
|
|
|
|
*/
|
|
|
|
|
struct orte_routed_module_t {
|
2007-10-16 20:09:41 +04:00
|
|
|
/** Startup/shutdown the communication system and clean up resources */
|
|
|
|
|
orte_routed_module_init_fn_t initialize;
|
2007-07-20 05:34:02 +04:00
|
|
|
orte_routed_module_finalize_fn_t finalize;
|
2007-10-16 20:09:41 +04:00
|
|
|
/* API functions */
|
2008-07-22 01:37:09 +04:00
|
|
|
orte_routed_module_delete_route_fn_t delete_route;
|
2007-07-20 05:34:02 +04:00
|
|
|
orte_routed_module_update_route_fn_t update_route;
|
|
|
|
|
orte_routed_module_get_route_fn_t get_route;
|
These changes were mostly captured in a prior RFC (except for #2 below) and are aimed specifically at improving startup performance and setting up the remaining modifications described in that RFC.
The commit has been tested for C/R and Cray operations, and on Odin (SLURM, rsh) and RoadRunner (TM). I tried to update all environments, but obviously could not test them. I know that Windows needs some work, and have highlighted what is know to be needed in the odls process component.
This represents a lot of work by Brian, Tim P, Josh, and myself, with much advice from Jeff and others. For posterity, I have appended a copy of the email describing the work that was done:
As we have repeatedly noted, the modex operation in MPI_Init is the single greatest consumer of time during startup. To-date, we have executed that operation as an ORTE stage gate that held the process until a startup message containing all required modex (and OOB contact info - see #3 below) info could be sent to it. Each process would send its data to the HNP's registry, which assembled and sent the message when all processes had reported in.
In addition, ORTE had taken responsibility for monitoring process status as it progressed through a series of "stage gates". The process reported its status at each gate, and ORTE would then send a "release" message once all procs had reported in.
The incoming changes revamp these procedures in three ways:
1. eliminating the ORTE stage gate system and cleanly delineating responsibility between the OMPI and ORTE layers for MPI init/finalize. The modex stage gate (STG1) has been replaced by a collective operation in the modex itself that performs an allgather on the required modex info. The allgather is implemented using the orte_grpcomm framework since the BTL's are not active at that point. At the moment, the grpcomm framework only has a "basic" component analogous to OMPI's "basic" coll framework - I would recommend that the MPI team create additional, more advanced components to improve performance of this step.
The other stage gates have been replaced by orte_grpcomm barrier functions. We tried to use MPI barriers instead (since the BTL's are active at that point), but - as we discussed on the telecon - these are not currently true barriers so the job would hang when we fell through while messages were still in process. Note that the grpcomm barrier doesn't actually resolve that problem, but Brian has pointed out that we are unlikely to ever see it violated. Again, you might want to spend a little time on an advanced barrier algorithm as the one in "basic" is very simplistic.
Summarizing this change: ORTE no longer tracks process state nor has direct responsibility for synchronizing jobs. This is now done via collective operations within the MPI layer, albeit using ORTE collective communication services. I -strongly- urge the MPI team to implement advanced collective algorithms to improve the performance of this critical procedure.
2. reducing the volume of data exchanged during modex. Data in the modex consisted of the process name, the name of the node where that process is located (expressed as a string), plus a string representation of all contact info. The nodename was required in order for the modex to determine if the process was local or not - in addition, some people like to have it to print pretty error messages when a connection failed.
The size of this data has been reduced in three ways:
(a) reducing the size of the process name itself. The process name consisted of two 32-bit fields for the jobid and vpid. This is far larger than any current system, or system likely to exist in the near future, can support. Accordingly, the default size of these fields has been reduced to 16-bits, which means you can have 32k procs in each of 32k jobs. Since the daemons must have a vpid, and we require one daemon/node, this also restricts the default configuration to 32k nodes.
To support any future "mega-clusters", a configuration option --enable-jumbo-apps has been added. This option increases the jobid and vpid field sizes to 32-bits. Someday, if necessary, someone can add yet another option to increase them to 64-bits, I suppose.
(b) replacing the string nodename with an integer nodeid. Since we have one daemon/node, the nodeid corresponds to the local daemon's vpid. This replaces an often lengthy string with only 2 (or at most 4) bytes, a substantial reduction.
(c) when the mca param requesting that nodenames be sent to support pretty error messages, a second mca param is now used to request FQDN - otherwise, the domain name is stripped (by default) from the message to save space. If someone wants to combine those into a single param somehow (perhaps with an argument?), they are welcome to do so - I didn't want to alter what people are already using.
While these may seem like small savings, they actually amount to a significant impact when aggregated across the entire modex operation. Since every proc must receive the modex data regardless of the collective used to send it, just reducing the size of the process name removes nearly 400MBytes of communication from a 32k proc job (admittedly, much of this comm may occur in parallel). So it does add up pretty quickly.
3. routing RML messages to reduce connections. The default messaging system remains point-to-point - i.e., each proc opens a socket to every proc it communicates with and sends its messages directly. A new option uses the orteds as routers - i.e., each proc only opens a single socket to its local orted. All messages are sent from the proc to the orted, which forwards the message to the orted on the node where the intended recipient proc is located - that orted then forwards the message to its local proc (the recipient). This greatly reduces the connection storm we have encountered during startup.
It also has the benefit of removing the sharing of every proc's OOB contact with every other proc. The orted routing tables are populated during launch since every orted gets a map of where every proc is being placed. Each proc, therefore, only needs to know the contact info for its local daemon, which is passed in via the environment when the proc is fork/exec'd by the daemon. This alone removes ~50 bytes/process of communication that was in the current STG1 startup message - so for our 32k proc job, this saves us roughly 32k*50 = 1.6MBytes sent to 32k procs = 51GBytes of messaging.
Note that you can use the new routing method by specifying -mca routed tree - if you so desire. This mode will become the default at some point in the future.
There are a few minor additional changes in the commit that I'll just note in passing:
* propagation of command line mca params to the orteds - fixes ticket #1073. See note there for details.
* requiring of "finalize" prior to "exit" for MPI procs - fixes ticket #1144. See note there for details.
* cleanup of some stale header files
This commit was SVN r16364.
2007-10-05 23:48:23 +04:00
|
|
|
orte_routed_module_init_routes_fn_t init_routes;
|
2008-03-26 04:00:24 +03:00
|
|
|
orte_routed_module_route_lost_fn_t route_lost;
|
2008-05-05 06:59:09 +04:00
|
|
|
orte_routed_module_route_is_defined_fn_t route_is_defined;
|
2008-12-21 02:23:11 +03:00
|
|
|
orte_routed_module_set_lifeline_fn_t set_lifeline;
|
2008-05-05 06:59:09 +04:00
|
|
|
/* fns for daemons */
|
2012-04-06 18:23:13 +04:00
|
|
|
orte_routed_module_update_routing_plan_fn_t update_routing_plan;
|
|
|
|
|
orte_routed_module_get_routing_list_fn_t get_routing_list;
|
2008-03-26 04:00:24 +03:00
|
|
|
orte_routed_module_get_wireup_info_fn_t get_wireup_info;
|
2010-07-18 01:03:27 +04:00
|
|
|
orte_routed_module_num_routes_fn_t num_routes;
|
2008-04-23 04:17:12 +04:00
|
|
|
/* FT Notification */
|
|
|
|
|
orte_routed_module_ft_event_fn_t ft_event;
|
2007-07-20 05:34:02 +04:00
|
|
|
};
|
2008-03-26 04:00:24 +03:00
|
|
|
/** Convenience typedef */
|
2007-07-20 05:34:02 +04:00
|
|
|
typedef struct orte_routed_module_t orte_routed_module_t;
|
|
|
|
|
|
|
|
|
|
/** Interface for routed communication */
|
|
|
|
|
ORTE_DECLSPEC extern orte_routed_module_t orte_routed;
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
/* ******************************************************************** */
|
|
|
|
|
|
|
|
|
|
|
2008-07-29 02:40:57 +04:00
|
|
|
/** Macro for use in components that are of type routed */
|
|
|
|
|
#define ORTE_ROUTED_BASE_VERSION_2_0_0 \
|
|
|
|
|
MCA_BASE_VERSION_2_0_0, \
|
|
|
|
|
"routed", 2, 0, 0
|
2007-07-20 05:34:02 +04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
/* ******************************************************************** */
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
END_C_DECLS
|
|
|
|
|
|
|
|
|
|
#endif
|