2011-10-17 20:51:22 +00:00
|
|
|
/*
|
|
|
|
|
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
|
|
|
|
* University Research and Technology
|
|
|
|
|
* Corporation. All rights reserved.
|
|
|
|
|
* Copyright (c) 2004-2011 The University of Tennessee and The University
|
|
|
|
|
* of Tennessee Research Foundation. All rights
|
|
|
|
|
* reserved.
|
2015-06-23 20:59:57 -07:00
|
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
2011-10-17 20:51:22 +00:00
|
|
|
* University of Stuttgart. All rights reserved.
|
|
|
|
|
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
|
|
|
* All rights reserved.
|
Per RFC, bring in the following changes:
* Remove paffinity, maffinity, and carto frameworks -- they've been
wholly replaced by hwloc.
* Move ompi_mpi_init() affinity-setting/checking code down to ORTE.
* Update sm, smcuda, wv, and openib components to no longer use carto.
Instead, use hwloc data. There are still optimizations possible in
the sm/smcuda BTLs (i.e., making multiple mpools). Also, the old
carto-based code found out how many NUMA nodes were ''available''
-- not how many were used ''in this job''. The new hwloc-using
code computes the same value -- it was not updated to calculate how
many NUMA nodes are used ''by this job.''
* Note that I cannot compile the smcuda and wv BTLs -- I ''think''
they're right, but they need to be verified by their owners.
* The openib component now does a bunch of stuff to figure out where
"near" OpenFabrics devices are. '''THIS IS A CHANGE IN DEFAULT
BEHAVIOR!!''' and still needs to be verified by OpenFabrics vendors
(I do not have a NUMA machine with an OpenFabrics device that is a
non-uniform distance from multiple different NUMA nodes).
* Completely rewrite the OMPI_Affinity_str() routine from the
"affinity" mpiext extension. This extension now understands
hyperthreads; the output format of it has changed a bit to reflect
this new information.
* Bunches of minor changes around the code base to update names/types
from maffinity/paffinity-based names to hwloc-based names.
* Add some helper functions into the hwloc base, mainly having to do
with the fact that we have the hwloc data reporting ''all''
topology information, but sometimes you really only want the
(online | available) data.
This commit was SVN r26391.
2012-05-07 14:52:54 +00:00
|
|
|
* Copyright (c) 2008-2012 Cisco Systems, Inc. All rights reserved.
|
2013-01-28 20:14:51 +00:00
|
|
|
* Copyright (c) 2012-2013 Los Alamos National Security, LLC.
|
2015-06-23 20:59:57 -07:00
|
|
|
* All rights reserved.
|
2016-03-02 14:30:57 -08:00
|
|
|
* Copyright (c) 2013-2016 Intel, Inc. All rights reserved.
|
2011-10-17 20:51:22 +00:00
|
|
|
* $COPYRIGHT$
|
2015-06-23 20:59:57 -07:00
|
|
|
*
|
2011-10-17 20:51:22 +00:00
|
|
|
* Additional copyrights may follow
|
2015-06-23 20:59:57 -07:00
|
|
|
*
|
2011-10-17 20:51:22 +00:00
|
|
|
* $HEADER$
|
|
|
|
|
*
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#include "orte_config.h"
|
|
|
|
|
#include "orte/constants.h"
|
|
|
|
|
|
|
|
|
|
#ifdef HAVE_UNISTD_H
|
|
|
|
|
#include <unistd.h>
|
|
|
|
|
#endif /* HAVE_UNISTD_H */
|
|
|
|
|
#include <string.h>
|
|
|
|
|
#include <ctype.h>
|
|
|
|
|
#ifdef HAVE_NETDB_H
|
|
|
|
|
#include <netdb.h>
|
|
|
|
|
#endif
|
|
|
|
|
#ifdef HAVE_IFADDRS_H
|
|
|
|
|
#include <ifaddrs.h>
|
|
|
|
|
#endif
|
|
|
|
|
|
|
|
|
|
#include "opal/util/opal_environ.h"
|
|
|
|
|
#include "opal/util/output.h"
|
|
|
|
|
#include "opal/util/argv.h"
|
2016-03-02 14:30:57 -08:00
|
|
|
#include "opal/runtime/opal_progress_threads.h"
|
2011-10-17 20:51:22 +00:00
|
|
|
#include "opal/class/opal_pointer_array.h"
|
2011-11-02 17:42:06 +00:00
|
|
|
#include "opal/mca/hwloc/base/base.h"
|
2011-10-17 20:51:22 +00:00
|
|
|
#include "opal/util/printf.h"
|
2015-06-18 09:53:20 -07:00
|
|
|
#include "opal/util/proc.h"
|
|
|
|
|
#include "opal/mca/pmix/pmix.h"
|
2014-12-01 21:19:11 -08:00
|
|
|
#include "opal/mca/pmix/base/base.h"
|
2011-10-17 20:51:22 +00:00
|
|
|
|
2012-06-27 14:53:55 +00:00
|
|
|
#include "orte/mca/errmgr/errmgr.h"
|
2016-03-18 18:50:24 -07:00
|
|
|
#include "orte/mca/grpcomm/grpcomm.h"
|
|
|
|
|
#include "orte/mca/rml/rml.h"
|
2011-10-17 20:51:22 +00:00
|
|
|
#include "orte/util/proc_info.h"
|
|
|
|
|
#include "orte/util/show_help.h"
|
|
|
|
|
#include "orte/util/name_fns.h"
|
|
|
|
|
#include "orte/util/pre_condition_transports.h"
|
|
|
|
|
#include "orte/util/regex.h"
|
2012-06-27 14:53:55 +00:00
|
|
|
#include "orte/runtime/orte_globals.h"
|
2011-10-17 20:51:22 +00:00
|
|
|
#include "orte/runtime/orte_wait.h"
|
|
|
|
|
|
|
|
|
|
#include "orte/mca/ess/ess.h"
|
|
|
|
|
#include "orte/mca/ess/base/base.h"
|
|
|
|
|
#include "orte/mca/ess/pmi/ess_pmi.h"
|
|
|
|
|
|
|
|
|
|
static int rte_init(void);
|
|
|
|
|
static int rte_finalize(void);
|
2012-04-06 14:23:13 +00:00
|
|
|
static void rte_abort(int error_code, bool report);
|
2011-10-17 20:51:22 +00:00
|
|
|
|
|
|
|
|
orte_ess_base_module_t orte_ess_pmi_module = {
|
|
|
|
|
rte_init,
|
|
|
|
|
rte_finalize,
|
|
|
|
|
rte_abort,
|
|
|
|
|
NULL /* ft_event */
|
|
|
|
|
};
|
|
|
|
|
|
2014-12-01 21:19:11 -08:00
|
|
|
static bool added_transport_keys=false;
|
|
|
|
|
static bool added_num_procs = false;
|
|
|
|
|
static bool added_app_ctx = false;
|
2016-03-02 14:30:57 -08:00
|
|
|
static bool progress_thread_running = false;
|
2011-10-17 20:51:22 +00:00
|
|
|
|
|
|
|
|
/**** MODULE FUNCTIONS ****/
|
|
|
|
|
|
|
|
|
|
static int rte_init(void)
|
|
|
|
|
{
|
Per the PMIx RFC:
WHAT: Merge the PMIx branch into the devel repo, creating a new
OPAL “lmix” framework to abstract PMI support for all RTEs.
Replace the ORTE daemon-level collectives with a new PMIx
server and update the ORTE grpcomm framework to support
server-to-server collectives
WHY: We’ve had problems dealing with variations in PMI implementations,
and need to extend the existing PMI definitions to meet exascale
requirements.
WHEN: Mon, Aug 25
WHERE: https://github.com/rhc54/ompi-svn-mirror.git
Several community members have been working on a refactoring of the current PMI support within OMPI. Although the APIs are common, Slurm and Cray implement a different range of capabilities, and package them differently. For example, Cray provides an integrated PMI-1/2 library, while Slurm separates the two and requires the user to specify the one to be used at runtime. In addition, several bugs in the Slurm implementations have caused problems requiring extra coding.
All this has led to a slew of #if’s in the PMI code and bugs when the corner-case logic for one implementation accidentally traps the other. Extending this support to other implementations would have increased this complexity to an unacceptable level.
Accordingly, we have:
* created a new OPAL “pmix” framework to abstract the PMI support, with separate components for Cray, Slurm PMI-1, and Slurm PMI-2 implementations.
* Replaced the current ORTE grpcomm daemon-based collective operation with an integrated PMIx server, and updated the grpcomm APIs to provide more flexible, multi-algorithm support for collective operations. At this time, only the xcast and allgather operations are supported.
* Replaced the current global collective id with a signature based on the names of the participating procs. The allows an unlimited number of collectives to be executed by any group of processes, subject to the requirement that only one collective can be active at a time for a unique combination of procs. Note that a proc can be involved in any number of simultaneous collectives - it is the specific combination of procs that is subject to the constraint
* removed the prior OMPI/OPAL modex code
* added new macros for executing modex send/recv to simplify use of the new APIs. The send macros allow the caller to specify whether or not the BTL supports async modex operations - if so, then the non-blocking “fence” operation is used, if the active PMIx component supports it. Otherwise, the default is a full blocking modex exchange as we currently perform.
* retained the current flag that directs us to use a blocking fence operation, but only to retrieve data upon demand
This commit was SVN r32570.
2014-08-21 18:56:47 +00:00
|
|
|
int ret;
|
|
|
|
|
char *error = NULL;
|
2012-08-12 01:28:23 +00:00
|
|
|
char *envar, *ev1, *ev2;
|
2011-10-17 20:51:22 +00:00
|
|
|
uint64_t unique_key[2];
|
2014-12-01 21:19:11 -08:00
|
|
|
char *string_key;
|
2016-03-18 18:50:24 -07:00
|
|
|
char *rmluri;
|
2015-06-18 09:53:20 -07:00
|
|
|
opal_value_t *kv;
|
|
|
|
|
char *val;
|
|
|
|
|
int u32, *u32ptr;
|
|
|
|
|
uint16_t u16, *u16ptr;
|
2015-09-11 06:38:47 -07:00
|
|
|
char **peers=NULL, *mycpuset, **cpusets=NULL;
|
2016-08-06 15:43:00 +06:00
|
|
|
opal_process_name_t wildcard_rank, pname;
|
2015-06-18 09:53:20 -07:00
|
|
|
size_t i;
|
2011-10-17 20:51:22 +00:00
|
|
|
|
|
|
|
|
/* run the prolog */
|
|
|
|
|
if (ORTE_SUCCESS != (ret = orte_ess_base_std_prolog())) {
|
|
|
|
|
error = "orte_ess_base_std_prolog";
|
|
|
|
|
goto error;
|
|
|
|
|
}
|
2015-06-23 20:59:57 -07:00
|
|
|
|
2016-03-02 14:30:57 -08:00
|
|
|
/* get an async event base - we use the opal_async one so
|
|
|
|
|
* we don't startup extra threads if not needed */
|
|
|
|
|
orte_event_base = opal_progress_thread_init(NULL);
|
|
|
|
|
progress_thread_running = true;
|
|
|
|
|
|
2016-02-25 11:05:38 -06:00
|
|
|
/* open and setup pmix */
|
|
|
|
|
if (OPAL_SUCCESS != (ret = mca_base_framework_open(&opal_pmix_base_framework, 0))) {
|
|
|
|
|
ORTE_ERROR_LOG(ret);
|
|
|
|
|
/* we cannot run */
|
|
|
|
|
error = "pmix init";
|
|
|
|
|
goto error;
|
|
|
|
|
}
|
|
|
|
|
if (OPAL_SUCCESS != (ret = opal_pmix_base_select())) {
|
|
|
|
|
/* we cannot run */
|
|
|
|
|
error = "pmix init";
|
|
|
|
|
goto error;
|
|
|
|
|
}
|
2016-03-02 14:30:57 -08:00
|
|
|
/* set the event base */
|
|
|
|
|
opal_pmix_base_set_evbase(orte_event_base);
|
2015-10-17 20:24:03 -07:00
|
|
|
/* initialize the selected module */
|
|
|
|
|
if (!opal_pmix.initialized() && (OPAL_SUCCESS != (ret = opal_pmix.init()))) {
|
|
|
|
|
/* we cannot run */
|
|
|
|
|
error = "pmix init";
|
|
|
|
|
goto error;
|
|
|
|
|
}
|
2015-06-18 09:53:20 -07:00
|
|
|
u32ptr = &u32;
|
|
|
|
|
u16ptr = &u16;
|
2011-10-17 20:51:22 +00:00
|
|
|
|
Per the PMIx RFC:
WHAT: Merge the PMIx branch into the devel repo, creating a new
OPAL “lmix” framework to abstract PMI support for all RTEs.
Replace the ORTE daemon-level collectives with a new PMIx
server and update the ORTE grpcomm framework to support
server-to-server collectives
WHY: We’ve had problems dealing with variations in PMI implementations,
and need to extend the existing PMI definitions to meet exascale
requirements.
WHEN: Mon, Aug 25
WHERE: https://github.com/rhc54/ompi-svn-mirror.git
Several community members have been working on a refactoring of the current PMI support within OMPI. Although the APIs are common, Slurm and Cray implement a different range of capabilities, and package them differently. For example, Cray provides an integrated PMI-1/2 library, while Slurm separates the two and requires the user to specify the one to be used at runtime. In addition, several bugs in the Slurm implementations have caused problems requiring extra coding.
All this has led to a slew of #if’s in the PMI code and bugs when the corner-case logic for one implementation accidentally traps the other. Extending this support to other implementations would have increased this complexity to an unacceptable level.
Accordingly, we have:
* created a new OPAL “pmix” framework to abstract the PMI support, with separate components for Cray, Slurm PMI-1, and Slurm PMI-2 implementations.
* Replaced the current ORTE grpcomm daemon-based collective operation with an integrated PMIx server, and updated the grpcomm APIs to provide more flexible, multi-algorithm support for collective operations. At this time, only the xcast and allgather operations are supported.
* Replaced the current global collective id with a signature based on the names of the participating procs. The allows an unlimited number of collectives to be executed by any group of processes, subject to the requirement that only one collective can be active at a time for a unique combination of procs. Note that a proc can be involved in any number of simultaneous collectives - it is the specific combination of procs that is subject to the constraint
* removed the prior OMPI/OPAL modex code
* added new macros for executing modex send/recv to simplify use of the new APIs. The send macros allow the caller to specify whether or not the BTL supports async modex operations - if so, then the non-blocking “fence” operation is used, if the active PMIx component supports it. Otherwise, the default is a full blocking modex exchange as we currently perform.
* retained the current flag that directs us to use a blocking fence operation, but only to retrieve data upon demand
This commit was SVN r32570.
2014-08-21 18:56:47 +00:00
|
|
|
/**** THE FOLLOWING ARE REQUIRED VALUES ***/
|
2015-06-18 09:53:20 -07:00
|
|
|
/* pmix.init set our process name down in the OPAL layer,
|
|
|
|
|
* so carry it forward here */
|
|
|
|
|
ORTE_PROC_MY_NAME->jobid = OPAL_PROC_MY_NAME.jobid;
|
|
|
|
|
ORTE_PROC_MY_NAME->vpid = OPAL_PROC_MY_NAME.vpid;
|
Per the PMIx RFC:
WHAT: Merge the PMIx branch into the devel repo, creating a new
OPAL “lmix” framework to abstract PMI support for all RTEs.
Replace the ORTE daemon-level collectives with a new PMIx
server and update the ORTE grpcomm framework to support
server-to-server collectives
WHY: We’ve had problems dealing with variations in PMI implementations,
and need to extend the existing PMI definitions to meet exascale
requirements.
WHEN: Mon, Aug 25
WHERE: https://github.com/rhc54/ompi-svn-mirror.git
Several community members have been working on a refactoring of the current PMI support within OMPI. Although the APIs are common, Slurm and Cray implement a different range of capabilities, and package them differently. For example, Cray provides an integrated PMI-1/2 library, while Slurm separates the two and requires the user to specify the one to be used at runtime. In addition, several bugs in the Slurm implementations have caused problems requiring extra coding.
All this has led to a slew of #if’s in the PMI code and bugs when the corner-case logic for one implementation accidentally traps the other. Extending this support to other implementations would have increased this complexity to an unacceptable level.
Accordingly, we have:
* created a new OPAL “pmix” framework to abstract the PMI support, with separate components for Cray, Slurm PMI-1, and Slurm PMI-2 implementations.
* Replaced the current ORTE grpcomm daemon-based collective operation with an integrated PMIx server, and updated the grpcomm APIs to provide more flexible, multi-algorithm support for collective operations. At this time, only the xcast and allgather operations are supported.
* Replaced the current global collective id with a signature based on the names of the participating procs. The allows an unlimited number of collectives to be executed by any group of processes, subject to the requirement that only one collective can be active at a time for a unique combination of procs. Note that a proc can be involved in any number of simultaneous collectives - it is the specific combination of procs that is subject to the constraint
* removed the prior OMPI/OPAL modex code
* added new macros for executing modex send/recv to simplify use of the new APIs. The send macros allow the caller to specify whether or not the BTL supports async modex operations - if so, then the non-blocking “fence” operation is used, if the active PMIx component supports it. Otherwise, the default is a full blocking modex exchange as we currently perform.
* retained the current flag that directs us to use a blocking fence operation, but only to retrieve data upon demand
This commit was SVN r32570.
2014-08-21 18:56:47 +00:00
|
|
|
|
2016-07-19 20:10:58 -07:00
|
|
|
/* setup a name for retrieving data associated with the job */
|
2016-08-06 15:43:00 +06:00
|
|
|
wildcard_rank.jobid = ORTE_PROC_MY_NAME->jobid;
|
|
|
|
|
wildcard_rank.vpid = ORTE_NAME_WILDCARD->vpid;
|
|
|
|
|
|
|
|
|
|
/* setup a name for retrieving proc-specific data */
|
|
|
|
|
pname.jobid = ORTE_PROC_MY_NAME->jobid;
|
|
|
|
|
pname.vpid = 0;
|
2016-07-21 09:02:00 -07:00
|
|
|
|
Per the PMIx RFC:
WHAT: Merge the PMIx branch into the devel repo, creating a new
OPAL “lmix” framework to abstract PMI support for all RTEs.
Replace the ORTE daemon-level collectives with a new PMIx
server and update the ORTE grpcomm framework to support
server-to-server collectives
WHY: We’ve had problems dealing with variations in PMI implementations,
and need to extend the existing PMI definitions to meet exascale
requirements.
WHEN: Mon, Aug 25
WHERE: https://github.com/rhc54/ompi-svn-mirror.git
Several community members have been working on a refactoring of the current PMI support within OMPI. Although the APIs are common, Slurm and Cray implement a different range of capabilities, and package them differently. For example, Cray provides an integrated PMI-1/2 library, while Slurm separates the two and requires the user to specify the one to be used at runtime. In addition, several bugs in the Slurm implementations have caused problems requiring extra coding.
All this has led to a slew of #if’s in the PMI code and bugs when the corner-case logic for one implementation accidentally traps the other. Extending this support to other implementations would have increased this complexity to an unacceptable level.
Accordingly, we have:
* created a new OPAL “pmix” framework to abstract the PMI support, with separate components for Cray, Slurm PMI-1, and Slurm PMI-2 implementations.
* Replaced the current ORTE grpcomm daemon-based collective operation with an integrated PMIx server, and updated the grpcomm APIs to provide more flexible, multi-algorithm support for collective operations. At this time, only the xcast and allgather operations are supported.
* Replaced the current global collective id with a signature based on the names of the participating procs. The allows an unlimited number of collectives to be executed by any group of processes, subject to the requirement that only one collective can be active at a time for a unique combination of procs. Note that a proc can be involved in any number of simultaneous collectives - it is the specific combination of procs that is subject to the constraint
* removed the prior OMPI/OPAL modex code
* added new macros for executing modex send/recv to simplify use of the new APIs. The send macros allow the caller to specify whether or not the BTL supports async modex operations - if so, then the non-blocking “fence” operation is used, if the active PMIx component supports it. Otherwise, the default is a full blocking modex exchange as we currently perform.
* retained the current flag that directs us to use a blocking fence operation, but only to retrieve data upon demand
This commit was SVN r32570.
2014-08-21 18:56:47 +00:00
|
|
|
/* get our local rank from PMI */
|
2015-06-18 09:53:20 -07:00
|
|
|
OPAL_MODEX_RECV_VALUE(ret, OPAL_PMIX_LOCAL_RANK,
|
|
|
|
|
ORTE_PROC_MY_NAME, &u16ptr, OPAL_UINT16);
|
|
|
|
|
if (OPAL_SUCCESS != ret) {
|
Per the PMIx RFC:
WHAT: Merge the PMIx branch into the devel repo, creating a new
OPAL “lmix” framework to abstract PMI support for all RTEs.
Replace the ORTE daemon-level collectives with a new PMIx
server and update the ORTE grpcomm framework to support
server-to-server collectives
WHY: We’ve had problems dealing with variations in PMI implementations,
and need to extend the existing PMI definitions to meet exascale
requirements.
WHEN: Mon, Aug 25
WHERE: https://github.com/rhc54/ompi-svn-mirror.git
Several community members have been working on a refactoring of the current PMI support within OMPI. Although the APIs are common, Slurm and Cray implement a different range of capabilities, and package them differently. For example, Cray provides an integrated PMI-1/2 library, while Slurm separates the two and requires the user to specify the one to be used at runtime. In addition, several bugs in the Slurm implementations have caused problems requiring extra coding.
All this has led to a slew of #if’s in the PMI code and bugs when the corner-case logic for one implementation accidentally traps the other. Extending this support to other implementations would have increased this complexity to an unacceptable level.
Accordingly, we have:
* created a new OPAL “pmix” framework to abstract the PMI support, with separate components for Cray, Slurm PMI-1, and Slurm PMI-2 implementations.
* Replaced the current ORTE grpcomm daemon-based collective operation with an integrated PMIx server, and updated the grpcomm APIs to provide more flexible, multi-algorithm support for collective operations. At this time, only the xcast and allgather operations are supported.
* Replaced the current global collective id with a signature based on the names of the participating procs. The allows an unlimited number of collectives to be executed by any group of processes, subject to the requirement that only one collective can be active at a time for a unique combination of procs. Note that a proc can be involved in any number of simultaneous collectives - it is the specific combination of procs that is subject to the constraint
* removed the prior OMPI/OPAL modex code
* added new macros for executing modex send/recv to simplify use of the new APIs. The send macros allow the caller to specify whether or not the BTL supports async modex operations - if so, then the non-blocking “fence” operation is used, if the active PMIx component supports it. Otherwise, the default is a full blocking modex exchange as we currently perform.
* retained the current flag that directs us to use a blocking fence operation, but only to retrieve data upon demand
This commit was SVN r32570.
2014-08-21 18:56:47 +00:00
|
|
|
error = "getting local rank";
|
|
|
|
|
goto error;
|
|
|
|
|
}
|
2015-06-18 09:53:20 -07:00
|
|
|
orte_process_info.my_local_rank = u16;
|
Per the PMIx RFC:
WHAT: Merge the PMIx branch into the devel repo, creating a new
OPAL “lmix” framework to abstract PMI support for all RTEs.
Replace the ORTE daemon-level collectives with a new PMIx
server and update the ORTE grpcomm framework to support
server-to-server collectives
WHY: We’ve had problems dealing with variations in PMI implementations,
and need to extend the existing PMI definitions to meet exascale
requirements.
WHEN: Mon, Aug 25
WHERE: https://github.com/rhc54/ompi-svn-mirror.git
Several community members have been working on a refactoring of the current PMI support within OMPI. Although the APIs are common, Slurm and Cray implement a different range of capabilities, and package them differently. For example, Cray provides an integrated PMI-1/2 library, while Slurm separates the two and requires the user to specify the one to be used at runtime. In addition, several bugs in the Slurm implementations have caused problems requiring extra coding.
All this has led to a slew of #if’s in the PMI code and bugs when the corner-case logic for one implementation accidentally traps the other. Extending this support to other implementations would have increased this complexity to an unacceptable level.
Accordingly, we have:
* created a new OPAL “pmix” framework to abstract the PMI support, with separate components for Cray, Slurm PMI-1, and Slurm PMI-2 implementations.
* Replaced the current ORTE grpcomm daemon-based collective operation with an integrated PMIx server, and updated the grpcomm APIs to provide more flexible, multi-algorithm support for collective operations. At this time, only the xcast and allgather operations are supported.
* Replaced the current global collective id with a signature based on the names of the participating procs. The allows an unlimited number of collectives to be executed by any group of processes, subject to the requirement that only one collective can be active at a time for a unique combination of procs. Note that a proc can be involved in any number of simultaneous collectives - it is the specific combination of procs that is subject to the constraint
* removed the prior OMPI/OPAL modex code
* added new macros for executing modex send/recv to simplify use of the new APIs. The send macros allow the caller to specify whether or not the BTL supports async modex operations - if so, then the non-blocking “fence” operation is used, if the active PMIx component supports it. Otherwise, the default is a full blocking modex exchange as we currently perform.
* retained the current flag that directs us to use a blocking fence operation, but only to retrieve data upon demand
This commit was SVN r32570.
2014-08-21 18:56:47 +00:00
|
|
|
|
|
|
|
|
/* get our node rank from PMI */
|
2015-06-18 09:53:20 -07:00
|
|
|
OPAL_MODEX_RECV_VALUE(ret, OPAL_PMIX_NODE_RANK,
|
|
|
|
|
ORTE_PROC_MY_NAME, &u16ptr, OPAL_UINT16);
|
|
|
|
|
if (OPAL_SUCCESS != ret) {
|
Per the PMIx RFC:
WHAT: Merge the PMIx branch into the devel repo, creating a new
OPAL “lmix” framework to abstract PMI support for all RTEs.
Replace the ORTE daemon-level collectives with a new PMIx
server and update the ORTE grpcomm framework to support
server-to-server collectives
WHY: We’ve had problems dealing with variations in PMI implementations,
and need to extend the existing PMI definitions to meet exascale
requirements.
WHEN: Mon, Aug 25
WHERE: https://github.com/rhc54/ompi-svn-mirror.git
Several community members have been working on a refactoring of the current PMI support within OMPI. Although the APIs are common, Slurm and Cray implement a different range of capabilities, and package them differently. For example, Cray provides an integrated PMI-1/2 library, while Slurm separates the two and requires the user to specify the one to be used at runtime. In addition, several bugs in the Slurm implementations have caused problems requiring extra coding.
All this has led to a slew of #if’s in the PMI code and bugs when the corner-case logic for one implementation accidentally traps the other. Extending this support to other implementations would have increased this complexity to an unacceptable level.
Accordingly, we have:
* created a new OPAL “pmix” framework to abstract the PMI support, with separate components for Cray, Slurm PMI-1, and Slurm PMI-2 implementations.
* Replaced the current ORTE grpcomm daemon-based collective operation with an integrated PMIx server, and updated the grpcomm APIs to provide more flexible, multi-algorithm support for collective operations. At this time, only the xcast and allgather operations are supported.
* Replaced the current global collective id with a signature based on the names of the participating procs. The allows an unlimited number of collectives to be executed by any group of processes, subject to the requirement that only one collective can be active at a time for a unique combination of procs. Note that a proc can be involved in any number of simultaneous collectives - it is the specific combination of procs that is subject to the constraint
* removed the prior OMPI/OPAL modex code
* added new macros for executing modex send/recv to simplify use of the new APIs. The send macros allow the caller to specify whether or not the BTL supports async modex operations - if so, then the non-blocking “fence” operation is used, if the active PMIx component supports it. Otherwise, the default is a full blocking modex exchange as we currently perform.
* retained the current flag that directs us to use a blocking fence operation, but only to retrieve data upon demand
This commit was SVN r32570.
2014-08-21 18:56:47 +00:00
|
|
|
error = "getting node rank";
|
|
|
|
|
goto error;
|
2014-05-14 15:23:30 +00:00
|
|
|
}
|
2015-06-18 09:53:20 -07:00
|
|
|
orte_process_info.my_node_rank = u16;
|
Per the PMIx RFC:
WHAT: Merge the PMIx branch into the devel repo, creating a new
OPAL “lmix” framework to abstract PMI support for all RTEs.
Replace the ORTE daemon-level collectives with a new PMIx
server and update the ORTE grpcomm framework to support
server-to-server collectives
WHY: We’ve had problems dealing with variations in PMI implementations,
and need to extend the existing PMI definitions to meet exascale
requirements.
WHEN: Mon, Aug 25
WHERE: https://github.com/rhc54/ompi-svn-mirror.git
Several community members have been working on a refactoring of the current PMI support within OMPI. Although the APIs are common, Slurm and Cray implement a different range of capabilities, and package them differently. For example, Cray provides an integrated PMI-1/2 library, while Slurm separates the two and requires the user to specify the one to be used at runtime. In addition, several bugs in the Slurm implementations have caused problems requiring extra coding.
All this has led to a slew of #if’s in the PMI code and bugs when the corner-case logic for one implementation accidentally traps the other. Extending this support to other implementations would have increased this complexity to an unacceptable level.
Accordingly, we have:
* created a new OPAL “pmix” framework to abstract the PMI support, with separate components for Cray, Slurm PMI-1, and Slurm PMI-2 implementations.
* Replaced the current ORTE grpcomm daemon-based collective operation with an integrated PMIx server, and updated the grpcomm APIs to provide more flexible, multi-algorithm support for collective operations. At this time, only the xcast and allgather operations are supported.
* Replaced the current global collective id with a signature based on the names of the participating procs. The allows an unlimited number of collectives to be executed by any group of processes, subject to the requirement that only one collective can be active at a time for a unique combination of procs. Note that a proc can be involved in any number of simultaneous collectives - it is the specific combination of procs that is subject to the constraint
* removed the prior OMPI/OPAL modex code
* added new macros for executing modex send/recv to simplify use of the new APIs. The send macros allow the caller to specify whether or not the BTL supports async modex operations - if so, then the non-blocking “fence” operation is used, if the active PMIx component supports it. Otherwise, the default is a full blocking modex exchange as we currently perform.
* retained the current flag that directs us to use a blocking fence operation, but only to retrieve data upon demand
This commit was SVN r32570.
2014-08-21 18:56:47 +00:00
|
|
|
|
2016-07-19 20:10:58 -07:00
|
|
|
/* get max procs for this application */
|
2016-04-28 20:21:01 -07:00
|
|
|
OPAL_MODEX_RECV_VALUE(ret, OPAL_PMIX_MAX_PROCS,
|
2016-08-06 15:43:00 +06:00
|
|
|
&wildcard_rank, &u32ptr, OPAL_UINT32);
|
2015-06-18 09:53:20 -07:00
|
|
|
if (OPAL_SUCCESS != ret) {
|
2016-04-28 20:21:01 -07:00
|
|
|
error = "getting max procs";
|
Per the PMIx RFC:
WHAT: Merge the PMIx branch into the devel repo, creating a new
OPAL “lmix” framework to abstract PMI support for all RTEs.
Replace the ORTE daemon-level collectives with a new PMIx
server and update the ORTE grpcomm framework to support
server-to-server collectives
WHY: We’ve had problems dealing with variations in PMI implementations,
and need to extend the existing PMI definitions to meet exascale
requirements.
WHEN: Mon, Aug 25
WHERE: https://github.com/rhc54/ompi-svn-mirror.git
Several community members have been working on a refactoring of the current PMI support within OMPI. Although the APIs are common, Slurm and Cray implement a different range of capabilities, and package them differently. For example, Cray provides an integrated PMI-1/2 library, while Slurm separates the two and requires the user to specify the one to be used at runtime. In addition, several bugs in the Slurm implementations have caused problems requiring extra coding.
All this has led to a slew of #if’s in the PMI code and bugs when the corner-case logic for one implementation accidentally traps the other. Extending this support to other implementations would have increased this complexity to an unacceptable level.
Accordingly, we have:
* created a new OPAL “pmix” framework to abstract the PMI support, with separate components for Cray, Slurm PMI-1, and Slurm PMI-2 implementations.
* Replaced the current ORTE grpcomm daemon-based collective operation with an integrated PMIx server, and updated the grpcomm APIs to provide more flexible, multi-algorithm support for collective operations. At this time, only the xcast and allgather operations are supported.
* Replaced the current global collective id with a signature based on the names of the participating procs. The allows an unlimited number of collectives to be executed by any group of processes, subject to the requirement that only one collective can be active at a time for a unique combination of procs. Note that a proc can be involved in any number of simultaneous collectives - it is the specific combination of procs that is subject to the constraint
* removed the prior OMPI/OPAL modex code
* added new macros for executing modex send/recv to simplify use of the new APIs. The send macros allow the caller to specify whether or not the BTL supports async modex operations - if so, then the non-blocking “fence” operation is used, if the active PMIx component supports it. Otherwise, the default is a full blocking modex exchange as we currently perform.
* retained the current flag that directs us to use a blocking fence operation, but only to retrieve data upon demand
This commit was SVN r32570.
2014-08-21 18:56:47 +00:00
|
|
|
goto error;
|
|
|
|
|
}
|
2016-04-27 12:00:19 -07:00
|
|
|
orte_process_info.max_procs = u32;
|
|
|
|
|
|
|
|
|
|
/* get job size */
|
|
|
|
|
OPAL_MODEX_RECV_VALUE(ret, OPAL_PMIX_JOB_SIZE,
|
2016-08-06 15:43:00 +06:00
|
|
|
&wildcard_rank, &u32ptr, OPAL_UINT32);
|
2016-04-27 12:00:19 -07:00
|
|
|
if (OPAL_SUCCESS != ret) {
|
|
|
|
|
error = "getting job size";
|
|
|
|
|
goto error;
|
|
|
|
|
}
|
2015-06-18 09:53:20 -07:00
|
|
|
orte_process_info.num_procs = u32;
|
2016-04-27 12:00:19 -07:00
|
|
|
|
2014-05-14 15:23:30 +00:00
|
|
|
/* push into the environ for pickup in MPI layer for
|
|
|
|
|
* MPI-3 required info key
|
|
|
|
|
*/
|
2014-12-01 21:19:11 -08:00
|
|
|
if (NULL == getenv(OPAL_MCA_PREFIX"orte_ess_num_procs")) {
|
|
|
|
|
asprintf(&ev1, OPAL_MCA_PREFIX"orte_ess_num_procs=%d", orte_process_info.num_procs);
|
|
|
|
|
putenv(ev1);
|
|
|
|
|
added_num_procs = true;
|
|
|
|
|
}
|
|
|
|
|
if (NULL == getenv("OMPI_APP_CTX_NUM_PROCS")) {
|
|
|
|
|
asprintf(&ev2, "OMPI_APP_CTX_NUM_PROCS=%d", orte_process_info.num_procs);
|
|
|
|
|
putenv(ev2);
|
|
|
|
|
added_app_ctx = true;
|
|
|
|
|
}
|
2014-05-14 15:23:30 +00:00
|
|
|
|
Per the PMIx RFC:
WHAT: Merge the PMIx branch into the devel repo, creating a new
OPAL “lmix” framework to abstract PMI support for all RTEs.
Replace the ORTE daemon-level collectives with a new PMIx
server and update the ORTE grpcomm framework to support
server-to-server collectives
WHY: We’ve had problems dealing with variations in PMI implementations,
and need to extend the existing PMI definitions to meet exascale
requirements.
WHEN: Mon, Aug 25
WHERE: https://github.com/rhc54/ompi-svn-mirror.git
Several community members have been working on a refactoring of the current PMI support within OMPI. Although the APIs are common, Slurm and Cray implement a different range of capabilities, and package them differently. For example, Cray provides an integrated PMI-1/2 library, while Slurm separates the two and requires the user to specify the one to be used at runtime. In addition, several bugs in the Slurm implementations have caused problems requiring extra coding.
All this has led to a slew of #if’s in the PMI code and bugs when the corner-case logic for one implementation accidentally traps the other. Extending this support to other implementations would have increased this complexity to an unacceptable level.
Accordingly, we have:
* created a new OPAL “pmix” framework to abstract the PMI support, with separate components for Cray, Slurm PMI-1, and Slurm PMI-2 implementations.
* Replaced the current ORTE grpcomm daemon-based collective operation with an integrated PMIx server, and updated the grpcomm APIs to provide more flexible, multi-algorithm support for collective operations. At this time, only the xcast and allgather operations are supported.
* Replaced the current global collective id with a signature based on the names of the participating procs. The allows an unlimited number of collectives to be executed by any group of processes, subject to the requirement that only one collective can be active at a time for a unique combination of procs. Note that a proc can be involved in any number of simultaneous collectives - it is the specific combination of procs that is subject to the constraint
* removed the prior OMPI/OPAL modex code
* added new macros for executing modex send/recv to simplify use of the new APIs. The send macros allow the caller to specify whether or not the BTL supports async modex operations - if so, then the non-blocking “fence” operation is used, if the active PMIx component supports it. Otherwise, the default is a full blocking modex exchange as we currently perform.
* retained the current flag that directs us to use a blocking fence operation, but only to retrieve data upon demand
This commit was SVN r32570.
2014-08-21 18:56:47 +00:00
|
|
|
|
|
|
|
|
/* get our app number from PMI - ok if not found */
|
2015-09-11 06:38:47 -07:00
|
|
|
OPAL_MODEX_RECV_VALUE_OPTIONAL(ret, OPAL_PMIX_APPNUM,
|
|
|
|
|
ORTE_PROC_MY_NAME, &u32ptr, OPAL_UINT32);
|
2015-06-18 09:53:20 -07:00
|
|
|
if (OPAL_SUCCESS == ret) {
|
|
|
|
|
orte_process_info.app_num = u32;
|
Per the PMIx RFC:
WHAT: Merge the PMIx branch into the devel repo, creating a new
OPAL “lmix” framework to abstract PMI support for all RTEs.
Replace the ORTE daemon-level collectives with a new PMIx
server and update the ORTE grpcomm framework to support
server-to-server collectives
WHY: We’ve had problems dealing with variations in PMI implementations,
and need to extend the existing PMI definitions to meet exascale
requirements.
WHEN: Mon, Aug 25
WHERE: https://github.com/rhc54/ompi-svn-mirror.git
Several community members have been working on a refactoring of the current PMI support within OMPI. Although the APIs are common, Slurm and Cray implement a different range of capabilities, and package them differently. For example, Cray provides an integrated PMI-1/2 library, while Slurm separates the two and requires the user to specify the one to be used at runtime. In addition, several bugs in the Slurm implementations have caused problems requiring extra coding.
All this has led to a slew of #if’s in the PMI code and bugs when the corner-case logic for one implementation accidentally traps the other. Extending this support to other implementations would have increased this complexity to an unacceptable level.
Accordingly, we have:
* created a new OPAL “pmix” framework to abstract the PMI support, with separate components for Cray, Slurm PMI-1, and Slurm PMI-2 implementations.
* Replaced the current ORTE grpcomm daemon-based collective operation with an integrated PMIx server, and updated the grpcomm APIs to provide more flexible, multi-algorithm support for collective operations. At this time, only the xcast and allgather operations are supported.
* Replaced the current global collective id with a signature based on the names of the participating procs. The allows an unlimited number of collectives to be executed by any group of processes, subject to the requirement that only one collective can be active at a time for a unique combination of procs. Note that a proc can be involved in any number of simultaneous collectives - it is the specific combination of procs that is subject to the constraint
* removed the prior OMPI/OPAL modex code
* added new macros for executing modex send/recv to simplify use of the new APIs. The send macros allow the caller to specify whether or not the BTL supports async modex operations - if so, then the non-blocking “fence” operation is used, if the active PMIx component supports it. Otherwise, the default is a full blocking modex exchange as we currently perform.
* retained the current flag that directs us to use a blocking fence operation, but only to retrieve data upon demand
This commit was SVN r32570.
2014-08-21 18:56:47 +00:00
|
|
|
} else {
|
|
|
|
|
orte_process_info.app_num = 0;
|
|
|
|
|
}
|
|
|
|
|
|
2014-08-22 05:17:51 +00:00
|
|
|
/* get the number of local peers - required for wireup of
|
|
|
|
|
* shared memory BTL */
|
2015-06-18 09:53:20 -07:00
|
|
|
OPAL_MODEX_RECV_VALUE(ret, OPAL_PMIX_LOCAL_SIZE,
|
2016-08-06 15:43:00 +06:00
|
|
|
&wildcard_rank, &u32ptr, OPAL_UINT32);
|
2015-06-18 09:53:20 -07:00
|
|
|
if (OPAL_SUCCESS == ret) {
|
|
|
|
|
orte_process_info.num_local_peers = u32 - 1; // want number besides ourselves
|
2014-08-22 05:17:51 +00:00
|
|
|
} else {
|
|
|
|
|
orte_process_info.num_local_peers = 0;
|
|
|
|
|
}
|
|
|
|
|
|
2014-05-14 15:23:30 +00:00
|
|
|
/* setup transport keys in case the MPI layer needs them -
|
|
|
|
|
* we can use the jobfam and stepid as unique keys
|
|
|
|
|
* because they are unique values assigned by the RM
|
|
|
|
|
*/
|
2014-12-01 21:19:11 -08:00
|
|
|
if (NULL == getenv(OPAL_MCA_PREFIX"orte_precondition_transports")) {
|
|
|
|
|
unique_key[0] = ORTE_JOB_FAMILY(ORTE_PROC_MY_NAME->jobid);
|
|
|
|
|
unique_key[1] = ORTE_LOCAL_JOBID(ORTE_PROC_MY_NAME->jobid);
|
|
|
|
|
if (NULL == (string_key = orte_pre_condition_transports_print(unique_key))) {
|
|
|
|
|
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
|
|
|
|
|
return ORTE_ERR_OUT_OF_RESOURCE;
|
|
|
|
|
}
|
2016-04-28 17:02:47 -07:00
|
|
|
opal_output_verbose(2, orte_ess_base_framework.framework_output,
|
|
|
|
|
"%s transport key %s",
|
|
|
|
|
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), string_key);
|
2014-12-01 21:19:11 -08:00
|
|
|
asprintf(&envar, OPAL_MCA_PREFIX"orte_precondition_transports=%s", string_key);
|
|
|
|
|
putenv(envar);
|
|
|
|
|
added_transport_keys = true;
|
|
|
|
|
/* cannot free the envar as that messes up our environ */
|
|
|
|
|
free(string_key);
|
2014-05-14 15:23:30 +00:00
|
|
|
}
|
|
|
|
|
|
2015-06-18 09:53:20 -07:00
|
|
|
/* retrieve our topology */
|
2015-09-23 15:21:04 -07:00
|
|
|
val = NULL;
|
2015-09-11 06:38:47 -07:00
|
|
|
OPAL_MODEX_RECV_VALUE_OPTIONAL(ret, OPAL_PMIX_LOCAL_TOPO,
|
2016-08-06 15:43:00 +06:00
|
|
|
&wildcard_rank, &val, OPAL_STRING);
|
2015-06-18 09:53:20 -07:00
|
|
|
if (OPAL_SUCCESS == ret && NULL != val) {
|
|
|
|
|
/* load the topology */
|
|
|
|
|
if (0 != hwloc_topology_init(&opal_hwloc_topology)) {
|
|
|
|
|
ret = OPAL_ERROR;
|
|
|
|
|
free(val);
|
|
|
|
|
error = "setting topology";
|
|
|
|
|
goto error;
|
|
|
|
|
}
|
|
|
|
|
if (0 != hwloc_topology_set_xmlbuffer(opal_hwloc_topology, val, strlen(val))) {
|
|
|
|
|
ret = OPAL_ERROR;
|
|
|
|
|
free(val);
|
|
|
|
|
hwloc_topology_destroy(opal_hwloc_topology);
|
|
|
|
|
error = "setting topology";
|
|
|
|
|
goto error;
|
|
|
|
|
}
|
|
|
|
|
/* since we are loading this from an external source, we have to
|
|
|
|
|
* explicitly set a flag so hwloc sets things up correctly
|
|
|
|
|
*/
|
|
|
|
|
if (0 != hwloc_topology_set_flags(opal_hwloc_topology,
|
|
|
|
|
(HWLOC_TOPOLOGY_FLAG_IS_THISSYSTEM |
|
|
|
|
|
HWLOC_TOPOLOGY_FLAG_WHOLE_SYSTEM |
|
|
|
|
|
HWLOC_TOPOLOGY_FLAG_IO_DEVICES))) {
|
|
|
|
|
ret = OPAL_ERROR;
|
|
|
|
|
hwloc_topology_destroy(opal_hwloc_topology);
|
|
|
|
|
free(val);
|
|
|
|
|
error = "setting topology";
|
|
|
|
|
goto error;
|
|
|
|
|
}
|
|
|
|
|
/* now load the topology */
|
|
|
|
|
if (0 != hwloc_topology_load(opal_hwloc_topology)) {
|
|
|
|
|
ret = OPAL_ERROR;
|
|
|
|
|
hwloc_topology_destroy(opal_hwloc_topology);
|
|
|
|
|
free(val);
|
|
|
|
|
error = "setting topology";
|
|
|
|
|
goto error;
|
|
|
|
|
}
|
|
|
|
|
free(val);
|
2015-08-30 12:00:22 -07:00
|
|
|
/* filter the cpus thru any default cpu set */
|
|
|
|
|
if (OPAL_SUCCESS != (ret = opal_hwloc_base_filter_cpus(opal_hwloc_topology))) {
|
|
|
|
|
error = "filtering topology";
|
|
|
|
|
goto error;
|
|
|
|
|
}
|
2015-06-18 09:53:20 -07:00
|
|
|
} else {
|
|
|
|
|
/* it wasn't passed down to us, so go get it */
|
2015-03-16 17:32:04 -07:00
|
|
|
if (OPAL_SUCCESS != (ret = opal_hwloc_base_get_topology())) {
|
|
|
|
|
error = "topology discovery";
|
|
|
|
|
goto error;
|
|
|
|
|
}
|
2015-06-18 09:53:20 -07:00
|
|
|
/* push it into the PMIx database in case someone
|
|
|
|
|
* tries to retrieve it so we avoid an attempt to
|
|
|
|
|
* get it again */
|
|
|
|
|
kv = OBJ_NEW(opal_value_t);
|
|
|
|
|
kv->key = strdup(OPAL_PMIX_LOCAL_TOPO);
|
|
|
|
|
kv->type = OPAL_STRING;
|
|
|
|
|
if (0 != (ret = hwloc_topology_export_xmlbuffer(opal_hwloc_topology, &kv->data.string, &u32))) {
|
|
|
|
|
error = "topology export";
|
|
|
|
|
goto error;
|
|
|
|
|
}
|
2016-08-06 15:43:00 +06:00
|
|
|
if (OPAL_SUCCESS != (ret = opal_pmix.store_local(&wildcard_rank, kv))) {
|
2015-06-18 09:53:20 -07:00
|
|
|
error = "topology store";
|
|
|
|
|
goto error;
|
|
|
|
|
}
|
|
|
|
|
OBJ_RELEASE(kv);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* get our local peers */
|
|
|
|
|
if (0 < orte_process_info.num_local_peers) {
|
2015-09-11 06:38:47 -07:00
|
|
|
/* if my local rank if too high, then that's an error */
|
|
|
|
|
if (orte_process_info.num_local_peers < orte_process_info.my_local_rank) {
|
|
|
|
|
ret = ORTE_ERR_BAD_PARAM;
|
|
|
|
|
error = "num local peers";
|
|
|
|
|
goto error;
|
|
|
|
|
}
|
2015-06-18 09:53:20 -07:00
|
|
|
/* retrieve the local peers */
|
2015-08-30 12:00:22 -07:00
|
|
|
OPAL_MODEX_RECV_VALUE(ret, OPAL_PMIX_LOCAL_PEERS,
|
2016-08-13 08:14:50 -07:00
|
|
|
ORTE_PROC_MY_NAME, &val, OPAL_STRING);
|
2015-06-18 09:53:20 -07:00
|
|
|
if (OPAL_SUCCESS == ret && NULL != val) {
|
|
|
|
|
peers = opal_argv_split(val, ',');
|
|
|
|
|
free(val);
|
2015-09-11 06:38:47 -07:00
|
|
|
/* and their cpusets, if available */
|
2016-08-13 08:14:50 -07:00
|
|
|
OPAL_MODEX_RECV_VALUE_OPTIONAL(ret, OPAL_PMIX_LOCAL_CPUSETS, ORTE_PROC_MY_NAME, &val, OPAL_STRING);
|
2015-09-11 06:38:47 -07:00
|
|
|
if (OPAL_SUCCESS == ret && NULL != val) {
|
|
|
|
|
cpusets = opal_argv_split(val, ':');
|
|
|
|
|
free(val);
|
|
|
|
|
} else {
|
|
|
|
|
cpusets = NULL;
|
|
|
|
|
}
|
2015-06-18 09:53:20 -07:00
|
|
|
} else {
|
|
|
|
|
peers = NULL;
|
2015-09-11 06:38:47 -07:00
|
|
|
cpusets = NULL;
|
2015-06-18 09:53:20 -07:00
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
peers = NULL;
|
2015-09-11 06:38:47 -07:00
|
|
|
cpusets = NULL;
|
2015-06-18 09:53:20 -07:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* set the locality */
|
2015-09-11 06:38:47 -07:00
|
|
|
if (NULL != peers) {
|
|
|
|
|
/* indentify our cpuset */
|
|
|
|
|
if (NULL != cpusets) {
|
|
|
|
|
mycpuset = cpusets[orte_process_info.my_local_rank];
|
2015-06-18 09:53:20 -07:00
|
|
|
} else {
|
2015-09-11 06:38:47 -07:00
|
|
|
mycpuset = NULL;
|
|
|
|
|
}
|
2016-08-06 15:43:00 +06:00
|
|
|
pname.jobid = ORTE_PROC_MY_NAME->jobid;
|
2015-09-11 06:38:47 -07:00
|
|
|
for (i=0; NULL != peers[i]; i++) {
|
|
|
|
|
kv = OBJ_NEW(opal_value_t);
|
|
|
|
|
kv->key = strdup(OPAL_PMIX_LOCALITY);
|
|
|
|
|
kv->type = OPAL_UINT16;
|
2016-08-06 15:43:00 +06:00
|
|
|
pname.vpid = strtoul(peers[i], NULL, 10);
|
|
|
|
|
if (pname.vpid == ORTE_PROC_MY_NAME->vpid) {
|
2015-09-11 06:38:47 -07:00
|
|
|
/* we are fully local to ourselves */
|
|
|
|
|
u16 = OPAL_PROC_ALL_LOCAL;
|
|
|
|
|
} else if (NULL == mycpuset || NULL == cpusets[i] ||
|
|
|
|
|
0 == strcmp(cpusets[i], "UNBOUND")) {
|
2015-08-30 12:00:22 -07:00
|
|
|
/* all we can say is that it shares our node */
|
2015-06-18 09:53:20 -07:00
|
|
|
u16 = OPAL_PROC_ON_CLUSTER | OPAL_PROC_ON_CU | OPAL_PROC_ON_NODE;
|
|
|
|
|
} else {
|
2015-09-11 06:38:47 -07:00
|
|
|
/* we have it, so compute the locality */
|
|
|
|
|
u16 = opal_hwloc_base_get_relative_locality(opal_hwloc_topology, mycpuset, cpusets[i]);
|
2015-06-18 09:53:20 -07:00
|
|
|
}
|
2015-09-11 06:38:47 -07:00
|
|
|
OPAL_OUTPUT_VERBOSE((1, orte_ess_base_framework.framework_output,
|
|
|
|
|
"%s ess:pmi:locality: proc %s locality %x",
|
|
|
|
|
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
|
2016-08-06 15:43:00 +06:00
|
|
|
ORTE_NAME_PRINT(&pname), u16));
|
2015-09-11 06:38:47 -07:00
|
|
|
kv->data.uint16 = u16;
|
2016-08-06 15:43:00 +06:00
|
|
|
ret = opal_pmix.store_local(&pname, kv);
|
2015-09-11 06:38:47 -07:00
|
|
|
if (OPAL_SUCCESS != ret) {
|
|
|
|
|
error = "local store of locality";
|
|
|
|
|
opal_argv_free(peers);
|
|
|
|
|
opal_argv_free(cpusets);
|
|
|
|
|
goto error;
|
2015-08-30 12:00:22 -07:00
|
|
|
}
|
2015-09-11 06:38:47 -07:00
|
|
|
OBJ_RELEASE(kv);
|
2015-06-18 09:53:20 -07:00
|
|
|
}
|
2015-09-11 06:38:47 -07:00
|
|
|
opal_argv_free(peers);
|
|
|
|
|
opal_argv_free(cpusets);
|
2015-08-30 12:00:22 -07:00
|
|
|
}
|
2015-03-16 17:32:04 -07:00
|
|
|
|
2014-12-01 21:19:11 -08:00
|
|
|
/* now that we have all required info, complete the setup */
|
2014-05-14 15:23:30 +00:00
|
|
|
if (ORTE_SUCCESS != (ret = orte_ess_base_app_setup(false))) {
|
|
|
|
|
ORTE_ERROR_LOG(ret);
|
|
|
|
|
error = "orte_ess_base_app_setup";
|
|
|
|
|
goto error;
|
|
|
|
|
}
|
2012-01-18 20:56:53 +00:00
|
|
|
|
2014-05-14 15:23:30 +00:00
|
|
|
/* setup process binding */
|
|
|
|
|
if (ORTE_SUCCESS != (ret = orte_ess_base_proc_binding())) {
|
|
|
|
|
error = "proc_binding";
|
|
|
|
|
goto error;
|
2011-10-17 20:51:22 +00:00
|
|
|
}
|
2012-01-18 20:56:53 +00:00
|
|
|
|
2014-05-14 15:23:30 +00:00
|
|
|
/* this needs to be set to enable debugger use when direct launched */
|
2014-08-22 03:00:56 +00:00
|
|
|
if (NULL == orte_process_info.my_daemon_uri) {
|
|
|
|
|
orte_standalone_operation = true;
|
|
|
|
|
}
|
2014-05-14 15:23:30 +00:00
|
|
|
|
2012-01-18 20:56:53 +00:00
|
|
|
/* set max procs */
|
|
|
|
|
if (orte_process_info.max_procs < orte_process_info.num_procs) {
|
|
|
|
|
orte_process_info.max_procs = orte_process_info.num_procs;
|
|
|
|
|
}
|
|
|
|
|
|
Per the PMIx RFC:
WHAT: Merge the PMIx branch into the devel repo, creating a new
OPAL “lmix” framework to abstract PMI support for all RTEs.
Replace the ORTE daemon-level collectives with a new PMIx
server and update the ORTE grpcomm framework to support
server-to-server collectives
WHY: We’ve had problems dealing with variations in PMI implementations,
and need to extend the existing PMI definitions to meet exascale
requirements.
WHEN: Mon, Aug 25
WHERE: https://github.com/rhc54/ompi-svn-mirror.git
Several community members have been working on a refactoring of the current PMI support within OMPI. Although the APIs are common, Slurm and Cray implement a different range of capabilities, and package them differently. For example, Cray provides an integrated PMI-1/2 library, while Slurm separates the two and requires the user to specify the one to be used at runtime. In addition, several bugs in the Slurm implementations have caused problems requiring extra coding.
All this has led to a slew of #if’s in the PMI code and bugs when the corner-case logic for one implementation accidentally traps the other. Extending this support to other implementations would have increased this complexity to an unacceptable level.
Accordingly, we have:
* created a new OPAL “pmix” framework to abstract the PMI support, with separate components for Cray, Slurm PMI-1, and Slurm PMI-2 implementations.
* Replaced the current ORTE grpcomm daemon-based collective operation with an integrated PMIx server, and updated the grpcomm APIs to provide more flexible, multi-algorithm support for collective operations. At this time, only the xcast and allgather operations are supported.
* Replaced the current global collective id with a signature based on the names of the participating procs. The allows an unlimited number of collectives to be executed by any group of processes, subject to the requirement that only one collective can be active at a time for a unique combination of procs. Note that a proc can be involved in any number of simultaneous collectives - it is the specific combination of procs that is subject to the constraint
* removed the prior OMPI/OPAL modex code
* added new macros for executing modex send/recv to simplify use of the new APIs. The send macros allow the caller to specify whether or not the BTL supports async modex operations - if so, then the non-blocking “fence” operation is used, if the active PMIx component supports it. Otherwise, the default is a full blocking modex exchange as we currently perform.
* retained the current flag that directs us to use a blocking fence operation, but only to retrieve data upon demand
This commit was SVN r32570.
2014-08-21 18:56:47 +00:00
|
|
|
/*** PUSH DATA FOR OTHERS TO FIND ***/
|
|
|
|
|
|
2016-03-18 18:50:24 -07:00
|
|
|
/* push our RML URI in case others need to talk directly to us */
|
|
|
|
|
rmluri = orte_rml.get_contact_info();
|
|
|
|
|
/* push it out for others to use */
|
|
|
|
|
OPAL_MODEX_SEND_VALUE(ret, OPAL_PMIX_GLOBAL, OPAL_PMIX_PROC_URI, rmluri, OPAL_STRING);
|
|
|
|
|
if (ORTE_SUCCESS != ret) {
|
|
|
|
|
error = "pmix put uri";
|
|
|
|
|
goto error;
|
|
|
|
|
}
|
|
|
|
|
free(rmluri);
|
|
|
|
|
|
Per the PMIx RFC:
WHAT: Merge the PMIx branch into the devel repo, creating a new
OPAL “lmix” framework to abstract PMI support for all RTEs.
Replace the ORTE daemon-level collectives with a new PMIx
server and update the ORTE grpcomm framework to support
server-to-server collectives
WHY: We’ve had problems dealing with variations in PMI implementations,
and need to extend the existing PMI definitions to meet exascale
requirements.
WHEN: Mon, Aug 25
WHERE: https://github.com/rhc54/ompi-svn-mirror.git
Several community members have been working on a refactoring of the current PMI support within OMPI. Although the APIs are common, Slurm and Cray implement a different range of capabilities, and package them differently. For example, Cray provides an integrated PMI-1/2 library, while Slurm separates the two and requires the user to specify the one to be used at runtime. In addition, several bugs in the Slurm implementations have caused problems requiring extra coding.
All this has led to a slew of #if’s in the PMI code and bugs when the corner-case logic for one implementation accidentally traps the other. Extending this support to other implementations would have increased this complexity to an unacceptable level.
Accordingly, we have:
* created a new OPAL “pmix” framework to abstract the PMI support, with separate components for Cray, Slurm PMI-1, and Slurm PMI-2 implementations.
* Replaced the current ORTE grpcomm daemon-based collective operation with an integrated PMIx server, and updated the grpcomm APIs to provide more flexible, multi-algorithm support for collective operations. At this time, only the xcast and allgather operations are supported.
* Replaced the current global collective id with a signature based on the names of the participating procs. The allows an unlimited number of collectives to be executed by any group of processes, subject to the requirement that only one collective can be active at a time for a unique combination of procs. Note that a proc can be involved in any number of simultaneous collectives - it is the specific combination of procs that is subject to the constraint
* removed the prior OMPI/OPAL modex code
* added new macros for executing modex send/recv to simplify use of the new APIs. The send macros allow the caller to specify whether or not the BTL supports async modex operations - if so, then the non-blocking “fence” operation is used, if the active PMIx component supports it. Otherwise, the default is a full blocking modex exchange as we currently perform.
* retained the current flag that directs us to use a blocking fence operation, but only to retrieve data upon demand
This commit was SVN r32570.
2014-08-21 18:56:47 +00:00
|
|
|
/* push our hostname so others can find us, if they need to */
|
2015-06-18 09:53:20 -07:00
|
|
|
OPAL_MODEX_SEND_VALUE(ret, OPAL_PMIX_GLOBAL, OPAL_PMIX_HOSTNAME, orte_process_info.nodename, OPAL_STRING);
|
|
|
|
|
if (ORTE_SUCCESS != ret) {
|
Per the meeting on moving the BTLs to OPAL, move the ORTE database "db" framework to OPAL so the relocated BTLs can access it. Because the data is indexed by process, this requires that we define a new "opal_identifier_t" that corresponds to the orte_process_name_t struct. In order to support multiple run-times, this is defined in opal/mca/db/db_types.h as a uint64_t without identifying the meaning of any part of that data.
A few changes were required to support this move:
1. the PMI component used to identify rte-related data (e.g., host name, bind level) and package them as a unit to reduce the number of PMI keys. This code was moved up to the ORTE layer as the OPAL layer has no understanding of these concepts. In addition, the component locally stored data based on process jobid/vpid - this could no longer be supported (see below for the solution).
2. the hash component was updated to use the new opal_identifier_t instead of orte_process_name_t as its index for storing data in the hash tables. Previously, we did a hash on the vpid and stored the data in a 32-bit hash table. In the revised system, we don't see a separate "vpid" field - we only have a 64-bit opaque value. The orte_process_name_t hash turned out to do nothing useful, so we now store the data in a 64-bit hash table. Preliminary tests didn't show any identifiable change in behavior or performance, but we'll have to see if a move back to the 32-bit table is required at some later time.
3. the db framework was a "select one" system. However, since the PMI component could no longer use its internal storage system, the framework has now been changed to a "select many" mode of operation. This allows the hash component to handle all internal storage, while the PMI component only handles pushing/pulling things from the PMI system. This was something we had planned for some time - when fetching data, we first check internal storage to see if we already have it, and then automatically go to the global system to look for it if we don't. Accordingly, the framework was provided with a custom query function used during "select" that lets you seperately specify the "store" and "fetch" ordering.
4. the ORTE grpcomm and ess/pmi components, and the nidmap code, were updated to work with the new db framework and to specify internal/global storage options.
No changes were made to the MPI layer, except for modifying the ORTE component of the OMPI/rte framework to support the new db framework.
This commit was SVN r28112.
2013-02-26 17:50:04 +00:00
|
|
|
error = "db store hostname";
|
|
|
|
|
goto error;
|
|
|
|
|
}
|
When we direct launch an application, we rely on PMI for wireup support. In doing so, we lose the de facto data compression we get from the ORTE modex since we no longer get all the wireup info from every proc in a single blob. Instead, we have to iterate over all the procs, calling PMI_KVS_get for every value we require.
This creates a really bad scaling behavior. Users have found a nearly 20% launch time differential between mpirun and PMI, with PMI being the slower method. Some of the problem is attributable to poor exchange algorithms in RM's like Slurm and Alps, but we make things worse by calling "get" so many times.
Nathan (with a tad advice from me) has attempted to alleviate this problem by reducing the number of "get" calls. This required the following changes:
* upon first request for data, have the OPAL db pmi component fetch and decode *all* the info from a given remote proc. It turned out we weren't caching the info, so we would continually request it and only decode the piece we needed for the immediate request. We now decode all the info and push it into the db hash component for local storage - and then all subsequent retrievals are fulfilled locally
* reduced the amount of data by eliminating the exchange of the OMPI_ARCH value if heterogeneity is not enabled. This was used solely as a check so we would error out if the system wasn't actually homogeneous, which was fine when we thought there was no cost in doing the check. Unfortunately, at large scale and with direct launch, there is a non-zero cost of making this test. We are open to finding a compromise (perhaps turning the test off if requested?), if people feel strongly about performing the test
* reduced the amount of RTE data being automatically fetched, and fetched the rest only upon request. In particular, we no longer immediately fetch the hostname (which is only used for error reporting), but instead get it when needed. Likewise for the RML uri as that info is only required for some (not all) environments. In addition, we no longer fetch the locality unless required, relying instead on the PMI clique info to tell us who is on our local node (if additional info is required, the fetch is performed when a modex_recv is issued).
Again, all this only impacts direct launch - all the info is provided when launched via mpirun as there is no added cost to getting it
Barring objections, we may move this (plus any required other pieces) to the 1.7 branch once it soaks for an appropriate time.
This commit was SVN r29040.
2013-08-17 00:49:18 +00:00
|
|
|
|
2013-10-08 18:37:59 +00:00
|
|
|
/* if we are an ORTE app - and not an MPI app - then
|
|
|
|
|
* we need to exchange our connection info here.
|
|
|
|
|
* MPI_Init has its own modex, so we don't need to do
|
|
|
|
|
* two of them. However, if we don't do a modex at all,
|
|
|
|
|
* then processes have no way to communicate
|
|
|
|
|
*
|
|
|
|
|
* NOTE: only do this when the process originally launches.
|
|
|
|
|
* Cannot do this on a restart as the rest of the processes
|
|
|
|
|
* in the job won't be executing this step, so we would hang
|
|
|
|
|
*/
|
|
|
|
|
if (ORTE_PROC_IS_NON_MPI && !orte_do_not_barrier) {
|
Per the PMIx RFC:
WHAT: Merge the PMIx branch into the devel repo, creating a new
OPAL “lmix” framework to abstract PMI support for all RTEs.
Replace the ORTE daemon-level collectives with a new PMIx
server and update the ORTE grpcomm framework to support
server-to-server collectives
WHY: We’ve had problems dealing with variations in PMI implementations,
and need to extend the existing PMI definitions to meet exascale
requirements.
WHEN: Mon, Aug 25
WHERE: https://github.com/rhc54/ompi-svn-mirror.git
Several community members have been working on a refactoring of the current PMI support within OMPI. Although the APIs are common, Slurm and Cray implement a different range of capabilities, and package them differently. For example, Cray provides an integrated PMI-1/2 library, while Slurm separates the two and requires the user to specify the one to be used at runtime. In addition, several bugs in the Slurm implementations have caused problems requiring extra coding.
All this has led to a slew of #if’s in the PMI code and bugs when the corner-case logic for one implementation accidentally traps the other. Extending this support to other implementations would have increased this complexity to an unacceptable level.
Accordingly, we have:
* created a new OPAL “pmix” framework to abstract the PMI support, with separate components for Cray, Slurm PMI-1, and Slurm PMI-2 implementations.
* Replaced the current ORTE grpcomm daemon-based collective operation with an integrated PMIx server, and updated the grpcomm APIs to provide more flexible, multi-algorithm support for collective operations. At this time, only the xcast and allgather operations are supported.
* Replaced the current global collective id with a signature based on the names of the participating procs. The allows an unlimited number of collectives to be executed by any group of processes, subject to the requirement that only one collective can be active at a time for a unique combination of procs. Note that a proc can be involved in any number of simultaneous collectives - it is the specific combination of procs that is subject to the constraint
* removed the prior OMPI/OPAL modex code
* added new macros for executing modex send/recv to simplify use of the new APIs. The send macros allow the caller to specify whether or not the BTL supports async modex operations - if so, then the non-blocking “fence” operation is used, if the active PMIx component supports it. Otherwise, the default is a full blocking modex exchange as we currently perform.
* retained the current flag that directs us to use a blocking fence operation, but only to retrieve data upon demand
This commit was SVN r32570.
2014-08-21 18:56:47 +00:00
|
|
|
opal_pmix.fence(NULL, 0);
|
2013-10-08 18:37:59 +00:00
|
|
|
}
|
|
|
|
|
|
2011-10-17 20:51:22 +00:00
|
|
|
return ORTE_SUCCESS;
|
|
|
|
|
|
2012-01-18 20:56:53 +00:00
|
|
|
error:
|
2016-03-02 14:30:57 -08:00
|
|
|
if (!progress_thread_running) {
|
|
|
|
|
/* can't send the help message, so ensure it
|
|
|
|
|
* comes out locally
|
|
|
|
|
*/
|
|
|
|
|
orte_show_help_finalize();
|
|
|
|
|
}
|
2011-11-22 21:24:35 +00:00
|
|
|
if (ORTE_ERR_SILENT != ret && !orte_report_silent_errors) {
|
|
|
|
|
orte_show_help("help-orte-runtime.txt",
|
|
|
|
|
"orte_init:startup:internal-failure",
|
|
|
|
|
true, error, ORTE_ERROR_NAME(ret), ret);
|
|
|
|
|
}
|
2011-10-17 20:51:22 +00:00
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static int rte_finalize(void)
|
|
|
|
|
{
|
As per the RFC, bring in the ORTE async progress code and the rewrite of OOB:
*** THIS RFC INCLUDES A MINOR CHANGE TO THE MPI-RTE INTERFACE ***
Note: during the course of this work, it was necessary to completely separate the MPI and RTE progress engines. There were multiple places in the MPI layer where ORTE_WAIT_FOR_COMPLETION was being used. A new OMPI_WAIT_FOR_COMPLETION macro was created (defined in ompi/mca/rte/rte.h) that simply cycles across opal_progress until the provided flag becomes false. Places where the MPI layer blocked waiting for RTE to complete an event have been modified to use this macro.
***************************************************************************************
I am reissuing this RFC because of the time that has passed since its original release. Since its initial release and review, I have debugged it further to ensure it fully supports tests like loop_spawn. It therefore seems ready for merge back to the trunk. Given its prior review, I have set the timeout for one week.
The code is in https://bitbucket.org/rhc/ompi-oob2
WHAT: Rewrite of ORTE OOB
WHY: Support asynchronous progress and a host of other features
WHEN: Wed, August 21
SYNOPSIS:
The current OOB has served us well, but a number of limitations have been identified over the years. Specifically:
* it is only progressed when called via opal_progress, which can lead to hangs or recursive calls into libevent (which is not supported by that code)
* we've had issues when multiple NICs are available as the code doesn't "shift" messages between transports - thus, all nodes had to be available via the same TCP interface.
* the OOB "unloads" incoming opal_buffer_t objects during the transmission, thus preventing use of OBJ_RETAIN in the code when repeatedly sending the same message to multiple recipients
* there is no failover mechanism across NICs - if the selected NIC (or its attached switch) fails, we are forced to abort
* only one transport (i.e., component) can be "active"
The revised OOB resolves these problems:
* async progress is used for all application processes, with the progress thread blocking in the event library
* each available TCP NIC is supported by its own TCP module. The ability to asynchronously progress each module independently is provided, but not enabled by default (a runtime MCA parameter turns it "on")
* multi-address TCP NICs (e.g., a NIC with both an IPv4 and IPv6 address, or with virtual interfaces) are supported - reachability is determined by comparing the contact info for a peer against all addresses within the range covered by the address/mask pairs for the NIC.
* a message that arrives on one TCP NIC is automatically shifted to whatever NIC that is connected to the next "hop" if that peer cannot be reached by the incoming NIC. If no TCP module will reach the peer, then the OOB attempts to send the message via all other available components - if none can reach the peer, then an "error" is reported back to the RML, which then calls the errmgr for instructions.
* opal_buffer_t now conforms to standard object rules re OBJ_RETAIN as we no longer "unload" the incoming object
* NIC failure is reported to the TCP component, which then tries to resend the message across any other available TCP NIC. If that doesn't work, then the message is given back to the OOB base to try using other components. If all that fails, then the error is reported to the RML, which reports to the errmgr for instructions
* obviously from the above, multiple OOB components (e.g., TCP and UD) can be active in parallel
* the matching code has been moved to the RML (and out of the OOB/TCP component) so it is independent of transport
* routing is done by the individual OOB modules (as opposed to the RML). Thus, both routed and non-routed transports can simultaneously be active
* all blocking send/recv APIs have been removed. Everything operates asynchronously.
KNOWN LIMITATIONS:
* although provision is made for component failover as described above, the code for doing so has not been fully implemented yet. At the moment, if all connections for a given peer fail, the errmgr is notified of a "lost connection", which by default results in termination of the job if it was a lifeline
* the IPv6 code is present and compiles, but is not complete. Since the current IPv6 support in the OOB doesn't work anyway, I don't consider this a blocker
* routing is performed at the individual module level, yet the active routed component is selected on a global basis. We probably should update that to reflect that different transports may need/choose to route in different ways
* obviously, not every error path has been tested nor necessarily covered
* determining abnormal termination is more challenging than in the old code as we now potentially have multiple ways of connecting to a process. Ideally, we would declare "connection failed" when *all* transports can no longer reach the process, but that requires some additional (possibly complex) code. For now, the code replicates the old behavior only somewhat modified - i.e., if a module sees its connection fail, it checks to see if it is a lifeline. If so, it notifies the errmgr that the lifeline is lost - otherwise, it notifies the errmgr that a non-lifeline connection was lost.
* reachability is determined solely on the basis of a shared subnet address/mask - more sophisticated algorithms (e.g., the one used in the tcp btl) are required to handle routing via gateways
* the RML needs to assign sequence numbers to each message on a per-peer basis. The receiving RML will then deliver messages in order, thus preventing out-of-order messaging in the case where messages travel across different transports or a message needs to be redirected/resent due to failure of a NIC
This commit was SVN r29058.
2013-08-22 16:37:40 +00:00
|
|
|
int ret;
|
|
|
|
|
|
2014-12-01 21:19:11 -08:00
|
|
|
/* remove the envars that we pushed into environ
|
|
|
|
|
* so we leave that structure intact
|
|
|
|
|
*/
|
|
|
|
|
if (added_transport_keys) {
|
|
|
|
|
unsetenv(OPAL_MCA_PREFIX"orte_precondition_transports");
|
|
|
|
|
}
|
|
|
|
|
if (added_num_procs) {
|
|
|
|
|
unsetenv(OPAL_MCA_PREFIX"orte_ess_num_procs");
|
|
|
|
|
}
|
|
|
|
|
if (added_app_ctx) {
|
|
|
|
|
unsetenv("OMPI_APP_CTX_NUM_PROCS");
|
|
|
|
|
}
|
2015-06-23 20:59:57 -07:00
|
|
|
|
2016-03-02 14:30:57 -08:00
|
|
|
/* use the default app procedure to finish */
|
|
|
|
|
if (ORTE_SUCCESS != (ret = orte_ess_base_app_finalize())) {
|
|
|
|
|
ORTE_ERROR_LOG(ret);
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
2014-12-03 12:19:00 -08:00
|
|
|
/* mark us as finalized */
|
|
|
|
|
if (NULL != opal_pmix.finalize) {
|
|
|
|
|
opal_pmix.finalize();
|
|
|
|
|
(void) mca_base_framework_close(&opal_pmix_base_framework);
|
|
|
|
|
}
|
2015-06-23 20:59:57 -07:00
|
|
|
|
2016-03-02 14:30:57 -08:00
|
|
|
/* release the event base */
|
|
|
|
|
if (progress_thread_running) {
|
|
|
|
|
opal_progress_thread_finalize(NULL);
|
|
|
|
|
progress_thread_running = false;
|
2015-06-18 09:53:20 -07:00
|
|
|
}
|
As per the RFC, bring in the ORTE async progress code and the rewrite of OOB:
*** THIS RFC INCLUDES A MINOR CHANGE TO THE MPI-RTE INTERFACE ***
Note: during the course of this work, it was necessary to completely separate the MPI and RTE progress engines. There were multiple places in the MPI layer where ORTE_WAIT_FOR_COMPLETION was being used. A new OMPI_WAIT_FOR_COMPLETION macro was created (defined in ompi/mca/rte/rte.h) that simply cycles across opal_progress until the provided flag becomes false. Places where the MPI layer blocked waiting for RTE to complete an event have been modified to use this macro.
***************************************************************************************
I am reissuing this RFC because of the time that has passed since its original release. Since its initial release and review, I have debugged it further to ensure it fully supports tests like loop_spawn. It therefore seems ready for merge back to the trunk. Given its prior review, I have set the timeout for one week.
The code is in https://bitbucket.org/rhc/ompi-oob2
WHAT: Rewrite of ORTE OOB
WHY: Support asynchronous progress and a host of other features
WHEN: Wed, August 21
SYNOPSIS:
The current OOB has served us well, but a number of limitations have been identified over the years. Specifically:
* it is only progressed when called via opal_progress, which can lead to hangs or recursive calls into libevent (which is not supported by that code)
* we've had issues when multiple NICs are available as the code doesn't "shift" messages between transports - thus, all nodes had to be available via the same TCP interface.
* the OOB "unloads" incoming opal_buffer_t objects during the transmission, thus preventing use of OBJ_RETAIN in the code when repeatedly sending the same message to multiple recipients
* there is no failover mechanism across NICs - if the selected NIC (or its attached switch) fails, we are forced to abort
* only one transport (i.e., component) can be "active"
The revised OOB resolves these problems:
* async progress is used for all application processes, with the progress thread blocking in the event library
* each available TCP NIC is supported by its own TCP module. The ability to asynchronously progress each module independently is provided, but not enabled by default (a runtime MCA parameter turns it "on")
* multi-address TCP NICs (e.g., a NIC with both an IPv4 and IPv6 address, or with virtual interfaces) are supported - reachability is determined by comparing the contact info for a peer against all addresses within the range covered by the address/mask pairs for the NIC.
* a message that arrives on one TCP NIC is automatically shifted to whatever NIC that is connected to the next "hop" if that peer cannot be reached by the incoming NIC. If no TCP module will reach the peer, then the OOB attempts to send the message via all other available components - if none can reach the peer, then an "error" is reported back to the RML, which then calls the errmgr for instructions.
* opal_buffer_t now conforms to standard object rules re OBJ_RETAIN as we no longer "unload" the incoming object
* NIC failure is reported to the TCP component, which then tries to resend the message across any other available TCP NIC. If that doesn't work, then the message is given back to the OOB base to try using other components. If all that fails, then the error is reported to the RML, which reports to the errmgr for instructions
* obviously from the above, multiple OOB components (e.g., TCP and UD) can be active in parallel
* the matching code has been moved to the RML (and out of the OOB/TCP component) so it is independent of transport
* routing is done by the individual OOB modules (as opposed to the RML). Thus, both routed and non-routed transports can simultaneously be active
* all blocking send/recv APIs have been removed. Everything operates asynchronously.
KNOWN LIMITATIONS:
* although provision is made for component failover as described above, the code for doing so has not been fully implemented yet. At the moment, if all connections for a given peer fail, the errmgr is notified of a "lost connection", which by default results in termination of the job if it was a lifeline
* the IPv6 code is present and compiles, but is not complete. Since the current IPv6 support in the OOB doesn't work anyway, I don't consider this a blocker
* routing is performed at the individual module level, yet the active routed component is selected on a global basis. We probably should update that to reflect that different transports may need/choose to route in different ways
* obviously, not every error path has been tested nor necessarily covered
* determining abnormal termination is more challenging than in the old code as we now potentially have multiple ways of connecting to a process. Ideally, we would declare "connection failed" when *all* transports can no longer reach the process, but that requires some additional (possibly complex) code. For now, the code replicates the old behavior only somewhat modified - i.e., if a module sees its connection fail, it checks to see if it is a lifeline. If so, it notifies the errmgr that the lifeline is lost - otherwise, it notifies the errmgr that a non-lifeline connection was lost.
* reachability is determined solely on the basis of a shared subnet address/mask - more sophisticated algorithms (e.g., the one used in the tcp btl) are required to handle routing via gateways
* the RML needs to assign sequence numbers to each message on a per-peer basis. The receiving RML will then deliver messages in order, thus preventing out-of-order messaging in the case where messages travel across different transports or a message needs to be redirected/resent due to failure of a NIC
This commit was SVN r29058.
2013-08-22 16:37:40 +00:00
|
|
|
return ORTE_SUCCESS;
|
2011-10-17 20:51:22 +00:00
|
|
|
}
|
|
|
|
|
|
2013-10-08 18:37:59 +00:00
|
|
|
static void rte_abort(int status, bool report)
|
2011-10-17 20:51:22 +00:00
|
|
|
{
|
2014-09-23 22:51:10 +00:00
|
|
|
struct timespec tp = {0, 100000};
|
|
|
|
|
|
2013-10-08 18:37:59 +00:00
|
|
|
OPAL_OUTPUT_VERBOSE((1, orte_ess_base_framework.framework_output,
|
|
|
|
|
"%s ess:pmi:abort: abort with status %d",
|
|
|
|
|
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
|
|
|
|
|
status));
|
|
|
|
|
|
|
|
|
|
/* PMI doesn't like NULL messages, but our interface
|
|
|
|
|
* doesn't provide one - so rig one up here
|
|
|
|
|
*/
|
2015-06-18 09:53:20 -07:00
|
|
|
opal_pmix.abort(status, "N/A", NULL);
|
2013-10-08 18:37:59 +00:00
|
|
|
|
2014-09-23 22:51:10 +00:00
|
|
|
/* provide a little delay for the PMIx thread to
|
|
|
|
|
* get the info out */
|
|
|
|
|
nanosleep(&tp, NULL);
|
|
|
|
|
|
2013-10-08 18:37:59 +00:00
|
|
|
/* Now Exit */
|
2014-09-23 22:51:10 +00:00
|
|
|
_exit(status);
|
2011-10-17 20:51:22 +00:00
|
|
|
}
|