2005-03-14 23:57:21 +03:00
|
|
|
/*
|
2007-03-29 04:50:56 +04:00
|
|
|
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
|
2005-11-05 22:57:48 +03:00
|
|
|
* University Research and Technology
|
|
|
|
|
* Corporation. All rights reserved.
|
2006-08-23 07:32:36 +04:00
|
|
|
* Copyright (c) 2004-2006 The University of Tennessee and The University
|
2005-11-05 22:57:48 +03:00
|
|
|
* of Tennessee Research Foundation. All rights
|
|
|
|
|
* reserved.
|
2015-06-24 06:59:57 +03:00
|
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
2005-03-14 23:57:21 +03:00
|
|
|
* University of Stuttgart. All rights reserved.
|
2005-03-24 15:43:37 +03:00
|
|
|
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
|
|
|
* All rights reserved.
|
2013-05-18 01:45:49 +04:00
|
|
|
* Copyright (c) 2006-2013 Cisco Systems, Inc. All rights reserved.
|
2012-09-01 01:28:49 +04:00
|
|
|
* Copyright (c) 2011-2012 Los Alamos National Security, LLC.
|
2012-04-06 18:23:13 +04:00
|
|
|
* All rights reserved.
|
2017-05-10 02:06:15 +03:00
|
|
|
* Copyright (c) 2014-2017 Intel, Inc. All rights reserved.
|
2005-03-14 23:57:21 +03:00
|
|
|
* $COPYRIGHT$
|
2015-06-24 06:59:57 +03:00
|
|
|
*
|
2005-03-14 23:57:21 +03:00
|
|
|
* Additional copyrights may follow
|
2015-06-24 06:59:57 +03:00
|
|
|
*
|
2005-03-14 23:57:21 +03:00
|
|
|
* $HEADER$
|
|
|
|
|
*/
|
2006-07-05 00:12:35 +04:00
|
|
|
|
2005-03-14 23:57:21 +03:00
|
|
|
#include "orte_config.h"
|
2008-02-28 04:57:57 +03:00
|
|
|
#include "orte/constants.h"
|
|
|
|
|
#include "orte/types.h"
|
2006-07-10 18:10:21 +04:00
|
|
|
|
2005-03-14 23:57:21 +03:00
|
|
|
#include <errno.h>
|
2005-12-12 23:04:00 +03:00
|
|
|
#ifdef HAVE_UNISTD_H
|
2005-03-14 23:57:21 +03:00
|
|
|
#include <unistd.h>
|
2005-12-12 23:04:00 +03:00
|
|
|
#endif /* HAVE_UNISTD_H */
|
2005-03-14 23:57:21 +03:00
|
|
|
#include <string.h>
|
|
|
|
|
|
2008-06-09 18:53:58 +04:00
|
|
|
#include "orte/util/show_help.h"
|
2006-07-10 18:10:21 +04:00
|
|
|
#include "orte/mca/errmgr/errmgr.h"
|
2011-02-16 02:24:31 +03:00
|
|
|
#include "orte/util/error_strings.h"
|
2006-09-15 01:29:51 +04:00
|
|
|
|
|
|
|
|
#include "orte/mca/rmaps/base/rmaps_private.h"
|
2006-10-07 23:50:12 +04:00
|
|
|
#include "orte/mca/rmaps/base/base.h"
|
2005-03-14 23:57:21 +03:00
|
|
|
#include "rmaps_rr.h"
|
|
|
|
|
|
|
|
|
|
/*
|
2006-07-10 18:10:21 +04:00
|
|
|
* Create a round-robin mapping for the job.
|
2005-03-14 23:57:21 +03:00
|
|
|
*/
|
2008-02-28 04:57:57 +03:00
|
|
|
static int orte_rmaps_rr_map(orte_job_t *jdata)
|
2005-03-14 23:57:21 +03:00
|
|
|
{
|
2009-06-12 21:52:17 +04:00
|
|
|
orte_app_context_t *app;
|
|
|
|
|
int i;
|
2008-04-17 17:50:59 +04:00
|
|
|
opal_list_t node_list;
|
2008-02-28 04:57:57 +03:00
|
|
|
opal_list_item_t *item;
|
2013-05-18 01:45:49 +04:00
|
|
|
orte_std_cntr_t num_slots;
|
2006-07-10 18:10:21 +04:00
|
|
|
int rc;
|
2011-03-12 08:30:09 +03:00
|
|
|
mca_base_component_t *c = &mca_rmaps_round_robin_component.base_version;
|
2011-12-06 02:01:08 +04:00
|
|
|
bool initial_map=true;
|
2011-03-12 08:30:09 +03:00
|
|
|
|
2011-02-16 02:24:31 +03:00
|
|
|
/* this mapper can only handle initial launch
|
|
|
|
|
* when rr mapping is desired - allow
|
|
|
|
|
* restarting of failed apps
|
|
|
|
|
*/
|
2014-06-01 20:14:10 +04:00
|
|
|
if (ORTE_FLAG_TEST(jdata, ORTE_JOB_FLAG_RESTART)) {
|
2013-03-28 01:14:43 +04:00
|
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
2012-04-06 18:23:13 +04:00
|
|
|
"mca:rmaps:rr: job %s is being restarted - rr cannot map",
|
|
|
|
|
ORTE_JOBID_PRINT(jdata->jobid));
|
2011-02-16 02:24:31 +03:00
|
|
|
return ORTE_ERR_TAKE_NEXT_OPTION;
|
|
|
|
|
}
|
2011-03-12 08:30:09 +03:00
|
|
|
if (NULL != jdata->map->req_mapper &&
|
|
|
|
|
0 != strcasecmp(jdata->map->req_mapper, c->mca_component_name)) {
|
2011-02-16 02:24:31 +03:00
|
|
|
/* a mapper has been specified, and it isn't me */
|
2013-03-28 01:14:43 +04:00
|
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
2011-02-16 02:24:31 +03:00
|
|
|
"mca:rmaps:rr: job %s not using rr mapper",
|
|
|
|
|
ORTE_JOBID_PRINT(jdata->jobid));
|
|
|
|
|
return ORTE_ERR_TAKE_NEXT_OPTION;
|
|
|
|
|
}
|
At long last, the fabled revision to the affinity system has arrived. A more detailed explanation of how this all works will be presented here:
https://svn.open-mpi.org/trac/ompi/wiki/ProcessPlacement
The wiki page is incomplete at the moment, but I hope to complete it over the next few days. I will provide updates on the devel list. As the wiki page states, the default and most commonly used options remain unchanged (except as noted below). New, esoteric and complex options have been added, but unless you are a true masochist, you are unlikely to use many of them beyond perhaps an initial curiosity-motivated experimentation.
In a nutshell, this commit revamps the map/rank/bind procedure to take into account topology info on the compute nodes. I have, for the most part, preserved the default behaviors, with three notable exceptions:
1. I have at long last bowed my head in submission to the system admin's of managed clusters. For years, they have complained about our default of allowing users to oversubscribe nodes - i.e., to run more processes on a node than allocated slots. Accordingly, I have modified the default behavior: if you are running off of hostfile/dash-host allocated nodes, then the default is to allow oversubscription. If you are running off of RM-allocated nodes, then the default is to NOT allow oversubscription. Flags to override these behaviors are provided, so this only affects the default behavior.
2. both cpus/rank and stride have been removed. The latter was demanded by those who didn't understand the purpose behind it - and I agreed as the users who requested it are no longer using it. The former was removed temporarily pending implementation.
3. vm launch is now the sole method for starting OMPI. It was just too darned hard to maintain multiple launch procedures - maybe someday, provided someone can demonstrate a reason to do so.
As Jeff stated, it is impossible to fully test a change of this size. I have tested it on Linux and Mac, covering all the default and simple options, singletons, and comm_spawn. That said, I'm sure others will find problems, so I'll be watching MTT results until this stabilizes.
This commit was SVN r25476.
2011-11-15 07:40:11 +04:00
|
|
|
if (ORTE_MAPPING_RR < ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
|
2011-03-12 08:30:09 +03:00
|
|
|
/* I don't know how to do these - defer */
|
2013-03-28 01:14:43 +04:00
|
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
2011-03-12 08:30:09 +03:00
|
|
|
"mca:rmaps:rr: job %s not using rr mapper",
|
|
|
|
|
ORTE_JOBID_PRINT(jdata->jobid));
|
|
|
|
|
return ORTE_ERR_TAKE_NEXT_OPTION;
|
|
|
|
|
}
|
2011-02-16 02:24:31 +03:00
|
|
|
|
2013-03-28 01:14:43 +04:00
|
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
2011-02-16 02:24:31 +03:00
|
|
|
"mca:rmaps:rr: mapping job %s",
|
|
|
|
|
ORTE_JOBID_PRINT(jdata->jobid));
|
2015-06-24 06:59:57 +03:00
|
|
|
|
2011-02-17 02:01:57 +03:00
|
|
|
/* flag that I did the mapping */
|
2011-03-12 18:37:33 +03:00
|
|
|
if (NULL != jdata->map->last_mapper) {
|
|
|
|
|
free(jdata->map->last_mapper);
|
|
|
|
|
}
|
2011-03-12 08:30:09 +03:00
|
|
|
jdata->map->last_mapper = strdup(c->mca_component_name);
|
2011-02-17 02:01:57 +03:00
|
|
|
|
2008-02-28 04:57:57 +03:00
|
|
|
/* start at the beginning... */
|
2009-05-16 08:15:55 +04:00
|
|
|
jdata->num_procs = 0;
|
2015-06-24 06:59:57 +03:00
|
|
|
|
2008-02-28 04:57:57 +03:00
|
|
|
/* cycle through the app_contexts, mapping them sequentially */
|
2009-06-12 21:52:17 +04:00
|
|
|
for(i=0; i < jdata->apps->size; i++) {
|
|
|
|
|
if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, i))) {
|
|
|
|
|
continue;
|
|
|
|
|
}
|
2015-06-24 06:59:57 +03:00
|
|
|
|
2011-07-13 19:00:22 +04:00
|
|
|
/* setup the nodelist here in case we jump to error */
|
|
|
|
|
OBJ_CONSTRUCT(&node_list, opal_list_t);
|
|
|
|
|
|
2008-02-28 04:57:57 +03:00
|
|
|
/* if the number of processes wasn't specified, then we know there can be only
|
|
|
|
|
* one app_context allowed in the launch, and that we are to launch it across
|
|
|
|
|
* all available slots. We'll double-check the single app_context rule first
|
|
|
|
|
*/
|
|
|
|
|
if (0 == app->num_procs && 1 < jdata->num_apps) {
|
This commit represents a bunch of work on a Mercurial side branch. As
such, the commit message back to the master SVN repository is fairly
long.
= ORTE Job-Level Output Messages =
Add two new interfaces that should be used for all new code throughout
the ORTE and OMPI layers (we already make the search-and-replace on
the existing ORTE / OMPI layers):
* orte_output(): (and corresponding friends ORTE_OUTPUT,
orte_output_verbose, etc.) This function sends the output directly
to the HNP for processing as part of a job-specific output
channel. It supports all the same outputs as opal_output()
(syslog, file, stdout, stderr), but for stdout/stderr, the output
is sent to the HNP for processing and output. More on this below.
* orte_show_help(): This function is a drop-in-replacement for
opal_show_help(), with two differences in functionality:
1. the rendered text help message output is sent to the HNP for
display (rather than outputting directly into the process' stderr
stream)
1. the HNP detects duplicate help messages and does not display them
(so that you don't see the same error message N times, once from
each of your N MPI processes); instead, it counts "new" instances
of the help message and displays a message every ~5 seconds when
there are new ones ("I got X new copies of the help message...")
opal_show_help and opal_output still exist, but they only output in
the current process. The intent for the new orte_* functions is that
they can apply job-level intelligence to the output. As such, we
recommend that all new ORTE and OMPI code use the new orte_*
functions, not thei opal_* functions.
=== New code ===
For ORTE and OMPI programmers, here's what you need to do differently
in new code:
* Do not include opal/util/show_help.h or opal/util/output.h.
Instead, include orte/util/output.h (this one header file has
declarations for both the orte_output() series of functions and
orte_show_help()).
* Effectively s/opal_output/orte_output/gi throughout your code.
Note that orte_output_open() takes a slightly different argument
list (as a way to pass data to the filtering stream -- see below),
so you if explicitly call opal_output_open(), you'll need to
slightly adapt to the new signature of orte_output_open().
* Literally s/opal_show_help/orte_show_help/. The function signature
is identical.
=== Notes ===
* orte_output'ing to stream 0 will do similar to what
opal_output'ing did, so leaving a hard-coded "0" as the first
argument is safe.
* For systems that do not use ORTE's RML or the HNP, the effect of
orte_output_* and orte_show_help will be identical to their opal
counterparts (the additional information passed to
orte_output_open() will be lost!). Indeed, the orte_* functions
simply become trivial wrappers to their opal_* counterparts. Note
that we have not tested this; the code is simple but it is quite
possible that we mucked something up.
= Filter Framework =
Messages sent view the new orte_* functions described above and
messages output via the IOF on the HNP will now optionally be passed
through a new "filter" framework before being output to
stdout/stderr. The "filter" OPAL MCA framework is intended to allow
preprocessing to messages before they are sent to their final
destinations. The first component that was written in the filter
framework was to create an XML stream, segregating all the messages
into different XML tags, etc. This will allow 3rd party tools to read
the stdout/stderr from the HNP and be able to know exactly what each
text message is (e.g., a help message, another OMPI infrastructure
message, stdout from the user process, stderr from the user process,
etc.).
Filtering is not active by default. Filter components must be
specifically requested, such as:
{{{
$ mpirun --mca filter xml ...
}}}
There can only be one filter component active.
= New MCA Parameters =
The new functionality described above introduces two new MCA
parameters:
* '''orte_base_help_aggregate''': Defaults to 1 (true), meaning that
help messages will be aggregated, as described above. If set to 0,
all help messages will be displayed, even if they are duplicates
(i.e., the original behavior).
* '''orte_base_show_output_recursions''': An MCA parameter to help
debug one of the known issues, described below. It is likely that
this MCA parameter will disappear before v1.3 final.
= Known Issues =
* The XML filter component is not complete. The current output from
this component is preliminary and not real XML. A bit more work
needs to be done to configure.m4 search for an appropriate XML
library/link it in/use it at run time.
* There are possible recursion loops in the orte_output() and
orte_show_help() functions -- e.g., if RML send calls orte_output()
or orte_show_help(). We have some ideas how to fix these, but
figured that it was ok to commit before feature freeze with known
issues. The code currently contains sub-optimal workarounds so
that this will not be a problem, but it would be good to actually
solve the problem rather than have hackish workarounds before v1.3 final.
This commit was SVN r18434.
2008-05-14 00:00:55 +04:00
|
|
|
orte_show_help("help-orte-rmaps-rr.txt", "orte-rmaps-rr:multi-apps-and-zero-np",
|
2008-02-28 04:57:57 +03:00
|
|
|
true, jdata->num_apps, NULL);
|
|
|
|
|
rc = ORTE_ERR_SILENT;
|
|
|
|
|
goto error;
|
2006-07-11 06:59:23 +04:00
|
|
|
}
|
|
|
|
|
|
2008-02-28 04:57:57 +03:00
|
|
|
/* for each app_context, we have to get the list of nodes that it can
|
|
|
|
|
* use since that can now be modified with a hostfile and/or -host
|
|
|
|
|
* option
|
|
|
|
|
*/
|
|
|
|
|
if(ORTE_SUCCESS != (rc = orte_rmaps_base_get_target_nodes(&node_list, &num_slots, app,
|
2012-09-01 01:28:49 +04:00
|
|
|
jdata->map->mapping, initial_map, false))) {
|
2008-02-28 04:57:57 +03:00
|
|
|
ORTE_ERROR_LOG(rc);
|
|
|
|
|
goto error;
|
2006-07-10 18:10:21 +04:00
|
|
|
}
|
2011-12-06 02:01:08 +04:00
|
|
|
/* flag that all subsequent requests should not reset the node->mapped flag */
|
|
|
|
|
initial_map = false;
|
2008-02-28 04:57:57 +03:00
|
|
|
|
|
|
|
|
/* if a bookmark exists from some prior mapping, set us to start there */
|
2011-12-06 02:01:08 +04:00
|
|
|
jdata->bookmark = orte_rmaps_base_get_starting_point(&node_list, jdata);
|
2015-06-24 06:59:57 +03:00
|
|
|
|
2011-02-16 02:24:31 +03:00
|
|
|
if (0 == app->num_procs) {
|
2014-01-23 09:01:07 +04:00
|
|
|
/* set the num_procs to equal the number of slots on these
|
|
|
|
|
* mapped nodes, taking into account the number of cpus/rank
|
|
|
|
|
*/
|
2016-08-23 01:54:41 +03:00
|
|
|
app->num_procs = num_slots;
|
2014-07-04 21:00:35 +04:00
|
|
|
/* sometimes, we have only one "slot" assigned, but may
|
|
|
|
|
* want more than one cpu/rank - so ensure we always wind
|
|
|
|
|
* up with at least one proc */
|
|
|
|
|
if (0 == app->num_procs) {
|
|
|
|
|
app->num_procs = 1;
|
|
|
|
|
}
|
2006-07-11 06:59:23 +04:00
|
|
|
}
|
2015-06-24 06:59:57 +03:00
|
|
|
|
2005-10-08 02:24:52 +04:00
|
|
|
/* Make assignments */
|
At long last, the fabled revision to the affinity system has arrived. A more detailed explanation of how this all works will be presented here:
https://svn.open-mpi.org/trac/ompi/wiki/ProcessPlacement
The wiki page is incomplete at the moment, but I hope to complete it over the next few days. I will provide updates on the devel list. As the wiki page states, the default and most commonly used options remain unchanged (except as noted below). New, esoteric and complex options have been added, but unless you are a true masochist, you are unlikely to use many of them beyond perhaps an initial curiosity-motivated experimentation.
In a nutshell, this commit revamps the map/rank/bind procedure to take into account topology info on the compute nodes. I have, for the most part, preserved the default behaviors, with three notable exceptions:
1. I have at long last bowed my head in submission to the system admin's of managed clusters. For years, they have complained about our default of allowing users to oversubscribe nodes - i.e., to run more processes on a node than allocated slots. Accordingly, I have modified the default behavior: if you are running off of hostfile/dash-host allocated nodes, then the default is to allow oversubscription. If you are running off of RM-allocated nodes, then the default is to NOT allow oversubscription. Flags to override these behaviors are provided, so this only affects the default behavior.
2. both cpus/rank and stride have been removed. The latter was demanded by those who didn't understand the purpose behind it - and I agreed as the users who requested it are no longer using it. The former was removed temporarily pending implementation.
3. vm launch is now the sole method for starting OMPI. It was just too darned hard to maintain multiple launch procedures - maybe someday, provided someone can demonstrate a reason to do so.
As Jeff stated, it is impossible to fully test a change of this size. I have tested it on Linux and Mac, covering all the default and simple options, singletons, and comm_spawn. That said, I'm sure others will find problems, so I'll be watching MTT results until this stabilizes.
This commit was SVN r25476.
2011-11-15 07:40:11 +04:00
|
|
|
if (ORTE_MAPPING_BYNODE == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
|
|
|
|
|
rc = orte_rmaps_rr_bynode(jdata, app, &node_list, num_slots,
|
|
|
|
|
app->num_procs);
|
|
|
|
|
} else if (ORTE_MAPPING_BYSLOT == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
|
|
|
|
|
rc = orte_rmaps_rr_byslot(jdata, app, &node_list, num_slots,
|
|
|
|
|
app->num_procs);
|
|
|
|
|
} else if (ORTE_MAPPING_BYHWTHREAD == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
|
|
|
|
|
rc = orte_rmaps_rr_byobj(jdata, app, &node_list, num_slots,
|
|
|
|
|
app->num_procs, HWLOC_OBJ_PU, 0);
|
2014-03-03 20:46:37 +04:00
|
|
|
if (ORTE_ERR_NOT_FOUND == rc) {
|
2013-12-17 18:50:10 +04:00
|
|
|
/* if the mapper couldn't map by this object because
|
|
|
|
|
* it isn't available, but the error allows us to try
|
|
|
|
|
* byslot, then do so
|
|
|
|
|
*/
|
2014-03-24 19:47:29 +04:00
|
|
|
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYSLOT);
|
2013-12-17 18:50:10 +04:00
|
|
|
rc = orte_rmaps_rr_byslot(jdata, app, &node_list, num_slots,
|
|
|
|
|
app->num_procs);
|
|
|
|
|
}
|
At long last, the fabled revision to the affinity system has arrived. A more detailed explanation of how this all works will be presented here:
https://svn.open-mpi.org/trac/ompi/wiki/ProcessPlacement
The wiki page is incomplete at the moment, but I hope to complete it over the next few days. I will provide updates on the devel list. As the wiki page states, the default and most commonly used options remain unchanged (except as noted below). New, esoteric and complex options have been added, but unless you are a true masochist, you are unlikely to use many of them beyond perhaps an initial curiosity-motivated experimentation.
In a nutshell, this commit revamps the map/rank/bind procedure to take into account topology info on the compute nodes. I have, for the most part, preserved the default behaviors, with three notable exceptions:
1. I have at long last bowed my head in submission to the system admin's of managed clusters. For years, they have complained about our default of allowing users to oversubscribe nodes - i.e., to run more processes on a node than allocated slots. Accordingly, I have modified the default behavior: if you are running off of hostfile/dash-host allocated nodes, then the default is to allow oversubscription. If you are running off of RM-allocated nodes, then the default is to NOT allow oversubscription. Flags to override these behaviors are provided, so this only affects the default behavior.
2. both cpus/rank and stride have been removed. The latter was demanded by those who didn't understand the purpose behind it - and I agreed as the users who requested it are no longer using it. The former was removed temporarily pending implementation.
3. vm launch is now the sole method for starting OMPI. It was just too darned hard to maintain multiple launch procedures - maybe someday, provided someone can demonstrate a reason to do so.
As Jeff stated, it is impossible to fully test a change of this size. I have tested it on Linux and Mac, covering all the default and simple options, singletons, and comm_spawn. That said, I'm sure others will find problems, so I'll be watching MTT results until this stabilizes.
This commit was SVN r25476.
2011-11-15 07:40:11 +04:00
|
|
|
} else if (ORTE_MAPPING_BYCORE == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
|
|
|
|
|
rc = orte_rmaps_rr_byobj(jdata, app, &node_list, num_slots,
|
|
|
|
|
app->num_procs, HWLOC_OBJ_CORE, 0);
|
2014-03-03 20:46:37 +04:00
|
|
|
if (ORTE_ERR_NOT_FOUND == rc) {
|
2013-12-17 18:50:10 +04:00
|
|
|
/* if the mapper couldn't map by this object because
|
|
|
|
|
* it isn't available, but the error allows us to try
|
|
|
|
|
* byslot, then do so
|
|
|
|
|
*/
|
2014-03-24 19:47:29 +04:00
|
|
|
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYSLOT);
|
2013-12-17 18:50:10 +04:00
|
|
|
rc = orte_rmaps_rr_byslot(jdata, app, &node_list, num_slots,
|
|
|
|
|
app->num_procs);
|
|
|
|
|
}
|
At long last, the fabled revision to the affinity system has arrived. A more detailed explanation of how this all works will be presented here:
https://svn.open-mpi.org/trac/ompi/wiki/ProcessPlacement
The wiki page is incomplete at the moment, but I hope to complete it over the next few days. I will provide updates on the devel list. As the wiki page states, the default and most commonly used options remain unchanged (except as noted below). New, esoteric and complex options have been added, but unless you are a true masochist, you are unlikely to use many of them beyond perhaps an initial curiosity-motivated experimentation.
In a nutshell, this commit revamps the map/rank/bind procedure to take into account topology info on the compute nodes. I have, for the most part, preserved the default behaviors, with three notable exceptions:
1. I have at long last bowed my head in submission to the system admin's of managed clusters. For years, they have complained about our default of allowing users to oversubscribe nodes - i.e., to run more processes on a node than allocated slots. Accordingly, I have modified the default behavior: if you are running off of hostfile/dash-host allocated nodes, then the default is to allow oversubscription. If you are running off of RM-allocated nodes, then the default is to NOT allow oversubscription. Flags to override these behaviors are provided, so this only affects the default behavior.
2. both cpus/rank and stride have been removed. The latter was demanded by those who didn't understand the purpose behind it - and I agreed as the users who requested it are no longer using it. The former was removed temporarily pending implementation.
3. vm launch is now the sole method for starting OMPI. It was just too darned hard to maintain multiple launch procedures - maybe someday, provided someone can demonstrate a reason to do so.
As Jeff stated, it is impossible to fully test a change of this size. I have tested it on Linux and Mac, covering all the default and simple options, singletons, and comm_spawn. That said, I'm sure others will find problems, so I'll be watching MTT results until this stabilizes.
This commit was SVN r25476.
2011-11-15 07:40:11 +04:00
|
|
|
} else if (ORTE_MAPPING_BYL1CACHE == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
|
|
|
|
|
rc = orte_rmaps_rr_byobj(jdata, app, &node_list, num_slots,
|
|
|
|
|
app->num_procs, HWLOC_OBJ_CACHE, 1);
|
2014-03-03 20:46:37 +04:00
|
|
|
if (ORTE_ERR_NOT_FOUND == rc) {
|
2013-12-17 18:50:10 +04:00
|
|
|
/* if the mapper couldn't map by this object because
|
|
|
|
|
* it isn't available, but the error allows us to try
|
|
|
|
|
* byslot, then do so
|
|
|
|
|
*/
|
2014-03-24 19:47:29 +04:00
|
|
|
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYSLOT);
|
2013-12-17 18:50:10 +04:00
|
|
|
rc = orte_rmaps_rr_byslot(jdata, app, &node_list, num_slots,
|
|
|
|
|
app->num_procs);
|
|
|
|
|
}
|
At long last, the fabled revision to the affinity system has arrived. A more detailed explanation of how this all works will be presented here:
https://svn.open-mpi.org/trac/ompi/wiki/ProcessPlacement
The wiki page is incomplete at the moment, but I hope to complete it over the next few days. I will provide updates on the devel list. As the wiki page states, the default and most commonly used options remain unchanged (except as noted below). New, esoteric and complex options have been added, but unless you are a true masochist, you are unlikely to use many of them beyond perhaps an initial curiosity-motivated experimentation.
In a nutshell, this commit revamps the map/rank/bind procedure to take into account topology info on the compute nodes. I have, for the most part, preserved the default behaviors, with three notable exceptions:
1. I have at long last bowed my head in submission to the system admin's of managed clusters. For years, they have complained about our default of allowing users to oversubscribe nodes - i.e., to run more processes on a node than allocated slots. Accordingly, I have modified the default behavior: if you are running off of hostfile/dash-host allocated nodes, then the default is to allow oversubscription. If you are running off of RM-allocated nodes, then the default is to NOT allow oversubscription. Flags to override these behaviors are provided, so this only affects the default behavior.
2. both cpus/rank and stride have been removed. The latter was demanded by those who didn't understand the purpose behind it - and I agreed as the users who requested it are no longer using it. The former was removed temporarily pending implementation.
3. vm launch is now the sole method for starting OMPI. It was just too darned hard to maintain multiple launch procedures - maybe someday, provided someone can demonstrate a reason to do so.
As Jeff stated, it is impossible to fully test a change of this size. I have tested it on Linux and Mac, covering all the default and simple options, singletons, and comm_spawn. That said, I'm sure others will find problems, so I'll be watching MTT results until this stabilizes.
This commit was SVN r25476.
2011-11-15 07:40:11 +04:00
|
|
|
} else if (ORTE_MAPPING_BYL2CACHE == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
|
|
|
|
|
rc = orte_rmaps_rr_byobj(jdata, app, &node_list, num_slots,
|
|
|
|
|
app->num_procs, HWLOC_OBJ_CACHE, 2);
|
2014-03-03 20:46:37 +04:00
|
|
|
if (ORTE_ERR_NOT_FOUND == rc) {
|
2013-12-17 18:50:10 +04:00
|
|
|
/* if the mapper couldn't map by this object because
|
|
|
|
|
* it isn't available, but the error allows us to try
|
|
|
|
|
* byslot, then do so
|
|
|
|
|
*/
|
2014-03-24 19:47:29 +04:00
|
|
|
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYSLOT);
|
2013-12-17 18:50:10 +04:00
|
|
|
rc = orte_rmaps_rr_byslot(jdata, app, &node_list, num_slots,
|
|
|
|
|
app->num_procs);
|
|
|
|
|
}
|
At long last, the fabled revision to the affinity system has arrived. A more detailed explanation of how this all works will be presented here:
https://svn.open-mpi.org/trac/ompi/wiki/ProcessPlacement
The wiki page is incomplete at the moment, but I hope to complete it over the next few days. I will provide updates on the devel list. As the wiki page states, the default and most commonly used options remain unchanged (except as noted below). New, esoteric and complex options have been added, but unless you are a true masochist, you are unlikely to use many of them beyond perhaps an initial curiosity-motivated experimentation.
In a nutshell, this commit revamps the map/rank/bind procedure to take into account topology info on the compute nodes. I have, for the most part, preserved the default behaviors, with three notable exceptions:
1. I have at long last bowed my head in submission to the system admin's of managed clusters. For years, they have complained about our default of allowing users to oversubscribe nodes - i.e., to run more processes on a node than allocated slots. Accordingly, I have modified the default behavior: if you are running off of hostfile/dash-host allocated nodes, then the default is to allow oversubscription. If you are running off of RM-allocated nodes, then the default is to NOT allow oversubscription. Flags to override these behaviors are provided, so this only affects the default behavior.
2. both cpus/rank and stride have been removed. The latter was demanded by those who didn't understand the purpose behind it - and I agreed as the users who requested it are no longer using it. The former was removed temporarily pending implementation.
3. vm launch is now the sole method for starting OMPI. It was just too darned hard to maintain multiple launch procedures - maybe someday, provided someone can demonstrate a reason to do so.
As Jeff stated, it is impossible to fully test a change of this size. I have tested it on Linux and Mac, covering all the default and simple options, singletons, and comm_spawn. That said, I'm sure others will find problems, so I'll be watching MTT results until this stabilizes.
This commit was SVN r25476.
2011-11-15 07:40:11 +04:00
|
|
|
} else if (ORTE_MAPPING_BYL3CACHE == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
|
|
|
|
|
rc = orte_rmaps_rr_byobj(jdata, app, &node_list, num_slots,
|
|
|
|
|
app->num_procs, HWLOC_OBJ_CACHE, 3);
|
2014-03-03 20:46:37 +04:00
|
|
|
if (ORTE_ERR_NOT_FOUND == rc) {
|
2013-12-17 18:50:10 +04:00
|
|
|
/* if the mapper couldn't map by this object because
|
|
|
|
|
* it isn't available, but the error allows us to try
|
|
|
|
|
* byslot, then do so
|
|
|
|
|
*/
|
2014-03-24 19:47:29 +04:00
|
|
|
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYSLOT);
|
2013-12-17 18:50:10 +04:00
|
|
|
rc = orte_rmaps_rr_byslot(jdata, app, &node_list, num_slots,
|
|
|
|
|
app->num_procs);
|
|
|
|
|
}
|
At long last, the fabled revision to the affinity system has arrived. A more detailed explanation of how this all works will be presented here:
https://svn.open-mpi.org/trac/ompi/wiki/ProcessPlacement
The wiki page is incomplete at the moment, but I hope to complete it over the next few days. I will provide updates on the devel list. As the wiki page states, the default and most commonly used options remain unchanged (except as noted below). New, esoteric and complex options have been added, but unless you are a true masochist, you are unlikely to use many of them beyond perhaps an initial curiosity-motivated experimentation.
In a nutshell, this commit revamps the map/rank/bind procedure to take into account topology info on the compute nodes. I have, for the most part, preserved the default behaviors, with three notable exceptions:
1. I have at long last bowed my head in submission to the system admin's of managed clusters. For years, they have complained about our default of allowing users to oversubscribe nodes - i.e., to run more processes on a node than allocated slots. Accordingly, I have modified the default behavior: if you are running off of hostfile/dash-host allocated nodes, then the default is to allow oversubscription. If you are running off of RM-allocated nodes, then the default is to NOT allow oversubscription. Flags to override these behaviors are provided, so this only affects the default behavior.
2. both cpus/rank and stride have been removed. The latter was demanded by those who didn't understand the purpose behind it - and I agreed as the users who requested it are no longer using it. The former was removed temporarily pending implementation.
3. vm launch is now the sole method for starting OMPI. It was just too darned hard to maintain multiple launch procedures - maybe someday, provided someone can demonstrate a reason to do so.
As Jeff stated, it is impossible to fully test a change of this size. I have tested it on Linux and Mac, covering all the default and simple options, singletons, and comm_spawn. That said, I'm sure others will find problems, so I'll be watching MTT results until this stabilizes.
This commit was SVN r25476.
2011-11-15 07:40:11 +04:00
|
|
|
} else if (ORTE_MAPPING_BYSOCKET == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
|
|
|
|
|
rc = orte_rmaps_rr_byobj(jdata, app, &node_list, num_slots,
|
|
|
|
|
app->num_procs, HWLOC_OBJ_SOCKET, 0);
|
2014-03-03 20:46:37 +04:00
|
|
|
if (ORTE_ERR_NOT_FOUND == rc) {
|
2013-12-17 18:50:10 +04:00
|
|
|
/* if the mapper couldn't map by this object because
|
|
|
|
|
* it isn't available, but the error allows us to try
|
|
|
|
|
* byslot, then do so
|
|
|
|
|
*/
|
2014-03-24 19:47:29 +04:00
|
|
|
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYSLOT);
|
2013-12-17 18:50:10 +04:00
|
|
|
rc = orte_rmaps_rr_byslot(jdata, app, &node_list, num_slots,
|
|
|
|
|
app->num_procs);
|
|
|
|
|
}
|
At long last, the fabled revision to the affinity system has arrived. A more detailed explanation of how this all works will be presented here:
https://svn.open-mpi.org/trac/ompi/wiki/ProcessPlacement
The wiki page is incomplete at the moment, but I hope to complete it over the next few days. I will provide updates on the devel list. As the wiki page states, the default and most commonly used options remain unchanged (except as noted below). New, esoteric and complex options have been added, but unless you are a true masochist, you are unlikely to use many of them beyond perhaps an initial curiosity-motivated experimentation.
In a nutshell, this commit revamps the map/rank/bind procedure to take into account topology info on the compute nodes. I have, for the most part, preserved the default behaviors, with three notable exceptions:
1. I have at long last bowed my head in submission to the system admin's of managed clusters. For years, they have complained about our default of allowing users to oversubscribe nodes - i.e., to run more processes on a node than allocated slots. Accordingly, I have modified the default behavior: if you are running off of hostfile/dash-host allocated nodes, then the default is to allow oversubscription. If you are running off of RM-allocated nodes, then the default is to NOT allow oversubscription. Flags to override these behaviors are provided, so this only affects the default behavior.
2. both cpus/rank and stride have been removed. The latter was demanded by those who didn't understand the purpose behind it - and I agreed as the users who requested it are no longer using it. The former was removed temporarily pending implementation.
3. vm launch is now the sole method for starting OMPI. It was just too darned hard to maintain multiple launch procedures - maybe someday, provided someone can demonstrate a reason to do so.
As Jeff stated, it is impossible to fully test a change of this size. I have tested it on Linux and Mac, covering all the default and simple options, singletons, and comm_spawn. That said, I'm sure others will find problems, so I'll be watching MTT results until this stabilizes.
This commit was SVN r25476.
2011-11-15 07:40:11 +04:00
|
|
|
} else if (ORTE_MAPPING_BYNUMA == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
|
|
|
|
|
rc = orte_rmaps_rr_byobj(jdata, app, &node_list, num_slots,
|
|
|
|
|
app->num_procs, HWLOC_OBJ_NODE, 0);
|
2014-03-03 20:46:37 +04:00
|
|
|
if (ORTE_ERR_NOT_FOUND == rc) {
|
2013-12-17 18:50:10 +04:00
|
|
|
/* if the mapper couldn't map by this object because
|
|
|
|
|
* it isn't available, but the error allows us to try
|
|
|
|
|
* byslot, then do so
|
|
|
|
|
*/
|
2014-03-24 19:47:29 +04:00
|
|
|
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYSLOT);
|
2013-12-17 18:50:10 +04:00
|
|
|
rc = orte_rmaps_rr_byslot(jdata, app, &node_list, num_slots,
|
|
|
|
|
app->num_procs);
|
|
|
|
|
}
|
(copied from a mail that has a lengthy description of this commit)
I spoke with Tim about this the other day -- he gave me the green
light to go ahead with this, but it turned into a bigger job than I
thought it would be. I revamped how the default RAS scheduling and
round_robin RMAPS mapping occurs. The previous algorithms were pretty
brain dead, and ignored the "slots" and "max_slots" tokens in
hostfiles. I considered this a big enough problem to fix it for the
beta (because there is currently no way to control where processes are
launched on SMPs).
There's still some more bells and whistles that I'd like to implement,
but there's no hurry, and they can go on the trunk at any time. My
patches below are for what I considered "essential", and do the
following:
- honor the "slots" and "max-slots" tokens in the hostfile (and all
their synonyms), meaning that we allocate/map until we fill slots,
and if there are still more processes to allocate/map, we keep going
until we fill max-slots (i.e., only oversubscribe a node if we have
to).
- offer two different algorithms, currently supported by two new
options to orterun. Remember that there are two parts here -- slot
allocation and process mapping. Slot allocation controls how many
processes we'll be running on a node. After that decision has been
made, process mapping effectively controls where the ranks of
MPI_COMM_WORLD (MCW) are placed. Some of the examples given below
don't make sense unless you remember that there is a difference
between the two (which makes total sense, but you have to think
about it in terms of both things):
1. "-bynode": allocates/maps one process per node in a round-robin
fashion until all slots on the node are taken. If we still have more
processes after all slots are taken, then keep going until all
max-slots are taken. Examples:
- The hostfile:
eddie slots=2 max-slots=4
vogon slots=4 max-slots=8
- orterun -bynode -np 6 -hostfile hostfile a.out
eddie: MCW ranks 0, 2
vogon: MCW ranks 1, 3, 4, 5
- orterun -bynode -np 8 -hostfile hostfile a.out
eddie: MCW ranks 0, 2, 4
vogon: MCW ranks 1, 3, 5, 6, 7
-> the algorithm oversubscribes all nodes "equally" (until each
node's max_slots is hit, of course)
- orterun -bynode -np 12 -hostfile hostfile a.out
eddie: MCW ranks 0, 2, 4, 6
vogon: MCW ranks 1, 3, 5, 7, 8, 9, 10, 11
2. "-byslot" (this is the default if you don't specify -bynode):
greedily takes all available slots on a node for a job before moving
on to the next node. If we still have processes to allocate/schedule,
then oversubscribe all nodes equally (i.e., go round robin on all
nodes until each node's max_slots is hit). Examples:
- The hostfile
eddie slots=2 max-slots=4
vogon slots=4 max-slots=8
- orterun -np 6 -hostfile hostfile a.out
eddie: MCW ranks 0, 1
vogon: MCW ranks 2, 3, 4, 5
- orterun -np 8 -hostfile hostfile a.out
eddie: MCW ranks 0, 1, 2
vogon: MCW ranks 3, 4, 5, 6, 7
-> the algorithm oversubscribes all nodes "equally" (until max_slots
is hit)
- orterun -np 12 -hostfile hostfile a.out
eddie: MCW ranks 0, 1, 2, 3
vogon: MCW ranks 4, 5, 6, 7, 8, 9, 10, 11
The above examples are fairly contrived, and it's not clear from them
that you can get different allocation answers in all cases (the
mapping differences are obvious). Consider the following allocation
example:
- The hostfile
eddie count=4
vogon count=4
earth count=4
deep-thought count=4
- orterun -np 8 -hostfile hostfile a.out
eddie: 4 slots will be allocated
vogon: 4 slots will be allocated
earth: no slots allocated
deep-thought: no slots allocated
- orterun -bynode -np 8 -hostfile hostfile a.out
eddie: 2 slots will be allocated
vogon: 2 slots will be allocated
earth: 2 slots will be allocated
deep-thought: 2 slots will be allocated
This commit was SVN r5894.
2005-05-31 20:36:53 +04:00
|
|
|
} else {
|
At long last, the fabled revision to the affinity system has arrived. A more detailed explanation of how this all works will be presented here:
https://svn.open-mpi.org/trac/ompi/wiki/ProcessPlacement
The wiki page is incomplete at the moment, but I hope to complete it over the next few days. I will provide updates on the devel list. As the wiki page states, the default and most commonly used options remain unchanged (except as noted below). New, esoteric and complex options have been added, but unless you are a true masochist, you are unlikely to use many of them beyond perhaps an initial curiosity-motivated experimentation.
In a nutshell, this commit revamps the map/rank/bind procedure to take into account topology info on the compute nodes. I have, for the most part, preserved the default behaviors, with three notable exceptions:
1. I have at long last bowed my head in submission to the system admin's of managed clusters. For years, they have complained about our default of allowing users to oversubscribe nodes - i.e., to run more processes on a node than allocated slots. Accordingly, I have modified the default behavior: if you are running off of hostfile/dash-host allocated nodes, then the default is to allow oversubscription. If you are running off of RM-allocated nodes, then the default is to NOT allow oversubscription. Flags to override these behaviors are provided, so this only affects the default behavior.
2. both cpus/rank and stride have been removed. The latter was demanded by those who didn't understand the purpose behind it - and I agreed as the users who requested it are no longer using it. The former was removed temporarily pending implementation.
3. vm launch is now the sole method for starting OMPI. It was just too darned hard to maintain multiple launch procedures - maybe someday, provided someone can demonstrate a reason to do so.
As Jeff stated, it is impossible to fully test a change of this size. I have tested it on Linux and Mac, covering all the default and simple options, singletons, and comm_spawn. That said, I'm sure others will find problems, so I'll be watching MTT results until this stabilizes.
This commit was SVN r25476.
2011-11-15 07:40:11 +04:00
|
|
|
/* unrecognized mapping directive */
|
|
|
|
|
orte_show_help("help-orte-rmaps-base.txt", "unrecognized-policy",
|
|
|
|
|
true, "mapping",
|
|
|
|
|
orte_rmaps_base_print_mapping(jdata->map->mapping));
|
|
|
|
|
rc = ORTE_ERR_SILENT;
|
|
|
|
|
goto error;
|
(copied from a mail that has a lengthy description of this commit)
I spoke with Tim about this the other day -- he gave me the green
light to go ahead with this, but it turned into a bigger job than I
thought it would be. I revamped how the default RAS scheduling and
round_robin RMAPS mapping occurs. The previous algorithms were pretty
brain dead, and ignored the "slots" and "max_slots" tokens in
hostfiles. I considered this a big enough problem to fix it for the
beta (because there is currently no way to control where processes are
launched on SMPs).
There's still some more bells and whistles that I'd like to implement,
but there's no hurry, and they can go on the trunk at any time. My
patches below are for what I considered "essential", and do the
following:
- honor the "slots" and "max-slots" tokens in the hostfile (and all
their synonyms), meaning that we allocate/map until we fill slots,
and if there are still more processes to allocate/map, we keep going
until we fill max-slots (i.e., only oversubscribe a node if we have
to).
- offer two different algorithms, currently supported by two new
options to orterun. Remember that there are two parts here -- slot
allocation and process mapping. Slot allocation controls how many
processes we'll be running on a node. After that decision has been
made, process mapping effectively controls where the ranks of
MPI_COMM_WORLD (MCW) are placed. Some of the examples given below
don't make sense unless you remember that there is a difference
between the two (which makes total sense, but you have to think
about it in terms of both things):
1. "-bynode": allocates/maps one process per node in a round-robin
fashion until all slots on the node are taken. If we still have more
processes after all slots are taken, then keep going until all
max-slots are taken. Examples:
- The hostfile:
eddie slots=2 max-slots=4
vogon slots=4 max-slots=8
- orterun -bynode -np 6 -hostfile hostfile a.out
eddie: MCW ranks 0, 2
vogon: MCW ranks 1, 3, 4, 5
- orterun -bynode -np 8 -hostfile hostfile a.out
eddie: MCW ranks 0, 2, 4
vogon: MCW ranks 1, 3, 5, 6, 7
-> the algorithm oversubscribes all nodes "equally" (until each
node's max_slots is hit, of course)
- orterun -bynode -np 12 -hostfile hostfile a.out
eddie: MCW ranks 0, 2, 4, 6
vogon: MCW ranks 1, 3, 5, 7, 8, 9, 10, 11
2. "-byslot" (this is the default if you don't specify -bynode):
greedily takes all available slots on a node for a job before moving
on to the next node. If we still have processes to allocate/schedule,
then oversubscribe all nodes equally (i.e., go round robin on all
nodes until each node's max_slots is hit). Examples:
- The hostfile
eddie slots=2 max-slots=4
vogon slots=4 max-slots=8
- orterun -np 6 -hostfile hostfile a.out
eddie: MCW ranks 0, 1
vogon: MCW ranks 2, 3, 4, 5
- orterun -np 8 -hostfile hostfile a.out
eddie: MCW ranks 0, 1, 2
vogon: MCW ranks 3, 4, 5, 6, 7
-> the algorithm oversubscribes all nodes "equally" (until max_slots
is hit)
- orterun -np 12 -hostfile hostfile a.out
eddie: MCW ranks 0, 1, 2, 3
vogon: MCW ranks 4, 5, 6, 7, 8, 9, 10, 11
The above examples are fairly contrived, and it's not clear from them
that you can get different allocation answers in all cases (the
mapping differences are obvious). Consider the following allocation
example:
- The hostfile
eddie count=4
vogon count=4
earth count=4
deep-thought count=4
- orterun -np 8 -hostfile hostfile a.out
eddie: 4 slots will be allocated
vogon: 4 slots will be allocated
earth: no slots allocated
deep-thought: no slots allocated
- orterun -bynode -np 8 -hostfile hostfile a.out
eddie: 2 slots will be allocated
vogon: 2 slots will be allocated
earth: 2 slots will be allocated
deep-thought: 2 slots will be allocated
This commit was SVN r5894.
2005-05-31 20:36:53 +04:00
|
|
|
}
|
|
|
|
|
if (ORTE_SUCCESS != rc) {
|
2006-07-10 18:10:21 +04:00
|
|
|
ORTE_ERROR_LOG(rc);
|
2008-02-28 04:57:57 +03:00
|
|
|
goto error;
|
2005-03-14 23:57:21 +03:00
|
|
|
}
|
2006-07-11 06:59:23 +04:00
|
|
|
|
2011-12-06 21:28:29 +04:00
|
|
|
/* track the total number of processes we mapped - must update
|
|
|
|
|
* this value AFTER we compute vpids so that computation
|
|
|
|
|
* is done correctly
|
|
|
|
|
*/
|
2009-10-02 07:16:18 +04:00
|
|
|
jdata->num_procs += app->num_procs;
|
2011-12-06 21:28:29 +04:00
|
|
|
|
2008-02-28 04:57:57 +03:00
|
|
|
/* cleanup the node list - it can differ from one app_context
|
|
|
|
|
* to another, so we have to get it every time
|
2006-10-19 22:57:29 +04:00
|
|
|
*/
|
At long last, the fabled revision to the affinity system has arrived. A more detailed explanation of how this all works will be presented here:
https://svn.open-mpi.org/trac/ompi/wiki/ProcessPlacement
The wiki page is incomplete at the moment, but I hope to complete it over the next few days. I will provide updates on the devel list. As the wiki page states, the default and most commonly used options remain unchanged (except as noted below). New, esoteric and complex options have been added, but unless you are a true masochist, you are unlikely to use many of them beyond perhaps an initial curiosity-motivated experimentation.
In a nutshell, this commit revamps the map/rank/bind procedure to take into account topology info on the compute nodes. I have, for the most part, preserved the default behaviors, with three notable exceptions:
1. I have at long last bowed my head in submission to the system admin's of managed clusters. For years, they have complained about our default of allowing users to oversubscribe nodes - i.e., to run more processes on a node than allocated slots. Accordingly, I have modified the default behavior: if you are running off of hostfile/dash-host allocated nodes, then the default is to allow oversubscription. If you are running off of RM-allocated nodes, then the default is to NOT allow oversubscription. Flags to override these behaviors are provided, so this only affects the default behavior.
2. both cpus/rank and stride have been removed. The latter was demanded by those who didn't understand the purpose behind it - and I agreed as the users who requested it are no longer using it. The former was removed temporarily pending implementation.
3. vm launch is now the sole method for starting OMPI. It was just too darned hard to maintain multiple launch procedures - maybe someday, provided someone can demonstrate a reason to do so.
As Jeff stated, it is impossible to fully test a change of this size. I have tested it on Linux and Mac, covering all the default and simple options, singletons, and comm_spawn. That said, I'm sure others will find problems, so I'll be watching MTT results until this stabilizes.
This commit was SVN r25476.
2011-11-15 07:40:11 +04:00
|
|
|
while (NULL != (item = opal_list_remove_first(&node_list))) {
|
2008-02-28 04:57:57 +03:00
|
|
|
OBJ_RELEASE(item);
|
2006-10-20 23:00:17 +04:00
|
|
|
}
|
2008-02-28 04:57:57 +03:00
|
|
|
OBJ_DESTRUCT(&node_list);
|
2005-03-14 23:57:21 +03:00
|
|
|
}
|
2006-07-10 18:10:21 +04:00
|
|
|
|
2008-02-28 04:57:57 +03:00
|
|
|
return ORTE_SUCCESS;
|
|
|
|
|
|
2011-02-16 02:24:31 +03:00
|
|
|
error:
|
2008-02-28 04:57:57 +03:00
|
|
|
while(NULL != (item = opal_list_remove_first(&node_list))) {
|
2005-03-14 23:57:21 +03:00
|
|
|
OBJ_RELEASE(item);
|
|
|
|
|
}
|
2008-02-28 04:57:57 +03:00
|
|
|
OBJ_DESTRUCT(&node_list);
|
2007-03-29 04:50:56 +04:00
|
|
|
|
2005-03-14 23:57:21 +03:00
|
|
|
return rc;
|
|
|
|
|
}
|
|
|
|
|
|
2017-05-13 02:16:47 +03:00
|
|
|
static int orte_rmaps_rr_assign_locations(orte_job_t *jdata)
|
|
|
|
|
{
|
|
|
|
|
mca_base_component_t *c = &mca_rmaps_round_robin_component.base_version;
|
|
|
|
|
int rc;
|
|
|
|
|
|
|
|
|
|
if (NULL == jdata->map->last_mapper ||
|
|
|
|
|
0 != strcasecmp(jdata->map->last_mapper, c->mca_component_name)) {
|
|
|
|
|
/* a mapper has been specified, and it isn't me */
|
|
|
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
|
|
|
"mca:rmaps:rr: job %s not using rr mapper",
|
|
|
|
|
ORTE_JOBID_PRINT(jdata->jobid));
|
|
|
|
|
return ORTE_ERR_TAKE_NEXT_OPTION;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
|
|
|
"mca:rmaps:rr: assign locations for job %s",
|
|
|
|
|
ORTE_JOBID_PRINT(jdata->jobid));
|
|
|
|
|
|
|
|
|
|
/* if the mapping directive was byslot or bynode, then we
|
|
|
|
|
* assign locations to the root object level */
|
|
|
|
|
if (ORTE_MAPPING_BYNODE == ORTE_GET_MAPPING_POLICY(jdata->map->mapping) ||
|
|
|
|
|
ORTE_MAPPING_BYSLOT == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
|
|
|
|
|
return orte_rmaps_rr_assign_root_level(jdata);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
/* otherwise, assign by object */
|
|
|
|
|
if (ORTE_MAPPING_BYHWTHREAD == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
|
|
|
|
|
rc = orte_rmaps_rr_assign_byobj(jdata, HWLOC_OBJ_PU, 0);
|
|
|
|
|
if (ORTE_ERR_NOT_FOUND == rc) {
|
|
|
|
|
/* if the mapper couldn't assign by this object because
|
|
|
|
|
* it isn't available, but the error allows us to try
|
|
|
|
|
* byslot, then do so
|
|
|
|
|
*/
|
|
|
|
|
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYSLOT);
|
|
|
|
|
rc = orte_rmaps_rr_assign_root_level(jdata);
|
|
|
|
|
}
|
|
|
|
|
} else if (ORTE_MAPPING_BYCORE == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
|
|
|
|
|
rc = orte_rmaps_rr_assign_byobj(jdata, HWLOC_OBJ_CORE, 0);
|
|
|
|
|
if (ORTE_ERR_NOT_FOUND == rc) {
|
|
|
|
|
/* if the mapper couldn't map by this object because
|
|
|
|
|
* it isn't available, but the error allows us to try
|
|
|
|
|
* byslot, then do so
|
|
|
|
|
*/
|
|
|
|
|
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYSLOT);
|
|
|
|
|
rc = orte_rmaps_rr_assign_root_level(jdata);
|
|
|
|
|
}
|
|
|
|
|
} else if (ORTE_MAPPING_BYL1CACHE == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
|
|
|
|
|
rc = orte_rmaps_rr_assign_byobj(jdata, HWLOC_OBJ_CACHE, 1);
|
|
|
|
|
if (ORTE_ERR_NOT_FOUND == rc) {
|
|
|
|
|
/* if the mapper couldn't map by this object because
|
|
|
|
|
* it isn't available, but the error allows us to try
|
|
|
|
|
* byslot, then do so
|
|
|
|
|
*/
|
|
|
|
|
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYSLOT);
|
|
|
|
|
rc = orte_rmaps_rr_assign_root_level(jdata);
|
|
|
|
|
}
|
|
|
|
|
} else if (ORTE_MAPPING_BYL2CACHE == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
|
|
|
|
|
rc = orte_rmaps_rr_assign_byobj(jdata, HWLOC_OBJ_CACHE, 2);
|
|
|
|
|
if (ORTE_ERR_NOT_FOUND == rc) {
|
|
|
|
|
/* if the mapper couldn't map by this object because
|
|
|
|
|
* it isn't available, but the error allows us to try
|
|
|
|
|
* byslot, then do so
|
|
|
|
|
*/
|
|
|
|
|
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYSLOT);
|
|
|
|
|
rc = orte_rmaps_rr_assign_root_level(jdata);
|
|
|
|
|
}
|
|
|
|
|
} else if (ORTE_MAPPING_BYL3CACHE == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
|
|
|
|
|
rc = orte_rmaps_rr_assign_byobj(jdata, HWLOC_OBJ_CACHE, 3);
|
|
|
|
|
if (ORTE_ERR_NOT_FOUND == rc) {
|
|
|
|
|
/* if the mapper couldn't map by this object because
|
|
|
|
|
* it isn't available, but the error allows us to try
|
|
|
|
|
* byslot, then do so
|
|
|
|
|
*/
|
|
|
|
|
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYSLOT);
|
|
|
|
|
rc = orte_rmaps_rr_assign_root_level(jdata);
|
|
|
|
|
}
|
|
|
|
|
} else if (ORTE_MAPPING_BYSOCKET == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
|
|
|
|
|
rc = orte_rmaps_rr_assign_byobj(jdata, HWLOC_OBJ_SOCKET, 0);
|
|
|
|
|
if (ORTE_ERR_NOT_FOUND == rc) {
|
|
|
|
|
/* if the mapper couldn't map by this object because
|
|
|
|
|
* it isn't available, but the error allows us to try
|
|
|
|
|
* byslot, then do so
|
|
|
|
|
*/
|
|
|
|
|
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYSLOT);
|
|
|
|
|
rc = orte_rmaps_rr_assign_root_level(jdata);
|
|
|
|
|
}
|
|
|
|
|
} else if (ORTE_MAPPING_BYNUMA == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) {
|
|
|
|
|
rc = orte_rmaps_rr_assign_byobj(jdata, HWLOC_OBJ_NODE, 0);
|
|
|
|
|
if (ORTE_ERR_NOT_FOUND == rc) {
|
|
|
|
|
/* if the mapper couldn't map by this object because
|
|
|
|
|
* it isn't available, but the error allows us to try
|
|
|
|
|
* byslot, then do so
|
|
|
|
|
*/
|
|
|
|
|
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYSLOT);
|
|
|
|
|
rc = orte_rmaps_rr_assign_root_level(jdata);
|
|
|
|
|
}
|
|
|
|
|
} else {
|
|
|
|
|
/* unrecognized mapping directive */
|
|
|
|
|
orte_show_help("help-orte-rmaps-base.txt", "unrecognized-policy",
|
|
|
|
|
true, "mapping",
|
|
|
|
|
orte_rmaps_base_print_mapping(jdata->map->mapping));
|
|
|
|
|
rc = ORTE_ERR_SILENT;
|
|
|
|
|
}
|
|
|
|
|
return rc;
|
|
|
|
|
}
|
|
|
|
|
|
2005-03-14 23:57:21 +03:00
|
|
|
orte_rmaps_base_module_t orte_rmaps_round_robin_module = {
|
2017-05-13 02:16:47 +03:00
|
|
|
.map_job = orte_rmaps_rr_map,
|
|
|
|
|
.assign_locations = orte_rmaps_rr_assign_locations
|
2005-03-14 23:57:21 +03:00
|
|
|
};
|