877 строки
37 KiB
C
877 строки
37 KiB
C
/*
|
|
* Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana
|
|
* University Research and Technology
|
|
* Corporation. All rights reserved.
|
|
* Copyright (c) 2004-2011 The University of Tennessee and The University
|
|
* of Tennessee Research Foundation. All rights
|
|
* reserved.
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
* University of Stuttgart. All rights reserved.
|
|
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
* All rights reserved.
|
|
* Copyright (c) 2011-2017 Cisco Systems, Inc. All rights reserved
|
|
* Copyright (c) 2014-2018 Intel, Inc. All rights reserved.
|
|
* Copyright (c) 2017 Research Organization for Information Science
|
|
* and Technology (RIST). All rights reserved.
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
|
|
#include "orte_config.h"
|
|
#include "orte/constants.h"
|
|
|
|
#include <sys/types.h>
|
|
#ifdef HAVE_UNISTD_H
|
|
#include <unistd.h>
|
|
#endif /* HAVE_UNISTD_H */
|
|
#include <string.h>
|
|
|
|
#include "opal/class/opal_pointer_array.h"
|
|
#include "opal/util/if.h"
|
|
#include "opal/util/output.h"
|
|
#include "orte/mca/mca.h"
|
|
#include "opal/mca/base/base.h"
|
|
#include "opal/mca/hwloc/base/base.h"
|
|
#include "opal/threads/tsd.h"
|
|
|
|
#include "orte/types.h"
|
|
#include "orte/util/show_help.h"
|
|
#include "orte/util/name_fns.h"
|
|
#include "orte/runtime/orte_globals.h"
|
|
#include "orte/util/hostfile/hostfile.h"
|
|
#include "orte/util/dash_host/dash_host.h"
|
|
#include "orte/mca/errmgr/errmgr.h"
|
|
#include "orte/mca/ess/ess.h"
|
|
#include "orte/runtime/data_type_support/orte_dt_support.h"
|
|
|
|
#include "orte/mca/rmaps/base/rmaps_private.h"
|
|
#include "orte/mca/rmaps/base/base.h"
|
|
|
|
static int rank_span(orte_job_t *jdata,
|
|
hwloc_obj_type_t target,
|
|
unsigned cache_level)
|
|
{
|
|
orte_app_context_t *app;
|
|
hwloc_obj_t obj;
|
|
int num_objs, i, j, m, n, rc;
|
|
orte_vpid_t num_ranked=0;
|
|
orte_node_t *node;
|
|
orte_proc_t *proc, *pptr;
|
|
orte_vpid_t vpid;
|
|
int cnt;
|
|
hwloc_obj_t locale;
|
|
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
"mca:rmaps:rank_span: for job %s",
|
|
ORTE_JOBID_PRINT(jdata->jobid));
|
|
|
|
/* if the ranking is spanned, then we perform the
|
|
* ranking as if it was one big node - i.e., we
|
|
* rank one proc on each object, step to the next object
|
|
* moving across all the nodes, then wrap around to the
|
|
* first object on the first node.
|
|
*
|
|
* Node 0 Node 1
|
|
* Obj 0 Obj 1 Obj 0 Obj 1
|
|
* 0 4 1 5 2 6 3 7
|
|
* 8 12 9 13 10 14 11 15
|
|
*/
|
|
|
|
/* In the interest of getting this committed in finite time,
|
|
* just loop across the nodes and objects until all procs
|
|
* are mapped
|
|
*/
|
|
|
|
vpid = 0;
|
|
for (n=0; n < jdata->apps->size; n++) {
|
|
if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, n))) {
|
|
continue;
|
|
}
|
|
|
|
cnt = 0;
|
|
while (cnt < app->num_procs) {
|
|
for (m=0; m < jdata->map->nodes->size; m++) {
|
|
if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(jdata->map->nodes, m))) {
|
|
continue;
|
|
}
|
|
/* get the number of objects - only consider those we can actually use */
|
|
num_objs = opal_hwloc_base_get_nbobjs_by_type(node->topology->topo, target,
|
|
cache_level, OPAL_HWLOC_AVAILABLE);
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
"mca:rmaps:rank_span: found %d objects on node %s with %d procs",
|
|
num_objs, node->name, (int)node->num_procs);
|
|
if (0 == num_objs) {
|
|
return ORTE_ERR_NOT_SUPPORTED;
|
|
}
|
|
|
|
/* for each object */
|
|
for (i=0; i < num_objs && cnt < app->num_procs; i++) {
|
|
obj = opal_hwloc_base_get_obj_by_type(node->topology->topo, target,
|
|
cache_level, i, OPAL_HWLOC_AVAILABLE);
|
|
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
"mca:rmaps:rank_span: working object %d", i);
|
|
|
|
/* cycle thru the procs on this node */
|
|
for (j=0; j < node->procs->size && cnt < app->num_procs; j++) {
|
|
if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(node->procs, j))) {
|
|
continue;
|
|
}
|
|
/* ignore procs from other jobs */
|
|
if (proc->name.jobid != jdata->jobid) {
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
"mca:rmaps:rank_span skipping proc %s - from another job, num_ranked %d",
|
|
ORTE_NAME_PRINT(&proc->name), num_ranked);
|
|
continue;
|
|
}
|
|
/* ignore procs that are already assigned */
|
|
if (ORTE_VPID_INVALID != proc->name.vpid) {
|
|
continue;
|
|
}
|
|
/* ignore procs from other apps */
|
|
if (proc->app_idx != app->idx) {
|
|
continue;
|
|
}
|
|
/* protect against bozo case */
|
|
locale = NULL;
|
|
if (!orte_get_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, (void**)&locale, OPAL_PTR)) {
|
|
ORTE_ERROR_LOG(ORTE_ERROR);
|
|
return ORTE_ERROR;
|
|
}
|
|
/* ignore procs not on this object */
|
|
if (!hwloc_bitmap_intersects(obj->cpuset, locale->cpuset)) {
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
"mca:rmaps:rank_span: proc at position %d is not on object %d",
|
|
j, i);
|
|
continue;
|
|
}
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
"mca:rmaps:rank_span: assigning vpid %s", ORTE_VPID_PRINT(vpid));
|
|
proc->name.vpid = vpid++;
|
|
if (0 == cnt) {
|
|
app->first_rank = proc->name.vpid;
|
|
}
|
|
cnt++;
|
|
|
|
/* insert the proc into the jdata array */
|
|
if (NULL != (pptr = (orte_proc_t*)opal_pointer_array_get_item(jdata->procs, proc->name.vpid))) {
|
|
OBJ_RELEASE(pptr);
|
|
}
|
|
OBJ_RETAIN(proc);
|
|
if (ORTE_SUCCESS != (rc = opal_pointer_array_set_item(jdata->procs, proc->name.vpid, proc))) {
|
|
ORTE_ERROR_LOG(rc);
|
|
return rc;
|
|
}
|
|
/* track where the highest vpid landed - this is our
|
|
* new bookmark
|
|
*/
|
|
jdata->bookmark = node;
|
|
/* move to next object */
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return ORTE_SUCCESS;
|
|
}
|
|
|
|
static int rank_fill(orte_job_t *jdata,
|
|
hwloc_obj_type_t target,
|
|
unsigned cache_level)
|
|
{
|
|
orte_app_context_t *app;
|
|
hwloc_obj_t obj;
|
|
int num_objs, i, j, m, n, rc;
|
|
orte_vpid_t num_ranked=0;
|
|
orte_node_t *node;
|
|
orte_proc_t *proc, *pptr;
|
|
orte_vpid_t vpid;
|
|
int cnt;
|
|
hwloc_obj_t locale;
|
|
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
"mca:rmaps:rank_fill: for job %s",
|
|
ORTE_JOBID_PRINT(jdata->jobid));
|
|
|
|
/* if the ranking is fill, then we rank all the procs
|
|
* within a given object before moving on to the next
|
|
*
|
|
* Node 0 Node 1
|
|
* Obj 0 Obj 1 Obj 0 Obj 1
|
|
* 0 1 4 5 8 9 12 13
|
|
* 2 3 6 7 10 11 14 15
|
|
*/
|
|
|
|
vpid = 0;
|
|
for (n=0; n < jdata->apps->size; n++) {
|
|
if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, n))) {
|
|
continue;
|
|
}
|
|
|
|
cnt = 0;
|
|
for (m=0; m < jdata->map->nodes->size; m++) {
|
|
if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(jdata->map->nodes, m))) {
|
|
continue;
|
|
}
|
|
/* get the number of objects - only consider those we can actually use */
|
|
num_objs = opal_hwloc_base_get_nbobjs_by_type(node->topology->topo, target,
|
|
cache_level, OPAL_HWLOC_AVAILABLE);
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
"mca:rmaps:rank_fill: found %d objects on node %s with %d procs",
|
|
num_objs, node->name, (int)node->num_procs);
|
|
if (0 == num_objs) {
|
|
return ORTE_ERR_NOT_SUPPORTED;
|
|
}
|
|
|
|
/* for each object */
|
|
for (i=0; i < num_objs && cnt < app->num_procs; i++) {
|
|
obj = opal_hwloc_base_get_obj_by_type(node->topology->topo, target,
|
|
cache_level, i, OPAL_HWLOC_AVAILABLE);
|
|
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
"mca:rmaps:rank_fill: working object %d", i);
|
|
|
|
/* cycle thru the procs on this node */
|
|
for (j=0; j < node->procs->size && cnt < app->num_procs; j++) {
|
|
if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(node->procs, j))) {
|
|
continue;
|
|
}
|
|
/* ignore procs from other jobs */
|
|
if (proc->name.jobid != jdata->jobid) {
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
"mca:rmaps:rank_fill skipping proc %s - from another job, num_ranked %d",
|
|
ORTE_NAME_PRINT(&proc->name), num_ranked);
|
|
continue;
|
|
}
|
|
/* ignore procs that are already assigned */
|
|
if (ORTE_VPID_INVALID != proc->name.vpid) {
|
|
continue;
|
|
}
|
|
/* ignore procs from other apps */
|
|
if (proc->app_idx != app->idx) {
|
|
continue;
|
|
}
|
|
/* protect against bozo case */
|
|
locale = NULL;
|
|
if (!orte_get_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, (void**)&locale, OPAL_PTR)) {
|
|
ORTE_ERROR_LOG(ORTE_ERROR);
|
|
return ORTE_ERROR;
|
|
}
|
|
/* ignore procs not on this object */
|
|
if (!hwloc_bitmap_intersects(obj->cpuset, locale->cpuset)) {
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
"mca:rmaps:rank_fill: proc at position %d is not on object %d",
|
|
j, i);
|
|
continue;
|
|
}
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
"mca:rmaps:rank_fill: assigning vpid %s", ORTE_VPID_PRINT(vpid));
|
|
proc->name.vpid = vpid++;
|
|
if (0 == cnt) {
|
|
app->first_rank = proc->name.vpid;
|
|
}
|
|
cnt++;
|
|
|
|
/* insert the proc into the jdata array */
|
|
if (NULL != (pptr = (orte_proc_t*)opal_pointer_array_get_item(jdata->procs, proc->name.vpid))) {
|
|
OBJ_RELEASE(pptr);
|
|
}
|
|
OBJ_RETAIN(proc);
|
|
if (ORTE_SUCCESS != (rc = opal_pointer_array_set_item(jdata->procs, proc->name.vpid, proc))) {
|
|
ORTE_ERROR_LOG(rc);
|
|
return rc;
|
|
}
|
|
/* track where the highest vpid landed - this is our
|
|
* new bookmark
|
|
*/
|
|
jdata->bookmark = node;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
return ORTE_SUCCESS;
|
|
}
|
|
|
|
static int rank_by(orte_job_t *jdata,
|
|
hwloc_obj_type_t target,
|
|
unsigned cache_level)
|
|
{
|
|
orte_app_context_t *app;
|
|
hwloc_obj_t obj;
|
|
int num_objs, i, j, m, n, rc, nn;
|
|
orte_vpid_t num_ranked=0;
|
|
orte_node_t *node;
|
|
orte_proc_t *proc, *pptr;
|
|
orte_vpid_t vpid, np;
|
|
int cnt;
|
|
opal_pointer_array_t objs;
|
|
hwloc_obj_t locale;
|
|
orte_app_idx_t napp;
|
|
|
|
if (ORTE_RANKING_SPAN & ORTE_GET_RANKING_DIRECTIVE(jdata->map->ranking)) {
|
|
return rank_span(jdata, target, cache_level);
|
|
} else if (ORTE_RANKING_FILL & ORTE_GET_RANKING_DIRECTIVE(jdata->map->ranking)) {
|
|
return rank_fill(jdata, target, cache_level);
|
|
}
|
|
|
|
/* if ranking is not spanned or filled, then we
|
|
* default to assign ranks sequentially across
|
|
* target objects within a node until that node
|
|
* is fully ranked, and then move on to the next
|
|
* node
|
|
*
|
|
* Node 0 Node 1
|
|
* Obj 0 Obj 1 Obj 0 Obj 1
|
|
* 0 2 1 3 8 10 9 11
|
|
* 4 6 5 7 12 14 13 15
|
|
*/
|
|
|
|
vpid = 0;
|
|
for (n=0, napp=0; napp < jdata->num_apps && n < jdata->apps->size; n++) {
|
|
if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, n))) {
|
|
continue;
|
|
}
|
|
napp++;
|
|
/* setup the pointer array */
|
|
OBJ_CONSTRUCT(&objs, opal_pointer_array_t);
|
|
opal_pointer_array_init(&objs, 2, INT_MAX, 2);
|
|
|
|
cnt = 0;
|
|
for (m=0, nn=0; nn < jdata->map->num_nodes && m < jdata->map->nodes->size; m++) {
|
|
if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(jdata->map->nodes, m))) {
|
|
continue;
|
|
}
|
|
nn++;
|
|
|
|
/* get the number of objects - only consider those we can actually use */
|
|
num_objs = opal_hwloc_base_get_nbobjs_by_type(node->topology->topo, target,
|
|
cache_level, OPAL_HWLOC_AVAILABLE);
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
"mca:rmaps:rank_by: found %d objects on node %s with %d procs",
|
|
num_objs, node->name, (int)node->num_procs);
|
|
if (0 == num_objs) {
|
|
OBJ_DESTRUCT(&objs);
|
|
return ORTE_ERR_NOT_SUPPORTED;
|
|
}
|
|
/* collect all the objects */
|
|
for (i=0; i < num_objs; i++) {
|
|
obj = opal_hwloc_base_get_obj_by_type(node->topology->topo, target,
|
|
cache_level, i, OPAL_HWLOC_AVAILABLE);
|
|
opal_pointer_array_set_item(&objs, i, obj);
|
|
}
|
|
|
|
/* cycle across the objects, assigning a proc to each one,
|
|
* until all procs have been assigned - unfortunately, since
|
|
* more than this job may be mapped onto a node, the number
|
|
* of procs on the node can't be used to tell us when we
|
|
* are done. Instead, we have to just keep going until all
|
|
* procs are ranked - which means we have to make one extra
|
|
* pass thru the loop
|
|
*
|
|
* Perhaps someday someone will come up with a more efficient
|
|
* algorithm, but this works for now.
|
|
*/
|
|
i = 0;
|
|
while (cnt < app->num_procs && i < (int)node->num_procs) {
|
|
/* get the next object */
|
|
obj = (hwloc_obj_t)opal_pointer_array_get_item(&objs, i % num_objs);
|
|
if (NULL == obj) {
|
|
break;
|
|
}
|
|
/* scan across the procs and find the one that is on this object */
|
|
np = 0;
|
|
for (j=0; np < node->num_procs && j < node->procs->size && cnt < app->num_procs; j++) {
|
|
if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(node->procs, j))) {
|
|
continue;
|
|
}
|
|
np++;
|
|
/* ignore procs from other jobs */
|
|
if (proc->name.jobid != jdata->jobid) {
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
"mca:rmaps:rank_by skipping proc %s - from another job, num_ranked %d",
|
|
ORTE_NAME_PRINT(&proc->name), num_ranked);
|
|
continue;
|
|
}
|
|
/* ignore procs that are already ranked */
|
|
if (ORTE_VPID_INVALID != proc->name.vpid) {
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
"mca:rmaps:rank_by skipping proc %s - already ranked, num_ranked %d",
|
|
ORTE_NAME_PRINT(&proc->name), num_ranked);
|
|
continue;
|
|
}
|
|
/* ignore procs from other apps */
|
|
if (proc->app_idx != app->idx) {
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
"mca:rmaps:rank_by skipping proc %s - from another app, num_ranked %d",
|
|
ORTE_NAME_PRINT(&proc->name), num_ranked);
|
|
continue;
|
|
}
|
|
/* protect against bozo case */
|
|
locale = NULL;
|
|
if (!orte_get_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, (void**)&locale, OPAL_PTR)) {
|
|
ORTE_ERROR_LOG(ORTE_ERROR);
|
|
return ORTE_ERROR;
|
|
}
|
|
/* ignore procs not on this object */
|
|
if (NULL == locale ||
|
|
!hwloc_bitmap_intersects(obj->cpuset, locale->cpuset)) {
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
"mca:rmaps:rank_by: proc at position %d is not on object %d",
|
|
j, i);
|
|
continue;
|
|
}
|
|
/* assign the vpid */
|
|
proc->name.vpid = vpid++;
|
|
if (0 == cnt) {
|
|
app->first_rank = proc->name.vpid;
|
|
}
|
|
cnt++;
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
"mca:rmaps:rank_by: proc in position %d is on object %d assigned rank %s",
|
|
j, i, ORTE_VPID_PRINT(proc->name.vpid));
|
|
/* insert the proc into the jdata array */
|
|
if (NULL != (pptr = (orte_proc_t*)opal_pointer_array_get_item(jdata->procs, proc->name.vpid))) {
|
|
OBJ_RELEASE(pptr);
|
|
}
|
|
OBJ_RETAIN(proc);
|
|
if (ORTE_SUCCESS != (rc = opal_pointer_array_set_item(jdata->procs, proc->name.vpid, proc))) {
|
|
ORTE_ERROR_LOG(rc);
|
|
OBJ_DESTRUCT(&objs);
|
|
return rc;
|
|
}
|
|
num_ranked++;
|
|
/* track where the highest vpid landed - this is our
|
|
* new bookmark
|
|
*/
|
|
jdata->bookmark = node;
|
|
/* move to next object */
|
|
break;
|
|
}
|
|
i++;
|
|
}
|
|
}
|
|
/* cleanup */
|
|
OBJ_DESTRUCT(&objs);
|
|
}
|
|
return ORTE_SUCCESS;
|
|
}
|
|
|
|
int orte_rmaps_base_compute_vpids(orte_job_t *jdata)
|
|
{
|
|
orte_job_map_t *map;
|
|
orte_app_context_t *app;
|
|
orte_vpid_t vpid;
|
|
int j, m, n, cnt;
|
|
orte_node_t *node;
|
|
orte_proc_t *proc, *pptr;
|
|
int rc;
|
|
bool one_found;
|
|
hwloc_obj_type_t target;
|
|
unsigned cache_level;
|
|
|
|
map = jdata->map;
|
|
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
"RANKING POLICY: %s", orte_rmaps_base_print_ranking(map->ranking));
|
|
|
|
/* start with the rank-by object options - if the object isn't
|
|
* included in the topology, then we obviously cannot rank by it.
|
|
* However, if this was the default ranking policy (as opposed to
|
|
* something given by the user), then fall back to rank-by slot
|
|
*/
|
|
if (ORTE_RANK_BY_NUMA == ORTE_GET_RANKING_POLICY(map->ranking)) {
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
"mca:rmaps: computing ranks by NUMA for job %s",
|
|
ORTE_JOBID_PRINT(jdata->jobid));
|
|
if (ORTE_SUCCESS != (rc = rank_by(jdata, HWLOC_OBJ_NODE, 0))) {
|
|
if (ORTE_ERR_NOT_SUPPORTED == rc &&
|
|
!(ORTE_RANKING_GIVEN & ORTE_GET_RANKING_DIRECTIVE(map->ranking))) {
|
|
ORTE_SET_RANKING_POLICY(map->ranking, ORTE_RANK_BY_SLOT);
|
|
goto rankbyslot;
|
|
}
|
|
ORTE_ERROR_LOG(rc);
|
|
}
|
|
return rc;
|
|
}
|
|
|
|
if (ORTE_RANK_BY_SOCKET == ORTE_GET_RANKING_POLICY(map->ranking)) {
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
"mca:rmaps: computing ranks by socket for job %s",
|
|
ORTE_JOBID_PRINT(jdata->jobid));
|
|
if (ORTE_SUCCESS != (rc = rank_by(jdata, HWLOC_OBJ_SOCKET, 0))) {
|
|
if (ORTE_ERR_NOT_SUPPORTED == rc &&
|
|
!(ORTE_RANKING_GIVEN & ORTE_GET_RANKING_DIRECTIVE(map->ranking))) {
|
|
ORTE_SET_RANKING_POLICY(map->ranking, ORTE_RANK_BY_SLOT);
|
|
goto rankbyslot;
|
|
}
|
|
ORTE_ERROR_LOG(rc);
|
|
}
|
|
return rc;
|
|
}
|
|
|
|
if (ORTE_RANK_BY_L3CACHE == ORTE_GET_RANKING_POLICY(map->ranking)) {
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
"mca:rmaps: computing ranks by L3cache for job %s",
|
|
ORTE_JOBID_PRINT(jdata->jobid));
|
|
OPAL_HWLOC_MAKE_OBJ_CACHE(3, target, cache_level);
|
|
if (ORTE_SUCCESS != (rc = rank_by(jdata, target, cache_level))) {
|
|
if (ORTE_ERR_NOT_SUPPORTED == rc &&
|
|
!(ORTE_RANKING_GIVEN & ORTE_GET_RANKING_DIRECTIVE(map->ranking))) {
|
|
ORTE_SET_RANKING_POLICY(map->ranking, ORTE_RANK_BY_SLOT);
|
|
goto rankbyslot;
|
|
}
|
|
ORTE_ERROR_LOG(rc);
|
|
}
|
|
return rc;
|
|
}
|
|
|
|
if (ORTE_RANK_BY_L2CACHE == ORTE_GET_RANKING_POLICY(map->ranking)) {
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
"mca:rmaps: computing ranks by L2cache for job %s",
|
|
ORTE_JOBID_PRINT(jdata->jobid));
|
|
OPAL_HWLOC_MAKE_OBJ_CACHE(2, target, cache_level);
|
|
if (ORTE_SUCCESS != (rc = rank_by(jdata, target, cache_level))) {
|
|
if (ORTE_ERR_NOT_SUPPORTED == rc &&
|
|
!(ORTE_RANKING_GIVEN & ORTE_GET_RANKING_DIRECTIVE(map->ranking))) {
|
|
ORTE_SET_RANKING_POLICY(map->ranking, ORTE_RANK_BY_SLOT);
|
|
goto rankbyslot;
|
|
}
|
|
ORTE_ERROR_LOG(rc);
|
|
}
|
|
return rc;
|
|
}
|
|
|
|
if (ORTE_RANK_BY_L1CACHE == ORTE_GET_RANKING_POLICY(map->ranking)) {
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
"mca:rmaps: computing ranks by L1cache for job %s",
|
|
ORTE_JOBID_PRINT(jdata->jobid));
|
|
OPAL_HWLOC_MAKE_OBJ_CACHE(1, target, cache_level);
|
|
if (ORTE_SUCCESS != (rc = rank_by(jdata, target, cache_level))) {
|
|
if (ORTE_ERR_NOT_SUPPORTED == rc &&
|
|
!(ORTE_RANKING_GIVEN & ORTE_GET_RANKING_DIRECTIVE(map->ranking))) {
|
|
ORTE_SET_RANKING_POLICY(map->ranking, ORTE_RANK_BY_SLOT);
|
|
goto rankbyslot;
|
|
}
|
|
ORTE_ERROR_LOG(rc);
|
|
}
|
|
return rc;
|
|
}
|
|
|
|
if (ORTE_RANK_BY_CORE == ORTE_GET_RANKING_POLICY(map->ranking)) {
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
"mca:rmaps: computing ranks by core for job %s",
|
|
ORTE_JOBID_PRINT(jdata->jobid));
|
|
if (ORTE_SUCCESS != (rc = rank_by(jdata, HWLOC_OBJ_CORE, 0))) {
|
|
if (ORTE_ERR_NOT_SUPPORTED == rc &&
|
|
!(ORTE_RANKING_GIVEN & ORTE_GET_RANKING_DIRECTIVE(map->ranking))) {
|
|
ORTE_SET_RANKING_POLICY(map->ranking, ORTE_RANK_BY_SLOT);
|
|
goto rankbyslot;
|
|
}
|
|
ORTE_ERROR_LOG(rc);
|
|
}
|
|
return rc;
|
|
}
|
|
|
|
if (ORTE_RANK_BY_HWTHREAD == ORTE_GET_RANKING_POLICY(map->ranking)) {
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
"mca:rmaps: computing ranks by hwthread for job %s",
|
|
ORTE_JOBID_PRINT(jdata->jobid));
|
|
if (ORTE_SUCCESS != (rc = rank_by(jdata, HWLOC_OBJ_PU, 0))) {
|
|
if (ORTE_ERR_NOT_SUPPORTED == rc &&
|
|
!(ORTE_RANKING_GIVEN & ORTE_GET_RANKING_DIRECTIVE(map->ranking))) {
|
|
ORTE_SET_RANKING_POLICY(map->ranking, ORTE_RANK_BY_SLOT);
|
|
goto rankbyslot;
|
|
}
|
|
ORTE_ERROR_LOG(rc);
|
|
}
|
|
return rc;
|
|
}
|
|
|
|
if (ORTE_RANK_BY_NODE == ORTE_GET_RANKING_POLICY(map->ranking) ||
|
|
ORTE_RANK_BY_BOARD == ORTE_GET_RANKING_POLICY(map->ranking)) {
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
"mca:rmaps:base: computing vpids by node for job %s",
|
|
ORTE_JOBID_PRINT(jdata->jobid));
|
|
/* assign the ranks round-robin across nodes - only one board/node
|
|
* at this time, so they are equivalent
|
|
*/
|
|
vpid=0;
|
|
for (n=0; n < jdata->apps->size; n++) {
|
|
if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, n))) {
|
|
continue;
|
|
}
|
|
cnt=0;
|
|
one_found = true;
|
|
while (cnt < app->num_procs && one_found) {
|
|
one_found = false;
|
|
for (m=0; m < jdata->map->nodes->size; m++) {
|
|
if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(jdata->map->nodes, m))) {
|
|
continue;
|
|
}
|
|
for (j=0; j < node->procs->size; j++) {
|
|
if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(node->procs, j))) {
|
|
continue;
|
|
}
|
|
/* ignore procs from other jobs */
|
|
if (proc->name.jobid != jdata->jobid) {
|
|
continue;
|
|
}
|
|
/* ignore procs from other apps */
|
|
if (proc->app_idx != app->idx) {
|
|
continue;
|
|
}
|
|
if (ORTE_VPID_INVALID != proc->name.vpid) {
|
|
continue;
|
|
}
|
|
proc->name.vpid = vpid++;
|
|
/* insert the proc into the jdata array */
|
|
if (NULL != (pptr = (orte_proc_t*)opal_pointer_array_get_item(jdata->procs, proc->name.vpid))) {
|
|
OBJ_RELEASE(pptr);
|
|
}
|
|
OBJ_RETAIN(proc);
|
|
if (ORTE_SUCCESS != (rc = opal_pointer_array_set_item(jdata->procs, proc->name.vpid, proc))) {
|
|
ORTE_ERROR_LOG(rc);
|
|
return rc;
|
|
}
|
|
cnt++;
|
|
one_found = true;
|
|
/* track where the highest vpid landed - this is our
|
|
* new bookmark
|
|
*/
|
|
jdata->bookmark = node;
|
|
break; /* move on to next node */
|
|
}
|
|
}
|
|
}
|
|
if (cnt < app->num_procs) {
|
|
ORTE_ERROR_LOG(ORTE_ERR_FATAL);
|
|
return ORTE_ERR_FATAL;
|
|
}
|
|
}
|
|
return ORTE_SUCCESS;
|
|
}
|
|
|
|
rankbyslot:
|
|
if (ORTE_RANK_BY_SLOT == ORTE_GET_RANKING_POLICY(map->ranking)) {
|
|
/* assign the ranks sequentially */
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
"mca:rmaps:base: computing vpids by slot for job %s",
|
|
ORTE_JOBID_PRINT(jdata->jobid));
|
|
vpid = 0;
|
|
for (n=0; n < jdata->apps->size; n++) {
|
|
if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, n))) {
|
|
continue;
|
|
}
|
|
for (m=0; m < jdata->map->nodes->size; m++) {
|
|
if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(jdata->map->nodes, m))) {
|
|
continue;
|
|
}
|
|
|
|
for (j=0; j < node->procs->size; j++) {
|
|
if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(node->procs, j))) {
|
|
continue;
|
|
}
|
|
/* ignore procs from other jobs */
|
|
if (proc->name.jobid != jdata->jobid) {
|
|
continue;
|
|
}
|
|
/* ignore procs from other apps */
|
|
if (proc->app_idx != app->idx) {
|
|
continue;
|
|
}
|
|
if (ORTE_VPID_INVALID == proc->name.vpid) {
|
|
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
|
|
"mca:rmaps:base: assigning rank %s to node %s",
|
|
ORTE_VPID_PRINT(vpid), node->name);
|
|
proc->name.vpid = vpid++;
|
|
/* track where the highest vpid landed - this is our
|
|
* new bookmark
|
|
*/
|
|
jdata->bookmark = node;
|
|
}
|
|
/* insert the proc into the jdata array */
|
|
if (NULL != (pptr = (orte_proc_t*)opal_pointer_array_get_item(jdata->procs, proc->name.vpid))) {
|
|
OBJ_RELEASE(pptr);
|
|
}
|
|
OBJ_RETAIN(proc);
|
|
if (ORTE_SUCCESS != (rc = opal_pointer_array_set_item(jdata->procs, proc->name.vpid, proc))) {
|
|
ORTE_ERROR_LOG(rc);
|
|
return rc;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return ORTE_SUCCESS;
|
|
}
|
|
|
|
return ORTE_ERR_NOT_IMPLEMENTED;
|
|
}
|
|
|
|
int orte_rmaps_base_compute_local_ranks(orte_job_t *jdata)
|
|
{
|
|
orte_std_cntr_t i;
|
|
int j, k;
|
|
orte_node_t *node;
|
|
orte_proc_t *proc, *psave, *psave2;
|
|
orte_vpid_t minv, minv2;
|
|
orte_local_rank_t local_rank;
|
|
orte_job_map_t *map;
|
|
orte_app_context_t *app;
|
|
|
|
OPAL_OUTPUT_VERBOSE((5, orte_rmaps_base_framework.framework_output,
|
|
"%s rmaps:base:compute_usage",
|
|
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
|
|
|
|
/* point to map */
|
|
map = jdata->map;
|
|
|
|
/* for each node in the map... */
|
|
for (i=0; i < map->nodes->size; i++) {
|
|
/* cycle through the array of procs on this node, setting
|
|
* local and node ranks, until we
|
|
* have done so for all procs on nodes in this map
|
|
*/
|
|
if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(map->nodes, i))) {
|
|
continue;
|
|
}
|
|
|
|
/* init search values */
|
|
local_rank = 0;
|
|
|
|
/* the proc map may have holes in it, so cycle
|
|
* all the way through and avoid the holes
|
|
*/
|
|
for (k=0; k < node->procs->size; k++) {
|
|
/* if this proc is NULL, skip it */
|
|
if (NULL == opal_pointer_array_get_item(node->procs, k)) {
|
|
continue;
|
|
}
|
|
minv = ORTE_VPID_MAX;
|
|
minv2 = ORTE_VPID_MAX;
|
|
psave = NULL;
|
|
psave2 = NULL;
|
|
/* find the minimum vpid proc */
|
|
for (j=0; j < node->procs->size; j++) {
|
|
/* if this proc is NULL, skip it */
|
|
if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(node->procs, j))) {
|
|
continue;
|
|
}
|
|
/* only look at procs for this job when
|
|
* determining local rank
|
|
*/
|
|
if (proc->name.jobid == jdata->jobid &&
|
|
ORTE_LOCAL_RANK_INVALID == proc->local_rank &&
|
|
proc->name.vpid < minv) {
|
|
minv = proc->name.vpid;
|
|
psave = proc;
|
|
}
|
|
/* no matter what job...still have to handle node_rank */
|
|
if (ORTE_NODE_RANK_INVALID == proc->node_rank &&
|
|
proc->name.vpid < minv2) {
|
|
minv2 = proc->name.vpid;
|
|
psave2 = proc;
|
|
}
|
|
}
|
|
if (NULL == psave && NULL == psave2) {
|
|
/* we must have processed them all for this node! */
|
|
break;
|
|
}
|
|
if (NULL != psave) {
|
|
psave->local_rank = local_rank;
|
|
++local_rank;
|
|
}
|
|
if (NULL != psave2) {
|
|
psave2->node_rank = node->next_node_rank;
|
|
node->next_node_rank++;
|
|
}
|
|
}
|
|
}
|
|
|
|
/* compute app_rank */
|
|
for (i=0; i < jdata->apps->size; i++) {
|
|
if (NULL == (app = (orte_app_context_t*)opal_pointer_array_get_item(jdata->apps, i))) {
|
|
continue;
|
|
}
|
|
k=0;
|
|
/* loop thru all procs in job to find those from this app_context */
|
|
for (j=0; j < jdata->procs->size; j++) {
|
|
if (NULL == (proc = (orte_proc_t*)opal_pointer_array_get_item(jdata->procs, j))) {
|
|
continue;
|
|
}
|
|
if (proc->app_idx != app->idx) {
|
|
continue;
|
|
}
|
|
proc->app_rank = k++;
|
|
}
|
|
}
|
|
|
|
return ORTE_SUCCESS;
|
|
}
|
|
|
|
/* when we restart a process on a different node, we have to
|
|
* ensure that the node and local ranks assigned to the proc
|
|
* don't overlap with any pre-existing proc on that node. If
|
|
* we don't, then it would be possible for procs to conflict
|
|
* when opening static ports, should that be enabled.
|
|
*/
|
|
void orte_rmaps_base_update_local_ranks(orte_job_t *jdata, orte_node_t *oldnode,
|
|
orte_node_t *newnode, orte_proc_t *newproc)
|
|
{
|
|
int k;
|
|
orte_node_rank_t node_rank;
|
|
orte_local_rank_t local_rank;
|
|
orte_proc_t *proc;
|
|
|
|
OPAL_OUTPUT_VERBOSE((5, orte_rmaps_base_framework.framework_output,
|
|
"%s rmaps:base:update_usage",
|
|
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
|
|
|
|
/* if the node hasn't changed, then we can just use the
|
|
* pre-defined values
|
|
*/
|
|
if (oldnode == newnode) {
|
|
return;
|
|
}
|
|
|
|
/* if the node has changed, then search the new node for the
|
|
* lowest unused local and node rank
|
|
*/
|
|
node_rank = 0;
|
|
retry_nr:
|
|
for (k=0; k < newnode->procs->size; k++) {
|
|
/* if this proc is NULL, skip it */
|
|
if (NULL == (proc = (orte_proc_t *) opal_pointer_array_get_item(newnode->procs, k))) {
|
|
continue;
|
|
}
|
|
if (node_rank == proc->node_rank) {
|
|
node_rank++;
|
|
goto retry_nr;
|
|
}
|
|
}
|
|
newproc->node_rank = node_rank;
|
|
|
|
local_rank = 0;
|
|
retry_lr:
|
|
for (k=0; k < newnode->procs->size; k++) {
|
|
/* if this proc is NULL, skip it */
|
|
if (NULL == (proc = (orte_proc_t *) opal_pointer_array_get_item(newnode->procs, k))) {
|
|
continue;
|
|
}
|
|
/* ignore procs from other jobs */
|
|
if (proc->name.jobid != jdata->jobid) {
|
|
continue;
|
|
}
|
|
if (local_rank == proc->local_rank) {
|
|
local_rank++;
|
|
goto retry_lr;
|
|
}
|
|
}
|
|
newproc->local_rank = local_rank;
|
|
}
|