ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

rmaps/mindist: reworked the job map binding

Просмотр исходного кода 
The following issues have been fixed for `mindist`:
- computing the job map on the backend nodes
- using slots count (`-host node1:<s1>,nodeN:<sN>`)
- fixed `dist:span` job mapping method
- fixed `oversubcribe` option with `-host`

Signed-off-by: Boris Karasev <karasev.b@gmail.com>
Этот коммит содержится в:
Boris Karasev 2017-11-06 00:02:27 +02:00
родитель 8a9ef3dc2d
Коммит d2a568afa5
1 изменённых файлов: 214 добавлений и 205 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

419
orte/mca/rmaps/mindist/rmaps_mindist_module.c
Просмотреть файл

@ -15,6 +15,8 @@
* Copyright (c) 2014-2017 Intel, Inc. All rights reserved. * Copyright (c) 2014-2017 Intel, Inc. All rights reserved.
* Copyright (c) 2014 Research Organization for Information Science * Copyright (c) 2014 Research Organization for Information Science
* and Technology (RIST). All rights reserved. * and Technology (RIST). All rights reserved.
* Copyright (c) 2017 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$ * $COPYRIGHT$
* *
* Additional copyrights may follow * Additional copyrights may follow
@ -43,9 +45,11 @@
#include "orte/mca/rmaps/mindist/rmaps_mindist.h" #include "orte/mca/rmaps/mindist/rmaps_mindist.h"
static int mindist_map(orte_job_t *jdata); static int mindist_map(orte_job_t *jdata);
static int assign_locations(orte_job_t *jdata);
orte_rmaps_base_module_t orte_rmaps_mindist_module = { orte_rmaps_base_module_t orte_rmaps_mindist_module = {
.map_job = mindist_map .map_job = mindist_map,
.assign_locations = assign_locations
}; };
/* /*
@ -65,13 +69,17 @@ static int mindist_map(orte_job_t *jdata)
orte_node_t *node; orte_node_t *node;
orte_proc_t *proc; orte_proc_t *proc;
int nprocs_mapped; int nprocs_mapped;
int extra_procs, navg, nextra=0; int navg=0, nextra=0;
orte_std_cntr_t num_nodes, num_slots; orte_std_cntr_t num_nodes, num_slots;
unsigned int npus, total_npus, num_procs_to_assign=0, required; unsigned int npus, total_npus, num_procs_to_assign=0, required;
int rc; int rc;
mca_base_component_t *c = &mca_rmaps_mindist_component.base_version; mca_base_component_t *c = &mca_rmaps_mindist_component.base_version;
bool initial_map=true; bool initial_map=true;
bool bynode = false; bool bynode = false;
float balance;
int extra_procs_to_assign=0, nxtra_nodes=0;
bool add_one=false;
bool oversubscribed=false;
int ret; int ret;
/* this mapper can only handle initial launch /* this mapper can only handle initial launch
@ -164,6 +172,18 @@ static int mindist_map(orte_job_t *jdata)
ORTE_ERROR_LOG(rc); ORTE_ERROR_LOG(rc);
goto error; goto error;
} }
/* quick check to see if we can map all the procs */
if (num_slots < (int)app->num_procs) {
if (ORTE_MAPPING_NO_OVERSUBSCRIBE & ORTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping)) {
orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:alloc-error",
true, app->num_procs, app->app, orte_process_info.nodename);
ORTE_UPDATE_EXIT_STATUS(ORTE_ERROR_DEFAULT_EXIT_CODE);
return ORTE_ERR_SILENT;
}
oversubscribed = true;
}
num_nodes = (orte_std_cntr_t)opal_list_get_size(&node_list); num_nodes = (orte_std_cntr_t)opal_list_get_size(&node_list);
/* flag that all subsequent requests should not reset the node->mapped flag */ /* flag that all subsequent requests should not reset the node->mapped flag */
initial_map = false; initial_map = false;
@ -181,215 +201,214 @@ static int mindist_map(orte_job_t *jdata)
rc = ORTE_ERR_SILENT; rc = ORTE_ERR_SILENT;
goto error; goto error;
} }
if (bynode) { do {
/* calculate num_procs_to_assign for bynode case */ if (bynode || (app->num_procs > num_slots)) {
navg = app->num_procs / num_nodes; /* if there is oversubscribe then uses bynode case */
nextra = app->num_procs - navg * num_nodes; bynode = true;
num_procs_to_assign = navg; /* calculate num_procs_to_assign for bynode case */
if (nextra > 0) navg = ((int)app->num_procs - nprocs_mapped) / num_nodes;
num_procs_to_assign++; nextra = app->num_procs - navg * num_nodes;
} num_procs_to_assign = navg;
if (nextra > 0) {
/* iterate through the list of nodes */ num_procs_to_assign++;
for (item = opal_list_get_first(&node_list); }
item != opal_list_get_end(&node_list); /* compute how many extra procs to put on each node */
item = opal_list_get_next(item)) { balance = (float)(((int)app->num_procs - nprocs_mapped) - (navg * num_nodes)) / (float)num_nodes;
node = (orte_node_t*)item; extra_procs_to_assign = (int)balance;
nxtra_nodes = 0;
if (NULL == node->topology || NULL == node->topology->topo) { add_one = false;
orte_show_help("help-orte-rmaps-base.txt", "rmaps:no-topology", if (0 < (balance - (float)extra_procs_to_assign)) {
true, node->name); /* compute how many nodes need an extra proc */
rc = ORTE_ERR_SILENT; nxtra_nodes = ((int)app->num_procs - nprocs_mapped) - ((navg + extra_procs_to_assign) * num_nodes);
goto error; /* add one so that we add an extra proc to the first nodes
} * until all procs are mapped
/* get the root object as we are not assigning */
* locale except at the node level extra_procs_to_assign++;
*/ /* flag that we added one */
obj = hwloc_get_root_obj(node->topology->topo); add_one = true;
if (NULL == obj) {
orte_show_help("help-orte-rmaps-base.txt", "rmaps:no-topology",
true, node->name);
rc = ORTE_ERR_SILENT;
goto error;
}
/* get the number of available pus */
if (opal_hwloc_use_hwthreads_as_cpus) {
total_npus = opal_hwloc_base_get_nbobjs_by_type(node->topology->topo, HWLOC_OBJ_PU, 0, OPAL_HWLOC_AVAILABLE);
} else {
total_npus = opal_hwloc_base_get_nbobjs_by_type(node->topology->topo, HWLOC_OBJ_CORE, 0, OPAL_HWLOC_AVAILABLE);
}
if (bynode) {
if (total_npus < num_procs_to_assign) {
/* check if oversubscribing is allowed */
if (ORTE_MAPPING_NO_OVERSUBSCRIBE & ORTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping)) {
orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:alloc-error",
true, app->num_procs, app->app);
rc = ORTE_ERR_SILENT;
ORTE_UPDATE_EXIT_STATUS(ORTE_ERROR_DEFAULT_EXIT_CODE);
goto error;
} else {
ORTE_FLAG_SET(node, ORTE_NODE_FLAG_OVERSUBSCRIBED);
ORTE_FLAG_SET(jdata, ORTE_JOB_FLAG_OVERSUBSCRIBED);
}
} }
} }
/* first we need to fill summary object for root with information about nodes
* so we call opal_hwloc_base_get_nbobjs_by_type */
opal_hwloc_base_get_nbobjs_by_type(node->topology->topo, HWLOC_OBJ_NODE, 0, OPAL_HWLOC_AVAILABLE);
OBJ_CONSTRUCT(&numa_list, opal_list_t);
ret = opal_hwloc_get_sorted_numa_list(node->topology->topo, orte_rmaps_base.device, &numa_list);
if (ret > 1) {
orte_show_help("help-orte-rmaps-md.txt", "orte-rmaps-mindist:several-devices",
true, orte_rmaps_base.device, ret, node->name);
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYSLOT);
rc = ORTE_ERR_TAKE_NEXT_OPTION;
goto error;
} else if (ret < 0) {
orte_show_help("help-orte-rmaps-md.txt", "orte-rmaps-mindist:device-not-found",
true, orte_rmaps_base.device, node->name);
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYSLOT);
rc = ORTE_ERR_TAKE_NEXT_OPTION;
goto error;
}
if (opal_list_get_size(&numa_list) > 0) {
j = 0;
required = 0;
OPAL_LIST_FOREACH(numa, &numa_list, opal_rmaps_numa_node_t) {
/* get the hwloc object for this numa */
if (NULL == (obj = opal_hwloc_base_get_obj_by_type(node->topology->topo, HWLOC_OBJ_NODE, 0, numa->index, OPAL_HWLOC_AVAILABLE))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return ORTE_ERR_NOT_FOUND;
}
npus = opal_hwloc_base_get_npus(node->topology->topo, obj);
if (bynode) {
required = ((num_procs_to_assign-j) > npus) ? (npus) : (num_procs_to_assign-j);
} else {
required = npus;
}
for (k = 0; (k < required) && (nprocs_mapped < app->num_procs); k++) {
if (NULL == (proc = orte_rmaps_base_setup_proc(jdata, node, i))) {
rc = ORTE_ERR_OUT_OF_RESOURCE;
goto error;
}
nprocs_mapped++;
j++;
orte_set_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, ORTE_ATTR_LOCAL, obj, OPAL_PTR);
}
if ((nprocs_mapped == (int)app->num_procs) || (bynode && ((int)num_procs_to_assign == j))) {
break;
}
}
if (0 != j) {
/* add the node to the map, if needed */
if (!ORTE_FLAG_TEST(node, ORTE_NODE_FLAG_MAPPED)) {
if (ORTE_SUCCESS > (rc = opal_pointer_array_add(jdata->map->nodes, (void*)node))) {
ORTE_ERROR_LOG(rc);
goto error;
}
ORTE_FLAG_SET(node, ORTE_NODE_FLAG_MAPPED);
OBJ_RETAIN(node); /* maintain accounting on object */
jdata->map->num_nodes++;
}
opal_output_verbose(2, orte_rmaps_base_framework.framework_output,
"mca:rmaps:mindist: assigned %d procs to node %s",
j, node->name);
}
} else {
if (hwloc_get_nbobjs_by_type(node->topology->topo, HWLOC_OBJ_SOCKET) > 1) {
/* don't have info about pci locality */
orte_show_help("help-orte-rmaps-md.txt", "orte-rmaps-mindist:no-pci-locality-info",
true, node->name);
}
/* else silently switch to byslot mapper since distance info is irrelevant for this machine configuration */
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYSLOT);
rc = ORTE_ERR_TAKE_NEXT_OPTION;
goto error;
}
while (NULL != (numa_item = opal_list_remove_first(&numa_list))) {
OBJ_RELEASE(numa_item);
}
OBJ_DESTRUCT(&numa_list);
if (bynode) {
nextra--;
if (nextra == 0) {
num_procs_to_assign--;
}
}
}
/* If we get to the end of all the nodes and still have procs remaining, then num_nodes = 0;
* we check the oversubscribed flag - if oversubscription is allowed, then /* iterate through the list of nodes */
* begin assigning procs round-robin *bynode* until all procs have been assigned.
* This ensures that the overload is evenly distributed across all nodes.
*/
extra_procs = app->num_procs - nprocs_mapped;
if (extra_procs > 0) {
/* check if oversubscribing is allowed */
if (ORTE_MAPPING_NO_OVERSUBSCRIBE & ORTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping)) {
orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:alloc-error",
true, app->num_procs, app->app);
rc = ORTE_ERR_SILENT;
ORTE_UPDATE_EXIT_STATUS(ORTE_ERROR_DEFAULT_EXIT_CODE);
goto error;
}
opal_output_verbose(2, orte_rmaps_base_framework.framework_output,
"mca:rmaps:mindist job %s is oversubscribed - performing second pass",
ORTE_JOBID_PRINT(jdata->jobid));
num_procs_to_assign = extra_procs/num_nodes;
nextra = extra_procs % num_nodes;
if (nextra > 0) {
num_procs_to_assign++;
}
for (item = opal_list_get_first(&node_list); for (item = opal_list_get_first(&node_list);
item != opal_list_get_end(&node_list); item != opal_list_get_end(&node_list);
item = opal_list_get_next(item)) { item = opal_list_get_next(item)) {
node = (orte_node_t*)item; node = (orte_node_t*)item;
if (nprocs_mapped == app->num_procs) if (NULL == node->topology || NULL == node->topology->topo) {
break; orte_show_help("help-orte-rmaps-base.txt", "rmaps:no-topology",
ORTE_FLAG_SET(node, ORTE_NODE_FLAG_OVERSUBSCRIBED); true, node->name);
ORTE_FLAG_SET(jdata, ORTE_JOB_FLAG_OVERSUBSCRIBED); rc = ORTE_ERR_SILENT;
opal_output_verbose(2, orte_rmaps_base_framework.framework_output, goto error;
"mca:rmaps:mindist: second pass assigning %d extra procs to node %s", }
(int)num_procs_to_assign, node->name); /* get the root object as we are not assigning
OBJ_CONSTRUCT(&numa_list, opal_list_t); * locale except at the node level
opal_hwloc_get_sorted_numa_list(node->topology->topo, orte_rmaps_base.device, &numa_list); */
if (opal_list_get_size(&numa_list) > 0) { obj = hwloc_get_root_obj(node->topology->topo);
numa_item = opal_list_get_first(&numa_list); if (NULL == obj) {
k = 0; orte_show_help("help-orte-rmaps-base.txt", "rmaps:no-topology",
obj = hwloc_get_obj_by_type(node->topology->topo, HWLOC_OBJ_NODE,((opal_rmaps_numa_node_t*)numa_item)->index); true, node->name);
npus = opal_hwloc_base_get_npus(node->topology->topo, obj); rc = ORTE_ERR_SILENT;
for (j = 0; j < (int)num_procs_to_assign && nprocs_mapped < (int)app->num_procs; j++) { goto error;
if (NULL == (proc = orte_rmaps_base_setup_proc(jdata, node, i))) { }
rc = ORTE_ERR_OUT_OF_RESOURCE;
goto error; num_nodes++;
}
nprocs_mapped++; /* get the number of available pus */
k++; if (opal_hwloc_use_hwthreads_as_cpus) {
orte_set_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, ORTE_ATTR_LOCAL, obj, OPAL_PTR); total_npus = opal_hwloc_base_get_nbobjs_by_type(node->topology->topo, HWLOC_OBJ_PU, 0, OPAL_HWLOC_AVAILABLE);
if (k > npus-1) { } else {
numa_item = opal_list_get_next(numa_item); total_npus = opal_hwloc_base_get_nbobjs_by_type(node->topology->topo, HWLOC_OBJ_CORE, 0, OPAL_HWLOC_AVAILABLE);
if (numa_item == opal_list_get_end(&numa_list)) { }
numa_item = opal_list_get_first(&numa_list);
if (bynode) {
if (oversubscribed) {
/* compute the number of procs to go on this node */
if (add_one) {
if (0 == nxtra_nodes) {
--extra_procs_to_assign;
add_one = false;
} else {
--nxtra_nodes;
} }
obj = hwloc_get_obj_by_type(node->topology->topo, HWLOC_OBJ_NODE,((opal_rmaps_numa_node_t*)numa_item)->index); }
npus = opal_hwloc_base_get_npus(node->topology->topo, obj); /* everybody just takes their share */
k = 0; num_procs_to_assign = navg + extra_procs_to_assign;
}else if (node->slots <= node->slots_inuse) {
/* since we are not oversubcribed, ignore this node */
continue;
} else {
/* if we are not oversubscribed, then there are enough
* slots to handle all the procs. However, not every
* node will have the same number of slots, so we
* have to track how many procs to "shift" elsewhere
* to make up the difference
*/
/* compute the number of procs to go on this node */
if (add_one) {
if (0 == nxtra_nodes) {
--extra_procs_to_assign;
add_one = false;
} else {
--nxtra_nodes;
}
}
/* if slots < avg + extra (adjusted for cpus/proc), then try to take all */
if ((node->slots - node->slots_inuse) < (navg + extra_procs_to_assign)) {
num_procs_to_assign = node->slots - node->slots_inuse;
/* if we can't take any proc, skip following steps */
if (num_procs_to_assign == 0) {
continue;
}
} else {
/* take the avg + extra */
num_procs_to_assign = navg + extra_procs_to_assign;
}
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:mindist: %s node %s avg %d assign %d extra %d",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), node->name,
navg, num_procs_to_assign, extra_procs_to_assign);
}
} else {
num_procs_to_assign = ((int)app->num_procs - nprocs_mapped) > node->slots ?
node->slots : ((int)app->num_procs - nprocs_mapped);
}
if (bynode) {
if (total_npus < num_procs_to_assign) {
/* check if oversubscribing is allowed */
if (ORTE_MAPPING_NO_OVERSUBSCRIBE & ORTE_GET_MAPPING_DIRECTIVE(jdata->map->mapping)) {
orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:alloc-error",
true, app->num_procs, app->app);
rc = ORTE_ERR_SILENT;
ORTE_UPDATE_EXIT_STATUS(ORTE_ERROR_DEFAULT_EXIT_CODE);
goto error;
} else {
ORTE_FLAG_SET(node, ORTE_NODE_FLAG_OVERSUBSCRIBED);
ORTE_FLAG_SET(jdata, ORTE_JOB_FLAG_OVERSUBSCRIBED);
} }
} }
} }
OBJ_CONSTRUCT(&numa_list, opal_list_t);
ret = opal_hwloc_get_sorted_numa_list(node->topology->topo, orte_rmaps_base.device, &numa_list);
if (ret > 1) {
orte_show_help("help-orte-rmaps-md.txt", "orte-rmaps-mindist:several-devices",
true, orte_rmaps_base.device, ret, node->name);
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYSLOT);
rc = ORTE_ERR_TAKE_NEXT_OPTION;
goto error;
} else if (ret < 0) {
orte_show_help("help-orte-rmaps-md.txt", "orte-rmaps-mindist:device-not-found",
true, orte_rmaps_base.device, node->name);
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYSLOT);
rc = ORTE_ERR_TAKE_NEXT_OPTION;
goto error;
}
if (opal_list_get_size(&numa_list) > 0) {
j = 0;
required = 0;
OPAL_LIST_FOREACH(numa, &numa_list, opal_rmaps_numa_node_t) {
/* get the hwloc object for this numa */
if (NULL == (obj = opal_hwloc_base_get_obj_by_type(node->topology->topo, HWLOC_OBJ_NODE, 0, numa->index, OPAL_HWLOC_AVAILABLE))) {
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
return ORTE_ERR_NOT_FOUND;
}
npus = opal_hwloc_base_get_npus(node->topology->topo, obj);
if (bynode) {
required = num_procs_to_assign;
} else {
required = (num_procs_to_assign-j) > npus ? npus : (num_procs_to_assign-j);
}
for (k = 0; (k < required) && (nprocs_mapped < app->num_procs); k++) {
if (NULL == (proc = orte_rmaps_base_setup_proc(jdata, node, i))) {
rc = ORTE_ERR_OUT_OF_RESOURCE;
goto error;
}
nprocs_mapped++;
j++;
orte_set_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, ORTE_ATTR_LOCAL, obj, OPAL_PTR);
}
if ((nprocs_mapped == (int)app->num_procs) || ((int)num_procs_to_assign == j)) {
break;
}
}
if (0 != j) {
/* add the node to the map, if needed */
if (!ORTE_FLAG_TEST(node, ORTE_NODE_FLAG_MAPPED)) {
ORTE_FLAG_SET(node, ORTE_NODE_FLAG_MAPPED);
OBJ_RETAIN(node); /* maintain accounting on object */
jdata->map->num_nodes++;
opal_pointer_array_add(jdata->map->nodes, node);
}
opal_output_verbose(2, orte_rmaps_base_framework.framework_output,
"mca:rmaps:mindist: assigned %d procs to node %s",
j, node->name);
}
} else {
if (hwloc_get_nbobjs_by_type(node->topology->topo, HWLOC_OBJ_SOCKET) > 1) {
/* don't have info about pci locality */
orte_show_help("help-orte-rmaps-md.txt", "orte-rmaps-mindist:no-pci-locality-info",
true, node->name);
}
/* else silently switch to byslot mapper since distance info is irrelevant for this machine configuration */
ORTE_SET_MAPPING_POLICY(jdata->map->mapping, ORTE_MAPPING_BYSLOT);
rc = ORTE_ERR_TAKE_NEXT_OPTION;
goto error;
}
while (NULL != (numa_item = opal_list_remove_first(&numa_list))) { while (NULL != (numa_item = opal_list_remove_first(&numa_list))) {
OBJ_RELEASE(numa_item); OBJ_RELEASE(numa_item);
} }
OBJ_DESTRUCT(&numa_list); OBJ_DESTRUCT(&numa_list);
nextra--; if (bynode) {
if (nextra == 0) { nextra--;
num_procs_to_assign--; if (nextra == 0) {
num_procs_to_assign--;
}
} }
} }
} } while(bynode && nprocs_mapped < app->num_procs && 0 < num_nodes);
/* track the total number of processes we mapped - must update /* track the total number of processes we mapped - must update
* this value AFTER we compute vpids so that computation * this value AFTER we compute vpids so that computation
@ -406,17 +425,7 @@ static int mindist_map(orte_job_t *jdata)
OBJ_DESTRUCT(&node_list); OBJ_DESTRUCT(&node_list);
} }
free(orte_rmaps_base.device); free(orte_rmaps_base.device);
/* compute vpids and add proc objects to the job - do this after
* each app_context so that the ranks within each context are
* contiguous
*/
if (ORTE_SUCCESS != (rc = orte_rmaps_base_compute_vpids(jdata))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* mark the job as fully described */
orte_set_attribute(&jdata->attributes, ORTE_JOB_FULLY_DESCRIBED, ORTE_ATTR_GLOBAL, NULL, OPAL_BOOL);
return ORTE_SUCCESS; return ORTE_SUCCESS;
error: error:
@ -428,7 +437,6 @@ error:
return rc; return rc;
} }
#if 0
static int assign_locations(orte_job_t *jdata) static int assign_locations(orte_job_t *jdata)
{ {
int j, k, m, n, npus; int j, k, m, n, npus;
@ -509,6 +517,8 @@ static int assign_locations(orte_job_t *jdata)
continue; continue;
} }
orte_set_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, ORTE_ATTR_LOCAL, obj, OPAL_PTR); orte_set_attribute(&proc->attributes, ORTE_PROC_HWLOC_LOCALE, ORTE_ATTR_LOCAL, obj, OPAL_PTR);
opal_output_verbose(5, orte_rmaps_base_framework.framework_output,
"mca:rmaps:mindist: assigning proc %d to numa %d", k, numa->index);
++j; ++j;
--npus; --npus;
} }
@ -519,4 +529,3 @@ static int assign_locations(orte_job_t *jdata)
return ORTE_SUCCESS; return ORTE_SUCCESS;
} }
#endif