diff --git a/orte/mca/odls/base/odls_base_default_fns.c b/orte/mca/odls/base/odls_base_default_fns.c index a9db7e3c7a..25b0bac3c9 100644 --- a/orte/mca/odls/base/odls_base_default_fns.c +++ b/orte/mca/odls/base/odls_base_default_fns.c @@ -14,7 +14,7 @@ * Copyright (c) 2011-2013 Los Alamos National Security, LLC. * All rights reserved. * Copyright (c) 2011-2013 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2013 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2014 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -778,6 +778,14 @@ static int odls_base_default_setup_fork(orte_app_context_t *context, free(param2); } + /* Set an info MCA param that tells the launched processes that + * any binding policy was applied by us (e.g., so that + * MPI_INIT doesn't try to bind itself) + */ + (void) mca_base_var_env_name ("orte_bound_at_launch", ¶m); + opal_setenv(param, "1", true, environ_copy); + free(param); + /* push data into environment - don't push any single proc * info, though. We are setting the environment up on a * per-context basis, and will add the individual proc diff --git a/orte/mca/odls/default/odls_default_module.c b/orte/mca/odls/default/odls_default_module.c index 01dd2afdd0..5952d155dd 100644 --- a/orte/mca/odls/default/odls_default_module.c +++ b/orte/mca/odls/default/odls_default_module.c @@ -478,11 +478,9 @@ static int do_child(orte_app_context_t* context, /* bind this proc to all available processors */ hwloc_set_cpubind(opal_hwloc_topology, sum->available, 0); } - /* Set an info MCA param that tells - the launched processes that it was bound by us (e.g., so that - MPI_INIT doesn't try to bind itself) */ - (void) mca_base_var_env_name ("orte_bound_at_launch", ¶m); - opal_setenv(param, "1", true, &environ_copy); + /* provide a nice string representation of what we bound to */ + (void) mca_base_var_env_name ("orte_base_applied_binding", ¶m); + opal_setenv(param, child->cpu_bitmap, true, &environ_copy); free(param); goto PROCEED; } @@ -602,17 +600,6 @@ static int do_child(orte_app_context_t* context, goto PROCEED; } } - /* Set an info MCA param that tells - the launched processes that it was bound by us (e.g., so that - MPI_INIT doesn't try to bind itself) */ - (void) mca_base_var_env_name ("orte_bound_at_launch", ¶m); - opal_setenv(param, "1", true, &environ_copy); - free(param); - /* ...and provide a nice string representation of what we - bound to */ - (void) mca_base_var_env_name ("orte_base_applied_binding", ¶m); - opal_setenv(param, child->cpu_bitmap, true, &environ_copy); - free (param); } } #endif diff --git a/orte/mca/plm/base/plm_base_launch_support.c b/orte/mca/plm/base/plm_base_launch_support.c index 3d493a98a9..3665b88ff3 100644 --- a/orte/mca/plm/base/plm_base_launch_support.c +++ b/orte/mca/plm/base/plm_base_launch_support.c @@ -113,7 +113,8 @@ void orte_plm_base_daemons_reported(int fd, short args, void *cbdata) * slots on each node as directed or using default */ if (!orte_managed_allocation) { - if (NULL != orte_set_slots) { + if (NULL != orte_set_slots && + 0 != strncmp(orte_set_slots, "none", strlen(orte_set_slots))) { for (i=0; i < orte_node_pool->size; i++) { if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, i))) { continue; diff --git a/orte/mca/rmaps/base/help-orte-rmaps-base.txt b/orte/mca/rmaps/base/help-orte-rmaps-base.txt index 87b723a46c..cf08ce7e0a 100644 --- a/orte/mca/rmaps/base/help-orte-rmaps-base.txt +++ b/orte/mca/rmaps/base/help-orte-rmaps-base.txt @@ -276,11 +276,11 @@ the required syntax of #:object Please check your request and try again. # [orte-rmaps-base:oversubscribed] -The requested number of processors exceeds the allocated +The requested number of processes exceeds the allocated number of slots: #slots: %d - #processors: %d + #processes: %d This creates an oversubscribed condition that may adversely impact performance when combined with the requested binding diff --git a/orte/mca/rmaps/round_robin/rmaps_rr.c b/orte/mca/rmaps/round_robin/rmaps_rr.c index dab17210da..acd70d8fb5 100644 --- a/orte/mca/rmaps/round_robin/rmaps_rr.c +++ b/orte/mca/rmaps/round_robin/rmaps_rr.c @@ -145,7 +145,7 @@ static int orte_rmaps_rr_map(orte_job_t *jdata) } else if (ORTE_MAPPING_BYHWTHREAD == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) { rc = orte_rmaps_rr_byobj(jdata, app, &node_list, num_slots, app->num_procs, HWLOC_OBJ_PU, 0); - if (ORTE_ERR_NOT_SUPPORTED == rc) { + if (ORTE_ERR_NOT_FOUND == rc) { /* if the mapper couldn't map by this object because * it isn't available, but the error allows us to try * byslot, then do so @@ -156,7 +156,7 @@ static int orte_rmaps_rr_map(orte_job_t *jdata) } else if (ORTE_MAPPING_BYCORE == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) { rc = orte_rmaps_rr_byobj(jdata, app, &node_list, num_slots, app->num_procs, HWLOC_OBJ_CORE, 0); - if (ORTE_ERR_NOT_SUPPORTED == rc) { + if (ORTE_ERR_NOT_FOUND == rc) { /* if the mapper couldn't map by this object because * it isn't available, but the error allows us to try * byslot, then do so @@ -167,7 +167,7 @@ static int orte_rmaps_rr_map(orte_job_t *jdata) } else if (ORTE_MAPPING_BYL1CACHE == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) { rc = orte_rmaps_rr_byobj(jdata, app, &node_list, num_slots, app->num_procs, HWLOC_OBJ_CACHE, 1); - if (ORTE_ERR_NOT_SUPPORTED == rc) { + if (ORTE_ERR_NOT_FOUND == rc) { /* if the mapper couldn't map by this object because * it isn't available, but the error allows us to try * byslot, then do so @@ -178,7 +178,7 @@ static int orte_rmaps_rr_map(orte_job_t *jdata) } else if (ORTE_MAPPING_BYL2CACHE == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) { rc = orte_rmaps_rr_byobj(jdata, app, &node_list, num_slots, app->num_procs, HWLOC_OBJ_CACHE, 2); - if (ORTE_ERR_NOT_SUPPORTED == rc) { + if (ORTE_ERR_NOT_FOUND == rc) { /* if the mapper couldn't map by this object because * it isn't available, but the error allows us to try * byslot, then do so @@ -189,7 +189,7 @@ static int orte_rmaps_rr_map(orte_job_t *jdata) } else if (ORTE_MAPPING_BYL3CACHE == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) { rc = orte_rmaps_rr_byobj(jdata, app, &node_list, num_slots, app->num_procs, HWLOC_OBJ_CACHE, 3); - if (ORTE_ERR_NOT_SUPPORTED == rc) { + if (ORTE_ERR_NOT_FOUND == rc) { /* if the mapper couldn't map by this object because * it isn't available, but the error allows us to try * byslot, then do so @@ -200,7 +200,7 @@ static int orte_rmaps_rr_map(orte_job_t *jdata) } else if (ORTE_MAPPING_BYSOCKET == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) { rc = orte_rmaps_rr_byobj(jdata, app, &node_list, num_slots, app->num_procs, HWLOC_OBJ_SOCKET, 0); - if (ORTE_ERR_NOT_SUPPORTED == rc) { + if (ORTE_ERR_NOT_FOUND == rc) { /* if the mapper couldn't map by this object because * it isn't available, but the error allows us to try * byslot, then do so @@ -211,7 +211,7 @@ static int orte_rmaps_rr_map(orte_job_t *jdata) } else if (ORTE_MAPPING_BYNUMA == ORTE_GET_MAPPING_POLICY(jdata->map->mapping)) { rc = orte_rmaps_rr_byobj(jdata, app, &node_list, num_slots, app->num_procs, HWLOC_OBJ_NODE, 0); - if (ORTE_ERR_NOT_SUPPORTED == rc) { + if (ORTE_ERR_NOT_FOUND == rc) { /* if the mapper couldn't map by this object because * it isn't available, but the error allows us to try * byslot, then do so diff --git a/orte/mca/rmaps/round_robin/rmaps_rr_mappers.c b/orte/mca/rmaps/round_robin/rmaps_rr_mappers.c index 3e3fb59ced..e00219ddc7 100644 --- a/orte/mca/rmaps/round_robin/rmaps_rr_mappers.c +++ b/orte/mca/rmaps/round_robin/rmaps_rr_mappers.c @@ -71,8 +71,8 @@ int orte_rmaps_rr_byslot(orte_job_t *jdata, !OPAL_BIND_OVERLOAD_ALLOWED(jdata->map->binding)){ orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:oversubscribed", true, num_slots, app->num_procs * orte_rmaps_base.cpus_per_rank); + OPAL_SET_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_NONE); } - OPAL_SET_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_NONE); } else { /* don't default to bound */ OPAL_SET_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_NONE); @@ -253,8 +253,8 @@ int orte_rmaps_rr_bynode(orte_job_t *jdata, !OPAL_BIND_OVERLOAD_ALLOWED(jdata->map->binding)){ orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:oversubscribed", true, num_slots, app->num_procs * orte_rmaps_base.cpus_per_rank); + OPAL_SET_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_NONE); } - OPAL_SET_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_NONE); } else { /* don't default to bound */ OPAL_SET_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_NONE); @@ -508,8 +508,8 @@ int orte_rmaps_rr_byobj(orte_job_t *jdata, !OPAL_BIND_OVERLOAD_ALLOWED(jdata->map->binding)){ orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:oversubscribed", true, num_slots, app->num_procs * orte_rmaps_base.cpus_per_rank); + OPAL_SET_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_NONE); } - OPAL_SET_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_NONE); } else { /* don't default to bound */ OPAL_SET_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_NONE); @@ -653,8 +653,8 @@ static int byobj_span(orte_job_t *jdata, !OPAL_BIND_OVERLOAD_ALLOWED(jdata->map->binding)){ orte_show_help("help-orte-rmaps-base.txt", "orte-rmaps-base:oversubscribed", true, num_slots, app->num_procs * orte_rmaps_base.cpus_per_rank); + OPAL_SET_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_NONE); } - OPAL_SET_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_NONE); } else { /* don't default to bound */ OPAL_SET_BINDING_POLICY(jdata->map->binding, OPAL_BIND_TO_NONE); diff --git a/orte/runtime/orte_mca_params.c b/orte/runtime/orte_mca_params.c index fb45e2c9eb..c5ce8ca4d4 100644 --- a/orte/runtime/orte_mca_params.c +++ b/orte/runtime/orte_mca_params.c @@ -712,9 +712,16 @@ int orte_register_params(void) OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &orte_max_vm_size); - orte_set_slots = NULL; + if (opal_hwloc_use_hwthreads_as_cpus) { + orte_set_slots = strdup("hwthreads"); + } else { + orte_set_slots = strdup("cores"); + } (void) mca_base_var_register ("orte", "orte", NULL, "set_default_slots", - "Set the number of slots on nodes that lack such info to the number of specified objects [a number, \"cores\", \"numas\", \"sockets\", or \"hwthreads\"]", + "Set the number of slots on nodes that lack such info to the" + " number of specified objects [a number, \"cores\" (default)," + " \"numas\", \"sockets\", \"hwthreads\" (default if hwthreads_as_cpus is set)," + " or \"none\" to skip this option]", MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY, &orte_set_slots); diff --git a/orte/tools/orterun/orterun.c b/orte/tools/orterun/orterun.c index c7e7afbb92..3843de722f 100644 --- a/orte/tools/orterun/orterun.c +++ b/orte/tools/orterun/orterun.c @@ -658,7 +658,7 @@ int orterun(int argc, char *argv[]) * depend upon opal_init_util() functionality. */ /* Need to initialize OPAL so that install_dirs are filled in */ - if (OPAL_SUCCESS != opal_init_util(&argc, &argv)) { + if (OPAL_SUCCESS != opal_init(&argc, &argv)) { exit(1); } @@ -833,10 +833,10 @@ int orterun(int argc, char *argv[]) */ return rc; } - /* finalize the OPAL utils. As they are opened again from orte_init->opal_init - * we continue to have a reference count on them. So we have to finalize them twice... + /* finalize OPAL. As it was opened again from orte_init->opal_init + * we continue to have a reference count on it. So we have to finalize it twice... */ - opal_finalize_util(); + opal_finalize(); /* get the daemon job object */