ports/openmpi
ports/openmpi
1
1
Fork 0
Code Releases Activity

Since the calls to "PMI get" scale by number of procs (not nodes), it makes more sense to have the MCA param be the cutoff based on number of procs. Also, it occurred to me that this shouldn't impact the nidmap process as that is built and circulated when we launch via mpirun, not during direct launch.

Browse Source 
So shift the cutoff param to the MPI layer, and have it solely determine whether or not we call modex_recv on the hostname. If comm_world is of size greater than the cutoff, then we don't automatically retrieve the hostname when we build the ompi_proc_t for a process - instead, we fill the hostname entry on first call to modex_recv for that process.

The param is now "ompi_hostname_cutoff=N", where N=number of procs for cutoff.

Refs trac:3729

This commit was SVN r29056.

The following Trac tickets were found above:
  Ticket 3729 --> https://svn.open-mpi.org/trac/ompi/ticket/3729
This commit is contained in:
Ralph Castain 2013-08-22 03:40:26 +00:00
parent 504fa2cda9
commit 16c5b30a1f
8 changed files with 163 additions and 166 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
ompi/mca/rte/orte/rte_orte.h
View File

@ -66,7 +66,6 @@ typedef orte_node_rank_t ompi_node_rank_t;
typedef orte_local_rank_t ompi_local_rank_t;
#define ompi_process_info orte_process_info
#define ompi_rte_proc_is_bound orte_proc_is_bound
#define ompi_rte_hostname_cutoff orte_hostname_cutoff
/* Error handling objects and operations */
OMPI_DECLSPEC void ompi_rte_abort(int error_code, char *fmt, ...);

5
ompi/proc/proc.c
View File

@ -35,6 +35,7 @@
#include "ompi/datatype/ompi_datatype.h"
#include "ompi/runtime/mpiruntime.h"
#include "ompi/runtime/ompi_module_exchange.h"
#include "ompi/runtime/params.h"
static opal_list_t ompi_proc_list;
static opal_mutex_t ompi_proc_lock;
@ -157,7 +158,7 @@ int ompi_proc_complete_init(void)
break;
}
if (ompi_process_info.num_daemons < ompi_rte_hostname_cutoff) {
if (ompi_process_info.num_procs < ompi_hostname_cutoff) {
/* retrieve the hostname */
ret = ompi_modex_recv_string_pointer(OMPI_DB_HOSTNAME, proc, (void**)&(proc->proc_hostname), OPAL_STRING);
if (OMPI_SUCCESS != ret) {
@ -394,7 +395,7 @@ int ompi_proc_refresh(void) {
if (OMPI_SUCCESS != ret) {
break;
}
if (ompi_process_info.num_daemons < ompi_rte_hostname_cutoff) {
if (ompi_process_info.num_procs < ompi_hostname_cutoff) {
/* retrieve the hostname */
ret = ompi_modex_recv_string_pointer(OMPI_DB_HOSTNAME, proc, (void**)&(proc->proc_hostname), OPAL_STRING);
if (OMPI_SUCCESS != ret) {

11
ompi/runtime/ompi_mpi_params.c
View File

@ -66,6 +66,7 @@ bool ompi_use_sparse_group_storage = OPAL_INT_TO_BOOL(OMPI_GROUP_SPARSE);
bool ompi_mpi_built_with_cuda_support = OPAL_INT_TO_BOOL(OMPI_CUDA_SUPPORT);
bool ompi_mpi_cuda_support;
uint32_t ompi_hostname_cutoff = UINT32_MAX;
bool ompi_mpi_yield_when_idle = true;
int ompi_mpi_event_tick_rate = -1;
char *ompi_mpi_show_mca_params_string = NULL;
@ -333,6 +334,16 @@ int ompi_mpi_register_params(void)
ompi_rte_abort(1, NULL);
}
/* cutoff for retrieving hostnames */
ompi_hostname_cutoff = UINT32_MAX;
(void) mca_base_var_register ("ompi", "ompi", NULL, "hostname_cutoff",
"If the number of processes in the application exceeds the provided value,"
"hostnames for remote processes will not be retrieved by applications [default: UINT32_MAX]",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&ompi_hostname_cutoff);
return OMPI_SUCCESS;
}

5
ompi/runtime/params.h
View File

@ -139,6 +139,11 @@ OMPI_DECLSPEC extern bool ompi_use_sparse_group_storage;
*/
OMPI_DECLSPEC extern bool ompi_mpi_cuda_support;
/*
* Cutoff point for retrieving hostnames
*/
OMPI_DECLSPEC extern uint32_t ompi_hostname_cutoff;
/**
* Register MCA parameters used by the MPI layer.
*

1
orte/runtime/orte_globals.c
View File

@ -79,7 +79,6 @@ bool orte_have_fqdn_allocation = false;
bool orte_show_resolved_nodenames;
bool orte_retain_aliases;
int orte_use_hostname_alias;
orte_vpid_t orte_hostname_cutoff;
int orted_debug_failure;
int orted_debug_failure_delay;

1
orte/runtime/orte_globals.h
View File

@ -607,7 +607,6 @@ ORTE_DECLSPEC extern bool orte_have_fqdn_allocation;
ORTE_DECLSPEC extern bool orte_show_resolved_nodenames;
ORTE_DECLSPEC extern bool orte_retain_aliases;
ORTE_DECLSPEC extern int orte_use_hostname_alias;
ORTE_DECLSPEC extern orte_vpid_t orte_hostname_cutoff;
/* debug flags */
ORTE_DECLSPEC extern int orted_debug_failure;

9
orte/runtime/orte_mca_params.c
View File

@ -436,15 +436,6 @@ int orte_register_params(void)
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_use_hostname_alias);
/* cutoff for including hostnames in modex */
orte_hostname_cutoff = UINT_MAX;
(void) mca_base_var_register ("orte", "orte", NULL, "hostname_cutoff",
"If the number of nodes in the allocation exceeds the provided value,"
"hostnames for remote processes will not be supplied to applications [default: UINT_MAX]",
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_hostname_cutoff);
orte_xml_output = false;
(void) mca_base_var_register ("orte", "orte", NULL, "xml_output",
"Display all output in XML format (default: false)",

296
orte/util/nidmap.c
View File

@ -306,42 +306,40 @@ int orte_util_encode_nodemap(opal_byte_object_t *boptr, bool update)
ORTE_ERROR_LOG(rc);
return rc;
}
if (daemons->num_procs < orte_hostname_cutoff) {
/* pack the name of the node */
if (!orte_keep_fqdn_hostnames) {
nodename = strdup(node->name);
/* if the nodename is an IP address, do not mess with it! */
if (!opal_net_isaddr(nodename)) {
/* not an IP address */
if (NULL != (ptr = strchr(nodename, '.'))) {
*ptr = '\0';
}
}
if (ORTE_SUCCESS != (rc = opal_dss.pack(&buf, &nodename, 1, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
return rc;
}
free(nodename);
} else {
if (ORTE_SUCCESS != (rc = opal_dss.pack(&buf, &node->name, 1, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
return rc;
/* pack the name of the node */
if (!orte_keep_fqdn_hostnames) {
nodename = strdup(node->name);
/* if the nodename is an IP address, do not mess with it! */
if (!opal_net_isaddr(nodename)) {
/* not an IP address */
if (NULL != (ptr = strchr(nodename, '.'))) {
*ptr = '\0';
}
}
/* if requested, pack any aliases */
if (orte_retain_aliases) {
uint8_t naliases, ni;
naliases = opal_argv_count(node->alias);
if (ORTE_SUCCESS != (rc = opal_dss.pack(&buf, &naliases, 1, OPAL_UINT8))) {
if (ORTE_SUCCESS != (rc = opal_dss.pack(&buf, &nodename, 1, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
return rc;
}
free(nodename);
} else {
if (ORTE_SUCCESS != (rc = opal_dss.pack(&buf, &node->name, 1, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
return rc;
}
}
/* if requested, pack any aliases */
if (orte_retain_aliases) {
uint8_t naliases, ni;
naliases = opal_argv_count(node->alias);
if (ORTE_SUCCESS != (rc = opal_dss.pack(&buf, &naliases, 1, OPAL_UINT8))) {
ORTE_ERROR_LOG(rc);
return rc;
}
for (ni=0; ni < naliases; ni++) {
if (ORTE_SUCCESS != (rc = opal_dss.pack(&buf, &node->alias[ni], 1, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
return rc;
}
for (ni=0; ni < naliases; ni++) {
if (ORTE_SUCCESS != (rc = opal_dss.pack(&buf, &node->alias[ni], 1, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
return rc;
}
}
}
}
@ -399,70 +397,68 @@ int orte_util_decode_nodemap(opal_byte_object_t *bo)
n=1;
while (OPAL_SUCCESS == (rc = opal_dss.unpack(&buf, &daemon.vpid, &n, ORTE_VPID))) {
if (num_daemons < orte_hostname_cutoff) {
/* unpack and store the node's name */
/* unpack and store the node's name */
n=1;
if (ORTE_SUCCESS != (rc = opal_dss.unpack(&buf, &nodename, &n, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
return rc;
}
if (ORTE_SUCCESS != (rc = opal_db.store((opal_identifier_t*)&daemon, OPAL_DB_INTERNAL, ORTE_DB_HOSTNAME, nodename, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* now store a direct reference so we can quickly lookup the daemon from a hostname */
opal_output_verbose(2, orte_nidmap_output,
"%s storing nodename %s for daemon %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
nodename, ORTE_VPID_PRINT(daemon.vpid));
if (ORTE_SUCCESS != (rc = opal_db.store((opal_identifier_t*)ORTE_NAME_WILDCARD, OPAL_DB_INTERNAL, nodename, &daemon.vpid, OPAL_UINT32))) {
ORTE_ERROR_LOG(rc);
return rc;
}
OPAL_OUTPUT_VERBOSE((2, orte_nidmap_output,
"%s orte:util:decode:nidmap daemon %s node %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_VPID_PRINT(daemon.vpid), nodename));
/* if this is my daemon, then store the data for me too */
if (daemon.vpid == ORTE_PROC_MY_DAEMON->vpid) {
if (ORTE_SUCCESS != (rc = opal_db.store((opal_identifier_t*)ORTE_PROC_MY_NAME, OPAL_DB_INTERNAL, ORTE_DB_HOSTNAME, nodename, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
return rc;
}
if (ORTE_SUCCESS != (rc = opal_db.store((opal_identifier_t*)ORTE_PROC_MY_NAME, OPAL_DB_INTERNAL, ORTE_DB_DAEMON_VPID, &daemon.vpid, OPAL_UINT32))) {
ORTE_ERROR_LOG(rc);
return rc;
}
}
/* if requested, unpack any aliases */
if (orte_retain_aliases) {
char *alias;
uint8_t naliases, ni;
n=1;
if (ORTE_SUCCESS != (rc = opal_dss.unpack(&buf, &nodename, &n, OPAL_STRING))) {
if (ORTE_SUCCESS != (rc = opal_dss.unpack(&buf, &naliases, &n, OPAL_UINT8))) {
ORTE_ERROR_LOG(rc);
return rc;
}
if (ORTE_SUCCESS != (rc = opal_db.store((opal_identifier_t*)&daemon, OPAL_DB_INTERNAL, ORTE_DB_HOSTNAME, nodename, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* now store a direct reference so we can quickly lookup the daemon from a hostname */
opal_output_verbose(2, orte_nidmap_output,
"%s storing nodename %s for daemon %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
nodename, ORTE_VPID_PRINT(daemon.vpid));
if (ORTE_SUCCESS != (rc = opal_db.store((opal_identifier_t*)ORTE_NAME_WILDCARD, OPAL_DB_INTERNAL, nodename, &daemon.vpid, OPAL_UINT32))) {
ORTE_ERROR_LOG(rc);
return rc;
}
OPAL_OUTPUT_VERBOSE((2, orte_nidmap_output,
"%s orte:util:decode:nidmap daemon %s node %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
ORTE_VPID_PRINT(daemon.vpid), nodename));
/* if this is my daemon, then store the data for me too */
if (daemon.vpid == ORTE_PROC_MY_DAEMON->vpid) {
if (ORTE_SUCCESS != (rc = opal_db.store((opal_identifier_t*)ORTE_PROC_MY_NAME, OPAL_DB_INTERNAL, ORTE_DB_HOSTNAME, nodename, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
return rc;
}
if (ORTE_SUCCESS != (rc = opal_db.store((opal_identifier_t*)ORTE_PROC_MY_NAME, OPAL_DB_INTERNAL, ORTE_DB_DAEMON_VPID, &daemon.vpid, OPAL_UINT32))) {
ORTE_ERROR_LOG(rc);
return rc;
}
}
/* if requested, unpack any aliases */
if (orte_retain_aliases) {
char *alias;
uint8_t naliases, ni;
for (ni=0; ni < naliases; ni++) {
n=1;
if (ORTE_SUCCESS != (rc = opal_dss.unpack(&buf, &naliases, &n, OPAL_UINT8))) {
if (ORTE_SUCCESS != (rc = opal_dss.unpack(&buf, &alias, &n, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
return rc;
}
for (ni=0; ni < naliases; ni++) {
n=1;
if (ORTE_SUCCESS != (rc = opal_dss.unpack(&buf, &alias, &n, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* store a cross-reference to the daemon for this nodename */
opal_output_verbose(2, orte_nidmap_output,
"%s storing alias %s for daemon %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
alias, ORTE_VPID_PRINT(daemon.vpid));
if (ORTE_SUCCESS != (rc = opal_db.store((opal_identifier_t*)ORTE_NAME_WILDCARD, OPAL_DB_INTERNAL, alias, &daemon.vpid, OPAL_UINT32))) {
ORTE_ERROR_LOG(rc);
return rc;
}
free(alias);
/* store a cross-reference to the daemon for this nodename */
opal_output_verbose(2, orte_nidmap_output,
"%s storing alias %s for daemon %s",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
alias, ORTE_VPID_PRINT(daemon.vpid));
if (ORTE_SUCCESS != (rc = opal_db.store((opal_identifier_t*)ORTE_NAME_WILDCARD, OPAL_DB_INTERNAL, alias, &daemon.vpid, OPAL_UINT32))) {
ORTE_ERROR_LOG(rc);
return rc;
}
free(alias);
}
}
@ -525,68 +521,66 @@ int orte_util_decode_daemon_nodemap(opal_byte_object_t *bo)
daemons->num_procs = num_daemons;
n=1;
while (OPAL_SUCCESS == (rc = opal_dss.unpack(&buf, &vpid, &n, ORTE_VPID))) {
if (daemons->num_procs < orte_hostname_cutoff) {
/* unpack and store the node's name */
n=1;
if (ORTE_SUCCESS != (rc = opal_dss.unpack(&buf, &name, &n, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
return rc;
}
/* do we already have this node? */
if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, vpid))) {
node = OBJ_NEW(orte_node_t);
node->name = name;
opal_pointer_array_set_item(orte_node_pool, vpid, node);
} else {
free(name);
}
/* if requested, unpack any aliases */
if (orte_retain_aliases) {
char *alias;
uint8_t naliases, ni;
n=1;
if (ORTE_SUCCESS != (rc = opal_dss.unpack(&buf, &naliases, &n, OPAL_UINT8))) {
ORTE_ERROR_LOG(rc);
return rc;
}
for (ni=0; ni < naliases; ni++) {
n=1;
if (ORTE_SUCCESS != (rc = opal_dss.unpack(&buf, &alias, &n, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
return rc;
}
opal_argv_append_nosize(&node->alias, alias);
free(alias);
}
}
}
/* unpack the oversubscribed flag */
/* unpack and store the node's name */
n=1;
if (ORTE_SUCCESS != (rc = opal_dss.unpack(&buf, &oversub, &n, OPAL_UINT8))) {
if (ORTE_SUCCESS != (rc = opal_dss.unpack(&buf, &name, &n, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
return rc;
}
if (NULL == (dptr = (orte_proc_t*)opal_pointer_array_get_item(daemons->procs, vpid))) {
dptr = OBJ_NEW(orte_proc_t);
dptr->name.jobid = ORTE_PROC_MY_NAME->jobid;
dptr->name.vpid = vpid;
opal_pointer_array_set_item(daemons->procs, vpid, dptr);
}
if (NULL != node->daemon) {
OBJ_RELEASE(node->daemon);
}
OBJ_RETAIN(dptr);
node->daemon = dptr;
if (NULL != dptr->node) {
OBJ_RELEASE(dptr->node);
}
OBJ_RETAIN(node);
dptr->node = node;
if (0 == oversub) {
node->oversubscribed = false;
/* do we already have this node? */
if (NULL == (node = (orte_node_t*)opal_pointer_array_get_item(orte_node_pool, vpid))) {
node = OBJ_NEW(orte_node_t);
node->name = name;
opal_pointer_array_set_item(orte_node_pool, vpid, node);
} else {
node->oversubscribed = true;
free(name);
}
/* if requested, unpack any aliases */
if (orte_retain_aliases) {
char *alias;
uint8_t naliases, ni;
n=1;
if (ORTE_SUCCESS != (rc = opal_dss.unpack(&buf, &naliases, &n, OPAL_UINT8))) {
ORTE_ERROR_LOG(rc);
return rc;
}
for (ni=0; ni < naliases; ni++) {
n=1;
if (ORTE_SUCCESS != (rc = opal_dss.unpack(&buf, &alias, &n, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
return rc;
}
opal_argv_append_nosize(&node->alias, alias);
free(alias);
}
}
}
/* unpack the oversubscribed flag */
n=1;
if (ORTE_SUCCESS != (rc = opal_dss.unpack(&buf, &oversub, &n, OPAL_UINT8))) {
ORTE_ERROR_LOG(rc);
return rc;
}
if (NULL == (dptr = (orte_proc_t*)opal_pointer_array_get_item(daemons->procs, vpid))) {
dptr = OBJ_NEW(orte_proc_t);
dptr->name.jobid = ORTE_PROC_MY_NAME->jobid;
dptr->name.vpid = vpid;
opal_pointer_array_set_item(daemons->procs, vpid, dptr);
}
if (NULL != node->daemon) {
OBJ_RELEASE(node->daemon);
}
OBJ_RETAIN(dptr);
node->daemon = dptr;
if (NULL != dptr->node) {
OBJ_RELEASE(dptr->node);
}
OBJ_RETAIN(node);
dptr->node = node;
if (0 == oversub) {
node->oversubscribed = false;
} else {
node->oversubscribed = true;
}
if (ORTE_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) {
ORTE_ERROR_LOG(rc);
@ -928,16 +922,14 @@ int orte_util_decode_pidmap(opal_byte_object_t *bo)
ORTE_ERROR_LOG(rc);
goto cleanup;
}
if (orte_process_info.num_daemons < orte_hostname_cutoff) {
/* lookup and store the hostname for this proc */
if (ORTE_SUCCESS != (rc = opal_db.fetch_pointer((opal_identifier_t*)&dmn, ORTE_DB_HOSTNAME, (void**)&hostname, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
if (ORTE_SUCCESS != (rc = opal_db.store((opal_identifier_t*)&proc, OPAL_DB_INTERNAL, ORTE_DB_HOSTNAME, hostname, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
/* lookup and store the hostname for this proc */
if (ORTE_SUCCESS != (rc = opal_db.fetch_pointer((opal_identifier_t*)&dmn, ORTE_DB_HOSTNAME, (void**)&hostname, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
if (ORTE_SUCCESS != (rc = opal_db.store((opal_identifier_t*)&proc, OPAL_DB_INTERNAL, ORTE_DB_HOSTNAME, hostname, OPAL_STRING))) {
ORTE_ERROR_LOG(rc);
goto cleanup;
}
}
}