ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность
0fae2caef3
openmpi/orte/util/proc_info.c
Ralph Castain 24c811805f **************************************************************** 
This change contains a non-mandatory modification
       of the MPI-RTE interface. Anyone wishing to support
       coprocessors such as the Xeon Phi may wish to add
       the required definition and underlying support
****************************************************************

Add locality support for coprocessors such as the Intel Xeon Phi.

Detecting that we are on a coprocessor inside of a host node isn't straightforward. There are no good "hooks" provided for programmatically detecting that "we are on a coprocessor running its own OS", and the ORTE daemon just thinks it is on another node. However, in order to properly use the Phi's public interface for MPI transport, it is necessary that the daemon detect that it is colocated with procs on the host.

So we have to split the locality to separately record "on the same host" vs "on the same board". We already have the board-level locality flag, but not quite enough flexibility to handle this use-case. Thus, do the following:

1. add OPAL_PROC_ON_HOST flag to indicate we share a host, but not necessarily the same board

2. modify OPAL_PROC_ON_NODE to indicate we share both a host AND the same board. Note that we have to modify the OPAL_PROC_ON_LOCAL_NODE macro to explicitly check both conditions

3. add support in opal/mca/hwloc/base/hwloc_base_util.c for the host to check for coprocessors, and for daemons to check to see if they are on a coprocessor. The former is done via hwloc, but support for the latter is not yet provided by hwloc. So the code for detecting we are on a coprocessor currently is Xeon Phi specific - hopefully, we will find more generic methods in the future.

4. modify the orted and the hnp startup so they check for coprocessors and to see if they are on a coprocessor, and have the orteds pass that info back in their callback message. Automatically detect that coprocessors have been found and identify which coprocessors are on which hosts. Note that this algo isn't scalable at the moment - this will hopefully be improved over time.

5. modify the ompi proc locality detection function to look for coprocessor host info IF the OMPI_RTE_HOST_ID database key has been defined. RTE's that choose not to provide this support do not have to do anything - the associated code will simply be ignored.

6. include some cleanup of the hwloc open/close code so it conforms to how we did things in other frameworks (e.g., having a single "frame" file instead of open/close). Also, fix the locality flags - e.g., being on the same node means you must also be on the same cluster/cu, so ensure those flags are also set.

cmr:v1.7.4:reviewer=hjelmn

This commit was SVN r29435.
2013-10-14 16:52:58 +00:00

347 строки
14 KiB
C
Исходник Ответственный История

/*
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
* University Research and Technology
* Corporation. All rights reserved.
* Copyright (c) 2004-2011 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2012 Los Alamos National Security, LLC.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "orte_config.h"
#include "orte/constants.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif
#include <ctype.h>
#include "opal/mca/base/base.h"
#include "opal/mca/base/mca_base_var.h"
#include "opal/util/argv.h"
#include "opal/util/net.h"
#include "opal/util/output.h"
#include "orte/util/proc_info.h"
/* provide a connection to a reqd variable */
extern bool orte_keep_fqdn_hostnames;
#define ORTE_NAME_INVALID {ORTE_JOBID_INVALID, ORTE_VPID_INVALID}
ORTE_DECLSPEC orte_proc_info_t orte_process_info = {
/* .my_name = */ ORTE_NAME_INVALID,
/* .my_daemon = */ ORTE_NAME_INVALID,
/* .my_daemon_uri = */ NULL,
/* .my_hnp = */ ORTE_NAME_INVALID,
/* .my_hnp_uri = */ NULL,
/* .my_parent = */ ORTE_NAME_INVALID,
/* .hnp_pid = */ 0,
/* .app_num = */ 0,
/* .num_procs = */ 1,
/* .max_procs = */ 1,
/* .num_daemons = */ 1,
/* .num_nodes = */ 1,
/* .nodename = */ NULL,
/* .pid = */ 0,
/* .proc_type = */ ORTE_PROC_TYPE_NONE,
/* .sync_buf = */ NULL,
/* .my_port = */ 0,
/* .num_restarts = */ 0,
/* .my_node_rank = */ ORTE_NODE_RANK_INVALID,
/* .my_local_rank = */ ORTE_LOCAL_RANK_INVALID,
/* .num_local_peers = */ 0,
/* .tmpdir_base = */ NULL,
/* .top_session_dir = */ NULL,
/* .job_session_dir = */ NULL,
/* .proc_session_dir = */ NULL,
/* .sock_stdin = */ NULL,
/* .sock_stdout = */ NULL,
/* .sock_stderr = */ NULL,
#if OPAL_HAVE_HWLOC
/* .cpuset = */ NULL,
#endif
/* .app_rank = */ -1,
/* .peer_modex = */ -1,
/* .peer_init_barrier = */ -1,
/* .peer_fini_barrier = */ -1,
/* .my_hostid = */ ORTE_VPID_INVALID
};
static bool init=false;
static int orte_ess_node_rank;
static int orte_peer_modex_id;
static int orte_peer_init_barrier_id;
static int orte_peer_fini_barrier_id;
static char *orte_strip_prefix;
int orte_proc_info(void)
{
int idx, i;
char *ptr;
char hostname[ORTE_MAX_HOSTNAME_SIZE];
char **prefixes;
bool match;
if (init) {
return ORTE_SUCCESS;
}
init = true;
orte_process_info.my_hnp_uri = NULL;
mca_base_var_register ("orte", "orte", NULL, "hnp_uri",
"HNP contact info",
MCA_BASE_VAR_TYPE_STRING, NULL, 0,
MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&orte_process_info.my_hnp_uri);
if (NULL != orte_process_info.my_hnp_uri) {
ptr = orte_process_info.my_hnp_uri;
/* the uri value passed to us will have quote marks around it to protect
* the value if passed on the command line. We must remove those
* to have a correct uri string
*/
if ('"' == ptr[0]) {
/* if the first char is a quote, then so will the last one be */
ptr[strlen(ptr)-1] = '\0';
memmove (ptr, ptr + 1, strlen (ptr));
}
}
orte_process_info.my_daemon_uri = NULL;
(void) mca_base_var_register ("orte", "orte", NULL, "local_daemon_uri",
"Daemon contact info",
MCA_BASE_VAR_TYPE_STRING, NULL, 0,
MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&orte_process_info.my_daemon_uri);
if (NULL != orte_process_info.my_daemon_uri) {
ptr = orte_process_info.my_daemon_uri;
/* the uri value passed to us may have quote marks around it to protect
* the value if passed on the command line. We must remove those
* to have a correct uri string
*/
if ('"' == ptr[0]) {
/* if the first char is a quote, then so will the last one be */
ptr[strlen(ptr)-1] = '\0';
memmove (ptr, ptr + 1, strlen (ptr) - 1);
}
}
orte_process_info.app_num = 0;
(void) mca_base_var_register ("orte", "orte", NULL, "app_num",
"Index of the app_context that defines this proc",
MCA_BASE_VAR_TYPE_INT, NULL, 0,
MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&orte_process_info.app_num);
/* get the process id */
orte_process_info.pid = getpid();
/* get the nodename */
gethostname(hostname, ORTE_MAX_HOSTNAME_SIZE);
if (!orte_keep_fqdn_hostnames) {
/* if the nodename is an IP address, do not mess with it! */
if (!opal_net_isaddr(hostname)) {
/* not an IP address, so remove any domain info */
if (NULL != (ptr = strchr(hostname, '.'))) {
*ptr = '\0';
}
}
}
orte_strip_prefix = NULL;
(void) mca_base_var_register ("orte", "orte", NULL, "strip_prefix",
"Prefix(es) to match when deciding whether to strip leading characters and zeroes from "
"node names returned by daemons", MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
OPAL_INFO_LVL_9, MCA_BASE_VAR_SCOPE_READONLY,
&orte_strip_prefix);
/* we have to strip node names here, if user directs, to ensure that
* the names exchanged in the modex match the names found locally
*/
if (NULL != orte_strip_prefix) {
prefixes = opal_argv_split(orte_strip_prefix, ',');
match = false;
for (i=0; NULL != prefixes[i]; i++) {
if (0 == strncmp(hostname, prefixes[i], strlen(prefixes[i]))) {
/* remove the prefix and leading zeroes */
idx = strlen(prefixes[i]);
while (idx < (int)strlen(hostname) &&
(hostname[idx] <= '0' || '9' < hostname[idx])) {
idx++;
}
if ((int)strlen(hostname) <= idx) {
/* there were no non-zero numbers in the name */
orte_process_info.nodename = strdup(&hostname[strlen(prefixes[i])]);
} else {
orte_process_info.nodename = strdup(&hostname[idx]);
}
match = true;
break;
}
}
/* if we didn't find a match, then just use the hostname as-is */
if (!match) {
orte_process_info.nodename = strdup(hostname);
}
opal_argv_free(prefixes);
} else {
orte_process_info.nodename = strdup(hostname);
}
/* get the number of nodes in the job */
orte_process_info.num_nodes = 1;
(void) mca_base_var_register ("orte", "orte", NULL, "num_nodes",
"Number of nodes in the job",
MCA_BASE_VAR_TYPE_INT, NULL, 0,
MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&orte_process_info.num_nodes);
/* get the number of times this proc has restarted */
orte_process_info.num_restarts = 0;
(void) mca_base_var_register ("orte", "orte", NULL, "num_restarts",
"Number of times this proc has restarted",
MCA_BASE_VAR_TYPE_INT, NULL, 0,
MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&orte_process_info.num_restarts);
orte_process_info.app_rank = 0;
(void) mca_base_var_register ("orte", "orte", NULL, "app_rank",
"Rank of this proc within its app_context",
MCA_BASE_VAR_TYPE_INT, NULL, 0,
MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_READONLY,
&orte_process_info.app_rank);
/* get my node rank in case we are using static ports - this won't
* be present for daemons, so don't error out if we don't have it
*/
orte_ess_node_rank = ORTE_NODE_RANK_INVALID;
(void) mca_base_var_register ("orte", "orte", NULL, "ess_node_rank", "Process node rank",
MCA_BASE_VAR_TYPE_INT, NULL, 0,
MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_CONSTANT,
&orte_ess_node_rank);
orte_process_info.my_node_rank = (orte_node_rank_t) orte_ess_node_rank;
/* setup the sync buffer */
orte_process_info.sync_buf = OBJ_NEW(opal_buffer_t);
/* get the collective id info */
orte_peer_modex_id = -1;
(void) mca_base_var_register ("orte", "orte", NULL, "peer_modex_id", "Peer modex collective id",
MCA_BASE_VAR_TYPE_INT, NULL, 0,
MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_CONSTANT,
&orte_peer_modex_id);
orte_process_info.peer_modex = (orte_grpcomm_coll_id_t) orte_peer_modex_id;
orte_peer_init_barrier_id = -1;
(void) mca_base_var_register ("orte", "orte", NULL, "peer_init_barrier_id", "Peer init barrier collective id",
MCA_BASE_VAR_TYPE_INT, NULL, 0,
MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_CONSTANT,
&orte_peer_init_barrier_id);
orte_process_info.peer_init_barrier = (orte_grpcomm_coll_id_t) orte_peer_init_barrier_id;
orte_peer_fini_barrier_id = -1;
(void) mca_base_var_register ("orte", "orte", NULL, "peer_fini_barrier_id", "Peer finalize barrier collective id",
MCA_BASE_VAR_TYPE_INT, NULL, 0,
MCA_BASE_VAR_FLAG_INTERNAL,
OPAL_INFO_LVL_9,
MCA_BASE_VAR_SCOPE_CONSTANT,
&orte_peer_fini_barrier_id);
orte_process_info.peer_fini_barrier = (orte_grpcomm_coll_id_t) orte_peer_fini_barrier_id;
return ORTE_SUCCESS;
}
int orte_proc_info_finalize(void)
{
if (!init) {
return ORTE_SUCCESS;
}
if (NULL != orte_process_info.tmpdir_base) {
free(orte_process_info.tmpdir_base);
orte_process_info.tmpdir_base = NULL;
}
if (NULL != orte_process_info.top_session_dir) {
free(orte_process_info.top_session_dir);
orte_process_info.top_session_dir = NULL;
}
if (NULL != orte_process_info.job_session_dir) {
free(orte_process_info.job_session_dir);
orte_process_info.job_session_dir = NULL;
}
if (NULL != orte_process_info.proc_session_dir) {
free(orte_process_info.proc_session_dir);
orte_process_info.proc_session_dir = NULL;
}
if (NULL != orte_process_info.nodename) {
free(orte_process_info.nodename);
orte_process_info.nodename = NULL;
}
if (NULL != orte_process_info.sock_stdin) {
free(orte_process_info.sock_stdin);
orte_process_info.sock_stdin = NULL;
}
if (NULL != orte_process_info.sock_stdout) {
free(orte_process_info.sock_stdout);
orte_process_info.sock_stdout = NULL;
}
if (NULL != orte_process_info.sock_stderr) {
free(orte_process_info.sock_stderr);
orte_process_info.sock_stderr = NULL;
}
orte_process_info.proc_type = ORTE_PROC_TYPE_NONE;
OBJ_RELEASE(orte_process_info.sync_buf);
orte_process_info.sync_buf = NULL;
init = false;
return ORTE_SUCCESS;
}
Показать git blame Копировать постоянную ссылку