ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Singletons currently start out by spawning an HNP - this is required solely in the cases where the singleton subsequently calls MPI_Comm_spawn or publishes port info without support from an external orte-server. In all other cases, the HNP is of no value and can actually be a detriment by creating additional overhead on the node. This is particularly concerning for async operations where processes may begin as singletons and then dynamically wireup to perform pt2pt communications.

Просмотр исходного кода 
So we now allow singletons to start on their own, only spawning an HNP when initiating an operation that actually requires it.

cmr:v1.7.4:reviewer=jsquyres

This commit was SVN r29354.
Этот коммит содержится в:
Ralph Castain 2013-10-04 02:58:26 +00:00
родитель 4dd1c86b36
Коммит f4f2287958
11 изменённых файлов: 417 добавлений и 321 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

18
ompi/mca/dpm/orte/dpm_orte.c
Просмотреть файл

@ -43,7 +43,7 @@
#include "orte/mca/errmgr/errmgr.h"
#include "orte/mca/grpcomm/base/base.h"
#include "orte/mca/plm/plm.h"
#include "orte/mca/plm/base/base.h"
#include "orte/mca/rml/rml.h"
#include "orte/mca/rml/rml_types.h"
#include "orte/mca/rmaps/rmaps.h"
@ -1422,6 +1422,22 @@ static int open_port(char *port_name, orte_rml_tag_t given_tag)
int rc, len;
char tag[12];
/* if we are a singleton and the supporting HNP hasn't
* been spawned, then do so now
*/
if ((orte_process_info.proc_type & ORTE_PROC_SINGLETON) &&
!orte_routing_is_enabled) {
if (ORTE_SUCCESS != orte_plm_base_fork_hnp()) {
ORTE_ERROR_LOG(ORTE_ERR_FATAL);
return ORTE_ERR_FATAL;
}
orte_routing_is_enabled = true;
/* need to init_routes again to redirect messages
* thru the HNP
*/
orte_routed.init_routes(ORTE_PROC_MY_NAME->jobid, NULL);
}
if (NULL == orte_process_info.my_hnp_uri) {
rc = OMPI_ERR_NOT_AVAILABLE;
ORTE_ERROR_LOG(rc);

19
orte/mca/ess/hnp/ess_hnp_module.c
Просмотреть файл

@ -284,12 +284,21 @@ static int rte_init(void)
error = "orte_plm_base_select";
goto error;
}
if (ORTE_SUCCESS != (ret = orte_plm.set_hnp_name())) {
ORTE_ERROR_LOG(ret);
error = "orte_plm_set_hnp_name";
goto error;
/* if we were spawned by a singleton, our jobid was given to us */
if (NULL != orte_ess_base_jobid) {
if (ORTE_SUCCESS != (ret = orte_util_convert_string_to_jobid(&ORTE_PROC_MY_NAME->jobid, orte_ess_base_jobid))) {
ORTE_ERROR_LOG(ret);
error = "convert_string_to_jobid";
goto error;
}
ORTE_PROC_MY_NAME->vpid = 0;
} else {
if (ORTE_SUCCESS != (ret = orte_plm.set_hnp_name())) {
ORTE_ERROR_LOG(ret);
error = "orte_plm_set_hnp_name";
goto error;
}
}
/* Setup the communication infrastructure */
/*

342
orte/mca/ess/singleton/ess_singleton_module.c
Просмотреть файл

@ -11,6 +11,7 @@
* All rights reserved.
* Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2011 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2013 Intel, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -51,18 +52,6 @@
#include "orte/mca/ess/base/base.h"
#include "orte/mca/ess/singleton/ess_singleton.h"
static int fork_hnp(void);
static void set_handler_default(int sig)
{
struct sigaction act;
act.sa_handler = SIG_DFL;
act.sa_flags = 0;
sigemptyset(&act.sa_mask);
sigaction(sig, &act, (struct sigaction *)0);
}
static int rte_init(void);
static int rte_finalize(void);
@ -136,82 +125,46 @@ static int rte_init(void)
}
/* save the daemon uri - we will process it later */
orte_process_info.my_daemon_uri = strdup(orte_process_info.my_hnp_uri);
/* indicate we are a singleton so orte_init knows what to do */
orte_process_info.proc_type |= ORTE_PROC_SINGLETON;
/* for convenience, push the pubsub version of this param into the environ */
asprintf(&param,"OMPI_MCA_pubsub_orte_server=%s",orte_process_info.my_hnp_uri);
putenv(param);
/* now define my own name */
/* hash the nodename */
OPAL_HASH_STR(orte_process_info.nodename, hash32);
bias = (uint32_t)orte_process_info.pid;
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_framework.framework_output,
"ess:singleton: initial bias %ld nodename hash %lu",
(long)bias, (unsigned long)hash32));
/* fold in the bias */
hash32 = hash32 ^ bias;
/* now compress to 16-bits */
jobfam = (uint16_t)(((0x0000ffff & (0xffff0000 & hash32) >> 16)) ^ (0x0000ffff & hash32));
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_framework.framework_output,
"ess:singleton:: final jobfam %lu",
(unsigned long)jobfam));
/* set the name */
ORTE_PROC_MY_NAME->jobid = 0xffff0000 & ((uint32_t)jobfam << 16);
ORTE_PROC_MY_NAME->vpid = 0;
} else {
/*
* If we are the selected module, then we must be a singleton
* as it means that no other method for discovering a name
* could be found. In this case, we need to start a daemon that
* can support our operation. We must do this for two reasons:
*
* (1) if we try to play the role of the HNP, then any child processes
* we might start via comm_spawn will rely on us for all ORTE-level
* support. However, we can only progress those requests when the
* the application calls into the OMPI/ORTE library! Thus, if this
* singleton just does computation, the other processes will "hang"
* in any calls into the ORTE layer that communicate with the HNP -
* and most calls on application procs *do*.
*
* (2) daemons are used to communicate messages for administrative
* purposes in a broadcast-like manner. Thus, daemons are expected
* to be able to interpret specific commands. Our application process
* doesn't have any idea how to handle those commands, thus causing
* the entire ORTE administrative system to break down.
*
* For those reasons, we choose to fork/exec a daemon at this time
* and then reconnect ourselves to it. We could just "fork" and declare
* the child to be a daemon, but that would require we place *all* of the
* daemon command processing code in the ORTE library, do some strange
* mojo in a few places, etc. This doesn't seem worth it, so we'll just
* do the old fork/exec here
*/
if (ORTE_SUCCESS != (rc= fork_hnp())) {
/* if this didn't work, then we cannot support operation any further.
* Abort the system and tell orte_init to exit
*/
ORTE_ERROR_LOG(rc);
return rc;
}
}
orte_process_info.num_procs = 1;
/* indicate we are a singleton so orte_init knows what to do */
orte_process_info.proc_type |= ORTE_PROC_SINGLETON;
/* now define my own name */
/* hash the nodename */
OPAL_HASH_STR(orte_process_info.nodename, hash32);
bias = (uint32_t)orte_process_info.pid;
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_framework.framework_output,
"ess:singleton: initial bias %ld nodename hash %lu",
(long)bias, (unsigned long)hash32));
/* fold in the bias */
hash32 = hash32 ^ bias;
/* now compress to 16-bits */
jobfam = (uint16_t)(((0x0000ffff & (0xffff0000 & hash32) >> 16)) ^ (0x0000ffff & hash32));
OPAL_OUTPUT_VERBOSE((5, orte_ess_base_framework.framework_output,
"ess:singleton:: final jobfam %lu",
(unsigned long)jobfam));
/* set the name - if we eventually spawn an HNP, it will use
* local jobid 0, so offset us by 1
*/
ORTE_PROC_MY_NAME->jobid = (0xffff0000 & ((uint32_t)jobfam << 16)) + 1;
ORTE_PROC_MY_NAME->vpid = 0;
orte_process_info.num_procs = 1;
if (orte_process_info.max_procs < orte_process_info.num_procs) {
orte_process_info.max_procs = orte_process_info.num_procs;
}
/* NOTE: do not wireup our io - let the fork'd orted serve
* as our io handler. This prevents issues with the event
* library wrt pty's and stdin
*/
/* flag that we are not routing since we have no HNP */
orte_routing_is_enabled = false;
/* use the std app init to complete the procedure */
if (ORTE_SUCCESS != (rc = orte_ess_base_app_setup(true))) {
@ -230,6 +183,10 @@ static int rte_init(void)
orte_process_info.peer_init_barrier = 1;
orte_process_info.peer_fini_barrier = 2;
/* to the best of our knowledge, we are alone */
orte_process_info.my_node_rank = 0;
orte_process_info.my_local_rank = 0;
/* set some envars */
putenv("OMPI_NUM_APP_CTX=1");
putenv("OMPI_FIRST_RANKS=0");
@ -250,226 +207,13 @@ static int rte_finalize(void)
if (ORTE_SUCCESS != (ret = orte_ess_base_app_finalize())) {
ORTE_ERROR_LOG(ret);
}
/* cleanup the environment */
unsetenv("OMPI_NUM_APP_CTX");
unsetenv("OMPI_FIRST_RANKS");
unsetenv("OMPI_APP_CTX_NUM_PROCS");
unsetenv("OMPI_MCA_orte_ess_num_procs");
unsetenv("OMPI_MCA_pubsub_orte_server"); // just in case it is there
return ret;
}
#define ORTE_URI_MSG_LGTH 256
static int fork_hnp(void)
{
int p[2], death_pipe[2];
char *cmd;
char **argv = NULL;
int argc;
char *param;
sigset_t sigs;
int buffer_length, num_chars_read, chunk;
char *orted_uri;
int rc;
/* A pipe is used to communicate between the parent and child to
indicate whether the exec ultimately succeeded or failed. The
child sets the pipe to be close-on-exec; the child only ever
writes anything to the pipe if there is an error (e.g.,
executable not found, exec() fails, etc.). The parent does a
blocking read on the pipe; if the pipe closed with no data,
then the exec() succeeded. If the parent reads something from
the pipe, then the child was letting us know that it failed.
*/
if (pipe(p) < 0) {
ORTE_ERROR_LOG(ORTE_ERR_SYS_LIMITS_PIPES);
return ORTE_ERR_SYS_LIMITS_PIPES;
}
/* we also have to give the HNP a pipe it can watch to know when
* we terminated. Since the HNP is going to be a child of us, it
* can't just use waitpid to see when we leave - so it will watch
* the pipe instead
*/
if (pipe(death_pipe) < 0) {
ORTE_ERROR_LOG(ORTE_ERR_SYS_LIMITS_PIPES);
return ORTE_ERR_SYS_LIMITS_PIPES;
}
/* find the orted binary using the install_dirs support - this also
* checks to ensure that we can see this executable and it *is* executable by us
*/
cmd = opal_path_access("orted", opal_install_dirs.bindir, X_OK);
if (NULL == cmd) {
/* guess we couldn't do it - best to abort */
ORTE_ERROR_LOG(ORTE_ERR_FILE_NOT_EXECUTABLE);
close(p[0]);
close(p[1]);
return ORTE_ERR_FILE_NOT_EXECUTABLE;
}
/* okay, setup an appropriate argv */
opal_argv_append(&argc, &argv, "orted");
/* tell the daemon it is to be the HNP */
opal_argv_append(&argc, &argv, "--hnp");
/* tell the daemon to get out of our process group */
opal_argv_append(&argc, &argv, "--set-sid");
/* tell the daemon to report back its uri so we can connect to it */
opal_argv_append(&argc, &argv, "--report-uri");
asprintf(&param, "%d", p[1]);
opal_argv_append(&argc, &argv, param);
free(param);
/* give the daemon a pipe it can watch to tell when we have died */
opal_argv_append(&argc, &argv, "--singleton-died-pipe");
asprintf(&param, "%d", death_pipe[0]);
opal_argv_append(&argc, &argv, param);
free(param);
/* add any debug flags */
if (orte_debug_flag) {
opal_argv_append(&argc, &argv, "--debug");
}
if (orte_debug_daemons_flag) {
opal_argv_append(&argc, &argv, "--debug-daemons");
}
if (orte_debug_daemons_file_flag) {
if (!orte_debug_daemons_flag) {
opal_argv_append(&argc, &argv, "--debug-daemons");
}
opal_argv_append(&argc, &argv, "--debug-daemons-file");
}
/* indicate that it must use the novm state machine */
opal_argv_append(&argc, &argv, "-mca");
opal_argv_append(&argc, &argv, "state_novm_select");
opal_argv_append(&argc, &argv, "1");
/* Fork off the child */
orte_process_info.hnp_pid = fork();
if(orte_process_info.hnp_pid < 0) {
ORTE_ERROR_LOG(ORTE_ERR_SYS_LIMITS_CHILDREN);
close(p[0]);
close(p[1]);
close(death_pipe[0]);
close(death_pipe[1]);
free(cmd);
opal_argv_free(argv);
return ORTE_ERR_SYS_LIMITS_CHILDREN;
}
if (orte_process_info.hnp_pid == 0) {
close(p[0]);
close(death_pipe[1]);
/* I am the child - exec me */
/* Set signal handlers back to the default. Do this close
to the execve() because the event library may (and likely
will) reset them. If we don't do this, the event
library may have left some set that, at least on some
OS's, don't get reset via fork() or exec(). Hence, the
orted could be unkillable (for example). */
set_handler_default(SIGTERM);
set_handler_default(SIGINT);
set_handler_default(SIGHUP);
set_handler_default(SIGPIPE);
set_handler_default(SIGCHLD);
/* Unblock all signals, for many of the same reasons that
we set the default handlers, above. This is noticable
on Linux where the event library blocks SIGTERM, but we
don't want that blocked by the orted (or, more
specifically, we don't want it to be blocked by the
orted and then inherited by the ORTE processes that it
forks, making them unkillable by SIGTERM). */
sigprocmask(0, 0, &sigs);
sigprocmask(SIG_UNBLOCK, &sigs, 0);
execv(cmd, argv);
/* if I get here, the execv failed! */
orte_show_help("help-ess-base.txt", "ess-base:execv-error",
true, cmd, strerror(errno));
exit(1);
} else {
/* I am the parent - wait to hear something back and
* report results
*/
close(p[1]); /* parent closes the write - orted will write its contact info to it*/
close(death_pipe[0]); /* parent closes the death_pipe's read */
opal_argv_free(argv);
/* setup the buffer to read the name + uri */
buffer_length = ORTE_URI_MSG_LGTH;
chunk = ORTE_URI_MSG_LGTH-1;
num_chars_read = 0;
orted_uri = (char*)malloc(buffer_length);
while (chunk == (rc = read(p[0], &orted_uri[num_chars_read], chunk))) {
/* we read an entire buffer - better get more */
num_chars_read += chunk;
buffer_length += ORTE_URI_MSG_LGTH;
orted_uri = realloc((void*)orted_uri, buffer_length);
}
num_chars_read += rc;
if (num_chars_read <= 0) {
/* we didn't get anything back - this is bad */
ORTE_ERROR_LOG(ORTE_ERR_HNP_COULD_NOT_START);
free(orted_uri);
return ORTE_ERR_HNP_COULD_NOT_START;
}
if (']' != orted_uri[strlen(orted_uri)-1]) {
ORTE_ERROR_LOG(ORTE_ERR_COMM_FAILURE);
free(orted_uri);
return ORTE_ERR_COMM_FAILURE;
}
orted_uri[strlen(orted_uri)-1] = '\0';
/* parse the sysinfo from the returned info */
if (NULL == (param = strrchr(orted_uri, '['))) {
ORTE_ERROR_LOG(ORTE_ERR_COMM_FAILURE);
free(orted_uri);
return ORTE_ERR_COMM_FAILURE;
}
param[-1] = '\0'; /* terminate the string */
if (ORTE_SUCCESS != (rc = orte_util_convert_string_to_sysinfo(&orte_local_cpu_type,
&orte_local_cpu_model, ++param))) {
ORTE_ERROR_LOG(rc);
free(orted_uri);
return rc;
}
/* parse the name from the returned info */
if (NULL == (param = strrchr(orted_uri, '['))) {
ORTE_ERROR_LOG(ORTE_ERR_COMM_FAILURE);
free(orted_uri);
return ORTE_ERR_COMM_FAILURE;
}
*param = '\0'; /* terminate the string */
param++;
if (ORTE_SUCCESS != (rc = orte_util_convert_string_to_process_name(ORTE_PROC_MY_NAME, param))) {
ORTE_ERROR_LOG(rc);
free(orted_uri);
return rc;
}
/* save the daemon uri - we will process it later */
orte_process_info.my_daemon_uri = strdup(orted_uri);
/* likewise, since this is also the HNP, set that uri too */
orte_process_info.my_hnp_uri = strdup(orted_uri);
/* indicate we are a singleton so orte_init knows what to do */
orte_process_info.proc_type |= ORTE_PROC_SINGLETON;
/* all done - report success */
free(orted_uri);
return ORTE_SUCCESS;
}
}

34
orte/mca/grpcomm/bad/grpcomm_bad_module.c
Просмотреть файл

@ -142,6 +142,23 @@ static void process_barrier(int fd, short args, void *cbdata)
opal_buffer_t *buf;
orte_namelist_t *nm;
/* if we are a singleton and routing isn't enabled,
* then we have nobody with which to communicate, so
* we can just declare success
*/
if ((orte_process_info.proc_type & ORTE_PROC_SINGLETON) &&
!orte_routing_is_enabled) {
if (NULL != coll->cbfunc) {
OPAL_OUTPUT_VERBOSE((2, orte_grpcomm_base_framework.framework_output,
"%s CALLING BARRIER RELEASE",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
coll->cbfunc(NULL, coll->cbdata);
}
/* flag the collective as complete */
coll->active = false;
return;
}
/* setup the collective */
opal_list_append(&orte_grpcomm_base.active_colls, &coll->super);
@ -199,6 +216,23 @@ static void process_allgather(int fd, short args, void *cbdata)
orte_namelist_t *nm;
opal_list_item_t *item;
/* if we are a singleton and routing isn't enabled,
* then we have nobody with which to communicate, so
* we can just declare success
*/
if ((orte_process_info.proc_type & ORTE_PROC_SINGLETON) &&
!orte_routing_is_enabled) {
if (NULL != gather->cbfunc) {
OPAL_OUTPUT_VERBOSE((2, orte_grpcomm_base_framework.framework_output,
"%s CALLING GATHER RELEASE",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
gather->cbfunc(NULL, gather->cbdata);
}
/* flag the collective as complete */
gather->active = false;
return;
}
/* if this is an original request, then record the collective */
if (NULL == gather->next_cb) {
opal_list_append(&orte_grpcomm_base.active_colls, &gather->super);

17
orte/mca/grpcomm/base/grpcomm_base_modex.c
Просмотреть файл

@ -79,6 +79,23 @@ void orte_grpcomm_base_modex(int fd, short args, void *cbdata)
"%s grpcomm:base:modex: performing modex",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* if we are a singleton and routing isn't enabled,
* then we have nobody with which to communicate, so
* we can just declare success
*/
if ((orte_process_info.proc_type & ORTE_PROC_SINGLETON) &&
!orte_routing_is_enabled) {
if (NULL != modex->cbfunc) {
OPAL_OUTPUT_VERBOSE((2, orte_grpcomm_base_framework.framework_output,
"%s CALLING MODEX RELEASE",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
modex->cbfunc(NULL, modex->cbdata);
}
/* flag the collective as complete */
modex->active = false;
return;
}
if (0 == opal_list_get_size(&modex->participants)) {
/* record the collective */
modex->next_cbdata = modex;

5
orte/mca/odls/base/odls_base_default_fns.c
Просмотреть файл

@ -14,6 +14,7 @@
* Copyright (c) 2011-2013 Los Alamos National Security, LLC.
* All rights reserved.
* Copyright (c) 2011-2013 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2013 Intel, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -794,7 +795,7 @@ static int setup_child(orte_proc_t *child,
ORTE_ERROR_LOG(rc);
return rc;
}
if (OPAL_SUCCESS != mca_base_var_env_name ("orte_ess_jobid", &param)) {
if (OPAL_SUCCESS != mca_base_var_env_name ("ess_base_jobid", &param)) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
rc = ORTE_ERR_OUT_OF_RESOURCE;
return rc;
@ -808,7 +809,7 @@ static int setup_child(orte_proc_t *child,
ORTE_ERROR_LOG(rc);
return rc;
}
if (OPAL_SUCCESS != mca_base_var_env_name ("orte_ess_vpid", &param)) {
if (OPAL_SUCCESS != mca_base_var_env_name ("ess_base_vpid", &param)) {
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
rc = ORTE_ERR_OUT_OF_RESOURCE;
return rc;

1
orte/mca/plm/base/base.h
Просмотреть файл

@ -63,6 +63,7 @@ ORTE_DECLSPEC void orte_plm_base_mapping_complete(int fd, short args, void *cbda
ORTE_DECLSPEC void orte_plm_base_launch_apps(int fd, short args, void *cbdata);
ORTE_DECLSPEC void orte_plm_base_post_launch(int fd, short args, void *cbdata);
ORTE_DECLSPEC void orte_plm_base_registered(int fd, short args, void *cbdata);
ORTE_DECLSPEC int orte_plm_base_fork_hnp(void);
END_C_DECLS

268
orte/mca/plm/base/plm_base_proxy.c
Просмотреть файл

@ -12,6 +12,7 @@
* Copyright (c) 2007 Cisco Systems, Inc. All rights reserved.
* Copyright (c) 2011-2012 Los Alamos National Security, LLC.
* All rights reserved.
* Copyright (c) 2013 Intel, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -24,12 +25,18 @@
#include "orte/constants.h"
#include "opal/dss/dss.h"
#include "opal/util/path.h"
#include "opal/mca/installdirs/installdirs.h"
#include "orte/util/name_fns.h"
#include "orte/util/show_help.h"
#include "orte/mca/errmgr/errmgr.h"
#include "orte/mca/rml/rml.h"
#include "orte/mca/rml/rml_types.h"
#include "orte/mca/errmgr/errmgr.h"
#include "orte/mca/routed/routed.h"
#include "orte/runtime/orte_globals.h"
#include "orte/mca/plm/base/base.h"
#include "orte/mca/plm/base/plm_private.h"
int orte_plm_proxy_init(void)
@ -94,6 +101,26 @@ int orte_plm_proxy_spawn(orte_job_t *jdata)
"%s plm:base:proxy spawn child job",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
/* if we are a singleton and the supporting HNP hasn't
* been spawned, then do so now
*/
if ((orte_process_info.proc_type & ORTE_PROC_SINGLETON) &&
!orte_routing_is_enabled) {
OPAL_OUTPUT_VERBOSE((5, orte_plm_base_framework.framework_output,
"%s plm:base:proxy spawn HNP for support",
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
if (ORTE_SUCCESS != orte_plm_base_fork_hnp()) {
ORTE_ERROR_LOG(ORTE_ERR_FATAL);
return ORTE_ERR_FATAL;
}
orte_routing_is_enabled = true;
/* need to init_routes again to redirect messages
* thru the HNP
*/
orte_routed.init_routes(ORTE_PROC_MY_NAME->jobid, NULL);
}
/* setup the buffer */
buf = OBJ_NEW(opal_buffer_t);
@ -143,7 +170,7 @@ int orte_plm_proxy_spawn(orte_job_t *jdata)
/* cleanup the memory */
OBJ_RELEASE(ps);
CLEANUP:
CLEANUP:
return rc;
}
@ -151,3 +178,240 @@ int orte_plm_proxy_finalize(void)
{
return ORTE_SUCCESS;
}
#define ORTE_URI_MSG_LGTH 256
static void set_handler_default(int sig)
{
struct sigaction act;
act.sa_handler = SIG_DFL;
act.sa_flags = 0;
sigemptyset(&act.sa_mask);
sigaction(sig, &act, (struct sigaction *)0);
}
int orte_plm_base_fork_hnp(void)
{
int p[2], death_pipe[2];
char *cmd;
char **argv = NULL;
int argc;
char *param;
sigset_t sigs;
int buffer_length, num_chars_read, chunk;
char *orted_uri;
int rc;
char *foo;
orte_jobid_t jobid;
/* A pipe is used to communicate between the parent and child to
indicate whether the exec ultimately succeeded or failed. The
child sets the pipe to be close-on-exec; the child only ever
writes anything to the pipe if there is an error (e.g.,
executable not found, exec() fails, etc.). The parent does a
blocking read on the pipe; if the pipe closed with no data,
then the exec() succeeded. If the parent reads something from
the pipe, then the child was letting us know that it failed.
*/
if (pipe(p) < 0) {
ORTE_ERROR_LOG(ORTE_ERR_SYS_LIMITS_PIPES);
return ORTE_ERR_SYS_LIMITS_PIPES;
}
/* we also have to give the HNP a pipe it can watch to know when
* we terminated. Since the HNP is going to be a child of us, it
* can't just use waitpid to see when we leave - so it will watch
* the pipe instead
*/
if (pipe(death_pipe) < 0) {
ORTE_ERROR_LOG(ORTE_ERR_SYS_LIMITS_PIPES);
return ORTE_ERR_SYS_LIMITS_PIPES;
}
/* find the orted binary using the install_dirs support - this also
* checks to ensure that we can see this executable and it *is* executable by us
*/
cmd = opal_path_access("orted", opal_install_dirs.bindir, X_OK);
if (NULL == cmd) {
/* guess we couldn't do it - best to abort */
ORTE_ERROR_LOG(ORTE_ERR_FILE_NOT_EXECUTABLE);
close(p[0]);
close(p[1]);
return ORTE_ERR_FILE_NOT_EXECUTABLE;
}
/* okay, setup an appropriate argv */
opal_argv_append(&argc, &argv, "orted");
/* tell the daemon it is to be the HNP */
opal_argv_append(&argc, &argv, "--hnp");
/* tell the daemon to get out of our process group */
opal_argv_append(&argc, &argv, "--set-sid");
/* tell the daemon to report back its uri so we can connect to it */
opal_argv_append(&argc, &argv, "--report-uri");
asprintf(&param, "%d", p[1]);
opal_argv_append(&argc, &argv, param);
free(param);
/* give the daemon a pipe it can watch to tell when we have died */
opal_argv_append(&argc, &argv, "--singleton-died-pipe");
asprintf(&param, "%d", death_pipe[0]);
opal_argv_append(&argc, &argv, param);
free(param);
/* add any debug flags */
if (orte_debug_flag) {
opal_argv_append(&argc, &argv, "--debug");
}
if (orte_debug_daemons_flag) {
opal_argv_append(&argc, &argv, "--debug-daemons");
}
if (orte_debug_daemons_file_flag) {
if (!orte_debug_daemons_flag) {
opal_argv_append(&argc, &argv, "--debug-daemons");
}
opal_argv_append(&argc, &argv, "--debug-daemons-file");
}
/* indicate that it must use the novm state machine */
opal_argv_append(&argc, &argv, "-mca");
opal_argv_append(&argc, &argv, "state_novm_select");
opal_argv_append(&argc, &argv, "1");
/* pass it a jobid to match my job family */
opal_argv_append(&argc, &argv, "-mca");
opal_argv_append(&argc, &argv, "ess_base_jobid");
jobid = ORTE_DAEMON_JOBID(ORTE_PROC_MY_NAME->jobid);
if (ORTE_SUCCESS != (rc = orte_util_convert_jobid_to_string(&param, jobid))) {
ORTE_ERROR_LOG(rc);
return rc;
}
opal_argv_append(&argc, &argv, param);
free(param);
foo = opal_argv_join(argv, ' ');
opal_output(0, "%s FORKING HNP: %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), foo);
free(foo);
/* Fork off the child */
orte_process_info.hnp_pid = fork();
if(orte_process_info.hnp_pid < 0) {
ORTE_ERROR_LOG(ORTE_ERR_SYS_LIMITS_CHILDREN);
close(p[0]);
close(p[1]);
close(death_pipe[0]);
close(death_pipe[1]);
free(cmd);
opal_argv_free(argv);
return ORTE_ERR_SYS_LIMITS_CHILDREN;
}
if (orte_process_info.hnp_pid == 0) {
close(p[0]);
close(death_pipe[1]);
/* I am the child - exec me */
/* Set signal handlers back to the default. Do this close
to the execve() because the event library may (and likely
will) reset them. If we don't do this, the event
library may have left some set that, at least on some
OS's, don't get reset via fork() or exec(). Hence, the
orted could be unkillable (for example). */
set_handler_default(SIGTERM);
set_handler_default(SIGINT);
set_handler_default(SIGHUP);
set_handler_default(SIGPIPE);
set_handler_default(SIGCHLD);
/* Unblock all signals, for many of the same reasons that
we set the default handlers, above. This is noticable
on Linux where the event library blocks SIGTERM, but we
don't want that blocked by the orted (or, more
specifically, we don't want it to be blocked by the
orted and then inherited by the ORTE processes that it
forks, making them unkillable by SIGTERM). */
sigprocmask(0, 0, &sigs);
sigprocmask(SIG_UNBLOCK, &sigs, 0);
execv(cmd, argv);
/* if I get here, the execv failed! */
orte_show_help("help-ess-base.txt", "ess-base:execv-error",
true, cmd, strerror(errno));
exit(1);
} else {
/* I am the parent - wait to hear something back and
* report results
*/
close(p[1]); /* parent closes the write - orted will write its contact info to it*/
close(death_pipe[0]); /* parent closes the death_pipe's read */
opal_argv_free(argv);
/* setup the buffer to read the HNP's uri */
buffer_length = ORTE_URI_MSG_LGTH;
chunk = ORTE_URI_MSG_LGTH-1;
num_chars_read = 0;
orted_uri = (char*)malloc(buffer_length);
while (chunk == (rc = read(p[0], &orted_uri[num_chars_read], chunk))) {
/* we read an entire buffer - better get more */
num_chars_read += chunk;
buffer_length += ORTE_URI_MSG_LGTH;
orted_uri = realloc((void*)orted_uri, buffer_length);
}
num_chars_read += rc;
if (num_chars_read <= 0) {
/* we didn't get anything back - this is bad */
ORTE_ERROR_LOG(ORTE_ERR_HNP_COULD_NOT_START);
free(orted_uri);
return ORTE_ERR_HNP_COULD_NOT_START;
}
if (']' != orted_uri[strlen(orted_uri)-1]) {
ORTE_ERROR_LOG(ORTE_ERR_COMM_FAILURE);
free(orted_uri);
return ORTE_ERR_COMM_FAILURE;
}
orted_uri[strlen(orted_uri)-1] = '\0';
/* parse the sysinfo from the returned info */
if (NULL == (param = strrchr(orted_uri, '['))) {
ORTE_ERROR_LOG(ORTE_ERR_COMM_FAILURE);
free(orted_uri);
return ORTE_ERR_COMM_FAILURE;
}
*param = '\0'; /* terminate the uri string */
if (ORTE_SUCCESS != (rc = orte_util_convert_string_to_sysinfo(&orte_local_cpu_type,
&orte_local_cpu_model, ++param))) {
ORTE_ERROR_LOG(rc);
free(orted_uri);
return rc;
}
/* save the daemon uri - we will process it later */
orte_process_info.my_daemon_uri = strdup(orted_uri);
/* Set the contact info in the RML - this won't actually establish
* the connection, but just tells the RML how to reach the daemon
* if/when we attempt to send to it
*/
orte_rml.set_contact_info(orte_process_info.my_daemon_uri);
/* likewise, since this is also the HNP, set that uri too */
orte_process_info.my_hnp_uri = strdup(orted_uri);
/* all done - report success */
free(orted_uri);
return ORTE_SUCCESS;
}
}

18
orte/mca/routed/binomial/routed_binomial.c
Просмотреть файл

@ -105,9 +105,10 @@ static int finalize(void)
opal_list_item_t *item;
/* if I am an application process, indicate that I am
* truly finalizing prior to departure
*/
if (ORTE_PROC_IS_APP) {
* truly finalizing prior to departure if I have
* an HNP/daemon monitoring me
*/
if (ORTE_PROC_IS_APP && orte_routing_is_enabled) {
if (ORTE_SUCCESS != (rc = orte_routed_base_register_sync(false))) {
ORTE_ERROR_LOG(rc);
return rc;
@ -640,6 +641,15 @@ static int init_routes(orte_jobid_t job, opal_buffer_t *ndat)
(NULL == orte_process_info.my_hnp_uri) ? "NULL" : orte_process_info.my_hnp_uri,
(NULL == orte_process_info.my_daemon_uri) ? "NULL" : orte_process_info.my_daemon_uri));
/* if we are a singleton and we have not spawned our
* supporting HNP, then we don't route and don't need
* the corresponding URIs
*/
if ((orte_process_info.proc_type & ORTE_PROC_SINGLETON) &&
!orte_routing_is_enabled) {
return ORTE_SUCCESS;
}
if (NULL == orte_process_info.my_daemon_uri) {
/* in this module, we absolutely MUST have this information - if
* we didn't get it, then error out
@ -700,7 +710,7 @@ static int init_routes(orte_jobid_t job, opal_buffer_t *ndat)
return rc;
}
/* no answer is expected or coming */
orte_routing_is_enabled = true;
return ORTE_SUCCESS;
}
}

11
orte/orted/orted_main.c
Просмотреть файл

@ -15,6 +15,7 @@
* Copyright (c) 2009 Institut National de Recherche en Informatique
* et Automatique. All rights reserved.
* Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2013 Intel, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
@ -465,7 +466,7 @@ int orte_daemon(int argc, char *argv[])
but what the heck... :-) */
opal_progress_set_event_flag(OPAL_EVLOOP_ONCE);
/* if requested, obtain and report a new process name and my uri to the indicated pipe */
/* if requested, report my uri to the indicated pipe */
if (orted_globals.uri_pipe > 0) {
orte_job_t *jdata;
orte_proc_t *proc;
@ -529,7 +530,7 @@ int orte_daemon(int argc, char *argv[])
proc->app_idx = 0;
proc->local_proc = true;
/* create the collectives for its modex/barriers */
/* account for the collectives in its modex/barriers */
jdata->peer_modex = orte_grpcomm_base_get_coll_id();
coll = orte_grpcomm_base_setup_collective(jdata->peer_modex);
nm = OBJ_NEW(orte_namelist_t);
@ -568,11 +569,9 @@ int orte_daemon(int argc, char *argv[])
goto DONE;
}
/* create a string that contains our uri + the singleton's name + sysinfo */
orte_util_convert_process_name_to_string(&nptr, &proc->name);
/* create a string that contains our uri + sysinfo */
orte_util_convert_sysinfo_to_string(&sysinfo, orte_local_cpu_type, orte_local_cpu_model);
asprintf(&tmp, "%s[%s][%s]", orte_process_info.my_daemon_uri, nptr, sysinfo);
free(nptr);
asprintf(&tmp, "%s[%s]", orte_process_info.my_daemon_uri, sysinfo);
free(sysinfo);
/* pass that info to the singleton */

1
orte/runtime/orte_globals.h
Просмотреть файл

@ -627,6 +627,7 @@ ORTE_DECLSPEC extern char *orte_set_slots;
ORTE_DECLSPEC extern bool orte_display_allocation;
ORTE_DECLSPEC extern bool orte_display_devel_allocation;
ORTE_DECLSPEC extern bool orte_soft_locations;
ORTE_DECLSPEC extern bool orte_hnp_connected;
/* launch agents */
ORTE_DECLSPEC extern char *orte_launch_agent;