From 57f6b94fa53166bc4d513be4507382e832a3a8c7 Mon Sep 17 00:00:00 2001 From: Ralph Castain Date: Wed, 3 Oct 2018 08:33:50 -0700 Subject: [PATCH] Cleanup race condition in finalize See https://github.com/open-mpi/ompi/issues/5798#issuecomment-426545893 for a lengthy explanation Signed-off-by: Ralph Castain --- orte/mca/ess/base/Makefile.am | 3 +- orte/mca/ess/base/base.h | 3 - orte/mca/ess/base/ess_base_std_app.c | 240 ------------------ orte/mca/ess/lsf/ess_lsf_module.c | 17 -- orte/mca/ess/pmi/ess_pmi_module.c | 179 ++++++++++++- orte/mca/ess/singleton/ess_singleton_module.c | 215 +++++++++++++++- orte/mca/ess/slurm/ess_slurm_module.c | 8 - orte/mca/ess/tm/ess_tm_module.c | 8 - 8 files changed, 368 insertions(+), 305 deletions(-) delete mode 100644 orte/mca/ess/base/ess_base_std_app.c diff --git a/orte/mca/ess/base/Makefile.am b/orte/mca/ess/base/Makefile.am index db1903699c..70528b9d8a 100644 --- a/orte/mca/ess/base/Makefile.am +++ b/orte/mca/ess/base/Makefile.am @@ -10,7 +10,7 @@ # Copyright (c) 2004-2005 The Regents of the University of California. # All rights reserved. # Copyright (c) 2013 Los Alamos National Security, LLC. All rights reserved. -# Copyright (c) 2015-2017 Intel, Inc. All rights reserved. +# Copyright (c) 2015-2018 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -28,7 +28,6 @@ libmca_ess_la_SOURCES += \ base/ess_base_select.c \ base/ess_base_get.c \ base/ess_base_std_tool.c \ - base/ess_base_std_app.c \ base/ess_base_std_orted.c \ base/ess_base_std_prolog.c \ base/ess_base_fns.c diff --git a/orte/mca/ess/base/base.h b/orte/mca/ess/base/base.h index d8706b7bab..139a6cff46 100644 --- a/orte/mca/ess/base/base.h +++ b/orte/mca/ess/base/base.h @@ -61,9 +61,6 @@ ORTE_DECLSPEC int orte_ess_env_get(void); ORTE_DECLSPEC int orte_ess_base_std_prolog(void); -ORTE_DECLSPEC int orte_ess_base_app_setup(bool db_restrict_local); -ORTE_DECLSPEC int orte_ess_base_app_finalize(void); - ORTE_DECLSPEC int orte_ess_base_tool_setup(opal_list_t *flags); ORTE_DECLSPEC int orte_ess_base_tool_finalize(void); diff --git a/orte/mca/ess/base/ess_base_std_app.c b/orte/mca/ess/base/ess_base_std_app.c deleted file mode 100644 index f0a7b848d2..0000000000 --- a/orte/mca/ess/base/ess_base_std_app.c +++ /dev/null @@ -1,240 +0,0 @@ -/* - * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2011 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2010-2012 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2011-2013 Los Alamos National Security, LLC. All rights - * reserved. - * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. - * Copyright (c) 2014-2016 Research Organization for Information Science - * and Technology (RIST). All rights reserved. - * Copyright (c) 2015 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2018 Mellanox Technologies, Inc. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "orte_config.h" -#include "orte/constants.h" - -#include -#include -#ifdef HAVE_FCNTL_H -#include -#endif -#ifdef HAVE_UNISTD_H -#include -#endif -#ifdef HAVE_SYS_STAT_H -#include -#endif - -#include "opal/mca/event/event.h" -#include "opal/mca/pmix/base/base.h" -#include "opal/util/arch.h" -#include "opal/util/os_path.h" -#include "opal/util/output.h" -#include "opal/util/proc.h" -#include "opal/runtime/opal.h" - -#include "orte/mca/odls/odls_types.h" -#include "orte/mca/filem/base/base.h" -#include "orte/mca/errmgr/base/base.h" -#include "orte/mca/rml/base/rml_contact.h" -#include "orte/mca/state/base/base.h" -#include "orte/util/proc_info.h" -#include "orte/util/session_dir.h" -#include "orte/util/name_fns.h" -#include "orte/util/show_help.h" -#include "opal/util/timings.h" - -#include "orte/runtime/orte_globals.h" -#include "orte/runtime/orte_wait.h" - -#include "orte/mca/ess/base/base.h" - -int orte_ess_base_app_setup(bool db_restrict_local) -{ - int ret; - char *error = NULL; - - OPAL_TIMING_ENV_INIT(ess_base_setup); - /* - * stdout/stderr buffering - * If the user requested to override the default setting then do - * as they wish. - */ - if( orte_ess_base_std_buffering > -1 ) { - if( 0 == orte_ess_base_std_buffering ) { - setvbuf(stdout, NULL, _IONBF, 0); - setvbuf(stderr, NULL, _IONBF, 0); - } - else if( 1 == orte_ess_base_std_buffering ) { - setvbuf(stdout, NULL, _IOLBF, 0); - setvbuf(stderr, NULL, _IOLBF, 0); - } - else if( 2 == orte_ess_base_std_buffering ) { - setvbuf(stdout, NULL, _IOFBF, 0); - setvbuf(stderr, NULL, _IOFBF, 0); - } - } - - /* if I am an MPI app, we will let the MPI layer define and - * control the opal_proc_t structure. Otherwise, we need to - * do so here */ - if (ORTE_PROC_NON_MPI) { - orte_process_info.super.proc_name = *(opal_process_name_t*)ORTE_PROC_MY_NAME; - orte_process_info.super.proc_hostname = orte_process_info.nodename; - orte_process_info.super.proc_flags = OPAL_PROC_ALL_LOCAL; - orte_process_info.super.proc_arch = opal_local_arch; - opal_proc_local_set(&orte_process_info.super); - } - - /* open and setup the state machine */ - if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_state_base_framework, 0))) { - ORTE_ERROR_LOG(ret); - error = "orte_state_base_open"; - goto error; - } - if (ORTE_SUCCESS != (ret = orte_state_base_select())) { - ORTE_ERROR_LOG(ret); - error = "orte_state_base_select"; - goto error; - } - OPAL_TIMING_ENV_NEXT(ess_base_setup, "state_framework_open"); - - /* open the errmgr */ - if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_errmgr_base_framework, 0))) { - ORTE_ERROR_LOG(ret); - error = "orte_errmgr_base_open"; - goto error; - } - OPAL_TIMING_ENV_NEXT(ess_base_setup, "errmgr_framework_open"); - - /* setup my session directory */ - if (orte_create_session_dirs) { - OPAL_OUTPUT_VERBOSE((2, orte_ess_base_framework.framework_output, - "%s setting up session dir with\n\ttmpdir: %s\n\thost %s", - ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), - (NULL == orte_process_info.tmpdir_base) ? "UNDEF" : orte_process_info.tmpdir_base, - orte_process_info.nodename)); - if (ORTE_SUCCESS != (ret = orte_session_dir(true, ORTE_PROC_MY_NAME))) { - ORTE_ERROR_LOG(ret); - error = "orte_session_dir"; - goto error; - } - /* Once the session directory location has been established, set - the opal_output env file location to be in the - proc-specific session directory. */ - opal_output_set_output_file_info(orte_process_info.proc_session_dir, - "output-", NULL, NULL); - /* register the directory for cleanup */ - if (NULL != opal_pmix.register_cleanup) { - if (orte_standalone_operation) { - if (OPAL_SUCCESS != (ret = opal_pmix.register_cleanup(orte_process_info.top_session_dir, true, false, true))) { - ORTE_ERROR_LOG(ret); - error = "register cleanup"; - goto error; - } - } else { - if (OPAL_SUCCESS != (ret = opal_pmix.register_cleanup(orte_process_info.job_session_dir, true, false, false))) { - ORTE_ERROR_LOG(ret); - error = "register cleanup"; - goto error; - } - } - } - } - OPAL_TIMING_ENV_NEXT(ess_base_setup, "create_session_dirs"); - - /* if we have info on the HNP and local daemon, process it */ - if (NULL != orte_process_info.my_hnp_uri) { - /* we have to set the HNP's name, even though we won't route messages directly - * to it. This is required to ensure that we -do- send messages to the correct - * HNP name - */ - if (ORTE_SUCCESS != (ret = orte_rml_base_parse_uris(orte_process_info.my_hnp_uri, - ORTE_PROC_MY_HNP, NULL))) { - ORTE_ERROR_LOG(ret); - error = "orte_rml_parse_HNP"; - goto error; - } - } - if (NULL != orte_process_info.my_daemon_uri) { - opal_value_t val; - - /* extract the daemon's name so we can update the routing table */ - if (ORTE_SUCCESS != (ret = orte_rml_base_parse_uris(orte_process_info.my_daemon_uri, - ORTE_PROC_MY_DAEMON, NULL))) { - ORTE_ERROR_LOG(ret); - error = "orte_rml_parse_daemon"; - goto error; - } - /* Set the contact info in the database - this won't actually establish - * the connection, but just tells us how to reach the daemon - * if/when we attempt to send to it - */ - OBJ_CONSTRUCT(&val, opal_value_t); - val.key = OPAL_PMIX_PROC_URI; - val.type = OPAL_STRING; - val.data.string = orte_process_info.my_daemon_uri; - if (OPAL_SUCCESS != (ret = opal_pmix.store_local(ORTE_PROC_MY_DAEMON, &val))) { - ORTE_ERROR_LOG(ret); - val.key = NULL; - val.data.string = NULL; - OBJ_DESTRUCT(&val); - error = "store DAEMON URI"; - goto error; - } - val.key = NULL; - val.data.string = NULL; - OBJ_DESTRUCT(&val); - } - - /* setup the errmgr */ - if (ORTE_SUCCESS != (ret = orte_errmgr_base_select())) { - ORTE_ERROR_LOG(ret); - error = "orte_errmgr_base_select"; - goto error; - } - OPAL_TIMING_ENV_NEXT(ess_base_setup, "errmgr_select"); - - return ORTE_SUCCESS; - error: - orte_show_help("help-orte-runtime.txt", - "orte_init:startup:internal-failure", - true, error, ORTE_ERROR_NAME(ret), ret); - return ret; -} - -int orte_ess_base_app_finalize(void) -{ - /* close frameworks */ - (void) mca_base_framework_close(&orte_filem_base_framework); - (void) mca_base_framework_close(&orte_errmgr_base_framework); - - if (NULL != opal_pmix.finalize) { - opal_pmix.finalize(); - (void) mca_base_framework_close(&opal_pmix_base_framework); - } - (void) mca_base_framework_close(&orte_state_base_framework); - - if (NULL == opal_pmix.register_cleanup) { - orte_session_dir_finalize(ORTE_PROC_MY_NAME); - } - /* cleanup the process info */ - orte_proc_info_finalize(); - - return ORTE_SUCCESS; -} diff --git a/orte/mca/ess/lsf/ess_lsf_module.c b/orte/mca/ess/lsf/ess_lsf_module.c index ec5fbfe724..32247aeeec 100644 --- a/orte/mca/ess/lsf/ess_lsf_module.c +++ b/orte/mca/ess/lsf/ess_lsf_module.c @@ -100,15 +100,6 @@ static int rte_init(void) } - /* otherwise, I must be an application process - use - * the default procedure to finish my setup - */ - if (ORTE_SUCCESS != (ret = orte_ess_base_app_setup(false))) { - ORTE_ERROR_LOG(ret); - error = "orte_ess_base_app_setup"; - goto error; - } - return ORTE_SUCCESS; error: @@ -137,14 +128,6 @@ static int rte_finalize(void) ORTE_ERROR_LOG(ret); } return ret; - } else { - /* otherwise, I must be an application process - * use the default procedure to finish - */ - if (ORTE_SUCCESS != (ret = orte_ess_base_app_finalize())) { - ORTE_ERROR_LOG(ret); - return ret; - } } return ORTE_SUCCESS;; diff --git a/orte/mca/ess/pmi/ess_pmi_module.c b/orte/mca/ess/pmi/ess_pmi_module.c index 1515ae9888..39518749ab 100644 --- a/orte/mca/ess/pmi/ess_pmi_module.c +++ b/orte/mca/ess/pmi/ess_pmi_module.c @@ -45,6 +45,7 @@ #include "opal/util/opal_environ.h" #include "opal/util/output.h" +#include "opal/util/arch.h" #include "opal/util/argv.h" #include "opal/runtime/opal_progress_threads.h" #include "opal/class/opal_pointer_array.h" @@ -55,11 +56,15 @@ #include "opal/mca/pmix/base/base.h" #include "opal/util/timings.h" -#include "orte/mca/errmgr/errmgr.h" +#include "orte/mca/errmgr/base/base.h" +#include "orte/mca/filem/base/base.h" #include "orte/mca/grpcomm/grpcomm.h" #include "orte/mca/rml/rml.h" +#include "orte/mca/rml/base/rml_contact.h" #include "orte/mca/schizo/schizo.h" +#include "orte/mca/state/base/base.h" #include "orte/util/proc_info.h" +#include "orte/util/session_dir.h" #include "orte/util/show_help.h" #include "orte/util/name_fns.h" #include "orte/util/pre_condition_transports.h" @@ -85,6 +90,7 @@ static bool added_transport_keys=false; static bool added_num_procs = false; static bool added_app_ctx = false; static bool progress_thread_running = false; +static bool direct_launched = false; /**** MODULE FUNCTIONS ****/ @@ -135,13 +141,17 @@ static int rte_init(void) opal_pmix_base_set_evbase(orte_event_base); OPAL_TIMING_ENV_NEXT(rte_init, "pmix_framework_open"); + /* see if we were direct launched */ + if (ORTE_SCHIZO_DIRECT_LAUNCHED == orte_schizo.check_launch_environment()) { + direct_launched = true; + } + /* initialize the selected module */ if (!opal_pmix.initialized() && (OPAL_SUCCESS != (ret = opal_pmix.init(NULL)))) { /* we cannot run - this could be due to being direct launched * without the required PMI support being built. Try to detect * that scenario and warn the user */ - if (ORTE_SCHIZO_DIRECT_LAUNCHED == orte_schizo.check_launch_environment() && - NULL != (envar = getenv("ORTE_SCHIZO_DETECTION"))) { + if (direct_launched && NULL != (envar = getenv("ORTE_SCHIZO_DETECTION"))) { if (0 == strcmp(envar, "SLURM")) { /* yes to both - so emit a hopefully helpful * error message and abort */ @@ -176,7 +186,7 @@ static int rte_init(void) pname.vpid = 0; OPAL_TIMING_ENV_NEXT(rte_init, "pmix_init"); - + /* get our local rank from PMI */ OPAL_MODEX_RECV_VALUE(ret, OPAL_PMIX_LOCAL_RANK, ORTE_PROC_MY_NAME, &u16ptr, OPAL_UINT16); @@ -412,12 +422,145 @@ static int rte_init(void) OPAL_TIMING_ENV_NEXT(rte_init, "pmix_set_locality"); /* now that we have all required info, complete the setup */ - if (ORTE_SUCCESS != (ret = orte_ess_base_app_setup(false))) { + /* + * stdout/stderr buffering + * If the user requested to override the default setting then do + * as they wish. + */ + if( orte_ess_base_std_buffering > -1 ) { + if( 0 == orte_ess_base_std_buffering ) { + setvbuf(stdout, NULL, _IONBF, 0); + setvbuf(stderr, NULL, _IONBF, 0); + } + else if( 1 == orte_ess_base_std_buffering ) { + setvbuf(stdout, NULL, _IOLBF, 0); + setvbuf(stderr, NULL, _IOLBF, 0); + } + else if( 2 == orte_ess_base_std_buffering ) { + setvbuf(stdout, NULL, _IOFBF, 0); + setvbuf(stderr, NULL, _IOFBF, 0); + } + } + + /* if I am an MPI app, we will let the MPI layer define and + * control the opal_proc_t structure. Otherwise, we need to + * do so here */ + if (ORTE_PROC_NON_MPI) { + orte_process_info.super.proc_name = *(opal_process_name_t*)ORTE_PROC_MY_NAME; + orte_process_info.super.proc_hostname = orte_process_info.nodename; + orte_process_info.super.proc_flags = OPAL_PROC_ALL_LOCAL; + orte_process_info.super.proc_arch = opal_local_arch; + opal_proc_local_set(&orte_process_info.super); + } + + /* open and setup the state machine */ + if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_state_base_framework, 0))) { ORTE_ERROR_LOG(ret); - error = "orte_ess_base_app_setup"; + error = "orte_state_base_open"; goto error; } - OPAL_TIMING_ENV_NEXT(rte_init, "ess_base_app_setup"); + if (ORTE_SUCCESS != (ret = orte_state_base_select())) { + ORTE_ERROR_LOG(ret); + error = "orte_state_base_select"; + goto error; + } + OPAL_TIMING_ENV_NEXT(ess_base_setup, "state_framework_open"); + + /* open the errmgr */ + if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_errmgr_base_framework, 0))) { + ORTE_ERROR_LOG(ret); + error = "orte_errmgr_base_open"; + goto error; + } + OPAL_TIMING_ENV_NEXT(ess_base_setup, "errmgr_framework_open"); + + /* setup my session directory */ + if (orte_create_session_dirs) { + OPAL_OUTPUT_VERBOSE((2, orte_ess_base_framework.framework_output, + "%s setting up session dir with\n\ttmpdir: %s\n\thost %s", + ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), + (NULL == orte_process_info.tmpdir_base) ? "UNDEF" : orte_process_info.tmpdir_base, + orte_process_info.nodename)); + if (ORTE_SUCCESS != (ret = orte_session_dir(true, ORTE_PROC_MY_NAME))) { + ORTE_ERROR_LOG(ret); + error = "orte_session_dir"; + goto error; + } + /* Once the session directory location has been established, set + the opal_output env file location to be in the + proc-specific session directory. */ + opal_output_set_output_file_info(orte_process_info.proc_session_dir, + "output-", NULL, NULL); + /* register the directory for cleanup */ + if (NULL != opal_pmix.register_cleanup) { + if (orte_standalone_operation) { + if (OPAL_SUCCESS != (ret = opal_pmix.register_cleanup(orte_process_info.top_session_dir, true, false, true))) { + ORTE_ERROR_LOG(ret); + error = "register cleanup"; + goto error; + } + } else { + if (OPAL_SUCCESS != (ret = opal_pmix.register_cleanup(orte_process_info.job_session_dir, true, false, false))) { + ORTE_ERROR_LOG(ret); + error = "register cleanup"; + goto error; + } + } + } + } + OPAL_TIMING_ENV_NEXT(ess_base_setup, "create_session_dirs"); + + /* if we have info on the HNP and local daemon, process it */ + if (NULL != orte_process_info.my_hnp_uri) { + /* we have to set the HNP's name, even though we won't route messages directly + * to it. This is required to ensure that we -do- send messages to the correct + * HNP name + */ + if (ORTE_SUCCESS != (ret = orte_rml_base_parse_uris(orte_process_info.my_hnp_uri, + ORTE_PROC_MY_HNP, NULL))) { + ORTE_ERROR_LOG(ret); + error = "orte_rml_parse_HNP"; + goto error; + } + } + if (NULL != orte_process_info.my_daemon_uri) { + opal_value_t val; + + /* extract the daemon's name so we can update the routing table */ + if (ORTE_SUCCESS != (ret = orte_rml_base_parse_uris(orte_process_info.my_daemon_uri, + ORTE_PROC_MY_DAEMON, NULL))) { + ORTE_ERROR_LOG(ret); + error = "orte_rml_parse_daemon"; + goto error; + } + /* Set the contact info in the database - this won't actually establish + * the connection, but just tells us how to reach the daemon + * if/when we attempt to send to it + */ + OBJ_CONSTRUCT(&val, opal_value_t); + val.key = OPAL_PMIX_PROC_URI; + val.type = OPAL_STRING; + val.data.string = orte_process_info.my_daemon_uri; + if (OPAL_SUCCESS != (ret = opal_pmix.store_local(ORTE_PROC_MY_DAEMON, &val))) { + ORTE_ERROR_LOG(ret); + val.key = NULL; + val.data.string = NULL; + OBJ_DESTRUCT(&val); + error = "store DAEMON URI"; + goto error; + } + val.key = NULL; + val.data.string = NULL; + OBJ_DESTRUCT(&val); + } + + /* setup the errmgr */ + if (ORTE_SUCCESS != (ret = orte_errmgr_base_select())) { + ORTE_ERROR_LOG(ret); + error = "orte_errmgr_base_select"; + goto error; + } + OPAL_TIMING_ENV_NEXT(ess_base_setup, "errmgr_select"); /* setup process binding */ if (ORTE_SUCCESS != (ret = orte_ess_base_proc_binding())) { @@ -464,7 +607,7 @@ static int rte_init(void) } } OPAL_TIMING_ENV_NEXT(rte_init, "rte_init_done"); - + return ORTE_SUCCESS; error: @@ -484,8 +627,6 @@ static int rte_init(void) static int rte_finalize(void) { - int ret; - /* remove the envars that we pushed into environ * so we leave that structure intact */ @@ -499,11 +640,21 @@ static int rte_finalize(void) unsetenv("OMPI_APP_CTX_NUM_PROCS"); } - /* use the default app procedure to finish */ - if (ORTE_SUCCESS != (ret = orte_ess_base_app_finalize())) { - ORTE_ERROR_LOG(ret); - return ret; + /* close frameworks */ + (void) mca_base_framework_close(&orte_filem_base_framework); + (void) mca_base_framework_close(&orte_errmgr_base_framework); + + if (NULL != opal_pmix.finalize) { + opal_pmix.finalize(); + (void) mca_base_framework_close(&opal_pmix_base_framework); } + (void) mca_base_framework_close(&orte_state_base_framework); + + if (direct_launched) { + orte_session_dir_finalize(ORTE_PROC_MY_NAME); + } + /* cleanup the process info */ + orte_proc_info_finalize(); /* release the event base */ if (progress_thread_running) { diff --git a/orte/mca/ess/singleton/ess_singleton_module.c b/orte/mca/ess/singleton/ess_singleton_module.c index b6b5262a89..c35909484b 100644 --- a/orte/mca/ess/singleton/ess_singleton_module.c +++ b/orte/mca/ess/singleton/ess_singleton_module.c @@ -39,9 +39,11 @@ #include #include "opal/hash_string.h" +#include "opal/util/arch.h" #include "opal/util/argv.h" #include "opal/util/opal_environ.h" #include "opal/util/path.h" +#include "opal/util/timings.h" #include "opal/runtime/opal_progress_threads.h" #include "opal/mca/installdirs/installdirs.h" #include "opal/mca/pmix/base/base.h" @@ -49,8 +51,11 @@ #include "orte/util/show_help.h" #include "orte/util/proc_info.h" -#include "orte/mca/errmgr/errmgr.h" +#include "orte/mca/errmgr/base/base.h" +#include "orte/mca/filem/base/base.h" #include "orte/mca/plm/base/base.h" +#include "orte/mca/rml/base/rml_contact.h" +#include "orte/mca/state/base/base.h" #include "orte/util/name_fns.h" #include "orte/runtime/orte_globals.h" #include "orte/util/session_dir.h" @@ -273,15 +278,196 @@ static int rte_init(void) } } - /* use the std app init to complete the procedure */ - if (ORTE_SUCCESS != (rc = orte_ess_base_app_setup(true))) { - ORTE_ERROR_LOG(rc); - return rc; + /* now that we have all required info, complete the setup */ + /* + * stdout/stderr buffering + * If the user requested to override the default setting then do + * as they wish. + */ + if( orte_ess_base_std_buffering > -1 ) { + if( 0 == orte_ess_base_std_buffering ) { + setvbuf(stdout, NULL, _IONBF, 0); + setvbuf(stderr, NULL, _IONBF, 0); + } + else if( 1 == orte_ess_base_std_buffering ) { + setvbuf(stdout, NULL, _IOLBF, 0); + setvbuf(stderr, NULL, _IOLBF, 0); + } + else if( 2 == orte_ess_base_std_buffering ) { + setvbuf(stdout, NULL, _IOFBF, 0); + setvbuf(stderr, NULL, _IOFBF, 0); + } } + /* if I am an MPI app, we will let the MPI layer define and + * control the opal_proc_t structure. Otherwise, we need to + * do so here */ + if (ORTE_PROC_NON_MPI) { + orte_process_info.super.proc_name = *(opal_process_name_t*)ORTE_PROC_MY_NAME; + orte_process_info.super.proc_hostname = orte_process_info.nodename; + orte_process_info.super.proc_flags = OPAL_PROC_ALL_LOCAL; + orte_process_info.super.proc_arch = opal_local_arch; + opal_proc_local_set(&orte_process_info.super); + } + + /* open and setup the state machine */ + if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_state_base_framework, 0))) { + ORTE_ERROR_LOG(ret); + error = "orte_state_base_open"; + goto error; + } + if (ORTE_SUCCESS != (ret = orte_state_base_select())) { + ORTE_ERROR_LOG(ret); + error = "orte_state_base_select"; + goto error; + } + OPAL_TIMING_ENV_NEXT(ess_base_setup, "state_framework_open"); + + /* open the errmgr */ + if (ORTE_SUCCESS != (ret = mca_base_framework_open(&orte_errmgr_base_framework, 0))) { + ORTE_ERROR_LOG(ret); + error = "orte_errmgr_base_open"; + goto error; + } + OPAL_TIMING_ENV_NEXT(ess_base_setup, "errmgr_framework_open"); + + /* setup my session directory */ + if (orte_create_session_dirs) { + OPAL_OUTPUT_VERBOSE((2, orte_ess_base_framework.framework_output, + "%s setting up session dir with\n\ttmpdir: %s\n\thost %s", + ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), + (NULL == orte_process_info.tmpdir_base) ? "UNDEF" : orte_process_info.tmpdir_base, + orte_process_info.nodename)); + if (ORTE_SUCCESS != (ret = orte_session_dir(true, ORTE_PROC_MY_NAME))) { + ORTE_ERROR_LOG(ret); + error = "orte_session_dir"; + goto error; + } + /* Once the session directory location has been established, set + the opal_output env file location to be in the + proc-specific session directory. */ + opal_output_set_output_file_info(orte_process_info.proc_session_dir, + "output-", NULL, NULL); + /* register the directory for cleanup */ + if (NULL != opal_pmix.register_cleanup) { + if (orte_standalone_operation) { + if (OPAL_SUCCESS != (ret = opal_pmix.register_cleanup(orte_process_info.top_session_dir, true, false, true))) { + ORTE_ERROR_LOG(ret); + error = "register cleanup"; + goto error; + } + } else { + if (OPAL_SUCCESS != (ret = opal_pmix.register_cleanup(orte_process_info.job_session_dir, true, false, false))) { + ORTE_ERROR_LOG(ret); + error = "register cleanup"; + goto error; + } + } + } + } + OPAL_TIMING_ENV_NEXT(ess_base_setup, "create_session_dirs"); + + /* if we have info on the HNP and local daemon, process it */ + if (NULL != orte_process_info.my_hnp_uri) { + /* we have to set the HNP's name, even though we won't route messages directly + * to it. This is required to ensure that we -do- send messages to the correct + * HNP name + */ + if (ORTE_SUCCESS != (ret = orte_rml_base_parse_uris(orte_process_info.my_hnp_uri, + ORTE_PROC_MY_HNP, NULL))) { + ORTE_ERROR_LOG(ret); + error = "orte_rml_parse_HNP"; + goto error; + } + } + if (NULL != orte_process_info.my_daemon_uri) { + opal_value_t val; + + /* extract the daemon's name so we can update the routing table */ + if (ORTE_SUCCESS != (ret = orte_rml_base_parse_uris(orte_process_info.my_daemon_uri, + ORTE_PROC_MY_DAEMON, NULL))) { + ORTE_ERROR_LOG(ret); + error = "orte_rml_parse_daemon"; + goto error; + } + /* Set the contact info in the database - this won't actually establish + * the connection, but just tells us how to reach the daemon + * if/when we attempt to send to it + */ + OBJ_CONSTRUCT(&val, opal_value_t); + val.key = OPAL_PMIX_PROC_URI; + val.type = OPAL_STRING; + val.data.string = orte_process_info.my_daemon_uri; + if (OPAL_SUCCESS != (ret = opal_pmix.store_local(ORTE_PROC_MY_DAEMON, &val))) { + ORTE_ERROR_LOG(ret); + val.key = NULL; + val.data.string = NULL; + OBJ_DESTRUCT(&val); + error = "store DAEMON URI"; + goto error; + } + val.key = NULL; + val.data.string = NULL; + OBJ_DESTRUCT(&val); + } + + /* setup the errmgr */ + if (ORTE_SUCCESS != (ret = orte_errmgr_base_select())) { + ORTE_ERROR_LOG(ret); + error = "orte_errmgr_base_select"; + goto error; + } + OPAL_TIMING_ENV_NEXT(ess_base_setup, "errmgr_select"); + + /* setup process binding */ + if (ORTE_SUCCESS != (ret = orte_ess_base_proc_binding())) { + error = "proc_binding"; + goto error; + } + OPAL_TIMING_ENV_NEXT(rte_init, "ess_base_proc_binding"); + + /* this needs to be set to enable debugger use when direct launched */ + if (NULL == orte_process_info.my_daemon_uri) { + orte_standalone_operation = true; + } + + /* set max procs */ + if (orte_process_info.max_procs < orte_process_info.num_procs) { + orte_process_info.max_procs = orte_process_info.num_procs; + } + + /* push our hostname so others can find us, if they need to - the + * native PMIx component will ignore this request as the hostname + * is provided by the system */ + OPAL_MODEX_SEND_VALUE(ret, OPAL_PMIX_GLOBAL, OPAL_PMIX_HOSTNAME, orte_process_info.nodename, OPAL_STRING); + if (ORTE_SUCCESS != ret) { + error = "db store hostname"; + goto error; + } + + /* if we are an ORTE app - and not an MPI app - then + * we need to exchange our connection info here. + * MPI_Init has its own modex, so we don't need to do + * two of them. However, if we don't do a modex at all, + * then processes have no way to communicate + * + * NOTE: only do this when the process originally launches. + * Cannot do this on a restart as the rest of the processes + * in the job won't be executing this step, so we would hang + */ + if (ORTE_PROC_IS_NON_MPI && !orte_do_not_barrier) { + /* need to commit the data before we fence */ + opal_pmix.commit(); + if (ORTE_SUCCESS != (ret = opal_pmix.fence(NULL, 0))) { + error = "opal_pmix.fence() failed"; + goto error; + } + } + OPAL_TIMING_ENV_NEXT(rte_init, "rte_init_done"); + return ORTE_SUCCESS; - error: + error: if (ORTE_ERR_SILENT != ret && !orte_report_silent_errors) { orte_show_help("help-orte-runtime.txt", "orte_init:startup:internal-failure", @@ -292,8 +478,6 @@ static int rte_init(void) static int rte_finalize(void) { - int ret; - /* remove the envars that we pushed into environ * so we leave that structure intact */ @@ -312,10 +496,9 @@ static int rte_finalize(void) unsetenv("PMIX_SERVER_URI"); unsetenv("PMIX_SECURITY_MODE"); } - /* use the default procedure to finish */ - if (ORTE_SUCCESS != (ret = orte_ess_base_app_finalize())) { - ORTE_ERROR_LOG(ret); - } + /* close frameworks */ + (void) mca_base_framework_close(&orte_filem_base_framework); + (void) mca_base_framework_close(&orte_errmgr_base_framework); /* mark us as finalized */ if (NULL != opal_pmix.finalize) { @@ -323,12 +506,18 @@ static int rte_finalize(void) (void) mca_base_framework_close(&opal_pmix_base_framework); } + (void) mca_base_framework_close(&orte_state_base_framework); + orte_session_dir_finalize(ORTE_PROC_MY_NAME); + + /* cleanup the process info */ + orte_proc_info_finalize(); + /* release the event base */ if (progress_thread_running) { opal_progress_thread_finalize(NULL); progress_thread_running = false; } - return ret; + return ORTE_SUCCESS; } #define ORTE_URI_MSG_LGTH 256 diff --git a/orte/mca/ess/slurm/ess_slurm_module.c b/orte/mca/ess/slurm/ess_slurm_module.c index de14bdc0e9..d400de3085 100644 --- a/orte/mca/ess/slurm/ess_slurm_module.c +++ b/orte/mca/ess/slurm/ess_slurm_module.c @@ -125,14 +125,6 @@ static int rte_finalize(void) ORTE_ERROR_LOG(ret); } return ret; - } else { - /* otherwise, I must be an application process - * use the default procedure to finish - */ - if (ORTE_SUCCESS != (ret = orte_ess_base_app_finalize())) { - ORTE_ERROR_LOG(ret); - return ret; - } } return ORTE_SUCCESS; diff --git a/orte/mca/ess/tm/ess_tm_module.c b/orte/mca/ess/tm/ess_tm_module.c index 16a6e74b1a..646caced77 100644 --- a/orte/mca/ess/tm/ess_tm_module.c +++ b/orte/mca/ess/tm/ess_tm_module.c @@ -129,14 +129,6 @@ static int rte_finalize(void) ORTE_ERROR_LOG(ret); } return ret; - } else { - /* otherwise, I must be an application process - * use the default procedure to finish - */ - if (ORTE_SUCCESS != (ret = orte_ess_base_app_finalize())) { - ORTE_ERROR_LOG(ret); - return ret; - } } return ORTE_SUCCESS;