ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность
5025628651
openmpi/ompi/runtime/ompi_rte.c
Gilles Gouaillardet 174e967dbc
Remove ORTE project 
Will be replaced by PRRTE. Ensure that OMPI and OPAL layers build
without reference to ORTE. Setup opal/pmix framework to be static.
Remove support for all PMI-1 and PMI-2 libraries. Add support for
"external" pmix component as well as internal v4 one.

remove orte: misc fixes

 - UCX fixes
 - VPATH issue
 - oshmem fixes
 - remove useless definition
 - Add PRRTE submodule
 - Get autogen.pl to traverse PRRTE submodule
 - Remove stale orcm reference
 - Configure embedded PRRTE
 - Correctly pass the prefix to PRRTE
 - Correctly set the OMPI_WANT_PRRTE am_conditional
 - Move prrte configuration to the end of OMPI's configure.ac
 - Make mpirun a symlink to prun, when available
 - Fix makedist with --no-orte/--no-prrte option
 - Add a `--no-prrte` option which is the same as the legacy
   `--no-orte` option.
 - Remove embedded PMIx tarball. Replace it with new submodule
   pointing to OpenPMIx master repo's master branch
 - Some cleanup in PRRTE integration and add config summary entry
 - Correctly set the hostname
 - Fix locality
 - Fix singleton operations
 - Fix support for "tune" and "am" options

Signed-off-by: Ralph Castain <rhc@pmix.org>
Signed-off-by: Gilles Gouaillardet <gilles@rist.or.jp>
Signed-off-by: Joshua Hursey <jhursey@us.ibm.com>
2020-02-07 18:20:06 -08:00

1033 строки
32 KiB
C
Исходник Ответственный История

/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2012-2013 Los Alamos National Security, LLC.
* All rights reserved.
* Copyright (c) 2013-2020 Intel, Inc. All rights reserved.
* Copyright (c) 2012-2014 The University of Tennessee and The University
* of Tennessee Research Foundation. All rights
* reserved.
* Copyright (c) 2014-2018 Cisco Systems, Inc. All rights reserved
* Copyright (c) 2018 Amazon.com, Inc. or its affiliates. All Rights reserved.
* Copyright (c) 2018 Triad National Security, LLC. All rights
* reserved.
* Copyright (c) 2019 Research Organization for Information Science
* and Technology (RIST). All rights reserved.
* $COPYRIGHT$
*/
#include "ompi_config.h"
#include "ompi/constants.h"
#include <string.h>
#include <stdio.h>
#include <ctype.h>
#ifdef HAVE_SYS_TYPES_H
#include <sys/types.h>
#endif /* HAVE_SYS_TYPES_H */
#include <sys/stat.h>
#ifdef HAVE_UNISTD_H
#include <unistd.h>
#endif /* HAVE_UNISTD_H */
#ifdef HAVE_DIRENT_H
#include <dirent.h>
#endif /* HAVE_DIRENT_H */
#ifdef HAVE_PWD_H
#include <pwd.h>
#endif /* HAVE_PWD_H */
#include "opal/dss/dss.h"
#include "opal/util/argv.h"
#include "opal/util/error.h"
#include "opal/util/opal_getcwd.h"
#include "opal/util/os_path.h"
#include "opal/util/os_dirpath.h"
#include "opal/util/printf.h"
#include "opal/util/proc.h"
#include "opal/util/show_help.h"
#include "opal/util/string_copy.h"
#include "opal/mca/hwloc/base/base.h"
#include "opal/mca/pmix/base/base.h"
#include "opal/threads/threads.h"
#include "opal/threads/tsd.h"
#include "opal/class/opal_list.h"
#include "opal/dss/dss.h"
#include "ompi/runtime/ompi_rte.h"
#include "ompi/debuggers/debuggers.h"
#include "ompi/proc/proc.h"
#include "ompi/runtime/params.h"
#include "ompi/communicator/communicator.h"
/* storage to support OMPI */
opal_process_name_t pmix_name_wildcard = {UINT32_MAX-1, UINT32_MAX-1};
opal_process_name_t pmix_name_invalid = {UINT32_MAX, UINT32_MAX};
hwloc_cpuset_t ompi_proc_applied_binding = NULL;
pmix_process_info_t pmix_process_info = {0};
bool pmix_proc_is_bound = false;
bool ompi_singleton = false;
static bool added_transport_keys = false;
static bool added_num_procs = false;
static bool added_app_ctx = false;
static char* pre_condition_transports_print(uint64_t *unique_key);
static int _setup_top_session_dir(char **sdir);
static int _setup_job_session_dir(char **sdir);
static int _setup_proc_session_dir(char **sdir);
#define OPAL_SCHEMA_DELIMITER_CHAR '.'
#define OPAL_SCHEMA_WILDCARD_CHAR '*'
#define OPAL_SCHEMA_WILDCARD_STRING "*"
#define OPAL_SCHEMA_INVALID_CHAR '$'
#define OPAL_SCHEMA_INVALID_STRING "$"
#define OPAL_PRINT_NAME_ARGS_MAX_SIZE 50
#define OPAL_PRINT_NAME_ARG_NUM_BUFS 16
static bool fns_init=false;
static opal_tsd_key_t print_args_tsd_key;
static char* opal_print_args_null = "NULL";
typedef struct {
char *buffers[OPAL_PRINT_NAME_ARG_NUM_BUFS];
int cntr;
} opal_print_args_buffers_t;
static void
buffer_cleanup(void *value)
{
int i;
opal_print_args_buffers_t *ptr;
if (NULL != value) {
ptr = (opal_print_args_buffers_t*)value;
for (i=0; i < OPAL_PRINT_NAME_ARG_NUM_BUFS; i++) {
free(ptr->buffers[i]);
}
free (ptr);
}
}
static opal_print_args_buffers_t*
get_print_name_buffer(void)
{
opal_print_args_buffers_t *ptr;
int ret, i;
if (!fns_init) {
/* setup the print_args function */
if (OPAL_SUCCESS != (ret = opal_tsd_key_create(&print_args_tsd_key, buffer_cleanup))) {
OPAL_ERROR_LOG(ret);
return NULL;
}
fns_init = true;
}
ret = opal_tsd_getspecific(print_args_tsd_key, (void**)&ptr);
if (OPAL_SUCCESS != ret) return NULL;
if (NULL == ptr) {
ptr = (opal_print_args_buffers_t*)malloc(sizeof(opal_print_args_buffers_t));
for (i=0; i < OPAL_PRINT_NAME_ARG_NUM_BUFS; i++) {
ptr->buffers[i] = (char *) malloc((OPAL_PRINT_NAME_ARGS_MAX_SIZE+1) * sizeof(char));
}
ptr->cntr = 0;
ret = opal_tsd_setspecific(print_args_tsd_key, (void*)ptr);
}
return (opal_print_args_buffers_t*) ptr;
}
static char* ompi_pmix_print_jobids(const opal_jobid_t job)
{
opal_print_args_buffers_t *ptr;
unsigned long tmp1, tmp2;
ptr = get_print_name_buffer();
if (NULL == ptr) {
OPAL_ERROR_LOG(OPAL_ERR_OUT_OF_RESOURCE);
return opal_print_args_null;
}
/* cycle around the ring */
if (OPAL_PRINT_NAME_ARG_NUM_BUFS == ptr->cntr) {
ptr->cntr = 0;
}
if (OPAL_JOBID_INVALID == job) {
snprintf(ptr->buffers[ptr->cntr++], OPAL_PRINT_NAME_ARGS_MAX_SIZE, "[INVALID]");
} else if (OPAL_JOBID_WILDCARD == job) {
snprintf(ptr->buffers[ptr->cntr++], OPAL_PRINT_NAME_ARGS_MAX_SIZE, "[WILDCARD]");
} else {
tmp1 = OMPI_JOB_FAMILY((unsigned long)job);
tmp2 = OMPI_LOCAL_JOBID((unsigned long)job);
snprintf(ptr->buffers[ptr->cntr++],
OPAL_PRINT_NAME_ARGS_MAX_SIZE,
"[%lu,%lu]", tmp1, tmp2);
}
return ptr->buffers[ptr->cntr-1];
}
static char* ompi_pmix_print_vpids(const opal_vpid_t vpid)
{
opal_print_args_buffers_t *ptr;
ptr = get_print_name_buffer();
if (NULL == ptr) {
OPAL_ERROR_LOG(OPAL_ERR_OUT_OF_RESOURCE);
return opal_print_args_null;
}
/* cycle around the ring */
if (OPAL_PRINT_NAME_ARG_NUM_BUFS == ptr->cntr) {
ptr->cntr = 0;
}
if (OPAL_VPID_INVALID == vpid) {
snprintf(ptr->buffers[ptr->cntr++], OPAL_PRINT_NAME_ARGS_MAX_SIZE, "INVALID");
} else if (OPAL_VPID_WILDCARD == vpid) {
snprintf(ptr->buffers[ptr->cntr++], OPAL_PRINT_NAME_ARGS_MAX_SIZE, "WILDCARD");
} else {
snprintf(ptr->buffers[ptr->cntr++],
OPAL_PRINT_NAME_ARGS_MAX_SIZE,
"%ld", (long)vpid);
}
return ptr->buffers[ptr->cntr-1];
}
char* ompi_pmix_print_name(const ompi_process_name_t *name)
{
opal_print_args_buffers_t *ptr;
char *job, *vpid;
/* protect against NULL names */
if (NULL == name) {
/* get the next buffer */
ptr = get_print_name_buffer();
if (NULL == ptr) {
OPAL_ERROR_LOG(OPAL_ERR_OUT_OF_RESOURCE);
return opal_print_args_null;
}
/* cycle around the ring */
if (OPAL_PRINT_NAME_ARG_NUM_BUFS == ptr->cntr) {
ptr->cntr = 0;
}
snprintf(ptr->buffers[ptr->cntr++], OPAL_PRINT_NAME_ARGS_MAX_SIZE, "[NO-NAME]");
return ptr->buffers[ptr->cntr-1];
}
/* get the jobid, vpid strings first - this will protect us from
* stepping on each other's buffer. This also guarantees
* that the print_args function has been initialized, so
* we don't need to duplicate that here
*/
job = ompi_pmix_print_jobids(name->jobid);
vpid = ompi_pmix_print_vpids(name->vpid);
/* get the next buffer */
ptr = get_print_name_buffer();
if (NULL == ptr) {
OPAL_ERROR_LOG(OPAL_ERR_OUT_OF_RESOURCE);
return opal_print_args_null;
}
/* cycle around the ring */
if (OPAL_PRINT_NAME_ARG_NUM_BUFS == ptr->cntr) {
ptr->cntr = 0;
}
snprintf(ptr->buffers[ptr->cntr++],
OPAL_PRINT_NAME_ARGS_MAX_SIZE,
"[%s,%s]", job, vpid);
return ptr->buffers[ptr->cntr-1];
}
int ompi_rte_compare_name_fields(ompi_rte_cmp_bitmask_t fields,
const opal_process_name_t* name1,
const opal_process_name_t* name2)
{
/* handle the NULL pointer case */
if (NULL == name1 && NULL == name2) {
return OPAL_EQUAL;
} else if (NULL == name1) {
return OPAL_VALUE2_GREATER;
} else if (NULL == name2) {
return OPAL_VALUE1_GREATER;
}
/* in this comparison function, we check for exact equalities.
* In the case of wildcards, we check to ensure that the fields
* actually match those values - thus, a "wildcard" in this
* function does not actually stand for a wildcard value, but
* rather a specific value - UNLESS the CMP_WILD bitmask value
* is set
*/
/* check job id */
if (OMPI_RTE_CMP_JOBID & fields) {
if (OMPI_RTE_CMP_WILD & fields &&
(pmix_name_wildcard.jobid == name1->jobid ||
pmix_name_wildcard.jobid == name2->jobid)) {
goto check_vpid;
}
if (name1->jobid < name2->jobid) {
return OPAL_VALUE2_GREATER;
} else if (name1->jobid > name2->jobid) {
return OPAL_VALUE1_GREATER;
}
}
/* get here if jobid's are equal, or not being checked
* now check vpid
*/
check_vpid:
if (OMPI_RTE_CMP_VPID & fields) {
if (OMPI_RTE_CMP_WILD & fields &&
(pmix_name_wildcard.vpid == name1->vpid ||
pmix_name_wildcard.vpid == name2->vpid)) {
return OPAL_EQUAL;
}
if (name1->vpid < name2->vpid) {
return OPAL_VALUE2_GREATER;
} else if (name1->vpid > name2->vpid) {
return OPAL_VALUE1_GREATER;
}
}
/* only way to get here is if all fields are being checked and are equal,
* or jobid not checked, but vpid equal,
* only vpid being checked, and equal
* return that fact
*/
return OPAL_EQUAL;
}
int ompi_rte_convert_string_to_process_name(opal_process_name_t *name,
const char* name_string)
{
char *temp, *token;
opal_jobid_t job;
opal_vpid_t vpid;
int return_code=OPAL_SUCCESS;
/* set default */
name->jobid = pmix_name_invalid.jobid;
name->vpid = pmix_name_invalid.vpid;
/* check for NULL string - error */
if (NULL == name_string) {
OPAL_ERROR_LOG(OPAL_ERR_BAD_PARAM);
return OPAL_ERR_BAD_PARAM;
}
temp = strdup(name_string); /** copy input string as the strtok process is destructive */
token = strchr(temp, OPAL_SCHEMA_DELIMITER_CHAR); /** get first field -> jobid */
/* check for error */
if (NULL == token) {
OPAL_ERROR_LOG(OPAL_ERR_BAD_PARAM);
free(temp);
return OPAL_ERR_BAD_PARAM;
}
*token = '\0';
token++;
/* check for WILDCARD character - assign
* value accordingly, if found
*/
if (0 == strcmp(temp, OPAL_SCHEMA_WILDCARD_STRING)) {
job = pmix_name_wildcard.jobid;
} else if (0 == strcmp(temp, OPAL_SCHEMA_INVALID_STRING)) {
job = pmix_name_invalid.jobid;
} else {
job = strtoul(temp, NULL, 10);
}
/* check for WILDCARD character - assign
* value accordingly, if found
*/
if (0 == strcmp(token, OPAL_SCHEMA_WILDCARD_STRING)) {
vpid = pmix_name_wildcard.vpid;
} else if (0 == strcmp(token, OPAL_SCHEMA_INVALID_STRING)) {
vpid = pmix_name_invalid.vpid;
} else {
vpid = strtoul(token, NULL, 10);
}
name->jobid = job;
name->vpid = vpid;
free(temp);
return return_code;
}
int ompi_rte_convert_process_name_to_string(char** name_string,
const opal_process_name_t *name)
{
char *tmp, *tmp2;
if (NULL == name) { /* got an error */
OPAL_ERROR_LOG(OPAL_ERR_BAD_PARAM);
return OPAL_ERR_BAD_PARAM;
}
/* check for wildcard and invalid values - where encountered, insert the
* corresponding string so we can correctly parse the name string when
* it is passed back to us later
*/
if (pmix_name_wildcard.jobid == name->jobid) {
opal_asprintf(&tmp, "%s", OPAL_SCHEMA_WILDCARD_STRING);
} else if (pmix_name_invalid.jobid == name->jobid) {
opal_asprintf(&tmp, "%s", OPAL_SCHEMA_INVALID_STRING);
} else {
opal_asprintf(&tmp, "%lu", (unsigned long)name->jobid);
}
if (pmix_name_wildcard.vpid == name->vpid) {
opal_asprintf(&tmp2, "%s%c%s", tmp, OPAL_SCHEMA_DELIMITER_CHAR, OPAL_SCHEMA_WILDCARD_STRING);
} else if (pmix_name_invalid.vpid == name->vpid) {
opal_asprintf(&tmp2, "%s%c%s", tmp, OPAL_SCHEMA_DELIMITER_CHAR, OPAL_SCHEMA_INVALID_STRING);
} else {
opal_asprintf(&tmp2, "%s%c%lu", tmp, OPAL_SCHEMA_DELIMITER_CHAR, (unsigned long)name->vpid);
}
opal_asprintf(name_string, "%s", tmp2);
free(tmp);
free(tmp2);
return OPAL_SUCCESS;
}
static int ompi_pmix_convert_string_to_jobid(opal_jobid_t *jobid, const char* jobidstring)
{
if (NULL == jobidstring) { /* got an error */
OPAL_ERROR_LOG(OPAL_ERR_BAD_PARAM);
*jobid = OPAL_JOBID_INVALID;
return OPAL_ERR_BAD_PARAM;
}
/** check for wildcard character - handle appropriately */
if (0 == strcmp(OPAL_SCHEMA_WILDCARD_STRING, jobidstring)) {
*jobid = OPAL_JOBID_WILDCARD;
return OPAL_SUCCESS;
}
/* check for invalid value */
if (0 == strcmp(OPAL_SCHEMA_INVALID_STRING, jobidstring)) {
*jobid = OPAL_JOBID_INVALID;
return OPAL_SUCCESS;
}
*jobid = strtoul(jobidstring, NULL, 10);
return OPAL_SUCCESS;
}
static int ompi_pmix_snprintf_jobid(char *jobid_string, size_t size, const opal_jobid_t jobid)
{
int rc;
/* check for wildcard value - handle appropriately */
if (OPAL_JOBID_WILDCARD == jobid) {
(void)opal_string_copy(jobid_string,
OPAL_SCHEMA_WILDCARD_STRING, size);
} else {
rc = snprintf(jobid_string, size, "%ld", (long) jobid);
if (0 > rc) {
return OPAL_ERROR;
}
}
return OPAL_SUCCESS;
}
/**
* Static functions used to configure the interactions between the OPAL and
* the runtime.
*/
static char*
_process_name_print_for_opal(const opal_process_name_t procname)
{
ompi_process_name_t* rte_name = (ompi_process_name_t*)&procname;
return ompi_pmix_print_name(rte_name);
}
static char*
_jobid_print_for_opal(const opal_jobid_t jobid)
{
return ompi_pmix_print_jobids(jobid);
}
static char*
_vpid_print_for_opal(const opal_vpid_t vpid)
{
return ompi_pmix_print_vpids(vpid);
}
static int
_process_name_compare(const opal_process_name_t p1, const opal_process_name_t p2)
{
return ompi_rte_compare_name_fields(OMPI_RTE_CMP_ALL, &p1, &p2);
}
static int _convert_string_to_process_name(opal_process_name_t *name,
const char* name_string)
{
return ompi_rte_convert_string_to_process_name(name, name_string);
}
static int _convert_process_name_to_string(char** name_string,
const opal_process_name_t *name)
{
return ompi_rte_convert_process_name_to_string(name_string, name);
}
static int
_convert_string_to_jobid(opal_jobid_t *jobid, const char *jobid_string)
{
return ompi_pmix_convert_string_to_jobid(jobid, jobid_string);
}
int ompi_rte_init(int *pargc, char ***pargv)
{
int ret;
char *error = NULL;
opal_process_name_t pname;
pmix_proc_t myproc, rproc;
int u32, *u32ptr;
uint16_t u16, *u16ptr;
char **peers=NULL;
char *envar, *ev1, *ev2;
char *val;
size_t i;
uint64_t unique_key[2];
char *string_key;
pmix_value_t pval;
pmix_status_t rc;
u32ptr = &u32;
u16ptr = &u16;
memset(&pmix_process_info, 0, sizeof(pmix_process_info));
/* Convince OPAL to use our naming scheme */
opal_process_name_print = _process_name_print_for_opal;
opal_vpid_print = _vpid_print_for_opal;
opal_jobid_print = _jobid_print_for_opal;
opal_compare_proc = _process_name_compare;
opal_convert_string_to_process_name = _convert_string_to_process_name;
opal_convert_process_name_to_string = _convert_process_name_to_string;
opal_snprintf_jobid = ompi_pmix_snprintf_jobid;
opal_convert_string_to_jobid = _convert_string_to_jobid;
/* initialize the opal layer */
if (OPAL_SUCCESS != (ret = opal_init(pargc, pargv))) {
error = "opal_init";
goto error;
}
/* initialize the selected module */
if (!PMIx_Initialized() && (PMIX_SUCCESS != (ret = PMIx_Init(&myproc, NULL, 0)))) {
/* we cannot run - this could be due to being direct launched
* without the required PMI support being built, so print
* out a help message indicating it */
opal_show_help("help-mpi-runtime.txt", "no-pmi", true, PMIx_Error_string(ret));
return OPAL_ERR_SILENT;
}
/* setup the process name fields */
OPAL_PMIX_CONVERT_PROCT(rc, &pname, &myproc);
if (OPAL_SUCCESS != rc) {
return rc;
}
OPAL_PROC_MY_NAME.jobid = pname.jobid;
OPAL_PROC_MY_NAME.vpid = pname.vpid;
pmix_process_info.my_name.jobid = OPAL_PROC_MY_NAME.jobid;
pmix_process_info.my_name.vpid = OPAL_PROC_MY_NAME.vpid;
/* set our hostname */
OPAL_MODEX_RECV_VALUE_OPTIONAL(ret, PMIX_HOSTNAME, &OPAL_PROC_MY_NAME,
(char**)&ev1, PMIX_STRING);
if (PMIX_SUCCESS == ret) {
if (NULL != opal_process_info.nodename) {
free(opal_process_info.nodename);
}
opal_process_info.nodename = ev1; // ev1 is an allocated string
}
ompi_process_info.nodename = opal_process_info.nodename;
/* get our local rank from PMI */
OPAL_MODEX_RECV_VALUE(rc, PMIX_LOCAL_RANK,
&pmix_process_info.my_name, &u16ptr, PMIX_UINT16);
if (PMIX_SUCCESS != rc) {
/* assume we are a singleton */
u16 = 0;
ompi_singleton = true;
}
pmix_process_info.my_local_rank = u16;
/* get our node rank from PMI */
OPAL_MODEX_RECV_VALUE(rc, PMIX_NODE_RANK,
&pmix_process_info.my_name, &u16ptr, PMIX_UINT16);
if (PMIX_SUCCESS != rc) {
/* assume we are a singleton */
u16 = 0;
}
pmix_process_info.my_node_rank = u16;
/* get job size */
pname.jobid = pmix_process_info.my_name.jobid;
pname.vpid = OPAL_VPID_WILDCARD;
OPAL_MODEX_RECV_VALUE(rc, PMIX_JOB_SIZE,
&pname, &u32ptr, PMIX_UINT32);
if (PMIX_SUCCESS != rc) {
/* assume we are a singleton */
u32 = 1;
}
pmix_process_info.num_procs = u32;
/* push into the environ for pickup in MPI layer for
* MPI-3 required info key
*/
if (NULL == getenv(OPAL_MCA_PREFIX"opal_ess_num_procs")) {
opal_asprintf(&ev1, OPAL_MCA_PREFIX"opal_ess_num_procs=%d", pmix_process_info.num_procs);
putenv(ev1);
added_num_procs = true;
}
if (NULL == getenv("OMPI_APP_CTX_NUM_PROCS")) {
opal_asprintf(&ev2, "OMPI_APP_CTX_NUM_PROCS=%d", pmix_process_info.num_procs);
putenv(ev2);
added_app_ctx = true;
}
/* get our app number from PMI - ok if not found */
OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_APPNUM,
&pmix_process_info.my_name, &u32ptr, PMIX_UINT32);
if (PMIX_SUCCESS == rc) {
pmix_process_info.app_num = u32;
} else {
pmix_process_info.app_num = 0;
}
/* get the number of local peers - required for wireup of
* shared memory BTL */
OPAL_MODEX_RECV_VALUE(rc, PMIX_LOCAL_SIZE,
&pname, &u32ptr, PMIX_UINT32);
if (PMIX_SUCCESS == rc) {
pmix_process_info.num_local_peers = u32 - 1; // want number besides ourselves
} else {
pmix_process_info.num_local_peers = 0;
}
/* setup transport keys in case the MPI layer needs them -
* we can use the jobfam and stepid as unique keys
* because they are unique values assigned by the RM
*/
if (NULL == getenv(OPAL_MCA_PREFIX"opal_precondition_transports")) {
unique_key[0] = (pmix_process_info.my_name.jobid & 0xff00) >> 16;
unique_key[1] = pmix_process_info.my_name.jobid & 0x00ff;
if (NULL == (string_key = pre_condition_transports_print(unique_key))) {
OPAL_ERROR_LOG(OPAL_ERR_OUT_OF_RESOURCE);
return OPAL_ERR_OUT_OF_RESOURCE;
}
opal_asprintf(&envar, OPAL_MCA_PREFIX"opal_precondition_transports=%s", string_key);
putenv(envar);
added_transport_keys = true;
/* cannot free the envar as that messes up our environ */
free(string_key);
}
/* retrieve temp directories info */
OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_TMPDIR, &pname, &val, PMIX_STRING);
if (OPAL_SUCCESS == rc && NULL != val) {
pmix_process_info.top_session_dir = val;
} else {
/* we need to create something */
rc = _setup_top_session_dir(&pmix_process_info.top_session_dir);
if (OPAL_SUCCESS != rc) {
error = "top session directory";
goto error;
}
}
/* retrieve job-session directory info */
OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_NSDIR, &pname, &val, PMIX_STRING);
if (PMIX_SUCCESS == rc && NULL != val) {
pmix_process_info.job_session_dir = val;
val = NULL;
} else {
/* we need to create something */
rc = _setup_job_session_dir(&pmix_process_info.job_session_dir);
if (OPAL_SUCCESS != rc) {
error = "job session directory";
goto error;
}
}
/* retrieve proc-session directory info */
OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_PROCDIR, &OPAL_PROC_MY_NAME, &val, OPAL_STRING);
if (OPAL_SUCCESS == rc && NULL != val) {
pmix_process_info.proc_session_dir = val;
} else {
/* we need to create something */
rc = _setup_proc_session_dir(&pmix_process_info.proc_session_dir);
if (OPAL_SUCCESS != rc) {
error = "proc session directory";
goto error;
}
}
/* identify our location */
val = NULL;
OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_LOCALITY_STRING,
&pmix_process_info.my_name, &val, PMIX_STRING);
if (PMIX_SUCCESS == rc && NULL != val) {
pmix_process_info.cpuset = val;
} else {
pmix_process_info.cpuset = NULL;
}
/* get our local peers */
if (0 < pmix_process_info.num_local_peers) {
/* if my local rank if too high, then that's an error */
if (pmix_process_info.num_local_peers < pmix_process_info.my_local_rank) {
ret = OPAL_ERR_BAD_PARAM;
error = "num local peers";
goto error;
}
/* retrieve the local peers */
OPAL_MODEX_RECV_VALUE(rc, PMIX_LOCAL_PEERS,
&pname, &val, PMIX_STRING);
if (PMIX_SUCCESS == rc && NULL != val) {
peers = opal_argv_split(val, ',');
free(val);
} else {
peers = NULL;
}
} else {
peers = NULL;
}
/* set the locality */
if (NULL != peers) {
pname.jobid = pmix_process_info.my_name.jobid;
for (i=0; NULL != peers[i]; i++) {
pname.vpid = strtoul(peers[i], NULL, 10);
if (pname.vpid == pmix_process_info.my_name.vpid) {
/* we are fully local to ourselves */
u16 = OPAL_PROC_ALL_LOCAL;
} else {
val = NULL;
OPAL_MODEX_RECV_VALUE_OPTIONAL(rc, PMIX_LOCALITY_STRING,
&pname, &val, PMIX_STRING);
if (PMIX_SUCCESS == rc && NULL != val) {
u16 = opal_hwloc_compute_relative_locality(pmix_process_info.cpuset, val);
free(val);
} else {
/* all we can say is that it shares our node */
u16 = OPAL_PROC_ON_CLUSTER | OPAL_PROC_ON_CU | OPAL_PROC_ON_NODE;
}
}
pval.type = PMIX_UINT16;
pval.data.uint16 = u16;
OPAL_PMIX_CONVERT_NAME(&rproc, &pname);
rc = PMIx_Store_internal(&rproc, PMIX_LOCALITY, &pval);
if (PMIX_SUCCESS != rc) {
ret = opal_pmix_convert_status(rc);
error = "local store of locality";
opal_argv_free(peers);
if (NULL != pmix_process_info.cpuset) {
free(pmix_process_info.cpuset);
}
goto error;
}
}
opal_argv_free(peers);
}
/* poor attempt to detect we are bound */
if (NULL != getenv("SLURM_CPU_BIND_TYPE")) {
pmix_proc_is_bound = true;
}
/* set the remaining opal_process_info fields. Note that
* the OPAL layer will have initialized these to NULL, and
* anyone between us would not have strdup'd the string, so
* we cannot free it here */
opal_process_info.top_session_dir = pmix_process_info.top_session_dir;
opal_process_info.job_session_dir = pmix_process_info.job_session_dir;
opal_process_info.proc_session_dir = pmix_process_info.proc_session_dir;
opal_process_info.num_local_peers = (int32_t)pmix_process_info.num_local_peers;
opal_process_info.my_local_rank = (int32_t)pmix_process_info.my_local_rank;
opal_process_info.cpuset = pmix_process_info.cpuset;
return OPAL_SUCCESS;
error:
if (OPAL_ERR_SILENT != ret ) {
opal_show_help("help-mpi-runtime.txt",
"mpi_init:startup:internal-failure",
true, "MPI runtime init", "RTE init",
error, opal_strerror(ret), ret);
}
opal_finalize();
return ret;
}
static bool check_file(const char *root, const char *path)
{
struct stat st;
char *fullpath;
/*
* Keep:
* - non-zero files starting with "output-"
*/
if (0 == strncmp(path, "output-", strlen("output-"))) {
fullpath = opal_os_path(false, &fullpath, root, path, NULL);
stat(fullpath, &st);
free(fullpath);
if (0 == st.st_size) {
return true;
}
return false;
}
return true;
}
int ompi_rte_finalize(void)
{
/* remove the envars that we pushed into environ
* so we leave that structure intact
*/
if (added_transport_keys) {
unsetenv(OPAL_MCA_PREFIX"opal_precondition_transports");
}
if (added_num_procs) {
unsetenv(OPAL_MCA_PREFIX"opal_ess_num_procs");
}
if (added_app_ctx) {
unsetenv("OMPI_APP_CTX_NUM_PROCS");
}
/* shutdown pmix */
PMIx_Finalize(NULL, 0);
/* cleanup the session directory we created */
if (NULL != pmix_process_info.job_session_dir) {
opal_os_dirpath_destroy(pmix_process_info.job_session_dir,
false, check_file);
free(pmix_process_info.job_session_dir);
}
free (pmix_process_info.cpuset);
pmix_process_info.cpuset = NULL;
return OMPI_SUCCESS;
}
void ompi_rte_abort(int error_code, char *fmt, ...)
{
va_list arglist;
char* buffer = NULL;
struct timespec tp = {0, 100000};
/* If there was a message, output it */
va_start(arglist, fmt);
if( NULL != fmt ) {
opal_vasprintf( &buffer, fmt, arglist );
}
va_end(arglist);
/* call abort */
PMIx_Abort(error_code, buffer, NULL, 0);
if (NULL != buffer) {
free(buffer);
}
/* provide a little delay for the PMIx thread to
* get the info out */
nanosleep(&tp, NULL);
/* Now Exit */
_exit(error_code);
}
void ompi_rte_abort_peers(opal_process_name_t *procs,
int32_t num_procs,
int error_code)
{
return;
}
static size_t handler = SIZE_MAX;
static bool debugger_event_active = true;
static void _release_fn(size_t refid, pmix_status_t status,
const pmix_proc_t *source,
pmix_info_t info[], size_t ninfo,
pmix_info_t *results, size_t nresults,
pmix_event_notification_cbfunc_fn_t cbfunc,
void *cbdata)
{
/* must let the notifier know we are done */
if (NULL != cbfunc) {
cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata);
}
debugger_event_active = false;
}
/*
* Wait for a debugger if asked. We support two ways of waiting for
* attaching debuggers -- see big comment in
* pmix/tools/pmixrun/debuggers.c explaining the two scenarios.
*/
void ompi_rte_wait_for_debugger(void)
{
pmix_info_t directive;
char *evar;
int time, code = PMIX_ERR_DEBUGGER_RELEASE;
/* check PMIx to see if we are under a debugger */
if (NULL == getenv("PMIX_DEBUG_WAIT_FOR_NOTIFY") &&
NULL == getenv("PMIX_TEST_DEBUGGER_ATTACH")) {
/* if not, just return */
return;
}
/* if we are being debugged, then we need to find
* the correct plug-ins
*/
ompi_debugger_setup_dlls();
if (NULL != (evar = getenv("PMIX_TEST_DEBUGGER_SLEEP"))) {
time = strtol(evar, NULL, 10);
sleep(time);
return;
}
/* register an event handler for the PMIX_ERR_DEBUGGER_RELEASE event */
PMIX_INFO_LOAD(&directive, PMIX_EVENT_HDLR_NAME, "MPI-DEBUGGER-ATTACH", PMIX_STRING);
PMIx_Register_event_handler(&code, 1, &directive, 1, _release_fn, NULL, NULL);
PMIX_INFO_DESTRUCT(&directive);
/* let the MPI progress engine run while we wait for debugger release */
OMPI_WAIT_FOR_COMPLETION(debugger_event_active);
/* deregister the event handler */
PMIx_Deregister_event_handler(handler, NULL, NULL);
}
static char* pre_condition_transports_print(uint64_t *unique_key)
{
unsigned int *int_ptr;
size_t i, j, string_key_len, written_len;
char *string_key = NULL, *format = NULL;
/* string is two 64 bit numbers printed in hex with a dash between
* and zero padding.
*/
string_key_len = (sizeof(uint64_t) * 2) * 2 + strlen("-") + 1;
string_key = (char*) malloc(string_key_len);
if (NULL == string_key) {
return NULL;
}
string_key[0] = '\0';
written_len = 0;
/* get a format string based on the length of an unsigned int. We
* want to have zero padding for sizeof(unsigned int) * 2
* characters -- when printing as a hex number, each byte is
* represented by 2 hex characters. Format will contain something
* that looks like %08lx, where the number 8 might be a different
* number if the system has a different sized long (8 would be for
* sizeof(int) == 4)).
*/
opal_asprintf(&format, "%%0%dx", (int)(sizeof(unsigned int)) * 2);
/* print the first number */
int_ptr = (unsigned int*) &unique_key[0];
for (i = 0 ; i < sizeof(uint64_t) / sizeof(unsigned int) ; ++i) {
if (0 == int_ptr[i]) {
/* inject some energy */
for (j=0; j < sizeof(unsigned int); j++) {
int_ptr[i] |= j << j;
}
}
snprintf(string_key + written_len,
string_key_len - written_len,
format, int_ptr[i]);
written_len = strlen(string_key);
}
/* print the middle dash */
snprintf(string_key + written_len, string_key_len - written_len, "-");
written_len = strlen(string_key);
/* print the second number */
int_ptr = (unsigned int*) &unique_key[1];
for (i = 0 ; i < sizeof(uint64_t) / sizeof(unsigned int) ; ++i) {
if (0 == int_ptr[i]) {
/* inject some energy */
for (j=0; j < sizeof(unsigned int); j++) {
int_ptr[i] |= j << j;
}
}
snprintf(string_key + written_len,
string_key_len - written_len,
format, int_ptr[i]);
written_len = strlen(string_key);
}
free(format);
return string_key;
}
static int _setup_top_session_dir(char **sdir)
{
char *tmpdir;
if( NULL == (tmpdir = getenv("TMPDIR")) )
if( NULL == (tmpdir = getenv("TEMP")) )
if( NULL == (tmpdir = getenv("TMP")) )
tmpdir = "/tmp";
*sdir = strdup(tmpdir);
return OPAL_SUCCESS;
}
static int _setup_job_session_dir(char **sdir)
{
/* get the effective uid */
uid_t uid = geteuid();
if (0 > opal_asprintf(sdir, "%s/ompi.%s.%lu/jf.0/%u",
pmix_process_info.top_session_dir,
pmix_process_info.nodename,
(unsigned long)uid,
pmix_process_info.my_name.jobid)) {
pmix_process_info.job_session_dir = NULL;
return OPAL_ERR_OUT_OF_RESOURCE;
}
return OPAL_SUCCESS;
}
static int _setup_proc_session_dir(char **sdir)
{
if (0 > opal_asprintf(sdir, "%s/%d",
pmix_process_info.job_session_dir,
pmix_process_info.my_name.vpid)) {
pmix_process_info.proc_session_dir = NULL;
return OPAL_ERR_OUT_OF_RESOURCE;
}
return OPAL_SUCCESS;
}
Показать git blame Копировать постоянную ссылку