Given total lack of interest/support from the folks behind these environments, and the fact that we can now scale so well with our own daemons, it seems unlikely that we will be able to pursue direct and/or standalone launch in these environments. If that situation ever changes, it is easy enough to revive the effort since little had really been done to-date.
Meantime, no reason to continue dragging these around. This commit was SVN r18119.
Этот коммит содержится в:
Ralph Castain
родитель
dc2f88b9f0
Коммит
22343e6e0b
@ -1,52 +0,0 @@
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
AM_CPPFLAGS = $(plm_md_CPPFLAGS)
|
||||
|
||||
dist_pkgdata_DATA = help-plm-md.txt
|
||||
|
||||
sources = \
|
||||
plm_md.h \
|
||||
plm_md_component.c \
|
||||
plm_md_module.c
|
||||
|
||||
# Make the output library in this directory, and name it either
|
||||
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
|
||||
# (for static builds).
|
||||
|
||||
if OMPI_BUILD_plm_md_DSO
|
||||
lib =
|
||||
lib_sources =
|
||||
component = mca_plm_md.la
|
||||
component_sources = $(sources)
|
||||
else
|
||||
lib = libmca_plm_md.la
|
||||
lib_sources = $(sources)
|
||||
component =
|
||||
component_sources =
|
||||
endif
|
||||
|
||||
mcacomponentdir = $(pkglibdir)
|
||||
mcacomponent_LTLIBRARIES = $(component)
|
||||
mca_plm_md_la_SOURCES = $(component_sources)
|
||||
mca_plm_md_la_LDFLAGS = -module -avoid-version $(plm_md_LDFLAGS)
|
||||
|
||||
noinst_LTLIBRARIES = $(lib)
|
||||
libmca_plm_md_la_SOURCES = $(lib_sources)
|
||||
libmca_plm_md_la_LDFLAGS = -module -avoid-version $(plm_md_LDFLAGS)
|
||||
libmca_plm_md_la_LIBADD = $(plm_md_LIBS)
|
||||
@ -1,37 +0,0 @@
|
||||
# -*- shell-script -*-
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
# MCA_plm_md_CONFIG([action-if-found], [action-if-not-found])
|
||||
# -----------------------------------------------------------
|
||||
AC_DEFUN([MCA_plm_md_CONFIG],[
|
||||
OMPI_CHECK_TM([plm_md], [plm_md_good=1], [plm_md_good=0])
|
||||
|
||||
# if check worked, set wrapper flags if so.
|
||||
# Evaluate succeed / fail
|
||||
AS_IF([test "$plm_md_good" = "1"],
|
||||
[plm_md_WRAPPER_EXTRA_LDFLAGS="$plm_md_LDFLAGS"
|
||||
plm_md_WRAPPER_EXTRA_LIBS="$plm_md_LIBS"
|
||||
$1],
|
||||
[$2])
|
||||
|
||||
# set build flags to use in makefile
|
||||
AC_SUBST([plm_md_CPPFLAGS])
|
||||
AC_SUBST([plm_md_LDFLAGS])
|
||||
AC_SUBST([plm_md_LIBS])
|
||||
])dnl
|
||||
@ -1,22 +0,0 @@
|
||||
# -*- shell-script -*-
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2007 Los Alamos National Security, LLC. All rights
|
||||
# reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
PARAM_CONFIG_FILES="Makefile"
|
||||
@ -1,52 +0,0 @@
|
||||
# -*- text -*-
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
[tm-bad-launchid]
|
||||
The TM (PBS / Torque) process starter cannot spawn the specified
|
||||
application on a remote node due to an invalid launch_id.
|
||||
|
||||
This is most likely due to use of the "--hostfile" option to the
|
||||
command line with one or more hosts in that file not having
|
||||
been allocated to this job.
|
||||
|
||||
Removing "--hostfile" from the command line will likely allow the
|
||||
application to be launched.
|
||||
#
|
||||
[multiple-prefixes]
|
||||
Multiple different --prefix options were specified to mpirun for the
|
||||
same node. This is a fatal error for the TM (PBS / Torque) process
|
||||
starter in Open MPI.
|
||||
|
||||
The first two prefix values supplied for node %s were:
|
||||
%s
|
||||
and %s
|
||||
#
|
||||
[tm-spawn-failed]
|
||||
The TM (PBS / Torque) process starter failed to spawn a daemon (orted)
|
||||
on a remote node.
|
||||
|
||||
Command line: %s
|
||||
Node name: %s
|
||||
Launch id: %d
|
||||
|
||||
If you do not understand this error mesage, please try the following:
|
||||
|
||||
1. Ensure that the executable "orted" is in your PATH
|
||||
2. Use the --prefix option to indicate where we can
|
||||
find that executable
|
||||
3. Talk to your local system administrator
|
||||
@ -1,36 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2006 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2006 Cisco Systems, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#ifndef ORTE_PLM_MD_EXPORT_H
|
||||
#define ORTE_PLM_MD_EXPORT_H
|
||||
|
||||
#include "orte_config.h"
|
||||
|
||||
#include "opal/mca/mca.h"
|
||||
#include "orte/mca/plm/plm.h"
|
||||
|
||||
BEGIN_C_DECLS
|
||||
|
||||
/* Globally exported variables */
|
||||
ORTE_DECLSPEC extern orte_plm_base_component_t mca_plm_md_component;
|
||||
extern orte_plm_base_module_t orte_plm_md_module;
|
||||
|
||||
END_C_DECLS
|
||||
|
||||
#endif /* ORTE_PLM_MD_EXPORT_H */
|
||||
@ -1,117 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2006 Cisco Systems, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*
|
||||
* These symbols are in a file by themselves to provide nice linker
|
||||
* semantics. Since linkers generally pull in symbols by object
|
||||
* files, keeping these symbols as the only symbols in this file
|
||||
* prevents utility programs such as "ompi_info" from having to import
|
||||
* entire components just to query their version and parameters.
|
||||
*/
|
||||
|
||||
#include "orte_config.h"
|
||||
#include "orte/constants.h"
|
||||
|
||||
#include "opal/mca/base/mca_base_param.h"
|
||||
#include "opal/util/output.h"
|
||||
#include "opal/util/argv.h"
|
||||
|
||||
#include "orte/util/proc_info.h"
|
||||
#include "orte/mca/errmgr/errmgr.h"
|
||||
|
||||
#include "orte/mca/plm/plm.h"
|
||||
#include "orte/mca/plm/base/base.h"
|
||||
#include "orte/mca/plm/base/plm_private.h"
|
||||
#include "plm_md.h"
|
||||
|
||||
|
||||
/*
|
||||
* Public string showing the plm ompi_tm component version number
|
||||
*/
|
||||
const char *mca_plm_md_component_version_string =
|
||||
"Open MPI tm plm MCA component version " ORTE_VERSION;
|
||||
|
||||
|
||||
|
||||
/*
|
||||
* Local function
|
||||
*/
|
||||
static int plm_md_open(void);
|
||||
static int plm_md_close(void);
|
||||
static orte_plm_base_module_t *plm_md_init(int *priority);
|
||||
|
||||
|
||||
/*
|
||||
* Instantiate the public struct with all of our public information
|
||||
* and pointers to our public functions in it
|
||||
*/
|
||||
|
||||
orte_plm_base_component_t mca_plm_md_component = {
|
||||
{
|
||||
/* Indicate that we are a plm v1.0.0 component (which also
|
||||
implies a specific MCA version) */
|
||||
ORTE_PLM_BASE_VERSION_1_0_0,
|
||||
|
||||
/* Component name and version */
|
||||
"md",
|
||||
ORTE_MAJOR_VERSION,
|
||||
ORTE_MINOR_VERSION,
|
||||
ORTE_RELEASE_VERSION,
|
||||
|
||||
/* Component open and close functions */
|
||||
plm_md_open,
|
||||
plm_md_close,
|
||||
},
|
||||
|
||||
/* Next the MCA v1.0.0 component meta data */
|
||||
{
|
||||
/* The component is checkpoint ready */
|
||||
MCA_BASE_METADATA_PARAM_CHECKPOINT
|
||||
},
|
||||
|
||||
/* Initialization / querying functions */
|
||||
plm_md_init
|
||||
};
|
||||
|
||||
|
||||
static int plm_md_open(void)
|
||||
{
|
||||
return ORTE_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
static int plm_md_close(void)
|
||||
{
|
||||
return ORTE_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
static orte_plm_base_module_t *plm_md_init(int *priority)
|
||||
{
|
||||
/* Are we running under a TM job? */
|
||||
|
||||
if (NULL != getenv("PBS_ENVIRONMENT") &&
|
||||
NULL != getenv("PBS_JOBID")) {
|
||||
|
||||
/* defer to the tm and tmd modules */
|
||||
*priority = 10;
|
||||
return &orte_plm_md_module;
|
||||
}
|
||||
|
||||
/* Sadly, no */
|
||||
return NULL;
|
||||
}
|
||||
@ -1,636 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2006 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2006-2007 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2007 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*
|
||||
* These symbols are in a file by themselves to provide nice linker
|
||||
* semantics. Since linkers generally pull in symbols by object
|
||||
* files, keeping these symbols as the only symbols in this file
|
||||
* prevents utility programs such as "ompi_info" from having to import
|
||||
* entire components just to query their version and parameters.
|
||||
*/
|
||||
|
||||
#include "orte_config.h"
|
||||
#include "orte/constants.h"
|
||||
#include "orte/types.h"
|
||||
|
||||
#include <sys/types.h>
|
||||
#ifdef HAVE_UNISTD_H
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
#include <signal.h>
|
||||
#ifdef HAVE_STDLIB_H
|
||||
#include <stdlib.h>
|
||||
#endif
|
||||
#ifdef HAVE_SYS_TYPES_H
|
||||
#include <sys/types.h>
|
||||
#endif
|
||||
#ifdef HAVE_SYS_TIME_H
|
||||
#include <sys/time.h>
|
||||
#endif
|
||||
#ifdef HAVE_SYS_STAT_H
|
||||
#include <sys/stat.h>
|
||||
#endif
|
||||
#ifdef HAVE_FCNTL_H
|
||||
#include <fcntl.h>
|
||||
#endif
|
||||
|
||||
#include "opal/mca/installdirs/installdirs.h"
|
||||
#include "opal/util/argv.h"
|
||||
#include "opal/util/output.h"
|
||||
#include "opal/util/opal_environ.h"
|
||||
#include "opal/util/path.h"
|
||||
#include "opal/util/show_help.h"
|
||||
#include "opal/util/basename.h"
|
||||
#include "opal/mca/base/mca_base_param.h"
|
||||
#include "opal/runtime/opal_progress.h"
|
||||
|
||||
#include "orte/util/name_fns.h"
|
||||
#include "orte/runtime/orte_globals.h"
|
||||
#include "orte/runtime/runtime.h"
|
||||
#include "orte/runtime/orte_wakeup.h"
|
||||
#include "orte/runtime/orte_wait.h"
|
||||
#include "orte/mca/rml/rml.h"
|
||||
#include "orte/mca/errmgr/errmgr.h"
|
||||
#include "orte/mca/rmaps/rmaps.h"
|
||||
|
||||
#include "orte/mca/plm/plm.h"
|
||||
#include "orte/mca/plm/base/plm_private.h"
|
||||
#include "plm_md.h"
|
||||
|
||||
|
||||
/*
|
||||
* Local functions
|
||||
*/
|
||||
static int plm_md_init(void);
|
||||
static int plm_md_launch_job(orte_job_t *jdata);
|
||||
static int plm_md_terminate_job(orte_jobid_t jobid);
|
||||
static int plm_md_terminate_orteds(void);
|
||||
static int plm_md_signal_job(orte_jobid_t jobid, int32_t signal);
|
||||
static int plm_md_finalize(void);
|
||||
|
||||
static int plm_md_start_proc(int argc, char **argv, char **env,
|
||||
char *prefix);
|
||||
|
||||
|
||||
/*
|
||||
* Global variable
|
||||
*/
|
||||
orte_plm_base_module_1_0_0_t orte_plm_md_module = {
|
||||
plm_md_init,
|
||||
orte_plm_base_set_hnp_name,
|
||||
plm_md_launch_job,
|
||||
plm_md_terminate_job,
|
||||
plm_md_terminate_orteds,
|
||||
plm_md_signal_job,
|
||||
plm_md_finalize
|
||||
};
|
||||
|
||||
/*
|
||||
* Local variables
|
||||
*/
|
||||
static pid_t srun_pid = 0;
|
||||
static orte_jobid_t active_job = ORTE_JOBID_INVALID;
|
||||
static bool failed_launch;
|
||||
|
||||
|
||||
/**
|
||||
* Init the module
|
||||
*/
|
||||
static int plm_md_init(void)
|
||||
{
|
||||
int rc;
|
||||
|
||||
if (ORTE_SUCCESS != (rc = orte_plm_base_comm_start())) {
|
||||
ORTE_ERROR_LOG(rc);
|
||||
}
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* When working in this function, ALWAYS jump to "cleanup" if
|
||||
* you encounter an error so that orterun will be woken up and
|
||||
* the job can cleanly terminate
|
||||
*/
|
||||
static int plm_md_launch_job(orte_job_t *jdata)
|
||||
{
|
||||
orte_app_context_t **apps;
|
||||
orte_node_t **nodes;
|
||||
orte_std_cntr_t n;
|
||||
orte_job_map_t *map;
|
||||
char *jobid_string = NULL;
|
||||
char *param;
|
||||
char **argv = NULL;
|
||||
int argc;
|
||||
int rc=ORTE_SUCCESS;
|
||||
char *tmp;
|
||||
char** env = NULL;
|
||||
char* var;
|
||||
char *nodelist_flat;
|
||||
char **nodelist_argv;
|
||||
char *rml_uri;
|
||||
int nodelist_argc;
|
||||
int i;
|
||||
char *cur_prefix;
|
||||
struct timeval launchstart, launchstop;
|
||||
|
||||
if (orte_timing) {
|
||||
if (0 != gettimeofday(&launchstart, NULL)) {
|
||||
opal_output(0, "plm_md: could not obtain job start time");
|
||||
launchstart.tv_sec = 0;
|
||||
launchstart.tv_usec = 0;
|
||||
}
|
||||
}
|
||||
|
||||
/* indicate the state of the launch */
|
||||
failed_launch = true;
|
||||
|
||||
/* if this jobid isn't invalid, then it already
|
||||
* has been setup, so skip the setup actions
|
||||
*/
|
||||
if (ORTE_JOBID_INVALID != jdata->jobid) {
|
||||
OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output,
|
||||
"%s plm:slurmd: launching job %s",
|
||||
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
|
||||
ORTE_JOBID_PRINT(jdata->jobid)));
|
||||
goto GETMAP;
|
||||
}
|
||||
|
||||
/* create a jobid for this job */
|
||||
if (ORTE_SUCCESS != (rc = orte_plm_base_create_jobid(&jdata->jobid))) {
|
||||
ORTE_ERROR_LOG(rc);
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output,
|
||||
"%s plm:slurmd: launching job %s",
|
||||
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
|
||||
ORTE_JOBID_PRINT(jdata->jobid)));
|
||||
|
||||
/* setup the job */
|
||||
if (ORTE_SUCCESS != (rc = orte_plm_base_setup_job(jdata))) {
|
||||
ORTE_ERROR_LOG(rc);
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
GETMAP:
|
||||
/* set the active jobid */
|
||||
active_job = jdata->jobid;
|
||||
|
||||
/* Get the map for this job */
|
||||
if (NULL == (map = orte_rmaps.get_job_map(active_job))) {
|
||||
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
|
||||
rc = ORTE_ERR_NOT_FOUND;
|
||||
goto cleanup;
|
||||
}
|
||||
apps = (orte_app_context_t**)jdata->apps->addr;
|
||||
nodes = (orte_node_t**)map->nodes->addr;
|
||||
|
||||
/* we are going to ignore the daemons and just launch
|
||||
* the apps directly. This means we cannot use the
|
||||
* base function to launch_apps as that function assumes
|
||||
* we are doing it via our daemons
|
||||
*/
|
||||
|
||||
/* are there any procs to actually launch? */
|
||||
if (0 == jdata->num_procs) {
|
||||
OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output,
|
||||
"%s plm:slurmd: no procs to launch!",
|
||||
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
/* need the jobid as a string for command line parameter */
|
||||
asprintf(&jobid_string, "%lu", (unsigned long) jdata->jobid);
|
||||
|
||||
/*
|
||||
* start building argv array
|
||||
*/
|
||||
argv = NULL;
|
||||
argc = 0;
|
||||
|
||||
/*
|
||||
* SLURM srun OPTIONS
|
||||
*/
|
||||
|
||||
/* add the srun command */
|
||||
opal_argv_append(&argc, &argv, "srun");
|
||||
|
||||
asprintf(&tmp, "--nodes=%lu", (unsigned long) map->num_new_daemons);
|
||||
opal_argv_append(&argc, &argv, tmp);
|
||||
free(tmp);
|
||||
|
||||
asprintf(&tmp, "--ntasks=%lu", (unsigned long) jdata->num_procs);
|
||||
opal_argv_append(&argc, &argv, tmp);
|
||||
free(tmp);
|
||||
|
||||
/* create nodelist */
|
||||
nodelist_argv = NULL;
|
||||
nodelist_argc = 0;
|
||||
|
||||
for (n=0; n < map->num_nodes; n++ ) {
|
||||
/* add it to the list of nodes upon which
|
||||
* we need to launch a proc
|
||||
*/
|
||||
opal_argv_append(&nodelist_argc, &nodelist_argv, nodes[n]->name);
|
||||
}
|
||||
if (0 == opal_argv_count(nodelist_argv)) {
|
||||
opal_show_help("help-plm-slurmd.txt", "no-hosts-in-list", true);
|
||||
rc = ORTE_ERR_FAILED_TO_START;
|
||||
goto cleanup;
|
||||
}
|
||||
nodelist_flat = opal_argv_join(nodelist_argv, ',');
|
||||
opal_argv_free(nodelist_argv);
|
||||
asprintf(&tmp, "--nodelist=%s", nodelist_flat);
|
||||
opal_argv_append(&argc, &argv, tmp);
|
||||
free(tmp);
|
||||
|
||||
OPAL_OUTPUT_VERBOSE((2, orte_plm_globals.output,
|
||||
"%s plm:slurmd: launching on nodes %s",
|
||||
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), nodelist_flat));
|
||||
|
||||
/*
|
||||
* APP OPTIONS
|
||||
*/
|
||||
|
||||
/* add the executable (as specified by user) */
|
||||
opal_argv_append(&argc, &argv, apps[0]->app);
|
||||
|
||||
/* add the user's cmd args */
|
||||
for (i=1; NULL != apps[0]->argv[i]; i++) {
|
||||
opal_argv_append(&argc, &argv, apps[0]->argv[i]);
|
||||
}
|
||||
|
||||
/* setup the environment - since there will be no daemon, we have
|
||||
* tell the the apps everything they need to know via the environment,
|
||||
* not command line args. Hence, we cannot use the plm base function
|
||||
* to setup the info
|
||||
*/
|
||||
if (NULL != apps[0]->env) {
|
||||
env = opal_environ_merge(environ, apps[0]->env);
|
||||
} else {
|
||||
env = opal_argv_copy(environ);
|
||||
}
|
||||
|
||||
/* add the nodelist */
|
||||
var = mca_base_param_environ_variable("orte", "slurm", "nodelist");
|
||||
opal_setenv(var, nodelist_flat, true, &env);
|
||||
free(nodelist_flat);
|
||||
free(var);
|
||||
|
||||
/* tell the proc what SDS component to use */
|
||||
var = mca_base_param_environ_variable("sds", NULL, NULL);
|
||||
opal_setenv(var, "slurm", true, &env);
|
||||
free(var);
|
||||
|
||||
/* pass the jobid */
|
||||
var = mca_base_param_environ_variable("orte", "sds", "jobid");
|
||||
orte_util_convert_jobid_to_string(¶m, jdata->jobid);
|
||||
opal_setenv(var, param, true, &env);
|
||||
free(var);
|
||||
free(param);
|
||||
|
||||
/* pass the starting vpid */
|
||||
var = mca_base_param_environ_variable("orte", "sds", "vpid");
|
||||
opal_setenv(var, "0", true, &env);
|
||||
free(var);
|
||||
|
||||
/* pass the total number of procs that will be in the system */
|
||||
var = mca_base_param_environ_variable("orte", "sds", "num_procs");
|
||||
asprintf(¶m, "%lu", (unsigned long)(jdata->num_procs));
|
||||
opal_setenv(var, param, true, &env);
|
||||
free(var);
|
||||
free(param);
|
||||
|
||||
/* pass the uri of the hnp */
|
||||
var = mca_base_param_environ_variable("orte", "hnp", "uri");
|
||||
rml_uri = orte_rml.get_contact_info();
|
||||
opal_setenv(var, rml_uri, true, &env);
|
||||
free(var);
|
||||
/* make the proc think the local daemon is the hnp */
|
||||
var = mca_base_param_environ_variable("orte", "local_daemon", "uri");
|
||||
opal_setenv(var, rml_uri, true, &env);
|
||||
free(var);
|
||||
free(rml_uri);
|
||||
|
||||
/* order the proc to use the unity routed component */
|
||||
var = mca_base_param_environ_variable("routed", NULL, NULL);
|
||||
opal_setenv(var, "unity", true, &env);
|
||||
free(var);
|
||||
|
||||
if (0 < opal_output_get_verbosity(orte_plm_globals.output)) {
|
||||
param = opal_argv_join(argv, ' ');
|
||||
OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output,
|
||||
"%s plm:slurmd: final top-level argv:\n\t%s",
|
||||
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
|
||||
(NULL == param) ? "NULL" : param));
|
||||
if (NULL != param) free(param);
|
||||
}
|
||||
|
||||
/* Copy the prefix-directory specified in the
|
||||
corresponding app_context. If there are multiple,
|
||||
different prefix's in the app context, complain (i.e., only
|
||||
allow one --prefix option for the entire slurmd run -- we
|
||||
don't support different --prefix'es for different nodes in
|
||||
the SLURM plm) */
|
||||
cur_prefix = NULL;
|
||||
for (n=0; n < jdata->num_apps; n++) {
|
||||
char * app_prefix_dir = apps[n]->prefix_dir;
|
||||
/* Check for already set cur_prefix_dir -- if different,
|
||||
complain */
|
||||
if (NULL != app_prefix_dir) {
|
||||
if (NULL != cur_prefix &&
|
||||
0 != strcmp (cur_prefix, app_prefix_dir)) {
|
||||
opal_show_help("help-plm-slurmd.txt", "multiple-prefixes",
|
||||
true, cur_prefix, app_prefix_dir);
|
||||
return ORTE_ERR_FATAL;
|
||||
}
|
||||
|
||||
/* If not yet set, copy it; iff set, then it's the
|
||||
same anyway */
|
||||
if (NULL == cur_prefix) {
|
||||
cur_prefix = strdup(app_prefix_dir);
|
||||
OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output,
|
||||
"%s plm:slurmd: Set prefix:%s",
|
||||
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
|
||||
cur_prefix));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* exec the proc(s) */
|
||||
if (ORTE_SUCCESS != (rc = plm_md_start_proc(argc, argv, env, cur_prefix))) {
|
||||
ORTE_ERROR_LOG(rc);
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
/* do NOT wait for srun to complete. Srun only completes when the processes
|
||||
* it starts complete. Instead, we'll catch
|
||||
* any srun failures and deal with them elsewhere
|
||||
*/
|
||||
|
||||
/* declare the launch a success */
|
||||
failed_launch = false;
|
||||
|
||||
if (orte_timing) {
|
||||
if (0 != gettimeofday(&launchstop, NULL)) {
|
||||
opal_output(0, "plm_md: could not obtain stop time");
|
||||
} else {
|
||||
opal_output(0, "plm_md: total job launch time is %ld usec",
|
||||
(launchstop.tv_sec - launchstart.tv_sec)*1000000 +
|
||||
(launchstop.tv_usec - launchstart.tv_usec));
|
||||
}
|
||||
}
|
||||
cleanup:
|
||||
if (NULL != argv) {
|
||||
opal_argv_free(argv);
|
||||
}
|
||||
if (NULL != env) {
|
||||
opal_argv_free(env);
|
||||
}
|
||||
|
||||
if(NULL != jobid_string) {
|
||||
free(jobid_string);
|
||||
}
|
||||
|
||||
/* check for failed launch - if so, force terminate */
|
||||
if (failed_launch) {
|
||||
jdata->state = ORTE_JOB_STATE_FAILED_TO_START;
|
||||
orte_wakeup();
|
||||
}
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
static int plm_md_terminate_job(orte_jobid_t jobid)
|
||||
{
|
||||
if (0 != srun_pid) {
|
||||
kill(srun_pid, -9);
|
||||
}
|
||||
return ORTE_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Terminate the orteds for a given job
|
||||
*/
|
||||
static int plm_md_terminate_orteds(void)
|
||||
{
|
||||
orte_job_t *daemons;
|
||||
int data=1;
|
||||
|
||||
/* get the daemon job object */
|
||||
if (NULL != (daemons = orte_get_job_data_object(ORTE_PROC_MY_NAME->jobid))) {
|
||||
/* record all daemons as terminated */
|
||||
daemons->num_terminated = daemons->num_procs;
|
||||
daemons->state = ORTE_JOB_STATE_TERMINATED;
|
||||
}
|
||||
|
||||
/* trip the orted exit trigger */
|
||||
write(orteds_exit, &data, sizeof(int));
|
||||
opal_progress();
|
||||
|
||||
return ORTE_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Signal all the processes in the child srun by sending the signal directly to it
|
||||
*/
|
||||
static int plm_md_signal_job(orte_jobid_t jobid, int32_t signal)
|
||||
{
|
||||
if (0 != srun_pid) {
|
||||
kill(srun_pid, (int)signal);
|
||||
}
|
||||
return ORTE_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
static int plm_md_finalize(void)
|
||||
{
|
||||
int rc;
|
||||
|
||||
/* cleanup any pending recvs */
|
||||
if (ORTE_SUCCESS != (rc = orte_plm_base_comm_stop())) {
|
||||
ORTE_ERROR_LOG(rc);
|
||||
}
|
||||
|
||||
return ORTE_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
static void srun_wait_cb(pid_t pid, int status, void* cbdata){
|
||||
/* According to the SLURM folks, srun always returns the highest exit
|
||||
code of our remote processes. Thus, a non-zero exit status doesn't
|
||||
necessarily mean that srun failed - it could be that a proc returned
|
||||
a non-zero exit status. Of course, that means the job failed(!), so
|
||||
the end result is the same.
|
||||
|
||||
As a result, we really can't do much with the exit status itself - it
|
||||
could be something in errno (if srun itself failed), or it could be
|
||||
something returned by a proc, or it could be something returned by
|
||||
the OS (e.g., couldn't find the executable). Somebody is welcome
|
||||
to sort out all the options and pretty-print a better error message. For
|
||||
now, though, the only thing that really matters is that
|
||||
srun terminated. Report the error (if there is one) and make sure that orterun
|
||||
wakes up!
|
||||
*/
|
||||
orte_job_t *jdata;
|
||||
|
||||
if (NULL == (jdata = orte_get_job_data_object(active_job))) {
|
||||
/* bad jobid */
|
||||
ORTE_ERROR_LOG(ORTE_ERR_BAD_PARAM);
|
||||
orte_wakeup();
|
||||
return;
|
||||
}
|
||||
|
||||
if (0 != status) {
|
||||
if (failed_launch) {
|
||||
/* we have a problem during launch */
|
||||
opal_output(0, "ERROR: srun failed to start the specified procs.");
|
||||
opal_output(0, "ERROR: This could be due to an inability to find the executable");
|
||||
opal_output(0, "ERROR: on one or more remote nodes, lack of authority to execute");
|
||||
opal_output(0, "ERROR: on one or more specified nodes, or other factors.");
|
||||
|
||||
/* report that the launch has failed so we break out of the daemon
|
||||
* callback receive and exit
|
||||
*/
|
||||
jdata->state = ORTE_JOB_STATE_FAILED_TO_START;
|
||||
|
||||
} else {
|
||||
/* a proc must have died unexpectedly after launch - report
|
||||
* that the job has failed so we exit
|
||||
*/
|
||||
jdata->state = ORTE_JOB_STATE_ABORTED;
|
||||
}
|
||||
} else {
|
||||
jdata->state = ORTE_JOB_STATE_TERMINATED;
|
||||
}
|
||||
|
||||
orte_wakeup();
|
||||
}
|
||||
|
||||
|
||||
static int plm_md_start_proc(int argc, char **argv, char **env,
|
||||
char *prefix)
|
||||
{
|
||||
int fd;
|
||||
char *exec_argv = opal_path_findv(argv[0], 0, env, NULL);
|
||||
|
||||
if (NULL == exec_argv) {
|
||||
return ORTE_ERR_NOT_FOUND;
|
||||
}
|
||||
|
||||
srun_pid = fork();
|
||||
if (-1 == srun_pid) {
|
||||
ORTE_ERROR_LOG(ORTE_ERR_SYS_LIMITS_CHILDREN);
|
||||
return ORTE_ERR_SYS_LIMITS_CHILDREN;
|
||||
}
|
||||
|
||||
if (0 == srun_pid) { /* child */
|
||||
char *bin_base = NULL, *lib_base = NULL;
|
||||
|
||||
/* Figure out the basenames for the libdir and bindir. There
|
||||
is a lengthy comment about this in plm_rsh_module.c
|
||||
explaining all the rationale for how / why we're doing
|
||||
this. */
|
||||
|
||||
lib_base = opal_basename(opal_install_dirs.libdir);
|
||||
bin_base = opal_basename(opal_install_dirs.bindir);
|
||||
|
||||
/* If we have a prefix, then modify the PATH and
|
||||
LD_LIBRARY_PATH environment variables. */
|
||||
if (NULL != prefix) {
|
||||
char *oldenv, *newenv;
|
||||
|
||||
/* Reset PATH */
|
||||
oldenv = getenv("PATH");
|
||||
if (NULL != oldenv) {
|
||||
asprintf(&newenv, "%s/%s:%s", prefix, bin_base, oldenv);
|
||||
} else {
|
||||
asprintf(&newenv, "%s/%s", prefix, bin_base);
|
||||
}
|
||||
opal_setenv("PATH", newenv, true, &env);
|
||||
OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output,
|
||||
"%s plm:slurmd: reset PATH: %s",
|
||||
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
|
||||
newenv));
|
||||
free(newenv);
|
||||
|
||||
/* Reset LD_LIBRARY_PATH */
|
||||
oldenv = getenv("LD_LIBRARY_PATH");
|
||||
if (NULL != oldenv) {
|
||||
asprintf(&newenv, "%s/%s:%s", prefix, lib_base, oldenv);
|
||||
} else {
|
||||
asprintf(&newenv, "%s/%s", prefix, lib_base);
|
||||
}
|
||||
opal_setenv("LD_LIBRARY_PATH", newenv, true, &env);
|
||||
OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output,
|
||||
"%s plm:slurmd: reset LD_LIBRARY_PATH: %s",
|
||||
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
|
||||
newenv));
|
||||
free(newenv);
|
||||
}
|
||||
|
||||
fd = open("/dev/null", O_CREAT|O_WRONLY|O_TRUNC, 0666);
|
||||
if(fd > 0) {
|
||||
dup2(fd, 0);
|
||||
}
|
||||
|
||||
/* When not in debug mode and --debug-daemons was not passed,
|
||||
* tie stdout/stderr to dev null so we don't see messages from orted */
|
||||
if (0 >= opal_output_get_verbosity(orte_plm_globals.output) &&
|
||||
!orte_debug_daemons_flag) {
|
||||
if (fd >= 0) {
|
||||
if (fd != 1) {
|
||||
dup2(fd,1);
|
||||
}
|
||||
if (fd != 2) {
|
||||
dup2(fd,2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (fd > 2) {
|
||||
close(fd);
|
||||
}
|
||||
|
||||
/* get the srun process out of orterun's process group so that
|
||||
signals sent from the shell (like those resulting from
|
||||
cntl-c) don't get sent to srun */
|
||||
setpgid(0, 0);
|
||||
|
||||
execve(exec_argv, argv, env);
|
||||
|
||||
opal_output(0, "plm:slurmd:start_proc: exec failed");
|
||||
/* don't return - need to exit - returning would be bad -
|
||||
we're not in the calling process anymore */
|
||||
exit(1);
|
||||
} else { /* parent */
|
||||
/* just in case, make sure that the srun process is not in our
|
||||
process group any more. Stevens says always do this on both
|
||||
sides of the fork... */
|
||||
setpgid(srun_pid, srun_pid);
|
||||
|
||||
/* setup the waitpid so we can find out if srun succeeds! */
|
||||
orte_wait_cb(srun_pid, srun_wait_cb, NULL);
|
||||
free(exec_argv);
|
||||
}
|
||||
|
||||
return ORTE_SUCCESS;
|
||||
}
|
||||
@ -1 +0,0 @@
|
||||
rhc
|
||||
@ -1,45 +0,0 @@
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
sources = \
|
||||
plm_slurmd.h \
|
||||
plm_slurmd_component.c \
|
||||
plm_slurmd_module.c
|
||||
|
||||
dist_pkgdata_DATA = help-plm-slurmd.txt
|
||||
|
||||
# Make the output library in this directory, and name it either
|
||||
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
|
||||
# (for static builds).
|
||||
|
||||
if OMPI_BUILD_plm_slurmd_DSO
|
||||
component_noinst =
|
||||
component_install = mca_plm_slurmd.la
|
||||
else
|
||||
component_noinst = libmca_plm_slurmd.la
|
||||
component_install =
|
||||
endif
|
||||
|
||||
mcacomponentdir = $(pkglibdir)
|
||||
mcacomponent_LTLIBRARIES = $(component_install)
|
||||
mca_plm_slurmd_la_SOURCES = $(sources)
|
||||
mca_plm_slurmd_la_LDFLAGS = -module -avoid-version
|
||||
|
||||
noinst_LTLIBRARIES = $(component_noinst)
|
||||
libmca_plm_slurmd_la_SOURCES =$(sources)
|
||||
libmca_plm_slurmd_la_LDFLAGS = -module -avoid-version
|
||||
@ -1,37 +0,0 @@
|
||||
# -*- shell-script -*-
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
# MCA_plm_slurmd_CONFIG([action-if-found], [action-if-not-found])
|
||||
# -----------------------------------------------------------
|
||||
AC_DEFUN([MCA_plm_slurmd_CONFIG],[
|
||||
OMPI_CHECK_SLURM([plm_slurmd], [plm_slurmd_good=1], [plm_slurmd_good=0])
|
||||
|
||||
# if check worked, set wrapper flags if so.
|
||||
# Evaluate succeed / fail
|
||||
AS_IF([test "$plm_slurmd_good" = "1"],
|
||||
[plm_slurmd_WRAPPER_EXTRA_LDFLAGS="$plm_slurmd_LDFLAGS"
|
||||
plm_slurmd_WRAPPER_EXTRA_LIBS="$plm_slurmd_LIBS"
|
||||
$1],
|
||||
[$2])
|
||||
|
||||
# set build flags to use in makefile
|
||||
AC_SUBST([plm_slurmd_CPPFLAGS])
|
||||
AC_SUBST([plm_slurmd_LDFLAGS])
|
||||
AC_SUBST([plm_slurmd_LIBS])
|
||||
])dnl
|
||||
@ -1,22 +0,0 @@
|
||||
# -*- shell-script -*-
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2007 Los Alamos National Security, LLC. All rights
|
||||
# reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
PARAM_CONFIG_FILES="Makefile"
|
||||
@ -1,41 +0,0 @@
|
||||
# -*- text -*-
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
[multiple-prefixes]
|
||||
The SLURM process starter for Open MPI does not support multiple
|
||||
different --prefix options to mpirun. You can specify at most one
|
||||
unique value for the --prefix option (in any of the application
|
||||
contexts); it will be applied to all the application contexts of your
|
||||
parallel job.
|
||||
|
||||
Put simply, you must have Open MPI installed in the same location on
|
||||
all of your SLURM nodes.
|
||||
|
||||
Multiple different --prefix options were specified to mpirun. This is
|
||||
a fatal error for the SLURM process starter in Open MPI.
|
||||
|
||||
The first two prefix values supplied were:
|
||||
%s
|
||||
and %s
|
||||
#
|
||||
[no-hosts-in-list]
|
||||
The SLURM process starter for Open MPI didn't find any hosts in
|
||||
the map for this application. This can be caused by a lack of
|
||||
an allocation, or by an error in the Open MPI code. Please check
|
||||
to ensure you have a SLURM allocation. If you do, then please pass
|
||||
the error to the Open MPI user's mailing list for assistance.
|
||||
@ -1,46 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#ifndef ORTE_PLM_SLURMD_EXPORT_H
|
||||
#define ORTE_PLM_SLURMD_EXPORT_H
|
||||
|
||||
#include "orte_config.h"
|
||||
|
||||
#include "opal/mca/mca.h"
|
||||
#include "orte/mca/plm/plm.h"
|
||||
|
||||
BEGIN_C_DECLS
|
||||
|
||||
struct orte_plm_slurmd_component_t {
|
||||
orte_plm_base_component_t super;
|
||||
int priority;
|
||||
char *orted;
|
||||
char *custom_args;
|
||||
};
|
||||
typedef struct orte_plm_slurmd_component_t orte_plm_slurmd_component_t;
|
||||
|
||||
/*
|
||||
* Globally exported variable
|
||||
*/
|
||||
|
||||
ORTE_MODULE_DECLSPEC extern orte_plm_slurmd_component_t mca_plm_slurmd_component;
|
||||
ORTE_DECLSPEC extern orte_plm_base_module_t orte_plm_slurmd_module;
|
||||
|
||||
END_C_DECLS
|
||||
|
||||
#endif /* ORTE_PLM_SLURMD_EXPORT_H */
|
||||
@ -1,154 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*
|
||||
* These symbols are in a file by themselves to provide nice linker
|
||||
* semantics. Since linkers generally pull in symbols by object
|
||||
* files, keeping these symbols as the only symbols in this file
|
||||
* prevents utility programs such as "ompi_info" from having to import
|
||||
* entire components just to query their version and parameters.
|
||||
*/
|
||||
|
||||
#include "orte_config.h"
|
||||
#include "orte/constants.h"
|
||||
|
||||
#include "opal/util/output.h"
|
||||
#include "opal/mca/base/mca_base_param.h"
|
||||
|
||||
#include "orte/util/name_fns.h"
|
||||
#include "orte/runtime/orte_globals.h"
|
||||
|
||||
#include "orte/mca/plm/plm.h"
|
||||
#include "orte/mca/plm/base/plm_private.h"
|
||||
#include "plm_slurmd.h"
|
||||
|
||||
|
||||
/*
|
||||
* Public string showing the plm ompi_slurmd component version number
|
||||
*/
|
||||
const char *mca_plm_slurmd_component_version_string =
|
||||
"Open MPI slurmd plm MCA component version " ORTE_VERSION;
|
||||
|
||||
|
||||
/*
|
||||
* Local functions
|
||||
*/
|
||||
static int plm_slurmd_open(void);
|
||||
static int plm_slurmd_close(void);
|
||||
static orte_plm_base_module_t *plm_slurmd_init(int *priority);
|
||||
|
||||
|
||||
/*
|
||||
* Instantiate the public struct with all of our public information
|
||||
* and pointers to our public functions in it
|
||||
*/
|
||||
|
||||
orte_plm_slurmd_component_t mca_plm_slurmd_component = {
|
||||
|
||||
{
|
||||
/* First, the mca_component_t struct containing meta
|
||||
information about the component itself */
|
||||
|
||||
{
|
||||
/* Indicate that we are a plm v1.0.0 component (which also
|
||||
implies a specific MCA version) */
|
||||
|
||||
ORTE_PLM_BASE_VERSION_1_0_0,
|
||||
|
||||
/* Component name and version */
|
||||
|
||||
"slurmd",
|
||||
ORTE_MAJOR_VERSION,
|
||||
ORTE_MINOR_VERSION,
|
||||
ORTE_RELEASE_VERSION,
|
||||
|
||||
/* Component open and close functions */
|
||||
|
||||
plm_slurmd_open,
|
||||
plm_slurmd_close
|
||||
},
|
||||
|
||||
/* Next the MCA v1.0.0 component meta data */
|
||||
|
||||
{
|
||||
/* The component is checkpoint ready */
|
||||
MCA_BASE_METADATA_PARAM_CHECKPOINT
|
||||
},
|
||||
|
||||
/* Initialization / querying functions */
|
||||
|
||||
plm_slurmd_init
|
||||
}
|
||||
|
||||
/* Other orte_plm_slurmd_component_t items -- left uninitialized
|
||||
here; will be initialized in plm_slurmd_open() */
|
||||
};
|
||||
|
||||
|
||||
static int plm_slurmd_open(void)
|
||||
{
|
||||
mca_base_component_t *comp = &mca_plm_slurmd_component.super.plm_version;
|
||||
|
||||
mca_base_param_reg_int(comp, "priority", "Default selection priority",
|
||||
false, false, 50, /* keep this below indirect slurm */
|
||||
&mca_plm_slurmd_component.priority);
|
||||
|
||||
mca_base_param_reg_string(comp, "orted",
|
||||
"Command to use to start proxy orted",
|
||||
false, false, "orted",
|
||||
&mca_plm_slurmd_component.orted);
|
||||
|
||||
mca_base_param_reg_string(comp, "args",
|
||||
"Custom arguments to srun",
|
||||
false, false, NULL,
|
||||
&mca_plm_slurmd_component.custom_args);
|
||||
|
||||
return ORTE_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
static orte_plm_base_module_t *plm_slurmd_init(int *priority)
|
||||
{
|
||||
/* Are we running under a SLURM job? */
|
||||
|
||||
if (NULL != getenv("SLURM_JOBID")) {
|
||||
*priority = mca_plm_slurmd_component.priority;
|
||||
|
||||
OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output,
|
||||
"%s plm:slurmd: available for selection",
|
||||
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
|
||||
|
||||
return &orte_plm_slurmd_module;
|
||||
}
|
||||
|
||||
/* Sadly, no */
|
||||
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
||||
static int plm_slurmd_close(void)
|
||||
{
|
||||
if (NULL != mca_plm_slurmd_component.orted) {
|
||||
free(mca_plm_slurmd_component.orted);
|
||||
}
|
||||
|
||||
if (NULL != mca_plm_slurmd_component.custom_args) {
|
||||
free(mca_plm_slurmd_component.custom_args);
|
||||
}
|
||||
|
||||
return ORTE_SUCCESS;
|
||||
}
|
||||
@ -1,627 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2006 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2006-2007 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2007 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*
|
||||
* These symbols are in a file by themselves to provide nice linker
|
||||
* semantics. Since linkers generally pull in symbols by object
|
||||
* files, keeping these symbols as the only symbols in this file
|
||||
* prevents utility programs such as "ompi_info" from having to import
|
||||
* entire components just to query their version and parameters.
|
||||
*/
|
||||
|
||||
#include "orte_config.h"
|
||||
#include "orte/constants.h"
|
||||
#include "orte/types.h"
|
||||
|
||||
#include <sys/types.h>
|
||||
#ifdef HAVE_UNISTD_H
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
#include <signal.h>
|
||||
#ifdef HAVE_STDLIB_H
|
||||
#include <stdlib.h>
|
||||
#endif
|
||||
#ifdef HAVE_SYS_TYPES_H
|
||||
#include <sys/types.h>
|
||||
#endif
|
||||
#ifdef HAVE_SYS_TIME_H
|
||||
#include <sys/time.h>
|
||||
#endif
|
||||
#ifdef HAVE_SYS_STAT_H
|
||||
#include <sys/stat.h>
|
||||
#endif
|
||||
#ifdef HAVE_FCNTL_H
|
||||
#include <fcntl.h>
|
||||
#endif
|
||||
|
||||
#include "opal/mca/installdirs/installdirs.h"
|
||||
#include "opal/util/argv.h"
|
||||
#include "opal/util/output.h"
|
||||
#include "opal/util/opal_environ.h"
|
||||
#include "opal/util/path.h"
|
||||
#include "opal/util/show_help.h"
|
||||
#include "opal/util/basename.h"
|
||||
#include "opal/mca/base/mca_base_param.h"
|
||||
#include "opal/runtime/opal_progress.h"
|
||||
|
||||
#include "orte/util/name_fns.h"
|
||||
#include "orte/runtime/orte_globals.h"
|
||||
#include "orte/runtime/runtime.h"
|
||||
#include "orte/runtime/orte_wakeup.h"
|
||||
#include "orte/runtime/orte_wait.h"
|
||||
#include "orte/mca/rml/rml.h"
|
||||
#include "orte/mca/errmgr/errmgr.h"
|
||||
#include "orte/mca/rmaps/rmaps.h"
|
||||
|
||||
#include "orte/mca/plm/plm.h"
|
||||
#include "orte/mca/plm/base/plm_private.h"
|
||||
#include "plm_slurmd.h"
|
||||
|
||||
|
||||
/*
|
||||
* Local functions
|
||||
*/
|
||||
static int plm_slurmd_init(void);
|
||||
static int plm_slurmd_launch_job(orte_job_t *jdata);
|
||||
static int plm_slurmd_terminate_job(orte_jobid_t jobid);
|
||||
static int plm_slurmd_terminate_orteds(void);
|
||||
static int plm_slurmd_signal_job(orte_jobid_t jobid, int32_t signal);
|
||||
static int plm_slurmd_finalize(void);
|
||||
|
||||
static int plm_slurmd_start_proc(int argc, char **argv, char **env,
|
||||
char *prefix);
|
||||
|
||||
|
||||
/*
|
||||
* Global variable
|
||||
*/
|
||||
orte_plm_base_module_1_0_0_t orte_plm_slurmd_module = {
|
||||
plm_slurmd_init,
|
||||
orte_plm_base_set_hnp_name,
|
||||
plm_slurmd_launch_job,
|
||||
plm_slurmd_terminate_job,
|
||||
plm_slurmd_terminate_orteds,
|
||||
plm_slurmd_signal_job,
|
||||
plm_slurmd_finalize
|
||||
};
|
||||
|
||||
/*
|
||||
* Local variables
|
||||
*/
|
||||
static pid_t srun_pid = 0;
|
||||
static orte_jobid_t active_job = ORTE_JOBID_INVALID;
|
||||
static bool failed_launch;
|
||||
|
||||
|
||||
/**
|
||||
* Init the module
|
||||
*/
|
||||
static int plm_slurmd_init(void)
|
||||
{
|
||||
int rc;
|
||||
|
||||
if (ORTE_SUCCESS != (rc = orte_plm_base_comm_start())) {
|
||||
ORTE_ERROR_LOG(rc);
|
||||
}
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* When working in this function, ALWAYS jump to "cleanup" if
|
||||
* you encounter an error so that orterun will be woken up and
|
||||
* the job can cleanly terminate
|
||||
*/
|
||||
static int plm_slurmd_launch_job(orte_job_t *jdata)
|
||||
{
|
||||
orte_app_context_t **apps;
|
||||
orte_node_t **nodes;
|
||||
orte_std_cntr_t n;
|
||||
orte_job_map_t *map;
|
||||
char *param;
|
||||
char **argv = NULL;
|
||||
int argc;
|
||||
int rc;
|
||||
char *tmp;
|
||||
char** env = NULL;
|
||||
char* var;
|
||||
char *nodelist_flat;
|
||||
char **nodelist_argv;
|
||||
char *rml_uri;
|
||||
int nodelist_argc;
|
||||
char **custom_strings;
|
||||
int num_args, i;
|
||||
char *cur_prefix;
|
||||
struct timeval launchstart, launchstop;
|
||||
|
||||
if (orte_timing) {
|
||||
if (0 != gettimeofday(&launchstart, NULL)) {
|
||||
opal_output(0, "plm_slurmd: could not obtain job start time");
|
||||
launchstart.tv_sec = 0;
|
||||
launchstart.tv_usec = 0;
|
||||
}
|
||||
}
|
||||
|
||||
/* indicate the state of the launch */
|
||||
failed_launch = true;
|
||||
|
||||
/* create a jobid for this job */
|
||||
if (ORTE_SUCCESS != (rc = orte_plm_base_create_jobid(&jdata->jobid))) {
|
||||
ORTE_ERROR_LOG(rc);
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output,
|
||||
"%s plm:slurmd: launching job %s",
|
||||
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
|
||||
ORTE_JOBID_PRINT(jdata->jobid)));
|
||||
|
||||
/* setup the job */
|
||||
if (ORTE_SUCCESS != (rc = orte_plm_base_setup_job(jdata))) {
|
||||
ORTE_ERROR_LOG(rc);
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
/* set the active jobid */
|
||||
active_job = jdata->jobid;
|
||||
|
||||
/* Get the map for this job */
|
||||
if (NULL == (map = orte_rmaps.get_job_map(active_job))) {
|
||||
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
|
||||
rc = ORTE_ERR_NOT_FOUND;
|
||||
goto cleanup;
|
||||
}
|
||||
apps = (orte_app_context_t**)jdata->apps->addr;
|
||||
nodes = (orte_node_t**)map->nodes->addr;
|
||||
|
||||
/* we are going to ignore the daemons and just launch
|
||||
* the apps directly. This means we cannot use the
|
||||
* base function to launch_apps as that function assumes
|
||||
* we are doing it via our daemons
|
||||
*/
|
||||
|
||||
/* are there any procs to actually launch? */
|
||||
if (0 == jdata->num_procs) {
|
||||
OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output,
|
||||
"%s plm:slurmd: no procs to launch!",
|
||||
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
/*
|
||||
* start building argv array
|
||||
*/
|
||||
argv = NULL;
|
||||
argc = 0;
|
||||
|
||||
/*
|
||||
* SLURM srun OPTIONS
|
||||
*/
|
||||
|
||||
/* add the srun command */
|
||||
opal_argv_append(&argc, &argv, "srun");
|
||||
|
||||
/* Append user defined arguments to srun */
|
||||
if ( NULL != mca_plm_slurmd_component.custom_args ) {
|
||||
custom_strings = opal_argv_split(mca_plm_slurmd_component.custom_args, ' ');
|
||||
num_args = opal_argv_count(custom_strings);
|
||||
for (i = 0; i < num_args; ++i) {
|
||||
opal_argv_append(&argc, &argv, custom_strings[i]);
|
||||
}
|
||||
opal_argv_free(custom_strings);
|
||||
}
|
||||
|
||||
asprintf(&tmp, "--nodes=%lu", (unsigned long) map->num_new_daemons);
|
||||
opal_argv_append(&argc, &argv, tmp);
|
||||
free(tmp);
|
||||
|
||||
asprintf(&tmp, "--ntasks=%lu", (unsigned long) jdata->num_procs);
|
||||
opal_argv_append(&argc, &argv, tmp);
|
||||
free(tmp);
|
||||
|
||||
/* create nodelist */
|
||||
nodelist_argv = NULL;
|
||||
nodelist_argc = 0;
|
||||
|
||||
for (n=0; n < map->num_nodes; n++ ) {
|
||||
/* add it to the list of nodes upon which
|
||||
* we need to launch a proc
|
||||
*/
|
||||
opal_argv_append(&nodelist_argc, &nodelist_argv, nodes[n]->name);
|
||||
}
|
||||
if (0 == opal_argv_count(nodelist_argv)) {
|
||||
opal_show_help("help-plm-slurmd.txt", "no-hosts-in-list", true);
|
||||
rc = ORTE_ERR_FAILED_TO_START;
|
||||
goto cleanup;
|
||||
}
|
||||
nodelist_flat = opal_argv_join(nodelist_argv, ',');
|
||||
opal_argv_free(nodelist_argv);
|
||||
asprintf(&tmp, "--nodelist=%s", nodelist_flat);
|
||||
opal_argv_append(&argc, &argv, tmp);
|
||||
free(tmp);
|
||||
|
||||
OPAL_OUTPUT_VERBOSE((2, orte_plm_globals.output,
|
||||
"%s plm:slurmd: launching on nodes %s",
|
||||
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), nodelist_flat));
|
||||
|
||||
/*
|
||||
* APP OPTIONS
|
||||
*/
|
||||
|
||||
/* add the executable (as specified by user) */
|
||||
opal_argv_append(&argc, &argv, apps[0]->app);
|
||||
|
||||
/* add the user's cmd args */
|
||||
for (i=1; NULL != apps[0]->argv[i]; i++) {
|
||||
opal_argv_append(&argc, &argv, apps[0]->argv[i]);
|
||||
}
|
||||
|
||||
/* setup the environment - since there will be no daemon, we have
|
||||
* tell the the apps everything they need to know via the environment,
|
||||
* not command line args. Hence, we cannot use the plm base function
|
||||
* to setup the info
|
||||
*/
|
||||
if (NULL != apps[0]->env) {
|
||||
env = opal_environ_merge(environ, apps[0]->env);
|
||||
} else {
|
||||
env = opal_argv_copy(environ);
|
||||
}
|
||||
|
||||
/* add the nodelist */
|
||||
var = mca_base_param_environ_variable("orte", "slurm", "nodelist");
|
||||
opal_setenv(var, nodelist_flat, true, &env);
|
||||
free(nodelist_flat);
|
||||
free(var);
|
||||
|
||||
/* tell the proc what SDS component to use */
|
||||
var = mca_base_param_environ_variable("sds", NULL, NULL);
|
||||
opal_setenv(var, "slurmd", true, &env);
|
||||
free(var);
|
||||
|
||||
/* pass the jobid */
|
||||
var = mca_base_param_environ_variable("orte", "sds", "jobid");
|
||||
orte_util_convert_jobid_to_string(¶m, jdata->jobid);
|
||||
opal_setenv(var, param, true, &env);
|
||||
free(var);
|
||||
free(param);
|
||||
|
||||
/* pass the starting vpid */
|
||||
var = mca_base_param_environ_variable("orte", "sds", "vpid");
|
||||
opal_setenv(var, "0", true, &env);
|
||||
free(var);
|
||||
|
||||
/* pass the total number of procs that will be in the system */
|
||||
var = mca_base_param_environ_variable("orte", "sds", "num_procs");
|
||||
asprintf(¶m, "%lu", (unsigned long)(jdata->num_procs));
|
||||
opal_setenv(var, param, true, &env);
|
||||
free(var);
|
||||
free(param);
|
||||
|
||||
/* pass the uri of the hnp */
|
||||
var = mca_base_param_environ_variable("orte", "hnp", "uri");
|
||||
rml_uri = orte_rml.get_contact_info();
|
||||
opal_setenv(var, rml_uri, true, &env);
|
||||
free(var);
|
||||
/* make the proc think the local daemon is the hnp */
|
||||
var = mca_base_param_environ_variable("orte", "local_daemon", "uri");
|
||||
opal_setenv(var, rml_uri, true, &env);
|
||||
free(var);
|
||||
free(rml_uri);
|
||||
|
||||
/* order the proc to use the unity routed component */
|
||||
var = mca_base_param_environ_variable("routed", NULL, NULL);
|
||||
opal_setenv(var, "unity", true, &env);
|
||||
free(var);
|
||||
|
||||
if (0 < opal_output_get_verbosity(orte_plm_globals.output)) {
|
||||
param = opal_argv_join(argv, ' ');
|
||||
OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output,
|
||||
"%s plm:slurmd: final top-level argv:\n\t%s",
|
||||
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
|
||||
(NULL == param) ? "NULL" : param));
|
||||
if (NULL != param) free(param);
|
||||
}
|
||||
|
||||
/* Copy the prefix-directory specified in the
|
||||
corresponding app_context. If there are multiple,
|
||||
different prefix's in the app context, complain (i.e., only
|
||||
allow one --prefix option for the entire slurmd run -- we
|
||||
don't support different --prefix'es for different nodes in
|
||||
the SLURM plm) */
|
||||
cur_prefix = NULL;
|
||||
for (n=0; n < jdata->num_apps; n++) {
|
||||
char * app_prefix_dir = apps[n]->prefix_dir;
|
||||
/* Check for already set cur_prefix_dir -- if different,
|
||||
complain */
|
||||
if (NULL != app_prefix_dir) {
|
||||
if (NULL != cur_prefix &&
|
||||
0 != strcmp (cur_prefix, app_prefix_dir)) {
|
||||
opal_show_help("help-plm-slurmd.txt", "multiple-prefixes",
|
||||
true, cur_prefix, app_prefix_dir);
|
||||
return ORTE_ERR_FATAL;
|
||||
}
|
||||
|
||||
/* If not yet set, copy it; iff set, then it's the
|
||||
same anyway */
|
||||
if (NULL == cur_prefix) {
|
||||
cur_prefix = strdup(app_prefix_dir);
|
||||
OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output,
|
||||
"%s plm:slurmd: Set prefix:%s",
|
||||
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
|
||||
cur_prefix));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* exec the proc(s) */
|
||||
if (ORTE_SUCCESS != (rc = plm_slurmd_start_proc(argc, argv, env, cur_prefix))) {
|
||||
ORTE_ERROR_LOG(rc);
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
/* do NOT wait for srun to complete. Srun only completes when the processes
|
||||
* it starts complete. Instead, we'll catch
|
||||
* any srun failures and deal with them elsewhere
|
||||
*/
|
||||
|
||||
/* declare the launch a success */
|
||||
failed_launch = false;
|
||||
|
||||
if (orte_timing) {
|
||||
if (0 != gettimeofday(&launchstop, NULL)) {
|
||||
opal_output(0, "plm_slurmd: could not obtain stop time");
|
||||
} else {
|
||||
opal_output(0, "plm_slurmd: total job launch time is %ld usec",
|
||||
(launchstop.tv_sec - launchstart.tv_sec)*1000000 +
|
||||
(launchstop.tv_usec - launchstart.tv_usec));
|
||||
}
|
||||
}
|
||||
cleanup:
|
||||
if (NULL != argv) {
|
||||
opal_argv_free(argv);
|
||||
}
|
||||
if (NULL != env) {
|
||||
opal_argv_free(env);
|
||||
}
|
||||
|
||||
/* check for failed launch - if so, force terminate */
|
||||
if (failed_launch) {
|
||||
jdata->state = ORTE_JOB_STATE_FAILED_TO_START;
|
||||
orte_wakeup();
|
||||
}
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
static int plm_slurmd_terminate_job(orte_jobid_t jobid)
|
||||
{
|
||||
if (0 != srun_pid) {
|
||||
kill(srun_pid, -9);
|
||||
}
|
||||
return ORTE_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Terminate the orteds for a given job
|
||||
*/
|
||||
static int plm_slurmd_terminate_orteds(void)
|
||||
{
|
||||
orte_job_t *daemons;
|
||||
int data=1;
|
||||
|
||||
/* get the daemon job object */
|
||||
if (NULL != (daemons = orte_get_job_data_object(ORTE_PROC_MY_NAME->jobid))) {
|
||||
/* record all daemons as terminated */
|
||||
daemons->num_terminated = daemons->num_procs;
|
||||
daemons->state = ORTE_JOB_STATE_TERMINATED;
|
||||
}
|
||||
|
||||
/* trip the orted exit trigger */
|
||||
write(orteds_exit, &data, sizeof(int));
|
||||
opal_progress();
|
||||
|
||||
return ORTE_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Signal all the processes in the child srun by sending the signal directly to it
|
||||
*/
|
||||
static int plm_slurmd_signal_job(orte_jobid_t jobid, int32_t signal)
|
||||
{
|
||||
if (0 != srun_pid) {
|
||||
kill(srun_pid, (int)signal);
|
||||
}
|
||||
return ORTE_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
static int plm_slurmd_finalize(void)
|
||||
{
|
||||
int rc;
|
||||
|
||||
/* cleanup any pending recvs */
|
||||
if (ORTE_SUCCESS != (rc = orte_plm_base_comm_stop())) {
|
||||
ORTE_ERROR_LOG(rc);
|
||||
}
|
||||
|
||||
return ORTE_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
static void srun_wait_cb(pid_t pid, int status, void* cbdata){
|
||||
/* According to the SLURM folks, srun always returns the highest exit
|
||||
code of our remote processes. Thus, a non-zero exit status doesn't
|
||||
necessarily mean that srun failed - it could be that a proc returned
|
||||
a non-zero exit status. Of course, that means the job failed(!), so
|
||||
the end result is the same.
|
||||
|
||||
As a result, we really can't do much with the exit status itself - it
|
||||
could be something in errno (if srun itself failed), or it could be
|
||||
something returned by a proc, or it could be something returned by
|
||||
the OS (e.g., couldn't find the executable). Somebody is welcome
|
||||
to sort out all the options and pretty-print a better error message. For
|
||||
now, though, the only thing that really matters is that
|
||||
srun terminated. Report the error (if there is one) and make sure that orterun
|
||||
wakes up!
|
||||
*/
|
||||
orte_job_t *jdata;
|
||||
|
||||
if (NULL == (jdata = orte_get_job_data_object(active_job))) {
|
||||
/* bad jobid */
|
||||
ORTE_ERROR_LOG(ORTE_ERR_BAD_PARAM);
|
||||
orte_wakeup();
|
||||
return;
|
||||
}
|
||||
|
||||
if (0 != status) {
|
||||
if (failed_launch) {
|
||||
/* we have a problem during launch */
|
||||
opal_output(0, "ERROR: srun failed to start the specified procs.");
|
||||
opal_output(0, "ERROR: This could be due to an inability to find the executable");
|
||||
opal_output(0, "ERROR: on one or more remote nodes, lack of authority to execute");
|
||||
opal_output(0, "ERROR: on one or more specified nodes, or other factors.");
|
||||
|
||||
/* report that the launch has failed so we break out of the daemon
|
||||
* callback receive and exit
|
||||
*/
|
||||
jdata->state = ORTE_JOB_STATE_FAILED_TO_START;
|
||||
|
||||
} else {
|
||||
/* a proc must have died unexpectedly after launch - report
|
||||
* that the job has failed so we exit
|
||||
*/
|
||||
jdata->state = ORTE_JOB_STATE_ABORTED;
|
||||
}
|
||||
} else {
|
||||
jdata->state = ORTE_JOB_STATE_TERMINATED;
|
||||
}
|
||||
|
||||
orte_wakeup();
|
||||
}
|
||||
|
||||
|
||||
static int plm_slurmd_start_proc(int argc, char **argv, char **env,
|
||||
char *prefix)
|
||||
{
|
||||
int fd;
|
||||
char *exec_argv = opal_path_findv(argv[0], 0, env, NULL);
|
||||
|
||||
if (NULL == exec_argv) {
|
||||
return ORTE_ERR_NOT_FOUND;
|
||||
}
|
||||
|
||||
srun_pid = fork();
|
||||
if (-1 == srun_pid) {
|
||||
ORTE_ERROR_LOG(ORTE_ERR_SYS_LIMITS_CHILDREN);
|
||||
return ORTE_ERR_SYS_LIMITS_CHILDREN;
|
||||
}
|
||||
|
||||
if (0 == srun_pid) { /* child */
|
||||
char *bin_base = NULL, *lib_base = NULL;
|
||||
|
||||
/* Figure out the basenames for the libdir and bindir. There
|
||||
is a lengthy comment about this in plm_rsh_module.c
|
||||
explaining all the rationale for how / why we're doing
|
||||
this. */
|
||||
|
||||
lib_base = opal_basename(opal_install_dirs.libdir);
|
||||
bin_base = opal_basename(opal_install_dirs.bindir);
|
||||
|
||||
/* If we have a prefix, then modify the PATH and
|
||||
LD_LIBRARY_PATH environment variables. */
|
||||
if (NULL != prefix) {
|
||||
char *oldenv, *newenv;
|
||||
|
||||
/* Reset PATH */
|
||||
oldenv = getenv("PATH");
|
||||
if (NULL != oldenv) {
|
||||
asprintf(&newenv, "%s/%s:%s", prefix, bin_base, oldenv);
|
||||
} else {
|
||||
asprintf(&newenv, "%s/%s", prefix, bin_base);
|
||||
}
|
||||
opal_setenv("PATH", newenv, true, &env);
|
||||
OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output,
|
||||
"%s plm:slurmd: reset PATH: %s",
|
||||
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
|
||||
newenv));
|
||||
free(newenv);
|
||||
|
||||
/* Reset LD_LIBRARY_PATH */
|
||||
oldenv = getenv("LD_LIBRARY_PATH");
|
||||
if (NULL != oldenv) {
|
||||
asprintf(&newenv, "%s/%s:%s", prefix, lib_base, oldenv);
|
||||
} else {
|
||||
asprintf(&newenv, "%s/%s", prefix, lib_base);
|
||||
}
|
||||
opal_setenv("LD_LIBRARY_PATH", newenv, true, &env);
|
||||
OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output,
|
||||
"%s plm:slurmd: reset LD_LIBRARY_PATH: %s",
|
||||
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
|
||||
newenv));
|
||||
free(newenv);
|
||||
}
|
||||
|
||||
fd = open("/dev/null", O_CREAT|O_WRONLY|O_TRUNC, 0666);
|
||||
if(fd > 0) {
|
||||
dup2(fd, 0);
|
||||
}
|
||||
|
||||
/* When not in debug mode and --debug-daemons was not passed,
|
||||
* tie stdout/stderr to dev null so we don't see messages from orted */
|
||||
if (0 >= opal_output_get_verbosity(orte_plm_globals.output) &&
|
||||
!orte_debug_daemons_flag) {
|
||||
if (fd >= 0) {
|
||||
if (fd != 1) {
|
||||
dup2(fd,1);
|
||||
}
|
||||
if (fd != 2) {
|
||||
dup2(fd,2);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (fd > 2) {
|
||||
close(fd);
|
||||
}
|
||||
|
||||
/* get the srun process out of orterun's process group so that
|
||||
signals sent from the shell (like those resulting from
|
||||
cntl-c) don't get sent to srun */
|
||||
setpgid(0, 0);
|
||||
|
||||
execve(exec_argv, argv, env);
|
||||
|
||||
opal_output(0, "plm:slurmd:start_proc: exec failed");
|
||||
/* don't return - need to exit - returning would be bad -
|
||||
we're not in the calling process anymore */
|
||||
exit(1);
|
||||
} else { /* parent */
|
||||
/* just in case, make sure that the srun process is not in our
|
||||
process group any more. Stevens says always do this on both
|
||||
sides of the fork... */
|
||||
setpgid(srun_pid, srun_pid);
|
||||
|
||||
/* setup the waitpid so we can find out if srun succeeds! */
|
||||
orte_wait_cb(srun_pid, srun_wait_cb, NULL);
|
||||
free(exec_argv);
|
||||
}
|
||||
|
||||
return ORTE_SUCCESS;
|
||||
}
|
||||
@ -1,52 +0,0 @@
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
AM_CPPFLAGS = $(plm_tmd_CPPFLAGS)
|
||||
|
||||
dist_pkgdata_DATA = help-plm-tmd.txt
|
||||
|
||||
sources = \
|
||||
plm_tmd.h \
|
||||
plm_tmd_component.c \
|
||||
plm_tmd_module.c
|
||||
|
||||
# Make the output library in this directory, and name it either
|
||||
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
|
||||
# (for static builds).
|
||||
|
||||
if OMPI_BUILD_plm_tmd_DSO
|
||||
lib =
|
||||
lib_sources =
|
||||
component = mca_plm_tmd.la
|
||||
component_sources = $(sources)
|
||||
else
|
||||
lib = libmca_plm_tmd.la
|
||||
lib_sources = $(sources)
|
||||
component =
|
||||
component_sources =
|
||||
endif
|
||||
|
||||
mcacomponentdir = $(pkglibdir)
|
||||
mcacomponent_LTLIBRARIES = $(component)
|
||||
mca_plm_tmd_la_SOURCES = $(component_sources)
|
||||
mca_plm_tmd_la_LDFLAGS = -module -avoid-version $(plm_tmd_LDFLAGS)
|
||||
|
||||
noinst_LTLIBRARIES = $(lib)
|
||||
libmca_plm_tmd_la_SOURCES = $(lib_sources)
|
||||
libmca_plm_tmd_la_LDFLAGS = -module -avoid-version $(plm_tmd_LDFLAGS)
|
||||
libmca_plm_tmd_la_LIBADD = $(plm_tmd_LIBS)
|
||||
@ -1,37 +0,0 @@
|
||||
# -*- shell-script -*-
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
# MCA_plm_tmd_CONFIG([action-if-found], [action-if-not-found])
|
||||
# -----------------------------------------------------------
|
||||
AC_DEFUN([MCA_plm_tmd_CONFIG],[
|
||||
OMPI_CHECK_TM([plm_tmd], [plm_tmd_good=1], [plm_tmd_good=0])
|
||||
|
||||
# if check worked, set wrapper flags if so.
|
||||
# Evaluate succeed / fail
|
||||
AS_IF([test "$plm_tmd_good" = "1"],
|
||||
[plm_tmd_WRAPPER_EXTRA_LDFLAGS="$plm_tmd_LDFLAGS"
|
||||
plm_tmd_WRAPPER_EXTRA_LIBS="$plm_tmd_LIBS"
|
||||
$1],
|
||||
[$2])
|
||||
|
||||
# set build flags to use in makefile
|
||||
AC_SUBST([plm_tmd_CPPFLAGS])
|
||||
AC_SUBST([plm_tmd_LDFLAGS])
|
||||
AC_SUBST([plm_tmd_LIBS])
|
||||
])dnl
|
||||
@ -1,22 +0,0 @@
|
||||
# -*- shell-script -*-
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2007 Los Alamos National Security, LLC. All rights
|
||||
# reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
PARAM_CONFIG_FILES="Makefile"
|
||||
@ -1,52 +0,0 @@
|
||||
# -*- text -*-
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
[tm-bad-launchid]
|
||||
The TM (PBS / Torque) process starter cannot spawn the specified
|
||||
application on a remote node due to an invalid launch_id.
|
||||
|
||||
This is most likely due to use of the "--hostfile" option to the
|
||||
command line with one or more hosts in that file not having
|
||||
been allocated to this job.
|
||||
|
||||
Removing "--hostfile" from the command line will likely allow the
|
||||
application to be launched.
|
||||
#
|
||||
[multiple-prefixes]
|
||||
Multiple different --prefix options were specified to mpirun for the
|
||||
same node. This is a fatal error for the TM (PBS / Torque) process
|
||||
starter in Open MPI.
|
||||
|
||||
The first two prefix values supplied for node %s were:
|
||||
%s
|
||||
and %s
|
||||
#
|
||||
[tm-spawn-failed]
|
||||
The TM (PBS / Torque) process starter failed to spawn a daemon (orted)
|
||||
on a remote node.
|
||||
|
||||
Command line: %s
|
||||
Node name: %s
|
||||
Launch id: %d
|
||||
|
||||
If you do not understand this error mesage, please try the following:
|
||||
|
||||
1. Ensure that the executable "orted" is in your PATH
|
||||
2. Use the --prefix option to indicate where we can
|
||||
find that executable
|
||||
3. Talk to your local system administrator
|
||||
@ -1,36 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2006 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2006 Cisco Systems, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#ifndef ORTE_PLM_TMD_EXPORT_H
|
||||
#define ORTE_PLM_TMD_EXPORT_H
|
||||
|
||||
#include "orte_config.h"
|
||||
|
||||
#include "opal/mca/mca.h"
|
||||
#include "orte/mca/plm/plm.h"
|
||||
|
||||
BEGIN_C_DECLS
|
||||
|
||||
/* Globally exported variables */
|
||||
ORTE_DECLSPEC extern orte_plm_base_component_t mca_plm_tmd_component;
|
||||
extern orte_plm_base_module_t orte_plm_tmd_module;
|
||||
|
||||
END_C_DECLS
|
||||
|
||||
#endif /* ORTE_PLM_TMD_EXPORT_H */
|
||||
@ -1,117 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2006 Cisco Systems, Inc. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*
|
||||
* These symbols are in a file by themselves to provide nice linker
|
||||
* semantics. Since linkers generally pull in symbols by object
|
||||
* files, keeping these symbols as the only symbols in this file
|
||||
* prevents utility programs such as "ompi_info" from having to import
|
||||
* entire components just to query their version and parameters.
|
||||
*/
|
||||
|
||||
#include "orte_config.h"
|
||||
#include "orte/constants.h"
|
||||
|
||||
#include "opal/mca/base/mca_base_param.h"
|
||||
#include "opal/util/output.h"
|
||||
#include "opal/util/argv.h"
|
||||
|
||||
#include "orte/util/proc_info.h"
|
||||
#include "orte/mca/errmgr/errmgr.h"
|
||||
|
||||
#include "orte/mca/plm/plm.h"
|
||||
#include "orte/mca/plm/base/base.h"
|
||||
#include "orte/mca/plm/base/plm_private.h"
|
||||
#include "plm_tmd.h"
|
||||
|
||||
|
||||
/*
|
||||
* Public string showing the plm ompi_tm component version number
|
||||
*/
|
||||
const char *mca_plm_tmd_component_version_string =
|
||||
"Open MPI tm plm MCA component version " ORTE_VERSION;
|
||||
|
||||
|
||||
|
||||
/*
|
||||
* Local function
|
||||
*/
|
||||
static int plm_tmd_open(void);
|
||||
static int plm_tmd_close(void);
|
||||
static orte_plm_base_module_t *plm_tmd_init(int *priority);
|
||||
|
||||
|
||||
/*
|
||||
* Instantiate the public struct with all of our public information
|
||||
* and pointers to our public functions in it
|
||||
*/
|
||||
|
||||
orte_plm_base_component_t mca_plm_tmd_component = {
|
||||
{
|
||||
/* Indicate that we are a plm v1.0.0 component (which also
|
||||
implies a specific MCA version) */
|
||||
ORTE_PLM_BASE_VERSION_1_0_0,
|
||||
|
||||
/* Component name and version */
|
||||
"tmd",
|
||||
ORTE_MAJOR_VERSION,
|
||||
ORTE_MINOR_VERSION,
|
||||
ORTE_RELEASE_VERSION,
|
||||
|
||||
/* Component open and close functions */
|
||||
plm_tmd_open,
|
||||
plm_tmd_close,
|
||||
},
|
||||
|
||||
/* Next the MCA v1.0.0 component meta data */
|
||||
{
|
||||
/* The component is checkpoint ready */
|
||||
MCA_BASE_METADATA_PARAM_CHECKPOINT
|
||||
},
|
||||
|
||||
/* Initialization / querying functions */
|
||||
plm_tmd_init
|
||||
};
|
||||
|
||||
|
||||
static int plm_tmd_open(void)
|
||||
{
|
||||
return ORTE_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
static int plm_tmd_close(void)
|
||||
{
|
||||
return ORTE_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
static orte_plm_base_module_t *plm_tmd_init(int *priority)
|
||||
{
|
||||
/* Are we running under a TM job? */
|
||||
|
||||
if (NULL != getenv("PBS_ENVIRONMENT") &&
|
||||
NULL != getenv("PBS_JOBID")) {
|
||||
|
||||
/* defer to tm module */
|
||||
*priority = 20;
|
||||
return &orte_plm_tmd_module;
|
||||
}
|
||||
|
||||
/* Sadly, no */
|
||||
return NULL;
|
||||
}
|
||||
@ -1,792 +0,0 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2006 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2006 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2007 Los Alamos National Security, LLC. All rights
|
||||
* reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*
|
||||
* These symbols are in a file by themselves to provide nice linker
|
||||
* semantics. Since linkers generally pull in symbols by object
|
||||
* files, keeping these symbols as the only symbols in this file
|
||||
* prevents utility programs such as "ompi_info" from having to import
|
||||
* entire components just to query their version and parameters.
|
||||
*/
|
||||
|
||||
#include "orte_config.h"
|
||||
#include "orte/constants.h"
|
||||
#include "orte/types.h"
|
||||
|
||||
#if HAVE_UNISTD_H
|
||||
#include <unistd.h>
|
||||
#endif
|
||||
#include <signal.h>
|
||||
#ifdef HAVE_SYS_TYPES_H
|
||||
#include <sys/types.h>
|
||||
#endif
|
||||
#ifdef HAVE_SYS_STAT_H
|
||||
#include <sys/stat.h>
|
||||
#endif
|
||||
#ifdef HAVE_SYS_WAIT_H
|
||||
#include <sys/wait.h>
|
||||
#endif
|
||||
#ifdef HAVE_SCHED_H
|
||||
#include <sched.h>
|
||||
#endif
|
||||
#ifdef HAVE_SYS_TIME_H
|
||||
#include <sys/time.h>
|
||||
#endif
|
||||
#include <errno.h>
|
||||
#include <tm.h>
|
||||
|
||||
#include "opal/mca/installdirs/installdirs.h"
|
||||
#include "opal/threads/condition.h"
|
||||
#include "opal/event/event.h"
|
||||
#include "opal/util/argv.h"
|
||||
#include "opal/util/output.h"
|
||||
#include "opal/util/opal_environ.h"
|
||||
#include "opal/util/show_help.h"
|
||||
#include "opal/util/path.h"
|
||||
#include "opal/util/basename.h"
|
||||
#include "opal/mca/base/mca_base_param.h"
|
||||
#include "opal/runtime/opal_progress.h"
|
||||
|
||||
#include "orte/util/name_fns.h"
|
||||
#include "orte/util/context_fns.h"
|
||||
#include "orte/runtime/orte_globals.h"
|
||||
#include "orte/runtime/orte_wait.h"
|
||||
#include "orte/runtime/orte_wakeup.h"
|
||||
#include "orte/mca/errmgr/errmgr.h"
|
||||
#include "orte/mca/rmaps/rmaps.h"
|
||||
|
||||
#include "orte/mca/plm/plm.h"
|
||||
#include "orte/mca/plm/base/plm_private.h"
|
||||
#include "plm_tmd.h"
|
||||
|
||||
/* define frequency of checking job complete */
|
||||
#define PLM_CHECK_COMPLETE_TIME 1
|
||||
|
||||
/*
|
||||
* Local functions
|
||||
*/
|
||||
static int plm_tmd_init(void);
|
||||
static int plm_tmd_launch_job(orte_job_t *jdata);
|
||||
static int plm_tmd_terminate_job(orte_jobid_t jobid);
|
||||
static int plm_tmd_terminate_orteds(void);
|
||||
static int plm_tmd_signal_job(orte_jobid_t jobid, int32_t signal);
|
||||
static int plm_tmd_finalize(void);
|
||||
|
||||
static int plm_tmd_connect(void);
|
||||
static int plm_tmd_disconnect(void);
|
||||
|
||||
/*
|
||||
* Local "global" variables
|
||||
*/
|
||||
static tm_task_id *tm_task_ids = NULL;
|
||||
static int num_tids;
|
||||
static orte_job_t *active_jdata;
|
||||
|
||||
/*
|
||||
* Global variable
|
||||
*/
|
||||
orte_plm_base_module_t orte_plm_tmd_module = {
|
||||
plm_tmd_init,
|
||||
orte_plm_base_set_hnp_name,
|
||||
plm_tmd_launch_job,
|
||||
plm_tmd_terminate_job,
|
||||
plm_tmd_terminate_orteds,
|
||||
plm_tmd_signal_job,
|
||||
plm_tmd_finalize
|
||||
};
|
||||
|
||||
/**
|
||||
* Init the module
|
||||
*/
|
||||
static int plm_tmd_init(void)
|
||||
{
|
||||
int rc;
|
||||
|
||||
if (ORTE_SUCCESS != (rc = orte_plm_base_comm_start())) {
|
||||
ORTE_ERROR_LOG(rc);
|
||||
}
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
/* check for job completion */
|
||||
static void check_job_complete(int fd, short evnt, void *arg)
|
||||
{
|
||||
tm_event_t event, *tm_events;
|
||||
int *exit_codes, i, rc, local_err;
|
||||
orte_vpid_t v;
|
||||
orte_proc_t **procs;
|
||||
|
||||
opal_output(0, "checking job completion");
|
||||
|
||||
tm_events = malloc(sizeof(tm_event_t) * num_tids);
|
||||
exit_codes = malloc(sizeof(int) * num_tids);
|
||||
|
||||
/* connect to the mom */
|
||||
rc = plm_tmd_connect();
|
||||
if (ORTE_SUCCESS != rc) {
|
||||
opal_output(0, "CANNOT CONNECT TO MOM");
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
/* cycle through the task_id's and check for complete */
|
||||
for (i=0; i < num_tids; i++) {
|
||||
if (ORTE_SUCCESS != (rc = tm_obit(tm_task_ids[i], exit_codes + i, tm_events + i))) {
|
||||
opal_output(0, "CANNOT CHECK PROC %d", i);
|
||||
}
|
||||
}
|
||||
opal_output(0, "obits requested");
|
||||
|
||||
|
||||
/* cycle through and poll the events */
|
||||
for (i=0; i < num_tids; i++) {
|
||||
rc = tm_poll(TM_NULL_EVENT, &event, 1, &local_err);
|
||||
if (TM_SUCCESS != rc) {
|
||||
errno = local_err;
|
||||
opal_output(0, "plm:tmd: failed to poll obit, return status = %d", rc);
|
||||
}
|
||||
}
|
||||
opal_output(0, "obits polled");
|
||||
|
||||
|
||||
/* store the exit codes */
|
||||
procs = (orte_proc_t**)active_jdata->procs->addr;
|
||||
for (v=0; v < active_jdata->num_procs; v++) {
|
||||
procs[v]->exit_code = exit_codes[v];
|
||||
ORTE_UPDATE_EXIT_STATUS(exit_codes[v]);
|
||||
opal_output(0, "rank %d ecode %d", (int)v, exit_codes[v]);
|
||||
if (WIFEXITED(exit_codes[v])) {
|
||||
if (procs[v]->state < ORTE_PROC_STATE_TERMINATED) {
|
||||
procs[v]->state = ORTE_PROC_STATE_TERMINATED;
|
||||
active_jdata->num_terminated++;
|
||||
}
|
||||
} else {
|
||||
procs[v]->state = ORTE_PROC_STATE_ABORTED_BY_SIG;
|
||||
active_jdata->abort = true;
|
||||
active_jdata->aborted_proc = procs[v];
|
||||
active_jdata->state = ORTE_JOB_STATE_ABORTED;
|
||||
}
|
||||
}
|
||||
|
||||
/* disconnect from mom */
|
||||
plm_tmd_disconnect();
|
||||
|
||||
cleanup:
|
||||
/* free tm_events */
|
||||
free(tm_events);
|
||||
free(exit_codes);
|
||||
|
||||
/* check for completion */
|
||||
if (active_jdata->num_terminated >= active_jdata->num_procs) {
|
||||
active_jdata->state = ORTE_JOB_STATE_TERMINATED;
|
||||
orte_wakeup();
|
||||
} else if (active_jdata->state == ORTE_JOB_STATE_ABORTED &&
|
||||
!orte_abnormal_term_ordered && !orte_abort_in_progress) {
|
||||
orte_errmgr.proc_aborted(&(active_jdata->aborted_proc->name),
|
||||
active_jdata->aborted_proc->exit_code);
|
||||
}
|
||||
|
||||
/* reset the timer */
|
||||
ORTE_TIMER_EVENT(1, check_job_complete);
|
||||
opal_output(0, "timer reset");
|
||||
|
||||
}
|
||||
|
||||
|
||||
/* When working in this function, ALWAYS jump to "cleanup" if
|
||||
* you encounter an error so that orterun will be woken up and
|
||||
* the job can cleanly terminate
|
||||
*/
|
||||
static int plm_tmd_launch_job(orte_job_t *jdata)
|
||||
{
|
||||
orte_job_map_t *map = NULL;
|
||||
orte_app_context_t **apps;
|
||||
orte_node_t **nodes;
|
||||
orte_proc_t **procs;
|
||||
char *param, *param2;
|
||||
char **env = NULL;
|
||||
int rc;
|
||||
bool connected = false;
|
||||
orte_std_cntr_t launched = 0, i, n;
|
||||
char *bin_base = NULL, *lib_base = NULL;
|
||||
int local_err;
|
||||
tm_event_t event;
|
||||
bool failed_launch = true;
|
||||
orte_vpid_t v;
|
||||
tm_event_t *tm_events = NULL;
|
||||
|
||||
/* record who we are working on */
|
||||
active_jdata = jdata;
|
||||
|
||||
/* create a jobid for this job */
|
||||
if (ORTE_SUCCESS != (rc = orte_plm_base_create_jobid(&jdata->jobid))) {
|
||||
ORTE_ERROR_LOG(rc);
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output,
|
||||
"%s plm:tm: launching job %s",
|
||||
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
|
||||
ORTE_JOBID_PRINT(jdata->jobid)));
|
||||
|
||||
/* setup the job */
|
||||
if (ORTE_SUCCESS != (rc = orte_plm_base_setup_job(jdata))) {
|
||||
ORTE_ERROR_LOG(rc);
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
/* Get the map for this job */
|
||||
if (NULL == (map = orte_rmaps.get_job_map(jdata->jobid))) {
|
||||
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
|
||||
rc = ORTE_ERR_NOT_FOUND;
|
||||
goto cleanup;
|
||||
}
|
||||
apps = (orte_app_context_t**)jdata->apps->addr;
|
||||
nodes = (orte_node_t**)map->nodes->addr;
|
||||
procs = (orte_proc_t**)jdata->procs->addr;
|
||||
|
||||
/* Allocate a bunch of TM events to use for tm_spawn()ing - we will
|
||||
* need one/process since we are not launching daemons
|
||||
*/
|
||||
tm_events = malloc(sizeof(tm_event_t) * jdata->num_procs);
|
||||
if (NULL == tm_events) {
|
||||
rc = ORTE_ERR_OUT_OF_RESOURCE;
|
||||
ORTE_ERROR_LOG(rc);
|
||||
goto cleanup;
|
||||
}
|
||||
tm_task_ids = malloc(sizeof(tm_task_id) * jdata->num_procs);
|
||||
if (NULL == tm_task_ids) {
|
||||
rc = ORTE_ERR_OUT_OF_RESOURCE;
|
||||
ORTE_ERROR_LOG(rc);
|
||||
goto cleanup;
|
||||
}
|
||||
num_tids = jdata->num_procs;
|
||||
|
||||
/* Figure out the basenames for the libdir and bindir. There is a
|
||||
lengthy comment about this in plm_rsh_module.c explaining all
|
||||
the rationale for how / why we're doing this. */
|
||||
lib_base = opal_basename(opal_install_dirs.libdir);
|
||||
bin_base = opal_basename(opal_install_dirs.bindir);
|
||||
|
||||
|
||||
/* setup the environment for each app_context - we need to insert enviro
|
||||
* values that tell the procs how to bootstrap themselves
|
||||
*/
|
||||
for (n=0; n < jdata->num_apps; n++) {
|
||||
orte_app_context_t *context = apps[n];
|
||||
char ***env = &context->env;
|
||||
|
||||
/* setup base environment: copy the current environ and merge
|
||||
* in the app context environ
|
||||
*/
|
||||
if (NULL != context->env) {
|
||||
*env = opal_environ_merge(orte_launch_environ, context->env);
|
||||
} else {
|
||||
*env = opal_argv_copy(orte_launch_environ);
|
||||
}
|
||||
|
||||
/* Try to change to the context cwd and check that the app
|
||||
* exists and is executable The function will
|
||||
* take care of outputting a pretty error message, if required
|
||||
*/
|
||||
if (ORTE_SUCCESS != (rc = orte_util_check_context_cwd(context, true))) {
|
||||
/* do not ERROR_LOG - it will be reported elsewhere */
|
||||
goto cleanup;
|
||||
}
|
||||
if (ORTE_SUCCESS != (rc = orte_util_check_context_app(context))) {
|
||||
/* do not ERROR_LOG - it will be reported elsewhere */
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
/* special case handling for --prefix: this is somewhat icky,
|
||||
* but at least some users do this. :-\ It is possible that
|
||||
* when using --prefix, the user will also "-x PATH" and/or
|
||||
* "-x LD_LIBRARY_PATH", which would therefore clobber the
|
||||
* work that was done in the prior pls to ensure that we have
|
||||
* the prefix at the beginning of the PATH and
|
||||
* LD_LIBRARY_PATH. So examine the context->env and see if we
|
||||
* find PATH or LD_LIBRARY_PATH. If found, that means the
|
||||
* prior work was clobbered, and we need to re-prefix those
|
||||
* variables.
|
||||
*/
|
||||
for (i = 0; NULL != context->env && NULL != context->env[i]; ++i) {
|
||||
char *newenv;
|
||||
|
||||
/* Reset PATH */
|
||||
if (0 == strncmp("PATH=", context->env[i], 5)) {
|
||||
asprintf(&newenv, "%s/%s:%s",
|
||||
apps[n]->prefix_dir, bin_base, context->env[i] + 5);
|
||||
OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output,
|
||||
"%s plm:tmd: resetting PATH: %s",
|
||||
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
|
||||
newenv));
|
||||
opal_setenv("PATH", newenv, true, env);
|
||||
free(newenv);
|
||||
}
|
||||
|
||||
/* Reset LD_LIBRARY_PATH */
|
||||
else if (0 == strncmp("LD_LIBRARY_PATH=", context->env[i], 16)) {
|
||||
asprintf(&newenv, "%s/%s:%s",
|
||||
apps[n]->prefix_dir, lib_base, context->env[i] + 16);
|
||||
OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output,
|
||||
"%s plm:tm: resetting LD_LIBRARY_PATH: %s",
|
||||
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
|
||||
newenv));
|
||||
opal_setenv("LD_LIBRARY_PATH", newenv, true, env);
|
||||
free(newenv);
|
||||
}
|
||||
}
|
||||
|
||||
/* there will be no local daemon */
|
||||
param = mca_base_param_environ_variable("orte","local_daemon","uri");
|
||||
opal_unsetenv(param, env);
|
||||
free(param);
|
||||
|
||||
/* pass the hnp's contact info to the local proc */
|
||||
param = mca_base_param_environ_variable("orte","hnp","uri");
|
||||
opal_setenv(param, orte_process_info.my_hnp_uri, true, env);
|
||||
free(param);
|
||||
|
||||
/* set the app_context number into the environment */
|
||||
param = mca_base_param_environ_variable("orte","app","num");
|
||||
asprintf(¶m2, "%ld", (long)context->idx);
|
||||
opal_setenv(param, param2, true, env);
|
||||
free(param);
|
||||
free(param2);
|
||||
|
||||
/* set the universe size in the environment */
|
||||
param = mca_base_param_environ_variable("orte","universe","size");
|
||||
asprintf(¶m2, "%ld", (long)jdata->total_slots_alloc);
|
||||
opal_setenv(param, param2, true, env);
|
||||
free(param);
|
||||
|
||||
/* although the total_slots_alloc is the universe size, users
|
||||
* would appreciate being given a public environmental variable
|
||||
* that also represents this value - something MPI specific - so
|
||||
* do that here.
|
||||
*
|
||||
* AND YES - THIS BREAKS THE ABSTRACTION BARRIER TO SOME EXTENT.
|
||||
* We know - just live with it
|
||||
*/
|
||||
opal_setenv("OMPI_UNIVERSE_SIZE", param2, true, env);
|
||||
free(param2);
|
||||
|
||||
/* tell the proc to use the "envd" - i.e., the environment direct
|
||||
* ESS module to set itself up
|
||||
*/
|
||||
param = mca_base_param_environ_variable("ess",NULL,"NULL");
|
||||
opal_setenv(param, "envd", true, env);
|
||||
free(param);
|
||||
|
||||
/* since we want to pass the name as separate components, make sure
|
||||
* that the "name" environmental variable is cleared!
|
||||
*/
|
||||
if(NULL == (param = mca_base_param_environ_variable("orte","ess","name"))) {
|
||||
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
|
||||
return ORTE_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
opal_unsetenv(param, env);
|
||||
free(param);
|
||||
|
||||
/* tell the proc the jobid - same for everyone */
|
||||
if (ORTE_SUCCESS != (rc = orte_util_convert_jobid_to_string(¶m2, jdata->jobid))) {
|
||||
ORTE_ERROR_LOG(rc);
|
||||
goto cleanup;
|
||||
}
|
||||
if(NULL == (param = mca_base_param_environ_variable("orte","ess","jobid"))) {
|
||||
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
|
||||
rc = ORTE_ERR_OUT_OF_RESOURCE;
|
||||
goto cleanup;
|
||||
}
|
||||
opal_setenv(param, param2, true, env);
|
||||
free(param);
|
||||
free(param2);
|
||||
|
||||
/* tell the proc the #procs in this job */
|
||||
asprintf(¶m2, "%ld", (long) jdata->num_procs);
|
||||
if(NULL == (param = mca_base_param_environ_variable("orte","ess","num_procs"))) {
|
||||
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
|
||||
return ORTE_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
opal_setenv(param, param2, true, env);
|
||||
free(param);
|
||||
|
||||
/* although the num_procs is the comm_world size, users
|
||||
* would appreciate being given a public environmental variable
|
||||
* that also represents this value - something MPI specific - so
|
||||
* do that here.
|
||||
*
|
||||
* AND YES - THIS BREAKS THE ABSTRACTION BARRIER TO SOME EXTENT.
|
||||
* We know - just live with it
|
||||
*/
|
||||
opal_setenv("OMPI_COMM_WORLD_SIZE", param2, true, env);
|
||||
free(param2);
|
||||
|
||||
}
|
||||
|
||||
rc = plm_tmd_connect();
|
||||
if (ORTE_SUCCESS != rc) {
|
||||
goto cleanup;
|
||||
}
|
||||
connected = true;
|
||||
|
||||
/* Iterate through each of the procs and launch it */
|
||||
for (v = 0; v < jdata->num_procs; v++) {
|
||||
orte_proc_t *proc = procs[v];
|
||||
orte_node_t* node = proc->node;
|
||||
orte_app_context_t *context = apps[proc->app_idx];
|
||||
char **environ_copy;
|
||||
int argc;
|
||||
|
||||
/* copy the environment */
|
||||
environ_copy = opal_argv_copy(context->env);
|
||||
|
||||
/* pass the proc's vpid */
|
||||
if (ORTE_SUCCESS != (rc = orte_util_convert_vpid_to_string(¶m2, proc->name.vpid))) {
|
||||
ORTE_ERROR_LOG(rc);
|
||||
return rc;
|
||||
}
|
||||
if(NULL == (param = mca_base_param_environ_variable("orte","ess","vpid"))) {
|
||||
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
|
||||
rc = ORTE_ERR_OUT_OF_RESOURCE;
|
||||
goto cleanup;
|
||||
}
|
||||
opal_setenv(param, param2, true, &environ_copy);
|
||||
free(param);
|
||||
/* although the vpid IS the process' rank within the job, users
|
||||
* would appreciate being given a public environmental variable
|
||||
* that also represents this value - something MPI specific - so
|
||||
* do that here.
|
||||
*
|
||||
* AND YES - THIS BREAKS THE ABSTRACTION BARRIER TO SOME EXTENT.
|
||||
* We know - just live with it
|
||||
*/
|
||||
opal_setenv("OMPI_COMM_WORLD_RANK", param2, true, &environ_copy);
|
||||
free(param2); /* done with this now */
|
||||
|
||||
/* pass the node's launch_id to the process as a "nodeid" so it can
|
||||
* identify which procs are local to it
|
||||
*/
|
||||
asprintf(¶m2, "%ld", (long)node->launch_id);
|
||||
param = mca_base_param_environ_variable("orte","nodeid",NULL);
|
||||
opal_setenv(param, param2, true, &environ_copy);
|
||||
free(param);
|
||||
free(param2);
|
||||
/* ensure we use the same nodename */
|
||||
param = mca_base_param_environ_variable("orte", "base", "nodename");
|
||||
opal_setenv(param, node->name, true, &environ_copy);
|
||||
free(param);
|
||||
|
||||
/* set the local rank */
|
||||
asprintf(¶m2, "%lu", (unsigned long) proc->local_rank);
|
||||
if(NULL == (param = mca_base_param_environ_variable("orte","ess","local_rank"))) {
|
||||
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
|
||||
rc = ORTE_ERR_OUT_OF_RESOURCE;
|
||||
goto cleanup;
|
||||
}
|
||||
opal_setenv(param, param2, true, &environ_copy);
|
||||
free(param);
|
||||
free(param2);
|
||||
|
||||
/* setup yield schedule and processor affinity
|
||||
* We default here to always setting the affinity processor if we want
|
||||
* it. The processor affinity system then determines
|
||||
* if processor affinity is enabled/requested - if so, it then uses
|
||||
* this value to select the process to which the proc is "assigned".
|
||||
* Otherwise, the paffinity subsystem just ignores this value anyway
|
||||
*/
|
||||
param = mca_base_param_environ_variable("mpi", NULL,
|
||||
"paffinity_processor");
|
||||
asprintf(¶m2, "%lu", (unsigned long) proc->local_rank);
|
||||
opal_setenv(param, param2, true, &environ_copy);
|
||||
free(param);
|
||||
free(param2);
|
||||
|
||||
if (node->oversubscribed) {
|
||||
param = mca_base_param_environ_variable("mpi", NULL, "yield_when_idle");
|
||||
opal_setenv(param, "1", false, &environ_copy);
|
||||
} else {
|
||||
param = mca_base_param_environ_variable("mpi", NULL, "yield_when_idle");
|
||||
opal_setenv(param, "0", false, &environ_copy);
|
||||
}
|
||||
free(param);
|
||||
|
||||
asprintf(¶m2, "%ld", (long) node->num_procs);
|
||||
if(NULL == (param = mca_base_param_environ_variable("orte","ess","num_local_procs"))) {
|
||||
ORTE_ERROR_LOG(ORTE_ERR_OUT_OF_RESOURCE);
|
||||
return ORTE_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
opal_setenv(param, param2, true, &environ_copy);
|
||||
free(param);
|
||||
free(param2);
|
||||
|
||||
OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output,
|
||||
"%s plm:tm: launching on node %s",
|
||||
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
|
||||
node->name));
|
||||
|
||||
/* set the argc count */
|
||||
argc = opal_argv_count(context->argv);
|
||||
|
||||
rc = tm_spawn(argc, context->argv, environ_copy, node->launch_id, tm_task_ids + launched, tm_events + launched);
|
||||
if (TM_SUCCESS != rc) {
|
||||
opal_show_help("help-plm-tm.txt", "tm-spawn-failed",
|
||||
true, context->argv[0], node->name, node->launch_id);
|
||||
rc = ORTE_ERROR;
|
||||
goto cleanup;
|
||||
}
|
||||
opal_argv_free(environ_copy);
|
||||
launched++;
|
||||
|
||||
/* Allow some progress to occur */
|
||||
opal_event_loop(OPAL_EVLOOP_NONBLOCK);
|
||||
}
|
||||
|
||||
OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output,
|
||||
"%s plm:tm:launch: finished spawning procs",
|
||||
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
|
||||
|
||||
/* TM poll for all the spawns */
|
||||
for (i = 0; i < launched; ++i) {
|
||||
rc = tm_poll(TM_NULL_EVENT, &event, 1, &local_err);
|
||||
if (TM_SUCCESS != rc) {
|
||||
errno = local_err;
|
||||
opal_output(0, "plm:tm: failed to poll for a spawned proc, return status = %d", rc);
|
||||
goto cleanup;
|
||||
}
|
||||
}
|
||||
#if 0
|
||||
if (ORTE_SUCCESS != (rc = orte_plm_base_launch_apps(jdata->jobid))) {
|
||||
OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output,
|
||||
"%s plm:tm: launch of apps failed for job %s on error %s",
|
||||
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME),
|
||||
ORTE_JOBID_PRINT(jdata->jobid), ORTE_ERROR_NAME(rc)));
|
||||
goto cleanup;
|
||||
}
|
||||
#endif
|
||||
/* if we get here, then everything launched okay - record that fact */
|
||||
failed_launch = false;
|
||||
|
||||
/* setup periodic timer so we check for job completion */
|
||||
ORTE_TIMER_EVENT(1, check_job_complete);
|
||||
|
||||
cleanup:
|
||||
/* can't reuse the events, so may as well get rid of them */
|
||||
if (NULL != tm_events) {
|
||||
free(tm_events);
|
||||
}
|
||||
|
||||
if (NULL != env) {
|
||||
opal_argv_free(env);
|
||||
}
|
||||
|
||||
if (connected) {
|
||||
plm_tmd_disconnect();
|
||||
}
|
||||
if (NULL != lib_base) {
|
||||
free(lib_base);
|
||||
}
|
||||
if (NULL != bin_base) {
|
||||
free(bin_base);
|
||||
}
|
||||
|
||||
/* check for failed launch - if so, force terminate */
|
||||
if (failed_launch) {
|
||||
orte_plm_base_launch_failed(jdata->jobid, false, -1, 0, ORTE_JOB_STATE_FAILED_TO_START);
|
||||
}
|
||||
|
||||
OPAL_OUTPUT_VERBOSE((1, orte_plm_globals.output,
|
||||
"%s plm:tm:launch: finished",
|
||||
ORTE_NAME_PRINT(ORTE_PROC_MY_NAME)));
|
||||
|
||||
return rc;
|
||||
}
|
||||
|
||||
|
||||
static int plm_tmd_terminate_job(orte_jobid_t jobid)
|
||||
{
|
||||
int rc;
|
||||
int i, local_err;
|
||||
tm_event_t event, *tm_events;
|
||||
|
||||
tm_events = malloc(sizeof(tm_event_t) * num_tids);
|
||||
if (NULL == tm_events) {
|
||||
rc = ORTE_ERR_OUT_OF_RESOURCE;
|
||||
ORTE_ERROR_LOG(rc);
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* connect to the mom */
|
||||
rc = plm_tmd_connect();
|
||||
if (ORTE_SUCCESS != rc) {
|
||||
opal_output(0, "CANNOT CONNECT TO MOM");
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
/* cycle through the task_id's and kill them */
|
||||
for (i=0; i < num_tids; i++) {
|
||||
if (ORTE_SUCCESS != (rc = tm_kill(tm_task_ids[i], SIGTERM, &tm_events[i]))) {
|
||||
opal_output(0, "CANNOT KILL PROC %d", i);
|
||||
}
|
||||
}
|
||||
|
||||
/* cycle through and poll the events */
|
||||
for (i=0; i < num_tids; i++) {
|
||||
rc = tm_poll(TM_NULL_EVENT, &event, 1, &local_err);
|
||||
if (TM_SUCCESS != rc) {
|
||||
errno = local_err;
|
||||
opal_output(0, "plm:tmd: failed to kill proc, return status = %d", rc);
|
||||
}
|
||||
}
|
||||
|
||||
/* disconnect from mom */
|
||||
plm_tmd_disconnect();
|
||||
|
||||
cleanup:
|
||||
/* free tm_events */
|
||||
free(tm_events);
|
||||
|
||||
return ORTE_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
/**
|
||||
* Terminate the orteds for a given job
|
||||
*/
|
||||
static int plm_tmd_terminate_orteds(void)
|
||||
{
|
||||
orte_job_t *daemons;
|
||||
int data=1;
|
||||
|
||||
/* fake the system into thinking the orteds
|
||||
* reported their clean termination
|
||||
*/
|
||||
if (NULL == (daemons = orte_get_job_data_object(ORTE_PROC_MY_NAME->jobid))) {
|
||||
ORTE_ERROR_LOG(ORTE_ERR_NOT_FOUND);
|
||||
return ORTE_ERR_NOT_FOUND;
|
||||
}
|
||||
daemons->num_terminated = daemons->num_procs;
|
||||
|
||||
/* fire the trigger indicating that the orteds are gone */
|
||||
write(orteds_exit, &data, sizeof(int));
|
||||
opal_progress();
|
||||
|
||||
return ORTE_SUCCESS;
|
||||
}
|
||||
|
||||
static int plm_tmd_signal_job(orte_jobid_t jobid, int32_t signal)
|
||||
{
|
||||
int rc;
|
||||
int i, local_err;
|
||||
tm_event_t event, *tm_events;
|
||||
|
||||
tm_events = malloc(sizeof(tm_event_t) * num_tids);
|
||||
if (NULL == tm_events) {
|
||||
rc = ORTE_ERR_OUT_OF_RESOURCE;
|
||||
ORTE_ERROR_LOG(rc);
|
||||
return rc;
|
||||
}
|
||||
|
||||
/* connect to the mom */
|
||||
rc = plm_tmd_connect();
|
||||
if (ORTE_SUCCESS != rc) {
|
||||
opal_output(0, "CANNOT CONNECT TO MOM");
|
||||
goto cleanup;
|
||||
}
|
||||
|
||||
/* cycle through the task_id's and kill them */
|
||||
for (i=0; i < num_tids; i++) {
|
||||
if (ORTE_SUCCESS != (rc = tm_kill(tm_task_ids[i], signal, &tm_events[i]))) {
|
||||
opal_output(0, "CANNOT SIGNAL PROC %d", i);
|
||||
}
|
||||
}
|
||||
|
||||
/* cycle through and poll the events */
|
||||
for (i=0; i < num_tids; i++) {
|
||||
rc = tm_poll(TM_NULL_EVENT, &event, 1, &local_err);
|
||||
if (TM_SUCCESS != rc) {
|
||||
errno = local_err;
|
||||
opal_output(0, "plm:tmd: failed to signal proc, return status = %d", rc);
|
||||
goto cleanup;
|
||||
}
|
||||
}
|
||||
|
||||
/* disconnect from mom */
|
||||
plm_tmd_disconnect();
|
||||
|
||||
cleanup:
|
||||
/* free tm_events */
|
||||
free(tm_events);
|
||||
|
||||
return ORTE_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
/*
|
||||
* Free stuff
|
||||
*/
|
||||
static int plm_tmd_finalize(void)
|
||||
{
|
||||
int rc;
|
||||
|
||||
/* cleanup any pending recvs */
|
||||
if (ORTE_SUCCESS != (rc = orte_plm_base_comm_stop())) {
|
||||
ORTE_ERROR_LOG(rc);
|
||||
}
|
||||
|
||||
if (NULL != tm_task_ids) {
|
||||
free(tm_task_ids);
|
||||
}
|
||||
|
||||
return ORTE_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
static int plm_tmd_connect(void)
|
||||
{
|
||||
int ret;
|
||||
struct tm_roots tm_root;
|
||||
int count, progress;
|
||||
|
||||
/* try a couple times to connect - might get busy signals every
|
||||
now and then */
|
||||
for (count = 0 ; count < 10; ++count) {
|
||||
ret = tm_init(NULL, &tm_root);
|
||||
if (TM_SUCCESS == ret) {
|
||||
return ORTE_SUCCESS;
|
||||
}
|
||||
|
||||
for (progress = 0 ; progress < 10 ; ++progress) {
|
||||
opal_progress();
|
||||
#if HAVE_SCHED_YIELD
|
||||
sched_yield();
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
||||
return ORTE_ERR_RESOURCE_BUSY;
|
||||
}
|
||||
|
||||
|
||||
static int plm_tmd_disconnect(void)
|
||||
{
|
||||
tm_finalize();
|
||||
|
||||
return ORTE_SUCCESS;
|
||||
}
|
||||
Загрузка…
x
Ссылка в новой задаче
Block a user