From 4d0cc27eb7c96a3ad50abf309b6f3186e2f05f58 Mon Sep 17 00:00:00 2001 From: Ralph Castain Date: Sat, 5 Mar 2016 21:47:42 -0800 Subject: [PATCH] Update the singularity support to match that of the latest singularity master. Remove the restriction on shared memory components by instructing singularity to not isolate the PID space. Add a new schizo API to allow setting up the original app_context. Ensure the container is installed prior to execution. --- opal/mca/shmem/mmap/shmem_mmap_component.c | 7 - opal/mca/shmem/posix/shmem_posix_component.c | 7 - opal/mca/shmem/sysv/shmem_sysv_component.c | 7 - orte/mca/schizo/alps/schizo_alps.c | 4 +- orte/mca/schizo/base/base.h | 2 + orte/mca/schizo/base/schizo_base_frame.c | 1 + orte/mca/schizo/base/schizo_base_stubs.c | 22 +++- orte/mca/schizo/schizo.h | 24 ++++ .../schizo/singularity/schizo_singularity.c | 121 ++++++++++++++---- orte/tools/orterun/orterun.c | 8 ++ orte/util/context_fns.c | 10 +- 11 files changed, 157 insertions(+), 56 deletions(-) diff --git a/opal/mca/shmem/mmap/shmem_mmap_component.c b/opal/mca/shmem/mmap/shmem_mmap_component.c index 8cdc958e2f..acac7bb8f3 100644 --- a/opal/mca/shmem/mmap/shmem_mmap_component.c +++ b/opal/mca/shmem/mmap/shmem_mmap_component.c @@ -176,13 +176,6 @@ mmap_open(void) static int mmap_query(mca_base_module_t **module, int *priority) { - /* if we are in a container, then we must disqualify ourselves */ - if (NULL != getenv("OPAL_PROC_CONTAINER")) { - *priority = 0; - *module = NULL; - return OPAL_ERROR; - } - *priority = mca_shmem_mmap_component.priority; *module = (mca_base_module_t *)&opal_shmem_mmap_module.super; return OPAL_SUCCESS; diff --git a/opal/mca/shmem/posix/shmem_posix_component.c b/opal/mca/shmem/posix/shmem_posix_component.c index 1cd7bb03ad..9ac4549bf1 100644 --- a/opal/mca/shmem/posix/shmem_posix_component.c +++ b/opal/mca/shmem/posix/shmem_posix_component.c @@ -201,13 +201,6 @@ posix_runtime_query(mca_base_module_t **module, static int posix_query(mca_base_module_t **module, int *priority) { - /* if we are in a container, then we must disqualify ourselves */ - if (NULL != getenv("OPAL_PROC_CONTAINER")) { - *priority = 0; - *module = NULL; - return OPAL_ERROR; - } - *priority = mca_shmem_posix_component.priority; *module = (mca_base_module_t *)&opal_shmem_posix_module.super; return OPAL_SUCCESS; diff --git a/opal/mca/shmem/sysv/shmem_sysv_component.c b/opal/mca/shmem/sysv/shmem_sysv_component.c index a03a88c31a..b53fd3bed1 100644 --- a/opal/mca/shmem/sysv/shmem_sysv_component.c +++ b/opal/mca/shmem/sysv/shmem_sysv_component.c @@ -210,13 +210,6 @@ out: static int sysv_query(mca_base_module_t **module, int *priority) { - /* if we are in a container, then we must disqualify ourselves */ - if (NULL != getenv("OPAL_PROC_CONTAINER")) { - *priority = 0; - *module = NULL; - return OPAL_ERROR; - } - *priority = mca_shmem_sysv_component.priority; *module = (mca_base_module_t *)&opal_shmem_sysv_module.super; return OPAL_SUCCESS; diff --git a/orte/mca/schizo/alps/schizo_alps.c b/orte/mca/schizo/alps/schizo_alps.c index c28b928bdb..b20ab9185d 100644 --- a/orte/mca/schizo/alps/schizo_alps.c +++ b/orte/mca/schizo/alps/schizo_alps.c @@ -29,9 +29,11 @@ #include "schizo_alps.h" static orte_schizo_launch_environ_t check_launch_environment(void); +static void finalize(void); orte_schizo_base_module_t orte_schizo_alps_module = { - .check_launch_environment = check_launch_environment + .check_launch_environment = check_launch_environment, + .finalize = finalize }; static char **pushed_envs = NULL; diff --git a/orte/mca/schizo/base/base.h b/orte/mca/schizo/base/base.h index 9cf4ffdb3d..1cb72d702c 100644 --- a/orte/mca/schizo/base/base.h +++ b/orte/mca/schizo/base/base.h @@ -68,6 +68,8 @@ ORTE_DECLSPEC int orte_schizo_base_parse_env(char **personality, opal_cmd_line_t *cmd_line, char **srcenv, char ***dstenv); +ORTE_DECLSPEC int orte_schizo_base_setup_app(char **personality, + orte_app_context_t *app); ORTE_DECLSPEC int orte_schizo_base_setup_fork(orte_job_t *jdata, orte_app_context_t *context); ORTE_DECLSPEC int orte_schizo_base_setup_child(orte_job_t *jobdat, diff --git a/orte/mca/schizo/base/schizo_base_frame.c b/orte/mca/schizo/base/schizo_base_frame.c index 33135060f2..c9fb70650f 100644 --- a/orte/mca/schizo/base/schizo_base_frame.c +++ b/orte/mca/schizo/base/schizo_base_frame.c @@ -39,6 +39,7 @@ orte_schizo_base_t orte_schizo_base = {{{0}}}; orte_schizo_base_module_t orte_schizo = { .parse_cli = orte_schizo_base_parse_cli, .parse_env = orte_schizo_base_parse_env, + .setup_app = orte_schizo_base_setup_app, .setup_fork = orte_schizo_base_setup_fork, .setup_child = orte_schizo_base_setup_child, .check_launch_environment = orte_schizo_base_check_launch_environment, diff --git a/orte/mca/schizo/base/schizo_base_stubs.c b/orte/mca/schizo/base/schizo_base_stubs.c index 76e35a0c73..21794c3d59 100644 --- a/orte/mca/schizo/base/schizo_base_stubs.c +++ b/orte/mca/schizo/base/schizo_base_stubs.c @@ -19,8 +19,8 @@ #include "orte/util/name_fns.h" #include "orte/mca/schizo/base/base.h" -const char* orte_schizo_base_print_env(orte_schizo_launch_environ_t env) -{ + const char* orte_schizo_base_print_env(orte_schizo_launch_environ_t env) + { switch(env) { case ORTE_SCHIZO_UNDETERMINED: return "UNDETERMINED"; @@ -80,6 +80,24 @@ int orte_schizo_base_parse_env(char **personality, return ORTE_SUCCESS; } +int orte_schizo_base_setup_app(char **personality, + orte_app_context_t *app) +{ + int rc; + orte_schizo_base_active_module_t *mod; + + OPAL_LIST_FOREACH(mod, &orte_schizo_base.active_modules, orte_schizo_base_active_module_t) { + if (NULL != mod->module->setup_app) { + rc = mod->module->setup_app(personality, app); + if (ORTE_SUCCESS != rc && ORTE_ERR_TAKE_NEXT_OPTION != rc) { + ORTE_ERROR_LOG(rc); + return rc; + } + } + } + return ORTE_SUCCESS; +} + int orte_schizo_base_setup_fork(orte_job_t *jdata, orte_app_context_t *context) { diff --git a/orte/mca/schizo/schizo.h b/orte/mca/schizo/schizo.h index 985f0f6a60..5906a1e22f 100644 --- a/orte/mca/schizo/schizo.h +++ b/orte/mca/schizo/schizo.h @@ -40,21 +40,44 @@ BEGIN_C_DECLS * the base stub functions */ +/* initialize the module - allow it to do whatever one-time + * things it requires */ typedef int (*orte_schizo_base_module_init_fn_t)(void); +/* given an argv-array of personalities, parse a tool command line + * starting from the given location according to the cmd line options + * known to this module's personality. First, of course, check that + * this module is included in the specified array of personalities! + * Only one command-line parser is allowed to operate - i.e., if */ typedef int (*orte_schizo_base_module_parse_cli_fn_t)(char **personality, int argc, int start, char **argv); +/* given an argv-array of personalities, parse the environment of the + * tool to extract any personality-specific envars that need to be + * forward to the app's environment upon execution */ typedef int (*orte_schizo_base_module_parse_env_fn_t)(char **personality, char *path, opal_cmd_line_t *cmd_line, char **srcenv, char ***dstenv); +/* given an argv-array of personalities, do whatever preparation work + * is required to setup the app for execution. This is intended to be + * used by orterun and other launcher tools to, for example, change + * an executable's relative-path to an absolute-path, or add a command + * required for starting a particular kind of application (e.g., adding + * "java" to start a Java application) */ +typedef int (*orte_schizo_base_module_setup_app_fn_t)(char **personality, + orte_app_context_t *app); + +/* add any personality-specific envars required at the job level prior + * to beginning to execute local procs */ typedef int (*orte_schizo_base_module_setup_fork_fn_t)(orte_job_t *jdata, orte_app_context_t *context); +/* add any personality-specific envars required for this specific local + * proc upon execution */ typedef int (*orte_schizo_base_module_setup_child_fn_t)(orte_job_t *jdata, orte_proc_t *child, orte_app_context_t *app); @@ -86,6 +109,7 @@ typedef struct { orte_schizo_base_module_init_fn_t init; orte_schizo_base_module_parse_cli_fn_t parse_cli; orte_schizo_base_module_parse_env_fn_t parse_env; + orte_schizo_base_module_setup_app_fn_t setup_app; orte_schizo_base_module_setup_fork_fn_t setup_fork; orte_schizo_base_module_setup_child_fn_t setup_child; orte_schizo_base_module_ck_launch_environ_fn_t check_launch_environment; diff --git a/orte/mca/schizo/singularity/schizo_singularity.c b/orte/mca/schizo/singularity/schizo_singularity.c index fbcb4bdf33..3e73db7a2d 100644 --- a/orte/mca/schizo/singularity/schizo_singularity.c +++ b/orte/mca/schizo/singularity/schizo_singularity.c @@ -20,6 +20,8 @@ #include "opal/util/argv.h" #include "opal/util/basename.h" #include "opal/util/opal_environ.h" +#include "opal/util/os_dirpath.h" +#include "opal/util/path.h" #include "orte/runtime/orte_globals.h" #include "orte/util/name_fns.h" @@ -27,24 +29,27 @@ #include "schizo_singularity.h" +static int setup_app(char **personality, + orte_app_context_t *context); static int setup_fork(orte_job_t *jdata, orte_app_context_t *context); orte_schizo_base_module_t orte_schizo_singularity_module = { + .setup_app = setup_app, .setup_fork = setup_fork }; -static int setup_fork(orte_job_t *jdata, - orte_app_context_t *app) +static int setup_app(char **personality, + orte_app_context_t *app) { int i; - char *newenv; + char *newenv, *pth; bool takeus = false; - char *cmd, *p, *t2; + char *p, *t2; /* see if we are included */ - for (i=0; NULL != jdata->personality[i]; i++) { - if (0 == strcmp(jdata->personality[i], "singularity")) { + for (i=0; NULL != personality[i]; i++) { + if (0 == strcmp(personality[i], "singularity")) { takeus = true; break; } @@ -64,44 +69,104 @@ static int setup_fork(orte_job_t *jdata, "%s schizo:singularity: checking app %s", ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), app->argv[0]); + if (0 < strlen(OPAL_SINGULARITY_PATH)) { + asprintf(&pth, "%s/singularity", OPAL_SINGULARITY_PATH); + } else { + /* since we allow for detecting singularity's presence, it + * is possible that we found it in the PATH, but not in a + * standard location. Check for that here */ + pth = opal_path_findv("singularity", X_OK, app->env, NULL); + if (NULL == pth) { + /* cannot execute */ + return ORTE_ERR_TAKE_NEXT_OPTION; + } + } /* find the path and prepend it with the path to Singularity */ for (i = 0; NULL != app->env && NULL != app->env[i]; ++i) { /* add to PATH */ if (0 == strncmp("PATH=", app->env[i], 5)) { - asprintf(&newenv, "%s:%s", OPAL_SINGULARITY_PATH, app->env[i] + 5); + t2 = opal_dirname(pth); + asprintf(&newenv, "%s:%s", t2, app->env[i] + 5); opal_setenv("PATH", newenv, true, &app->env); free(newenv); + free(t2); break; } } - /* flag that the app is in a container */ - opal_setenv("OPAL_PROC_CONTAINER", "1", true, &app->env); - /* ensure that we use "singularity run" to execute this app */ if (0 != strcmp(app->app, "singularity")) { opal_output_verbose(1, orte_schizo_base_framework.framework_output, - "%s schizo:singularity: adding singularity cmds at %s", - ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), OPAL_SINGULARITY_PATH); + "%s schizo:singularity: adding singularity cmd %s", + ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), pth); /* change the app to the "singularity" command */ free(app->app); - if (0 < strlen(OPAL_SINGULARITY_PATH)) { - asprintf(&app->app, "%s/singularity", OPAL_SINGULARITY_PATH); - } else { - app->app = strdup("singularity"); - } - /* if the app contains .sapp, then we need to strip that - * extension so singularity doesn't bark at us */ - if (NULL != (p = strstr(app->argv[0], ".sapp"))) { - t2 = opal_basename(app->argv[0]); - p = strstr(t2, ".sapp"); - *p = '\0'; // strip the extension - free(app->argv[0]); - app->argv[0] = t2; - } - opal_argv_prepend_nosize(&app->argv, "run"); - opal_argv_prepend_nosize(&app->argv, "singularity"); + app->app = pth; + } else { + free(pth); } + /* if the app contains .sapp, then we need to strip that + * extension so singularity doesn't bark at us */ + if (NULL != (p = strstr(app->argv[0], ".sapp"))) { + t2 = opal_basename(app->argv[0]); + p = strstr(t2, ".sapp"); + *p = '\0'; // strip the extension + free(app->argv[0]); + app->argv[0] = t2; + } + opal_argv_prepend_nosize(&app->argv, "run"); + opal_argv_prepend_nosize(&app->argv, "singularity"); + + /* export an envar to permit shared memory operations */ + opal_setenv("SINGULARITY_NO_NAMESPACE_PID", "1", true, &app->env); + return ORTE_SUCCESS; } + +static int setup_fork(orte_job_t *jdata, + orte_app_context_t *app) +{ + int i; + bool takeus = false; + char *p; + char dir[MAXPATHLEN]; + + /* see if we are included */ + for (i=0; NULL != jdata->personality[i]; i++) { + if (0 == strcmp(jdata->personality[i], "singularity")) { + takeus = true; + break; + } + } + if (!takeus) { + /* even if they didn't specify, check to see if + * this involves a singularity container */ + if (0 != strcmp(app->argv[0],"singularity") && + 0 != strcmp(app->argv[0],"sapprun") && + NULL == strstr(app->argv[0], ".sapp")) { + /* guess not! */ + return ORTE_ERR_TAKE_NEXT_OPTION; + } + } + + /* save our current directory */ + getcwd(dir, sizeof(dir)); + + /* change to the working directory for this context */ + chdir(app->cwd); + + /* ensure the app is installed */ + opal_output_verbose(1, orte_schizo_base_framework.framework_output, + "%s schizo:singularity: installing app %s", + ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), app->argv[2]); + (void)asprintf(&p, "%s install %s.sapp &> /dev/null", app->app, app->argv[2]); + system(p); + free(p); + + /* return to the original directory */ + chdir(dir); + + return ORTE_SUCCESS; +} + diff --git a/orte/tools/orterun/orterun.c b/orte/tools/orterun/orterun.c index d63008e49a..6dde2ed177 100644 --- a/orte/tools/orterun/orterun.c +++ b/orte/tools/orterun/orterun.c @@ -1258,6 +1258,9 @@ static int parse_locals(orte_job_t *jdata, int argc, char* argv[]) ++app_num; opal_pointer_array_add(jdata->apps, app); ++jdata->num_apps; + if (ORTE_SUCCESS != (rc = orte_schizo.setup_app(jdata->personality, app))) { + return rc; + } } /* Reset the temps */ @@ -1284,6 +1287,9 @@ static int parse_locals(orte_job_t *jdata, int argc, char* argv[]) ++app_num; opal_pointer_array_add(jdata->apps, app); ++jdata->num_apps; + if (ORTE_SUCCESS != (rc = orte_schizo.setup_app(jdata->personality, app))) { + return rc; + } } } if (NULL != env) { @@ -1670,6 +1676,8 @@ static int create_app(int argc, char* argv[], rc = ORTE_ERR_NOT_FOUND; goto cleanup; } + free(app->argv[0]); + app->argv[0] = opal_basename(app->app); /* if this is a Java application, we have a bit more work to do. Such * applications actually need to be run under the Java virtual machine diff --git a/orte/util/context_fns.c b/orte/util/context_fns.c index 8e3e486c8e..a5a4a54efd 100644 --- a/orte/util/context_fns.c +++ b/orte/util/context_fns.c @@ -130,16 +130,18 @@ int orte_util_check_context_app(orte_app_context_t *context, char **env) bproc, for example, does not use the fork pls for launching, so it does this same work over there. */ - tmp = opal_basename(context->argv[0]); - if (strlen(tmp) == strlen(context->argv[0])) { + tmp = opal_basename(context->app); + if (strlen(tmp) == strlen(context->app)) { /* If this is a naked executable -- no relative or absolute pathname -- then search the PATH for it */ free(tmp); - tmp = opal_path_findv(context->argv[0], X_OK, env, context->cwd); + tmp = opal_path_findv(context->app, X_OK, env, context->cwd); if (NULL == tmp) { return ORTE_ERR_EXE_NOT_FOUND; } - if (NULL != context->app) free(context->app); + if (NULL != context->app) { + free(context->app); + } context->app = tmp; } else { free(tmp);