diff --git a/orte/mca/state/base/state_base_fns.c b/orte/mca/state/base/state_base_fns.c index 38c27ba08a..cfc258d7d1 100644 --- a/orte/mca/state/base/state_base_fns.c +++ b/orte/mca/state/base/state_base_fns.c @@ -13,6 +13,13 @@ #include "orte_config.h" #include "orte/constants.h" +#if HAVE_UNISTD_H +#include +#endif +#if HAVE_FCNTL_H +#include +#endif + #include "opal/class/opal_list.h" #include "opal/mca/event/event.h" #include "opal/mca/pmix/pmix.h" @@ -714,6 +721,10 @@ void orte_state_base_track_procs(int fd, short argc, void *cbdata) /* track job status */ jdata->num_terminated++; if (jdata->num_terminated == jdata->num_procs) { + /* if requested, check fd status for leaks */ + if (orte_state_base_run_fdcheck) { + orte_state_base_check_fds(jdata); + } ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_TERMINATED); /* if they requested notification upon completion, provide it */ if (orte_get_attribute(&jdata->attributes, ORTE_JOB_NOTIFY_COMPLETION, NULL, OPAL_BOOL)) { @@ -1016,3 +1027,99 @@ void orte_state_base_check_all_complete(int fd, short args, void *cbdata) OBJ_RELEASE(caddy); } + + +void orte_state_base_check_fds(orte_job_t *jdata) +{ + int nfds, i, fdflags, flflags; + char path[1024], info[256], **list=NULL, *status, *result, *r2; + ssize_t rc; + struct flock fl; + int cnt = 0; + + /* get the number of available file descriptors + * for this daemon */ + nfds = getdtablesize(); + result = NULL; + /* loop over them and get their info */ + for (i=0; i < nfds; i++) { + fdflags = fcntl(i, F_GETFD); + if (-1 == fdflags) { + /* no open fd in that slot */ + continue; + } + flflags = fcntl(i, F_GETFL); + if (-1 == flflags) { + /* no open fd in that slot */ + continue; + } + snprintf(path, 1024, "/proc/self/fd/%d", i); + memset(info, 0, 256); + /* read the info about this fd */ + rc = readlink(path, info, 256); + if (-1 == rc) { + /* this fd is unavailable */ + continue; + } + /* get any file locking status */ + fl.l_type = F_WRLCK; + fl.l_whence = 0; + fl.l_start = 0; + fl.l_len = 0; + fcntl(i, F_GETLK, &fl); + /* construct the list of capabilities */ + if (fdflags & FD_CLOEXEC) { + opal_argv_append_nosize(&list, "cloexec"); + } + if (flflags & O_APPEND) { + opal_argv_append_nosize(&list, "append"); + } + if (flflags & O_NONBLOCK) { + opal_argv_append_nosize(&list, "nonblock"); + } + if (flflags & O_RDONLY) { + opal_argv_append_nosize(&list, "rdonly"); + } + if (flflags & O_RDWR) { + opal_argv_append_nosize(&list, "rdwr"); + } + if (flflags & O_WRONLY) { + opal_argv_append_nosize(&list, "wronly"); + } + if (flflags & O_DSYNC) { + opal_argv_append_nosize(&list, "dsync"); + } + if (flflags & O_RSYNC) { + opal_argv_append_nosize(&list, "rsync"); + } + if (flflags & O_SYNC) { + opal_argv_append_nosize(&list, "sync"); + } + if (F_UNLCK != fl.l_type) { + if (F_WRLCK == fl.l_type) { + opal_argv_append_nosize(&list, "wrlock"); + } else { + opal_argv_append_nosize(&list, "rdlock"); + } + } + if (NULL != list) { + status = opal_argv_join(list, ' '); + opal_argv_free(list); + list = NULL; + if (NULL == result) { + asprintf(&result, " %d\t(%s)\t%s\n", i, info, status); + } else { + asprintf(&r2, "%s %d\t(%s)\t%s\n", result, i, info, status); + free(result); + result = r2; + } + free(status); + } + ++cnt; + } + asprintf(&r2, "%s: %d open file descriptors after job %d completed\n%s", + ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), cnt, ORTE_LOCAL_JOBID(jdata->jobid), result); + opal_output(0, "%s", r2); + free(result); + free(r2); +} diff --git a/orte/mca/state/base/state_base_frame.c b/orte/mca/state/base/state_base_frame.c index 3838d901dd..74c009d46f 100644 --- a/orte/mca/state/base/state_base_frame.c +++ b/orte/mca/state/base/state_base_frame.c @@ -4,6 +4,7 @@ * All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2017 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -41,6 +42,20 @@ * Globals */ orte_state_base_module_t orte_state = {0}; +bool orte_state_base_run_fdcheck = false; + +static int orte_state_base_register(mca_base_register_flag_t flags) +{ + orte_state_base_run_fdcheck = false; + mca_base_var_register("orte", "state", "base", "check_fds", + "Daemons should check fds for leaks after each job completes", + MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, + OPAL_INFO_LVL_9, + MCA_BASE_VAR_SCOPE_READONLY, + &orte_state_base_run_fdcheck); + + return ORTE_SUCCESS; +} static int orte_state_base_close(void) { @@ -62,7 +77,8 @@ static int orte_state_base_open(mca_base_open_flag_t flags) return mca_base_framework_components_open(&orte_state_base_framework, flags); } -MCA_BASE_FRAMEWORK_DECLARE(orte, state, "ORTE State Machine", NULL, +MCA_BASE_FRAMEWORK_DECLARE(orte, state, "ORTE State Machine", + orte_state_base_register, orte_state_base_open, orte_state_base_close, mca_state_base_static_components, 0); @@ -95,4 +111,3 @@ OBJ_CLASS_INSTANCE(orte_state_caddy_t, opal_object_t, orte_state_caddy_construct, orte_state_caddy_destruct); - diff --git a/orte/mca/state/base/state_private.h b/orte/mca/state/base/state_private.h index 0c9db094ad..3ba3bcc1dd 100644 --- a/orte/mca/state/base/state_private.h +++ b/orte/mca/state/base/state_private.h @@ -1,6 +1,7 @@ /* * Copyright (c) 2011-2013 Los Alamos National Security, LLC. * All rights reserved. + * Copyright (c) 2017 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -31,6 +32,7 @@ BEGIN_C_DECLS +extern bool orte_state_base_run_fdcheck; /* * Base functions */ @@ -75,7 +77,7 @@ ORTE_DECLSPEC void orte_state_base_cleanup_job(int fd, short argc, void *cbdata) ORTE_DECLSPEC void orte_state_base_report_progress(int fd, short argc, void *cbdata); ORTE_DECLSPEC void orte_state_base_track_procs(int fd, short argc, void *cbdata); ORTE_DECLSPEC void orte_state_base_check_all_complete(int fd, short args, void *cbdata); - +ORTE_DECLSPEC void orte_state_base_check_fds(orte_job_t *jdata); END_C_DECLS #endif diff --git a/orte/mca/state/orted/state_orted.c b/orte/mca/state/orted/state_orted.c index 708d69fca2..55ad8082e1 100644 --- a/orte/mca/state/orted/state_orted.c +++ b/orte/mca/state/orted/state_orted.c @@ -484,6 +484,11 @@ static void track_procs(int fd, short argc, void *cbdata) jdata->map = NULL; } + /* if requested, check fd status for leaks */ + if (orte_state_base_run_fdcheck) { + orte_state_base_check_fds(jdata); + } + /* cleanup the job info */ opal_hash_table_set_value_uint32(orte_job_data, jdata->jobid, NULL); OBJ_RELEASE(jdata);