Merge pull request #3603 from rhc54/topic/fds

Add some debug code for detecting leaking file descriptors.

orte/mca/state/base/state_base_fns.c

@@ -13,6 +13,13 @@
 #include "orte_config.h"
 #include "orte/constants.h"
 
+#if HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+#if HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+
 #include "opal/class/opal_list.h"
 #include "opal/mca/event/event.h"
 #include "opal/mca/pmix/pmix.h"
@@ -714,6 +721,10 @@ void orte_state_base_track_procs(int fd, short argc, void *cbdata)
         /* track job status */
         jdata->num_terminated++;
         if (jdata->num_terminated == jdata->num_procs) {
+            /* if requested, check fd status for leaks */
+            if (orte_state_base_run_fdcheck) {
+                orte_state_base_check_fds(jdata);
+            }
             ORTE_ACTIVATE_JOB_STATE(jdata, ORTE_JOB_STATE_TERMINATED);
             /* if they requested notification upon completion, provide it */
             if (orte_get_attribute(&jdata->attributes, ORTE_JOB_NOTIFY_COMPLETION, NULL, OPAL_BOOL)) {
@@ -1016,3 +1027,99 @@ void orte_state_base_check_all_complete(int fd, short args, void *cbdata)
 
     OBJ_RELEASE(caddy);
 }
+
+
+void orte_state_base_check_fds(orte_job_t *jdata)
+{
+    int nfds, i, fdflags, flflags;
+    char path[1024], info[256], **list=NULL, *status, *result, *r2;
+    ssize_t rc;
+    struct flock fl;
+    int cnt = 0;
+
+    /* get the number of available file descriptors
+     * for this daemon */
+    nfds = getdtablesize();
+    result = NULL;
+    /* loop over them and get their info */
+    for (i=0; i < nfds; i++) {
+        fdflags = fcntl(i, F_GETFD);
+        if (-1 == fdflags) {
+            /* no open fd in that slot */
+            continue;
+        }
+        flflags = fcntl(i, F_GETFL);
+        if (-1 == flflags) {
+            /* no open fd in that slot */
+            continue;
+        }
+        snprintf(path, 1024, "/proc/self/fd/%d", i);
+        memset(info, 0, 256);
+        /* read the info about this fd */
+        rc = readlink(path, info, 256);
+        if (-1 == rc) {
+            /* this fd is unavailable */
+            continue;
+        }
+        /* get any file locking status */
+        fl.l_type = F_WRLCK;
+        fl.l_whence = 0;
+        fl.l_start = 0;
+        fl.l_len = 0;
+        fcntl(i, F_GETLK, &fl);
+        /* construct the list of capabilities */
+        if (fdflags & FD_CLOEXEC) {
+            opal_argv_append_nosize(&list, "cloexec");
+        }
+        if (flflags & O_APPEND) {
+            opal_argv_append_nosize(&list, "append");
+        }
+        if (flflags & O_NONBLOCK) {
+            opal_argv_append_nosize(&list, "nonblock");
+        }
+        if (flflags & O_RDONLY) {
+            opal_argv_append_nosize(&list, "rdonly");
+        }
+        if (flflags & O_RDWR) {
+            opal_argv_append_nosize(&list, "rdwr");
+        }
+        if (flflags & O_WRONLY) {
+            opal_argv_append_nosize(&list, "wronly");
+        }
+        if (flflags & O_DSYNC) {
+            opal_argv_append_nosize(&list, "dsync");
+        }
+        if (flflags & O_RSYNC) {
+            opal_argv_append_nosize(&list, "rsync");
+        }
+        if (flflags & O_SYNC) {
+            opal_argv_append_nosize(&list, "sync");
+        }
+        if (F_UNLCK != fl.l_type) {
+            if (F_WRLCK == fl.l_type) {
+                opal_argv_append_nosize(&list, "wrlock");
+            } else {
+                opal_argv_append_nosize(&list, "rdlock");
+            }
+        }
+        if (NULL != list) {
+            status = opal_argv_join(list, ' ');
+            opal_argv_free(list);
+            list = NULL;
+            if (NULL == result) {
+                asprintf(&result, "    %d\t(%s)\t%s\n", i, info, status);
+            } else {
+                asprintf(&r2, "%s    %d\t(%s)\t%s\n", result, i, info, status);
+                free(result);
+                result = r2;
+            }
+            free(status);
+        }
+        ++cnt;
+    }
+    asprintf(&r2, "%s: %d open file descriptors after job %d completed\n%s",
+             ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), cnt, ORTE_LOCAL_JOBID(jdata->jobid), result);
+    opal_output(0, "%s", r2);
+    free(result);
+    free(r2);
+}

orte/mca/state/base/state_base_frame.c

@@ -4,6 +4,7 @@
  *                         All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2017      Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -41,6 +42,20 @@
  * Globals
  */
 orte_state_base_module_t orte_state = {0};
+bool orte_state_base_run_fdcheck = false;
+
+static int orte_state_base_register(mca_base_register_flag_t flags)
+{
+    orte_state_base_run_fdcheck = false;
+    mca_base_var_register("orte", "state", "base", "check_fds",
+                          "Daemons should check fds for leaks after each job completes",
+                          MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
+                          OPAL_INFO_LVL_9,
+                          MCA_BASE_VAR_SCOPE_READONLY,
+                          &orte_state_base_run_fdcheck);
+
+    return ORTE_SUCCESS;
+}
 
 static int orte_state_base_close(void)
 {
@@ -62,7 +77,8 @@ static int orte_state_base_open(mca_base_open_flag_t flags)
     return mca_base_framework_components_open(&orte_state_base_framework, flags);
 }
 
-MCA_BASE_FRAMEWORK_DECLARE(orte, state, "ORTE State Machine", NULL,
+MCA_BASE_FRAMEWORK_DECLARE(orte, state, "ORTE State Machine",
+                           orte_state_base_register,
                            orte_state_base_open, orte_state_base_close,
                            mca_state_base_static_components, 0);
 
@@ -95,4 +111,3 @@ OBJ_CLASS_INSTANCE(orte_state_caddy_t,
                    opal_object_t,
                    orte_state_caddy_construct,
                    orte_state_caddy_destruct);
-

orte/mca/state/base/state_private.h

@@ -1,6 +1,7 @@
 /*
  * Copyright (c) 2011-2013 Los Alamos National Security, LLC.
  *                         All rights reserved.
+ * Copyright (c) 2017      Intel, Inc. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -31,6 +32,7 @@
 
 BEGIN_C_DECLS
 
+extern bool orte_state_base_run_fdcheck;
 /*
  * Base functions
  */
@@ -75,7 +77,7 @@ ORTE_DECLSPEC void orte_state_base_cleanup_job(int fd, short argc, void *cbdata)
 ORTE_DECLSPEC void orte_state_base_report_progress(int fd, short argc, void *cbdata);
 ORTE_DECLSPEC void orte_state_base_track_procs(int fd, short argc, void *cbdata);
 ORTE_DECLSPEC void orte_state_base_check_all_complete(int fd, short args, void *cbdata);
-
+ORTE_DECLSPEC void orte_state_base_check_fds(orte_job_t *jdata);
 
 END_C_DECLS
 #endif

orte/mca/state/orted/state_orted.c

@@ -484,6 +484,11 @@ static void track_procs(int fd, short argc, void *cbdata)
                 jdata->map = NULL;
             }
 
+            /* if requested, check fd status for leaks */
+            if (orte_state_base_run_fdcheck) {
+                orte_state_base_check_fds(jdata);
+            }
+
             /* cleanup the job info */
             opal_hash_table_set_value_uint32(orte_job_data, jdata->jobid, NULL);
             OBJ_RELEASE(jdata);