Merge pull request #4698 from rhc54/topic/cl2

Ensure the epilog gets executed in PMIx server

opal/mca/pmix/pmix3x/pmix/src/include/pmix_globals.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2017 Intel, Inc. All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc.  All rights reserved.
  * Copyright (c) 2014-2017 Research Organization for Information Science
  * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
  *                         and Technology (RIST). All rights reserved.
@@ -53,7 +53,6 @@
 #include "src/util/error.h"
 #include "src/util/os_path.h"
 
-static void cleanup(pmix_epilog_t *epi);
 static void dirpath_destroy(char *path, pmix_cleanup_dir_t *cd,
                             pmix_epilog_t *epi);
 static bool dirpath_is_empty(const char *path);
@@ -121,7 +120,7 @@ static void nsdes(pmix_nspace_t *p)
     }
     PMIX_LIST_DESTRUCT(&p->ranks);
     /* perform any epilog */
-    cleanup(&p->epilog);
+    pmix_execute_epilog(&p->epilog);
     /* cleanup the epilog */
     PMIX_LIST_DESTRUCT(&p->epilog.cleanup_dirs);
     PMIX_LIST_DESTRUCT(&p->epilog.cleanup_files);
@@ -208,7 +207,7 @@ static void pdes(pmix_peer_t *p)
         PMIX_RELEASE(p->recv_msg);
     }
     /* perform any epilog */
-    cleanup(&p->epilog);
+    pmix_execute_epilog(&p->epilog);
     /* cleanup the epilog */
     PMIX_LIST_DESTRUCT(&p->epilog.cleanup_dirs);
     PMIX_LIST_DESTRUCT(&p->epilog.cleanup_files);
@@ -318,15 +317,15 @@ PMIX_EXPORT PMIX_CLASS_INSTANCE(pmix_query_caddy_t,
                                 pmix_object_t,
                                 qcon, qdes);
 
-static void cleanup(pmix_epilog_t *epi)
+void pmix_execute_epilog(pmix_epilog_t *epi)
 {
-    pmix_cleanup_file_t *cf;
+    pmix_cleanup_file_t *cf, *cfnext;
-    pmix_cleanup_dir_t *cd;
+    pmix_cleanup_dir_t *cd, *cdnext;
     struct stat statbuf;
     int rc;
 
     /* start with any specified files */
-    PMIX_LIST_FOREACH(cf, &epi->cleanup_files, pmix_cleanup_file_t) {
+    PMIX_LIST_FOREACH_SAFE(cf, cfnext, &epi->cleanup_files, pmix_cleanup_file_t) {
         /* check the effective uid/gid of the file and ensure it
          * matches that of the peer - we do this to provide at least
          * some minimum level of protection */
@@ -350,10 +349,12 @@ static void cleanup(pmix_epilog_t *epi)
             pmix_output_verbose(10, pmix_globals.debug_output,
                                 "File %s failed to unlink: %s", cf->path, strerror(rc));
         }
+        pmix_list_remove_item(&epi->cleanup_files, &cf->super);
+        PMIX_RELEASE(cf);
     }
 
     /* now cleanup the directories */
-    PMIX_LIST_FOREACH(cd, &epi->cleanup_dirs, pmix_cleanup_dir_t) {
+    PMIX_LIST_FOREACH_SAFE(cd, cdnext, &epi->cleanup_dirs, pmix_cleanup_dir_t) {
         /* check the effective uid/gid of the file and ensure it
          * matches that of the peer - we do this to provide at least
          * some minimum level of protection */
@@ -378,6 +379,8 @@ static void cleanup(pmix_epilog_t *epi)
             pmix_output_verbose(10, pmix_globals.debug_output,
                                 "Directory %s lacks permissions", cd->path);
         }
+        pmix_list_remove_item(&epi->cleanup_dirs, &cd->super);
+        PMIX_RELEASE(cd);
     }
 }
 

opal/mca/pmix/pmix3x/pmix/src/include/pmix_globals.h

@@ -10,7 +10,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -408,6 +408,8 @@ typedef struct {
     pmix_gds_base_module_t *mygds;
 } pmix_globals_t;
 
+/* provide access to a function to cleanup epilogs */
+PMIX_EXPORT void pmix_execute_epilog(pmix_epilog_t *ep);
 
 PMIX_EXPORT extern pmix_globals_t pmix_globals;
 PMIX_EXPORT extern pmix_lock_t pmix_global_lock;

opal/mca/pmix/pmix3x/pmix/src/server/pmix_server.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2014-2017 Intel, Inc.  All rights reserved.
+ * Copyright (c) 2014-2018 Intel, Inc.  All rights reserved.
  * Copyright (c) 2014-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2014-2015 Artem Y. Polyakov <artpol84@gmail.com>.
@@ -346,6 +346,7 @@ PMIX_EXPORT pmix_status_t PMIx_server_finalize(void)
 {
     int i;
     pmix_peer_t *peer;
+    pmix_nspace_t *ns;
 
     PMIX_ACQUIRE_THREAD(&pmix_global_lock);
     if (pmix_globals.init_cntr <= 0) {
@@ -359,7 +360,6 @@ PMIX_EXPORT pmix_status_t PMIx_server_finalize(void)
         return PMIX_SUCCESS;
     }
     pmix_globals.init_cntr = 0;
-    PMIX_RELEASE_THREAD(&pmix_global_lock);
 
     pmix_output_verbose(2, pmix_server_globals.base_output,
                         "pmix:server finalize called");
@@ -376,6 +376,10 @@ PMIX_EXPORT pmix_status_t PMIx_server_finalize(void)
 
     for (i=0; i < pmix_server_globals.clients.size; i++) {
         if (NULL != (peer = (pmix_peer_t*)pmix_pointer_array_get_item(&pmix_server_globals.clients, i))) {
+            /* ensure that we do the specified cleanup - if this is an
+             * abnormal termination, then the peer object may not be
+             * at zero refcount */
+            pmix_execute_epilog(&peer->epilog);
             PMIX_RELEASE(peer);
         }
     }
@@ -385,6 +389,12 @@ PMIX_EXPORT pmix_status_t PMIx_server_finalize(void)
     PMIX_LIST_DESTRUCT(&pmix_server_globals.local_reqs);
     PMIX_LIST_DESTRUCT(&pmix_server_globals.gdata);
     PMIX_LIST_DESTRUCT(&pmix_server_globals.events);
+    PMIX_LIST_FOREACH(ns, &pmix_server_globals.nspaces, pmix_nspace_t) {
+        /* ensure that we do the specified cleanup - if this is an
+         * abnormal termination, then the nspace object may not be
+         * at zero refcount */
+        pmix_execute_epilog(&ns->epilog);
+    }
     PMIX_LIST_DESTRUCT(&pmix_server_globals.nspaces);
 
     if (NULL != security_mode) {
@@ -406,6 +416,8 @@ PMIX_EXPORT pmix_status_t PMIx_server_finalize(void)
 
     pmix_output_verbose(2, pmix_server_globals.base_output,
                         "pmix:server finalize complete");
+    PMIX_RELEASE_THREAD(&pmix_global_lock);
+    PMIX_DESTRUCT_LOCK(&pmix_global_lock);
 
     return PMIX_SUCCESS;
 }