opal/threads: manually invoke thread-specific key destructors on the main thread.

there is no such thing as pthread_join(main_thread), so key destructors
are never invoked on the main thread, which causes valgrind report
some memory leaks. Manually store and then invoke the key destructors and
make valgrind happy.

Signed-off-by: Gilles Gouaillardet <gilles@rist.or.jp>

opal/runtime/opal_finalize.c

@@ -13,7 +13,7 @@
  * Copyright (c) 2010-2015 Los Alamos National Security, LLC.
  *                         All rights reserved.
  * Copyright (c) 2013-2015 Intel, Inc. All rights reserved
- * Copyright (c) 2016      Research Organization for Information Science
+ * Copyright (c) 2016-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
  *
@@ -55,6 +55,7 @@
 
 #include "opal/runtime/opal_cr.h"
 #include "opal/mca/crs/base/base.h"
+#include "opal/threads/tsd.h"
 
 extern int opal_initialized;
 extern int opal_util_initialized;
@@ -160,6 +161,9 @@ opal_finalize(void)
     /* close the sec framework */
     (void) mca_base_framework_close(&opal_sec_base_framework);
 
+    /* cleanup the main thread specific stuff */
+    opal_tsd_keys_destruct();
+
     /* finalize util code */
     opal_finalize_util();
 

opal/runtime/opal_init.c

@@ -16,7 +16,7 @@
  * Copyright (c) 2010-2015 Los Alamos National Security, LLC.
  *                         All rights reserved.
  * Copyright (c) 2013-2016 Intel, Inc. All rights reserved
- * Copyright (c) 2015-2016 Research Organization for Information Science
+ * Copyright (c) 2015-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
  *
@@ -344,6 +344,8 @@ opal_init_util(int* pargc, char*** pargv)
         return OPAL_SUCCESS;
     }
 
+    opal_thread_set_main();
+
     opal_init_called = true;
 
     /* set the nodename right away so anyone who needs it has it. Note

opal/threads/thread.c

@@ -10,7 +10,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2010      Cisco Systems, Inc. All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
+ * Copyright (c) 2015-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
  *
@@ -22,12 +22,23 @@
 #include "opal_config.h"
 
 #include "opal/threads/threads.h"
+#include "opal/threads/tsd.h"
 #include "opal/constants.h"
 
 bool opal_debug_threads = false;
 
 static void opal_thread_construct(opal_thread_t *t);
 
+static pthread_t opal_main_thread;
+
+struct opal_tsd_key_value {
+    opal_tsd_key_t key;
+    opal_tsd_destructor_t destructor;
+};
+
+static struct opal_tsd_key_value *opal_tsd_key_values = NULL;
+static int opal_tsd_key_values_count = 0;
+
 OBJ_CLASS_INSTANCE(opal_thread_t,
                    opal_object_t,
                    opal_thread_construct, NULL);
@@ -83,3 +94,39 @@ void opal_thread_kill(opal_thread_t *t, int sig)
 {
     pthread_kill(t->t_handle, sig);
 }
+
+int opal_tsd_key_create(opal_tsd_key_t *key,
+                    opal_tsd_destructor_t destructor)
+{
+    int rc;
+    rc = pthread_key_create(key, destructor);
+    if ((0 == rc) && (pthread_self() == opal_main_thread)) {
+        opal_tsd_key_values = (struct opal_tsd_key_value *)realloc(opal_tsd_key_values, (opal_tsd_key_values_count+1) * sizeof(struct opal_tsd_key_value));
+        opal_tsd_key_values[opal_tsd_key_values_count].key = *key;
+        opal_tsd_key_values[opal_tsd_key_values_count].destructor = destructor;
+        opal_tsd_key_values_count ++;
+    }
+    return rc;
+}
+
+int opal_tsd_keys_destruct()
+{
+    int i;
+    void * ptr;
+    for (i=0; i<opal_tsd_key_values_count; i++) {
+        if(OPAL_SUCCESS == opal_tsd_getspecific(opal_tsd_key_values[i].key, &ptr)) {
+            if (NULL != opal_tsd_key_values[i].destructor) {
+                opal_tsd_key_values[i].destructor(ptr);
+            }
+        }
+    }
+    if (0 < opal_tsd_key_values_count) {
+        free(opal_tsd_key_values);
+        opal_tsd_key_values_count = 0;
+    }
+    return OPAL_SUCCESS;
+}
+
+void opal_thread_set_main() {
+    opal_main_thread = pthread_self();
+}

opal/threads/threads.h

@@ -11,7 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2010      Cisco Systems, Inc. All rights reserved.
  * Copyright (c) 2010      Oracle and/or its affiliates.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
+ * Copyright (c) 2015-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
  *
@@ -120,6 +120,7 @@ OPAL_DECLSPEC int  opal_thread_join(opal_thread_t *, void **thread_return);
 OPAL_DECLSPEC bool opal_thread_self_compare(opal_thread_t*);
 OPAL_DECLSPEC opal_thread_t *opal_thread_get_self(void);
 OPAL_DECLSPEC void opal_thread_kill(opal_thread_t *, int sig);
+OPAL_DECLSPEC void opal_thread_set_main(void);
 
 END_C_DECLS
 

opal/threads/tsd.h

@@ -2,7 +2,7 @@
  * Copyright (c) 2007-2013 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2008      Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2015      Research Organization for Information Science
+ * Copyright (c) 2015-2017 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * $COPYRIGHT$
  *
@@ -45,36 +45,6 @@ typedef void (*opal_tsd_destructor_t)(void *value);
 typedef void* opal_tsd_key_t;
 
 
-/**
- * Create thread-specific data key
- *
- * Create a thread-specific data key visible to all threads in the
- * current process.  The returned key is valid in all threads,
- * although the values bound to the key by opal_tsd_setspecific() are
- * allocated on a per-thread basis and persist for the life of the
- * calling thread.
- *
- * Upon key creation, the value NULL is associated with the new key in
- * all active threads.  When a new thread is created, the value NULL
- * is associated with all defined keys in the new thread.
- *
- * The destructor parameter may be NULL.  At thread exit, if
- * destructor is non-NULL AND the thread has a non-NULL value
- * associated with the key, the function is called with the current
- * value as its argument.
- *
- * @param key[out]       The key for accessing thread-specific data
- * @param destructor[in] Cleanup function to call when a thread exits
- *
- * @retval OPAL_SUCCESS  Success
- * @retval EAGAIN        The system lacked the necessary resource to
- *                       create another thread specific data key
- * @retval ENOMEM        Insufficient memory exists to create the key
- */
-OPAL_DECLSPEC int opal_tsd_key_create(opal_tsd_key_t *key,
-                                      opal_tsd_destructor_t destructor);
-
-
 /**
  * Delete a thread-specific data key
  *
@@ -138,13 +108,6 @@ OPAL_DECLSPEC int opal_tsd_getspecific(opal_tsd_key_t key, void **valuep);
 
 typedef pthread_key_t opal_tsd_key_t;
 
-static inline int
-opal_tsd_key_create(opal_tsd_key_t *key,
-                    opal_tsd_destructor_t destructor)
-{
-    return pthread_key_create(key, destructor);
-}
-
 static inline int
 opal_tsd_key_delete(opal_tsd_key_t key)
 {
@@ -166,6 +129,50 @@ opal_tsd_getspecific(opal_tsd_key_t key, void **valuep)
 
 #endif
 
+/**
+ * Create thread-specific data key
+ *
+ * Create a thread-specific data key visible to all threads in the
+ * current process.  The returned key is valid in all threads,
+ * although the values bound to the key by opal_tsd_setspecific() are
+ * allocated on a per-thread basis and persist for the life of the
+ * calling thread.
+ *
+ * Upon key creation, the value NULL is associated with the new key in
+ * all active threads.  When a new thread is created, the value NULL
+ * is associated with all defined keys in the new thread.
+ *
+ * The destructor parameter may be NULL.  At thread exit, if
+ * destructor is non-NULL AND the thread has a non-NULL value
+ * associated with the key, the function is called with the current
+ * value as its argument.
+ *
+ * @param key[out]       The key for accessing thread-specific data
+ * @param destructor[in] Cleanup function to call when a thread exits
+ *
+ * @retval OPAL_SUCCESS  Success
+ * @retval EAGAIN        The system lacked the necessary resource to
+ *                       create another thread specific data key
+ * @retval ENOMEM        Insufficient memory exists to create the key
+ */
+OPAL_DECLSPEC int opal_tsd_key_create(opal_tsd_key_t *key,
+                                      opal_tsd_destructor_t destructor);
+
+
+/**
+ * Destruct all thread-specific data keys
+ *
+ * Destruct all thread-specific data keys and invoke the destructor
+ *
+ * This should only be invoked in the main thread.
+ * This is made necessary since destructors are not invoked on the
+ * keys of the main thread, since there is no such thing as
+ * pthread_join(main_thread)
+ *
+ * @retval OPAL_SUCCESS  Success
+ */
+OPAL_DECLSPEC int opal_tsd_keys_destruct(void);
+
 END_C_DECLS
 
 #endif /* OPAL_MTHREADS_TSD_H */