opal/common/ucx: Switch to opal/tsd

Signed-off-by: Artem Polyakov <artpol84@gmail.com>

ompi/mca/osc/ucx/osc_ucx_component.c

@@ -26,8 +26,6 @@
     memcpy(((char*)(_dst)) + (_off), _src, _len); \
     (_off) += (_len);
 
-static int dbg_level = 0;
-
 static int component_open(void);
 static int component_register(void);
 static int component_init(bool enable_progress_threads, bool enable_mpi_threads);

ompi/mca/osc/ucx/osc_ucx_passive_target.c

@@ -18,8 +18,6 @@
 
 OBJ_CLASS_INSTANCE(ompi_osc_ucx_lock_t, opal_object_t, NULL, NULL);
 
-static int dbg_level = 0;
-
 static inline int start_shared(ompi_osc_ucx_module_t *module, int target) {
     uint64_t result_value = -1;
     uint64_t remote_addr = (module->state_addrs)[target] + OSC_UCX_STATE_LOCK_OFFSET;

opal/mca/common/ucx/common_ucx_wpool.c

@@ -201,7 +201,7 @@ opal_common_ucx_wpool_init(opal_common_ucx_wpool_t *wpool,
         goto err_wpool_add;
     }
 
-    pthread_key_create(&wpool->tls_key, _tlocal_cleanup);
+    opal_tsd_key_create(&wpool->tls_key, _tlocal_cleanup);
 
     return rc;
 
@@ -229,7 +229,7 @@ void opal_common_ucx_wpool_finalize(opal_common_ucx_wpool_t *wpool)
 
     /* After this have been called no thread cleanup callback
      * will be called */
-    pthread_key_delete(wpool->tls_key);
+    opal_tsd_key_delete(wpool->tls_key);
 
     /* Go over remaining TLS structures and release it */
     OPAL_LIST_FOREACH_SAFE(tls_item, tls_next, &wpool->tls_list,
@@ -553,7 +553,7 @@ int opal_common_ucx_wpmem_create(opal_common_ucx_ctx_t *ctx,
 
     /* Dont need the destructor here, will use
      * wpool-level destructor */
-    pthread_key_create(&mem->mem_tls_key, NULL);
+    opal_tsd_key_create(&mem->mem_tls_key, NULL);
 
     (*mem_ptr) = mem;
     (*my_mem_addr) = rkey_addr;
@@ -641,7 +641,7 @@ static int _comm_ucx_wpmem_map(opal_common_ucx_wpool_t *wpool,
 
 static void _common_ucx_wpmem_free(opal_common_ucx_wpmem_t *mem)
 {
-    pthread_key_delete(mem->mem_tls_key);
+    opal_tsd_key_delete(mem->mem_tls_key);
     free(mem->mem_addrs);
     free(mem->mem_displs);
     ucp_mem_unmap(mem->ctx->wpool->ucp_ctx, mem->memh);
@@ -714,15 +714,21 @@ static _tlocal_table_t* _common_ucx_tls_init(opal_common_ucx_wpool_t *wpool)
         return NULL;
     }
 
-    pthread_setspecific(wpool->tls_key, tls);
+    opal_tsd_setspecific(wpool->tls_key, tls);
 
     return tls;
 }
 
 static inline _tlocal_table_t *
 _tlocal_get_tls(opal_common_ucx_wpool_t *wpool){
-    _tlocal_table_t *tls = pthread_getspecific(wpool->tls_key);
-    if( OPAL_UNLIKELY(NULL == tls) ) {
+    _tlocal_table_t *tls;
+    int rc = opal_tsd_getspecific(wpool->tls_key, (void**)&tls);
+
+    if (OPAL_SUCCESS != rc) {
+        return NULL;
+    }
+
+    if (OPAL_UNLIKELY(NULL == tls)) {
         tls = _common_ucx_tls_init(wpool);
     }
     return tls;
@@ -777,7 +783,7 @@ static void _common_ucx_tls_cleanup(_tlocal_table_t *tls)
         free(tls->ctx_tbl[i]);
     }
 
-    pthread_setspecific(tls->wpool->tls_key, NULL);
+    opal_tsd_setspecific(tls->wpool->tls_key, NULL);
 
     OBJ_RELEASE(tls);
     return;
@@ -1033,7 +1039,7 @@ static _tlocal_mem_t *_tlocal_add_mem(_tlocal_table_t *tls,
             calloc(1, sizeof(*tls->mem_tbl[free_idx]->mem_tls_ptr));
     tls->mem_tbl[free_idx]->mem_tls_ptr->winfo = ctx_rec->winfo;
     tls->mem_tbl[free_idx]->mem_tls_ptr->rkeys = tls->mem_tbl[free_idx]->mem->rkeys;
-    pthread_setspecific(mem->mem_tls_key, tls->mem_tbl[free_idx]->mem_tls_ptr);
+    opal_tsd_setspecific(mem->mem_tls_key, tls->mem_tbl[free_idx]->mem_tls_ptr);
 
     /* Make sure that we completed all the data structures before
      * placing the item to the list

opal/mca/common/ucx/common_ucx_wpool.h

@@ -9,13 +9,13 @@
 #include <string.h>
 
 #include <ucp/api/ucp.h>
-#include <pthread.h>
 
 #include "opal/mca/mca.h"
 #include "opal/util/output.h"
 #include "opal/runtime/opal_progress.h"
 #include "opal/include/opal/constants.h"
 #include "opal/class/opal_list.h"
+#include "opal/threads/tsd.h"
 
 BEGIN_C_DECLS
 
@@ -32,7 +32,7 @@ typedef struct {
 
     /* Thread-local key to allow each thread to have
      * local information assisiated with this wpool */
-    pthread_key_t tls_key;
+    opal_tsd_key_t tls_key;
 
     /* Bookkeeping information */
     opal_list_t idle_workers;
@@ -75,7 +75,7 @@ typedef struct {
     /* TLS item that allows each thread to
      * store endpoints and rkey arrays
      * for faster access */
-    pthread_key_t mem_tls_key;
+    opal_tsd_key_t mem_tls_key;
 } opal_common_ucx_wpmem_t;
 
 typedef struct opal_common_ucx_winfo {
@@ -159,7 +159,10 @@ opal_common_ucx_tlocal_fetch(opal_common_ucx_wpmem_t *mem, int target,
     int rc = OPAL_SUCCESS;
 
     /* First check the fast-path */
-    fp = pthread_getspecific(mem->mem_tls_key);
+    rc = opal_tsd_getspecific(mem->mem_tls_key, (void**)&fp);
+    if (OPAL_SUCCESS != rc) {
+        return rc;
+    }
     expr = fp && (NULL != fp->winfo) && (fp->winfo->endpoints[target]) &&
             (NULL != fp->rkeys[target]);
     if (OPAL_UNLIKELY(!expr)) {
@@ -167,7 +170,10 @@ opal_common_ucx_tlocal_fetch(opal_common_ucx_wpmem_t *mem, int target,
         if (OPAL_SUCCESS != rc) {
             return rc;
         }
-        fp = pthread_getspecific(mem->mem_tls_key);
+        rc = opal_tsd_getspecific(mem->mem_tls_key, (void**)&fp);
+        if (OPAL_SUCCESS != rc) {
+            return rc;
+        }
     }
     MCA_COMMON_UCX_ASSERT(fp && (NULL != fp->winfo) &&
                           (fp->winfo->endpoints[target])