Protect the tcp_endpoints list from concurrent accesses.

Thanks Gilles for your help.

Signed-off-by: George Bosilca <bosilca@icl.utk.edu>

opal/mca/btl/tcp/btl_tcp.c

@@ -50,7 +50,8 @@ mca_btl_tcp_module_t mca_btl_tcp_module = {
         .btl_put = mca_btl_tcp_put,
         .btl_dump = mca_btl_base_dump,
         .btl_ft_event = mca_btl_tcp_ft_event
-    }
+    },
+    .tcp_endpoints_mutex = OPAL_MUTEX_STATIC_INIT
 };
 
 /**
@@ -122,7 +123,9 @@ int mca_btl_tcp_add_procs( struct mca_btl_base_module_t* btl,
                 continue;
             }
 
+            OPAL_THREAD_LOCK(&tcp_btl->tcp_endpoints_mutex);
             opal_list_append(&tcp_btl->tcp_endpoints, (opal_list_item_t*)tcp_endpoint);
+            OPAL_THREAD_UNLOCK(&tcp_btl->tcp_endpoints_mutex);
         }
 
         OPAL_THREAD_UNLOCK(&tcp_proc->proc_lock);
@@ -143,12 +146,14 @@ int mca_btl_tcp_add_procs( struct mca_btl_base_module_t* btl,
 }
 
 int mca_btl_tcp_del_procs(struct mca_btl_base_module_t* btl,
-        size_t nprocs,
+                          size_t nprocs,
-        struct opal_proc_t **procs,
+                          struct opal_proc_t **procs,
-        struct mca_btl_base_endpoint_t ** endpoints)
+                          struct mca_btl_base_endpoint_t ** endpoints)
 {
     mca_btl_tcp_module_t* tcp_btl = (mca_btl_tcp_module_t*)btl;
     size_t i;
+
+    OPAL_THREAD_LOCK(&tcp_btl->tcp_endpoints_mutex);
     for( i = 0; i < nprocs; i++ ) {
         mca_btl_tcp_endpoint_t* tcp_endpoint = endpoints[i];
         if(tcp_endpoint->endpoint_proc != mca_btl_tcp_proc_local()) {
@@ -157,6 +162,7 @@ int mca_btl_tcp_del_procs(struct mca_btl_base_module_t* btl,
         }
         opal_progress_event_users_decrement();
     }
+    OPAL_THREAD_UNLOCK(&tcp_btl->tcp_endpoints_mutex);
     return OPAL_SUCCESS;
 }
 
@@ -479,6 +485,10 @@ int mca_btl_tcp_finalize(struct mca_btl_base_module_t* btl)
 {
     mca_btl_tcp_module_t* tcp_btl = (mca_btl_tcp_module_t*) btl;
     opal_list_item_t* item;
+
+    /* Don't lock the tcp_endpoints_mutex, at this point a single
+     * thread should be active.
+     */
     for( item = opal_list_remove_first(&tcp_btl->tcp_endpoints);
          item != NULL;
          item = opal_list_remove_first(&tcp_btl->tcp_endpoints)) {
@@ -512,11 +522,13 @@ void mca_btl_tcp_dump(struct mca_btl_base_module_t* base_btl,
     } else if( verbose ) {
         opal_list_item_t *item;
 
+        OPAL_THREAD_LOCK(&btl->tcp_endpoints_mutex);
         for(item =  opal_list_get_first(&btl->tcp_endpoints);
             item != opal_list_get_end(&btl->tcp_endpoints);
             item = opal_list_get_next(item)) {
             MCA_BTL_TCP_ENDPOINT_DUMP(10, (mca_btl_base_endpoint_t*)item, false, "TCP");
         }
+        OPAL_THREAD_UNLOCK(&btl->tcp_endpoints_mutex);
     }
 #endif /* OPAL_ENABLE_DEBUG && WANT_PEER_DUMP */
 }

opal/mca/btl/tcp/btl_tcp.h

@@ -170,7 +170,10 @@ struct mca_btl_tcp_module_t {
 #endif
     struct sockaddr_storage tcp_ifaddr; /**< BTL interface address */
     uint32_t           tcp_ifmask;  /**< BTL interface netmask */
+
+    opal_mutex_t       tcp_endpoints_mutex;
     opal_list_t        tcp_endpoints;
+
     mca_btl_base_module_error_cb_fn_t tcp_error_cb;  /**< Upper layer error callback */
 #if MCA_BTL_TCP_STATISTICS
     size_t tcp_bytes_sent;

opal/mca/btl/tcp/btl_tcp_component.c

@@ -474,6 +474,7 @@ static int mca_btl_tcp_create(int if_kindex, const char* if_name)
             return OPAL_ERR_OUT_OF_RESOURCE;
         memcpy(btl, &mca_btl_tcp_module, sizeof(mca_btl_tcp_module));
         OBJ_CONSTRUCT(&btl->tcp_endpoints, opal_list_t);
+        OBJ_CONSTRUCT(&btl->tcp_endpoints_mutex, opal_mutex_t);
         mca_btl_tcp_component.tcp_btls[mca_btl_tcp_component.tcp_num_btls++] = btl;
 
         /* initialize the btl */