dont allow callbacks to processed recursively - appear to be blowing away the stack

This commit was SVN r7862.

orte/mca/oob/tcp/oob_tcp.c

@@ -177,7 +177,7 @@ int mca_oob_tcp_component_open(void)
 
     OBJ_CONSTRUCT(&mca_oob_tcp_component.tcp_subscriptions, opal_list_t);
     OBJ_CONSTRUCT(&mca_oob_tcp_component.tcp_peer_list,     opal_list_t);
-    OBJ_CONSTRUCT(&mca_oob_tcp_component.tcp_peers,     opal_hash_table_t);
+    OBJ_CONSTRUCT(&mca_oob_tcp_component.tcp_peers,         opal_hash_table_t);
     OBJ_CONSTRUCT(&mca_oob_tcp_component.tcp_peer_names,    opal_hash_table_t);
     OBJ_CONSTRUCT(&mca_oob_tcp_component.tcp_peer_free,     opal_free_list_t);
     OBJ_CONSTRUCT(&mca_oob_tcp_component.tcp_msgs,          opal_free_list_t);
@@ -185,6 +185,7 @@ int mca_oob_tcp_component_open(void)
     OBJ_CONSTRUCT(&mca_oob_tcp_component.tcp_events,        opal_list_t);
     OBJ_CONSTRUCT(&mca_oob_tcp_component.tcp_msg_post,      opal_list_t);
     OBJ_CONSTRUCT(&mca_oob_tcp_component.tcp_msg_recv,      opal_list_t);
+    OBJ_CONSTRUCT(&mca_oob_tcp_component.tcp_msg_completed, opal_list_t);
     OBJ_CONSTRUCT(&mca_oob_tcp_component.tcp_match_lock,    opal_mutex_t);
     OBJ_CONSTRUCT(&mca_oob_tcp_component.tcp_match_cond,    opal_condition_t);
 
@@ -228,6 +229,7 @@ int mca_oob_tcp_component_close(void)
     OBJ_DESTRUCT(&mca_oob_tcp_component.tcp_lock);
     OBJ_DESTRUCT(&mca_oob_tcp_component.tcp_msg_post);
     OBJ_DESTRUCT(&mca_oob_tcp_component.tcp_msg_recv);
+    OBJ_DESTRUCT(&mca_oob_tcp_component.tcp_msg_completed);
     OBJ_DESTRUCT(&mca_oob_tcp_component.tcp_match_lock);
     OBJ_DESTRUCT(&mca_oob_tcp_component.tcp_match_cond);
     return OMPI_SUCCESS;

orte/mca/oob/tcp/oob_tcp.h

@@ -323,6 +323,7 @@ struct mca_oob_tcp_component_t {
     opal_list_t        tcp_events;           /**< list of pending events (accepts) */
     opal_list_t        tcp_msg_post;         /**< list of recieves user has posted */
     opal_list_t        tcp_msg_recv;         /**< list of recieved messages */
+    opal_list_t        tcp_msg_completed;    /**< list of completed messages */
     opal_mutex_t       tcp_match_lock;       /**< lock held while searching/posting messages */
     opal_condition_t   tcp_match_cond;       /**< condition variable used in finalize */
     int                tcp_match_count;      /**< number of matched recvs in progress */

orte/mca/oob/tcp/oob_tcp_msg.c

@@ -147,15 +147,47 @@ int mca_oob_tcp_msg_timedwait(mca_oob_tcp_msg_t* msg, int* rc, struct timespec*
  */
 int mca_oob_tcp_msg_complete(mca_oob_tcp_msg_t* msg, orte_process_name_t * peer)
 {
-    opal_mutex_lock(&msg->msg_lock);
+    OPAL_THREAD_LOCK(&msg->msg_lock);
     msg->msg_complete = true;
     if(NULL != msg->msg_cbfunc) {
+        opal_list_item_t* item;
+        OPAL_THREAD_UNLOCK(&msg->msg_lock);
+
+        /* post to a global list of completed messages */
+        OPAL_THREAD_LOCK(&mca_oob_tcp_component.tcp_lock);
+        opal_list_append(&mca_oob_tcp_component.tcp_msg_completed, (opal_list_item_t*)msg);
+        if(opal_list_get_size(&mca_oob_tcp_component.tcp_msg_completed) > 1) {
+            OPAL_THREAD_UNLOCK(&mca_oob_tcp_component.tcp_lock);
+            return OMPI_SUCCESS;
+        }
+        OPAL_THREAD_UNLOCK(&mca_oob_tcp_component.tcp_lock);
+
+        /* invoke message callback */
         msg->msg_cbfunc(msg->msg_rc, peer, msg->msg_uiov, msg->msg_ucnt, msg->msg_hdr.msg_tag, msg->msg_cbdata);
-        opal_mutex_unlock(&msg->msg_lock);
+
+        /* dispatch any completed events */
+        OPAL_THREAD_LOCK(&mca_oob_tcp_component.tcp_lock);
+        opal_list_remove_item(&mca_oob_tcp_component.tcp_msg_completed, (opal_list_item_t*)msg);
         MCA_OOB_TCP_MSG_RETURN(msg);
+        while(NULL != 
+            (item = opal_list_remove_first(&mca_oob_tcp_component.tcp_msg_completed))) {
+            msg = (mca_oob_tcp_msg_t*)item;
+            OPAL_THREAD_UNLOCK(&mca_oob_tcp_component.tcp_lock);
+            msg->msg_cbfunc(
+                msg->msg_rc, 
+                &msg->msg_peer, 
+                msg->msg_uiov, 
+                msg->msg_ucnt, 
+                msg->msg_hdr.msg_tag, 
+                msg->msg_cbdata);
+            MCA_OOB_TCP_MSG_RETURN(msg);
+            OPAL_THREAD_LOCK(&mca_oob_tcp_component.tcp_lock);
+        }
+        OPAL_THREAD_UNLOCK(&mca_oob_tcp_component.tcp_lock);
+
     } else {
         opal_condition_broadcast(&msg->msg_condition);
-        opal_mutex_unlock(&msg->msg_lock);
+        OPAL_THREAD_UNLOCK(&msg->msg_lock);
     }
     return OMPI_SUCCESS;
 }