A more Windows friendly version. As the socket event will be generated

through the win dll using multiple threads, we have to insure that
the oob callbacks happens only in a synchronous way or really bad
things happens with the current design (blocking messages from a receive
callback).

This commit was SVN r15069.

orte/mca/oob/tcp/oob_tcp.c

@@ -347,6 +347,31 @@ int mca_oob_tcp_component_open(void)
     return ORTE_SUCCESS;
 }
 
+#if defined(__WINDOWS__)
+static int oob_tcp_windows_progress_callback( void )
+{
+    opal_list_item_t* item;
+    mca_oob_tcp_msg_t* msg;
+
+    OPAL_THREAD_LOCK(&mca_oob_tcp_component.tcp_lock);
+    while(NULL != 
+         (item = opal_list_remove_first(&mca_oob_tcp_component.tcp_msg_completed))) {
+        msg = (mca_oob_tcp_msg_t*)item;
+        OPAL_THREAD_UNLOCK(&mca_oob_tcp_component.tcp_lock);
+        msg->msg_cbfunc( msg->msg_rc, 
+                         &msg->msg_peer, 
+                         msg->msg_uiov, 
+                         msg->msg_ucnt, 
+                         msg->msg_hdr.msg_tag, 
+                         msg->msg_cbdata);
+        OPAL_THREAD_LOCK(&mca_oob_tcp_component.tcp_lock);
+        MCA_OOB_TCP_MSG_RETURN(msg);
+    }
+    OPAL_THREAD_UNLOCK(&mca_oob_tcp_component.tcp_lock);
+
+    return 0;
+}
+#endif  /* defined(__WINDOWS__) */
 
 /*
  * Cleanup of global variables used by this module.
@@ -354,9 +379,10 @@ int mca_oob_tcp_component_open(void)
 
 int mca_oob_tcp_component_close(void)
 {
-#ifdef __WINDOWS__
+#if defined(__WINDOWS__)
+    opal_progress_unregister(oob_tcp_windows_progress_callback);
     WSACleanup();
-#endif
+#endif  /* defined(__WINDOWS__) */
 
     /* cleanup resources */
 
@@ -1010,11 +1036,16 @@ mca_oob_t* mca_oob_tcp_component_init(int* priority)
     memset(&mca_oob_tcp_component.tcp6_recv_event, 0, sizeof(opal_event_t));
     memset(&mca_oob_tcp_component.tcp6_send_event, 0, sizeof(opal_event_t));
 #endif
-                        
+
+#if defined(__WINDOWS__)
+    /* Register the libevent callback which will trigger the OOB
+     * completion callbacks. */
+     opal_progress_register(oob_tcp_windows_progress_callback);
+#endif  /* defined(__WINDOWS__) */
+
     return &mca_oob_tcp;
 }
 
-
 /*
  * Callback from registry on change to subscribed segments.
  */

orte/mca/oob/tcp/oob_tcp_msg.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2007 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
@@ -186,6 +186,19 @@ int mca_oob_tcp_msg_complete(mca_oob_tcp_msg_t* msg, orte_process_name_t * peer)
         /* post to a global list of completed messages */
         OPAL_THREAD_LOCK(&mca_oob_tcp_component.tcp_lock);
         opal_list_append(&mca_oob_tcp_component.tcp_msg_completed, (opal_list_item_t*)msg);
+#if defined(__WINDOWS__)
+        /**
+         * In order to be able to generate TCP events reccursively, Windows need
+         * to get out of the callback attached to a specific socket. Therefore,
+         * as our OOB allow to block on a connection from a callback on the same
+         * connection, we have to trigger the completion callbacks outside of the
+         * OOB callbacks. We add them to the completed list here, and the progress
+         * engine will call our progress function later once all socket related
+         * callbacks have been triggered.
+         */
+         OPAL_THREAD_UNLOCK(&mca_oob_tcp_component.tcp_lock);
+         return ORTE_SUCCESS;        
+#else
         if(opal_list_get_size(&mca_oob_tcp_component.tcp_msg_completed) > 1) {
             OPAL_THREAD_UNLOCK(&mca_oob_tcp_component.tcp_lock);
             return ORTE_SUCCESS;
@@ -214,7 +227,7 @@ int mca_oob_tcp_msg_complete(mca_oob_tcp_msg_t* msg, orte_process_name_t * peer)
             MCA_OOB_TCP_MSG_RETURN(msg);
         }
         OPAL_THREAD_UNLOCK(&mca_oob_tcp_component.tcp_lock);
-
+#endif  /* defined(__WINDOWS__) */
     } else {
         opal_condition_broadcast(&msg->msg_condition);
         OPAL_THREAD_UNLOCK(&msg->msg_lock);