* AC_CONFIG_LINKS has a minor problem in that it always uses ln -s, rather

than $(LN_S).  This causes problems with with Windows and probably
  elsewhere (re: #200).  So use a slightly different trick to get the
  right header selected for the MEMCPY and TIMER components.

* Using the same trick used to solve the AC_CONFIG_LINKS problem, 
  stop using a separate header file for direct calling in the
  PML and MTL.  This lets me remove some icky code in ompi_mca.m4
  that was more fragile than I really liked.

This commit was SVN r10841.

config/ompi_mca.m4

@@ -717,7 +717,7 @@ AC_DEFUN([MCA_PROCESS_COMPONENT],[
         if test "$DIRECT_$2" = "$component" ; then
             if test "`grep DIRECT_CALL_HEADER $infile`" != "" ; then
                 line="`grep DIRECT_CALL_HEADER $infile | cut -d= -f2-`"
-                str="MCA_${framework}_DIRECT_CALL_HEADER=\\\"$line\\\""
+                str="MCA_${framework}_DIRECT_CALL_HEADER=$line"
                 eval $str
             else
 AC_MSG_ERROR([*** ${framework} component ${component} was supposed to be direct-called, but
@@ -886,24 +886,9 @@ AC_DEFUN([MCA_SETUP_DIRECT_CALL],[
             [Defined to 1 if $1 should use direct calls instead of components])
     AC_DEFINE_UNQUOTED([MCA_$1_DIRECT_CALL_COMPONENT], [$MCA_$1_DIRECT_CALL_COMPONENT],
             [name of component to use for direct calls, if MCA_$1_DIRECT_CALL is 1])
-    MCA_WRITE_DIRECT_CALL_HEADER($1, $2)
+    AC_DEFINE_UNQUOTED([MCA_$1_DIRECT_CALL_HEADER],
-])
+                       ["[$MCA_]$1[_DIRECT_CALL_HEADER]"],
-AC_DEFUN([MCA_WRITE_DIRECT_CALL_HEADER],[
+                       [Header $1 includes to be direct called])
-    AC_CONFIG_FILES($2/mca/$1/$1_direct_call.h.tmp:$2/mca/$1/$1_direct_call.h.in)
-    AC_CONFIG_COMMANDS($1-direct,
-[if test -f "$2/mca/$1/$1_direct_call"; then
-  diff "$2/mca/$1/$1_direct_call.h" "$2/mca/$1/$1_direct_call.h.tmp" > /dev/null 2>&1
-  if test "$?" != "0"; then
-    cp "$2/mca/$1/$1_direct_call.h.tmp" "$2/mca/$1/$1_direct_call.h"
-    echo "config.status: regenerating $2/mca/$1/$1_direct_call.h"
-  else
-    echo "config.status: $2/mca/$1/$1_direct_call.h unchanged"
-  fi
-else
-  cp "$2/mca/$1/$1_direct_call.h.tmp" "$2/mca/$1/$1_direct_call.h"
-  echo "config.status: creating $2/mca/$1/$1_direct_call.h"
-fi
-rm $2/mca/$1/$1_direct_call.h.tmp])
 ])
 
 

ompi/mca/mtl/Makefile.am

@@ -26,7 +26,6 @@ nobase_nodist_ompi_HEADERS =
 
 # local files
 headers = mtl.h
-nodist_headers = 
 libmca_mtl_la_SOURCES += $(headers) $(nodist_headers)
 
 # Conditionally install the header files
@@ -41,4 +40,4 @@ endif
 include base/Makefile.am
 
 distclean-local:
-	rm -f base/static-components.h mtl_direct_call.h
+	rm -f base/static-components.h

ompi/mca/mtl/mtl.h

@@ -395,7 +395,7 @@ typedef struct mca_mtl_base_module_t mca_mtl_base_module_t;
  */
 #if MCA_mtl_DIRECT_CALL
 
-#include "ompi/mca/mtl/mtl_direct_call.h"
+#include MCA_mtl_DIRECT_CALL_HEADER
 
 #define OMPI_MTL_CALL_STAMP(a, b) ompi_mtl_ ## a ## _ ## b
 #define OMPI_MTL_CALL_EXPANDER(a, b) OMPI_MTL_CALL_STAMP(a,b)

ompi/mca/mtl/mtl_direct_call.h.in

@@ -1,20 +0,0 @@
-/*
- * Copyright (c) 2006 The Regents of the University of California.
- *                    All rights reserved.
- *
- * $COPYRIGHT$
- * 
- * Additional copyrights may follow
- * 
- * $HEADER$
- */
-
-
-#ifndef MCA_MTL_DIRECT_CALL_H_
-#define MCA_MTL_DIRECT_CALL_H_
-
-#if MCA_mtl_DIRECT_CALL
-#include @MCA_mtl_DIRECT_CALL_HEADER@
-#endif
-
-#endif

ompi/mca/pml/Makefile.am

@@ -26,7 +26,6 @@ nobase_nodist_ompi_HEADERS =
 
 # local files
 headers = pml.h
-nodist_headers = pml_direct_call.h
 libmca_pml_la_SOURCES += $(headers) $(nodist_headers)
 
 # Conditionally install the header files
@@ -41,4 +40,4 @@ endif
 include base/Makefile.am
 
 distclean-local:
-	rm -f base/static-components.h pml_direct_call.h
+	rm -f base/static-components.h 

ompi/mca/pml/pml.h

@@ -530,7 +530,7 @@ typedef mca_pml_base_module_1_0_0_t mca_pml_base_module_t;
      */
 #if MCA_pml_DIRECT_CALL
 
-#include "ompi/mca/pml/pml_direct_call.h"
+#include MCA_pml_DIRECT_CALL_HEADER
 
 #define MCA_PML_CALL_STAMP(a, b) mca_pml_ ## a ## _ ## b
 #define MCA_PML_CALL_EXPANDER(a, b) MCA_PML_CALL_STAMP(a,b)

ompi/mca/pml/pml_direct_call.h.in

@@ -1,27 +0,0 @@
-/*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
- *                         University Research and Technology
- *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
- *                         of Tennessee Research Foundation.  All rights
- *                         reserved.
- * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
- *                         University of Stuttgart.  All rights reserved.
- * Copyright (c) 2004-2005 The Regents of the University of California.
- *                         All rights reserved.
- * $COPYRIGHT$
- * 
- * Additional copyrights may follow
- * 
- * $HEADER$
- */
-
-
-#ifndef MCA_PML_DIRECT_CALL_H_
-#define MCA_PML_DIRECT_CALL_H_
-
-#if MCA_pml_DIRECT_CALL
-#include @MCA_pml_DIRECT_CALL_HEADER@
-#endif
-
-#endif

opal/class/opal_free_list.h

@@ -124,7 +124,7 @@ OMPI_DECLSPEC int opal_free_list_grow(opal_free_list_t* flist, size_t num_elemen
 #define OPAL_FREE_LIST_WAIT(fl, item, rc)                                  \
 {                                                                          \
     OPAL_THREAD_LOCK(&((fl)->fl_lock));                                    \
-    while( NULL == (item = opal_list_remove_first(&((fl)->super))) ) {     \
+    while( NULL == (item = (opal_free_list_item_t*) opal_list_remove_first(&((fl)->super))) ) { \
         if((fl)->fl_max_to_alloc <= (fl)->fl_num_allocated) {              \
             (fl)->fl_num_waiting++;                                        \
             opal_condition_wait(&((fl)->fl_condition), &((fl)->fl_lock));  \
@@ -149,7 +149,7 @@ OMPI_DECLSPEC int opal_free_list_grow(opal_free_list_t* flist, size_t num_elemen
 #define OPAL_FREE_LIST_RETURN(fl, item)                                    \
 {                                                                          \
     OPAL_THREAD_LOCK(&(fl)->fl_lock);                                      \
-    opal_list_prepend(&((fl)->super), (item));                            \
+    opal_list_prepend(&((fl)->super), ((opal_list_item_t*) item));      \
     if((fl)->fl_num_waiting > 0) {                                         \
         opal_condition_signal(&((fl)->fl_condition));                      \
     }                                                                      \

opal/mca/memcpy/base/Makefile.am

@@ -16,5 +16,3 @@ headers += \
 libmca_memcpy_la_SOURCES += \
         base/memcpy_base_close.c \
         base/memcpy_base_open.c
-
-nodist_headers += base/base_impl.h

opal/mca/memcpy/base/base.h

@@ -69,6 +69,6 @@ extern "C" {
 #endif
 
 /* include implementation to call */
-#include "opal/mca/memcpy/base/base_impl.h"
+#include MCA_timer_IMPLEMENTATION_HEADER
 
 #endif /* OPAL_BASE_MEMCPY_H */

opal/mca/memcpy/configure.m4

@@ -3,6 +3,8 @@ dnl
 dnl Copyright (c) 2004-2006 The University of Tennessee and The University
 dnl                         of Tennessee Research Foundation.  All rights
 dnl                         reserved.
+dnl Copyright (c) 2004-2006 The Regents of the University of California.
+dnl                         All rights reserved.
 dnl $COPYRIGHT$
 dnl 
 dnl Additional copyrights may follow
@@ -10,6 +12,7 @@ dnl
 dnl $HEADER$
 dnl
 
+dnl we only want one
 m4_define(MCA_memcpy_CONFIGURE_MODE, STOP_AT_FIRST)
 
 AC_DEFUN([MCA_memcpy_CONFIG],[
@@ -23,5 +26,7 @@ AC_DEFUN([MCA_memcpy_CONFIG],[
             memcpy_base_include="base/memcpy_base_default.h"
         fi
 
-        AC_CONFIG_LINKS([opal/mca/memcpy/base/base_impl.h:opal/mca/memcpy/$memcpy_base_include])
+        AC_DEFINE_UNQUOTED([MCA_memcpy_IMPLEMENTATION_HEADER],
+                           ["opal/mca/memcpy/$memcpy_base_include"],
+                           [Header to include for memcpy implementation])
 ])

opal/mca/timer/base/Makefile.am

@@ -23,5 +23,3 @@ headers += \
 libmca_timer_la_SOURCES += \
         base/timer_base_close.c \
         base/timer_base_open.c
-
-nodist_headers += base/base_impl.h

opal/mca/timer/base/base.h

@@ -76,6 +76,6 @@ extern "C" {
 #endif
 
 /* include implementation to call */
-#include "opal/mca/timer/base/base_impl.h"
+#include MCA_timer_IMPLEMENTATION_HEADER
 
 #endif /* OPAL_BASE_TIMER_H */

opal/mca/timer/configure.m4

@@ -31,5 +31,7 @@ AC_DEFUN([MCA_timer_CONFIG],[
             timer_base_include="base/timer_base_null.h"
         fi
 
-        AC_CONFIG_LINKS([opal/mca/timer/base/base_impl.h:opal/mca/timer/$timer_base_include])
+        AC_DEFINE_UNQUOTED([MCA_timer_IMPLEMENTATION_HEADER],
+                           ["opal/mca/timer/$timer_base_include"],
+                           [Header to include for timer implementation])
 ])

orte/mca/oob/tcp/oob_tcp.c

@@ -83,6 +83,7 @@ OBJ_CLASS_INSTANCE(
  */
 
 static int  mca_oob_tcp_create_listen(void);
+static int  mca_oob_tcp_create_listen_thread(void);
 static void mca_oob_tcp_recv_handler(int sd, short flags, void* user);
 static void mca_oob_tcp_accept(void);
 
@@ -169,6 +170,8 @@ static inline char* mca_oob_tcp_param_register_str(
  */
 int mca_oob_tcp_component_open(void)
 {
+    char *listen_type;
+
 #ifdef __WINDOWS__
     WSADATA win_sock_data;
     if (WSAStartup(MAKEWORD(2,2), &win_sock_data) != 0) {
@@ -207,7 +210,26 @@ int mca_oob_tcp_component_open(void)
     mca_oob_tcp_component.tcp_rcvbuf =
         mca_oob_tcp_param_register_int("rcvbuf", 128*1024);
 
+    mca_base_param_reg_string(&mca_oob_tcp_component.super.oob_base,
+                              "listen_mode",
+                              "Mode for HNP to accept incoming connections: event, listen_thread",
+                              false,
+                              false,
+                              "event",
+                              &listen_type);
+
+    if (0 == strcmp(listen_type, "event")) {
+        mca_oob_tcp_component.tcp_listen_type = OOB_TCP_EVENT;
+    } else if (0 == strcmp(listen_type, "listen_thread")) {
+        mca_oob_tcp_component.tcp_listen_type = OOB_TCP_LISTEN_THREAD;
+    } else {
+        opal_output(0, "Invalid value for oob_tcp_listen_mode parameter: %s",
+                    listen_type);
+        return ORTE_ERROR;
+    }
+
     /* initialize state */
+    mca_oob_tcp_component.tcp_shutdown = false;
     mca_oob_tcp_component.tcp_listen_sd = -1;
     mca_oob_tcp_component.tcp_match_count = 0;
     return ORTE_SUCCESS;
@@ -352,6 +374,131 @@ static int mca_oob_tcp_create_listen(void)
 }
 
 
+static void* mca_oob_tcp_listen_thread(opal_object_t *obj)
+{
+    int sd, rc;
+    ompi_socklen_t addrlen = sizeof(struct sockaddr_in);
+    struct sockaddr_in addr;
+    opal_free_list_item_t *fl_item;
+    mca_oob_tcp_pending_connection_t *item;
+
+    while (false == mca_oob_tcp_component.tcp_shutdown) {
+        sd = accept(mca_oob_tcp_component.tcp_listen_sd, 
+                    (struct sockaddr*)&addr, &addrlen);
+        if(sd < 0) {
+            if (mca_oob_tcp_component.tcp_shutdown) return NULL;
+
+            if(ompi_socket_errno == EINTR)
+                continue;
+            if(ompi_socket_errno != EAGAIN || ompi_socket_errno != EWOULDBLOCK)
+                opal_output(0, "mca_oob_tcp_accept: accept() failed with errno %d.", ompi_socket_errno);
+            close(sd);
+            /* start a new listen socket */
+            mca_oob_tcp_create_listen_thread();
+            return NULL;
+        }
+
+        OPAL_FREE_LIST_WAIT(&mca_oob_tcp_component.tcp_pending_connections_fl, 
+                            fl_item, rc);
+        item = (mca_oob_tcp_pending_connection_t*) fl_item;
+        item->fd = sd;
+        opal_mutex_lock(&mca_oob_tcp_component.tcp_pending_connections_lock);
+        opal_list_append(&mca_oob_tcp_component.tcp_pending_connections,
+                         (opal_list_item_t*) item);
+        opal_mutex_unlock(&mca_oob_tcp_component.tcp_pending_connections_lock);
+    }
+
+    return NULL;
+}
+
+static int mca_oob_tcp_listen_progress(void)
+{
+    int count = 0;
+    mca_oob_tcp_pending_connection_t *item;
+    struct sockaddr_in addr;
+    mca_oob_tcp_event_t* event;
+
+    if (opal_list_get_size(&mca_oob_tcp_component.tcp_pending_connections)) {
+        opal_mutex_lock(&mca_oob_tcp_component.tcp_pending_connections_lock);
+        while (NULL != (item = (opal_list_item_t*) 
+                        opal_list_remove_first(&mca_oob_tcp_component.tcp_pending_connections))) {
+
+            /* setup socket options */
+            mca_oob_tcp_set_socket_options(item->fd);
+
+            /* log the accept */
+            if(mca_oob_tcp_component.tcp_debug) {
+                opal_output(0, "[%lu,%lu,%lu] mca_oob_tcp_accept: %s:%d\n",
+                            ORTE_NAME_ARGS(orte_process_info.my_name),
+                            inet_ntoa(addr.sin_addr),
+                            addr.sin_port);
+            }
+
+            /* wait for receipt of peers process identifier to
+               complete this connection */
+            event = OBJ_NEW(mca_oob_tcp_event_t);
+            opal_event_set(&event->event, item->fd, OPAL_EV_READ, mca_oob_tcp_recv_handler, event);
+            opal_event_add(&event->event, 0);
+            OPAL_FREE_LIST_RETURN(&mca_oob_tcp_component.tcp_pending_connections_fl, 
+                                (opal_free_list_item_t *) item);
+
+            count++;
+        }
+        opal_mutex_unlock(&mca_oob_tcp_component.tcp_pending_connections_lock);
+    }
+
+    return count;
+}
+
+
+static int mca_oob_tcp_create_listen_thread(void)
+{
+    struct sockaddr_in inaddr;
+    ompi_socklen_t addrlen;
+
+    /* create a listen socket for incoming connections */
+    mca_oob_tcp_component.tcp_listen_sd = socket(AF_INET, SOCK_STREAM, 0);
+    if(mca_oob_tcp_component.tcp_listen_sd < 0) {
+        opal_output(0,"mca_oob_tcp_component_init: socket() failed with errno=%d", ompi_socket_errno);
+        return ORTE_ERROR;
+    }
+
+    /* setup socket options */
+    mca_oob_tcp_set_socket_options(mca_oob_tcp_component.tcp_listen_sd);
+
+    /* bind address */
+    memset(&inaddr, 0, sizeof(inaddr));
+    inaddr.sin_family = AF_INET;
+    inaddr.sin_addr.s_addr = INADDR_ANY;
+    inaddr.sin_port = 0;
+
+    if(bind(mca_oob_tcp_component.tcp_listen_sd, (struct sockaddr*)&inaddr, sizeof(inaddr)) < 0) {
+        opal_output(0,"mca_oob_tcp_create_listen: bind() failed with errno=%d", ompi_socket_errno);
+        return ORTE_ERROR;
+    }
+
+    /* resolve system assigned port */
+    addrlen = sizeof(struct sockaddr_in);
+    if(getsockname(mca_oob_tcp_component.tcp_listen_sd, (struct sockaddr*)&inaddr, &addrlen) < 0) {
+        opal_output(0, "mca_oob_tcp_create_listen: getsockname() failed with errno=%d", ompi_socket_errno);
+        return ORTE_ERROR;
+    }
+    mca_oob_tcp_component.tcp_listen_port = inaddr.sin_port;
+
+    /* setup listen backlog to maximum allowed by kernel */
+    if(listen(mca_oob_tcp_component.tcp_listen_sd, SOMAXCONN) < 0) {
+        opal_output(0, "mca_oob_tcp_component_init: listen() failed with errno=%d", ompi_socket_errno);
+        return ORTE_ERROR;
+    }
+
+    /* start the listen thread */
+    mca_oob_tcp_component.tcp_listen_thread.t_run = mca_oob_tcp_listen_thread;
+    mca_oob_tcp_component.tcp_listen_thread.t_arg = NULL;
+
+    return opal_thread_start(&mca_oob_tcp_component.tcp_listen_thread);
+}
+
+
 /*
  * Handle probe
  */
@@ -537,9 +684,17 @@ mca_oob_t* mca_oob_tcp_component_init(int* priority)
     memset(&mca_oob_tcp_component.tcp_send_event, 0, sizeof(opal_event_t));
 
     /* create a listen socket */
-    if(mca_oob_tcp_create_listen() != ORTE_SUCCESS) {
+    if (OOB_TCP_EVENT == mca_oob_tcp_component.tcp_listen_type) {
-        opal_output(0, "mca_oob_tcp_init: unable to create listen socket\n");
+        if(mca_oob_tcp_create_listen() != ORTE_SUCCESS) {
-        return NULL;
+            opal_output(0, "mca_oob_tcp_init: unable to create listen socket");
+            return NULL;
+        }
+    } else if (OOB_TCP_LISTEN_THREAD == mca_oob_tcp_component.tcp_listen_type) {
+        if (mca_oob_tcp_create_listen_thread() != ORTE_SUCCESS) {
+            opal_output(0, "mca_oob_tcp_init: unable to create listen thread");
+            return NULL;
+        }
+        opal_progress_register(mca_oob_tcp_listen_progress);
     }
     return &mca_oob_tcp;
 }
@@ -932,10 +1087,19 @@ int mca_oob_tcp_fini(void)
 
     /* close listen socket */
     if (mca_oob_tcp_component.tcp_listen_sd >= 0) {
-        opal_event_del(&mca_oob_tcp_component.tcp_recv_event);
+        if (OOB_TCP_EVENT == mca_oob_tcp_component.tcp_listen_type) {
-        close(mca_oob_tcp_component.tcp_listen_sd);
+            opal_event_del(&mca_oob_tcp_component.tcp_recv_event);
+            close(mca_oob_tcp_component.tcp_listen_sd);
+        } else if (OOB_TCP_LISTEN_THREAD == mca_oob_tcp_component.tcp_listen_type) {
+            void *data;
+            close(mca_oob_tcp_component.tcp_listen_sd);
+            opal_thread_join(&mca_oob_tcp_component.tcp_listen_thread, data);
+            opal_progress_unregister(mca_oob_tcp_listen_progress);
+        }
+
         mca_oob_tcp_component.tcp_listen_sd = -1;
     }
+    opal_progress_unregister(mca_oob_tcp_listen_progress);
 
     /* cleanup all peers */
     for(item = opal_list_remove_first(&mca_oob_tcp_component.tcp_peer_list);

orte/mca/oob/tcp/oob_tcp.h

@@ -255,6 +255,12 @@ struct mca_oob_tcp_component_t {
     opal_condition_t   tcp_match_cond;       /**< condition variable used in finalize */
     int                tcp_match_count;      /**< number of matched recvs in progress */
     int                tcp_debug;            /**< debug level */
+    bool               tcp_shutdown;
+    enum { OOB_TCP_EVENT, OOB_TCP_LISTEN_THREAD } tcp_listen_type;
+    opal_thread_t tcp_listen_thread;
+    opal_free_list_t tcp_pending_connections_fl;
+    opal_list_t tcp_pending_connections;
+    opal_mutex_t tcp_pending_connections_lock;
 };
 
 /**
@@ -264,6 +270,13 @@ typedef struct mca_oob_tcp_component_t mca_oob_tcp_component_t;
 
 OMPI_COMP_EXPORT extern mca_oob_tcp_component_t mca_oob_tcp_component;
 
+    struct mca_oob_tcp_pending_connection_t {
+        opal_free_list_item_t super;
+        int fd;
+    };
+    typedef struct mca_oob_tcp_pending_connection_t mca_oob_tcp_pending_connection_t;
+    OBJ_CLASS_DECLARATION(mca_oob_tcp_pending_connection_t);
+
 
 #if defined(c_plusplus) || defined(__cplusplus)
 }