diff --git a/config/ompi_mca.m4 b/config/ompi_mca.m4 index 663ba5ae03..896e4fd042 100644 --- a/config/ompi_mca.m4 +++ b/config/ompi_mca.m4 @@ -717,7 +717,7 @@ AC_DEFUN([MCA_PROCESS_COMPONENT],[ if test "$DIRECT_$2" = "$component" ; then if test "`grep DIRECT_CALL_HEADER $infile`" != "" ; then line="`grep DIRECT_CALL_HEADER $infile | cut -d= -f2-`" - str="MCA_${framework}_DIRECT_CALL_HEADER=\\\"$line\\\"" + str="MCA_${framework}_DIRECT_CALL_HEADER=$line" eval $str else AC_MSG_ERROR([*** ${framework} component ${component} was supposed to be direct-called, but @@ -886,24 +886,9 @@ AC_DEFUN([MCA_SETUP_DIRECT_CALL],[ [Defined to 1 if $1 should use direct calls instead of components]) AC_DEFINE_UNQUOTED([MCA_$1_DIRECT_CALL_COMPONENT], [$MCA_$1_DIRECT_CALL_COMPONENT], [name of component to use for direct calls, if MCA_$1_DIRECT_CALL is 1]) - MCA_WRITE_DIRECT_CALL_HEADER($1, $2) -]) -AC_DEFUN([MCA_WRITE_DIRECT_CALL_HEADER],[ - AC_CONFIG_FILES($2/mca/$1/$1_direct_call.h.tmp:$2/mca/$1/$1_direct_call.h.in) - AC_CONFIG_COMMANDS($1-direct, -[if test -f "$2/mca/$1/$1_direct_call"; then - diff "$2/mca/$1/$1_direct_call.h" "$2/mca/$1/$1_direct_call.h.tmp" > /dev/null 2>&1 - if test "$?" != "0"; then - cp "$2/mca/$1/$1_direct_call.h.tmp" "$2/mca/$1/$1_direct_call.h" - echo "config.status: regenerating $2/mca/$1/$1_direct_call.h" - else - echo "config.status: $2/mca/$1/$1_direct_call.h unchanged" - fi -else - cp "$2/mca/$1/$1_direct_call.h.tmp" "$2/mca/$1/$1_direct_call.h" - echo "config.status: creating $2/mca/$1/$1_direct_call.h" -fi -rm $2/mca/$1/$1_direct_call.h.tmp]) + AC_DEFINE_UNQUOTED([MCA_$1_DIRECT_CALL_HEADER], + ["[$MCA_]$1[_DIRECT_CALL_HEADER]"], + [Header $1 includes to be direct called]) ]) diff --git a/ompi/mca/mtl/Makefile.am b/ompi/mca/mtl/Makefile.am index a5a80be3fd..dd623f981e 100644 --- a/ompi/mca/mtl/Makefile.am +++ b/ompi/mca/mtl/Makefile.am @@ -26,7 +26,6 @@ nobase_nodist_ompi_HEADERS = # local files headers = mtl.h -nodist_headers = libmca_mtl_la_SOURCES += $(headers) $(nodist_headers) # Conditionally install the header files @@ -41,4 +40,4 @@ endif include base/Makefile.am distclean-local: - rm -f base/static-components.h mtl_direct_call.h + rm -f base/static-components.h diff --git a/ompi/mca/mtl/mtl.h b/ompi/mca/mtl/mtl.h index 823c678929..7f1f87f21c 100644 --- a/ompi/mca/mtl/mtl.h +++ b/ompi/mca/mtl/mtl.h @@ -395,7 +395,7 @@ typedef struct mca_mtl_base_module_t mca_mtl_base_module_t; */ #if MCA_mtl_DIRECT_CALL -#include "ompi/mca/mtl/mtl_direct_call.h" +#include MCA_mtl_DIRECT_CALL_HEADER #define OMPI_MTL_CALL_STAMP(a, b) ompi_mtl_ ## a ## _ ## b #define OMPI_MTL_CALL_EXPANDER(a, b) OMPI_MTL_CALL_STAMP(a,b) diff --git a/ompi/mca/mtl/mtl_direct_call.h.in b/ompi/mca/mtl/mtl_direct_call.h.in deleted file mode 100644 index 7196ad4f30..0000000000 --- a/ompi/mca/mtl/mtl_direct_call.h.in +++ /dev/null @@ -1,20 +0,0 @@ -/* - * Copyright (c) 2006 The Regents of the University of California. - * All rights reserved. - * - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - - -#ifndef MCA_MTL_DIRECT_CALL_H_ -#define MCA_MTL_DIRECT_CALL_H_ - -#if MCA_mtl_DIRECT_CALL -#include @MCA_mtl_DIRECT_CALL_HEADER@ -#endif - -#endif diff --git a/ompi/mca/pml/Makefile.am b/ompi/mca/pml/Makefile.am index fce7293810..b3d69b5d6d 100644 --- a/ompi/mca/pml/Makefile.am +++ b/ompi/mca/pml/Makefile.am @@ -26,7 +26,6 @@ nobase_nodist_ompi_HEADERS = # local files headers = pml.h -nodist_headers = pml_direct_call.h libmca_pml_la_SOURCES += $(headers) $(nodist_headers) # Conditionally install the header files @@ -41,4 +40,4 @@ endif include base/Makefile.am distclean-local: - rm -f base/static-components.h pml_direct_call.h + rm -f base/static-components.h diff --git a/ompi/mca/pml/pml.h b/ompi/mca/pml/pml.h index 22fae0be19..cfbdb16abe 100644 --- a/ompi/mca/pml/pml.h +++ b/ompi/mca/pml/pml.h @@ -530,7 +530,7 @@ typedef mca_pml_base_module_1_0_0_t mca_pml_base_module_t; */ #if MCA_pml_DIRECT_CALL -#include "ompi/mca/pml/pml_direct_call.h" +#include MCA_pml_DIRECT_CALL_HEADER #define MCA_PML_CALL_STAMP(a, b) mca_pml_ ## a ## _ ## b #define MCA_PML_CALL_EXPANDER(a, b) MCA_PML_CALL_STAMP(a,b) diff --git a/ompi/mca/pml/pml_direct_call.h.in b/ompi/mca/pml/pml_direct_call.h.in deleted file mode 100644 index a99436fc8a..0000000000 --- a/ompi/mca/pml/pml_direct_call.h.in +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - - -#ifndef MCA_PML_DIRECT_CALL_H_ -#define MCA_PML_DIRECT_CALL_H_ - -#if MCA_pml_DIRECT_CALL -#include @MCA_pml_DIRECT_CALL_HEADER@ -#endif - -#endif diff --git a/opal/class/opal_free_list.h b/opal/class/opal_free_list.h index 75e60e7af6..138b59a302 100644 --- a/opal/class/opal_free_list.h +++ b/opal/class/opal_free_list.h @@ -124,7 +124,7 @@ OMPI_DECLSPEC int opal_free_list_grow(opal_free_list_t* flist, size_t num_elemen #define OPAL_FREE_LIST_WAIT(fl, item, rc) \ { \ OPAL_THREAD_LOCK(&((fl)->fl_lock)); \ - while( NULL == (item = opal_list_remove_first(&((fl)->super))) ) { \ + while( NULL == (item = (opal_free_list_item_t*) opal_list_remove_first(&((fl)->super))) ) { \ if((fl)->fl_max_to_alloc <= (fl)->fl_num_allocated) { \ (fl)->fl_num_waiting++; \ opal_condition_wait(&((fl)->fl_condition), &((fl)->fl_lock)); \ @@ -149,7 +149,7 @@ OMPI_DECLSPEC int opal_free_list_grow(opal_free_list_t* flist, size_t num_elemen #define OPAL_FREE_LIST_RETURN(fl, item) \ { \ OPAL_THREAD_LOCK(&(fl)->fl_lock); \ - opal_list_prepend(&((fl)->super), (item)); \ + opal_list_prepend(&((fl)->super), ((opal_list_item_t*) item)); \ if((fl)->fl_num_waiting > 0) { \ opal_condition_signal(&((fl)->fl_condition)); \ } \ diff --git a/opal/mca/memcpy/base/Makefile.am b/opal/mca/memcpy/base/Makefile.am index 30a3ba638f..db093b5dbb 100644 --- a/opal/mca/memcpy/base/Makefile.am +++ b/opal/mca/memcpy/base/Makefile.am @@ -16,5 +16,3 @@ headers += \ libmca_memcpy_la_SOURCES += \ base/memcpy_base_close.c \ base/memcpy_base_open.c - -nodist_headers += base/base_impl.h diff --git a/opal/mca/memcpy/base/base.h b/opal/mca/memcpy/base/base.h index 4996f372bd..8b3585f794 100644 --- a/opal/mca/memcpy/base/base.h +++ b/opal/mca/memcpy/base/base.h @@ -69,6 +69,6 @@ extern "C" { #endif /* include implementation to call */ -#include "opal/mca/memcpy/base/base_impl.h" +#include MCA_timer_IMPLEMENTATION_HEADER #endif /* OPAL_BASE_MEMCPY_H */ diff --git a/opal/mca/memcpy/configure.m4 b/opal/mca/memcpy/configure.m4 index 5e3c98f1b8..f77c564da5 100644 --- a/opal/mca/memcpy/configure.m4 +++ b/opal/mca/memcpy/configure.m4 @@ -3,6 +3,8 @@ dnl dnl Copyright (c) 2004-2006 The University of Tennessee and The University dnl of Tennessee Research Foundation. All rights dnl reserved. +dnl Copyright (c) 2004-2006 The Regents of the University of California. +dnl All rights reserved. dnl $COPYRIGHT$ dnl dnl Additional copyrights may follow @@ -10,6 +12,7 @@ dnl dnl $HEADER$ dnl +dnl we only want one m4_define(MCA_memcpy_CONFIGURE_MODE, STOP_AT_FIRST) AC_DEFUN([MCA_memcpy_CONFIG],[ @@ -23,5 +26,7 @@ AC_DEFUN([MCA_memcpy_CONFIG],[ memcpy_base_include="base/memcpy_base_default.h" fi - AC_CONFIG_LINKS([opal/mca/memcpy/base/base_impl.h:opal/mca/memcpy/$memcpy_base_include]) + AC_DEFINE_UNQUOTED([MCA_memcpy_IMPLEMENTATION_HEADER], + ["opal/mca/memcpy/$memcpy_base_include"], + [Header to include for memcpy implementation]) ]) diff --git a/opal/mca/timer/base/Makefile.am b/opal/mca/timer/base/Makefile.am index 8841b43af6..71cd53a621 100644 --- a/opal/mca/timer/base/Makefile.am +++ b/opal/mca/timer/base/Makefile.am @@ -23,5 +23,3 @@ headers += \ libmca_timer_la_SOURCES += \ base/timer_base_close.c \ base/timer_base_open.c - -nodist_headers += base/base_impl.h diff --git a/opal/mca/timer/base/base.h b/opal/mca/timer/base/base.h index 31d9d081fc..ef73d9f724 100644 --- a/opal/mca/timer/base/base.h +++ b/opal/mca/timer/base/base.h @@ -76,6 +76,6 @@ extern "C" { #endif /* include implementation to call */ -#include "opal/mca/timer/base/base_impl.h" +#include MCA_timer_IMPLEMENTATION_HEADER #endif /* OPAL_BASE_TIMER_H */ diff --git a/opal/mca/timer/configure.m4 b/opal/mca/timer/configure.m4 index 1a0d3878fb..f5253bc238 100644 --- a/opal/mca/timer/configure.m4 +++ b/opal/mca/timer/configure.m4 @@ -31,5 +31,7 @@ AC_DEFUN([MCA_timer_CONFIG],[ timer_base_include="base/timer_base_null.h" fi - AC_CONFIG_LINKS([opal/mca/timer/base/base_impl.h:opal/mca/timer/$timer_base_include]) + AC_DEFINE_UNQUOTED([MCA_timer_IMPLEMENTATION_HEADER], + ["opal/mca/timer/$timer_base_include"], + [Header to include for timer implementation]) ]) diff --git a/orte/mca/oob/tcp/oob_tcp.c b/orte/mca/oob/tcp/oob_tcp.c index 1c38404fbd..77548c32b7 100644 --- a/orte/mca/oob/tcp/oob_tcp.c +++ b/orte/mca/oob/tcp/oob_tcp.c @@ -83,6 +83,7 @@ OBJ_CLASS_INSTANCE( */ static int mca_oob_tcp_create_listen(void); +static int mca_oob_tcp_create_listen_thread(void); static void mca_oob_tcp_recv_handler(int sd, short flags, void* user); static void mca_oob_tcp_accept(void); @@ -169,6 +170,8 @@ static inline char* mca_oob_tcp_param_register_str( */ int mca_oob_tcp_component_open(void) { + char *listen_type; + #ifdef __WINDOWS__ WSADATA win_sock_data; if (WSAStartup(MAKEWORD(2,2), &win_sock_data) != 0) { @@ -207,7 +210,26 @@ int mca_oob_tcp_component_open(void) mca_oob_tcp_component.tcp_rcvbuf = mca_oob_tcp_param_register_int("rcvbuf", 128*1024); + mca_base_param_reg_string(&mca_oob_tcp_component.super.oob_base, + "listen_mode", + "Mode for HNP to accept incoming connections: event, listen_thread", + false, + false, + "event", + &listen_type); + + if (0 == strcmp(listen_type, "event")) { + mca_oob_tcp_component.tcp_listen_type = OOB_TCP_EVENT; + } else if (0 == strcmp(listen_type, "listen_thread")) { + mca_oob_tcp_component.tcp_listen_type = OOB_TCP_LISTEN_THREAD; + } else { + opal_output(0, "Invalid value for oob_tcp_listen_mode parameter: %s", + listen_type); + return ORTE_ERROR; + } + /* initialize state */ + mca_oob_tcp_component.tcp_shutdown = false; mca_oob_tcp_component.tcp_listen_sd = -1; mca_oob_tcp_component.tcp_match_count = 0; return ORTE_SUCCESS; @@ -352,6 +374,131 @@ static int mca_oob_tcp_create_listen(void) } +static void* mca_oob_tcp_listen_thread(opal_object_t *obj) +{ + int sd, rc; + ompi_socklen_t addrlen = sizeof(struct sockaddr_in); + struct sockaddr_in addr; + opal_free_list_item_t *fl_item; + mca_oob_tcp_pending_connection_t *item; + + while (false == mca_oob_tcp_component.tcp_shutdown) { + sd = accept(mca_oob_tcp_component.tcp_listen_sd, + (struct sockaddr*)&addr, &addrlen); + if(sd < 0) { + if (mca_oob_tcp_component.tcp_shutdown) return NULL; + + if(ompi_socket_errno == EINTR) + continue; + if(ompi_socket_errno != EAGAIN || ompi_socket_errno != EWOULDBLOCK) + opal_output(0, "mca_oob_tcp_accept: accept() failed with errno %d.", ompi_socket_errno); + close(sd); + /* start a new listen socket */ + mca_oob_tcp_create_listen_thread(); + return NULL; + } + + OPAL_FREE_LIST_WAIT(&mca_oob_tcp_component.tcp_pending_connections_fl, + fl_item, rc); + item = (mca_oob_tcp_pending_connection_t*) fl_item; + item->fd = sd; + opal_mutex_lock(&mca_oob_tcp_component.tcp_pending_connections_lock); + opal_list_append(&mca_oob_tcp_component.tcp_pending_connections, + (opal_list_item_t*) item); + opal_mutex_unlock(&mca_oob_tcp_component.tcp_pending_connections_lock); + } + + return NULL; +} + +static int mca_oob_tcp_listen_progress(void) +{ + int count = 0; + mca_oob_tcp_pending_connection_t *item; + struct sockaddr_in addr; + mca_oob_tcp_event_t* event; + + if (opal_list_get_size(&mca_oob_tcp_component.tcp_pending_connections)) { + opal_mutex_lock(&mca_oob_tcp_component.tcp_pending_connections_lock); + while (NULL != (item = (opal_list_item_t*) + opal_list_remove_first(&mca_oob_tcp_component.tcp_pending_connections))) { + + /* setup socket options */ + mca_oob_tcp_set_socket_options(item->fd); + + /* log the accept */ + if(mca_oob_tcp_component.tcp_debug) { + opal_output(0, "[%lu,%lu,%lu] mca_oob_tcp_accept: %s:%d\n", + ORTE_NAME_ARGS(orte_process_info.my_name), + inet_ntoa(addr.sin_addr), + addr.sin_port); + } + + /* wait for receipt of peers process identifier to + complete this connection */ + event = OBJ_NEW(mca_oob_tcp_event_t); + opal_event_set(&event->event, item->fd, OPAL_EV_READ, mca_oob_tcp_recv_handler, event); + opal_event_add(&event->event, 0); + OPAL_FREE_LIST_RETURN(&mca_oob_tcp_component.tcp_pending_connections_fl, + (opal_free_list_item_t *) item); + + count++; + } + opal_mutex_unlock(&mca_oob_tcp_component.tcp_pending_connections_lock); + } + + return count; +} + + +static int mca_oob_tcp_create_listen_thread(void) +{ + struct sockaddr_in inaddr; + ompi_socklen_t addrlen; + + /* create a listen socket for incoming connections */ + mca_oob_tcp_component.tcp_listen_sd = socket(AF_INET, SOCK_STREAM, 0); + if(mca_oob_tcp_component.tcp_listen_sd < 0) { + opal_output(0,"mca_oob_tcp_component_init: socket() failed with errno=%d", ompi_socket_errno); + return ORTE_ERROR; + } + + /* setup socket options */ + mca_oob_tcp_set_socket_options(mca_oob_tcp_component.tcp_listen_sd); + + /* bind address */ + memset(&inaddr, 0, sizeof(inaddr)); + inaddr.sin_family = AF_INET; + inaddr.sin_addr.s_addr = INADDR_ANY; + inaddr.sin_port = 0; + + if(bind(mca_oob_tcp_component.tcp_listen_sd, (struct sockaddr*)&inaddr, sizeof(inaddr)) < 0) { + opal_output(0,"mca_oob_tcp_create_listen: bind() failed with errno=%d", ompi_socket_errno); + return ORTE_ERROR; + } + + /* resolve system assigned port */ + addrlen = sizeof(struct sockaddr_in); + if(getsockname(mca_oob_tcp_component.tcp_listen_sd, (struct sockaddr*)&inaddr, &addrlen) < 0) { + opal_output(0, "mca_oob_tcp_create_listen: getsockname() failed with errno=%d", ompi_socket_errno); + return ORTE_ERROR; + } + mca_oob_tcp_component.tcp_listen_port = inaddr.sin_port; + + /* setup listen backlog to maximum allowed by kernel */ + if(listen(mca_oob_tcp_component.tcp_listen_sd, SOMAXCONN) < 0) { + opal_output(0, "mca_oob_tcp_component_init: listen() failed with errno=%d", ompi_socket_errno); + return ORTE_ERROR; + } + + /* start the listen thread */ + mca_oob_tcp_component.tcp_listen_thread.t_run = mca_oob_tcp_listen_thread; + mca_oob_tcp_component.tcp_listen_thread.t_arg = NULL; + + return opal_thread_start(&mca_oob_tcp_component.tcp_listen_thread); +} + + /* * Handle probe */ @@ -537,9 +684,17 @@ mca_oob_t* mca_oob_tcp_component_init(int* priority) memset(&mca_oob_tcp_component.tcp_send_event, 0, sizeof(opal_event_t)); /* create a listen socket */ - if(mca_oob_tcp_create_listen() != ORTE_SUCCESS) { - opal_output(0, "mca_oob_tcp_init: unable to create listen socket\n"); - return NULL; + if (OOB_TCP_EVENT == mca_oob_tcp_component.tcp_listen_type) { + if(mca_oob_tcp_create_listen() != ORTE_SUCCESS) { + opal_output(0, "mca_oob_tcp_init: unable to create listen socket"); + return NULL; + } + } else if (OOB_TCP_LISTEN_THREAD == mca_oob_tcp_component.tcp_listen_type) { + if (mca_oob_tcp_create_listen_thread() != ORTE_SUCCESS) { + opal_output(0, "mca_oob_tcp_init: unable to create listen thread"); + return NULL; + } + opal_progress_register(mca_oob_tcp_listen_progress); } return &mca_oob_tcp; } @@ -932,10 +1087,19 @@ int mca_oob_tcp_fini(void) /* close listen socket */ if (mca_oob_tcp_component.tcp_listen_sd >= 0) { - opal_event_del(&mca_oob_tcp_component.tcp_recv_event); - close(mca_oob_tcp_component.tcp_listen_sd); + if (OOB_TCP_EVENT == mca_oob_tcp_component.tcp_listen_type) { + opal_event_del(&mca_oob_tcp_component.tcp_recv_event); + close(mca_oob_tcp_component.tcp_listen_sd); + } else if (OOB_TCP_LISTEN_THREAD == mca_oob_tcp_component.tcp_listen_type) { + void *data; + close(mca_oob_tcp_component.tcp_listen_sd); + opal_thread_join(&mca_oob_tcp_component.tcp_listen_thread, data); + opal_progress_unregister(mca_oob_tcp_listen_progress); + } + mca_oob_tcp_component.tcp_listen_sd = -1; } + opal_progress_unregister(mca_oob_tcp_listen_progress); /* cleanup all peers */ for(item = opal_list_remove_first(&mca_oob_tcp_component.tcp_peer_list); diff --git a/orte/mca/oob/tcp/oob_tcp.h b/orte/mca/oob/tcp/oob_tcp.h index f66baffc5e..5ea8510b73 100644 --- a/orte/mca/oob/tcp/oob_tcp.h +++ b/orte/mca/oob/tcp/oob_tcp.h @@ -255,6 +255,12 @@ struct mca_oob_tcp_component_t { opal_condition_t tcp_match_cond; /**< condition variable used in finalize */ int tcp_match_count; /**< number of matched recvs in progress */ int tcp_debug; /**< debug level */ + bool tcp_shutdown; + enum { OOB_TCP_EVENT, OOB_TCP_LISTEN_THREAD } tcp_listen_type; + opal_thread_t tcp_listen_thread; + opal_free_list_t tcp_pending_connections_fl; + opal_list_t tcp_pending_connections; + opal_mutex_t tcp_pending_connections_lock; }; /** @@ -264,6 +270,13 @@ typedef struct mca_oob_tcp_component_t mca_oob_tcp_component_t; OMPI_COMP_EXPORT extern mca_oob_tcp_component_t mca_oob_tcp_component; + struct mca_oob_tcp_pending_connection_t { + opal_free_list_item_t super; + int fd; + }; + typedef struct mca_oob_tcp_pending_connection_t mca_oob_tcp_pending_connection_t; + OBJ_CLASS_DECLARATION(mca_oob_tcp_pending_connection_t); + #if defined(c_plusplus) || defined(__cplusplus) }