From 540d483dd3f3f2733bf8eeb9bd3e2ae6cea51201 Mon Sep 17 00:00:00 2001 From: Brad Penoff Date: Sat, 15 Dec 2007 03:28:10 +0000 Subject: [PATCH] 64 bit fix and initial Solaris support This commit was SVN r16967. --- ompi/mca/btl/sctp/btl_sctp_component.c | 6 ++-- ompi/mca/btl/sctp/btl_sctp_proc.c | 9 ++++-- ompi/mca/btl/sctp/btl_sctp_proc.h | 9 +++--- ompi/mca/btl/sctp/btl_sctp_recv_handler.c | 4 +-- ompi/mca/btl/sctp/btl_sctp_utils.c | 34 +++++++++++++++++++++++ ompi/mca/btl/sctp/configure.m4 | 23 ++++++++++++++- 6 files changed, 73 insertions(+), 12 deletions(-) diff --git a/ompi/mca/btl/sctp/btl_sctp_component.c b/ompi/mca/btl/sctp/btl_sctp_component.c index c98d8f5f4f..e19765ea53 100644 --- a/ompi/mca/btl/sctp/btl_sctp_component.c +++ b/ompi/mca/btl/sctp/btl_sctp_component.c @@ -237,9 +237,11 @@ int mca_btl_sctp_component_open(void) !mca_btl_sctp_param_register_int ("use_nagle", 0); /* port_min */ /* port_range */ - /* use a single one-to-many socket by default */ + /* use a single one-to-many socket by default except in Solaris (see + * the configure.m4 file) + */ mca_btl_sctp_component.sctp_if_11 = - mca_btl_sctp_param_register_int ("if_11", 0); + mca_btl_sctp_param_register_int ("if_11", OMPI_MCA_BTL_SCTP_USE_ONE_TO_ONE_SOCKET); /* have lower exclusivity than tcp */ mca_btl_sctp_module.super.btl_exclusivity = MCA_BTL_EXCLUSIVITY_LOW; diff --git a/ompi/mca/btl/sctp/btl_sctp_proc.c b/ompi/mca/btl/sctp/btl_sctp_proc.c index 5fa340ae35..f88ffd6200 100644 --- a/ompi/mca/btl/sctp/btl_sctp_proc.c +++ b/ompi/mca/btl/sctp/btl_sctp_proc.c @@ -362,7 +362,7 @@ bool mca_btl_sctp_proc_accept(mca_btl_sctp_proc_t* btl_proc, struct sockaddr_in* * * TODO - change this to use a hash for constant time performance */ -int mca_btl_sctp_proc_check(uint32_t id, struct mca_btl_sctp_proc_table_node *table) { +int mca_btl_sctp_proc_check(sctp_assoc_t id, struct mca_btl_sctp_proc_table_node *table) { #if MCA_BTL_SCTP_DONT_USE_HASH int i; for(i = 0; i < MCA_BTL_SCTP_PROC_TABLE_SIZE; i++) { @@ -376,6 +376,7 @@ int mca_btl_sctp_proc_check(uint32_t id, struct mca_btl_sctp_proc_table_node *ta return INVALID_ENTRY; #else mca_btl_sctp_proc_t *val; + /* TODO fix if sctp_assoc_t is 64 bit (once we change to hash) */ int rc = opal_hash_table_get_value_uint32(&mca_btl_sctp_component.sctp_assocID_hash, id, &val); if(OPAL_SUCCESS == rc) { return VALID_ENTRY; @@ -394,7 +395,7 @@ int mca_btl_sctp_proc_check(uint32_t id, struct mca_btl_sctp_proc_table_node *ta * TODO change this to a hash table that can expand to eliminate * MCA_BTL_SCTP_PROC_TABLE_SIZE limitation */ -void mca_btl_sctp_proc_add(uint32_t id, struct mca_btl_sctp_proc_t *proc, struct mca_btl_sctp_proc_table_node *table) { +void mca_btl_sctp_proc_add(sctp_assoc_t id, struct mca_btl_sctp_proc_t *proc, struct mca_btl_sctp_proc_table_node *table) { #if MCA_BTL_SCTP_DONT_USE_HASH int i; for(i = 0; i < MCA_BTL_SCTP_PROC_TABLE_SIZE; i++) { @@ -406,6 +407,7 @@ void mca_btl_sctp_proc_add(uint32_t id, struct mca_btl_sctp_proc_t *proc, struct } } #else + /* TODO fix if sctp_assoc_t is 64 bit (once we change to hash) */ int rc = opal_hash_table_set_value_uint32(&mca_btl_sctp_component.sctp_assocID_hash, id, proc); /* TODO handle return code */ #endif @@ -417,7 +419,7 @@ void mca_btl_sctp_proc_add(uint32_t id, struct mca_btl_sctp_proc_t *proc, struct * ------------------------------------------ * Returns pointer to a proc that is indexed by the association id. */ -mca_btl_sctp_proc_t *mca_btl_sctp_proc_get(uint32_t id, struct mca_btl_sctp_proc_table_node *table) { +mca_btl_sctp_proc_t *mca_btl_sctp_proc_get(sctp_assoc_t id, struct mca_btl_sctp_proc_table_node *table) { #if MCA_BTL_SCTP_DONT_USE_HASH int i; for(i = 0; i < MCA_BTL_SCTP_PROC_TABLE_SIZE; i++){ @@ -428,6 +430,7 @@ mca_btl_sctp_proc_t *mca_btl_sctp_proc_get(uint32_t id, struct mca_btl_sctp_proc return NULL; #else mca_btl_sctp_proc_t *val; + /* TODO fix if sctp_assoc_t is 64 bit (once we change to hash) */ int rc = opal_hash_table_get_value_uint32(&mca_btl_sctp_component.sctp_assocID_hash, id, &val); if(OPAL_SUCCESS == rc) { return val; diff --git a/ompi/mca/btl/sctp/btl_sctp_proc.h b/ompi/mca/btl/sctp/btl_sctp_proc.h index 910abff384..82cf40122f 100644 --- a/ompi/mca/btl/sctp/btl_sctp_proc.h +++ b/ompi/mca/btl/sctp/btl_sctp_proc.h @@ -25,6 +25,7 @@ #include "btl_sctp.h" #include "btl_sctp_addr.h" #include "btl_sctp_endpoint.h" +#include #if defined(c_plusplus) || defined(__cplusplus) extern "C" { @@ -94,7 +95,7 @@ enum { #define MCA_BTL_SCTP_PROC_TABLE_SIZE 256 struct mca_btl_sctp_proc_table_node { int valid; - uint32_t sctp_assoc_id; + sctp_assoc_t sctp_assoc_id; struct mca_btl_sctp_proc_t *proc; }; typedef struct mca_btl_sctp_proc_table_node mca_btl_sctp_proc_table_node; @@ -102,9 +103,9 @@ typedef struct mca_btl_sctp_proc_table_node mca_btl_sctp_proc_table_node; extern struct mca_btl_sctp_proc_table_node *recvr_proc_table; extern struct mca_btl_sctp_proc_table_node *sender_proc_table; -int mca_btl_sctp_proc_check(uint32_t id, struct mca_btl_sctp_proc_table_node *table); -void mca_btl_sctp_proc_add(uint32_t id, struct mca_btl_sctp_proc_t *proc, struct mca_btl_sctp_proc_table_node *table); -mca_btl_sctp_proc_t *mca_btl_sctp_proc_get(uint32_t id, struct mca_btl_sctp_proc_table_node *table); +int mca_btl_sctp_proc_check(sctp_assoc_t id, struct mca_btl_sctp_proc_table_node *table); +void mca_btl_sctp_proc_add(sctp_assoc_t id, struct mca_btl_sctp_proc_t *proc, struct mca_btl_sctp_proc_table_node *table); +mca_btl_sctp_proc_t *mca_btl_sctp_proc_get(sctp_assoc_t id, struct mca_btl_sctp_proc_table_node *table); #if defined(c_plusplus) || defined(__cplusplus) } diff --git a/ompi/mca/btl/sctp/btl_sctp_recv_handler.c b/ompi/mca/btl/sctp/btl_sctp_recv_handler.c index 00abc26115..4d578d48c6 100644 --- a/ompi/mca/btl/sctp/btl_sctp_recv_handler.c +++ b/ompi/mca/btl/sctp/btl_sctp_recv_handler.c @@ -141,7 +141,7 @@ void mca_btl_sctp_recv_handler(int sd, short flags, void *user) { /* Check if sender is known to us. */ - if((mca_btl_sctp_proc_check(((uint32_t)(sri.sinfo_assoc_id)), recvr_proc_table)) == VALID_ENTRY) { + if((mca_btl_sctp_proc_check(sri.sinfo_assoc_id, recvr_proc_table)) == VALID_ENTRY) { mca_btl_base_endpoint_t *btl_endpoint; mca_btl_sctp_frag_t* frag; @@ -233,7 +233,7 @@ data_still_pending_on_endpoint: CLOSE_THE_SOCKET(sd); return; } - mca_btl_sctp_proc_add((uint32_t)sri.sinfo_assoc_id, btl_proc, recvr_proc_table); + mca_btl_sctp_proc_add(sri.sinfo_assoc_id, btl_proc, recvr_proc_table); /* are there any existing peer instances will to accept this connection */ diff --git a/ompi/mca/btl/sctp/btl_sctp_utils.c b/ompi/mca/btl/sctp/btl_sctp_utils.c index 112d252869..43170e0799 100644 --- a/ompi/mca/btl/sctp/btl_sctp_utils.c +++ b/ompi/mca/btl/sctp/btl_sctp_utils.c @@ -63,6 +63,39 @@ struct sockaddr_in mca_btl_sctp_utils_sockaddr_from_endpoint(struct mca_btl_base int mca_btl_sctp_utils_writev(int sd, struct iovec *vec, size_t len, struct sockaddr *to_addr, socklen_t to_len, uint16_t stream_no) { +#if OMPI_MCA_BTL_SCTP_CONCATENATES_IOVS + /* required for Solaris since struct msghdr has no msg_control field */ + + int current_cnt=0, total_offset=0, total=0, byte_sent; + char *send_buf; + + /* count the total and allocate space to copy to */ + while(current_cnt < len) { + total += vec[current_cnt].iov_len; + current_cnt++; + } + if(NULL == (send_buf = (char *) malloc(total))) { + BTL_ERROR(("Ran out of memory.")); + return 0; + } + + /* combine all iovcnt's into one message. write all or nothing */ + current_cnt=0; + while(current_cnt < len) { + memcpy(send_buf+total_offset, vec[current_cnt].iov_base, vec[current_cnt].iov_len); + total_offset += vec[current_cnt].iov_len; + current_cnt++; + } + + byte_sent = sctp_sendmsg(sd, send_buf, total, (struct sockaddr *) to_addr, + sizeof(*to_addr), 0, 0, stream_no, 0, 0); + free(send_buf); + + return byte_sent; + +#else + /* uses iovec directly */ + char outcmesg[CMSG_SPACE(sizeof(struct sctp_sndrcvinfo))]; struct cmsghdr *cmesg; struct msghdr outmesg; @@ -97,4 +130,5 @@ int mca_btl_sctp_utils_writev(int sd, struct iovec *vec, size_t len, srinfo->sinfo_context = cxt; return sendmsg(sd, &outmesg, 0); +#endif } diff --git a/ompi/mca/btl/sctp/configure.m4 b/ompi/mca/btl/sctp/configure.m4 index 69f707ee13..958a733a9e 100644 --- a/ompi/mca/btl/sctp/configure.m4 +++ b/ompi/mca/btl/sctp/configure.m4 @@ -37,12 +37,20 @@ AC_DEFUN([OMPI_CHECK_SCTP],[ case "$host" in *linux*) ompi_sctp_try_to_build="yes" + AC_DEFINE(OMPI_MCA_BTL_SCTP_USE_ONE_TO_ONE_SOCKET, 0, + [Default value for socket style to use with SCTP BTL]) + AC_DEFINE(OMPI_MCA_BTL_SCTP_CONCATENATES_IOVS, 0, + [False if you can use iovec's directly with SCTP BTL]) ;; *bsd*) # only add -DFREEBSD once to get extra sin_len field btl_sctp_CFLAGS="`echo $btl_sctp_CFLAGS | sed 's/-DFREEBSD//g'`" btl_sctp_CFLAGS="$btl_sctp_CFLAGS -DFREEBSD" ompi_sctp_try_to_build="yes" + AC_DEFINE(OMPI_MCA_BTL_SCTP_USE_ONE_TO_ONE_SOCKET, 0, + [Default value for socket style to use with SCTP BTL]) + AC_DEFINE(OMPI_MCA_BTL_SCTP_CONCATENATES_IOVS, 0, + [False if you can use iovec's directly with SCTP BTL]) AC_MSG_WARN([Adding -DFREEBSD to set extra sin_len field in sockaddr.]) ;; # Mac OS X support for SCTP NKE. Adjustments should look like *bsd*... @@ -51,10 +59,23 @@ AC_DEFUN([OMPI_CHECK_SCTP],[ btl_sctp_CFLAGS="`echo $btl_sctp_CFLAGS | sed 's/-DFREEBSD//g'`" btl_sctp_CFLAGS="$btl_sctp_CFLAGS -DFREEBSD" ompi_sctp_try_to_build="yes" + AC_DEFINE(OMPI_MCA_BTL_SCTP_USE_ONE_TO_ONE_SOCKET, 0, + [Default value for socket style to use with SCTP BTL]) + AC_DEFINE(OMPI_MCA_BTL_SCTP_CONCATENATES_IOVS, 0, + [False if you can use iovec's directly with SCTP BTL]) AC_MSG_WARN([Adding -DFREEBSD to set extra sin_len field in sockaddr.]) ;; + *solaris*) + # Solaris SCTP stack makes different assumptions about one-to-many + # sockets so change the default to use one-to-one sockets + ompi_sctp_try_to_build="yes" + AC_DEFINE(OMPI_MCA_BTL_SCTP_USE_ONE_TO_ONE_SOCKET, 1, + [Default value for socket style to use with SCTP BTL]) + AC_DEFINE(OMPI_MCA_BTL_SCTP_CONCATENATES_IOVS, 1, + [False if you can use iovec's directly with SCTP BTL]) + ;; *) - AC_MSG_WARN([Only build sctp BTL on Linux, Mac OS X, and BSD variants]) + AC_MSG_WARN([Only build sctp BTL on Solaris, Linux, Mac OS X, and BSD variants]) ;; esac