From fb5536f11dd7a997f95dcf9e9a47c3ea35f59178 Mon Sep 17 00:00:00 2001 From: Brad Penoff Date: Wed, 21 Nov 2007 10:13:41 +0000 Subject: [PATCH] conforming SCTP BTL to Open MPI naming conventions and IP requirements This commit was SVN r16764. --- ompi/mca/btl/sctp/Makefile.am | 10 ++- ompi/mca/btl/sctp/btl_sctp_endpoint.c | 4 +- ompi/mca/btl/sctp/btl_sctp_frag.c | 35 ++++------ ompi/mca/btl/sctp/btl_sctp_utils.c | 45 ++++++++++++ ompi/mca/btl/sctp/btl_sctp_utils.h | 5 ++ ompi/mca/btl/sctp/sctp_writev.c | 98 --------------------------- ompi/mca/btl/sctp/sctp_writev.h | 35 ---------- 7 files changed, 69 insertions(+), 163 deletions(-) delete mode 100644 ompi/mca/btl/sctp/sctp_writev.c delete mode 100644 ompi/mca/btl/sctp/sctp_writev.h diff --git a/ompi/mca/btl/sctp/Makefile.am b/ompi/mca/btl/sctp/Makefile.am index ddb4ddaf11..aab9895346 100644 --- a/ompi/mca/btl/sctp/Makefile.am +++ b/ompi/mca/btl/sctp/Makefile.am @@ -35,12 +35,10 @@ sources = \ btl_sctp_hdr.h \ btl_sctp_proc.c \ btl_sctp_proc.h \ - sctp_writev.c \ - sctp_writev.h \ - btl_sctp_recv_handler.c \ - btl_sctp_recv_handler.h \ - btl_sctp_utils.c \ - btl_sctp_utils.h + btl_sctp_recv_handler.c \ + btl_sctp_recv_handler.h \ + btl_sctp_utils.c \ + btl_sctp_utils.h # Make the output library in this directory, and name it either # mca__.la (for DSO builds) or libmca__.la diff --git a/ompi/mca/btl/sctp/btl_sctp_endpoint.c b/ompi/mca/btl/sctp/btl_sctp_endpoint.c index e3e0b93afa..0b7c6ce116 100644 --- a/ompi/mca/btl/sctp/btl_sctp_endpoint.c +++ b/ompi/mca/btl/sctp/btl_sctp_endpoint.c @@ -883,12 +883,12 @@ int mca_btl_sctp_set_socket_options(int sd) #endif /* set socket up to be non-blocking */ - if((flags = fcntl(mca_btl_sctp_component.sctp_listen_sd, F_GETFL, 0)) < 0) { + if((flags = fcntl(sd, F_GETFL, 0)) < 0) { BTL_ERROR(("fcntl(F_GETFL) failed with errno=%d", opal_socket_errno)); return OMPI_ERROR; } else { flags |= O_NONBLOCK; - if(fcntl(mca_btl_sctp_component.sctp_listen_sd, F_SETFL, flags) < 0) { + if(fcntl(sd, F_SETFL, flags) < 0) { BTL_ERROR(("fcntl(F_SETFL) failed with errno=%d", opal_socket_errno)); return OMPI_ERROR; } diff --git a/ompi/mca/btl/sctp/btl_sctp_frag.c b/ompi/mca/btl/sctp/btl_sctp_frag.c index c2b346319d..46e17d65f4 100644 --- a/ompi/mca/btl/sctp/btl_sctp_frag.c +++ b/ompi/mca/btl/sctp/btl_sctp_frag.c @@ -39,10 +39,6 @@ #include "btl_sctp_endpoint.h" #include "orte/util/proc_info.h" -/* Needed to use sctp_writev() which is a slightly modified sctp_sendmsg() - * call. - */ -#include "sctp_writev.h" #include "btl_sctp.h" #include "btl_sctp_addr.h" #include "btl_sctp_utils.h" @@ -128,10 +124,7 @@ int mca_btl_sctp_frag_get_msg_size(mca_btl_sctp_frag_t *frag) { /** * mca_btl_sctp_frag_large_send(mca_btl_sctp_frag_t* frag, int sd) * --------------------------------------------------------------- - * Hopefully will fragment the message and send it on the wire. One approach is - * to fragment the message, then call mca_btl_sctp_frag_send() with each - * smaller fragment from a while loop until the entire message is sent. This is - * approach 1. + * Send a frag that is too large to send in one call to a vector write. */ bool mca_btl_sctp_frag_large_send(mca_btl_sctp_frag_t* frag, int sd, int iov_fragment, int *amt_sent) { @@ -168,9 +161,8 @@ bool mca_btl_sctp_frag_large_send(mca_btl_sctp_frag_t* frag, int sd, int iov_fra if(mca_btl_sctp_component.sctp_if_11) { cnt = sctp_sendmsg(sd, frag->iov_ptr->iov_base, to_send, 0, 0, 0, 0, 0, 0, 0 ); } else { - cnt = sctp_sendmsg(sd, frag->iov_ptr->iov_base, to_send, - (struct sockaddr *)&btl_sockaddr, sizeof(btl_sockaddr), 0, 0, - 0, 0, 0 ); + cnt = sctp_sendmsg(sd, frag->iov_ptr->iov_base, to_send, (struct sockaddr *)&btl_sockaddr, + sizeof(btl_sockaddr), 0, 0, 0, 0, 0 ); } @@ -185,12 +177,12 @@ bool mca_btl_sctp_frag_large_send(mca_btl_sctp_frag_t* frag, int sd, int iov_fra cnt=0; break; case EFAULT: - BTL_ERROR(("writev error (%p, %d)\n\t%s(%d)\n", + BTL_ERROR(("sctp_sendmsg error (%p, %d)\n\t%s(%d)\n", frag->iov_ptr[0].iov_base, frag->iov_ptr[0].iov_len, strerror(opal_socket_errno), frag->iov_cnt)); default: { - BTL_ERROR(("writev failed with errno=%d", opal_socket_errno)); + BTL_ERROR(("sctp_sendmsg failed with errno=%d", opal_socket_errno)); mca_btl_sctp_endpoint_close(frag->endpoint); return false; } @@ -295,10 +287,9 @@ bool mca_btl_sctp_frag_send(mca_btl_sctp_frag_t* frag, int sd) while(cnt < 0) { if(mca_btl_sctp_component.sctp_if_11) { - cnt = sctp_writev(sd, frag->iov_ptr, frag->iov_cnt, 0, 0, 0, 0, 0, 0, 0); + cnt = mca_btl_sctp_utils_writev(sd, frag->iov_ptr, frag->iov_cnt, 0, 0, 0); } else { - cnt = sctp_writev(sd, frag->iov_ptr, frag->iov_cnt, (struct sockaddr *)&btl_sockaddr, len, 0, - 0, 0, 0, 0); + cnt = mca_btl_sctp_utils_writev(sd, frag->iov_ptr, frag->iov_cnt, (struct sockaddr *)&btl_sockaddr, len, 0); } if(cnt >= 0) { @@ -310,12 +301,12 @@ bool mca_btl_sctp_frag_send(mca_btl_sctp_frag_t* frag, int sd) case EWOULDBLOCK: return false; case EFAULT: - BTL_ERROR(("writev error (%p, %d)\n\t%s(%d)\n", + BTL_ERROR(("mca_btl_sctp_utils_writev error (%p, %d)\n\t%s(%d)\n", frag->iov_ptr[0].iov_base, frag->iov_ptr[0].iov_len, strerror(opal_socket_errno), frag->iov_cnt)); default: { - BTL_ERROR(("writev failed with errno=%d", opal_socket_errno)); + BTL_ERROR(("mca_btl_sctp_utils_writev failed with errno=%d", opal_socket_errno)); mca_btl_sctp_endpoint_close(frag->endpoint); return false; } @@ -407,11 +398,11 @@ repeat11: case EWOULDBLOCK: return false; case EFAULT: - opal_output( 0, "mca_btl_sctp_frag_send: writev error (%p, %d)\n\t%s(%d)\n", + opal_output( 0, "mca_btl_sctp_frag_recv: readv error (%p, %d)\n\t%s(%d)\n", frag->iov_ptr[0].iov_base, (int) frag->iov_ptr[0].iov_len, strerror(opal_socket_errno), (int) frag->iov_cnt ); default: - opal_output(0, "mca_btl_sctp_frag_send: writev failed with errno=%d", + opal_output(0, "mca_btl_sctp_frag_recv: readv failed with errno=%d", opal_socket_errno); mca_btl_sctp_endpoint_close(btl_endpoint); return false; @@ -551,11 +542,11 @@ repeat: case EWOULDBLOCK: return false; case EFAULT: - opal_output( 0, "mca_btl_sctp_frag_send: writev error (%p, %d)\n\t%s(%d)\n", + opal_output( 0, "mca_btl_sctp_frag_recv: error (%p, %d)\n\t%s(%d)\n", frag->iov_ptr[0].iov_base, (int) frag->iov_ptr[0].iov_len, strerror(opal_socket_errno), (int) frag->iov_cnt ); default: - opal_output(0, "mca_btl_sctp_frag_send: writev failed with errno=%d", + opal_output(0, "mca_btl_sctp_frag_recv: failed with errno=%d", opal_socket_errno); mca_btl_sctp_endpoint_close(btl_endpoint); return false; diff --git a/ompi/mca/btl/sctp/btl_sctp_utils.c b/ompi/mca/btl/sctp/btl_sctp_utils.c index 6440f5b28c..112d252869 100644 --- a/ompi/mca/btl/sctp/btl_sctp_utils.c +++ b/ompi/mca/btl/sctp/btl_sctp_utils.c @@ -53,3 +53,48 @@ struct sockaddr_in mca_btl_sctp_utils_sockaddr_from_endpoint(struct mca_btl_base return btl_sockaddr; } + +/** + * int mca_btl_sctp_utils_writev(int sd, struct iovec *vec, size_t len, + * struct sockaddr *to_addr, socklen_t to_len, uint16_t stream_no) + * ----------------------------------------------------------------------- + * Vector write. + */ +int mca_btl_sctp_utils_writev(int sd, struct iovec *vec, size_t len, + struct sockaddr *to_addr, socklen_t to_len, uint16_t stream_no) { + + char outcmesg[CMSG_SPACE(sizeof(struct sctp_sndrcvinfo))]; + struct cmsghdr *cmesg; + struct msghdr outmesg; + struct sctp_sndrcvinfo *srinfo; + + /* Default these to 0. */ + uint32_t ppid = 0; + uint32_t flags = 0; + uint32_t ttl = 0; + uint32_t cxt = 0; + + outmesg.msg_name = to_addr; + outmesg.msg_namelen = to_len; + outmesg.msg_iov = vec; + outmesg.msg_iovlen = 1; + outmesg.msg_flags = 0; + outmesg.msg_control = outcmesg; + outmesg.msg_controllen = sizeof(outcmesg); + + cmesg = CMSG_FIRSTHDR(&outmesg); + cmesg->cmsg_len = CMSG_LEN(sizeof(struct sctp_sndrcvinfo)); + outmesg.msg_controllen = cmesg->cmsg_len; + cmesg->cmsg_level = IPPROTO_SCTP; + cmesg->cmsg_type = SCTP_SNDRCV; + + srinfo = (struct sctp_sndrcvinfo *)CMSG_DATA(cmesg); + memset(srinfo, 0, sizeof(struct sctp_sndrcvinfo)); + srinfo->sinfo_ppid = ppid; + srinfo->sinfo_flags = flags; + srinfo->sinfo_stream = stream_no; + srinfo->sinfo_timetolive = ttl; + srinfo->sinfo_context = cxt; + + return sendmsg(sd, &outmesg, 0); +} diff --git a/ompi/mca/btl/sctp/btl_sctp_utils.h b/ompi/mca/btl/sctp/btl_sctp_utils.h index 217b763734..b265d84426 100644 --- a/ompi/mca/btl/sctp/btl_sctp_utils.h +++ b/ompi/mca/btl/sctp/btl_sctp_utils.h @@ -23,8 +23,13 @@ #include "btl_sctp_frag.h" #include "btl_sctp_endpoint.h" #include "btl_sctp_addr.h" +#include +#include +#include struct sockaddr_in mca_btl_sctp_utils_sockaddr_from_frag(struct mca_btl_sctp_frag_t *frag); struct sockaddr_in mca_btl_sctp_utils_sockaddr_from_endpoint(struct mca_btl_base_endpoint_t *ep); +int mca_btl_sctp_utils_writev(int sd, struct iovec *vec, size_t len, + struct sockaddr *to_addr, socklen_t to_len, uint16_t stream_no); #endif diff --git a/ompi/mca/btl/sctp/sctp_writev.c b/ompi/mca/btl/sctp/sctp_writev.c deleted file mode 100644 index 323b51f601..0000000000 --- a/ompi/mca/btl/sctp/sctp_writev.c +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2006 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -/* SCTP kernel reference Implementation: User API extensions. - * - * sendmsg.c - * - * Distributed under the terms of the LGPL v2.1 as described in - * http://www.gnu.org/copyleft/lesser.txt - * - * This file is part of the user library that offers support for the - * SCTP kernel reference Implementation. The main purpose of this - * code is to provide the SCTP Socket API mappings for user - * application to interface with the SCTP in kernel. - * - * This implementation is based on the Socket API Extensions for SCTP - * defined in - * - * Copyright (c) 2003 Intel Corp. - * - * Written or modified by: - * Ardelle Fan - */ - - -/* Small modifications made by Karol Mroz (kmroz). */ - -/* #include - * #include - * #include - */ -#include "sctp_writev.h" - -/* This library function assists the user with the advanced features - * of SCTP. This is a new SCTP API described in the section 8.7 of the - * Sockets API Extensions for SCTP. This is implemented using the - * sendmsg() interface. - * - * kmroz: Modification to this was trivial. const char* was replaced with const - * struct iovec* and the sendmsg call was replaced with a call to writev that - * takes *vector as a parameter. - */ - -int -sctp_writev(int s, /* const */ struct iovec *vector, size_t len, struct sockaddr *to, - socklen_t tolen, uint32_t ppid, uint32_t flags, - uint16_t stream_no, uint32_t timetolive, uint32_t context) -{ - struct msghdr outmsg; - struct iovec iov; - char outcmsg[CMSG_SPACE(sizeof(struct sctp_sndrcvinfo))]; - struct cmsghdr *cmsg; - struct sctp_sndrcvinfo *sinfo; - - outmsg.msg_name = to; - outmsg.msg_namelen = tolen; - /* outmsg.msg_iov = &iov; */ - outmsg.msg_iov = vector; - iov.iov_base = vector->iov_base; - iov.iov_len = vector->iov_len; - outmsg.msg_iovlen = 1; - - outmsg.msg_control = outcmsg; - outmsg.msg_controllen = sizeof(outcmsg); - outmsg.msg_flags = 0; - - cmsg = CMSG_FIRSTHDR(&outmsg); - cmsg->cmsg_level = IPPROTO_SCTP; - cmsg->cmsg_type = SCTP_SNDRCV; - cmsg->cmsg_len = CMSG_LEN(sizeof(struct sctp_sndrcvinfo)); - - outmsg.msg_controllen = cmsg->cmsg_len; - sinfo = (struct sctp_sndrcvinfo *)CMSG_DATA(cmsg); - memset(sinfo, 0, sizeof(struct sctp_sndrcvinfo)); - sinfo->sinfo_ppid = ppid; - sinfo->sinfo_flags = flags; - sinfo->sinfo_stream = stream_no; - sinfo->sinfo_timetolive = timetolive; - sinfo->sinfo_context = context; - - return sendmsg(s, &outmsg, 0); - /* return writev(s, vector, len); */ -} diff --git a/ompi/mca/btl/sctp/sctp_writev.h b/ompi/mca/btl/sctp/sctp_writev.h deleted file mode 100644 index 47a46c87c9..0000000000 --- a/ompi/mca/btl/sctp/sctp_writev.h +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2006 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef MCA_BTL_SCTP_WRITEV_H -#define MCA_BTL_SCTP_WRITEV_H - -#include -#include -#include - -/* - * sctp_writev.h - * ------------- - */ - -int sctp_writev(int s, /* const */ struct iovec *vector, size_t len, struct sockaddr *to, - socklen_t tolen, uint32_t ppid, uint32_t flags, - uint16_t stream_no, uint32_t timetolive, uint32_t context); - -#endif