From 457a4b632cc64f564039c18e5b7c61d028a5267a Mon Sep 17 00:00:00 2001 From: Tim Prins Date: Wed, 4 Aug 2004 23:42:51 +0000 Subject: [PATCH] Some more changes to the oob: All the user interface functions are now in mca/oob/base/base.h Anyone who uses the oob should just include this file. All component related functions have been moved to mca/oob/oob.h The reason for this change was to make the user interface more clear. This commit was SVN r1884. --- src/mca/oob/base/base.h | 337 +++++++++++++++++++--- src/mca/oob/base/oob_base_close.c | 2 +- src/mca/oob/base/oob_base_open.c | 2 +- src/mca/oob/base/oob_base_recv.c | 4 +- src/mca/oob/base/oob_base_recv_nb.c | 5 +- src/mca/oob/base/oob_base_send.c | 1 + src/mca/oob/base/oob_base_send_nb.c | 1 + src/mca/oob/cofs/src/oob_cofs.h | 1 + src/mca/oob/oob.h | 431 +++++++--------------------- src/mca/oob/tcp/oob_tcp.c | 3 + src/mca/oob/tcp/oob_tcp.h | 5 +- src/mca/oob/tcp/oob_tcp_recv.c | 4 +- src/runtime/ompi_rte_finalize.c | 2 +- src/runtime/ompi_rte_init.c | 2 +- test/mca/oob/oob_test.c | 24 +- test/mca/oob/oob_test_self.c | 1 + 16 files changed, 438 insertions(+), 387 deletions(-) diff --git a/src/mca/oob/base/base.h b/src/mca/oob/base/base.h index 5313789e22..b74b735337 100644 --- a/src/mca/oob/base/base.h +++ b/src/mca/oob/base/base.h @@ -3,49 +3,311 @@ */ /** @file: * - * Contains packing functions provided by the oob + * the oob framework */ -#ifndef MCA_OOB_BASE_H_ -#define MCA_OOB_BASE_H_ - -#include "ompi_config.h" - -#include "include/types.h" +#ifndef _MCA_OOB_BASE_H_ +#define _MCA_OOB_BASE_H_ #include "mca/mca.h" -#include "mca/oob/oob.h" - -/* - * This is the first module on the list. This is here temporarily - * to make things work - */ -extern mca_oob_t mca_oob; - -/** - * associate a component and a module that belongs to it - */ -struct mca_oob_base_info_t { - ompi_list_item_t super; - mca_oob_base_component_t *oob_component; - mca_oob_t *oob_module; -}; -typedef struct mca_oob_base_info_t mca_oob_base_info_t; - -/** - * declare the association structure as a class - */ -OBJ_CLASS_DECLARATION(mca_oob_base_info_t); +#include "mca/ns/ns.h" +#include /* - * Global functions for MCA overall collective open and close + * Well known address */ + +extern ompi_process_name_t mca_oob_name_any; +extern ompi_process_name_t mca_oob_name_seed; +extern ompi_process_name_t mca_oob_name_self; + +#define MCA_OOB_NAME_ANY &mca_oob_name_any +#define MCA_OOB_NAME_SEED &mca_oob_name_seed +#define MCA_OOB_NAME_SELF &mca_oob_name_self + +/* + * Other constants + */ + +#define MCA_OOB_TAG_ANY 0 + +/* + * OOB API + */ + +/** + * Supported datatypes for conversion operations. + */ +typedef enum { + MCA_OOB_BASE_BYTE, /**< a byte of data */ + MCA_OOB_BASE_INT16, /**< a 16 bit integer */ + MCA_OOB_BASE_INT32, /**< a 32 bit integer */ + MCA_OOB_BASE_STRING, /**< a NULL terminated string */ + MCA_OOB_BASE_PACKED /**< already packed data. */ +} mca_oob_base_type_t; + +/** +* General flags for send/recv +* +* An example of usage - to determine the size of the next available message w/out receiving it: +* +* int size = mca_oob_recv(name, 0, 0, MSG_OOB_PEEK|MSG_OOB_TRUNC); +*/ + +#define MCA_OOB_PEEK 0x01 /* flag to oob_recv to allow caller to peek a portion of the next available + * message w/out removing the message from the queue. */ +#define MCA_OOB_TRUNC 0x02 /* flag to oob_recv to return the actual size of the message even if the receive + buffer is smaller than the number of bytes available */ + #if defined(c_plusplus) || defined(__cplusplus) extern "C" { #endif - int mca_oob_base_open(void); - int mca_oob_base_init(bool *allow_multi_user_threads, bool *have_hidden_threads); - int mca_oob_base_close(void); + +/** +* Similiar to unix writev(2). +* +* @param peer (IN) Opaque name of peer process. +* @param msg (IN) Array of iovecs describing user buffers and lengths. +* @param count (IN) Number of elements in iovec array. +* @param tag (IN) User defined tag for matching send/recv. +* @param flags (IN) Currently unused. +* @return OMPI error code (<0) on error number of bytes actually sent. +* +* This routine provides semantics similar to unix send/writev with the addition of +* a tag parameter that can be used by the application to match the send w/ a specific +* receive. In other words - a recv call by the specified peer will only succeed when +* the corresponding (or wildcard) tag is used. +* +* The peer parameter represents an opaque handle to the peer process that +* is resolved by the oob layer (using the registry) to an actual physical network +* address. +*/ + +int mca_oob_send( + const ompi_process_name_t* peer, + const struct iovec *msg, + int count, + int tag, + int flags); + +/** +* Convert data (if required) to network byte order prior to sending to peer. +* +* @param peer (IN) Opaque name of peer process. +* @param msg (IN) Array of iovecs describing user buffers and lengths. +* @param types (IN) Parallel array to iovecs describing data type of each iovec element. +* @param count (IN) Number of elements in iovec array. +* @param tag (IN) User defined tag for matching send/recv. +* @param flags (IN) Currently unused. +* @return OMPI error code (<0) on error number of bytes actually sent. +* +* This routine is equivalent to mca_oob_send, with the exception that it excepts +* an additional array of type codes describing the data types contained within the +* iovec array. This information is used to convert the data to network byte order +* (if required) prior to transmission over the underlying network transport. +*/ + +int mca_oob_send_hton( + const ompi_process_name_t* peer, + const struct iovec *msg, + const mca_oob_base_type_t *types, + int count, + int tag, + int flags); + + +/** +* Similiar to unix readv(2) +* +* @param peer (IN) Opaque name of peer process or MCA_OOB_NAME_ANY for wildcard receive. +* @param msg (IN) Array of iovecs describing user buffers and lengths. +* @param count (IN) Number of elements in iovec array. +* @param tag (IN) User defined tag for matching send/recv. +* @param flags (IN) May be MCA_OOB_PEEK to return up to the number of bytes provided in the +* iovec array without removing the message from the queue. +* @return OMPI error code (<0) on error or number of bytes actually received. +* +* The OOB recv call is similar to unix recv/readv in that it requires the caller to manage +* memory associated w/ the message. The routine accepts an array of iovecs (msg); however, +* the caller must determine the appropriate number of elements (count) and allocate the +* buffer space for each entry. +* +* The tag parameter is provided to facilitate this. The user may define tags based on message +* type to determine the message layout and size, as the mca_oob_recv call will block until a message +* with the matching tag is received. +* +* Alternately, the flags parameter may be used to peek (MCA_OOB_PEEK) a portion of the message +* (e.g. a standard message header) or determine the overall message size (MCA_OOB_TRUNC|MCA_OOB_PEEK) +* without removing the message from the queue. +* +*/ + +int mca_oob_recv( + ompi_process_name_t* peer, + const struct iovec *msg, + int count, + int tag, + int flags); + +/** +* Receive data and convert (if required) to host byte order. +* +* @param peer (IN) Opaque name of peer process or MCA_OOB_NAME_ANY for wildcard receive. +* @param msg (IN) Array of iovecs describing user buffers and lengths. +* @param types (IN) Parallel array to iovecs describing data type of each iovec element. +* @param count (IN) Number of elements in iovec array. +* @param tag (IN) User defined tag for matching send/recv. +* @param flags (IN) May be MCA_OOB_PEEK to return up to the number of bytes provided in the +* iovec array without removing the message from the queue. +* @return OMPI error code (<0) on error or number of bytes actually received. +* +* This routine is equivalent to mca_oob_recv, with the exception that it accepts +* an additional array of type codes describing the data types contained within the +* iovec array. This information is used to convert the data from network byte order +* (if required) to host byte order prior to receiving into the users buffer. +* +*/ + +int mca_oob_recv_ntoh( + ompi_process_name_t* peer, + const struct iovec *msg, + const mca_oob_base_type_t *types, + int count, + int tag, + int flags); + +/* + * Non-blocking versions of send/recv. +*/ + + +/** +* Callback function on send/recv completion. +* +* @param status (IN) Completion status - equivalent to the return value from blocking send/recv. +* @param peer (IN) Opaque name of peer process. +* @param msg (IN) Array of iovecs describing user buffers and lengths. +* @param count (IN) Number of elements in iovec array. +* @param tag (IN) User defined tag for matching send/recv. +* @param cbdata (IN) User data. +*/ + +typedef void (*mca_oob_callback_fn_t)( + int status, + const ompi_process_name_t* peer, + const struct iovec* msg, + int count, + int tag, + void* cbdata); + +/** +* Non-blocking version of mca_oob_send(). +* +* @param peer (IN) Opaque name of peer process. +* @param msg (IN) Array of iovecs describing user buffers and lengths. +* @param count (IN) Number of elements in iovec array. +* @param tag (IN) User defined tag for matching send/recv. +* @param flags (IN) Currently unused. +* @param cbfunc (IN) Callback function on send completion. +* @param cbdata (IN) User data that is passed to callback function. +* @return OMPI error code (<0) on error number of bytes actually sent. +* +* The user supplied callback function is called when the send completes. Note that +* the callback may occur before the call to mca_oob_send returns to the caller, +* if the send completes during the call. +* +*/ + +int mca_oob_send_nb( + const ompi_process_name_t* peer, + const struct iovec* msg, + int count, + int tag, + int flags, + mca_oob_callback_fn_t cbfunc, + void* cbdata); + +/** +* Non-blocking version of mca_oob_send_hton(). +* +* @param peer (IN) Opaque name of peer process. +* @param msg (IN) Array of iovecs describing user buffers and lengths. +* @param types (IN) Parallel array to iovecs describing data type of each iovec element. +* @param count (IN) Number of elements in iovec array. +* @param tag (IN) User defined tag for matching send/recv. +* @param flags (IN) Currently unused. +* @param cbfunc (IN) Callback function on send completion. +* @param cbdata (IN) User data that is passed to callback function. +* @return OMPI error code (<0) on error number of bytes actually sent. +* +* The user supplied callback function is called when the send completes. Note that +* the callback may occur before the call to mca_oob_send returns to the caller, +* if the send completes during the call. +* +*/ + +int mca_oob_send_hton_nb( + const ompi_process_name_t* peer, + const struct iovec* msg, + const mca_oob_base_type_t* types, + int count, + int tag, + int flags, + mca_oob_callback_fn_t cbfunc, + void* cbdata); + +/** +* Non-blocking version of mca_oob_recv(). +* +* @param peer (IN) Opaque name of peer process or MCA_OOB_NAME_ANY for wildcard receive. +* @param msg (IN) Array of iovecs describing user buffers and lengths. +* @param count (IN) Number of elements in iovec array. +* @param tag (IN) User defined tag for matching send/recv. +* @param flags (IN) May be MCA_OOB_PEEK to return up to size bytes of msg w/out removing it from the queue, +* @param cbfunc (IN) Callback function on recv completion. +* @param cbdata (IN) User data that is passed to callback function. +* @return OMPI error code (<0) on error or number of bytes actually received. +* +* The user supplied callback function is called asynchronously when a message is received +* that matches the call parameters. +*/ + +int mca_oob_recv_nb( + ompi_process_name_t* peer, + const struct iovec* msg, + int count, + int tag, + int flags, + mca_oob_callback_fn_t cbfunc, + void* cbdata); + +/** +* Non-blocking version of mca_oob_recv_ntoh(). +* +* @param peer (IN/OUT) Opaque name of peer process or MCA_OOB_NAME_ANY for wildcard receive. +* @param msg (IN) Array of iovecs describing user buffers and lengths. +* @param types (IN) Parallel array to iovecs describing data type of each iovec element. +* @param count (IN) Number of elements in iovec array. +* @param tag (IN) User defined tag for matching send/recv. +* @param flags (IN) May be MCA_OOB_PEEK to return up to size bytes of msg w/out removing it from the queue, +* @param cbfunc (IN) Callback function on recv completion. +* @param cbdata (IN) User data that is passed to callback function. +* @return OMPI error code (<0) on error or number of bytes actually received. +* +* The user supplied callback function is called asynchronously when a message is received +* that matches the call parameters. +*/ + +int mca_oob_recv_ntoh_nb( + ompi_process_name_t* peer, + const struct iovec* msg, + const mca_oob_base_type_t* types, + int count, + int tag, + int flags, + mca_oob_callback_fn_t cbfunc, + void* cbdata); + /* * functions for pack and unpack routines @@ -81,12 +343,5 @@ extern "C" { } #endif - -/* - * Global struct holding the selected module's function pointers - */ -extern int mca_oob_base_output; -extern ompi_list_t mca_oob_base_components; -extern ompi_list_t mca_oob_base_modules; - #endif + diff --git a/src/mca/oob/base/oob_base_close.c b/src/mca/oob/base/oob_base_close.c index 6b5eb0716e..2d9cb4f2d0 100644 --- a/src/mca/oob/base/oob_base_close.c +++ b/src/mca/oob/base/oob_base_close.c @@ -9,7 +9,7 @@ #include "include/constants.h" #include "mca/mca.h" #include "mca/base/base.h" -#include "mca/oob/base/base.h" +#include "mca/oob/oob.h" int mca_oob_base_close(void) diff --git a/src/mca/oob/base/oob_base_open.c b/src/mca/oob/base/oob_base_open.c index 11a322acdd..1eeb322962 100644 --- a/src/mca/oob/base/oob_base_open.c +++ b/src/mca/oob/base/oob_base_open.c @@ -6,7 +6,7 @@ #include "mca/mca.h" #include "mca/base/base.h" -#include "mca/oob/base/base.h" +#include "mca/oob/oob.h" /* diff --git a/src/mca/oob/base/oob_base_recv.c b/src/mca/oob/base/oob_base_recv.c index bb91a31f22..584b9b98ae 100644 --- a/src/mca/oob/base/oob_base_recv.c +++ b/src/mca/oob/base/oob_base_recv.c @@ -6,7 +6,7 @@ /* * Similiar to unix recv(2) * -* @param peer (IN) Opaque name of peer process or MCA_OOB_BASE_ANY for wildcard receive. +* @param peer (IN) Opaque name of peer process or MCA_OOB_NAME_ANY for wildcard receive. * @param msg (IN) Array of iovecs describing user buffers and lengths. * @param types (IN) Parallel array to iovecs describing data type of each iovec element. * @param count (IN) Number of elements in iovec array. @@ -23,7 +23,7 @@ int mca_oob_recv(ompi_process_name_t* peer, const struct iovec *msg, int count, /* * Receive data and convert (if required) to host byte order. * - * @param peer (IN) Opaque name of peer process or MCA_OOB_BASE_ANY for wildcard receive. + * @param peer (IN) Opaque name of peer process or MCA_OOB_NAME_ANY for wildcard receive. * @param msg (IN) Array of iovecs describing user buffers and lengths. * @param types (IN) Parallel array to iovecs describing data type of each iovec element. * @param count (IN) Number of elements in iovec array. diff --git a/src/mca/oob/base/oob_base_recv_nb.c b/src/mca/oob/base/oob_base_recv_nb.c index fb66d13adf..7cba1c6a65 100644 --- a/src/mca/oob/base/oob_base_recv_nb.c +++ b/src/mca/oob/base/oob_base_recv_nb.c @@ -1,5 +1,6 @@ #include "mca/oob/oob.h" #include "mca/oob/base/base.h" +#include #include struct mca_oob_base_cb_data_t { @@ -52,7 +53,7 @@ static void mca_oob_base_recv_cb(int status, const ompi_process_name_t* peer, /* * Non-blocking version of mca_oob_recv_nb(). * - * @param peer (IN) Opaque name of peer process or MCA_OOB_BASE_ANY for wildcard receive. + * @param peer (IN) Opaque name of peer process or MCA_OOB_NAME_ANY for wildcard receive. * @param msg (IN) Array of iovecs describing user buffers and lengths. * @param count (IN) Number of elements in iovec array. * @param flags (IN) May be MCA_OOB_PEEK to return up to size bytes of msg w/out removing it from the queue, @@ -69,7 +70,7 @@ int mca_oob_recv_nb(ompi_process_name_t* peer, const struct iovec* msg, int coun /* * Non-blocking version of mca_oob_recv_ntoh(). * - * @param peer (IN/OUT) Opaque name of peer process or MCA_OOB_BASE_ANY for wildcard receive. + * @param peer (IN/OUT) Opaque name of peer process or MCA_OOB_NAME_ANY for wildcard receive. * @param msg (IN) Array of iovecs describing user buffers and lengths. * @param types (IN) Parallel array to iovecs describing data type of each iovec element. * @param count (IN) Number of elements in iovec array. diff --git a/src/mca/oob/base/oob_base_send.c b/src/mca/oob/base/oob_base_send.c index d55e82f6a4..b300cad8e1 100644 --- a/src/mca/oob/base/oob_base_send.c +++ b/src/mca/oob/base/oob_base_send.c @@ -1,5 +1,6 @@ #include "mca/oob/oob.h" #include "mca/oob/base/base.h" +#include #include /* * Similiar to unix send(2). diff --git a/src/mca/oob/base/oob_base_send_nb.c b/src/mca/oob/base/oob_base_send_nb.c index 2e9b6dcd5b..7dfe3d1d80 100644 --- a/src/mca/oob/base/oob_base_send_nb.c +++ b/src/mca/oob/base/oob_base_send_nb.c @@ -1,5 +1,6 @@ #include "mca/oob/oob.h" #include "mca/oob/base/base.h" +#include #include struct mca_oob_base_cb_data_t { diff --git a/src/mca/oob/cofs/src/oob_cofs.h b/src/mca/oob/cofs/src/oob_cofs.h index 801a73401b..a0d8235d31 100644 --- a/src/mca/oob/cofs/src/oob_cofs.h +++ b/src/mca/oob/cofs/src/oob_cofs.h @@ -8,6 +8,7 @@ #include "ompi_config.h" #include "mca/oob/oob.h" +#include "mca/oob/base/base.h" #include "include/types.h" /* diff --git a/src/mca/oob/oob.h b/src/mca/oob/oob.h index a075701480..3c2844db1a 100644 --- a/src/mca/oob/oob.h +++ b/src/mca/oob/oob.h @@ -3,322 +3,31 @@ */ /** @file: * - * the oob framework + * Contains the internal functions and typedefs for the use of the oob */ -#ifndef _MCA_OOB_H_ -#define _MCA_OOB_H_ +#ifndef MCA_OOB_H_ +#define MCA_OOB_H_ + +#include "ompi_config.h" + +#include "include/types.h" #include "mca/mca.h" -#include "mca/base/base.h" -#include "mca/ns/ns.h" -#include - - -/* - * Well known address +#include "mca/oob/base/base.h" +/******** + * NOTE: these are functions and prototypes for the use of the modules + * and components. + * THESE ARE NOT USER INTERFACE FUNCTIONS. + * the user interface is in mca/oob/base/base.h */ -extern ompi_process_name_t mca_oob_name_any; -extern ompi_process_name_t mca_oob_name_seed; -extern ompi_process_name_t mca_oob_name_self; - -#define MCA_OOB_NAME_ANY &mca_oob_name_any -#define MCA_OOB_NAME_SEED &mca_oob_name_seed -#define MCA_OOB_NAME_SELF &mca_oob_name_self - -/* - * Other constants - */ - -#define MCA_OOB_TAG_ANY 0 - -/* - * OOB API - */ - -/** - * Supported datatypes for conversion operations. - */ -typedef enum { - MCA_OOB_BASE_BYTE, /**< a byte of data */ - MCA_OOB_BASE_INT16, /**< a 16 bit integer */ - MCA_OOB_BASE_INT32, /**< a 32 bit integer */ - MCA_OOB_BASE_STRING, /**< a NULL terminated string */ - MCA_OOB_BASE_PACKED /**< already packed data. */ -} mca_oob_base_type_t; - -/** -* General flags for send/recv -* -* An example of usage - to determine the size of the next available message w/out receiving it: -* -* int size = mca_oob_recv(name, 0, 0, MSG_OOB_PEEK|MSG_OOB_TRUNC); -*/ - -#define MCA_OOB_PEEK 0x01 /* flag to oob_recv to allow caller to peek a portion of the next available - * message w/out removing the message from the queue. */ -#define MCA_OOB_TRUNC 0x02 /* flag to oob_recv to return the actual size of the message even if the receive - buffer is smaller than the number of bytes available */ - -#if defined(c_plusplus) || defined(__cplusplus) -extern "C" { -#endif - -/** -* Similiar to unix writev(2). -* -* @param peer (IN) Opaque name of peer process. -* @param msg (IN) Array of iovecs describing user buffers and lengths. -* @param count (IN) Number of elements in iovec array. -* @param tag (IN) User defined tag for matching send/recv. -* @param flags (IN) Currently unused. -* @return OMPI error code (<0) on error number of bytes actually sent. -* -* This routine provides semantics similar to unix send/writev with the addition of -* a tag parameter that can be used by the application to match the send w/ a specific -* receive. In other words - a recv call by the specified peer will only succeed when -* the corresponding (or wildcard) tag is used. -* -* The peer parameter represents an opaque handle to the peer process that -* is resolved by the oob layer (using the registry) to an actual physical network -* address. -*/ - -int mca_oob_send( - const ompi_process_name_t* peer, - const struct iovec *msg, - int count, - int tag, - int flags); - -/** -* Convert data (if required) to network byte order prior to sending to peer. -* -* @param peer (IN) Opaque name of peer process. -* @param msg (IN) Array of iovecs describing user buffers and lengths. -* @param types (IN) Parallel array to iovecs describing data type of each iovec element. -* @param count (IN) Number of elements in iovec array. -* @param tag (IN) User defined tag for matching send/recv. -* @param flags (IN) Currently unused. -* @return OMPI error code (<0) on error number of bytes actually sent. -* -* This routine is equivalent to mca_oob_send, with the exception that it excepts -* an additional array of type codes describing the data types contained within the -* iovec array. This information is used to convert the data to network byte order -* (if required) prior to transmission over the underlying network transport. -*/ - -int mca_oob_send_hton( - const ompi_process_name_t* peer, - const struct iovec *msg, - const mca_oob_base_type_t *types, - int count, - int tag, - int flags); - - -/** -* Similiar to unix readv(2) -* -* @param peer (IN) Opaque name of peer process or MCA_OOB_BASE_ANY for wildcard receive. -* @param msg (IN) Array of iovecs describing user buffers and lengths. -* @param count (IN) Number of elements in iovec array. -* @param tag (IN) User defined tag for matching send/recv. -* @param flags (IN) May be MCA_OOB_PEEK to return up to the number of bytes provided in the -* iovec array without removing the message from the queue. -* @return OMPI error code (<0) on error or number of bytes actually received. -* -* The OOB recv call is similar to unix recv/readv in that it requires the caller to manage -* memory associated w/ the message. The routine accepts an array of iovecs (msg); however, -* the caller must determine the appropriate number of elements (count) and allocate the -* buffer space for each entry. -* -* The tag parameter is provided to facilitate this. The user may define tags based on message -* type to determine the message layout and size, as the mca_oob_recv call will block until a message -* with the matching tag is received. -* -* Alternately, the flags parameter may be used to peek (MCA_OOB_PEEK) a portion of the message -* (e.g. a standard message header) or determine the overall message size (MCA_OOB_TRUNC|MCA_OOB_PEEK) -* without removing the message from the queue. -* -*/ - -int mca_oob_recv( - ompi_process_name_t* peer, - const struct iovec *msg, - int count, - int tag, - int flags); - -/** -* Receive data and convert (if required) to host byte order. -* -* @param peer (IN) Opaque name of peer process or MCA_OOB_BASE_ANY for wildcard receive. -* @param msg (IN) Array of iovecs describing user buffers and lengths. -* @param types (IN) Parallel array to iovecs describing data type of each iovec element. -* @param count (IN) Number of elements in iovec array. -* @param tag (IN) User defined tag for matching send/recv. -* @param flags (IN) May be MCA_OOB_PEEK to return up to the number of bytes provided in the -* iovec array without removing the message from the queue. -* @return OMPI error code (<0) on error or number of bytes actually received. -* -* This routine is equivalent to mca_oob_recv, with the exception that it accepts -* an additional array of type codes describing the data types contained within the -* iovec array. This information is used to convert the data from network byte order -* (if required) to host byte order prior to receiving into the users buffer. -* -*/ - -int mca_oob_recv_ntoh( - ompi_process_name_t* peer, - const struct iovec *msg, - const mca_oob_base_type_t *types, - int count, - int tag, - int flags); - -/* - * Non-blocking versions of send/recv. -*/ - - -/** -* Callback function on send/recv completion. -* -* @param status (IN) Completion status - equivalent to the return value from blocking send/recv. -* @param peer (IN) Opaque name of peer process. -* @param msg (IN) Array of iovecs describing user buffers and lengths. -* @param count (IN) Number of elements in iovec array. -* @param tag (IN) User defined tag for matching send/recv. -* @param cbdata (IN) User data. -*/ - -typedef void (*mca_oob_callback_fn_t)( - int status, - const ompi_process_name_t* peer, - const struct iovec* msg, - int count, - int tag, - void* cbdata); - -/** -* Non-blocking version of mca_oob_send(). -* -* @param peer (IN) Opaque name of peer process. -* @param msg (IN) Array of iovecs describing user buffers and lengths. -* @param count (IN) Number of elements in iovec array. -* @param tag (IN) User defined tag for matching send/recv. -* @param flags (IN) Currently unused. -* @param cbfunc (IN) Callback function on send completion. -* @param cbdata (IN) User data that is passed to callback function. -* @return OMPI error code (<0) on error number of bytes actually sent. -* -* The user supplied callback function is called when the send completes. Note that -* the callback may occur before the call to mca_oob_send returns to the caller, -* if the send completes during the call. -* -*/ - -int mca_oob_send_nb( - const ompi_process_name_t* peer, - const struct iovec* msg, - int count, - int tag, - int flags, - mca_oob_callback_fn_t cbfunc, - void* cbdata); - -/** -* Non-blocking version of mca_oob_send_hton(). -* -* @param peer (IN) Opaque name of peer process. -* @param msg (IN) Array of iovecs describing user buffers and lengths. -* @param types (IN) Parallel array to iovecs describing data type of each iovec element. -* @param count (IN) Number of elements in iovec array. -* @param tag (IN) User defined tag for matching send/recv. -* @param flags (IN) Currently unused. -* @param cbfunc (IN) Callback function on send completion. -* @param cbdata (IN) User data that is passed to callback function. -* @return OMPI error code (<0) on error number of bytes actually sent. -* -* The user supplied callback function is called when the send completes. Note that -* the callback may occur before the call to mca_oob_send returns to the caller, -* if the send completes during the call. -* -*/ - -int mca_oob_send_hton_nb( - const ompi_process_name_t* peer, - const struct iovec* msg, - const mca_oob_base_type_t* types, - int count, - int tag, - int flags, - mca_oob_callback_fn_t cbfunc, - void* cbdata); - -/** -* Non-blocking version of mca_oob_recv(). -* -* @param peer (IN) Opaque name of peer process or MCA_OOB_BASE_ANY for wildcard receive. -* @param msg (IN) Array of iovecs describing user buffers and lengths. -* @param count (IN) Number of elements in iovec array. -* @param tag (IN) User defined tag for matching send/recv. -* @param flags (IN) May be MCA_OOB_PEEK to return up to size bytes of msg w/out removing it from the queue, -* @param cbfunc (IN) Callback function on recv completion. -* @param cbdata (IN) User data that is passed to callback function. -* @return OMPI error code (<0) on error or number of bytes actually received. -* -* The user supplied callback function is called asynchronously when a message is received -* that matches the call parameters. -*/ - -int mca_oob_recv_nb( - ompi_process_name_t* peer, - const struct iovec* msg, - int count, - int tag, - int flags, - mca_oob_callback_fn_t cbfunc, - void* cbdata); - -/** -* Non-blocking version of mca_oob_recv_ntoh(). -* -* @param peer (IN/OUT) Opaque name of peer process or MCA_OOB_BASE_ANY for wildcard receive. -* @param msg (IN) Array of iovecs describing user buffers and lengths. -* @param types (IN) Parallel array to iovecs describing data type of each iovec element. -* @param count (IN) Number of elements in iovec array. -* @param tag (IN) User defined tag for matching send/recv. -* @param flags (IN) May be MCA_OOB_PEEK to return up to size bytes of msg w/out removing it from the queue, -* @param cbfunc (IN) Callback function on recv completion. -* @param cbdata (IN) User data that is passed to callback function. -* @return OMPI error code (<0) on error or number of bytes actually received. -* -* The user supplied callback function is called asynchronously when a message is received -* that matches the call parameters. -*/ - -int mca_oob_recv_ntoh_nb( - ompi_process_name_t* peer, - const struct iovec* msg, - const mca_oob_base_type_t* types, - int count, - int tag, - int flags, - mca_oob_callback_fn_t cbfunc, - void* cbdata); - -#if defined(c_plusplus) || defined(__cplusplus) -} -#endif /* * OOB Component/Module function prototypes. */ /** * Implementation of mca_oob_send(). -* +* * @param peer (IN) Opaque name of peer process. * @param msg (IN) Array of iovecs describing user buffers and lengths. * @param count (IN) Number of elements in iovec array. @@ -328,17 +37,16 @@ int mca_oob_recv_ntoh_nb( */ typedef int (*mca_oob_base_module_send_fn_t)( - const ompi_process_name_t* peer, - const struct iovec *msg, - int count, + const ompi_process_name_t* peer, + const struct iovec *msg, + int count, int tag, int flags); - /** * Implementation of mca_oob_recv(). * -* @param peer (IN) Opaque name of peer process or MCA_OOB_BASE_ANY for wildcard receive. +* @param peer (IN) Opaque name of peer process or MCA_OOB_NAME_ANY for wildcard receive. * @param msg (IN) Array of iovecs describing user buffers and lengths. * @param types (IN) Parallel array to iovecs describing data type of each iovec element. * @param count (IN) Number of elements in iovec array. @@ -349,9 +57,9 @@ typedef int (*mca_oob_base_module_send_fn_t)( */ typedef int (*mca_oob_base_module_recv_fn_t)( - ompi_process_name_t* peer, - const struct iovec *msg, - int count, + ompi_process_name_t* peer, + const struct iovec *msg, + int count, int tag, int flags); @@ -370,18 +78,18 @@ typedef int (*mca_oob_base_module_recv_fn_t)( */ typedef int (*mca_oob_base_module_send_nb_fn_t)( - const ompi_process_name_t* peer, - const struct iovec* msg, - int count, + const ompi_process_name_t* peer, + const struct iovec* msg, + int count, int tag, - int flags, + int flags, mca_oob_callback_fn_t cbfunc, void* cbdata); /** * Implementation of mca_oob_recv_nb(). * -* @param peer (IN) Opaque name of peer process or MCA_OOB_BASE_ANY for wildcard receive. +* @param peer (IN) Opaque name of peer process or MCA_OOB_NAME_ANY for wildcard receive. * @param msg (IN) Array of iovecs describing user buffers and lengths. * @param count (IN) Number of elements in iovec array. * @param tag (IN) User defined tag for matching send/recv. @@ -392,11 +100,11 @@ typedef int (*mca_oob_base_module_send_nb_fn_t)( */ typedef int (*mca_oob_base_module_recv_nb_fn_t)( - ompi_process_name_t* peer, - const struct iovec* msg, - int count, + ompi_process_name_t* peer, + const struct iovec* msg, + int count, int tag, - int flags, + int flags, mca_oob_callback_fn_t cbfunc, void* cbdata); @@ -405,7 +113,7 @@ typedef int (*mca_oob_base_module_recv_nb_fn_t)( */ struct mca_oob_1_0_0_t { - mca_oob_base_module_send_fn_t oob_send; + mca_oob_base_module_send_fn_t oob_send; mca_oob_base_module_recv_fn_t oob_recv; mca_oob_base_module_send_nb_fn_t oob_send_nb; mca_oob_base_module_recv_nb_fn_t oob_recv_nb; @@ -441,10 +149,83 @@ typedef mca_oob_base_component_1_0_0_t mca_oob_base_component_t; * Macro for use in components that are of type oob v1.0.0 */ #define MCA_OOB_BASE_VERSION_1_0_0 \ - /* pml v1.0 is chained to MCA v1.0 */ \ + /* oob v1.0 is chained to MCA v1.0 */ \ MCA_BASE_VERSION_1_0_0, \ - /* pml v1.0 */ \ + /* oob v1.0 */ \ "oob", 1, 0, 0 +/* + * This is the first module on the list. This is here temporarily + * to make things work + */ +extern mca_oob_t mca_oob; + +/** + * associate a component and a module that belongs to it + */ +struct mca_oob_base_info_t { + ompi_list_item_t super; + mca_oob_base_component_t *oob_component; + mca_oob_t *oob_module; +}; +typedef struct mca_oob_base_info_t mca_oob_base_info_t; + +/** + * declare the association structure as a class + */ +OBJ_CLASS_DECLARATION(mca_oob_base_info_t); + + +/* + * Global functions for MCA overall collective open and close + */ +#if defined(c_plusplus) || defined(__cplusplus) +extern "C" { +#endif + int mca_oob_base_open(void); + int mca_oob_base_init(bool *allow_multi_user_threads, bool *have_hidden_threads); + int mca_oob_base_close(void); + +/* + * functions for pack and unpack routines + */ +/** + * This function packs the passed data according to the type enum. + * + * @param dest the destination for the packed data + * @param src the source of the data + * @param n the number of elements in the src + * @param type the type of data + * + * @retval OMPI_SUCCESS + * @retval OMPI_ERROR + */ + int mca_oob_base_pack(void * dest, void * src, size_t n, mca_oob_base_type_t type); + +/** + * This function unpacks the passed data according to the type enum. + * + * @param dest the destination for the unpacked data + * @param src the source of the packed data + * @param n the number of elements in the src + * @param type the type of the data to unpack + * + * @retval OMPI_SUCCESS + * @retval OMPI_ERROR + */ + int mca_oob_base_unpack(void * dest, void * src, size_t n, mca_oob_base_type_t type); + + +#if defined(c_plusplus) || defined(__cplusplus) +} #endif + +/* + * Global struct holding the selected module's function pointers + */ +extern int mca_oob_base_output; +extern ompi_list_t mca_oob_base_components; +extern ompi_list_t mca_oob_base_modules; + +#endif diff --git a/src/mca/oob/tcp/oob_tcp.c b/src/mca/oob/tcp/oob_tcp.c index 2150337a70..a41434823f 100644 --- a/src/mca/oob/tcp/oob_tcp.c +++ b/src/mca/oob/tcp/oob_tcp.c @@ -287,6 +287,8 @@ mca_oob_t* mca_oob_tcp_init(bool *allow_multi_user_threads, bool *have_hidden_th #if 0 /* intialize event library */ + memset(&mca_oob_tcp_component.tcp_recv_event, 0, sizeof(ompi_event_t)); + memset(&mca_oob_tcp_component.tcp_send_event, 0, sizeof(ompi_event_t)); if(ompi_event_init() != OMPI_SUCCESS) { ompi_output(0, "mca_oob_tcp_init: unable to initialize event library\n"); return NULL; @@ -316,6 +318,7 @@ int mca_oob_tcp_finalize(void) mca_oob_tcp_peer_close(peer); OBJ_DESTRUCT(peer); } + ompi_event_fini(); return OMPI_SUCCESS; } diff --git a/src/mca/oob/tcp/oob_tcp.h b/src/mca/oob/tcp/oob_tcp.h index af395cef1e..34c860f34d 100644 --- a/src/mca/oob/tcp/oob_tcp.h +++ b/src/mca/oob/tcp/oob_tcp.h @@ -11,6 +11,7 @@ #include "mca/oob/oob.h" #include "mca/oob/base/base.h" +#include "mca/base/base.h" #include "class/ompi_free_list.h" #include "class/ompi_rb_tree.h" #include "event/event.h" @@ -105,7 +106,7 @@ int mca_oob_tcp_send( /** * Similiar to unix readv(2) * - * @param peer (IN) Opaque name of peer process or MCA_OOB_BASE_ANY for wildcard receive. + * @param peer (IN) Opaque name of peer process or MCA_OOB_NAME_ANY for wildcard receive. * @param msg (IN) Array of iovecs describing user buffers and lengths. * @param count (IN) Number of elements in iovec array. * @param tag (IN) User defined tag for matching send/recv. @@ -152,7 +153,7 @@ int mca_oob_tcp_send_nb( /** * Non-blocking version of mca_oob_recv(). * - * @param peer (IN) Opaque name of peer process or MCA_OOB_BASE_ANY for wildcard receive. + * @param peer (IN) Opaque name of peer process or MCA_OOB_NAME_ANY for wildcard receive. * @param msg (IN) Array of iovecs describing user buffers and lengths. * @param count (IN) Number of elements in iovec array. * @param tag (IN) User defined tag for matching send/recv. diff --git a/src/mca/oob/tcp/oob_tcp_recv.c b/src/mca/oob/tcp/oob_tcp_recv.c index e7f885ac26..f0fe9d27eb 100644 --- a/src/mca/oob/tcp/oob_tcp_recv.c +++ b/src/mca/oob/tcp/oob_tcp_recv.c @@ -3,7 +3,7 @@ /* * Similiar to unix readv(2) * - * @param peer (IN) Opaque name of peer process or MCA_OOB_BASE_ANY for wildcard receive. + * @param peer (IN) Opaque name of peer process or MCA_OOB_NAME_ANY for wildcard receive. * @param msg (IN) Array of iovecs describing user buffers and lengths. * @param types (IN) Parallel array to iovecs describing data type of each iovec element. * @param count (IN) Number of elements in iovec array. @@ -89,7 +89,7 @@ int mca_oob_tcp_recv( /* * Non-blocking version of mca_oob_recv(). * - * @param peer (IN) Opaque name of peer process or MCA_OOB_BASE_ANY for wildcard receive. + * @param peer (IN) Opaque name of peer process or MCA_OOB_NAME_ANY for wildcard receive. * @param msg (IN) Array of iovecs describing user buffers and lengths. * @param count (IN) Number of elements in iovec array. * @param tag (IN) User supplied tag for matching send/recv. diff --git a/src/runtime/ompi_rte_finalize.c b/src/runtime/ompi_rte_finalize.c index f9839ded8e..70c46d2e6c 100644 --- a/src/runtime/ompi_rte_finalize.c +++ b/src/runtime/ompi_rte_finalize.c @@ -11,7 +11,7 @@ #include "util/output.h" #include "threads/mutex.h" #include "mca/pcm/base/base.h" -#include "mca/oob/base/base.h" +#include "mca/oob/oob.h" #include "mca/ns/base/base.h" diff --git a/src/runtime/ompi_rte_init.c b/src/runtime/ompi_rte_init.c index b24e36f608..635dc35a90 100644 --- a/src/runtime/ompi_rte_init.c +++ b/src/runtime/ompi_rte_init.c @@ -11,7 +11,7 @@ #include "util/output.h" #include "threads/mutex.h" #include "mca/pcm/base/base.h" -#include "mca/oob/base/base.h" +#include "mca/oob/oob.h" #include "mca/ns/base/base.h" diff --git a/test/mca/oob/oob_test.c b/test/mca/oob/oob_test.c index 25385afb14..20bbc92455 100644 --- a/test/mca/oob/oob_test.c +++ b/test/mca/oob/oob_test.c @@ -10,6 +10,7 @@ #include #include #include +#include #include "mpi.h" #include "support.h" #include "mca/oob/oob.h" @@ -17,8 +18,10 @@ #define MSG_TYPE_1 1 #define MSG_TYPE_2 2 #define NUM_TESTS 8 +#define NUM_TIMES 1 -bool testdone[NUM_TESTS]; +int i; +bool testdone[NUM_TESTS * NUM_TIMES]; void do_sends(ompi_process_name_t * peer); void do_recvs(ompi_process_name_t * peer); @@ -29,7 +32,6 @@ bool compare_iovec(const struct iovec * msg1, const struct iovec * msg2, int i; for(i = 0; i < n; i++) { if(msg1[i].iov_len != msg2[i].iov_len) { - fprintf(stderr, "len problem %d", i); return false; } if(0 != memcmp(msg1[i].iov_base, msg2[i].iov_base, msg1[i].iov_len)) { @@ -49,12 +51,12 @@ void callback(int status, const ompi_process_name_t * peer, if(0 != tag) { test_failure("Bad tag."); } - if((int) cbdata >= NUM_TESTS) { + if(((int) cbdata + (NUM_TESTS * i)) >= (NUM_TESTS * NUM_TIMES)) { test_failure("Bad value in callback function."); - } else if (testdone[(int) cbdata]) { + } else if (testdone[(int) cbdata + (NUM_TESTS * i)]) { test_failure("Callback function called on an already completed test."); } else { - testdone[(int) cbdata] = true; + testdone[(int) cbdata + (NUM_TESTS * i)] = true; test_success(); } } @@ -112,14 +114,18 @@ int main(int argc, char ** argv) test_init("oob send then recieve"); /* local vpid is 1 - peer is 0 */ peer.vpid = 0; - do_sends(&peer); - do_recvs(&peer); + for(i = 0; i < NUM_TIMES; i++) { + do_sends(&peer); + do_recvs(&peer); + } } else { test_init("oob recieve then send"); /* local vpid is 0 - peer is 1 */ peer.vpid = 1; - do_recvs(&peer); - do_sends(&peer); + for(i = 0; i < NUM_TIMES; i++) { + do_recvs(&peer); + do_sends(&peer); + } } /* done */ test_finalize(); diff --git a/test/mca/oob/oob_test_self.c b/test/mca/oob/oob_test_self.c index e99d2f2d8c..8f7a671258 100755 --- a/test/mca/oob/oob_test_self.c +++ b/test/mca/oob/oob_test_self.c @@ -6,6 +6,7 @@ #include #include #include +#include #include "mpi.h" #include "support.h" #include "mca/oob/oob.h"