Some more changes to the oob:

All the user interface functions are now in mca/oob/base/base.h
Anyone who uses the oob should just include this file.

All component related functions have been moved to mca/oob/oob.h

The reason for this change was to make the user interface more clear.

This commit was SVN r1884.

src/mca/oob/base/base.h

@@ -3,49 +3,311 @@
  */
 /** @file:
  *
- * Contains packing functions provided by the oob
+ * the oob framework
  */
 
-#ifndef MCA_OOB_BASE_H_
+#ifndef _MCA_OOB_BASE_H_
-#define MCA_OOB_BASE_H_
+#define _MCA_OOB_BASE_H_
-
-#include "ompi_config.h"
-
-#include "include/types.h"
 #include "mca/mca.h"
-#include "mca/oob/oob.h"
+#include "mca/ns/ns.h"
-
+#include <sys/uio.h>
-/*
- * This is the first module on the list. This is here temporarily
- * to make things work
- */
-extern mca_oob_t mca_oob;
-
-/**
- * associate a component and a module that belongs to it
- */
-struct mca_oob_base_info_t {
-  ompi_list_item_t super;
-  mca_oob_base_component_t *oob_component;
-  mca_oob_t *oob_module;
-};
-typedef struct mca_oob_base_info_t mca_oob_base_info_t;
-
-/**
- * declare the association structure as a class
- */
-OBJ_CLASS_DECLARATION(mca_oob_base_info_t);
 
 
 /*
- * Global functions for MCA overall collective open and close
+ * Well known address
  */
+
+extern ompi_process_name_t mca_oob_name_any;
+extern ompi_process_name_t mca_oob_name_seed;
+extern ompi_process_name_t mca_oob_name_self;
+
+#define MCA_OOB_NAME_ANY  &mca_oob_name_any
+#define MCA_OOB_NAME_SEED &mca_oob_name_seed
+#define MCA_OOB_NAME_SELF &mca_oob_name_self
+
+/*
+ * Other constants
+ */
+
+#define MCA_OOB_TAG_ANY 0
+
+/*
+ * OOB API 
+ */
+
+/**
+ * Supported datatypes for conversion operations.
+ */
+typedef enum {
+    MCA_OOB_BASE_BYTE, /**< a byte of data */
+    MCA_OOB_BASE_INT16, /**< a 16 bit integer */
+    MCA_OOB_BASE_INT32, /**< a 32 bit integer */
+    MCA_OOB_BASE_STRING, /**< a NULL terminated string */
+    MCA_OOB_BASE_PACKED /**< already packed data. */
+} mca_oob_base_type_t;
+
+/**
+*   General flags for send/recv
+* 
+*   An example of usage - to determine the size of the next available message w/out receiving it:
+*
+*   int size = mca_oob_recv(name, 0, 0, MSG_OOB_PEEK|MSG_OOB_TRUNC);
+*/
+
+#define MCA_OOB_PEEK  0x01   /* flag to oob_recv to allow caller to peek a portion of the next available
+                              * message w/out removing the message from the queue.  */
+#define MCA_OOB_TRUNC 0x02   /* flag to oob_recv to return the actual size of the message even if the receive
+                                buffer is smaller than the number of bytes available */
+
 #if defined(c_plusplus) || defined(__cplusplus)
 extern "C" {
 #endif
-    int mca_oob_base_open(void);
+        
-    int mca_oob_base_init(bool *allow_multi_user_threads, bool *have_hidden_threads);
+/**
-    int mca_oob_base_close(void);
+*  Similiar to unix writev(2).
+*
+*  @param peer (IN)    Opaque name of peer process.
+*  @param msg (IN)     Array of iovecs describing user buffers and lengths.
+*  @param count (IN)   Number of elements in iovec array.
+*  @param tag (IN)     User defined tag for matching send/recv.
+*  @param flags (IN)   Currently unused.
+*  @return             OMPI error code (<0) on error number of bytes actually sent.
+*
+*  This routine provides semantics similar to unix send/writev with the addition of
+*  a tag parameter that can be used by the application to match the send w/ a specific
+*  receive. In other words - a recv call by the specified peer will only succeed when
+*  the corresponding (or wildcard) tag is used.
+*
+*  The <i>peer</i> parameter represents an opaque handle to the peer process that
+*  is resolved by the oob layer (using the registry) to an actual physical network
+*  address.
+*/
+
+int mca_oob_send(
+    const ompi_process_name_t* peer, 
+    const struct iovec *msg, 
+    int count, 
+    int tag,
+    int flags);
+
+/**
+* Convert data (if required) to network byte order prior to sending to peer.
+*
+* @param peer (IN)    Opaque name of peer process.
+* @param msg (IN)     Array of iovecs describing user buffers and lengths.
+* @param types (IN)   Parallel array to iovecs describing data type of each iovec element.
+* @param count (IN)   Number of elements in iovec array.
+* @param tag (IN)     User defined tag for matching send/recv.
+* @param flags (IN)   Currently unused.
+* @return             OMPI error code (<0) on error number of bytes actually sent.
+*
+* This routine is equivalent to mca_oob_send, with the exception that it excepts
+* an additional array of type codes describing the data types contained within the
+* iovec array. This information is used to convert the data to network byte order
+* (if required) prior to transmission over the underlying network transport.
+*/
+
+int mca_oob_send_hton(
+    const ompi_process_name_t* peer, 
+    const struct iovec *msg, 
+    const mca_oob_base_type_t *types, 
+    int count, 
+    int tag,
+    int flags);
+
+
+/**
+* Similiar to unix readv(2)
+*
+* @param peer (IN)    Opaque name of peer process or MCA_OOB_NAME_ANY for wildcard receive.
+* @param msg (IN)     Array of iovecs describing user buffers and lengths.
+* @param count (IN)   Number of elements in iovec array.
+* @param tag (IN)     User defined tag for matching send/recv.
+* @param flags (IN)   May be MCA_OOB_PEEK to return up to the number of bytes provided in the
+*                     iovec array without removing the message from the queue.
+* @return             OMPI error code (<0) on error or number of bytes actually received.
+*
+* The OOB recv call is similar to unix recv/readv in that it requires the caller to manage
+* memory associated w/ the message. The routine accepts an array of iovecs (<i>msg</i>); however,
+* the caller must determine the appropriate number of elements (<i>count</i>) and allocate the
+* buffer space for each entry. 
+*
+* The <i>tag</i> parameter is provided to facilitate this. The user may define tags based on message 
+* type to determine the message layout and size, as the mca_oob_recv call will block until a message
+* with the matching tag is received.
+*
+* Alternately, the <i>flags</i> parameter may be used to peek (MCA_OOB_PEEK) a portion of the message 
+* (e.g. a standard message header) or determine the overall message size (MCA_OOB_TRUNC|MCA_OOB_PEEK)
+* without removing the message from the queue. 
+*
+*/
+
+int mca_oob_recv(
+    ompi_process_name_t* peer, 
+    const struct iovec *msg, 
+    int count, 
+    int tag, 
+    int flags);
+
+/**
+* Receive data and convert (if required) to host byte order.
+*
+* @param peer (IN)    Opaque name of peer process or MCA_OOB_NAME_ANY for wildcard receive.
+* @param msg (IN)     Array of iovecs describing user buffers and lengths.
+* @param types (IN)   Parallel array to iovecs describing data type of each iovec element.
+* @param count (IN)   Number of elements in iovec array.
+* @param tag (IN)     User defined tag for matching send/recv.
+* @param flags (IN)   May be MCA_OOB_PEEK to return up to the number of bytes provided in the
+*                     iovec array without removing the message from the queue.
+* @return             OMPI error code (<0) on error or number of bytes actually received.
+*
+* This routine is equivalent to mca_oob_recv, with the exception that it accepts
+* an additional array of type codes describing the data types contained within the
+* iovec array. This information is used to convert the data from network byte order
+* (if required) to host byte order prior to receiving into the users buffer.
+* 
+*/
+
+int mca_oob_recv_ntoh(
+   ompi_process_name_t* peer, 
+   const struct iovec *msg, 
+   const mca_oob_base_type_t *types, 
+   int count, 
+   int tag,
+   int flags);
+
+/*
+ * Non-blocking versions of send/recv.
+*/
+
+
+/**
+*  Callback function on send/recv completion.
+*
+*  @param status (IN)  Completion status - equivalent to the return value from blocking send/recv.
+*  @param peer (IN)    Opaque name of peer process.
+*  @param msg (IN)     Array of iovecs describing user buffers and lengths.
+*  @param count (IN)   Number of elements in iovec array.
+*  @param tag (IN)     User defined tag for matching send/recv.
+*  @param cbdata (IN)  User data.
+*/
+
+typedef void (*mca_oob_callback_fn_t)(
+    int status,
+    const ompi_process_name_t* peer, 
+    const struct iovec* msg, 
+    int count,
+    int tag,
+    void* cbdata);
+
+/**
+*  Non-blocking version of mca_oob_send().
+*
+*  @param peer (IN)    Opaque name of peer process.
+*  @param msg (IN)     Array of iovecs describing user buffers and lengths.
+*  @param count (IN)   Number of elements in iovec array.
+*  @param tag (IN)     User defined tag for matching send/recv.
+*  @param flags (IN)   Currently unused.
+*  @param cbfunc (IN)  Callback function on send completion.
+*  @param cbdata (IN)  User data that is passed to callback function.
+*  @return             OMPI error code (<0) on error number of bytes actually sent.
+*
+*  The user supplied callback function is called when the send completes. Note that
+*  the callback may occur before the call to mca_oob_send returns to the caller, 
+*  if the send completes during the call.
+*
+*/
+
+int mca_oob_send_nb(
+    const ompi_process_name_t* peer, 
+    const struct iovec* msg, 
+    int count, 
+    int tag,
+    int flags, 
+    mca_oob_callback_fn_t cbfunc,
+    void* cbdata);
+
+/**
+*  Non-blocking version of mca_oob_send_hton().
+*
+*  @param peer (IN)    Opaque name of peer process.
+*  @param msg (IN)     Array of iovecs describing user buffers and lengths.
+*  @param types (IN)   Parallel array to iovecs describing data type of each iovec element.
+*  @param count (IN)   Number of elements in iovec array.
+*  @param tag (IN)     User defined tag for matching send/recv.
+*  @param flags (IN)   Currently unused.
+*  @param cbfunc (IN)  Callback function on send completion.
+*  @param cbdata (IN)  User data that is passed to callback function.
+*  @return             OMPI error code (<0) on error number of bytes actually sent.
+*
+*  The user supplied callback function is called when the send completes. Note that
+*  the callback may occur before the call to mca_oob_send returns to the caller, 
+*  if the send completes during the call.
+*
+*/
+
+int mca_oob_send_hton_nb(
+    const ompi_process_name_t* peer, 
+    const struct iovec* msg, 
+    const mca_oob_base_type_t* types,
+    int count, 
+    int tag,
+    int flags, 
+    mca_oob_callback_fn_t cbfunc,
+    void* cbdata);
+
+/**
+* Non-blocking version of mca_oob_recv().
+*
+* @param peer (IN)    Opaque name of peer process or MCA_OOB_NAME_ANY for wildcard receive.
+* @param msg (IN)     Array of iovecs describing user buffers and lengths.
+* @param count (IN)   Number of elements in iovec array.
+* @param tag (IN)     User defined tag for matching send/recv.
+* @param flags (IN)   May be MCA_OOB_PEEK to return up to size bytes of msg w/out removing it from the queue,
+* @param cbfunc (IN)  Callback function on recv completion.
+* @param cbdata (IN)  User data that is passed to callback function.
+* @return             OMPI error code (<0) on error or number of bytes actually received.
+*
+* The user supplied callback function is called asynchronously when a message is received 
+* that matches the call parameters.
+*/
+
+int mca_oob_recv_nb(
+    ompi_process_name_t* peer, 
+    const struct iovec* msg,  
+    int count, 
+    int tag,
+    int flags, 
+    mca_oob_callback_fn_t cbfunc,
+    void* cbdata);
+
+/**
+* Non-blocking version of mca_oob_recv_ntoh().
+*
+* @param peer (IN/OUT) Opaque name of peer process or MCA_OOB_NAME_ANY for wildcard receive.
+* @param msg (IN)      Array of iovecs describing user buffers and lengths.
+* @param types (IN)    Parallel array to iovecs describing data type of each iovec element.
+* @param count (IN)    Number of elements in iovec array.
+* @param tag (IN)      User defined tag for matching send/recv.
+* @param flags (IN)    May be MCA_OOB_PEEK to return up to size bytes of msg w/out removing it from the queue,
+* @param cbfunc (IN)   Callback function on recv completion.
+* @param cbdata (IN)   User data that is passed to callback function.
+* @return              OMPI error code (<0) on error or number of bytes actually received.
+* 
+* The user supplied callback function is called asynchronously when a message is received 
+* that matches the call parameters.
+*/
+
+int mca_oob_recv_ntoh_nb(
+    ompi_process_name_t* peer, 
+    const struct iovec* msg,  
+    const mca_oob_base_type_t* types,
+    int count, 
+    int tag,
+    int flags, 
+    mca_oob_callback_fn_t cbfunc,
+    void* cbdata);
+
 
 /*
  * functions for pack and unpack routines
@@ -81,12 +343,5 @@ extern "C" {
 }
 #endif
 
-
-/*
- * Global struct holding the selected module's function pointers
- */
-extern int mca_oob_base_output;
-extern ompi_list_t mca_oob_base_components;
-extern ompi_list_t mca_oob_base_modules;
-
 #endif
+

src/mca/oob/base/oob_base_close.c

@@ -9,7 +9,7 @@
 #include "include/constants.h"
 #include "mca/mca.h"
 #include "mca/base/base.h"
-#include "mca/oob/base/base.h"
+#include "mca/oob/oob.h"
 
 
 int mca_oob_base_close(void)

src/mca/oob/base/oob_base_open.c

@@ -6,7 +6,7 @@
 
 #include "mca/mca.h"
 #include "mca/base/base.h"
-#include "mca/oob/base/base.h"
+#include "mca/oob/oob.h"
 
 
 /*

src/mca/oob/base/oob_base_recv.c

@@ -6,7 +6,7 @@
 /*
 * Similiar to unix recv(2)
 *
-* @param peer (IN)    Opaque name of peer process or MCA_OOB_BASE_ANY for wildcard receive.
+* @param peer (IN)    Opaque name of peer process or MCA_OOB_NAME_ANY for wildcard receive.
 * @param msg (IN)     Array of iovecs describing user buffers and lengths.
 * @param types (IN)   Parallel array to iovecs describing data type of each iovec element.
 * @param count (IN)   Number of elements in iovec array.
@@ -23,7 +23,7 @@ int mca_oob_recv(ompi_process_name_t* peer, const struct iovec *msg, int count,
 /*
  * Receive data and convert (if required) to host byte order.
  *
- * @param peer (IN)    Opaque name of peer process or MCA_OOB_BASE_ANY for wildcard receive.
+ * @param peer (IN)    Opaque name of peer process or MCA_OOB_NAME_ANY for wildcard receive.
  * @param msg (IN)     Array of iovecs describing user buffers and lengths.
  * @param types (IN)   Parallel array to iovecs describing data type of each iovec element.
  * @param count (IN)   Number of elements in iovec array.

src/mca/oob/base/oob_base_recv_nb.c

@@ -1,5 +1,6 @@
 #include "mca/oob/oob.h"
 #include "mca/oob/base/base.h"
+#include <string.h>
 #include <netinet/in.h>
 
 struct mca_oob_base_cb_data_t {
@@ -52,7 +53,7 @@ static void mca_oob_base_recv_cb(int status, const ompi_process_name_t* peer,
 /*
  * Non-blocking version of mca_oob_recv_nb().
  *
- * @param peer (IN)    Opaque name of peer process or MCA_OOB_BASE_ANY for wildcard receive.
+ * @param peer (IN)    Opaque name of peer process or MCA_OOB_NAME_ANY for wildcard receive.
  * @param msg (IN)     Array of iovecs describing user buffers and lengths.
  * @param count (IN)   Number of elements in iovec array.
  * @param flags (IN)   May be MCA_OOB_PEEK to return up to size bytes of msg w/out removing it from the queue,
@@ -69,7 +70,7 @@ int mca_oob_recv_nb(ompi_process_name_t* peer, const struct iovec* msg, int coun
 /*
  * Non-blocking version of mca_oob_recv_ntoh().
  *
- * @param peer (IN/OUT) Opaque name of peer process or MCA_OOB_BASE_ANY for wildcard receive.
+ * @param peer (IN/OUT) Opaque name of peer process or MCA_OOB_NAME_ANY for wildcard receive.
  * @param msg (IN)      Array of iovecs describing user buffers and lengths.
  * @param types (IN)    Parallel array to iovecs describing data type of each iovec element.
  * @param count (IN)    Number of elements in iovec array.

src/mca/oob/base/oob_base_send.c

@@ -1,5 +1,6 @@
 #include "mca/oob/oob.h"
 #include "mca/oob/base/base.h"
+#include <string.h>
 #include <netinet/in.h>
 /*
 *  Similiar to unix send(2).

src/mca/oob/base/oob_base_send_nb.c

@@ -1,5 +1,6 @@
 #include "mca/oob/oob.h"
 #include "mca/oob/base/base.h"
+#include <string.h>
 #include <netinet/in.h>
 
 struct mca_oob_base_cb_data_t {

src/mca/oob/cofs/src/oob_cofs.h

@@ -8,6 +8,7 @@
 #include "ompi_config.h"
 
 #include "mca/oob/oob.h"
+#include "mca/oob/base/base.h"
 #include "include/types.h"
 
 /*

src/mca/oob/oob.h

@@ -3,315 +3,24 @@
  */
 /** @file:
  *
- * the oob framework
+ * Contains the internal functions and typedefs for the use of the oob
  */
 
-#ifndef _MCA_OOB_H_
+#ifndef MCA_OOB_H_
-#define _MCA_OOB_H_
+#define MCA_OOB_H_
+
+#include "ompi_config.h"
+
+#include "include/types.h"
 #include "mca/mca.h"
-#include "mca/base/base.h"
+#include "mca/oob/base/base.h"
-#include "mca/ns/ns.h"
+/********
-#include <sys/uio.h>
+ * NOTE: these are functions and prototypes for the use of the modules
-
+ *       and components. 
-
+ *       THESE ARE NOT USER INTERFACE FUNCTIONS.
-/*
+ *       the user interface is in mca/oob/base/base.h
- * Well known address
  */
 
-extern ompi_process_name_t mca_oob_name_any;
-extern ompi_process_name_t mca_oob_name_seed;
-extern ompi_process_name_t mca_oob_name_self;
-
-#define MCA_OOB_NAME_ANY  &mca_oob_name_any
-#define MCA_OOB_NAME_SEED &mca_oob_name_seed
-#define MCA_OOB_NAME_SELF &mca_oob_name_self
-
-/*
- * Other constants
- */
-
-#define MCA_OOB_TAG_ANY 0
-
-/*
- * OOB API 
- */
-
-/**
- * Supported datatypes for conversion operations.
- */
-typedef enum {
-    MCA_OOB_BASE_BYTE, /**< a byte of data */
-    MCA_OOB_BASE_INT16, /**< a 16 bit integer */
-    MCA_OOB_BASE_INT32, /**< a 32 bit integer */
-    MCA_OOB_BASE_STRING, /**< a NULL terminated string */
-    MCA_OOB_BASE_PACKED /**< already packed data. */
-} mca_oob_base_type_t;
-
-/**
-*   General flags for send/recv
-* 
-*   An example of usage - to determine the size of the next available message w/out receiving it:
-*
-*   int size = mca_oob_recv(name, 0, 0, MSG_OOB_PEEK|MSG_OOB_TRUNC);
-*/
-
-#define MCA_OOB_PEEK  0x01   /* flag to oob_recv to allow caller to peek a portion of the next available
-                              * message w/out removing the message from the queue.  */
-#define MCA_OOB_TRUNC 0x02   /* flag to oob_recv to return the actual size of the message even if the receive
-                                buffer is smaller than the number of bytes available */
-
-#if defined(c_plusplus) || defined(__cplusplus)
-extern "C" {
-#endif
-        
-/**
-*  Similiar to unix writev(2).
-*
-*  @param peer (IN)    Opaque name of peer process.
-*  @param msg (IN)     Array of iovecs describing user buffers and lengths.
-*  @param count (IN)   Number of elements in iovec array.
-*  @param tag (IN)     User defined tag for matching send/recv.
-*  @param flags (IN)   Currently unused.
-*  @return             OMPI error code (<0) on error number of bytes actually sent.
-*
-*  This routine provides semantics similar to unix send/writev with the addition of
-*  a tag parameter that can be used by the application to match the send w/ a specific
-*  receive. In other words - a recv call by the specified peer will only succeed when
-*  the corresponding (or wildcard) tag is used.
-*
-*  The <i>peer</i> parameter represents an opaque handle to the peer process that
-*  is resolved by the oob layer (using the registry) to an actual physical network
-*  address.
-*/
-
-int mca_oob_send(
-    const ompi_process_name_t* peer, 
-    const struct iovec *msg, 
-    int count, 
-    int tag,
-    int flags);
-
-/**
-* Convert data (if required) to network byte order prior to sending to peer.
-*
-* @param peer (IN)    Opaque name of peer process.
-* @param msg (IN)     Array of iovecs describing user buffers and lengths.
-* @param types (IN)   Parallel array to iovecs describing data type of each iovec element.
-* @param count (IN)   Number of elements in iovec array.
-* @param tag (IN)     User defined tag for matching send/recv.
-* @param flags (IN)   Currently unused.
-* @return             OMPI error code (<0) on error number of bytes actually sent.
-*
-* This routine is equivalent to mca_oob_send, with the exception that it excepts
-* an additional array of type codes describing the data types contained within the
-* iovec array. This information is used to convert the data to network byte order
-* (if required) prior to transmission over the underlying network transport.
-*/
-
-int mca_oob_send_hton(
-    const ompi_process_name_t* peer, 
-    const struct iovec *msg, 
-    const mca_oob_base_type_t *types, 
-    int count, 
-    int tag,
-    int flags);
-
-
-/**
-* Similiar to unix readv(2)
-*
-* @param peer (IN)    Opaque name of peer process or MCA_OOB_BASE_ANY for wildcard receive.
-* @param msg (IN)     Array of iovecs describing user buffers and lengths.
-* @param count (IN)   Number of elements in iovec array.
-* @param tag (IN)     User defined tag for matching send/recv.
-* @param flags (IN)   May be MCA_OOB_PEEK to return up to the number of bytes provided in the
-*                     iovec array without removing the message from the queue.
-* @return             OMPI error code (<0) on error or number of bytes actually received.
-*
-* The OOB recv call is similar to unix recv/readv in that it requires the caller to manage
-* memory associated w/ the message. The routine accepts an array of iovecs (<i>msg</i>); however,
-* the caller must determine the appropriate number of elements (<i>count</i>) and allocate the
-* buffer space for each entry. 
-*
-* The <i>tag</i> parameter is provided to facilitate this. The user may define tags based on message 
-* type to determine the message layout and size, as the mca_oob_recv call will block until a message
-* with the matching tag is received.
-*
-* Alternately, the <i>flags</i> parameter may be used to peek (MCA_OOB_PEEK) a portion of the message 
-* (e.g. a standard message header) or determine the overall message size (MCA_OOB_TRUNC|MCA_OOB_PEEK)
-* without removing the message from the queue. 
-*
-*/
-
-int mca_oob_recv(
-    ompi_process_name_t* peer, 
-    const struct iovec *msg, 
-    int count, 
-    int tag, 
-    int flags);
-
-/**
-* Receive data and convert (if required) to host byte order.
-*
-* @param peer (IN)    Opaque name of peer process or MCA_OOB_BASE_ANY for wildcard receive.
-* @param msg (IN)     Array of iovecs describing user buffers and lengths.
-* @param types (IN)   Parallel array to iovecs describing data type of each iovec element.
-* @param count (IN)   Number of elements in iovec array.
-* @param tag (IN)     User defined tag for matching send/recv.
-* @param flags (IN)   May be MCA_OOB_PEEK to return up to the number of bytes provided in the
-*                     iovec array without removing the message from the queue.
-* @return             OMPI error code (<0) on error or number of bytes actually received.
-*
-* This routine is equivalent to mca_oob_recv, with the exception that it accepts
-* an additional array of type codes describing the data types contained within the
-* iovec array. This information is used to convert the data from network byte order
-* (if required) to host byte order prior to receiving into the users buffer.
-* 
-*/
-
-int mca_oob_recv_ntoh(
-   ompi_process_name_t* peer, 
-   const struct iovec *msg, 
-   const mca_oob_base_type_t *types, 
-   int count, 
-   int tag,
-   int flags);
-
-/*
- * Non-blocking versions of send/recv.
-*/
-
-
-/**
-*  Callback function on send/recv completion.
-*
-*  @param status (IN)  Completion status - equivalent to the return value from blocking send/recv.
-*  @param peer (IN)    Opaque name of peer process.
-*  @param msg (IN)     Array of iovecs describing user buffers and lengths.
-*  @param count (IN)   Number of elements in iovec array.
-*  @param tag (IN)     User defined tag for matching send/recv.
-*  @param cbdata (IN)  User data.
-*/
-
-typedef void (*mca_oob_callback_fn_t)(
-    int status,
-    const ompi_process_name_t* peer, 
-    const struct iovec* msg, 
-    int count,
-    int tag,
-    void* cbdata);
-
-/**
-*  Non-blocking version of mca_oob_send().
-*
-*  @param peer (IN)    Opaque name of peer process.
-*  @param msg (IN)     Array of iovecs describing user buffers and lengths.
-*  @param count (IN)   Number of elements in iovec array.
-*  @param tag (IN)     User defined tag for matching send/recv.
-*  @param flags (IN)   Currently unused.
-*  @param cbfunc (IN)  Callback function on send completion.
-*  @param cbdata (IN)  User data that is passed to callback function.
-*  @return             OMPI error code (<0) on error number of bytes actually sent.
-*
-*  The user supplied callback function is called when the send completes. Note that
-*  the callback may occur before the call to mca_oob_send returns to the caller, 
-*  if the send completes during the call.
-*
-*/
-
-int mca_oob_send_nb(
-    const ompi_process_name_t* peer, 
-    const struct iovec* msg, 
-    int count, 
-    int tag,
-    int flags, 
-    mca_oob_callback_fn_t cbfunc,
-    void* cbdata);
-
-/**
-*  Non-blocking version of mca_oob_send_hton().
-*
-*  @param peer (IN)    Opaque name of peer process.
-*  @param msg (IN)     Array of iovecs describing user buffers and lengths.
-*  @param types (IN)   Parallel array to iovecs describing data type of each iovec element.
-*  @param count (IN)   Number of elements in iovec array.
-*  @param tag (IN)     User defined tag for matching send/recv.
-*  @param flags (IN)   Currently unused.
-*  @param cbfunc (IN)  Callback function on send completion.
-*  @param cbdata (IN)  User data that is passed to callback function.
-*  @return             OMPI error code (<0) on error number of bytes actually sent.
-*
-*  The user supplied callback function is called when the send completes. Note that
-*  the callback may occur before the call to mca_oob_send returns to the caller, 
-*  if the send completes during the call.
-*
-*/
-
-int mca_oob_send_hton_nb(
-    const ompi_process_name_t* peer, 
-    const struct iovec* msg, 
-    const mca_oob_base_type_t* types,
-    int count, 
-    int tag,
-    int flags, 
-    mca_oob_callback_fn_t cbfunc,
-    void* cbdata);
-
-/**
-* Non-blocking version of mca_oob_recv().
-*
-* @param peer (IN)    Opaque name of peer process or MCA_OOB_BASE_ANY for wildcard receive.
-* @param msg (IN)     Array of iovecs describing user buffers and lengths.
-* @param count (IN)   Number of elements in iovec array.
-* @param tag (IN)     User defined tag for matching send/recv.
-* @param flags (IN)   May be MCA_OOB_PEEK to return up to size bytes of msg w/out removing it from the queue,
-* @param cbfunc (IN)  Callback function on recv completion.
-* @param cbdata (IN)  User data that is passed to callback function.
-* @return             OMPI error code (<0) on error or number of bytes actually received.
-*
-* The user supplied callback function is called asynchronously when a message is received 
-* that matches the call parameters.
-*/
-
-int mca_oob_recv_nb(
-    ompi_process_name_t* peer, 
-    const struct iovec* msg,  
-    int count, 
-    int tag,
-    int flags, 
-    mca_oob_callback_fn_t cbfunc,
-    void* cbdata);
-
-/**
-* Non-blocking version of mca_oob_recv_ntoh().
-*
-* @param peer (IN/OUT) Opaque name of peer process or MCA_OOB_BASE_ANY for wildcard receive.
-* @param msg (IN)      Array of iovecs describing user buffers and lengths.
-* @param types (IN)    Parallel array to iovecs describing data type of each iovec element.
-* @param count (IN)    Number of elements in iovec array.
-* @param tag (IN)      User defined tag for matching send/recv.
-* @param flags (IN)    May be MCA_OOB_PEEK to return up to size bytes of msg w/out removing it from the queue,
-* @param cbfunc (IN)   Callback function on recv completion.
-* @param cbdata (IN)   User data that is passed to callback function.
-* @return              OMPI error code (<0) on error or number of bytes actually received.
-* 
-* The user supplied callback function is called asynchronously when a message is received 
-* that matches the call parameters.
-*/
-
-int mca_oob_recv_ntoh_nb(
-    ompi_process_name_t* peer, 
-    const struct iovec* msg,  
-    const mca_oob_base_type_t* types,
-    int count, 
-    int tag,
-    int flags, 
-    mca_oob_callback_fn_t cbfunc,
-    void* cbdata);
-
-#if defined(c_plusplus) || defined(__cplusplus)
-}
-#endif
 /*
  * OOB Component/Module function prototypes.
  */
@@ -334,11 +43,10 @@ typedef int (*mca_oob_base_module_send_fn_t)(
     int tag,
     int flags);
 
-
 /**
 *  Implementation of mca_oob_recv().
 *
-*  @param peer (IN)    Opaque name of peer process or MCA_OOB_BASE_ANY for wildcard receive.
+*  @param peer (IN)    Opaque name of peer process or MCA_OOB_NAME_ANY for wildcard receive.
 *  @param msg (IN)     Array of iovecs describing user buffers and lengths.
 *  @param types (IN)   Parallel array to iovecs describing data type of each iovec element.
 *  @param count (IN)   Number of elements in iovec array.
@@ -381,7 +89,7 @@ typedef int (*mca_oob_base_module_send_nb_fn_t)(
 /**
 * Implementation of mca_oob_recv_nb().
 *
-* @param peer (IN)    Opaque name of peer process or MCA_OOB_BASE_ANY for wildcard receive.
+* @param peer (IN)    Opaque name of peer process or MCA_OOB_NAME_ANY for wildcard receive.
 * @param msg (IN)     Array of iovecs describing user buffers and lengths.
 * @param count (IN)   Number of elements in iovec array.
 * @param tag (IN)     User defined tag for matching send/recv.
@@ -441,10 +149,83 @@ typedef mca_oob_base_component_1_0_0_t mca_oob_base_component_t;
  * Macro for use in components that are of type oob v1.0.0
  */
 #define MCA_OOB_BASE_VERSION_1_0_0 \
-  /* pml v1.0 is chained to MCA v1.0 */ \
+  /* oob v1.0 is chained to MCA v1.0 */ \
   MCA_BASE_VERSION_1_0_0, \
-  /* pml v1.0 */ \
+  /* oob v1.0 */ \
   "oob", 1, 0, 0
 
+/*
+ * This is the first module on the list. This is here temporarily
+ * to make things work
+ */
+extern mca_oob_t mca_oob;
+
+/**
+ * associate a component and a module that belongs to it
+ */
+struct mca_oob_base_info_t {
+  ompi_list_item_t super;
+  mca_oob_base_component_t *oob_component;
+  mca_oob_t *oob_module;
+};
+typedef struct mca_oob_base_info_t mca_oob_base_info_t;
+
+/**
+ * declare the association structure as a class
+ */
+OBJ_CLASS_DECLARATION(mca_oob_base_info_t);
+
+
+/*
+ * Global functions for MCA overall collective open and close
+ */
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+    int mca_oob_base_open(void);
+    int mca_oob_base_init(bool *allow_multi_user_threads, bool *have_hidden_threads);
+    int mca_oob_base_close(void);
+
+/*
+ * functions for pack and unpack routines
+ */
+/**
+ * This function packs the passed data according to the type enum.
+ *
+ * @param dest the destination for the packed data
+ * @param src the source of the data
+ * @param n the number of elements in the src
+ * @param type the type of data 
+ *
+ * @retval OMPI_SUCCESS
+ * @retval OMPI_ERROR
+ */
+    int mca_oob_base_pack(void * dest, void * src, size_t n, mca_oob_base_type_t type);
+
+/**
+ * This function unpacks the passed data according to the type enum.
+ *
+ * @param dest the destination for the unpacked data
+ * @param src the source of the packed data
+ * @param n the number of elements in the src
+ * @param type the type of the data to unpack
+ * 
+ * @retval OMPI_SUCCESS
+ * @retval OMPI_ERROR
+ */
+    int mca_oob_base_unpack(void * dest, void * src, size_t n, mca_oob_base_type_t type);
+
+
+#if defined(c_plusplus) || defined(__cplusplus)
+}
 #endif
 
+
+/*
+ * Global struct holding the selected module's function pointers
+ */
+extern int mca_oob_base_output;
+extern ompi_list_t mca_oob_base_components;
+extern ompi_list_t mca_oob_base_modules;
+
+#endif

src/mca/oob/tcp/oob_tcp.c

@@ -287,6 +287,8 @@ mca_oob_t* mca_oob_tcp_init(bool *allow_multi_user_threads, bool *have_hidden_th
 
 #if 0
     /* intialize event library */
+    memset(&mca_oob_tcp_component.tcp_recv_event, 0, sizeof(ompi_event_t));
+    memset(&mca_oob_tcp_component.tcp_send_event, 0, sizeof(ompi_event_t));
     if(ompi_event_init() != OMPI_SUCCESS) {
         ompi_output(0, "mca_oob_tcp_init: unable to initialize event library\n");
         return NULL;
@@ -316,6 +318,7 @@ int mca_oob_tcp_finalize(void)
         mca_oob_tcp_peer_close(peer);
         OBJ_DESTRUCT(peer);
     }
+    ompi_event_fini();
     return OMPI_SUCCESS;
 }
 

src/mca/oob/tcp/oob_tcp.h

@@ -11,6 +11,7 @@
 
 #include "mca/oob/oob.h"
 #include "mca/oob/base/base.h"
+#include "mca/base/base.h"
 #include "class/ompi_free_list.h"
 #include "class/ompi_rb_tree.h"
 #include "event/event.h"
@@ -105,7 +106,7 @@ int mca_oob_tcp_send(
 /**
  * Similiar to unix readv(2)
  *
- * @param peer (IN)    Opaque name of peer process or MCA_OOB_BASE_ANY for wildcard receive.
+ * @param peer (IN)    Opaque name of peer process or MCA_OOB_NAME_ANY for wildcard receive.
  * @param msg (IN)     Array of iovecs describing user buffers and lengths.
  * @param count (IN)   Number of elements in iovec array.
  * @param tag (IN)     User defined tag for matching send/recv.
@@ -152,7 +153,7 @@ int mca_oob_tcp_send_nb(
 /**
  * Non-blocking version of mca_oob_recv().
  *
- * @param peer (IN)    Opaque name of peer process or MCA_OOB_BASE_ANY for wildcard receive.
+ * @param peer (IN)    Opaque name of peer process or MCA_OOB_NAME_ANY for wildcard receive.
  * @param msg (IN)     Array of iovecs describing user buffers and lengths.
  * @param count (IN)   Number of elements in iovec array.
  * @param tag (IN)     User defined tag for matching send/recv.

src/mca/oob/tcp/oob_tcp_recv.c

@@ -3,7 +3,7 @@
 /*
  * Similiar to unix readv(2)
  *
- * @param peer (IN)    Opaque name of peer process or MCA_OOB_BASE_ANY for wildcard receive.
+ * @param peer (IN)    Opaque name of peer process or MCA_OOB_NAME_ANY for wildcard receive.
  * @param msg (IN)     Array of iovecs describing user buffers and lengths.
  * @param types (IN)   Parallel array to iovecs describing data type of each iovec element.
  * @param count (IN)   Number of elements in iovec array.
@@ -89,7 +89,7 @@ int mca_oob_tcp_recv(
 /*
  * Non-blocking version of mca_oob_recv().
  *
- * @param peer (IN)    Opaque name of peer process or MCA_OOB_BASE_ANY for wildcard receive.
+ * @param peer (IN)    Opaque name of peer process or MCA_OOB_NAME_ANY for wildcard receive.
  * @param msg (IN)     Array of iovecs describing user buffers and lengths.
  * @param count (IN)   Number of elements in iovec array.
  * @param tag (IN)     User supplied tag for matching send/recv.

src/runtime/ompi_rte_finalize.c

@@ -11,7 +11,7 @@
 #include "util/output.h"
 #include "threads/mutex.h"
 #include "mca/pcm/base/base.h"
-#include "mca/oob/base/base.h"
+#include "mca/oob/oob.h"
 #include "mca/ns/base/base.h"
 
 

src/runtime/ompi_rte_init.c

@@ -11,7 +11,7 @@
 #include "util/output.h"
 #include "threads/mutex.h"
 #include "mca/pcm/base/base.h"
-#include "mca/oob/base/base.h"
+#include "mca/oob/oob.h"
 #include "mca/ns/base/base.h"
 
 

test/mca/oob/oob_test.c

@@ -10,6 +10,7 @@
 #include <sys/uio.h>
 #include <unistd.h>
 #include <stdio.h>
+#include <string.h>
 #include "mpi.h"
 #include "support.h"
 #include "mca/oob/oob.h"
@@ -17,8 +18,10 @@
 #define MSG_TYPE_1   1
 #define MSG_TYPE_2   2
 #define NUM_TESTS    8
+#define NUM_TIMES    1
 
-bool testdone[NUM_TESTS];
+int i;
+bool testdone[NUM_TESTS * NUM_TIMES];
 void do_sends(ompi_process_name_t * peer);
 void do_recvs(ompi_process_name_t * peer);
 
@@ -29,7 +32,6 @@ bool compare_iovec(const struct iovec * msg1, const struct iovec * msg2,
     int i;
     for(i = 0; i < n; i++) {
         if(msg1[i].iov_len != msg2[i].iov_len) {
-            fprintf(stderr, "len problem %d", i);
             return false;
         }
         if(0 != memcmp(msg1[i].iov_base, msg2[i].iov_base, msg1[i].iov_len)) {
@@ -49,12 +51,12 @@ void callback(int status, const ompi_process_name_t * peer,
     if(0 != tag) {
         test_failure("Bad tag.");
     }
-    if((int) cbdata >= NUM_TESTS) {
+    if(((int) cbdata + (NUM_TESTS * i)) >= (NUM_TESTS * NUM_TIMES)) {
         test_failure("Bad value in callback function.");
-    } else if (testdone[(int) cbdata]) {
+    } else if (testdone[(int) cbdata + (NUM_TESTS * i)]) {
         test_failure("Callback function called on an already completed test.");
     } else {
-        testdone[(int) cbdata] = true;
+        testdone[(int) cbdata + (NUM_TESTS * i)] = true;
         test_success();
     }
 }
@@ -112,15 +114,19 @@ int main(int argc, char ** argv)
         test_init("oob send then recieve");
         /* local vpid is 1 - peer is 0 */
         peer.vpid = 0;
+        for(i = 0; i < NUM_TIMES; i++) {
             do_sends(&peer);
             do_recvs(&peer);
+        }
     } else {
         test_init("oob recieve then send");
         /* local vpid is 0 - peer is 1 */
         peer.vpid = 1;
+        for(i = 0; i < NUM_TIMES; i++) {
             do_recvs(&peer);
             do_sends(&peer);
         }
+    }
     /* done */
     test_finalize();
     MPI_Finalize();

test/mca/oob/oob_test_self.c

@@ -6,6 +6,7 @@
 #include <sys/uio.h>
 #include <unistd.h>
 #include <stdio.h>
+#include <string.h>
 #include "mpi.h"
 #include "support.h"
 #include "mca/oob/oob.h"