template for mx

This commit was SVN r3303.

src/mca/ptl/mx/Makefile.am

@@ -0,0 +1,41 @@
+#
+# $HEADER$
+#
+
+# Use the top-level Makefile.options
+
+include $(top_ompi_srcdir)/config/Makefile.options
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if OMPI_BUILD_ptl_mx_DSO
+component_noinst =
+component_install = mca_ptl_mx.la
+else
+component_noinst = libmca_ptl_mx.la
+component_install =
+endif
+
+mx_SOURCES = \
+    ptl_mx.c \
+    ptl_mx.h \
+    ptl_mx_component.c \
+    ptl_mx_peer.c \
+    ptl_mx_peer.h \
+    ptl_mx_recvfrag.c \
+    ptl_mx_recvfrag.h \
+    ptl_mx_sendfrag.c \
+    ptl_mx_sendfrag.h
+
+mcacomponentdir = $(libdir)/openmpi
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_ptl_mx_la_SOURCES = $(mx_SOURCES)
+mca_ptl_mx_la_LIBADD = 
+mca_ptl_mx_la_LDFLAGS = -module -avoid-version
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_ptl_mx_la_SOURCES = $(mx_SOURCES)
+libmca_ptl_mx_la_LIBADD =
+libmca_ptl_mx_la_LDFLAGS = -module -avoid-version

src/mca/ptl/mx/configure.stub

@@ -0,0 +1,77 @@
+# -*- shell-script -*-
+#
+# $HEADER$
+#
+
+#
+# Main function.  This will be invoked in the middle of the templated
+# configure script.
+#
+AC_DEFUN([MCA_CONFIGURE_STUB],[
+
+    # Additional --with flags that can be specified
+
+    AC_ARG_WITH(ptl-mx, 
+        AC_HELP_STRING([--with-ptl-mx=DIR],
+                       [Specify the installation directory of MX]))
+    AC_ARG_WITH(ptl-mx-libdir,
+        AC_HELP_STRING([--with-ptl-mx-libdir=DIR],
+                       [directory where the MX library can be found, if it is not in \$MXDIR/lib or \$MXDIR/binary/lib]))
+
+    # Add to CPPFLAGS if necessary
+
+    EXTRA_CPPFLAGS=
+    if test -n "$with_ptl_mx"; then
+        if test -d "$with_ptl_mx/include"; then
+            EXTRA_CPPFLAGS="-I$with_ptl_mx/include"
+        else
+            AC_MSG_WARN([*** Warning: cannot find $with_ptl_mx/include])
+            AC_MSG_WARN([*** Will still try to configure mx ptl anyway...])
+        fi
+    fi
+
+    # See if we can find mx.h
+
+    CPPFLAGS="$CPPFLAGS $EXTRA_CPPFLAGS"
+    AC_CHECK_HEADERS(myriexpress.h,,
+        AC_MSG_ERROR([*** Cannot find working myriexpress.h.]))
+
+    # Add to LDFLAGS if necessary
+
+    EXTRA_LDFLAGS=
+    if test -n "$with_ptl_mx_libdir"; then
+        if test -d "$with_ptl_mx_libdir/lib"; then
+            EXTRA_LDFLAGS="-L$with_ptl_mx_libdir/lib"
+        elif test -d "$with_ptl_mx_libdir/binary/lib"; then
+            EXTRA_LDFLAGS="-L$with_ptl_mx_libdir/binary/lib"
+        else
+            AC_MSG_WARN([*** Warning: cannot find $with_ptl_mx_libdir/lib])
+            AC_MSG_WARN([*** or $with_ptl_mx_libdir/binary/lib])
+            AC_MSG_WARN([*** Will still try to configure mx ptl anyway...])
+        fi
+    elif test -n "$with_ptl_mx"; then
+        if test -d "$with_ptl_mx/lib"; then
+            EXTRA_LDFLAGS="-L$with_ptl_mx/lib"
+        elif test -d "$with_ptl_mx/binary/lib"; then
+            EXTRA_LDFLAGS="-L$with_ptl_mx/binary/lib"
+        else
+            AC_MSG_WARN([*** Warning: cannot find $with_ptl_mx/lib])
+            AC_MSG_WARN([*** or $with_ptl_mx/binary/lib])
+            AC_MSG_WARN([*** Will still try to configure mx ptl anyway...])
+        fi
+    fi
+
+    # Try to find libmyriexpress
+
+    LDFLAGS="$LDFLAGS $EXTRA_LDFLAGS"
+    AC_CHECK_LIB([myriexpress], [main], [],
+        AC_MSG_ERROR([*** Cannot find libmyriexpress]))
+
+    #
+    # Save extra compiler/linker flags so that they can be added in
+    # the wrapper compilers, if necessary
+    #
+
+    WRAPPER_EXTRA_LDFLAGS="$EXTRA_LDFLAGS"
+    WRAPPER_EXTRA_LIBS="-lmx"
+])dnl

src/mca/ptl/mx/ptl_mx.c

@@ -0,0 +1,173 @@
+/*
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include <errno.h>
+#include <unistd.h>
+#include <string.h>
+
+#include "include/constants.h"
+#include "util/argv.h"
+#include "util/output.h"
+#include "mca/pml/pml.h"
+#include "mca/ptl/ptl.h"
+#include "ptl_mx.h"
+#include "ptl_mx_sendfrag.h"
+#include "ptl_mx_recvfrag.h"
+
+
+mca_ptl_mx_module_t mca_ptl_mx_module = {
+    {
+    &mca_ptl_mx_component.super,
+    8, /* ptl_cache_size */
+    sizeof(mca_ptl_mx_send_frag_t), /* ptl_cache_bytes */
+    0, /* ptl_frag_first_size */
+    0, /* ptl_frag_min_size */
+    0, /* ptl_frag_max_size */
+    0, /* ptl_exclusivity */
+    0, /* ptl_latency */
+    0, /* ptl_bandwidth */
+    MCA_PTL_PUT, /* ptl flags */
+    mca_ptl_mx_add_procs,
+    mca_ptl_mx_del_procs,
+    mca_ptl_mx_finalize,
+    mca_ptl_mx_send,  /* put */
+    mca_ptl_mx_send,  /* put */
+    NULL, /* get */
+    mca_ptl_mx_matched, /* matched */
+    mca_ptl_mx_request_init,
+    mca_ptl_mx_request_fini,
+    NULL, /* match */
+    NULL,
+    NULL
+    }
+};
+
+
+/**
+ * PML->PTL notification of addition to the process list.
+ *
+ * @param ptl (IN)
+ * @param nprocs (IN)     Number of processes
+ * @param procs (IN)      Set of processes
+ * @param peers (OUT)     Set of (optional) peer addressing info.
+ * @param peers (IN/OUT)  Set of processes that are reachable via this PTL.
+ * @return                OMPI_SUCCESS or error status on failure.
+ *
+ */
+                                                                                
+int mca_ptl_mx_add_procs(
+    struct mca_ptl_base_module_t* ptl, 
+    size_t nprocs, 
+    struct ompi_proc_t **ompi_proc, 
+    struct mca_ptl_base_peer_t** peer_ret, 
+    ompi_bitmap_t* reachable)
+{
+    size_t i;
+    for( i = 0; i < nprocs; i++ ) {
+    }
+    return OMPI_SUCCESS;
+}
+
+
+/**
+ * PML->PTL notification of change in the process list.
+ *
+ * @param ptl (IN)     PTL instance
+ * @param nproc (IN)   Number of processes.
+ * @param procs (IN)   Set of processes.
+ * @param peers (IN)   Set of peer data structures.
+ * @return             Status indicating if cleanup was successful
+ *
+ */
+
+int mca_ptl_mx_del_procs(
+    struct mca_ptl_base_module_t* ptl, 
+    size_t nprocs, 
+    struct ompi_proc_t **proc, 
+    struct mca_ptl_base_peer_t** ptl_peer)
+{
+    return OMPI_SUCCESS;
+}
+
+int mca_ptl_mx_finalize(struct mca_ptl_base_module_t* ptl)
+{
+    return OMPI_SUCCESS;
+}
+
+/**
+ * PML->PTL Initialize a send request for use by the PTL.
+ *
+ * @param ptl (IN)       PTL instance
+ * @param request (IN)   Pointer to allocated request.
+ *
+ * To reduce latency (number of required allocations), the PML allocates up
+ * to ptl_cache_bytes of additional space contigous w/ the base send request.
+ * This space may be used by the PTL for additional control information (e.g.
+ * first fragment descriptor).
+ *
+ * The ptl_request_init() function is called by the PML when requests are
+ * allocated to the PTLs cache. These requests will be cached by the PML
+ * on completion and re-used by the same PTL w/out additional calls to
+ * ptl_request_init().
+ *
+ * If the cache size is exceeded, the PML may pass requests to ptl_send/ptl_put
+ * that have been taken from the global pool and have not been initialized by the
+ * PTL. These requests will have the req_cached attribute set to false.
+ *
+ */
+
+int mca_ptl_mx_request_init(struct mca_ptl_base_module_t* ptl, mca_pml_base_send_request_t* request)
+{
+    OBJ_CONSTRUCT(request+1, mca_ptl_mx_send_frag_t);
+    return OMPI_SUCCESS;
+}
+
+
+/**
+ * PML->PTL Cleanup any resources that may have been associated with the
+ *          request by the PTL.
+ *
+ * @param ptl (IN)       PTL instance
+ * @param request (IN)   Pointer to allocated request.
+ *
+ * The ptl_request_fini function is called when the PML removes a request
+ * from the PTLs cache (due to resource constraints).  This routine provides
+ * the PTL the chance to cleanup/release any resources cached on the send
+ * descriptor by the PTL.
+ */
+
+void mca_ptl_mx_request_fini(struct mca_ptl_base_module_t* ptl, mca_pml_base_send_request_t* request)
+{
+    OBJ_DESTRUCT(request+1);
+}
+
+
+/**
+ * PML->PTL Notification from the PML to the PTL that a receive
+ * has been posted and matched against the indicated fragment.
+ *
+ * @param ptl (IN)       PTL instance
+ * @param recv_frag      Matched fragment
+ *
+ * The ptl_matched() function is called by the PML when a fragment
+ * is matched to a posted receive. This may occur during a call to
+ * ptl_match() if the receive is matched, or at a later point in time
+ * when a matching receive is posted.
+ *
+ * When this routine is called, the PTL is responsible for generating
+ * an acknowledgment to the peer if the MCA_PTL_FLAGS_ACK_MATCHED
+ * bit is set in the original fragment header. Additionally, the PTL
+ * is responsible for transferring any data associated with the fragment
+ * into the users buffer utilizing the datatype engine, and notifying
+ * the PML that the fragment has completed via the ptl_recv_progress()
+ * function.
+ */
+
+void mca_ptl_mx_matched( 
+    mca_ptl_base_module_t* ptl,
+    mca_ptl_base_recv_frag_t* frag)
+{
+}
+
+

src/mca/ptl/mx/ptl_mx.h

@@ -0,0 +1,298 @@
+/*
+ * $HEADER$
+ */
+/**
+ * @file
+ */
+#ifndef MCA_PTL_MX_H
+#define MCA_PTL_MX_H
+
+#include "ompi_config.h"
+#include "mca/pml/pml.h"
+#include "mca/ptl/ptl.h"
+#include "class/ompi_free_list.h"
+
+#define MCA_PTL_MX_STATISTICS 0
+
+/**
+ * Myricom MX PTL component.
+ */
+struct mca_ptl_mx_component_t {
+    mca_ptl_base_component_1_0_0_t super;         /**< base PTL component */
+    struct mca_ptl_mx_module_t** mx_ptl_modules;  /**< array of available PTL moduless */
+    size_t mx_num_ptl_modules;                    /**< number of ptls actually used */
+    int    mx_free_list_num;                      /**< initial size of free lists */
+    int    mx_free_list_max;                      /**< maximum size of free lists */
+    int    mx_free_list_inc;                      /**< number of elements to growing free lists by */
+    ompi_free_list_t mx_send_frags;               /**< free list of mx send fragments */
+    ompi_free_list_t mx_recv_frags;               /**< free list of mx recv fragments */
+    ompi_mutex_t mx_lock;                         /**< lock for accessing module state */
+};
+
+typedef struct mca_ptl_mx_component_t mca_ptl_mx_component_t;
+struct mca_ptl_mx_recv_frag_t;
+struct mca_ptl_mx_send_frag_t;
+
+extern mca_ptl_mx_component_t mca_ptl_mx_component;
+
+/**
+ * Register MX module parameters with the MCA framework
+ */
+extern int mca_ptl_mx_component_open(void);
+
+/**
+ * Any final cleanup before being unloaded.
+ */
+extern int mca_ptl_mx_component_close(void);
+
+/**
+ * MCA->PTL Intializes the PTL component and creates specific PTL
+ * module(s).
+ *
+ * @param num_ptls (OUT) Returns the number of ptl instances created, or 0
+ *                       if the transport is not available.
+ *
+ * @param allow_multi_user_threads (OUT) Indicated wether this component can
+ * run at MPI_THREAD_MULTIPLE or not.
+ *
+ * @param have_hidden_threads (OUT) Whether this component uses
+ * hidden threads (e.g., progress threads) or not.
+ *
+ * @return Array of pointers to PTL modules, or NULL if the transport
+ *         is not available.
+ *
+ * During component initialization, the PTL component should discover
+ * the physical devices that are available for the given transport,
+ * and create a PTL instance to represent each device. Any addressing
+ * information required by peers to reach the device should be published
+ * during this function via the mca_base_modex_send() interface.
+ *
+ */
+
+extern mca_ptl_base_module_t** mca_ptl_mx_component_init(
+    int *num_ptls, 
+    bool *allow_multi_user_threads,
+    bool *have_hidden_threads
+);
+
+
+/**
+ * MCA->PTL Called to dynamically change a component parameter.
+ *
+ * @param flag (IN)   Parameter to change.
+ * @param value (IN)  Optional parameter value.
+ *
+ * @return            OMPI_SUCCESS or error code on failure.
+ *
+ * The only supported parameter is currently MCA_PTL_ENABLE,
+ * which can be used by the PML to enable/disable forwarding
+ * by the PTL.
+ */
+
+extern int mca_ptl_mx_component_control(
+    int param,
+    void* value,
+    size_t size
+);
+
+/**
+ * MCA->PTL Called to progress outstanding requests for
+ * non-threaded polling environments.
+ *
+ * @param tstamp     Current time.
+ * @return           OMPI_SUCCESS or error code on failure.
+ */
+
+extern int mca_ptl_mx_component_progress(
+   mca_ptl_tstamp_t tstamp
+);
+
+/**
+ * Myricom MX PTL module.
+ */
+struct mca_ptl_mx_module_t {
+    mca_ptl_base_module_t super;    /**< base PTL module interface */
+    ompi_list_item_t mx_peers;      /**< list of peers */
+};
+typedef struct mca_ptl_mx_module_t mca_ptl_mx_module_t;
+
+
+extern mca_ptl_mx_module_t mca_ptl_mx_module;
+
+
+/**
+ * Cleanup any resources held by the PTL.
+ * 
+ * @param ptl  PTL instance.
+ * @return     OMPI_SUCCESS or error status on failure.
+ */
+
+extern int mca_ptl_mx_finalize(
+    struct mca_ptl_base_module_t* ptl
+);
+
+
+/**
+ * PML->PTL notification of change in the process list.
+ *
+ * @param ptl (IN)            PTL instance
+ * @param nprocs (IN)         Number of processes
+ * @param procs (IN)          Set of processes
+ * @param peer (OUT)          Set of (optional) mca_ptl_base_peer_t instances returned by PTL. * @param reachable (OUT)     Bitmask indicating set of peer processes that are reachable by this PTL.
+ * @return                    OMPI_SUCCESS or error status on failure.
+ *
+ * The mca_ptl_base_module_add_procs_fn_t() is called by the PML to
+ * determine the set of PTLs that should be used to reach each process.
+ * Any addressing information exported by the peer via the mca_base_modex_send()
+ * function should be available during this call via the corresponding
+ * mca_base_modex_recv() function. The PTL may utilize this information to
+ * determine reachability of each peer process.
+ *
+ * For each process that is reachable by the PTL, the bit corresponding to the index
+ * into the proc array (nprocs) should be set in the reachable bitmask. The PML
+ * provides the PTL the option to return a pointer to a data structure defined
+ * by the PTL that is returned to the PTL on subsequent calls to the PTL data
+ * transfer functions (e.g ptl_send). This may be used by the PTL to cache any addressing
+ * or connection information (e.g. TCP socket, IP queue pair).
+ */
+
+extern int mca_ptl_mx_add_procs(
+    struct mca_ptl_base_module_t* ptl,
+    size_t nprocs,
+    struct ompi_proc_t **procs,
+    struct mca_ptl_base_peer_t** peers,
+    ompi_bitmap_t* reachable
+);
+
+
+/**
+ * PML->PTL notification of change to the process list.
+ *
+ * @param ptl (IN)     PTL instance
+ * @param nprocs (IN)  Number of processes
+ * @param proc (IN)    Set of processes
+ * @param peer (IN)    Set of peer addressing information.
+ * @return             Status indicating if cleanup was successful
+ *
+ * When the process list changes, the PML notifies the PTL of the
+ * change, to provide the opportunity to cleanup or release any
+ * resources associated with the peer.
+ */
+
+extern int mca_ptl_mx_del_procs(
+    struct mca_ptl_base_module_t* ptl,
+    size_t nprocs,
+    struct ompi_proc_t **procs,
+    struct mca_ptl_base_peer_t** peers
+);
+
+
+/**
+ * PML->PTL Initialize a send request for use by the PTL.
+ *
+ * @param ptl (IN)       PTL instance
+ * @param request (IN)   Pointer to allocated request.
+ *
+ * To reduce latency (number of required allocations), the PML allocates up
+ * to ptl_cache_bytes of additional space contigous w/ the base send request.
+ * This space may be used by the PTL for additional control information (e.g.
+ * first fragment descriptor).
+ *
+ * The ptl_request_init() function is called by the PML when requests are
+ * allocated to the PTLs cache. These requests will be cached by the PML
+ * on completion and re-used by the same PTL w/out additional calls to
+ * ptl_request_init().
+ *
+ * If the cache size is exceeded, the PML may pass requests to ptl_send/ptl_put
+ * that have been taken from the global pool and have not been initialized by the
+ * PTL. These requests will have the req_cached attribute set to false.
+ *
+ */
+
+extern int mca_ptl_mx_request_init(
+    struct mca_ptl_base_module_t* ptl,
+    struct mca_pml_base_send_request_t*
+);
+
+
+/**
+ * PML->PTL Cleanup any resources that may have been associated with the
+ *          request by the PTL.
+ *
+ * @param ptl (IN)       PTL instance
+ * @param request (IN)   Pointer to allocated request.
+ *
+ * The ptl_request_fini function is called when the PML removes a request
+ * from the PTLs cache (due to resource constraints).  This routine provides
+ * the PTL the chance to cleanup/release any resources cached on the send
+ * descriptor by the PTL.
+ */
+                                                                                              
+extern void mca_ptl_mx_request_fini(
+    struct mca_ptl_base_module_t* ptl,
+    struct mca_pml_base_send_request_t*
+);
+
+
+/**
+ * PML->PTL Notification from the PML to the PTL that a receive
+ * has been posted and matched against the indicated fragment.
+ *
+ * @param ptl (IN)       PTL instance
+ * @param recv_frag      Matched fragment
+ *
+ * The ptl_matched() function is called by the PML when a fragment
+ * is matched to a posted receive. This may occur during a call to
+ * ptl_match() if the receive is matched, or at a later point in time
+ * when a matching receive is posted.
+ *
+ * When this routine is called, the PTL is responsible for generating
+ * an acknowledgment to the peer if the MCA_PTL_FLAGS_ACK_MATCHED
+ * bit is set in the original fragment header. Additionally, the PTL
+ * is responsible for transferring any data associated with the fragment
+ * into the users buffer utilizing the datatype engine, and notifying
+ * the PML that the fragment has completed via the ptl_recv_progress()
+ * function.
+ */
+
+extern void mca_ptl_mx_matched(
+    struct mca_ptl_base_module_t* ptl,
+    struct mca_ptl_base_recv_frag_t* frag
+);
+
+/**
+ * PML->PTL Initiate a send to the peer.
+ *
+ * @param ptl (IN)               PTL instance
+ * @param ptl_base_peer (IN)     PTL peer addressing
+ * @param request (IN)           Send request
+ * @param offset                 Current offset into packed/contiguous buffer.
+ * @param size (IN)              Number of bytes PML is requesting PTL to deliver,
+ * @param flags (IN)             Flags that should be passed to the peer via the message header.
+ * @param request (OUT)          OMPI_SUCCESS if the PTL was able to queue one or more fragments
+ *
+ * The PML implements a rendevouz protocol, with up to the PTL threshold
+ * (ptl_first_frag_size) bytes of the message sent in eager send mode. The ptl_send()
+ * function is called by the PML to initiate the send of the first message fragment.
+ *
+ * The PTL is responsible for updating the current data offset (req_offset) in the
+ * request to reflect the actual number of bytes fragmented.  This may be less than
+ * the requested size, due to resource constraints or datatype alighnment/offset. If
+ * an acknowledgment is required, the MCA_PTL_FLAGS_ACK_MATCHED bit will be set in the
+ * flags parameter. In this case, the PTL should not call ptl_send_progress() function
+ * to indicate completion of the fragment until the ack is received. For all other
+ * fragments ptl_send_progress() may be called based on local completion semantics.
+ */
+
+extern int mca_ptl_mx_send(
+    struct mca_ptl_base_module_t* ptl,
+    struct mca_ptl_base_peer_t* ptl_peer,
+    struct mca_pml_base_send_request_t*,
+    size_t offset,
+    size_t size,
+    int flags
+);
+
+
+#endif
+

src/mca/ptl/mx/ptl_mx_component.c

@@ -0,0 +1,216 @@
+/*
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include "include/constants.h"
+#include "util/output.h"
+#include "ptl_mx.h"
+#include "ptl_mx_recvfrag.h"
+#include "ptl_mx_sendfrag.h"
+#include "ptl_mx_peer.h"
+
+
+/*
+ * The MX component
+ */
+
+mca_ptl_mx_component_t mca_ptl_mx_component = {
+    {
+      /* First, the mca_base_module_t struct containing meta
+         information about the module itself */
+      {
+        /* Indicate that we are a pml v1.0.0 module (which also
+           implies a specific MCA version) */
+
+        MCA_PTL_BASE_VERSION_1_0_0,
+
+        "mx", /* MCA module name */
+        1,  /* MCA module major version */
+        0,  /* MCA module minor version */
+        0,  /* MCA module release version */
+        mca_ptl_mx_component_open,  /* module open */
+        mca_ptl_mx_component_close  /* module close */
+      },
+      
+      /* Next the MCA v1.0.0 module meta data */
+      
+      {
+        /* Whether the module is checkpointable or not */
+        
+        false
+      },
+      
+      mca_ptl_mx_component_init,  
+      mca_ptl_mx_component_control,
+      mca_ptl_mx_component_progress,
+    }
+};
+
+
+/*
+ * utility routines for parameter registration
+ */
+
+static inline char* mca_ptl_mx_param_register_string(
+    const char* param_name, 
+    const char* default_value)
+{
+    char *param_value;
+    int id = mca_base_param_register_string("ptl","mx",param_name,NULL,default_value);
+    mca_base_param_lookup_string(id, &param_value);
+    return param_value;
+}
+                                                                                                                            
+static inline int mca_ptl_mx_param_register_int(
+    const char* param_name, 
+    int default_value)
+{
+    int id = mca_base_param_register_int("ptl","mx",param_name,NULL,default_value);
+    int param_value = default_value;
+    mca_base_param_lookup_int(id,&param_value);
+    return param_value;
+}
+
+/*
+ *  Called by MCA framework to open the module, registers
+ *  module parameters.
+ */
+
+int mca_ptl_mx_component_open(void)
+{
+    /* initialize state */
+    mca_ptl_mx_component.mx_ptl_modules = NULL;
+    mca_ptl_mx_component.mx_num_ptl_modules = 0;
+
+    /* initialize objects */
+    OBJ_CONSTRUCT(&mca_ptl_mx_component.mx_lock, ompi_mutex_t);
+    OBJ_CONSTRUCT(&mca_ptl_mx_component.mx_send_frags, ompi_free_list_t);
+    OBJ_CONSTRUCT(&mca_ptl_mx_component.mx_recv_frags, ompi_free_list_t);
+
+    /* register MX module parameters */
+    mca_ptl_mx_component.mx_free_list_num =
+        mca_ptl_mx_param_register_int("free_list_num", 256);
+    mca_ptl_mx_component.mx_free_list_max =
+        mca_ptl_mx_param_register_int("free_list_max", -1);
+    mca_ptl_mx_component.mx_free_list_inc =
+        mca_ptl_mx_param_register_int("free_list_inc", 256);
+    mca_ptl_mx_module.super.ptl_exclusivity =
+        mca_ptl_mx_param_register_int("exclusivity", 0);
+    mca_ptl_mx_module.super.ptl_first_frag_size =
+        mca_ptl_mx_param_register_int("first_frag_size", 64*1024);
+    mca_ptl_mx_module.super.ptl_min_frag_size = 
+        mca_ptl_mx_param_register_int("min_frag_size", 64*1024);
+    mca_ptl_mx_module.super.ptl_max_frag_size =
+        mca_ptl_mx_param_register_int("max_frag_size", -1);
+    return OMPI_SUCCESS;
+}
+
+/*
+ * module cleanup - sanity checking of queue lengths
+ */
+
+int mca_ptl_mx_component_close(void)
+{
+    if (mca_ptl_mx_component.mx_send_frags.fl_num_allocated != 
+        mca_ptl_mx_component.mx_send_frags.super.ompi_list_length) {
+        ompi_output(0, "mx send frags: %d allocated %d returned\n",
+            mca_ptl_mx_component.mx_send_frags.fl_num_allocated, 
+            mca_ptl_mx_component.mx_send_frags.super.ompi_list_length);
+    }
+    if (mca_ptl_mx_component.mx_recv_frags.fl_num_allocated != 
+        mca_ptl_mx_component.mx_recv_frags.super.ompi_list_length) {
+        ompi_output(0, "mx recv frags: %d allocated %d returned\n",
+            mca_ptl_mx_component.mx_recv_frags.fl_num_allocated, 
+            mca_ptl_mx_component.mx_recv_frags.super.ompi_list_length);
+    }
+
+    /* release resources */
+    OBJ_DESTRUCT(&mca_ptl_mx_component.mx_send_frags);
+    OBJ_DESTRUCT(&mca_ptl_mx_component.mx_recv_frags);
+    OBJ_DESTRUCT(&mca_ptl_mx_component.mx_lock);
+    return OMPI_SUCCESS;
+}
+
+
+/*
+ * Open NIC and initialize resources.
+ */
+
+static int ptl_mx_open_endpoints(void)
+{
+    return OMPI_SUCCESS;
+}
+
+
+/*
+ *  MX module initialization.
+ */
+mca_ptl_base_module_t** mca_ptl_mx_component_init(int *num_ptl_modules, 
+                                                bool *allow_multi_user_threads,
+                                                bool *have_hidden_threads)
+{
+    mca_ptl_base_module_t** ptls;
+    *num_ptl_modules = 0;
+    *allow_multi_user_threads = true;
+    *have_hidden_threads = OMPI_HAVE_THREADS;
+
+    ompi_free_list_init(&mca_ptl_mx_component.mx_send_frags, 
+        sizeof(mca_ptl_mx_send_frag_t),
+        OBJ_CLASS(mca_ptl_mx_send_frag_t),
+        mca_ptl_mx_component.mx_free_list_num,
+        mca_ptl_mx_component.mx_free_list_max,
+        mca_ptl_mx_component.mx_free_list_inc,
+        NULL); /* use default allocator */
+
+    ompi_free_list_init(&mca_ptl_mx_component.mx_recv_frags, 
+        sizeof(mca_ptl_mx_recv_frag_t),
+        OBJ_CLASS(mca_ptl_mx_recv_frag_t),
+        mca_ptl_mx_component.mx_free_list_num,
+        mca_ptl_mx_component.mx_free_list_max,
+        mca_ptl_mx_component.mx_free_list_inc,
+        NULL); /* use default allocator */
+
+    ptls = malloc(mca_ptl_mx_component.mx_num_ptl_modules * 
+                  sizeof(mca_ptl_base_module_t*));
+    if(NULL == ptls)
+        return NULL;
+
+    if(OMPI_SUCCESS != ptl_mx_open_endpoints())
+        return NULL;
+
+    memcpy(ptls, 
+        mca_ptl_mx_component.mx_ptl_modules, 
+        mca_ptl_mx_component.mx_num_ptl_modules*sizeof(mca_ptl_mx_module_t*));
+    *num_ptl_modules = mca_ptl_mx_component.mx_num_ptl_modules;
+    return ptls;
+}
+
+/*
+ *  MX module control
+ */
+
+int mca_ptl_mx_component_control(int param, void* value, size_t size)
+{
+    switch(param) {
+        case MCA_PTL_ENABLE:
+            if(*(int*)value)
+                ;  /* enable forwarding */
+            else
+                ;  /* disable forwarding */
+            break;
+        default:
+            break;
+    }
+    return OMPI_SUCCESS;
+}
+
+
+/*
+ *  MX module progress.
+ */
+
+int mca_ptl_mx_component_progress(mca_ptl_tstamp_t tstamp)
+{
+    return OMPI_SUCCESS;
+}
+

src/mca/ptl/mx/ptl_mx_peer.c

@@ -0,0 +1,33 @@
+/*
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include "ptl_mx.h"
+#include "ptl_mx_peer.h"
+
+static void mca_ptl_mx_peer_construct(mca_ptl_base_peer_t* ptl_peer);
+static void mca_ptl_mx_peer_destruct(mca_ptl_base_peer_t* ptl_peer);
+
+OBJ_CLASS_INSTANCE(
+    mca_ptl_mx_peer_t,
+    ompi_list_item_t,
+    mca_ptl_mx_peer_construct,
+    mca_ptl_mx_peer_destruct);
+
+
+/*
+ * Initialize state of the peer instance.
+ */
+
+static void mca_ptl_mx_peer_construct(mca_ptl_base_peer_t* ptl_peer)
+{
+}
+
+/*
+ * Cleanup any resources held by the peer.
+ */
+
+static void mca_ptl_mx_peer_destruct(mca_ptl_base_peer_t* ptl_peer)
+{
+}
+

src/mca/ptl/mx/ptl_mx_peer.h

@@ -0,0 +1,26 @@
+/*
+ * $HEADER$
+ */
+/**
+ * @file
+ */
+#ifndef MCA_PTL_MX_PEER_H
+#define MCA_PTL_MX_PEER_H
+
+#include "ompi_config.h"
+
+
+/**
+ *  An abstraction that represents a a peer process.
+*/
+struct mca_ptl_base_peer_t {
+    ompi_list_item_t peer_item;
+};
+typedef struct mca_ptl_base_peer_t mca_ptl_base_peer_t;
+typedef struct mca_ptl_base_peer_t mca_ptl_mx_peer_t;
+
+OBJ_CLASS_DECLARATION(mca_ptl_mx_peer_t);
+
+
+#endif
+

src/mca/ptl/mx/ptl_mx_recvfrag.c

@@ -0,0 +1,35 @@
+/*
+ * $HEADER$
+ */
+#include "ompi_config.h"
+#include "ptl_mx.h"
+#include "ptl_mx_recvfrag.h"
+
+
+static void mca_ptl_mx_recv_frag_construct(mca_ptl_mx_recv_frag_t* frag);
+static void mca_ptl_mx_recv_frag_destruct(mca_ptl_mx_recv_frag_t* frag);
+
+
+OBJ_CLASS_INSTANCE(
+    mca_ptl_mx_recv_frag_t,
+    mca_ptl_base_recv_frag_t,
+    mca_ptl_mx_recv_frag_construct,
+    mca_ptl_mx_recv_frag_destruct);
+                                                                                                           
+/*
+ * MX recv fragment constructor
+ */
+
+static void mca_ptl_mx_recv_frag_construct(mca_ptl_mx_recv_frag_t* frag)
+{
+}
+
+
+/*
+ * MX recv fragment destructor
+ */
+
+static void mca_ptl_mx_recv_frag_destruct(mca_ptl_mx_recv_frag_t* frag)
+{
+}
+

src/mca/ptl/mx/ptl_mx_recvfrag.h

@@ -0,0 +1,32 @@
+/* 
+ * $HEADER$
+ */
+/**
+ * @file
+ */
+
+#ifndef MCA_PTL_MX_RECV_FRAG_H
+#define MCA_PTL_MX_RECV_FRAG_H
+
+#include "ptl_mx.h"
+#include "mca/ptl/base/ptl_base_recvfrag.h"
+
+/**
+ *  MX received fragment derived type.
+ */
+struct mca_ptl_mx_recv_frag_t {
+    mca_ptl_base_recv_frag_t frag_recv;  /**< base receive fragment descriptor */
+};
+typedef struct mca_ptl_mx_recv_frag_t mca_ptl_mx_recv_frag_t;
+
+OBJ_CLASS_DECLARATION(mca_ptl_mx_recv_frag_t);
+
+#define MCA_PTL_MX_RECV_FRAG_ALLOC(frag, rc) \
+    { \
+    ompi_list_item_t* item; \
+    OMPI_FREE_LIST_GET(&mca_ptl_mx_component.mx_recv_frags, item, rc); \
+    frag = (mca_ptl_mx_recv_frag_t*)item; \
+    }
+
+#endif
+

src/mca/ptl/mx/ptl_mx_sendfrag.c

@@ -0,0 +1,31 @@
+/*
+ * $HEADER$
+ */
+#include "ptl_mx.h"
+#include "ptl_mx_sendfrag.h"
+
+
+static void mca_ptl_mx_send_frag_construct(mca_ptl_mx_send_frag_t* frag);
+static void mca_ptl_mx_send_frag_destruct(mca_ptl_mx_send_frag_t* frag);
+
+
+OBJ_CLASS_INSTANCE(
+    mca_ptl_mx_send_frag_t,
+    mca_ptl_base_send_frag_t,
+    mca_ptl_mx_send_frag_construct,
+    mca_ptl_mx_send_frag_destruct);
+
+                                                                                                           
+/*
+ * Placeholders for send fragment constructor/destructors.
+ */
+
+static void mca_ptl_mx_send_frag_construct(mca_ptl_mx_send_frag_t* frag)
+{
+}
+
+
+static void mca_ptl_mx_send_frag_destruct(mca_ptl_mx_send_frag_t* frag)
+{
+}
+

src/mca/ptl/mx/ptl_mx_sendfrag.h

@@ -0,0 +1,34 @@
+/*
+ * $HEADER$
+ */
+/**
+ * @file
+ */
+#ifndef MCA_PTL_MX_SEND_FRAG_H
+#define MCA_PTL_MX_SEND_FRAG_H
+
+#include "ompi_config.h"
+#include "include/sys/atomic.h"
+#include "mca/pml/base/pml_base_sendreq.h"
+#include "mca/ptl/base/ptl_base_sendfrag.h"
+#include "ptl_mx.h"
+
+
+/**
+ * MX send fragment derived type.
+ */
+struct mca_ptl_mx_send_frag_t {
+   mca_ptl_base_send_frag_t frag_send;  /**< base send fragment descriptor */
+};
+typedef struct mca_ptl_mx_send_frag_t mca_ptl_mx_send_frag_t;
+
+#define MCA_PTL_MX_SEND_FRAG_ALLOC(item, rc)  \
+    OMPI_FREE_LIST_GET(&mca_ptl_tcp_component.mx_send_frags, item, rc);
+
+#define MCA_PTL_MX_SEND_FRAG_RETURN(item)  \
+    OMPI_FREE_LIST_RETURN(&mca_ptl_mx_component.mx_send_frags, item);
+
+OBJ_CLASS_DECLARATION(mca_ptl_mx_send_frag_t);
+
+#endif
+