reorg to support new pml - move ptl/teg specific data

from pml_base_* data structures
*** NOTE ***
this will break gm/ib - I will fix these shortly

This commit was SVN r5655.

src/communicator/comm.c

@@ -972,11 +972,6 @@ int ompi_comm_determine_first ( ompi_communicator_t *intercomm, int high )
 /********************************************************************************/
 int ompi_comm_dump ( ompi_communicator_t *comm )
 {
-    mca_pml_ptl_comm_t *pml_comm;
-    mca_ptl_sequence_t *seq;
-    ompi_list_t *list;
-    int i;
-
     ompi_output(0, "Dumping information for comm_cid %d\n", comm->c_contextid);
     ompi_output(0,"  f2c index:%d cube_dim: %d\n", comm->c_f_to_c_index,  
            comm->c_cube_dim);
@@ -997,23 +992,6 @@ int ompi_comm_dump ( ompi_communicator_t *comm )
     if (OMPI_COMM_IS_INTER(comm)) {
         ompi_output(0,"  Remote group size:%d\n", comm->c_remote_group->grp_proc_count);
     }
-
-
-    /* Dump the c_pml_comm->c_unexpexted_frags for each process */
-    pml_comm = (mca_pml_ptl_comm_t *)comm->c_pml_comm;
-    seq      = (mca_ptl_sequence_t *) pml_comm->c_frags_cant_match;
-    for ( i = 0; i < comm->c_local_group->grp_proc_count; i++ ){
-        list = (ompi_list_t *)seq+i;
-        ompi_output(0,"%d: head->list_next:%p head->list_prev:%p"
-               "    tail->list_next:%p tail->list_next:%p\n",
-               i,
-               (char*)list->ompi_list_head.ompi_list_next,
-               (char*)list->ompi_list_head.ompi_list_prev, 
-               (char*)list->ompi_list_tail.ompi_list_next, 
-               (char*)list->ompi_list_tail.ompi_list_prev );
-        fflush(stdout);
-    }
-    
     return MPI_SUCCESS;
 }
 /********************************************************************************/

src/communicator/comm_init.c

@@ -272,6 +272,7 @@ static void ompi_comm_construct(ompi_communicator_t* comm)
     comm->c_remote_group = NULL;
     comm->error_handler  = NULL;
     comm->c_pml_comm     = NULL;
+    comm->c_pml_procs    = NULL;
     comm->c_topo         = NULL;
     comm->c_topo_component = NULL;
     comm->c_topo_comm    = NULL; 

src/communicator/communicator.h

@@ -131,8 +131,8 @@ struct ompi_communicator_t {
     ompi_errhandler_type_t             errhandler_type;
 
     /* Hooks for PML to hang things */
-
     struct mca_pml_comm_t  *c_pml_comm;
+    struct mca_pml_proc_t **c_pml_procs;
 
     mca_coll_base_module_1_0_0_t c_coll;
     /**< Selected collective module, saved by value for speed (instead

src/mca/pml/base/pml_base_recvreq.h

@@ -30,8 +30,6 @@ OMPI_DECLSPEC extern ompi_class_t mca_pml_base_recv_request_t_class;
 struct mca_pml_base_recv_request_t {
    mca_pml_base_request_t req_base;  /**< base request */
    size_t req_bytes_packed;          /**< size of message being received */
-   size_t req_bytes_received;        /**< number of bytes received from network */
-   size_t req_bytes_delivered;       /**< number of bytes delivered to user */
 };
 typedef struct mca_pml_base_recv_request_t mca_pml_base_recv_request_t;
 

src/mca/pml/base/pml_base_sendreq.c

@@ -34,11 +34,11 @@ static void mca_pml_base_send_request_construct(mca_pml_base_send_request_t* req
 {
     /* no need to reinit for every send -- never changes */
     request->req_base.req_type = MCA_PML_REQUEST_SEND;
-    request->req_cached = false;
     OBJ_CONSTRUCT(&request->req_convertor, ompi_convertor_t);
 }
 
 static void mca_pml_base_send_request_destruct(mca_pml_base_send_request_t* req)
 {
+    OBJ_DESTRUCT(&req->req_convertor);
 }
 

src/mca/pml/base/pml_base_sendreq.h

@@ -35,21 +35,12 @@ OMPI_DECLSPEC extern ompi_class_t mca_pml_base_send_request_t_class;
  */
 struct mca_pml_base_send_request_t {
     mca_pml_base_request_t req_base;         /** base request type - common data structure for use by wait/test */
+    ompi_datatype_t* req_datatype;           /**< pointer to datatype */
     void *req_addr;                          /**< pointer to send buffer - may not be application buffer */
     size_t req_count;                        /**< number of elements in send buffer */
-    ompi_datatype_t* req_datatype;           /**< pointer to datatype */
-    size_t req_offset;                       /**< number of bytes that have already been assigned to a fragment */
     size_t req_bytes_packed;                 /**< packed size of a message given the datatype and count */
-    size_t req_bytes_sent;                   /**< number of bytes that have been sent */
     mca_pml_base_send_mode_t req_send_mode;  /**< type of send */
-    struct mca_ptl_base_module_t* req_ptl;   /**< PTL that is selected for first fragment */
-    struct mca_ptl_base_peer_t* req_peer;    /**< PTL peer instance that will be used for first fragment */
-    ompi_ptr_t req_peer_match;               /**< matched receive at peer */
-    ompi_ptr_t req_peer_addr;                /**< peers remote buffer address */
-    uint64_t req_peer_size;                    /**< size of peers remote buffer */
-    bool req_cached;                         /**< has this request been obtained from the ptls cache */
     ompi_convertor_t req_convertor;          /**< convertor that describes this datatype */
-    volatile int32_t req_lock;               /**< lock used by the scheduler */
 };
 typedef struct mca_pml_base_send_request_t mca_pml_base_send_request_t;
 
@@ -137,32 +128,6 @@ typedef struct mca_pml_base_send_request_t mca_pml_base_send_request_t;
         OBJ_RELEASE(request->req_base.req_datatype);                      \
     } while (0)
 
-/**
- * Test to check if an acknowledgment has been received, with the match.
- *
- * @param  request (IN)  Send request.
- * return                TRUE if an ack/match has been received from peer.
- */
-
-static inline bool mca_pml_base_send_request_matched(
-    mca_pml_base_send_request_t* request)
-{
-    return (NULL != request->req_peer_match.pval);
-}
-
-/**
- * Atomically increase the request offset.
- *
- * @param  request (IN)  Send request.
- * @param  offset (IN)   Increment.
- */
-
-static inline void mca_pml_base_send_request_offset(
-    mca_pml_base_send_request_t* request,
-    size_t offset)
-{
-    OMPI_THREAD_ADD32((int32_t*)(&request->req_offset), offset);
-}
 
 #if defined(c_plusplus) || defined(__cplusplus)
 }

src/mca/pml/teg/Makefile.am

@@ -14,34 +14,28 @@
 # $HEADER$
 #
 
-# Use the top-level Makefile.options
-
 include $(top_ompi_srcdir)/config/Makefile.options
 
-sources =
+noinst_LTLIBRARIES = libmca_pml_teg.la
-include src/Makefile.extra
+libmca_pml_teg_la_SOURCES = \
-
+	pml_teg.c \
-# Make the output library in this directory, and name it either
+	pml_teg.h \
-# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+	pml_teg_cancel.c \
-# (for static builds).
+	pml_teg_component.c \
-
+	pml_teg_component.h \
-if OMPI_BUILD_pml_teg_DSO
+	pml_teg_iprobe.c \
-lib =
+	pml_teg_irecv.c \
-lib_sources = 
+	pml_teg_isend.c \
-component = mca_pml_teg.la
+	pml_teg_ptl.c \
-component_sources = $(sources)
+	pml_teg_ptl.h \
-else
+	pml_teg_proc.c \
-lib = libmca_pml_teg.la
+	pml_teg_proc.h \
-lib_sources = $(sources)
+	pml_teg_progress.c \
-component =
+	pml_teg_recvfrag.c \
-component_sources =
+	pml_teg_recvfrag.h \
-endif
+	pml_teg_recvreq.c \
-
+	pml_teg_recvreq.h \
-mcacomponentdir = $(libdir)/openmpi
+	pml_teg_sendreq.c \
-mcacomponent_LTLIBRARIES = $(component)
+	pml_teg_sendreq.h \
-mca_pml_teg_la_SOURCES = $(component_sources)
+	pml_teg_start.c \
-mca_pml_teg_la_LDFLAGS = -module -avoid-version
+    pml_ptl_array.c
-
-noinst_LTLIBRARIES = $(lib)
-libmca_pml_teg_la_SOURCES = $(lib_sources)
-libmca_pml_teg_la_LDFLAGS = -module -avoid-version

src/mca/pml/teg/configure.params

@@ -17,6 +17,6 @@
 
 # Specific to this module
 
-PARAM_INIT_FILE=src/pml_teg.c
+PARAM_INIT_FILE=pml_teg.c
-PARAM_CONFIG_HEADER_FILE="src/teg_config.h"
+PARAM_CONFIG_HEADER_FILE="teg_config.h"
 PARAM_CONFIG_FILES="Makefile"

src/mca/pml/teg/pml_ptl_array.c

@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include <string.h>
+
+#include "mca/pml/pml.h"
+#include "pml_ptl_array.h"
+
+
+static void mca_ptl_array_construct(mca_ptl_array_t* array)
+{
+    array->ptl_procs = 0;
+    array->ptl_size = 0;
+    array->ptl_index = 0;
+    array->ptl_reserve = 0;
+}
+
+
+static void mca_ptl_array_destruct(mca_ptl_array_t* array)
+{
+    if(array->ptl_procs != 0)
+        free(array->ptl_procs);
+}
+
+OBJ_CLASS_INSTANCE(
+    mca_pml_teg_ptl_array_t,
+    ompi_object_t,
+    mca_ptl_array_construct,
+    mca_ptl_array_destruct
+);
+
+int mca_ptl_array_reserve(mca_ptl_array_t* array, size_t size)
+{
+    mca_ptl_proc_t *procs;
+    if(array->ptl_reserve >= size)
+        return OMPI_SUCCESS;
+    
+    procs = (mca_ptl_proc_t *)realloc(array->ptl_procs, sizeof(mca_ptl_proc_t)*size);
+    if(NULL == procs)
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    array->ptl_procs = procs;
+    array->ptl_reserve = size;
+    memset(array->ptl_procs+array->ptl_size, 0, (size-array->ptl_size)*sizeof(mca_ptl_proc_t));
+    return OMPI_SUCCESS;
+}
+

src/mca/pml/teg/pml_ptl_array.h

@@ -0,0 +1,146 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+/**
+ * @file
+ */
+#ifndef OMPI_PTL_ARRAY_H
+#define OMPI_PTL_ARRAY_H
+
+#include "util/output.h"
+#include "mca/ptl/ptl.h"
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+
+extern ompi_class_t mca_pml_teg_ptl_array_t_class;
+
+/**
+ * A data structure associated with a ompi_proc_t that caches
+ * addressing/scheduling attributes for a specific PTL instance
+ * that can be used to reach the process.
+ */
+struct mca_ptl_proc_t {
+    int ptl_weight;                       /**< PTL weight for scheduling */
+    struct mca_ptl_base_peer_t* ptl_peer; /**< PTL addressing info */
+    struct mca_pml_base_ptl_t* ptl_base;  /**< PML specific PTL info */
+    mca_ptl_base_module_t *ptl;           /**< PTL module */
+};
+typedef struct mca_ptl_proc_t mca_ptl_proc_t;
+
+/**
+ * A dynamically growable array of mca_ptl_proc_t instances.
+ * Maintains an index into the array that is used for round-robin
+ * scheduling across contents.
+ */
+struct mca_ptl_array_t {
+    ompi_object_t     super;
+    mca_ptl_proc_t*  ptl_procs;   /**< array of ptl procs */
+    size_t           ptl_size;    /**< number available */
+    size_t           ptl_reserve; /**< size of allocated ptl_proc array */
+    size_t           ptl_index;   /**< last used index*/
+};
+typedef struct mca_ptl_array_t mca_ptl_array_t;
+typedef struct mca_ptl_array_t mca_pml_teg_ptl_array_t;
+
+
+/**
+ * If required, reallocate (grow) the array to the indicate size.
+ * 
+ * @param array (IN)
+ * @param size (IN)
+ */
+int mca_ptl_array_reserve(mca_ptl_array_t*, size_t);
+
+static inline size_t mca_ptl_array_get_size(mca_ptl_array_t* array)
+{
+    return array->ptl_size;
+}
+
+/**
+ * Grow the array if required, and set the size.
+ * 
+ * @param array (IN)
+ * @param size (IN)
+ */
+static inline void mca_ptl_array_set_size(mca_ptl_array_t* array, size_t size)
+{
+    if(array->ptl_size > array->ptl_reserve)
+        mca_ptl_array_reserve(array, size);
+    array->ptl_size = size;
+}
+
+/**
+ * Grow the array size by one and return the item at that index.
+ * 
+ * @param array (IN)
+ */
+static inline mca_ptl_proc_t* mca_ptl_array_insert(mca_ptl_array_t* array)
+{
+#if OMPI_ENABLE_DEBUG
+    if(array->ptl_size >= array->ptl_reserve) {
+        ompi_output(0, "mca_ptl_array_insert: invalid array index %d >= %d", 
+            array->ptl_size, array->ptl_reserve);
+        return 0;
+    }
+#endif
+    return &array->ptl_procs[array->ptl_size++];
+}
+
+/**
+ * Return an array item at the specified index.
+ * 
+ * @param array (IN)
+ * @param index (IN)
+ */
+static inline mca_ptl_proc_t* mca_ptl_array_get_index(mca_ptl_array_t* array, size_t index)
+{
+#if OMPI_ENABLE_DEBUG
+    if(index >= array->ptl_size) {
+        ompi_output(0, "mca_ptl_array_get_index: invalid array index %d >= %d",
+            index, array->ptl_size);
+        return 0;
+    }
+#endif
+    return &array->ptl_procs[index];
+}
+
+/**
+ * Return the next LRU index in the array.
+ * 
+ * @param array (IN)
+ * @param index (IN)
+ */
+static inline mca_ptl_proc_t* mca_ptl_array_get_next(mca_ptl_array_t* array)
+{
+    mca_ptl_proc_t* ptl_proc;
+#if OMPI_ENABLE_DEBUG
+    if(array->ptl_size == 0) {
+        ompi_output(0, "mca_ptl_array_get_next: invalid array size");
+        return 0;
+    }
+#endif
+    ptl_proc = &array->ptl_procs[array->ptl_index++];
+    if(array->ptl_index == array->ptl_size)
+        array->ptl_index = 0;
+    return ptl_proc;
+}
+
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+
+#endif
+

src/mca/pml/teg/pml_teg.c

@@ -0,0 +1,436 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+#include "class/ompi_bitmap.h"
+#include "mca/pml/pml.h"
+#include "mca/ptl/ptl.h"
+#include "mca/ptl/base/base.h"
+#include "mca/ptl/base/ptl_base_comm.h"
+#include "mca/ptl/base/ptl_base_header.h"
+#include "mca/ptl/base/ptl_base_recvfrag.h"
+#include "mca/ptl/base/ptl_base_sendfrag.h"
+#include "pml_teg.h"
+#include "pml_teg_component.h"
+#include "pml_teg_proc.h"
+#include "pml_teg_ptl.h"
+#include "pml_teg_recvreq.h"
+#include "pml_teg_sendreq.h"
+#include "pml_teg_recvfrag.h"
+
+
+mca_pml_teg_t mca_pml_teg = {
+    {
+    mca_pml_teg_add_procs,
+    mca_pml_teg_del_procs,
+    mca_pml_teg_add_ptls,
+    mca_pml_teg_control,
+    mca_pml_teg_progress,
+    mca_pml_teg_add_comm,
+    mca_pml_teg_del_comm,
+    mca_pml_teg_irecv_init,
+    mca_pml_teg_irecv,
+    mca_pml_teg_recv,
+    mca_pml_teg_isend_init,
+    mca_pml_teg_isend,
+    mca_pml_teg_send,
+    mca_pml_teg_iprobe,
+    mca_pml_teg_probe,
+    mca_pml_teg_start
+    }
+};
+
+
+int mca_pml_teg_add_comm(ompi_communicator_t* comm)
+{
+    /* allocate pml specific comm data */
+    mca_pml_ptl_comm_t* pml_comm = OBJ_NEW(mca_pml_ptl_comm_t);
+    if (NULL == pml_comm) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+    mca_pml_ptl_comm_init_size(pml_comm, comm->c_remote_group->grp_proc_count);
+    comm->c_pml_comm = pml_comm;
+    return OMPI_SUCCESS;
+}
+
+int mca_pml_teg_del_comm(ompi_communicator_t* comm)
+{
+    OBJ_RELEASE(comm->c_pml_comm);
+    comm->c_pml_comm = 0;
+    return OMPI_SUCCESS;
+}
+
+static int ptl_exclusivity_compare(const void* arg1, const void* arg2)
+{
+    mca_ptl_base_module_t* ptl1 = *(struct mca_ptl_base_module_t**)arg1;
+    mca_ptl_base_module_t* ptl2 = *(struct mca_ptl_base_module_t**)arg2;
+    if( ptl1->ptl_exclusivity > ptl2->ptl_exclusivity ) {
+        return -1;
+    } else if (ptl1->ptl_exclusivity == ptl2->ptl_exclusivity ) {
+        return 0;
+    } else {
+        return 1;
+    }
+}
+
+
+int mca_pml_teg_add_ptls(ompi_list_t *ptls)
+{
+    /* build an array of ptls and ptl modules */
+    mca_ptl_base_selected_module_t* selected_ptl;
+    size_t num_ptls = ompi_list_get_size(ptls);
+    size_t cache_bytes = 0;
+    mca_pml_teg.teg_num_ptl_modules = 0;
+    mca_pml_teg.teg_num_ptl_progress = 0;
+    mca_pml_teg.teg_num_ptl_components = 0;
+    mca_pml_teg.teg_ptl_modules = (mca_ptl_base_module_t **)malloc(sizeof(mca_ptl_base_module_t*) * num_ptls);
+    mca_pml_teg.teg_ptl_progress = (mca_ptl_base_component_progress_fn_t*)malloc(sizeof(mca_ptl_base_component_progress_fn_t) * num_ptls);
+    mca_pml_teg.teg_ptl_components = (mca_ptl_base_component_t **)malloc(sizeof(mca_ptl_base_component_t*) * num_ptls);
+    if (NULL == mca_pml_teg.teg_ptl_modules || 
+        NULL == mca_pml_teg.teg_ptl_progress ||
+        NULL == mca_pml_teg.teg_ptl_components) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    for(selected_ptl =  (mca_ptl_base_selected_module_t*)ompi_list_get_first(ptls);
+        selected_ptl != (mca_ptl_base_selected_module_t*)ompi_list_get_end(ptls);
+        selected_ptl =  (mca_ptl_base_selected_module_t*)ompi_list_get_next(selected_ptl)) {
+        mca_ptl_base_module_t *ptl = selected_ptl->pbsm_module;
+        size_t i;
+
+        mca_pml_teg.teg_ptl_modules[mca_pml_teg.teg_num_ptl_modules++] = ptl;
+        for(i=0; i<mca_pml_teg.teg_num_ptl_components; i++) {
+          if(mca_pml_teg.teg_ptl_components[i] == ptl->ptl_component) {
+                break;
+          }
+        }
+        if(i == mca_pml_teg.teg_num_ptl_components) {
+            mca_pml_teg.teg_ptl_components[mca_pml_teg.teg_num_ptl_components++] = ptl->ptl_component;
+        }
+
+        /* 
+         *setup ptl 
+         */
+
+        /* set pointer to fragment matching logic routine, if this
+         *   not already set by the ptl */
+       if( NULL == ptl->ptl_match)
+           ptl->ptl_match = mca_pml_teg_recv_frag_match;
+         ptl->ptl_send_progress = mca_pml_teg_send_request_progress;
+         ptl->ptl_recv_progress = mca_pml_teg_recv_request_progress;
+         ptl->ptl_stack = ptl;
+         ptl->ptl_base = NULL;
+
+         /* find maximum required size for cache */
+         if(ptl->ptl_cache_bytes > cache_bytes) {
+             cache_bytes = ptl->ptl_cache_bytes;
+         }
+    }
+
+    /* setup send fragments based on largest required send request */
+    ompi_free_list_init(
+        &mca_pml_teg.teg_send_requests,
+        sizeof(mca_pml_teg_send_request_t) + cache_bytes,
+        OBJ_CLASS(mca_pml_teg_send_request_t),
+        mca_pml_teg.teg_free_list_num,
+        mca_pml_teg.teg_free_list_max,
+        mca_pml_teg.teg_free_list_inc,
+        NULL);
+
+    /* sort ptl list by exclusivity */
+    qsort(mca_pml_teg.teg_ptl_modules, mca_pml_teg.teg_num_ptl_modules, sizeof(struct mca_ptl_t*), ptl_exclusivity_compare);
+    return OMPI_SUCCESS;
+}
+
+/*
+ * Pass control information through to all PTL modules.
+ */
+
+int mca_pml_teg_control(int param, void* value, size_t size)
+{
+    size_t i=0;
+    for(i=0; i<mca_pml_teg.teg_num_ptl_components; i++) {
+        if(NULL != mca_pml_teg.teg_ptl_components[i]->ptlm_control) {
+            int rc = mca_pml_teg.teg_ptl_components[i]->ptlm_control(param,value,size);
+            if(rc != OMPI_SUCCESS)
+                return rc;
+        }
+    }
+    return OMPI_SUCCESS;
+}
+
+/*
+ *   For each proc setup a datastructure that indicates the PTLs
+ *   that can be used to reach the destination.
+ *
+ */
+
+int mca_pml_teg_add_procs(ompi_proc_t** procs, size_t nprocs)
+{
+    size_t p;
+    ompi_bitmap_t reachable;
+    struct mca_ptl_base_peer_t** ptl_peers = NULL;
+    int rc;
+    size_t p_index;
+    
+    if(nprocs == 0)
+        return OMPI_SUCCESS;
+
+    OBJ_CONSTRUCT(&reachable, ompi_bitmap_t);
+    rc = ompi_bitmap_init(&reachable, nprocs);
+    if(OMPI_SUCCESS != rc)
+        return rc;
+
+    /* iterate through each of the procs and set the peers architecture */
+    for(p=0; p<nprocs; p++) {
+        uint32_t* proc_arch;
+        size_t size = sizeof(uint32_t);
+        rc = mca_base_modex_recv(&mca_pml_teg_component.pmlm_version, procs[p], 
+            (void**)&proc_arch, &size);
+        if(rc != OMPI_SUCCESS) 
+            return rc;
+        if(size != sizeof(uint32_t))
+            return OMPI_ERROR;
+        procs[p]->proc_arch = ntohl(*proc_arch);
+        free(proc_arch);
+    }
+    
+    /* attempt to add all procs to each ptl */
+    ptl_peers = (struct mca_ptl_base_peer_t **)malloc(nprocs * sizeof(struct mca_ptl_base_peer_t*));
+    for(p_index = 0; p_index < mca_pml_teg.teg_num_ptl_modules; p_index++) {
+        mca_ptl_base_module_t* ptl = mca_pml_teg.teg_ptl_modules[p_index];
+        int ptl_inuse = 0;
+
+        /* if the ptl can reach the destination proc it sets the
+         * corresponding bit (proc index) in the reachable bitmap
+         * and can return addressing information for each proc
+         * that is passed back to the ptl on data transfer calls
+         */
+        ompi_bitmap_clear_all_bits(&reachable);
+        memset(ptl_peers, 0, nprocs * sizeof(struct mca_ptl_base_peer_t*));
+        rc = ptl->ptl_add_procs(ptl, nprocs, procs, ptl_peers, &reachable);
+        if(OMPI_SUCCESS != rc) {
+            free(ptl_peers);
+            return rc;
+        }
+
+        /* for each proc that is reachable - add the ptl to the procs array(s) */
+        for(p=0; p<nprocs; p++) {
+            if(ompi_bitmap_is_set_bit(&reachable, p)) {
+                ompi_proc_t *proc = procs[p];
+                mca_pml_proc_t* proc_pml = proc->proc_pml;
+                mca_ptl_proc_t* proc_ptl;
+                size_t size;
+
+                /* this ptl can be used */
+                ptl_inuse++;
+
+                /* initialize each proc */
+                if(NULL == proc_pml) {
+
+                    /* allocate pml specific proc data */
+                    proc_pml = OBJ_NEW(mca_pml_teg_proc_t);
+                    if (NULL == proc_pml) {
+                        ompi_output(0, "mca_pml_teg_add_procs: unable to allocate resources");
+                        free(ptl_peers);
+                        return OMPI_ERR_OUT_OF_RESOURCE;
+                    }
+
+                    /* preallocate space in array for max number of ptls */
+                    mca_ptl_array_reserve(&proc_pml->proc_ptl_first, mca_pml_teg.teg_num_ptl_modules);
+                    mca_ptl_array_reserve(&proc_pml->proc_ptl_next, mca_pml_teg.teg_num_ptl_modules);
+                    proc_pml->proc_ompi = proc;
+                    proc->proc_pml = proc_pml;
+                }
+
+                /* dont allow an additional PTL with a lower exclusivity ranking */
+                size = mca_ptl_array_get_size(&proc_pml->proc_ptl_next);
+                if(size > 0) {
+                    proc_ptl = mca_ptl_array_get_index(&proc_pml->proc_ptl_next, size-1);
+                    /* skip this ptl if the exclusivity is less than the previous */
+                    if(proc_ptl->ptl->ptl_exclusivity > ptl->ptl_exclusivity) {
+                        if(ptl_peers[p] != NULL) {
+                            ptl->ptl_del_procs(ptl, 1, &proc, &ptl_peers[p]);
+                        }
+                        continue;
+                    }
+                }
+               
+                /* cache the ptl on the proc */
+                proc_ptl = mca_ptl_array_insert(&proc_pml->proc_ptl_next);
+                proc_ptl->ptl = ptl;
+                proc_ptl->ptl_peer = ptl_peers[p];
+                proc_ptl->ptl_weight = 0;
+                proc_pml->proc_ptl_flags |= ptl->ptl_flags;
+            }
+        }
+        if(ptl_inuse > 0 && NULL != ptl->ptl_component->ptlm_progress) {
+            size_t p;
+            bool found = false;
+            for(p=0; p<mca_pml_teg.teg_num_ptl_progress; p++) {
+                if(mca_pml_teg.teg_ptl_progress[p] == ptl->ptl_component->ptlm_progress) {
+                    found = true;
+                    break;
+                }
+            }
+            if(found == false) {
+                mca_pml_teg.teg_ptl_progress[mca_pml_teg.teg_num_ptl_progress] = 
+                    ptl->ptl_component->ptlm_progress;
+                mca_pml_teg.teg_num_ptl_progress++;
+            }
+        }
+    }
+    free(ptl_peers);
+
+    /* iterate back through procs and compute metrics for registered ptls */
+    for(p=0; p<nprocs; p++) {
+        ompi_proc_t *proc = procs[p];
+        mca_pml_proc_t* proc_pml = proc->proc_pml;
+        double total_bandwidth = 0;
+        uint32_t latency = 0;
+        size_t n_index;
+        size_t n_size;
+
+        /* skip over procs w/ no ptls registered */
+        if(NULL == proc_pml)
+            continue;
+
+        /* (1) determine the total bandwidth available across all ptls
+         *     note that we need to do this here, as we may already have ptls configured
+         * (2) determine the highest priority ranking for latency
+         */
+        n_size = mca_ptl_array_get_size(&proc_pml->proc_ptl_next); 
+        for(n_index = 0; n_index < n_size; n_index++) {
+            struct mca_ptl_proc_t* proc_ptl = mca_ptl_array_get_index(&proc_pml->proc_ptl_next, n_index);
+            struct mca_ptl_base_module_t* ptl = proc_ptl->ptl;
+            total_bandwidth += proc_ptl->ptl->ptl_bandwidth; 
+            if(ptl->ptl_latency > latency)
+                latency = ptl->ptl_latency;
+        }
+
+        /* (1) set the weight of each ptl as a percentage of overall bandwidth
+         * (2) copy all ptl instances at the highest priority ranking into the
+         *     list of ptls used for first fragments
+         */
+
+        for(n_index = 0; n_index < n_size; n_index++) {
+            struct mca_ptl_proc_t* proc_ptl = mca_ptl_array_get_index(&proc_pml->proc_ptl_next, n_index);
+            struct mca_ptl_base_module_t *ptl = proc_ptl->ptl;
+            double weight;
+
+            /* compute weighting factor for this ptl */
+            if(ptl->ptl_bandwidth)
+                weight = proc_ptl->ptl->ptl_bandwidth / total_bandwidth;
+            else
+                weight = 1.0 / n_size;
+            proc_ptl->ptl_weight = (int)(weight * 100);
+
+            /*
+             * save/create ptl extension for use by pml
+             */
+            proc_ptl->ptl_base = ptl->ptl_base;
+            if (NULL == proc_ptl->ptl_base && 
+                ptl->ptl_cache_bytes > 0 &&
+                NULL != ptl->ptl_request_init &&
+                NULL != ptl->ptl_request_fini) {
+
+                mca_pml_base_ptl_t* ptl_base = OBJ_NEW(mca_pml_base_ptl_t);
+                ptl_base->ptl = ptl;
+                ptl_base->ptl_cache_size = ptl->ptl_cache_size;
+                proc_ptl->ptl_base = ptl->ptl_base = ptl_base;
+            }
+
+            /* check to see if this ptl is already in the array of ptls used for first
+             * fragments - if not add it.
+             */
+            if(ptl->ptl_latency == latency) {
+                struct mca_ptl_proc_t* proc_new = mca_ptl_array_insert(&proc_pml->proc_ptl_first);
+                *proc_new = *proc_ptl;
+            }
+        
+        }
+    }
+    return OMPI_SUCCESS;
+}
+
+/*
+ * iterate through each proc and notify any PTLs associated
+ * with the proc that it is/has gone away
+ */
+
+int mca_pml_teg_del_procs(ompi_proc_t** procs, size_t nprocs)
+{
+    size_t p;
+    int rc;
+    for(p = 0; p < nprocs; p++) {
+        ompi_proc_t *proc = procs[p];
+        mca_pml_proc_t* proc_pml = proc->proc_pml;
+        size_t f_index, f_size;
+        size_t n_index, n_size;
+ 
+        /* notify each ptl that the proc is going away */
+        f_size = mca_ptl_array_get_size(&proc_pml->proc_ptl_first);
+        for(f_index = 0; f_index < f_size; f_index++) {
+            mca_ptl_proc_t* ptl_proc = mca_ptl_array_get_index(&proc_pml->proc_ptl_first, f_index);
+            mca_ptl_base_module_t* ptl = ptl_proc->ptl;
+            
+            rc = ptl->ptl_del_procs(ptl,1,&proc,&ptl_proc->ptl_peer);
+            if(OMPI_SUCCESS != rc) {
+                return rc;
+            }
+
+            /* remove this from next array so that we dont call it twice w/ 
+             * the same address pointer
+             */
+            n_size = mca_ptl_array_get_size(&proc_pml->proc_ptl_first);
+            for(n_index = 0; n_index < n_size; n_index++) {
+                mca_ptl_proc_t* next_proc = mca_ptl_array_get_index(&proc_pml->proc_ptl_next, n_index);
+                if(next_proc->ptl == ptl) {
+                    memset(next_proc, 0, sizeof(mca_ptl_proc_t));
+                    break;
+                }
+            }
+        }
+
+        /* notify each ptl that was not in the array of ptls for first fragments */
+        n_size = mca_ptl_array_get_size(&proc_pml->proc_ptl_next);
+        for(n_index = 0; n_index < n_size; n_index++) {
+            mca_ptl_proc_t* ptl_proc = mca_ptl_array_get_index(&proc_pml->proc_ptl_first, n_index);
+            mca_ptl_base_module_t* ptl = ptl_proc->ptl;
+            if (ptl != 0) {
+                rc = ptl->ptl_del_procs(ptl,1,&proc,&ptl_proc->ptl_peer);
+                if(OMPI_SUCCESS != rc)
+                    return rc;
+            }
+        }
+        
+        /* do any required cleanup */
+        OBJ_RELEASE(proc_pml);
+        proc->proc_pml = NULL;
+    }
+    return OMPI_SUCCESS;
+}
+
+int mca_pml_teg_component_fini(void)
+{
+    /* FIX */
+    return OMPI_SUCCESS;
+}
+

src/mca/pml/teg/pml_teg.h

@@ -0,0 +1,268 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+/**
+ *  @file 
+ */
+
+#ifndef MCA_PML_TEG_H
+#define MCA_PML_TEG_H
+
+#include "threads/thread.h"
+#include "threads/condition.h"
+#include "class/ompi_free_list.h"
+#include "util/cmd_line.h"
+#include "request/request.h"
+#include "mca/pml/pml.h"
+#include "mca/pml/base/pml_base_request.h"
+#include "mca/pml/base/pml_base_bsend.h"
+#include "mca/ptl/base/ptl_base_sendreq.h"
+#include "mca/ptl/ptl.h"
+
+
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+/**
+ * TEG PML module
+ */
+
+struct mca_pml_teg_t {
+    mca_pml_base_module_t super; 
+
+    mca_ptl_base_component_t **teg_ptl_components;
+    size_t teg_num_ptl_components;
+
+    mca_ptl_base_module_t** teg_ptl_modules;
+    size_t teg_num_ptl_modules;
+
+    mca_ptl_base_component_progress_fn_t* teg_ptl_progress;
+    size_t teg_num_ptl_progress;
+
+    ompi_list_t  teg_procs;
+    ompi_mutex_t teg_lock;
+
+    int teg_priority;
+
+    int teg_free_list_num;   /* initial size of free list */
+    int teg_free_list_max;   /* maximum size of free list */
+    int teg_free_list_inc;   /* number of elements to grow free list */
+    int teg_poll_iterations; /* number of iterations to poll for completion */
+
+    /* free list of requests */
+    ompi_free_list_t teg_send_requests;
+    ompi_free_list_t teg_recv_requests;
+
+    /* list of pending send requests */
+    ompi_list_t teg_send_pending;
+};
+typedef struct mca_pml_teg_t mca_pml_teg_t; 
+
+extern mca_pml_teg_t mca_pml_teg;
+
+
+/*
+ * PML module functions.
+ */
+
+
+extern int mca_pml_teg_component_open(void);
+extern int mca_pml_teg_component_close(void);
+
+extern mca_pml_base_module_t* mca_pml_teg_component_init(
+    int *priority, 
+    bool enable_progress_threads,
+    bool enable_mpi_threads
+);
+
+extern int mca_pml_teg_component_fini(void);
+
+
+
+/*
+ * PML interface functions.
+ */
+
+extern int mca_pml_teg_add_comm(
+    struct ompi_communicator_t* comm
+);
+
+extern int mca_pml_teg_del_comm(
+    struct ompi_communicator_t* comm
+);
+
+extern int mca_pml_teg_add_procs(
+    struct ompi_proc_t **procs,
+    size_t nprocs
+);
+
+extern int mca_pml_teg_del_procs(
+    struct ompi_proc_t **procs,
+    size_t nprocs
+);
+
+extern int mca_pml_teg_add_ptls(
+    ompi_list_t *ptls
+);
+
+extern int mca_pml_teg_control(
+    int param,
+    void *size,
+    size_t value
+);
+
+extern int mca_pml_teg_progress(void);
+
+extern int mca_pml_teg_iprobe(
+    int dst,
+    int tag,
+    struct ompi_communicator_t* comm,
+    int *matched,
+    ompi_status_public_t* status
+);
+
+extern int mca_pml_teg_probe(
+    int dst,
+    int tag,
+    struct ompi_communicator_t* comm,
+    ompi_status_public_t* status
+);
+
+extern int mca_pml_teg_cancelled(
+    ompi_request_t* request,
+    int *flag
+);
+
+
+extern int mca_pml_teg_isend_init(
+    void *buf,
+    size_t count,
+    ompi_datatype_t *datatype,
+    int dst,
+    int tag,
+    mca_pml_base_send_mode_t mode,
+    struct ompi_communicator_t* comm,
+    struct ompi_request_t **request
+);
+
+extern int mca_pml_teg_isend(
+    void *buf,
+    size_t count,
+    ompi_datatype_t *datatype,
+    int dst,
+    int tag,
+    mca_pml_base_send_mode_t mode,
+    struct ompi_communicator_t* comm,
+    struct ompi_request_t **request
+);
+
+extern int mca_pml_teg_send(
+    void *buf,
+    size_t count,
+    ompi_datatype_t *datatype,
+    int dst,
+    int tag,
+    mca_pml_base_send_mode_t mode,
+    struct ompi_communicator_t* comm
+);
+
+extern int mca_pml_teg_irecv_init(
+    void *buf,
+    size_t count,
+    ompi_datatype_t *datatype,
+    int src,
+    int tag,
+    struct ompi_communicator_t* comm,
+    struct ompi_request_t **request
+);
+
+extern int mca_pml_teg_irecv(
+    void *buf,
+    size_t count,
+    ompi_datatype_t *datatype,
+    int src,
+    int tag,
+    struct ompi_communicator_t* comm,
+    struct ompi_request_t **request
+);
+
+extern int mca_pml_teg_recv(
+    void *buf,
+    size_t count,
+    ompi_datatype_t *datatype,
+    int src,
+    int tag,
+    struct ompi_communicator_t* comm,
+    ompi_status_public_t* status
+);
+
+extern int mca_pml_teg_progress(void);
+
+extern int mca_pml_teg_start(
+    size_t count,
+    ompi_request_t** requests
+);
+
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+#define MCA_PML_TEG_FINI(request) \
+{ \
+    mca_pml_base_request_t* pml_request = *(mca_pml_base_request_t**)(request); \
+    if(pml_request->req_persistent) { \
+       if(pml_request->req_free_called) { \
+           MCA_PML_TEG_FREE(request); \
+       } else { \
+           pml_request->req_ompi.req_state = OMPI_REQUEST_INACTIVE; \
+       } \
+    } else { \
+        MCA_PML_TEG_FREE(request); \
+    } \
+}
+
+ 
+#define MCA_PML_TEG_FREE(request) \
+{ \
+    mca_pml_base_request_t* pml_request = *(mca_pml_base_request_t**)(request); \
+    pml_request->req_free_called = true; \
+    if( pml_request->req_pml_complete == true) \
+    { \
+        OMPI_REQUEST_FINI(*(request)); \
+        switch(pml_request->req_type) { \
+        case MCA_PML_REQUEST_SEND: \
+            { \
+                mca_pml_teg_send_request_t* sendreq = (mca_pml_teg_send_request_t*)pml_request; \
+                while(sendreq->req_lock > 0); \
+                if(sendreq->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED) { \
+                    mca_pml_base_bsend_request_fini((ompi_request_t*)sendreq); \
+                } \
+                MCA_PML_TEG_SEND_REQUEST_RETURN(sendreq); \
+                break; \
+            } \
+        case MCA_PML_REQUEST_RECV: \
+            { \
+                mca_pml_teg_recv_request_t* recvreq = (mca_pml_teg_recv_request_t*)pml_request; \
+                MCA_PML_TEG_RECV_REQUEST_RETURN(recvreq); \
+                break; \
+            } \
+        default: \
+            break; \
+        } \
+    } \
+    *(request) = MPI_REQUEST_NULL; \
+}
+
+#endif
+

src/mca/pml/teg/pml_teg_cancel.c

@@ -0,0 +1,28 @@
+/* -*- Mode: C; c-basic-offset:4 ; -*- */
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "pml_teg.h"
+
+int mca_pml_teg_cancelled(ompi_request_t* request, int* flag)
+{
+    if(NULL != flag)
+        *flag = (true == request->req_status._cancelled ? 1 : 0);
+    return OMPI_SUCCESS;
+}
+

src/mca/pml/teg/pml_teg_component.c

@@ -0,0 +1,177 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "event/event.h"
+#include "mpi.h"
+#include "mca/pml/pml.h"
+#include "mca/ptl/ptl.h"
+#include "mca/base/mca_base_param.h"
+#include "mca/pml/base/pml_base_bsend.h"
+#include "pml_teg.h"
+#include "pml_teg_proc.h"
+#include "pml_teg_sendreq.h"
+#include "pml_teg_recvreq.h"
+
+
+mca_pml_base_component_1_0_0_t mca_pml_teg_component = {
+
+    /* First, the mca_base_component_t struct containing meta
+       information about the component itself */
+
+    {
+      /* Indicate that we are a pml v1.0.0 component (which also implies
+         a specific MCA version) */
+
+      MCA_PML_BASE_VERSION_1_0_0,
+    
+      "teg", /* MCA component name */
+      1,  /* MCA component major version */
+      0,  /* MCA component minor version */
+      0,  /* MCA component release version */
+      mca_pml_teg_component_open,  /* component open */
+      mca_pml_teg_component_close  /* component close */
+    },
+
+    /* Next the MCA v1.0.0 component meta data */
+
+    {
+      /* Whether the component is checkpointable or not */
+      false
+    },
+
+    mca_pml_teg_component_init,  /* component init */
+    mca_pml_teg_component_fini   /* component finalize */
+};
+
+
+
+static inline int mca_pml_teg_param_register_int(
+    const char* param_name,
+    int default_value)
+{
+    int id = mca_base_param_register_int("pml","teg",param_name,NULL,default_value);
+    int param_value = default_value;
+    mca_base_param_lookup_int(id,&param_value);
+    return param_value;
+}
+                                                                                                                        
+
+int mca_pml_teg_component_open(void)
+{
+#ifdef WIN32
+     WSADATA win_sock_data;
+     if (WSAStartup(MAKEWORD(2,2), &win_sock_data) != 0) {
+         ompi_output (0, "mca_oob_tcp_component_init: failed to initialise windows sockets: %d\n", WSAGetLastError());
+         return OMPI_ERROR;
+      }
+#endif
+    OBJ_CONSTRUCT(&mca_pml_teg.teg_lock, ompi_mutex_t);
+    OBJ_CONSTRUCT(&mca_pml_teg.teg_send_requests, ompi_free_list_t);
+    OBJ_CONSTRUCT(&mca_pml_teg.teg_recv_requests, ompi_free_list_t);
+    OBJ_CONSTRUCT(&mca_pml_teg.teg_procs, ompi_list_t);
+    OBJ_CONSTRUCT(&mca_pml_teg.teg_send_pending, ompi_list_t);
+
+    mca_pml_teg.teg_ptl_components = NULL;
+    mca_pml_teg.teg_num_ptl_components = 0;
+    mca_pml_teg.teg_ptl_modules = NULL;
+    mca_pml_teg.teg_num_ptl_modules = 0;
+    mca_pml_teg.teg_ptl_progress = NULL;
+    mca_pml_teg.teg_num_ptl_progress = 0;
+
+    mca_pml_teg.teg_free_list_num =
+        mca_pml_teg_param_register_int("free_list_num", 256);
+    mca_pml_teg.teg_free_list_max =
+        mca_pml_teg_param_register_int("free_list_max", -1);
+    mca_pml_teg.teg_free_list_inc =
+        mca_pml_teg_param_register_int("free_list_inc", 256);
+    mca_pml_teg.teg_poll_iterations =
+        mca_pml_teg_param_register_int("poll_iterations", 100000);
+    mca_pml_teg.teg_priority =
+        mca_pml_teg_param_register_int("priority", 0);
+
+    return OMPI_SUCCESS;
+}
+
+
+int mca_pml_teg_component_close(void)
+{
+#ifdef WIN32
+    WSACleanup();
+#endif
+
+#if OMPI_ENABLE_DEBUG
+    if (mca_pml_teg.teg_recv_requests.fl_num_allocated !=
+        mca_pml_teg.teg_recv_requests.super.ompi_list_length) {
+        ompi_output(0, "teg recv requests: %d allocated %d returned\n",
+            mca_pml_teg.teg_recv_requests.fl_num_allocated,
+            mca_pml_teg.teg_recv_requests.super.ompi_list_length);
+    }
+#endif
+
+    if(NULL != mca_pml_teg.teg_ptl_components) {
+        free(mca_pml_teg.teg_ptl_components);
+    }
+    if(NULL != mca_pml_teg.teg_ptl_modules) {
+        free(mca_pml_teg.teg_ptl_modules);
+    }
+    if(NULL != mca_pml_teg.teg_ptl_progress) {
+        free(mca_pml_teg.teg_ptl_progress);
+    }
+    OBJ_DESTRUCT(&mca_pml_teg.teg_send_pending);
+    OBJ_DESTRUCT(&mca_pml_teg.teg_send_requests);
+    OBJ_DESTRUCT(&mca_pml_teg.teg_recv_requests);
+    OBJ_DESTRUCT(&mca_pml_teg.teg_procs);
+    OBJ_DESTRUCT(&mca_pml_teg.teg_lock);
+    return OMPI_SUCCESS;
+}
+
+
+mca_pml_base_module_t* mca_pml_teg_component_init(int* priority, 
+                                                  bool enable_progress_threads,
+                                                  bool enable_mpi_threads)
+{
+    uint32_t proc_arch;
+    int rc;
+    *priority = mca_pml_teg.teg_priority;
+
+    /* recv requests */
+    ompi_free_list_init(
+        &mca_pml_teg.teg_recv_requests,
+        sizeof(mca_pml_teg_recv_request_t),
+        OBJ_CLASS(mca_pml_teg_recv_request_t), 
+        mca_pml_teg.teg_free_list_num,
+        mca_pml_teg.teg_free_list_max,
+        mca_pml_teg.teg_free_list_inc,
+        NULL);
+
+    /* buffered send */
+    if(OMPI_SUCCESS != mca_pml_base_bsend_init(enable_mpi_threads)) {
+        ompi_output(0, "mca_pml_teg_component_init: mca_pml_bsend_init failed\n");
+        return NULL;
+    }
+
+    /* post this processes datatype */
+    proc_arch = ompi_proc_local()->proc_arch;
+    proc_arch = htonl(proc_arch);
+    rc = mca_base_modex_send(&mca_pml_teg_component.pmlm_version, &proc_arch, sizeof(proc_arch));
+    if(rc != OMPI_SUCCESS)
+        return NULL;
+    
+    return &mca_pml_teg.super;
+}
+

src/mca/pml/teg/pml_teg_component.h

@@ -0,0 +1,27 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+/**
+ *  @file 
+ */
+
+#ifndef MCA_PML_TEG_COMPONENT_H
+#define MCA_PML_TEG_COMPONENT_H
+
+/*
+ * PML module functions.
+ */
+
+OMPI_COMP_EXPORT extern mca_pml_base_component_1_0_0_t mca_pml_teg_component;
+
+#endif

src/mca/pml/teg/pml_teg_iprobe.c

@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "request/request.h"
+#include "pml_teg_recvreq.h"
+
+
+int mca_pml_teg_iprobe(int src,
+                       int tag,
+                       struct ompi_communicator_t *comm,
+                       int *matched, ompi_status_public_t * status)
+{
+    int rc;
+    mca_ptl_base_recv_request_t recvreq;
+
+    OBJ_CONSTRUCT( &(recvreq), mca_ptl_base_recv_request_t );
+    recvreq.req_recv.req_base.req_ompi.req_type = OMPI_REQUEST_PML;
+    recvreq.req_recv.req_base.req_type = MCA_PML_REQUEST_IPROBE;
+    MCA_PML_TEG_RECV_REQUEST_INIT(&recvreq, NULL, 0, &ompi_mpi_char, src, tag, comm, true);
+
+    *matched = 0;
+    if ((rc = mca_pml_teg_recv_request_start(&recvreq)) == OMPI_SUCCESS) {
+        if( recvreq.req_recv.req_base.req_ompi.req_complete == true ) {
+            if( NULL != status ) {
+                *status = recvreq.req_recv.req_base.req_ompi.req_status;
+            }
+            *matched = 1;
+        } else {
+            /* we are supposed to progress ... */
+            ompi_progress();
+        }
+    }
+    MCA_PML_BASE_RECV_REQUEST_RETURN((&recvreq.req_recv));
+    return rc;
+}
+
+
+int mca_pml_teg_probe(int src,
+                      int tag,
+                      struct ompi_communicator_t *comm,
+                      ompi_status_public_t * status)
+{
+    int rc;
+    mca_ptl_base_recv_request_t recvreq;
+
+    OBJ_CONSTRUCT( &(recvreq), mca_ptl_base_recv_request_t );
+    recvreq.req_recv.req_base.req_ompi.req_type = OMPI_REQUEST_PML;
+    recvreq.req_recv.req_base.req_type = MCA_PML_REQUEST_PROBE;
+    MCA_PML_TEG_RECV_REQUEST_INIT(&recvreq, NULL, 0, &ompi_mpi_char, src, tag, comm, true);
+
+    if ((rc = mca_pml_teg_recv_request_start(&recvreq)) != OMPI_SUCCESS) {
+        MCA_PML_BASE_RECV_REQUEST_RETURN((&recvreq.req_recv));
+        return rc;
+    }
+
+    if (recvreq.req_recv.req_base.req_ompi.req_complete == false) {
+        /* give up and sleep until completion */
+        if (ompi_using_threads()) {
+            ompi_mutex_lock(&ompi_request_lock);
+            ompi_request_waiting++;
+            while (recvreq.req_recv.req_base.req_ompi.req_complete == false)
+                ompi_condition_wait(&ompi_request_cond, &ompi_request_lock);
+            ompi_request_waiting--;
+            ompi_mutex_unlock(&ompi_request_lock);
+        } else {
+            ompi_request_waiting++;
+            while (recvreq.req_recv.req_base.req_ompi.req_complete == false)
+                ompi_condition_wait(&ompi_request_cond, &ompi_request_lock);
+            ompi_request_waiting--;
+        }
+    }
+
+    if (NULL != status) {
+        *status = recvreq.req_recv.req_base.req_ompi.req_status;
+    }
+    MCA_PML_BASE_RECV_REQUEST_RETURN(&recvreq.req_recv);
+    return OMPI_SUCCESS;
+}
+

src/mca/pml/teg/pml_teg_irecv.c

@@ -0,0 +1,117 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+#include "request/request.h"
+#include "pml_teg_recvreq.h"
+
+
+int mca_pml_teg_irecv_init(void *addr,
+                           size_t count,
+                           ompi_datatype_t * datatype,
+                           int src,
+                           int tag,
+                           struct ompi_communicator_t *comm,
+                           struct ompi_request_t **request)
+{
+    int rc;
+    mca_ptl_base_recv_request_t *recvreq;
+    MCA_PML_TEG_RECV_REQUEST_ALLOC(recvreq, rc);
+    if (NULL == recvreq)
+        return rc;
+
+    MCA_PML_TEG_RECV_REQUEST_INIT(recvreq,
+                                   addr,
+                                   count, datatype, src, tag, comm, true);
+
+    *request = (ompi_request_t *) recvreq;
+    return OMPI_SUCCESS;
+}
+
+int mca_pml_teg_irecv(void *addr,
+                      size_t count,
+                      ompi_datatype_t * datatype,
+                      int src,
+                      int tag,
+                      struct ompi_communicator_t *comm,
+                      struct ompi_request_t **request)
+{
+    int rc;
+
+    mca_ptl_base_recv_request_t *recvreq;
+    MCA_PML_TEG_RECV_REQUEST_ALLOC(recvreq, rc);
+    if (NULL == recvreq)
+        return rc;
+
+    MCA_PML_TEG_RECV_REQUEST_INIT(recvreq,
+                                   addr,
+                                   count, datatype, src, tag, comm, false);
+
+    if ((rc = mca_pml_teg_recv_request_start(recvreq)) != OMPI_SUCCESS) {
+        MCA_PML_TEG_RECV_REQUEST_RETURN(recvreq);
+        return rc;
+    }
+    *request = (ompi_request_t *) recvreq;
+    return OMPI_SUCCESS;
+}
+
+
+int mca_pml_teg_recv(void *addr,
+                     size_t count,
+                     ompi_datatype_t * datatype,
+                     int src,
+                     int tag,
+                     struct ompi_communicator_t *comm,
+                     ompi_status_public_t * status)
+{
+    int rc;
+    mca_ptl_base_recv_request_t *recvreq;
+    MCA_PML_TEG_RECV_REQUEST_ALLOC(recvreq, rc);
+    if (NULL == recvreq)
+        return rc;
+
+    MCA_PML_TEG_RECV_REQUEST_INIT(recvreq,
+                                   addr,
+                                   count, datatype, src, tag, comm, false);
+
+    if ((rc = mca_pml_teg_recv_request_start(recvreq)) != OMPI_SUCCESS) {
+        goto recv_finish;
+    }
+
+    if (recvreq->req_recv.req_base.req_ompi.req_complete == false) {
+        /* give up and sleep until completion */
+        if (ompi_using_threads()) {
+            ompi_mutex_lock(&ompi_request_lock);
+            ompi_request_waiting++;
+            while (recvreq->req_recv.req_base.req_ompi.req_complete == false)
+                ompi_condition_wait(&ompi_request_cond, &ompi_request_lock);
+            ompi_request_waiting--;
+            ompi_mutex_unlock(&ompi_request_lock);
+        } else {
+            ompi_request_waiting++;
+            while (recvreq->req_recv.req_base.req_ompi.req_complete == false)
+                ompi_condition_wait(&ompi_request_cond, &ompi_request_lock);
+            ompi_request_waiting--;
+        }
+    }
+ recv_finish:
+    if (NULL != status) {  /* return status */
+        *status = recvreq->req_recv.req_base.req_ompi.req_status;
+    }
+
+    MCA_PML_TEG_RECV_REQUEST_RETURN(recvreq);
+    return recvreq->req_recv.req_base.req_ompi.req_status.MPI_ERROR;
+}

src/mca/pml/teg/pml_teg_isend.c

@@ -0,0 +1,128 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "pml_teg.h"
+#include "pml_teg_proc.h"
+#include "pml_teg_sendreq.h"
+#include "pml_teg_recvreq.h"
+
+
+int mca_pml_teg_isend_init(void *buf,
+                           size_t count,
+                           ompi_datatype_t * datatype,
+                           int dst,
+                           int tag,
+                           mca_pml_base_send_mode_t sendmode,
+                           ompi_communicator_t * comm,
+                           ompi_request_t ** request)
+{
+    int rc;
+
+    mca_pml_teg_send_request_t *sendreq;
+    MCA_PML_TEG_SEND_REQUEST_ALLOC(comm, dst, sendreq, rc);
+    if (rc != OMPI_SUCCESS)
+        return rc;
+
+    MCA_PML_TEG_SEND_REQUEST_INIT(sendreq,
+                                  buf,
+                                  count,
+                                  datatype,
+                                  dst, tag,
+                                  comm, sendmode, true);
+
+    *request = (ompi_request_t *) sendreq;
+    return OMPI_SUCCESS;
+}
+
+
+int mca_pml_teg_isend(void *buf,
+                      size_t count,
+                      ompi_datatype_t * datatype,
+                      int dst,
+                      int tag,
+                      mca_pml_base_send_mode_t sendmode,
+                      ompi_communicator_t * comm,
+                      ompi_request_t ** request)
+{
+    int rc;
+    mca_pml_teg_send_request_t *sendreq;
+    MCA_PML_TEG_SEND_REQUEST_ALLOC(comm, dst, sendreq, rc);
+    if (rc != OMPI_SUCCESS)
+        return rc;
+    MCA_PML_TEG_SEND_REQUEST_INIT(sendreq,
+                                  buf,
+                                  count,
+                                  datatype,
+                                  dst, tag,
+                                  comm, sendmode, false);
+
+    MCA_PML_TEG_SEND_REQUEST_START(sendreq, rc);
+    *request = (ompi_request_t *) sendreq;
+    return rc;
+}
+
+
+int mca_pml_teg_send(void *buf,
+                     size_t count,
+                     ompi_datatype_t * datatype,
+                     int dst,
+                     int tag,
+                     mca_pml_base_send_mode_t sendmode,
+                     ompi_communicator_t * comm)
+{
+    int rc;
+    mca_pml_teg_send_request_t *sendreq;
+    MCA_PML_TEG_SEND_REQUEST_ALLOC(comm, dst, sendreq, rc);
+    if (rc != OMPI_SUCCESS)
+        return rc;
+
+    MCA_PML_TEG_SEND_REQUEST_INIT(sendreq,
+                                  buf,
+                                  count,
+                                  datatype,
+                                  dst, tag,
+                                  comm, sendmode, false);
+
+    MCA_PML_TEG_SEND_REQUEST_START(sendreq, rc);
+    if (rc != OMPI_SUCCESS) {
+        MCA_PML_TEG_FREE((ompi_request_t **) & sendreq);
+        return rc;
+    }
+
+    if (sendreq->req_send.req_base.req_ompi.req_complete == false) {
+        /* give up and sleep until completion */
+        if (ompi_using_threads()) {
+            ompi_mutex_lock(&ompi_request_lock);
+            ompi_request_waiting++;
+            while (sendreq->req_send.req_base.req_ompi.req_complete == false)
+                ompi_condition_wait(&ompi_request_cond, &ompi_request_lock);
+            ompi_request_waiting--;
+            ompi_mutex_unlock(&ompi_request_lock);
+        } else {
+            ompi_request_waiting++;
+            while (sendreq->req_send.req_base.req_ompi.req_complete == false)
+                ompi_condition_wait(&ompi_request_cond, &ompi_request_lock);
+            ompi_request_waiting--;
+        }
+    }
+
+    /* return request to pool */
+    MCA_PML_TEG_FREE((ompi_request_t **) & sendreq);
+    return OMPI_SUCCESS;
+}
+

src/mca/pml/teg/pml_teg_proc.c

@@ -0,0 +1,56 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "include/sys/atomic.h"
+#include "pml_teg.h"
+#include "pml_teg_proc.h"
+#include "pml_ptl_array.h"
+
+
+static void mca_pml_teg_proc_construct(mca_pml_proc_t* proc)
+{
+    proc->proc_ompi = NULL;
+    proc->proc_ptl_flags = 0;
+    OBJ_CONSTRUCT(&proc->proc_lock, ompi_mutex_t);
+    OBJ_CONSTRUCT(&proc->proc_ptl_first, mca_pml_teg_ptl_array_t);
+    OBJ_CONSTRUCT(&proc->proc_ptl_next,  mca_pml_teg_ptl_array_t);
+
+    OMPI_THREAD_LOCK(&mca_pml_teg.teg_lock);
+    ompi_list_append(&mca_pml_teg.teg_procs, (ompi_list_item_t*)proc);
+    OMPI_THREAD_UNLOCK(&mca_pml_teg.teg_lock);
+}
+
+
+static void mca_pml_teg_proc_destruct(mca_pml_proc_t* proc)
+{
+    OMPI_THREAD_LOCK(&mca_pml_teg.teg_lock);
+    ompi_list_remove_item(&mca_pml_teg.teg_procs, (ompi_list_item_t*)proc);
+    OMPI_THREAD_UNLOCK(&mca_pml_teg.teg_lock);
+
+    OBJ_DESTRUCT(&proc->proc_lock);
+    OBJ_DESTRUCT(&proc->proc_ptl_first);
+    OBJ_DESTRUCT(&proc->proc_ptl_next);
+}
+
+OBJ_CLASS_INSTANCE(
+    mca_pml_teg_proc_t,
+    ompi_list_item_t,
+    mca_pml_teg_proc_construct, 
+    mca_pml_teg_proc_destruct 
+);
+

src/mca/pml/teg/pml_teg_proc.h

@@ -0,0 +1,107 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+/**
+ * @file
+ */
+#ifndef MCA_PML_PROC_H
+#define MCA_PML_PROC_H
+
+#include "threads/mutex.h"
+#include "communicator/communicator.h"
+#include "group/group.h"
+#include "proc/proc.h"
+#include "pml_ptl_array.h"
+
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+/**
+ *  Structure associated w/ ompi_proc_t that contains data specific
+ *  to the PML. Note that this name is not PML specific.
+ */
+struct mca_pml_proc_t {
+   ompi_list_item_t super;
+   ompi_proc_t *proc_ompi;           /**< back-pointer to ompi_proc_t */
+   ompi_mutex_t proc_lock;           /**< lock to protect against concurrent access */
+   mca_ptl_array_t proc_ptl_first;   /**< array of ptls to use for first fragments */
+   mca_ptl_array_t proc_ptl_next;    /**< array of ptls to use for remaining fragments */
+   uint32_t proc_ptl_flags;          /**< aggregate ptl flags */
+};
+typedef struct mca_pml_proc_t mca_pml_proc_t;
+
+
+OMPI_COMP_EXPORT extern ompi_class_t mca_pml_teg_proc_t_class;
+typedef struct mca_pml_proc_t mca_pml_teg_proc_t;
+
+/**
+ * Return the mca_pml_proc_t instance cached in the communicators local group.
+ * 
+ * @param comm   Communicator
+ * @param rank   Peer rank
+ * @return       mca_pml_proc_t instance
+ */
+
+static inline mca_pml_proc_t* mca_pml_teg_proc_lookup_local(ompi_communicator_t* comm, int rank)
+{
+    ompi_proc_t* proc = comm->c_local_group->grp_proc_pointers[rank];
+    return proc->proc_pml;
+}
+
+/**
+ * Return the mca_pml_proc_t instance cached on the communicators remote group.
+ * 
+ * @param comm   Communicator
+ * @param rank   Peer rank
+ * @return       mca_pml_proc_t instance
+ */
+
+static inline mca_pml_proc_t* mca_pml_teg_proc_lookup_remote(ompi_communicator_t* comm, int rank)
+{
+    ompi_proc_t* proc = comm->c_remote_group->grp_proc_pointers[rank];
+    return proc->proc_pml;
+}
+
+/**
+ * Return the mca_ptl_peer_t instance corresponding to the process/ptl combination.
+ * 
+ * @param comm   Communicator
+ * @param rank   Peer rank
+ * @return       mca_pml_proc_t instance
+ */
+
+static inline struct mca_ptl_base_peer_t* mca_pml_teg_proc_lookup_remote_peer(
+    ompi_communicator_t* comm, 
+    int rank, 
+    struct mca_ptl_base_module_t* ptl)
+{
+    ompi_proc_t* proc = comm->c_remote_group->grp_proc_pointers[rank];
+    mca_pml_proc_t* proc_pml = proc->proc_pml;
+    size_t i, size = mca_ptl_array_get_size(&proc_pml->proc_ptl_first);
+    mca_ptl_proc_t* proc_ptl = proc_pml->proc_ptl_first.ptl_procs;
+    for(i = 0; i < size; i++) {
+        if(proc_ptl->ptl == ptl) {
+            return proc_ptl->ptl_peer;
+        }
+        proc_ptl++;
+    }
+    return NULL;
+}
+
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+#endif
+

src/mca/pml/teg/pml_teg_progress.c

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "pml_teg.h"
+#include "pml_teg_sendreq.h"
+
+
+int mca_pml_teg_progress(void)
+{
+    mca_ptl_tstamp_t tstamp = 0;
+    size_t i;
+    int count = 0;
+
+    /*
+     * Progress each of the PTL modules
+     */
+    for(i=0; i<mca_pml_teg.teg_num_ptl_progress; i++) {
+        int rc = mca_pml_teg.teg_ptl_progress[i](tstamp);
+        if(rc > 0) {
+            count += rc;
+        }
+    }
+    return count;
+}
+

src/mca/pml/teg/pml_teg_ptl.c

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "pml_teg_ptl.h"
+
+
+static void mca_pml_base_ptl_construct(mca_pml_base_ptl_t* ptl)
+{
+    OBJ_CONSTRUCT(&ptl->ptl_cache, ompi_list_t);
+    OBJ_CONSTRUCT(&ptl->ptl_cache_lock, ompi_mutex_t);
+    ptl->ptl = NULL;
+    ptl->ptl_cache_size = 0;
+    ptl->ptl_cache_alloc = 0;
+}
+
+static void mca_pml_base_ptl_destruct(mca_pml_base_ptl_t* ptl)
+{
+    OBJ_DESTRUCT(&ptl->ptl_cache);
+    OBJ_DESTRUCT(&ptl->ptl_cache_lock);
+}
+
+OBJ_CLASS_INSTANCE(
+    mca_pml_base_ptl_t,
+    ompi_list_t,
+    mca_pml_base_ptl_construct,
+    mca_pml_base_ptl_destruct
+);
+

src/mca/pml/teg/pml_teg_ptl.h

@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#ifndef _MCA_PML_BASE_PTL_
+#define _MCA_PML_BASE_PTL_
+
+#include "mca/pml/pml.h"
+#include "mca/ptl/ptl.h"
+#include "threads/condition.h"
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+
+
+struct mca_pml_base_ptl_t {
+    ompi_list_t       ptl_cache;       /**< cache of send requests */
+    size_t            ptl_cache_size;  /**< maximum size of cache */
+    size_t            ptl_cache_alloc; /**< current number of allocated items */
+    ompi_mutex_t      ptl_cache_lock;  /**< lock for queue access */
+    struct mca_ptl_base_module_t* ptl; /**< back pointer to ptl */
+};
+typedef struct mca_pml_base_ptl_t mca_pml_base_ptl_t;
+
+OBJ_CLASS_DECLARATION(mca_pml_base_ptl_t);
+
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+
+#endif
+

src/mca/pml/teg/pml_teg_recvfrag.c

@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+/**
+ * @file
+ */
+
+#include "ompi_config.h"
+
+#include "mca/pml/pml.h"
+#include "pml_teg_recvfrag.h"
+#include "pml_teg_proc.h"
+
+
+OMPI_DECLSPEC extern ompi_class_t mca_ptl_base_recv_frag_t_class;
+
+
+/**
+ * Called by the PTL to match attempt a match for new fragments.
+ * 
+ * @param ptl (IN)      The PTL pointer
+ * @param frag (IN)     Receive fragment descriptor.
+ * @param header (IN)   Header corresponding to the receive fragment.
+ * @return              OMPI_SUCCESS or error status on failure.
+ */
+bool mca_pml_teg_recv_frag_match(
+    mca_ptl_base_module_t* ptl,
+    mca_ptl_base_recv_frag_t* frag, 
+    mca_ptl_base_match_header_t* header)
+{
+    bool matched;
+    bool matches = false;
+    ompi_list_t matched_frags;
+    if((matched = mca_ptl_base_match(header, frag, &matched_frags, &matches)) == false) {
+        frag = (matches ? (mca_ptl_base_recv_frag_t*)ompi_list_remove_first(&matched_frags) : NULL);
+    }
+
+    while(NULL != frag) {
+        mca_ptl_base_module_t* ptl = frag->frag_base.frag_owner;
+        mca_ptl_base_recv_request_t *request = frag->frag_request;
+        mca_ptl_base_match_header_t *header = &frag->frag_base.frag_header.hdr_match;
+
+        /*
+         * Initialize request status.
+         */
+        request->req_recv.req_bytes_packed = header->hdr_msg_length;
+        request->req_recv.req_base.req_ompi.req_status.MPI_SOURCE = header->hdr_src;
+        request->req_recv.req_base.req_ompi.req_status.MPI_TAG = header->hdr_tag;
+
+        /*
+         * If probe - signal request is complete - but don't notify PTL
+         */
+        if(request->req_recv.req_base.req_type == MCA_PML_REQUEST_PROBE) {
+
+             ptl->ptl_recv_progress( ptl, 
+                                     request,
+                                     header->hdr_msg_length,
+                                     header->hdr_msg_length );
+             matched = mca_pml_teg_recv_frag_match( ptl, frag, header );
+
+        } else {
+
+            /* if required - setup pointer to ptls peer */
+            if (NULL == frag->frag_base.frag_peer) {
+                frag->frag_base.frag_peer = mca_pml_teg_proc_lookup_remote_peer(
+                    request->req_recv.req_base.req_comm,header->hdr_src,ptl);
+            }
+
+            /* notify ptl of match */
+            ptl->ptl_matched(ptl, frag);
+
+        };
+
+        /* process any additional fragments that arrived out of order */
+        frag = (matches ? (mca_ptl_base_recv_frag_t*)ompi_list_remove_first(&matched_frags) : NULL);
+    };
+    return matched;
+}
+
+

src/mca/pml/teg/pml_teg_recvfrag.h

@@ -0,0 +1,42 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+/**
+ *  @file
+ */
+                                                                                                                                                 
+#ifndef MCA_PML_TEG_RECVFRAG_H
+#define MCA_PML_TEG_RECVFRAG_H
+
+#include "mca/ptl/ptl.h"
+#include "mca/pml/base/pml_base_recvreq.h"
+#include "mca/ptl/base/ptl_base_recvfrag.h"
+
+/**
+ * Called by the PTL to match attempt a match for new fragments.
+ *
+ * @param ptl (IN)      The PTL pointer
+ * @param frag (IN)     Receive fragment descriptor.
+ * @param header (IN)   Header corresponding to the receive fragment.
+ * @return              OMPI_SUCCESS or error status on failure.
+ */
+bool mca_pml_teg_recv_frag_match( 
+    mca_ptl_base_module_t* ptl,
+    mca_ptl_base_recv_frag_t* frag,
+    mca_ptl_base_match_header_t* header
+);
+
+#endif
+

src/mca/pml/teg/pml_teg_recvreq.c

@@ -0,0 +1,280 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "mca/pml/pml.h"
+#include "mca/ptl/ptl.h"
+#include "mca/ptl/base/ptl_base_comm.h"
+#include "pml_teg_recvreq.h"
+#include "pml_teg_sendreq.h"
+
+                                                                                                               
+static mca_ptl_base_recv_frag_t* mca_pml_teg_recv_request_match_specific_proc(
+    mca_ptl_base_recv_request_t* request, int proc);
+
+
+static int mca_pml_teg_recv_request_fini(struct ompi_request_t** request)
+{
+    MCA_PML_TEG_FINI(request);
+    return OMPI_SUCCESS;
+}
+
+static int mca_pml_teg_recv_request_free(struct ompi_request_t** request)
+{
+    MCA_PML_TEG_FREE(request);
+    return OMPI_SUCCESS;
+}
+
+
+static int mca_pml_teg_recv_request_cancel(struct ompi_request_t* request, int complete)
+{
+    mca_pml_base_request_t* teg_request = (mca_pml_base_request_t*)request;
+    ompi_communicator_t* ompi_comm = teg_request->req_comm;
+    mca_pml_ptl_comm_t* pml_comm = (mca_pml_ptl_comm_t*)ompi_comm->c_pml_comm;
+
+    if( true == request->req_complete ) {  /* way to late to cancel this one */
+       return OMPI_SUCCESS;
+    }
+    
+    /* The rest should be protected behind the match logic lock */
+    OMPI_THREAD_LOCK(&pml_comm->c_matching_lock);
+    
+    if( OMPI_ANY_TAG == request->req_status.MPI_TAG ) { /* the match have not been already done */
+       
+       if( teg_request->req_peer == OMPI_ANY_SOURCE ) {
+          ompi_list_remove_item( &(pml_comm->c_wild_receives),
+                                 (ompi_list_item_t*)request );
+       } else {
+          ompi_list_remove_item( pml_comm->c_specific_receives + teg_request->req_peer,
+                                 (ompi_list_item_t*)request );
+       }
+    }
+    
+    OMPI_THREAD_UNLOCK(&pml_comm->c_matching_lock);
+    
+    request->req_status._cancelled = true;
+    request->req_complete = true;  /* mark it as completed so all the test/wait  functions
+                                    * on this particular request will finish */
+    /* Now we have a problem if we are in a multi-threaded environment. We shou ld
+     * broadcast the condition on the request in order to allow the other threa ds
+     * to complete their test/wait functions.
+     */
+    if(ompi_request_waiting) {
+       ompi_condition_broadcast(&ompi_request_cond);
+    }
+    return OMPI_SUCCESS;
+}
+
+static void mca_pml_teg_recv_request_construct(mca_ptl_base_recv_request_t* request)
+{
+    request->req_recv.req_base.req_type = MCA_PML_REQUEST_RECV;
+    request->req_recv.req_base.req_ompi.req_fini = mca_pml_teg_recv_request_fini;
+    request->req_recv.req_base.req_ompi.req_free = mca_pml_teg_recv_request_free;
+    request->req_recv.req_base.req_ompi.req_cancel = mca_pml_teg_recv_request_cancel;
+}
+
+static void mca_pml_teg_recv_request_destruct(mca_ptl_base_recv_request_t* request)
+{
+}
+
+OBJ_CLASS_INSTANCE(
+    mca_pml_teg_recv_request_t,
+    mca_ptl_base_recv_request_t,
+    mca_pml_teg_recv_request_construct,
+    mca_pml_teg_recv_request_destruct);
+                                                                                                                                                     
+
+/*
+ * Update the recv request status to reflect the number of bytes
+ * received and actually delivered to the application. 
+ */
+
+void mca_pml_teg_recv_request_progress(
+    struct mca_ptl_base_module_t* ptl,
+    mca_ptl_base_recv_request_t* req,
+    size_t bytes_received,
+    size_t bytes_delivered)
+{
+    OMPI_THREAD_LOCK(&ompi_request_lock);
+    req->req_bytes_received += bytes_received;
+    req->req_bytes_delivered += bytes_delivered;
+    if (req->req_bytes_received >= req->req_recv.req_bytes_packed) {
+        /* initialize request status */
+        req->req_recv.req_base.req_ompi.req_status._count = req->req_bytes_delivered;
+        req->req_recv.req_base.req_pml_complete = true; 
+        req->req_recv.req_base.req_ompi.req_complete = true;
+        if(ompi_request_waiting) {
+            ompi_condition_broadcast(&ompi_request_cond);
+        }
+    }
+    OMPI_THREAD_UNLOCK(&ompi_request_lock);
+}
+
+
+                                                                                                                    
+/*
+ * This routine is used to match a posted receive when the source process 
+ * is specified.
+*/
+
+void mca_pml_teg_recv_request_match_specific(mca_ptl_base_recv_request_t* request)
+{
+    ompi_communicator_t *comm = request->req_recv.req_base.req_comm;
+    mca_pml_ptl_comm_t* pml_comm = comm->c_pml_comm;
+    int req_peer = request->req_recv.req_base.req_peer;
+    mca_ptl_base_recv_frag_t* frag;
+   
+    /* check for a specific match */
+    OMPI_THREAD_LOCK(&pml_comm->c_matching_lock);
+
+    /* assign sequence number */
+    request->req_recv.req_base.req_sequence = pml_comm->c_recv_seq++;
+
+    if (ompi_list_get_size(&pml_comm->c_unexpected_frags[req_peer]) > 0 &&
+        (frag = mca_pml_teg_recv_request_match_specific_proc(request, req_peer)) != NULL) {
+        mca_ptl_base_module_t* ptl = frag->frag_base.frag_owner;
+        /* setup pointer to ptls peer */
+        if(NULL == frag->frag_base.frag_peer) 
+            frag->frag_base.frag_peer = mca_pml_teg_proc_lookup_remote_peer(comm,req_peer,ptl);
+        OMPI_THREAD_UNLOCK(&pml_comm->c_matching_lock);
+        if( !((MCA_PML_REQUEST_IPROBE == request->req_recv.req_base.req_type) ||
+              (MCA_PML_REQUEST_PROBE == request->req_recv.req_base.req_type)) ) {
+            ptl->ptl_matched(ptl, frag);
+        }
+        return; /* match found */
+    }
+
+    /* We didn't find any matches.  Record this irecv so we can match 
+     * it when the message comes in.
+    */
+    if(request->req_recv.req_base.req_type != MCA_PML_REQUEST_IPROBE) { 
+        ompi_list_append(pml_comm->c_specific_receives+req_peer, (ompi_list_item_t*)request);
+    }
+    OMPI_THREAD_UNLOCK(&pml_comm->c_matching_lock);
+}
+
+
+/*
+ * this routine is used to try and match a wild posted receive - where
+ * wild is determined by the value assigned to the source process
+*/
+
+void mca_pml_teg_recv_request_match_wild(mca_ptl_base_recv_request_t* request)
+{
+    ompi_communicator_t *comm = request->req_recv.req_base.req_comm;
+    mca_pml_ptl_comm_t* pml_comm = comm->c_pml_comm;
+    int proc_count = comm->c_remote_group->grp_proc_count;
+    int proc;
+
+    /*
+     * Loop over all the outstanding messages to find one that matches.
+     * There is an outer loop over lists of messages from each
+     * process, then an inner loop over the messages from the
+     * process.
+    */
+    OMPI_THREAD_LOCK(&pml_comm->c_matching_lock);
+
+    /* assign sequence number */
+    request->req_recv.req_base.req_sequence = pml_comm->c_recv_seq++;
+
+    for (proc = 0; proc < proc_count; proc++) {
+        mca_ptl_base_recv_frag_t* frag;
+
+        /* continue if no frags to match */
+        if (ompi_list_get_size(&pml_comm->c_unexpected_frags[proc]) == 0)
+            continue;
+
+        /* loop over messages from the current proc */
+        if ((frag = mca_pml_teg_recv_request_match_specific_proc(request, proc)) != NULL) {
+            mca_ptl_base_module_t* ptl = frag->frag_base.frag_owner;
+            /* if required - setup pointer to ptls peer */
+            if(NULL == frag->frag_base.frag_peer) 
+                frag->frag_base.frag_peer = mca_pml_teg_proc_lookup_remote_peer(comm,proc,ptl);
+            OMPI_THREAD_UNLOCK(&pml_comm->c_matching_lock);
+            if( !((MCA_PML_REQUEST_IPROBE == request->req_recv.req_base.req_type) ||
+                  (MCA_PML_REQUEST_PROBE == request->req_recv.req_base.req_type)) ) {
+                ptl->ptl_matched(ptl, frag);
+            }
+            return; /* match found */
+        }
+    } 
+
+    /* We didn't find any matches.  Record this irecv so we can match to
+     * it when the message comes in.
+    */
+ 
+    if(request->req_recv.req_base.req_type != MCA_PML_REQUEST_IPROBE)
+        ompi_list_append(&pml_comm->c_wild_receives, (ompi_list_item_t*)request);
+    OMPI_THREAD_UNLOCK(&pml_comm->c_matching_lock);
+}
+
+
+/*
+ *  this routine tries to match a posted receive.  If a match is found,
+ *  it places the request in the appropriate matched receive list. 
+*/
+
+static mca_ptl_base_recv_frag_t* mca_pml_teg_recv_request_match_specific_proc(
+    mca_ptl_base_recv_request_t* request, int proc)
+{
+    mca_pml_ptl_comm_t *pml_comm = request->req_recv.req_base.req_comm->c_pml_comm;
+    ompi_list_t* unexpected_frags = pml_comm->c_unexpected_frags+proc;
+    mca_ptl_base_recv_frag_t* frag;
+    mca_ptl_base_match_header_t* header;
+    int tag = request->req_recv.req_base.req_tag;
+
+    if( OMPI_ANY_TAG == tag ) {
+        for (frag =  (mca_ptl_base_recv_frag_t*)ompi_list_get_first(unexpected_frags);
+             frag != (mca_ptl_base_recv_frag_t*)ompi_list_get_end(unexpected_frags);
+             frag =  (mca_ptl_base_recv_frag_t*)ompi_list_get_next(frag)) {
+            header = &(frag->frag_base.frag_header.hdr_match);
+            
+            /* check first frag - we assume that process matching has been done already */
+            if( header->hdr_tag >= 0 ) {
+                goto find_fragment;
+            } 
+        }
+    } else {
+        for (frag =  (mca_ptl_base_recv_frag_t*)ompi_list_get_first(unexpected_frags);
+             frag != (mca_ptl_base_recv_frag_t*)ompi_list_get_end(unexpected_frags);
+             frag =  (mca_ptl_base_recv_frag_t*)ompi_list_get_next(frag)) {
+            header = &(frag->frag_base.frag_header.hdr_match);
+            
+            /* check first frag - we assume that process matching has been done already */
+            if ( tag == header->hdr_tag ) {
+                /* we assume that the tag is correct from MPI point of view (ie. >= 0 ) */
+                goto find_fragment;
+            } 
+        }
+    }
+    return NULL;
+ find_fragment:
+    request->req_recv.req_bytes_packed = header->hdr_msg_length;
+    request->req_recv.req_base.req_ompi.req_status.MPI_TAG = header->hdr_tag;
+    request->req_recv.req_base.req_ompi.req_status.MPI_SOURCE = header->hdr_src;
+
+    if( !((MCA_PML_REQUEST_IPROBE == request->req_recv.req_base.req_type) ||
+          (MCA_PML_REQUEST_PROBE == request->req_recv.req_base.req_type)) ) {
+        ompi_list_remove_item(unexpected_frags, (ompi_list_item_t*)frag);
+        frag->frag_request = request;
+    } else {
+        /* it's a probe, therefore report it's completion */
+        mca_pml_teg_recv_request_progress( NULL, request, header->hdr_msg_length, header->hdr_msg_length );
+    }
+    return frag;
+}
+

src/mca/pml/teg/pml_teg_recvreq.h

@@ -0,0 +1,152 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+/**
+ * @file
+ */
+#ifndef OMPI_PML_TEG_RECV_REQUEST_H
+#define OMPI_PML_TEG_RECV_REQUEST_H
+
+#include "pml_teg.h"
+#include "pml_teg_proc.h"
+#include "mca/ptl/base/ptl_base_recvreq.h"
+#include "mca/ptl/base/ptl_base_recvfrag.h"
+
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+typedef mca_ptl_base_recv_request_t mca_pml_teg_recv_request_t;
+
+OBJ_CLASS_DECLARATION(mca_pml_teg_recv_request_t);
+
+
+/**
+ *  Allocate a recv request from the modules free list.
+ *
+ *  @param rc (OUT)  OMPI_SUCCESS or error status on failure.
+ *  @return          Receive request.
+ */
+#define MCA_PML_TEG_RECV_REQUEST_ALLOC(recvreq, rc)                   \
+    do {                                                              \
+        ompi_list_item_t* item;                                       \
+        OMPI_FREE_LIST_GET(&mca_pml_teg.teg_recv_requests, item, rc); \
+        recvreq = (mca_ptl_base_recv_request_t*)item;                 \
+    } while(0)
+        
+
+/**
+ *  Initialize a recv request.
+ */
+#define MCA_PML_TEG_RECV_REQUEST_INIT(                                \
+    request,                                                          \
+    addr,                                                             \
+    count,                                                            \
+    datatype,                                                         \
+    src,                                                              \
+    tag,                                                              \
+    comm,                                                             \
+    persistent)                                                       \
+{                                                                     \
+    MCA_PML_BASE_RECV_REQUEST_INIT(                                   \
+        (&(request)->req_recv),                                       \
+        addr,                                                         \
+        count,                                                        \
+        datatype,                                                     \
+        src,                                                          \
+        tag,                                                          \
+        comm,                                                         \
+        persistent                                                    \
+    );                                                                \
+}
+
+
+/**
+ *  Return a recv request to the modules free list.
+ *
+ *  @param request (IN)  Receive request.
+ */
+#define MCA_PML_TEG_RECV_REQUEST_RETURN(request)                                           \
+    do {                                                                                   \
+        MCA_PML_BASE_RECV_REQUEST_RETURN( &request->req_recv );                            \
+        OMPI_FREE_LIST_RETURN(&mca_pml_teg.teg_recv_requests, (ompi_list_item_t*)request); \
+    } while(0)
+
+/**
+ * Attempt to match the request against the unexpected fragment list
+ * for all source ranks w/in the communicator.
+ *
+ * @param request (IN)   Request to match.
+ */
+void mca_pml_teg_recv_request_match_wild(mca_ptl_base_recv_request_t* request);
+                                                                                                                                 
+/**
+ * Attempt to match the request against the unexpected fragment list
+ * for a specific source rank.
+ *
+ * @param request (IN)   Request to match.
+ */
+void mca_pml_teg_recv_request_match_specific(mca_ptl_base_recv_request_t* request);
+                                                                                                                                 
+/**
+ * Start an initialized request.
+ *
+ * @param request  Receive request.
+ * @return         OMPI_SUCESS or error status on failure.
+ */
+static inline int mca_pml_teg_recv_request_start(mca_ptl_base_recv_request_t* request)
+{
+    /* init/re-init the request */
+    request->req_bytes_received = 0;
+    request->req_bytes_delivered = 0;
+    request->req_recv.req_base.req_pml_complete = false;
+    request->req_recv.req_base.req_ompi.req_complete = false;
+    request->req_recv.req_base.req_ompi.req_state = OMPI_REQUEST_ACTIVE;
+    /* always set the req_status.MPI_TAG to ANY_TAG before starting the request. This field
+     * is used on the cancel part in order to find out if the request has been matched or not.
+     */
+    request->req_recv.req_base.req_ompi.req_status.MPI_TAG = OMPI_ANY_TAG;
+    request->req_recv.req_base.req_ompi.req_status.MPI_ERROR = OMPI_SUCCESS;
+    request->req_recv.req_base.req_ompi.req_status._cancelled = 0;
+
+    /* attempt to match posted recv */
+    if(request->req_recv.req_base.req_peer == OMPI_ANY_SOURCE) {
+        mca_pml_teg_recv_request_match_wild(request);
+    } else {
+        mca_pml_teg_recv_request_match_specific(request);
+    }
+    return OMPI_SUCCESS;
+}
+
+/**
+ *  Update status of a recv request based on the completion status of 
+ *  the receive fragment.
+ *
+ *  @param ptl (IN)              The PTL pointer.
+ *  @param request (IN)          Receive request.
+ *  @param bytes_received (IN)   Bytes received from peer.
+ *  @param bytes_delivered (IN)  Bytes delivered to application.
+ */
+void mca_pml_teg_recv_request_progress(
+    struct mca_ptl_base_module_t* ptl,
+    mca_ptl_base_recv_request_t* request,
+    size_t bytes_received,
+    size_t bytes_delivered
+);
+
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+#endif
+

src/mca/pml/teg/pml_teg_sendreq.c

@@ -0,0 +1,215 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+
+/*%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%*/
+
+#include "ompi_config.h"
+#include "include/constants.h"
+#include "mca/pml/pml.h"
+#include "mca/ptl/ptl.h"
+#include "pml_teg.h"
+#include "pml_teg_proc.h"
+#include "pml_teg_sendreq.h"
+#include "pml_teg_recvreq.h"
+
+
+                                                                                                         
+static int mca_pml_teg_send_request_fini(struct ompi_request_t** request)
+{
+    MCA_PML_TEG_FINI(request);
+    return OMPI_SUCCESS;
+}
+
+static int mca_pml_teg_send_request_free(struct ompi_request_t** request)
+{
+    MCA_PML_TEG_FREE(request);
+    return OMPI_SUCCESS;
+}
+
+static int mca_pml_teg_send_request_cancel(struct ompi_request_t* request, int complete)
+{
+    /* we dont cancel send requests by now */
+    return OMPI_SUCCESS;
+}
+                                                                                                             
+
+static void mca_pml_teg_send_request_construct(mca_pml_teg_send_request_t* req)
+{
+    req->req_cached = false;
+    req->req_send.req_base.req_ompi.req_fini = mca_pml_teg_send_request_fini;
+    req->req_send.req_base.req_ompi.req_free = mca_pml_teg_send_request_free;
+    req->req_send.req_base.req_ompi.req_cancel = mca_pml_teg_send_request_cancel;
+}
+
+
+static void mca_pml_teg_send_request_destruct(mca_pml_teg_send_request_t* req)
+{
+}
+
+
+OBJ_CLASS_INSTANCE(
+    mca_pml_teg_send_request_t,
+    mca_ptl_base_send_request_t,
+    mca_pml_teg_send_request_construct,
+    mca_pml_teg_send_request_destruct);
+                                                                                                                                                     
+
+/**
+ *  Schedule message delivery across potentially multiple PTLs. 
+ *
+ *  @param request (IN)    Request to schedule
+ *  @return status         Error status
+ *
+ */
+
+
+int mca_pml_teg_send_request_schedule(mca_ptl_base_send_request_t* req)
+{
+    ompi_proc_t *proc = ompi_comm_peer_lookup(req->req_send.req_base.req_comm, req->req_send.req_base.req_peer);
+    mca_pml_proc_t* proc_pml = proc->proc_pml;
+    int send_count = 0;
+    size_t bytes_remaining;
+    size_t num_ptl_avail;
+    size_t num_ptl;
+
+    /*
+     * Only allow one thread in this routine for a given request. 
+     * However, we cannot block callers on a mutex, so simply keep track 
+     * of the number of times the routine has been called and run through
+     * the scheduling logic once for every call.
+    */
+    if(OMPI_THREAD_ADD32(&req->req_lock,1) == 1) {
+        do {
+            /* allocate remaining bytes to PTLs */
+            bytes_remaining = req->req_send.req_bytes_packed - req->req_offset;
+            num_ptl_avail = proc_pml->proc_ptl_next.ptl_size;
+            num_ptl = 0;
+            while(bytes_remaining > 0 && num_ptl++ < num_ptl_avail) {
+                mca_ptl_proc_t* ptl_proc = mca_ptl_array_get_next(&proc_pml->proc_ptl_next);
+                mca_ptl_base_module_t* ptl = ptl_proc->ptl;
+                int rc;
+        
+                /* if this is the last PTL that is available to use, or the number of 
+                 * bytes remaining in the message is less than the PTLs minimum fragment 
+                 * size, then go ahead and give the rest of the message to this PTL.
+                 */
+                size_t bytes_to_frag;
+                if(num_ptl == num_ptl_avail || bytes_remaining < ptl->ptl_min_frag_size) {
+                    bytes_to_frag = bytes_remaining;
+        
+                /* otherwise attempt to give the PTL a percentage of the message
+                 * based on a weighting factor. for simplicity calculate this as  
+                 * a percentage of the overall message length (regardless of amount 
+                 * previously assigned)
+                 */
+                } else {
+                    bytes_to_frag = (ptl_proc->ptl_weight * bytes_remaining) / 100;
+                }
+    
+                /* makes sure that we don't exceed ptl_max_frag_size */
+                if(ptl->ptl_max_frag_size != 0 && bytes_to_frag > ptl->ptl_max_frag_size)
+                    bytes_to_frag = ptl->ptl_max_frag_size;
+        
+                rc = ptl->ptl_put(ptl, ptl_proc->ptl_peer, req, req->req_offset, bytes_to_frag, 0);
+                if(rc == OMPI_SUCCESS) {
+                    send_count++;
+                    bytes_remaining = req->req_send.req_bytes_packed - req->req_offset;
+                }
+            }
+    
+            /* unable to complete send - queue for later */
+            if(send_count == 0) {
+                OMPI_THREAD_LOCK(&mca_pml_teg.teg_lock);
+                ompi_list_append(&mca_pml_teg.teg_send_pending, (ompi_list_item_t*)req);
+                OMPI_THREAD_UNLOCK(&mca_pml_teg.teg_lock);
+                req->req_lock = 0;
+                return OMPI_ERR_OUT_OF_RESOURCE;
+            }
+
+        /* fragments completed while scheduling - so retry */
+        } while(OMPI_THREAD_ADD32(&req->req_lock,-1) > 0);
+
+        /* free the request if completed while in the scheduler */
+        if (req->req_send.req_base.req_free_called && req->req_send.req_base.req_pml_complete) {
+            MCA_PML_TEG_FREE((ompi_request_t**)&req);
+        }
+    }
+    return OMPI_SUCCESS;
+}
+
+
+/**
+ *  Update the status of the send request to reflect the number of bytes
+ *  "actually" sent (and acknowledged). This should be called by the
+ *  lower layer PTL after the fragment is actually delivered and has been
+ *  acknowledged (if required). Note that this routine should NOT be called
+ *  directly by the PTL, a function pointer is setup on the PTL at init to 
+ *  enable upcalls into the PML w/out directly linking to a specific PML 
+ *  implementation.
+ */
+
+void mca_pml_teg_send_request_progress(
+    struct mca_ptl_base_module_t* ptl,
+    mca_ptl_base_send_request_t* req,
+    size_t bytes_sent)
+{
+    bool schedule = false;
+
+    OMPI_THREAD_LOCK(&ompi_request_lock);
+    req->req_bytes_sent += bytes_sent;
+    if (req->req_bytes_sent >= req->req_send.req_bytes_packed) {
+        req->req_send.req_base.req_pml_complete = true;
+        if (req->req_send.req_base.req_ompi.req_complete == false) {
+            req->req_send.req_base.req_ompi.req_status.MPI_SOURCE = req->req_send.req_base.req_comm->c_my_rank;
+            req->req_send.req_base.req_ompi.req_status.MPI_TAG = req->req_send.req_base.req_tag;
+            req->req_send.req_base.req_ompi.req_status.MPI_ERROR = OMPI_SUCCESS;
+            req->req_send.req_base.req_ompi.req_status._count = req->req_bytes_sent;
+            req->req_send.req_base.req_ompi.req_complete = true;
+            if(ompi_request_waiting) {
+                ompi_condition_broadcast(&ompi_request_cond);
+            }
+        } else if(req->req_send.req_base.req_free_called) {
+            /* don't free the request if in the scheduler */
+            if(req->req_lock == 0) {
+                MCA_PML_TEG_FREE((ompi_request_t**)&req);
+            }
+        } else if (req->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED) {
+            mca_pml_base_bsend_request_fini((ompi_request_t*)req);
+        }
+    /* test to see if we have scheduled the entire request */
+    } else if (req->req_offset < req->req_send.req_bytes_packed) {
+        schedule = true;
+    }
+    OMPI_THREAD_UNLOCK(&ompi_request_lock);
+
+    /* schedule remaining fragments of this request */
+    if(schedule) {
+        mca_pml_teg_send_request_schedule(req);
+    }
+
+    /* check for pending requests that need to be progressed */
+    while(ompi_list_get_size(&mca_pml_teg.teg_send_pending) != 0) {
+        OMPI_THREAD_LOCK(&mca_pml_teg.teg_lock);
+        req = (mca_ptl_base_send_request_t*)ompi_list_remove_first(&mca_pml_teg.teg_send_pending);
+        OMPI_THREAD_UNLOCK(&mca_pml_teg.teg_lock);
+        if(req == NULL)
+            break;
+        if(mca_pml_teg_send_request_schedule(req) != OMPI_SUCCESS)
+            break;
+    }
+}
+

src/mca/pml/teg/pml_teg_sendreq.h

@@ -0,0 +1,211 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+/**
+ * @file
+ */
+#ifndef OMPI_PML_TEG_SEND_REQUEST_H
+#define OMPI_PML_TEG_SEND_REQUEST_H
+
+#include "mca/ptl/ptl.h"
+#include "mca/ptl/base/ptl_base_sendreq.h"
+#include "mca/ptl/base/ptl_base_sendfrag.h"
+#include "mca/ptl/base/ptl_base_comm.h"
+#include "pml_teg_proc.h"
+#include "pml_teg_ptl.h"
+
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+
+
+typedef mca_ptl_base_send_request_t mca_pml_teg_send_request_t;
+OBJ_CLASS_DECLARATION(mca_pml_teg_send_request_t);
+
+
+#define MCA_PML_TEG_SEND_REQUEST_ALLOC(                                    \
+    comm,                                                                  \
+    dst,                                                                   \
+    sendreq,                                                               \
+    rc)                                                                    \
+{                                                                          \
+    mca_pml_proc_t *proc = mca_pml_teg_proc_lookup_remote(comm,dst);       \
+    mca_ptl_proc_t* ptl_proc;                                              \
+    mca_pml_base_ptl_t* ptl_base;                                          \
+                                                                           \
+    if(NULL == proc) {                                                     \
+       return OMPI_ERR_OUT_OF_RESOURCE;                                    \
+    }                                                                      \
+    OMPI_THREAD_SCOPED_LOCK(&proc->proc_lock,                              \
+       (ptl_proc = mca_ptl_array_get_next(&proc->proc_ptl_first)));        \
+    ptl_base = ptl_proc->ptl_base;                                         \
+    /*                                                                     \
+     * check to see if there is a cache of send requests associated with   \
+     * this ptl - if so try the allocation from there.                     \
+    */                                                                     \
+    if(NULL != ptl_base) {                                                 \
+        OMPI_THREAD_LOCK(&ptl_base->ptl_cache_lock);                       \
+        sendreq = (mca_pml_teg_send_request_t*)                            \
+            ompi_list_remove_first(&ptl_base->ptl_cache);                  \
+        if(NULL != sendreq) {                                              \
+            OMPI_THREAD_UNLOCK(&ptl_base->ptl_cache_lock);                 \
+            rc = OMPI_SUCCESS;                                             \
+        } else if (ptl_base->ptl_cache_alloc < ptl_base->ptl_cache_size) { \
+            /*                                                             \
+             * allocate an additional request to the cache                 \
+            */                                                             \
+            mca_ptl_base_module_t* ptl = ptl_base->ptl;                    \
+            ompi_list_item_t* item;                                        \
+            OMPI_FREE_LIST_WAIT(&mca_pml_teg.teg_send_requests, item, rc); \
+            sendreq = (mca_pml_teg_send_request_t*)item;                   \
+            sendreq->req_ptl = ptl;                                        \
+            if(ptl->ptl_request_init(ptl, sendreq) == OMPI_SUCCESS) {      \
+                sendreq->req_cached = true;                                \
+                ptl_base->ptl_cache_alloc++;                               \
+            }                                                              \
+            OMPI_THREAD_UNLOCK(&ptl_base->ptl_cache_lock);                 \
+        } else {                                                           \
+            /*                                                             \
+             * take a request from the global pool                         \
+            */                                                             \
+            ompi_list_item_t* item;                                        \
+            OMPI_THREAD_UNLOCK(&ptl_base->ptl_cache_lock);                 \
+            OMPI_FREE_LIST_WAIT(&mca_pml_teg.teg_send_requests, item, rc); \
+            sendreq = (mca_pml_teg_send_request_t*)item;                   \
+            sendreq->req_ptl = ptl_proc->ptl;                              \
+        }                                                                  \
+                                                                           \
+    /* otherwise - take the allocation from the global list */             \
+    } else {                                                               \
+        ompi_list_item_t* item;                                            \
+        OMPI_FREE_LIST_WAIT(&mca_pml_teg.teg_send_requests, item, rc);     \
+        sendreq = (mca_pml_teg_send_request_t*)item;                       \
+        sendreq->req_ptl = ptl_proc->ptl;                                  \
+    }                                                                      \
+    /* update request to point to current peer */                          \
+    sendreq->req_peer = ptl_proc->ptl_peer;                                \
+}
+
+
+#define MCA_PML_TEG_SEND_REQUEST_INIT( request,                            \
+                                        addr,                              \
+                                        count,                             \
+                                        datatype,                          \
+                                        peer,                              \
+                                        tag,                               \
+                                        comm,                              \
+                                        mode,                              \
+                                        persistent)                        \
+{                                                                          \
+   MCA_PML_BASE_SEND_REQUEST_INIT((&request->req_send),                    \
+       addr,                                                               \
+       count,                                                              \
+       datatype,                                                           \
+       peer,                                                               \
+       tag,                                                                \
+       comm,                                                               \
+       mode,                                                               \
+       persistent                                                          \
+    );                                                                     \
+}
+    
+
+#define MCA_PML_TEG_SEND_REQUEST_RETURN(sendreq)                           \
+{                                                                          \
+    mca_ptl_base_module_t* ptl = (sendreq)->req_ptl;                       \
+    mca_pml_base_ptl_t* ptl_base = ptl->ptl_base;                          \
+                                                                           \
+    /*  Let the base handle the reference counts */                        \
+    MCA_PML_BASE_SEND_REQUEST_RETURN((&sendreq->req_send));                \
+                                                                           \
+    /*                                                                     \
+     * If there is a cache associated with the ptl - first attempt         \
+     * to return the send descriptor to the cache.                         \
+     */                                                                    \
+    if(NULL != ptl->ptl_base && (sendreq)->req_cached) {                   \
+        OMPI_THREAD_LOCK(&ptl_base->ptl_cache_lock);                       \
+        ompi_list_prepend(&ptl_base->ptl_cache,                            \
+                          (ompi_list_item_t*)sendreq);                     \
+        OMPI_THREAD_UNLOCK(&ptl_base->ptl_cache_lock);                     \
+    } else {                                                               \
+        OMPI_FREE_LIST_RETURN(                                             \
+            &mca_pml_teg.teg_send_requests, (ompi_list_item_t*)sendreq);   \
+    }                                                                      \
+}
+
+
+/**
+ * Start a send request. 
+ */
+#define MCA_PML_TEG_SEND_REQUEST_START(req, rc)                            \
+{                                                                          \
+    mca_ptl_base_module_t* ptl = req->req_ptl;                             \
+    size_t first_fragment_size = ptl->ptl_first_frag_size;                 \
+    int flags;                                                             \
+                                                                           \
+    req->req_lock = 0;                                                     \
+    req->req_bytes_sent = 0;                                               \
+    req->req_peer_match.lval = 0;                                          \
+    req->req_peer_addr.lval = 0;                                           \
+    req->req_peer_size = 0;                                                \
+    req->req_offset = 0;                                                   \
+    req->req_send.req_base.req_pml_complete = false;                       \
+    req->req_send.req_base.req_ompi.req_complete = false;                  \
+    req->req_send.req_base.req_ompi.req_state = OMPI_REQUEST_ACTIVE;       \
+    req->req_send.req_base.req_sequence = mca_pml_ptl_comm_send_sequence(  \
+        req->req_send.req_base.req_comm->c_pml_comm, req->req_send.req_base.req_peer); \
+                                                                           \
+    /* handle buffered send */                                             \
+    if(req->req_send.req_send_mode == MCA_PML_BASE_SEND_BUFFERED) {        \
+        mca_pml_base_bsend_request_start(&req->req_send.req_base.req_ompi); \
+    }                                                                      \
+                                                                           \
+    /* start the first fragment */                                         \
+    if (first_fragment_size == 0 ||                                        \
+        req->req_send.req_bytes_packed <= first_fragment_size) {           \
+        first_fragment_size = req->req_send.req_bytes_packed;              \
+        flags = (req->req_send.req_send_mode == MCA_PML_BASE_SEND_SYNCHRONOUS) ? \
+        MCA_PTL_FLAGS_ACK : 0;                                             \
+    } else {                                                               \
+        /* require match for first fragment of a multi-fragment */         \
+        flags = MCA_PTL_FLAGS_ACK;                                         \
+    }                                                                      \
+    rc = ptl->ptl_send(ptl, req->req_peer, req, 0, first_fragment_size,    \
+        flags);                                                            \
+}
+
+
+/**
+ *  Schedule any data that was not delivered in the first fragment
+ *  across the available PTLs.
+ */
+int mca_pml_teg_send_request_schedule(mca_ptl_base_send_request_t* req);
+
+
+/**
+ *  Update the request to reflect the number of bytes delivered. If this
+ *  was the first fragment - schedule the rest of the data.
+ */
+void mca_pml_teg_send_request_progress(
+    struct mca_ptl_base_module_t* ptl,
+    mca_ptl_base_send_request_t* send_request,
+    size_t bytes_sent
+);
+
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+#endif
+

src/mca/pml/teg/pml_teg_start.c

@@ -0,0 +1,122 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+
+#include "ompi_config.h"
+
+#include "pml_teg.h"
+#include "pml_teg_recvreq.h"
+#include "pml_teg_sendreq.h"
+
+
+int mca_pml_teg_start(size_t count, ompi_request_t** requests)
+{
+    int rc;
+    size_t i;
+    for(i=0; i<count; i++) {
+        mca_pml_base_request_t *pml_request = (mca_pml_base_request_t*)requests[i];
+        if(NULL == pml_request)
+            continue;
+
+        /* If the persistent request is currently active - obtain the
+         * request lock and verify the status is incomplete. if the
+         * pml layer has not completed the request - mark the request
+         * as free called - so that it will be freed when the request
+         * completes - and create a new request.
+         */
+
+        switch(pml_request->req_ompi.req_state) {
+            case OMPI_REQUEST_INACTIVE:
+                if(pml_request->req_pml_complete == true)
+                    break;
+                /* otherwise fall through */
+            case OMPI_REQUEST_ACTIVE: {
+            
+                ompi_request_t *request;
+                OMPI_THREAD_LOCK(&ompi_request_lock);
+                if (pml_request->req_pml_complete == false) {
+                    /* free request after it completes */
+                    pml_request->req_free_called = true;
+                } else {
+                    /* can reuse the existing request */
+                    OMPI_THREAD_UNLOCK(&ompi_request_lock);
+                    break;
+                }
+
+                /* allocate a new request */
+                switch(pml_request->req_type) {
+                    case MCA_PML_REQUEST_SEND: {
+                         mca_pml_base_send_mode_t sendmode = 
+                             ((mca_pml_base_send_request_t*)pml_request)->req_send_mode;
+                         rc = mca_pml_teg_isend_init(
+                              pml_request->req_addr,
+                              pml_request->req_count,
+                              pml_request->req_datatype,
+                              pml_request->req_peer,
+                              pml_request->req_tag,
+                              sendmode,
+                              pml_request->req_comm,
+                              &request);
+                         break;
+                    }
+                    case MCA_PML_REQUEST_RECV:
+                         rc = mca_pml_teg_irecv_init(
+                              pml_request->req_addr,
+                              pml_request->req_count,
+                              pml_request->req_datatype,
+                              pml_request->req_peer,
+                              pml_request->req_tag,
+                              pml_request->req_comm,
+                              &request);
+                         break;
+                    default:
+                         rc = OMPI_ERR_REQUEST;
+                         break;
+                }
+                OMPI_THREAD_UNLOCK(&ompi_request_lock);
+                if(OMPI_SUCCESS != rc)
+                    return rc;
+                pml_request = (mca_pml_base_request_t*)request;
+                requests[i] = request;
+                break;
+            }
+            default:
+                return OMPI_ERR_REQUEST;
+        }
+
+        /* start the request */
+        switch(pml_request->req_type) {
+            case MCA_PML_REQUEST_SEND: 
+            {
+                mca_ptl_base_send_request_t* sendreq = (mca_ptl_base_send_request_t*)pml_request;
+                MCA_PML_TEG_SEND_REQUEST_START(sendreq, rc);
+                if(rc != OMPI_SUCCESS)
+                    return rc;
+                break;
+            }
+            case MCA_PML_REQUEST_RECV:
+            {
+                mca_ptl_base_recv_request_t* recvreq = (mca_ptl_base_recv_request_t*)pml_request;
+                if((rc = mca_pml_teg_recv_request_start(recvreq)) != OMPI_SUCCESS)
+                    return rc;
+                break;
+            }
+            default:
+                return OMPI_ERR_REQUEST;
+        }
+    }
+    return OMPI_SUCCESS;
+}
+

src/mca/pml/teg/src/pml_teg_recvreq.c

@@ -81,10 +81,10 @@ static int mca_pml_teg_recv_request_cancel(struct ompi_request_t* request, int c
 
 static void mca_pml_teg_recv_request_construct(mca_pml_base_recv_request_t* request)
 {
-    request->req_base.req_type = MCA_PML_REQUEST_RECV;
+    request->req_recv.req_base.req_type = MCA_PML_REQUEST_RECV;
-    request->req_base.req_ompi.req_fini = mca_pml_teg_recv_request_fini;
+    request->req_recv.req_base.req_ompi.req_fini = mca_pml_teg_recv_request_fini;
-    request->req_base.req_ompi.req_free = mca_pml_teg_recv_request_free;
+    request->req_recv.req_base.req_ompi.req_free = mca_pml_teg_recv_request_free;
-    request->req_base.req_ompi.req_cancel = mca_pml_teg_recv_request_cancel;
+    request->req_recv.req_base.req_ompi.req_cancel = mca_pml_teg_recv_request_cancel;
 }
 
 static void mca_pml_teg_recv_request_destruct(mca_pml_base_recv_request_t* request)
@@ -93,7 +93,7 @@ static void mca_pml_teg_recv_request_destruct(mca_pml_base_recv_request_t* reque
 
 OBJ_CLASS_INSTANCE(
     mca_pml_teg_recv_request_t,
-    mca_pml_base_recv_request_t,
+    mca_ptl_base_recv_request_t,
     mca_pml_teg_recv_request_construct,
     mca_pml_teg_recv_request_destruct);
                                                                                                                                                      

src/mca/pml/teg/src/pml_teg_recvreq.h

@@ -21,13 +21,13 @@
 
 #include "pml_teg.h"
 #include "pml_teg_proc.h"
-#include "mca/pml/base/pml_base_recvreq.h"
+#include "mca/ptl/base/ptl_base_recvreq.h"
 #include "mca/ptl/base/ptl_base_recvfrag.h"
 
 #if defined(c_plusplus) || defined(__cplusplus)
 extern "C" {
 #endif
-typedef mca_pml_base_recv_request_t mca_pml_teg_recv_request_t;
+typedef mca_ptl_base_recv_request_t mca_pml_teg_recv_request_t;
 
 OBJ_CLASS_DECLARATION(mca_pml_teg_recv_request_t);
 

src/mca/ptl/base/Makefile.am

@@ -31,7 +31,9 @@ headers = \
 	ptl_base_header.h \
 	ptl_base_match.h \
 	ptl_base_recvfrag.h \
-	ptl_base_sendfrag.h
+	ptl_base_recvreq.h \
+	ptl_base_sendfrag.h \
+    ptl_base_sendreq.h
 
 libmca_ptl_base_la_SOURCES = \
     $(headers) \
@@ -41,8 +43,10 @@ libmca_ptl_base_la_SOURCES = \
 	ptl_base_match.c \
     ptl_base_open.c \
     ptl_base_recvfrag.c \
+    ptl_base_recvreq.c \
     ptl_base_select.c \
-        ptl_base_sendfrag.c
+    ptl_base_sendfrag.c \
+    ptl_base_sendreq.c
 
 # Conditionally install the header files
 

src/mca/ptl/base/ptl_base_match.c

@@ -50,7 +50,7 @@
 do { \
     /* local parameters */ \
     ompi_list_t* wild_receives = &pml_comm->c_wild_receives; \
-    mca_pml_base_recv_request_t *wild_recv; \
+    mca_ptl_base_recv_request_t *wild_recv; \
     int frag_tag,recv_tag; \
  \
     /* initialization */ \
@@ -61,14 +61,14 @@ do { \
      * locking is protecting from having other threads trying to \
      * change this list. \
      */ \
-    for(wild_recv = (mca_pml_base_recv_request_t *)  \
+    for(wild_recv = (mca_ptl_base_recv_request_t *)  \
             ompi_list_get_first(wild_receives); \
-            wild_recv != (mca_pml_base_recv_request_t *) \
+            wild_recv != (mca_ptl_base_recv_request_t *) \
             ompi_list_get_end(wild_receives); \
-            wild_recv = (mca_pml_base_recv_request_t *)  \
+            wild_recv = (mca_ptl_base_recv_request_t *)  \
             ((ompi_list_item_t *)wild_recv)->ompi_list_next) { \
  \
-        recv_tag = wild_recv->req_base.req_tag; \
+        recv_tag = wild_recv->req_recv.req_base.req_tag; \
         if (  \
                 /* exact tag match */ \
                 (frag_tag == recv_tag) || \
@@ -112,7 +112,7 @@ do { \
 do { \
     /* local variables */ \
     ompi_list_t* specific_receives = (pml_comm->c_specific_receives)+frag_src; \
-    mca_pml_base_recv_request_t *specific_recv; \
+    mca_ptl_base_recv_request_t *specific_recv; \
     int frag_src,recv_tag,frag_tag; \
  \
     /* initialization */ \
@@ -122,16 +122,16 @@ do { \
     /* \
      * Loop over the specific irecvs. \
      */ \
-    for(specific_recv = (mca_pml_base_recv_request_t *)  \
+    for(specific_recv = (mca_ptl_base_recv_request_t *)  \
             ompi_list_get_first(specific_receives); \
-            specific_recv != (mca_pml_base_recv_request_t *) \
+            specific_recv != (mca_ptl_base_recv_request_t *) \
             ompi_list_get_end(specific_receives); \
-            specific_recv = (mca_pml_base_recv_request_t *)  \
+            specific_recv = (mca_ptl_base_recv_request_t *)  \
             ((ompi_list_item_t *)specific_recv)->ompi_list_next) { \
         /* \
          * Check for a match \
          */ \
-        recv_tag = specific_recv->req_base.req_tag; \
+        recv_tag = specific_recv->req_recv.req_base.req_tag; \
         if ( (frag_tag == recv_tag) || \
              ( (recv_tag == OMPI_ANY_TAG) && (0 <= frag_tag) ) ) { \
  \
@@ -170,7 +170,7 @@ do { \
     frag_header, pml_comm, return_match) \
 do {  \
     /* local variables */  \
-    mca_pml_base_recv_request_t *specific_recv, *wild_recv; \
+    mca_ptl_base_recv_request_t *specific_recv, *wild_recv; \
     mca_ptl_sequence_t wild_recv_seq, specific_recv_seq;  \
     int frag_src,frag_tag, wild_recv_tag, specific_recv_tag;  \
   \
@@ -182,13 +182,13 @@ do {  \
      * We know that when this is called, both specific and wild irecvs  \
      *  have been posted.  \
      */  \
-    specific_recv = (mca_pml_base_recv_request_t *)   \
+    specific_recv = (mca_ptl_base_recv_request_t *)   \
             ompi_list_get_first((pml_comm->c_specific_receives)+frag_src);  \
-    wild_recv = (mca_pml_base_recv_request_t *)  \
+    wild_recv = (mca_ptl_base_recv_request_t *)  \
             ompi_list_get_first(&(pml_comm->c_wild_receives));  \
   \
-    specific_recv_seq = specific_recv->req_base.req_sequence;  \
+    specific_recv_seq = specific_recv->req_recv.req_base.req_sequence;  \
-    wild_recv_seq = wild_recv->req_base.req_sequence;  \
+    wild_recv_seq = wild_recv->req_recv.req_base.req_sequence;  \
   \
     while (true) {  \
         if (wild_recv_seq < specific_recv_seq) {  \
@@ -198,7 +198,7 @@ do {  \
             /*  \
              * try and match  \
              */  \
-            wild_recv_tag = wild_recv->req_base.req_tag;  \
+            wild_recv_tag = wild_recv->req_recv.req_base.req_tag;  \
             if ( (frag_tag == wild_recv_tag) ||  \
                  ( (wild_recv_tag == OMPI_ANY_TAG) && (0 <= frag_tag) ) ) {  \
                 /*  \
@@ -215,14 +215,14 @@ do {  \
             /*  \
              * No match, go to the next.  \
              */  \
-            wild_recv=(mca_pml_base_recv_request_t *)  \
+            wild_recv=(mca_ptl_base_recv_request_t *)  \
                 ((ompi_list_item_t *)wild_recv)->ompi_list_next;  \
   \
             /*  \
              * If that was the last wild one, just look at the  \
              * rest of the specific ones.  \
              */  \
-            if (wild_recv == (mca_pml_base_recv_request_t *)  \
+            if (wild_recv == (mca_ptl_base_recv_request_t *)  \
                     ompi_list_get_end(&(pml_comm->c_wild_receives)) )   \
             {   \
                 MCA_PTL_BASE_CHECK_SPECIFIC_RECEIVES_FOR_MATCH(frag_header, pml_comm, return_match);  \
@@ -233,13 +233,13 @@ do {  \
              * Get the sequence number for this recv, and go  \
              * back to the top of the loop.  \
              */  \
-            wild_recv_seq = wild_recv->req_base.req_sequence;  \
+            wild_recv_seq = wild_recv->req_recv.req_base.req_sequence;  \
   \
         } else {  \
             /*  \
              * specific recv is earlier than the wild one.  \
              */  \
-            specific_recv_tag=specific_recv->req_base.req_tag;  \
+            specific_recv_tag=specific_recv->req_recv.req_base.req_tag;  \
             if ( (frag_tag == specific_recv_tag) || \
                  ( (specific_recv_tag == OMPI_ANY_TAG) && (0<=frag_tag)) )   \
             {  \
@@ -256,14 +256,14 @@ do {  \
             /*  \
              * No match, go on to the next specific irecv.  \
              */  \
-            specific_recv = (mca_pml_base_recv_request_t *)  \
+            specific_recv = (mca_ptl_base_recv_request_t *)  \
                 ((ompi_list_item_t *)specific_recv)->ompi_list_next;  \
   \
             /*  \
              * If that was the last specific irecv, process the  \
              * rest of the wild ones.  \
              */  \
-            if (specific_recv == (mca_pml_base_recv_request_t *)  \
+            if (specific_recv == (mca_ptl_base_recv_request_t *)  \
                     ompi_list_get_end((pml_comm->c_specific_receives)+frag_src) )  \
             {  \
                 MCA_PTL_BASE_CHECK_WILD_RECEIVES_FOR_MATCH(frag_header, pml_comm, return_match);  \
@@ -273,7 +273,7 @@ do {  \
              * Get the sequence number for this recv, and go  \
              * back to the top of the loop.  \
              */ \
-            specific_recv_seq = specific_recv->req_base.req_sequence;  \
+            specific_recv_seq = specific_recv->req_recv.req_base.req_sequence;  \
         }  \
     }  \
 } while(0)
@@ -328,7 +328,7 @@ bool mca_ptl_base_match(
     uint16_t next_msg_seq_expected, frag_msg_seq;
 
     ompi_communicator_t *comm_ptr;
-    mca_pml_base_recv_request_t *matched_receive = NULL;
+    mca_ptl_base_recv_request_t *matched_receive = NULL;
     mca_pml_ptl_comm_t *pml_comm;
     int frag_src;
     bool match_made=false;
@@ -398,12 +398,12 @@ bool mca_ptl_base_match(
             frag_desc->frag_request=matched_receive;
 
             /* set lenght of incoming message */
-            matched_receive->req_bytes_packed = frag_header->hdr_msg_length;
+            matched_receive->req_recv.req_bytes_packed = frag_header->hdr_msg_length;
 
             /*
              * update delivered sequence number information, if needed.
              */
-            if( (matched_receive->req_base.req_type == MCA_PML_REQUEST_PROBE) ) {
+            if( (matched_receive->req_recv.req_base.req_type == MCA_PML_REQUEST_PROBE) ) {
                 /* Match a probe, rollback the next expected sequence number */
                 (pml_comm->c_next_msg_seq[frag_src])--;
             }
@@ -461,7 +461,7 @@ static bool mca_ptl_base_check_cantmatch_for_match(ompi_list_t *additional_match
     int match_found;
     uint16_t next_msg_seq_expected, frag_seq;
     mca_ptl_base_recv_frag_t *frag_desc;
-    mca_pml_base_recv_request_t *matched_receive = NULL;
+    mca_ptl_base_recv_request_t *matched_receive = NULL;
     bool match_made = false;
 
     /*
@@ -604,7 +604,7 @@ bool mca_ptl_base_match_in_order_network_delivery(
 {
     /* local variables */
     ompi_communicator_t *comm_ptr;
-    mca_pml_base_recv_request_t *matched_receive = NULL;
+    mca_ptl_base_recv_request_t *matched_receive = NULL;
     mca_pml_ptl_comm_t *pml_comm;
     int frag_src;
 
@@ -658,7 +658,7 @@ bool mca_ptl_base_match_in_order_network_delivery(
         frag_desc->frag_request=matched_receive;
 
         /* set lenght of incoming message */
-        matched_receive->req_bytes_packed=frag_header->hdr_msg_length;
+        matched_receive->req_recv.req_bytes_packed=frag_header->hdr_msg_length;
 
     } else {
         /* if no match found, place on unexpected queue */

src/mca/ptl/base/ptl_base_recvfrag.h

@@ -20,7 +20,7 @@
 #define MCA_PTL_BASE_RECVFRAG_H
 
 #include "mca/ptl/ptl.h"
-#include "mca/pml/base/pml_base_recvreq.h"
+#include "mca/ptl/base/ptl_base_recvreq.h"
 #include "mca/ptl/base/ptl_base_fragment.h"
 #include "mca/ptl/base/ptl_base_match.h"
 
@@ -34,7 +34,7 @@ OMPI_DECLSPEC extern ompi_class_t mca_ptl_base_recv_frag_t_class;
  */
 struct mca_ptl_base_recv_frag_t {
     mca_ptl_base_frag_t frag_base; /**< base fragment descriptor */
-    mca_pml_base_recv_request_t *frag_request; /**< matched posted receive */
+    mca_ptl_base_recv_request_t *frag_request; /**< matched posted receive */
     bool frag_is_buffered; /**< does fragment need to be unpacked into users buffer */
 };
 typedef struct mca_ptl_base_recv_frag_t mca_ptl_base_recv_frag_t;

src/mca/ptl/base/ptl_base_recvreq.c

@@ -0,0 +1,10 @@
+#include "ptl_base_recvreq.h"
+
+
+OBJ_CLASS_INSTANCE(
+    mca_ptl_base_recv_request_t,
+    mca_pml_base_recv_request_t,
+    NULL,
+    NULL
+);
+    

src/mca/ptl/base/ptl_base_recvreq.h

@@ -0,0 +1,47 @@
+/* 
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+/**
+ * @file
+ */
+#ifndef MCA_PTL_BASE_RECV_REQUEST_H
+#define MCA_PTL_BASE_RECV_REQUEST_H
+
+#include "mca/ptl/ptl.h"
+#include "mca/pml/base/pml_base_recvreq.h"
+
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+
+/**
+ * Base type for recv requests
+ */
+struct mca_ptl_base_recv_request_t {
+    mca_pml_base_recv_request_t req_recv;
+    size_t req_bytes_received;        /**< number of bytes received from network */
+    size_t req_bytes_delivered;       /**< number of bytes delivered to user */
+};
+typedef struct mca_ptl_base_recv_request_t mca_ptl_base_recv_request_t;
+
+
+OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_ptl_base_recv_request_t);
+
+
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+#endif
+

src/mca/ptl/base/ptl_base_sendfrag.h

@@ -32,7 +32,7 @@ OMPI_DECLSPEC extern ompi_class_t mca_ptl_base_send_frag_t_class;
  */
 struct mca_ptl_base_send_frag_t {
     mca_ptl_base_frag_t frag_base;  /**< base fragment descriptor */
-    struct mca_pml_base_send_request_t *frag_request;  /**< pointer to send request */
+    struct mca_ptl_base_send_request_t *frag_request;  /**< pointer to send request */
 };
 typedef struct mca_ptl_base_send_frag_t mca_ptl_base_send_frag_t;
 

src/mca/ptl/base/ptl_base_sendreq.c

@@ -0,0 +1,10 @@
+#include "ptl_base_sendreq.h"
+
+
+OBJ_CLASS_INSTANCE(
+    mca_ptl_base_send_request_t,
+    mca_pml_base_send_request_t,
+    NULL,
+    NULL
+);
+    

src/mca/ptl/base/ptl_base_sendreq.h

@@ -0,0 +1,69 @@
+/* 
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+/**
+ * @file
+ */
+#ifndef MCA_PTL_BASE_SEND_REQUEST_H
+#define MCA_PTL_BASE_SEND_REQUEST_H
+
+#include "mca/ptl/ptl.h"
+#include "mca/pml/base/pml_base_sendreq.h"
+
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+
+/**
+ * Base type for send requests 
+ */
+struct mca_ptl_base_send_request_t {
+    mca_pml_base_send_request_t req_send;
+    size_t req_offset;                       /**< number of bytes that have been scheduled */
+    size_t req_bytes_sent;                   /**< number of bytes that have been sent */
+    ompi_ptr_t req_peer_match;               /**< matched receive at peer */
+    ompi_ptr_t req_peer_addr;                /**< peers remote buffer address */
+    uint64_t req_peer_size;                  /**< size of peers remote buffer */
+    bool req_cached;                         /**< has this request been obtained from the ptls cache */
+    volatile int32_t req_lock;               /**< lock used by the scheduler */
+    struct mca_ptl_base_module_t* req_ptl;   /**< ptl allocated for first fragment */
+    struct mca_ptl_base_peer_t* req_peer;    /**< peer associated w/ this ptl */
+};
+typedef struct mca_ptl_base_send_request_t mca_ptl_base_send_request_t;
+
+
+OMPI_DECLSPEC OBJ_CLASS_DECLARATION(mca_ptl_base_send_request_t);
+
+
+/**
+ * Atomically increase the request offset.
+ *
+ * @param  request (IN)  Send request.
+ * @param  offset (IN)   Increment.
+ */
+
+static inline void mca_ptl_base_send_request_offset(
+    mca_ptl_base_send_request_t* request,
+    size_t offset)
+{
+    OMPI_THREAD_ADD32((int32_t*)(&request->req_offset), offset);
+}
+
+
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+#endif
+

src/mca/ptl/mx/ptl_mx.c

@@ -89,7 +89,7 @@ static void *mca_ptl_mx_alloc(size_t *size)
  *
  */
 
-int mca_ptl_mx_request_init(struct mca_ptl_base_module_t* ptl, mca_pml_base_send_request_t* request)
+int mca_ptl_mx_request_init(struct mca_ptl_base_module_t* ptl, mca_ptl_base_send_request_t* request)
 {
     OBJ_CONSTRUCT(request+1, mca_ptl_mx_send_frag_t);
     return OMPI_SUCCESS;
@@ -109,7 +109,7 @@ int mca_ptl_mx_request_init(struct mca_ptl_base_module_t* ptl, mca_pml_base_send
  * descriptor by the PTL.
  */
 
-void mca_ptl_mx_request_fini(struct mca_ptl_base_module_t* ptl, mca_pml_base_send_request_t* request)
+void mca_ptl_mx_request_fini(struct mca_ptl_base_module_t* ptl, mca_ptl_base_send_request_t* request)
 {
     OBJ_DESTRUCT(request+1);
 }
@@ -142,7 +142,7 @@ void mca_ptl_mx_request_fini(struct mca_ptl_base_module_t* ptl, mca_pml_base_sen
 int mca_ptl_mx_send(
     struct mca_ptl_base_module_t* ptl,
     struct mca_ptl_base_peer_t* ptl_peer,
-    struct mca_pml_base_send_request_t* sendreq,
+    struct mca_ptl_base_send_request_t* sendreq,
     size_t offset,
     size_t size,
     int flags)
@@ -177,13 +177,13 @@ int mca_ptl_mx_send(
        int rc;
 
        convertor = &sendfrag->frag_send.frag_base.frag_convertor;
-       ompi_convertor_copy(&sendreq->req_convertor, convertor);
+       ompi_convertor_copy(&sendreq->req_send.req_convertor, convertor);
        ompi_convertor_init_for_send(
                     convertor,
                     0,
-                    sendreq->req_datatype,
+                    sendreq->req_send.req_datatype,
-                    sendreq->req_count,
+                    sendreq->req_send.req_count,
-                    sendreq->req_addr,
+                    sendreq->req_send.req_addr,
                     offset,
                     mca_ptl_mx_alloc );
                                                                                                                       
@@ -218,12 +218,12 @@ int mca_ptl_mx_send(
     /* first fragment - need to try and match at the receiver */
     if(offset == 0) {
         hdr->hdr_common.hdr_flags = flags;
-        hdr->hdr_match.hdr_contextid = sendreq->req_base.req_comm->c_contextid;
+        hdr->hdr_match.hdr_contextid = sendreq->req_send.req_base.req_comm->c_contextid;
-        hdr->hdr_match.hdr_src = sendreq->req_base.req_comm->c_my_rank;
+        hdr->hdr_match.hdr_src = sendreq->req_send.req_base.req_comm->c_my_rank;
-        hdr->hdr_match.hdr_dst = sendreq->req_base.req_peer;
+        hdr->hdr_match.hdr_dst = sendreq->req_send.req_base.req_peer;
-        hdr->hdr_match.hdr_tag = sendreq->req_base.req_tag;
+        hdr->hdr_match.hdr_tag = sendreq->req_send.req_base.req_tag;
-        hdr->hdr_match.hdr_msg_length = sendreq->req_bytes_packed;
+        hdr->hdr_match.hdr_msg_length = sendreq->req_send.req_bytes_packed;
-        hdr->hdr_match.hdr_msg_seq = sendreq->req_base.req_sequence;
+        hdr->hdr_match.hdr_msg_seq = sendreq->req_send.req_base.req_sequence;
 
         /* for the first 32K - send header for matching + data */
         segments = sendfrag->frag_segments;
@@ -292,7 +292,7 @@ int mca_ptl_mx_send(
     /* must update the offset after actual fragment size is determined 
      * before attempting to send the fragment
      */
-    mca_pml_base_send_request_offset(sendreq,
+    mca_ptl_base_send_request_offset(sendreq,
         sendfrag->frag_send.frag_base.frag_size);
 
     /* start the fragment */
@@ -329,7 +329,7 @@ int mca_ptl_mx_send(
 int mca_ptl_mx_send_continue(
     struct mca_ptl_base_module_t* ptl,
     struct mca_ptl_base_peer_t* ptl_peer,
-    struct mca_pml_base_send_request_t* sendreq,
+    struct mca_ptl_base_send_request_t* sendreq,
     size_t offset,
     size_t size,
     int flags)
@@ -353,13 +353,13 @@ int mca_ptl_mx_send_continue(
 
     /* initialize convertor */
     convertor = &sendfrag->frag_send.frag_base.frag_convertor;
-    ompi_convertor_copy(&sendreq->req_convertor, convertor);
+    ompi_convertor_copy(&sendreq->req_send.req_convertor, convertor);
     ompi_convertor_init_for_send(
         convertor,
         0,
-        sendreq->req_datatype,
+        sendreq->req_send.req_datatype,
-        sendreq->req_count,
+        sendreq->req_send.req_count,
-        sendreq->req_addr,
+        sendreq->req_send.req_addr,
         offset,
         mca_ptl_mx_alloc );
                                                                                                                       
@@ -400,7 +400,7 @@ int mca_ptl_mx_send_continue(
     /* must update the offset after actual fragment size is determined 
      * before attempting to send the fragment
      */
-    mca_pml_base_send_request_offset(sendreq, size);
+    mca_ptl_base_send_request_offset(sendreq, size);
 
     /* start the fragment */
     match.sval.uval = sendreq->req_peer_match.ival;
@@ -446,7 +446,7 @@ void mca_ptl_mx_matched(
     mca_ptl_base_recv_frag_t* frag)
 {
     mca_ptl_base_header_t* hdr = &frag->frag_base.frag_header;
-    mca_pml_base_recv_request_t* request = frag->frag_request;
+    mca_ptl_base_recv_request_t* request = frag->frag_request;
     mca_ptl_mx_module_t* mx_ptl = (mca_ptl_mx_module_t*)ptl;
     mca_ptl_mx_recv_frag_t* mx_frag = (mca_ptl_mx_recv_frag_t*)frag;
     unsigned int bytes_delivered = mx_frag->frag_size;
@@ -493,8 +493,8 @@ void mca_ptl_mx_matched(
         struct iovec iov;
         unsigned int iov_count = 1;
         int free_after = 0;
-        ompi_proc_t *proc = ompi_comm_peer_lookup(request->req_base.req_comm,
+        ompi_proc_t *proc = ompi_comm_peer_lookup(request->req_recv.req_base.req_comm,
-            request->req_base.req_ompi.req_status.MPI_SOURCE);
+            request->req_recv.req_base.req_ompi.req_status.MPI_SOURCE);
         ompi_convertor_t* convertor = &frag->frag_base.frag_convertor;
 
         /* initialize receive convertor */
@@ -502,9 +502,9 @@ void mca_ptl_mx_matched(
         ompi_convertor_init_for_recv(
             convertor,                      /* convertor */
             0,                              /* flags */
-            request->req_base.req_datatype, /* datatype */
+            request->req_recv.req_base.req_datatype, /* datatype */
-            request->req_base.req_count,    /* count elements */
+            request->req_recv.req_base.req_count,    /* count elements */
-            request->req_base.req_addr,     /* users buffer */
+            request->req_recv.req_base.req_addr,     /* users buffer */
             0,                              /* offset in bytes into packed buffer */
             NULL );                         /* not allocating memory */
         /*ompi_convertor_get_packed_size(convertor, &request->req_bytes_packed); */

src/mca/ptl/mx/ptl_mx.h

@@ -249,7 +249,7 @@ extern int mca_ptl_mx_del_procs(
 
 extern int mca_ptl_mx_request_init(
     struct mca_ptl_base_module_t* ptl,
-    struct mca_pml_base_send_request_t*
+    struct mca_ptl_base_send_request_t*
 );
 
 
@@ -268,7 +268,7 @@ extern int mca_ptl_mx_request_init(
                                                                                               
 extern void mca_ptl_mx_request_fini(
     struct mca_ptl_base_module_t* ptl,
-    struct mca_pml_base_send_request_t*
+    struct mca_ptl_base_send_request_t*
 );
 
 
@@ -325,7 +325,7 @@ extern void mca_ptl_mx_matched(
 extern int mca_ptl_mx_send(
     struct mca_ptl_base_module_t* ptl,
     struct mca_ptl_base_peer_t* ptl_peer,
-    struct mca_pml_base_send_request_t*,
+    struct mca_ptl_base_send_request_t*,
     size_t offset,
     size_t size,
     int flags
@@ -348,7 +348,7 @@ extern int mca_ptl_mx_send(
 extern int mca_ptl_mx_send_continue(
     struct mca_ptl_base_module_t* ptl,
     struct mca_ptl_base_peer_t* ptl_peer,
-    struct mca_pml_base_send_request_t*,
+    struct mca_ptl_base_send_request_t*,
     size_t offset,
     size_t size,
     int flags

src/mca/ptl/mx/ptl_mx_module.c

@@ -178,7 +178,7 @@ static void* mca_ptl_mx_thread(ompi_object_t *arg)
 static void mca_ptl_mx_match(void* context, uint64_t match_value, int size)
 {
     mca_ptl_mx_module_t* ptl = (mca_ptl_mx_module_t*)context;
-	mca_pml_base_recv_request_t* request;
+	mca_ptl_base_recv_request_t* request;
     mca_ptl_mx_recv_frag_t *frag;
     mx_return_t mx_return;
     ompi_ptr_t match;
@@ -193,10 +193,10 @@ static void mca_ptl_mx_match(void* context, uint64_t match_value, int size)
 
     /* otherwise extract request pointer and offset */
     match.lval = match_value;
-    request = (mca_pml_base_recv_request_t*)match.sval.uval;
+    request = (mca_ptl_base_recv_request_t*)match.sval.uval;
     offset = match.sval.lval;
     proc = ompi_comm_peer_lookup(request->req_base.req_comm,
-    	request->req_base.req_ompi.req_status.MPI_SOURCE);
+    	request->req_recv.req_base.req_ompi.req_status.MPI_SOURCE);
 
     /* allocate a fragment for receive */
     MCA_PTL_MX_RECV_FRAG_ALLOC(frag, rc); 
@@ -220,15 +220,15 @@ static void mca_ptl_mx_match(void* context, uint64_t match_value, int size)
     ompi_convertor_init_for_recv(
     	convertor,
     	0,                              /* flags */
-    	request->req_base.req_datatype, /* datatype */
+    	request->req_recv.req_base.req_datatype, /* datatype */
-    	request->req_base.req_count,    /* count elements */
+    	request->req_recv.req_base.req_count,    /* count elements */
-    	request->req_base.req_addr,     /* users buffer */
+    	request->req_recv.req_base.req_addr,     /* users buffer */
     	offset,                         /* offset in bytes into packed buffer */
     	mca_ptl_mx_mem_alloc );         /* not allocating memory */
                                                       
     /* non-contiguous - allocate buffer for receive */
     if( 1 == ompi_convertor_need_buffers( convertor )  ||
-    	request->req_bytes_packed < offset + frag->frag_size ) {
+    	request->req_recv.req_bytes_packed < offset + frag->frag_size ) {
 
     	/* TODO - use a fixed fragment size for non-contigous and convert 
     	 * this to a free-list of buffers.
@@ -242,14 +242,14 @@ static void mca_ptl_mx_match(void* context, uint64_t match_value, int size)
     	}
 
     	/* check for sending more than receiving */
-    	if( offset > request->req_bytes_packed ) {
+    	if( offset > request->req_recv.req_bytes_packed ) {
             frag->frag_recv.frag_base.frag_size = 0;
-    	} else if (offset + frag->frag_size > request->req_bytes_packed ) {
+    	} else if (offset + frag->frag_size > request->req_recv.req_bytes_packed ) {
-            frag->frag_recv.frag_base.frag_size = request->req_bytes_packed - offset;
+            frag->frag_recv.frag_base.frag_size = request->req_recv.req_bytes_packed - offset;
     	}
     /* calculate offset into users buffer */
     } else {
-    	frag->frag_recv.frag_base.frag_addr = ((unsigned char*)request->req_base.req_addr) + offset;
+    	frag->frag_recv.frag_base.frag_addr = ((unsigned char*)request->req_recv.req_base.req_addr) + offset;
     	frag->frag_recv.frag_is_buffered = false;
     }
 

src/mca/ptl/mx/ptl_mx_module.h

@@ -110,7 +110,7 @@ do {
                 case MCA_PTL_HDR_TYPE_ACK:                                          \
                 {                                                                   \
                     mca_ptl_mx_send_frag_t* sendfrag;                               \
-                    mca_pml_base_send_request_t* sendreq;                           \
+                    mca_ptl_base_send_request_t* sendreq;                           \
                     sendfrag = (mca_ptl_mx_send_frag_t*)                            \
                         hdr->hdr_ack.hdr_src_ptr.pval;                              \
                     sendreq = sendfrag->frag_send.frag_request;                     \

src/mca/ptl/mx/ptl_mx_sendfrag.h

@@ -21,7 +21,7 @@
 
 #include "ompi_config.h"
 #include "include/sys/atomic.h"
-#include "mca/pml/base/pml_base_sendreq.h"
+#include "mca/ptl/base/ptl_base_sendreq.h"
 #include "mca/ptl/base/ptl_base_sendfrag.h"
 #include "ptl_mx.h"
 #include "ptl_mx_peer.h"
@@ -88,7 +88,7 @@ OBJ_CLASS_DECLARATION(mca_ptl_mx_send_frag_t);
                                                                                                                   
 #define MCA_PTL_MX_SEND_FRAG_PROGRESS(frag) \
 do { \
-    mca_pml_base_send_request_t* request = frag->frag_send.frag_request; \
+    mca_ptl_base_send_request_t* request = frag->frag_send.frag_request; \
     bool frag_ack; \
     \
     /* if this is an ack - simply return to pool */ \

src/mca/ptl/ptl.h

@@ -105,7 +105,7 @@
  * PTLs.
  *
  * The PML is responsible for managing the state (allocation, initialization, 
- * and release) of send request descriptors (mca_pml_base_send_request_t). 
+ * and release) of send request descriptors (mca_ptl_base_send_request_t). 
  * However, to minimize the latency associated with allocating resources to 
  * a request, the PML provides the capability to cache send requests 
  * descriptors on a per-PTL basis. Each PTL exports two variables 
@@ -117,7 +117,7 @@
  * is grown by the PML up to the specified limit. The PTL variable, 
  * ptl_cache_bytes, can be used to specify that additional memory should be 
  * allocated by the PML in one contigous block along with the base send request 
- * (mca_pml_base_send_request_t) for use by the PTL. The PTLs ptl_request_init() 
+ * (mca_ptl_base_send_request_t) for use by the PTL. The PTLs ptl_request_init() 
  * method is then called to initialize this additional memory and associating 
  * any PTL specific resources with the request.  If a request is removed from 
  * the cache, the ptl_request_fini() method will be called to allow the PTL 
@@ -129,7 +129,7 @@
  * number of bytes. However, this may not be possible due to resource
  * contraints or datatype alignment/offset. The PTL is responsible for
  * updating the number of bytes actually fragmented and queued for delivery 
- * on the send request (mca_pml_base_send_request.req_offset) to reflect 
+ * on the send request (mca_ptl_base_send_request.req_offset) to reflect 
  * the current offset into the send buffer.
  *
  * If the request is larger than ptl_first_frag_size, the remainder of
@@ -258,8 +258,8 @@
 struct mca_ptl_base_module_t;
 struct mca_ptl_base_peer_t;
 struct mca_ptl_base_fragment_t;
-struct mca_pml_base_recv_request_t;
+struct mca_ptl_base_recv_request_t;
-struct mca_pml_base_send_request_t;
+struct mca_ptl_base_send_request_t;
 struct mca_ptl_base_recv_frag_t;
 struct mca_ptl_base_send_frag_t;
 struct mca_ptl_base_match_header_t;
@@ -458,7 +458,7 @@ typedef int (*mca_ptl_base_module_del_procs_fn_t)(
  */
 typedef int (*mca_ptl_base_module_request_init_fn_t)(
     struct mca_ptl_base_module_t* ptl, 
-    struct mca_pml_base_send_request_t* request
+    struct mca_ptl_base_send_request_t* request
 );
 
 
@@ -477,7 +477,7 @@ typedef int (*mca_ptl_base_module_request_init_fn_t)(
 
 typedef void (*mca_ptl_base_module_request_fini_fn_t)(
     struct mca_ptl_base_module_t* ptl, 
-    struct mca_pml_base_send_request_t* request
+    struct mca_ptl_base_send_request_t* request
 );
 
 /**
@@ -506,7 +506,7 @@ typedef void (*mca_ptl_base_module_request_fini_fn_t)(
 typedef int (*mca_ptl_base_module_send_fn_t)(
     struct mca_ptl_base_module_t* ptl,
     struct mca_ptl_base_peer_t* ptl_base_peer,
-    struct mca_pml_base_send_request_t* request,
+    struct mca_ptl_base_send_request_t* request,
     size_t offset,
     size_t size,
     int flags
@@ -541,7 +541,7 @@ typedef int (*mca_ptl_base_module_send_fn_t)(
 typedef int (*mca_ptl_base_module_put_fn_t)(
     struct mca_ptl_base_module_t* ptl,
     struct mca_ptl_base_peer_t* ptl_base_peer,
-    struct mca_pml_base_send_request_t* request,
+    struct mca_ptl_base_send_request_t* request,
     size_t offset,
     size_t size,
     int flags
@@ -566,7 +566,7 @@ typedef int (*mca_ptl_base_module_put_fn_t)(
 typedef int (*mca_ptl_base_module_get_fn_t)(
     struct mca_ptl_base_module_t* ptl, 
     struct mca_ptl_base_peer_t* ptl_base_peer, 
-    struct mca_pml_base_recv_request_t* request,
+    struct mca_ptl_base_recv_request_t* request,
     size_t offset,
     size_t size,
     int flags
@@ -637,7 +637,7 @@ typedef void (*mca_ptl_base_module_matched_fn_t)(
  */
 typedef void (*mca_ptl_base_module_recv_progress_fn_t)(
     struct mca_ptl_base_module_t* ptl,
-    struct mca_pml_base_recv_request_t* recv_request,
+    struct mca_ptl_base_recv_request_t* recv_request,
     size_t bytes_received,
     size_t bytes_delivered
 );
@@ -657,7 +657,7 @@ typedef void (*mca_ptl_base_module_recv_progress_fn_t)(
 
 typedef void (*mca_ptl_base_module_send_progress_fn_t)(
     struct mca_ptl_base_module_t* ptl,
-    struct mca_pml_base_send_request_t* send_request,
+    struct mca_ptl_base_send_request_t* send_request,
     size_t bytes_sent
 );
 

src/mca/ptl/self/ptl_self.c

@@ -101,13 +101,13 @@ int mca_ptl_self_finalize(struct mca_ptl_base_module_t* ptl)
     return OMPI_SUCCESS;
 }
 
-int mca_ptl_self_request_init(struct mca_ptl_base_module_t* ptl, mca_pml_base_send_request_t* request)
+int mca_ptl_self_request_init(struct mca_ptl_base_module_t* ptl, mca_ptl_base_send_request_t* request)
 {
     OBJ_CONSTRUCT(request+1, mca_ptl_base_recv_frag_t);
     return OMPI_SUCCESS;
 }
 
-void mca_ptl_self_request_fini(struct mca_ptl_base_module_t* ptl, mca_pml_base_send_request_t* request)
+void mca_ptl_self_request_fini(struct mca_ptl_base_module_t* ptl, mca_ptl_base_send_request_t* request)
 {
     OBJ_DESTRUCT(request+1);
 }
@@ -122,7 +122,7 @@ void mca_ptl_self_request_fini(struct mca_ptl_base_module_t* ptl, mca_pml_base_s
 int mca_ptl_self_send(
                       struct mca_ptl_base_module_t* ptl,
                       struct mca_ptl_base_peer_t* ptl_base_peer,
-                      struct mca_pml_base_send_request_t* request,
+                      struct mca_ptl_base_send_request_t* request,
                       size_t offset,
                       size_t size,
                       int flags )
@@ -133,16 +133,16 @@ int mca_ptl_self_send(
 
     hdr->hdr_common.hdr_type = MCA_PTL_HDR_TYPE_MATCH;
     hdr->hdr_common.hdr_flags = flags;
-    hdr->hdr_match.hdr_contextid = request->req_base.req_comm->c_contextid;
+    hdr->hdr_match.hdr_contextid = request->req_send.req_base.req_comm->c_contextid;
-    hdr->hdr_match.hdr_src = request->req_base.req_comm->c_my_rank;
+    hdr->hdr_match.hdr_src = request->req_send.req_base.req_comm->c_my_rank;
-    hdr->hdr_match.hdr_dst = request->req_base.req_peer;
+    hdr->hdr_match.hdr_dst = request->req_send.req_base.req_peer;
-    hdr->hdr_match.hdr_tag = request->req_base.req_tag;
+    hdr->hdr_match.hdr_tag = request->req_send.req_base.req_tag;
-    hdr->hdr_match.hdr_msg_length = request->req_bytes_packed;
+    hdr->hdr_match.hdr_msg_length = request->req_send.req_bytes_packed;
-    hdr->hdr_match.hdr_msg_seq = request->req_base.req_sequence;
+    hdr->hdr_match.hdr_msg_seq = request->req_send.req_base.req_sequence;
     hdr->hdr_rndv.hdr_src_ptr.lval = 0;
     hdr->hdr_rndv.hdr_src_ptr.pval = request;
     req->req_frag.frag_base.frag_peer = ptl_base_peer;
-    req->req_frag.frag_base.frag_size = request->req_bytes_packed;
+    req->req_frag.frag_base.frag_size = request->req_send.req_bytes_packed;
     req->req_frag.frag_base.frag_owner = &mca_ptl_self_module;
     req->req_frag.frag_request = NULL;
     req->req_frag.frag_is_buffered = 0;
@@ -169,10 +169,10 @@ void mca_ptl_self_matched( mca_ptl_base_module_t* ptl,
 {
     mca_ptl_self_send_request_t* sendreq = (mca_ptl_self_send_request_t*)
         frag->frag_base.frag_header.hdr_rndv.hdr_src_ptr.pval;
-    mca_pml_base_recv_request_t* recvreq = frag->frag_request;
+    mca_ptl_base_recv_request_t* recvreq = frag->frag_request;
 
-    if( (recvreq->req_base.req_count != 0) &&
+    if( (recvreq->req_recv.req_base.req_count != 0) &&
-        (sendreq->req_send.req_base.req_count != 0) ) {
+        (sendreq->req_ptl.req_send.req_base.req_count != 0) ) {
         /* Did you have the same datatype or not ? If yes we can use an optimized version
          * for the copy function, if not we have to use a temporary buffer to pack/unpack
          * 
@@ -180,12 +180,12 @@ void mca_ptl_self_matched( mca_ptl_base_module_t* ptl,
          * a contigous buffer and the convertor on the send request initialized to point
          * into this buffer.
          */
-        if( sendreq->req_send.req_datatype == recvreq->req_base.req_datatype ) {
+        if( sendreq->req_ptl.req_send.req_datatype == recvreq->req_recv.req_base.req_datatype ) {
-            ompi_ddt_copy_content_same_ddt( recvreq->req_base.req_datatype, 
+            ompi_ddt_copy_content_same_ddt( recvreq->req_recv.req_base.req_datatype, 
-                                            recvreq->req_base.req_count > sendreq->req_send.req_count ?
+                                            recvreq->req_recv.req_base.req_count > sendreq->req_ptl.req_send.req_count ?
-                                            sendreq->req_send.req_count : recvreq->req_base.req_count,
+                                            sendreq->req_ptl.req_send.req_count : recvreq->req_recv.req_base.req_count,
-                                            (char *)recvreq->req_base.req_addr, 
+                                            (char *)recvreq->req_recv.req_base.req_addr, 
-                                            (const char *)sendreq->req_send.req_addr );
+                                            (const char *)sendreq->req_ptl.req_send.req_addr );
         } else {
             ompi_convertor_t *pSendConvertor, *pRecvConvertor;
             struct iovec iov[1];
@@ -199,10 +199,10 @@ void mca_ptl_self_matched( mca_ptl_base_module_t* ptl,
             
             ompi_convertor_init_for_recv( &frag->frag_base.frag_convertor, 
                                           0, 
-                                          recvreq->req_base.req_datatype, 
+                                          recvreq->req_recv.req_base.req_datatype, 
-                                          recvreq->req_base.req_count, 
+                                          recvreq->req_recv.req_base.req_count, 
-                                          recvreq->req_base.req_addr, 0, NULL );
+                                          recvreq->req_recv.req_base.req_addr, 0, NULL );
-            pSendConvertor = &(sendreq->req_send.req_convertor);
+            pSendConvertor = &(sendreq->req_ptl.req_send.req_convertor);
             pRecvConvertor = &(frag->frag_base.frag_convertor);
             completed = 0;
             freeAfter = 0;
@@ -221,8 +221,8 @@ void mca_ptl_self_matched( mca_ptl_base_module_t* ptl,
             free( buf );
         }
     }
-    ptl->ptl_send_progress( ptl, &sendreq->req_send,
+    ptl->ptl_send_progress( ptl, &sendreq->req_ptl,
-                            sendreq->req_send.req_bytes_packed );
+                            sendreq->req_ptl.req_send.req_bytes_packed );
     ptl->ptl_recv_progress( ptl, 
                             recvreq, 
                             frag->frag_base.frag_header.hdr_match.hdr_msg_length,

src/mca/ptl/self/ptl_self.h

@@ -24,8 +24,8 @@
 #include "event/event.h"
 #include "mca/pml/pml.h"
 #include "mca/ptl/ptl.h"
-#include "mca/pml/base/pml_base_sendreq.h"
+#include "mca/ptl/base/ptl_base_sendreq.h"
-#include "mca/pml/base/pml_base_recvreq.h"
+#include "mca/ptl/base/ptl_base_recvreq.h"
 #include "mca/ptl/base/ptl_base_recvfrag.h"
 #if defined(c_plusplus) || defined(__cplusplus)
 extern "C" {
@@ -54,7 +54,7 @@ typedef struct mca_ptl_self_component_t mca_ptl_self_component_t;
  * base send request, and the base receive fragment which will be used to do the match.
  */
 struct mca_ptl_self_send_request_t {
-   mca_pml_base_send_request_t req_send;
+   mca_ptl_base_send_request_t req_ptl;
    mca_ptl_base_recv_frag_t req_frag; /* first fragment */
 };
 typedef struct mca_ptl_self_send_request_t mca_ptl_self_send_request_t;
@@ -91,9 +91,9 @@ extern mca_ptl_base_module_t** mca_ptl_self_component_init(
 int  mca_ptl_self_add_proc(struct mca_ptl_base_module_t* ptl, size_t nprocs, struct ompi_proc_t **ompi_proc, struct mca_ptl_base_peer_t** peer_ret, struct ompi_bitmap_t* reachable);
 int  mca_ptl_self_del_proc(struct mca_ptl_base_module_t* ptl, size_t nprocs, struct ompi_proc_t **proc, struct mca_ptl_base_peer_t** ptl_peer);
 int  mca_ptl_self_finalize(struct mca_ptl_base_module_t* ptl);
-int mca_ptl_self_request_init(struct mca_ptl_base_module_t* ptl, struct mca_pml_base_send_request_t* request);
+int mca_ptl_self_request_init(struct mca_ptl_base_module_t* ptl, struct mca_ptl_base_send_request_t* request);
-void mca_ptl_self_request_fini(struct mca_ptl_base_module_t* ptl, struct mca_pml_base_send_request_t* request);
+void mca_ptl_self_request_fini(struct mca_ptl_base_module_t* ptl, struct mca_ptl_base_send_request_t* request);
-int  mca_ptl_self_send( struct mca_ptl_base_module_t* ptl, struct mca_ptl_base_peer_t* ptl_base_peer, struct mca_pml_base_send_request_t* request,
+int  mca_ptl_self_send( struct mca_ptl_base_module_t* ptl, struct mca_ptl_base_peer_t* ptl_base_peer, struct mca_ptl_base_send_request_t* request,
                       size_t offset, size_t size, int flags );
 void mca_ptl_self_matched( mca_ptl_base_module_t* ptl, mca_ptl_base_recv_frag_t* frag );
 

src/mca/ptl/sm/src/ptl_sm.c

@@ -27,8 +27,8 @@
 #include "util/if.h"
 #include "mca/pml/pml.h"
 #include "mca/ptl/ptl.h"
-#include "mca/pml/base/pml_base_sendreq.h"
+#include "mca/ptl/base/ptl_base_sendreq.h"
-#include "mca/pml/base/pml_base_recvreq.h"
+#include "mca/ptl/base/ptl_base_recvreq.h"
 #include "mca/ptl/base/ptl_base_header.h"
 #include "mca/ptl/base/ptl_base_sendfrag.h"
 #include "mca/ptl/base/ptl_base_recvfrag.h"
@@ -55,7 +55,7 @@ mca_ptl_sm_t mca_ptl_sm[2] = {
         &mca_ptl_sm_component.super,
         20, /* number of elements in the send descriptor cache */
         sizeof(mca_ptl_sm_send_request_t) -
-            sizeof(mca_pml_base_send_request_t),  /* size of shared memory send
+            sizeof(mca_ptl_base_send_request_t),  /* size of shared memory send
                                                      descriptor */
         1, /* ptl_first_frag_size */
         0, /* ptl_min_frag_size */
@@ -80,7 +80,7 @@ mca_ptl_sm_t mca_ptl_sm[2] = {
             &mca_ptl_sm_component.super,
             20, /* number of elements in the send descriptor cache */
             sizeof(mca_ptl_sm_send_request_t) -
-                sizeof(mca_pml_base_send_request_t),  /* size of shared memory 
+                sizeof(mca_ptl_base_send_request_t),  /* size of shared memory 
                                                          send descriptor */
             1, /* ptl_first_frag_size */
             0, /* ptl_min_frag_size */
@@ -711,7 +711,7 @@ int mca_ptl_sm_finalize(struct mca_ptl_base_module_t* ptl)
 }
 
 
-int mca_ptl_sm_request_alloc(struct mca_ptl_base_module_t* ptl, struct mca_pml_base_send_request_t* request)
+int mca_ptl_sm_request_alloc(struct mca_ptl_base_module_t* ptl, struct mca_ptl_base_send_request_t* request)
 {
     mca_ptl_sm_send_request_t *sm_request;
     ompi_list_item_t* item;
@@ -731,7 +731,7 @@ int mca_ptl_sm_request_alloc(struct mca_ptl_base_module_t* ptl, struct mca_pml_b
 }
 
 
-void mca_ptl_sm_request_return(struct mca_ptl_base_module_t* ptl, struct mca_pml_base_send_request_t* request)
+void mca_ptl_sm_request_return(struct mca_ptl_base_module_t* ptl, struct mca_ptl_base_send_request_t* request)
 {
     mca_ptl_sm_send_request_t *sm_request;
     ompi_list_item_t* item;
@@ -763,7 +763,7 @@ void mca_ptl_sm_request_return(struct mca_ptl_base_module_t* ptl, struct mca_pml
 int mca_ptl_sm_send(
     struct mca_ptl_base_module_t* ptl,
     struct mca_ptl_base_peer_t* ptl_peer,
-    struct mca_pml_base_send_request_t* sendreq,
+    struct mca_ptl_base_send_request_t* sendreq,
     size_t offset,
     size_t size,
     int flags)
@@ -794,11 +794,11 @@ int mca_ptl_sm_send(
         int free_after=0;
         struct iovec address;
 
-        convertor = &sendreq->req_convertor;
+        convertor = &sendreq->req_send.req_convertor;
         ompi_convertor_init_for_send( convertor, 0, 
-                sendreq->req_datatype,
+                sendreq->req_send.req_datatype,
-                sendreq->req_count, 
+                sendreq->req_send.req_count, 
-                sendreq->req_addr,
+                sendreq->req_send.req_addr,
                 offset, NULL);
 
         sm_data_ptr=sm_request->req_frag->buff;
@@ -822,11 +822,11 @@ int mca_ptl_sm_send(
     hdr = &(send_frag->super.frag_base.frag_header);
     hdr->hdr_common.hdr_type = MCA_PTL_HDR_TYPE_MATCH;
     hdr->hdr_common.hdr_flags = flags;
-    hdr->hdr_match.hdr_contextid = sendreq->req_base.req_comm->c_contextid;
+    hdr->hdr_match.hdr_contextid = sendreq->req_send.req_base.req_comm->c_contextid;
-    hdr->hdr_match.hdr_src = sendreq->req_base.req_comm->c_my_rank;
+    hdr->hdr_match.hdr_src = sendreq->req_send.req_base.req_comm->c_my_rank;
-    hdr->hdr_match.hdr_dst = sendreq->req_base.req_peer;
+    hdr->hdr_match.hdr_dst = sendreq->req_send.req_base.req_peer;
-    hdr->hdr_match.hdr_tag = sendreq->req_base.req_tag;
+    hdr->hdr_match.hdr_tag = sendreq->req_send.req_base.req_tag;
-    hdr->hdr_match.hdr_msg_length = sendreq->req_bytes_packed;
+    hdr->hdr_match.hdr_msg_length = sendreq->req_send.req_bytes_packed;
 
     /* update the offset within the payload */
     sendreq->req_offset += size;
@@ -879,8 +879,8 @@ int mca_ptl_sm_send(
         ompi_atomic_unlock(&send_fifo->head_lock);
 
     /* if this is the entire message - signal request is complete */
-    if(sendreq->req_bytes_packed == size) {
+    if(sendreq->req_send.req_bytes_packed == size) {
-        ompi_request_complete( &(sendreq->req_base.req_ompi) );
+        ompi_request_complete( &(sendreq->req_send.req_base.req_ompi) );
     }
 
     /* return */
@@ -897,7 +897,7 @@ int mca_ptl_sm_send(
 int mca_ptl_sm_send_continue(
     struct mca_ptl_base_module_t* ptl,
     struct mca_ptl_base_peer_t* ptl_peer,
-    struct mca_pml_base_send_request_t* sendreq,
+    struct mca_ptl_base_send_request_t* sendreq,
     size_t offset,
     size_t size,
     int flags)
@@ -928,11 +928,11 @@ int mca_ptl_sm_send_continue(
     }
 
     /* pack data in payload buffer */
-    convertor = &sendreq->req_convertor;
+    convertor = &sendreq->req_send.req_convertor;
     ompi_convertor_init_for_send( convertor, 0, 
-            sendreq->req_datatype,
+            sendreq->req_send.req_datatype,
-            sendreq->req_count, 
+            sendreq->req_send.req_count, 
-            sendreq->req_addr,
+            sendreq->req_send.req_addr,
             offset, NULL);
     sm_data_ptr=send_frag->buff;
 

src/mca/ptl/sm/src/ptl_sm.h

@@ -274,7 +274,7 @@ extern int mca_ptl_sm_del_procs(
  */
 extern int mca_ptl_sm_request_alloc(
     struct mca_ptl_base_module_t* ptl,
-    struct mca_pml_base_send_request_t*
+    struct mca_ptl_base_send_request_t*
 );
 
 /**
@@ -286,7 +286,7 @@ extern int mca_ptl_sm_request_alloc(
  */
 extern void mca_ptl_sm_request_return(
     struct mca_ptl_base_module_t* ptl,
-    struct mca_pml_base_send_request_t*
+    struct mca_ptl_base_send_request_t*
 );
 
 /**
@@ -330,7 +330,7 @@ extern int mca_ptl_sm_del_procs(
  */
 extern int mca_ptl_sm_request_alloc(
     struct mca_ptl_base_module_t* ptl,
-    struct mca_pml_base_send_request_t*
+    struct mca_ptl_base_send_request_t*
 );
 
 /**
@@ -342,7 +342,7 @@ extern int mca_ptl_sm_request_alloc(
  */
 extern void mca_ptl_sm_request_return(
     struct mca_ptl_base_module_t* ptl,
-    struct mca_pml_base_send_request_t*
+    struct mca_ptl_base_send_request_t*
 );
 
 /**
@@ -388,7 +388,7 @@ extern void mca_ptl_sm_matched_common_base_addr(
 extern int mca_ptl_sm_send(
     struct mca_ptl_base_module_t* ptl,
     struct mca_ptl_base_peer_t* ptl_peer,
-    struct mca_pml_base_send_request_t*,
+    struct mca_ptl_base_send_request_t*,
     size_t offset,
     size_t size,
     int flags
@@ -407,7 +407,7 @@ extern int mca_ptl_sm_send(
 extern int mca_ptl_sm_send_continue(
     struct mca_ptl_base_module_t* ptl,
     struct mca_ptl_base_peer_t* ptl_peer,
-    struct mca_pml_base_send_request_t*,
+    struct mca_ptl_base_send_request_t*,
     size_t offset,
     size_t size,
     int flags

src/mca/ptl/sm/src/ptl_sm_component.c

@@ -293,7 +293,7 @@ mca_ptl_base_module_t** mca_ptl_sm_component_init(
     for( i=0 ; i < 2 ; i++ ) {
         mca_ptl_sm[i].super.ptl_cache_size=mca_ptl_sm_component.sm_first_frag_free_list_max;
         mca_ptl_sm[i].super.ptl_cache_bytes=sizeof(mca_ptl_sm_send_request_t) -
-                sizeof(mca_pml_base_send_request_t);
+                sizeof(mca_ptl_base_send_request_t);
         mca_ptl_sm[i].super.ptl_first_frag_size=mca_ptl_sm_component.first_fragment_size;
         mca_ptl_sm[i].super.ptl_min_frag_size=mca_ptl_sm_component.max_fragment_size;
         mca_ptl_sm[i].super.ptl_max_frag_size=mca_ptl_sm_component.max_fragment_size;
@@ -363,7 +363,7 @@ int mca_ptl_sm_component_progress(mca_ptl_tstamp_t tstamp)
     ompi_fifo_t *send_fifo = NULL;
     bool frag_matched;
     mca_ptl_base_match_header_t *matching_header;
-    mca_pml_base_send_request_t *base_send_req;
+    mca_ptl_base_send_request_t *base_send_req;
     ompi_list_item_t *item;
     int return_status = 0;
 
@@ -434,10 +434,10 @@ int mca_ptl_sm_component_progress(mca_ptl_tstamp_t tstamp)
                     if( frag_matched ) {
                         /* deliver data, and ack */
                         mca_ptl_base_match_header_t *hdr = &header_ptr->super.frag_base.frag_header.hdr_match;
-                        mca_pml_base_recv_request_t *request = header_ptr->super.frag_request;
+                        mca_ptl_base_recv_request_t *request = header_ptr->super.frag_request;
-                        request->req_bytes_packed = hdr->hdr_msg_length;
+                        request->req_recv.req_bytes_packed = hdr->hdr_msg_length;
-                        request->req_base.req_ompi.req_status.MPI_SOURCE = hdr->hdr_src;
+                        request->req_recv.req_base.req_ompi.req_status.MPI_SOURCE = hdr->hdr_src;
-                        request->req_base.req_ompi.req_status.MPI_TAG = hdr->hdr_tag;
+                        request->req_recv.req_base.req_ompi.req_status.MPI_TAG = hdr->hdr_tag;
                         mca_ptl_sm_matched_same_base_addr(
                                 (mca_ptl_base_module_t *)&mca_ptl_sm,
                                 (mca_ptl_base_recv_frag_t *)header_ptr);
@@ -548,10 +548,10 @@ int mca_ptl_sm_component_progress(mca_ptl_tstamp_t tstamp)
                     if( frag_matched ) {
                         /* deliver data, and ack */
                         mca_ptl_base_match_header_t *hdr = &header_ptr->super.frag_base.frag_header.hdr_match;
-                        mca_pml_base_recv_request_t *request = header_ptr->super.frag_request;
+                        mca_ptl_base_recv_request_t *request = header_ptr->super.frag_request;
-                        request->req_bytes_packed = hdr->hdr_msg_length;
+                        request->req_recv.req_bytes_packed = hdr->hdr_msg_length;
-                        request->req_base.req_ompi.req_status.MPI_SOURCE = hdr->hdr_src;
+                        request->req_recv.req_base.req_ompi.req_status.MPI_SOURCE = hdr->hdr_src;
-                        request->req_base.req_ompi.req_status.MPI_TAG = hdr->hdr_tag;
+                        request->req_recv.req_base.req_ompi.req_status.MPI_TAG = hdr->hdr_tag;
                         mca_ptl_sm_matched((mca_ptl_base_module_t *)&mca_ptl_sm,
                                 (mca_ptl_base_recv_frag_t *)header_ptr);
                     }

src/mca/ptl/sm/src/ptl_sm_frag.h

@@ -47,7 +47,7 @@ struct mca_ptl_sm_frag_t {
     int queue_index;      /**< local process index, cached for fast
                             acking */
     struct mca_ptl_base_module_t* send_ptl;   /**< PTL that is selected for first fragment */
-    struct mca_pml_base_send_request_t* send_req;
+    struct mca_ptl_base_send_request_t* send_req;
     size_t send_offset;
     void *buff;           /**< pointer to buffer */
 };

src/mca/ptl/sm/src/ptl_sm_send.c

@@ -67,7 +67,7 @@ void mca_ptl_sm_matched(
     mca_ptl_base_module_t* ptl,
     mca_ptl_base_recv_frag_t* frag)
 {
-    mca_pml_base_recv_request_t* recv_desc;
+    mca_ptl_base_recv_request_t* recv_desc;
     mca_ptl_sm_frag_t *sm_frag_desc;
     struct iovec iov; 
     ompi_convertor_t frag_convertor;
@@ -92,7 +92,7 @@ void mca_ptl_sm_matched(
          * Initialize convertor and use it to unpack data  
          */ 
         OBJ_CONSTRUCT(&frag_convertor, ompi_convertor_t);
-        proc = ompi_comm_peer_lookup(recv_desc->req_base.req_comm,
+        proc = ompi_comm_peer_lookup(recv_desc->req_recv.req_base.req_comm,
                     frag->frag_base.frag_header.hdr_match.hdr_src);
         /* write over converter set on the send side */
         ompi_convertor_copy(proc->proc_convertor,
@@ -100,9 +100,9 @@ void mca_ptl_sm_matched(
         ompi_convertor_init_for_recv( 
                 &frag_convertor,                   /* convertor */ 
                 0,                                 /* flags */ 
-                recv_desc->req_base.req_datatype,  /* datatype */ 
+                recv_desc->req_recv.req_base.req_datatype,  /* datatype */ 
-                recv_desc->req_base.req_count,     /* count elements */ 
+                recv_desc->req_recv.req_base.req_count,     /* count elements */ 
-                recv_desc->req_base.req_addr,      /* users buffer */ 
+                recv_desc->req_recv.req_base.req_addr,      /* users buffer */ 
                 sm_frag_desc->send_offset,         /* offset in bytes into packed buffer */ 
                 NULL );                            /* dont allocate memory */
                 

src/mca/ptl/sm/src/ptl_sm_sendfrag.h

@@ -21,7 +21,7 @@
 
 #include <sys/types.h>
 #include "ompi_config.h"
-#include "mca/pml/base/pml_base_sendreq.h"
+#include "mca/ptl/base/ptl_base_sendreq.h"
 #include "mca/ptl/base/ptl_base_sendfrag.h"
 #include "ptl_sm.h"
 
@@ -56,7 +56,7 @@ typedef struct mca_ptl_sm_send_frag_t mca_ptl_sm_send_frag_t;
 int mca_ptl_sm_send_frag_init(
     mca_ptl_sm_send_frag_t*, 
     struct mca_ptl_base_peer_t*, 
-    struct mca_pml_base_send_request_t*, 
+    struct mca_ptl_base_send_request_t*, 
     size_t offset,
     size_t* size,
     int flags);

src/mca/ptl/sm/src/ptl_sm_sendreq.c

@@ -30,7 +30,7 @@ static void mca_ptl_sm_send_request_destruct(mca_ptl_sm_send_request_t*);
 
 OBJ_CLASS_INSTANCE(
     mca_ptl_sm_send_request_t,
-    mca_pml_base_send_request_t,
+    mca_ptl_base_send_request_t,
     mca_ptl_sm_send_request_construct,
     mca_ptl_sm_send_request_destruct
 );
@@ -54,7 +54,7 @@ void mca_ptl_sm_send_request_destruct(mca_ptl_sm_send_request_t* request)
  * to allocate fragment descriptor and payload memory
  */
 int mca_ptl_sm_send_request_init(struct mca_ptl_base_module_t* ptl,
-        struct mca_pml_base_send_request_t* request)
+        struct mca_ptl_base_send_request_t* request)
 {
 
     mca_ptl_sm_send_request_t *sm_request;

src/mca/ptl/sm/src/ptl_sm_sendreq.h

@@ -22,7 +22,7 @@
 
 #include <sys/types.h>
 #include "ompi_config.h"
-#include "mca/pml/base/pml_base_sendreq.h"
+#include "mca/ptl/base/ptl_base_sendreq.h"
 #include "ptl_sm_frag.h"
 
 #if defined(c_plusplus) || defined(__cplusplus)
@@ -38,7 +38,7 @@ OBJ_CLASS_DECLARATION(mca_ptl_sm_send_request_t);
 struct mca_ptl_sm_send_request_t {
 
     /* base send descriptor */
-    mca_pml_base_send_request_t super;
+    mca_ptl_base_send_request_t super;
   
     /* pointer to first fragment descriptor */
     mca_ptl_sm_frag_t *req_frag;
@@ -50,7 +50,7 @@ typedef struct mca_ptl_sm_send_request_t mca_ptl_sm_send_request_t;
  * memory send request is initialized.
  */
 int mca_ptl_sm_send_request_init(struct mca_ptl_base_module_t* ptl,
-        struct mca_pml_base_send_request_t* request);
+        struct mca_ptl_base_send_request_t* request);
 
 #if defined(c_plusplus) || defined(__cplusplus)
 }

src/mca/ptl/tcp/src/ptl_tcp.c

@@ -22,9 +22,9 @@
 #include "mca/pml/pml.h"
 #include "mca/ptl/ptl.h"
 #include "mca/ptl/base/ptl_base_header.h"
-#include "mca/pml/base/pml_base_sendreq.h"
+#include "mca/ptl/base/ptl_base_sendreq.h"
 #include "mca/ptl/base/ptl_base_sendfrag.h"
-#include "mca/pml/base/pml_base_recvreq.h"
+#include "mca/ptl/base/ptl_base_recvreq.h"
 #include "mca/ptl/base/ptl_base_recvfrag.h"
 #include "mca/base/mca_base_module_exchange.h"
 #include "ptl_tcp.h"
@@ -39,7 +39,7 @@ mca_ptl_tcp_module_t mca_ptl_tcp_module = {
     {
     &mca_ptl_tcp_component.super,
     16, /* max size of request cache */
-    sizeof(mca_ptl_tcp_send_request_t) - sizeof(mca_pml_base_send_request_t), /* bytes required by ptl for a request */
+    sizeof(mca_ptl_tcp_send_request_t) - sizeof(mca_ptl_base_send_request_t), /* bytes required by ptl for a request */
     0, /* max size of first fragment */
     0, /* min fragment size */
     0, /* max fragment size */
@@ -179,7 +179,7 @@ int mca_ptl_tcp_finalize(struct mca_ptl_base_module_t* ptl)
  * along w/ the ptl to cache the first fragment control information.
  */
 
-int mca_ptl_tcp_request_init(struct mca_ptl_base_module_t* ptl, struct mca_pml_base_send_request_t* request)
+int mca_ptl_tcp_request_init(struct mca_ptl_base_module_t* ptl, struct mca_ptl_base_send_request_t* request)
 {
     OBJ_CONSTRUCT(request+1, mca_ptl_tcp_send_frag_t);
     return OMPI_SUCCESS;
@@ -190,7 +190,7 @@ int mca_ptl_tcp_request_init(struct mca_ptl_base_module_t* ptl, struct mca_pml_b
  * Cleanup any resources cached along w/ the request.
  */
 
-void mca_ptl_tcp_request_fini(struct mca_ptl_base_module_t* ptl, struct mca_pml_base_send_request_t* request)
+void mca_ptl_tcp_request_fini(struct mca_ptl_base_module_t* ptl, struct mca_ptl_base_send_request_t* request)
 {
     OBJ_DESTRUCT(request+1);
 }
@@ -240,7 +240,7 @@ void mca_ptl_tcp_send_frag_return(struct mca_ptl_base_module_t* ptl, struct mca_
 int mca_ptl_tcp_send(
     struct mca_ptl_base_module_t* ptl,
     struct mca_ptl_base_peer_t* ptl_peer,
-    struct mca_pml_base_send_request_t* sendreq,
+    struct mca_ptl_base_send_request_t* sendreq,
     size_t offset,
     size_t size,
     int flags)
@@ -261,7 +261,7 @@ int mca_ptl_tcp_send(
     /* must update the offset after actual fragment size is determined -- and very important --
      * before attempting to send the fragment 
      */
-    mca_pml_base_send_request_offset(sendreq, size);
+    mca_ptl_base_send_request_offset(sendreq, size);
     return mca_ptl_tcp_peer_send(ptl_peer, sendfrag, offset);
 }
 

src/mca/ptl/tcp/src/ptl_tcp.h

@@ -197,7 +197,7 @@ extern int mca_ptl_tcp_del_procs(
  */
 extern int mca_ptl_tcp_request_init(
     struct mca_ptl_base_module_t* ptl,
-    struct mca_pml_base_send_request_t*
+    struct mca_ptl_base_send_request_t*
 );
 
 /**
@@ -209,7 +209,7 @@ extern int mca_ptl_tcp_request_init(
  */
 extern void mca_ptl_tcp_request_fini(
     struct mca_ptl_base_module_t* ptl,
-    struct mca_pml_base_send_request_t*
+    struct mca_ptl_base_send_request_t*
 );
 
 /**
@@ -237,7 +237,7 @@ extern void mca_ptl_tcp_matched(
 extern int mca_ptl_tcp_send(
     struct mca_ptl_base_module_t* ptl,
     struct mca_ptl_base_peer_t* ptl_peer,
-    struct mca_pml_base_send_request_t*,
+    struct mca_ptl_base_send_request_t*,
     size_t offset,
     size_t size,
     int flags

src/mca/ptl/tcp/src/ptl_tcp_recvfrag.c

@@ -21,7 +21,7 @@
 #include <sys/types.h>
 #endif
 #include "include/ompi_socket_errno.h"
-#include "mca/pml/base/pml_base_sendreq.h"
+#include "mca/ptl/base/ptl_base_sendreq.h"
 #include "ptl_tcp.h"
 #include "ptl_tcp_peer.h"
 #include "ptl_tcp_recvfrag.h"
@@ -184,7 +184,7 @@ static bool mca_ptl_tcp_recv_frag_header(mca_ptl_tcp_recv_frag_t* frag, int sd,
 static bool mca_ptl_tcp_recv_frag_ack(mca_ptl_tcp_recv_frag_t* frag, int sd)
 {
     mca_ptl_tcp_send_frag_t* sendfrag;
-    mca_pml_base_send_request_t* sendreq;
+    mca_ptl_base_send_request_t* sendreq;
     sendfrag = (mca_ptl_tcp_send_frag_t*)frag->frag_recv.frag_base.frag_header.hdr_ack.hdr_src_ptr.pval;
     sendreq = sendfrag->frag_send.frag_request;
     sendreq->req_peer_match = frag->frag_recv.frag_base.frag_header.hdr_ack.hdr_dst_match;
@@ -227,7 +227,7 @@ static bool mca_ptl_tcp_recv_frag_frag(mca_ptl_tcp_recv_frag_t* frag, int sd)
 {
     /* get request from header */
     if(frag->frag_msg_cnt == 0) {
-        frag->frag_recv.frag_request = (mca_pml_base_recv_request_t *)frag->frag_recv.frag_base.frag_header.hdr_frag.hdr_dst_ptr.pval;
+        frag->frag_recv.frag_request = (mca_ptl_base_recv_request_t *)frag->frag_recv.frag_base.frag_header.hdr_frag.hdr_dst_ptr.pval;
         mca_ptl_tcp_recv_frag_matched(frag, 
             frag->frag_recv.frag_base.frag_header.hdr_frag.hdr_frag_offset,
             frag->frag_recv.frag_base.frag_header.hdr_frag.hdr_frag_length);

src/mca/ptl/tcp/src/ptl_tcp_recvfrag.h

@@ -89,21 +89,21 @@ static inline void mca_ptl_tcp_recv_frag_matched(
     size_t frag_offset, 
     size_t frag_length)
 {
-    mca_pml_base_recv_request_t* request = frag->frag_recv.frag_request;
+    mca_ptl_base_recv_request_t* request = frag->frag_recv.frag_request;
   
     /* if there is data associated with the fragment -- setup to receive */
     if(frag_length > 0) {
         /* initialize receive convertor */
         ompi_proc_t *proc =
-            ompi_comm_peer_lookup(request->req_base.req_comm, 
+            ompi_comm_peer_lookup(request->req_recv.req_base.req_comm, 
-                                  request->req_base.req_ompi.req_status.MPI_SOURCE);
+                                  request->req_recv.req_base.req_ompi.req_status.MPI_SOURCE);
         ompi_convertor_copy(proc->proc_convertor, &frag->frag_recv.frag_base.frag_convertor);
         ompi_convertor_init_for_recv(
             &frag->frag_recv.frag_base.frag_convertor,  /* convertor */
             0,                              /* flags */
-            request->req_base.req_datatype, /* datatype */
+            request->req_recv.req_base.req_datatype, /* datatype */
-            request->req_base.req_count,    /* count elements */
+            request->req_recv.req_base.req_count,    /* count elements */
-            request->req_base.req_addr,     /* users buffer */
+            request->req_recv.req_base.req_addr,     /* users buffer */
             0,                              /* offset in bytes into packed buffer */
             mca_ptl_tcp_memalloc );         /* not allocating memory */
 
@@ -115,16 +115,16 @@ static inline void mca_ptl_tcp_recv_frag_matched(
         } else {
             long true_lb, true_extent;
 
-            ompi_ddt_get_true_extent( request->req_base.req_datatype, &true_lb, &true_extent );
+            ompi_ddt_get_true_extent( request->req_recv.req_base.req_datatype, &true_lb, &true_extent );
-            frag->frag_recv.frag_base.frag_addr = ((unsigned char*)request->req_base.req_addr) + 
+            frag->frag_recv.frag_base.frag_addr = ((unsigned char*)request->req_recv.req_base.req_addr) + 
                frag_offset + true_lb;
         }
 	frag->frag_size = frag_length;
-        if(frag_offset + frag_length > request->req_bytes_packed) {
+        if(frag_offset + frag_length > request->req_recv.req_bytes_packed) {
-            if(frag_offset > request->req_bytes_packed)
+            if(frag_offset > request->req_recv.req_bytes_packed)
                 frag->frag_recv.frag_base.frag_size = 0;
             else
-                frag->frag_recv.frag_base.frag_size = request->req_bytes_packed - frag_offset;
+                frag->frag_recv.frag_base.frag_size = request->req_recv.req_bytes_packed - frag_offset;
         } else {
             frag->frag_recv.frag_base.frag_size = frag_length;
         }
@@ -151,7 +151,7 @@ static inline void mca_ptl_tcp_recv_frag_progress(mca_ptl_tcp_recv_frag_t* frag)
        (frag->frag_recv.frag_base.frag_header.hdr_common.hdr_type != MCA_PTL_HDR_TYPE_FRAG &&
         frag_progressed == 2 )) {
 
-        mca_pml_base_recv_request_t* request = frag->frag_recv.frag_request; 
+        mca_ptl_base_recv_request_t* request = frag->frag_recv.frag_request; 
         if(frag->frag_recv.frag_is_buffered) { 
 	    mca_ptl_base_frag_header_t* header = &(frag)->frag_recv.frag_base.frag_header.hdr_frag; 
 	    size_t offset = (header->hdr_common.hdr_type == MCA_PTL_HDR_TYPE_FRAG) ?
@@ -162,15 +162,15 @@ static inline void mca_ptl_tcp_recv_frag_progress(mca_ptl_tcp_recv_frag_t* frag)
 	     */ 
 	    struct iovec iov; 
 	    ompi_proc_t *proc = 
-		    ompi_comm_peer_lookup(request->req_base.req_comm, 
+		    ompi_comm_peer_lookup(request->req_recv.req_base.req_comm, 
-				          request->req_base.req_ompi.req_status.MPI_SOURCE); 
+				          request->req_recv.req_base.req_ompi.req_status.MPI_SOURCE); 
 	    ompi_convertor_copy(proc->proc_convertor, &frag->frag_recv.frag_base.frag_convertor); 
 	    ompi_convertor_init_for_recv( 
 		    &frag->frag_recv.frag_base.frag_convertor,  /* convertor */ 
 		    0,                                 /* flags */ 
-		    request->req_base.req_datatype,    /* datatype */ 
+		    request->req_recv.req_base.req_datatype,    /* datatype */ 
-		    request->req_base.req_count,       /* count elements */ 
+		    request->req_recv.req_base.req_count,       /* count elements */ 
-		    request->req_base.req_addr,        /* users buffer */ 
+		    request->req_recv.req_base.req_addr,        /* users buffer */ 
 		    offset,                            /* offset in bytes into packed buffer */ 
 		NULL );                            /* dont allocate memory */
 	

src/mca/ptl/tcp/src/ptl_tcp_sendfrag.c

@@ -78,7 +78,7 @@ void* mca_ptl_tcp_memalloc( size_t* length )
 int mca_ptl_tcp_send_frag_init(
     mca_ptl_tcp_send_frag_t* sendfrag,
     mca_ptl_base_peer_t* ptl_peer,
-    mca_pml_base_send_request_t* sendreq,
+    mca_ptl_base_send_request_t* sendreq,
     size_t offset,
     size_t* size,
     int flags)
@@ -96,13 +96,13 @@ int mca_ptl_tcp_send_frag_init(
        int rc;
 
        convertor = &sendfrag->frag_convertor;
-       ompi_convertor_copy(&sendreq->req_convertor, convertor);
+       ompi_convertor_copy(&sendreq->req_send.req_convertor, convertor);
        ompi_convertor_init_for_send( 
 				    convertor,
 				    0, 
-				    sendreq->req_datatype,
+				    sendreq->req_send.req_datatype,
-				    sendreq->req_count,
+				    sendreq->req_send.req_count,
-				    sendreq->req_addr,
+				    sendreq->req_send.req_addr,
 				    offset,
 				    mca_ptl_tcp_memalloc );
 
@@ -130,12 +130,12 @@ int mca_ptl_tcp_send_frag_init(
     if(offset == 0) {
         hdr->hdr_common.hdr_type = (flags & MCA_PTL_FLAGS_ACK) ? MCA_PTL_HDR_TYPE_MATCH : MCA_PTL_HDR_TYPE_RNDV;
         hdr->hdr_common.hdr_flags = flags;
-        hdr->hdr_match.hdr_contextid = sendreq->req_base.req_comm->c_contextid;
+        hdr->hdr_match.hdr_contextid = sendreq->req_send.req_base.req_comm->c_contextid;
-        hdr->hdr_match.hdr_src = sendreq->req_base.req_comm->c_my_rank;
+        hdr->hdr_match.hdr_src = sendreq->req_send.req_base.req_comm->c_my_rank;
-        hdr->hdr_match.hdr_dst = sendreq->req_base.req_peer;
+        hdr->hdr_match.hdr_dst = sendreq->req_send.req_base.req_peer;
-        hdr->hdr_match.hdr_tag = sendreq->req_base.req_tag;
+        hdr->hdr_match.hdr_tag = sendreq->req_send.req_base.req_tag;
-        hdr->hdr_match.hdr_msg_length = sendreq->req_bytes_packed;
+        hdr->hdr_match.hdr_msg_length = sendreq->req_send.req_bytes_packed;
-        hdr->hdr_match.hdr_msg_seq = sendreq->req_base.req_sequence;
+        hdr->hdr_match.hdr_msg_seq = sendreq->req_send.req_base.req_sequence;
         hdr->hdr_rndv.hdr_frag_length = size_out;
         hdr->hdr_rndv.hdr_src_ptr.lval = 0; /* for VALGRIND/PURIFY - REPLACE WITH MACRO */
         hdr->hdr_rndv.hdr_src_ptr.pval = sendfrag;

src/mca/ptl/tcp/src/ptl_tcp_sendfrag.h

@@ -30,7 +30,7 @@
 #include <netinet/in.h>
 #endif
 #include "include/sys/atomic.h"
-#include "mca/pml/base/pml_base_sendreq.h"
+#include "mca/ptl/base/ptl_base_sendreq.h"
 #include "mca/ptl/base/ptl_base_sendfrag.h"
 #include "ptl_tcp.h"
 #include "ptl_tcp_recvfrag.h"
@@ -77,7 +77,7 @@ bool mca_ptl_tcp_send_frag_handler(mca_ptl_tcp_send_frag_t*, int sd);
 int mca_ptl_tcp_send_frag_init(
     mca_ptl_tcp_send_frag_t*, 
     struct mca_ptl_base_peer_t*, 
-    struct mca_pml_base_send_request_t*, 
+    struct mca_ptl_base_send_request_t*, 
     size_t offset,
     size_t* size,
     int flags);
@@ -92,7 +92,7 @@ int mca_ptl_tcp_send_frag_init(
 
 static inline void mca_ptl_tcp_send_frag_progress(mca_ptl_tcp_send_frag_t* frag) 
 { 
-    mca_pml_base_send_request_t* request = frag->frag_send.frag_request; 
+    mca_ptl_base_send_request_t* request = frag->frag_send.frag_request; 
     bool frag_ack;
 
     /* if this is an ack - simply return to pool */ 
@@ -133,15 +133,15 @@ static inline void mca_ptl_tcp_send_frag_init_ack(
     mca_ptl_tcp_recv_frag_t* frag)
 {
     mca_ptl_base_header_t* hdr = &ack->frag_send.frag_base.frag_header;
-    mca_pml_base_recv_request_t* request = frag->frag_recv.frag_request;
+    mca_ptl_base_recv_request_t* request = frag->frag_recv.frag_request;
     hdr->hdr_common.hdr_type = MCA_PTL_HDR_TYPE_ACK;
     hdr->hdr_common.hdr_flags = 0;
     hdr->hdr_ack.hdr_src_ptr = frag->frag_recv.frag_base.frag_header.hdr_rndv.hdr_src_ptr;
     hdr->hdr_ack.hdr_dst_match.lval = 0; /* for VALGRIND/PURIFY - REPLACE WITH MACRO */
     hdr->hdr_ack.hdr_dst_match.pval = request;
     hdr->hdr_ack.hdr_dst_addr.lval = 0; /* for VALGRIND/PURIFY - REPLACE WITH MACRO */
-    hdr->hdr_ack.hdr_dst_addr.pval = request->req_base.req_addr;
+    hdr->hdr_ack.hdr_dst_addr.pval = request->req_recv.req_base.req_addr;
-    hdr->hdr_ack.hdr_dst_size = request->req_bytes_packed;
+    hdr->hdr_ack.hdr_dst_size = request->req_recv.req_bytes_packed;
     ack->frag_send.frag_request = 0;
     ack->frag_send.frag_base.frag_peer = ptl_peer;
     ack->frag_send.frag_base.frag_owner = ptl;

src/mca/ptl/tcp/src/ptl_tcp_sendreq.c

@@ -22,7 +22,7 @@
 #endif
 #include "include/ompi_socket_errno.h"
 #include "include/types.h"
-#include "mca/pml/base/pml_base_sendreq.h"
+#include "mca/ptl/base/ptl_base_sendreq.h"
 #include "ptl_tcp.h"
 #include "ptl_tcp_sendreq.h"
 
@@ -33,7 +33,7 @@ static void mca_ptl_tcp_send_request_destruct(mca_ptl_tcp_send_request_t*);
 
 ompi_class_t  mca_ptl_tcp_send_request_t_class = {
     "mca_ptl_tcp_send_request_t",
-    OBJ_CLASS(mca_pml_base_send_request_t),
+    OBJ_CLASS(mca_ptl_base_send_request_t),
     (ompi_construct_t)mca_ptl_tcp_send_request_construct,
     (ompi_destruct_t)mca_ptl_tcp_send_request_destruct
 };

src/mca/ptl/tcp/src/ptl_tcp_sendreq.h

@@ -30,7 +30,7 @@
 #include <netinet/in.h>
 #endif
 #include "ompi_config.h"
-#include "mca/pml/base/pml_base_sendreq.h"
+#include "mca/ptl/base/ptl_base_sendreq.h"
 #include "ptl_tcp_sendfrag.h"
 #if defined(c_plusplus) || defined(__cplusplus)
 extern "C" {
@@ -43,7 +43,7 @@ extern ompi_class_t mca_ptl_tcp_send_request_t_class;
  * fragment on every send request.
  */
 struct mca_ptl_tcp_send_request_t {
-   mca_pml_base_send_request_t super;
+   mca_ptl_base_send_request_t super;
    mca_ptl_tcp_send_frag_t req_frag; /* first fragment */
 };
 typedef struct mca_ptl_tcp_send_request_t mca_ptl_tcp_send_request_t;