allocating fragement descriptor and aligned payload buffer.

payload size and alignment are mca parameters.

This commit was SVN r2503.

src/mca/ptl/sm/src/Makefile.am

@@ -31,6 +31,7 @@ libmca_ptl_sm_la_SOURCES = \
 	ptl_sm.h \
 	ptl_sm_component.c \
 	ptl_sm_frag.h \
+	ptl_sm_frag.c \
 	ptl_sm_sendreq.c \
 	ptl_sm_sendfrag.c \
 	ptl_sm_recvfrag.c

src/mca/ptl/sm/src/ptl_sm.h

@@ -46,8 +46,9 @@ struct mca_ptl_sm_component_t {
     mca_mpool_base_module_t* sm_mpool; /**< shared memory pool */
     void* sm_mpool_base;                  /**< base address of shared memory pool */
     ompi_free_list_t sm_send_requests;    /**< free list of sm send requests -- sendreq + sendfrag */
-    ompi_free_list_t sm_send_frags;       /**< free list of sm send fragments */
-    ompi_free_list_t sm_recv_frags;       /**< free list of sm recv fragments */
+    ompi_free_list_t sm_frags;       /**< free list of sm recv fragments */
+    size_t fragment_size;            /**< fragment size */
+    size_t fragment_alignment;       /**< fragment alignment */
     ompi_mutex_t sm_lock;
     char* sm_resouce_ctl_file;     /**< name of shared memory file used 
                                             to coordinate resource usage */

src/mca/ptl/sm/src/ptl_sm_component.c

@@ -9,6 +9,7 @@
 #include <sys/mman.h>
 
 #include "include/constants.h"
+#include "include/sys/cache.h"
 #include "event/event.h"
 #include "util/if.h"
 #include "util/argv.h"
@@ -112,6 +113,11 @@ int mca_ptl_sm_component_open(void)
         mca_ptl_sm_param_register_int("max_procs", -1);
     mca_ptl_sm_component.sm_mpool_name =
         mca_ptl_sm_param_register_string("mpool", "sm");
+    mca_ptl_sm_component.fragment_size =
+        mca_ptl_sm_param_register_int("fragment_size", 8192);
+    mca_ptl_sm_component.fragment_alignment =
+        mca_ptl_sm_param_register_int("fragment_alignment",
+                CACHE_LINE_SIZE);
 
     /* default number of extra procs to allow for future growth */
     mca_ptl_sm_component.sm_extra_procs =
@@ -120,8 +126,7 @@ int mca_ptl_sm_component_open(void)
     /* initialize objects */
     OBJ_CONSTRUCT(&mca_ptl_sm_component.sm_lock, ompi_mutex_t);
     OBJ_CONSTRUCT(&mca_ptl_sm_component.sm_send_requests, ompi_free_list_t);
-    OBJ_CONSTRUCT(&mca_ptl_sm_component.sm_send_frags, ompi_free_list_t);
-    OBJ_CONSTRUCT(&mca_ptl_sm_component.sm_recv_frags, ompi_free_list_t);
+    OBJ_CONSTRUCT(&mca_ptl_sm_component.sm_frags, ompi_free_list_t);
    
     return OMPI_SUCCESS;
 }
@@ -135,8 +140,7 @@ int mca_ptl_sm_component_close(void)
 {
     OBJ_DESTRUCT(&mca_ptl_sm_component.sm_lock);
     OBJ_DESTRUCT(&mca_ptl_sm_component.sm_send_requests);
-    OBJ_DESTRUCT(&mca_ptl_sm_component.sm_send_frags);
-    OBJ_DESTRUCT(&mca_ptl_sm_component.sm_recv_frags);
+    OBJ_DESTRUCT(&mca_ptl_sm_component.sm_frags);
     return OMPI_SUCCESS;
 }
 
@@ -150,7 +154,7 @@ mca_ptl_base_module_t** mca_ptl_sm_component_init(
     bool *have_hidden_threads)
 {
     mca_ptl_base_module_t **ptls = NULL;
-    mca_mpool_base_component_t *sm_mpool_component; /* RLG */
+    size_t length;
 
     *num_ptls = 0;
     *allow_multi_user_threads = true;
@@ -162,7 +166,7 @@ mca_ptl_base_module_t** mca_ptl_sm_component_init(
 
     mca_ptl_sm_component.sm_mpool_base = mca_ptl_sm_component.sm_mpool->mpool_base();
 
-    /* initialize free lists */
+    /* initialize send descriptor free list */
     ompi_free_list_init(&mca_ptl_sm_component.sm_send_requests, 
         sizeof(mca_ptl_sm_send_request_t),
         OBJ_CLASS(mca_ptl_sm_send_request_t),
@@ -171,9 +175,15 @@ mca_ptl_base_module_t** mca_ptl_sm_component_init(
         mca_ptl_sm_component.sm_free_list_inc,
         mca_ptl_sm_component.sm_mpool); /* use shared-memory pool */
 
-    ompi_free_list_init(&mca_ptl_sm_component.sm_recv_frags, 
-        sizeof(mca_ptl_sm_recv_frag_t),
-        OBJ_CLASS(mca_ptl_sm_recv_frag_t),
+    /* initialize fragment descriptor free list */
+
+    /* allocation will be for the fragment descriptor, payload buffer,
+     * and padding to ensure proper alignment can be acheived */
+    length=sizeof(mca_ptl_sm_frag_t)+mca_ptl_sm_component.fragment_alignment+
+        mca_ptl_sm_component.fragment_size;
+
+    ompi_free_list_init(&mca_ptl_sm_component.sm_frags, length,
+        OBJ_CLASS(mca_ptl_sm_frag_t),
         mca_ptl_sm_component.sm_free_list_num,
         mca_ptl_sm_component.sm_free_list_max,
         mca_ptl_sm_component.sm_free_list_inc,

src/mca/ptl/sm/src/ptl_sm_frag.c

@@ -0,0 +1,48 @@
+/*
+ * $HEADER$
+ */
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/errno.h>
+#include "ptl_sm.h"
+#include "ptl_sm_frag.h"
+
+
+static void mca_ptl_sm_frag_construct(mca_ptl_sm_frag_t* frag);
+static void mca_ptl_sm_frag_destruct(mca_ptl_sm_frag_t* frag);
+
+OBJ_CLASS_INSTANCE(
+    mca_ptl_sm_frag_t,
+    mca_ptl_base_recv_frag_t,
+    mca_ptl_sm_frag_construct,
+    mca_ptl_sm_frag_destruct
+);
+                                                                                                           
+
+/*
+ * shared memory recv fragment constructor
+ */
+
+static void mca_ptl_sm_frag_construct(mca_ptl_sm_frag_t* frag)
+{
+    char *ptr;
+
+    /* set the buffer length */
+    frag->buff_length=(size_t)mca_ptl_sm_component.fragment_size;
+
+    /* set buffer pointer */
+    ptr=((char *)frag)+sizeof(mca_ptl_sm_frag_t)+
+        mca_ptl_sm_component.fragment_alignment;
+    /* align */
+    ptr=ptr-(((size_t)ptr)%(mca_ptl_sm_component.fragment_alignment));
+}
+
+
+/*
+ * shared memory recv fragment destructor
+ */
+
+static void mca_ptl_sm_frag_destruct(mca_ptl_sm_frag_t* frag)
+{
+}
+

src/mca/ptl/sm/src/ptl_sm_frag.h

@@ -24,6 +24,8 @@ OBJ_CLASS_DECLARATION(mca_ptl_sm_frag_t);
  */
 struct mca_ptl_sm_frag_t {
     mca_ptl_base_recv_frag_t super; /**< base receive fragment descriptor */
+    size_t buff_length;   /**< size of buffer */
+    void *buff;           /**< pointer to buffer */
 };
 typedef struct mca_ptl_sm_frag_t mca_ptl_sm_frag_t;
 

src/mca/ptl/sm/src/ptl_sm_sendreq.c

@@ -24,7 +24,7 @@ OBJ_CLASS_INSTANCE(
 
 void mca_ptl_sm_send_request_construct(mca_ptl_sm_send_request_t* request)
 {
-    OBJ_CONSTRUCT(&request->req_frag, mca_ptl_sm_send_frag_t);
+    OBJ_CONSTRUCT(&request->req_frag, mca_ptl_sm_frag_t);
 }
 
 

src/mca/ptl/sm/src/ptl_sm_sendreq.h

@@ -11,20 +11,23 @@
 #include <sys/types.h>
 #include "ompi_config.h"
 #include "mca/pml/base/pml_base_sendreq.h"
-#include "ptl_sm_sendfrag.h"
+#include "ptl_sm_frag.h"
 
 OBJ_CLASS_DECLARATION(mca_ptl_sm_send_request_t);
 
 
 /**
- * Shared Memory (SM) send request derived type. The send request contains both the
- * base send request, and space for the first send fragment descriptor.
- * This avoids the overhead of a second allocation for the initial send 
- * fragment on every send request.
+ * Shared Memory (SM) send request derived type. The send request contains 
+ * both base send request, and a pointer to the first fragment descriptor.
  */
 struct mca_ptl_sm_send_request_t {
-   mca_pml_base_send_request_t super;
-   mca_ptl_sm_send_frag_t req_frag; /* first fragment */
+
+    /* base send descriptor */
+    mca_pml_base_send_request_t super;
+  
+    /* pointer to first fragment descriptor */
+    mca_ptl_sm_frag_t *req_frag;
+
 };
 typedef struct mca_ptl_sm_send_request_t mca_ptl_sm_send_request_t;