Changed ptl_first_frag_size. For totally supporting dynamic

connection management, the first *REAL* infiniband send has
to be triggered from the progress engine. Taking care of this
case with this code.

This commit was SVN r2674.

src/mca/ptl/ib/src/ptl_ib.c

@@ -119,6 +119,7 @@ int mca_ptl_ib_finalize(struct mca_ptl_base_module_t* ptl)
 int mca_ptl_ib_request_init( struct mca_ptl_base_module_t* ptl,
         struct mca_pml_base_send_request_t* request)
 {
+    mca_ptl_ib_send_request_t *ib_send_req;
     mca_ptl_ib_send_frag_t *ib_send_frag;
 
     ib_send_frag = mca_ptl_ib_alloc_send_frag(ptl,
@@ -128,13 +129,10 @@ int mca_ptl_ib_request_init( struct mca_ptl_base_module_t* ptl,
         D_PRINT("Unable to allocate ib_send_frag");
         return OMPI_ERR_OUT_OF_RESOURCE;
     } else {
-        ((mca_ptl_ib_send_request_t *)request)->req_frag = 
-            ib_send_frag;
-    }
 
-    D_PRINT("sendfrag = %p, lkey = %d", 
+        ib_send_req = (mca_ptl_ib_send_request_t *) request;
-            ib_send_frag,
+        ib_send_req->req_frag = ib_send_frag;
-            ib_send_frag->ib_buf.hndl.lkey);
+    }
 
     return OMPI_SUCCESS;
 }
@@ -165,7 +163,7 @@ int mca_ptl_ib_send( struct mca_ptl_base_module_t* ptl,
 
     if (0 == offset) {
         sendfrag = (mca_ptl_ib_send_frag_t *)
-            &((mca_ptl_ib_send_request_t*)sendreq)->req_frag;
+            ((mca_ptl_ib_send_request_t*)sendreq)->req_frag;
     } else {
 
         /* TODO: Implementation for messages > frag size */
@@ -177,10 +175,6 @@ int mca_ptl_ib_send( struct mca_ptl_base_module_t* ptl,
         }
     }
 
-    D_PRINT("Sendfrag = %p, Lkey = %d", 
-            sendfrag,
-            sendfrag->ib_buf.hndl.lkey);
-
     rc = mca_ptl_ib_send_frag_init(sendfrag, ptl_peer,
             sendreq, offset, &size, flags);
     if(rc != OMPI_SUCCESS) {

src/mca/ptl/ib/src/ptl_ib_component.c

@@ -95,10 +95,10 @@ int mca_ptl_ib_component_open(void)
         mca_ptl_ib_param_register_int ("exclusivity", 0);
     mca_ptl_ib_module.super.ptl_first_frag_size =
         mca_ptl_ib_param_register_int ("first_frag_size",
-                (2048 - sizeof(mca_ptl_base_header_t))/*magic*/);
+                (16384 - sizeof(mca_ptl_base_header_t))/*magic*/);
     mca_ptl_ib_module.super.ptl_min_frag_size =
         mca_ptl_ib_param_register_int ("min_frag_size",
-                (2048 - sizeof(mca_ptl_base_header_t))/*magic*/);
+                (4096 - sizeof(mca_ptl_base_header_t))/*magic*/);
     mca_ptl_ib_module.super.ptl_max_frag_size =
         mca_ptl_ib_param_register_int ("max_frag_size", 2<<30);
 

src/mca/ptl/ib/src/ptl_ib_peer.c

@@ -116,12 +116,16 @@ static int mca_ptl_ib_peer_send_conn_info(mca_ptl_base_peer_t* peer)
 {
     int rc;
     ompi_process_name_t *name;
-    char sendbuf[50];
+    char* sendbuf;
-
-    D_PRINT("");
 
     name = &peer->peer_proc->proc_guid;
 
+    sendbuf = (char*) malloc(sizeof(char)*50);
+
+    if(NULL == sendbuf) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
     /* Zero out the send buffer */
     memset(sendbuf, 0, 50);
 
@@ -146,6 +150,9 @@ static int mca_ptl_ib_peer_send_conn_info(mca_ptl_base_peer_t* peer)
     if(rc != OMPI_SUCCESS) {
         return rc;
     }
+
+    D_PRINT("Sent buffer : %s", sendbuf);
+
     return OMPI_SUCCESS;
 }
 
@@ -211,8 +218,6 @@ static int mca_ptl_ib_peer_start_connect(mca_ptl_base_peer_t* peer)
     /* Update status of peer to as connecting */
     peer->peer_state = MCA_PTL_IB_CONNECTING;
 
-    DUMP_PEER(peer);
-
     return rc;
 }
 
@@ -257,8 +262,6 @@ static int mca_ptl_ib_peer_reply_start_connect(mca_ptl_ib_peer_t *peer,
     /* Update status of peer to as connected */
     peer->peer_state = MCA_PTL_IB_CONNECTED;
 
-    DUMP_PEER(peer);
-
     return rc;
 }
 
@@ -269,6 +272,8 @@ static int mca_ptl_ib_peer_reply_start_connect(mca_ptl_ib_peer_t *peer,
 static void mca_ptl_ib_peer_connected(mca_ptl_ib_peer_t *peer)
 {
     peer->peer_state = MCA_PTL_IB_CONNECTED;
+
+    DUMP_PEER(peer);
 }
 
 /*

src/mca/ptl/ib/src/ptl_ib_peer.h

@@ -87,7 +87,7 @@ void mca_ptl_ib_post_oob_recv_nb(void);
             peer_ptr->peer_conn->lres->qp_prop.qp_num);             \
     ompi_output(0, "Remote QP num : %d",                            \
             peer_ptr->peer_conn->rres->qp_num);                     \
-    ompi_output(0, "Remte LID : %d",                                \
+    ompi_output(0, "Remote LID : %d",                               \
             peer_ptr->peer_conn->rres->lid);                        \
 }
 

src/mca/ptl/ib/src/ptl_ib_priv.c

@@ -511,9 +511,9 @@ int mca_ptl_ib_post_send(mca_ptl_ib_state_t *ib_state,
 {
     VAPI_ret_t ret;
 
-    IB_PREPARE_SEND_DESC(ib_buf, (peer_conn->rres->qp_num));
+    IB_SET_REMOTE_QP_NUM(ib_buf, (peer_conn->rres->qp_num));
 
-    D_PRINT("lkey = %d", ib_buf->hndl.lkey);
+    D_PRINT("length : %d", ib_buf->desc.sg_entry.len);
 
     ret = VAPI_post_sr(ib_state->nic,
             peer_conn->lres->qp_hndl,

src/mca/ptl/ib/src/ptl_ib_priv.h

@@ -149,21 +149,25 @@ typedef struct mca_ptl_ib_peer_conn_t mca_ptl_ib_peer_conn_t;
         (MT_virt_addr_t) ib_buf_ptr->buf;                           \
 }
 
-#define IB_PREPARE_SEND_DESC(ib_buf_ptr, qp) {                      \
+#define IB_PREPARE_SEND_DESC(ib_buf_ptr, qp, msg_len) {             \
     ib_buf_ptr->desc.sr.comp_type = VAPI_SIGNALED;                  \
     ib_buf_ptr->desc.sr.opcode = VAPI_SEND;                         \
     ib_buf_ptr->desc.sr.remote_qkey = 0;                            \
     ib_buf_ptr->desc.sr.remote_qp = qp;                             \
     ib_buf_ptr->desc.sr.id = (VAPI_virt_addr_t)                     \
         (MT_virt_addr_t) ib_buf_ptr;                                \
-    ib_buf_ptr->desc.rr.sg_lst_len = 1;                             \
+    ib_buf_ptr->desc.sr.sg_lst_len = 1;                             \
-    ib_buf_ptr->desc.rr.sg_lst_p = &ib_buf_ptr->desc.sg_entry;      \
+    ib_buf_ptr->desc.sr.sg_lst_p = &ib_buf_ptr->desc.sg_entry;      \
-    ib_buf_ptr->desc.sg_entry.len = 4096;                           \
+    ib_buf_ptr->desc.sg_entry.len = msg_len;                        \
     ib_buf_ptr->desc.sg_entry.lkey = ib_buf_ptr->hndl.lkey;         \
     ib_buf_ptr->desc.sg_entry.addr = (VAPI_virt_addr_t)             \
         (MT_virt_addr_t) ib_buf_ptr->buf;                           \
 }
 
+#define IB_SET_REMOTE_QP_NUM(ib_buf_ptr, qp) {                      \
+    ib_buf_ptr->desc.sr.remote_qp = qp;                             \
+}
+
 int mca_ptl_ib_init_module(mca_ptl_ib_state_t*, int);
 int mca_ptl_ib_get_num_hcas(uint32_t*);
 int mca_ptl_ib_init_peer(mca_ptl_ib_state_t*, mca_ptl_ib_peer_conn_t*);

src/mca/ptl/ib/src/ptl_ib_sendfrag.c

@@ -5,6 +5,7 @@
 #include "ptl_ib_peer.h"
 #include "ptl_ib_proc.h"
 #include "ptl_ib_sendfrag.h"
+#include "ptl_ib_priv.h"
 
 static void mca_ptl_ib_send_frag_construct(mca_ptl_ib_send_frag_t* frag);
 static void mca_ptl_ib_send_frag_destruct(mca_ptl_ib_send_frag_t* frag);
@@ -104,7 +105,7 @@ int mca_ptl_ib_send_frag_init(mca_ptl_ib_send_frag_t* sendfrag,
         }
 
 
-        /* if data is contigous convertor will return an offset
+        /* if data is contigous, convertor will return an offset
          * into users buffer - otherwise will return an allocated buffer 
          * that holds the packed data
          */
@@ -123,8 +124,12 @@ int mca_ptl_ib_send_frag_init(mca_ptl_ib_send_frag_t* sendfrag,
         /* adjust size and request offset to reflect actual 
          * number of bytes packed by convertor */
         size_out = iov.iov_len;
+        IB_PREPARE_SEND_DESC((&sendfrag->ib_buf), 0, 
+                (header_length + iov.iov_len));
     } else {
         size_out = size_in;
+        IB_PREPARE_SEND_DESC((&sendfrag->ib_buf), 0, 
+                (header_length + size_in));
     }
 
     hdr->hdr_frag.hdr_frag_length = size_out;
@@ -185,8 +190,6 @@ int mca_ptl_ib_register_send_frags(mca_ptl_base_module_t *ptl)
     mca_ptl_ib_send_frag_t *ib_send_frag;
     mca_ptl_ib_state_t *ib_state;
 
-    D_PRINT("");
-
     flist = &((mca_ptl_ib_module_t *)ptl)->send_free;
 
     ib_state = ((mca_ptl_ib_module_t *)ptl)->ib_state;
@@ -212,11 +215,7 @@ int mca_ptl_ib_register_send_frags(mca_ptl_base_module_t *ptl)
             return OMPI_ERROR;
         }
 
-        if(i == 0) {
+        IB_PREPARE_SEND_DESC(ib_buf_ptr, 0, 4096);
-            D_PRINT("lkey = %d", ib_buf_ptr->hndl.lkey);
-        }
-
-        IB_PREPARE_SEND_DESC(ib_buf_ptr, 0);
     }
 
     return OMPI_SUCCESS;

src/mca/ptl/ib/src/ptl_ib_sendreq.c

@@ -15,12 +15,19 @@ OBJ_CLASS_INSTANCE(mca_ptl_ib_send_request_t,
 void mca_ptl_ib_send_request_construct(mca_ptl_ib_send_request_t* request)
 {
     D_PRINT("\n");
+    request->req_frag = NULL;
+
+    /*
     OBJ_CONSTRUCT(&request->req_frag, mca_ptl_ib_send_frag_t);
+    */
 }
 
 
 void mca_ptl_ib_send_request_destruct(mca_ptl_ib_send_request_t* request)
 {
     D_PRINT("\n");
+
+    /*
     OBJ_DESTRUCT(&request->req_frag);
+    */
 }