Furthur cleanups. Added new IB send/recv buffer data structures.

This commit was SVN r2585.

src/mca/ptl/ib/src/ptl_ib.c

@@ -50,8 +50,6 @@ int mca_ptl_ib_add_procs(struct mca_ptl_base_module_t* base_module,
     mca_ptl_ib_proc_t* module_proc;
     mca_ptl_base_peer_t* module_peer;
 
-    D_PRINT("Adding %d procs\n", nprocs);
-
     for(i = 0; i < nprocs; i++) {
 
         ompi_proc = ompi_procs[i];
@@ -98,8 +96,6 @@ int mca_ptl_ib_add_procs(struct mca_ptl_base_module_t* base_module,
         peers[i] = module_peer;
     }
 
-    D_PRINT("Added %d procs\n", nprocs);
-
     return OMPI_SUCCESS;
 }
 

src/mca/ptl/ib/src/ptl_ib_peer.c

@@ -107,6 +107,7 @@ static int mca_ptl_ib_alloc_peer_conn(mca_ptl_base_peer_t* peer)
     if(NULL == peer->peer_conn) {
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
+
     return OMPI_SUCCESS;
 }
 
@@ -249,10 +250,6 @@ static int mca_ptl_ib_peer_reply_start_connect(mca_ptl_ib_peer_t *peer,
         return rc;
     }
 
-    /* Register Buffers */
-
-    /* Post receives */
-
     /* Send connection info over to remote peer */
     rc = mca_ptl_ib_peer_send_conn_info(peer);
     if(rc != OMPI_SUCCESS) {

src/mca/ptl/ib/src/ptl_ib_priv.c

@@ -403,7 +403,7 @@ static int mca_ptl_ib_register_mem(VAPI_hca_hndl_t nic, VAPI_pd_hndl_t ptag,
     memhandle->lkey = mem_handle.lkey;
     memhandle->rkey = mem_handle.rkey;
 
-    D_PRINT("addr = %p, lkey = %d\n", buf, memhandle->lkey);
+    /* D_PRINT("addr = %p, lkey = %d\n", buf, memhandle->lkey); */
 
     memhandle->hndl = mem_handle.hndl;
 
@@ -413,20 +413,19 @@ static int mca_ptl_ib_register_mem(VAPI_hca_hndl_t nic, VAPI_pd_hndl_t ptag,
 int mca_ptl_ib_init_peer(mca_ptl_ib_state_t *ib_state,
         mca_ptl_ib_peer_conn_t *peer_conn)
 {
-    /* Allocate resources for the peer connection */
-
+    /* Local resources */
     peer_conn->lres = (mca_ptl_ib_peer_local_res_t *)
         malloc(sizeof(mca_ptl_ib_peer_local_res_t));
     if(NULL == peer_conn->lres) {
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
 
+    /* Remote resources */
     peer_conn->rres = (mca_ptl_ib_peer_remote_res_t *)
         malloc(sizeof(mca_ptl_ib_peer_remote_res_t));
     if(NULL == peer_conn->rres) {
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
-
     /* Create the Queue Pair */
     if(mca_ptl_ib_create_qp(ib_state->nic,
                     ib_state->ptag,
@@ -443,15 +442,81 @@ int mca_ptl_ib_init_peer(mca_ptl_ib_state_t *ib_state,
 }
 
 /*
- * Establish Reliable Connection with peer
+ * 1. Establish Reliable Connection with peer
+ * 2. Allocate resources to this connection
+ * 3. Post receives for this connection
  *
  */
 
 int mca_ptl_ib_peer_connect(mca_ptl_ib_state_t *ib_state,
         mca_ptl_ib_peer_conn_t *peer_conn)
 {
-    return(mca_ptl_ib_rc_qp_init(ib_state->nic,
+    int rc, i;
+    VAPI_ret_t ret;
+
+    /* Establish Reliable Connection */
+    rc = mca_ptl_ib_rc_qp_init(ib_state->nic,
                 peer_conn->lres->qp_hndl,
                 peer_conn->rres->qp_num,
-                peer_conn->rres->lid));
+                peer_conn->rres->lid);
+
+    if(rc != OMPI_SUCCESS) {
+        return rc;
+    }
+
+    /* Allocate resources to this connection */
+    peer_conn->lres->send = (ib_buffer_t*)
+        malloc(sizeof(ib_buffer_t) * NUM_IB_SEND_BUF);
+
+    if(NULL == peer_conn->lres->send) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    peer_conn->lres->recv = (ib_buffer_t*)
+        malloc(sizeof(ib_buffer_t) * NUM_IB_RECV_BUF);
+
+    if(NULL == peer_conn->lres->recv) {
+        return OMPI_ERR_OUT_OF_RESOURCE;
+    }
+
+    /* Register the buffers */
+    for(i = 0; i < NUM_IB_SEND_BUF; i++) {
+
+        rc = mca_ptl_ib_register_mem(ib_state->nic, ib_state->ptag,
+                (void*) peer_conn->lres->send[i].buf, 
+                4096, &peer_conn->lres->send[i].hndl);
+        if(rc != OMPI_SUCCESS) {
+            return OMPI_ERROR;
+        }
+    }
+
+    for(i = 0; i < NUM_IB_RECV_BUF; i++) {
+
+        ib_buffer_t *ib_buf_ptr;
+
+        rc = mca_ptl_ib_register_mem(ib_state->nic, ib_state->ptag,
+                (void*) peer_conn->lres->recv[i].buf, 
+                4096, &peer_conn->lres->recv[i].hndl);
+        if(rc != OMPI_SUCCESS) {
+            return OMPI_ERROR;
+        }
+
+        IB_PREPARE_RECV_DESC((&peer_conn->lres->recv[i]));
+    }
+
+    /* Post receives */
+    for(i = 0; i < NUM_IB_RECV_BUF; i++) {
+
+        ret = VAPI_post_rr(ib_state->nic,
+                peer_conn->lres->qp_hndl,
+                &peer_conn->lres->recv[i].desc.rr);
+
+        if(VAPI_OK != ret) {
+            MCA_PTL_IB_VAPI_RET(ret, "VAPI_post_rr");
+        }
+    }
+
+    D_PRINT("Done posting recvs");
+
+    return OMPI_SUCCESS;
 }

src/mca/ptl/ib/src/ptl_ib_priv.h

@@ -4,21 +4,28 @@
 #include <stdint.h>
 #include "ptl_ib_vapi.h"
 
-struct mca_ptl_ib_state_t {
-    VAPI_hca_id_t                   hca_id;     /* ID of HCA */
-    VAPI_hca_port_t                 port;       /* IB port of this PTL */
-    VAPI_hca_hndl_t                 nic;        /* NIC handle */  
-    VAPI_pd_hndl_t                  ptag;       /* Protection Domain tag */
+#define NUM_IB_SEND_BUF             (10)
+#define NUM_IB_RECV_BUF             (10)
 
-    VAPI_cq_hndl_t                  cq_hndl;    /* Completion Queue handle */
-                                                /* At present Send & Recv
-                                                 * are tied to the same
-                                                 * completion queue */
+struct mca_ptl_ib_state_t {
+    VAPI_hca_id_t                   hca_id;     
+    /* ID of HCA */
+
+    VAPI_hca_port_t                 port;       
+    /* IB port of this PTL */
+
+    VAPI_hca_hndl_t                 nic;        
+    /* NIC handle */  
+
+    VAPI_pd_hndl_t                  ptag;       
+    /* Protection Domain tag */
+
+    VAPI_cq_hndl_t                  cq_hndl;    
+    /* Completion Queue handle */
+    /* At present Send & Recv are tied to the same completion queue */
 
     EVAPI_async_handler_hndl_t      async_handler; 
-                                                /* Async event handler used
-                                                 * to detect weird/unknown
-                                                 * events */
+    /* Async event handler used to detect weird/unknown events */
 };
 
 typedef struct mca_ptl_ib_state_t mca_ptl_ib_state_t;
@@ -58,6 +65,19 @@ struct vapi_descriptor_t {
 
 typedef struct vapi_descriptor_t vapi_descriptor_t;
 
+struct ib_buffer_t {
+    vapi_descriptor_t               desc;
+    /* Descriptor of the buffer */
+
+    vapi_memhandle_t                hndl;
+    /* Buffer handle */
+
+    char                            buf[4096];
+    /* Buffer space */
+};
+
+typedef struct ib_buffer_t ib_buffer_t;
+
 /* mca_ptl_ib_peer_local_res_t contains information
  * regarding local resources dedicated to this
  * connection */
@@ -68,6 +88,12 @@ struct mca_ptl_ib_peer_local_res_t {
 
     VAPI_qp_prop_t                  qp_prop;
     /* Local QP properties */
+
+    ib_buffer_t                     *send;
+    /* Pointer to send buffers */
+
+    ib_buffer_t                     *recv;
+    /* Pointer to recv buffers */
 };
 
 typedef struct mca_ptl_ib_peer_local_res_t mca_ptl_ib_peer_local_res_t;
@@ -112,6 +138,18 @@ typedef struct mca_ptl_ib_peer_conn_t mca_ptl_ib_peer_conn_t;
     ompi_output(0, "Async hndl : %d", ib_state_ptr->async_handler); \
 }
 
+#define IB_PREPARE_RECV_DESC(ib_buf_ptr) {                          \
+    ib_buf_ptr->desc.rr.comp_type = VAPI_SIGNALED;                  \
+    ib_buf_ptr->desc.rr.opcode = VAPI_RECEIVE;                      \
+    ib_buf_ptr->desc.rr.id = (VAPI_virt_addr_t)                     \
+        (MT_virt_addr_t) ib_buf_ptr;                                \
+    ib_buf_ptr->desc.rr.sg_lst_len = 1;                             \
+    ib_buf_ptr->desc.rr.sg_lst_p = &ib_buf_ptr->desc.sg_entry;      \
+    ib_buf_ptr->desc.sg_entry.len = 4096;                           \
+    ib_buf_ptr->desc.sg_entry.lkey = ib_buf_ptr->hndl.lkey;         \
+    ib_buf_ptr->desc.sg_entry.addr = (VAPI_virt_addr_t)             \
+        (MT_virt_addr_t) ib_buf_ptr->buf;                           \
+}
 
 int mca_ptl_ib_init_module(mca_ptl_ib_state_t*, int);
 int mca_ptl_ib_get_num_hcas(uint32_t*);

src/mca/ptl/ib/src/ptl_ib_proc.c

@@ -110,8 +110,6 @@ mca_ptl_ib_proc_t* mca_ptl_ib_proc_create(ompi_proc_t* ompi_proc)
      * size) to represent the proc */
     module_proc->proc_guid = ompi_proc->proc_name;
 
-    D_PRINT("Creating proc for %d\n", ompi_proc->proc_name.vpid);
-
     /* IB module doesn't have addresses exported at
      * initialization, so the addr_count is set to one. */
     module_proc->proc_addr_count = 1;