reset sg_entry->addr to point at header when sending control messages.

cast to uint64_t (the correct datatype per verbs.h) instead of uintptr_t. 

This commit was SVN r8175.

ompi/mca/btl/openib/btl_openib.c

@@ -308,7 +308,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_src(
         frag->mr = openib_reg->mr; 
         frag->sg_entry.length = max_data; 
         frag->sg_entry.lkey = frag->mr->lkey; 
-        frag->sg_entry.addr = (uintptr_t) iov.iov_base; 
+        frag->sg_entry.addr = (uint64_t) iov.iov_base; 
         
         frag->segment.seg_key.key32[0] = (uint32_t) frag->sg_entry.lkey; 
         
@@ -352,7 +352,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_src(
         frag->mr = openib_reg->mr; 
         frag->sg_entry.length = max_data; 
         frag->sg_entry.lkey = openib_reg->mr->lkey;
-        frag->sg_entry.addr = (uintptr_t) iov.iov_base; 
+        frag->sg_entry.addr = (uint64_t) iov.iov_base; 
         
         frag->segment.seg_key.key32[0] = (uint32_t) frag->mr->rkey; 
             
@@ -492,7 +492,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_dst(
     frag->mr = openib_reg->mr; 
     frag->sg_entry.length = *size; 
     frag->sg_entry.lkey = openib_reg->mr->lkey; 
-    frag->sg_entry.addr = (uintptr_t) frag->segment.seg_addr.pval; 
+    frag->sg_entry.addr = (uint64_t) frag->segment.seg_addr.pval; 
     
     frag->segment.seg_key.key32[0] = frag->mr->rkey; 
     
@@ -566,6 +566,7 @@ int mca_btl_openib_send(
     frag->endpoint = endpoint; 
     frag->hdr->tag = tag;
     frag->type = MCA_BTL_IB_FRAG_SEND; 
+    
     return mca_btl_openib_endpoint_send(endpoint, frag);
 }
 
@@ -604,9 +605,9 @@ int mca_btl_openib_put( mca_btl_base_module_t* btl,
     } else { 
         
         frag->wr_desc.sr_desc.send_flags = IBV_SEND_SIGNALED; 
-        frag->wr_desc.sr_desc.wr.rdma.remote_addr = (uintptr_t) frag->base.des_dst->seg_addr.pval; 
+        frag->wr_desc.sr_desc.wr.rdma.remote_addr = (uint64_t) frag->base.des_dst->seg_addr.pval; 
         frag->wr_desc.sr_desc.wr.rdma.rkey = frag->base.des_dst->seg_key.key32[0]; 
-        frag->sg_entry.addr = (uintptr_t) frag->base.des_src->seg_addr.pval; 
+        frag->sg_entry.addr = (uint64_t) frag->base.des_src->seg_addr.pval; 
         frag->sg_entry.length  = frag->base.des_src->seg_len; 
         
         BTL_VERBOSE(("frag->wr_desc.sr_desc.wr.rdma.remote_addr = %llu .rkey = %lu frag->sg_entry.addr = %llu .length = %lu" 
@@ -684,9 +685,9 @@ int mca_btl_openib_get( mca_btl_base_module_t* btl,
     } else { 
     
         frag->wr_desc.sr_desc.send_flags = IBV_SEND_SIGNALED; 
-        frag->wr_desc.sr_desc.wr.rdma.remote_addr = (uintptr_t) frag->base.des_src->seg_addr.pval; 
+        frag->wr_desc.sr_desc.wr.rdma.remote_addr = (uint64_t) frag->base.des_src->seg_addr.pval; 
         frag->wr_desc.sr_desc.wr.rdma.rkey = frag->base.des_src->seg_key.key32[0]; 
-        frag->sg_entry.addr = (uintptr_t) frag->base.des_dst->seg_addr.pval; 
+        frag->sg_entry.addr = (uint64_t) frag->base.des_dst->seg_addr.pval; 
         frag->sg_entry.length  = frag->base.des_dst->seg_len; 
         
         BTL_VERBOSE(("frag->wr_desc.sr_desc.wr.rdma.remote_addr = %llu .rkey = %lu frag->sg_entry.addr = %llu .length = %lu" 
@@ -802,5 +803,6 @@ int mca_btl_openib_module_init(mca_btl_openib_module_t *openib_btl)
         return OMPI_ERROR;
     }
         
+    
     return OMPI_SUCCESS;
 }

ompi/mca/btl/openib/btl_openib_component.c

@@ -590,7 +590,7 @@ int mca_btl_openib_component_progress()
             case IBV_WC_SEND :
                     
                 /* Process a completed send or rdma write*/
-                frag = (mca_btl_openib_frag_t*) (void*) (unsigned long) wc.wr_id; 
+                frag = (mca_btl_openib_frag_t*) (void*) (uint64_t) wc.wr_id; 
                 endpoint = frag->endpoint;
                 frag->base.des_cbfunc(&openib_btl->super, frag->endpoint, &frag->base, OMPI_SUCCESS); 
 
@@ -611,7 +611,7 @@ int mca_btl_openib_component_progress()
             case IBV_WC_RECV: 
 
                 /* Process a RECV */ 
-                frag = (mca_btl_openib_frag_t*) (void*) (unsigned long) wc.wr_id;
+                frag = (mca_btl_openib_frag_t*) (void*) (uint64_t) wc.wr_id;
                 endpoint = (mca_btl_openib_endpoint_t*) frag->endpoint; 
                 credits = frag->hdr->credits;
 
@@ -698,7 +698,7 @@ int mca_btl_openib_component_progress()
             case IBV_WC_SEND:
 
                 /* Process a completed send - receiver must return tokens */
-                frag = (mca_btl_openib_frag_t*) (unsigned long) wc.wr_id;
+                frag = (mca_btl_openib_frag_t*) (uint64_t) wc.wr_id;
                 frag->base.des_cbfunc(&openib_btl->super, frag->endpoint, &frag->base, OMPI_SUCCESS);
 
                 /* if we have tokens, process pending sends */
@@ -715,14 +715,14 @@ int mca_btl_openib_component_progress()
 
             case IBV_WC_RDMA_READ: 
                                     
-                frag = (mca_btl_openib_frag_t*) (unsigned long) wc.wr_id;
+                frag = (mca_btl_openib_frag_t*) (uint64_t) wc.wr_id;
                 OPAL_THREAD_ADD32(&frag->endpoint->get_tokens, 1);
                 /* fall through */
 
             case IBV_WC_RDMA_WRITE: 
 
                 /* Process a completed write - returns tokens immediately */
-                frag = (mca_btl_openib_frag_t*) (unsigned long) wc.wr_id;
+                frag = (mca_btl_openib_frag_t*) (uint64_t) wc.wr_id;
                 endpoint = frag->endpoint;
                 frag->base.des_cbfunc(&openib_btl->super, frag->endpoint, &frag->base, OMPI_SUCCESS);
 
@@ -749,7 +749,7 @@ int mca_btl_openib_component_progress()
                 
             case IBV_WC_RECV: 
 
-                frag = (mca_btl_openib_frag_t*) (unsigned long) wc.wr_id;
+                frag = (mca_btl_openib_frag_t*) (uint64_t) wc.wr_id;
                 endpoint = (mca_btl_openib_endpoint_t*) frag->endpoint;
                 credits = frag->hdr->credits;
                 
@@ -769,7 +769,7 @@ int mca_btl_openib_component_progress()
 #endif
                     OPAL_THREAD_ADD32((int32_t*) &endpoint->rd_posted_lp, -1); 
                     MCA_BTL_OPENIB_ENDPOINT_POST_RR_LOW(((mca_btl_openib_frag_t*) (void*) 
-                                                         (unsigned long)wc.wr_id)->endpoint, 0); 
+                                                         (uint64_t)wc.wr_id)->endpoint, 0); 
 #ifdef OMPI_MCA_BTL_OPENIB_HAVE_SRQ
                 }
 #endif 

ompi/mca/btl/openib/btl_openib_endpoint.c

@@ -73,7 +73,7 @@ static inline int mca_btl_openib_endpoint_post_send(mca_btl_openib_module_t* ope
     
     struct ibv_qp* ib_qp; 
     struct ibv_send_wr* bad_wr; 
-    frag->sg_entry.addr = (uintptr_t) frag->hdr; 
+    frag->sg_entry.addr = (uint64_t) frag->hdr; 
     frag->wr_desc.sr_desc.opcode = IBV_WR_SEND; 
     
     if(frag->base.des_flags & MCA_BTL_DES_FLAGS_PRIORITY  && frag->size <= openib_btl->super.btl_eager_limit){ 
@@ -120,9 +120,7 @@ static inline int mca_btl_openib_endpoint_post_send(mca_btl_openib_module_t* ope
     
     
     frag->sg_entry.length = 
-        frag->segment.seg_len + 
+        frag->segment.seg_len + sizeof(mca_btl_openib_header_t);
-        ((unsigned char*) frag->segment.seg_addr.pval - (unsigned char*) frag->hdr);  
-
         
     if(frag->sg_entry.length <= openib_btl->ib_inline_max) { 
         frag->wr_desc.sr_desc.send_flags |= IBV_SEND_INLINE;
@@ -986,15 +984,12 @@ void mca_btl_openib_endpoint_send_credits(
     OPAL_THREAD_ADD32(credits, -frag->hdr->credits);
 
     frag->wr_desc.sr_desc.opcode = IBV_WR_SEND;
-    frag->sg_entry.length =
+    frag->sg_entry.length = sizeof(mca_btl_openib_header_t);
-        frag->segment.seg_len +
+                                                                                                                     
-        ((unsigned char*) frag->segment.seg_addr.pval - (unsigned char*) frag->hdr);
+    frag->wr_desc.sr_desc.send_flags = IBV_SEND_INLINE | 
+        IBV_SEND_SIGNALED; 
+    frag->sg_entry.addr = (uint64_t) frag->hdr; 
     
-    if(frag->sg_entry.length <= openib_btl->ib_inline_max) {
-        frag->wr_desc.sr_desc.send_flags |= IBV_SEND_INLINE;
-    } else {
-        frag->wr_desc.sr_desc.send_flags = IBV_SEND_SIGNALED;
-    }
 
     if(ibv_post_send(ib_qp,
                      &frag->wr_desc.sr_desc,

ompi/mca/btl/openib/btl_openib_frag.c

@@ -33,7 +33,7 @@ static void mca_btl_openib_frag_common_constructor( mca_btl_openib_frag_t* frag)
     frag->mr = registration->mr; 
     frag->segment.seg_len = frag->size;
     frag->segment.seg_key.key32[0] = (uint32_t) frag->mr->lkey; 
-    frag->sg_entry.addr = (uintptr_t) frag->hdr; 
+    frag->sg_entry.addr = (uint64_t) frag->hdr; 
     frag->sg_entry.length = frag->size; 
     frag->sg_entry.lkey = frag->mr->lkey; 
     frag->base.des_flags = 0; 
@@ -49,7 +49,7 @@ static void mca_btl_openib_send_frag_common_constructor(mca_btl_openib_frag_t* f
     frag->base.des_dst = NULL;
     frag->base.des_dst_cnt = 0;
     
-    frag->wr_desc.sr_desc.wr_id = (unsigned long) frag; 
+    frag->wr_desc.sr_desc.wr_id = (uint64_t) frag; 
     frag->wr_desc.sr_desc.sg_list = &frag->sg_entry; 
     frag->wr_desc.sr_desc.num_sge = 1; 
     frag->wr_desc.sr_desc.opcode = IBV_WR_SEND; 
@@ -66,7 +66,7 @@ static void mca_btl_openib_recv_frag_common_constructor(mca_btl_openib_frag_t* f
     frag->base.des_src = NULL;
     frag->base.des_src_cnt = 0;
    
-    frag->wr_desc.rd_desc.wr_id = (unsigned long) frag; 
+    frag->wr_desc.rd_desc.wr_id = (uint64_t) frag; 
     frag->wr_desc.rd_desc.sg_list = &frag->sg_entry; 
     frag->wr_desc.rd_desc.num_sge = 1; 
     frag->wr_desc.rd_desc.next = NULL;