post_send() function is called without endpoint lock held from explicit credits

update function so eager_rdma_remote.head have to be updated in a thread safe
manner.

This commit was SVN r15966.

ompi/mca/btl/openib/btl_openib.h

@@ -149,7 +149,7 @@ struct mca_btl_openib_component_t {
     uint32_t ib_service_level; 
     uint32_t use_eager_rdma;
     int32_t eager_rdma_threshold; /**< After this number of msg, use RDMA for short messages, always */
-    uint32_t eager_rdma_num;
+    int32_t eager_rdma_num;
     int32_t max_eager_rdma;
     uint32_t btls_per_lid;
     uint32_t max_lmc;

ompi/mca/btl/openib/btl_openib_eager_rdma.h

@@ -36,7 +36,7 @@ typedef struct mca_btl_openib_eager_rdma_local_t mca_btl_openib_eager_rdma_local
 struct mca_btl_openib_eager_rdma_remote_t {
 	ompi_ptr_t base; /**< address of remote buffer */
 	uint32_t rkey; /**< RKey for accessing remote buffer */
-	uint16_t head; /**< RDMA buffer to post to */
+	int32_t head; /**< RDMA buffer to post to */
 	int32_t tokens; /**< number of rdam tokens */
 #if OMPI_ENABLE_DEBUG
     uint32_t seq;
@@ -83,6 +83,18 @@ typedef struct mca_btl_openib_eager_rdma_remote_t mca_btl_openib_eager_rdma_remo
                                    mca_btl_openib_component.eager_rdma_num) \
                                 (I) = 0;                                    \
                         } while (0)
+#define MCA_BTL_OPENIB_RDMA_MOVE_INDEX(HEAD, OLD_HEAD)                      \
+    do {                                                                    \
+        int32_t new_head;                                                   \
+        do {                                                                \
+            OLD_HEAD = HEAD;                                                \
+            new_head = OLD_HEAD + 1;                                        \
+            if(new_head == mca_btl_openib_component.eager_rdma_num)         \
+                new_head = 0;                                               \
+        } while(!OPAL_ATOMIC_CMPSET_32(&HEAD, OLD_HEAD, new_head));         \
+    } while(0)
+
+
 #if defined(c_plusplus) || defined(__cplusplus)
 }
 #endif

ompi/mca/btl/openib/btl_openib_endpoint.c

@@ -68,6 +68,7 @@ static int post_send(mca_btl_openib_module_t *openib_btl,
         BTL_OPENIB_HEADER_HTON((*(frag->hdr)));
 
     if(do_rdma) {
+        int32_t head;
         mca_btl_openib_footer_t* ftr =
             (mca_btl_openib_footer_t*)(((char*)frag->segment.seg_addr.pval) +
                                        frag->segment.seg_len);
@@ -83,15 +84,14 @@ static int post_send(mca_btl_openib_module_t *openib_btl,
             BTL_OPENIB_FOOTER_HTON((*ftr));
 
         frag->wr_desc.sr_desc.wr.rdma.rkey = endpoint->eager_rdma_remote.rkey;
+        MCA_BTL_OPENIB_RDMA_MOVE_INDEX(endpoint->eager_rdma_remote.head, head);
         frag->wr_desc.sr_desc.wr.rdma.remote_addr =
             endpoint->eager_rdma_remote.base.lval +
-            endpoint->eager_rdma_remote.head *
-            openib_btl->eager_rdma_frag_size +
+            head * openib_btl->eager_rdma_frag_size +
             sizeof(mca_btl_openib_header_t) +
             mca_btl_openib_component.eager_limit +
             sizeof(mca_btl_openib_footer_t);
         frag->wr_desc.sr_desc.wr.rdma.remote_addr -= frag->sg_entry.length;
-        MCA_BTL_OPENIB_RDMA_NEXT_INDEX(endpoint->eager_rdma_remote.head);
     } else {
         if(MCA_BTL_OPENIB_SRQ_QP == endpoint->qps[qp].qp_type) { 
             frag->wr_desc.sr_desc.opcode = IBV_WR_SEND_WITH_IMM;
@@ -708,7 +708,7 @@ void mca_btl_openib_endpoint_connect_eager_rdma(
     mca_btl_openib_module_t* openib_btl = endpoint->endpoint_btl;
     char *buf;
     mca_btl_openib_recv_frag_t *headers_buf;
-    unsigned int i;
+    int i;
     orte_std_cntr_t index;
 
     /* Set local rdma pointer to 1 temporarily so other threads will not try

ompi/mca/btl/openib/btl_openib_mca.c

@@ -383,7 +383,7 @@ int btl_openib_register_mca_params(void)
                   "for small messages"
                   "(must be >= 1)",
                   16, &ival, REGINT_GE_ONE));
-    mca_btl_openib_component.eager_rdma_num = (uint32_t) (ival + 1);
+    mca_btl_openib_component.eager_rdma_num = (int32_t) (ival + 1);
 
     CHECK(reg_int("btls_per_lid", "Number of BTLs to create for each "
                   "InfiniBand LID "