Add type to frag. Sometimes we need to know that a frag is from short rdma area.

I used hack for this that doesn't work for mvapi, so changing it to something more sane.

This commit was SVN r9477.

ompi/mca/btl/openib/btl_openib_component.c

@@ -653,10 +653,8 @@ int mca_btl_openib_handle_incoming_hp(
     /* repost receive descriptors */
 #ifdef OMPI_MCA_BTL_OPENIB_HAVE_SRQ
     if(mca_btl_openib_component.use_srq) { 
-        if (!MCA_BTL_OPENIB_RDMA_FRAG(frag)) {
+        OPAL_THREAD_ADD32((int32_t*) &openib_btl->srd_posted_hp, -1); 
-            OPAL_THREAD_ADD32((int32_t*) &openib_btl->srd_posted_hp, -1); 
+        MCA_BTL_OPENIB_POST_SRR_HIGH(openib_btl, 0); 
-            MCA_BTL_OPENIB_POST_SRR_HIGH(openib_btl, 0); 
-        }
     } else { 
 #endif
         if (!MCA_BTL_OPENIB_RDMA_FRAG(frag)) {

ompi/mca/btl/openib/btl_openib_eager_rdma.h

@@ -43,10 +43,8 @@ struct mca_btl_openib_eager_rdma_remote_t {
 };
 typedef struct mca_btl_openib_eager_rdma_remote_t mca_btl_openib_eager_rdma_remote_t;
 
-#define MCA_BTL_OPENIB_RDMA_FRAG(F)                           \
+#define MCA_BTL_OPENIB_RDMA_FRAG(F) \
-            ((F)->endpoint &&                                 \
+    ((F)->type == MCA_BTL_OPENIB_FRAG_EAGER_RDMA)
-             (F)->endpoint->eager_rdma_local.reg &&           \
-             (F)->mr == (F)->endpoint->eager_rdma_local.reg->mr)
 
 #define EAGER_RDMA_BUFFER_REMOTE (0)
 #define EAGER_RDMA_BUFFER_LOCAL (0xff)

ompi/mca/btl/openib/btl_openib_endpoint.c

@@ -1238,6 +1238,7 @@ void mca_btl_openib_endpoint_connect_eager_rdma(
         item->user_data = endpoint->eager_rdma_local.reg;
         OBJ_CONSTRUCT(item, mca_btl_openib_recv_frag_eager_t);
         ((mca_btl_openib_frag_t*)item)->endpoint = endpoint;
+        ((mca_btl_openib_frag_t*)item)->type = MCA_BTL_OPENIB_FRAG_EAGER_RDMA;
     }
 
     OPAL_THREAD_LOCK(&openib_btl->eager_rdma_lock);

ompi/mca/btl/openib/btl_openib_frag.c

@@ -75,6 +75,7 @@ static void mca_btl_openib_recv_frag_common_constructor(mca_btl_openib_frag_t* f
 static void mca_btl_openib_send_frag_eager_constructor(mca_btl_openib_frag_t* frag) 
 { 
     frag->size = mca_btl_openib_component.eager_limit;  
+    frag->type = MCA_BTL_OPENIB_FRAG_EAGER;
     mca_btl_openib_send_frag_common_constructor(frag); 
 }
 
@@ -82,12 +83,14 @@ static void mca_btl_openib_send_frag_eager_constructor(mca_btl_openib_frag_t* fr
 static void mca_btl_openib_send_frag_max_constructor(mca_btl_openib_frag_t* frag) 
 { 
     frag->size = mca_btl_openib_component.max_send_size; 
+    frag->type = MCA_BTL_OPENIB_FRAG_MAX;
     mca_btl_openib_send_frag_common_constructor(frag); 
 }
 
 static void mca_btl_openib_recv_frag_max_constructor(mca_btl_openib_frag_t* frag) 
 {
     frag->size = mca_btl_openib_component.max_send_size; 
+    frag->type = MCA_BTL_OPENIB_FRAG_MAX;
     mca_btl_openib_recv_frag_common_constructor(frag); 
 }
 
@@ -95,6 +98,7 @@ static void mca_btl_openib_recv_frag_max_constructor(mca_btl_openib_frag_t* frag
 static void mca_btl_openib_recv_frag_eager_constructor(mca_btl_openib_frag_t* frag) 
 {
     frag->size = mca_btl_openib_component.eager_limit; 
+    frag->type = MCA_BTL_OPENIB_FRAG_EAGER;
     mca_btl_openib_recv_frag_common_constructor(frag); 
     frag->ftr = (mca_btl_openib_footer_t*)((char*)frag->segment.seg_addr.pval 
 		    + frag->size);
@@ -104,6 +108,7 @@ static void mca_btl_openib_recv_frag_eager_constructor(mca_btl_openib_frag_t* fr
 static void mca_btl_openib_send_frag_frag_constructor(mca_btl_openib_frag_t* frag) 
 { 
     frag->size = 0; 
+    frag->type = MCA_BTL_OPENIB_FRAG_FRAG;
     mca_btl_openib_send_frag_common_constructor(frag); 
 }
 

ompi/mca/btl/openib/btl_openib_frag.h

@@ -67,6 +67,14 @@ struct mca_btl_openib_eager_rdma_header_t {
 };
 typedef struct mca_btl_openib_eager_rdma_header_t mca_btl_openib_eager_rdma_header_t;
 
+enum mca_btl_openib_frag_type_t {
+    MCA_BTL_OPENIB_FRAG_EAGER,
+    MCA_BTL_OPENIB_FRAG_MAX,
+    MCA_BTL_OPENIB_FRAG_FRAG,
+    MCA_BTL_OPENIB_FRAG_EAGER_RDMA
+};
+typedef enum mca_btl_openib_frag_type_t mca_btl_openib_frag_type_t;
+
 /**
  * IB send fragment derived type.
  */
@@ -75,7 +83,8 @@ struct mca_btl_openib_frag_t {
     mca_btl_base_segment_t segment; 
     struct mca_btl_base_endpoint_t *endpoint; 
     size_t size; 
-    int rc; 
+    int rc;
+    mca_btl_openib_frag_type_t type; 
     union{ 
         struct ibv_recv_wr rd_desc; 
         struct ibv_send_wr sr_desc;