* implement btl_put for Portals

This commit was SVN r7320.

ompi/mca/btl/portals/btl_portals.c

@@ -249,6 +249,10 @@ mca_btl_portals_alloc(struct mca_btl_base_module_t* btl_base,
     frag->base.des_src_cnt = 1;
     frag->base.des_flags = 0; 
 
+    /* can't setup off an alloc right now - we don't know how much the
+       caller will actually use */
+    frag->md_h = PTL_INVALID_HANDLE;
+
     return &frag->base;
 }
 
@@ -261,6 +265,10 @@ mca_btl_portals_free(struct mca_btl_base_module_t* btl_base,
 
     assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
 
+    if (frag->md_h != PTL_INVALID_HANDLE) {
+        PtlMDUnlink(frag->md_h);
+    }
+
     if (frag->size == 0) {
         OMPI_BTL_PORTALS_FRAG_RETURN_USER(&mca_btl_portals_module.super, frag); 
     } else if (frag->size == mca_btl_portals_module.super.btl_eager_limit){ 
@@ -344,12 +352,19 @@ mca_btl_portals_prepare_src(struct mca_btl_base_module_t* btl_base,
             frag->base.des_src_cnt = 1;
         }
 
+        /* clearly a send - delay setup of memory descriptor until send */
+        frag->md_h = PTL_INVALID_HANDLE;
+        
+
     } else {
         /* no need to pack - we can send directly out of the user's
            buffer.  If we have reserve space, use an eager fragment
            and give the caller the eager space as reserve.  If we have
            no reserve space needs, use a user frag */
         if (0 == reserve) {
+            ptl_md_t md;
+            ptl_handle_me_t me_h;
+
             /* user frags are always setup to use only one fragment */
             OMPI_BTL_PORTALS_FRAG_ALLOC_USER(&mca_btl_portals_module.super, frag, ret);
             if(NULL == frag){
@@ -363,8 +378,55 @@ mca_btl_portals_prepare_src(struct mca_btl_base_module_t* btl_base,
 
             frag->segments[0].seg_len = max_data;
             frag->segments[0].seg_addr.pval = iov.iov_base;
+            frag->segments[0].seg_key.key64 = OPAL_THREAD_ADD64(&(mca_btl_portals_module.portals_rdma_key), 1);
             frag->base.des_src_cnt = 1;
 
+            /* either a put or get.  figure out which later */
+
+            OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output,
+                                 "rdma src posted for frag 0x%x, callback 0x%x, bits %lld",
+                                 frag, frag->base.des_cbfunc, frag->segments[0].seg_key.key64));
+
+            /* create a match entry */
+            ret = PtlMEAttach(mca_btl_portals_module.portals_ni_h,
+                              OMPI_BTL_PORTALS_RDMA_TABLE_ID,
+                              *((mca_btl_base_endpoint_t*) peer),
+                              frag->segments[0].seg_key.key64, /* match */
+                              0, /* ignore */
+                              PTL_UNLINK,
+                              PTL_INS_AFTER,
+                              &me_h);
+            if (PTL_OK != ret) {
+                opal_output(mca_btl_portals_component.portals_output,
+                            "Error creating rdma src ME: %d", ret);
+                OMPI_BTL_PORTALS_FRAG_RETURN_USER(&mca_btl_portals_module.super, frag);
+                return NULL;
+            }
+
+            /* setup the memory descriptor.  RDMA should never need to be
+               retransmitted, so we set the threshold for the event it will
+               receive (PUT/GET START and END).  No need to track the unlinks
+               later :) */
+            md.start = frag->segments[0].seg_addr.pval;
+            md.length = frag->segments[0].seg_len;
+            md.threshold = PTL_MD_THRESH_INF;
+            md.max_size = 0;
+            md.options = PTL_MD_OP_PUT | PTL_MD_OP_GET | PTL_MD_EVENT_START_DISABLE;
+            md.user_ptr = frag; /* keep a pointer to ourselves */
+            md.eq_handle = mca_btl_portals_module.portals_eq_handles[OMPI_BTL_PORTALS_EQ];
+
+            ret = PtlMDAttach(me_h, 
+                              md,
+                              PTL_UNLINK,
+                              &(frag->md_h));
+            if (PTL_OK != ret) {
+                opal_output(mca_btl_portals_component.portals_output,
+                            "Error creating rdma src MD: %d", ret);
+                PtlMEUnlink(me_h);
+                OMPI_BTL_PORTALS_FRAG_RETURN_USER(&mca_btl_portals_module.super, frag);
+                return NULL;
+            }
+
         } else {
             OMPI_BTL_PORTALS_FRAG_ALLOC_EAGER(&mca_btl_portals_module, frag, ret);
             if (NULL == frag) {
@@ -391,6 +453,9 @@ mca_btl_portals_prepare_src(struct mca_btl_base_module_t* btl_base,
             frag->iov[0].iov_len = frag->segments[0].seg_len;
             frag->iov[1].iov_base = frag->segments[1].seg_addr.pval;
             frag->iov[1].iov_len = frag->segments[1].seg_len;
+
+            /* clearly a send - delay setup of memory descriptor until send */
+            frag->md_h = PTL_INVALID_HANDLE;
         }
     }
 
@@ -414,7 +479,6 @@ mca_btl_portals_prepare_dst(struct mca_btl_base_module_t* btl_base,
     mca_btl_portals_frag_t* frag;
     ptl_md_t md;
     ptl_handle_me_t me_h;
-    ptl_handle_md_t md_h;
     int ret;
 
     assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
@@ -461,7 +525,7 @@ mca_btl_portals_prepare_dst(struct mca_btl_base_module_t* btl_base,
        later :) */
     md.start = frag->segments[0].seg_addr.pval;
     md.length = frag->segments[0].seg_len;
-    md.threshold = 1; /* unlink after put */
+    md.threshold = PTL_MD_THRESH_INF;
     md.max_size = 0;
     md.options = PTL_MD_OP_PUT | PTL_MD_OP_GET | PTL_MD_EVENT_START_DISABLE;
     md.user_ptr = frag; /* keep a pointer to ourselves */
@@ -470,7 +534,7 @@ mca_btl_portals_prepare_dst(struct mca_btl_base_module_t* btl_base,
     ret = PtlMDAttach(me_h, 
                       md,
                       PTL_UNLINK,
-                      &md_h);
+                      &(frag->md_h));
     if (PTL_OK != ret) {
         opal_output(mca_btl_portals_component.portals_output,
                     "Error creating rdma dest MD: %d", ret);

ompi/mca/btl/portals/btl_portals_component.c

@@ -194,7 +194,7 @@ mca_btl_portals_component_open(void)
                            &dummy);
     mca_btl_portals_module.super.btl_bandwidth = dummy;
 
-    mca_btl_portals_module.super.btl_flags = MCA_BTL_FLAGS_PUT | MCA_BTL_FLAGS_SEND_INPLACE;
+    mca_btl_portals_module.super.btl_flags = MCA_BTL_FLAGS_PUT | MCA_BTL_FLAGS_GET | MCA_BTL_FLAGS_SEND_INPLACE;
 
     mca_btl_portals_module.portals_num_procs = 0;
     bzero(&(mca_btl_portals_module.portals_reg),

ompi/mca/btl/portals/btl_portals_frag.h

@@ -36,6 +36,8 @@ struct mca_btl_portals_frag_t {
     enum { mca_btl_portals_frag_type_send, 
            mca_btl_portals_frag_type_recv, 
            mca_btl_portals_frag_type_rdma} type;
+    /* handle to use for communication */
+    ptl_handle_md_t md_h;
     size_t size; 
 
 };

ompi/mca/btl/portals/btl_portals_rdma.c

@@ -28,11 +28,10 @@ mca_btl_portals_put(struct mca_btl_base_module_t* btl_base,
                     struct mca_btl_base_descriptor_t* descriptor)
 {
     mca_btl_portals_frag_t *frag = (mca_btl_portals_frag_t*) descriptor;
-    ptl_md_t md;
-    ptl_handle_md_t md_h;
     int ret;
 
     assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
+    assert(frag->md_h != PTL_INVALID_HANDLE);
 
     frag->endpoint = btl_peer;
     frag->hdr.tag = MCA_BTL_TAG_MAX;
@@ -41,26 +40,7 @@ mca_btl_portals_put(struct mca_btl_base_module_t* btl_base,
     /* setup the send */
     assert(1 == frag->base.des_src_cnt);
 
-    md.start = frag->segments[0].seg_addr.pval;
+    ret = PtlPut(frag->md_h,
-    md.length = frag->segments[0].seg_len;
-    md.threshold = 2; /* unlink after send & ack */
-    md.max_size = 0;
-    md.options = PTL_MD_EVENT_START_DISABLE;
-    md.user_ptr = frag; /* keep a pointer to ourselves */
-    md.eq_handle = mca_btl_portals_module.portals_eq_handles[OMPI_BTL_PORTALS_EQ];
-
-    /* make a free-floater */
-    ret = PtlMDBind(mca_btl_portals_module.portals_ni_h,
-                    md,
-                    PTL_UNLINK,
-                    &md_h);
-    if (ret != PTL_OK) {
-        opal_output(mca_btl_portals_component.portals_output,
-                    "PtlMDBind failed with error %d", ret);
-        return OMPI_ERROR;
-    }
-
-    ret = PtlPut(md_h,
                  PTL_ACK_REQ,
                  *((mca_btl_base_endpoint_t*) btl_peer),
                  OMPI_BTL_PORTALS_RDMA_TABLE_ID,
@@ -71,7 +51,7 @@ mca_btl_portals_put(struct mca_btl_base_module_t* btl_base,
     if (ret != PTL_OK) {
         opal_output(mca_btl_portals_component.portals_output,
                     "PtlPut failed with error %d", ret);
-        PtlMDUnlink(md_h);
+        PtlMDUnlink(frag->md_h);
         return OMPI_ERROR;
     }
 
@@ -85,37 +65,16 @@ mca_btl_portals_get(struct mca_btl_base_module_t* btl_base,
                     struct mca_btl_base_descriptor_t* descriptor)
 {
     mca_btl_portals_frag_t *frag = (mca_btl_portals_frag_t*) descriptor;
-    ptl_md_t md;
-    ptl_handle_md_t md_h;
     int ret;
 
     assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
+    assert(frag->md_h != PTL_INVALID_HANDLE);
 
     frag->endpoint = btl_peer;
     frag->hdr.tag = MCA_BTL_TAG_MAX;
     frag->type = mca_btl_portals_frag_type_rdma;
 
-    /* setup the get */
+    ret = PtlGet(frag->md_h,
-    md.start = frag->segments[0].seg_addr.pval;
-    md.length = frag->segments[0].seg_len;
-    md.threshold = 2; /* unlink after send & ack */
-    md.max_size = 0;
-    md.options = PTL_MD_EVENT_START_DISABLE;
-    md.user_ptr = frag; /* keep a pointer to ourselves */
-    md.eq_handle = mca_btl_portals_module.portals_eq_handles[OMPI_BTL_PORTALS_EQ];
-
-    /* make a free-floater */
-    ret = PtlMDBind(mca_btl_portals_module.portals_ni_h,
-                    md,
-                    PTL_UNLINK,
-                    &md_h);
-    if (ret != PTL_OK) {
-        opal_output(mca_btl_portals_component.portals_output,
-                    "PtlMDBind failed with error %d", ret);
-        return OMPI_ERROR;
-    }
-
-    ret = PtlGet(md_h,
                  *((mca_btl_base_endpoint_t*) btl_peer),
                  OMPI_BTL_PORTALS_RDMA_TABLE_ID,
                  0, /* ac_index - not used*/
@@ -124,7 +83,7 @@ mca_btl_portals_get(struct mca_btl_base_module_t* btl_base,
     if (ret != PTL_OK) {
         opal_output(mca_btl_portals_component.portals_output,
                     "PtlGet failed with error %d", ret);
-        PtlMDUnlink(md_h);
+        PtlMDUnlink(frag->md_h);
         return OMPI_ERROR;
     }
 

ompi/mca/btl/portals/btl_portals_send.c

@@ -36,6 +36,7 @@ mca_btl_portals_send(struct mca_btl_base_module_t* btl_base,
     int ret;
 
     assert(&mca_btl_portals_module == (mca_btl_portals_module_t*) btl_base);
+    assert(frag->md_h == PTL_INVALID_HANDLE);
 
     frag->endpoint = endpoint;
     frag->hdr.tag = tag;
@@ -57,8 +58,8 @@ mca_btl_portals_send(struct mca_btl_base_module_t* btl_base,
 
         ret = OMPI_SUCCESS;
     } else {
-        ptl_handle_md_t md_h;
         int ret;
+        ptl_handle_md_t md_h;
 
         /* setup the send */
         if (1 == frag->base.des_src_cnt) {
@@ -95,7 +96,7 @@ mca_btl_portals_send(struct mca_btl_base_module_t* btl_base,
                      frag->hdr.tag); /* hdr_data - tag */
         if (ret != PTL_OK) {
             opal_output(mca_btl_portals_component.portals_output,
-                        "PtlPut failed with error %d", ret);
+                        "send: PtlPut failed with error %d", ret);
             PtlMDUnlink(md_h);
             return OMPI_ERROR;
         }