ugni: fix eager get

This commit was SVN r26699.

ompi/mca/btl/ugni/Makefile.am

@@ -33,7 +33,6 @@ ugni_SOURCES = \
         btl_ugni_frag.h \
         btl_ugni_rdma.h \
         btl_ugni_send.c \
-        btl_ugni_sendi.c \
         btl_ugni_put.c \
         btl_ugni_get.c \
         btl_ugni.h \

ompi/mca/btl/ugni/btl_ugni_endpoint.c

@@ -145,6 +145,7 @@ static inline int mca_btl_ugni_ep_connect_finish (mca_btl_base_endpoint_t *ep) {
 
     rc = mca_btl_progress_send_wait_list (ep);
     if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+        ep->wait_listed = true;
         opal_list_append (&ep->btl->ep_wait_list, &ep->super);
     }
 

ompi/mca/btl/ugni/btl_ugni_frag.h

@@ -16,6 +16,13 @@
 #include "btl_ugni.h"
 #include "btl_ugni_endpoint.h"
 
+typedef struct mca_btl_ugni_segment_t {
+    mca_btl_base_segment_t base;
+    gni_mem_handle_t       memory_handle;
+    uint8_t                extra_bytes[3];
+    uint8_t                extra_byte_count;
+} mca_btl_ugni_segment_t;
+
 typedef struct mca_btl_ugni_send_frag_hdr_t {
     uint32_t lag;
 } mca_btl_ugni_send_frag_hdr_t;
@@ -26,14 +33,12 @@ typedef struct mca_btl_ugni_send_ex_frag_hdr_t {
 } mca_btl_ugni_send_ex_frag_hdr_t;
 
 typedef struct mca_btl_ugni_rdma_frag_hdr_t {
-    mca_btl_base_segment_t src_seg;
-    mca_btl_base_segment_t dst_seg;
     void *ctx;
 } mca_btl_ugni_rdma_frag_hdr_t;
 
 typedef struct mca_btl_ugni_eager_frag_hdr_t {
     mca_btl_ugni_send_frag_hdr_t send;
-    mca_btl_base_segment_t src_seg;
+    mca_btl_ugni_segment_t src_seg;
     void *ctx;
 } mca_btl_ugni_eager_frag_hdr_t;
 
@@ -61,13 +66,6 @@ struct mca_btl_ugni_base_frag_t;
 
 typedef void (*frag_cb_t) (struct mca_btl_ugni_base_frag_t *, int);
 
-typedef struct mca_btl_ugni_segment_t {
-    mca_btl_base_segment_t base;
-    gni_mem_handle_t       memory_handle;
-    uint8_t                extra_bytes[3];
-    uint8_t                extra_byte_count;
-} mca_btl_ugni_segment_t;
-
 typedef struct mca_btl_ugni_base_frag_t {
     mca_btl_base_descriptor_t    base;
     size_t                       hdr_size;

ompi/mca/btl/ugni/btl_ugni_get.c

@@ -43,7 +43,7 @@ int mca_btl_ugni_get (struct mca_btl_base_module_t *btl,
     }
 
     if (src_seg->extra_byte_count) {
-        memmove (dst_seg->base.seg_addr.pval + size, src_seg->extra_bytes, src_seg->extra_byte_count);
+        memmove ((char *) dst_seg->base.seg_addr.pval + size, src_seg->extra_bytes, src_seg->extra_byte_count);
         src_seg->base.seg_len = size;
         dst_seg->base.seg_len = size;
     }
@@ -53,19 +53,11 @@ int mca_btl_ugni_get (struct mca_btl_base_module_t *btl,
     return mca_btl_ugni_post (frag, true, dst_seg, src_seg);
 }
 
-static void mca_btl_ugni_frag_set_ownership (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
-                                             mca_btl_base_descriptor_t *desc, int rc) {
-    desc->des_flags |= MCA_BTL_DES_FLAGS_BTL_OWNERSHIP;
-}
-
 static void mca_btl_ugni_callback_rdma_complete (mca_btl_ugni_base_frag_t *frag, int rc)
 {
     BTL_VERBOSE(("rdma operation for rem_ctx %p complete", frag->hdr.rdma.ctx));
 
-    frag->base.des_cbfunc = mca_btl_ugni_frag_set_ownership;
+    /* tell peer the get is complete */
-    frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
-
-    /* tell peer the put is complete */
     rc = ompi_mca_btl_ugni_smsg_send (frag, &frag->hdr.rdma, sizeof (frag->hdr.rdma),
                                       NULL, 0, MCA_BTL_UGNI_TAG_RDMA_COMPLETE);
     if (OPAL_UNLIKELY(0 > rc)) {
@@ -81,13 +73,11 @@ static void mca_btl_ugni_callback_eager_get_retry (mca_btl_ugni_base_frag_t *fra
     (void) mca_btl_ugni_start_eager_get(frag->endpoint, frag->hdr.eager_ex, frag);
 }
 
-static void mca_btl_ugni_callback_eager_get (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint,
+static void mca_btl_ugni_callback_eager_get (mca_btl_ugni_base_frag_t *frag, int rc)
-                                             mca_btl_base_descriptor_t *desc, int rc)
 {
-    mca_btl_ugni_base_frag_t *frag = (mca_btl_ugni_base_frag_t *) desc;
     uint32_t len = frag->hdr.eager.send.lag & 0x00ffffff;
     uint8_t tag = frag->hdr.eager.send.lag >> 24;
-    size_t payload_len = frag->hdr.eager.src_seg.seg_len;
+    size_t payload_len = frag->hdr.eager.src_seg.base.seg_len;
     size_t hdr_len = len - payload_len;
     mca_btl_active_message_callback_t *reg;
     mca_btl_base_segment_t segs[2];
@@ -144,24 +134,26 @@ int mca_btl_ugni_start_eager_get (mca_btl_base_endpoint_t *ep,
         }
 
         frag->hdr.eager_ex = hdr;
+        frag->flags = 0;
 
-        frag->base.des_cbfunc = mca_btl_ugni_callback_eager_get;
+        frag->base.des_flags  = MCA_BTL_DES_FLAGS_BTL_OWNERSHIP;
-        frag->base.des_flags  = MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
 
-        frag->segments[1].base = hdr.eager.src_seg;
+        frag->segments[1] = hdr.eager.src_seg;
 
         /* increase size to a multiple of 4 bytes (required for get) */
         frag->segments[0].base.seg_len = frag->segments[1].base.seg_len =
-            (hdr.eager.src_seg.seg_len + 3) & ~3;
+            (hdr.eager.src_seg.base.seg_len + 3) & ~3;
 
-        rc = mca_btl_ugni_post (frag, true, frag->segments, frag->segments + 1);
+        frag->base.des_src = &frag->segments[1].base;
+
+        rc = mca_btl_ugni_post_wcb (frag, GNI_POST_RDMA_GET, frag->segments, frag->segments + 1,
+                                    mca_btl_ugni_callback_eager_get);
         if (OPAL_UNLIKELY(OMPI_SUCCESS == rc)) {
             return OMPI_SUCCESS;
         }
     } while (0);
 
     frag->cbfunc = mca_btl_ugni_callback_eager_get_retry;
-
     opal_list_append (&ep->btl->failed_frags, (opal_list_item_t *) frag);
 
     return rc;

ompi/mca/btl/ugni/btl_ugni_module.c

@@ -66,7 +66,7 @@ mca_btl_ugni_module_t mca_btl_ugni_module = {
         mca_btl_ugni_prepare_src,
         mca_btl_ugni_prepare_dst,
         mca_btl_ugni_send,
-        mca_btl_ugni_sendi,
+        NULL, /* sendi */
         mca_btl_ugni_put,
         mca_btl_ugni_get,
         NULL, /* mca_btl_base_dump, */

ompi/mca/btl/ugni/btl_ugni_prepare.h

@@ -153,7 +153,7 @@ mca_btl_ugni_prepare_src_send (struct mca_btl_base_module_t *btl,
 
     opal_convertor_get_current_pointer (convertor, &data_ptr);
 
-    send_in_place = !(*size >= 4096 || opal_convertor_need_buffers(convertor) ||
+    send_in_place = !(opal_convertor_need_buffers(convertor) ||
                       (use_eager_get && ((uintptr_t)data_ptr & 3)));
 
     if (send_in_place) {

ompi/mca/btl/ugni/btl_ugni_rdma.h

@@ -78,9 +78,9 @@ static inline int mca_btl_ugni_post_bte (mca_btl_ugni_base_frag_t *frag, gni_pos
     return OMPI_SUCCESS;
 }
 
-static inline int mca_btl_ugni_post (mca_btl_ugni_base_frag_t *frag, bool get, mca_btl_ugni_segment_t *lcl_seg,
+static inline int mca_btl_ugni_post_wcb (mca_btl_ugni_base_frag_t *frag, bool get, mca_btl_ugni_segment_t *lcl_seg,
-                                     mca_btl_ugni_segment_t *rem_seg) {
+                                         mca_btl_ugni_segment_t *rem_seg, frag_cb_t cb) {
-    frag->cbfunc = mca_btl_ugni_frag_complete;
+    frag->cbfunc = cb;
 
     if (frag->base.des_src->seg_len <= mca_btl_ugni_component.ugni_fma_limit) {
         return mca_btl_ugni_post_fma (frag, get ? GNI_POST_FMA_GET : GNI_POST_FMA_PUT, lcl_seg, rem_seg);
@@ -89,6 +89,11 @@ static inline int mca_btl_ugni_post (mca_btl_ugni_base_frag_t *frag, bool get, m
     return mca_btl_ugni_post_bte (frag, get ? GNI_POST_RDMA_GET : GNI_POST_RDMA_PUT, lcl_seg, rem_seg);
 }
 
+static inline int mca_btl_ugni_post (mca_btl_ugni_base_frag_t *frag, bool get, mca_btl_ugni_segment_t *lcl_seg,
+                                     mca_btl_ugni_segment_t *rem_seg) {
+    return mca_btl_ugni_post_wcb (frag, get, lcl_seg, rem_seg, mca_btl_ugni_frag_complete);
+}
+
 static inline void mca_btl_ugni_repost (mca_btl_ugni_base_frag_t *frag, int rc) {
     gni_return_t grc;
 

ompi/mca/btl/ugni/btl_ugni_sendi.c

@@ -1,81 +0,0 @@
-/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
-/*
- * Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights
- *                         reserved.
- * Copyright (c) 2011      UT-Battelle, LLC. All rights reserved.
- * $COPYRIGHT$
- *
- * Additional copyrights may follow
- *
- * $HEADER$
- */
-
-#include "btl_ugni.h"
-#include "btl_ugni_frag.h"
-#include "btl_ugni_smsg.h"
-#include "btl_ugni_prepare.h"
-
-int mca_btl_ugni_sendi (struct mca_btl_base_module_t *btl,
-                        struct mca_btl_base_endpoint_t *endpoint,
-                        struct opal_convertor_t *convertor,
-                        void *header, size_t header_size,
-                        size_t payload_size, uint8_t order,
-                        uint32_t flags, mca_btl_base_tag_t tag,
-                        mca_btl_base_descriptor_t **descriptor)
-{
-    const size_t length = header_size + payload_size;
-    mca_btl_ugni_base_frag_t *frag;
-    size_t max_data;
-    int rc;
-
-    assert (length <= btl->btl_eager_limit && !(flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK));
-
-    max_data = payload_size;
-    frag = (mca_btl_ugni_base_frag_t *)
-        mca_btl_ugni_prepare_src_send_buffered (btl, endpoint, convertor,
-                                                order, header_size, &max_data,
-                                                flags | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP);
-
-    if (OPAL_UNLIKELY(NULL == frag || OMPI_SUCCESS != mca_btl_ugni_check_endpoint_state (endpoint))) {
-        /* can't complete inline send if the endpoint is not already connected */
-        /* go ahead and start the connection */
-        if (NULL != frag) {
-            mca_btl_ugni_frag_return (frag);
-        }
-
-        *descriptor = NULL;
-
-        return !frag ? OMPI_ERR_OUT_OF_RESOURCE : OMPI_ERR_RESOURCE_BUSY;
-    }
-
-    assert (payload_size == max_data);
-
-    BTL_VERBOSE(("btl/ugni sending inline descriptor %p from %d -> %d. length = %u", (void *) frag,
-                 ORTE_PROC_MY_NAME->vpid, endpoint->common->ep_rem_id, (unsigned int) length));
-
-    frag->hdr.send.lag = (tag << 24) | length;
-
-    /* write match header (with MPI comm/tag/etc. info) */
-    memmove (frag->base.des_src[0].seg_addr.pval, header, header_size);
-
-    /* send message */
-    rc = mca_btl_ugni_send_frag (endpoint, frag);
-    if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
-        if (OPAL_UNLIKELY(OMPI_ERR_OUT_OF_RESOURCE == rc)) {
-            mca_btl_ugni_module_t *ugni_module = (mca_btl_ugni_module_t *) btl;
-
-            /* queue up request */
-            if (0 == opal_list_get_size (&endpoint->frag_wait_list)) {
-                opal_list_append (&ugni_module->ep_wait_list, &endpoint->super);
-            }
-            opal_list_append (&endpoint->frag_wait_list, (opal_list_item_t *) frag);
-            rc = OMPI_SUCCESS;
-        } else {
-            /* return this frag */
-            mca_btl_ugni_frag_return (frag);
-            *descriptor = NULL;
-        }
-    }
-
-    return rc;
-}

ompi/mca/btl/ugni/btl_ugni_smsg.h

@@ -92,8 +92,6 @@ static inline int ompi_mca_btl_ugni_smsg_send (mca_btl_ugni_base_frag_t *frag,
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
 
-    (void) mca_btl_ugni_progress_remote_smsg ((mca_btl_ugni_module_t *) frag->endpoint->btl);
-
     BTL_ERROR(("GNI_SmsgSendWTag failed with rc = %d. handle = %lu, hdr_len = %d, payload_len = %d",
                grc, (uintptr_t) frag->endpoint->smsg_ep_handle, (int) hdr_len, (int) payload_len));
 
@@ -109,7 +107,7 @@ static inline int mca_btl_ugni_send_frag (struct mca_btl_base_endpoint_t *btl_pe
                                             MCA_BTL_UGNI_TAG_SEND);
     }
 
-    frag->hdr.eager.src_seg = frag->segments[1].base;
+    frag->hdr.eager.src_seg = frag->segments[1];
     frag->hdr.eager.ctx     = (void *) frag;
 
     return ompi_mca_btl_ugni_smsg_send (frag, &frag->hdr.eager, frag->hdr_size,