ugni: cleanup frag setup code

This commit was SVN r26316.

ompi/mca/btl/ugni/Makefile.am

@@ -38,7 +38,8 @@ ugni_SOURCES = \
         btl_ugni_get.c \
         btl_ugni.h \
         btl_ugni_smsg.h \
-        btl_ugni_smsg.c
+        btl_ugni_smsg.c \
+        btl_ugni_prepare.h
 
 mcacomponentdir = $(pkglibdir)
 mcacomponent_LTLIBRARIES = $(component_install)

ompi/mca/btl/ugni/btl_ugni_frag.h

@@ -60,7 +60,7 @@ typedef struct mca_btl_ugni_base_frag_t {
     mca_btl_ugni_reg_t *registration;
     ompi_free_list_t *my_list;
     uint32_t msg_id;
-
+    bool is_buffered;
     void (*cbfunc) (struct mca_btl_ugni_base_frag_t*, int);
 } mca_btl_ugni_base_frag_t;
 
@@ -109,7 +109,7 @@ static inline int mca_btl_ugni_frag_return (mca_btl_ugni_base_frag_t *frag)
 
 static inline void mca_btl_ugni_frag_complete (mca_btl_ugni_base_frag_t *frag, int rc) {
     /* call callback if specified */
-    if (frag->base.des_flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK) {
+    if (OPAL_LIKELY(frag->base.des_flags & MCA_BTL_DES_SEND_ALWAYS_CALLBACK)) {
         frag->base.des_cbfunc(&frag->endpoint->btl->super, frag->endpoint, &frag->base, rc);
     }
 

ompi/mca/btl/ugni/btl_ugni_module.c

@@ -26,6 +26,7 @@
 #include "btl_ugni_frag.h"
 #include "btl_ugni_endpoint.h"
 #include "btl_ugni_smsg.h"
+#include "btl_ugni_prepare.h"
 
 #include <unistd.h>
 #include <sys/types.h>
@@ -274,101 +275,6 @@ mca_btl_ugni_free (struct mca_btl_base_module_t *btl,
     return mca_btl_ugni_frag_return ((mca_btl_ugni_base_frag_t *) des);
 }
 
-static inline struct mca_btl_base_descriptor_t *
-mca_btl_ugni_prepare_src_send (struct mca_btl_base_module_t *btl,
-                          mca_btl_base_endpoint_t *endpoint,
-                          struct opal_convertor_t *convertor,
-                          uint8_t order, size_t reserve, size_t *size,
-                          uint32_t flags)
-{
-    bool use_eager_get = (*size + reserve) > mca_btl_ugni_component.smsg_max_data;
-    mca_mpool_base_registration_t *registration = NULL;
-    mca_btl_ugni_base_frag_t *frag = NULL;
-    bool send_in_place;
-    void *data_ptr;
-    int rc;
-
-    opal_convertor_get_current_pointer (convertor, &data_ptr);
-
-    send_in_place = !(opal_convertor_need_buffers(convertor) ||
-                      (use_eager_get && ((uintptr_t)data_ptr & 3)));
-
-    if (OPAL_UNLIKELY(*size > btl->btl_eager_limit)) {
-        *size = btl->btl_eager_limit;
-    }
-
-    if (OPAL_LIKELY(send_in_place)) {
-        (void) MCA_BTL_UGNI_FRAG_ALLOC_RDMA(endpoint, frag);
-
-        if (OPAL_UNLIKELY(NULL == frag)) {
-            return NULL;
-        }
-
-        BTL_VERBOSE(("preparing src for send fragment. size = %u",
-                     (unsigned int)(*size + reserve)));
-
-        if (OPAL_UNLIKELY(true == use_eager_get)) {
-            rc = btl->btl_mpool->mpool_register(btl->btl_mpool, data_ptr,
-                                                *size, 0, &registration);
-            if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
-                mca_btl_ugni_frag_return (frag);
-                return NULL;
-            }
-
-            frag->registration = (mca_btl_ugni_reg_t *) registration;
-            memcpy ((void *) frag->segments[1].seg_key.key64,
-                    (void *)&((mca_btl_ugni_reg_t *)registration)->memory_hdl,
-                    sizeof (((mca_btl_ugni_reg_t *)registration)->memory_hdl));        
-        }
-    } else {
-        uint32_t iov_count = 1;
-        struct iovec iov;
-
-        /* buffer the user's data */
-        if (OPAL_LIKELY(!use_eager_get)) {
-            (void) MCA_BTL_UGNI_FRAG_ALLOC_SMSG(endpoint, frag);
-        } else {
-            (void) MCA_BTL_UGNI_FRAG_ALLOC_EAGER_SEND(endpoint, frag);
-        }
-
-        if (OPAL_UNLIKELY(NULL == frag)) {
-            return NULL;
-        }
-
-        data_ptr = frag->base.super.ptr;
-
-        iov.iov_len  = *size;
-        iov.iov_base = (IOVBASE_TYPE *) data_ptr;
-
-        rc = opal_convertor_pack (convertor, &iov, &iov_count, size);
-        if (OPAL_UNLIKELY(rc < 0)) {
-            mca_btl_ugni_frag_return (frag);
-            return NULL;
-        }
-
-        if (true == use_eager_get) {
-            registration = frag->base.super.registration;
-            memcpy ((void *) frag->segments[1].seg_key.key64,
-                    (void *)&((mca_btl_ugni_reg_t *)registration)->memory_hdl,
-                    sizeof (((mca_btl_ugni_reg_t *)registration)->memory_hdl));        
-        }
-    }
-
-    frag->hdr_size = reserve + (use_eager_get ? sizeof (frag->hdr.eager) : sizeof (frag->hdr.send));
-    frag->segments[0].seg_addr.pval = use_eager_get ? frag->hdr.eager_ex.pml_header : frag->hdr.send_ex.pml_header;
-    frag->segments[0].seg_len       = reserve;
-
-    frag->segments[1].seg_addr.pval = data_ptr;
-    frag->segments[1].seg_len       = *size;
-
-    frag->base.des_src     = frag->segments;
-    frag->base.des_src_cnt = 2;
-    frag->base.order       = order;
-    frag->base.des_flags   = flags;
-
-    return &frag->base;
-}
-
 static struct mca_btl_base_descriptor_t *
 mca_btl_ugni_prepare_src (struct mca_btl_base_module_t *btl,
                           mca_btl_base_endpoint_t *endpoint,
@@ -384,50 +290,10 @@ mca_btl_ugni_prepare_src (struct mca_btl_base_module_t *btl,
     if (OPAL_LIKELY(reserve)) {
         return mca_btl_ugni_prepare_src_send (btl, endpoint, convertor,
                                               order, reserve, size, flags);
-    }
-
-    opal_convertor_get_current_pointer (convertor, &data_ptr);
-
-    (void) MCA_BTL_UGNI_FRAG_ALLOC_RDMA(endpoint, frag);
-    if (OPAL_UNLIKELY(NULL == frag)) {
-        return NULL;
-    }
-
-    /*
-     * For medium message use FMA protocols and for large message
-     * use BTE protocols
-     */
-    /* No need to register while using FMA Put (registration is
-     * non-null in get-- is this always true?) */
-    if (*size >= mca_btl_ugni_component.ugni_fma_limit || (flags & MCA_BTL_DES_FLAGS_GET)) {
-        if (NULL == registration) {
-            rc = btl->btl_mpool->mpool_register(btl->btl_mpool, data_ptr,
-                                                *size, 0, &registration);
-            if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
-                mca_btl_ugni_frag_return (frag);
-                return NULL;
-            }
-
-            frag->registration = (mca_btl_ugni_reg_t *) registration;
-        }
-
-        memcpy ((void *) frag->segments[0].seg_key.key64,
-                (void *)&((mca_btl_ugni_reg_t *)registration)->memory_hdl,
-                sizeof (((mca_btl_ugni_reg_t *)registration)->memory_hdl));
     } else {
-        memset ((void *) frag->segments[0].seg_key.key64, 0,
-                sizeof (frag->segments[0].seg_key.key64));
+        return mca_btl_ugni_prepare_src_rdma (btl, endpoint, registration,
+                                              convertor, order, size, flags);
     }
-
-    frag->segments[0].seg_addr.pval = data_ptr;
-    frag->segments[0].seg_len = reserve + *size;
-
-    frag->base.des_src     = frag->segments;
-    frag->base.des_src_cnt = 1;
-    frag->base.order       = order;
-    frag->base.des_flags   = flags;
-
-    return &frag->base;
 }
 
 static mca_btl_base_descriptor_t *

ompi/mca/btl/ugni/btl_ugni_prepare.h

@@ -0,0 +1,225 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights
+ *                         reserved.
+ * Copyright (c) 2011      UT-Battelle, LLC. All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#if !defined(MCA_BTL_UGNI_PREPARE_H)
+#define MCA_BTL_UGNI_PREPARE_H
+
+#include "ompi_config.h"
+
+#include "btl_ugni.h"
+#include "btl_ugni_frag.h"
+
+static inline struct mca_btl_base_descriptor_t *
+mca_btl_ugni_prepare_src_send_inplace (struct mca_btl_base_module_t *btl,
+				       mca_btl_base_endpoint_t *endpoint,
+				       struct opal_convertor_t *convertor,
+				       uint8_t order, size_t reserve, size_t *size,
+				       uint32_t flags)
+{
+    bool use_eager_get = (*size + reserve) > mca_btl_ugni_component.smsg_max_data;
+    mca_btl_ugni_base_frag_t *frag = NULL;
+    mca_btl_ugni_reg_t *registration = NULL;
+    void *data_ptr;
+    int rc;
+
+    opal_convertor_get_current_pointer (convertor, &data_ptr);
+
+    (void) MCA_BTL_UGNI_FRAG_ALLOC_RDMA(endpoint, frag);
+
+    if (OPAL_UNLIKELY(NULL == frag)) {
+        return NULL;
+    }
+
+    frag->is_buffered = false;
+
+    BTL_VERBOSE(("preparing src for send fragment. size = %u",
+                 (unsigned int)(*size + reserve)));
+
+    if (OPAL_UNLIKELY(true == use_eager_get)) {
+        rc = btl->btl_mpool->mpool_register(btl->btl_mpool, data_ptr, *size, 0,
+                                            (mca_mpool_base_registration_t **)&registration);
+        if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+            mca_btl_ugni_frag_return (frag);
+            return NULL;
+        }
+
+        frag->registration = registration;
+        memcpy ((void *) frag->segments[1].seg_key.key64,
+                (void *)&registration->memory_hdl,
+                sizeof (registration->memory_hdl));        
+
+        frag->hdr_size = reserve + sizeof (frag->hdr.eager);
+        frag->segments[0].seg_addr.pval = frag->hdr.eager_ex.pml_header;
+    } else {
+        frag->hdr_size = reserve + sizeof (frag->hdr.send);
+        frag->segments[0].seg_addr.pval = frag->hdr.send_ex.pml_header;
+    }
+
+    frag->segments[0].seg_len       = reserve;
+
+    frag->segments[1].seg_addr.pval = data_ptr;
+    frag->segments[1].seg_len       = *size;
+
+    frag->base.des_src     = frag->segments;
+    frag->base.des_src_cnt = 2;
+    frag->base.order       = order;
+    frag->base.des_flags   = flags;
+
+    return &frag->base;
+}
+
+static inline struct mca_btl_base_descriptor_t *
+mca_btl_ugni_prepare_src_send_buffered (struct mca_btl_base_module_t *btl,
+                                        mca_btl_base_endpoint_t *endpoint,
+                                        struct opal_convertor_t *convertor,
+                                        uint8_t order, size_t reserve, size_t *size,
+                                        uint32_t flags)
+{
+    bool use_eager_get = (*size + reserve) > mca_btl_ugni_component.smsg_max_data;
+    mca_btl_ugni_reg_t *registration = NULL;
+    mca_btl_ugni_base_frag_t *frag = NULL;
+    uint32_t iov_count = 1;
+    struct iovec iov;
+    int rc;
+
+    /* buffer user data */
+    if (OPAL_LIKELY(!use_eager_get)) {
+        (void) MCA_BTL_UGNI_FRAG_ALLOC_SMSG(endpoint, frag);
+    } else {
+        (void) MCA_BTL_UGNI_FRAG_ALLOC_EAGER_SEND(endpoint, frag);
+    }
+
+    if (OPAL_UNLIKELY(NULL == frag)) {
+        return NULL;
+    }
+
+    frag->is_buffered = true;
+
+    if (OPAL_UNLIKELY(true == use_eager_get)) {
+        registration = (mca_btl_ugni_reg_t *) frag->base.super.registration;
+        memcpy ((void *) frag->segments[1].seg_key.key64,
+                (void *)&registration->memory_hdl,
+                sizeof (registration->memory_hdl));
+
+        frag->hdr_size = reserve + sizeof (frag->hdr.eager);
+        frag->segments[0].seg_addr.pval = frag->hdr.eager_ex.pml_header;
+    } else {
+        frag->hdr_size = reserve + sizeof (frag->hdr.send);
+        frag->segments[0].seg_addr.pval = frag->hdr.send_ex.pml_header;
+    }
+
+    if (*size) {
+        iov.iov_len  = *size;
+        iov.iov_base = (IOVBASE_TYPE *) frag->base.super.ptr;
+
+        rc = opal_convertor_pack (convertor, &iov, &iov_count, size);
+        if (OPAL_UNLIKELY(rc < 0)) {
+            mca_btl_ugni_frag_return (frag);
+            return NULL;
+        }
+    }
+
+    frag->segments[0].seg_len       = reserve;
+
+    frag->segments[1].seg_addr.pval = *size ? frag->base.super.ptr : NULL;
+    frag->segments[1].seg_len       = *size;
+
+    frag->base.des_src     = frag->segments;
+    frag->base.des_src_cnt = 2;
+    frag->base.order       = order;
+    frag->base.des_flags   = flags;
+
+    return &frag->base;
+}
+
+static inline struct mca_btl_base_descriptor_t *
+mca_btl_ugni_prepare_src_send (struct mca_btl_base_module_t *btl,
+			       mca_btl_base_endpoint_t *endpoint,
+			       struct opal_convertor_t *convertor,
+			       uint8_t order, size_t reserve, size_t *size,
+			       uint32_t flags)
+{
+    bool use_eager_get = (*size + reserve) > mca_btl_ugni_component.smsg_max_data;
+    bool send_in_place;
+    void *data_ptr;
+
+    opal_convertor_get_current_pointer (convertor, &data_ptr);
+
+    send_in_place = !(*size >= 4096 || opal_convertor_need_buffers(convertor) ||
+                      (use_eager_get && ((uintptr_t)data_ptr & 3)));
+
+    if (send_in_place) {
+        return mca_btl_ugni_prepare_src_send_inplace (btl, endpoint, convertor, order,
+                                                      reserve, size, flags);
+    } else {
+        return mca_btl_ugni_prepare_src_send_buffered (btl, endpoint, convertor, order,
+                                                       reserve, size, flags);
+    }
+}
+
+static inline struct mca_btl_base_descriptor_t *
+mca_btl_ugni_prepare_src_rdma (struct mca_btl_base_module_t *btl,
+			       mca_btl_base_endpoint_t *endpoint,
+                               mca_mpool_base_registration_t *registration,
+			       struct opal_convertor_t *convertor,
+			       uint8_t order, size_t *size,
+			       uint32_t flags)
+{
+    mca_btl_ugni_base_frag_t *frag;
+    void *data_ptr;
+    int rc;
+
+    opal_convertor_get_current_pointer (convertor, &data_ptr);
+
+    (void) MCA_BTL_UGNI_FRAG_ALLOC_RDMA(endpoint, frag);
+    if (OPAL_UNLIKELY(NULL == frag)) {
+        return NULL;
+    }
+
+    /*
+     * For medium message use FMA protocols and for large message
+     * use BTE protocols
+     */
+    /* No need to register while using FMA Put (registration is
+     * non-null in get-- is this always true?) */
+    if (*size >= mca_btl_ugni_component.ugni_fma_limit || (flags & MCA_BTL_DES_FLAGS_GET)) {
+        if (NULL == registration) {
+            rc = btl->btl_mpool->mpool_register(btl->btl_mpool, data_ptr, *size, 0,
+                                                (mca_mpool_base_registration_t **) &registration);
+            if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
+                mca_btl_ugni_frag_return (frag);
+                return NULL;
+            }
+
+            frag->registration = (mca_btl_ugni_reg_t *) registration;
+        }
+
+        memcpy ((void *) frag->segments[0].seg_key.key64,
+                (void *)&((mca_btl_ugni_reg_t *)registration)->memory_hdl,
+                sizeof (((mca_btl_ugni_reg_t *)registration)->memory_hdl));
+    } else {
+        memset ((void *) frag->segments[0].seg_key.key64, 0,
+                sizeof (frag->segments[0].seg_key.key64));
+    }
+
+    frag->segments[0].seg_addr.pval = data_ptr;
+    frag->segments[0].seg_len       = *size;
+
+    frag->base.des_src     = frag->segments;
+    frag->base.des_src_cnt = 1;
+    frag->base.order       = order;
+    frag->base.des_flags   = flags;
+
+    return &frag->base;
+}
+
+#endif

ompi/mca/btl/ugni/btl_ugni_send.c

@@ -35,7 +35,10 @@ int mca_btl_ugni_send (struct mca_btl_base_module_t *btl,
         frag->hdr.eager.ctx     = (void *) frag;
     }
 
-    frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
+    if (false == frag->is_buffered && (frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP)) {
+        frag->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
+    }
+
     frag->endpoint = btl_peer;
 
     rc = mca_btl_ugni_check_endpoint_state (btl_peer);
@@ -51,7 +54,9 @@ int mca_btl_ugni_send (struct mca_btl_base_module_t *btl,
         return rc;
     }
 
-    (void) mca_btl_ugni_progress_local_smsg ((mca_btl_ugni_module_t *) btl);
+    if (frag->is_buffered && (frag->base.des_flags & MCA_BTL_DES_FLAGS_BTL_OWNERSHIP)) {
+        return 1;
+    }
 
     return OMPI_SUCCESS;
 }

ompi/mca/btl/ugni/btl_ugni_sendi.c

@@ -13,6 +13,7 @@
 #include "btl_ugni.h"
 #include "btl_ugni_frag.h"
 #include "btl_ugni_smsg.h"
+#include "btl_ugni_prepare.h"
 
 int mca_btl_ugni_sendi (struct mca_btl_base_module_t *btl,
                         struct mca_btl_base_endpoint_t *endpoint,
@@ -39,53 +40,38 @@ int mca_btl_ugni_sendi (struct mca_btl_base_module_t *btl,
         return OMPI_ERR_RESOURCE_BUSY;
     }
 
-    if (OPAL_LIKELY(length <= mca_btl_ugni_component.smsg_max_data)) {
-        (void) MCA_BTL_UGNI_FRAG_ALLOC_SMSG(endpoint, frag);
-        frag->segments[0].seg_addr.pval = frag->base.super.ptr;
-    } else {
-        (void) MCA_BTL_UGNI_FRAG_ALLOC_EAGER_SEND(endpoint, frag);
-    }
-
+    max_data = payload_size;
+    frag = (mca_btl_ugni_base_frag_t *) mca_btl_ugni_prepare_src_send_buffered (btl, endpoint,
+                                                                                convertor,
+                                                                                order, header_size,
+                                                                                &max_data, flags);
     if (OPAL_UNLIKELY(NULL == frag)) {
         *descriptor = NULL;
         return OMPI_ERR_OUT_OF_RESOURCE;
     }
 
+    assert (payload_size == max_data);
+
     BTL_VERBOSE(("btl/ugni sending inline descriptor %p from %d -> %d. length = %u", (void *) frag,
                  ORTE_PROC_MY_NAME->vpid, endpoint->common->ep_rem_id, (unsigned int) length));
 
-    frag->base.des_cbfunc = NULL;
     frag->base.des_flags  = flags | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP;
 
     frag->hdr.send.lag = (tag << 24) | length;
 
     /* write match header (with MPI comm/tag/etc. info) */
-    memmove (frag->segments[0].seg_addr.pval, header, header_size);
-
-    /*
-      We can add MEMCHECKER calls before and after the packing.
-    */
-    if (payload_size) {
-        /* pack the data into the supplied buffer */
-        iov.iov_base = (IOVBASE_TYPE *)((uintptr_t)frag->segments[0].seg_addr.pval + header_size);
-        iov.iov_len  = max_data = payload_size;
-
-        (void) opal_convertor_pack (convertor, &iov, &iov_count, &max_data);
-
-        assert (max_data == payload_size);
-    }
+    memmove (frag->base.des_src[0].seg_addr.pval, header, header_size);
 
     /* send message */
-    if (OPAL_LIKELY(length <= mca_btl_ugni_component.smsg_max_data)) {
-        rc = ompi_mca_btl_ugni_smsg_send (frag, false, &frag->hdr.send, sizeof (frag->hdr.send),
-                                          frag->segments[0].seg_addr.pval, length, MCA_BTL_UGNI_TAG_SEND);
+    if (OPAL_LIKELY(length <= mca_btl_ugni_component.smsg_max_data)) { 
+        rc = ompi_mca_btl_ugni_smsg_send (frag, false, &frag->hdr.send_ex, frag->hdr_size,
+                                          frag->segments[1].seg_addr.pval, frag->segments[1].seg_len,
+                                          MCA_BTL_UGNI_TAG_SEND);
     } else {
-        frag->segments[0].seg_len = length;
+        frag->hdr.eager.src_seg = frag->segments[1];
+        frag->hdr.eager.ctx     = (void *) frag;
 
-        frag->hdr.eager.src_seg = frag->segments[0];
-        frag->hdr.eager.ctx     = (void *) &frag->post_desc;
-
-        rc = ompi_mca_btl_ugni_smsg_send (frag, true, &frag->hdr.eager, sizeof (frag->hdr.eager),
+        rc = ompi_mca_btl_ugni_smsg_send (frag, true, &frag->hdr.eager_ex, frag->hdr_size,
                                           NULL, 0, MCA_BTL_UGNI_TAG_GET_INIT);
     }
 

ompi/mca/btl/ugni/btl_ugni_smsg.h

@@ -87,9 +87,12 @@ static inline int ompi_mca_btl_ugni_smsg_send (mca_btl_ugni_base_frag_t *frag,
 
     grc = GNI_SmsgSendWTag (frag->endpoint->smsg_ep_handle, hdr, hdr_len, payload, payload_len,
                            ignore_local_comp ? (uint32_t) -1 : frag->msg_id, tag);
+
+    (void) mca_btl_ugni_progress_local_smsg ((mca_btl_ugni_module_t *) frag->endpoint->btl);
+
     if (OPAL_UNLIKELY(GNI_RC_SUCCESS != grc)) {
         /* see if we can free up some credits */
-        mca_btl_ugni_progress_remote_smsg (frag->endpoint->btl);
+        (void) mca_btl_ugni_progress_remote_smsg ((mca_btl_ugni_module_t *) frag->endpoint->btl);
 
         if (OPAL_LIKELY(GNI_RC_NOT_DONE == grc)) {
             BTL_VERBOSE(("out of credits"));