From 707e594d137d5a3280433cbcc5d34420e3bd1d00 Mon Sep 17 00:00:00 2001 From: Dave Goodell Date: Wed, 26 Feb 2014 07:39:53 +0000 Subject: [PATCH] usnic: Use INLINE flag more often, saving the DMA is useful. Authored-by: Reese Faucette Reviewed-by: Jeff Squyres cmr=v1.7.5:ticket=trac:4253 This commit was SVN r30833. The following Trac tickets were found above: Ticket 4253 --> https://svn.open-mpi.org/trac/ompi/ticket/4253 --- ompi/mca/btl/usnic/btl_usnic_frag.c | 1 + ompi/mca/btl/usnic/btl_usnic_frag.h | 9 ++++++--- ompi/mca/btl/usnic/btl_usnic_send.c | 4 ++++ 3 files changed, 11 insertions(+), 3 deletions(-) diff --git a/ompi/mca/btl/usnic/btl_usnic_frag.c b/ompi/mca/btl/usnic/btl_usnic_frag.c index 0936e42011..297d8d26ef 100644 --- a/ompi/mca/btl/usnic/btl_usnic_frag.c +++ b/ompi/mca/btl/usnic/btl_usnic_frag.c @@ -203,6 +203,7 @@ small_send_frag_constructor(ompi_btl_usnic_small_send_frag_t *frag) fseg->ss_parent_frag = (struct ompi_btl_usnic_send_frag_t *)frag; frag->ssf_base.sf_base.uf_type = OMPI_BTL_USNIC_FRAG_SMALL_SEND; + frag->ssf_segment.ss_send_desc.send_flags = IBV_SEND_SIGNALED; /* save data pointer for PML */ frag->ssf_base.sf_base.uf_src_seg[0].seg_addr.pval = diff --git a/ompi/mca/btl/usnic/btl_usnic_frag.h b/ompi/mca/btl/usnic/btl_usnic_frag.h index 5c7f1dea95..1a9a70250f 100644 --- a/ompi/mca/btl/usnic/btl_usnic_frag.h +++ b/ompi/mca/btl/usnic/btl_usnic_frag.h @@ -365,9 +365,6 @@ ompi_btl_usnic_small_send_frag_alloc(ompi_btl_usnic_module_t *module) /* this belongs in constructor... */ frag->ssf_base.sf_base.uf_freelist = &(module->small_send_frags); - /* always clear flag */ - frag->ssf_segment.ss_send_desc.send_flags = IBV_SEND_SIGNALED; - assert(frag); assert(OMPI_BTL_USNIC_FRAG_SMALL_SEND == frag->ssf_base.sf_base.uf_type); @@ -474,6 +471,10 @@ ompi_btl_usnic_frag_return( NULL == lfrag->lsf_des_src[1].seg_addr.pval) { opal_convertor_cleanup(&lfrag->lsf_base.sf_convertor); } + } else if (frag->uf_type == OMPI_BTL_USNIC_FRAG_SMALL_SEND) { + ompi_btl_usnic_small_send_frag_t *sfrag; + sfrag = (ompi_btl_usnic_small_send_frag_t *)frag; + sfrag->ssf_segment.ss_send_desc.send_flags &= ~IBV_SEND_INLINE; } OMPI_FREE_LIST_RETURN_MT(frag->uf_freelist, &(frag->uf_base.super)); @@ -528,6 +529,7 @@ ompi_btl_usnic_chunk_segment_alloc( seg = (ompi_btl_usnic_send_segment_t*) item; seg->ss_channel = USNIC_DATA_CHANNEL; + seg->ss_send_desc.send_flags = IBV_SEND_SIGNALED; assert(seg); assert(OMPI_BTL_USNIC_SEG_CHUNK == seg->ss_base.us_type); @@ -562,6 +564,7 @@ ompi_btl_usnic_ack_segment_alloc(ompi_btl_usnic_module_t *module) ack = (ompi_btl_usnic_ack_segment_t*) item; ack->ss_channel = USNIC_PRIORITY_CHANNEL; + ack->ss_send_desc.send_flags = IBV_SEND_SIGNALED; assert(ack); assert(OMPI_BTL_USNIC_SEG_ACK == ack->ss_base.us_type); diff --git a/ompi/mca/btl/usnic/btl_usnic_send.c b/ompi/mca/btl/usnic/btl_usnic_send.c index 6cae3c65e7..413ac0abd3 100644 --- a/ompi/mca/btl/usnic/btl_usnic_send.c +++ b/ompi/mca/btl/usnic/btl_usnic_send.c @@ -152,6 +152,10 @@ ompi_btl_usnic_finish_put_or_send( /* use standard channel */ sseg->ss_channel = USNIC_DATA_CHANNEL; sseg->ss_base.us_btl_header->tag = tag; + + if (frag->sf_base.uf_src_seg[0].seg_len < module->tiny_mtu) { + sseg->ss_send_desc.send_flags |= IBV_SEND_INLINE; + } } else { ompi_btl_usnic_large_send_frag_t *lfrag;