diff --git a/ompi/mca/btl/portals/btl_portals.h b/ompi/mca/btl/portals/btl_portals.h index ea050ee4be..c81035d573 100644 --- a/ompi/mca/btl/portals/btl_portals.h +++ b/ompi/mca/btl/portals/btl_portals.h @@ -9,6 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. + * Copyright (c) 2008 UT-Battelle, LLC. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -63,6 +64,10 @@ struct mca_btl_portals_component_t { /* shall I use portals to send to thyself? */ int portals_support_self; + + /* do I need a portals ACK? */ + int portals_need_ack; + }; typedef struct mca_btl_portals_component_t mca_btl_portals_component_t; diff --git a/ompi/mca/btl/portals/btl_portals_component.c b/ompi/mca/btl/portals/btl_portals_component.c index c9417854dc..4d6d73eec8 100644 --- a/ompi/mca/btl/portals/btl_portals_component.c +++ b/ompi/mca/btl/portals/btl_portals_component.c @@ -125,7 +125,7 @@ mca_btl_portals_component_open(void) false, 32, &(mca_btl_portals_component.portals_free_list_eager_max_num)); - + mca_base_param_reg_int(&mca_btl_portals_component.super.btl_version, "support_self", "Use portals for send to self", @@ -134,6 +134,14 @@ mca_btl_portals_component_open(void) 1, /* default to true.. */ &(mca_btl_portals_component.portals_support_self)); + mca_base_param_reg_int(&mca_btl_portals_component.super.btl_version, + "needs_ack", + "Require a portals level ACK", + false, + false, + 1, /* default to true.. */ + &(mca_btl_portals_component.portals_need_ack)); + /* * fill default module state */ @@ -490,7 +498,7 @@ mca_btl_portals_component_progress(void) OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output, "PTL_EVENT_SEND_END for 0x%lx, %d", (unsigned long) frag, (int) ev.hdr_data)); - + if (ev.ni_fail_type != PTL_NI_OK) { opal_output(mca_btl_portals_component.portals_output, "Failure to end send event\n"); @@ -504,6 +512,22 @@ mca_btl_portals_component_progress(void) } } #endif + if(!mca_btl_portals_component.portals_need_ack) { + /* my part's done, in portals we trust! */ + frag->base.des_cbfunc(&mca_btl_portals_module.super, + frag->endpoint, + &frag->base, + OMPI_SUCCESS); + if( btl_ownership ) { + mca_btl_portals_free(&mca_btl_portals_module.super, + &frag->base); + } + if (0 != frag->size) { + OPAL_THREAD_ADD32(&mca_btl_portals_module.portals_outstanding_ops, + -1); + MCA_BTL_PORTALS_PROGRESS_QUEUED_SENDS(); + } + } break; case PTL_EVENT_ACK: @@ -516,8 +540,13 @@ mca_btl_portals_component_progress(void) OPAL_OUTPUT_VERBOSE((90, mca_btl_portals_component.portals_output, "PTL_EVENT_ACK for 0x%lx", (unsigned long) frag)); - + #if OMPI_ENABLE_DEBUG + if(!mca_btl_portals_component.portals_need_ack) { + opal_output(mca_btl_portals_component.portals_output, + "Received PTL_EVENT_ACK but ACK's are disabled!\n"); + abort(); + } if (ev.ni_fail_type != PTL_NI_OK) { opal_output(mca_btl_portals_component.portals_output, "Failure to ack event\n"); diff --git a/ompi/mca/btl/portals/btl_portals_rdma.c b/ompi/mca/btl/portals/btl_portals_rdma.c index ab0e02b0b1..f1a060e6e6 100644 --- a/ompi/mca/btl/portals/btl_portals_rdma.c +++ b/ompi/mca/btl/portals/btl_portals_rdma.c @@ -9,6 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. + * Copyright (c) 2008 UT-Battelle, LLC. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -48,9 +49,9 @@ mca_btl_portals_put(struct mca_btl_base_module_t* btl_base, /* setup the send */ assert(1 == frag->base.des_src_cnt); - + ret = PtlPut(frag->md_h, - PTL_ACK_REQ, + (mca_btl_portals_component.portals_need_ack ? PTL_ACK_REQ : PTL_NO_ACK_REQ), *((mca_btl_base_endpoint_t*) btl_peer), OMPI_BTL_PORTALS_RDMA_TABLE_ID, 0, /* ac_index - not used*/ diff --git a/ompi/mca/btl/portals/btl_portals_send.c b/ompi/mca/btl/portals/btl_portals_send.c index 78740df7fc..87afd487f1 100644 --- a/ompi/mca/btl/portals/btl_portals_send.c +++ b/ompi/mca/btl/portals/btl_portals_send.c @@ -9,6 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. + * Copyright (c) 2008 UT-Battelle, LLC. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -62,8 +63,16 @@ mca_btl_portals_send(struct mca_btl_base_module_t* btl_base, mca_btl_portals_module.md_send.start = frag->segments[0].seg_addr.pval; mca_btl_portals_module.md_send.length = 0 == frag->size ? frag->segments[0].seg_len : frag->size; +#if OMPI_ENABLE_DEBUG mca_btl_portals_module.md_send.options = PTL_MD_EVENT_START_DISABLE; +#else + /* optimized build, we can get rid of the END event */ + /* if we are using portals ACK's for completion */ + mca_btl_portals_module.md_send.options = + (PTL_MD_EVENT_START_DISABLE | + (mca_btl_portals_component.portals_need_ack ? PTL_MD_EVENT_END_DISABLE : 0)); +#endif mca_btl_portals_module.md_send.user_ptr = frag; /* keep a pointer to ourselves */ /* make a free-floater */ @@ -84,11 +93,11 @@ mca_btl_portals_send(struct mca_btl_base_module_t* btl_base, "\tlen: %d", (unsigned long) frag->segments[0].seg_addr.pval, frag->segments[0].seg_len)); - + ret = PtlPutRegion(frag->md_h, /* memory descriptor */ 0, /* fragment offset */ frag->segments[0].seg_len, /* fragment length */ - PTL_ACK_REQ, + (mca_btl_portals_component.portals_need_ack ? PTL_ACK_REQ : PTL_NO_ACK_REQ), *((mca_btl_base_endpoint_t*) endpoint), OMPI_BTL_PORTALS_SEND_TABLE_ID, 0, /* ac_index - not used */