From 9d3a79925b4fb3c9f52ba7d75ce641d073260839 Mon Sep 17 00:00:00 2001 From: Nathan Hjelm Date: Thu, 12 Jul 2018 10:53:10 -0600 Subject: [PATCH] btl/vader: fix bugs in rma emulation This commit fixes two bugs in the RMA/atomic emulation code: 1) Fix a fragment leak when using AMO emulation. 2) Always initialize the single-copy emulation code. This is required to use the AMO support. Signed-off-by: Nathan Hjelm --- opal/mca/btl/vader/btl_vader_atomic.c | 16 ++++++++++++++-- opal/mca/btl/vader/btl_vader_component.c | 4 +++- 2 files changed, 17 insertions(+), 3 deletions(-) diff --git a/opal/mca/btl/vader/btl_vader_atomic.c b/opal/mca/btl/vader/btl_vader_atomic.c index 085fa1f3f0..df3c2664be 100644 --- a/opal/mca/btl/vader/btl_vader_atomic.c +++ b/opal/mca/btl/vader/btl_vader_atomic.c @@ -21,8 +21,14 @@ static void mca_btl_vader_sc_emu_aop_complete (mca_btl_base_module_t *btl, mca_b { mca_btl_vader_frag_t *frag = (mca_btl_vader_frag_t *) desc; void *local_address = frag->rdma.local_address; + void *context = frag->rdma.context; + void *cbdata = frag->rdma.cbdata; + mca_btl_base_rdma_completion_fn_t cbfunc = frag->rdma.cbfunc; - frag->rdma.cbfunc (btl, endpoint, local_address, NULL, frag->rdma.context, frag->rdma.cbdata, status); + /* return the fragment first since the callback may call put/get/amo and could use this fragment */ + MCA_BTL_VADER_FRAG_RETURN(frag); + + cbfunc (btl, endpoint, local_address, NULL, context, cbdata, status); } int mca_btl_vader_emu_aop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, @@ -50,12 +56,18 @@ static void mca_btl_vader_sc_emu_afop_complete (mca_btl_base_module_t *btl, mca_ mca_btl_vader_frag_t *frag = (mca_btl_vader_frag_t *) desc; mca_btl_vader_sc_emu_hdr_t *hdr; void *local_address = frag->rdma.local_address; + void *context = frag->rdma.context; + void *cbdata = frag->rdma.cbdata; + mca_btl_base_rdma_completion_fn_t cbfunc = frag->rdma.cbfunc; hdr = (mca_btl_vader_sc_emu_hdr_t *) frag->segments[0].seg_addr.pval; *((int64_t *) frag->rdma.local_address) = hdr->operand[0]; - frag->rdma.cbfunc (btl, endpoint, local_address, NULL, frag->rdma.context, frag->rdma.cbdata, status); + /* return the fragment first since the callback may call put/get/amo and could use this fragment */ + MCA_BTL_VADER_FRAG_RETURN(frag); + + cbfunc (btl, endpoint, local_address, NULL, context, cbdata, status); } int mca_btl_vader_emu_afop (struct mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, diff --git a/opal/mca/btl/vader/btl_vader_component.c b/opal/mca/btl/vader/btl_vader_component.c index 1c97fc0746..3fc35c6a46 100644 --- a/opal/mca/btl/vader/btl_vader_component.c +++ b/opal/mca/btl/vader/btl_vader_component.c @@ -379,6 +379,9 @@ static void mca_btl_vader_check_single_copy (void) { int initial_mechanism = mca_btl_vader_component.single_copy_mechanism; + /* single-copy emulation is always used to support AMO's right now */ + mca_btl_vader_sc_emu_init (); + #if OPAL_BTL_VADER_HAVE_XPMEM if (MCA_BTL_VADER_XPMEM == mca_btl_vader_component.single_copy_mechanism) { /* try to create an xpmem segment for the entire address space */ @@ -462,7 +465,6 @@ static void mca_btl_vader_check_single_copy (void) } if (MCA_BTL_VADER_EMUL == mca_btl_vader_component.single_copy_mechanism) { - mca_btl_vader_sc_emu_init (); /* limit to the maximum fragment size */ mca_btl_vader.super.btl_put_limit = mca_btl_vader.super.btl_max_send_size - sizeof (mca_btl_vader_sc_emu_hdr_t); mca_btl_vader.super.btl_get_limit = mca_btl_vader.super.btl_max_send_size - sizeof (mca_btl_vader_sc_emu_hdr_t);