From 1b231f7e73dc7aadbc938d28d9809d8fc03dce30 Mon Sep 17 00:00:00 2001 From: Rolf vandeVaart Date: Tue, 2 Nov 2010 20:11:47 +0000 Subject: [PATCH] More miscellaneous cleanup of bfo. This commit was SVN r23986. --- ompi/mca/pml/bfo/pml_bfo_failover.c | 7 ++----- ompi/mca/pml/bfo/pml_bfo_failover.h | 13 ++++++++++++- ompi/mca/pml/bfo/pml_bfo_recvfrag.c | 5 +---- ompi/mca/pml/bfo/pml_bfo_recvreq.c | 2 +- ompi/mca/pml/bfo/pml_bfo_recvreq.h | 3 +-- ompi/mca/pml/bfo/pml_bfo_sendreq.c | 12 ++---------- ompi/mca/pml/bfo/pml_bfo_sendreq.h | 9 +-------- 7 files changed, 20 insertions(+), 31 deletions(-) diff --git a/ompi/mca/pml/bfo/pml_bfo_failover.c b/ompi/mca/pml/bfo/pml_bfo_failover.c index 6cfa52d033..083e3237af 100644 --- a/ompi/mca/pml/bfo/pml_bfo_failover.c +++ b/ompi/mca/pml/bfo/pml_bfo_failover.c @@ -1779,7 +1779,6 @@ void mca_pml_bfo_check_recv_ctl_completion_status(mca_btl_base_module_t* btl, int status) { mca_pml_bfo_common_hdr_t * common = des->des_src->seg_addr.pval; - mca_pml_bfo_ack_hdr_t* ack; /* ACK header */ mca_pml_bfo_rdma_hdr_t* hdr; /* PUT header */ struct mca_btl_base_descriptor_t* rdma_des; mca_pml_bfo_recv_request_t* recvreq; @@ -1787,8 +1786,7 @@ void mca_pml_bfo_check_recv_ctl_completion_status(mca_btl_base_module_t* btl, if(OPAL_UNLIKELY(OMPI_SUCCESS != status)) { switch (common->hdr_type) { case MCA_PML_BFO_HDR_TYPE_ACK: - ack = (mca_pml_bfo_ack_hdr_t*)des->des_src->seg_addr.pval; - recvreq = (mca_pml_bfo_recv_request_t*) ack->hdr_dst_req.pval; + recvreq = des->des_cbdata; /* Record the error. Send RECVERRNOTIFY if necessary. */ if (recvreq->req_errstate) { @@ -1860,8 +1858,7 @@ void mca_pml_bfo_check_recv_ctl_completion_status(mca_btl_base_module_t* btl, switch (common->hdr_type) { case MCA_PML_BFO_HDR_TYPE_ACK: - ack = (mca_pml_bfo_ack_hdr_t*)des->des_src->seg_addr.pval; - recvreq = (mca_pml_bfo_recv_request_t*) ack->hdr_dst_req.pval; + recvreq = des->des_cbdata; recvreq->req_events--; assert(recvreq->req_events >= 0); if(OPAL_UNLIKELY (recvreq->req_errstate & RECVREQ_RNDVRESTART_RECVED)) { diff --git a/ompi/mca/pml/bfo/pml_bfo_failover.h b/ompi/mca/pml/bfo/pml_bfo_failover.h index 364dc75a94..22b7f39949 100644 --- a/ompi/mca/pml/bfo/pml_bfo_failover.h +++ b/ompi/mca/pml/bfo/pml_bfo_failover.h @@ -253,6 +253,18 @@ extern void mca_pml_bfo_recv_frag_callback_recverrnotify( mca_btl_base_module_t * Macros for pml_bfo_sendreq.c file. */ +/* This macro is called on the sending side after receiving + * a PUT message. There is a chance that this PUT message + * has shown up and is attempting to modify the state of + * the req_state, but the req_state is no longer being tracked + * because the RNDV message has turned into a RGET message + * because it got an error on the RNDV completion. + */ +#define MCA_PML_BFO_VERIFY_SENDREQ_REQ_STATE_VALUE(sendreq) \ + if (sendreq->req_state == -1) { \ + OPAL_THREAD_ADD32(&sendreq->req_state, 1); \ + } + /* Now check the error state. This request can be in error if the * RNDV message made it over, but the receiver got an error trying to * send the ACK back and therefore sent a RECVERRNOTIFY message. In @@ -326,7 +338,6 @@ extern void mca_pml_bfo_recv_frag_callback_recverrnotify( mca_btl_base_module_t mca_bml_base_endpoint_t* bml_endpoint = (mca_bml_base_endpoint_t*) proc->proc_bml; \ bml_btl = mca_bml_base_btl_array_find(&bml_endpoint->btl_eager, btl); \ } - #define MCA_PML_BFO_CHECK_SENDREQ_EAGER_BML_BTL(bml_btl, btl, sendreq, type) \ if (bml_btl->btl != btl) { \ mca_pml_bfo_find_sendreq_eager_bml_btl(&bml_btl, btl, sendreq, type); \ diff --git a/ompi/mca/pml/bfo/pml_bfo_recvfrag.c b/ompi/mca/pml/bfo/pml_bfo_recvfrag.c index 071064682d..7997c34bfb 100644 --- a/ompi/mca/pml/bfo/pml_bfo_recvfrag.c +++ b/ompi/mca/pml/bfo/pml_bfo_recvfrag.c @@ -243,8 +243,6 @@ void mca_pml_bfo_recv_frag_callback_match(mca_btl_base_module_t* btl, slow_path: OPAL_THREAD_UNLOCK(&comm->matching_lock); #ifdef PML_BFO - /* Check for duplicate messages. If message is duplicate, then just - * return as that essentially drops the message. */ if (true == mca_pml_bfo_is_duplicate_msg(proc, hdr)) { return; } @@ -626,6 +624,7 @@ static int mca_pml_bfo_recv_frag_match( mca_btl_base_module_t *btl, * the fragment. */ OPAL_THREAD_LOCK(&comm->matching_lock); + #ifdef PML_BFO /* In case of network failover, we may get a message telling us to * restart. In that case, we already have a pointer to the receive @@ -718,8 +717,6 @@ wrong_seq: * is ahead of sequence. Save it for later. */ #ifdef PML_BFO - /* Check for duplicate messages. If message is duplicate, then just - * return as that essentially drops the message. */ if (true == mca_pml_bfo_is_duplicate_msg(proc, hdr)) { return OMPI_SUCCESS; } diff --git a/ompi/mca/pml/bfo/pml_bfo_recvreq.c b/ompi/mca/pml/bfo/pml_bfo_recvreq.c index 66093a8bc9..1e2096ecfb 100644 --- a/ompi/mca/pml/bfo/pml_bfo_recvreq.c +++ b/ompi/mca/pml/bfo/pml_bfo_recvreq.c @@ -251,7 +251,7 @@ int mca_pml_bfo_recv_request_ack_send_btl( /* initialize descriptor */ des->des_cbfunc = mca_pml_bfo_recv_ctl_completion; #ifdef PML_BFO - des->des_cbdata = (void *)proc; + des->des_cbdata = hdr_dst_req; #endif rc = mca_bml_base_send(bml_btl, des, MCA_PML_BFO_HDR_TYPE_ACK); diff --git a/ompi/mca/pml/bfo/pml_bfo_recvreq.h b/ompi/mca/pml/bfo/pml_bfo_recvreq.h index 08a0391fb9..23f3c8bb92 100644 --- a/ompi/mca/pml/bfo/pml_bfo_recvreq.h +++ b/ompi/mca/pml/bfo/pml_bfo_recvreq.h @@ -169,8 +169,7 @@ recv_request_pml_complete(mca_pml_bfo_recv_request_t *recvreq) } recvreq->req_rdma_cnt = 0; #ifdef PML_BFO - /* Reset to a value that to indicate it is invalid. */ - recvreq->req_msgseq = recvreq->req_msgseq - 100; + recvreq->req_msgseq -= 100; #endif OPAL_THREAD_LOCK(&ompi_request_lock); diff --git a/ompi/mca/pml/bfo/pml_bfo_sendreq.c b/ompi/mca/pml/bfo/pml_bfo_sendreq.c index 88888d5928..7bdbcc2bfc 100644 --- a/ompi/mca/pml/bfo/pml_bfo_sendreq.c +++ b/ompi/mca/pml/bfo/pml_bfo_sendreq.c @@ -1343,18 +1343,10 @@ void mca_pml_bfo_send_request_put( mca_pml_bfo_send_request_t* sendreq, size_t i, size = 0; if(hdr->hdr_common.hdr_flags & MCA_PML_BFO_HDR_TYPE_ACK) { -#ifdef PML_BFO - /* Handle the failover case where a RNDV request may - * have turned into a RGET and therefore the state - * is not being tracked. */ - if (sendreq->req_state != 0) { - OPAL_THREAD_ADD32(&sendreq->req_state, -1); - } -#else OPAL_THREAD_ADD32(&sendreq->req_state, -1); -#endif } #ifdef PML_BFO + MCA_PML_BFO_VERIFY_SENDREQ_REQ_STATE_VALUE(sendreq); sendreq->req_recv = hdr->hdr_dst_req; /* only needed once, but it is OK */ #endif @@ -1385,8 +1377,8 @@ void mca_pml_bfo_send_request_put( mca_pml_bfo_send_request_t* sendreq, frag->rdma_bml = mca_bml_base_btl_array_find(&bml_endpoint->btl_rdma, btl); #ifdef PML_BFO - frag->rdma_btl = btl; MCA_PML_BFO_CHECK_FOR_REMOVED_BML(sendreq, frag, btl); + frag->rdma_btl = btl; /* in case frag ends up on pending */ #endif frag->rdma_hdr.hdr_rdma = *hdr; frag->rdma_req = sendreq; diff --git a/ompi/mca/pml/bfo/pml_bfo_sendreq.h b/ompi/mca/pml/bfo/pml_bfo_sendreq.h index 75cd106652..45a5b61dc4 100644 --- a/ompi/mca/pml/bfo/pml_bfo_sendreq.h +++ b/ompi/mca/pml/bfo/pml_bfo_sendreq.h @@ -250,14 +250,7 @@ send_request_pml_complete(mca_pml_bfo_send_request_t *sendreq) } sendreq->req_send.req_base.req_pml_complete = true; #ifdef PML_BFO - assert(0 == sendreq->req_events); - sendreq->req_restartseq = 0; - /* Since sequence numbers increase monotonically and - * roll over, initialize it to a value far away from - * what it was. I cannot set it to something like -1 - * as that is not within the valid range. */ - sendreq->req_send.req_base.req_sequence = - sendreq->req_send.req_base.req_sequence - 10; + sendreq->req_send.req_base.req_sequence -= 100; #endif if(sendreq->req_send.req_base.req_free_called) {