From e3b267a8fe9af863a524b83e58c5f5f4db198fdd Mon Sep 17 00:00:00 2001 From: Thananon Patinyasakdikul Date: Sun, 29 Oct 2017 14:06:48 -0400 Subject: [PATCH] pml/ob1: match callback will now queue wrong sequence frag and return. In multithreaded case, it is expensive to release the lock, call the slow match and retake the lock again just to queue the frag. This patch will eliminate number of lock taken by queueing the frag right away and return. Signed-off-by: Thananon Patinyasakdikul --- ompi/mca/pml/ob1/pml_ob1_recvfrag.c | 23 +++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/ompi/mca/pml/ob1/pml_ob1_recvfrag.c b/ompi/mca/pml/ob1/pml_ob1_recvfrag.c index 1b59e3aae1..c9df9f318c 100644 --- a/ompi/mca/pml/ob1/pml_ob1_recvfrag.c +++ b/ompi/mca/pml/ob1/pml_ob1_recvfrag.c @@ -164,15 +164,30 @@ void mca_pml_ob1_recv_frag_callback_match(mca_btl_base_module_t* btl, OB1_MATCHING_LOCK(&comm->matching_lock); if (!OMPI_COMM_CHECK_ASSERT_ALLOW_OVERTAKE(comm_ptr)) { - /* get sequence number of next message that can be processed */ - if(OPAL_UNLIKELY((((uint16_t) hdr->hdr_seq) != ((uint16_t) proc->expected_sequence)) || - (opal_list_get_size(&proc->frags_cant_match) > 0 ))) { - goto slow_path; + /* get sequence number of next message that can be processed. + * If this frag is out of sequence, queue it up in the list + * now as we still have the lock. + */ + if(OPAL_UNLIKELY(((uint16_t) hdr->hdr_seq) != ((uint16_t) proc->expected_sequence))) { + /* We generate the MSG_ARRIVED event as soon as the PML is aware of a matching + * fragment arrival. Independing if it is received on the correct order or not. + * This will allow the tools to figure out if the messages are not received in the + * correct order (if multiple network interfaces). + */ + PERUSE_TRACE_MSG_EVENT(PERUSE_COMM_MSG_ARRIVED, comm_ptr, + hdr->hdr_src, hdr->hdr_tag, PERUSE_RECV); + + append_frag_to_list(&proc->frags_cant_match, btl, + hdr, segments, num_segments, NULL); + OB1_MATCHING_UNLOCK(&comm->matching_lock); + return; } /* This is the sequence number we were expecting, so we can try * matching it to already posted receives. */ + if(opal_list_get_size(&proc->frags_cant_match) > 0) + goto slow_path; /* We're now expecting the next sequence number. */ proc->expected_sequence++;