pml/ob1: match callback will now queue wrong sequence frag and return.

In multithreaded case, it is expensive to release the lock, call the slow match
and retake the lock again just to queue the frag. This patch will eliminate number of
lock taken by queueing the frag right away and return.

Signed-off-by: Thananon Patinyasakdikul <tpatinya@utk.edu>

ompi/mca/pml/ob1/pml_ob1_recvfrag.c

@@ -164,15 +164,30 @@ void mca_pml_ob1_recv_frag_callback_match(mca_btl_base_module_t* btl,
     OB1_MATCHING_LOCK(&comm->matching_lock);
 
     if (!OMPI_COMM_CHECK_ASSERT_ALLOW_OVERTAKE(comm_ptr)) {
-        /* get sequence number of next message that can be processed */
-        if(OPAL_UNLIKELY((((uint16_t) hdr->hdr_seq) != ((uint16_t) proc->expected_sequence)) ||
-                         (opal_list_get_size(&proc->frags_cant_match) > 0 ))) {
-            goto slow_path;
+        /* get sequence number of next message that can be processed.
+         * If this frag is out of sequence, queue it up in the list
+         * now as we still have the lock.
+         */
+        if(OPAL_UNLIKELY(((uint16_t) hdr->hdr_seq) != ((uint16_t) proc->expected_sequence))) {
+             /* We generate the MSG_ARRIVED event as soon as the PML is aware of a matching
+             *  fragment arrival. Independing if it is received on the correct order or not.
+             *  This will allow the tools to figure out if the messages are not received in the
+             *  correct order (if multiple network interfaces).
+             */
+            PERUSE_TRACE_MSG_EVENT(PERUSE_COMM_MSG_ARRIVED, comm_ptr,
+                                   hdr->hdr_src, hdr->hdr_tag, PERUSE_RECV);
+
+            append_frag_to_list(&proc->frags_cant_match, btl,
+                                hdr, segments, num_segments, NULL);
+            OB1_MATCHING_UNLOCK(&comm->matching_lock);
+            return;
         }
 
         /* This is the sequence number we were expecting, so we can try
          * matching it to already posted receives.
          */
+        if(opal_list_get_size(&proc->frags_cant_match) > 0)
+            goto slow_path;
 
         /* We're now expecting the next sequence number. */
         proc->expected_sequence++;