From b81bcd4b0d71ad02637d469b356ba2e7134bb1ef Mon Sep 17 00:00:00 2001
From: Matias Cabral <matias.a.cabral@intel.com>
Date: Tue, 17 Oct 2017 06:13:04 -0700
Subject: [PATCH] MTL PSM2: add a thread lock while peeking and completing the
 psm2 requests. Reviewed-by: Gopalakrishnan, Aravind
 <aravind.gopalakrishnan@intel.com> Signed-off-by: Matias Cabral
 <matias.a.cabral@intel.com>

---
 ompi/mca/mtl/psm2/mtl_psm2.c           | 80 ++++++++++++++------------
 ompi/mca/mtl/psm2/mtl_psm2.h           |  2 +
 ompi/mca/mtl/psm2/mtl_psm2_component.c |  2 +
 3 files changed, 46 insertions(+), 38 deletions(-)

diff --git a/ompi/mca/mtl/psm2/mtl_psm2.c b/ompi/mca/mtl/psm2/mtl_psm2.c
index f0d04a2159..4b5fc9cfd9 100644
--- a/ompi/mca/mtl/psm2/mtl_psm2.c
+++ b/ompi/mca/mtl/psm2/mtl_psm2.c
@@ -406,58 +406,62 @@ int ompi_mtl_psm2_progress( void ) {
     int completed = 1;
 
     do {
+        OPAL_THREAD_LOCK(&mtl_psm2_mq_mutex);
         err = psm2_mq_ipeek2(ompi_mtl_psm2.mq, &req, NULL);
-	if (err == PSM2_MQ_INCOMPLETE) {
-	    return completed;
-	} else if (err != PSM2_OK) {
-	    goto error;
-	}
+        if (err == PSM2_MQ_INCOMPLETE) {
+            OPAL_THREAD_UNLOCK(&mtl_psm2_mq_mutex);
+            return completed;
+        } else if (OPAL_UNLIKELY(err != PSM2_OK)) {
+            OPAL_THREAD_UNLOCK(&mtl_psm2_mq_mutex);
+            goto error;
+        }
 
-	completed++;
+        err = psm2_mq_test2(&req, &psm2_status);
+        OPAL_THREAD_UNLOCK(&mtl_psm2_mq_mutex);
 
-	err = psm2_mq_test2(&req, &psm2_status);
-	if (err != PSM2_OK) {
-	    goto error;
-	}
+        if (OPAL_UNLIKELY (err != PSM2_OK)) {
+            goto error;
+        }
+
+        completed++;
 
         mtl_psm2_request = (mca_mtl_psm2_request_t*) psm2_status.context;
 
-	if (mtl_psm2_request->type == OMPI_mtl_psm2_IRECV) {
+        if (mtl_psm2_request->type == OMPI_mtl_psm2_IRECV) {
 
-        mtl_psm2_request->super.ompi_req->req_status.MPI_SOURCE =
-            psm2_status.msg_tag.tag1;
-	    mtl_psm2_request->super.ompi_req->req_status.MPI_TAG =
-		    psm2_status.msg_tag.tag0;
+            mtl_psm2_request->super.ompi_req->req_status.MPI_SOURCE =
+                psm2_status.msg_tag.tag1;
+            mtl_psm2_request->super.ompi_req->req_status.MPI_TAG =
+                psm2_status.msg_tag.tag0;
             mtl_psm2_request->super.ompi_req->req_status._ucount =
                 psm2_status.nbytes;
 
             ompi_mtl_datatype_unpack(mtl_psm2_request->convertor,
-                                     mtl_psm2_request->buf,
-                                     psm2_status.msg_length);
-	}
+                mtl_psm2_request->buf,
+                psm2_status.msg_length);
+        }
 
-	if(mtl_psm2_request->type == OMPI_mtl_psm2_ISEND) {
-	  if (mtl_psm2_request->free_after) {
-	    free(mtl_psm2_request->buf);
-	  }
-	}
+        if(mtl_psm2_request->type == OMPI_mtl_psm2_ISEND) {
+          if (mtl_psm2_request->free_after) {
+            free(mtl_psm2_request->buf);
+          }
+        }
 
-	switch (psm2_status.error_code) {
-	    case PSM2_OK:
-		mtl_psm2_request->super.ompi_req->req_status.MPI_ERROR =
-		    OMPI_SUCCESS;
-		break;
-	    case PSM2_MQ_TRUNCATION:
-		mtl_psm2_request->super.ompi_req->req_status.MPI_ERROR =
-		    MPI_ERR_TRUNCATE;
-		break;
-	    default:
-		mtl_psm2_request->super.ompi_req->req_status.MPI_ERROR =
-                        MPI_ERR_INTERN;
-	}
-
-	mtl_psm2_request->super.completion_callback(&mtl_psm2_request->super);
+        switch (psm2_status.error_code) {
+            case PSM2_OK:
+            mtl_psm2_request->super.ompi_req->req_status.MPI_ERROR =
+                OMPI_SUCCESS;
+            break;
+            case PSM2_MQ_TRUNCATION:
+            mtl_psm2_request->super.ompi_req->req_status.MPI_ERROR =
+                MPI_ERR_TRUNCATE;
+            break;
+            default:
+            mtl_psm2_request->super.ompi_req->req_status.MPI_ERROR =
+                MPI_ERR_INTERN;
+        }
 
+        mtl_psm2_request->super.completion_callback(&mtl_psm2_request->super);
     }
     while (1);
 
diff --git a/ompi/mca/mtl/psm2/mtl_psm2.h b/ompi/mca/mtl/psm2/mtl_psm2.h
index cea7d323a0..3b62d8c193 100644
--- a/ompi/mca/mtl/psm2/mtl_psm2.h
+++ b/ompi/mca/mtl/psm2/mtl_psm2.h
@@ -34,6 +34,8 @@
 
 BEGIN_C_DECLS
 
+/* MPI_THREAD_MULTIPLE_SUPPORT */
+extern opal_mutex_t mtl_psm2_mq_mutex;
 
 /* MTL interface functions */
 extern int ompi_mtl_psm2_add_procs(struct mca_mtl_base_module_t* mtl,
diff --git a/ompi/mca/mtl/psm2/mtl_psm2_component.c b/ompi/mca/mtl/psm2/mtl_psm2_component.c
index b2d74aeaf2..2c183d48ea 100644
--- a/ompi/mca/mtl/psm2/mtl_psm2_component.c
+++ b/ompi/mca/mtl/psm2/mtl_psm2_component.c
@@ -42,6 +42,8 @@
 #include <glob.h>
 
 static int param_priority;
+/* MPI_THREAD_MULTIPLE_SUPPORT */
+opal_mutex_t mtl_psm2_mq_mutex = OPAL_MUTEX_STATIC_INIT;
 
 static int ompi_mtl_psm2_component_open(void);
 static int ompi_mtl_psm2_component_close(void);