From b3d8ead76e6d4989812266451dfc06a25afd9807 Mon Sep 17 00:00:00 2001
From: yohann <yohann.burette@intel.com>
Date: Fri, 12 Feb 2016 10:05:52 -0800
Subject: [PATCH] mtl/ofi: Fix dynamic add_procs.

---
 ompi/mca/mtl/ofi/mtl_ofi.h          | 15 +++++----------
 ompi/mca/mtl/ofi/mtl_ofi_endpoint.h | 15 +++++++++++++--
 ompi/mca/mtl/ofi/mtl_ofi_request.h  |  4 ++++
 3 files changed, 22 insertions(+), 12 deletions(-)

diff --git a/ompi/mca/mtl/ofi/mtl_ofi.h b/ompi/mca/mtl/ofi/mtl_ofi.h
index bb577051fc..da3f7d27ef 100644
--- a/ompi/mca/mtl/ofi/mtl_ofi.h
+++ b/ompi/mca/mtl/ofi/mtl_ofi.h
@@ -50,10 +50,6 @@ BEGIN_C_DECLS
 extern mca_mtl_ofi_module_t ompi_mtl_ofi;
 extern mca_base_framework_t ompi_mtl_base_framework;
 
-extern int ompi_mtl_ofi_add_procs(struct mca_mtl_base_module_t *mtl,
-                                  size_t nprocs,
-                                  struct ompi_proc_t **procs);
-
 extern int ompi_mtl_ofi_del_procs(struct mca_mtl_base_module_t *mtl,
                                   size_t nprocs,
                                   struct ompi_proc_t **procs);
@@ -235,7 +231,7 @@ ompi_mtl_ofi_send_start(struct mca_mtl_base_module_t *mtl,
     ompi_mtl_ofi_request_t *ack_req = NULL; /* For synchronous send */
 
     ompi_proc = ompi_comm_peer_lookup(comm, dest);
-    endpoint = ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL];
+    endpoint = ompi_mtl_ofi_get_endpoint(mtl, ompi_proc);
 
     ompi_ret = ompi_mtl_datatype_pack(convertor, &start, &length, &free_after);
     if (OMPI_SUCCESS != ompi_ret) return ompi_ret;
@@ -460,7 +456,7 @@ ompi_mtl_ofi_recv_callback(struct fi_cq_tagged_entry *wc,
         if (ompi_mtl_ofi.any_addr == ofi_req->remote_addr) {
             src = MTL_OFI_GET_SOURCE(wc->tag);
             ompi_proc = ompi_comm_peer_lookup(ofi_req->comm, src);
-            endpoint = ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL];
+            endpoint = ompi_mtl_ofi_get_endpoint(ofi_req->mtl, ompi_proc);
             ofi_req->remote_addr = endpoint->peer_fiaddr;
         }
 	    MTL_OFI_RETRY_UNTIL_DONE(fi_tsend(ompi_mtl_ofi.ep,
@@ -532,7 +528,7 @@ ompi_mtl_ofi_irecv(struct mca_mtl_base_module_t *mtl,
 
     if (MPI_ANY_SOURCE != src) {
         ompi_proc = ompi_comm_peer_lookup(comm, src);
-        endpoint = ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL];
+        endpoint = ompi_mtl_ofi_get_endpoint(mtl, ompi_proc);
         remote_addr = endpoint->peer_fiaddr;
     } else {
         remote_addr = ompi_mtl_ofi.any_addr;
@@ -744,7 +740,7 @@ ompi_mtl_ofi_iprobe(struct mca_mtl_base_module_t *mtl,
      */
     if (MPI_ANY_SOURCE != src) {
         ompi_proc = ompi_comm_peer_lookup( comm, src );
-        endpoint = ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL];
+        endpoint = ompi_mtl_ofi_get_endpoint(mtl, ompi_proc);
         remote_proc = endpoint->peer_fiaddr;
     }
 
@@ -829,7 +825,7 @@ ompi_mtl_ofi_improbe(struct mca_mtl_base_module_t *mtl,
      */
     if (MPI_ANY_SOURCE != src) {
         ompi_proc = ompi_comm_peer_lookup( comm, src );
-        endpoint = ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL];
+        endpoint = ompi_mtl_ofi_get_endpoint(mtl, ompi_proc);
         remote_proc = endpoint->peer_fiaddr;
     }
 
@@ -961,7 +957,6 @@ ompi_mtl_ofi_del_comm(struct mca_mtl_base_module_t *mtl,
     return OMPI_SUCCESS;
 }
 
-
 END_C_DECLS
 
 #endif  /* MTL_OFI_H_HAS_BEEN_INCLUDED */
diff --git a/ompi/mca/mtl/ofi/mtl_ofi_endpoint.h b/ompi/mca/mtl/ofi/mtl_ofi_endpoint.h
index 2799d495b5..e886dcea23 100644
--- a/ompi/mca/mtl/ofi/mtl_ofi_endpoint.h
+++ b/ompi/mca/mtl/ofi/mtl_ofi_endpoint.h
@@ -11,10 +11,12 @@
 #ifndef OMPI_MTL_OFI_ENDPOINT_H
 #define OMPI_MTL_OFI_ENDPOINT_H
 
-#include "mtl_ofi.h"
-
 BEGIN_C_DECLS
 
+extern int ompi_mtl_ofi_add_procs(struct mca_mtl_base_module_t *mtl,
+                                  size_t nprocs,
+                                  struct ompi_proc_t **procs);
+
 OBJ_CLASS_DECLARATION(mca_mtl_ofi_endpoint_t);
 
 /**
@@ -37,5 +39,14 @@ struct mca_mtl_ofi_endpoint_t {
 typedef struct mca_mtl_ofi_endpoint_t  mca_mtl_ofi_endpoint_t;
 OBJ_CLASS_DECLARATION(mca_mtl_ofi_endpoint);
 
+static inline mca_mtl_ofi_endpoint_t *ompi_mtl_ofi_get_endpoint (struct mca_mtl_base_module_t* mtl, ompi_proc_t *ompi_proc)
+{
+    if (OPAL_UNLIKELY(NULL == ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL])) {
+        ompi_mtl_ofi_add_procs(mtl, 1, &ompi_proc);
+    }
+
+    return ompi_proc->proc_endpoints[OMPI_PROC_ENDPOINT_TAG_MTL];
+}
+
 END_C_DECLS
 #endif
diff --git a/ompi/mca/mtl/ofi/mtl_ofi_request.h b/ompi/mca/mtl/ofi/mtl_ofi_request.h
index ee544073cc..dc35a31f07 100644
--- a/ompi/mca/mtl/ofi/mtl_ofi_request.h
+++ b/ompi/mca/mtl/ofi/mtl_ofi_request.h
@@ -55,6 +55,10 @@ struct ompi_mtl_ofi_request_t {
     /*  lookup source of an ANY_SOURCE Recv    */
     struct ompi_communicator_t *comm;
 
+    /** Reference to the MTL used to lookup */
+    /*  source of an ANY_SOURCE Recv        */
+    struct mca_mtl_base_module_t* mtl;
+
     /** Pack buffer */
     void *buffer;