From da8df859b3e2f89d9dd8cbe01cedb98096adbf41 Mon Sep 17 00:00:00 2001
From: Mike Dubman <miked@mellanox.com>
Date: Thu, 17 Jul 2014 08:35:55 +0000
Subject: [PATCH] MXM: use builk connection establishment API
fixed by Vasily, reviewed by Yossi/Miked
cmr=v1.8.2:reviwer=ompi-rm1.8
This commit was SVN r32256.
---
ompi/mca/mtl/mxm/mtl_mxm.c | 13 ++++++++++
ompi/mca/mtl/mxm/mtl_mxm_component.c | 38 ++++++++++++++++++++++++++++
ompi/mca/mtl/mxm/mtl_mxm_types.h | 4 +++
3 files changed, 55 insertions(+)
diff --git a/ompi/mca/mtl/mxm/mtl_mxm.c b/ompi/mca/mtl/mxm/mtl_mxm.c
index c94fde0fe2..f583df273a 100644
--- a/ompi/mca/mtl/mxm/mtl_mxm.c
+++ b/ompi/mca/mtl/mxm/mtl_mxm.c
@@ -484,6 +484,12 @@ int ompi_mtl_mxm_add_procs(struct mca_mtl_base_module_t *mtl, size_t nprocs,
#endif
+#if MXM_API >= MXM_VERSION(3,1)
+ if (ompi_mtl_mxm.bulk_connect) {
+ mxm_ep_wireup(ompi_mtl_mxm.ep);
+ }
+#endif
+
rc = OMPI_SUCCESS;
bail:
@@ -499,6 +505,13 @@ int ompi_mtl_mxm_del_procs(struct mca_mtl_base_module_t *mtl, size_t nprocs,
{
size_t i;
+#if MXM_API >= MXM_VERSION(3,1)
+ if (ompi_mtl_mxm.bulk_disconnect &&
+ nprocs == ompi_comm_size(&ompi_mpi_comm_world.comm)) {
+ mxm_ep_powerdown(ompi_mtl_mxm.ep);
+ }
+#endif
+
/* XXX: Directly accessing the obj_reference_count is an abstraction
* violation of the object system. We know this needs to be fixed, but
* are deferring the fix to a later time as it involves a design issue
diff --git a/ompi/mca/mtl/mxm/mtl_mxm_component.c b/ompi/mca/mtl/mxm/mtl_mxm_component.c
index a6e5f16e73..492c5a3523 100644
--- a/ompi/mca/mtl/mxm/mtl_mxm_component.c
+++ b/ompi/mca/mtl/mxm/mtl_mxm_component.c
@@ -125,6 +125,44 @@ static int ompi_mtl_mxm_component_register(void)
free(runtime_version);
#endif
+#if MXM_API >= MXM_VERSION(3,1)
+{
+ unsigned long cur_ver = mxm_get_version();
+
+ if (cur_ver < MXM_VERSION(3,2)) {
+ ompi_mtl_mxm.bulk_connect = 0;
+ ompi_mtl_mxm.bulk_disconnect = 0;
+ } else {
+ ompi_mtl_mxm.bulk_connect = 1;
+ ompi_mtl_mxm.bulk_disconnect = 1;
+ }
+
+ (void) mca_base_component_var_register(c, "bulk_connect",
+ "[integer] use bulk connect",
+ MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+ OPAL_INFO_LVL_9,
+ MCA_BASE_VAR_SCOPE_READONLY,
+ &ompi_mtl_mxm.bulk_connect);
+
+ (void) mca_base_component_var_register(c, "bulk_disconnect",
+ "[integer] use bulk disconnect",
+ MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
+ OPAL_INFO_LVL_9,
+ MCA_BASE_VAR_SCOPE_READONLY,
+ &ompi_mtl_mxm.bulk_disconnect);
+
+ if (cur_ver < MXM_VERSION(3,2) &&
+ (ompi_mtl_mxm.bulk_connect || ompi_mtl_mxm.bulk_disconnect)) {
+ ompi_mtl_mxm.bulk_connect = 0;
+ ompi_mtl_mxm.bulk_disconnect = 0;
+
+ MXM_VERBOSE(1, "WARNING: OMPI runs with %s version of MXM that is less than 3.2, "
+ "so bulk connect/disconnect cannot work properly and will be turn off.",
+ ompi_mtl_mxm.runtime_version);
+ }
+}
+#endif
+
return OMPI_SUCCESS;
}
diff --git a/ompi/mca/mtl/mxm/mtl_mxm_types.h b/ompi/mca/mtl/mxm/mtl_mxm_types.h
index 2a6410d7a5..42358a8a8f 100644
--- a/ompi/mca/mtl/mxm/mtl_mxm_types.h
+++ b/ompi/mca/mtl/mxm/mtl_mxm_types.h
@@ -34,6 +34,10 @@ typedef struct mca_mtl_mxm_module_t {
mxm_ep_opts_t *mxm_ep_opts;
#if MXM_API >= MXM_VERSION(2,0)
int using_mem_hooks;
+#endif
+#if MXM_API >= MXM_VERSION(3,1)
+ int bulk_connect; /* use bulk connect */
+ int bulk_disconnect; /* use bulk disconnect */
#endif
char* runtime_version;
char* compiletime_version;