MCA/ATOMICS/UCX: workaround for abstraction violation

- some spml calls are marked as inline to exclude cross-module
  dependency
- updated get-key call to get link to local module

Signed-off-by: Sergey Oblomov <sergeyo@mellanox.com>

oshmem/mca/atomic/ucx/Makefile.am

@@ -35,7 +35,8 @@ mcacomponentdir = $(ompilibdir)
 mcacomponent_LTLIBRARIES = $(component_install)
 mca_atomic_ucx_la_SOURCES = $(ucx_sources)
 mca_atomic_ucx_la_LIBADD = $(top_builddir)/oshmem/liboshmem.la \
-	$(atomic_ucx_LIBS)
+	$(atomic_ucx_LIBS) \
+	$(OPAL_TOP_BUILDDIR)/opal/mca/common/ucx/lib@OPAL_LIB_PREFIX@mca_common_ucx.la
 mca_atomic_ucx_la_LDFLAGS = -module -avoid-version $(atomic_ucx_LDFLAGS)
 
 noinst_LTLIBRARIES = $(component_noinst)

oshmem/mca/atomic/ucx/atomic_ucx_cswap.c

@@ -37,7 +37,7 @@ int mca_atomic_ucx_cswap_inner(void *target,
     uint64_t cmp;
 
     val = (4 == nlong) ? *(uint32_t*)value : *(uint64_t*)value;
-    ucx_mkey = mca_spml_ucx_get_mkey(pe, target, (void *)&rva); 
+    ucx_mkey = mca_spml_ucx_get_mkey(pe, target, (void *)&rva, mca_spml_self);
     if (NULL == cond) {
         status_ptr = ucp_atomic_fetch_nb(mca_spml_self->ucp_peers[pe].ucp_conn,
                                          UCP_ATOMIC_FETCH_OP_SWAP, val, prev, nlong,

oshmem/mca/atomic/ucx/atomic_ucx_fadd.c

@@ -40,7 +40,7 @@ int mca_atomic_ucx_fadd(void *target,
         return OSHMEM_ERROR;
     }
 
-    ucx_mkey = mca_spml_ucx_get_mkey(pe, target, (void *)&rva);
+    ucx_mkey = mca_spml_ucx_get_mkey(pe, target, (void *)&rva, mca_spml_self);
     if (NULL == prev) {
         status = ucp_atomic_post(mca_spml_self->ucp_peers[pe].ucp_conn, 
                                  UCP_ATOMIC_POST_OP_ADD, val, nlong, rva,

oshmem/mca/spml/ucx/spml_ucx.c

@@ -330,20 +330,6 @@ error:
 }
 
 
-spml_ucx_mkey_t * mca_spml_ucx_get_mkey_slow(int pe, void *va, void **rva)
-{
-    sshmem_mkey_t *r_mkey;
-
-    r_mkey = mca_memheap_base_get_cached_mkey(pe, va, 0, rva);
-    if (OPAL_UNLIKELY(!r_mkey)) {
-        SPML_ERROR("pe=%d: %p is not address of symmetric variable",
-                   pe, va);
-        oshmem_shmem_abort(-1);
-        return NULL;
-    }
-    return (spml_ucx_mkey_t *)(r_mkey->spml_context);
-}
-
 void mca_spml_ucx_rmkey_free(sshmem_mkey_t *mkey)
 {
     spml_ucx_mkey_t   *ucx_mkey;
@@ -555,7 +541,7 @@ int mca_spml_ucx_get(void *src_addr, size_t size, void *dst_addr, int src)
     ucs_status_t status;
     spml_ucx_mkey_t *ucx_mkey;
 
-    ucx_mkey = mca_spml_ucx_get_mkey(src, src_addr, &rva);
+    ucx_mkey = mca_spml_ucx_get_mkey(src, src_addr, &rva, &mca_spml_ucx);
     status = ucp_get(mca_spml_ucx.ucp_peers[src].ucp_conn, dst_addr, size,
                      (uint64_t)rva, ucx_mkey->rkey);
 
@@ -568,7 +554,7 @@ int mca_spml_ucx_get_nb(void *src_addr, size_t size, void *dst_addr, int src, vo
     ucs_status_t status;
     spml_ucx_mkey_t *ucx_mkey;
 
-    ucx_mkey = mca_spml_ucx_get_mkey(src, src_addr, &rva);
+    ucx_mkey = mca_spml_ucx_get_mkey(src, src_addr, &rva, &mca_spml_ucx);
     status = ucp_get_nbi(mca_spml_ucx.ucp_peers[src].ucp_conn, dst_addr, size,
                      (uint64_t)rva, ucx_mkey->rkey);
 
@@ -581,7 +567,7 @@ int mca_spml_ucx_put(void* dst_addr, size_t size, void* src_addr, int dst)
     ucs_status_t status;
     spml_ucx_mkey_t *ucx_mkey;
 
-    ucx_mkey = mca_spml_ucx_get_mkey(dst, dst_addr, &rva);
+    ucx_mkey = mca_spml_ucx_get_mkey(dst, dst_addr, &rva, &mca_spml_ucx);
     status = ucp_put(mca_spml_ucx.ucp_peers[dst].ucp_conn, src_addr, size,
                      (uint64_t)rva, ucx_mkey->rkey);
 
@@ -594,7 +580,7 @@ int mca_spml_ucx_put_nb(void* dst_addr, size_t size, void* src_addr, int dst, vo
     ucs_status_t status;
     spml_ucx_mkey_t *ucx_mkey;
 
-    ucx_mkey = mca_spml_ucx_get_mkey(dst, dst_addr, &rva);
+    ucx_mkey = mca_spml_ucx_get_mkey(dst, dst_addr, &rva, &mca_spml_ucx);
     status = ucp_put_nbi(mca_spml_ucx.ucp_peers[dst].ucp_conn, src_addr, size,
                      (uint64_t)rva, ucx_mkey->rkey);
 

oshmem/mca/spml/ucx/spml_ucx.h

@@ -121,14 +121,28 @@ extern int mca_spml_ucx_quiet(void);
 extern int spml_ucx_progress(void);
 
 
-spml_ucx_mkey_t * mca_spml_ucx_get_mkey_slow(int pe, void *va, void **rva);
+static inline
+spml_ucx_mkey_t * mca_spml_ucx_get_mkey_slow(int pe, void *va, void **rva)
+{
+    sshmem_mkey_t *r_mkey;
+
+    r_mkey = mca_memheap_base_get_cached_mkey(pe, va, 0, rva);
+    if (OPAL_UNLIKELY(!r_mkey)) {
+        SPML_ERROR("pe=%d: %p is not address of symmetric variable",
+                   pe, va);
+        oshmem_shmem_abort(-1);
+        return NULL;
+    }
+    return (spml_ucx_mkey_t *)(r_mkey->spml_context);
+}
+
 
 static inline spml_ucx_mkey_t * 
-mca_spml_ucx_get_mkey(int pe, void *va, void **rva)
+mca_spml_ucx_get_mkey(int pe, void *va, void **rva, mca_spml_ucx_t* module)
 {
     spml_ucx_cached_mkey_t *mkey;
 
-    mkey = mca_spml_ucx.ucp_peers[pe].mkeys;
+    mkey = module->ucp_peers[pe].mkeys;
     mkey = (spml_ucx_cached_mkey_t *)map_segment_find_va(&mkey->super.super, sizeof(*mkey), va);
     if (OPAL_UNLIKELY(NULL == mkey)) {
         return mca_spml_ucx_get_mkey_slow(pe, va, rva);