Merge pull request #2717 from alex-mikheev/topic/sshmem_ucx

oshmem: sshmem: adds UCX allocator

oshmem/mca/spml/ucx/spml_ucx.c

@@ -420,13 +420,14 @@ sshmem_mkey_t *mca_spml_ucx_register(void* addr,
                                          int *count)
 {
     sshmem_mkey_t *mkeys;
-    ucs_status_t err;
+    ucs_status_t status;
     spml_ucx_mkey_t   *ucx_mkey;
     size_t len;
-    int my_pe = oshmem_my_proc_id();
     ucp_mem_map_params_t mem_map_params;
-    int seg;
+    int segno;
+    map_segment_t *mem_seg;
     unsigned flags;
+    int my_pe = oshmem_my_proc_id();
 
     *count = 0;
     mkeys = (sshmem_mkey_t *) calloc(1, sizeof(*mkeys));
@@ -434,11 +435,14 @@ sshmem_mkey_t *mca_spml_ucx_register(void* addr,
         return NULL;
     }
 
-    seg = memheap_find_segnum(addr);
+    segno   = memheap_find_segnum(addr);
+    mem_seg = memheap_find_seg(segno);
 
-    ucx_mkey = &mca_spml_ucx.ucp_peers[my_pe].mkeys[seg].key;
+    ucx_mkey = &mca_spml_ucx.ucp_peers[my_pe].mkeys[segno].key;
     mkeys[0].spml_context = ucx_mkey;
 
+    /* if possible use mem handle already created by ucx allocator */
+    if (MAP_SEGMENT_ALLOC_UCX != mem_seg->type) {
         flags = 0;
         if (mca_spml_ucx.heap_reg_nb && memheap_is_va_in_segment(addr, HEAP_SEG_INDEX)) {
             flags = UCP_MEM_MAP_NONBLOCK;
@@ -451,14 +455,18 @@ sshmem_mkey_t *mca_spml_ucx_register(void* addr,
         mem_map_params.length     = size;
         mem_map_params.flags      = flags;
 
-    err = ucp_mem_map(mca_spml_ucx.ucp_context, &mem_map_params, &ucx_mkey->mem_h);
+        status = ucp_mem_map(mca_spml_ucx.ucp_context, &mem_map_params, &ucx_mkey->mem_h);
-    if (UCS_OK != err) {
+        if (UCS_OK != status) {
             goto error_out;
         }
 
-    err = ucp_rkey_pack(mca_spml_ucx.ucp_context, ucx_mkey->mem_h, 
+    } else {
+        ucx_mkey->mem_h = (ucp_mem_h)mem_seg->context;
+    }
+
+    status = ucp_rkey_pack(mca_spml_ucx.ucp_context, ucx_mkey->mem_h, 
                            &mkeys[0].u.data, &len); 
-    if (UCS_OK != err) {
+    if (UCS_OK != status) {
         goto error_unmap;
     }
     if (len >= 0xffff) {
@@ -468,18 +476,18 @@ sshmem_mkey_t *mca_spml_ucx_register(void* addr,
         oshmem_shmem_abort(-1);
     }
 
-    err = ucp_ep_rkey_unpack(mca_spml_ucx.ucp_peers[oshmem_group_self->my_pe].ucp_conn,
+    status = ucp_ep_rkey_unpack(mca_spml_ucx.ucp_peers[oshmem_group_self->my_pe].ucp_conn,
                                 mkeys[0].u.data,
                                 &ucx_mkey->rkey);
-    if (UCS_OK != err) {
+    if (UCS_OK != status) {
         SPML_ERROR("failed to unpack rkey");
         goto error_unmap;
     }
 
     mkeys[0].len     = len;
-    mkeys[0].va_base = mem_map_params.address;
+    mkeys[0].va_base = addr;
     *count = 1;
-    mca_spml_ucx_cache_mkey(&mkeys[0], seg, my_pe);
+    mca_spml_ucx_cache_mkey(&mkeys[0], segno, my_pe);
     return mkeys;
 
 error_unmap:
@@ -493,6 +501,7 @@ error_out:
 int mca_spml_ucx_deregister(sshmem_mkey_t *mkeys)
 {
     spml_ucx_mkey_t   *ucx_mkey;
+    map_segment_t *mem_seg;
 
     MCA_SPML_CALL(fence());
     if (!mkeys)
@@ -501,8 +510,12 @@ int mca_spml_ucx_deregister(sshmem_mkey_t *mkeys)
     if (!mkeys[0].spml_context) 
         return OSHMEM_SUCCESS;
 
+    mem_seg = memheap_find_va(mkeys[0].va_base);
+    
+    if (MAP_SEGMENT_ALLOC_UCX != mem_seg->type) {
         ucx_mkey = (spml_ucx_mkey_t *)mkeys[0].spml_context;
         ucp_mem_unmap(mca_spml_ucx.ucp_context, ucx_mkey->mem_h);
+    }
 
     if (0 < mkeys[0].len) {
         ucp_rkey_buffer_release(mkeys[0].u.data);

oshmem/mca/sshmem/base/base.h

@@ -33,10 +33,6 @@ mca_sshmem_segment_create(map_segment_t *ds_buf,
                           const char *file_name,
                           size_t size);
 
-OSHMEM_DECLSPEC int
-mca_sshmem_ds_copy(const map_segment_t *from,
-                   map_segment_t *to);
-
 OSHMEM_DECLSPEC void *
 mca_sshmem_segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey);
 
@@ -148,23 +144,12 @@ OSHMEM_DECLSPEC extern mca_base_framework_t oshmem_sshmem_base_framework;
         "Warning %s:%d - %s()", __SSHMEM_FILE__, __LINE__, __func__, __VA_ARGS__)
 
 
+OSHMEM_DECLSPEC extern void shmem_ds_reset(map_segment_t *ds_buf);
+
 /*
  * Get unique file name
  */
-static inline char * oshmem_get_unique_file_name(uint64_t pe)
+OSHMEM_DECLSPEC extern char * oshmem_get_unique_file_name(uint64_t pe);
-{
-    char *file_name = NULL;
-
-    assert(mca_sshmem_base_backing_file_dir);
-
-    if (NULL == (file_name = calloc(OPAL_PATH_MAX, sizeof(char)))) {
-        return NULL;
-    }
-
-    snprintf(file_name, OPAL_PATH_MAX, "%s/shmem_job_%u_pe_%llu", mca_sshmem_base_backing_file_dir, ORTE_PROC_MY_NAME->jobid, (unsigned long long)pe);
-
-    return file_name;
-}
 
 END_C_DECLS
 

oshmem/mca/sshmem/base/sshmem_base_wrappers.c

@@ -15,7 +15,6 @@
 #include "oshmem/mca/sshmem/sshmem.h"
 #include "oshmem/mca/sshmem/base/base.h"
 
-/* ////////////////////////////////////////////////////////////////////////// */
 int
 mca_sshmem_segment_create(map_segment_t *ds_buf,
                           const char *file_name,
@@ -28,19 +27,6 @@ mca_sshmem_segment_create(map_segment_t *ds_buf,
     return mca_sshmem_base_module->segment_create(ds_buf, file_name, size);
 }
 
-/* ////////////////////////////////////////////////////////////////////////// */
-int
-mca_sshmem_ds_copy(const map_segment_t *from,
-                   map_segment_t *to)
-{
-    if (!mca_sshmem_base_selected) {
-        return OSHMEM_ERROR;
-    }
-
-    return mca_sshmem_base_module->ds_copy(from, to);
-}
-
-/* ////////////////////////////////////////////////////////////////////////// */
 void *
 mca_sshmem_segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
 {
@@ -51,7 +37,6 @@ mca_sshmem_segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
     return mca_sshmem_base_module->segment_attach(ds_buf, mkey);
 }
 
-/* ////////////////////////////////////////////////////////////////////////// */
 int
 mca_sshmem_segment_detach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
 {
@@ -62,7 +47,6 @@ mca_sshmem_segment_detach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
     return mca_sshmem_base_module->segment_detach(ds_buf, mkey);
 }
 
-/* ////////////////////////////////////////////////////////////////////////// */
 int
 mca_sshmem_unlink(map_segment_t *ds_buf)
 {
@@ -73,3 +57,31 @@ mca_sshmem_unlink(map_segment_t *ds_buf)
     return mca_sshmem_base_module->unlink(ds_buf);
 }
 
+
+char * oshmem_get_unique_file_name(uint64_t pe)
+{
+    char *file_name = NULL;
+
+    assert(mca_sshmem_base_backing_file_dir);
+
+    if (NULL == (file_name = calloc(OPAL_PATH_MAX, sizeof(char)))) {
+        return NULL;
+    }
+
+    snprintf(file_name, OPAL_PATH_MAX, "%s/shmem_job_%u_pe_%llu", mca_sshmem_base_backing_file_dir, ORTE_PROC_MY_NAME->jobid, (unsigned long long)pe);
+
+    return file_name;
+}
+
+
+void
+shmem_ds_reset(map_segment_t *ds_buf)
+{
+    MAP_SEGMENT_RESET_FLAGS(ds_buf);
+    ds_buf->seg_id = MAP_SEGMENT_SHM_INVALID;
+    ds_buf->super.va_base = 0;
+    ds_buf->super.va_end = 0;
+    ds_buf->seg_size = 0;
+    ds_buf->type = MAP_SEGMENT_UNKNOWN;
+}
+

oshmem/mca/sshmem/mmap/sshmem_mmap_module.c

@@ -64,10 +64,6 @@ segment_create(map_segment_t *ds_buf,
                const char *file_name,
                size_t size);
 
-static int
-ds_copy(const map_segment_t *from,
-        map_segment_t *to);
-
 static void *
 segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey);
 
@@ -88,7 +84,6 @@ mca_sshmem_mmap_module_t mca_sshmem_mmap_module = {
     {
         module_init,
         segment_create,
-        ds_copy,
         segment_attach,
         segment_detach,
         segment_unlink,
@@ -96,36 +91,6 @@ mca_sshmem_mmap_module_t mca_sshmem_mmap_module = {
     }
 };
 
-/* ////////////////////////////////////////////////////////////////////////// */
-/* private utility functions */
-/* ////////////////////////////////////////////////////////////////////////// */
-
-/* ////////////////////////////////////////////////////////////////////////// */
-/**
- * completely resets the contents of *ds_buf
- */
-static inline void
-shmem_ds_reset(map_segment_t *ds_buf)
-{
-    OPAL_OUTPUT_VERBOSE(
-        (70, oshmem_sshmem_base_framework.framework_output,
-         "%s: %s: shmem_ds_resetting "
-         "(id: %d, size: %lu, name: %s)\n",
-         mca_sshmem_mmap_component.super.base_version.mca_type_name,
-         mca_sshmem_mmap_component.super.base_version.mca_component_name,
-         ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
-    );
-
-    MAP_SEGMENT_RESET_FLAGS(ds_buf);
-    ds_buf->seg_id = MAP_SEGMENT_SHM_INVALID;
-    ds_buf->super.va_base = 0;
-    ds_buf->super.va_end = 0;
-    ds_buf->seg_size = 0;
-    ds_buf->type = MAP_SEGMENT_UNKNOWN;
-    unlink(ds_buf->seg_name);
-    memset(ds_buf->seg_name, '\0', sizeof(ds_buf->seg_name));
-}
-
 /* ////////////////////////////////////////////////////////////////////////// */
 static int
 module_init(void)
@@ -142,30 +107,6 @@ module_finalize(void)
     return OSHMEM_SUCCESS;
 }
 
-/* ////////////////////////////////////////////////////////////////////////// */
-static int
-ds_copy(const map_segment_t *from,
-        map_segment_t *to)
-{
-    memcpy(to, from, sizeof(map_segment_t));
-
-    OPAL_OUTPUT_VERBOSE(
-        (70, oshmem_sshmem_base_framework.framework_output,
-         "%s: %s: ds_copy complete "
-         "from: (id: %d, size: %lu, "
-         "name: %s flags: 0x%02x) "
-         "to: (id: %d, size: %lu, "
-         "name: %s flags: 0x%02x)\n",
-         mca_sshmem_mmap_component.super.base_version.mca_type_name,
-         mca_sshmem_mmap_component.super.base_version.mca_component_name,
-         from->seg_id, (unsigned long)from->seg_size, from->seg_name,
-         from->flags, to->seg_id, (unsigned long)to->seg_size, to->seg_name,
-         to->flags)
-    );
-
-    return OSHMEM_SUCCESS;
-}
-/* ////////////////////////////////////////////////////////////////////////// */
 
 static int
 segment_create(map_segment_t *ds_buf,
@@ -225,11 +166,11 @@ segment_create(map_segment_t *ds_buf,
     OPAL_OUTPUT_VERBOSE(
           (70, oshmem_sshmem_base_framework.framework_output,
            "%s: %s: create %s "
-           "(id: %d, addr: %p size: %lu, name: %s)\n",
+           "(id: %d, addr: %p size: %lu)\n",
            mca_sshmem_mmap_component.super.base_version.mca_type_name,
            mca_sshmem_mmap_component.super.base_version.mca_component_name,
            (rc ? "failure" : "successful"),
-           ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
+           ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size)
       );
 
     return rc;
@@ -316,10 +257,10 @@ segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
     OPAL_OUTPUT_VERBOSE(
         (70, oshmem_sshmem_base_framework.framework_output,
          "%s: %s: attach successful "
-            "(id: %d, addr: %p size: %lu, name: %s | va_base: 0x%p len: %d key %llx)\n",
+            "(id: %d, addr: %p size: %lu | va_base: 0x%p len: %d key %llx)\n",
             mca_sshmem_mmap_component.super.base_version.mca_type_name,
             mca_sshmem_mmap_component.super.base_version.mca_component_name,
-            ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name,
+            ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size,
             mkey->va_base, mkey->len, (unsigned long long)mkey->u.key)
         );
 
@@ -338,10 +279,10 @@ segment_detach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
     OPAL_OUTPUT_VERBOSE(
         (70, oshmem_sshmem_base_framework.framework_output,
          "%s: %s: detaching "
-            "(id: %d, addr: %p size: %lu, name: %s)\n",
+            "(id: %d, addr: %p size: %lu)\n",
             mca_sshmem_mmap_component.super.base_version.mca_type_name,
             mca_sshmem_mmap_component.super.base_version.mca_component_name,
-            ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
+            ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size)
     );
 
     munmap((void *)ds_buf->super.va_base, ds_buf->seg_size);
@@ -363,10 +304,10 @@ segment_unlink(map_segment_t *ds_buf)
     OPAL_OUTPUT_VERBOSE(
         (70, oshmem_sshmem_base_framework.framework_output,
          "%s: %s: unlinking "
-            "(id: %d, addr: %p size: %lu, name: %s)\n",
+            "(id: %d, addr: %p size: %lu)\n",
             mca_sshmem_mmap_component.super.base_version.mca_type_name,
             mca_sshmem_mmap_component.super.base_version.mca_component_name,
-            ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
+            ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size)
     );
 
     /* don't completely reset.  in particular, only reset

oshmem/mca/sshmem/sshmem.h

@@ -21,7 +21,6 @@
  *
  * - module_init
  * - segment_create
- * - ds_copy
  * - segment_attach
  * - segment_detach
  * - unlink
@@ -74,20 +73,6 @@ typedef struct mca_sshmem_base_component_2_0_0_t mca_sshmem_base_component_t;
 typedef int
 (*mca_sshmem_base_module_init_fn_t)(void);
 
-/**
- * copy shmem data structure information pointed to by from to the structure
- * pointed to by to.
- *
- * @param from  source pointer (IN).
- *
- * @param to    destination pointer (OUT).
- *
- * @return OSHMEM_SUCCESS on success.
- */
-typedef int
-(*mca_sshmem_base_ds_copy_fn_t)(const map_segment_t *from,
-                                map_segment_t *to);
-
 /**
  * create a new shared memory segment and initialize members in structure
  * pointed to by ds_buf.
@@ -153,7 +138,6 @@ typedef int (*mca_sshmem_base_module_finalize_fn_t)(void);
 struct mca_sshmem_base_module_2_0_0_t {
     mca_sshmem_base_module_init_fn_t            module_init;
     mca_sshmem_base_module_segment_create_fn_t  segment_create;
-    mca_sshmem_base_ds_copy_fn_t                ds_copy;
     mca_sshmem_base_module_segment_attach_fn_t  segment_attach;
     mca_sshmem_base_module_segment_detach_fn_t  segment_detach;
     mca_sshmem_base_module_unlink_fn_t          unlink;

oshmem/mca/sshmem/sshmem_types.h

@@ -72,6 +72,7 @@ typedef enum {
     MAP_SEGMENT_ALLOC_SHM,
     MAP_SEGMENT_ALLOC_IBV,
     MAP_SEGMENT_ALLOC_IBV_NOSHMR,
+    MAP_SEGMENT_ALLOC_UCX,
     MAP_SEGMENT_UNKNOWN
 } segment_type_t;
 
@@ -112,9 +113,10 @@ typedef struct map_segment {
     sshmem_mkey_t       *mkeys;          /* includes local segment bases in va_base */
     segment_flag_t       flags;          /* enable/disable flag */
     int                  seg_id;
-    char                 seg_name[OPAL_PATH_MAX];
     size_t               seg_size;       /* length of the segment */
     segment_type_t       type;           /* type of the segment */
+    void                *context;        /* allocator can use this field to store
+                                            its own private data */
 } map_segment_t;
 
 END_C_DECLS

oshmem/mca/sshmem/sysv/sshmem_sysv_module.c

@@ -62,10 +62,6 @@ segment_create(map_segment_t *ds_buf,
                const char *file_name,
                size_t size);
 
-static int
-ds_copy(const map_segment_t *from,
-        map_segment_t *to);
-
 static void *
 segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey);
 
@@ -84,7 +80,6 @@ mca_sshmem_sysv_module_t mca_sshmem_sysv_module = {
     {
         module_init,
         segment_create,
-        ds_copy,
         segment_attach,
         segment_detach,
         segment_unlink,
@@ -92,34 +87,6 @@ mca_sshmem_sysv_module_t mca_sshmem_sysv_module = {
     }
 };
 
-/* ////////////////////////////////////////////////////////////////////////// */
-/* private utility functions */
-/* ////////////////////////////////////////////////////////////////////////// */
-
-/* ////////////////////////////////////////////////////////////////////////// */
-/**
- * completely resets the contents of *ds_buf
- */
-static inline void
-shmem_ds_reset(map_segment_t *ds_buf)
-{
-    OPAL_OUTPUT_VERBOSE(
-          (70, oshmem_sshmem_base_framework.framework_output,
-           "%s: %s: shmem_ds_resetting "
-           "(id: %d, size: %lu, name: %s)\n",
-           mca_sshmem_sysv_component.super.base_version.mca_type_name,
-           mca_sshmem_sysv_component.super.base_version.mca_component_name,
-           ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
-      );
-
-      MAP_SEGMENT_RESET_FLAGS(ds_buf);
-      ds_buf->seg_id = MAP_SEGMENT_SHM_INVALID;
-      ds_buf->super.va_base = 0;
-      ds_buf->super.va_end = 0;
-      ds_buf->seg_size = 0;
-      ds_buf->type = MAP_SEGMENT_UNKNOWN;
-      memset(ds_buf->seg_name, '\0', sizeof(ds_buf->seg_name));
-}
 
 /* ////////////////////////////////////////////////////////////////////////// */
 static int
@@ -137,29 +104,6 @@ module_finalize(void)
     return OSHMEM_SUCCESS;
 }
 
-/* ////////////////////////////////////////////////////////////////////////// */
-static int
-ds_copy(const map_segment_t *from,
-        map_segment_t *to)
-{
-    memcpy(to, from, sizeof(map_segment_t));
-
-    OPAL_OUTPUT_VERBOSE(
-        (70, oshmem_sshmem_base_framework.framework_output,
-         "%s: %s: ds_copy complete "
-         "from: (id: %d, size: %lu, "
-         "name: %s flags: 0x%02x) "
-         "to: (id: %d, size: %lu, "
-         "name: %s flags: 0x%02x)\n",
-         mca_sshmem_sysv_component.super.base_version.mca_type_name,
-         mca_sshmem_sysv_component.super.base_version.mca_component_name,
-         from->seg_id, (unsigned long)from->seg_size, from->seg_name,
-         from->flags, to->seg_id, (unsigned long)to->seg_size, to->seg_name,
-         to->flags)
-    );
-
-    return OSHMEM_SUCCESS;
-}
 
 /* ////////////////////////////////////////////////////////////////////////// */
 static int
@@ -232,11 +176,11 @@ segment_create(map_segment_t *ds_buf,
     OPAL_OUTPUT_VERBOSE(
           (70, oshmem_sshmem_base_framework.framework_output,
            "%s: %s: create %s "
-           "(id: %d, addr: %p size: %lu, name: %s)\n",
+           "(id: %d, addr: %p size: %lu)\n",
            mca_sshmem_sysv_component.super.base_version.mca_type_name,
            mca_sshmem_sysv_component.super.base_version.mca_component_name,
            (rc ? "failure" : "successful"),
-           ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
+           ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size)
       );
 
     return rc;
@@ -261,10 +205,10 @@ segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
     OPAL_OUTPUT_VERBOSE(
         (70, oshmem_sshmem_base_framework.framework_output,
          "%s: %s: attach successful "
-            "(id: %d, addr: %p size: %lu, name: %s | va_base: 0x%p len: %d key %llx)\n",
+            "(id: %d, addr: %p size: %lu | va_base: 0x%p len: %d key %llx)\n",
             mca_sshmem_sysv_component.super.base_version.mca_type_name,
             mca_sshmem_sysv_component.super.base_version.mca_component_name,
-            ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name,
+            ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size,
             mkey->va_base, mkey->len, (unsigned long long)mkey->u.key)
     );
 
@@ -283,10 +227,10 @@ segment_detach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
     OPAL_OUTPUT_VERBOSE(
         (70, oshmem_sshmem_base_framework.framework_output,
          "%s: %s: detaching "
-            "(id: %d, addr: %p size: %lu, name: %s)\n",
+            "(id: %d, addr: %p size: %lu)\n",
             mca_sshmem_sysv_component.super.base_version.mca_type_name,
             mca_sshmem_sysv_component.super.base_version.mca_component_name,
-            ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
+            ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size)
     );
 
     if (ds_buf->seg_id != MAP_SEGMENT_SHM_INVALID) {
@@ -318,10 +262,10 @@ segment_unlink(map_segment_t *ds_buf)
     OPAL_OUTPUT_VERBOSE(
         (70, oshmem_sshmem_base_framework.framework_output,
          "%s: %s: unlinking "
-         "(id: %d, size: %lu, name: %s)\n",
+         "(id: %d, size: %lu)\n",
          mca_sshmem_sysv_component.super.base_version.mca_type_name,
          mca_sshmem_sysv_component.super.base_version.mca_component_name,
-         ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
+         ds_buf->seg_id, (unsigned long)ds_buf->seg_size)
     );
 
     /* don't completely reset.  in particular, only reset

oshmem/mca/sshmem/ucx/Makefile.am

@@ -0,0 +1,40 @@
+# Copyright (c) 2014      Mellanox Technologies, Inc.
+#                         All rights reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+#dist_oshmemdata_DATA = help-oshmem-sshmem-mmap.txt
+
+AM_CPPFLAGS = $(sshmem_ucx_CPPFLAGS)
+
+sources = \
+        sshmem_ucx.h \
+        sshmem_ucx_component.c \
+        sshmem_ucx_module.c
+
+# Make the output library in this directory, and name it either
+# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
+# (for static builds).
+
+if MCA_BUILD_oshmem_sshmem_ucx_DSO
+component_noinst =
+component_install = mca_sshmem_ucx.la
+else
+component_noinst = libmca_sshmem_ucx.la
+component_install =
+endif
+
+mcacomponentdir = $(oshmemlibdir)
+mcacomponent_LTLIBRARIES = $(component_install)
+mca_sshmem_ucx_la_SOURCES = $(sources)
+mca_sshmem_ucx_la_LDFLAGS = -module -avoid-version $(sshmem_ucx_LDFLAGS)
+mca_sshmem_ucx_la_LIBADD = $(sshmem_ucx_LIBS)
+
+noinst_LTLIBRARIES = $(component_noinst)
+libmca_sshmem_ucx_la_SOURCES =$(sources)
+libmca_sshmem_ucx_la_LDFLAGS = -module -avoid-version $(sshmem_ucx_LDFLAGS)
+

oshmem/mca/sshmem/ucx/configure.m4

@@ -0,0 +1,32 @@
+/*
+ * Copyright (c) 2017      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+# MCA_oshmem_sshmem_ucx_CONFIG([action-if-can-compile],
+#                    [action-if-cant-compile])
+# ------------------------------------------------
+AC_DEFUN([MCA_oshmem_sshmem_ucx_CONFIG],[
+    AC_CONFIG_FILES([oshmem/mca/sshmem/ucx/Makefile])
+
+    OMPI_CHECK_UCX([sshmem_ucx],
+                   [sshmem_ucx_happy="yes"],
+                   [sshmem_ucx_happy="no"])
+
+    AS_IF([test "$sshmem_ucx_happy" = "yes"],
+          [$1],
+          [$2])
+
+
+    # substitute in the things needed to build ucx 
+    AC_SUBST([sshmem_ucx_CFLAGS])
+    AC_SUBST([sshmem_ucx_CPPFLAGS])
+    AC_SUBST([sshmem_ucx_LDFLAGS])
+    AC_SUBST([sshmem_ucx_LIBS])
+])dnl
+

oshmem/mca/sshmem/ucx/sshmem_ucx.h

@@ -0,0 +1,40 @@
+/*
+ * Copyright (c) 2017      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef MCA_SSHMEM_UCX_EXPORT_H
+#define MCA_SSHMEM_UCX_EXPORT_H
+
+#include "oshmem_config.h"
+
+#include "oshmem/mca/sshmem/sshmem.h"
+
+BEGIN_C_DECLS
+
+/**
+ * globally exported variable to hold the ucx component.
+ */
+typedef struct mca_sshmem_ucx_component_t {
+    /* base component struct */
+    mca_sshmem_base_component_t super;
+    /* priority for ucx component */
+    int priority;
+} mca_sshmem_ucx_component_t;
+
+OSHMEM_MODULE_DECLSPEC extern mca_sshmem_ucx_component_t
+mca_sshmem_ucx_component;
+
+typedef struct mca_sshmem_ucx_module_t {
+    mca_sshmem_base_module_t super;
+} mca_sshmem_ucx_module_t;
+extern mca_sshmem_ucx_module_t mca_sshmem_ucx_module;
+
+END_C_DECLS
+
+#endif /* MCA_SHMEM_UCX_EXPORT_H */

oshmem/mca/sshmem/ucx/sshmem_ucx_component.c

@@ -0,0 +1,125 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2017      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "oshmem_config.h"
+
+#include "opal/constants.h"
+
+#include "oshmem/mca/sshmem/sshmem.h"
+#include "oshmem/mca/sshmem/base/base.h"
+#include "oshmem/mca/spml/base/base.h"
+
+#include "sshmem_ucx.h"
+
+/**
+ * public string showing the shmem ompi_ucx component version number
+ */
+const char *mca_sshmem_ucx_component_version_string =
+    "OSHMEM ucx sshmem MCA component version " OSHMEM_VERSION;
+
+
+/**
+ * local functions
+ */
+static int ucx_register(void);
+static int ucx_open(void);
+static int ucx_close(void);
+static int ucx_query(mca_base_module_t **module, int *priority);
+static int ucx_runtime_query(mca_base_module_t **module,
+                             int *priority,
+                             const char *hint);
+
+/**
+ * instantiate the public struct with all of our public information
+ * and pointers to our public functions in it
+ */
+mca_sshmem_ucx_component_t mca_sshmem_ucx_component = {
+    /* ////////////////////////////////////////////////////////////////////// */
+    /* super */
+    /* ////////////////////////////////////////////////////////////////////// */
+    {
+        /**
+         * common MCA component data
+         */
+        .base_version = {
+            MCA_SSHMEM_BASE_VERSION_2_0_0,
+
+            /* component name and version */
+            .mca_component_name = "ucx",
+            MCA_BASE_MAKE_VERSION(component, OSHMEM_MAJOR_VERSION, OSHMEM_MINOR_VERSION,
+                                  OSHMEM_RELEASE_VERSION),
+
+            .mca_open_component = ucx_open,
+            .mca_close_component = ucx_close,
+            .mca_query_component = ucx_query,
+            .mca_register_component_params = ucx_register,
+        },
+        /* MCA v2.0.0 component meta data */
+        .base_data = {
+            /* the component is checkpoint ready */
+            MCA_BASE_METADATA_PARAM_CHECKPOINT
+        },
+        .runtime_query = ucx_runtime_query,
+    },
+};
+
+static int
+ucx_runtime_query(mca_base_module_t **module,
+                   int *priority,
+                   const char *hint)
+{
+    /* check that spml ucx was selected. Otherwise disqualify */
+    if (strcmp(mca_spml_base_selected_component.spmlm_version.mca_component_name, "ucx")) {
+        *module = NULL;
+        return OSHMEM_ERR_NOT_AVAILABLE;
+    }
+
+    *priority = mca_sshmem_ucx_component.priority;
+    *module = (mca_base_module_t *)&mca_sshmem_ucx_module.super;
+    return OPAL_SUCCESS;
+}
+
+static int
+ucx_register(void)
+{
+    /* (default) priority - set high to make ucx the default */
+    mca_sshmem_ucx_component.priority = 100;
+    mca_base_component_var_register (&mca_sshmem_ucx_component.super.base_version,
+                                     "priority", "Priority for sshmem ucx "
+                                     "component (default: 100)", MCA_BASE_VAR_TYPE_INT,
+                                     NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
+                                     OPAL_INFO_LVL_3,
+                                     MCA_BASE_VAR_SCOPE_ALL_EQ,
+                                     &mca_sshmem_ucx_component.priority);
+
+    return OSHMEM_SUCCESS;
+}
+
+static int
+ucx_open(void)
+{
+    return OSHMEM_SUCCESS;
+}
+
+static int
+ucx_query(mca_base_module_t **module, int *priority)
+{
+    *priority = mca_sshmem_ucx_component.priority;
+    *module = (mca_base_module_t *)&mca_sshmem_ucx_module.super;
+    return OSHMEM_SUCCESS;
+}
+
+static int
+ucx_close(void)
+{
+    return OSHMEM_SUCCESS;
+}
+

oshmem/mca/sshmem/ucx/sshmem_ucx_module.c

@@ -0,0 +1,190 @@
+/*
+ * Copyright (c) 2017      Mellanox Technologies, Inc.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "oshmem_config.h"
+
+#include "opal/constants.h"
+#include "opal/util/output.h"
+#include "opal/util/path.h"
+#include "opal/util/show_help.h"
+#include "orte/util/show_help.h"
+
+#include "oshmem/proc/proc.h"
+#include "oshmem/mca/sshmem/sshmem.h"
+#include "oshmem/mca/sshmem/base/base.h"
+#include "oshmem/util/oshmem_util.h"
+#include "oshmem/mca/spml/ucx/spml_ucx.h"
+
+#include "sshmem_ucx.h"
+
+/* ////////////////////////////////////////////////////////////////////////// */
+/*local functions */
+/* local functions */
+static int
+module_init(void);
+
+static int
+segment_create(map_segment_t *ds_buf,
+               const char *file_name,
+               size_t size);
+
+static void *
+segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey);
+
+static int
+segment_detach(map_segment_t *ds_buf, sshmem_mkey_t *mkey);
+
+static int
+segment_unlink(map_segment_t *ds_buf);
+
+static int
+module_finalize(void);
+
+/*
+ * ucx shmem module
+ */
+mca_sshmem_ucx_module_t mca_sshmem_ucx_module = {
+    /* super */
+    {
+        module_init,
+        segment_create,
+        segment_attach,
+        segment_detach,
+        segment_unlink,
+        module_finalize
+    }
+};
+
+static int
+module_init(void)
+{
+    /* nothing to do */
+    return OSHMEM_SUCCESS;
+}
+
+/* ////////////////////////////////////////////////////////////////////////// */
+static int
+module_finalize(void)
+{
+    /* nothing to do */
+    return OSHMEM_SUCCESS;
+}
+
+/* ////////////////////////////////////////////////////////////////////////// */
+
+static int
+segment_create(map_segment_t *ds_buf,
+               const char *file_name,
+               size_t size)
+{
+    int rc = OSHMEM_SUCCESS;
+    mca_spml_ucx_t *spml = (mca_spml_ucx_t *)mca_spml.self;
+    ucp_mem_map_params_t mem_map_params;
+    ucp_mem_h mem_h;
+    ucs_status_t status;
+
+    assert(ds_buf);
+
+    /* init the contents of map_segment_t */
+    shmem_ds_reset(ds_buf);
+
+    mem_map_params.field_mask = UCP_MEM_MAP_PARAM_FIELD_ADDRESS |
+                                UCP_MEM_MAP_PARAM_FIELD_LENGTH |
+                                UCP_MEM_MAP_PARAM_FIELD_FLAGS;
+
+    mem_map_params.address    = (void *)mca_sshmem_base_start_address;
+    mem_map_params.length     = size;
+    mem_map_params.flags      = UCP_MEM_MAP_ALLOCATE|UCP_MEM_MAP_FIXED;
+
+    if (spml->heap_reg_nb) {
+        mem_map_params.flags |= UCP_MEM_MAP_NONBLOCK;
+    }
+
+    status = ucp_mem_map(spml->ucp_context, &mem_map_params, &mem_h);
+    if (UCS_OK != status) {
+        rc = OSHMEM_ERROR;
+        goto out;
+    }
+
+    ds_buf->super.va_base = mem_map_params.address;
+    ds_buf->seg_size      = size;
+    ds_buf->super.va_end  = (void*)((uintptr_t)ds_buf->super.va_base + ds_buf->seg_size);
+    ds_buf->context       = mem_h;
+    ds_buf->type          = MAP_SEGMENT_ALLOC_UCX;
+
+out:
+    OPAL_OUTPUT_VERBOSE(
+          (70, oshmem_sshmem_base_framework.framework_output,
+           "%s: %s: create %s "
+           "(id: %d, addr: %p size: %lu)\n",
+           mca_sshmem_ucx_component.super.base_version.mca_type_name,
+           mca_sshmem_ucx_component.super.base_version.mca_component_name,
+           (rc ? "failure" : "successful"),
+           ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size)
+      );
+    return rc;
+}
+
+static void *
+segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
+{
+    assert(ds_buf);
+    assert(mkey->va_base == 0);
+
+    OPAL_OUTPUT((oshmem_sshmem_base_framework.framework_output,
+                "can not attach to ucx segment"));
+    oshmem_shmem_abort(-1);
+    return NULL;
+}
+
+static int
+segment_detach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
+{
+    OPAL_OUTPUT_VERBOSE(
+        (70, oshmem_sshmem_base_framework.framework_output,
+         "%s: %s: detaching "
+            "(id: %d, addr: %p size: %lu)\n",
+            mca_sshmem_ucx_component.super.base_version.mca_type_name,
+            mca_sshmem_ucx_component.super.base_version.mca_component_name,
+            ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size)
+    );
+
+    /* reset the contents of the map_segment_t associated with this
+     * shared memory segment.
+     */
+    shmem_ds_reset(ds_buf);
+
+    return OSHMEM_SUCCESS;
+}
+
+static int
+segment_unlink(map_segment_t *ds_buf)
+{
+    mca_spml_ucx_t *spml = (mca_spml_ucx_t *)mca_spml.self;
+
+    assert(ds_buf);
+
+    ucp_mem_unmap(spml->ucp_context, (ucp_mem_h)ds_buf->context);
+
+    OPAL_OUTPUT_VERBOSE(
+        (70, oshmem_sshmem_base_framework.framework_output,
+         "%s: %s: unlinking "
+            "(id: %d, addr: %p size: %lu)\n",
+            mca_sshmem_ucx_component.super.base_version.mca_type_name,
+            mca_sshmem_ucx_component.super.base_version.mca_component_name,
+            ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size)
+    );
+
+    ds_buf->seg_id = MAP_SEGMENT_SHM_INVALID;
+    MAP_SEGMENT_INVALIDATE(ds_buf);
+
+    return OSHMEM_SUCCESS;
+}
+

oshmem/mca/sshmem/verbs/sshmem_verbs_module.c

@@ -56,10 +56,6 @@ segment_create(map_segment_t *ds_buf,
                const char *file_name,
                size_t size);
 
-static int
-ds_copy(const map_segment_t *from,
-        map_segment_t *to);
-
 static void *
 segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey);
 
@@ -80,7 +76,6 @@ mca_sshmem_verbs_module_t mca_sshmem_verbs_module = {
     {
         module_init,
         segment_create,
-        ds_copy,
         segment_attach,
         segment_detach,
         segment_unlink,
@@ -88,35 +83,6 @@ mca_sshmem_verbs_module_t mca_sshmem_verbs_module = {
     }
 };
 
-/* ////////////////////////////////////////////////////////////////////////// */
-/* private utility functions */
-/* ////////////////////////////////////////////////////////////////////////// */
-
-/* ////////////////////////////////////////////////////////////////////////// */
-/**
- * completely resets the contents of *ds_buf
- */
-static inline void
-shmem_ds_reset(map_segment_t *ds_buf)
-{
-  OPAL_OUTPUT_VERBOSE(
-        (70, oshmem_sshmem_base_framework.framework_output,
-         "%s: %s: shmem_ds_resetting "
-         "(id: %d, size: %lu, name: %s)\n",
-         mca_sshmem_verbs_component.super.base_version.mca_type_name,
-         mca_sshmem_verbs_component.super.base_version.mca_component_name,
-         ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
-    );
-
-    MAP_SEGMENT_RESET_FLAGS(ds_buf);
-    ds_buf->seg_id = MAP_SEGMENT_SHM_INVALID;
-    ds_buf->super.va_base = 0;
-    ds_buf->super.va_end = 0;
-    ds_buf->seg_size = 0;
-    ds_buf->type = MAP_SEGMENT_UNKNOWN;
-    memset(ds_buf->seg_name, '\0', sizeof(ds_buf->seg_name));
-}
-
 /* ////////////////////////////////////////////////////////////////////////// */
 static int
 module_init(void)
@@ -133,29 +99,6 @@ module_finalize(void)
     return OSHMEM_SUCCESS;
 }
 
-/* ////////////////////////////////////////////////////////////////////////// */
-static int
-ds_copy(const map_segment_t *from,
-        map_segment_t *to)
-{
-    memcpy(to, from, sizeof(map_segment_t));
-
-    OPAL_OUTPUT_VERBOSE(
-        (70, oshmem_sshmem_base_framework.framework_output,
-         "%s: %s: ds_copy complete "
-         "from: (id: %d, size: %lu, "
-         "name: %s flags: 0x%02x) "
-         "to: (id: %d, size: %lu, "
-         "name: %s flags: 0x%02x)\n",
-         mca_sshmem_verbs_component.super.base_version.mca_type_name,
-         mca_sshmem_verbs_component.super.base_version.mca_component_name,
-         from->seg_id, (unsigned long)from->seg_size, from->seg_name,
-         from->flags, to->seg_id, (unsigned long)to->seg_size, to->seg_name,
-         to->flags)
-    );
-
-    return OSHMEM_SUCCESS;
-}
 
 /* ////////////////////////////////////////////////////////////////////////// */
 static int
@@ -329,11 +272,11 @@ segment_create(map_segment_t *ds_buf,
     OPAL_OUTPUT_VERBOSE(
           (70, oshmem_sshmem_base_framework.framework_output,
            "%s: %s: create %s "
-           "(id: %d, addr: %p size: %lu, name: %s)\n",
+           "(id: %d, addr: %p size: %lu)\n",
            mca_sshmem_verbs_component.super.base_version.mca_type_name,
            mca_sshmem_verbs_component.super.base_version.mca_component_name,
            (rc ? "failure" : "successful"),
-           ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
+           ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size)
       );
 
     return rc;
@@ -395,10 +338,10 @@ segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
     OPAL_OUTPUT_VERBOSE(
         (70, oshmem_sshmem_base_framework.framework_output,
          "%s: %s: attach successful "
-            "(id: %d, addr: %p size: %lu, name: %s | va_base: 0x%p len: %d key %llx)\n",
+            "(id: %d, addr: %p size: %lu | va_base: 0x%p len: %d key %llx)\n",
             mca_sshmem_verbs_component.super.base_version.mca_type_name,
             mca_sshmem_verbs_component.super.base_version.mca_component_name,
-            ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name,
+            ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size,
             mkey->va_base, mkey->len, (unsigned long long)mkey->u.key)
     );
 
@@ -419,10 +362,10 @@ segment_detach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
     OPAL_OUTPUT_VERBOSE(
         (70, oshmem_sshmem_base_framework.framework_output,
          "%s: %s: detaching "
-            "(id: %d, addr: %p size: %lu, name: %s)\n",
+            "(id: %d, addr: %p size: %lu)\n",
             mca_sshmem_verbs_component.super.base_version.mca_type_name,
             mca_sshmem_verbs_component.super.base_version.mca_component_name,
-            ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
+            ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size)
     );
 
     if (device) {
@@ -498,10 +441,10 @@ segment_unlink(map_segment_t *ds_buf)
     OPAL_OUTPUT_VERBOSE(
         (70, oshmem_sshmem_base_framework.framework_output,
          "%s: %s: unlinking "
-            "(id: %d, addr: %p size: %lu, name: %s)\n",
+            "(id: %d, addr: %p size: %lu)\n",
             mca_sshmem_verbs_component.super.base_version.mca_type_name,
             mca_sshmem_verbs_component.super.base_version.mca_component_name,
-            ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
+            ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size)
     );
 
     /* don't completely reset.  in particular, only reset