From e038e3f9e0aa95083297d3d479abb7f29cedeee2 Mon Sep 17 00:00:00 2001 From: Alex Mikheev Date: Thu, 12 Jan 2017 16:18:02 +0200 Subject: [PATCH 1/4] oshmem: sshmem: code cleaunp The commit removes unused code and interface function, moves common code to the base. Signed-off-by: Alex Mikheev --- oshmem/mca/sshmem/base/base.h | 21 +----- oshmem/mca/sshmem/base/sshmem_base_wrappers.c | 44 +++++++---- oshmem/mca/sshmem/mmap/sshmem_mmap_module.c | 75 ++----------------- oshmem/mca/sshmem/sshmem.h | 16 ---- oshmem/mca/sshmem/sshmem_types.h | 4 +- oshmem/mca/sshmem/sysv/sshmem_sysv_module.c | 72 ++---------------- oshmem/mca/sshmem/verbs/sshmem_verbs_module.c | 73 ++---------------- 7 files changed, 58 insertions(+), 247 deletions(-) diff --git a/oshmem/mca/sshmem/base/base.h b/oshmem/mca/sshmem/base/base.h index 27c86014fd..ea44ff5096 100644 --- a/oshmem/mca/sshmem/base/base.h +++ b/oshmem/mca/sshmem/base/base.h @@ -33,10 +33,6 @@ mca_sshmem_segment_create(map_segment_t *ds_buf, const char *file_name, size_t size); -OSHMEM_DECLSPEC int -mca_sshmem_ds_copy(const map_segment_t *from, - map_segment_t *to); - OSHMEM_DECLSPEC void * mca_sshmem_segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey); @@ -148,23 +144,12 @@ OSHMEM_DECLSPEC extern mca_base_framework_t oshmem_sshmem_base_framework; "Warning %s:%d - %s()", __SSHMEM_FILE__, __LINE__, __func__, __VA_ARGS__) +OSHMEM_DECLSPEC extern void shmem_ds_reset(map_segment_t *ds_buf); + /* * Get unique file name */ -static inline char * oshmem_get_unique_file_name(uint64_t pe) -{ - char *file_name = NULL; - - assert(mca_sshmem_base_backing_file_dir); - - if (NULL == (file_name = calloc(OPAL_PATH_MAX, sizeof(char)))) { - return NULL; - } - - snprintf(file_name, OPAL_PATH_MAX, "%s/shmem_job_%u_pe_%llu", mca_sshmem_base_backing_file_dir, ORTE_PROC_MY_NAME->jobid, (unsigned long long)pe); - - return file_name; -} +OSHMEM_DECLSPEC extern char * oshmem_get_unique_file_name(uint64_t pe); END_C_DECLS diff --git a/oshmem/mca/sshmem/base/sshmem_base_wrappers.c b/oshmem/mca/sshmem/base/sshmem_base_wrappers.c index 0c01a37fcc..d70490cb4e 100644 --- a/oshmem/mca/sshmem/base/sshmem_base_wrappers.c +++ b/oshmem/mca/sshmem/base/sshmem_base_wrappers.c @@ -15,7 +15,6 @@ #include "oshmem/mca/sshmem/sshmem.h" #include "oshmem/mca/sshmem/base/base.h" -/* ////////////////////////////////////////////////////////////////////////// */ int mca_sshmem_segment_create(map_segment_t *ds_buf, const char *file_name, @@ -28,19 +27,6 @@ mca_sshmem_segment_create(map_segment_t *ds_buf, return mca_sshmem_base_module->segment_create(ds_buf, file_name, size); } -/* ////////////////////////////////////////////////////////////////////////// */ -int -mca_sshmem_ds_copy(const map_segment_t *from, - map_segment_t *to) -{ - if (!mca_sshmem_base_selected) { - return OSHMEM_ERROR; - } - - return mca_sshmem_base_module->ds_copy(from, to); -} - -/* ////////////////////////////////////////////////////////////////////////// */ void * mca_sshmem_segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey) { @@ -51,7 +37,6 @@ mca_sshmem_segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey) return mca_sshmem_base_module->segment_attach(ds_buf, mkey); } -/* ////////////////////////////////////////////////////////////////////////// */ int mca_sshmem_segment_detach(map_segment_t *ds_buf, sshmem_mkey_t *mkey) { @@ -62,7 +47,6 @@ mca_sshmem_segment_detach(map_segment_t *ds_buf, sshmem_mkey_t *mkey) return mca_sshmem_base_module->segment_detach(ds_buf, mkey); } -/* ////////////////////////////////////////////////////////////////////////// */ int mca_sshmem_unlink(map_segment_t *ds_buf) { @@ -73,3 +57,31 @@ mca_sshmem_unlink(map_segment_t *ds_buf) return mca_sshmem_base_module->unlink(ds_buf); } + +char * oshmem_get_unique_file_name(uint64_t pe) +{ + char *file_name = NULL; + + assert(mca_sshmem_base_backing_file_dir); + + if (NULL == (file_name = calloc(OPAL_PATH_MAX, sizeof(char)))) { + return NULL; + } + + snprintf(file_name, OPAL_PATH_MAX, "%s/shmem_job_%u_pe_%llu", mca_sshmem_base_backing_file_dir, ORTE_PROC_MY_NAME->jobid, (unsigned long long)pe); + + return file_name; +} + + +void +shmem_ds_reset(map_segment_t *ds_buf) +{ + MAP_SEGMENT_RESET_FLAGS(ds_buf); + ds_buf->seg_id = MAP_SEGMENT_SHM_INVALID; + ds_buf->super.va_base = 0; + ds_buf->super.va_end = 0; + ds_buf->seg_size = 0; + ds_buf->type = MAP_SEGMENT_UNKNOWN; +} + diff --git a/oshmem/mca/sshmem/mmap/sshmem_mmap_module.c b/oshmem/mca/sshmem/mmap/sshmem_mmap_module.c index 07de1e6d58..9a30322107 100644 --- a/oshmem/mca/sshmem/mmap/sshmem_mmap_module.c +++ b/oshmem/mca/sshmem/mmap/sshmem_mmap_module.c @@ -64,10 +64,6 @@ segment_create(map_segment_t *ds_buf, const char *file_name, size_t size); -static int -ds_copy(const map_segment_t *from, - map_segment_t *to); - static void * segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey); @@ -88,7 +84,6 @@ mca_sshmem_mmap_module_t mca_sshmem_mmap_module = { { module_init, segment_create, - ds_copy, segment_attach, segment_detach, segment_unlink, @@ -96,36 +91,6 @@ mca_sshmem_mmap_module_t mca_sshmem_mmap_module = { } }; -/* ////////////////////////////////////////////////////////////////////////// */ -/* private utility functions */ -/* ////////////////////////////////////////////////////////////////////////// */ - -/* ////////////////////////////////////////////////////////////////////////// */ -/** - * completely resets the contents of *ds_buf - */ -static inline void -shmem_ds_reset(map_segment_t *ds_buf) -{ - OPAL_OUTPUT_VERBOSE( - (70, oshmem_sshmem_base_framework.framework_output, - "%s: %s: shmem_ds_resetting " - "(id: %d, size: %lu, name: %s)\n", - mca_sshmem_mmap_component.super.base_version.mca_type_name, - mca_sshmem_mmap_component.super.base_version.mca_component_name, - ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name) - ); - - MAP_SEGMENT_RESET_FLAGS(ds_buf); - ds_buf->seg_id = MAP_SEGMENT_SHM_INVALID; - ds_buf->super.va_base = 0; - ds_buf->super.va_end = 0; - ds_buf->seg_size = 0; - ds_buf->type = MAP_SEGMENT_UNKNOWN; - unlink(ds_buf->seg_name); - memset(ds_buf->seg_name, '\0', sizeof(ds_buf->seg_name)); -} - /* ////////////////////////////////////////////////////////////////////////// */ static int module_init(void) @@ -142,30 +107,6 @@ module_finalize(void) return OSHMEM_SUCCESS; } -/* ////////////////////////////////////////////////////////////////////////// */ -static int -ds_copy(const map_segment_t *from, - map_segment_t *to) -{ - memcpy(to, from, sizeof(map_segment_t)); - - OPAL_OUTPUT_VERBOSE( - (70, oshmem_sshmem_base_framework.framework_output, - "%s: %s: ds_copy complete " - "from: (id: %d, size: %lu, " - "name: %s flags: 0x%02x) " - "to: (id: %d, size: %lu, " - "name: %s flags: 0x%02x)\n", - mca_sshmem_mmap_component.super.base_version.mca_type_name, - mca_sshmem_mmap_component.super.base_version.mca_component_name, - from->seg_id, (unsigned long)from->seg_size, from->seg_name, - from->flags, to->seg_id, (unsigned long)to->seg_size, to->seg_name, - to->flags) - ); - - return OSHMEM_SUCCESS; -} -/* ////////////////////////////////////////////////////////////////////////// */ static int segment_create(map_segment_t *ds_buf, @@ -225,11 +166,11 @@ segment_create(map_segment_t *ds_buf, OPAL_OUTPUT_VERBOSE( (70, oshmem_sshmem_base_framework.framework_output, "%s: %s: create %s " - "(id: %d, addr: %p size: %lu, name: %s)\n", + "(id: %d, addr: %p size: %lu)\n", mca_sshmem_mmap_component.super.base_version.mca_type_name, mca_sshmem_mmap_component.super.base_version.mca_component_name, (rc ? "failure" : "successful"), - ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name) + ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size) ); return rc; @@ -316,10 +257,10 @@ segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey) OPAL_OUTPUT_VERBOSE( (70, oshmem_sshmem_base_framework.framework_output, "%s: %s: attach successful " - "(id: %d, addr: %p size: %lu, name: %s | va_base: 0x%p len: %d key %llx)\n", + "(id: %d, addr: %p size: %lu | va_base: 0x%p len: %d key %llx)\n", mca_sshmem_mmap_component.super.base_version.mca_type_name, mca_sshmem_mmap_component.super.base_version.mca_component_name, - ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name, + ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, mkey->va_base, mkey->len, (unsigned long long)mkey->u.key) ); @@ -338,10 +279,10 @@ segment_detach(map_segment_t *ds_buf, sshmem_mkey_t *mkey) OPAL_OUTPUT_VERBOSE( (70, oshmem_sshmem_base_framework.framework_output, "%s: %s: detaching " - "(id: %d, addr: %p size: %lu, name: %s)\n", + "(id: %d, addr: %p size: %lu)\n", mca_sshmem_mmap_component.super.base_version.mca_type_name, mca_sshmem_mmap_component.super.base_version.mca_component_name, - ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name) + ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size) ); munmap((void *)ds_buf->super.va_base, ds_buf->seg_size); @@ -363,10 +304,10 @@ segment_unlink(map_segment_t *ds_buf) OPAL_OUTPUT_VERBOSE( (70, oshmem_sshmem_base_framework.framework_output, "%s: %s: unlinking " - "(id: %d, addr: %p size: %lu, name: %s)\n", + "(id: %d, addr: %p size: %lu)\n", mca_sshmem_mmap_component.super.base_version.mca_type_name, mca_sshmem_mmap_component.super.base_version.mca_component_name, - ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name) + ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size) ); /* don't completely reset. in particular, only reset diff --git a/oshmem/mca/sshmem/sshmem.h b/oshmem/mca/sshmem/sshmem.h index 2957c22281..a2b570aab8 100644 --- a/oshmem/mca/sshmem/sshmem.h +++ b/oshmem/mca/sshmem/sshmem.h @@ -21,7 +21,6 @@ * * - module_init * - segment_create - * - ds_copy * - segment_attach * - segment_detach * - unlink @@ -74,20 +73,6 @@ typedef struct mca_sshmem_base_component_2_0_0_t mca_sshmem_base_component_t; typedef int (*mca_sshmem_base_module_init_fn_t)(void); -/** - * copy shmem data structure information pointed to by from to the structure - * pointed to by to. - * - * @param from source pointer (IN). - * - * @param to destination pointer (OUT). - * - * @return OSHMEM_SUCCESS on success. - */ -typedef int -(*mca_sshmem_base_ds_copy_fn_t)(const map_segment_t *from, - map_segment_t *to); - /** * create a new shared memory segment and initialize members in structure * pointed to by ds_buf. @@ -153,7 +138,6 @@ typedef int (*mca_sshmem_base_module_finalize_fn_t)(void); struct mca_sshmem_base_module_2_0_0_t { mca_sshmem_base_module_init_fn_t module_init; mca_sshmem_base_module_segment_create_fn_t segment_create; - mca_sshmem_base_ds_copy_fn_t ds_copy; mca_sshmem_base_module_segment_attach_fn_t segment_attach; mca_sshmem_base_module_segment_detach_fn_t segment_detach; mca_sshmem_base_module_unlink_fn_t unlink; diff --git a/oshmem/mca/sshmem/sshmem_types.h b/oshmem/mca/sshmem/sshmem_types.h index 3081892e5f..ccdf8995b5 100644 --- a/oshmem/mca/sshmem/sshmem_types.h +++ b/oshmem/mca/sshmem/sshmem_types.h @@ -72,6 +72,7 @@ typedef enum { MAP_SEGMENT_ALLOC_SHM, MAP_SEGMENT_ALLOC_IBV, MAP_SEGMENT_ALLOC_IBV_NOSHMR, + MAP_SEGMENT_ALLOC_UCX, MAP_SEGMENT_UNKNOWN } segment_type_t; @@ -112,9 +113,10 @@ typedef struct map_segment { sshmem_mkey_t *mkeys; /* includes local segment bases in va_base */ segment_flag_t flags; /* enable/disable flag */ int seg_id; - char seg_name[OPAL_PATH_MAX]; size_t seg_size; /* length of the segment */ segment_type_t type; /* type of the segment */ + void *context; /* allocator can use this field to store + its own private data */ } map_segment_t; END_C_DECLS diff --git a/oshmem/mca/sshmem/sysv/sshmem_sysv_module.c b/oshmem/mca/sshmem/sysv/sshmem_sysv_module.c index 625ef44534..dec0bee0be 100644 --- a/oshmem/mca/sshmem/sysv/sshmem_sysv_module.c +++ b/oshmem/mca/sshmem/sysv/sshmem_sysv_module.c @@ -62,10 +62,6 @@ segment_create(map_segment_t *ds_buf, const char *file_name, size_t size); -static int -ds_copy(const map_segment_t *from, - map_segment_t *to); - static void * segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey); @@ -84,7 +80,6 @@ mca_sshmem_sysv_module_t mca_sshmem_sysv_module = { { module_init, segment_create, - ds_copy, segment_attach, segment_detach, segment_unlink, @@ -92,34 +87,6 @@ mca_sshmem_sysv_module_t mca_sshmem_sysv_module = { } }; -/* ////////////////////////////////////////////////////////////////////////// */ -/* private utility functions */ -/* ////////////////////////////////////////////////////////////////////////// */ - -/* ////////////////////////////////////////////////////////////////////////// */ -/** - * completely resets the contents of *ds_buf - */ -static inline void -shmem_ds_reset(map_segment_t *ds_buf) -{ - OPAL_OUTPUT_VERBOSE( - (70, oshmem_sshmem_base_framework.framework_output, - "%s: %s: shmem_ds_resetting " - "(id: %d, size: %lu, name: %s)\n", - mca_sshmem_sysv_component.super.base_version.mca_type_name, - mca_sshmem_sysv_component.super.base_version.mca_component_name, - ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name) - ); - - MAP_SEGMENT_RESET_FLAGS(ds_buf); - ds_buf->seg_id = MAP_SEGMENT_SHM_INVALID; - ds_buf->super.va_base = 0; - ds_buf->super.va_end = 0; - ds_buf->seg_size = 0; - ds_buf->type = MAP_SEGMENT_UNKNOWN; - memset(ds_buf->seg_name, '\0', sizeof(ds_buf->seg_name)); -} /* ////////////////////////////////////////////////////////////////////////// */ static int @@ -137,29 +104,6 @@ module_finalize(void) return OSHMEM_SUCCESS; } -/* ////////////////////////////////////////////////////////////////////////// */ -static int -ds_copy(const map_segment_t *from, - map_segment_t *to) -{ - memcpy(to, from, sizeof(map_segment_t)); - - OPAL_OUTPUT_VERBOSE( - (70, oshmem_sshmem_base_framework.framework_output, - "%s: %s: ds_copy complete " - "from: (id: %d, size: %lu, " - "name: %s flags: 0x%02x) " - "to: (id: %d, size: %lu, " - "name: %s flags: 0x%02x)\n", - mca_sshmem_sysv_component.super.base_version.mca_type_name, - mca_sshmem_sysv_component.super.base_version.mca_component_name, - from->seg_id, (unsigned long)from->seg_size, from->seg_name, - from->flags, to->seg_id, (unsigned long)to->seg_size, to->seg_name, - to->flags) - ); - - return OSHMEM_SUCCESS; -} /* ////////////////////////////////////////////////////////////////////////// */ static int @@ -232,11 +176,11 @@ segment_create(map_segment_t *ds_buf, OPAL_OUTPUT_VERBOSE( (70, oshmem_sshmem_base_framework.framework_output, "%s: %s: create %s " - "(id: %d, addr: %p size: %lu, name: %s)\n", + "(id: %d, addr: %p size: %lu)\n", mca_sshmem_sysv_component.super.base_version.mca_type_name, mca_sshmem_sysv_component.super.base_version.mca_component_name, (rc ? "failure" : "successful"), - ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name) + ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size) ); return rc; @@ -261,10 +205,10 @@ segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey) OPAL_OUTPUT_VERBOSE( (70, oshmem_sshmem_base_framework.framework_output, "%s: %s: attach successful " - "(id: %d, addr: %p size: %lu, name: %s | va_base: 0x%p len: %d key %llx)\n", + "(id: %d, addr: %p size: %lu | va_base: 0x%p len: %d key %llx)\n", mca_sshmem_sysv_component.super.base_version.mca_type_name, mca_sshmem_sysv_component.super.base_version.mca_component_name, - ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name, + ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, mkey->va_base, mkey->len, (unsigned long long)mkey->u.key) ); @@ -283,10 +227,10 @@ segment_detach(map_segment_t *ds_buf, sshmem_mkey_t *mkey) OPAL_OUTPUT_VERBOSE( (70, oshmem_sshmem_base_framework.framework_output, "%s: %s: detaching " - "(id: %d, addr: %p size: %lu, name: %s)\n", + "(id: %d, addr: %p size: %lu)\n", mca_sshmem_sysv_component.super.base_version.mca_type_name, mca_sshmem_sysv_component.super.base_version.mca_component_name, - ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name) + ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size) ); if (ds_buf->seg_id != MAP_SEGMENT_SHM_INVALID) { @@ -318,10 +262,10 @@ segment_unlink(map_segment_t *ds_buf) OPAL_OUTPUT_VERBOSE( (70, oshmem_sshmem_base_framework.framework_output, "%s: %s: unlinking " - "(id: %d, size: %lu, name: %s)\n", + "(id: %d, size: %lu)\n", mca_sshmem_sysv_component.super.base_version.mca_type_name, mca_sshmem_sysv_component.super.base_version.mca_component_name, - ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name) + ds_buf->seg_id, (unsigned long)ds_buf->seg_size) ); /* don't completely reset. in particular, only reset diff --git a/oshmem/mca/sshmem/verbs/sshmem_verbs_module.c b/oshmem/mca/sshmem/verbs/sshmem_verbs_module.c index f182876413..04d925e700 100644 --- a/oshmem/mca/sshmem/verbs/sshmem_verbs_module.c +++ b/oshmem/mca/sshmem/verbs/sshmem_verbs_module.c @@ -56,10 +56,6 @@ segment_create(map_segment_t *ds_buf, const char *file_name, size_t size); -static int -ds_copy(const map_segment_t *from, - map_segment_t *to); - static void * segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey); @@ -80,7 +76,6 @@ mca_sshmem_verbs_module_t mca_sshmem_verbs_module = { { module_init, segment_create, - ds_copy, segment_attach, segment_detach, segment_unlink, @@ -88,35 +83,6 @@ mca_sshmem_verbs_module_t mca_sshmem_verbs_module = { } }; -/* ////////////////////////////////////////////////////////////////////////// */ -/* private utility functions */ -/* ////////////////////////////////////////////////////////////////////////// */ - -/* ////////////////////////////////////////////////////////////////////////// */ -/** - * completely resets the contents of *ds_buf - */ -static inline void -shmem_ds_reset(map_segment_t *ds_buf) -{ - OPAL_OUTPUT_VERBOSE( - (70, oshmem_sshmem_base_framework.framework_output, - "%s: %s: shmem_ds_resetting " - "(id: %d, size: %lu, name: %s)\n", - mca_sshmem_verbs_component.super.base_version.mca_type_name, - mca_sshmem_verbs_component.super.base_version.mca_component_name, - ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name) - ); - - MAP_SEGMENT_RESET_FLAGS(ds_buf); - ds_buf->seg_id = MAP_SEGMENT_SHM_INVALID; - ds_buf->super.va_base = 0; - ds_buf->super.va_end = 0; - ds_buf->seg_size = 0; - ds_buf->type = MAP_SEGMENT_UNKNOWN; - memset(ds_buf->seg_name, '\0', sizeof(ds_buf->seg_name)); -} - /* ////////////////////////////////////////////////////////////////////////// */ static int module_init(void) @@ -133,29 +99,6 @@ module_finalize(void) return OSHMEM_SUCCESS; } -/* ////////////////////////////////////////////////////////////////////////// */ -static int -ds_copy(const map_segment_t *from, - map_segment_t *to) -{ - memcpy(to, from, sizeof(map_segment_t)); - - OPAL_OUTPUT_VERBOSE( - (70, oshmem_sshmem_base_framework.framework_output, - "%s: %s: ds_copy complete " - "from: (id: %d, size: %lu, " - "name: %s flags: 0x%02x) " - "to: (id: %d, size: %lu, " - "name: %s flags: 0x%02x)\n", - mca_sshmem_verbs_component.super.base_version.mca_type_name, - mca_sshmem_verbs_component.super.base_version.mca_component_name, - from->seg_id, (unsigned long)from->seg_size, from->seg_name, - from->flags, to->seg_id, (unsigned long)to->seg_size, to->seg_name, - to->flags) - ); - - return OSHMEM_SUCCESS; -} /* ////////////////////////////////////////////////////////////////////////// */ static int @@ -329,11 +272,11 @@ segment_create(map_segment_t *ds_buf, OPAL_OUTPUT_VERBOSE( (70, oshmem_sshmem_base_framework.framework_output, "%s: %s: create %s " - "(id: %d, addr: %p size: %lu, name: %s)\n", + "(id: %d, addr: %p size: %lu)\n", mca_sshmem_verbs_component.super.base_version.mca_type_name, mca_sshmem_verbs_component.super.base_version.mca_component_name, (rc ? "failure" : "successful"), - ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name) + ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size) ); return rc; @@ -395,10 +338,10 @@ segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey) OPAL_OUTPUT_VERBOSE( (70, oshmem_sshmem_base_framework.framework_output, "%s: %s: attach successful " - "(id: %d, addr: %p size: %lu, name: %s | va_base: 0x%p len: %d key %llx)\n", + "(id: %d, addr: %p size: %lu | va_base: 0x%p len: %d key %llx)\n", mca_sshmem_verbs_component.super.base_version.mca_type_name, mca_sshmem_verbs_component.super.base_version.mca_component_name, - ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name, + ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, mkey->va_base, mkey->len, (unsigned long long)mkey->u.key) ); @@ -419,10 +362,10 @@ segment_detach(map_segment_t *ds_buf, sshmem_mkey_t *mkey) OPAL_OUTPUT_VERBOSE( (70, oshmem_sshmem_base_framework.framework_output, "%s: %s: detaching " - "(id: %d, addr: %p size: %lu, name: %s)\n", + "(id: %d, addr: %p size: %lu)\n", mca_sshmem_verbs_component.super.base_version.mca_type_name, mca_sshmem_verbs_component.super.base_version.mca_component_name, - ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name) + ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size) ); if (device) { @@ -498,10 +441,10 @@ segment_unlink(map_segment_t *ds_buf) OPAL_OUTPUT_VERBOSE( (70, oshmem_sshmem_base_framework.framework_output, "%s: %s: unlinking " - "(id: %d, addr: %p size: %lu, name: %s)\n", + "(id: %d, addr: %p size: %lu)\n", mca_sshmem_verbs_component.super.base_version.mca_type_name, mca_sshmem_verbs_component.super.base_version.mca_component_name, - ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name) + ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size) ); /* don't completely reset. in particular, only reset From 132fbd9ae98eca66f0d6c0d33fa4d76b6b0d3c29 Mon Sep 17 00:00:00 2001 From: Alex Mikheev Date: Sun, 15 Jan 2017 12:45:37 +0200 Subject: [PATCH 2/4] oshmem: sshmem: add UCX allocator Signed-off-by: Alex Mikheev --- oshmem/mca/spml/ucx/spml_ucx.c | 52 +++-- oshmem/mca/sshmem/ucx/Makefile.am | 40 ++++ oshmem/mca/sshmem/ucx/configure.m4 | 32 ++++ oshmem/mca/sshmem/ucx/sshmem_ucx.h | 40 ++++ oshmem/mca/sshmem/ucx/sshmem_ucx_component.c | 129 +++++++++++++ oshmem/mca/sshmem/ucx/sshmem_ucx_module.c | 191 +++++++++++++++++++ 6 files changed, 464 insertions(+), 20 deletions(-) create mode 100644 oshmem/mca/sshmem/ucx/Makefile.am create mode 100644 oshmem/mca/sshmem/ucx/configure.m4 create mode 100644 oshmem/mca/sshmem/ucx/sshmem_ucx.h create mode 100644 oshmem/mca/sshmem/ucx/sshmem_ucx_component.c create mode 100644 oshmem/mca/sshmem/ucx/sshmem_ucx_module.c diff --git a/oshmem/mca/spml/ucx/spml_ucx.c b/oshmem/mca/spml/ucx/spml_ucx.c index 5b9066564f..5cf473efc8 100644 --- a/oshmem/mca/spml/ucx/spml_ucx.c +++ b/oshmem/mca/spml/ucx/spml_ucx.c @@ -425,7 +425,8 @@ sshmem_mkey_t *mca_spml_ucx_register(void* addr, size_t len; int my_pe = oshmem_my_proc_id(); ucp_mem_map_params_t mem_map_params; - int seg; + int segno; + map_segment_t *mem_seg; unsigned flags; *count = 0; @@ -434,26 +435,32 @@ sshmem_mkey_t *mca_spml_ucx_register(void* addr, return NULL; } - seg = memheap_find_segnum(addr); + segno = memheap_find_segnum(addr); + mem_seg = memheap_find_seg(segno); - ucx_mkey = &mca_spml_ucx.ucp_peers[my_pe].mkeys[seg].key; + ucx_mkey = &mca_spml_ucx.ucp_peers[my_pe].mkeys[segno].key; mkeys[0].spml_context = ucx_mkey; - flags = 0; - if (mca_spml_ucx.heap_reg_nb && memheap_is_va_in_segment(addr, HEAP_SEG_INDEX)) { - flags = UCP_MEM_MAP_NONBLOCK; - } + /* if possible use mem handle already created by ucx allocator */ + if (MAP_SEGMENT_ALLOC_UCX != mem_seg->type) { + flags = 0; + if (mca_spml_ucx.heap_reg_nb && memheap_is_va_in_segment(addr, HEAP_SEG_INDEX)) { + flags = UCP_MEM_MAP_NONBLOCK; + } - mem_map_params.field_mask = UCP_MEM_MAP_PARAM_FIELD_ADDRESS | - UCP_MEM_MAP_PARAM_FIELD_LENGTH | - UCP_MEM_MAP_PARAM_FIELD_FLAGS; - mem_map_params.address = addr; - mem_map_params.length = size; - mem_map_params.flags = flags; + mem_map_params.field_mask = UCP_MEM_MAP_PARAM_FIELD_ADDRESS | + UCP_MEM_MAP_PARAM_FIELD_LENGTH | + UCP_MEM_MAP_PARAM_FIELD_FLAGS; + mem_map_params.address = addr; + mem_map_params.length = size; + mem_map_params.flags = flags; - err = ucp_mem_map(mca_spml_ucx.ucp_context, &mem_map_params, &ucx_mkey->mem_h); - if (UCS_OK != err) { - goto error_out; + err = ucp_mem_map(mca_spml_ucx.ucp_context, &mem_map_params, &ucx_mkey->mem_h); + if (UCS_OK != err) { + goto error_out; + } + } else { + ucx_mkey->mem_h = (ucp_mem_h)mem_seg->context; } err = ucp_rkey_pack(mca_spml_ucx.ucp_context, ucx_mkey->mem_h, @@ -477,9 +484,9 @@ sshmem_mkey_t *mca_spml_ucx_register(void* addr, } mkeys[0].len = len; - mkeys[0].va_base = mem_map_params.address; + mkeys[0].va_base = addr; *count = 1; - mca_spml_ucx_cache_mkey(&mkeys[0], seg, my_pe); + mca_spml_ucx_cache_mkey(&mkeys[0], segno, my_pe); return mkeys; error_unmap: @@ -493,6 +500,7 @@ error_out: int mca_spml_ucx_deregister(sshmem_mkey_t *mkeys) { spml_ucx_mkey_t *ucx_mkey; + map_segment_t *mem_seg; MCA_SPML_CALL(fence()); if (!mkeys) @@ -501,8 +509,12 @@ int mca_spml_ucx_deregister(sshmem_mkey_t *mkeys) if (!mkeys[0].spml_context) return OSHMEM_SUCCESS; - ucx_mkey = (spml_ucx_mkey_t *)mkeys[0].spml_context; - ucp_mem_unmap(mca_spml_ucx.ucp_context, ucx_mkey->mem_h); + mem_seg = memheap_find_va(mkeys[0].va_base); + + if (MAP_SEGMENT_ALLOC_UCX != mem_seg->type) { + ucx_mkey = (spml_ucx_mkey_t *)mkeys[0].spml_context; + ucp_mem_unmap(mca_spml_ucx.ucp_context, ucx_mkey->mem_h); + } if (0 < mkeys[0].len) { ucp_rkey_buffer_release(mkeys[0].u.data); diff --git a/oshmem/mca/sshmem/ucx/Makefile.am b/oshmem/mca/sshmem/ucx/Makefile.am new file mode 100644 index 0000000000..2bc2205679 --- /dev/null +++ b/oshmem/mca/sshmem/ucx/Makefile.am @@ -0,0 +1,40 @@ +# Copyright (c) 2014 Mellanox Technologies, Inc. +# All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +#dist_oshmemdata_DATA = help-oshmem-sshmem-mmap.txt + +AM_CPPFLAGS = $(sshmem_ucx_CPPFLAGS) + +sources = \ + sshmem_ucx.h \ + sshmem_ucx_component.c \ + sshmem_ucx_module.c + +# Make the output library in this directory, and name it either +# mca__.la (for DSO builds) or libmca__.la +# (for static builds). + +if MCA_BUILD_oshmem_sshmem_ucx_DSO +component_noinst = +component_install = mca_sshmem_ucx.la +else +component_noinst = libmca_sshmem_ucx.la +component_install = +endif + +mcacomponentdir = $(oshmemlibdir) +mcacomponent_LTLIBRARIES = $(component_install) +mca_sshmem_ucx_la_SOURCES = $(sources) +mca_sshmem_ucx_la_LDFLAGS = -module -avoid-version $(sshmem_ucx_LDFLAGS) +mca_sshmem_ucx_la_LIBADD = $(sshmem_ucx_LIBS) + +noinst_LTLIBRARIES = $(component_noinst) +libmca_sshmem_ucx_la_SOURCES =$(sources) +libmca_sshmem_ucx_la_LDFLAGS = -module -avoid-version $(sshmem_ucx_LDFLAGS) + diff --git a/oshmem/mca/sshmem/ucx/configure.m4 b/oshmem/mca/sshmem/ucx/configure.m4 new file mode 100644 index 0000000000..aafa4f4e02 --- /dev/null +++ b/oshmem/mca/sshmem/ucx/configure.m4 @@ -0,0 +1,32 @@ +/* + * Copyright (c) 2017 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +# MCA_oshmem_sshmem_ucx_CONFIG([action-if-can-compile], +# [action-if-cant-compile]) +# ------------------------------------------------ +AC_DEFUN([MCA_oshmem_sshmem_ucx_CONFIG],[ + AC_CONFIG_FILES([oshmem/mca/sshmem/ucx/Makefile]) + + OMPI_CHECK_UCX([sshmem_ucx], + [sshmem_ucx_happy="yes"], + [sshmem_ucx_happy="no"]) + + AS_IF([test "$sshmem_ucx_happy" = "yes"], + [$1], + [$2]) + + + # substitute in the things needed to build ucx + AC_SUBST([sshmem_ucx_CFLAGS]) + AC_SUBST([sshmem_ucx_CPPFLAGS]) + AC_SUBST([sshmem_ucx_LDFLAGS]) + AC_SUBST([sshmem_ucx_LIBS]) +])dnl + diff --git a/oshmem/mca/sshmem/ucx/sshmem_ucx.h b/oshmem/mca/sshmem/ucx/sshmem_ucx.h new file mode 100644 index 0000000000..0b625fcc46 --- /dev/null +++ b/oshmem/mca/sshmem/ucx/sshmem_ucx.h @@ -0,0 +1,40 @@ +/* + * Copyright (c) 2017 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef MCA_SSHMEM_UCX_EXPORT_H +#define MCA_SSHMEM_UCX_EXPORT_H + +#include "oshmem_config.h" + +#include "oshmem/mca/sshmem/sshmem.h" + +BEGIN_C_DECLS + +/** + * globally exported variable to hold the ucx component. + */ +typedef struct mca_sshmem_ucx_component_t { + /* base component struct */ + mca_sshmem_base_component_t super; + /* priority for ucx component */ + int priority; +} mca_sshmem_ucx_component_t; + +OSHMEM_MODULE_DECLSPEC extern mca_sshmem_ucx_component_t +mca_sshmem_ucx_component; + +typedef struct mca_sshmem_ucx_module_t { + mca_sshmem_base_module_t super; +} mca_sshmem_ucx_module_t; +extern mca_sshmem_ucx_module_t mca_sshmem_ucx_module; + +END_C_DECLS + +#endif /* MCA_SHMEM_UCX_EXPORT_H */ diff --git a/oshmem/mca/sshmem/ucx/sshmem_ucx_component.c b/oshmem/mca/sshmem/ucx/sshmem_ucx_component.c new file mode 100644 index 0000000000..72278236f1 --- /dev/null +++ b/oshmem/mca/sshmem/ucx/sshmem_ucx_component.c @@ -0,0 +1,129 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2017 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "oshmem_config.h" + +#include "opal/constants.h" + +#include "oshmem/mca/sshmem/sshmem.h" +#include "oshmem/mca/sshmem/base/base.h" +#include "oshmem/mca/spml/base/base.h" + +#include "sshmem_ucx.h" + +/** + * public string showing the shmem ompi_ucx component version number + */ +const char *mca_sshmem_ucx_component_version_string = + "OSHMEM ucx sshmem MCA component version " OSHMEM_VERSION; + + +/** + * local functions + */ +static int ucx_register(void); +static int ucx_open(void); +static int ucx_close(void); +static int ucx_query(mca_base_module_t **module, int *priority); +static int ucx_runtime_query(mca_base_module_t **module, + int *priority, + const char *hint); + +/** + * instantiate the public struct with all of our public information + * and pointers to our public functions in it + */ +mca_sshmem_ucx_component_t mca_sshmem_ucx_component = { + /* ////////////////////////////////////////////////////////////////////// */ + /* super */ + /* ////////////////////////////////////////////////////////////////////// */ + { + /** + * common MCA component data + */ + .base_version = { + MCA_SSHMEM_BASE_VERSION_2_0_0, + + /* component name and version */ + .mca_component_name = "ucx", + MCA_BASE_MAKE_VERSION(component, OSHMEM_MAJOR_VERSION, OSHMEM_MINOR_VERSION, + OSHMEM_RELEASE_VERSION), + + .mca_open_component = ucx_open, + .mca_close_component = ucx_close, + .mca_query_component = ucx_query, + .mca_register_component_params = ucx_register, + }, + /* MCA v2.0.0 component meta data */ + .base_data = { + /* the component is checkpoint ready */ + MCA_BASE_METADATA_PARAM_CHECKPOINT + }, + .runtime_query = ucx_runtime_query, + }, +}; + +static int +ucx_runtime_query(mca_base_module_t **module, + int *priority, + const char *hint) +{ + /* check that spml ucx was selected. Otherwise disqualify */ + if (strcmp(mca_spml_base_selected_component.spmlm_version.mca_component_name, "ucx")) { + *module = NULL; + return OSHMEM_ERR_NOT_AVAILABLE; + } + + *priority = mca_sshmem_ucx_component.priority; + /* use lowest priority because UCX does not support + * fixed address mapping yet + */ + *priority = 0; + *module = (mca_base_module_t *)&mca_sshmem_ucx_module.super; + return OPAL_SUCCESS; +} + +static int +ucx_register(void) +{ + /* (default) priority - set high to make ucx the default */ + mca_sshmem_ucx_component.priority = 100; + mca_base_component_var_register (&mca_sshmem_ucx_component.super.base_version, + "priority", "Priority for sshmem ucx " + "component (default: 100)", MCA_BASE_VAR_TYPE_INT, + NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE, + OPAL_INFO_LVL_3, + MCA_BASE_VAR_SCOPE_ALL_EQ, + &mca_sshmem_ucx_component.priority); + + return OSHMEM_SUCCESS; +} + +static int +ucx_open(void) +{ + return OSHMEM_SUCCESS; +} + +static int +ucx_query(mca_base_module_t **module, int *priority) +{ + *priority = mca_sshmem_ucx_component.priority; + *module = (mca_base_module_t *)&mca_sshmem_ucx_module.super; + return OSHMEM_SUCCESS; +} + +static int +ucx_close(void) +{ + return OSHMEM_SUCCESS; +} + diff --git a/oshmem/mca/sshmem/ucx/sshmem_ucx_module.c b/oshmem/mca/sshmem/ucx/sshmem_ucx_module.c new file mode 100644 index 0000000000..fc4c9a666f --- /dev/null +++ b/oshmem/mca/sshmem/ucx/sshmem_ucx_module.c @@ -0,0 +1,191 @@ +/* + * Copyright (c) 2017 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "oshmem_config.h" + +#include "opal/constants.h" +#include "opal/util/output.h" +#include "opal/util/path.h" +#include "opal/util/show_help.h" +#include "orte/util/show_help.h" + +#include "oshmem/proc/proc.h" +#include "oshmem/mca/sshmem/sshmem.h" +#include "oshmem/mca/sshmem/base/base.h" +#include "oshmem/util/oshmem_util.h" +#include "oshmem/mca/spml/ucx/spml_ucx.h" + +#include "sshmem_ucx.h" + +/* ////////////////////////////////////////////////////////////////////////// */ +/*local functions */ +/* local functions */ +static int +module_init(void); + +static int +segment_create(map_segment_t *ds_buf, + const char *file_name, + size_t size); + +static void * +segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey); + +static int +segment_detach(map_segment_t *ds_buf, sshmem_mkey_t *mkey); + +static int +segment_unlink(map_segment_t *ds_buf); + +static int +module_finalize(void); + +/* + * ucx shmem module + */ +mca_sshmem_ucx_module_t mca_sshmem_ucx_module = { + /* super */ + { + module_init, + segment_create, + segment_attach, + segment_detach, + segment_unlink, + module_finalize + } +}; + +static int +module_init(void) +{ + /* nothing to do */ + return OSHMEM_SUCCESS; +} + +/* ////////////////////////////////////////////////////////////////////////// */ +static int +module_finalize(void) +{ + /* nothing to do */ + return OSHMEM_SUCCESS; +} + +/* ////////////////////////////////////////////////////////////////////////// */ + +static int +segment_create(map_segment_t *ds_buf, + const char *file_name, + size_t size) +{ + int rc = OSHMEM_SUCCESS; + mca_spml_ucx_t *spml = (mca_spml_ucx_t *)mca_spml.self; + ucp_mem_map_params_t mem_map_params; + ucp_mem_h mem_h; + ucs_status_t status; + + assert(ds_buf); + + /* init the contents of map_segment_t */ + shmem_ds_reset(ds_buf); + + mem_map_params.field_mask = UCP_MEM_MAP_PARAM_FIELD_ADDRESS | + UCP_MEM_MAP_PARAM_FIELD_LENGTH | + UCP_MEM_MAP_PARAM_FIELD_FLAGS; + /* TODO: use default base address */ + mem_map_params.address = NULL; + mem_map_params.length = size; + mem_map_params.flags = 0; + if (spml->heap_reg_nb) { + mem_map_params.flags |= UCP_MEM_MAP_NONBLOCK; + } + + status = ucp_mem_map(spml->ucp_context, &mem_map_params, &mem_h); + if (UCS_OK != status) { + rc = OSHMEM_ERROR; + goto out; + } + + ds_buf->super.va_base = mem_map_params.address; + ds_buf->seg_size = size; + ds_buf->super.va_end = (void*)((uintptr_t)ds_buf->super.va_base + ds_buf->seg_size); + ds_buf->context = mem_h; + ds_buf->type = MAP_SEGMENT_ALLOC_UCX; + + /* TODO: ucp_mem_advise when UCX has it */ + +out: + OPAL_OUTPUT_VERBOSE( + (70, oshmem_sshmem_base_framework.framework_output, + "%s: %s: create %s " + "(id: %d, addr: %p size: %lu)\n", + mca_sshmem_ucx_component.super.base_version.mca_type_name, + mca_sshmem_ucx_component.super.base_version.mca_component_name, + (rc ? "failure" : "successful"), + ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size) + ); + return rc; +} + +static void * +segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey) +{ + assert(ds_buf); + assert(mkey->va_base == 0); + + OPAL_OUTPUT((oshmem_sshmem_base_framework.framework_output, + "can not attach to ucx segment")); + oshmem_shmem_abort(-1); + return NULL; +} + +static int +segment_detach(map_segment_t *ds_buf, sshmem_mkey_t *mkey) +{ + OPAL_OUTPUT_VERBOSE( + (70, oshmem_sshmem_base_framework.framework_output, + "%s: %s: detaching " + "(id: %d, addr: %p size: %lu)\n", + mca_sshmem_ucx_component.super.base_version.mca_type_name, + mca_sshmem_ucx_component.super.base_version.mca_component_name, + ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size) + ); + + /* reset the contents of the map_segment_t associated with this + * shared memory segment. + */ + shmem_ds_reset(ds_buf); + + return OSHMEM_SUCCESS; +} + +static int +segment_unlink(map_segment_t *ds_buf) +{ + mca_spml_ucx_t *spml = (mca_spml_ucx_t *)mca_spml.self; + + assert(ds_buf); + + ucp_mem_unmap(spml->ucp_context, (ucp_mem_h)ds_buf->context); + + OPAL_OUTPUT_VERBOSE( + (70, oshmem_sshmem_base_framework.framework_output, + "%s: %s: unlinking " + "(id: %d, addr: %p size: %lu)\n", + mca_sshmem_ucx_component.super.base_version.mca_type_name, + mca_sshmem_ucx_component.super.base_version.mca_component_name, + ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size) + ); + + ds_buf->seg_id = MAP_SEGMENT_SHM_INVALID; + MAP_SEGMENT_INVALIDATE(ds_buf); + + return OSHMEM_SUCCESS; +} + From c63137e1c0c1e222f1dbae1b17d476ab6a7bf711 Mon Sep 17 00:00:00 2001 From: Alex Mikheev Date: Thu, 26 Jan 2017 15:32:11 +0200 Subject: [PATCH 3/4] oshmem: sshmem ucx: minor code cleanup Signed-off-by: Alex Mikheev --- oshmem/mca/spml/ucx/spml_ucx.c | 23 ++++++++++++----------- oshmem/mca/sshmem/ucx/sshmem_ucx_module.c | 2 -- 2 files changed, 12 insertions(+), 13 deletions(-) diff --git a/oshmem/mca/spml/ucx/spml_ucx.c b/oshmem/mca/spml/ucx/spml_ucx.c index 5cf473efc8..1e7ac7f537 100644 --- a/oshmem/mca/spml/ucx/spml_ucx.c +++ b/oshmem/mca/spml/ucx/spml_ucx.c @@ -420,14 +420,14 @@ sshmem_mkey_t *mca_spml_ucx_register(void* addr, int *count) { sshmem_mkey_t *mkeys; - ucs_status_t err; + ucs_status_t status; spml_ucx_mkey_t *ucx_mkey; size_t len; - int my_pe = oshmem_my_proc_id(); ucp_mem_map_params_t mem_map_params; int segno; map_segment_t *mem_seg; unsigned flags; + int my_pe = oshmem_my_proc_id(); *count = 0; mkeys = (sshmem_mkey_t *) calloc(1, sizeof(*mkeys)); @@ -455,17 +455,18 @@ sshmem_mkey_t *mca_spml_ucx_register(void* addr, mem_map_params.length = size; mem_map_params.flags = flags; - err = ucp_mem_map(mca_spml_ucx.ucp_context, &mem_map_params, &ucx_mkey->mem_h); - if (UCS_OK != err) { + status = ucp_mem_map(mca_spml_ucx.ucp_context, &mem_map_params, &ucx_mkey->mem_h); + if (UCS_OK != status) { goto error_out; } + } else { ucx_mkey->mem_h = (ucp_mem_h)mem_seg->context; } - err = ucp_rkey_pack(mca_spml_ucx.ucp_context, ucx_mkey->mem_h, - &mkeys[0].u.data, &len); - if (UCS_OK != err) { + status = ucp_rkey_pack(mca_spml_ucx.ucp_context, ucx_mkey->mem_h, + &mkeys[0].u.data, &len); + if (UCS_OK != status) { goto error_unmap; } if (len >= 0xffff) { @@ -475,10 +476,10 @@ sshmem_mkey_t *mca_spml_ucx_register(void* addr, oshmem_shmem_abort(-1); } - err = ucp_ep_rkey_unpack(mca_spml_ucx.ucp_peers[oshmem_group_self->my_pe].ucp_conn, - mkeys[0].u.data, - &ucx_mkey->rkey); - if (UCS_OK != err) { + status = ucp_ep_rkey_unpack(mca_spml_ucx.ucp_peers[oshmem_group_self->my_pe].ucp_conn, + mkeys[0].u.data, + &ucx_mkey->rkey); + if (UCS_OK != status) { SPML_ERROR("failed to unpack rkey"); goto error_unmap; } diff --git a/oshmem/mca/sshmem/ucx/sshmem_ucx_module.c b/oshmem/mca/sshmem/ucx/sshmem_ucx_module.c index fc4c9a666f..19072c1856 100644 --- a/oshmem/mca/sshmem/ucx/sshmem_ucx_module.c +++ b/oshmem/mca/sshmem/ucx/sshmem_ucx_module.c @@ -118,8 +118,6 @@ segment_create(map_segment_t *ds_buf, ds_buf->context = mem_h; ds_buf->type = MAP_SEGMENT_ALLOC_UCX; - /* TODO: ucp_mem_advise when UCX has it */ - out: OPAL_OUTPUT_VERBOSE( (70, oshmem_sshmem_base_framework.framework_output, From c9b5b12af4da9ff50928fd090910a9972b78590c Mon Sep 17 00:00:00 2001 From: Alex Mikheev Date: Sun, 26 Feb 2017 15:16:28 +0200 Subject: [PATCH 4/4] oshmem: sshmem ucx: use fixed base address Signed-off-by: Alex Mikheev --- oshmem/mca/sshmem/ucx/sshmem_ucx_component.c | 4 ---- oshmem/mca/sshmem/ucx/sshmem_ucx_module.c | 7 ++++--- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/oshmem/mca/sshmem/ucx/sshmem_ucx_component.c b/oshmem/mca/sshmem/ucx/sshmem_ucx_component.c index 72278236f1..46c6c61c88 100644 --- a/oshmem/mca/sshmem/ucx/sshmem_ucx_component.c +++ b/oshmem/mca/sshmem/ucx/sshmem_ucx_component.c @@ -83,10 +83,6 @@ ucx_runtime_query(mca_base_module_t **module, } *priority = mca_sshmem_ucx_component.priority; - /* use lowest priority because UCX does not support - * fixed address mapping yet - */ - *priority = 0; *module = (mca_base_module_t *)&mca_sshmem_ucx_module.super; return OPAL_SUCCESS; } diff --git a/oshmem/mca/sshmem/ucx/sshmem_ucx_module.c b/oshmem/mca/sshmem/ucx/sshmem_ucx_module.c index 19072c1856..44f7374331 100644 --- a/oshmem/mca/sshmem/ucx/sshmem_ucx_module.c +++ b/oshmem/mca/sshmem/ucx/sshmem_ucx_module.c @@ -98,10 +98,11 @@ segment_create(map_segment_t *ds_buf, mem_map_params.field_mask = UCP_MEM_MAP_PARAM_FIELD_ADDRESS | UCP_MEM_MAP_PARAM_FIELD_LENGTH | UCP_MEM_MAP_PARAM_FIELD_FLAGS; - /* TODO: use default base address */ - mem_map_params.address = NULL; + + mem_map_params.address = (void *)mca_sshmem_base_start_address; mem_map_params.length = size; - mem_map_params.flags = 0; + mem_map_params.flags = UCP_MEM_MAP_ALLOCATE|UCP_MEM_MAP_FIXED; + if (spml->heap_reg_nb) { mem_map_params.flags |= UCP_MEM_MAP_NONBLOCK; }