ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Merge pull request #2717 from alex-mikheev/topic/sshmem_ucx

Просмотр исходного кода 
oshmem: sshmem: adds UCX allocator
Этот коммит содержится в:
Yossi 2017-03-09 12:58:06 +02:00 коммит произвёл GitHub
родитель 16ee880c4e c9b5b12af4
Коммит 1a95633e40
13 изменённых файлов: 529 добавлений и 278 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

39
oshmem/mca/spml/ucx/spml_ucx.c
Просмотреть файл

@ -420,13 +420,14 @@ sshmem_mkey_t *mca_spml_ucx_register(void* addr,
int *count)
{
sshmem_mkey_t *mkeys;
ucs_status_t err;
ucs_status_t status;
spml_ucx_mkey_t *ucx_mkey;
size_t len;
int my_pe = oshmem_my_proc_id();
ucp_mem_map_params_t mem_map_params;
int seg;
int segno;
map_segment_t *mem_seg;
unsigned flags;
int my_pe = oshmem_my_proc_id();
*count = 0;
mkeys = (sshmem_mkey_t *) calloc(1, sizeof(*mkeys));
@ -434,11 +435,14 @@ sshmem_mkey_t *mca_spml_ucx_register(void* addr,
return NULL;
}
seg = memheap_find_segnum(addr);
segno = memheap_find_segnum(addr);
mem_seg = memheap_find_seg(segno);
ucx_mkey = &mca_spml_ucx.ucp_peers[my_pe].mkeys[seg].key;
ucx_mkey = &mca_spml_ucx.ucp_peers[my_pe].mkeys[segno].key;
mkeys[0].spml_context = ucx_mkey;
/* if possible use mem handle already created by ucx allocator */
if (MAP_SEGMENT_ALLOC_UCX != mem_seg->type) {
flags = 0;
if (mca_spml_ucx.heap_reg_nb && memheap_is_va_in_segment(addr, HEAP_SEG_INDEX)) {
flags = UCP_MEM_MAP_NONBLOCK;
@ -451,14 +455,18 @@ sshmem_mkey_t *mca_spml_ucx_register(void* addr,
mem_map_params.length = size;
mem_map_params.flags = flags;
err = ucp_mem_map(mca_spml_ucx.ucp_context, &mem_map_params, &ucx_mkey->mem_h);
if (UCS_OK != err) {
status = ucp_mem_map(mca_spml_ucx.ucp_context, &mem_map_params, &ucx_mkey->mem_h);
if (UCS_OK != status) {
goto error_out;
}
err = ucp_rkey_pack(mca_spml_ucx.ucp_context, ucx_mkey->mem_h,
} else {
ucx_mkey->mem_h = (ucp_mem_h)mem_seg->context;
}
status = ucp_rkey_pack(mca_spml_ucx.ucp_context, ucx_mkey->mem_h,
&mkeys[0].u.data, &len);
if (UCS_OK != err) {
if (UCS_OK != status) {
goto error_unmap;
}
if (len >= 0xffff) {
@ -468,18 +476,18 @@ sshmem_mkey_t *mca_spml_ucx_register(void* addr,
oshmem_shmem_abort(-1);
}
err = ucp_ep_rkey_unpack(mca_spml_ucx.ucp_peers[oshmem_group_self->my_pe].ucp_conn,
status = ucp_ep_rkey_unpack(mca_spml_ucx.ucp_peers[oshmem_group_self->my_pe].ucp_conn,
mkeys[0].u.data,
&ucx_mkey->rkey);
if (UCS_OK != err) {
if (UCS_OK != status) {
SPML_ERROR("failed to unpack rkey");
goto error_unmap;
}
mkeys[0].len = len;
mkeys[0].va_base = mem_map_params.address;
mkeys[0].va_base = addr;
*count = 1;
mca_spml_ucx_cache_mkey(&mkeys[0], seg, my_pe);
mca_spml_ucx_cache_mkey(&mkeys[0], segno, my_pe);
return mkeys;
error_unmap:
@ -493,6 +501,7 @@ error_out:
int mca_spml_ucx_deregister(sshmem_mkey_t *mkeys)
{
spml_ucx_mkey_t *ucx_mkey;
map_segment_t *mem_seg;
MCA_SPML_CALL(fence());
if (!mkeys)
@ -501,8 +510,12 @@ int mca_spml_ucx_deregister(sshmem_mkey_t *mkeys)
if (!mkeys[0].spml_context)
return OSHMEM_SUCCESS;
mem_seg = memheap_find_va(mkeys[0].va_base);
if (MAP_SEGMENT_ALLOC_UCX != mem_seg->type) {
ucx_mkey = (spml_ucx_mkey_t *)mkeys[0].spml_context;
ucp_mem_unmap(mca_spml_ucx.ucp_context, ucx_mkey->mem_h);
}
if (0 < mkeys[0].len) {
ucp_rkey_buffer_release(mkeys[0].u.data);

21
oshmem/mca/sshmem/base/base.h
Просмотреть файл

@ -33,10 +33,6 @@ mca_sshmem_segment_create(map_segment_t *ds_buf,
const char *file_name,
size_t size);
OSHMEM_DECLSPEC int
mca_sshmem_ds_copy(const map_segment_t *from,
map_segment_t *to);
OSHMEM_DECLSPEC void *
mca_sshmem_segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey);
@ -148,23 +144,12 @@ OSHMEM_DECLSPEC extern mca_base_framework_t oshmem_sshmem_base_framework;
"Warning %s:%d - %s()", __SSHMEM_FILE__, __LINE__, __func__, __VA_ARGS__)
OSHMEM_DECLSPEC extern void shmem_ds_reset(map_segment_t *ds_buf);
/*
* Get unique file name
*/
static inline char * oshmem_get_unique_file_name(uint64_t pe)
{
char *file_name = NULL;
assert(mca_sshmem_base_backing_file_dir);
if (NULL == (file_name = calloc(OPAL_PATH_MAX, sizeof(char)))) {
return NULL;
}
snprintf(file_name, OPAL_PATH_MAX, "%s/shmem_job_%u_pe_%llu", mca_sshmem_base_backing_file_dir, ORTE_PROC_MY_NAME->jobid, (unsigned long long)pe);
return file_name;
}
OSHMEM_DECLSPEC extern char * oshmem_get_unique_file_name(uint64_t pe);
END_C_DECLS

44
oshmem/mca/sshmem/base/sshmem_base_wrappers.c
Просмотреть файл

@ -15,7 +15,6 @@
#include "oshmem/mca/sshmem/sshmem.h"
#include "oshmem/mca/sshmem/base/base.h"
/* ////////////////////////////////////////////////////////////////////////// */
int
mca_sshmem_segment_create(map_segment_t *ds_buf,
const char *file_name,
@ -28,19 +27,6 @@ mca_sshmem_segment_create(map_segment_t *ds_buf,
return mca_sshmem_base_module->segment_create(ds_buf, file_name, size);
}
/* ////////////////////////////////////////////////////////////////////////// */
int
mca_sshmem_ds_copy(const map_segment_t *from,
map_segment_t *to)
{
if (!mca_sshmem_base_selected) {
return OSHMEM_ERROR;
}
return mca_sshmem_base_module->ds_copy(from, to);
}
/* ////////////////////////////////////////////////////////////////////////// */
void *
mca_sshmem_segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
{
@ -51,7 +37,6 @@ mca_sshmem_segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
return mca_sshmem_base_module->segment_attach(ds_buf, mkey);
}
/* ////////////////////////////////////////////////////////////////////////// */
int
mca_sshmem_segment_detach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
{
@ -62,7 +47,6 @@ mca_sshmem_segment_detach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
return mca_sshmem_base_module->segment_detach(ds_buf, mkey);
}
/* ////////////////////////////////////////////////////////////////////////// */
int
mca_sshmem_unlink(map_segment_t *ds_buf)
{
@ -73,3 +57,31 @@ mca_sshmem_unlink(map_segment_t *ds_buf)
return mca_sshmem_base_module->unlink(ds_buf);
}
char * oshmem_get_unique_file_name(uint64_t pe)
{
char *file_name = NULL;
assert(mca_sshmem_base_backing_file_dir);
if (NULL == (file_name = calloc(OPAL_PATH_MAX, sizeof(char)))) {
return NULL;
}
snprintf(file_name, OPAL_PATH_MAX, "%s/shmem_job_%u_pe_%llu", mca_sshmem_base_backing_file_dir, ORTE_PROC_MY_NAME->jobid, (unsigned long long)pe);
return file_name;
}
void
shmem_ds_reset(map_segment_t *ds_buf)
{
MAP_SEGMENT_RESET_FLAGS(ds_buf);
ds_buf->seg_id = MAP_SEGMENT_SHM_INVALID;
ds_buf->super.va_base = 0;
ds_buf->super.va_end = 0;
ds_buf->seg_size = 0;
ds_buf->type = MAP_SEGMENT_UNKNOWN;
}

75
oshmem/mca/sshmem/mmap/sshmem_mmap_module.c
Просмотреть файл

@ -64,10 +64,6 @@ segment_create(map_segment_t *ds_buf,
const char *file_name,
size_t size);
static int
ds_copy(const map_segment_t *from,
map_segment_t *to);
static void *
segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey);
@ -88,7 +84,6 @@ mca_sshmem_mmap_module_t mca_sshmem_mmap_module = {
{
module_init,
segment_create,
ds_copy,
segment_attach,
segment_detach,
segment_unlink,
@ -96,36 +91,6 @@ mca_sshmem_mmap_module_t mca_sshmem_mmap_module = {
}
};
/* ////////////////////////////////////////////////////////////////////////// */
/* private utility functions */
/* ////////////////////////////////////////////////////////////////////////// */
/* ////////////////////////////////////////////////////////////////////////// */
/**
* completely resets the contents of *ds_buf
*/
static inline void
shmem_ds_reset(map_segment_t *ds_buf)
{
OPAL_OUTPUT_VERBOSE(
(70, oshmem_sshmem_base_framework.framework_output,
"%s: %s: shmem_ds_resetting "
"(id: %d, size: %lu, name: %s)\n",
mca_sshmem_mmap_component.super.base_version.mca_type_name,
mca_sshmem_mmap_component.super.base_version.mca_component_name,
ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
);
MAP_SEGMENT_RESET_FLAGS(ds_buf);
ds_buf->seg_id = MAP_SEGMENT_SHM_INVALID;
ds_buf->super.va_base = 0;
ds_buf->super.va_end = 0;
ds_buf->seg_size = 0;
ds_buf->type = MAP_SEGMENT_UNKNOWN;
unlink(ds_buf->seg_name);
memset(ds_buf->seg_name, '\0', sizeof(ds_buf->seg_name));
}
/* ////////////////////////////////////////////////////////////////////////// */
static int
module_init(void)
@ -142,30 +107,6 @@ module_finalize(void)
return OSHMEM_SUCCESS;
}
/* ////////////////////////////////////////////////////////////////////////// */
static int
ds_copy(const map_segment_t *from,
map_segment_t *to)
{
memcpy(to, from, sizeof(map_segment_t));
OPAL_OUTPUT_VERBOSE(
(70, oshmem_sshmem_base_framework.framework_output,
"%s: %s: ds_copy complete "
"from: (id: %d, size: %lu, "
"name: %s flags: 0x%02x) "
"to: (id: %d, size: %lu, "
"name: %s flags: 0x%02x)\n",
mca_sshmem_mmap_component.super.base_version.mca_type_name,
mca_sshmem_mmap_component.super.base_version.mca_component_name,
from->seg_id, (unsigned long)from->seg_size, from->seg_name,
from->flags, to->seg_id, (unsigned long)to->seg_size, to->seg_name,
to->flags)
);
return OSHMEM_SUCCESS;
}
/* ////////////////////////////////////////////////////////////////////////// */
static int
segment_create(map_segment_t *ds_buf,
@ -225,11 +166,11 @@ segment_create(map_segment_t *ds_buf,
OPAL_OUTPUT_VERBOSE(
(70, oshmem_sshmem_base_framework.framework_output,
"%s: %s: create %s "
"(id: %d, addr: %p size: %lu, name: %s)\n",
"(id: %d, addr: %p size: %lu)\n",
mca_sshmem_mmap_component.super.base_version.mca_type_name,
mca_sshmem_mmap_component.super.base_version.mca_component_name,
(rc ? "failure" : "successful"),
ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size)
);
return rc;
@ -316,10 +257,10 @@ segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
OPAL_OUTPUT_VERBOSE(
(70, oshmem_sshmem_base_framework.framework_output,
"%s: %s: attach successful "
"(id: %d, addr: %p size: %lu, name: %s | va_base: 0x%p len: %d key %llx)\n",
"(id: %d, addr: %p size: %lu | va_base: 0x%p len: %d key %llx)\n",
mca_sshmem_mmap_component.super.base_version.mca_type_name,
mca_sshmem_mmap_component.super.base_version.mca_component_name,
ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name,
ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size,
mkey->va_base, mkey->len, (unsigned long long)mkey->u.key)
);
@ -338,10 +279,10 @@ segment_detach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
OPAL_OUTPUT_VERBOSE(
(70, oshmem_sshmem_base_framework.framework_output,
"%s: %s: detaching "
"(id: %d, addr: %p size: %lu, name: %s)\n",
"(id: %d, addr: %p size: %lu)\n",
mca_sshmem_mmap_component.super.base_version.mca_type_name,
mca_sshmem_mmap_component.super.base_version.mca_component_name,
ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size)
);
munmap((void *)ds_buf->super.va_base, ds_buf->seg_size);
@ -363,10 +304,10 @@ segment_unlink(map_segment_t *ds_buf)
OPAL_OUTPUT_VERBOSE(
(70, oshmem_sshmem_base_framework.framework_output,
"%s: %s: unlinking "
"(id: %d, addr: %p size: %lu, name: %s)\n",
"(id: %d, addr: %p size: %lu)\n",
mca_sshmem_mmap_component.super.base_version.mca_type_name,
mca_sshmem_mmap_component.super.base_version.mca_component_name,
ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size)
);
/* don't completely reset. in particular, only reset

16
oshmem/mca/sshmem/sshmem.h
Просмотреть файл

@ -21,7 +21,6 @@
*
* - module_init
* - segment_create
* - ds_copy
* - segment_attach
* - segment_detach
* - unlink
@ -74,20 +73,6 @@ typedef struct mca_sshmem_base_component_2_0_0_t mca_sshmem_base_component_t;
typedef int
(*mca_sshmem_base_module_init_fn_t)(void);
/**
* copy shmem data structure information pointed to by from to the structure
* pointed to by to.
*
* @param from source pointer (IN).
*
* @param to destination pointer (OUT).
*
* @return OSHMEM_SUCCESS on success.
*/
typedef int
(*mca_sshmem_base_ds_copy_fn_t)(const map_segment_t *from,
map_segment_t *to);
/**
* create a new shared memory segment and initialize members in structure
* pointed to by ds_buf.
@ -153,7 +138,6 @@ typedef int (*mca_sshmem_base_module_finalize_fn_t)(void);
struct mca_sshmem_base_module_2_0_0_t {
mca_sshmem_base_module_init_fn_t module_init;
mca_sshmem_base_module_segment_create_fn_t segment_create;
mca_sshmem_base_ds_copy_fn_t ds_copy;
mca_sshmem_base_module_segment_attach_fn_t segment_attach;
mca_sshmem_base_module_segment_detach_fn_t segment_detach;
mca_sshmem_base_module_unlink_fn_t unlink;

4
oshmem/mca/sshmem/sshmem_types.h
Просмотреть файл

@ -72,6 +72,7 @@ typedef enum {
MAP_SEGMENT_ALLOC_SHM,
MAP_SEGMENT_ALLOC_IBV,
MAP_SEGMENT_ALLOC_IBV_NOSHMR,
MAP_SEGMENT_ALLOC_UCX,
MAP_SEGMENT_UNKNOWN
} segment_type_t;
@ -112,9 +113,10 @@ typedef struct map_segment {
sshmem_mkey_t *mkeys; /* includes local segment bases in va_base */
segment_flag_t flags; /* enable/disable flag */
int seg_id;
char seg_name[OPAL_PATH_MAX];
size_t seg_size; /* length of the segment */
segment_type_t type; /* type of the segment */
void *context; /* allocator can use this field to store
its own private data */
} map_segment_t;
END_C_DECLS

72
oshmem/mca/sshmem/sysv/sshmem_sysv_module.c
Просмотреть файл

@ -62,10 +62,6 @@ segment_create(map_segment_t *ds_buf,
const char *file_name,
size_t size);
static int
ds_copy(const map_segment_t *from,
map_segment_t *to);
static void *
segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey);
@ -84,7 +80,6 @@ mca_sshmem_sysv_module_t mca_sshmem_sysv_module = {
{
module_init,
segment_create,
ds_copy,
segment_attach,
segment_detach,
segment_unlink,
@ -92,34 +87,6 @@ mca_sshmem_sysv_module_t mca_sshmem_sysv_module = {
}
};
/* ////////////////////////////////////////////////////////////////////////// */
/* private utility functions */
/* ////////////////////////////////////////////////////////////////////////// */
/* ////////////////////////////////////////////////////////////////////////// */
/**
* completely resets the contents of *ds_buf
*/
static inline void
shmem_ds_reset(map_segment_t *ds_buf)
{
OPAL_OUTPUT_VERBOSE(
(70, oshmem_sshmem_base_framework.framework_output,
"%s: %s: shmem_ds_resetting "
"(id: %d, size: %lu, name: %s)\n",
mca_sshmem_sysv_component.super.base_version.mca_type_name,
mca_sshmem_sysv_component.super.base_version.mca_component_name,
ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
);
MAP_SEGMENT_RESET_FLAGS(ds_buf);
ds_buf->seg_id = MAP_SEGMENT_SHM_INVALID;
ds_buf->super.va_base = 0;
ds_buf->super.va_end = 0;
ds_buf->seg_size = 0;
ds_buf->type = MAP_SEGMENT_UNKNOWN;
memset(ds_buf->seg_name, '\0', sizeof(ds_buf->seg_name));
}
/* ////////////////////////////////////////////////////////////////////////// */
static int
@ -137,29 +104,6 @@ module_finalize(void)
return OSHMEM_SUCCESS;
}
/* ////////////////////////////////////////////////////////////////////////// */
static int
ds_copy(const map_segment_t *from,
map_segment_t *to)
{
memcpy(to, from, sizeof(map_segment_t));
OPAL_OUTPUT_VERBOSE(
(70, oshmem_sshmem_base_framework.framework_output,
"%s: %s: ds_copy complete "
"from: (id: %d, size: %lu, "
"name: %s flags: 0x%02x) "
"to: (id: %d, size: %lu, "
"name: %s flags: 0x%02x)\n",
mca_sshmem_sysv_component.super.base_version.mca_type_name,
mca_sshmem_sysv_component.super.base_version.mca_component_name,
from->seg_id, (unsigned long)from->seg_size, from->seg_name,
from->flags, to->seg_id, (unsigned long)to->seg_size, to->seg_name,
to->flags)
);
return OSHMEM_SUCCESS;
}
/* ////////////////////////////////////////////////////////////////////////// */
static int
@ -232,11 +176,11 @@ segment_create(map_segment_t *ds_buf,
OPAL_OUTPUT_VERBOSE(
(70, oshmem_sshmem_base_framework.framework_output,
"%s: %s: create %s "
"(id: %d, addr: %p size: %lu, name: %s)\n",
"(id: %d, addr: %p size: %lu)\n",
mca_sshmem_sysv_component.super.base_version.mca_type_name,
mca_sshmem_sysv_component.super.base_version.mca_component_name,
(rc ? "failure" : "successful"),
ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size)
);
return rc;
@ -261,10 +205,10 @@ segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
OPAL_OUTPUT_VERBOSE(
(70, oshmem_sshmem_base_framework.framework_output,
"%s: %s: attach successful "
"(id: %d, addr: %p size: %lu, name: %s | va_base: 0x%p len: %d key %llx)\n",
"(id: %d, addr: %p size: %lu | va_base: 0x%p len: %d key %llx)\n",
mca_sshmem_sysv_component.super.base_version.mca_type_name,
mca_sshmem_sysv_component.super.base_version.mca_component_name,
ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name,
ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size,
mkey->va_base, mkey->len, (unsigned long long)mkey->u.key)
);
@ -283,10 +227,10 @@ segment_detach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
OPAL_OUTPUT_VERBOSE(
(70, oshmem_sshmem_base_framework.framework_output,
"%s: %s: detaching "
"(id: %d, addr: %p size: %lu, name: %s)\n",
"(id: %d, addr: %p size: %lu)\n",
mca_sshmem_sysv_component.super.base_version.mca_type_name,
mca_sshmem_sysv_component.super.base_version.mca_component_name,
ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size)
);
if (ds_buf->seg_id != MAP_SEGMENT_SHM_INVALID) {
@ -318,10 +262,10 @@ segment_unlink(map_segment_t *ds_buf)
OPAL_OUTPUT_VERBOSE(
(70, oshmem_sshmem_base_framework.framework_output,
"%s: %s: unlinking "
"(id: %d, size: %lu, name: %s)\n",
"(id: %d, size: %lu)\n",
mca_sshmem_sysv_component.super.base_version.mca_type_name,
mca_sshmem_sysv_component.super.base_version.mca_component_name,
ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
ds_buf->seg_id, (unsigned long)ds_buf->seg_size)
);
/* don't completely reset. in particular, only reset

40
oshmem/mca/sshmem/ucx/Makefile.am Обычный файл
Просмотреть файл

@ -0,0 +1,40 @@
# Copyright (c) 2014 Mellanox Technologies, Inc.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
#dist_oshmemdata_DATA = help-oshmem-sshmem-mmap.txt
AM_CPPFLAGS = $(sshmem_ucx_CPPFLAGS)
sources = \
sshmem_ucx.h \
sshmem_ucx_component.c \
sshmem_ucx_module.c
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds).
if MCA_BUILD_oshmem_sshmem_ucx_DSO
component_noinst =
component_install = mca_sshmem_ucx.la
else
component_noinst = libmca_sshmem_ucx.la
component_install =
endif
mcacomponentdir = $(oshmemlibdir)
mcacomponent_LTLIBRARIES = $(component_install)
mca_sshmem_ucx_la_SOURCES = $(sources)
mca_sshmem_ucx_la_LDFLAGS = -module -avoid-version $(sshmem_ucx_LDFLAGS)
mca_sshmem_ucx_la_LIBADD = $(sshmem_ucx_LIBS)
noinst_LTLIBRARIES = $(component_noinst)
libmca_sshmem_ucx_la_SOURCES =$(sources)
libmca_sshmem_ucx_la_LDFLAGS = -module -avoid-version $(sshmem_ucx_LDFLAGS)

32
oshmem/mca/sshmem/ucx/configure.m4 Обычный файл
Просмотреть файл

@ -0,0 +1,32 @@
/*
* Copyright (c) 2017 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
# MCA_oshmem_sshmem_ucx_CONFIG([action-if-can-compile],
# [action-if-cant-compile])
# ------------------------------------------------
AC_DEFUN([MCA_oshmem_sshmem_ucx_CONFIG],[
AC_CONFIG_FILES([oshmem/mca/sshmem/ucx/Makefile])
OMPI_CHECK_UCX([sshmem_ucx],
[sshmem_ucx_happy="yes"],
[sshmem_ucx_happy="no"])
AS_IF([test "$sshmem_ucx_happy" = "yes"],
[$1],
[$2])
# substitute in the things needed to build ucx
AC_SUBST([sshmem_ucx_CFLAGS])
AC_SUBST([sshmem_ucx_CPPFLAGS])
AC_SUBST([sshmem_ucx_LDFLAGS])
AC_SUBST([sshmem_ucx_LIBS])
])dnl

40
oshmem/mca/sshmem/ucx/sshmem_ucx.h Обычный файл
Просмотреть файл

@ -0,0 +1,40 @@
/*
* Copyright (c) 2017 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_SSHMEM_UCX_EXPORT_H
#define MCA_SSHMEM_UCX_EXPORT_H
#include "oshmem_config.h"
#include "oshmem/mca/sshmem/sshmem.h"
BEGIN_C_DECLS
/**
* globally exported variable to hold the ucx component.
*/
typedef struct mca_sshmem_ucx_component_t {
/* base component struct */
mca_sshmem_base_component_t super;
/* priority for ucx component */
int priority;
} mca_sshmem_ucx_component_t;
OSHMEM_MODULE_DECLSPEC extern mca_sshmem_ucx_component_t
mca_sshmem_ucx_component;
typedef struct mca_sshmem_ucx_module_t {
mca_sshmem_base_module_t super;
} mca_sshmem_ucx_module_t;
extern mca_sshmem_ucx_module_t mca_sshmem_ucx_module;
END_C_DECLS
#endif /* MCA_SHMEM_UCX_EXPORT_H */

125
oshmem/mca/sshmem/ucx/sshmem_ucx_component.c Обычный файл
Просмотреть файл

@ -0,0 +1,125 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2017 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "oshmem_config.h"
#include "opal/constants.h"
#include "oshmem/mca/sshmem/sshmem.h"
#include "oshmem/mca/sshmem/base/base.h"
#include "oshmem/mca/spml/base/base.h"
#include "sshmem_ucx.h"
/**
* public string showing the shmem ompi_ucx component version number
*/
const char *mca_sshmem_ucx_component_version_string =
"OSHMEM ucx sshmem MCA component version " OSHMEM_VERSION;
/**
* local functions
*/
static int ucx_register(void);
static int ucx_open(void);
static int ucx_close(void);
static int ucx_query(mca_base_module_t **module, int *priority);
static int ucx_runtime_query(mca_base_module_t **module,
int *priority,
const char *hint);
/**
* instantiate the public struct with all of our public information
* and pointers to our public functions in it
*/
mca_sshmem_ucx_component_t mca_sshmem_ucx_component = {
/* ////////////////////////////////////////////////////////////////////// */
/* super */
/* ////////////////////////////////////////////////////////////////////// */
{
/**
* common MCA component data
*/
.base_version = {
MCA_SSHMEM_BASE_VERSION_2_0_0,
/* component name and version */
.mca_component_name = "ucx",
MCA_BASE_MAKE_VERSION(component, OSHMEM_MAJOR_VERSION, OSHMEM_MINOR_VERSION,
OSHMEM_RELEASE_VERSION),
.mca_open_component = ucx_open,
.mca_close_component = ucx_close,
.mca_query_component = ucx_query,
.mca_register_component_params = ucx_register,
},
/* MCA v2.0.0 component meta data */
.base_data = {
/* the component is checkpoint ready */
MCA_BASE_METADATA_PARAM_CHECKPOINT
},
.runtime_query = ucx_runtime_query,
},
};
static int
ucx_runtime_query(mca_base_module_t **module,
int *priority,
const char *hint)
{
/* check that spml ucx was selected. Otherwise disqualify */
if (strcmp(mca_spml_base_selected_component.spmlm_version.mca_component_name, "ucx")) {
*module = NULL;
return OSHMEM_ERR_NOT_AVAILABLE;
}
*priority = mca_sshmem_ucx_component.priority;
*module = (mca_base_module_t *)&mca_sshmem_ucx_module.super;
return OPAL_SUCCESS;
}
static int
ucx_register(void)
{
/* (default) priority - set high to make ucx the default */
mca_sshmem_ucx_component.priority = 100;
mca_base_component_var_register (&mca_sshmem_ucx_component.super.base_version,
"priority", "Priority for sshmem ucx "
"component (default: 100)", MCA_BASE_VAR_TYPE_INT,
NULL, 0, MCA_BASE_VAR_FLAG_SETTABLE,
OPAL_INFO_LVL_3,
MCA_BASE_VAR_SCOPE_ALL_EQ,
&mca_sshmem_ucx_component.priority);
return OSHMEM_SUCCESS;
}
static int
ucx_open(void)
{
return OSHMEM_SUCCESS;
}
static int
ucx_query(mca_base_module_t **module, int *priority)
{
*priority = mca_sshmem_ucx_component.priority;
*module = (mca_base_module_t *)&mca_sshmem_ucx_module.super;
return OSHMEM_SUCCESS;
}
static int
ucx_close(void)
{
return OSHMEM_SUCCESS;
}

190
oshmem/mca/sshmem/ucx/sshmem_ucx_module.c Обычный файл
Просмотреть файл

@ -0,0 +1,190 @@
/*
* Copyright (c) 2017 Mellanox Technologies, Inc.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "oshmem_config.h"
#include "opal/constants.h"
#include "opal/util/output.h"
#include "opal/util/path.h"
#include "opal/util/show_help.h"
#include "orte/util/show_help.h"
#include "oshmem/proc/proc.h"
#include "oshmem/mca/sshmem/sshmem.h"
#include "oshmem/mca/sshmem/base/base.h"
#include "oshmem/util/oshmem_util.h"
#include "oshmem/mca/spml/ucx/spml_ucx.h"
#include "sshmem_ucx.h"
/* ////////////////////////////////////////////////////////////////////////// */
/*local functions */
/* local functions */
static int
module_init(void);
static int
segment_create(map_segment_t *ds_buf,
const char *file_name,
size_t size);
static void *
segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey);
static int
segment_detach(map_segment_t *ds_buf, sshmem_mkey_t *mkey);
static int
segment_unlink(map_segment_t *ds_buf);
static int
module_finalize(void);
/*
* ucx shmem module
*/
mca_sshmem_ucx_module_t mca_sshmem_ucx_module = {
/* super */
{
module_init,
segment_create,
segment_attach,
segment_detach,
segment_unlink,
module_finalize
}
};
static int
module_init(void)
{
/* nothing to do */
return OSHMEM_SUCCESS;
}
/* ////////////////////////////////////////////////////////////////////////// */
static int
module_finalize(void)
{
/* nothing to do */
return OSHMEM_SUCCESS;
}
/* ////////////////////////////////////////////////////////////////////////// */
static int
segment_create(map_segment_t *ds_buf,
const char *file_name,
size_t size)
{
int rc = OSHMEM_SUCCESS;
mca_spml_ucx_t *spml = (mca_spml_ucx_t *)mca_spml.self;
ucp_mem_map_params_t mem_map_params;
ucp_mem_h mem_h;
ucs_status_t status;
assert(ds_buf);
/* init the contents of map_segment_t */
shmem_ds_reset(ds_buf);
mem_map_params.field_mask = UCP_MEM_MAP_PARAM_FIELD_ADDRESS |
UCP_MEM_MAP_PARAM_FIELD_LENGTH |
UCP_MEM_MAP_PARAM_FIELD_FLAGS;
mem_map_params.address = (void *)mca_sshmem_base_start_address;
mem_map_params.length = size;
mem_map_params.flags = UCP_MEM_MAP_ALLOCATE|UCP_MEM_MAP_FIXED;
if (spml->heap_reg_nb) {
mem_map_params.flags |= UCP_MEM_MAP_NONBLOCK;
}
status = ucp_mem_map(spml->ucp_context, &mem_map_params, &mem_h);
if (UCS_OK != status) {
rc = OSHMEM_ERROR;
goto out;
}
ds_buf->super.va_base = mem_map_params.address;
ds_buf->seg_size = size;
ds_buf->super.va_end = (void*)((uintptr_t)ds_buf->super.va_base + ds_buf->seg_size);
ds_buf->context = mem_h;
ds_buf->type = MAP_SEGMENT_ALLOC_UCX;
out:
OPAL_OUTPUT_VERBOSE(
(70, oshmem_sshmem_base_framework.framework_output,
"%s: %s: create %s "
"(id: %d, addr: %p size: %lu)\n",
mca_sshmem_ucx_component.super.base_version.mca_type_name,
mca_sshmem_ucx_component.super.base_version.mca_component_name,
(rc ? "failure" : "successful"),
ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size)
);
return rc;
}
static void *
segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
{
assert(ds_buf);
assert(mkey->va_base == 0);
OPAL_OUTPUT((oshmem_sshmem_base_framework.framework_output,
"can not attach to ucx segment"));
oshmem_shmem_abort(-1);
return NULL;
}
static int
segment_detach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
{
OPAL_OUTPUT_VERBOSE(
(70, oshmem_sshmem_base_framework.framework_output,
"%s: %s: detaching "
"(id: %d, addr: %p size: %lu)\n",
mca_sshmem_ucx_component.super.base_version.mca_type_name,
mca_sshmem_ucx_component.super.base_version.mca_component_name,
ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size)
);
/* reset the contents of the map_segment_t associated with this
* shared memory segment.
*/
shmem_ds_reset(ds_buf);
return OSHMEM_SUCCESS;
}
static int
segment_unlink(map_segment_t *ds_buf)
{
mca_spml_ucx_t *spml = (mca_spml_ucx_t *)mca_spml.self;
assert(ds_buf);
ucp_mem_unmap(spml->ucp_context, (ucp_mem_h)ds_buf->context);
OPAL_OUTPUT_VERBOSE(
(70, oshmem_sshmem_base_framework.framework_output,
"%s: %s: unlinking "
"(id: %d, addr: %p size: %lu)\n",
mca_sshmem_ucx_component.super.base_version.mca_type_name,
mca_sshmem_ucx_component.super.base_version.mca_component_name,
ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size)
);
ds_buf->seg_id = MAP_SEGMENT_SHM_INVALID;
MAP_SEGMENT_INVALIDATE(ds_buf);
return OSHMEM_SUCCESS;
}

73
oshmem/mca/sshmem/verbs/sshmem_verbs_module.c
Просмотреть файл

@ -56,10 +56,6 @@ segment_create(map_segment_t *ds_buf,
const char *file_name,
size_t size);
static int
ds_copy(const map_segment_t *from,
map_segment_t *to);
static void *
segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey);
@ -80,7 +76,6 @@ mca_sshmem_verbs_module_t mca_sshmem_verbs_module = {
{
module_init,
segment_create,
ds_copy,
segment_attach,
segment_detach,
segment_unlink,
@ -88,35 +83,6 @@ mca_sshmem_verbs_module_t mca_sshmem_verbs_module = {
}
};
/* ////////////////////////////////////////////////////////////////////////// */
/* private utility functions */
/* ////////////////////////////////////////////////////////////////////////// */
/* ////////////////////////////////////////////////////////////////////////// */
/**
* completely resets the contents of *ds_buf
*/
static inline void
shmem_ds_reset(map_segment_t *ds_buf)
{
OPAL_OUTPUT_VERBOSE(
(70, oshmem_sshmem_base_framework.framework_output,
"%s: %s: shmem_ds_resetting "
"(id: %d, size: %lu, name: %s)\n",
mca_sshmem_verbs_component.super.base_version.mca_type_name,
mca_sshmem_verbs_component.super.base_version.mca_component_name,
ds_buf->seg_id, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
);
MAP_SEGMENT_RESET_FLAGS(ds_buf);
ds_buf->seg_id = MAP_SEGMENT_SHM_INVALID;
ds_buf->super.va_base = 0;
ds_buf->super.va_end = 0;
ds_buf->seg_size = 0;
ds_buf->type = MAP_SEGMENT_UNKNOWN;
memset(ds_buf->seg_name, '\0', sizeof(ds_buf->seg_name));
}
/* ////////////////////////////////////////////////////////////////////////// */
static int
module_init(void)
@ -133,29 +99,6 @@ module_finalize(void)
return OSHMEM_SUCCESS;
}
/* ////////////////////////////////////////////////////////////////////////// */
static int
ds_copy(const map_segment_t *from,
map_segment_t *to)
{
memcpy(to, from, sizeof(map_segment_t));
OPAL_OUTPUT_VERBOSE(
(70, oshmem_sshmem_base_framework.framework_output,
"%s: %s: ds_copy complete "
"from: (id: %d, size: %lu, "
"name: %s flags: 0x%02x) "
"to: (id: %d, size: %lu, "
"name: %s flags: 0x%02x)\n",
mca_sshmem_verbs_component.super.base_version.mca_type_name,
mca_sshmem_verbs_component.super.base_version.mca_component_name,
from->seg_id, (unsigned long)from->seg_size, from->seg_name,
from->flags, to->seg_id, (unsigned long)to->seg_size, to->seg_name,
to->flags)
);
return OSHMEM_SUCCESS;
}
/* ////////////////////////////////////////////////////////////////////////// */
static int
@ -329,11 +272,11 @@ segment_create(map_segment_t *ds_buf,
OPAL_OUTPUT_VERBOSE(
(70, oshmem_sshmem_base_framework.framework_output,
"%s: %s: create %s "
"(id: %d, addr: %p size: %lu, name: %s)\n",
"(id: %d, addr: %p size: %lu)\n",
mca_sshmem_verbs_component.super.base_version.mca_type_name,
mca_sshmem_verbs_component.super.base_version.mca_component_name,
(rc ? "failure" : "successful"),
ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size)
);
return rc;
@ -395,10 +338,10 @@ segment_attach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
OPAL_OUTPUT_VERBOSE(
(70, oshmem_sshmem_base_framework.framework_output,
"%s: %s: attach successful "
"(id: %d, addr: %p size: %lu, name: %s | va_base: 0x%p len: %d key %llx)\n",
"(id: %d, addr: %p size: %lu | va_base: 0x%p len: %d key %llx)\n",
mca_sshmem_verbs_component.super.base_version.mca_type_name,
mca_sshmem_verbs_component.super.base_version.mca_component_name,
ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name,
ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size,
mkey->va_base, mkey->len, (unsigned long long)mkey->u.key)
);
@ -419,10 +362,10 @@ segment_detach(map_segment_t *ds_buf, sshmem_mkey_t *mkey)
OPAL_OUTPUT_VERBOSE(
(70, oshmem_sshmem_base_framework.framework_output,
"%s: %s: detaching "
"(id: %d, addr: %p size: %lu, name: %s)\n",
"(id: %d, addr: %p size: %lu)\n",
mca_sshmem_verbs_component.super.base_version.mca_type_name,
mca_sshmem_verbs_component.super.base_version.mca_component_name,
ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size)
);
if (device) {
@ -498,10 +441,10 @@ segment_unlink(map_segment_t *ds_buf)
OPAL_OUTPUT_VERBOSE(
(70, oshmem_sshmem_base_framework.framework_output,
"%s: %s: unlinking "
"(id: %d, addr: %p size: %lu, name: %s)\n",
"(id: %d, addr: %p size: %lu)\n",
mca_sshmem_verbs_component.super.base_version.mca_type_name,
mca_sshmem_verbs_component.super.base_version.mca_component_name,
ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size, ds_buf->seg_name)
ds_buf->seg_id, ds_buf->super.va_base, (unsigned long)ds_buf->seg_size)
);
/* don't completely reset. in particular, only reset