ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Fix knem support in both sm and vader

Просмотр исходного кода
Этот коммит содержится в:
Nathan Hjelm 2014-11-02 09:36:36 -08:00 коммит произвёл Nathan Hjelm
родитель ee2b111011
Коммит 249e5e009f
8 изменённых файлов: 126 добавлений и 100 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

61
opal/mca/btl/sm/btl_sm.c
Просмотреть файл

@ -57,6 +57,9 @@
#include "opal/mca/mpool/base/base.h"
#include "opal/mca/mpool/sm/mpool_sm.h"
#include "opal/align.h"
#include "opal/util/sys_limits.h"
#if OPAL_ENABLE_FT_CR == 1
#include "opal/util/basename.h"
#include "opal/mca/crs/base/base.h"
@ -81,9 +84,6 @@ mca_btl_sm_t mca_btl_sm = {
.btl_alloc = mca_btl_sm_alloc,
.btl_free = mca_btl_sm_free,
.btl_prepare_src = mca_btl_sm_prepare_src,
#if OPAL_BTL_SM_HAVE_KNEM || OPAL_BTL_SM_HAVE_CMA
.btl_prepare_dst = mca_btl_sm_prepare_dst,
#endif /* OPAL_BTL_SM_HAVE_KNEM || OPAL_BTL_SM_HAVE_CMA */
.btl_send = mca_btl_sm_send,
.btl_sendi = mca_btl_sm_sendi,
.btl_dump = mca_btl_sm_dump,
@ -1002,19 +1002,26 @@ mca_btl_base_registration_handle_t *mca_btl_sm_register_mem (struct mca_btl_base
struct mca_btl_base_endpoint_t* endpoint,
void *base, size_t size, uint32_t flags)
{
mca_btl_sm_registration_handle_t *handle = NULL;
mca_btl_sm_registration_handle_t *handle;
mca_btl_sm_t *sm_btl = (mca_btl_sm_t *) btl;
ompi_free_list_item_t *item = NULL;
OMPI_FREE_LIST_GET_MT(&mca_btl_sm_component.registration_handles, &handle);
if (OPAL_UNLIKELY(NULL == handle)) {
OMPI_FREE_LIST_GET_MT(&mca_btl_sm_component.registration_handles, item);
if (OPAL_UNLIKELY(NULL == item)) {
return NULL;
}
handle = (mca_btl_sm_registration_handle_t *) item;
#if OPAL_BTL_SM_HAVE_KNEM
if (OPAL_LIKELY(mca_btl_sm_component.use_knem)) {
knem_iov.base = (uintptr_t)base & (opal_getpagesize() - 1);
knem_iov.len = OPAL_ALIGN(size + ((intptr_t) base - knem_iov.base), opal_getpagesize());
struct knem_cmd_create_region knem_cr;
struct knem_cmd_param_iovec knem_iov;
knem_iov.base = (uintptr_t)base & ~(opal_getpagesize() - 1);
knem_iov.len = OPAL_ALIGN(size + ((intptr_t) base - knem_iov.base), opal_getpagesize(), intptr_t);
knem_cr.iovec_array = (uintptr_t)&knem_iov;
knem_cr.iovec_nr = iov_count;
knem_cr.iovec_nr = 1;
knem_cr.flags = 0;
knem_cr.protection = 0;
@ -1026,7 +1033,8 @@ mca_btl_base_registration_handle_t *mca_btl_sm_register_mem (struct mca_btl_base
}
if (OPAL_UNLIKELY(ioctl(sm_btl->knem_fd, KNEM_CMD_CREATE_REGION, &knem_cr) < 0)) {
return NULL;
OMPI_FREE_LIST_RETURN_MT(&mca_btl_sm_component.registration_handles, item);
return NULL;
}
handle->btl_handle.data.knem.cookie = knem_cr.cookie;
@ -1043,14 +1051,15 @@ mca_btl_base_registration_handle_t *mca_btl_sm_register_mem (struct mca_btl_base
return &handle->btl_handle;
}
void mca_btl_sm_deregister_mem (struct mca_btl_base_module_t* btl, mca_btl_base_registration_handle_t *handle)
int mca_btl_sm_deregister_mem (struct mca_btl_base_module_t* btl, mca_btl_base_registration_handle_t *handle)
{
mca_btl_sm_registration_handle_t *sm_handle =
(mca_btl_sm_registration_handle_t *)((intptr_t) handle - offsetof (mca_btl_sm_registration_handle_t, btl_handle));
mca_btl_sm_t* sm_btl = (mca_btl_sm_t*) btl;
#if OPAL_BTL_SM_HAVE_KNEM
if (OPAL_LIKELY(mca_btl_sm_component.use_knem)) {
(void) ioctl(sm_btl->knem_fd, KNEM_CMD_DESTROY_REGION, &handle->cookie);
(void) ioctl(sm_btl->knem_fd, KNEM_CMD_DESTROY_REGION, &handle->data.knem.cookie);
}
#endif
@ -1065,15 +1074,11 @@ void mca_btl_sm_deregister_mem (struct mca_btl_base_module_t* btl, mca_btl_base_
/**
* Initiate an synchronous get.
*/
int mca_btl_sm_get_sync (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
int mca_btl_sm_get_sync (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, void *local_address,
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
{
int btl_ownership;
mca_btl_sm_frag_t* frag = (mca_btl_sm_frag_t*)des;
mca_btl_sm_segment_t *src = (mca_btl_sm_segment_t*)des->des_remote;
mca_btl_sm_segment_t *dst = (mca_btl_sm_segment_t*)des->des_segments;
#if OPAL_BTL_SM_HAVE_KNEM
mca_btl_sm_t* sm_btl = (mca_btl_sm_t*) btl;
if (OPAL_LIKELY(mca_btl_sm_component.use_knem)) {
@ -1087,7 +1092,7 @@ int mca_btl_sm_get_sync (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *en
icopy.local_iovec_array = (uintptr_t)&recv_iovec;
icopy.local_iovec_nr = 1;
icopy.remote_cookie = remote_handle->data.knem.cookie;
icopy.remote_offset = remote_address - remote_handle->base_addr;
icopy.remote_offset = remote_address - remote_handle->data.knem.base_addr;
icopy.write = 0;
/* Use the DMA flag if knem supports it *and* the segment length
@ -1117,18 +1122,15 @@ int mca_btl_sm_get_sync (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *en
pid_t remote_pid;
int val;
local_address = (char *)(uintptr_t) dst->base.seg_addr.lval;
local_length = dst->base.seg_len;
remote_pid = remote_handle->data.pid;
remote.iov_base = remote_address;
remote.iov_base = (void *) (intptr_t) remote_address;
remote.iov_len = size;
local.iov_base = local_address;
local.iov_len = size;
val = process_vm_readv(remote_pid, &local, 1, &remote, 1, 0);
if (val != local_length) {
if (val != size) {
if (val<0) {
opal_output(0, "mca_btl_sm_get_sync: process_vm_readv failed: %i",
errno);
@ -1155,16 +1157,13 @@ int mca_btl_sm_get_sync (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *en
/**
* Initiate an asynchronous get.
*/
int mca_btl_sm_get_async (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
int mca_btl_sm_get_async (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, void *local_address,
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata)
{
int btl_ownership;
mca_btl_sm_t* sm_btl = (mca_btl_sm_t*) btl;
mca_btl_sm_frag_t* frag;
mca_btl_sm_segment_t *src = (mca_btl_sm_segment_t*)des->des_remote;
mca_btl_sm_segment_t *dst = (mca_btl_sm_segment_t*)des->des_segments;
struct knem_cmd_inline_copy icopy;
struct knem_cmd_param_iovec recv_iovec;
@ -1172,14 +1171,14 @@ int mca_btl_sm_get_async (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *e
if (sm_btl->knem_status_num_used >=
mca_btl_sm_component.knem_max_simultaneous) {
return mca_btl_sm_get_sync (btl, endpoint, local_address, remote_address, local_handle,
remote_handle, size, flags, cbfunc, cbcontext, cbdata);
remote_handle, size, flags, order, cbfunc, cbcontext, cbdata);
}
/* allocate a fragment to keep track of this transaction */
MCA_BTL_SM_FRAG_ALLOC_USER(frag);
if (OPAL_UNLIKELY(NULL == frag)) {
return mca_btl_sm_get_sync (btl, endpoint, local_address, remote_address, local_handle,
remote_handle, size, flags, cbfunc, cbcontext, cbdata);
remote_handle, size, flags, order, cbfunc, cbcontext, cbdata);
}
/* fill in callback data */
@ -1192,7 +1191,7 @@ int mca_btl_sm_get_async (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *e
/* We have a slot, so fill in the data fields. Bump the
first_avail and num_used counters. */
recv_iovec.base = (uintptr_t) local_address;
recv_iovec.len = dst->base.seg_len;
recv_iovec.len = size;
icopy.local_iovec_array = (uintptr_t)&recv_iovec;
icopy.local_iovec_nr = 1;
icopy.write = 0;
@ -1208,7 +1207,7 @@ int mca_btl_sm_get_async (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *e
/* Use the DMA flag if knem supports it *and* the segment length
is greater than the cutoff */
icopy.flags = KNEM_FLAG_ASYNCDMACOMPLETE;
if (mca_btl_sm_component.knem_dma_min <= dst->base.seg_len) {
if (mca_btl_sm_component.knem_dma_min <= size) {
icopy.flags = mca_btl_sm_component.knem_dma_flag;
}

9
opal/mca/btl/sm/btl_sm.h
Просмотреть файл

@ -183,6 +183,8 @@ struct mca_btl_sm_component_t {
#if OPAL_BTL_SM_HAVE_KNEM
/* Knem capabilities info */
struct knem_cmd_info knem_info;
#endif
#if OPAL_BTL_SM_HAVE_KNEM || OPAL_BTL_SM_HAVE_CMA
/** registration handles to hold knem cookies */
ompi_free_list_t registration_handles;
#endif /* OPAL_BTL_SM_HAVE_KNEM */
@ -506,7 +508,7 @@ extern int mca_btl_sm_send(
/*
* Synchronous knem/cma get
*/
int mca_btl_sm_get_sync (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
int mca_btl_sm_get_sync (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, void *local_address,
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
@ -516,7 +518,7 @@ int mca_btl_sm_get_sync (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *en
/*
* Asynchronous knem get
*/
int mca_btl_sm_get_async (mca_btl_base_module_t *btl, mca_btl_base_endpoint_t *endpoint, void *local_address,
int mca_btl_sm_get_async (mca_btl_base_module_t *btl, struct mca_btl_base_endpoint_t *endpoint, void *local_address,
uint64_t remote_address, mca_btl_base_registration_handle_t *local_handle,
mca_btl_base_registration_handle_t *remote_handle, size_t size, int flags,
int order, mca_btl_base_rdma_completion_fn_t cbfunc, void *cbcontext, void *cbdata);
@ -566,13 +568,12 @@ struct mca_btl_sm_registration_handle_t {
mca_btl_base_registration_handle_t btl_handle;
};
typedef struct mca_btl_sm_registration_handle_t mca_btl_sm_registration_handle_t;
OBJ_CLASS_DECLARATION(mca_btl_sm_registration_handle_t);
mca_btl_base_registration_handle_t *mca_btl_sm_register_mem (struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t* endpoint,
void *base, size_t size, uint32_t flags);
void mca_btl_sm_deregister_mem (struct mca_btl_base_module_t* btl, mca_btl_base_registration_handle_t *handle);
int mca_btl_sm_deregister_mem (struct mca_btl_base_module_t* btl, mca_btl_base_registration_handle_t *handle);
#endif

37
opal/mca/btl/sm/btl_sm_component.c
Просмотреть файл

@ -67,6 +67,8 @@
#include "opal/mca/common/cuda/common_cuda.h"
#endif /* OPAL_CUDA_SUPPORT */
static OBJ_CLASS_INSTANCE(mca_btl_sm_registration_handle_t, ompi_free_list_item_t, NULL, NULL);
static int mca_btl_sm_component_open(void);
static int mca_btl_sm_component_close(void);
static int sm_register(void);
@ -298,9 +300,12 @@ static int mca_btl_sm_component_open(void)
OBJ_CONSTRUCT(&mca_btl_sm_component.pending_send_fl, opal_free_list_t);
mca_btl_sm_component.sm_seg = NULL;
#if OPAL_BTL_SM_HAVE_KNEM
OBJ_CONSTRICT(&mca_btl_sm_component.registration_handles, ompi_free_list_t);
#if OPAL_BTL_SM_HAVE_KNEM || OPAL_BTL_SM_HAVE_CMA
OBJ_CONSTRUCT(&mca_btl_sm_component.registration_handles, ompi_free_list_t);
#endif
#if OPAL_BTL_SM_HAVE_KNEM
mca_btl_sm.knem_fd = -1;
mca_btl_sm.knem_status_array = NULL;
mca_btl_sm.knem_frag_array = NULL;
@ -335,10 +340,12 @@ static int mca_btl_sm_component_close(void)
close(mca_btl_sm.knem_fd);
mca_btl_sm.knem_fd = -1;
}
OBJ_DESTRUCT(&mca_btl_sm_component.registration_handles);
#endif /* OPAL_BTL_SM_HAVE_KNEM */
#if OPAL_BTL_SM_HAVE_KNEM || OPAL_BTL_SM_HAVE_CMA
OBJ_DESTRUCT(&mca_btl_sm_component.registration_handles);
#endif
OBJ_DESTRUCT(&mca_btl_sm_component.sm_lock);
/**
* We don't have to destroy the fragment lists. They are allocated
@ -943,6 +950,21 @@ mca_btl_sm_component_init(int *num_btls,
}
#endif /* OPAL_BTL_SM_HAVE_CMA */
#if OPAL_BTL_SM_HAVE_KNEM | OPAL_BTL_SM_HAVE_CMA
if (mca_btl_sm_component.use_cma || mca_btl_sm_component.use_knem) {
rc = ompi_free_list_init_new (&mca_btl_sm_component.registration_handles,
sizeof (mca_btl_sm_registration_handle_t),
8, OBJ_CLASS(mca_btl_sm_registration_handle_t),
0, 0, mca_btl_sm_component.sm_free_list_num,
mca_btl_sm_component.sm_free_list_max,
mca_btl_sm_component.sm_free_list_inc, NULL);
if (OPAL_SUCCESS != rc) {
free (btls);
return NULL;
}
}
#endif
return btls;
no_knem:
@ -975,6 +997,7 @@ mca_btl_sm_component_init(int *num_btls,
/* disable get when not using knem or cma */
mca_btl_sm.super.btl_get = NULL;
mca_btl_sm.super.btl_flags &= ~MCA_BTL_FLAGS_GET;
mca_btl_sm_component.use_knem = 0;
}
/* Otherwise, use_knem was 0 (and we didn't get here) or use_knem
@ -1192,9 +1215,9 @@ int mca_btl_sm_component_progress(void)
/* Handle the completed fragment */
frag =
mca_btl_sm.knem_frag_array[mca_btl_sm.knem_status_first_used];
frag.cb.func (&mca_btl_sm.super, frag->endpoint,
frag->cb.local_address, frag->cb.local_handle,
frag->cb.context, frag->cb.data, OPAL_SUCCESS);
frag->cb.func (&mca_btl_sm.super, frag->endpoint,
frag->cb.local_address, frag->cb.local_handle,
frag->cb.context, frag->cb.data, OPAL_SUCCESS);
MCA_BTL_SM_FRAG_RETURN(frag);
/* Bump counters, loop around the circular buffer if

3
opal/mca/btl/vader/btl_vader.h
Просмотреть файл

@ -115,9 +115,6 @@ struct mca_btl_vader_component_t {
ompi_free_list_t vader_frags_eager; /**< free list of vader send frags */
ompi_free_list_t vader_frags_max_send; /**< free list of vader max send frags (large fragments) */
ompi_free_list_t vader_frags_user; /**< free list of small inline frags */
#if OPAL_BTL_VADER_HAVE_KNEM
ompi_free_list_t registration_handles; /**< registration handles for knem segments */
#endif
unsigned int fbox_threshold; /**< number of sends required before we setup a send fast box for a peer */
unsigned int fbox_max; /**< maximum number of send fast boxes to allocate */

8
opal/mca/btl/vader/btl_vader_component.c
Просмотреть файл

@ -271,9 +271,6 @@ static int mca_btl_vader_component_open(void)
OBJ_CONSTRUCT(&mca_btl_vader_component.vader_frags_eager, ompi_free_list_t);
OBJ_CONSTRUCT(&mca_btl_vader_component.vader_frags_user, ompi_free_list_t);
OBJ_CONSTRUCT(&mca_btl_vader_component.vader_frags_max_send, ompi_free_list_t);
#if OPAL_BTL_VADER_HAVE_KNEM
OBJ_CONSTRUCT(&mca_btl_vader_component.registration_handles, ompi_free_list_t);
#endif
OBJ_CONSTRUCT(&mca_btl_vader_component.lock, opal_mutex_t);
OBJ_CONSTRUCT(&mca_btl_vader_component.pending_endpoints, opal_list_t);
OBJ_CONSTRUCT(&mca_btl_vader_component.pending_fragments, opal_list_t);
@ -294,9 +291,6 @@ static int mca_btl_vader_component_close(void)
OBJ_DESTRUCT(&mca_btl_vader_component.vader_frags_eager);
OBJ_DESTRUCT(&mca_btl_vader_component.vader_frags_user);
OBJ_DESTRUCT(&mca_btl_vader_component.vader_frags_max_send);
#if OPAL_BTL_VADER_HAVE_KNEM
OBJ_DESTRUCT(&mca_btl_vader_component.registration_handles);
#endif
OBJ_DESTRUCT(&mca_btl_vader_component.lock);
OBJ_DESTRUCT(&mca_btl_vader_component.pending_endpoints);
OBJ_DESTRUCT(&mca_btl_vader_component.pending_fragments);
@ -421,7 +415,7 @@ static void mca_btl_vader_check_single_copy (void)
#if OPAL_BTL_VADER_HAVE_KNEM
if (MCA_BTL_VADER_KNEM == mca_btl_vader_component.single_copy_mechanism) {
/* mca_btl_vader_knem_init will set the appropriate get/put functions */
rc = mca_btl_vader_knem_init ();
int rc = mca_btl_vader_knem_init ();
if (OPAL_SUCCESS != rc) {
if (MCA_BTL_VADER_KNEM == initial_mechanism) {
opal_show_help("help-btl-vader.txt", "knem requested but not available",

87
opal/mca/btl/vader/btl_vader_knem.c
Просмотреть файл

@ -19,6 +19,7 @@
#include <unistd.h>
#include "opal/util/show_help.h"
#include "opal/mca/mpool/grdma/mpool_grdma.h"
struct mca_btl_vader_registration_handle_t {
ompi_free_list_item_t super;
@ -28,65 +29,79 @@ typedef struct mca_btl_vader_registration_handle_t mca_btl_vader_registration_ha
OBJ_CLASS_INSTANCE(mca_btl_vader_registration_handle_t, ompi_free_list_item_t, NULL, NULL);
static mca_btl_base_registration_handle_t *
mca_btl_vader_register_mem_knem (struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t *endpoint,
void *base, size_t size, uint32_t flags)
static int mca_btl_vader_knem_reg (void *reg_data, void *base, size_t size,
mca_mpool_base_registration_t *reg)
{
mca_btl_vader_registration_handle_t *handle = NULL;
mca_btl_vader_reg_t *knem_reg = (mca_btl_vader_reg_t *) reg;
struct knem_cmd_create_region knem_cr;
struct knem_cmd_param_iovec knem_iov;
/* NTH: TODO -- Replace this with just using an mpool once we can pass the
* protection flags through. */
OMPI_FREE_LIST_GET_MT(&mca_btl_vader.registration_handles, &handle);
if (OPAL_UNLIKELY(NULL == handle)) {
return NULL;
}
knem_iov.base = (uintptr_t) base;
knem_iov.len = size;
knem_cr.iovec_array = (uintptr_t) &knem_iov;
knem_cr.iovec_nr = 1;
knem_cr.protection = 0;
if (flags & MCA_BTL_REG_FLAG_REMOTE_READ) {
knem_cr.protection |= PROT_READ;
}
if (flags & MCA_BTL_REG_FLAG_REMOTE_WRITE) {
knem_cr.protection |= PROT_WRITE;
}
/* TODO -- set proper access flags when the protection is passed down */
knem_cr.protection = PROT_READ | PROT_WRITE;
/* Vader will explicitly destroy this cookie */
knem_cr.flags = 0;
if (OPAL_UNLIKELY(ioctl(mca_btl_vader.knem_fd, KNEM_CMD_CREATE_REGION, &knem_cr) < 0)) {
OMPI_FREE_LIST_RETURN_MT(&mca_btl_vader.registration_handles, handle);
return OPAL_ERROR;
}
knem_reg->btl_handle.cookie = knem_cr.cookie;
knem_reg->btl_handle.base_addr = (intptr_t) base;
return OPAL_SUCCESS;
}
static int mca_btl_vader_knem_dereg (void *reg_data, mca_mpool_base_registration_t *reg)
{
mca_btl_vader_reg_t *knem_reg = (mca_btl_vader_reg_t *) reg;
/* NTH: explicity ignore the return code. Don't care about this cookie anymore anyway. */
(void) ioctl(mca_btl_vader.knem_fd, KNEM_CMD_DESTROY_REGION, &knem_reg->btl_handle.cookie);
return OPAL_SUCCESS;
}
static mca_btl_base_registration_handle_t *
mca_btl_vader_register_mem_knem (struct mca_btl_base_module_t* btl,
struct mca_btl_base_endpoint_t *endpoint,
void *base, size_t size, uint32_t flags)
{
mca_btl_vader_reg_t *reg = NULL;
int rc;
rc = btl->btl_mpool->mpool_register (btl->btl_mpool, base, size, 0,
(mca_mpool_base_registration_t **) &reg);
if (OPAL_UNLIKELY(OPAL_SUCCESS != rc)) {
return NULL;
}
handle->btl_handle.cookie = knem_cr.cookie;
handle->btl_handle.base_addr = (intptr_t) base;
return &handle->btl_handle;
return &reg->btl_handle;
}
static int
mca_btl_vader_deregister_mem_knem (struct mca_btl_base_module_t* btl, struct mca_btl_base_registration_handle_t *handle)
{
mca_btl_vader_registration_handle_t *vader_handle =
(mca_btl_vader_registration_handle_t *)((intptr_t) handle - offsetof (mca_btl_vader_registration_handle_t, btl_handle));
mca_btl_vader_reg_t *reg =
(mca_btl_vader_reg_t *)((intptr_t) handle - offsetof (mca_btl_vader_reg_t, btl_handle));
/* NTH: explicity ignore the return code. Don't care about this cookie anymore anyway. */
(void) ioctl(mca_btl_vader.knem_fd, KNEM_CMD_DESTROY_REGION, &vader_handle->cookie);
btl->btl_mpool->mpool_deregister (btl->btl_mpool, &reg->base);
return OPAL_SUCCESS;
}
int mca_btl_vader_knem_init (void)
{
mca_mpool_base_resources_t mpool_resources = {
.pool_name = "vader", .reg_data = NULL,
.sizeof_reg = sizeof (mca_btl_vader_reg_t),
.register_mem = mca_btl_vader_knem_reg,
.deregister_mem = mca_btl_vader_knem_dereg
};
struct knem_cmd_info knem_info;
int rc;
@ -140,10 +155,16 @@ int mca_btl_vader_knem_init (void)
mca_btl_vader.super.btl_put = mca_btl_vader_put_knem;
/* knem requires registration */
mca_btl_vader.super.btl_register_mem = mca_btl_vader_vader_register_mem_kem;
mca_btl_vader.super.btl_deregister_mem = mca_btl_vader_vader_deregister_mem_kem;
mca_btl_vader.super.btl_register_mem = mca_btl_vader_register_mem_knem;
mca_btl_vader.super.btl_deregister_mem = mca_btl_vader_deregister_mem_knem;
mca_btl_vader.super.btl_registration_handle_size = sizeof (mca_btl_base_registration_handle_t);
mca_btl_vader.super.btl_mpool = mca_mpool_base_module_create ("grdma", NULL,
&mpool_resources);
if (NULL == mca_btl_vader.super.btl_mpool) {
return OPAL_ERR_OUT_OF_RESOURCE;
}
return OPAL_SUCCESS;
} while (0);

6
opal/mca/btl/vader/btl_vader_knem.h
Просмотреть файл

@ -23,6 +23,12 @@ struct mca_btl_base_registration_handle_t {
intptr_t base_addr;
};
struct mca_btl_vader_reg_t {
mca_mpool_base_registration_t base;
mca_btl_base_registration_handle_t btl_handle;
};
typedef struct mca_btl_vader_reg_t mca_btl_vader_reg_t;
int mca_btl_vader_knem_init (void);
int mca_btl_vader_knem_fini (void);
int mca_btl_vader_knem_progress (void);

15
opal/mca/btl/vader/btl_vader_module.c
Просмотреть файл

@ -96,21 +96,6 @@ static int vader_btl_first_time_init(mca_btl_vader_t *vader_btl, int n)
component->segment_offset = MCA_BTL_VADER_FIFO_SIZE;
/* initialize fragment descriptor free lists */
/* initialize free list for single copy (get, put) */
#if OPAL_BTL_VADER_HAVE_KNEM
if (MCA_BTL_VADER_KNEM != mca_btl_vader_component.single_copy_mechanism) {
rc = ompi_free_list_init_new (&component->registration_handles,
sizeof(mca_btl_vader_registration_handle_t), 8,
OBJ_CLASS(mca_btl_vader_registration_handle_t),
0, 8, component->vader_free_list_num,
component->vader_free_list_max,
component->vader_free_list_inc, NULL);
if (OPAL_SUCCESS != rc) {
return rc;
}
}
#endif
/* initialize free list for small send and inline fragments */
rc = ompi_free_list_init_ex_new(&component->vader_frags_user,
sizeof(mca_btl_vader_frag_t),