ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

spml ikrit: update configuration parsing api to mxm 2.1, drop

Просмотр исходного кода 
older version support (1.0 and 1.1), and cleanup the code.

reviewed by miked.

This commit was SVN r29798.
Этот коммит содержится в:
Yossi Etigin 2013-12-04 09:13:17 +00:00
родитель a913b00f89
Коммит 97b0a560cb
3 изменённых файлов: 38 добавлений и 113 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

51
oshmem/mca/spml/ikrit/spml_ikrit.c
Просмотреть файл

@ -558,9 +558,7 @@ mca_spml_mkey_t *mca_spml_ikrit_register(void* addr,
case MXM_PTL_RDMA:
mkeys[i].va_base = addr;
mkeys[i].spml_context = 0;
#if MXM_API < MXM_VERSION(1,5)
mkeys[i].handle.ib.lkey = mkeys[i].handle.ib.rkey = MXM_MKEY_NONE;
#elif MXM_API < MXM_VERSION(2,0)
#if MXM_API < MXM_VERSION(2,0)
mkeys[i].handle.ib.lkey = mkeys[i].handle.ib.rkey = 0;
#else
mkeys[i].handle.ib.lkey = mkeys[i].handle.ib.rkey = 0;
@ -608,12 +606,7 @@ int mca_spml_ikrit_deregister(mca_spml_mkey_t *mkeys)
/* dereg memory */
if (!mkeys[i].spml_context)
break;
#if MXM_API < MXM_VERSION(1,5)
mxm_dereg_mr(mca_spml_ikrit.mxm_ep,
MXM_PTL_RDMA,
(void *) mkeys[i].va_base,
(unsigned long) mkeys[i].spml_context);
#elif MXM_API >= MXM_VERSION(2,0)
#if MXM_API >= MXM_VERSION(2,0)
mxm_mem_unmap(mca_spml_ikrit.mxm_context,
(void *)mkeys[i].va_base,
(unsigned long)mkeys[i].spml_context,
@ -654,10 +647,6 @@ int mca_spml_ikrit_oob_get_mkeys(int pe, uint32_t seg, mca_spml_mkey_t *mkeys)
if (seg > 1)
return OSHMEM_ERROR;
#if MXM_API < MXM_VERSION(1,5)
mkeys[ptl].ib.rkey = MXM_MKEY_NONE;
#endif
return OSHMEM_SUCCESS;
}
@ -702,10 +691,7 @@ static int mca_spml_ikrit_get_helper(mxm_send_req_t *sreq,
sreq->base.data_type = MXM_REQ_DATA_BUFFER;
sreq->base.data.buffer.ptr = dst_addr;
sreq->base.data.buffer.length = size;
#if MXM_API < MXM_VERSION(1,5)
sreq->base.data.buffer.mkey = MXM_MKEY_NONE;
sreq->op.mem.remote_mkey = r_mkey->handle.ib.rkey;
#elif MXM_API < MXM_VERSION(2,0)
#if MXM_API < MXM_VERSION(2,0)
sreq->base.data.buffer.memh = NULL;
sreq->op.mem.remote_memh = NULL;
#else
@ -1029,7 +1015,7 @@ static inline int mca_spml_ikrit_put_internal(void* dst_addr,
put_req->mxm_req.base.mq = mca_spml_ikrit.mxm_mq;
/* request immediate responce if we are getting low on send buffers. We only get responce from remote on ack timeout.
* Also request explicit ack once in a while */
#if MXM_API < MXM_VERSION(2, 0)
#if MXM_API < MXM_VERSION(2,0)
put_req->mxm_req.opcode = MXM_REQ_OP_PUT;
if (mca_spml_ikrit.free_list_max - mca_spml_ikrit.n_active_puts <= SPML_IKRIT_PUT_LOW_WATER ||
(mca_spml_ikrit.mxm_peers[dst]->n_active_puts + 1) % SPML_IKRIT_PACKETS_PER_SYNC == 0) {
@ -1063,10 +1049,7 @@ static inline int mca_spml_ikrit_put_internal(void* dst_addr,
put_req->mxm_req.base.state = MXM_REQ_NEW;
put_req->pe = dst;
#if MXM_API < MXM_VERSION(1,5)
put_req->mxm_req.base.data.buffer.mkey = MXM_MKEY_NONE;
put_req->mxm_req.op.mem.remote_mkey = r_mkey->handle.ib.rkey;
#elif MXM_API < MXM_VERSION(2, 0)
#if MXM_API < MXM_VERSION(2,0)
put_req->mxm_req.base.data.buffer.memh = NULL;
put_req->mxm_req.op.mem.remote_memh = NULL;
#else
@ -1095,13 +1078,8 @@ static inline int mca_spml_ikrit_put_internal(void* dst_addr,
put_req->iov[1].ptr = src_addr;
put_req->iov[1].length = size;
#if MXM_API < MXM_VERSION(1,5)
put_req->iov[0].mkey = MXM_MKEY_NONE;
put_req->iov[1].mkey = MXM_MKEY_NONE;
#else
put_req->iov[0].memh = NULL;
put_req->iov[1].memh = NULL;
#endif
}
OPAL_THREAD_ADD32(&mca_spml_ikrit.n_active_puts, 1);
@ -1195,7 +1173,7 @@ int mca_spml_ikrit_put_simple(void* dst_addr,
/* fill out request */
mxm_req.base.mq = mca_spml_ikrit.mxm_mq;
#if MXM_API < MXM_VERSION(2, 0)
#if MXM_API < MXM_VERSION(2,0)
mxm_req.base.flags = MXM_REQ_FLAG_BLOCKING;
#else
mxm_req.flags = MXM_REQ_SEND_FLAG_BLOCKING;
@ -1211,10 +1189,7 @@ int mca_spml_ikrit_put_simple(void* dst_addr,
mxm_req.base.state = MXM_REQ_NEW;
mxm_req.base.error = MXM_OK;
#if MXM_API < MXM_VERSION(1,5)
mxm_req.base.data.buffer.mkey = MXM_MKEY_NONE;
mxm_req.op.mem.remote_mkey = MXM_MKEY_NONE;
#elif MXM_API < MXM_VERSION(2, 0)
#if MXM_API < MXM_VERSION(2, 0)
mxm_req.base.data.buffer.memh = NULL;
mxm_req.op.mem.remote_memh = NULL;
#else
@ -1445,7 +1420,7 @@ int mca_spml_ikrit_recv(void* buf, size_t size, int src)
req.base.state = MXM_REQ_NEW;
req.base.mq = mca_spml_ikrit.mxm_mq;
req.base.conn = NULL;
#if MXM_API < MXM_VERSION(2, 0)
#if MXM_API < MXM_VERSION(2,0)
req.base.flags = MXM_REQ_FLAG_BLOCKING;
#endif
req.base.completed_cb = NULL;
@ -1453,11 +1428,7 @@ int mca_spml_ikrit_recv(void* buf, size_t size, int src)
req.base.data_type = MXM_REQ_DATA_BUFFER;
req.base.data.buffer.ptr = buf == NULL ? dummy_buf : buf;
req.base.data.buffer.length = size == 0 ? sizeof(dummy_buf) : size;
#if MXM_API < MXM_VERSION(1,5)
req.base.data.buffer.mkey = MXM_MKEY_NONE;
#else
req.base.data.buffer.memh = NULL;
#endif
mxm_req_recv(&req);
mca_spml_irkit_req_wait(&req.base);
@ -1490,7 +1461,7 @@ int mca_spml_ikrit_send(void* buf,
req.base.state = MXM_REQ_NEW;
req.base.mq = mca_spml_ikrit.mxm_mq;
req.base.conn = mca_spml_ikrit.mxm_peers[dst]->mxm_conn;
#if MXM_API < MXM_VERSION(2, 0)
#if MXM_API < MXM_VERSION(2,0)
req.base.flags = MXM_REQ_FLAG_BLOCKING;
#else
req.flags = MXM_REQ_SEND_FLAG_BLOCKING;
@ -1500,11 +1471,7 @@ int mca_spml_ikrit_send(void* buf,
req.base.data_type = MXM_REQ_DATA_BUFFER;
req.base.data.buffer.ptr = buf == NULL ? dummy_buf : buf;
req.base.data.buffer.length = size == 0 ? sizeof(dummy_buf) : size;
#if MXM_API < MXM_VERSION(1,5)
req.base.data.buffer.mkey = MXM_MKEY_NONE;
#else
req.base.data.buffer.memh = NULL;
#endif
mxm_req_send(&req);
mca_spml_irkit_req_wait(&req.base);

2
oshmem/mca/spml/ikrit/spml_ikrit.h
Просмотреть файл

@ -72,6 +72,8 @@ OBJ_CLASS_DECLARATION(mxm_peer_t);
struct mca_spml_ikrit_t {
mca_spml_base_module_t super;
mxm_context_opts_t *mxm_ctx_opts;
mxm_ep_opts_t *mxm_ep_opts;
mxm_h mxm_context;
mxm_ep_h mxm_ep;
mxm_mq_h mxm_mq;

98
oshmem/mca/spml/ikrit/spml_ikrit_component.c
Просмотреть файл

@ -98,7 +98,7 @@ static int mca_spml_ikrit_component_register(void)
"[integer] First N ranks on host will receive and forward put messages to other ranks running on it. Can be used to as work around Sandy Bridge far socket problem");
np = mca_spml_ikrit_param_register_int("np",
#if MXM_API > MXM_VERSION(1,5)
#if MXM_API <= MXM_VERSION(2,0)
128,
#else
0,
@ -150,37 +150,24 @@ static int mca_spml_ikrit_component_open(void)
return OSHMEM_ERROR;
}
#if MXM_API < MXM_VERSION(1,5)
mxm_context_opts_t mxm_opts;
mxm_fill_context_opts(&mxm_opts);
/* only enable rmda and self ptls */
mxm_opts.ptl_bitmap = (MXM_BIT(MXM_PTL_SELF) | MXM_BIT(MXM_PTL_RDMA));
#if MXM_API < MXM_VERSION(2,1)
if ((MXM_OK != mxm_config_read_context_opts(&mca_spml_ikrit.mxm_ctx_opts)) ||
(MXM_OK != mxm_config_read_ep_opts(&mca_spml_ikrit.mxm_ep_opts)))
#else
mxm_context_opts_t *mxm_opts;
err = mxm_config_read_context_opts(&mxm_opts);
if (MXM_OK != err) {
if (MXM_OK != mxm_config_read_opts(&mca_spml_ikrit.mxm_ctx_opts,
&mca_spml_ikrit.mxm_ep_opts,
"OSHMEM", NULL, 0))
#endif
{
SPML_ERROR("Failed to parse MXM configuration");
return OSHMEM_ERROR;
}
#if MXM_API < MXM_VERSION(2, 0)
mxm_opts->ptl_bitmap = (MXM_BIT(MXM_PTL_SELF) | MXM_BIT(MXM_PTL_RDMA));
#endif
#endif
#if MXM_API < MXM_VERSION(1,5)
err = mxm_init(&mxm_opts, &mca_spml_ikrit.mxm_context);
#else
err = mxm_init(mxm_opts, &mca_spml_ikrit.mxm_context);
#if MXM_API < MXM_VERSION(2, 0)
mxm_config_free(mxm_opts);
#else
mxm_config_free_context_opts(mxm_opts);
#endif
#if MXM_API < MXM_VERSION(2,0)
mca_spml_ikrit.mxm_ctx_opts->ptl_bitmap = (MXM_BIT(MXM_PTL_SELF) | MXM_BIT(MXM_PTL_RDMA));
#endif
err = mxm_init(mca_spml_ikrit.mxm_ctx_opts, &mca_spml_ikrit.mxm_context);
if (MXM_OK != err) {
if (MXM_ERR_NO_DEVICE == err) {
SPML_VERBOSE(1,
@ -210,8 +197,16 @@ static int mca_spml_ikrit_component_open(void)
static int mca_spml_ikrit_component_close(void)
{
if (mca_spml_ikrit.mxm_context)
if (mca_spml_ikrit.mxm_context) {
mxm_cleanup(mca_spml_ikrit.mxm_context);
#if MXM_API < MXM_VERSION(2,0)
mxm_config_free(mca_spml_ikrit.mxm_ep_opts);
mxm_config_free(mca_spml_ikrit.mxm_ctx_opts);
#else
mxm_config_free_ep_opts(mca_spml_ikrit.mxm_ep_opts);
mxm_config_free_context_opts(mca_spml_ikrit.mxm_ctx_opts);
#endif
}
mca_spml_ikrit.mxm_context = NULL;
return OSHMEM_SUCCESS;
}
@ -219,49 +214,18 @@ static int mca_spml_ikrit_component_close(void)
static int spml_ikrit_mxm_init(void)
{
mxm_error_t err;
mxm_ep_opts_t *p_ep_opts;
#if MXM_API < MXM_VERSION(1,5)
mxm_ep_opts_t ep_opt;
struct sockaddr_mxm_local_proc sa_bind_self;
struct sockaddr_mxm_ib_local sa_bind_rdma;
p_ep_opts = &ep_opt;
/* Setup the endpoint options and local addresses to bind to. */
mxm_fill_ep_opts(&ep_opt);
sa_bind_self.sa_family = AF_MXM_LOCAL_PROC;
sa_bind_self.context_id = 0;
sa_bind_self.process_id = oshmem_proc_local()->proc_name.vpid;
sa_bind_rdma.sa_family = AF_MXM_IB_LOCAL;
sa_bind_rdma.lid = 0;
sa_bind_rdma.pkey = 0;
sa_bind_rdma.qp_num = 0;
sa_bind_rdma.sl = 0;
ep_opt.ptl_bind_addr[MXM_PTL_SELF] = (struct sockaddr*) &sa_bind_self;
ep_opt.ptl_bind_addr[MXM_PTL_RDMA] = (struct sockaddr*) &sa_bind_rdma;
#else
err = mxm_config_read_ep_opts(&p_ep_opts);
if (err != MXM_OK) {
SPML_ERROR("Failed to parse MXM configuration");
return OSHMEM_ERROR;
}
#if MXM_API < MXM_VERSION(2,0)
/* Only relevant for SHM PTL - ignore */
p_ep_opts->job_id = 0;
p_ep_opts->local_rank = 0;
p_ep_opts->num_local_procs = 0;
p_ep_opts->rdma.drain_cq = 1;
#endif
mca_spml_ikrit.mxm_ep_opts->job_id = 0;
mca_spml_ikrit.mxm_ep_opts->local_rank = 0;
mca_spml_ikrit.mxm_ep_opts->num_local_procs = 0;
mca_spml_ikrit.mxm_ep_opts->rdma.drain_cq = 1;
#endif
/* Open MXM endpoint */
err = mxm_ep_create(mca_spml_ikrit.mxm_context,
p_ep_opts,
mca_spml_ikrit.mxm_ep_opts,
&mca_spml_ikrit.mxm_ep);
if (MXM_OK != err) {
orte_show_help("help-shmem-spml-ikrit.txt",
@ -271,14 +235,6 @@ static int spml_ikrit_mxm_init(void)
return OSHMEM_ERROR;
}
#if MXM_API >= MXM_VERSION(1,5)
#if MXM_API < MXM_VERSION(2,0)
mxm_config_free(p_ep_opts);
#else
mxm_config_free_ep_opts(p_ep_opts);
#endif
#endif
return OSHMEM_SUCCESS;
}