spml ikrit: update configuration parsing api to mxm 2.1, drop

older version support (1.0 and 1.1), and cleanup the code.

reviewed by miked.

This commit was SVN r29798.

oshmem/mca/spml/ikrit/spml_ikrit.c

@@ -558,9 +558,7 @@ mca_spml_mkey_t *mca_spml_ikrit_register(void* addr,
         case MXM_PTL_RDMA:
             mkeys[i].va_base = addr;
             mkeys[i].spml_context = 0;
-#if MXM_API < MXM_VERSION(1,5)
+#if MXM_API < MXM_VERSION(2,0)
-            mkeys[i].handle.ib.lkey = mkeys[i].handle.ib.rkey = MXM_MKEY_NONE;
-#elif MXM_API < MXM_VERSION(2,0)
             mkeys[i].handle.ib.lkey = mkeys[i].handle.ib.rkey = 0;
 #else
             mkeys[i].handle.ib.lkey = mkeys[i].handle.ib.rkey = 0;
@@ -608,12 +606,7 @@ int mca_spml_ikrit_deregister(mca_spml_mkey_t *mkeys)
             /* dereg memory */
             if (!mkeys[i].spml_context)
                 break;
-#if MXM_API < MXM_VERSION(1,5)
+#if MXM_API >= MXM_VERSION(2,0)
-            mxm_dereg_mr(mca_spml_ikrit.mxm_ep,
-                         MXM_PTL_RDMA,
-                         (void *) mkeys[i].va_base,
-                         (unsigned long) mkeys[i].spml_context);
-#elif MXM_API >= MXM_VERSION(2,0)
             mxm_mem_unmap(mca_spml_ikrit.mxm_context,
                     (void *)mkeys[i].va_base,
                     (unsigned long)mkeys[i].spml_context,
@@ -654,10 +647,6 @@ int mca_spml_ikrit_oob_get_mkeys(int pe, uint32_t seg, mca_spml_mkey_t *mkeys)
     if (seg > 1)
         return OSHMEM_ERROR;
 
-#if MXM_API < MXM_VERSION(1,5)
-    mkeys[ptl].ib.rkey = MXM_MKEY_NONE;
-#endif
-
     return OSHMEM_SUCCESS;
 }
 
@@ -702,10 +691,7 @@ static int mca_spml_ikrit_get_helper(mxm_send_req_t *sreq,
     sreq->base.data_type = MXM_REQ_DATA_BUFFER;
     sreq->base.data.buffer.ptr = dst_addr;
     sreq->base.data.buffer.length = size;
-#if MXM_API < MXM_VERSION(1,5)
+#if MXM_API < MXM_VERSION(2,0)
-    sreq->base.data.buffer.mkey = MXM_MKEY_NONE;
-    sreq->op.mem.remote_mkey = r_mkey->handle.ib.rkey;
-#elif MXM_API < MXM_VERSION(2,0)
     sreq->base.data.buffer.memh = NULL;
     sreq->op.mem.remote_memh = NULL;
 #else
@@ -1029,7 +1015,7 @@ static inline int mca_spml_ikrit_put_internal(void* dst_addr,
     put_req->mxm_req.base.mq = mca_spml_ikrit.mxm_mq;
     /* request immediate responce if we are getting low on send buffers. We only get responce from remote on ack timeout.
      * Also request explicit ack once in a while  */
-#if MXM_API < MXM_VERSION(2, 0)
+#if MXM_API < MXM_VERSION(2,0)
     put_req->mxm_req.opcode = MXM_REQ_OP_PUT;
     if (mca_spml_ikrit.free_list_max - mca_spml_ikrit.n_active_puts <= SPML_IKRIT_PUT_LOW_WATER ||
             (mca_spml_ikrit.mxm_peers[dst]->n_active_puts + 1) % SPML_IKRIT_PACKETS_PER_SYNC == 0) {
@@ -1063,10 +1049,7 @@ static inline int mca_spml_ikrit_put_internal(void* dst_addr,
     put_req->mxm_req.base.state = MXM_REQ_NEW;
     put_req->pe = dst;
 
-#if MXM_API < MXM_VERSION(1,5)
+#if MXM_API < MXM_VERSION(2,0)
-    put_req->mxm_req.base.data.buffer.mkey = MXM_MKEY_NONE;
-    put_req->mxm_req.op.mem.remote_mkey = r_mkey->handle.ib.rkey;
-#elif MXM_API < MXM_VERSION(2, 0)
     put_req->mxm_req.base.data.buffer.memh = NULL;
     put_req->mxm_req.op.mem.remote_memh = NULL;
 #else
@@ -1095,13 +1078,8 @@ static inline int mca_spml_ikrit_put_internal(void* dst_addr,
         put_req->iov[1].ptr = src_addr;
         put_req->iov[1].length = size;
 
-#if MXM_API < MXM_VERSION(1,5)
-        put_req->iov[0].mkey = MXM_MKEY_NONE;
-        put_req->iov[1].mkey = MXM_MKEY_NONE;
-#else
         put_req->iov[0].memh = NULL;
         put_req->iov[1].memh = NULL;
-#endif
     }
 
     OPAL_THREAD_ADD32(&mca_spml_ikrit.n_active_puts, 1);
@@ -1195,7 +1173,7 @@ int mca_spml_ikrit_put_simple(void* dst_addr,
 
     /* fill out request */
     mxm_req.base.mq = mca_spml_ikrit.mxm_mq;
-#if MXM_API <  MXM_VERSION(2, 0)
+#if MXM_API <  MXM_VERSION(2,0)
     mxm_req.base.flags = MXM_REQ_FLAG_BLOCKING;
 #else
     mxm_req.flags = MXM_REQ_SEND_FLAG_BLOCKING;
@@ -1211,10 +1189,7 @@ int mca_spml_ikrit_put_simple(void* dst_addr,
     mxm_req.base.state = MXM_REQ_NEW;
     mxm_req.base.error = MXM_OK;
 
-#if MXM_API < MXM_VERSION(1,5)
+#if MXM_API < MXM_VERSION(2, 0)
-    mxm_req.base.data.buffer.mkey = MXM_MKEY_NONE;
-    mxm_req.op.mem.remote_mkey = MXM_MKEY_NONE;
-#elif MXM_API < MXM_VERSION(2, 0)
     mxm_req.base.data.buffer.memh = NULL;
     mxm_req.op.mem.remote_memh = NULL;
 #else
@@ -1445,7 +1420,7 @@ int mca_spml_ikrit_recv(void* buf, size_t size, int src)
     req.base.state = MXM_REQ_NEW;
     req.base.mq = mca_spml_ikrit.mxm_mq;
     req.base.conn = NULL;
-#if MXM_API < MXM_VERSION(2, 0)
+#if MXM_API < MXM_VERSION(2,0)
     req.base.flags           = MXM_REQ_FLAG_BLOCKING;
 #endif
     req.base.completed_cb = NULL;
@@ -1453,11 +1428,7 @@ int mca_spml_ikrit_recv(void* buf, size_t size, int src)
     req.base.data_type = MXM_REQ_DATA_BUFFER;
     req.base.data.buffer.ptr = buf == NULL ? dummy_buf : buf;
     req.base.data.buffer.length = size == 0 ? sizeof(dummy_buf) : size;
-#if MXM_API < MXM_VERSION(1,5)
-    req.base.data.buffer.mkey = MXM_MKEY_NONE;
-#else
     req.base.data.buffer.memh = NULL;
-#endif
 
     mxm_req_recv(&req);
     mca_spml_irkit_req_wait(&req.base);
@@ -1490,7 +1461,7 @@ int mca_spml_ikrit_send(void* buf,
     req.base.state = MXM_REQ_NEW;
     req.base.mq = mca_spml_ikrit.mxm_mq;
     req.base.conn = mca_spml_ikrit.mxm_peers[dst]->mxm_conn;
-#if MXM_API < MXM_VERSION(2, 0)
+#if MXM_API < MXM_VERSION(2,0)
     req.base.flags           = MXM_REQ_FLAG_BLOCKING;
 #else
     req.flags                = MXM_REQ_SEND_FLAG_BLOCKING;
@@ -1500,11 +1471,7 @@ int mca_spml_ikrit_send(void* buf,
     req.base.data_type = MXM_REQ_DATA_BUFFER;
     req.base.data.buffer.ptr = buf == NULL ? dummy_buf : buf;
     req.base.data.buffer.length = size == 0 ? sizeof(dummy_buf) : size;
-#if MXM_API < MXM_VERSION(1,5)
-    req.base.data.buffer.mkey = MXM_MKEY_NONE;
-#else
     req.base.data.buffer.memh = NULL;
-#endif
 
     mxm_req_send(&req);
     mca_spml_irkit_req_wait(&req.base);

oshmem/mca/spml/ikrit/spml_ikrit.h

@@ -72,6 +72,8 @@ OBJ_CLASS_DECLARATION(mxm_peer_t);
 struct mca_spml_ikrit_t {
     mca_spml_base_module_t super;
 
+    mxm_context_opts_t *mxm_ctx_opts;
+    mxm_ep_opts_t *mxm_ep_opts;
     mxm_h mxm_context;
     mxm_ep_h mxm_ep;
     mxm_mq_h mxm_mq;

oshmem/mca/spml/ikrit/spml_ikrit_component.c

@@ -98,7 +98,7 @@ static int mca_spml_ikrit_component_register(void)
                                               "[integer] First N ranks on host will receive and forward put messages to other ranks running on it. Can be used to as work around Sandy Bridge far socket problem");
 
     np = mca_spml_ikrit_param_register_int("np",
-#if MXM_API > MXM_VERSION(1,5)
+#if MXM_API <= MXM_VERSION(2,0)
                                            128,
 #else
                                            0,
@@ -150,37 +150,24 @@ static int mca_spml_ikrit_component_open(void)
         return OSHMEM_ERROR;
     }
 
-#if MXM_API < MXM_VERSION(1,5)
+#if MXM_API < MXM_VERSION(2,1)
-    mxm_context_opts_t mxm_opts;
+    if ((MXM_OK != mxm_config_read_context_opts(&mca_spml_ikrit.mxm_ctx_opts)) ||
-
+        (MXM_OK != mxm_config_read_ep_opts(&mca_spml_ikrit.mxm_ep_opts)))
-    mxm_fill_context_opts(&mxm_opts);
-    /* only enable rmda and self ptls */
-    mxm_opts.ptl_bitmap = (MXM_BIT(MXM_PTL_SELF) | MXM_BIT(MXM_PTL_RDMA));
-
 #else
-    mxm_context_opts_t *mxm_opts;
+    if (MXM_OK != mxm_config_read_opts(&mca_spml_ikrit.mxm_ctx_opts,
-
+                                       &mca_spml_ikrit.mxm_ep_opts,
-    err = mxm_config_read_context_opts(&mxm_opts);
+                                       "OSHMEM", NULL, 0))
-    if (MXM_OK != err) {
+#endif
+    {
         SPML_ERROR("Failed to parse MXM configuration");
         return OSHMEM_ERROR;
     }
-#if MXM_API < MXM_VERSION(2, 0)
+
-    mxm_opts->ptl_bitmap = (MXM_BIT(MXM_PTL_SELF) | MXM_BIT(MXM_PTL_RDMA));
+#if MXM_API < MXM_VERSION(2,0)
-#endif
+    mca_spml_ikrit.mxm_ctx_opts->ptl_bitmap = (MXM_BIT(MXM_PTL_SELF) | MXM_BIT(MXM_PTL_RDMA));
-#endif
-
-#if MXM_API < MXM_VERSION(1,5)
-    err = mxm_init(&mxm_opts, &mca_spml_ikrit.mxm_context);
-#else
-    err = mxm_init(mxm_opts, &mca_spml_ikrit.mxm_context);
-#if MXM_API < MXM_VERSION(2, 0)
-    mxm_config_free(mxm_opts);
-#else
-    mxm_config_free_context_opts(mxm_opts);
-#endif
 #endif
 
+    err = mxm_init(mca_spml_ikrit.mxm_ctx_opts, &mca_spml_ikrit.mxm_context);
     if (MXM_OK != err) {
         if (MXM_ERR_NO_DEVICE == err) {
             SPML_VERBOSE(1,
@@ -210,8 +197,16 @@ static int mca_spml_ikrit_component_open(void)
 
 static int mca_spml_ikrit_component_close(void)
 {
-    if (mca_spml_ikrit.mxm_context)
+    if (mca_spml_ikrit.mxm_context) {
         mxm_cleanup(mca_spml_ikrit.mxm_context);
+#if MXM_API < MXM_VERSION(2,0)
+        mxm_config_free(mca_spml_ikrit.mxm_ep_opts);
+        mxm_config_free(mca_spml_ikrit.mxm_ctx_opts);
+#else
+        mxm_config_free_ep_opts(mca_spml_ikrit.mxm_ep_opts);
+        mxm_config_free_context_opts(mca_spml_ikrit.mxm_ctx_opts);
+#endif
+    }
     mca_spml_ikrit.mxm_context = NULL;
     return OSHMEM_SUCCESS;
 }
@@ -219,49 +214,18 @@ static int mca_spml_ikrit_component_close(void)
 static int spml_ikrit_mxm_init(void)
 {
     mxm_error_t err;
-    mxm_ep_opts_t *p_ep_opts;
-
-#if MXM_API < MXM_VERSION(1,5)
-    mxm_ep_opts_t ep_opt;
-    struct sockaddr_mxm_local_proc sa_bind_self;
-    struct sockaddr_mxm_ib_local sa_bind_rdma;
-
-    p_ep_opts = &ep_opt;
-    /* Setup the endpoint options and local addresses to bind to. */
-    mxm_fill_ep_opts(&ep_opt);
-
-    sa_bind_self.sa_family = AF_MXM_LOCAL_PROC;
-    sa_bind_self.context_id = 0;
-    sa_bind_self.process_id = oshmem_proc_local()->proc_name.vpid;
-
-    sa_bind_rdma.sa_family = AF_MXM_IB_LOCAL;
-    sa_bind_rdma.lid = 0;
-    sa_bind_rdma.pkey = 0;
-    sa_bind_rdma.qp_num = 0;
-    sa_bind_rdma.sl = 0;
-
-    ep_opt.ptl_bind_addr[MXM_PTL_SELF] = (struct sockaddr*) &sa_bind_self;
-    ep_opt.ptl_bind_addr[MXM_PTL_RDMA] = (struct sockaddr*) &sa_bind_rdma;
-
-#else
-    err = mxm_config_read_ep_opts(&p_ep_opts);
-    if (err != MXM_OK) {
-        SPML_ERROR("Failed to parse MXM configuration");
-        return OSHMEM_ERROR;
-    }
 
 #if MXM_API < MXM_VERSION(2,0)
     /* Only relevant for SHM PTL - ignore */
-    p_ep_opts->job_id = 0;
+    mca_spml_ikrit.mxm_ep_opts->job_id = 0;
-    p_ep_opts->local_rank = 0;
+    mca_spml_ikrit.mxm_ep_opts->local_rank = 0;
-    p_ep_opts->num_local_procs = 0;
+    mca_spml_ikrit.mxm_ep_opts->num_local_procs = 0;
-    p_ep_opts->rdma.drain_cq = 1;
+    mca_spml_ikrit.mxm_ep_opts->rdma.drain_cq = 1;
-#endif
 #endif
 
     /* Open MXM endpoint */
     err = mxm_ep_create(mca_spml_ikrit.mxm_context,
-                        p_ep_opts,
+                        mca_spml_ikrit.mxm_ep_opts,
                         &mca_spml_ikrit.mxm_ep);
     if (MXM_OK != err) {
         orte_show_help("help-shmem-spml-ikrit.txt",
@@ -271,14 +235,6 @@ static int spml_ikrit_mxm_init(void)
         return OSHMEM_ERROR;
     }
 
-#if MXM_API >= MXM_VERSION(1,5)
-#if MXM_API < MXM_VERSION(2,0)
-    mxm_config_free(p_ep_opts);
-#else
-    mxm_config_free_ep_opts(p_ep_opts);
-#endif
-#endif
-
     return OSHMEM_SUCCESS;
 }