diff --git a/ompi/mca/btl/wv/btl_wv.c b/ompi/mca/btl/wv/btl_wv.c index 1eec7ebd4f..af9a43dcd7 100644 --- a/ompi/mca/btl/wv/btl_wv.c +++ b/ompi/mca/btl/wv/btl_wv.c @@ -80,6 +80,7 @@ mca_btl_wv_module_t mca_btl_wv_module = { 0, /* latency */ 0, /* bandwidth */ 0, /* TODO this should be PUT btl flags */ + 0, /* segment size */ mca_btl_wv_add_procs, mca_btl_wv_del_procs, NULL, @@ -629,7 +630,7 @@ ib_frag_alloc(mca_btl_wv_module_t *btl, size_t size, uint8_t order, return NULL; /* not all upper layer users set this */ - to_base_frag(item)->segment.seg_len = size; + to_base_frag(item)->segment.base.seg_len = size; to_base_frag(item)->base.order = order; to_base_frag(item)->base.des_flags = flags; @@ -661,7 +662,7 @@ static mca_btl_wv_send_frag_t *check_coalescing(opal_list_t *frag_list, } total_length = size + frag->coalesced_length + - to_base_frag(frag)->segment.seg_len + + to_base_frag(frag)->segment.base.seg_len + sizeof(mca_btl_wv_header_coalesced_t); qp = to_base_frag(frag)->base.order; @@ -737,8 +738,8 @@ mca_btl_base_descriptor_t* mca_btl_wv_alloc(struct mca_btl_base_module_t* btl, sfrag->hdr->tag = MCA_BTL_TAG_BTL; ctrl_hdr->type = MCA_BTL_WV_CONTROL_COALESCED; clsc_hdr->tag = org_tag; - clsc_hdr->size = to_base_frag(sfrag)->segment.seg_len; - clsc_hdr->alloc_size = to_base_frag(sfrag)->segment.seg_len; + clsc_hdr->size = to_base_frag(sfrag)->segment.base.seg_len; + clsc_hdr->alloc_size = to_base_frag(sfrag)->segment.base.seg_len; if(ep->nbo) BTL_WV_HEADER_COALESCED_HTON(*clsc_hdr); sfrag->coalesced_length = sizeof(mca_btl_wv_control_header_t) + @@ -748,12 +749,12 @@ mca_btl_base_descriptor_t* mca_btl_wv_alloc(struct mca_btl_base_module_t* btl, cfrag->hdr = (mca_btl_wv_header_coalesced_t*) (((unsigned char*)(sfrag->hdr + 1)) + sfrag->coalesced_length + - to_base_frag(sfrag)->segment.seg_len); + to_base_frag(sfrag)->segment.base.seg_len); cfrag->hdr->alloc_size = size; /* point coalesced frag pointer into a data buffer */ - to_base_frag(cfrag)->segment.seg_addr.pval = cfrag->hdr + 1; - to_base_frag(cfrag)->segment.seg_len = size; + to_base_frag(cfrag)->segment.base.seg_addr.pval = cfrag->hdr + 1; + to_base_frag(cfrag)->segment.base.seg_len = size; /* save coalesced fragment on a main fragment; we will need it after send * completion to free it and to call upper layer callback */ @@ -802,6 +803,8 @@ int mca_btl_wv_free(struct mca_btl_base_module_t* btl, to_com_frag(des)->sg_entry.pAddress = (void*)(uintptr_t)to_send_frag(des)->hdr; to_send_frag(des)->coalesced_length = 0; + to_base_frag(des)->segment.base.seg_addr.pval = + to_send_frag(des)->hdr + 1; assert(!opal_list_get_size(&to_send_frag(des)->coalesced_frags)); /* fall throug */ case MCA_BTL_WV_FRAG_SEND_USER: @@ -854,6 +857,7 @@ mca_btl_base_descriptor_t* mca_btl_wv_prepare_src(struct mca_btl_base_module_t* struct iovec iov; uint32_t iov_count = 1; size_t max_data = *size; + void *ptr; int rc; wv_btl = (mca_btl_wv_module_t*)btl; @@ -890,10 +894,9 @@ mca_btl_base_descriptor_t* mca_btl_wv_prepare_src(struct mca_btl_base_module_t* frag->sg_entry.pAddress = (void*)(uintptr_t)iov.iov_base; to_base_frag(frag)->base.order = order; to_base_frag(frag)->base.des_flags = flags; - to_base_frag(frag)->segment.seg_len = max_data; - to_base_frag(frag)->segment.seg_addr.pval = iov.iov_base; - to_base_frag(frag)->segment.seg_key.key32[0] = - (uint32_t)frag->sg_entry.Lkey; + to_base_frag(frag)->segment.base.seg_len = max_data; + to_base_frag(frag)->segment.base.seg_addr.lval = (uint64_t)(uintptr_t) iov.iov_base; + to_base_frag(frag)->segment.key = (uint32_t)frag->sg_entry.Lkey; assert(MCA_BTL_NO_ORDER == order); BTL_VERBOSE(("frag->sg_entry.lkey = %" PRIu32 " .addr = %" PRIx64 " frag->segment.seg_key.key32[0] = %" PRIu32, @@ -910,21 +913,32 @@ mca_btl_base_descriptor_t* mca_btl_wv_prepare_src(struct mca_btl_base_module_t* max_data = btl->btl_max_send_size - reserve; } - frag = (mca_btl_wv_com_frag_t*)(reserve ? - mca_btl_wv_alloc(btl, endpoint, order, max_data + reserve, - flags) : - ib_frag_alloc(wv_btl, max_data, order, flags)); - - if(NULL == frag) - return NULL; + if (OPAL_UNLIKELY(0 == reserve)) { + frag = (mca_btl_wv_com_frag_t *) ib_frag_alloc(wv_btl, max_data, order, flags); + if(NULL == frag) + return NULL; + + /* NTH: this frag will be ue used for either a get or put so we need to set the lval to be + consistent with the usage in get and put. the pval will be restored in mca_btl_openib_free */ + ptr = to_base_frag(frag)->segment.base.seg_addr.pval; + to_base_frag(frag)->segment.base.seg_addr.lval = + (uint64_t)(uintptr_t) ptr; + } else { + frag = + (mca_btl_wv_com_frag_t *) mca_btl_wv_alloc(btl, endpoint, order, + max_data + reserve, flags); + if(NULL == frag) + return NULL; + + ptr = to_base_frag(frag)->segment.base.seg_addr.pval; + } iov.iov_len = max_data; - iov.iov_base = (IOVBASE_TYPE *) ( (unsigned char*)to_base_frag(frag)->segment.seg_addr.pval + - reserve ); + iov.iov_base = (IOVBASE_TYPE *) ( (unsigned char*) ptr + reserve ); rc = opal_convertor_pack(convertor, &iov, &iov_count, &max_data); *size = max_data; /* not all upper layer users set this */ - to_base_frag(frag)->segment.seg_len = max_data + reserve; + to_base_frag(frag)->segment.base.seg_len = max_data + reserve; return &to_base_frag(frag)->base; } @@ -982,7 +996,7 @@ mca_btl_base_descriptor_t* mca_btl_wv_prepare_dst(struct mca_btl_base_module_t* /* limit the message so to max_msg_size */ if(*size > max_msg_sz) { *size = max_msg_sz; - BTL_VERBOSE("message size limited to %d",*size); + BTL_VERBOSE(("message size limited to %d",*size)); } opal_convertor_get_current_pointer(convertor, &buffer); @@ -1004,9 +1018,9 @@ mca_btl_base_descriptor_t* mca_btl_wv_prepare_dst(struct mca_btl_base_module_t* frag->sg_entry.Length = *size; frag->sg_entry.Lkey = wv_reg->mr->lkey; frag->sg_entry.pAddress = (void*)(uintptr_t)buffer; - to_base_frag(frag)->segment.seg_addr.pval = buffer; - to_base_frag(frag)->segment.seg_len = *size; - to_base_frag(frag)->segment.seg_key.key32[0] = wv_reg->mr->rkey; + to_base_frag(frag)->segment.base.seg_addr.lval = (uint64_t)(uintptr_t) buffer; + to_base_frag(frag)->segment.base.seg_len = *size; + to_base_frag(frag)->segment.key = wv_reg->mr->rkey; to_base_frag(frag)->base.order = order; to_base_frag(frag)->base.des_flags = flags; BTL_VERBOSE(("frag->sg_entry.lkey = %" PRIu32 " .addr = %" PRIx64 " " @@ -1205,14 +1219,14 @@ int mca_btl_wv_sendi(struct mca_btl_base_module_t* btl, } frag = to_base_frag(item); hdr = to_send_frag(item)->hdr; - frag->segment.seg_len = size; + frag->segment.base.seg_len = size; frag->base.order = qp; frag->base.des_flags = flags; hdr->tag = tag; to_com_frag(item)->endpoint = ep; /* put match header */ - memcpy(frag->segment.seg_addr.pval, header, header_size); + memcpy(frag->segment.base.seg_addr.pval, header, header_size); /* Pack data */ if(payload_size) { @@ -1220,7 +1234,7 @@ int mca_btl_wv_sendi(struct mca_btl_base_module_t* btl, struct iovec iov; uint32_t iov_count; /* pack the data into the supplied buffer */ - iov.iov_base = (IOVBASE_TYPE*)((unsigned char*)frag->segment.seg_addr.pval + header_size); + iov.iov_base = (IOVBASE_TYPE*)((unsigned char*)frag->segment.base.seg_addr.pval + header_size); iov.iov_len = max_data = payload_size; iov_count = 1; @@ -1334,8 +1348,12 @@ int mca_btl_wv_put(mca_btl_base_module_t* btl, HRESULT hr = 0; mca_btl_wv_out_frag_t* frag = to_out_frag(descriptor); int qp = descriptor->order; + + mca_btl_openib_segment_t *src_seg = (mca_btl_openib_segment_t *) descriptor->des_src; + mca_btl_openib_segment_t *dst_seg = (mca_btl_openib_segment_t *) descriptor->des_dst; + uint64_t rem_addr = descriptor->des_dst->seg_addr.lval; - uint32_t rkey = descriptor->des_dst->seg_key.key32[0]; + uint32_t rkey = dst_seg->key; assert(wv_frag_type(frag) == MCA_BTL_WV_FRAG_SEND_USER || wv_frag_type(frag) == MCA_BTL_WV_FRAG_SEND); descriptor->des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK; @@ -1366,7 +1384,7 @@ int mca_btl_wv_put(mca_btl_base_module_t* btl, frag->sr_desc.Wr.Rdma.RemoteAddress = htonll(rem_addr); frag->sr_desc.Wr.Rdma.Rkey = htonl(rkey); to_com_frag(frag)->sg_entry.pAddress = - (void*)(uintptr_t)descriptor->des_src->seg_addr.pval; + (void*)(uintptr_t)descriptor->des_src->seg_addr.lval; to_com_frag(frag)->sg_entry.Length = descriptor->des_src->seg_len; to_com_frag(frag)->endpoint = ep; descriptor->order = qp; @@ -1394,8 +1412,12 @@ int mca_btl_wv_get(mca_btl_base_module_t* btl, HRESULT hr = 0; mca_btl_wv_get_frag_t* frag = to_get_frag(descriptor); int qp = descriptor->order; + + mca_btl_openib_segment_t *src_seg = (mca_btl_openib_segment_t *) descriptor->des_src; + mca_btl_openib_segment_t *dst_seg = (mca_btl_openib_segment_t *) descriptor->des_dst; + uint64_t rem_addr = descriptor->des_src->seg_addr.lval; - uint32_t rkey = descriptor->des_src->seg_key.key32[0]; + uint32_t rkey = dst_seg->key; assert(wv_frag_type(frag) == MCA_BTL_WV_FRAG_RECV_USER); descriptor->des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK; @@ -1435,7 +1457,7 @@ int mca_btl_wv_get(mca_btl_base_module_t* btl, frag->sr_desc.Wr.Rdma.RemoteAddress = htonll(rem_addr); frag->sr_desc.Wr.Rdma.Rkey = htonl(rkey); to_com_frag(frag)->sg_entry.pAddress = - (void*)(uintptr_t)descriptor->des_dst->seg_addr.pval; + (void*)(uintptr_t)descriptor->des_dst->seg_addr.lval; to_com_frag(frag)->sg_entry.Length = descriptor->des_dst->seg_len; to_com_frag(frag)->endpoint = ep; descriptor->order = qp; diff --git a/ompi/mca/btl/wv/btl_wv_component.c b/ompi/mca/btl/wv/btl_wv_component.c index 7b6163cd7e..b784693d2d 100644 --- a/ompi/mca/btl/wv/btl_wv_component.c +++ b/ompi/mca/btl/wv/btl_wv_component.c @@ -42,6 +42,16 @@ #include "opal/memoryhooks/memory.h" #include "opal/mca/base/mca_base_param.h" #include "opal/mca/hwloc/base/base.h" +/* Define this before including hwloc.h so that we also get the hwloc + verbs helper header file, too. We have to do this level of + indirection because the hwloc subsystem is a component -- we don't + know its exact path. We have to rely on the framework header files + to find the right hwloc verbs helper file for us. */ +#define OPAL_HWLOC_WANT_VERBS_HELPER 1 +#include "opal/mca/hwloc/hwloc.h" +#include "opal/mca/hwloc/base/base.h" +#include "hwloc/openfabrics-verbs.h" + #include "opal/mca/installdirs/installdirs.h" #include "opal_stdint.h" @@ -349,7 +359,7 @@ static void btl_wv_control(mca_btl_base_module_t* btl, mca_btl_wv_module_t *obtl = (mca_btl_wv_module_t*)btl; mca_btl_wv_endpoint_t* ep = to_com_frag(des)->endpoint; mca_btl_wv_control_header_t *ctl_hdr = - (mca_btl_wv_control_header_t *) to_base_frag(des)->segment.seg_addr.pval; + (mca_btl_wv_control_header_t *) to_base_frag(des)->segment.base.seg_addr.pval; mca_btl_wv_eager_rdma_header_t *rdma_hdr; mca_btl_wv_header_coalesced_t *clsc_hdr = (mca_btl_wv_header_coalesced_t*)(ctl_hdr + 1); @@ -990,7 +1000,7 @@ static void merge_values(ompi_btl_wv_ini_values_t *target, static bool inline is_credit_message(const mca_btl_wv_recv_frag_t *frag) { mca_btl_wv_control_header_t* chdr = - (mca_btl_wv_control_header_t *) to_base_frag(frag)->segment.seg_addr.pval; + (mca_btl_wv_control_header_t *) to_base_frag(frag)->segment.base.seg_addr.pval; return (MCA_BTL_TAG_BTL == frag->hdr->tag) && (MCA_BTL_WV_CONTROL_CREDITS == chdr->type); } @@ -998,7 +1008,7 @@ static bool inline is_credit_message(const mca_btl_wv_recv_frag_t *frag) static bool inline is_cts_message(const mca_btl_wv_recv_frag_t *frag) { mca_btl_wv_control_header_t* chdr = - (mca_btl_wv_control_header_t *) to_base_frag(frag)->segment.seg_addr.pval; + (mca_btl_wv_control_header_t *) to_base_frag(frag)->segment.base.seg_addr.pval; return (MCA_BTL_TAG_BTL == frag->hdr->tag) && (MCA_BTL_WV_CONTROL_CTS == chdr->type); } @@ -1416,7 +1426,7 @@ static int init_one_device(opal_list_t *btl_list, struct wv_device* ib_dev) device->use_eager_rdma = values.use_eager_rdma; } - mpool_resources.pool_name = "$Sverbs"; + asprintf (&mpool_resources.pool_name, "verbs.%" PRIu64, device->ib_dev_attr.NodeGuid); mpool_resources.reg_data = (void*)device; mpool_resources.sizeof_reg = sizeof(mca_btl_wv_reg_t); mpool_resources.register_mem = wv_reg_mr; @@ -2096,7 +2106,7 @@ sort_devs_by_distance(struct wv_device **ib_devs, int count) if (orte_proc_is_bound) { /* If this process is bound to one or more PUs, we can get an accurate distance. */ - devs[i].distance = get_ib_dev_distance(ib_devs[i]); + devs[i].distance = get_ib_dev_distance((ibv_device *)ib_devs[i]); } else { /* Since we're not bound, just assume that the device is close. */ @@ -2106,8 +2116,6 @@ sort_devs_by_distance(struct wv_device **ib_devs, int count) qsort(devs, count, sizeof(struct dev_distance), compare_distance); - opal_carto_base_free_graph(host_topo); - return devs; } @@ -3067,7 +3075,7 @@ static int progress_one_device(mca_btl_wv_device_t *device) OPAL_THREAD_UNLOCK(&endpoint->eager_rdma_local.lock); frag->hdr = (mca_btl_wv_header_t*)(((char*)frag->ftr) - size + sizeof(mca_btl_wv_footer_t)); - to_base_frag(frag)->segment.seg_addr.pval = + to_base_frag(frag)->segment.base.seg_addr.pval = ((unsigned char* )frag->hdr) + sizeof(mca_btl_wv_header_t); ret = btl_wv_handle_incoming(btl, to_com_frag(frag)->endpoint, frag, size - sizeof(mca_btl_wv_footer_t)); diff --git a/ompi/mca/btl/wv/btl_wv_endpoint.c b/ompi/mca/btl/wv/btl_wv_endpoint.c index 2a37877e50..3b290b2c35 100644 --- a/ompi/mca/btl/wv/btl_wv_endpoint.c +++ b/ompi/mca/btl/wv/btl_wv_endpoint.c @@ -492,7 +492,7 @@ void mca_btl_wv_endpoint_send_cts(mca_btl_wv_endpoint_t *endpoint) base_des->des_cbdata = NULL; base_des->des_flags |= MCA_BTL_DES_FLAGS_PRIORITY|MCA_BTL_DES_SEND_ALWAYS_CALLBACK; base_des->order = mca_btl_wv_component.credits_qp; - wv_frag->segment.seg_len = sizeof(mca_btl_wv_control_header_t); + wv_frag->segment.base.seg_len = sizeof(mca_btl_wv_control_header_t); com_frag->endpoint = endpoint; sc_frag->hdr->tag = MCA_BTL_TAG_BTL; @@ -500,7 +500,7 @@ void mca_btl_wv_endpoint_send_cts(mca_btl_wv_endpoint_t *endpoint) sc_frag->hdr->credits = 0; ctl_hdr = (mca_btl_wv_control_header_t*) - wv_frag->segment.seg_addr.pval; + wv_frag->segment.base.seg_addr.pval; ctl_hdr->type = MCA_BTL_WV_CONTROL_CTS; /* Send the fragment */ @@ -680,13 +680,13 @@ void mca_btl_wv_endpoint_send_credits(mca_btl_wv_endpoint_t* endpoint, to_base_frag(frag)->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;; to_com_frag(frag)->endpoint = endpoint; frag->hdr->tag = MCA_BTL_TAG_BTL; - to_base_frag(frag)->segment.seg_len = + to_base_frag(frag)->segment.base.seg_len = sizeof(mca_btl_wv_rdma_credits_header_t); } assert(frag->qp_idx == qp); credits_hdr = (mca_btl_wv_rdma_credits_header_t*) - to_base_frag(frag)->segment.seg_addr.pval; + to_base_frag(frag)->segment.base.seg_addr.pval; if(OMPI_SUCCESS == acquire_eager_rdma_send_credit(endpoint)) { do_rdma = true; } else { @@ -770,12 +770,12 @@ static int mca_btl_wv_endpoint_send_eager_rdma( to_base_frag(frag)->base.des_cbdata = NULL; to_base_frag(frag)->base.des_flags |= MCA_BTL_DES_FLAGS_PRIORITY|MCA_BTL_DES_SEND_ALWAYS_CALLBACK; to_base_frag(frag)->base.order = mca_btl_wv_component.credits_qp; - to_base_frag(frag)->segment.seg_len = + to_base_frag(frag)->segment.base.seg_len = sizeof(mca_btl_wv_eager_rdma_header_t); to_com_frag(frag)->endpoint = endpoint; frag->hdr->tag = MCA_BTL_TAG_BTL; - rdma_hdr = (mca_btl_wv_eager_rdma_header_t*)to_base_frag(frag)->segment.seg_addr.pval; + rdma_hdr = (mca_btl_wv_eager_rdma_header_t*)to_base_frag(frag)->segment.base.seg_addr.pval; rdma_hdr->control.type = MCA_BTL_WV_CONTROL_RDMA; rdma_hdr->rkey = endpoint->eager_rdma_local.reg->mr->rkey; rdma_hdr->rdma_start.lval = ompi_ptr_ptol(endpoint->eager_rdma_local.base.pval); @@ -862,7 +862,7 @@ void mca_btl_wv_endpoint_connect_eager_rdma( to_base_frag(frag)->type = MCA_BTL_WV_FRAG_EAGER_RDMA; to_com_frag(frag)->endpoint = endpoint; frag->ftr = (mca_btl_wv_footer_t*) - ((char*)to_base_frag(frag)->segment.seg_addr.pval + + ((char*)to_base_frag(frag)->segment.base.seg_addr.pval + mca_btl_wv_component.eager_limit); MCA_BTL_WV_RDMA_MAKE_REMOTE(frag->ftr); diff --git a/ompi/mca/btl/wv/btl_wv_endpoint.h b/ompi/mca/btl/wv/btl_wv_endpoint.h index 83cb4e14b3..e9f8e4a44b 100644 --- a/ompi/mca/btl/wv/btl_wv_endpoint.h +++ b/ompi/mca/btl/wv/btl_wv_endpoint.h @@ -465,7 +465,7 @@ static inline int post_send(mca_btl_wv_endpoint_t *ep, mca_btl_wv_send_frag_t *frag, const bool rdma) { mca_btl_wv_module_t *wv_btl = ep->endpoint_btl; - mca_btl_base_segment_t *seg = &to_base_frag(frag)->segment; + mca_btl_base_segment_t *seg = (mca_btl_base_segment_t *) &to_base_frag(frag)->segment; WV_SGE *sg = &to_com_frag(frag)->sg_entry; WV_SEND_REQUEST *sr_desc = &to_out_frag(frag)->sr_desc; WV_SEND_REQUEST *bad_wr; diff --git a/ompi/mca/btl/wv/btl_wv_frag.c b/ompi/mca/btl/wv/btl_wv_frag.c index e242b28af1..0fb76ed7ee 100644 --- a/ompi/mca/btl/wv/btl_wv_frag.c +++ b/ompi/mca/btl/wv/btl_wv_frag.c @@ -58,7 +58,7 @@ static void com_constructor(mca_btl_wv_com_frag_t *frag) if(reg) { frag->sg_entry.Lkey = reg->mr->lkey; - base_frag->segment.seg_key.key32[0] = reg->mr->lkey; + base_frag->segment.key = reg->mr->lkey; } } @@ -66,7 +66,7 @@ static void out_constructor(mca_btl_wv_out_frag_t *frag) { mca_btl_wv_frag_t *base_frag = to_base_frag(frag); - base_frag->base.des_src = &base_frag->segment; + base_frag->base.des_src = (mca_btl_base_segment_t *) &base_frag->segment; base_frag->base.des_src_cnt = 1; base_frag->base.des_dst = NULL; base_frag->base.des_dst_cnt = 0; @@ -83,7 +83,7 @@ static void in_constructor(mca_btl_wv_in_frag_t *frag) { mca_btl_wv_frag_t *base_frag = to_base_frag(frag); - base_frag->base.des_dst = &base_frag->segment; + base_frag->base.des_dst = (mca_btl_base_segment_t *) &base_frag->segment; base_frag->base.des_dst_cnt = 1; base_frag->base.des_src = NULL; base_frag->base.des_src_cnt = 0; @@ -100,7 +100,7 @@ static void send_constructor(mca_btl_wv_send_frag_t *frag) (((unsigned char*)base_frag->base.super.ptr) + sizeof(mca_btl_wv_header_coalesced_t) + sizeof(mca_btl_wv_control_header_t)); - base_frag->segment.seg_addr.pval = frag->hdr + 1; + base_frag->segment.base.seg_addr.pval = frag->hdr + 1; to_com_frag(frag)->sg_entry.pAddress = (void*)(uintptr_t)frag->hdr; frag->coalesced_length = 0; OBJ_CONSTRUCT(&frag->coalesced_frags, opal_list_t); @@ -113,7 +113,7 @@ static void recv_constructor(mca_btl_wv_recv_frag_t *frag) base_frag->type = MCA_BTL_WV_FRAG_RECV; frag->hdr = (mca_btl_wv_header_t*)base_frag->base.super.ptr; - base_frag->segment.seg_addr.pval = + base_frag->segment.base.seg_addr.pval = ((unsigned char* )frag->hdr) + sizeof(mca_btl_wv_header_t); to_com_frag(frag)->sg_entry.pAddress = (void*)(uintptr_t)frag->hdr; @@ -152,7 +152,7 @@ static void coalesced_constructor(mca_btl_wv_coalesced_frag_t *frag) base_frag->type = MCA_BTL_WV_FRAG_COALESCED; - base_frag->base.des_src = &base_frag->segment; + base_frag->base.des_src = (mca_btl_base_segment_t *) &base_frag->segment; base_frag->base.des_src_cnt = 1; base_frag->base.des_dst = NULL; base_frag->base.des_dst_cnt = 0; diff --git a/ompi/mca/btl/wv/btl_wv_frag.h b/ompi/mca/btl/wv/btl_wv_frag.h index 925f2ca6a6..d99cde8d1d 100644 --- a/ompi/mca/btl/wv/btl_wv_frag.h +++ b/ompi/mca/btl/wv/btl_wv_frag.h @@ -206,13 +206,18 @@ typedef enum mca_btl_wv_frag_type_t mca_btl_wv_frag_type_t; #define wv_frag_type(f) (to_base_frag(f)->type) /** - * IB fragment derived type. + * WV fragment derived type. */ +typedef struct mca_btl_wv_segment_t { + mca_btl_base_segment_t base; + uint32_t key; +} mca_btl_openib_segment_t; + /* base wv frag */ typedef struct mca_btl_wv_frag_t { mca_btl_base_descriptor_t base; - mca_btl_base_segment_t segment; + mca_btl_wv_segment_t segment; mca_btl_wv_frag_type_t type; ompi_free_list_t* list; } mca_btl_wv_frag_t; diff --git a/ompi/mca/btl/wv/connect/btl_wv_connect_base.c b/ompi/mca/btl/wv/connect/btl_wv_connect_base.c index 2030144d7c..23b0efaaa8 100644 --- a/ompi/mca/btl/wv/connect/btl_wv_connect_base.c +++ b/ompi/mca/btl/wv/connect/btl_wv_connect_base.c @@ -413,7 +413,7 @@ int ompi_btl_wv_connect_base_alloc_cts(mca_btl_base_endpoint_t *endpoint) /* Copy the lkey where it needs to go */ endpoint->endpoint_cts_frag.super.sg_entry.Lkey = - endpoint->endpoint_cts_frag.super.super.segment.seg_key.key32[0] = + endpoint->endpoint_cts_frag.super.super.segment.key = endpoint->endpoint_cts_mr->lkey; endpoint->endpoint_cts_frag.super.sg_entry.Length = length; @@ -437,8 +437,8 @@ int ompi_btl_wv_connect_base_free_cts(mca_btl_base_endpoint_t *endpoint) because it was not registered there in the first place (see comment above, near call to ibv_reg_mr). */ if (NULL != endpoint->endpoint_cts_mr) { - if(SUCCEEDED(endpoint->endpoint_cts_mr->pd->handle->DeregisterMemory(endpoint->endpoint_cts_mr->lkey,NULL))) - free(endpoint->endpoint_cts_mr); + if(SUCCEEDED(endpoint->endpoint_cts_mr->pd->handle->DeregisterMemory(endpoint->endpoint_cts_mr->lkey,NULL))) + free(endpoint->endpoint_cts_mr); endpoint->endpoint_cts_mr = NULL; } if (NULL != endpoint->endpoint_cts_frag.super.super.base.super.ptr) { diff --git a/ompi/mca/common/sm/common_sm.c b/ompi/mca/common/sm/common_sm.c index b06ed0aeb7..889d9c3faa 100644 --- a/ompi/mca/common/sm/common_sm.c +++ b/ompi/mca/common/sm/common_sm.c @@ -170,7 +170,7 @@ mca_common_sm_module_create(size_t size, mca_common_sm_module_t *map = NULL; opal_shmem_ds_t *seg_meta = NULL; - if (NULL == (seg_meta = malloc(sizeof(*seg_meta)))) { + if (NULL == (seg_meta = (opal_shmem_ds_t *) malloc(sizeof(*seg_meta)))) { /* out of resources */ return NULL; } @@ -263,7 +263,7 @@ mca_common_sm_init(ompi_proc_t **procs, return NULL; } - if (NULL == (seg_meta = malloc(sizeof(*seg_meta)))) { + if (NULL == (seg_meta = (opal_shmem_ds_t *) malloc(sizeof(*seg_meta)))) { /* out of resources - just bail */ return NULL; } diff --git a/ompi/mca/mpool/grdma/mpool_grdma.h b/ompi/mca/mpool/grdma/mpool_grdma.h index 0cbad727f6..7e370e860a 100644 --- a/ompi/mca/mpool/grdma/mpool_grdma.h +++ b/ompi/mca/mpool/grdma/mpool_grdma.h @@ -31,7 +31,9 @@ #include "ompi/class/ompi_free_list.h" #include "opal/mca/event/event.h" #include "ompi/mca/mpool/mpool.h" +#if HAVE_SYS_MMAN_H #include +#endif BEGIN_C_DECLS