From b308afa8fd6ea3e4ce207fc46ddf65a56f1b88ae Mon Sep 17 00:00:00 2001 From: Nathan Hjelm Date: Tue, 10 Mar 2015 14:21:42 -0600 Subject: [PATCH] btl/openib: remove derived btl segment type The derived segment type (btl_openib_segment_t) was intended to store the registration info needed for put and get. In BTL 3.0 this is no longer required. I intended to remove this type as part of open-mpi/ompi@74f1af45484628c3262a18e1ffff9e3295016ed5 . Signed-off-by: Nathan Hjelm --- opal/mca/btl/openib/btl_openib.c | 26 +++++++++---------- opal/mca/btl/openib/btl_openib_component.c | 8 +++--- opal/mca/btl/openib/btl_openib_endpoint.c | 14 +++++----- opal/mca/btl/openib/btl_openib_endpoint.h | 4 +-- opal/mca/btl/openib/btl_openib_failover.c | 11 ++++---- opal/mca/btl/openib/btl_openib_frag.c | 13 +++++----- opal/mca/btl/openib/btl_openib_frag.h | 8 +----- .../openib/connect/btl_openib_connect_base.c | 1 - 8 files changed, 39 insertions(+), 46 deletions(-) diff --git a/opal/mca/btl/openib/btl_openib.c b/opal/mca/btl/openib/btl_openib.c index ffb43b4d96..a6ebeff4b7 100644 --- a/opal/mca/btl/openib/btl_openib.c +++ b/opal/mca/btl/openib/btl_openib.c @@ -1130,7 +1130,7 @@ ib_frag_alloc(mca_btl_openib_module_t *btl, size_t size, uint8_t order, return NULL; /* not all upper layer users set this */ - to_base_frag(item)->segment.base.seg_len = size; + to_base_frag(item)->segment.seg_len = size; to_base_frag(item)->base.order = order; to_base_frag(item)->base.des_flags = flags; @@ -1161,7 +1161,7 @@ static mca_btl_openib_send_frag_t *check_coalescing(opal_list_t *frag_list, } total_length = size + frag->coalesced_length + - to_base_frag(frag)->segment.base.seg_len + + to_base_frag(frag)->segment.seg_len + sizeof(mca_btl_openib_header_coalesced_t); qp = to_base_frag(frag)->base.order; @@ -1248,8 +1248,8 @@ mca_btl_base_descriptor_t* mca_btl_openib_alloc( sfrag->hdr->tag = MCA_BTL_TAG_IB; ctrl_hdr->type = MCA_BTL_OPENIB_CONTROL_COALESCED; clsc_hdr->tag = org_tag; - clsc_hdr->size = to_base_frag(sfrag)->segment.base.seg_len; - clsc_hdr->alloc_size = to_base_frag(sfrag)->segment.base.seg_len; + clsc_hdr->size = to_base_frag(sfrag)->segment.seg_len; + clsc_hdr->alloc_size = to_base_frag(sfrag)->segment.seg_len; if(ep->nbo) BTL_OPENIB_HEADER_COALESCED_HTON(*clsc_hdr); sfrag->coalesced_length = sizeof(mca_btl_openib_control_header_t) + @@ -1259,13 +1259,13 @@ mca_btl_base_descriptor_t* mca_btl_openib_alloc( cfrag->hdr = (mca_btl_openib_header_coalesced_t*)((unsigned char*)(sfrag->hdr + 1) + sfrag->coalesced_length + - to_base_frag(sfrag)->segment.base.seg_len); + to_base_frag(sfrag)->segment.seg_len); cfrag->hdr = (mca_btl_openib_header_coalesced_t*)BTL_OPENIB_ALIGN_COALESCE_HDR(cfrag->hdr); cfrag->hdr->alloc_size = size; /* point coalesced frag pointer into a data buffer */ - to_base_frag(cfrag)->segment.base.seg_addr.pval = cfrag->hdr + 1; - to_base_frag(cfrag)->segment.base.seg_len = size; + to_base_frag(cfrag)->segment.seg_addr.pval = cfrag->hdr + 1; + to_base_frag(cfrag)->segment.seg_len = size; /* NTH: there is no reason to append the coalesced fragment here. No more * fragments will be added until either send or free has been called on @@ -1297,7 +1297,7 @@ int mca_btl_openib_free( to_com_frag(des)->sg_entry.addr = (uint64_t)(uintptr_t)to_send_frag(des)->hdr; to_send_frag(des)->coalesced_length = 0; - to_base_frag(des)->segment.base.seg_addr.pval = + to_base_frag(des)->segment.seg_addr.pval = to_send_frag(des)->hdr + 1; assert(!opal_list_get_size(&to_send_frag(des)->coalesced_frags)); /* fall through */ @@ -1369,7 +1369,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_src( return NULL; } - ptr = to_base_frag(frag)->segment.base.seg_addr.pval; + ptr = to_base_frag(frag)->segment.seg_addr.pval; iov.iov_len = max_data; iov.iov_base = (IOVBASE_TYPE *) ( (unsigned char*) ptr + reserve ); @@ -1388,7 +1388,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_src( *size = max_data; /* not all upper layer users set this */ - to_base_frag(frag)->segment.base.seg_len = max_data + reserve; + to_base_frag(frag)->segment.seg_len = max_data + reserve; return &to_base_frag(frag)->base; } @@ -1593,14 +1593,14 @@ int mca_btl_openib_sendi( struct mca_btl_base_module_t* btl, goto cant_send_frag; } - frag->segment.base.seg_len = size; + frag->segment.seg_len = size; frag->base.order = qp; frag->base.des_flags = flags; hdr->tag = tag; to_com_frag(item)->endpoint = ep; /* put match header */ - memcpy(frag->segment.base.seg_addr.pval, header, header_size); + memcpy(frag->segment.seg_addr.pval, header, header_size); /* Pack data */ if(payload_size) { @@ -1608,7 +1608,7 @@ int mca_btl_openib_sendi( struct mca_btl_base_module_t* btl, struct iovec iov; uint32_t iov_count; /* pack the data into the supplied buffer */ - iov.iov_base = (IOVBASE_TYPE*)((unsigned char*)frag->segment.base.seg_addr.pval + header_size); + iov.iov_base = (IOVBASE_TYPE*)((unsigned char*)frag->segment.seg_addr.pval + header_size); iov.iov_len = max_data = payload_size; iov_count = 1; diff --git a/opal/mca/btl/openib/btl_openib_component.c b/opal/mca/btl/openib/btl_openib_component.c index 1d3a50e68d..8331714abd 100644 --- a/opal/mca/btl/openib/btl_openib_component.c +++ b/opal/mca/btl/openib/btl_openib_component.c @@ -463,7 +463,7 @@ static void btl_openib_control(mca_btl_base_module_t* btl, mca_btl_openib_module_t *obtl = (mca_btl_openib_module_t*)btl; mca_btl_openib_endpoint_t* ep = to_com_frag(des)->endpoint; mca_btl_openib_control_header_t *ctl_hdr = - (mca_btl_openib_control_header_t *) to_base_frag(des)->segment.base.seg_addr.pval; + (mca_btl_openib_control_header_t *) to_base_frag(des)->segment.seg_addr.pval; mca_btl_openib_eager_rdma_header_t *rdma_hdr; mca_btl_openib_header_coalesced_t *clsc_hdr = (mca_btl_openib_header_coalesced_t*)(ctl_hdr + 1); @@ -1179,7 +1179,7 @@ static void merge_values(opal_btl_openib_ini_values_t *target, static bool inline is_credit_message(const mca_btl_openib_recv_frag_t *frag) { mca_btl_openib_control_header_t* chdr = - (mca_btl_openib_control_header_t *) to_base_frag(frag)->segment.base.seg_addr.pval; + (mca_btl_openib_control_header_t *) to_base_frag(frag)->segment.seg_addr.pval; return (MCA_BTL_TAG_IB == frag->hdr->tag) && (MCA_BTL_OPENIB_CONTROL_CREDITS == chdr->type); } @@ -1187,7 +1187,7 @@ static bool inline is_credit_message(const mca_btl_openib_recv_frag_t *frag) static bool inline is_cts_message(const mca_btl_openib_recv_frag_t *frag) { mca_btl_openib_control_header_t* chdr = - (mca_btl_openib_control_header_t *) to_base_frag(frag)->segment.base.seg_addr.pval; + (mca_btl_openib_control_header_t *) to_base_frag(frag)->segment.seg_addr.pval; return (MCA_BTL_TAG_IB == frag->hdr->tag) && (MCA_BTL_OPENIB_CONTROL_CTS == chdr->type); } @@ -3703,7 +3703,7 @@ static int progress_one_device(mca_btl_openib_device_t *device) OPAL_THREAD_UNLOCK(&endpoint->eager_rdma_local.lock); frag->hdr = (mca_btl_openib_header_t*)(((char*)frag->ftr) - size - BTL_OPENIB_FTR_PADDING(size) + sizeof(mca_btl_openib_footer_t)); - to_base_frag(frag)->segment.base.seg_addr.pval = + to_base_frag(frag)->segment.seg_addr.pval = ((unsigned char* )frag->hdr) + sizeof(mca_btl_openib_header_t); ret = btl_openib_handle_incoming(btl, to_com_frag(frag)->endpoint, diff --git a/opal/mca/btl/openib/btl_openib_endpoint.c b/opal/mca/btl/openib/btl_openib_endpoint.c index dbb9613639..c117acd1c2 100644 --- a/opal/mca/btl/openib/btl_openib_endpoint.c +++ b/opal/mca/btl/openib/btl_openib_endpoint.c @@ -492,7 +492,7 @@ void mca_btl_openib_endpoint_send_cts(mca_btl_openib_endpoint_t *endpoint) base_des->des_cbdata = NULL; base_des->des_flags |= MCA_BTL_DES_FLAGS_PRIORITY|MCA_BTL_DES_SEND_ALWAYS_CALLBACK; base_des->order = mca_btl_openib_component.credits_qp; - openib_frag->segment.base.seg_len = sizeof(mca_btl_openib_control_header_t); + openib_frag->segment.seg_len = sizeof(mca_btl_openib_control_header_t); com_frag->endpoint = endpoint; sc_frag->hdr->tag = MCA_BTL_TAG_IB; @@ -500,7 +500,7 @@ void mca_btl_openib_endpoint_send_cts(mca_btl_openib_endpoint_t *endpoint) sc_frag->hdr->credits = 0; ctl_hdr = (mca_btl_openib_control_header_t*) - openib_frag->segment.base.seg_addr.pval; + openib_frag->segment.seg_addr.pval; ctl_hdr->type = MCA_BTL_OPENIB_CONTROL_CTS; /* Send the fragment */ @@ -727,13 +727,13 @@ void mca_btl_openib_endpoint_send_credits(mca_btl_openib_endpoint_t* endpoint, to_base_frag(frag)->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;; to_com_frag(frag)->endpoint = endpoint; frag->hdr->tag = MCA_BTL_TAG_IB; - to_base_frag(frag)->segment.base.seg_len = + to_base_frag(frag)->segment.seg_len = sizeof(mca_btl_openib_rdma_credits_header_t); } assert(frag->qp_idx == qp); credits_hdr = (mca_btl_openib_rdma_credits_header_t*) - to_base_frag(frag)->segment.base.seg_addr.pval; + to_base_frag(frag)->segment.seg_addr.pval; if(OPAL_SUCCESS == acquire_eager_rdma_send_credit(endpoint)) { do_rdma = true; } else { @@ -818,12 +818,12 @@ static int mca_btl_openib_endpoint_send_eager_rdma( to_base_frag(frag)->base.des_cbdata = NULL; to_base_frag(frag)->base.des_flags |= MCA_BTL_DES_FLAGS_PRIORITY|MCA_BTL_DES_SEND_ALWAYS_CALLBACK; to_base_frag(frag)->base.order = mca_btl_openib_component.credits_qp; - to_base_frag(frag)->segment.base.seg_len = + to_base_frag(frag)->segment.seg_len = sizeof(mca_btl_openib_eager_rdma_header_t); to_com_frag(frag)->endpoint = endpoint; frag->hdr->tag = MCA_BTL_TAG_IB; - rdma_hdr = (mca_btl_openib_eager_rdma_header_t*)to_base_frag(frag)->segment.base.seg_addr.pval; + rdma_hdr = (mca_btl_openib_eager_rdma_header_t*)to_base_frag(frag)->segment.seg_addr.pval; rdma_hdr->control.type = MCA_BTL_OPENIB_CONTROL_RDMA; rdma_hdr->rkey = endpoint->eager_rdma_local.reg->mr->rkey; rdma_hdr->rdma_start.lval = opal_ptr_ptol(endpoint->eager_rdma_local.base.pval); @@ -925,7 +925,7 @@ void mca_btl_openib_endpoint_connect_eager_rdma( to_base_frag(frag)->type = MCA_BTL_OPENIB_FRAG_EAGER_RDMA; to_com_frag(frag)->endpoint = endpoint; frag->ftr = (mca_btl_openib_footer_t*) - ((char*)to_base_frag(frag)->segment.base.seg_addr.pval + + ((char*)to_base_frag(frag)->segment.seg_addr.pval + mca_btl_openib_component.eager_limit); MCA_BTL_OPENIB_RDMA_MAKE_REMOTE(frag->ftr); diff --git a/opal/mca/btl/openib/btl_openib_endpoint.h b/opal/mca/btl/openib/btl_openib_endpoint.h index 610759f571..3b55c57019 100644 --- a/opal/mca/btl/openib/btl_openib_endpoint.h +++ b/opal/mca/btl/openib/btl_openib_endpoint.h @@ -546,13 +546,13 @@ static inline int post_send(mca_btl_openib_endpoint_t *ep, mca_btl_openib_send_frag_t *frag, const bool rdma, int do_signal) { mca_btl_openib_module_t *openib_btl = ep->endpoint_btl; - mca_btl_openib_segment_t *seg = &to_base_frag(frag)->segment; + mca_btl_base_segment_t *seg = &to_base_frag(frag)->segment; struct ibv_sge *sg = &to_com_frag(frag)->sg_entry; struct ibv_send_wr *sr_desc = &to_out_frag(frag)->sr_desc; struct ibv_send_wr *bad_wr; int qp = to_base_frag(frag)->base.order; - sg->length = seg->base.seg_len + sizeof(mca_btl_openib_header_t) + + sg->length = seg->seg_len + sizeof(mca_btl_openib_header_t) + (rdma ? sizeof(mca_btl_openib_footer_t) : 0) + frag->coalesced_length; sr_desc->send_flags = ib_send_flags(sg->length, &(ep->qps[qp]), do_signal); diff --git a/opal/mca/btl/openib/btl_openib_failover.c b/opal/mca/btl/openib/btl_openib_failover.c index d5f9c15874..7e38953a25 100644 --- a/opal/mca/btl/openib/btl_openib_failover.c +++ b/opal/mca/btl/openib/btl_openib_failover.c @@ -1,7 +1,8 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ /* * Copyright (c) 2010-2011 Oracle and/or its affiliates. All rights reserved. * Copyright (c) 2011 NVIDIA Corporation. All rights reserved. - * Copyright (c) 2012 Los Alamos National Security, LLC. All rights + * Copyright (c) 2012-2015 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2013 NVIDIA Corporation. All rights reserved. * Copyright (c) 2014 Research Organization for Information Science @@ -678,12 +679,12 @@ static void mca_btl_openib_endpoint_notify(mca_btl_base_endpoint_t* endpoint, ui to_base_frag(frag)->base.des_cbdata = NULL; to_base_frag(frag)->base.des_flags |= MCA_BTL_DES_FLAGS_PRIORITY|MCA_BTL_DES_SEND_ALWAYS_CALLBACK; to_base_frag(frag)->base.order = mca_btl_openib_component.credits_qp; - to_base_frag(frag)->segment.base.seg_len = + to_base_frag(frag)->segment.seg_len = sizeof(mca_btl_openib_broken_connection_header_t); to_com_frag(frag)->endpoint = newep; frag->hdr->tag = MCA_BTL_TAG_IB; - bc_hdr = (mca_btl_openib_broken_connection_header_t*)to_base_frag(frag)->segment.base.seg_addr.pval; + bc_hdr = (mca_btl_openib_broken_connection_header_t*)to_base_frag(frag)->segment.seg_addr.pval; bc_hdr->control.type = type; bc_hdr->lid = endpoint->endpoint_btl->port_info.lid; bc_hdr->subnet_id = endpoint->endpoint_btl->port_info.subnet_id; @@ -720,10 +721,10 @@ static void dump_local_rdma_frags(mca_btl_openib_endpoint_t * endpoint) { frag->hdr = (mca_btl_openib_header_t*)(((char*)frag->ftr) - size + sizeof(mca_btl_openib_footer_t)); - to_base_frag(frag)->segment.base.seg_addr.pval = + to_base_frag(frag)->segment.seg_addr.pval = ((unsigned char* )frag->hdr) + sizeof(mca_btl_openib_header_t); - chdr = to_base_frag(frag)->segment.base.seg_addr.pval; + chdr = to_base_frag(frag)->segment.seg_addr.pval; if ((MCA_BTL_TAG_IB == frag->hdr->tag) && (MCA_BTL_OPENIB_CONTROL_CREDITS == chdr->type)) { opal_output(0, "tag[%d] is credit message", i); diff --git a/opal/mca/btl/openib/btl_openib_frag.c b/opal/mca/btl/openib/btl_openib_frag.c index 1cd4d93685..d8d94b939b 100644 --- a/opal/mca/btl/openib/btl_openib_frag.c +++ b/opal/mca/btl/openib/btl_openib_frag.c @@ -62,7 +62,6 @@ static void com_constructor(mca_btl_openib_com_frag_t *frag) if(reg) { frag->sg_entry.lkey = reg->mr->lkey; - base_frag->segment.key = reg->mr->lkey; } frag->n_wqes_inflight = 0; } @@ -71,7 +70,7 @@ static void out_constructor(mca_btl_openib_out_frag_t *frag) { mca_btl_openib_frag_t *base_frag = to_base_frag(frag); - base_frag->base.des_segments = &base_frag->segment.base; + base_frag->base.des_segments = &base_frag->segment; base_frag->base.des_segment_count = 1; frag->sr_desc.wr_id = (uint64_t)(uintptr_t)frag; @@ -86,7 +85,7 @@ static void in_constructor(mca_btl_openib_in_frag_t *frag) { mca_btl_openib_frag_t *base_frag = to_base_frag(frag); - base_frag->base.des_segments = &base_frag->segment.base; + base_frag->base.des_segments = &base_frag->segment; base_frag->base.des_segment_count = 1; } @@ -101,7 +100,7 @@ static void send_constructor(mca_btl_openib_send_frag_t *frag) (((unsigned char*)base_frag->base.super.ptr) + sizeof(mca_btl_openib_header_coalesced_t) + sizeof(mca_btl_openib_control_header_t)); - base_frag->segment.base.seg_addr.pval = frag->hdr + 1; + base_frag->segment.seg_addr.pval = frag->hdr + 1; to_com_frag(frag)->sg_entry.addr = (uint64_t)(uintptr_t)frag->hdr; frag->coalesced_length = 0; OBJ_CONSTRUCT(&frag->coalesced_frags, opal_list_t); @@ -114,7 +113,7 @@ static void recv_constructor(mca_btl_openib_recv_frag_t *frag) base_frag->type = MCA_BTL_OPENIB_FRAG_RECV; frag->hdr = (mca_btl_openib_header_t*)base_frag->base.super.ptr; - base_frag->segment.base.seg_addr.pval = + base_frag->segment.seg_addr.pval = ((unsigned char* )frag->hdr) + sizeof(mca_btl_openib_header_t); to_com_frag(frag)->sg_entry.addr = (uint64_t)(uintptr_t)frag->hdr; @@ -129,7 +128,7 @@ static void send_control_constructor(mca_btl_openib_send_control_frag_t *frag) to_base_frag(frag)->type = MCA_BTL_OPENIB_FRAG_CONTROL; /* adjusting headers because there is no coalesce header in control messages */ frag->hdr = frag->chdr; - to_base_frag(frag)->segment.base.seg_addr.pval = frag->hdr + 1; + to_base_frag(frag)->segment.seg_addr.pval = frag->hdr + 1; to_com_frag(frag)->sg_entry.addr = (uint64_t)(uintptr_t)frag->hdr; } @@ -158,7 +157,7 @@ static void coalesced_constructor(mca_btl_openib_coalesced_frag_t *frag) base_frag->type = MCA_BTL_OPENIB_FRAG_COALESCED; - base_frag->base.des_segments = &base_frag->segment.base; + base_frag->base.des_segments = &base_frag->segment; base_frag->base.des_segment_count = 1; } diff --git a/opal/mca/btl/openib/btl_openib_frag.h b/opal/mca/btl/openib/btl_openib_frag.h index b61b900703..392728f9d3 100644 --- a/opal/mca/btl/openib/btl_openib_frag.h +++ b/opal/mca/btl/openib/btl_openib_frag.h @@ -284,16 +284,10 @@ typedef enum mca_btl_openib_frag_type_t mca_btl_openib_frag_type_t; /** * IB fragment derived type. */ - -typedef struct mca_btl_openib_segment_t { - mca_btl_base_segment_t base; - uint32_t key; -} mca_btl_openib_segment_t; - /* base openib frag */ typedef struct mca_btl_openib_frag_t { mca_btl_base_descriptor_t base; - mca_btl_openib_segment_t segment; + mca_btl_base_segment_t segment; mca_btl_openib_frag_type_t type; opal_free_list_t* list; } mca_btl_openib_frag_t; diff --git a/opal/mca/btl/openib/connect/btl_openib_connect_base.c b/opal/mca/btl/openib/connect/btl_openib_connect_base.c index 6865651497..eadde14983 100644 --- a/opal/mca/btl/openib/connect/btl_openib_connect_base.c +++ b/opal/mca/btl/openib/connect/btl_openib_connect_base.c @@ -448,7 +448,6 @@ int opal_btl_openib_connect_base_alloc_cts(mca_btl_base_endpoint_t *endpoint) /* Copy the lkey where it needs to go */ endpoint->endpoint_cts_frag.super.sg_entry.lkey = - endpoint->endpoint_cts_frag.super.super.segment.key = endpoint->endpoint_cts_mr->lkey; endpoint->endpoint_cts_frag.super.sg_entry.length = length;