ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

btl/openib: remove derived btl segment type

Просмотр исходного кода 
The derived segment type (btl_openib_segment_t) was intended to store
the registration info needed for put and get. In BTL 3.0 this is no
longer required. I intended to remove this type as part of
open-mpi/ompi@74f1af4548 .

Signed-off-by: Nathan Hjelm <hjelmn@lanl.gov>
Этот коммит содержится в:
Nathan Hjelm 2015-03-10 14:21:42 -06:00
родитель d929137768
Коммит b308afa8fd
8 изменённых файлов: 39 добавлений и 46 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

26
opal/mca/btl/openib/btl_openib.c
Просмотреть файл

@ -1130,7 +1130,7 @@ ib_frag_alloc(mca_btl_openib_module_t *btl, size_t size, uint8_t order,
return NULL;
/* not all upper layer users set this */
to_base_frag(item)->segment.base.seg_len = size;
to_base_frag(item)->segment.seg_len = size;
to_base_frag(item)->base.order = order;
to_base_frag(item)->base.des_flags = flags;
@ -1161,7 +1161,7 @@ static mca_btl_openib_send_frag_t *check_coalescing(opal_list_t *frag_list,
}
total_length = size + frag->coalesced_length +
to_base_frag(frag)->segment.base.seg_len +
to_base_frag(frag)->segment.seg_len +
sizeof(mca_btl_openib_header_coalesced_t);
qp = to_base_frag(frag)->base.order;
@ -1248,8 +1248,8 @@ mca_btl_base_descriptor_t* mca_btl_openib_alloc(
sfrag->hdr->tag = MCA_BTL_TAG_IB;
ctrl_hdr->type = MCA_BTL_OPENIB_CONTROL_COALESCED;
clsc_hdr->tag = org_tag;
clsc_hdr->size = to_base_frag(sfrag)->segment.base.seg_len;
clsc_hdr->alloc_size = to_base_frag(sfrag)->segment.base.seg_len;
clsc_hdr->size = to_base_frag(sfrag)->segment.seg_len;
clsc_hdr->alloc_size = to_base_frag(sfrag)->segment.seg_len;
if(ep->nbo)
BTL_OPENIB_HEADER_COALESCED_HTON(*clsc_hdr);
sfrag->coalesced_length = sizeof(mca_btl_openib_control_header_t) +
@ -1259,13 +1259,13 @@ mca_btl_base_descriptor_t* mca_btl_openib_alloc(
cfrag->hdr = (mca_btl_openib_header_coalesced_t*)((unsigned char*)(sfrag->hdr + 1) +
sfrag->coalesced_length +
to_base_frag(sfrag)->segment.base.seg_len);
to_base_frag(sfrag)->segment.seg_len);
cfrag->hdr = (mca_btl_openib_header_coalesced_t*)BTL_OPENIB_ALIGN_COALESCE_HDR(cfrag->hdr);
cfrag->hdr->alloc_size = size;
/* point coalesced frag pointer into a data buffer */
to_base_frag(cfrag)->segment.base.seg_addr.pval = cfrag->hdr + 1;
to_base_frag(cfrag)->segment.base.seg_len = size;
to_base_frag(cfrag)->segment.seg_addr.pval = cfrag->hdr + 1;
to_base_frag(cfrag)->segment.seg_len = size;
/* NTH: there is no reason to append the coalesced fragment here. No more
* fragments will be added until either send or free has been called on
@ -1297,7 +1297,7 @@ int mca_btl_openib_free(
to_com_frag(des)->sg_entry.addr =
(uint64_t)(uintptr_t)to_send_frag(des)->hdr;
to_send_frag(des)->coalesced_length = 0;
to_base_frag(des)->segment.base.seg_addr.pval =
to_base_frag(des)->segment.seg_addr.pval =
to_send_frag(des)->hdr + 1;
assert(!opal_list_get_size(&to_send_frag(des)->coalesced_frags));
/* fall through */
@ -1369,7 +1369,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_src(
return NULL;
}
ptr = to_base_frag(frag)->segment.base.seg_addr.pval;
ptr = to_base_frag(frag)->segment.seg_addr.pval;
iov.iov_len = max_data;
iov.iov_base = (IOVBASE_TYPE *) ( (unsigned char*) ptr + reserve );
@ -1388,7 +1388,7 @@ mca_btl_base_descriptor_t* mca_btl_openib_prepare_src(
*size = max_data;
/* not all upper layer users set this */
to_base_frag(frag)->segment.base.seg_len = max_data + reserve;
to_base_frag(frag)->segment.seg_len = max_data + reserve;
return &to_base_frag(frag)->base;
}
@ -1593,14 +1593,14 @@ int mca_btl_openib_sendi( struct mca_btl_base_module_t* btl,
goto cant_send_frag;
}
frag->segment.base.seg_len = size;
frag->segment.seg_len = size;
frag->base.order = qp;
frag->base.des_flags = flags;
hdr->tag = tag;
to_com_frag(item)->endpoint = ep;
/* put match header */
memcpy(frag->segment.base.seg_addr.pval, header, header_size);
memcpy(frag->segment.seg_addr.pval, header, header_size);
/* Pack data */
if(payload_size) {
@ -1608,7 +1608,7 @@ int mca_btl_openib_sendi( struct mca_btl_base_module_t* btl,
struct iovec iov;
uint32_t iov_count;
/* pack the data into the supplied buffer */
iov.iov_base = (IOVBASE_TYPE*)((unsigned char*)frag->segment.base.seg_addr.pval + header_size);
iov.iov_base = (IOVBASE_TYPE*)((unsigned char*)frag->segment.seg_addr.pval + header_size);
iov.iov_len = max_data = payload_size;
iov_count = 1;

8
opal/mca/btl/openib/btl_openib_component.c
Просмотреть файл

@ -463,7 +463,7 @@ static void btl_openib_control(mca_btl_base_module_t* btl,
mca_btl_openib_module_t *obtl = (mca_btl_openib_module_t*)btl;
mca_btl_openib_endpoint_t* ep = to_com_frag(des)->endpoint;
mca_btl_openib_control_header_t *ctl_hdr =
(mca_btl_openib_control_header_t *) to_base_frag(des)->segment.base.seg_addr.pval;
(mca_btl_openib_control_header_t *) to_base_frag(des)->segment.seg_addr.pval;
mca_btl_openib_eager_rdma_header_t *rdma_hdr;
mca_btl_openib_header_coalesced_t *clsc_hdr =
(mca_btl_openib_header_coalesced_t*)(ctl_hdr + 1);
@ -1179,7 +1179,7 @@ static void merge_values(opal_btl_openib_ini_values_t *target,
static bool inline is_credit_message(const mca_btl_openib_recv_frag_t *frag)
{
mca_btl_openib_control_header_t* chdr =
(mca_btl_openib_control_header_t *) to_base_frag(frag)->segment.base.seg_addr.pval;
(mca_btl_openib_control_header_t *) to_base_frag(frag)->segment.seg_addr.pval;
return (MCA_BTL_TAG_IB == frag->hdr->tag) &&
(MCA_BTL_OPENIB_CONTROL_CREDITS == chdr->type);
}
@ -1187,7 +1187,7 @@ static bool inline is_credit_message(const mca_btl_openib_recv_frag_t *frag)
static bool inline is_cts_message(const mca_btl_openib_recv_frag_t *frag)
{
mca_btl_openib_control_header_t* chdr =
(mca_btl_openib_control_header_t *) to_base_frag(frag)->segment.base.seg_addr.pval;
(mca_btl_openib_control_header_t *) to_base_frag(frag)->segment.seg_addr.pval;
return (MCA_BTL_TAG_IB == frag->hdr->tag) &&
(MCA_BTL_OPENIB_CONTROL_CTS == chdr->type);
}
@ -3703,7 +3703,7 @@ static int progress_one_device(mca_btl_openib_device_t *device)
OPAL_THREAD_UNLOCK(&endpoint->eager_rdma_local.lock);
frag->hdr = (mca_btl_openib_header_t*)(((char*)frag->ftr) -
size - BTL_OPENIB_FTR_PADDING(size) + sizeof(mca_btl_openib_footer_t));
to_base_frag(frag)->segment.base.seg_addr.pval =
to_base_frag(frag)->segment.seg_addr.pval =
((unsigned char* )frag->hdr) + sizeof(mca_btl_openib_header_t);
ret = btl_openib_handle_incoming(btl, to_com_frag(frag)->endpoint,

14
opal/mca/btl/openib/btl_openib_endpoint.c
Просмотреть файл

@ -492,7 +492,7 @@ void mca_btl_openib_endpoint_send_cts(mca_btl_openib_endpoint_t *endpoint)
base_des->des_cbdata = NULL;
base_des->des_flags |= MCA_BTL_DES_FLAGS_PRIORITY|MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
base_des->order = mca_btl_openib_component.credits_qp;
openib_frag->segment.base.seg_len = sizeof(mca_btl_openib_control_header_t);
openib_frag->segment.seg_len = sizeof(mca_btl_openib_control_header_t);
com_frag->endpoint = endpoint;
sc_frag->hdr->tag = MCA_BTL_TAG_IB;
@ -500,7 +500,7 @@ void mca_btl_openib_endpoint_send_cts(mca_btl_openib_endpoint_t *endpoint)
sc_frag->hdr->credits = 0;
ctl_hdr = (mca_btl_openib_control_header_t*)
openib_frag->segment.base.seg_addr.pval;
openib_frag->segment.seg_addr.pval;
ctl_hdr->type = MCA_BTL_OPENIB_CONTROL_CTS;
/* Send the fragment */
@ -727,13 +727,13 @@ void mca_btl_openib_endpoint_send_credits(mca_btl_openib_endpoint_t* endpoint,
to_base_frag(frag)->base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;;
to_com_frag(frag)->endpoint = endpoint;
frag->hdr->tag = MCA_BTL_TAG_IB;
to_base_frag(frag)->segment.base.seg_len =
to_base_frag(frag)->segment.seg_len =
sizeof(mca_btl_openib_rdma_credits_header_t);
}
assert(frag->qp_idx == qp);
credits_hdr = (mca_btl_openib_rdma_credits_header_t*)
to_base_frag(frag)->segment.base.seg_addr.pval;
to_base_frag(frag)->segment.seg_addr.pval;
if(OPAL_SUCCESS == acquire_eager_rdma_send_credit(endpoint)) {
do_rdma = true;
} else {
@ -818,12 +818,12 @@ static int mca_btl_openib_endpoint_send_eager_rdma(
to_base_frag(frag)->base.des_cbdata = NULL;
to_base_frag(frag)->base.des_flags |= MCA_BTL_DES_FLAGS_PRIORITY|MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
to_base_frag(frag)->base.order = mca_btl_openib_component.credits_qp;
to_base_frag(frag)->segment.base.seg_len =
to_base_frag(frag)->segment.seg_len =
sizeof(mca_btl_openib_eager_rdma_header_t);
to_com_frag(frag)->endpoint = endpoint;
frag->hdr->tag = MCA_BTL_TAG_IB;
rdma_hdr = (mca_btl_openib_eager_rdma_header_t*)to_base_frag(frag)->segment.base.seg_addr.pval;
rdma_hdr = (mca_btl_openib_eager_rdma_header_t*)to_base_frag(frag)->segment.seg_addr.pval;
rdma_hdr->control.type = MCA_BTL_OPENIB_CONTROL_RDMA;
rdma_hdr->rkey = endpoint->eager_rdma_local.reg->mr->rkey;
rdma_hdr->rdma_start.lval = opal_ptr_ptol(endpoint->eager_rdma_local.base.pval);
@ -925,7 +925,7 @@ void mca_btl_openib_endpoint_connect_eager_rdma(
to_base_frag(frag)->type = MCA_BTL_OPENIB_FRAG_EAGER_RDMA;
to_com_frag(frag)->endpoint = endpoint;
frag->ftr = (mca_btl_openib_footer_t*)
((char*)to_base_frag(frag)->segment.base.seg_addr.pval +
((char*)to_base_frag(frag)->segment.seg_addr.pval +
mca_btl_openib_component.eager_limit);
MCA_BTL_OPENIB_RDMA_MAKE_REMOTE(frag->ftr);

4
opal/mca/btl/openib/btl_openib_endpoint.h
Просмотреть файл

@ -546,13 +546,13 @@ static inline int post_send(mca_btl_openib_endpoint_t *ep,
mca_btl_openib_send_frag_t *frag, const bool rdma, int do_signal)
{
mca_btl_openib_module_t *openib_btl = ep->endpoint_btl;
mca_btl_openib_segment_t *seg = &to_base_frag(frag)->segment;
mca_btl_base_segment_t *seg = &to_base_frag(frag)->segment;
struct ibv_sge *sg = &to_com_frag(frag)->sg_entry;
struct ibv_send_wr *sr_desc = &to_out_frag(frag)->sr_desc;
struct ibv_send_wr *bad_wr;
int qp = to_base_frag(frag)->base.order;
sg->length = seg->base.seg_len + sizeof(mca_btl_openib_header_t) +
sg->length = seg->seg_len + sizeof(mca_btl_openib_header_t) +
(rdma ? sizeof(mca_btl_openib_footer_t) : 0) + frag->coalesced_length;
sr_desc->send_flags = ib_send_flags(sg->length, &(ep->qps[qp]), do_signal);

11
opal/mca/btl/openib/btl_openib_failover.c
Просмотреть файл

@ -1,7 +1,8 @@
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
/*
* Copyright (c) 2010-2011 Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2011 NVIDIA Corporation. All rights reserved.
* Copyright (c) 2012 Los Alamos National Security, LLC. All rights
* Copyright (c) 2012-2015 Los Alamos National Security, LLC. All rights
* reserved.
* Copyright (c) 2013 NVIDIA Corporation. All rights reserved.
* Copyright (c) 2014 Research Organization for Information Science
@ -678,12 +679,12 @@ static void mca_btl_openib_endpoint_notify(mca_btl_base_endpoint_t* endpoint, ui
to_base_frag(frag)->base.des_cbdata = NULL;
to_base_frag(frag)->base.des_flags |= MCA_BTL_DES_FLAGS_PRIORITY|MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
to_base_frag(frag)->base.order = mca_btl_openib_component.credits_qp;
to_base_frag(frag)->segment.base.seg_len =
to_base_frag(frag)->segment.seg_len =
sizeof(mca_btl_openib_broken_connection_header_t);
to_com_frag(frag)->endpoint = newep;
frag->hdr->tag = MCA_BTL_TAG_IB;
bc_hdr = (mca_btl_openib_broken_connection_header_t*)to_base_frag(frag)->segment.base.seg_addr.pval;
bc_hdr = (mca_btl_openib_broken_connection_header_t*)to_base_frag(frag)->segment.seg_addr.pval;
bc_hdr->control.type = type;
bc_hdr->lid = endpoint->endpoint_btl->port_info.lid;
bc_hdr->subnet_id = endpoint->endpoint_btl->port_info.subnet_id;
@ -720,10 +721,10 @@ static void dump_local_rdma_frags(mca_btl_openib_endpoint_t * endpoint) {
frag->hdr = (mca_btl_openib_header_t*)(((char*)frag->ftr) -
size + sizeof(mca_btl_openib_footer_t));
to_base_frag(frag)->segment.base.seg_addr.pval =
to_base_frag(frag)->segment.seg_addr.pval =
((unsigned char* )frag->hdr) + sizeof(mca_btl_openib_header_t);
chdr = to_base_frag(frag)->segment.base.seg_addr.pval;
chdr = to_base_frag(frag)->segment.seg_addr.pval;
if ((MCA_BTL_TAG_IB == frag->hdr->tag) &&
(MCA_BTL_OPENIB_CONTROL_CREDITS == chdr->type)) {
opal_output(0, "tag[%d] is credit message", i);

13
opal/mca/btl/openib/btl_openib_frag.c
Просмотреть файл

@ -62,7 +62,6 @@ static void com_constructor(mca_btl_openib_com_frag_t *frag)
if(reg) {
frag->sg_entry.lkey = reg->mr->lkey;
base_frag->segment.key = reg->mr->lkey;
}
frag->n_wqes_inflight = 0;
}
@ -71,7 +70,7 @@ static void out_constructor(mca_btl_openib_out_frag_t *frag)
{
mca_btl_openib_frag_t *base_frag = to_base_frag(frag);
base_frag->base.des_segments = &base_frag->segment.base;
base_frag->base.des_segments = &base_frag->segment;
base_frag->base.des_segment_count = 1;
frag->sr_desc.wr_id = (uint64_t)(uintptr_t)frag;
@ -86,7 +85,7 @@ static void in_constructor(mca_btl_openib_in_frag_t *frag)
{
mca_btl_openib_frag_t *base_frag = to_base_frag(frag);
base_frag->base.des_segments = &base_frag->segment.base;
base_frag->base.des_segments = &base_frag->segment;
base_frag->base.des_segment_count = 1;
}
@ -101,7 +100,7 @@ static void send_constructor(mca_btl_openib_send_frag_t *frag)
(((unsigned char*)base_frag->base.super.ptr) +
sizeof(mca_btl_openib_header_coalesced_t) +
sizeof(mca_btl_openib_control_header_t));
base_frag->segment.base.seg_addr.pval = frag->hdr + 1;
base_frag->segment.seg_addr.pval = frag->hdr + 1;
to_com_frag(frag)->sg_entry.addr = (uint64_t)(uintptr_t)frag->hdr;
frag->coalesced_length = 0;
OBJ_CONSTRUCT(&frag->coalesced_frags, opal_list_t);
@ -114,7 +113,7 @@ static void recv_constructor(mca_btl_openib_recv_frag_t *frag)
base_frag->type = MCA_BTL_OPENIB_FRAG_RECV;
frag->hdr = (mca_btl_openib_header_t*)base_frag->base.super.ptr;
base_frag->segment.base.seg_addr.pval =
base_frag->segment.seg_addr.pval =
((unsigned char* )frag->hdr) + sizeof(mca_btl_openib_header_t);
to_com_frag(frag)->sg_entry.addr = (uint64_t)(uintptr_t)frag->hdr;
@ -129,7 +128,7 @@ static void send_control_constructor(mca_btl_openib_send_control_frag_t *frag)
to_base_frag(frag)->type = MCA_BTL_OPENIB_FRAG_CONTROL;
/* adjusting headers because there is no coalesce header in control messages */
frag->hdr = frag->chdr;
to_base_frag(frag)->segment.base.seg_addr.pval = frag->hdr + 1;
to_base_frag(frag)->segment.seg_addr.pval = frag->hdr + 1;
to_com_frag(frag)->sg_entry.addr = (uint64_t)(uintptr_t)frag->hdr;
}
@ -158,7 +157,7 @@ static void coalesced_constructor(mca_btl_openib_coalesced_frag_t *frag)
base_frag->type = MCA_BTL_OPENIB_FRAG_COALESCED;
base_frag->base.des_segments = &base_frag->segment.base;
base_frag->base.des_segments = &base_frag->segment;
base_frag->base.des_segment_count = 1;
}

8
opal/mca/btl/openib/btl_openib_frag.h
Просмотреть файл

@ -284,16 +284,10 @@ typedef enum mca_btl_openib_frag_type_t mca_btl_openib_frag_type_t;
/**
* IB fragment derived type.
*/
typedef struct mca_btl_openib_segment_t {
mca_btl_base_segment_t base;
uint32_t key;
} mca_btl_openib_segment_t;
/* base openib frag */
typedef struct mca_btl_openib_frag_t {
mca_btl_base_descriptor_t base;
mca_btl_openib_segment_t segment;
mca_btl_base_segment_t segment;
mca_btl_openib_frag_type_t type;
opal_free_list_t* list;
} mca_btl_openib_frag_t;

1
opal/mca/btl/openib/connect/btl_openib_connect_base.c
Просмотреть файл

@ -448,7 +448,6 @@ int opal_btl_openib_connect_base_alloc_cts(mca_btl_base_endpoint_t *endpoint)
/* Copy the lkey where it needs to go */
endpoint->endpoint_cts_frag.super.sg_entry.lkey =
endpoint->endpoint_cts_frag.super.super.segment.key =
endpoint->endpoint_cts_mr->lkey;
endpoint->endpoint_cts_frag.super.sg_entry.length = length;