ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

More rdma work..

Просмотр исходного кода 
This commit was SVN r6036.
Этот коммит содержится в:
Galen Shipman 2005-06-10 20:40:46 +00:00
родитель 215ba87116
Коммит b3c8d47f62
6 изменённых файлов: 60 добавлений и 30 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

50
src/mca/bmi/ib/bmi_ib.c
Просмотреть файл

@ -169,14 +169,26 @@ int mca_bmi_ib_free(
mca_bmi_base_descriptor_t* des)
{
mca_bmi_ib_frag_t* frag = (mca_bmi_ib_frag_t*)des;
mca_bmi_ib_module_t * ib_bmi = (mca_bmi_ib_module_t*) bmi;
if(frag->size == 0) {
MCA_BMI_IB_FRAG_RETURN_FRAG(bmi, frag);
/* we also need to unregister the associated memory */
frag->ret = VAPI_deregister_mr(
ib_bmi->nic,
frag->mem_hndl
);
if(frag->ret!=VAPI_OK){
ompi_output(0, "%s:error deregistering memory region", __func__);
return OMPI_ERROR;
}
} else if(frag->size == mca_bmi_ib_component.max_send_size){
MCA_BMI_IB_FRAG_RETURN_MAX(bmi, frag);
} else if(frag->size == mca_bmi_ib_component.eager_limit){
MCA_BMI_IB_FRAG_RETURN_EAGER(bmi, frag);
}
return frag->rc;
}
@ -245,6 +257,14 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
frag->segment.seg_len = max_data + reserve;
*size = max_data;
frag->sg_entry.len = max_data;
frag->segment.seg_key.key32[0] = (uint32_t) frag->sg_entry.lkey;
frag->base.des_src = &frag->segment;
frag->base.des_src_cnt = 1;
frag->base.des_dst = NULL;
frag->base.des_dst_cnt = 0;
return &frag->base;
} else {
VAPI_mrw_t mr_in, mr_out;
@ -292,7 +312,7 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
mem_hndl.l_key = mr_out.l_key;
mem_hndl.r_key = mr_out.r_key;
frag->mem_hndl = mem_hndl.hndl;
frag->sg_entry.len = max_data;
frag->sg_entry.lkey = mem_hndl.l_key;
@ -300,6 +320,12 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_src(
frag->segment.seg_key.key32[0] = (uint32_t) mem_hndl.l_key;
frag->base.des_src = &frag->segment;
frag->base.des_src_cnt = 1;
frag->base.des_dst = NULL;
frag->base.des_dst_cnt = 0;
return &frag->base;
}
@ -323,10 +349,7 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_dst(
{
mca_bmi_ib_module_t* ib_bmi;
mca_bmi_ib_frag_t* frag;
struct iovec iov;
int32_t iov_count = 1;
size_t max_data = *size;
int32_t free_after;
int rc;
ib_bmi = (mca_bmi_ib_module_t*) bmi;
@ -351,15 +374,12 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_dst(
return NULL;
}
iov.iov_len = max_data;
iov.iov_base = NULL;
ompi_convertor_pack(convertor, &iov, &iov_count, &max_data, &free_after);
frag->segment.seg_len = max_data;
frag->segment.seg_addr.pval = iov.iov_base;
frag->segment.seg_addr.pval = convertor->pBaseBuf + convertor->bConverted;
mr_in.size = max_data;
mr_in.start = (VAPI_virt_addr_t) (MT_virt_addr_t) iov.iov_base;
mr_in.start = (VAPI_virt_addr_t) (MT_virt_addr_t) frag->segment.seg_addr.pval;
ret = VAPI_register_mr(
ib_bmi->nic,
@ -379,10 +399,14 @@ mca_bmi_base_descriptor_t* mca_bmi_ib_prepare_dst(
frag->mem_hndl = mem_hndl.hndl;
frag->sg_entry.len = max_data;
frag->sg_entry.lkey = mem_hndl.l_key;
frag->sg_entry.addr = (VAPI_virt_addr_t) (MT_virt_addr_t) iov.iov_base;
frag->sg_entry.addr = (VAPI_virt_addr_t) (MT_virt_addr_t) frag->segment.seg_addr.pval;
frag->segment.seg_key.key32[0] = (uint32_t) mem_hndl.l_key;
frag->base.des_dst = &frag->segment;
frag->base.des_dst_cnt = 1;
frag->base.des_src = NULL;
frag->base.des_src_cnt = 0;
return &frag->base;
}
@ -455,9 +479,11 @@ int mca_bmi_ib_put( mca_bmi_base_module_t* bmi,
{
mca_bmi_ib_module_t* ib_bmi = (mca_bmi_ib_module_t*) bmi;
mca_bmi_ib_frag_t* frag = (mca_bmi_ib_frag_t*) descriptor;
frag->endpoint = endpoint;
frag->sr_desc.opcode = VAPI_RDMA_WRITE;
frag->sr_desc.remote_qp = endpoint->rem_qp_num;
frag->sr_desc.remote_addr = (VAPI_virt_addr_t) (MT_virt_addr_t) frag->base.des_dst->seg_addr.pval;
frag->sr_desc.r_key = frag->base.des_dst->seg_key.key32[0];
frag->ret = VAPI_post_sr(ib_bmi->nic,
endpoint->lcl_qp_hndl,

15
src/mca/bmi/ib/bmi_ib_component.c
Просмотреть файл

@ -187,7 +187,7 @@ int mca_bmi_ib_component_open(void)
mca_bmi_ib_param_register_int("ib_src_path_bits",
0);
mca_bmi_ib_module.super.bmi_flags = MCA_BMI_FLAGS_RDMA;
@ -505,16 +505,17 @@ int mca_bmi_ib_component_progress()
ompi_output(0, "Got error : %s, Vendor code : %d Frag : %p",
VAPI_wc_status_sym(comp.status),
comp.vendor_err_syndrome, comp.id);
frag->rc = OMPI_ERROR;
}
/* Handle n/w completions */
switch(comp.opcode) {
case VAPI_CQE_SQ_RDMA_WRITE:
case VAPI_CQE_SQ_SEND_DATA :
/* Process a completed send */
frag = (mca_bmi_ib_frag_t*) comp.id;
frag->base.des_cbfunc(&ib_bmi->super,(mca_bmi_base_endpoint_t*) &frag->endpoint, &frag->base, frag->rc);
frag->base.des_cbfunc(&ib_bmi->super, frag->endpoint, &frag->base, frag->rc);
count++;
break;
@ -522,7 +523,7 @@ int mca_bmi_ib_component_progress()
DEBUG_OUT(0, "%s:%d ib recv under redesign\n", __FILE__, __LINE__);
frag = (mca_bmi_ib_frag_t*) comp.id;
frag->segment.seg_len = comp.byte_len-sizeof(mca_bmi_ib_header_t);
frag->segment.seg_len = comp.byte_len-((unsigned char*) frag->segment.seg_len - (unsigned char*) frag->hdr);
/* advance the segment address past the header and subtract from the length..*/
ib_bmi->ib_reg[frag->hdr->tag].cbfunc(&ib_bmi->super, frag->hdr->tag, &frag->base, ib_bmi->ib_reg[frag->hdr->tag].cbdata);
@ -537,12 +538,6 @@ int mca_bmi_ib_component_progress()
count++;
break;
case VAPI_CQE_SQ_RDMA_WRITE:
ompi_output(0, "%s:%d RDMA not implemented\n", __FILE__,__LINE__);
count++;
break;
default:
ompi_output(0, "Errorneous network completion");
break;

18
src/mca/bmi/ib/bmi_ib_frag.c
Просмотреть файл

@ -27,12 +27,8 @@ static void mca_bmi_ib_frag_common_constructor( mca_bmi_ib_frag_t* frag)
#endif
frag->segment.seg_len = frag->size;
frag->base.des_src = &frag->segment;
frag->base.des_src_cnt = 1;
frag->base.des_dst = NULL;
frag->base.des_dst_cnt = 0;
frag->base.des_flags = 0;
frag->base.des_src->seg_key.key32[0] = (uint64_t) mem_hndl->l_key;
frag->segment.seg_key.key32[0] = (uint64_t) mem_hndl->l_key;
frag->sg_entry.lkey = mem_hndl->l_key;
frag->sg_entry.addr = (VAPI_virt_addr_t) (MT_virt_addr_t) frag->hdr;
}
@ -42,7 +38,11 @@ static void mca_bmi_ib_send_frag_common_constructor(mca_bmi_ib_frag_t* frag)
{
mca_bmi_ib_frag_common_constructor(frag);
frag->base.des_src = &frag->segment;
frag->base.des_src_cnt = 1;
frag->base.des_dst = NULL;
frag->base.des_dst_cnt = 0;
frag->sr_desc.comp_type = VAPI_SIGNALED;
frag->sr_desc.opcode = VAPI_SEND;
frag->sr_desc.remote_qkey = 0;
@ -56,7 +56,11 @@ static void mca_bmi_ib_recv_frag_common_constructor(mca_bmi_ib_frag_t* frag)
{
mca_bmi_ib_frag_common_constructor(frag);
frag->base.des_dst = &frag->segment;
frag->base.des_dst_cnt = 1;
frag->base.des_src = NULL;
frag->base.des_src_cnt = 0;
frag->rr_desc.comp_type = VAPI_SIGNALED;
frag->rr_desc.opcode = VAPI_RECEIVE;
frag->rr_desc.sg_lst_len = 1;

4
src/mca/pml/ob1/pml_ob1_component.c
Просмотреть файл

@ -114,6 +114,10 @@ int mca_pml_ob1_component_open(void)
mca_pml_ob1_param_register_int("send_pipeline_depth", 3);
mca_pml_ob1.recv_pipeline_depth =
mca_pml_ob1_param_register_int("recv_pipeline_depth", 3);
mca_pml_ob1.rdma_threshold =
mca_pml_ob1_param_register_int("rdma_threshold", 512*1024);
mca_pml_ob1.rdma_offset =
mca_pml_ob1_param_register_int("rdma_offset", 256*1024);
return mca_bmi_base_open();
}

2
src/mca/pml/ob1/pml_ob1_recvfrag.c
Просмотреть файл

@ -60,7 +60,7 @@ void mca_pml_ob1_recv_frag_callback(
mca_bmi_base_descriptor_t* des,
void* cbdata)
{
mca_bmi_base_segment_t* segments = des->des_src;
mca_bmi_base_segment_t* segments = des->des_dst;
mca_pml_ob1_hdr_t* hdr = (mca_pml_ob1_hdr_t*)segments->seg_addr.pval;
if(segments->seg_len < sizeof(mca_pml_ob1_common_hdr_t)) {
return;

1
src/mca/pml/ob1/pml_ob1_sendreq.c
Просмотреть файл

@ -588,6 +588,7 @@ void mca_pml_ob1_send_request_put(
}
frag->rdma_state = MCA_PML_OB1_RDMA_PUT;
frag->rdma_length = size;
frag->rdma_req = sendreq;
des->des_dst = frag->rdma_segs;
des->des_dst_cnt = hdr->hdr_seg_cnt;