diff --git a/ompi/mca/btl/mvapi/btl_mvapi.c b/ompi/mca/btl/mvapi/btl_mvapi.c index e6e3f711f2..df7c4368cd 100644 --- a/ompi/mca/btl/mvapi/btl_mvapi.c +++ b/ompi/mca/btl/mvapi/btl_mvapi.c @@ -793,7 +793,7 @@ int mca_btl_mvapi_module_init(mca_btl_mvapi_module_t *mvapi_btl) return OMPI_ERROR; } - ret = VAPI_create_cq(mvapi_btl->nic, mvapi_btl->ib_cq_size, + ret = VAPI_create_cq(mvapi_btl->nic, mca_btl_mvapi_component.ib_cq_size, &mvapi_btl->cq_hndl_low, &cqe_cnt); @@ -802,7 +802,7 @@ int mca_btl_mvapi_module_init(mca_btl_mvapi_module_t *mvapi_btl) return OMPI_ERROR; } - ret = VAPI_create_cq(mvapi_btl->nic, mvapi_btl->ib_cq_size, + ret = VAPI_create_cq(mvapi_btl->nic, mca_btl_mvapi_component.ib_cq_size, &mvapi_btl->cq_hndl_high, &cqe_cnt); diff --git a/ompi/mca/btl/mvapi/btl_mvapi.h b/ompi/mca/btl/mvapi/btl_mvapi.h index a7ac09981d..9f7f40d558 100644 --- a/ompi/mca/btl/mvapi/btl_mvapi.h +++ b/ompi/mca/btl/mvapi/btl_mvapi.h @@ -96,6 +96,24 @@ struct mca_btl_mvapi_component_t { uint32_t reg_mru_len; + uint32_t ib_cq_size; /**< Max outstanding CQE on the CQ */ + uint32_t ib_wq_size; /**< Max outstanding WR on the WQ */ + uint32_t ib_sg_list_size; /**< Max scatter/gather descriptor entries on the WQ*/ + uint32_t ib_pkey_ix; + uint32_t ib_psn; + uint32_t ib_qp_ous_rd_atom; + uint32_t ib_mtu; + uint32_t ib_min_rnr_timer; + uint32_t ib_timeout; + uint32_t ib_retry_count; + uint32_t ib_rnr_retry; + uint32_t ib_max_rdma_dst_ops; + uint32_t ib_service_level; + uint32_t ib_static_rate; + uint32_t ib_src_path_bits; + + + }; typedef struct mca_btl_mvapi_component_t mca_btl_mvapi_component_t; extern mca_btl_mvapi_component_t mca_btl_mvapi_component; @@ -148,24 +166,7 @@ struct mca_btl_mvapi_module_t { /**< an array to allow posting of rr in one swoop */ size_t ib_inline_max; /**< max size of inline send*/ - size_t ib_pin_min; /**< min size to pin memory*/ - uint32_t ib_cq_size; /**< Max outstanding CQE on the CQ */ - uint32_t ib_wq_size; /**< Max outstanding WR on the WQ */ - uint32_t ib_sg_list_size; /**< Max scatter/gather descriptor entries on the WQ*/ - uint32_t ib_pkey_ix; - uint32_t ib_psn; - uint32_t ib_qp_ous_rd_atom; - uint32_t ib_mtu; - uint32_t ib_min_rnr_timer; - uint32_t ib_timeout; - uint32_t ib_retry_count; - uint32_t ib_rnr_retry; - uint32_t ib_max_rdma_dst_ops; - uint32_t ib_service_level; - uint32_t ib_static_rate; - uint32_t ib_src_path_bits; - }; typedef struct mca_btl_mvapi_module_t mca_btl_mvapi_module_t; diff --git a/ompi/mca/btl/mvapi/btl_mvapi_component.c b/ompi/mca/btl/mvapi/btl_mvapi_component.c index 7fb44e9aac..4322d75bcc 100644 --- a/ompi/mca/btl/mvapi/btl_mvapi_component.c +++ b/ompi/mca/btl/mvapi/btl_mvapi_component.c @@ -129,6 +129,53 @@ int mca_btl_mvapi_component_open(void) mca_btl_mvapi_param_register_int("rr_buf_min", 8); mca_btl_mvapi_component.reg_mru_len = mca_btl_mvapi_param_register_int("reg_mru_len", 16); + mca_btl_mvapi_component.ib_cq_size = + mca_btl_mvapi_param_register_int("ib_cq_size", + 40000); + mca_btl_mvapi_component.ib_wq_size = + mca_btl_mvapi_param_register_int("ib_wq_size", + 10000); + mca_btl_mvapi_component.ib_sg_list_size = + mca_btl_mvapi_param_register_int("ib_sg_list_size", + 1); + mca_btl_mvapi_component.ib_pkey_ix = + mca_btl_mvapi_param_register_int("ib_pkey_ix", + 0); + mca_btl_mvapi_component.ib_psn = + mca_btl_mvapi_param_register_int("ib_psn", + 0); + mca_btl_mvapi_component.ib_qp_ous_rd_atom = + mca_btl_mvapi_param_register_int("ib_qp_ous_rd_atom", + 1); + mca_btl_mvapi_component.ib_mtu = + mca_btl_mvapi_param_register_int("ib_mtu", + MTU1024); + mca_btl_mvapi_component.ib_min_rnr_timer = + mca_btl_mvapi_param_register_int("ib_min_rnr_timer", + 5); + mca_btl_mvapi_component.ib_timeout = + mca_btl_mvapi_param_register_int("ib_timeout", + 10); + mca_btl_mvapi_component.ib_retry_count = + mca_btl_mvapi_param_register_int("ib_retry_count", + 7); + mca_btl_mvapi_component.ib_rnr_retry = + mca_btl_mvapi_param_register_int("ib_rnr_retry", + 7); + mca_btl_mvapi_component.ib_max_rdma_dst_ops = + mca_btl_mvapi_param_register_int("ib_max_rdma_dst_ops", + 16); + + mca_btl_mvapi_component.ib_service_level = + mca_btl_mvapi_param_register_int("ib_service_level", + 0); + mca_btl_mvapi_component.ib_static_rate = + mca_btl_mvapi_param_register_int("ib_static_rate", + 0); + mca_btl_mvapi_component.ib_src_path_bits = + mca_btl_mvapi_param_register_int("ib_src_path_bits", + 0); + mca_btl_mvapi_module.super.btl_exclusivity = mca_btl_mvapi_param_register_int ("exclusivity", 0); @@ -144,54 +191,6 @@ int mca_btl_mvapi_component_open(void) mca_btl_mvapi_param_register_int ("max_send_size", (128*1024)) - sizeof(mca_btl_mvapi_header_t); - mca_btl_mvapi_module.ib_pin_min = - mca_btl_mvapi_param_register_int("ib_pin_min", 128*1024); - mca_btl_mvapi_module.ib_cq_size = - mca_btl_mvapi_param_register_int("ib_cq_size", - 40000); - mca_btl_mvapi_module.ib_wq_size = - mca_btl_mvapi_param_register_int("ib_wq_size", - 10000); - mca_btl_mvapi_module.ib_sg_list_size = - mca_btl_mvapi_param_register_int("ib_sg_list_size", - 1); - mca_btl_mvapi_module.ib_pkey_ix = - mca_btl_mvapi_param_register_int("ib_pkey_ix", - 0); - mca_btl_mvapi_module.ib_psn = - mca_btl_mvapi_param_register_int("ib_psn", - 0); - mca_btl_mvapi_module.ib_qp_ous_rd_atom = - mca_btl_mvapi_param_register_int("ib_qp_ous_rd_atom", - 1); - mca_btl_mvapi_module.ib_mtu = - mca_btl_mvapi_param_register_int("ib_mtu", - MTU1024); - mca_btl_mvapi_module.ib_min_rnr_timer = - mca_btl_mvapi_param_register_int("ib_min_rnr_timer", - 5); - mca_btl_mvapi_module.ib_timeout = - mca_btl_mvapi_param_register_int("ib_timeout", - 10); - mca_btl_mvapi_module.ib_retry_count = - mca_btl_mvapi_param_register_int("ib_retry_count", - 7); - mca_btl_mvapi_module.ib_rnr_retry = - mca_btl_mvapi_param_register_int("ib_rnr_retry", - 7); - mca_btl_mvapi_module.ib_max_rdma_dst_ops = - mca_btl_mvapi_param_register_int("ib_max_rdma_dst_ops", - 16); - - mca_btl_mvapi_module.ib_service_level = - mca_btl_mvapi_param_register_int("ib_service_level", - 0); - mca_btl_mvapi_module.ib_static_rate = - mca_btl_mvapi_param_register_int("ib_static_rate", - 0); - mca_btl_mvapi_module.ib_src_path_bits = - mca_btl_mvapi_param_register_int("ib_src_path_bits", - 0); mca_btl_mvapi_module.super.btl_min_rdma_size = mca_btl_mvapi_param_register_int("min_rdma_size", 1024*1024); diff --git a/ompi/mca/btl/mvapi/btl_mvapi_endpoint.c b/ompi/mca/btl/mvapi/btl_mvapi_endpoint.c index e886cf7969..34af110f93 100644 --- a/ompi/mca/btl/mvapi/btl_mvapi_endpoint.c +++ b/ompi/mca/btl/mvapi/btl_mvapi_endpoint.c @@ -651,10 +651,10 @@ int mca_btl_mvapi_endpoint_create_qp( switch(transport_type) { case VAPI_TS_RC: /* Set up RC qp parameters */ - qp_init_attr.cap.max_oust_wr_rq = mvapi_btl->ib_wq_size; - qp_init_attr.cap.max_oust_wr_sq = mvapi_btl->ib_wq_size; - qp_init_attr.cap.max_sg_size_rq = mvapi_btl->ib_sg_list_size; - qp_init_attr.cap.max_sg_size_sq = mvapi_btl->ib_sg_list_size; + qp_init_attr.cap.max_oust_wr_rq = mca_btl_mvapi_component.ib_wq_size; + qp_init_attr.cap.max_oust_wr_sq = mca_btl_mvapi_component.ib_wq_size; + qp_init_attr.cap.max_sg_size_rq = mca_btl_mvapi_component.ib_sg_list_size; + qp_init_attr.cap.max_sg_size_sq = mca_btl_mvapi_component.ib_sg_list_size; qp_init_attr.pd_hndl = ptag; /* We don't have Reliable Datagram Handle right now */ qp_init_attr.rdd_hndl = 0; @@ -712,7 +712,7 @@ int mca_btl_mvapi_endpoint_qp_init_query( QP_ATTR_MASK_CLR_ALL(qp_attr_mask); qp_attr.qp_state = VAPI_INIT; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE); - qp_attr.pkey_ix = mvapi_btl->ib_pkey_ix; + qp_attr.pkey_ix = mca_btl_mvapi_component.ib_pkey_ix; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PKEY_IX); qp_attr.port = port_id; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PORT); @@ -733,21 +733,21 @@ int mca_btl_mvapi_endpoint_qp_init_query( QP_ATTR_MASK_CLR_ALL(qp_attr_mask); qp_attr.qp_state = VAPI_RTR; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE); - qp_attr.qp_ous_rd_atom = mvapi_btl->ib_qp_ous_rd_atom; + qp_attr.qp_ous_rd_atom = mca_btl_mvapi_component.ib_qp_ous_rd_atom; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_OUS_RD_ATOM); - qp_attr.path_mtu = mvapi_btl->ib_mtu; + qp_attr.path_mtu = mca_btl_mvapi_component.ib_mtu; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PATH_MTU); - qp_attr.rq_psn = mvapi_btl->ib_psn; + qp_attr.rq_psn = mca_btl_mvapi_component.ib_psn; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_RQ_PSN); - qp_attr.pkey_ix = mvapi_btl->ib_pkey_ix; + qp_attr.pkey_ix = mca_btl_mvapi_component.ib_pkey_ix; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_PKEY_IX); - qp_attr.min_rnr_timer = mvapi_btl->ib_min_rnr_timer; + qp_attr.min_rnr_timer = mca_btl_mvapi_component.ib_min_rnr_timer; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_MIN_RNR_TIMER); - qp_attr.av.sl = mvapi_btl->ib_service_level; + qp_attr.av.sl = mca_btl_mvapi_component.ib_service_level; qp_attr.av.grh_flag = FALSE; - qp_attr.av.static_rate = mvapi_btl->ib_static_rate; - qp_attr.av.src_path_bits = mvapi_btl->ib_src_path_bits; + qp_attr.av.static_rate = mca_btl_mvapi_component.ib_static_rate; + qp_attr.av.src_path_bits = mca_btl_mvapi_component.ib_src_path_bits; qp_attr.dest_qp_num = remote_qp_num; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_DEST_QP_NUM); @@ -768,15 +768,15 @@ int mca_btl_mvapi_endpoint_qp_init_query( QP_ATTR_MASK_CLR_ALL(qp_attr_mask); qp_attr.qp_state = VAPI_RTS; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_QP_STATE); - qp_attr.sq_psn = mvapi_btl->ib_psn; + qp_attr.sq_psn = mca_btl_mvapi_component.ib_psn; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_SQ_PSN); - qp_attr.timeout = mvapi_btl->ib_timeout; + qp_attr.timeout = mca_btl_mvapi_component.ib_timeout; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_TIMEOUT); - qp_attr.retry_count = mvapi_btl->ib_retry_count; + qp_attr.retry_count = mca_btl_mvapi_component.ib_retry_count; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_RETRY_COUNT); - qp_attr.rnr_retry = mvapi_btl->ib_rnr_retry; + qp_attr.rnr_retry = mca_btl_mvapi_component.ib_rnr_retry; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_RNR_RETRY); - qp_attr.ous_dst_rd_atom = mvapi_btl->ib_max_rdma_dst_ops; + qp_attr.ous_dst_rd_atom = mca_btl_mvapi_component.ib_max_rdma_dst_ops; QP_ATTR_MASK_SET(qp_attr_mask, QP_ATTR_OUS_DST_RD_ATOM); ret = VAPI_modify_qp(nic, qp_hndl,