diff --git a/ompi/mca/btl/openib/btl_openib.h b/ompi/mca/btl/openib/btl_openib.h index a81310d830..d8af5ec476 100644 --- a/ompi/mca/btl/openib/btl_openib.h +++ b/ompi/mca/btl/openib/btl_openib.h @@ -108,6 +108,7 @@ struct mca_btl_openib_component_t { uint32_t ib_cq_size; /**< Max outstanding CQE on the CQ */ uint32_t ib_sg_list_size; /**< Max scatter/gather descriptor entries on the WQ*/ uint32_t ib_pkey_ix; + uint32_t ib_pkey_val; uint32_t ib_psn; uint32_t ib_qp_ous_rd_atom; uint32_t ib_mtu; @@ -197,6 +198,7 @@ struct mca_btl_openib_module_t { mca_btl_openib_port_info_t port_info; /* contains only the subnet id right now */ mca_btl_openib_hca_t *hca; uint8_t port_num; /**< ID of the PORT */ + uint16_t pkey_index; struct ibv_cq *ib_cq[2]; struct ibv_port_attr ib_port_attr; uint16_t lid; /**< lid that is actually used (for LMC) */ diff --git a/ompi/mca/btl/openib/btl_openib_component.c b/ompi/mca/btl/openib/btl_openib_component.c index eb68619eec..934c345bbb 100644 --- a/ompi/mca/btl/openib/btl_openib_component.c +++ b/ompi/mca/btl/openib/btl_openib_component.c @@ -66,7 +66,8 @@ static void btl_openib_control(struct mca_btl_base_module_t* btl, mca_btl_base_descriptor_t* descriptor, void* cbdata); static int init_one_port(opal_list_t *btl_list, mca_btl_openib_hca_t *hca, - uint8_t port_num, struct ibv_port_attr *ib_port_attr); + uint8_t port_num, uint16_t pkey_index, + struct ibv_port_attr *ib_port_attr); static int init_one_hca(opal_list_t *btl_list, struct ibv_device* ib_dev); static mca_btl_base_module_t **btl_openib_component_init( int *num_btl_modules, bool enable_progress_threads, @@ -297,7 +298,8 @@ static int openib_dereg_mr(void *reg_data, mca_mpool_base_registration_t *reg) } static int init_one_port(opal_list_t *btl_list, mca_btl_openib_hca_t *hca, - uint8_t port_num, struct ibv_port_attr *ib_port_attr) + uint8_t port_num, uint16_t pkey_index, + struct ibv_port_attr *ib_port_attr) { uint16_t lid, i, lmc; mca_btl_openib_module_t *openib_btl; @@ -339,6 +341,7 @@ static int init_one_port(opal_list_t *btl_list, mca_btl_openib_hca_t *hca, ib_selected->btl_module = (mca_btl_base_module_t*) openib_btl; openib_btl->hca = hca; openib_btl->port_num = (uint8_t) port_num; + openib_btl->pkey_index = pkey_index; openib_btl->lid = lid; openib_btl->src_path_bits = lid - ib_port_attr->lid; /* store the subnet for multi-nic support */ @@ -546,8 +549,22 @@ static int init_one_hca(opal_list_t *btl_list, struct ibv_device* ib_dev) } if(IBV_PORT_ACTIVE == ib_port_attr.state){ - ret = init_one_port(btl_list, hca, i, &ib_port_attr); + if (0 == mca_btl_openib_component.ib_pkey_val) { + ret = init_one_port(btl_list, hca, i, mca_btl_openib_component.ib_pkey_ix, + &ib_port_attr); + } + else { + uint16_t pkey,j; + for (j=0; j < hca->ib_dev_attr.max_pkeys; j++) { + ibv_query_pkey(hca->ib_dev_context, i, j, &pkey); + pkey=ntohs(pkey); + if(pkey == mca_btl_openib_component.ib_pkey_val){ + ret = init_one_port(btl_list, hca, i, j, &ib_port_attr); + break; + } + } + } if (OMPI_SUCCESS != ret) { /* Out of bounds error indicates that we hit max btl number * don't propagate the error to the caller */ diff --git a/ompi/mca/btl/openib/btl_openib_endpoint.c b/ompi/mca/btl/openib/btl_openib_endpoint.c index aa8ec06414..6c93639279 100644 --- a/ompi/mca/btl/openib/btl_openib_endpoint.c +++ b/ompi/mca/btl/openib/btl_openib_endpoint.c @@ -969,10 +969,10 @@ int mca_btl_openib_endpoint_create_qp( { qp_attr->qp_state = IBV_QPS_INIT; - qp_attr->pkey_index = mca_btl_openib_component.ib_pkey_ix; + qp_attr->pkey_index = openib_btl->pkey_index; /*mca_btl_openib_component.ib_pkey_ix; */ qp_attr->port_num = openib_btl->port_num; qp_attr->qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ; - + if(ibv_modify_qp((*qp), qp_attr, IBV_QP_STATE | IBV_QP_PKEY_INDEX | diff --git a/ompi/mca/btl/openib/btl_openib_mca.c b/ompi/mca/btl/openib/btl_openib_mca.c index 71c9581d33..932342a021 100644 --- a/ompi/mca/btl/openib/btl_openib_mca.c +++ b/ompi/mca/btl/openib/btl_openib_mca.c @@ -194,6 +194,14 @@ int btl_openib_register_mca_params(void) 0, &ival, REGINT_GE_ZERO)); mca_btl_openib_component.ib_pkey_ix = (uint32_t) ival; + CHECK(reg_int("ib_pkey_val", "InfiniBand pkey value" + "(must be > 0 and < 0xffff)", + 0, &ival, REGINT_GE_ZERO)); + if (ival > 0xffff) { + ret = OMPI_ERR_BAD_PARAM; + } + mca_btl_openib_component.ib_pkey_val = (uint32_t) ival; + CHECK(reg_int("ib_psn", "InfiniBand packet sequence starting number " "(must be >= 0)", 0, &ival, REGINT_GE_ZERO));