Add pkey value MCA parameter. if this param is used,

only ports with the actual pkey value will be initiate.

This commit was SVN r14463.

ompi/mca/btl/openib/btl_openib.h

@@ -108,6 +108,7 @@ struct mca_btl_openib_component_t {
     uint32_t ib_cq_size;   /**< Max outstanding CQE on the CQ */  
     uint32_t ib_sg_list_size; /**< Max scatter/gather descriptor entries on the WQ*/ 
     uint32_t ib_pkey_ix; 
+    uint32_t ib_pkey_val;
     uint32_t ib_psn; 
     uint32_t ib_qp_ous_rd_atom; 
     uint32_t ib_mtu; 
@@ -197,6 +198,7 @@ struct mca_btl_openib_module_t {
     mca_btl_openib_port_info_t port_info;  /* contains only the subnet id right now */ 
     mca_btl_openib_hca_t *hca;
     uint8_t port_num;           /**< ID of the PORT */ 
+    uint16_t pkey_index;
     struct ibv_cq *ib_cq[2];
     struct ibv_port_attr ib_port_attr; 
     uint16_t lid;                      /**< lid that is actually used (for LMC) */

ompi/mca/btl/openib/btl_openib_component.c

@@ -66,7 +66,8 @@ static void btl_openib_control(struct mca_btl_base_module_t* btl,
                                mca_btl_base_descriptor_t* descriptor,
                                void* cbdata);
 static int init_one_port(opal_list_t *btl_list, mca_btl_openib_hca_t *hca,
-                         uint8_t port_num, struct ibv_port_attr *ib_port_attr);
+                         uint8_t port_num, uint16_t pkey_index,
+                         struct ibv_port_attr *ib_port_attr);
 static int init_one_hca(opal_list_t *btl_list, struct ibv_device* ib_dev);
 static mca_btl_base_module_t **btl_openib_component_init(
     int *num_btl_modules, bool enable_progress_threads,
@@ -297,7 +298,8 @@ static int openib_dereg_mr(void *reg_data, mca_mpool_base_registration_t *reg)
 }
 
 static int init_one_port(opal_list_t *btl_list, mca_btl_openib_hca_t *hca,
-                         uint8_t port_num, struct ibv_port_attr *ib_port_attr)
+                         uint8_t port_num, uint16_t pkey_index,
+                         struct ibv_port_attr *ib_port_attr)
 {
     uint16_t lid, i, lmc;
     mca_btl_openib_module_t *openib_btl;
@@ -339,6 +341,7 @@ static int init_one_port(opal_list_t *btl_list, mca_btl_openib_hca_t *hca,
             ib_selected->btl_module = (mca_btl_base_module_t*) openib_btl;
             openib_btl->hca = hca;
             openib_btl->port_num = (uint8_t) port_num;
+            openib_btl->pkey_index = pkey_index;
             openib_btl->lid = lid;
             openib_btl->src_path_bits = lid - ib_port_attr->lid;
             /* store the subnet for multi-nic support */
@@ -546,8 +549,22 @@ static int init_one_hca(opal_list_t *btl_list, struct ibv_device* ib_dev)
         }
 
         if(IBV_PORT_ACTIVE == ib_port_attr.state){
-            ret = init_one_port(btl_list, hca, i, &ib_port_attr);
 
+            if (0 == mca_btl_openib_component.ib_pkey_val) {
+                ret = init_one_port(btl_list, hca, i, mca_btl_openib_component.ib_pkey_ix,
+                                    &ib_port_attr);
+            }
+            else {
+                uint16_t pkey,j;
+                for (j=0; j < hca->ib_dev_attr.max_pkeys; j++) {
+                    ibv_query_pkey(hca->ib_dev_context, i, j, &pkey);
+                    pkey=ntohs(pkey);
+                    if(pkey == mca_btl_openib_component.ib_pkey_val){
+                        ret = init_one_port(btl_list, hca, i, j, &ib_port_attr);
+                        break;
+                    }
+                }
+            }
             if (OMPI_SUCCESS != ret) {
                 /* Out of bounds error indicates that we hit max btl number 
                  * don't propagate the error to the caller */

ompi/mca/btl/openib/btl_openib_endpoint.c

@@ -969,10 +969,10 @@ int mca_btl_openib_endpoint_create_qp(
     
     {
         qp_attr->qp_state = IBV_QPS_INIT; 
-        qp_attr->pkey_index = mca_btl_openib_component.ib_pkey_ix; 
+        qp_attr->pkey_index = openib_btl->pkey_index; /*mca_btl_openib_component.ib_pkey_ix; */
         qp_attr->port_num = openib_btl->port_num; 
         qp_attr->qp_access_flags = IBV_ACCESS_REMOTE_WRITE | IBV_ACCESS_REMOTE_READ; 
-        
+
         if(ibv_modify_qp((*qp), qp_attr, 
                          IBV_QP_STATE | 
                          IBV_QP_PKEY_INDEX | 

ompi/mca/btl/openib/btl_openib_mca.c

@@ -194,6 +194,14 @@ int btl_openib_register_mca_params(void)
                   0, &ival, REGINT_GE_ZERO));
     mca_btl_openib_component.ib_pkey_ix = (uint32_t) ival;
 
+    CHECK(reg_int("ib_pkey_val", "InfiniBand pkey value"
+                  "(must be > 0 and < 0xffff)",
+                  0, &ival, REGINT_GE_ZERO));
+    if (ival > 0xffff) {
+        ret = OMPI_ERR_BAD_PARAM;
+    }
+    mca_btl_openib_component.ib_pkey_val = (uint32_t) ival;
+
     CHECK(reg_int("ib_psn", "InfiniBand packet sequence starting number "
                   "(must be >= 0)", 
                   0, &ival, REGINT_GE_ZERO));