Fixing openib partition support.

This commit was SVN r19705.

ompi/mca/btl/openib/btl_openib.h

@@ -54,6 +54,7 @@ BEGIN_C_DECLS
 
 #define MCA_BTL_IB_LEAVE_PINNED 1
 #define IB_DEFAULT_GID_PREFIX 0xfe80000000000000ll
+#define MCA_BTL_IB_PKEY_MASK 0x7fff
 
 
 /*--------------------------------------------------------------------*/

ompi/mca/btl/openib/btl_openib_component.c

@@ -1726,9 +1726,13 @@ static int init_one_device(opal_list_t *btl_list, struct ibv_device* ib_dev)
                                     &ib_port_attr);
             } else {
                 uint16_t pkey,j;
-                for (j=0; j < device->ib_dev_attr.max_pkeys; j++) {
-                    ibv_query_pkey(device->ib_dev_context, i, j, &pkey);
-                    pkey=ntohs(pkey);
+                for (j = 0; j < device->ib_dev_attr.max_pkeys; j++) {
+                    if(ibv_query_pkey(device->ib_dev_context, i, j, &pkey)){
+                        BTL_ERROR(("error getting pkey for index %d, device %s "
+                                    "port number %d errno says %s",
+                                    j, ibv_get_device_name(device->ib_dev), i, strerror(errno)));
+                    }
+                    pkey = ntohs(pkey) & MCA_BTL_IB_PKEY_MASK;
                     if(pkey == mca_btl_openib_component.ib_pkey_val){
                         ret = init_one_port(btl_list, device, i, j, &ib_port_attr);
                         break;

ompi/mca/btl/openib/btl_openib_ini.c

@@ -10,6 +10,7 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2006-2008 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2008      Mellanox Technologies. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -91,8 +92,6 @@ static int parse_line(parsed_section_values_t *item);
 static void reset_section(bool had_previous_value, parsed_section_values_t *s);
 static void reset_values(ompi_btl_openib_ini_values_t *v);
 static int save_section(parsed_section_values_t *s);
-static int intify(char *string);
-static int intify_list(char *str, uint32_t **values, int *len);
 static inline void show_help(const char *topic);
 
 
@@ -365,14 +364,14 @@ static int parse_line(parsed_section_values_t *sv)
        all whitespace at the beginning and ending of the value. */
 
     if (0 == strcasecmp(key_buffer, "vendor_id")) {
-        if (OMPI_SUCCESS != (ret = intify_list(value, &sv->vendor_ids,
+        if (OMPI_SUCCESS != (ret = ompi_btl_openib_ini_intify_list(value, &sv->vendor_ids,
                                                &sv->vendor_ids_len))) {
             return ret;
         }
     }
 
     else if (0 == strcasecmp(key_buffer, "vendor_part_id")) {
-        if (OMPI_SUCCESS != (ret = intify_list(value, &sv->vendor_part_ids,
+        if (OMPI_SUCCESS != (ret = ompi_btl_openib_ini_intify_list(value, &sv->vendor_part_ids,
                                                &sv->vendor_part_ids_len))) {
             return ret;
         }
@@ -380,13 +379,13 @@ static int parse_line(parsed_section_values_t *sv)
 
     else if (0 == strcasecmp(key_buffer, "mtu")) {
         /* Single value */
-        sv->values.mtu = (uint32_t) intify(value);
+        sv->values.mtu = (uint32_t) ompi_btl_openib_ini_intify(value);
         sv->values.mtu_set = true;
     }
 
     else if (0 == strcasecmp(key_buffer, "use_eager_rdma")) {
         /* Single value */
-        sv->values.use_eager_rdma = (uint32_t) intify(value);
+        sv->values.use_eager_rdma = (uint32_t) ompi_btl_openib_ini_intify(value);
         sv->values.use_eager_rdma_set = true;
     }
 
@@ -398,7 +397,7 @@ static int parse_line(parsed_section_values_t *sv)
 
     else if (0 == strcasecmp(key_buffer, "max_inline_data")) {
         /* Single value */
-        sv->values.max_inline_data = (uint32_t) intify(value);
+        sv->values.max_inline_data = (uint32_t) ompi_btl_openib_ini_intify(value);
         sv->values.max_inline_data_set = true;
     }
 
@@ -582,7 +581,7 @@ static int save_section(parsed_section_values_t *s)
 /*
  * Do string-to-integer conversion, for both hex and decimal numbers
  */
-static int intify(char *str)
+int ompi_btl_openib_ini_intify(char *str)
 {
     while (isspace(*str)) {
         ++str;
@@ -603,7 +602,7 @@ static int intify(char *str)
 /*
  * Take a comma-delimited list and infity them all
  */
-static int intify_list(char *value, uint32_t **values, int *len)
+int ompi_btl_openib_ini_intify_list(char *value, uint32_t **values, int *len)
 {
     char *comma;
     char *str = value;
@@ -619,7 +618,7 @@ static int intify_list(char *value, uint32_t **values, int *len)
         if (NULL == *values) {
             return OMPI_ERR_OUT_OF_RESOURCE;
         }
-        *values[0] = (uint32_t) intify(str);
+        *values[0] = (uint32_t) ompi_btl_openib_ini_intify(str);
         *len = 1;
     } else {
         int newsize = 1;
@@ -641,7 +640,7 @@ static int intify_list(char *value, uint32_t **values, int *len)
         comma = strchr(str, ',');
         do {
             *comma = '\0';
-            (*values)[*len] = (uint32_t) intify(str);
+            (*values)[*len] = (uint32_t) ompi_btl_openib_ini_intify(str);
             ++(*len);
             str = comma + 1;
             comma = strchr(str, ',');
@@ -649,7 +648,7 @@ static int intify_list(char *value, uint32_t **values, int *len)
         /* Get the last value (i.e., the value after the last
            comma, because it won't have been snarfed in the
            loop) */
-        (*values)[*len] = (uint32_t) intify(str);
+        (*values)[*len] = (uint32_t) ompi_btl_openib_ini_intify(str);
         ++(*len);
     }
 

ompi/mca/btl/openib/btl_openib_ini.h

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2006-2008 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2008      Mellanox Technologies. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -52,6 +53,12 @@ BEGIN_C_DECLS
      */
     int ompi_btl_openib_ini_finalize(void);
 
+    /**
+     * string to int convertors with dec/hex autodetection
+     */
+    int ompi_btl_openib_ini_intify(char *string);
+    int ompi_btl_openib_ini_intify_list(char *str, uint32_t **values, int *len);
+
 END_C_DECLS
 
 #endif

ompi/mca/btl/openib/btl_openib_mca.c

@@ -31,6 +31,7 @@
 #include "opal/mca/base/mca_base_param.h"
 #include "btl_openib.h"
 #include "btl_openib_mca.h"
+#include "btl_openib_ini.h"
 #include "connect/base.h"
 
 #ifdef HAVE_IBV_FORK_INIT
@@ -132,7 +133,7 @@ int btl_openib_register_mca_params(void)
     char default_qps[100];
     uint32_t mid_qp_size;
     int i;
-    char *msg, *str;
+    char *msg, *str, *pkey;
     int ival, ival2, ret, tmp;
 
     ret = OMPI_SUCCESS;
@@ -280,16 +281,18 @@ int btl_openib_register_mca_params(void)
                   0, &ival, REGINT_GE_ZERO));
     mca_btl_openib_component.ib_pkey_ix = (uint32_t) ival;
 
-    CHECK(reg_int("of_pkey_val", "ib_pkey_val", "OpenFabrics pkey value"
-                  "(must be > 0 and < 0xffff)",
-                  0, &ival, REGINT_GE_ZERO));
-    if (ival > 0xffff) {
+    CHECK(reg_string("of_pkey_val", "ib_pkey_val", "OpenFabrics pkey value"
+                "(must be > 0 and < 0xffff)",
+                "0", &pkey, 0));
+    mca_btl_openib_component.ib_pkey_val = ompi_btl_openib_ini_intify(pkey) & MCA_BTL_IB_PKEY_MASK;
+    if (mca_btl_openib_component.ib_pkey_val > 0x7fff ||
+            mca_btl_openib_component.ib_pkey_val < 0) {
         orte_show_help("help-mpi-btl-openib.txt", "invalid mca param value",
                        true, "invalid value for btl_openib_ib_pkey_val",
                        "btl_openib_ib_pkey_val ignored");
-    } else {
-        mca_btl_openib_component.ib_pkey_val = (uint32_t) ival;
+        mca_btl_openib_component.ib_pkey_val = 0;
     }
+    free(pkey);
 
     CHECK(reg_int("of_psn", "ib_psn",
                   "OpenFabrics packet sequence starting number "