Merge pull request #5938 from bwbarrett/bugfix/tcp-multi-address

Always use same IP address for module and modex

opal/mca/btl/tcp/btl_tcp.h

@@ -164,13 +164,11 @@ OPAL_MODULE_DECLSPEC extern mca_btl_tcp_component_t mca_btl_tcp_component;
 struct mca_btl_tcp_module_t {
     mca_btl_base_module_t  super;  /**< base BTL interface */
     uint16_t           tcp_ifkindex; /** <BTL kernel interface index */
-#if 0
+    struct sockaddr_storage tcp_ifaddr;   /**< First address
-    int                tcp_ifindex; /**< BTL interface index */
+                                             discovered for this
-#endif
+                                             interface, bound as
-    struct sockaddr_storage tcp_ifaddr; /**< First IPv4 address discovered for this interface, bound as sending address for this BTL */
+                                             sending address for this
-#if OPAL_ENABLE_IPV6
+                                             BTL */
-    struct sockaddr_storage tcp_ifaddr_6; /**< First IPv6 address discovered for this interface, bound as sending address for this BTL  */
-#endif
     uint32_t           tcp_ifmask;  /**< BTL interface netmask */
 
     opal_mutex_t       tcp_endpoints_mutex;

opal/mca/btl/tcp/btl_tcp_addr.h

@@ -33,40 +33,52 @@
 
 
 /**
- * Structure used to publish TCP connection information to peers.
+ * Modex address structure.
+ *
+ * One of these structures will be sent for every btl module in use by
+ * the local BTL TCP component.
  */
+struct mca_btl_tcp_modex_addr_t {
+    uint8_t     addr[16];       /* endpoint address.  for addr_family
+                                   of MCA_BTL_TCP_AF_INET, only the
+                                   first 4 bytes have meaning. */
+    uint32_t    addr_ifkindex;  /* endpoint kernel index */
+    uint16_t    addr_port;      /* endpoint listen port */
+    uint8_t     addr_family;    /* endpoint address family.  Note that
+                                   this is
+                                   MCA_BTL_TCP_AF_{INET,INET6}, not
+                                   the traditional
+                                   AF_INET/AF_INET6. */
+    uint8_t     padding[1];     /* padd out to an 8-byte word */
+};
+typedef struct mca_btl_tcp_modex_addr_t mca_btl_tcp_modex_addr_t;
 
+
+/**
+ * Remote peer address structure
+ *
+ * One of these structures will be allocated for every remote endpoint
+ * associated with a remote proc.  The data is pulled from the
+ * mca_btl_tcp_modex_addr_t structure, except for the addr_inuse
+ * field, which is local.
+ */
 struct mca_btl_tcp_addr_t {
-    /* the following information is exchanged between different
+    union {
-       machines (read: byte order), so use network byte order
+        struct in_addr  addr_inet;      /* IPv6 listen address */
-       for everything and don't add padding
-    */
 #if OPAL_ENABLE_IPV6
-    struct in6_addr addr_inet;    /**< IPv4/IPv6 listen address > */
+        struct in6_addr addr_inet6;     /* IPv6 listen address */
-#else
-    /* Bug, FIXME: needs testing */
-    struct my_in6_addr {
-        union {
-            uint32_t u6_addr32[4];
-            struct _my_in6_addr {
-                struct in_addr _addr_inet;
-                uint32_t _pad[3];
-            } _addr__inet;
-        } _union_inet;
-    } addr_inet;
 #endif
-    in_port_t      addr_port;     /**< listen port */
+    };
-    uint16_t       addr_ifkindex; /**< remote interface index assigned with
+    in_port_t   addr_port;     /**< listen port */
+    int         addr_ifkindex; /**< remote interface index assigned with
                                     this address */
-    unsigned short addr_inuse;    /**< local meaning only */
+    uint8_t     addr_family;   /**< AF_INET or AF_INET6 */
-    uint8_t        addr_family;   /**< AF_INET or AF_INET6 */
+    bool        addr_inuse;    /**< local meaning only */
 };
 typedef struct mca_btl_tcp_addr_t mca_btl_tcp_addr_t;
 
 #define MCA_BTL_TCP_AF_INET     0
-#if OPAL_ENABLE_IPV6
+#define MCA_BTL_TCP_AF_INET6    1
-# define MCA_BTL_TCP_AF_INET6   1
-#endif
 
 #endif
 

opal/mca/btl/tcp/btl_tcp_component.c

@@ -490,11 +490,59 @@ static int mca_btl_tcp_component_close(void)
  *  Create a btl instance and add to modules list.
  */
 
-static int mca_btl_tcp_create(int if_kindex, const char* if_name)
+static int mca_btl_tcp_create(const int if_kindex, const char* if_name)
 {
     struct mca_btl_tcp_module_t* btl;
     char param[256];
     int i;
+    struct sockaddr_storage addr;
+    bool found = false;
+
+    /*
+     * Look for an address on the given device (ie, kindex) which
+     * isn't disabled by the disable_family option.  If there isn't
+     * one, skip creating the modules for this interface.  We store
+     * the address on the module both to publish in the modex and to
+     * use as the source address of all packets sent by this module.
+     *
+     * This still isn't quite right.  Better would be to pull apart
+     * split_and_resolve and pass the address used to select the
+     * device into mca_btl_tcp_create().  This is a cleanup of the
+     * logic that's been in use for years, but the case it doesn't
+     * cover is (say) only specifing mca_btl_if_include 10.0.0.0/16
+     * when the interface has addresses of both 10.0.0.1 and 10.1.0.1;
+     * there's absolutely nothing that keeps this code from picking
+     * 10.1.0.1 as the one that is published in the modex and used for
+     * connection.
+     */
+    for (i = opal_ifbegin() ; i >= 0 ; i = opal_ifnext(i)) {
+        int ret;
+
+        if (if_kindex != opal_ifindextokindex(i)) {
+            continue;
+        }
+
+        ret = opal_ifindextoaddr(i, (struct sockaddr*)&addr,
+                                 sizeof(struct sockaddr_storage));
+        if (OPAL_SUCCESS != ret) {
+            return ret;
+        }
+
+        if (addr.ss_family == AF_INET &&
+            4 != mca_btl_tcp_component.tcp_disable_family) {
+            found = true;
+            break;
+        } else if (addr.ss_family == AF_INET6 &&
+                   6 != mca_btl_tcp_component.tcp_disable_family) {
+            found = true;
+            break;
+        }
+    }
+    /* if we didn't find an address that works for us on this
+       interface, just move on. */
+    if (!found) {
+        return OPAL_SUCCESS;
+    }
 
     for( i = 0; i < (int)mca_btl_tcp_component.tcp_num_links; i++ ) {
         btl = (struct mca_btl_tcp_module_t *)malloc(sizeof(mca_btl_tcp_module_t));
@@ -513,17 +561,8 @@ static int mca_btl_tcp_create(int if_kindex, const char* if_name)
         btl->tcp_send_handler = 0;
 #endif
 
-       struct sockaddr_storage addr;
+        memcpy(&btl->tcp_ifaddr, &addr, sizeof(struct sockaddr_storage));
-       opal_ifkindextoaddr(if_kindex, (struct sockaddr*) &addr,
+
-                                          sizeof (struct sockaddr_storage));
-#if OPAL_ENABLE_IPV6
-        if (addr.ss_family == AF_INET6) {
-            btl->tcp_ifaddr_6 =  addr;
-        }
-#endif
-        if (addr.ss_family == AF_INET) {
-           btl->tcp_ifaddr = addr;
-        }
         /* allow user to specify interface bandwidth */
         sprintf(param, "bandwidth_%s", if_name);
         mca_btl_tcp_param_register_uint(param, NULL, btl->super.btl_bandwidth, OPAL_INFO_LVL_5, &btl->super.btl_bandwidth);
@@ -567,8 +606,8 @@ static int mca_btl_tcp_create(int if_kindex, const char* if_name)
         opal_output_verbose(5, opal_btl_base_framework.framework_output,
                             "btl:tcp: %p: if %s kidx %d cnt %i addr %s %s bw %d lt %d\n",
                             (void*)btl, if_name, (int) btl->tcp_ifkindex, i,
-                            opal_net_get_hostname((struct sockaddr*)&addr),
+                            opal_net_get_hostname((struct sockaddr*)&btl->tcp_ifaddr),
-                            (addr.ss_family == AF_INET) ? "IPv4" : "IPv6",
+                            (btl->tcp_ifaddr.ss_family == AF_INET) ? "IPv4" : "IPv6",
                             btl->super.btl_bandwidth, btl->super.btl_latency);
     }
     return OPAL_SUCCESS;
@@ -721,6 +760,7 @@ static int mca_btl_tcp_component_create_instances(void)
          */
         for(if_index = opal_ifbegin(); if_index >= 0; if_index = opal_ifnext(if_index)){
             int index = opal_ifindextokindex (if_index);
+
             if (index > 0) {
                 bool want_this_if = true;
 
@@ -1121,97 +1161,64 @@ static int mca_btl_tcp_component_create_listen(uint16_t af_family)
 
 static int mca_btl_tcp_component_exchange(void)
 {
-     int rc = 0, index;
+     int rc;
-     size_t i = 0;
+     size_t i;
-     size_t size = mca_btl_tcp_component.tcp_addr_count *
+     size_t num_btls = mca_btl_tcp_component.tcp_num_btls;
-                   mca_btl_tcp_component.tcp_num_links * sizeof(mca_btl_tcp_addr_t);
+     size_t size = num_btls * sizeof(mca_btl_tcp_modex_addr_t);
-     /* adi@2007-04-12:
+     mca_btl_tcp_modex_addr_t *addrs;
-      *
-      * We'll need to explain things a bit here:
-      *    1. We normally have as many BTLs as physical NICs.
-      *    2. With num_links, we now have num_btl = num_links * #NICs
-      *    3. we might have more than one address per NIC
-      */
-     size_t xfer_size = 0; /* real size to transfer (may differ from 'size') */
-     size_t current_addr = 0;
 
-     if(mca_btl_tcp_component.tcp_num_btls != 0) {
+     if (num_btls <= 0) {
-         char ifn[32];
+         return 0;
-         mca_btl_tcp_addr_t *addrs = (mca_btl_tcp_addr_t *)malloc(size);
+     }
-         memset(addrs, 0, size);
 
-         /* here we start populating our addresses */
+     addrs = (mca_btl_tcp_modex_addr_t*)malloc(size);
-         for( i = 0; i < mca_btl_tcp_component.tcp_num_btls; i++ ) {
+     if (NULL == addrs) {
-             for (index = opal_ifbegin(); index >= 0;
+         return OPAL_ERR_OUT_OF_RESOURCE;
-                     index = opal_ifnext(index)) {
+     }
-                 struct sockaddr_storage my_ss;
+     memset(addrs, 0, size);
 
-                 /* Look if the module's address belongs to this
+     for (i = 0 ; i < num_btls ; i++) {
-                  * kernel IP interface.  If not, go to next address.
+         struct mca_btl_tcp_module_t *btl = mca_btl_tcp_component.tcp_btls[i];
-                  */
+         struct sockaddr *addr = (struct sockaddr*)&(btl->tcp_ifaddr);
-                 if (opal_ifindextokindex (index) !=
-                     mca_btl_tcp_component.tcp_btls[i]->tcp_ifkindex) {
-                     continue;
-                 }
-
-                 opal_ifindextoname(index, ifn, sizeof(ifn));
-                 opal_output_verbose(30, opal_btl_base_framework.framework_output,
-                                     "btl: tcp: component_exchange: examining interface %s",
-                                     ifn);
-                 if (OPAL_SUCCESS !=
-                     opal_ifindextoaddr(index, (struct sockaddr*) &my_ss,
-                                        sizeof (my_ss))) {
-                     opal_output (0,
-                             "btl_tcp_component: problems getting address for index %i (kernel index %i)\n",
-                             index, opal_ifindextokindex (index));
-                     continue;
-                 }
 
 #if OPAL_ENABLE_IPV6
-                 if ((AF_INET6 == my_ss.ss_family) &&
+         if (AF_INET6 == addr->sa_family) {
-                     (6 != mca_btl_tcp_component.tcp_disable_family)) {
+             struct sockaddr_in6 *inaddr6 = (struct sockaddr_in6*)addr;
-                     memcpy(&addrs[current_addr].addr_inet,
+
-                             &((struct sockaddr_in6*)&my_ss)->sin6_addr,
+             memcpy(&addrs[i].addr, &(inaddr6->sin6_addr),
-                             sizeof(addrs[0].addr_inet));
+                    sizeof(struct in6_addr));
-                     addrs[current_addr].addr_port =
+             addrs[i].addr_port = mca_btl_tcp_component.tcp6_listen_port;
-                         mca_btl_tcp_component.tcp6_listen_port;
+             addrs[i].addr_ifkindex = btl->tcp_ifkindex;
-                     addrs[current_addr].addr_family = MCA_BTL_TCP_AF_INET6;
+             addrs[i].addr_family = MCA_BTL_TCP_AF_INET6;
-                     xfer_size += sizeof (mca_btl_tcp_addr_t);
+             opal_output_verbose(5, opal_btl_base_framework.framework_output,
-                     addrs[current_addr].addr_inuse   = 0;
+                                 "btl: tcp: exchange: %d %d IPv6 %s",
-                     addrs[current_addr].addr_ifkindex =
+                                 (int)i, btl->tcp_ifkindex,
-                         opal_ifindextokindex (index);
+                                 opal_net_get_hostname(addr));
-                     current_addr++;
+         } else
-                     opal_output_verbose(30, opal_btl_base_framework.framework_output,
-                                         "btl: tcp: component_exchange: "
-                                         "%s IPv6 %s", ifn,
-                                         opal_net_get_hostname((struct sockaddr*)&my_ss));
-                 } else
 #endif
-                 if ((AF_INET == my_ss.ss_family) &&
+         if (AF_INET == addr->sa_family) {
-                     (4 != mca_btl_tcp_component.tcp_disable_family)) {
+             struct sockaddr_in *inaddr = (struct sockaddr_in*)addr;
-                     memcpy(&addrs[current_addr].addr_inet,
+
-                            &((struct sockaddr_in*)&my_ss)->sin_addr,
+             memcpy(&addrs[i].addr, &(inaddr->sin_addr),
-                            sizeof(struct in_addr));
+                    sizeof(struct in_addr));
-                     addrs[current_addr].addr_port =
+             addrs[i].addr_port = mca_btl_tcp_component.tcp_listen_port;
-                         mca_btl_tcp_component.tcp_listen_port;
+             addrs[i].addr_ifkindex = btl->tcp_ifkindex;
-                     addrs[current_addr].addr_family = MCA_BTL_TCP_AF_INET;
+             addrs[i].addr_family = MCA_BTL_TCP_AF_INET;
-                     xfer_size += sizeof (mca_btl_tcp_addr_t);
+             opal_output_verbose(5, opal_btl_base_framework.framework_output,
-                     addrs[current_addr].addr_inuse   = 0;
+                                 "btl: tcp: exchange: %d %d IPv4 %s",
-                     addrs[current_addr].addr_ifkindex =
+                                 (int)i, btl->tcp_ifkindex,
-                         opal_ifindextokindex (index);
+                                 opal_net_get_hostname(addr));
-                     current_addr++;
+         } else {
-                     opal_output_verbose(30, opal_btl_base_framework.framework_output,
+             BTL_ERROR(("Unexpected address family: %d", addr->sa_family));
-                                         "btl: tcp: component_exchange: "
+             return OPAL_ERR_BAD_PARAM;
-                                         "%s IPv4 %s", ifn,
+         }
-                                         opal_net_get_hostname((struct sockaddr*)&my_ss));
+     }
-                 }
+
-             } /* end of for opal_ifbegin() */
+     OPAL_MODEX_SEND(rc, OPAL_PMIX_GLOBAL,
-         } /* end of for tcp_num_btls */
+                     &mca_btl_tcp_component.super.btl_version,
-         OPAL_MODEX_SEND(rc, OPAL_PMIX_GLOBAL,
+                     addrs, size);
-                         &mca_btl_tcp_component.super.btl_version,
+     free(addrs);
-                         addrs, xfer_size);
+
-         free(addrs);
-     } /* end if */
      return rc;
 }
 

opal/mca/btl/tcp/btl_tcp_endpoint.c

@@ -743,11 +743,11 @@ static int mca_btl_tcp_endpoint_start_connect(mca_btl_base_endpoint_t* btl_endpo
 #if OPAL_ENABLE_IPV6
     if (endpoint_addr.ss_family == AF_INET6) {
         assert(NULL != &btl_endpoint->endpoint_btl->tcp_ifaddr_6);
-        if (bind(btl_endpoint->endpoint_sd, (struct sockaddr*) &btl_endpoint->endpoint_btl->tcp_ifaddr_6,
+        if (bind(btl_endpoint->endpoint_sd, (struct sockaddr*) &btl_endpoint->endpoint_btl->tcp_ifaddr,
                  sizeof(struct sockaddr_in6)) < 0) {
             BTL_ERROR(("bind on local address (%s:%d) failed: %s (%d)",
                        opal_net_get_hostname((struct sockaddr*) &btl_endpoint->endpoint_btl->tcp_ifaddr),
-                       htons(((struct sockaddr_in*)&btl_endpoint->endpoint_btl->tcp_ifaddr)->sin_port),
+                       htons(((struct sockaddr_in6*)&btl_endpoint->endpoint_btl->tcp_ifaddr)->sin6_port),
                        strerror(opal_socket_errno), opal_socket_errno));
 
             CLOSE_THE_SOCKET(btl_endpoint->endpoint_sd);

opal/mca/btl/tcp/btl_tcp_proc.c

@@ -117,73 +117,99 @@ void mca_btl_tcp_proc_destruct(mca_btl_tcp_proc_t* tcp_proc)
 mca_btl_tcp_proc_t* mca_btl_tcp_proc_create(opal_proc_t* proc)
 {
     mca_btl_tcp_proc_t* btl_proc;
-    size_t size;
     int rc;
+    mca_btl_tcp_modex_addr_t *remote_addrs = NULL;
+    size_t i, size;
 
     OPAL_THREAD_LOCK(&mca_btl_tcp_component.tcp_lock);
     rc = opal_proc_table_get_value(&mca_btl_tcp_component.tcp_procs,
                                    proc->proc_name, (void**)&btl_proc);
-    if(OPAL_SUCCESS == rc) {
+    if (OPAL_SUCCESS == rc) {
         OPAL_THREAD_UNLOCK(&mca_btl_tcp_component.tcp_lock);
         return btl_proc;
     }
 
-    do {  /* This loop is only necessary so that we can break out of the serial code */
+    /* proc was not found, so create one */
-        btl_proc = OBJ_NEW(mca_btl_tcp_proc_t);
+    btl_proc = OBJ_NEW(mca_btl_tcp_proc_t);
-        if(NULL == btl_proc) {
+    if (NULL == btl_proc) {
-            rc = OPAL_ERR_OUT_OF_RESOURCE;
+        rc = OPAL_ERR_OUT_OF_RESOURCE;
-            break;
+        goto cleanup;
+    }
+
+    /* Retain the proc, but don't store the ref into the btl_proc just yet. This
+     * provides a way to release the btl_proc in case of failure without having to
+     * unlock the mutex.
+     */
+    OBJ_RETAIN(proc);
+
+    /* lookup tcp parameters exported by this proc */
+    OPAL_MODEX_RECV(rc, &mca_btl_tcp_component.super.btl_version,
+                    &proc->proc_name, (uint8_t**)&remote_addrs, &size);
+    if (OPAL_SUCCESS != rc) {
+        if (OPAL_ERR_NOT_FOUND != rc) {
+            BTL_ERROR(("opal_modex_recv: failed with return value=%d", rc));
         }
+        goto cleanup;
+    }
 
-        /* Retain the proc, but don't store the ref into the btl_proc just yet. This
+    if (0 != (size % sizeof(mca_btl_tcp_modex_addr_t))) {
-         * provides a way to release the btl_proc in case of failure without having to
+        BTL_ERROR(("opal_modex_recv: invalid size %lu: btl-size: %lu\n",
-         * unlock the mutex.
+                   (unsigned long)size,
-         */
+                   (unsigned long)sizeof(mca_btl_tcp_modex_addr_t)));
-        OBJ_RETAIN(proc);
+        rc = OPAL_ERROR;
+        goto cleanup;
+    }
 
-        /* lookup tcp parameters exported by this proc */
+    btl_proc->proc_addr_count = size / sizeof(mca_btl_tcp_modex_addr_t);
-        OPAL_MODEX_RECV(rc, &mca_btl_tcp_component.super.btl_version,
+    btl_proc->proc_addrs = malloc(btl_proc->proc_addr_count *
-                        &proc->proc_name, (uint8_t**)&btl_proc->proc_addrs, &size);
+                                  sizeof(mca_btl_tcp_addr_t));
-        if(rc != OPAL_SUCCESS) {
+    if (NULL == btl_proc->proc_addrs) {
-            if(OPAL_ERR_NOT_FOUND != rc)
+        rc = OPAL_ERR_OUT_OF_RESOURCE;
-                BTL_ERROR(("opal_modex_recv: failed with return value=%d", rc));
+        goto cleanup;
-            break;
+    }
-        }
 
-        if(0 != (size % sizeof(mca_btl_tcp_addr_t))) {
+    /* the modex and proc structures differ slightly, so copy the
-            BTL_ERROR(("opal_modex_recv: invalid size %lu: btl-size: %lu\n",
+       fields needed in the proc version */
-                       (unsigned long) size, (unsigned long)sizeof(mca_btl_tcp_addr_t)));
+    for (i = 0 ; i < btl_proc->proc_addr_count ; i++) {
-            rc = OPAL_ERROR;
+        if (MCA_BTL_TCP_AF_INET == remote_addrs[i].addr_family) {
-            break;
+            memcpy(&btl_proc->proc_addrs[i].addr_inet,
-        }
+                   remote_addrs[i].addr, sizeof(struct in_addr));
-
+            btl_proc->proc_addrs[i].addr_port = remote_addrs[i].addr_port;
-        btl_proc->proc_addr_count = size / sizeof(mca_btl_tcp_addr_t);
+            btl_proc->proc_addrs[i].addr_ifkindex = remote_addrs[i].addr_ifkindex;
-
+            btl_proc->proc_addrs[i].addr_family = AF_INET;
-        /* allocate space for endpoint array - one for each exported address */
+            btl_proc->proc_addrs[i].addr_inuse = false;
-        btl_proc->proc_endpoints = (mca_btl_base_endpoint_t**)
+        } else if (MCA_BTL_TCP_AF_INET6 == remote_addrs[i].addr_family) {
-            malloc((1 + btl_proc->proc_addr_count) *
-                   sizeof(mca_btl_base_endpoint_t*));
-        if(NULL == btl_proc->proc_endpoints) {
-            rc = OPAL_ERR_OUT_OF_RESOURCE;
-            break;
-        }
-
-        /* convert the OPAL addr_family field to OS constants,
-         * so we can check for AF_INET (or AF_INET6) and don't have
-         * to deal with byte ordering anymore.
-         */
-        for (unsigned int i = 0; i < btl_proc->proc_addr_count; i++) {
-            if (MCA_BTL_TCP_AF_INET == btl_proc->proc_addrs[i].addr_family) {
-                btl_proc->proc_addrs[i].addr_family = AF_INET;
-            }
 #if OPAL_ENABLE_IPV6
-            if (MCA_BTL_TCP_AF_INET6 == btl_proc->proc_addrs[i].addr_family) {
+            memcpy(&btl_proc->proc_addrs[i].addr_inet6,
-                btl_proc->proc_addrs[i].addr_family = AF_INET6;
+                   remote_addrs[i].addr, sizeof(struct in6_addr));
-            }
+            btl_proc->proc_addrs[i].addr_port = remote_addrs[i].addr_port;
+            btl_proc->proc_addrs[i].addr_ifkindex = remote_addrs[i].addr_ifkindex;
+            btl_proc->proc_addrs[i].addr_family = AF_INET6;
+            btl_proc->proc_addrs[i].addr_inuse = false;
+#else
+            rc = OPAL_ERR_NOT_SUPPORTED;
+            goto cleanup;
 #endif
+        } else {
+            BTL_ERROR(("Unexpected address family %d",
+                       (int)remote_addrs[i].addr_family));
+            rc = OPAL_ERR_BAD_PARAM;
+            goto cleanup;
         }
-    } while (0);
+    }
 
+    free(remote_addrs);
+
+    /* allocate space for endpoint array - one for each exported address */
+    btl_proc->proc_endpoints = (mca_btl_base_endpoint_t**)
+        malloc((1 + btl_proc->proc_addr_count) *
+               sizeof(mca_btl_base_endpoint_t*));
+    if (NULL == btl_proc->proc_endpoints) {
+        rc = OPAL_ERR_OUT_OF_RESOURCE;
+        goto cleanup;
+    }
+
+cleanup:
     if (OPAL_SUCCESS == rc) {
         btl_proc->proc_opal = proc;  /* link with the proc */
         /* add to hash table of all proc instance. */
@@ -669,7 +695,7 @@ int mca_btl_tcp_proc_insert( mca_btl_tcp_proc_t* btl_proc,
             }
             peer_interfaces[best]->inuse++;
             btl_endpoint->endpoint_addr = proc_data->best_addr[i][best];
-            btl_endpoint->endpoint_addr->addr_inuse++;
+            btl_endpoint->endpoint_addr->addr_inuse = true;
             rc = OPAL_SUCCESS;
             break;
         }
@@ -695,7 +721,7 @@ int mca_btl_tcp_proc_insert( mca_btl_tcp_proc_t* btl_proc,
         if (CQ_NO_CONNECTION != max) {
             peer_interfaces[j_max]->inuse++;
             btl_endpoint->endpoint_addr = proc_data->best_addr[i_max][j_max];
-            btl_endpoint->endpoint_addr->addr_inuse++;
+            btl_endpoint->endpoint_addr->addr_inuse = true;
             rc = OPAL_SUCCESS;
         }
     }
@@ -772,7 +798,7 @@ int mca_btl_tcp_proc_remove(mca_btl_tcp_proc_t* btl_proc, mca_btl_base_endpoint_
                    being removed early in the wireup sequence (e.g., if it
                    is unreachable by all other procs) */
                 if (NULL != btl_endpoint->endpoint_addr) {
-                    btl_endpoint->endpoint_addr->addr_inuse--;
+                    btl_endpoint->endpoint_addr->addr_inuse = false;
                 }
                 break;
             }