usnic: check connectivity on first communication to a peer

Previously, we were only checking connectivity upon first ''send'' to
a peer.  But this ignores the case where the first communication to a
peer is actually an ACK -- i.e., we successfully received something
from the peer and we need to send an ACK back.  So we need to verify
that the ACK will actually get there.

Specifically, certain asymmetric routing cases can lead to a hang if
we don't check the connectivity in both directions.  E.g., if the
sender is able to get traffic to the receiver, but the receiver is
unable to get traffic back to the sender because it made a different
routing decision than the sender.

In this case, the connectivity checker from the sender could succeed
(because the connectivity checker will ACK along the same path in
which the ping was received), but sending a BTL ACK could fail
(because the BTL ACK will be sent back along the path chosen by the
graph algorithm, which, in an erroneous asymmetric routing scenario,
may be different/wrong).

Hence, we want to trigger the connectivity checker at the first
communication from A->B, which may either be a BTL send or an ACK.

Reviewed by Dave Goodell.

cmr=v1.8.2:reviewer=ompi-rm1.8

This commit was SVN r32309.

ompi/mca/btl/usnic/btl_usnic_ack.c

@@ -23,6 +23,7 @@
 #include "btl_usnic_ack.h"
 #include "btl_usnic_util.h"
 #include "btl_usnic_send.h"
+#include "btl_usnic_connectivity.h"
 
 /*
  * Force a retrans of a segment
@@ -220,6 +221,11 @@ ompi_btl_usnic_ack_send(
                 endpoint->endpoint_remote_addr.qp_num[ack->ss_channel]);
 #endif
 
+    /* Do we need to check the connectivity?  If enabled, we'll check
+       the connectivity at either first send to peer X or first ACK to
+       peer X. */
+    ompi_btl_usnic_check_connectivity(module, endpoint);
+
     /* send the ACK */
     ompi_btl_usnic_post_segment(module, endpoint, ack);
 

ompi/mca/btl/usnic/btl_usnic_connectivity.h

@@ -12,13 +12,12 @@
 
 #include "ompi_config.h"
 
-#include <netinet/in.h>
-
 #include "opal/util/show_help.h"
 
 #include "ompi/runtime/mpiruntime.h"
 
 #include "btl_usnic_util.h"
+#include "btl_usnic_proc.h"
 
 
 /**
@@ -245,4 +244,27 @@ int ompi_btl_usnic_connectivity_agent_init(void);
  */
 int ompi_btl_usnic_connectivity_agent_finalize(void);
 
+
+/**
+ * Helper function invoked in the BTL that will invoke a ping, if the
+ * ping hasn't already been invoked.
+ */
+static inline void
+ompi_btl_usnic_check_connectivity(ompi_btl_usnic_module_t *module,
+				  ompi_btl_usnic_endpoint_t *endpoint)
+{
+    if (OPAL_LIKELY(mca_btl_usnic_component.connectivity_enabled) &&
+        OPAL_UNLIKELY(!endpoint->endpoint_connectivity_checked)) {
+        ompi_btl_usnic_connectivity_ping(module->local_addr.ipv4_addr,
+                                         module->local_addr.connectivity_udp_port,
+                                         endpoint->endpoint_remote_addr.ipv4_addr,
+                                         endpoint->endpoint_remote_addr.cidrmask,
+                                         endpoint->endpoint_remote_addr.connectivity_udp_port,
+                                         endpoint->endpoint_remote_addr.mac,
+					 endpoint->endpoint_proc->proc_ompi->proc_hostname,
+                                         endpoint->endpoint_remote_addr.mtu);
+        endpoint->endpoint_connectivity_checked = true;
+    }
+}
+
 #endif /* OMPI_BTL_USNIC_CONNECITIVITY_H */

ompi/mca/btl/usnic/btl_usnic_module.c

@@ -981,19 +981,10 @@ usnic_prepare_src(
     size_t osize = *size;
 #endif
 
-    /* Do we need to check the connectivity? */
-    if (mca_btl_usnic_component.connectivity_enabled &&
-        OPAL_UNLIKELY(!endpoint->endpoint_connectivity_checked)) {
-        ompi_btl_usnic_connectivity_ping(module->local_addr.ipv4_addr,
-                                         module->local_addr.connectivity_udp_port,
-                                         endpoint->endpoint_remote_addr.ipv4_addr,
-                                         endpoint->endpoint_remote_addr.cidrmask,
-                                         endpoint->endpoint_remote_addr.connectivity_udp_port,
-                                         endpoint->endpoint_remote_addr.mac,
-                                         endpoint->endpoint_proc->proc_ompi->proc_hostname,
-                                         endpoint->endpoint_remote_addr.mtu);
-        endpoint->endpoint_connectivity_checked = true;
-    }
+    /* Do we need to check the connectivity?  If enabled, we'll check
+       the connectivity at either first send to peer X or first ACK to
+       peer X. */
+    ompi_btl_usnic_check_connectivity(module, endpoint);
 
     /*
      * if total payload len fits in one MTU use small send, else large