usnic: properly check src iface in route queries

rtnetlink doesn't check the source address when determining whether to
return route info for a query.  So we need to check that the OIF matches
the OIF of the source interface name.  Without this check, OMPI might
pair a local interface which does not have a route to a particular
remote interface.

Fixes Cisco bug CSCup55797.

Reviewed-by: Jeff Squyres <jsquyres@cisco.com>

cmr=v1.8.2:reviewer=ompi-rm1.8

This commit was SVN r32090.

ompi/mca/btl/usnic/btl_usnic_libnl1_utils.c

@@ -17,6 +17,7 @@
 #include <errno.h>
 #include <arpa/inet.h>
 #include <time.h>
+#include <net/if.h>
 #include <netlink/netlink.h>
 #include <netlink/addr.h>
 #include <netlink/route/rtnl.h>
@@ -45,6 +46,7 @@ struct usnic_rtnl_sk {
 
 struct nl_lookup_arg {
 	uint32_t	nh_addr;
+	int		oif;
 	int		found;
 	int		replied;
 	int		msg_count;
@@ -130,9 +132,17 @@ static int rtnl_raw_parse_cb(struct nl_msg *msg, void *arg)
 		return NL_STOP;
 	}
 
-        if (tb[RTA_METRICS]) {
+	if (tb[RTA_OIF]) {
+		if (nla_get_u32(tb[RTA_OIF]) == (uint32_t)lookup_arg->oif)
+			found = 1;
+		else
+			usnic_err("Retrieved route has a different outgoing interface %d (expected %d)\n",
+					nla_get_u32(tb[RTA_OIF]),
+					lookup_arg->oif);
+	}
+
+        if (found && tb[RTA_METRICS]) {
             lookup_arg->metric = (int)nla_get_u32(tb[RTA_METRICS]);
-            found = 1;
         }
 
 	if (found && tb[RTA_GATEWAY])
@@ -173,7 +183,9 @@ static int nl_set_recv_timeout(struct nl_handle *handle)
 	return err;
 }
 
-int ompi_btl_usnic_nl_ip_rt_lookup(struct usnic_rtnl_sk *unlsk, uint32_t src_addr,
+int ompi_btl_usnic_nl_ip_rt_lookup(struct usnic_rtnl_sk *unlsk,
+                                   const char *src_ifname,
+                                   uint32_t src_addr,
                                    uint32_t dst_addr, int *metric)
 {
 	struct nl_msg *nlm;
@@ -181,8 +193,15 @@ int ompi_btl_usnic_nl_ip_rt_lookup(struct usnic_rtnl_sk *unlsk, uint32_t src_add
 	struct nl_lookup_arg arg;
 	int	msg_cnt;
 	int err;
+	int oif;
+
+	oif = if_nametoindex(src_ifname);
+	if (0 == oif) {
+	    return errno;
+	}
 
 	arg.nh_addr 	= 0;
+	arg.oif		= oif;
 	arg.found	= 0;
 	arg.replied	= 0;
 	arg.unlsk	= unlsk;

ompi/mca/btl/usnic/btl_usnic_libnl3_utils.c

@@ -16,6 +16,7 @@
 
 #include <errno.h>
 #include <arpa/inet.h>
+#include <net/if.h>
 
 /* the netlink headers have many C99-isms that cause warnings on the v1.6
  * branch when --enable-picky is passed to configure :( */
@@ -44,6 +45,7 @@ struct usnic_rtnl_sk {
 
 struct nl_lookup_arg {
 	uint32_t	nh_addr;
+	int		oif;
 	int		found;
 	int		replied;
 	int		msg_count;
@@ -128,9 +130,17 @@ static int rtnl_raw_parse_cb(struct nl_msg *msg, void *arg)
 		return NL_STOP;
 	}
 
-        if (tb[RTA_METRICS]) {
+	if (tb[RTA_OIF]) {
+		if (nla_get_u32(tb[RTA_OIF]) == (uint32_t)lookup_arg->oif)
+			found = 1;
+		else
+			usnic_err("Retrieved route has a different outgoing interface %d (expected %d)\n",
+					nla_get_u32(tb[RTA_OIF]),
+					lookup_arg->oif);
+	}
+
+        if (found && tb[RTA_METRICS]) {
             lookup_arg->metric = (int)nla_get_u32(tb[RTA_METRICS]);
-            found = 1;
         }
 
 	if (found && tb[RTA_GATEWAY])
@@ -171,7 +181,9 @@ static int nl_set_recv_timeout(struct nl_sock *sock)
 	return err;
 }
 
-int ompi_btl_usnic_nl_ip_rt_lookup(struct usnic_rtnl_sk *unlsk, uint32_t src_addr,
+int ompi_btl_usnic_nl_ip_rt_lookup(struct usnic_rtnl_sk *unlsk,
+                                   const char *src_ifname,
+                                   uint32_t src_addr,
                                    uint32_t dst_addr, int *metric)
 {
 	struct nl_msg 		*nlm;
@@ -179,8 +191,15 @@ int ompi_btl_usnic_nl_ip_rt_lookup(struct usnic_rtnl_sk *unlsk, uint32_t src_add
 	struct nl_lookup_arg    arg;
 	int	msg_cnt;
 	int		     	err;
+	int oif;
+
+	oif = if_nametoindex(src_ifname);
+	if (0 == oif) {
+	    return errno;
+	}
 
 	arg.nh_addr 	= 0;
+	arg.oif		= oif;
 	arg.found	= 0;
 	arg.replied	= 0;
 	arg.unlsk	= unlsk;

ompi/mca/btl/usnic/btl_usnic_libnl_utils.h

@@ -30,7 +30,9 @@ void ompi_btl_usnic_rtnl_sk_free(struct usnic_rtnl_sk* u_nlsk);
 
 /* src_addr and dst_addr are IPv4 addresses in network byte order.  Returns
  * zero on successful route lookup, -1 otherwise. */
-int ompi_btl_usnic_nl_ip_rt_lookup(struct usnic_rtnl_sk *unlsk, uint32_t src_addr,
+int ompi_btl_usnic_nl_ip_rt_lookup(struct usnic_rtnl_sk *unlsk,
+                                   const char *src_ifname,
+                                   uint32_t src_addr,
                                    uint32_t dst_addr, int *metric);
 
 #endif /* LIBNL_UTILS_H */

ompi/mca/btl/usnic/btl_usnic_proc.c

@@ -310,6 +310,7 @@ static uint64_t compute_weight(
 
     metric = 0;
     err = ompi_btl_usnic_nl_ip_rt_lookup(mca_btl_usnic_component.unlsk,
+                                         module->if_name,
                                          module->if_ipv4_addr,
                                          proc_modex_addr->ipv4_addr,
                                          &metric);

ompi/mca/btl/usnic/configure.m4

@@ -127,6 +127,9 @@ AC_DEFUN([MCA_ompi_btl_usnic_CONFIG],[
                        [define to 1 if usnic BTL unit tests are enabled, 0 otherwise])
     unset unit_tests
 
+    # for if_nametoindex(3)
+    AC_CHECK_HEADERS([net/if.h])
+
     OMPI_CHECK_OPENFABRICS([btl_usnic],
                         [btl_usnic_happy="yes"],
                         [btl_usnic_happy="no"])