Btl tcp: Using magic string to verify mpi connection

As part of improvement towards handling failure case
in btl tcp we are using magic string to verify mpi
connection. In case if there is mismatch or missing
magic string we can identify that we are trying to
connect with someother process.

Signed-off-by: Mohan Gandhi <mohgan@amazon.com>

opal/mca/btl/tcp/btl_tcp_component.c

@@ -1300,11 +1300,62 @@ static void mca_btl_tcp_component_recv_handler(int sd, short flags, void* user)
     opal_socklen_t addr_len = sizeof(addr);
     mca_btl_tcp_proc_t* btl_proc;
     int retval;
+    char str[128];
+    size_t len = strlen(mca_btl_tcp_magic_id_string);
 
     OBJ_RELEASE(event);
 
+    /* Receive the magic string */
+    assert(len < sizeof(str));
+    str[0] = '\0';
+    /* TODO: recv_blocking() will block forever, without timeout
+     * There is chance of spin forever if it tries to connect to old version
+     * as older version will send just process id which won't be long enough
+     * to cross sizeof(str) length. Probably better to have a timeout
+     * (say, 10 seconds) for receiving the magic string, and giving up after that.
+     * Will be reiterating this logic in my next iteration
+     */
+    retval = mca_btl_tcp_recv_blocking(sd, str, len);
+    if (retval > 0) {
+        str[retval] = '\0';
+    }
+
+    /* An unknown process attempted to connect to Open MPI via TCP.
+     * Open MPI uses a "magic" string to trivially verify that the connecting
+     * process is a fellow Open MPI process.  An MPI process accepted a TCP
+     * connection but did not receive the correct magic string.  This might
+     * indicate an Open MPI version mismatch between different MPI processes
+     * in the same job, or it may indicate that some other agent is
+     * mistakenly attempting to connect to Open MPI's TCP listening sockets.
+
+     * This attempted connection will be ignored; your MPI job may or may not
+     * continue properly.
+     */
+    if (retval != (int) len) {
+        opal_output_verbose(20, opal_btl_base_framework.framework_output,
+                            "server did not receive magic string. "
+                            "Local_host:%s PID:%d Role:%s String_received:%s Test_fail:%s",
+                            opal_process_info.nodename,
+                            getpid(), "server",
+                            (len > 0) ? str : "<nothing>", "string length");
+
+        /* The other side probably isn't OMPI, so just hang up */
+        CLOSE_THE_SOCKET(sd);
+        return;
+    }
+    if (0 != strncmp(str, mca_btl_tcp_magic_id_string, len)) {
+        opal_output_verbose(20, opal_btl_base_framework.framework_output,
+                            "server did not receive right magic string. "
+                            "Local_host:%s PID:%d Role:%s String_received:%s Test_fail:%s",
+                            opal_process_info.nodename,
+                            getpid(), "server", str,
+                            "string value");
+        /* The other side probably isn't OMPI, so just hang up */
+        CLOSE_THE_SOCKET(sd);
+        return;
+    }
     /* recv the process identifier */
-    retval = recv(sd, (char *)&guid, sizeof(guid), 0);
+    retval = mca_btl_tcp_recv_blocking(sd, (char *)&guid, sizeof(guid));
     if(retval != sizeof(guid)) {
         CLOSE_THE_SOCKET(sd);
         return;

opal/mca/btl/tcp/btl_tcp_endpoint.c

@@ -62,6 +62,10 @@
 #include "btl_tcp_frag.h"
 #include "btl_tcp_addr.h"
 
+/*
+ * Magic ID string send during connect/accept handshake
+ */
+const char mca_btl_tcp_magic_id_string[] = "OPAL-TCP-BTL";
 
 /*
  * Initialize state of the endpoint instance.
@@ -393,9 +397,24 @@ mca_btl_tcp_endpoint_send_blocking(mca_btl_base_endpoint_t* btl_endpoint,
 
 static int mca_btl_tcp_endpoint_send_connect_ack(mca_btl_base_endpoint_t* btl_endpoint)
 {
-    /* send process identifier to remote endpoint */
+
     opal_process_name_t guid = opal_proc_local_get()->proc_name;
 
+    int len = (int) strlen(mca_btl_tcp_magic_id_string);
+
+    /* send magic string to the remote endpoint, identifying me as a
+       fellow Open MPI TCP BTL */
+    if (mca_btl_tcp_endpoint_send_blocking(btl_endpoint,
+                                           mca_btl_tcp_magic_id_string,
+                                           len) != len) {
+        opal_show_help("help-mpi-btl-tcp.txt", "client handshake fail",
+                       true, opal_process_info.nodename,
+                       getpid(),
+                       "failed to send magic ID string");
+        return OPAL_ERR_UNREACH;
+    }
+
+    /* send process identifier to remote endpoint */
     OPAL_PROCESS_NAME_HTON(guid);
     if(mca_btl_tcp_endpoint_send_blocking(btl_endpoint, &guid, sizeof(guid)) !=
           sizeof(guid)) {
@@ -572,12 +591,39 @@ static int mca_btl_tcp_endpoint_recv_blocking(mca_btl_base_endpoint_t* btl_endpo
  *  Receive the endpoints globally unique process identification from a newly
  *  connected socket and verify the expected response. If so, move the
  *  socket to a connected state.
+ *
+ *  NOTE: The return codes from this function are checked in
+ *  mca_btl_tcp_endpoint_recv_handler().  Don't change them here
+ *  without also changing the handling in _recv_handler()!
  */
 static int mca_btl_tcp_endpoint_recv_connect_ack(mca_btl_base_endpoint_t* btl_endpoint)
 {
-    size_t s;
+    size_t s, len = strlen(mca_btl_tcp_magic_id_string);;
     opal_process_name_t guid;
     mca_btl_tcp_proc_t* btl_proc = btl_endpoint->endpoint_proc;
+    char msg[1024];
+
+    /* First get magic string indicating that the connector is an Open MPI TCP BTL */
+    assert(len < sizeof(msg));
+    msg[0] = '\0';
+    s = mca_btl_tcp_endpoint_recv_blocking(btl_endpoint, msg, len);
+    if (s > 0) {
+        msg[s] = '\0';
+    }
+    if (s != len) {
+        opal_show_help("help-mpi-btl-tcp.txt", "did not receive magic string",
+                       true, opal_process_info.nodename,
+                       getpid(), "client",
+                       (s > 0) ? msg : "<nothing>", "string length");
+        return OPAL_ERR_BAD_PARAM;
+    }
+    if (0 != strncmp(msg, mca_btl_tcp_magic_id_string, len)) {
+        opal_show_help("help-mpi-btl-tcp.txt", "did not receive magic string",
+                       true, opal_process_info.nodename,
+                       getpid(), "client", msg,
+                       "string value");
+        return OPAL_ERR_BAD_PARAM;
+    }
 
     s = mca_btl_tcp_endpoint_recv_blocking(btl_endpoint,
                                            &guid, sizeof(opal_process_name_t));

opal/mca/btl/tcp/btl_tcp_endpoint.h

@@ -75,6 +75,9 @@ typedef struct mca_btl_base_endpoint_t mca_btl_base_endpoint_t;
 typedef mca_btl_base_endpoint_t  mca_btl_tcp_endpoint_t;
 OBJ_CLASS_DECLARATION(mca_btl_tcp_endpoint_t);
 
+/* Magic socket handshake string */
+extern const char mca_btl_tcp_magic_id_string[];
+
 void mca_btl_tcp_set_socket_options(int sd);
 void mca_btl_tcp_endpoint_close(mca_btl_base_endpoint_t*);
 int  mca_btl_tcp_endpoint_send(mca_btl_base_endpoint_t*, struct mca_btl_tcp_frag_t*);