coll/basic: fix neighbor alltoall message ordering

This commit updates the coll/basic component to correctly order sends
and receives for cartesian communicators with cyclic boundaries. This
addresses an issue identified by mpi-forum/mpi-issues#153. This issue
occurs when the size in any dimension is 1. This gives the same
neighbor in the positive and negative directions. The old code was
sending and receiving in the same order so the -1 buffer contained
the +1 result and vise-versa. The problem is addressed by using
unique tags for each send. This should cover both the case where
overtaking is allowed and is not allowed. The former case will be
possible is a MPI_Cart_create_with_info() call is added to the
standard.

Signed-off-by: Nathan Hjelm <hjelmn@google.com>
(cherry picked from commit 196a91e604885d7aae9ac9dfbd9b2e846b3015b7)

ompi/mca/coll/basic/coll_basic_neighbor_alltoall.c

@@ -15,6 +15,7 @@
  * Copyright (c) 2014-2015 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
  * Copyright (c) 2017      IBM Corporation. All rights reserved.
+ * Copyright (c) 2019      Google, LLC. All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -67,7 +68,7 @@ mca_coll_basic_neighbor_alltoall_cart(const void *sbuf, int scount, struct ompi_
         if (MPI_PROC_NULL != srank) {
             nreqs++;
             rc = MCA_PML_CALL(irecv(rbuf, rcount, rdtype, srank,
-                                    MCA_COLL_BASE_TAG_ALLTOALL,
+                                    MCA_COLL_BASE_TAG_NONBLOCKING_BASE - 2 * dim,
                                     comm, preqs++));
             if (OMPI_SUCCESS != rc) break;
         }
@@ -77,7 +78,7 @@ mca_coll_basic_neighbor_alltoall_cart(const void *sbuf, int scount, struct ompi_
         if (MPI_PROC_NULL != drank) {
             nreqs++;
             rc = MCA_PML_CALL(irecv(rbuf, rcount, rdtype, drank,
-                                    MCA_COLL_BASE_TAG_ALLTOALL,
+                                    MCA_COLL_BASE_TAG_NONBLOCKING_BASE - 2 * dim - 1,
                                     comm, preqs++));
             if (OMPI_SUCCESS != rc) break;
         }
@@ -104,7 +105,7 @@ mca_coll_basic_neighbor_alltoall_cart(const void *sbuf, int scount, struct ompi_
              * a const for the send buffer. */
             nreqs++;
             rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, srank,
-                                    MCA_COLL_BASE_TAG_ALLTOALL,
+                                    MCA_COLL_BASE_TAG_NONBLOCKING_BASE - 2 * dim - 1,
                                     MCA_PML_BASE_SEND_STANDARD,
                                     comm, preqs++));
             if (OMPI_SUCCESS != rc) break;
@@ -115,7 +116,7 @@ mca_coll_basic_neighbor_alltoall_cart(const void *sbuf, int scount, struct ompi_
         if (MPI_PROC_NULL != drank) {
             nreqs++;
             rc = MCA_PML_CALL(isend((void *) sbuf, scount, sdtype, drank,
-                                    MCA_COLL_BASE_TAG_ALLTOALL,
+                                    MCA_COLL_BASE_TAG_NONBLOCKING_BASE - 2 * dim,
                                     MCA_PML_BASE_SEND_STANDARD,
                                     comm, preqs++));
             if (OMPI_SUCCESS != rc) break;