/* -*- C -*-
 *
 * Copyright (c) 2008 Los Alamos National Security, LLC.  All rights reserved.
 *
 * Copyright (c) 2015 Cisco Systems, Inc.  All rights reserved.
 * $COPYRIGHT$
 *
 * Additional copyrights may follow
 *
 * $HEADER$
 *
 */
#include <stdio.h>
#include <stdbool.h>
#include <sys/types.h>
#include <unistd.h>
#include <stdlib.h>
#include <time.h>
#include <sys/time.h>

#include <mpi.h>

int main(int argc, char* argv[])
{
    int msg;
    int rank, size, my_twin;
    int ppn, my_node;
    struct timeval tv;
    unsigned long my_timestamp[2];
    long *timestamps;
    int i, maxrank;
    unsigned long maxsec, maxusec, minutes, seconds;
    unsigned long start_sec, start_usec;
    float fsecs;
    int nnodes;
    bool odd_nnodes;
    bool recvit;
    char *ppnstr;

    if (argc < 3) {
        fprintf(stderr, "start times must be provided\n");
        return 1;
    }

    ppnstr = getenv("OMPI_COMM_WORLD_LOCAL_SIZE");
    ppn = strtol(ppnstr, NULL, 10);
    start_sec = strtol(argv[1], NULL, 10);
    start_usec = strtol(argv[2], NULL, 10);

    MPI_Init(NULL, NULL);
    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
    MPI_Comm_size(MPI_COMM_WORLD, &size);

    /* this program requires that the size be an integer multiple of ppn */
    if (0 != (size % ppn)) {
        if (0 == rank) {
            fprintf(stderr, "The number of procs must be an integer multiple of the ppn\n"
                    "Given: num_procs %d ppn %d\n", size, ppn);
            MPI_Abort(MPI_COMM_WORLD, 1);
        } else {
            goto cleanup;
        }
    }

    /* see how many nodes we have */
    nnodes = size / ppn;

    odd_nnodes = false;
    if (0 != (nnodes % 2)) {
        /* we have an odd # of nodes */
        odd_nnodes = true;
    }

    /* compute the rank of the rank with which I am to exchange a message.
     * Per requirements, this proc must be on another node. To accomplish
     * this with max efficiency, we take advantage of knowing that the ppn
     * on every node will be the same. We therefore pair up the nodes, and
     * pair up the procs on each node, so that only one connection is setup
     * for each proc. We also want to ensure that the node pairs are
     * "neighboring" - i.e., that they hopefully share a switch so that the
     * hop count of sending the messages is minimized.
     */

    /* first, determine if my node is odd or even */
    my_node = rank / ppn;

     if (0 != (my_node % 2)) {
        /* compute my twin's rank - as I am an odd numbered node, my
         * twin will be on the node below me. Thus, its rank will be
         * my rank - ppn
         */
        my_twin = rank - ppn;
        /* if I am an odd numbered node, then I will receive first */
        MPI_Recv(&msg, 1, MPI_INT, my_twin, 1, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
        /* receive the return message so that we meet the stated requirement
         * that -every- proc send a message
         */
        MPI_Send(&msg, 1, MPI_INT, my_twin, 1, MPI_COMM_WORLD);
    } else {
        /* compute my twin's rank - as I am an even numbered node, my
         * twin will be on the node above me. Thus, its rank will be
         * my rank + ppn
         */
        my_twin = rank + ppn;
        /* if we have an odd number of nodes, then the last node will be
         * even and will have no one above them. In this case, we wrap around
         * and ask that node=0 take the additional connections
         */
        recvit = true;
        if (my_twin >= size) {
            my_twin = my_twin - size;
            recvit = false;
        }
        /* I am an even numbered node, so I send first */
        MPI_Send(&msg, 1, MPI_INT, my_twin, 1, MPI_COMM_WORLD);
        /* now receive the reply so my twin also meets the requirement - but only
         * if we don't have an odd number of nodes. If we have an odd number of
         * nodes, then the node=0 procs will already have met their requirement
         */
        if (recvit) {
            MPI_Recv(&msg, 1, MPI_INT, my_twin, 1, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
        }
    }

    /* if we have an odd number of nodes and I am on node=0, then I have
     * to take the extra recv
     */
    if (odd_nnodes && 0 == my_node) {
        my_twin = size - ppn + rank;
        MPI_Recv(&msg, 1, MPI_INT, my_twin, 1, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
    }

    /* get a completion time stamp */
    gettimeofday(&tv, NULL);
    my_timestamp[0] = tv.tv_sec;
    my_timestamp[1] = tv.tv_usec;

    /* THIS COMPLETES THE OFFICIAL TIMING POINT */

    /* Gather to get all the timestamps to rank 0 */
    timestamps = NULL;
    if (0 == rank) {
        timestamps = malloc(2 * size * sizeof(unsigned long));
        if (NULL == timestamps) {
            MPI_Abort(MPI_COMM_WORLD, 1);
        }
    }
    MPI_Gather(&my_timestamp, 2, MPI_LONG,
               timestamps, 2, MPI_LONG, 0, MPI_COMM_WORLD);
    if (0 == rank) {
        /* The "timestamps" array will now have everyone's timestamp
         (i.e., rank 0's timestamp will be in pos 0 & 1,, rank 1's timestamp
         will be in 2 & 3, ...etc. */
        /* find the maximum timestamp */
        maxsec = start_sec;
        maxusec = start_usec;
        maxrank = -1;
        for (i=0; i < 2*size; i+=2) {
            if (timestamps[i] < maxsec) {
                continue;
            }
            if (timestamps[i] == maxsec &&
                timestamps[i+1] < maxusec) {
                continue;
            }
            maxsec = timestamps[i];
            maxusec = timestamps[i+1];
            maxrank = i/2;
        }
        free(timestamps);
        /* subtract starting time to get time in microsecs for test */
        maxsec = maxsec - start_sec;
        if (maxusec >= start_usec) {
            maxusec = maxusec - start_usec;
        } else {
            maxsec--;
            maxusec = 1000000 - start_usec + maxusec;
        }
        /* pretty-print the result */
        seconds = maxsec + (maxusec / 1000000l);
        minutes = seconds / 60l;
        seconds = seconds % 60l;
        if (0 == minutes && 0 == seconds) {
            fsecs = ((float)(maxsec)*1000000.0 + (float)maxusec) / 1000.0;
            fprintf(stderr, "Time test was completed in %8.2f millisecs\nSlowest rank: %d\n",
                    fsecs, maxrank);
        } else {
            fprintf(stderr, "Time test was completed in %3lu:%02lu min:sec\nSlowest rank: %d\n",
                    minutes, seconds, maxrank);
        }
    }

cleanup:
    /* this completes the test */
    MPI_Finalize();

    return 0;
}