Merge pull request #4885 from davideberius/spc_pr

Initial Software-based Performance Counters PR

configure.ac

@@ -3,7 +3,7 @@
 # Copyright (c) 2004-2009 The Trustees of Indiana University and Indiana
 #                         University Research and Technology
 #                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2015 The University of Tennessee and The University
+# Copyright (c) 2004-2018 The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
 # Copyright (c) 2004-2007 High Performance Computing Center Stuttgart,
@@ -275,6 +275,24 @@ m4_ifndef([project_orte], [project_orte_amc=false])
 AS_IF([test "$enable_oshmem" != "no"], [project_oshmem_amc=true], [project_oshmem_amc="no (disabled)"])
 m4_ifndef([project_oshmem], [project_oshmem_amc="no (not available)"])
 
+# Enable/Disable Software-Based Performance Counters Capability
+AC_ARG_ENABLE(spc,
+    AC_HELP_STRING([--enable-spc],
+                   [Enable software-based performance counters capability (default: disabled)]))
+if test "$enable_spc" = "yes"; then
+    AC_MSG_RESULT([yes])
+    SPC_ENABLE=1
+else
+    AC_MSG_RESULT([no])
+    SPC_ENABLE=0
+fi
+AC_DEFINE_UNQUOTED([SPC_ENABLE],
+                   [$SPC_ENABLE],
+                   [If the software-based performance counters capability should be enabled.])
+AM_CONDITIONAL(SPC_ENABLE, test "$SPC_ENABLE" = "1")
+
+AS_IF([test "$enable_spc" != "no"], [project_spc_amc=true], [project_spc_amc=false])
+
 if test "$enable_binaries" = "no" && test "$enable_dist" = "yes"; then
     AC_MSG_WARN([--disable-binaries is incompatible with --enable dist])
     AC_MSG_ERROR([Cannot continue])
@@ -1441,7 +1459,7 @@ AC_CONFIG_FILES([
     test/util/Makefile
 ])
 
-m4_ifdef([project_ompi], [AC_CONFIG_FILES([test/monitoring/Makefile])])
+m4_ifdef([project_ompi], [AC_CONFIG_FILES([test/monitoring/Makefile test/spc/Makefile])])
 
 AC_CONFIG_FILES([contrib/dist/mofed/debian/rules],
                 [chmod +x contrib/dist/mofed/debian/rules])

examples/Makefile

@@ -2,7 +2,7 @@
 # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
 #                         University Research and Technology
 #                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
+# Copyright (c) 2004-2018 The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
 # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -13,7 +13,7 @@
 # Copyright (c) 2011-2016 Cisco Systems, Inc.  All rights reserved.
 # Copyright (c) 2012      Los Alamos National Security, Inc.  All rights reserved.
 # Copyright (c) 2013      Mellanox Technologies, Inc.  All rights reserved.
-# Copyright (c) 2017      Research Organization for Information Science
+# Copyright (c) 2017-2018 Research Organization for Information Science
 #                         and Technology (RIST). All rights reserved.
 # $COPYRIGHT$
 #
@@ -67,14 +67,15 @@ EXAMPLES = \
 	 oshmem_circular_shift \
 	 oshmem_max_reduction \
 	 oshmem_strided_puts \
-	 oshmem_symmetric_data
+	 oshmem_symmetric_data \
+	 spc_example
 
 
 # Default target.  Always build the C MPI examples.  Only build the
 # others if we have the appropriate Open MPI / OpenSHMEM language
 # bindings.
 
-all: hello_c ring_c connectivity_c
+all: hello_c ring_c connectivity_c spc_example
 	@ if which ompi_info >/dev/null 2>&1 ; then \
 	    $(MAKE) mpi; \
 	fi
@@ -132,6 +133,8 @@ ring_c: ring_c.c
 	$(MPICC) $(CFLAGS) $(LDFLAGS) $? $(LDLIBS) -o $@
 connectivity_c: connectivity_c.c
 	$(MPICC) $(CFLAGS) $(LDFLAGS) $? $(LDLIBS) -o $@
+spc_example: spc_example.c
+	$(MPICC) $(CFLAGS) $(LDFLAGS) $? $(LDLIBS) -o $@
 
 hello_cxx: hello_cxx.cc
 	$(MPICXX) $(CXXFLAGS) $(LDFLAGS) $? $(LDLIBS) -o $@

examples/Makefile.include

@@ -3,7 +3,7 @@
 # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
 #                         University Research and Technology
 #                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
+# Copyright (c) 2004-2018 The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
 # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -57,4 +57,5 @@ EXTRA_DIST += \
         examples/oshmem_strided_puts.c \
         examples/oshmem_symmetric_data.c \
         examples/Hello.java \
-        examples/Ring.java
+        examples/Ring.java \
+        examples/spc_example.c

examples/spc_example.c

@@ -0,0 +1,120 @@
+/*
+ * Copyright (c) 2018      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ *
+ * Simple example usage of SPCs through MPI_T.
+ */
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "mpi.h"
+
+/* Sends 'num_messages' messages of 'message_size' bytes from rank 0 to rank 1.
+ * All messages are send synchronously and with the same tag in MPI_COMM_WORLD.
+ */
+void message_exchange(int num_messages, int message_size)
+{
+    int i, rank;
+    /* Use calloc to initialize data to 0's */
+    char *data = (char*)calloc(message_size, sizeof(char));
+    MPI_Status status;
+
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+
+    if(rank == 0) {
+        for(i = 0; i < num_messages; i++)
+            MPI_Send(data, message_size, MPI_BYTE, 1, 123, MPI_COMM_WORLD);
+    } else if(rank == 1) {
+        for(i = 0; i < num_messages; i++)
+            MPI_Recv(data, message_size, MPI_BYTE, 0, 123, MPI_COMM_WORLD, &status);
+    }
+
+    free(data);
+}
+
+int main(int argc, char **argv)
+{
+    int num_messages, message_size;
+
+    if(argc < 3) {
+        printf("Usage: mpirun -np 2 --mca mpi_spc_attach all --mca mpi_spc_dump_enabled true ./spc_example [num_messages] [message_size]\n");
+        return -1;
+    } else {
+        num_messages = atoi(argv[1]);
+        message_size = atoi(argv[2]);
+    }
+
+    int i, rank, size, provided, num, name_len, desc_len, verbosity, bind, var_class, readonly, continuous, atomic, count, index;
+    MPI_Datatype datatype;
+    MPI_T_enum enumtype;
+    MPI_Comm comm;
+    char name[256], description[256];
+
+    /* Counter names to be read by ranks 0 and 1 */
+    char *counter_names[] = {"runtime_spc_OMPI_BYTES_SENT_USER",
+                             "runtime_spc_OMPI_BYTES_RECEIVED_USER" };
+
+    MPI_Init(NULL, NULL);
+    MPI_T_init_thread(MPI_THREAD_SINGLE, &provided);
+
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    MPI_Comm_size(MPI_COMM_WORLD, &size);
+    if(size != 2) {
+        fprintf(stderr, "ERROR: This test should be run with two MPI processes.\n");
+        MPI_Abort(MPI_COMM_WORLD, -1);
+    }
+
+    /* Determine the MPI_T pvar indices for the OMPI_BYTES_SENT/RECIEVED_USER SPCs */
+    index = -1;
+    MPI_T_pvar_get_num(&num);
+    for(i = 0; i < num; i++) {
+        name_len = desc_len = 256;
+        PMPI_T_pvar_get_info(i, name, &name_len, &verbosity,
+                             &var_class, &datatype, &enumtype, description, &desc_len, &bind,
+                             &readonly, &continuous, &atomic);
+        if(strcmp(name, counter_names[rank]) == 0) {
+            index = i;
+            printf("[%d] %s -> %s\n", rank, name, description);
+        }
+    }
+
+    /* Make sure we found the counters */
+    if(index == -1) {
+        fprintf(stderr, "ERROR: Couldn't find the appropriate SPC counter in the MPI_T pvars.\n");
+        MPI_Abort(MPI_COMM_WORLD, -1);
+    }
+
+    int ret;
+    long long value;
+
+    MPI_T_pvar_session session;
+    MPI_T_pvar_handle handle;
+    /* Create the MPI_T sessions/handles for the counters and start the counters */
+    ret = MPI_T_pvar_session_create(&session);
+    ret = MPI_T_pvar_handle_alloc(session, index, NULL, &handle, &count);
+    ret = MPI_T_pvar_start(session, handle);
+
+    message_exchange(num_messages, message_size);
+
+    ret = MPI_T_pvar_read(session, handle, &value);
+    /* Print the counter values in order by rank */
+    for(i = 0; i < 2; i++) {
+        if(i == rank) {
+            printf("[%d] Value Read: %lld\n", rank, value);
+            fflush(stdout);
+        }
+        MPI_Barrier(MPI_COMM_WORLD);
+    }
+    /* Stop the MPI_T session, free the handle, and then free the session */
+    ret = MPI_T_pvar_stop(session, handle);
+    ret = MPI_T_pvar_handle_free(session, &handle);
+    ret = MPI_T_pvar_session_free(&session);
+
+    MPI_T_finalize();
+    MPI_Finalize();
+
+    return 0;
+}

ompi/mca/pml/ob1/pml_ob1_isend.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2015 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -28,6 +28,7 @@
 #include "pml_ob1_sendreq.h"
 #include "pml_ob1_recvreq.h"
 #include "ompi/peruse/peruse-internal.h"
+#include "ompi/runtime/ompi_spc.h"
 
 /**
  * Single usage request. As we allow recursive calls (as an
@@ -119,6 +120,19 @@ static inline int mca_pml_ob1_send_inline (const void *buf, size_t count,
     rc = mca_bml_base_sendi (bml_btl, &convertor, &match, OMPI_PML_OB1_MATCH_HDR_LEN,
                              size, MCA_BTL_NO_ORDER, MCA_BTL_DES_FLAGS_PRIORITY | MCA_BTL_DES_FLAGS_BTL_OWNERSHIP,
                              MCA_PML_OB1_HDR_TYPE_MATCH, NULL);
+
+    /* This #if is required due to an issue that arises with the IBM CI (XL Compiler).
+     * The compiler doesn't seem to like having a compiler hint attached to an if
+     * statement that only has a no-op inside and has the following error:
+     *
+     * 1586-494 (U) INTERNAL COMPILER ERROR: Signal 11.
+     */
+#if SPC_ENABLE == 1
+    if(OPAL_LIKELY(rc == OPAL_SUCCESS)) {
+        SPC_USER_OR_MPI(tag, (ompi_spc_value_t)size, OMPI_SPC_BYTES_SENT_USER, OMPI_SPC_BYTES_SENT_MPI);
+    }
+#endif
+
     if (count > 0) {
         opal_convertor_cleanup (&convertor);
     }

ompi/mca/pml/ob1/pml_ob1_recvfrag.c

@@ -39,6 +39,7 @@
 #include "ompi/mca/pml/pml.h"
 #include "ompi/peruse/peruse-internal.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #include "pml_ob1.h"
 #include "pml_ob1_comm.h"
@@ -388,6 +389,7 @@ void mca_pml_ob1_recv_frag_callback_match(mca_btl_base_module_t* btl,
             MCA_PML_OB1_RECV_FRAG_ALLOC(frag);
             MCA_PML_OB1_RECV_FRAG_INIT(frag, hdr, segments, num_segments, btl);
             append_frag_to_ordered_list(&proc->frags_cant_match, frag, proc->expected_sequence);
+            SPC_RECORD(OMPI_SPC_OUT_OF_SEQUENCE, 1);
             OB1_MATCHING_UNLOCK(&comm->matching_lock);
             return;
         }
@@ -453,6 +455,8 @@ void mca_pml_ob1_recv_frag_callback_match(mca_btl_base_module_t* btl,
                                    &iov_count,
                                    &bytes_received );
             match->req_bytes_received = bytes_received;
+            SPC_USER_OR_MPI(match->req_recv.req_base.req_ompi.req_status.MPI_TAG, (ompi_spc_value_t)bytes_received,
+                            OMPI_SPC_BYTES_RECEIVED_USER, OMPI_SPC_BYTES_RECEIVED_MPI);
             /*
              *  Unpacking finished, make the user buffer unaccessable again.
              */
@@ -777,6 +781,11 @@ match_one(mca_btl_base_module_t *btl,
           mca_pml_ob1_comm_proc_t *proc,
           mca_pml_ob1_recv_frag_t* frag)
 {
+#if SPC_ENABLE == 1
+    opal_timer_t timer = 0;
+#endif
+    SPC_TIMER_START(OMPI_SPC_MATCH_TIME, &timer);
+
     mca_pml_ob1_recv_request_t *match;
     mca_pml_ob1_comm_t *comm = (mca_pml_ob1_comm_t *)comm_ptr->c_pml_comm;
 
@@ -814,19 +823,25 @@ match_one(mca_btl_base_module_t *btl,
                                                        num_segments);
                 /* this frag is already processed, so we want to break out
                    of the loop and not end up back on the unexpected queue. */
+                SPC_TIMER_STOP(OMPI_SPC_MATCH_TIME, &timer);
                 return NULL;
             }
 
             PERUSE_TRACE_COMM_EVENT(PERUSE_COMM_MSG_MATCH_POSTED_REQ,
                                     &(match->req_recv.req_base), PERUSE_RECV);
+            SPC_TIMER_STOP(OMPI_SPC_MATCH_TIME, &timer);
             return match;
         }
 
         /* if no match found, place on unexpected queue */
         append_frag_to_list(&proc->unexpected_frags, btl, hdr, segments,
                             num_segments, frag);
+        SPC_RECORD(OMPI_SPC_UNEXPECTED, 1);
+        SPC_RECORD(OMPI_SPC_UNEXPECTED_IN_QUEUE, 1);
+        SPC_UPDATE_WATERMARK(OMPI_SPC_MAX_UNEXPECTED_IN_QUEUE, OMPI_SPC_UNEXPECTED_IN_QUEUE);
         PERUSE_TRACE_MSG_EVENT(PERUSE_COMM_MSG_INSERT_IN_UNEX_Q, comm_ptr,
                                hdr->hdr_src, hdr->hdr_tag, PERUSE_RECV);
+        SPC_TIMER_STOP(OMPI_SPC_MATCH_TIME, &timer);
         return NULL;
     } while(true);
 }
@@ -920,6 +935,11 @@ static int mca_pml_ob1_recv_frag_match( mca_btl_base_module_t *btl,
         MCA_PML_OB1_RECV_FRAG_ALLOC(frag);
         MCA_PML_OB1_RECV_FRAG_INIT(frag, hdr, segments, num_segments, btl);
         append_frag_to_ordered_list(&proc->frags_cant_match, frag, next_msg_seq_expected);
+
+        SPC_RECORD(OMPI_SPC_OUT_OF_SEQUENCE, 1);
+        SPC_RECORD(OMPI_SPC_OOS_IN_QUEUE, 1);
+        SPC_UPDATE_WATERMARK(OMPI_SPC_MAX_OOS_IN_QUEUE, OMPI_SPC_OOS_IN_QUEUE);
+
         OB1_MATCHING_UNLOCK(&comm->matching_lock);
         return OMPI_SUCCESS;
     }

ompi/mca/pml/ob1/pml_ob1_recvreq.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -29,6 +29,7 @@
 
 #include "opal/mca/mpool/mpool.h"
 #include "opal/util/arch.h"
+#include "ompi/runtime/ompi_spc.h"
 #include "ompi/mca/pml/pml.h"
 #include "ompi/mca/bml/bml.h"
 #include "pml_ob1_comm.h"
@@ -199,6 +200,8 @@ static void mca_pml_ob1_put_completion (mca_pml_ob1_rdma_frag_t *frag, int64_t r
 
         /* check completion status */
         OPAL_THREAD_ADD_FETCH_SIZE_T(&recvreq->req_bytes_received, rdma_size);
+        SPC_USER_OR_MPI(recvreq->req_recv.req_base.req_ompi.req_status.MPI_TAG, (ompi_spc_value_t)rdma_size,
+                        OMPI_SPC_BYTES_RECEIVED_USER, OMPI_SPC_BYTES_RECEIVED_MPI);
         if (recv_request_pml_complete_check(recvreq) == false &&
             recvreq->req_rdma_offset < recvreq->req_send_offset) {
             /* schedule additional rdma operations */
@@ -242,6 +245,7 @@ int mca_pml_ob1_recv_request_ack_send_btl(
     des->des_cbfunc = mca_pml_ob1_recv_ctl_completion;
 
     rc = mca_bml_base_send(bml_btl, des, MCA_PML_OB1_HDR_TYPE_ACK);
+    SPC_RECORD(OMPI_SPC_BYTES_RECEIVED_MPI, (ompi_spc_value_t)size);
     if( OPAL_LIKELY( rc >= 0 ) ) {
         return OMPI_SUCCESS;
     }
@@ -374,6 +378,8 @@ static void mca_pml_ob1_rget_completion (mca_btl_base_module_t* btl, struct mca_
     } else {
         /* is receive request complete */
         OPAL_THREAD_ADD_FETCH_SIZE_T(&recvreq->req_bytes_received, frag->rdma_length);
+        SPC_USER_OR_MPI(recvreq->req_recv.req_base.req_tag, (ompi_spc_value_t)frag->rdma_length,
+                        OMPI_SPC_BYTES_RECEIVED_USER, OMPI_SPC_BYTES_RECEIVED_MPI);
         /* TODO: re-add order */
         mca_pml_ob1_send_fin (recvreq->req_recv.req_base.req_proc,
                               bml_btl, frag->rdma_hdr.hdr_rget.hdr_frag,
@@ -429,6 +435,8 @@ static int mca_pml_ob1_recv_request_put_frag (mca_pml_ob1_rdma_frag_t *frag)
 
     /* send rdma request to peer */
     rc = mca_bml_base_send (bml_btl, ctl, MCA_PML_OB1_HDR_TYPE_PUT);
+    /* Increment counter for bytes_put even though they probably haven't all been received yet */
+    SPC_RECORD(OMPI_SPC_BYTES_PUT, (ompi_spc_value_t)frag->rdma_length);
     if (OPAL_UNLIKELY(rc < 0)) {
         mca_bml_base_free (bml_btl, ctl);
         return rc;
@@ -470,6 +478,8 @@ int mca_pml_ob1_recv_request_get_frag (mca_pml_ob1_rdma_frag_t *frag)
     rc = mca_bml_base_get (bml_btl, frag->local_address, frag->remote_address, local_handle,
                            (mca_btl_base_registration_handle_t *) frag->remote_handle, frag->rdma_length,
                            0, MCA_BTL_NO_ORDER, mca_pml_ob1_rget_completion, frag);
+    /* Increment counter for bytes_get even though they probably haven't all been received yet */
+    SPC_RECORD(OMPI_SPC_BYTES_GET, (ompi_spc_value_t)frag->rdma_length);
     if( OPAL_UNLIKELY(OMPI_SUCCESS > rc) ) {
         return mca_pml_ob1_recv_request_get_frag_failed (frag, OMPI_ERR_OUT_OF_RESOURCE);
     }
@@ -525,6 +535,8 @@ void mca_pml_ob1_recv_request_progress_frag( mca_pml_ob1_recv_request_t* recvreq
                );
 
     OPAL_THREAD_ADD_FETCH_SIZE_T(&recvreq->req_bytes_received, bytes_received);
+    SPC_USER_OR_MPI(recvreq->req_recv.req_base.req_ompi.req_status.MPI_TAG, (ompi_spc_value_t)bytes_received,
+                    OMPI_SPC_BYTES_RECEIVED_USER, OMPI_SPC_BYTES_RECEIVED_MPI);
     /* check completion status */
     if(recv_request_pml_complete_check(recvreq) == false &&
             recvreq->req_rdma_offset < recvreq->req_send_offset) {
@@ -602,7 +614,8 @@ void mca_pml_ob1_recv_request_frag_copy_finished( mca_btl_base_module_t* btl,
     des->des_cbfunc(NULL, NULL, des, 0);
 
     OPAL_THREAD_ADD_FETCH_SIZE_T(&recvreq->req_bytes_received, bytes_received);
-
+    SPC_USER_OR_MPI(recvreq->req_recv.req_base.req_ompi.req_status.MPI_TAG, (ompi_spc_value_t)bytes_received,
+                    OMPI_SPC_BYTES_RECEIVED_USER, OMPI_SPC_BYTES_RECEIVED_MPI);
     /* check completion status */
     if(recv_request_pml_complete_check(recvreq) == false &&
             recvreq->req_rdma_offset < recvreq->req_send_offset) {
@@ -816,6 +829,8 @@ void mca_pml_ob1_recv_request_progress_rndv( mca_pml_ob1_recv_request_t* recvreq
                                    recvreq->req_recv.req_base.req_datatype);
                    );
         OPAL_THREAD_ADD_FETCH_SIZE_T(&recvreq->req_bytes_received, bytes_received);
+        SPC_USER_OR_MPI(recvreq->req_recv.req_base.req_ompi.req_status.MPI_TAG, (ompi_spc_value_t)bytes_received,
+                        OMPI_SPC_BYTES_RECEIVED_USER, OMPI_SPC_BYTES_RECEIVED_MPI);
     }
     /* check completion status */
     if(recv_request_pml_complete_check(recvreq) == false &&
@@ -886,6 +901,8 @@ void mca_pml_ob1_recv_request_progress_match( mca_pml_ob1_recv_request_t* recvre
      * for this request.
      */
     recvreq->req_bytes_received += bytes_received;
+    SPC_USER_OR_MPI(recvreq->req_recv.req_base.req_ompi.req_status.MPI_TAG, (ompi_spc_value_t)bytes_received,
+                    OMPI_SPC_BYTES_RECEIVED_USER, OMPI_SPC_BYTES_RECEIVED_MPI);
     recv_request_pml_complete(recvreq);
 }
 
@@ -1228,6 +1245,7 @@ void mca_pml_ob1_recv_req_start(mca_pml_ob1_recv_request_t *req)
 
             opal_list_remove_item(&proc->unexpected_frags,
                                   (opal_list_item_t*)frag);
+            SPC_RECORD(OMPI_SPC_UNEXPECTED_IN_QUEUE, -1);
             OB1_MATCHING_UNLOCK(&ob1_comm->matching_lock);
 
             switch(hdr->hdr_common.hdr_type) {
@@ -1258,6 +1276,7 @@ void mca_pml_ob1_recv_req_start(mca_pml_ob1_recv_request_t *req)
                restarted with this request during mrecv */
             opal_list_remove_item(&proc->unexpected_frags,
                                   (opal_list_item_t*)frag);
+            SPC_RECORD(OMPI_SPC_UNEXPECTED_IN_QUEUE, -1);
             OB1_MATCHING_UNLOCK(&ob1_comm->matching_lock);
 
             req->req_recv.req_base.req_addr = frag;

ompi/mca/pml/ob1/pml_ob1_sendreq.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2016 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -29,6 +29,7 @@
 #include "ompi_config.h"
 #include "opal/prefetch.h"
 #include "opal/mca/mpool/mpool.h"
+#include "ompi/runtime/ompi_spc.h"
 #include "ompi/constants.h"
 #include "ompi/mca/pml/pml.h"
 #include "pml_ob1.h"
@@ -206,6 +207,8 @@ mca_pml_ob1_rndv_completion_request( mca_bml_base_btl_t* bml_btl,
     }
 
     OPAL_THREAD_ADD_FETCH_SIZE_T(&sendreq->req_bytes_delivered, req_bytes_delivered);
+    SPC_USER_OR_MPI(sendreq->req_send.req_base.req_ompi.req_status.MPI_TAG, (ompi_spc_value_t)req_bytes_delivered,
+                    OMPI_SPC_BYTES_SENT_USER, OMPI_SPC_BYTES_SENT_MPI);
 
     /* advance the request */
     OPAL_THREAD_ADD_FETCH32(&sendreq->req_state, -1);
@@ -262,6 +265,8 @@ mca_pml_ob1_rget_completion (mca_pml_ob1_rdma_frag_t *frag, int64_t rdma_length)
     /* count bytes of user data actually delivered and check for request completion */
     if (OPAL_LIKELY(0 < rdma_length)) {
         OPAL_THREAD_ADD_FETCH_SIZE_T(&sendreq->req_bytes_delivered, (size_t) rdma_length);
+        SPC_USER_OR_MPI(sendreq->req_send.req_base.req_ompi.req_status.MPI_TAG, (ompi_spc_value_t)rdma_length,
+                        OMPI_SPC_BYTES_SENT_USER, OMPI_SPC_BYTES_SENT_MPI);
     }
 
     send_request_pml_complete_check(sendreq);
@@ -315,6 +320,8 @@ mca_pml_ob1_frag_completion( mca_btl_base_module_t* btl,
 
     OPAL_THREAD_ADD_FETCH32(&sendreq->req_pipeline_depth, -1);
     OPAL_THREAD_ADD_FETCH_SIZE_T(&sendreq->req_bytes_delivered, req_bytes_delivered);
+    SPC_USER_OR_MPI(sendreq->req_send.req_base.req_ompi.req_status.MPI_TAG, (ompi_spc_value_t)req_bytes_delivered,
+                    OMPI_SPC_BYTES_SENT_USER, OMPI_SPC_BYTES_SENT_MPI);
 
     if(send_request_pml_complete_check(sendreq) == false) {
         mca_pml_ob1_send_request_schedule(sendreq);
@@ -497,6 +504,8 @@ int mca_pml_ob1_send_request_start_copy( mca_pml_ob1_send_request_t* sendreq,
                                  &des);
         if( OPAL_LIKELY(OMPI_SUCCESS == rc) ) {
             /* signal request completion */
+            SPC_USER_OR_MPI(sendreq->req_send.req_base.req_ompi.req_status.MPI_TAG, (ompi_spc_value_t)size,
+                            OMPI_SPC_BYTES_SENT_USER, OMPI_SPC_BYTES_SENT_MPI);
             send_request_pml_complete(sendreq);
             return OMPI_SUCCESS;
         }
@@ -567,6 +576,8 @@ int mca_pml_ob1_send_request_start_copy( mca_pml_ob1_send_request_t* sendreq,
 
     /* send */
     rc = mca_bml_base_send_status(bml_btl, des, MCA_PML_OB1_HDR_TYPE_MATCH);
+    SPC_USER_OR_MPI(sendreq->req_send.req_base.req_ompi.req_status.MPI_TAG, (ompi_spc_value_t)size,
+                    OMPI_SPC_BYTES_SENT_USER, OMPI_SPC_BYTES_SENT_MPI);
     if( OPAL_LIKELY( rc >= OPAL_SUCCESS ) ) {
         if( OPAL_LIKELY( 1 == rc ) ) {
             mca_pml_ob1_match_completion_free_request( bml_btl, sendreq );
@@ -627,6 +638,8 @@ int mca_pml_ob1_send_request_start_prepare( mca_pml_ob1_send_request_t* sendreq,
 
     /* send */
     rc = mca_bml_base_send(bml_btl, des, MCA_PML_OB1_HDR_TYPE_MATCH);
+    SPC_USER_OR_MPI(sendreq->req_send.req_base.req_ompi.req_status.MPI_TAG, (ompi_spc_value_t)size,
+                    OMPI_SPC_BYTES_SENT_USER, OMPI_SPC_BYTES_SENT_MPI);
     if( OPAL_LIKELY( rc >= OPAL_SUCCESS ) ) {
         if( OPAL_LIKELY( 1 == rc ) ) {
             mca_pml_ob1_match_completion_free_request( bml_btl, sendreq );
@@ -1127,6 +1140,8 @@ static void mca_pml_ob1_put_completion (mca_btl_base_module_t* btl, struct mca_b
 
         /* check for request completion */
         OPAL_THREAD_ADD_FETCH_SIZE_T(&sendreq->req_bytes_delivered, frag->rdma_length);
+        SPC_USER_OR_MPI(sendreq->req_send.req_base.req_ompi.req_status.MPI_TAG, (ompi_spc_value_t)frag->rdma_length,
+                        OMPI_SPC_BYTES_SENT_USER, OMPI_SPC_BYTES_SENT_MPI);
 
         send_request_pml_complete_check(sendreq);
     } else {
@@ -1177,6 +1192,8 @@ int mca_pml_ob1_send_request_put_frag( mca_pml_ob1_rdma_frag_t *frag )
     rc = mca_bml_base_put (bml_btl, frag->local_address, frag->remote_address, local_handle,
                            (mca_btl_base_registration_handle_t *) frag->remote_handle, frag->rdma_length,
                            0, MCA_BTL_NO_ORDER, mca_pml_ob1_put_completion, frag);
+    /* Count the bytes put even though they probably haven't been sent yet */
+    SPC_RECORD(OMPI_SPC_BYTES_PUT, (ompi_spc_value_t)frag->rdma_length);
     if (OPAL_UNLIKELY(OMPI_SUCCESS != rc)) {
         mca_pml_ob1_send_request_put_frag_failed (frag, rc);
         return rc;

ompi/mpi/c/allgather.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -32,6 +32,7 @@
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -49,6 +50,8 @@ int MPI_Allgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
 {
     int err;
 
+    SPC_RECORD(OMPI_SPC_ALLGATHER, 1);
+
     MEMCHECKER(
         int rank;
         ptrdiff_t ext;

ompi/mpi/c/allgatherv.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -32,6 +32,7 @@
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -49,6 +50,8 @@ int MPI_Allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
 {
     int i, size, err;
 
+    SPC_RECORD(OMPI_SPC_ALLGATHERV, 1);
+
     MEMCHECKER(
         int rank;
         ptrdiff_t ext;

ompi/mpi/c/allreduce.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -31,6 +31,7 @@
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/op/op.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -47,6 +48,8 @@ int MPI_Allreduce(const void *sendbuf, void *recvbuf, int count,
 {
     int err;
 
+    SPC_RECORD(OMPI_SPC_ALLREDUCE, 1);
+
     MEMCHECKER(
         memchecker_datatype(datatype);
         memchecker_comm(comm);

ompi/mpi/c/alltoall.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -33,6 +33,7 @@
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -51,6 +52,8 @@ int MPI_Alltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
     int err;
     size_t recvtype_size;
 
+    SPC_RECORD(OMPI_SPC_ALLTOALL, 1);
+
     MEMCHECKER(
         memchecker_comm(comm);
         if (MPI_IN_PLACE != sendbuf) {

ompi/mpi/c/barrier.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -25,6 +25,7 @@
 #include "ompi/communicator/communicator.h"
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -40,6 +41,8 @@ int MPI_Barrier(MPI_Comm comm)
 {
   int err = MPI_SUCCESS;
 
+  SPC_RECORD(OMPI_SPC_BARRIER, 1);
+
   MEMCHECKER(
     memchecker_comm(comm);
   );

ompi/mpi/c/bcast.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -26,6 +26,7 @@
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -42,6 +43,8 @@ int MPI_Bcast(void *buffer, int count, MPI_Datatype datatype,
 {
     int err;
 
+    SPC_RECORD(OMPI_SPC_BCAST, 1);
+
     MEMCHECKER(
         memchecker_datatype(datatype);
         memchecker_comm(comm);

ompi/mpi/c/bsend.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -31,6 +31,7 @@
 #include "ompi/mca/pml/pml.h"
 #include "ompi/mca/pml/base/pml_base_bsend.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -46,6 +47,8 @@ int MPI_Bsend(const void *buf, int count, MPI_Datatype type, int dest, int tag,
 {
     int rc = MPI_SUCCESS;
 
+    SPC_RECORD(OMPI_SPC_BSEND, 1);
+
     MEMCHECKER(
         memchecker_datatype(type);
         memchecker_call(&opal_memchecker_base_isdefined, buf, count, type);

ompi/mpi/c/cancel.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -28,6 +28,7 @@
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/request/request.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -43,6 +44,8 @@ int MPI_Cancel(MPI_Request *request)
 {
     int rc;
 
+    SPC_RECORD(OMPI_SPC_CANCEL, 1);
+
     MEMCHECKER(
         memchecker_request(request);
     );

ompi/mpi/c/finalize.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -23,6 +23,7 @@
 #include "ompi/mpi/c/bindings.h"
 #include "ompi/runtime/params.h"
 #include "ompi/errhandler/errhandler.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -36,6 +37,12 @@ static const char FUNC_NAME[] = "MPI_Finalize";
 
 int MPI_Finalize(void)
 {
+    /* If --with-spc and ompi_mpi_spc_dump_enabled were specified, print
+     * all of the final SPC values aggregated across the whole MPI run.
+     * Also, free all SPC memory.
+     */
+    SPC_FINI();
+
     OPAL_CR_FINALIZE_LIBRARY();
 
     if (MPI_PARAM_CHECK) {

ompi/mpi/c/gather.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -32,6 +32,7 @@
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -49,6 +50,8 @@ int MPI_Gather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
 {
     int err;
 
+    SPC_RECORD(OMPI_SPC_GATHER, 1);
+
     MEMCHECKER(
         int rank;
         ptrdiff_t ext;

ompi/mpi/c/gatherv.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -30,6 +30,7 @@
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -47,6 +48,8 @@ int MPI_Gatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
 {
     int i, size, err;
 
+    SPC_RECORD(OMPI_SPC_GATHERV, 1);
+
     MEMCHECKER(
         ptrdiff_t ext;
 

ompi/mpi/c/get.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -30,6 +30,7 @@
 #include "ompi/win/win.h"
 #include "ompi/mca/osc/osc.h"
 #include "ompi/datatype/ompi_datatype.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -48,6 +49,8 @@ int MPI_Get(void *origin_addr, int origin_count,
 {
     int rc;
 
+    SPC_RECORD(OMPI_SPC_GET, 1);
+
     if (MPI_PARAM_CHECK) {
         rc = OMPI_SUCCESS;
 

ompi/mpi/c/iallgather.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -32,6 +32,7 @@
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -49,6 +50,8 @@ int MPI_Iallgather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
 {
     int err;
 
+    SPC_RECORD(OMPI_SPC_IALLGATHER, 1);
+
     MEMCHECKER(
         int rank;
         ptrdiff_t ext;

ompi/mpi/c/iallgatherv.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -32,6 +32,7 @@
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -49,6 +50,8 @@ int MPI_Iallgatherv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
 {
     int i, size, err;
 
+    SPC_RECORD(OMPI_SPC_IALLGATHERV, 1);
+
     MEMCHECKER(
         int rank;
         ptrdiff_t ext;

ompi/mpi/c/iallreduce.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -32,6 +32,7 @@
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/op/op.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -48,6 +49,8 @@ int MPI_Iallreduce(const void *sendbuf, void *recvbuf, int count,
 {
     int err;
 
+    SPC_RECORD(OMPI_SPC_IALLREDUCE, 1);
+
     MEMCHECKER(
         memchecker_datatype(datatype);
         memchecker_comm(comm);

ompi/mpi/c/ialltoall.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -32,6 +32,7 @@
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -50,6 +51,8 @@ int MPI_Ialltoall(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
     size_t sendtype_size, recvtype_size;
     int err;
 
+    SPC_RECORD(OMPI_SPC_IALLTOALL, 1);
+
     MEMCHECKER(
         memchecker_comm(comm);
         if (MPI_IN_PLACE != sendbuf) {

ompi/mpi/c/ialltoallv.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -31,6 +31,7 @@
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -49,6 +50,8 @@ int MPI_Ialltoallv(const void *sendbuf, const int sendcounts[], const int sdispl
 {
     int i, size, err;
 
+    SPC_RECORD(OMPI_SPC_IALLTOALLV, 1);
+
     MEMCHECKER(
         ptrdiff_t recv_ext;
         ptrdiff_t send_ext;

ompi/mpi/c/ialltoallw.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -31,6 +31,7 @@
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -49,6 +50,8 @@ int MPI_Ialltoallw(const void *sendbuf, const int sendcounts[], const int sdispl
 {
     int i, size, err;
 
+    SPC_RECORD(OMPI_SPC_IALLTOALLW, 1);
+
     MEMCHECKER(
         ptrdiff_t recv_ext;
         ptrdiff_t send_ext;

ompi/mpi/c/ibarrier.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -27,6 +27,7 @@
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -42,6 +43,8 @@ int MPI_Ibarrier(MPI_Comm comm, MPI_Request *request)
 {
     int err = MPI_SUCCESS;
 
+    SPC_RECORD(OMPI_SPC_IBARRIER, 1);
+
     MEMCHECKER(
             memchecker_comm(comm);
             );

ompi/mpi/c/ibcast.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2012      Oak Rigde National Laboratory. All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
- * Copyright (c) 2017      The University of Tennessee and The University
+ * Copyright (c) 2017-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * $COPYRIGHT$
@@ -20,6 +20,7 @@
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -36,6 +37,8 @@ int MPI_Ibcast(void *buffer, int count, MPI_Datatype datatype,
 {
     int err;
 
+    SPC_RECORD(OMPI_SPC_IBCAST, 1);
+
     MEMCHECKER(
         memchecker_datatype(datatype);
         memchecker_call(&opal_memchecker_base_isdefined, buffer, count, datatype);

ompi/mpi/c/ibsend.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -31,6 +31,7 @@
 #include "ompi/mca/pml/pml.h"
 #include "ompi/mca/pml/base/pml_base_bsend.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -47,6 +48,8 @@ int MPI_Ibsend(const void *buf, int count, MPI_Datatype type, int dest,
 {
     int rc = MPI_SUCCESS;
 
+    SPC_RECORD(OMPI_SPC_IBSEND, 1);
+
     MEMCHECKER(
         memchecker_datatype(type);
         memchecker_call(&opal_memchecker_base_isdefined, buf, count, type);

ompi/mpi/c/igather.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -32,6 +32,7 @@
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -49,6 +50,8 @@ int MPI_Igather(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
 {
     int err;
 
+    SPC_RECORD(OMPI_SPC_IGATHER, 1);
+
     MEMCHECKER(
         int rank;
         ptrdiff_t ext;

ompi/mpi/c/init.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2006 High Performance Computing Center Stuttgart,
@@ -25,6 +25,7 @@
 #include <stdlib.h>
 
 #include "opal/util/show_help.h"
+#include "ompi/runtime/ompi_spc.h"
 #include "ompi/mpi/c/bindings.h"
 #include "ompi/communicator/communicator.h"
 #include "ompi/errhandler/errhandler.h"
@@ -83,5 +84,7 @@ int MPI_Init(int *argc, char ***argv)
 
     OPAL_CR_INIT_LIBRARY();
 
+    SPC_INIT();
+
     return MPI_SUCCESS;
 }

ompi/mpi/c/init_thread.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2006 High Performance Computing Center Stuttgart,
@@ -26,6 +26,7 @@
 #include "ompi_config.h"
 
 #include "opal/util/show_help.h"
+#include "ompi/runtime/ompi_spc.h"
 #include "ompi/mpi/c/bindings.h"
 #include "ompi/runtime/params.h"
 #include "ompi/communicator/communicator.h"
@@ -81,6 +82,8 @@ int MPI_Init_thread(int *argc, char ***argv, int required,
 
     OPAL_CR_INIT_LIBRARY();
 
+    SPC_INIT();
+
     ompi_hook_base_mpi_init_thread_bottom(argc, argv, required, provided);
 
     return MPI_SUCCESS;

ompi/mpi/c/iprobe.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -27,6 +27,7 @@
 #include "ompi/mca/pml/pml.h"
 #include "ompi/memchecker.h"
 #include "ompi/request/request.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -42,6 +43,8 @@ int MPI_Iprobe(int source, int tag, MPI_Comm comm, int *flag, MPI_Status *status
 {
     int rc;
 
+    SPC_RECORD(OMPI_SPC_IPROBE, 1);
+
     MEMCHECKER(
         memchecker_comm(comm);
     );

ompi/mpi/c/irecv.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -27,6 +27,7 @@
 #include "ompi/mca/pml/pml.h"
 #include "ompi/request/request.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -43,6 +44,8 @@ int MPI_Irecv(void *buf, int count, MPI_Datatype type, int source,
 {
     int rc = MPI_SUCCESS;
 
+    SPC_RECORD(OMPI_SPC_IRECV, 1);
+
     MEMCHECKER(
         memchecker_datatype(type);
         memchecker_comm(comm);

ompi/mpi/c/ireduce.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -32,6 +32,7 @@
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/op/op.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -48,6 +49,8 @@ int MPI_Ireduce(const void *sendbuf, void *recvbuf, int count,
 {
     int err;
 
+    SPC_RECORD(OMPI_SPC_IREDUCE, 1);
+
     MEMCHECKER(
         memchecker_datatype(datatype);
         memchecker_comm(comm);

ompi/mpi/c/ireduce_scatter.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -32,6 +32,7 @@
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/op/op.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -48,6 +49,8 @@ int MPI_Ireduce_scatter(const void *sendbuf, void *recvbuf, const int recvcounts
 {
     int i, err, size, count;
 
+    SPC_RECORD(OMPI_SPC_IREDUCE_SCATTER, 1);
+
     MEMCHECKER(
         int rank;
         int count;

ompi/mpi/c/ireduce_scatter_block.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -32,6 +32,7 @@
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/op/op.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -49,6 +50,8 @@ int MPI_Ireduce_scatter_block(const void *sendbuf, void *recvbuf, int recvcount,
 {
     int err;
 
+    SPC_RECORD(OMPI_SPC_IREDUCE_SCATTER_BLOCK, 1);
+
     MEMCHECKER(
         memchecker_comm(comm);
         memchecker_datatype(datatype);

ompi/mpi/c/irsend.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -31,6 +31,7 @@
 #include "ompi/mca/pml/pml.h"
 #include "ompi/request/request.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -47,6 +48,8 @@ int MPI_Irsend(const void *buf, int count, MPI_Datatype type, int dest,
 {
     int rc;
 
+    SPC_RECORD(OMPI_SPC_IRSEND, 1);
+
     MEMCHECKER(
         memchecker_datatype(type);
         memchecker_call(&opal_memchecker_base_isdefined, buf, count, type);

ompi/mpi/c/iscan.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -31,6 +31,7 @@
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/op/op.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -47,6 +48,8 @@ int MPI_Iscan(const void *sendbuf, void *recvbuf, int count,
 {
     int err;
 
+    SPC_RECORD(OMPI_SPC_ISCAN, 1);
+
     MEMCHECKER(
         memchecker_datatype(datatype);
         memchecker_comm(comm);

ompi/mpi/c/iscatter.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -32,6 +32,7 @@
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -49,6 +50,8 @@ int MPI_Iscatter(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
 {
     int err;
 
+    SPC_RECORD(OMPI_SPC_ISCATTER, 1);
+
     MEMCHECKER(
         memchecker_comm(comm);
         if(OMPI_COMM_IS_INTRA(comm)) {

ompi/mpi/c/iscatterv.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -30,6 +30,7 @@
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -47,6 +48,8 @@ int MPI_Iscatterv(const void *sendbuf, const int sendcounts[], const int displs[
 {
     int i, size, err;
 
+    SPC_RECORD(OMPI_SPC_ISCATTERV, 1);
+
     MEMCHECKER(
         ptrdiff_t ext;
 

ompi/mpi/c/isend.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -31,6 +31,7 @@
 #include "ompi/mca/pml/pml.h"
 #include "ompi/request/request.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -47,6 +48,8 @@ int MPI_Isend(const void *buf, int count, MPI_Datatype type, int dest,
 {
     int rc = MPI_SUCCESS;
 
+    SPC_RECORD(OMPI_SPC_ISEND, 1);
+
     MEMCHECKER(
         memchecker_datatype(type);
         memchecker_call(&opal_memchecker_base_isdefined, buf, count, type);

ompi/mpi/c/issend.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -30,6 +30,7 @@
 #include "ompi/mca/pml/pml.h"
 #include "ompi/request/request.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -46,6 +47,8 @@ int MPI_Issend(const void *buf, int count, MPI_Datatype type, int dest,
 {
     int rc = MPI_SUCCESS;
 
+    SPC_RECORD(OMPI_SPC_ISSEND, 1);
+
     MEMCHECKER(
         memchecker_datatype(type);
         memchecker_call(&opal_memchecker_base_isdefined, buf, count, type);

ompi/mpi/c/mrecv.c

@@ -3,6 +3,9 @@
  * Copyright (c) 2012-2013 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
+ * Copyright (c) 2018      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -18,6 +21,7 @@
 #include "ompi/memchecker.h"
 #include "ompi/request/request.h"
 #include "ompi/message/message.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -35,6 +39,8 @@ int MPI_Mrecv(void *buf, int count, MPI_Datatype type,
     int rc = MPI_SUCCESS;
     ompi_communicator_t *comm;
 
+    SPC_RECORD(OMPI_SPC_MRECV, 1);
+
     MEMCHECKER(
         memchecker_datatype(type);
         memchecker_message(message);

ompi/mpi/c/neighbor_allgather.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -35,6 +35,7 @@
 #include "ompi/memchecker.h"
 #include "ompi/mca/topo/topo.h"
 #include "ompi/mca/topo/base/base.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -52,6 +53,8 @@ int MPI_Neighbor_allgather(const void *sendbuf, int sendcount, MPI_Datatype send
 {
     int err;
 
+    SPC_RECORD(OMPI_SPC_NEIGHBOR_ALLGATHER, 1);
+
     MEMCHECKER(
         int rank;
         ptrdiff_t ext;

ompi/mpi/c/neighbor_allgatherv.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -36,6 +36,7 @@
 #include "ompi/memchecker.h"
 #include "ompi/mca/topo/topo.h"
 #include "ompi/mca/topo/base/base.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -53,6 +54,8 @@ int MPI_Neighbor_allgatherv(const void *sendbuf, int sendcount, MPI_Datatype sen
 {
     int in_size, out_size, err;
 
+    SPC_RECORD(OMPI_SPC_NEIGHBOR_ALLGATHERV, 1);
+
     MEMCHECKER(
         int rank;
         ptrdiff_t ext;

ompi/mpi/c/neighbor_alltoall.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -34,6 +34,7 @@
 #include "ompi/memchecker.h"
 #include "ompi/mca/topo/topo.h"
 #include "ompi/mca/topo/base/base.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -52,6 +53,8 @@ int MPI_Neighbor_alltoall(const void *sendbuf, int sendcount, MPI_Datatype sendt
     size_t sendtype_size, recvtype_size;
     int err;
 
+    SPC_RECORD(OMPI_SPC_NEIGHBOR_ALLTOALL, 1);
+
     MEMCHECKER(
         memchecker_comm(comm);
         if (MPI_IN_PLACE != sendbuf) {

ompi/mpi/c/neighbor_alltoallv.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -34,6 +34,7 @@
 #include "ompi/memchecker.h"
 #include "ompi/mca/topo/topo.h"
 #include "ompi/mca/topo/base/base.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -53,6 +54,8 @@ int MPI_Neighbor_alltoallv(const void *sendbuf, const int sendcounts[], const in
     int i, err;
     int indegree, outdegree;
 
+    SPC_RECORD(OMPI_SPC_NEIGHBOR_ALLTOALLV, 1);
+
     MEMCHECKER(
         ptrdiff_t recv_ext;
         ptrdiff_t send_ext;

ompi/mpi/c/neighbor_alltoallw.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -34,6 +34,7 @@
 #include "ompi/memchecker.h"
 #include "ompi/mca/topo/topo.h"
 #include "ompi/mca/topo/base/base.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -53,6 +54,8 @@ int MPI_Neighbor_alltoallw(const void *sendbuf, const int sendcounts[], const MP
     int i, err;
     int indegree, outdegree;
 
+    SPC_RECORD(OMPI_SPC_NEIGHBOR_ALLTOALLW, 1);
+
     MEMCHECKER(
         ptrdiff_t recv_ext;
         ptrdiff_t send_ext;

ompi/mpi/c/probe.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -27,6 +27,7 @@
 #include "ompi/mca/pml/pml.h"
 #include "ompi/memchecker.h"
 #include "ompi/request/request.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -42,6 +43,8 @@ int MPI_Probe(int source, int tag, MPI_Comm comm, MPI_Status *status)
 {
     int rc;
 
+    SPC_RECORD(OMPI_SPC_PROBE, 1);
+
     MEMCHECKER(
         memchecker_comm(comm);
     );

ompi/mpi/c/put.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -31,6 +31,7 @@
 #include "ompi/win/win.h"
 #include "ompi/mca/osc/osc.h"
 #include "ompi/datatype/ompi_datatype.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -48,6 +49,8 @@ int MPI_Put(const void *origin_addr, int origin_count, MPI_Datatype origin_datat
 {
     int rc;
 
+    SPC_RECORD(OMPI_SPC_PUT, 1);
+
     if (MPI_PARAM_CHECK) {
         rc = OMPI_SUCCESS;
 

ompi/mpi/c/recv.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -27,6 +27,7 @@
 #include "ompi/mca/pml/pml.h"
 #include "ompi/memchecker.h"
 #include "ompi/request/request.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -43,6 +44,8 @@ int MPI_Recv(void *buf, int count, MPI_Datatype type, int source,
 {
     int rc = MPI_SUCCESS;
 
+    SPC_RECORD(OMPI_SPC_RECV, 1);
+
     MEMCHECKER(
         memchecker_datatype(type);
         memchecker_call(&opal_memchecker_base_isaddressable, buf, count, type);

ompi/mpi/c/reduce.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -31,6 +31,7 @@
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/op/op.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -47,6 +48,8 @@ int MPI_Reduce(const void *sendbuf, void *recvbuf, int count,
 {
     int err;
 
+    SPC_RECORD(OMPI_SPC_REDUCE, 1);
+
     MEMCHECKER(
         memchecker_datatype(datatype);
         memchecker_comm(comm);

ompi/mpi/c/rget.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -30,6 +30,7 @@
 #include "ompi/win/win.h"
 #include "ompi/mca/osc/osc.h"
 #include "ompi/datatype/ompi_datatype.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -48,6 +49,8 @@ int MPI_Rget(void *origin_addr, int origin_count,
 {
     int rc;
 
+    SPC_RECORD(OMPI_SPC_RGET, 1);
+
     if (MPI_PARAM_CHECK) {
         rc = OMPI_SUCCESS;
 

ompi/mpi/c/rput.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -31,6 +31,7 @@
 #include "ompi/win/win.h"
 #include "ompi/mca/osc/osc.h"
 #include "ompi/datatype/ompi_datatype.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -48,6 +49,8 @@ int MPI_Rput(const void *origin_addr, int origin_count, MPI_Datatype origin_data
 {
     int rc;
 
+    SPC_RECORD(OMPI_SPC_RPUT, 1);
+
     if (MPI_PARAM_CHECK) {
         rc = OMPI_SUCCESS;
 

ompi/mpi/c/rsend.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -30,7 +30,7 @@
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/mca/pml/pml.h"
 #include "ompi/memchecker.h"
-
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -46,6 +46,8 @@ int MPI_Rsend(const void *buf, int count, MPI_Datatype type, int dest, int tag,
 {
     int rc = MPI_SUCCESS;
 
+    SPC_RECORD(OMPI_SPC_RSEND, 1);
+
     MEMCHECKER(
         memchecker_datatype(type);
         memchecker_call(&opal_memchecker_base_isdefined, buf, count, type);

ompi/mpi/c/scan.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -31,6 +31,7 @@
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/op/op.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -47,6 +48,8 @@ int MPI_Scan(const void *sendbuf, void *recvbuf, int count,
 {
     int err;
 
+    SPC_RECORD(OMPI_SPC_SCAN, 1);
+
     MEMCHECKER(
         memchecker_datatype(datatype);
         memchecker_comm(comm);

ompi/mpi/c/scatter.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -32,6 +32,7 @@
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -49,6 +50,8 @@ int MPI_Scatter(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
 {
     int err;
 
+    SPC_RECORD(OMPI_SPC_SCATTER, 1);
+
     MEMCHECKER(
         memchecker_comm(comm);
         if(OMPI_COMM_IS_INTRA(comm)) {

ompi/mpi/c/scatterv.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -30,6 +30,7 @@
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -47,6 +48,8 @@ int MPI_Scatterv(const void *sendbuf, const int sendcounts[], const int displs[]
 {
     int i, size, err;
 
+    SPC_RECORD(OMPI_SPC_SCATTERV, 1);
+
     MEMCHECKER(
         ptrdiff_t ext;
 

ompi/mpi/c/send.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -30,6 +30,7 @@
 #include "ompi/mca/pml/pml.h"
 #include "ompi/datatype/ompi_datatype.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -46,6 +47,8 @@ int MPI_Send(const void *buf, int count, MPI_Datatype type, int dest,
 {
     int rc = MPI_SUCCESS;
 
+    SPC_RECORD(OMPI_SPC_SEND, 1);
+
     MEMCHECKER(
         memchecker_datatype(type);
         memchecker_call(&opal_memchecker_base_isdefined, buf, count, type);

ompi/mpi/c/sendrecv.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -29,6 +29,7 @@
 #include "ompi/mca/pml/pml.h"
 #include "ompi/request/request.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -48,6 +49,8 @@ int MPI_Sendrecv(const void *sendbuf, int sendcount, MPI_Datatype sendtype,
     ompi_request_t* req;
     int rc = MPI_SUCCESS;
 
+    SPC_RECORD(OMPI_SPC_SENDRECV, 1);
+
     MEMCHECKER(
         memchecker_datatype(sendtype);
         memchecker_datatype(recvtype);

ompi/mpi/c/sendrecv_replace.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2017 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -30,6 +30,7 @@
 #include "ompi/mca/pml/pml.h"
 #include "ompi/proc/proc.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -48,6 +49,8 @@ int MPI_Sendrecv_replace(void * buf, int count, MPI_Datatype datatype,
 {
     int rc = MPI_SUCCESS;
 
+    SPC_RECORD(OMPI_SPC_SENDRECV_REPLACE, 1);
+
     MEMCHECKER(
                memchecker_datatype(datatype);
                memchecker_call(&opal_memchecker_base_isdefined, buf, count, datatype);

ompi/mpi/c/ssend.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -30,6 +30,7 @@
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/mca/pml/pml.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -45,6 +46,8 @@ int MPI_Ssend(const void *buf, int count, MPI_Datatype type, int dest, int tag,
 {
     int rc = MPI_SUCCESS;
 
+    SPC_RECORD(OMPI_SPC_SSEND, 1);
+
     MEMCHECKER(
         memchecker_datatype(type);
         memchecker_call(&opal_memchecker_base_isdefined, buf, count, type);

ompi/mpi/c/test.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -27,6 +27,7 @@
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/request/request.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -42,6 +43,8 @@ int MPI_Test(MPI_Request *request, int *completed, MPI_Status *status)
 {
     int rc;
 
+    SPC_RECORD(OMPI_SPC_TEST, 1);
+
     MEMCHECKER(
         memchecker_request (request);
     );

ompi/mpi/c/testall.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2006 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -30,6 +30,7 @@
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/request/request.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -44,6 +45,8 @@ static const char FUNC_NAME[] = "MPI_Testall";
 int MPI_Testall(int count, MPI_Request requests[], int *flag,
                 MPI_Status statuses[])
 {
+    SPC_RECORD(OMPI_SPC_TESTALL, 1);
+
     MEMCHECKER(
         int j;
         for (j = 0; j < count; j++){

ompi/mpi/c/testany.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2006 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -30,6 +30,7 @@
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/request/request.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -43,6 +44,8 @@ static const char FUNC_NAME[] = "MPI_Testany";
 
 int MPI_Testany(int count, MPI_Request requests[], int *indx, int *completed, MPI_Status *status)
 {
+    SPC_RECORD(OMPI_SPC_TESTANY, 1);
+
     MEMCHECKER(
         int j;
         for (j = 0; j < count; j++){

ompi/mpi/c/testsome.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2006 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -30,6 +30,7 @@
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/request/request.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -45,6 +46,8 @@ int MPI_Testsome(int incount, MPI_Request requests[],
                  int *outcount, int indices[],
                  MPI_Status statuses[])
 {
+    SPC_RECORD(OMPI_SPC_TESTSOME, 1);
+
     MEMCHECKER(
         int j;
         for (j = 0; j < incount; j++){

ompi/mpi/c/wait.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2013 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -27,6 +27,7 @@
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/request/request.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -40,6 +41,8 @@ static const char FUNC_NAME[] = "MPI_Wait";
 
 int MPI_Wait(MPI_Request *request, MPI_Status *status)
 {
+    SPC_RECORD(OMPI_SPC_WAIT, 1);
+
     MEMCHECKER(
         memchecker_request(request);
     );

ompi/mpi/c/waitall.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2006 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -29,6 +29,7 @@
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/request/request.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -42,6 +43,8 @@ static const char FUNC_NAME[] = "MPI_Waitall";
 
 int MPI_Waitall(int count, MPI_Request requests[], MPI_Status statuses[])
 {
+    SPC_RECORD(OMPI_SPC_WAITALL, 1);
+
     MEMCHECKER(
         int j;
         for (j = 0; j < count; j++){

ompi/mpi/c/waitany.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2006 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -30,6 +30,7 @@
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/request/request.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -43,6 +44,8 @@ static const char FUNC_NAME[] = "MPI_Waitany";
 
 int MPI_Waitany(int count, MPI_Request requests[], int *indx, MPI_Status *status)
 {
+    SPC_RECORD(OMPI_SPC_WAITANY, 1);
+
     MEMCHECKER(
         int j;
         for (j = 0; j < count; j++){

ompi/mpi/c/waitsome.c

@@ -2,7 +2,7 @@
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2006 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2008 High Performance Computing Center Stuttgart,
@@ -30,6 +30,7 @@
 #include "ompi/errhandler/errhandler.h"
 #include "ompi/request/request.h"
 #include "ompi/memchecker.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -45,6 +46,8 @@ int MPI_Waitsome(int incount, MPI_Request requests[],
                  int *outcount, int indices[],
                  MPI_Status statuses[])
 {
+    SPC_RECORD(OMPI_SPC_WAITSOME, 1);
+
     MEMCHECKER(
         int j;
         for (j = 0; j < incount; j++){

ompi/mpi/c/wtime.c

@@ -34,6 +34,7 @@
 #include MCA_timer_IMPLEMENTATION_HEADER
 #include "ompi/mpi/c/bindings.h"
 #include "ompi/runtime/mpiruntime.h"
+#include "ompi/runtime/ompi_spc.h"
 
 #if OMPI_BUILD_MPI_PROFILING
 #if OPAL_HAVE_WEAK_SYMBOLS
@@ -62,6 +63,8 @@ double MPI_Wtime(void)
 {
     double wtime;
 
+    SPC_RECORD(OMPI_SPC_WTIME, 1);
+
     /*
      * See https://github.com/open-mpi/ompi/issues/3003 to find out
      * what's happening here.

ompi/runtime/Makefile.am

@@ -2,7 +2,7 @@
 # Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
 #                         University Research and Technology
 #                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2005 The University of Tennessee and The University
+# Copyright (c) 2004-2018 The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
 # Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -27,7 +27,8 @@ headers += \
         runtime/mpiruntime.h \
 	runtime/ompi_cr.h \
         runtime/params.h \
-	runtime/ompi_info_support.h
+	runtime/ompi_info_support.h \
+	runtime/ompi_spc.h
 
 lib@OMPI_LIBMPI_NAME@_la_SOURCES += \
         runtime/ompi_mpi_abort.c \
@@ -37,4 +38,5 @@ lib@OMPI_LIBMPI_NAME@_la_SOURCES += \
         runtime/ompi_mpi_params.c \
         runtime/ompi_mpi_preconnect.c \
 	runtime/ompi_cr.c \
-	runtime/ompi_info_support.c
+	runtime/ompi_info_support.c \
+	runtime/ompi_spc.c

ompi/runtime/help-mpi-runtime.txt

@@ -10,7 +10,7 @@
 #                         University of Stuttgart.  All rights reserved.
 # Copyright (c) 2004-2005 The Regents of the University of California.
 #                         All rights reserved.
-# Copyright (c) 2007-2015 Cisco Systems, Inc.  All rights reserved.
+# Copyright (c) 2007-2018 Cisco Systems, Inc.  All rights reserved
 # Copyright (c) 2013      NVIDIA Corporation.  All rights reserved.
 # Copyright (c) 2017      Intel, Inc. All rights reserved.
 # $COPYRIGHT$
@@ -104,3 +104,18 @@ its predecessors) APIs for this operation.
 This typically happens when launching outside of mpirun where the underlying
 resource manager does not provide publish/lookup support. One way of solving
 the problem is to simply use mpirun to start the application.
+#
+[lib-call-fail]
+A library call unexpectedly failed.  This is a terminal error; please
+show this message to an Open MPI wizard:
+
+        Library call: %s
+         Source file: %s
+  Source line number: %d
+
+Aborting...
+#
+[spc: MPI_T disabled]
+There was an error registering software performance counters (SPCs) as
+MPI_T performance variables.  Your job will continue, but SPCs will be
+disabled for MPI_T.

ompi/runtime/ompi_mpi_params.c

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2006 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -75,6 +75,9 @@ bool ompi_async_mpi_finalize = false;
 uint32_t ompi_add_procs_cutoff = OMPI_ADD_PROCS_CUTOFF_DEFAULT;
 bool ompi_mpi_dynamics_enabled = true;
 
+char *ompi_mpi_spc_attach_string = NULL;
+bool ompi_mpi_spc_dump_enabled = false;
+
 static bool show_default_mca_params = false;
 static bool show_file_mca_params = false;
 static bool show_enviro_mca_params = false;
@@ -315,6 +318,22 @@ int ompi_mpi_register_params(void)
                                       MCA_BASE_VAR_SYN_FLAG_DEPRECATED);
     }
 
+    ompi_mpi_spc_attach_string = NULL;
+    (void) mca_base_var_register("ompi", "mpi", NULL, "spc_attach",
+                                 "A comma delimeted string listing the software-based performance counters (SPCs) to enable.",
+                                 MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0,
+                                 OPAL_INFO_LVL_4,
+                                 MCA_BASE_VAR_SCOPE_READONLY,
+                                 &ompi_mpi_spc_attach_string);
+
+    ompi_mpi_spc_dump_enabled = false;
+    (void) mca_base_var_register("ompi", "mpi", NULL, "spc_dump_enabled",
+                                 "A boolean value for whether (true) or not (false) to enable dumping SPC counters in MPI_Finalize.",
+                                 MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0,
+                                 OPAL_INFO_LVL_4,
+                                 MCA_BASE_VAR_SCOPE_READONLY,
+                                 &ompi_mpi_spc_dump_enabled);
+
     return OMPI_SUCCESS;
 }
 

ompi/runtime/ompi_spc.c

@@ -0,0 +1,458 @@
+/*
+ * Copyright (c) 2018      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ *
+ * Copyright (c) 2018      Cisco Systems, Inc.  All rights reserved
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#include "ompi_spc.h"
+
+opal_timer_t sys_clock_freq_mhz = 0;
+
+static void ompi_spc_dump(void);
+
+/* Array for converting from SPC indices to MPI_T indices */
+OMPI_DECLSPEC int mpi_t_offset = -1;
+OMPI_DECLSPEC bool mpi_t_enabled = false;
+
+OPAL_DECLSPEC ompi_communicator_t *comm = NULL;
+
+typedef struct ompi_spc_event_t {
+    const char* counter_name;
+    const char* counter_description;
+} ompi_spc_event_t;
+
+#define SET_COUNTER_ARRAY(NAME, DESC)   [NAME] = { .counter_name = #NAME, .counter_description = DESC }
+
+static ompi_spc_event_t ompi_spc_events_names[OMPI_SPC_NUM_COUNTERS] = {
+    SET_COUNTER_ARRAY(OMPI_SPC_SEND, "The number of times MPI_Send was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_BSEND, "The number of times MPI_Bsend was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_RSEND, "The number of times MPI_Rsend was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_SSEND, "The number of times MPI_Ssend was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_RECV, "The number of times MPI_Recv was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_MRECV, "The number of times MPI_Mrecv was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_ISEND, "The number of times MPI_Isend was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_IBSEND, "The number of times MPI_Ibsend was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_IRSEND, "The number of times MPI_Irsend was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_ISSEND, "The number of times MPI_Issend was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_IRECV, "The number of times MPI_Irecv was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_SENDRECV, "The number of times MPI_Sendrecv was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_SENDRECV_REPLACE, "The number of times MPI_Sendrecv_replace was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_PUT, "The number of times MPI_Put was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_RPUT, "The number of times MPI_Rput was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_GET, "The number of times MPI_Get was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_RGET, "The number of times MPI_Rget was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_PROBE, "The number of times MPI_Probe was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_IPROBE, "The number of times MPI_Iprobe was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_BCAST, "The number of times MPI_Bcast was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_IBCAST, "The number of times MPI_Ibcast was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_REDUCE, "The number of times MPI_Reduce was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_IREDUCE, "The number of times MPI_Ireduce was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_IREDUCE_SCATTER, "The number of times MPI_Ireduce_scatter was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_IREDUCE_SCATTER_BLOCK, "The number of times MPI_Ireduce_scatter_block was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_ALLREDUCE, "The number of times MPI_Allreduce was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_IALLREDUCE, "The number of times MPI_Iallreduce was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_SCAN, "The number of times MPI_Scan was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_ISCAN, "The number of times MPI_Iscan was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_SCATTER, "The number of times MPI_Scatter was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_SCATTERV, "The number of times MPI_Scatterv was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_ISCATTER, "The number of times MPI_Iscatter was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_ISCATTERV, "The number of times MPI_Iscatterv was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_GATHER, "The number of times MPI_Gather was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_GATHERV, "The number of times MPI_Gatherv was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_IGATHER, "The number of times MPI_Igather was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_ALLTOALL, "The number of times MPI_Alltoall was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_IALLTOALL, "The number of times MPI_Ialltoall was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_IALLTOALLV, "The number of times MPI_Ialltoallv was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_IALLTOALLW, "The number of times MPI_Ialltoallw was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_NEIGHBOR_ALLTOALL, "The number of times MPI_Neighbor_alltoall was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_NEIGHBOR_ALLTOALLV, "The number of times MPI_Neighbor_alltoallv was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_NEIGHBOR_ALLTOALLW, "The number of times MPI_Neighbor_alltoallw was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_ALLGATHER, "The number of times MPI_Allgather was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_ALLGATHERV, "The number of times MPI_Allgatherv was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_IALLGATHER, "The number of times MPI_Iallgather was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_IALLGATHERV, "The number of times MPI_Iallgatherv was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_NEIGHBOR_ALLGATHER, "The number of times MPI_Neighbor_allgather was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_NEIGHBOR_ALLGATHERV, "The number of times MPI_Neighbor_allgatherv was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_TEST, "The number of times MPI_Test was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_TESTALL, "The number of times MPI_Testall was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_TESTANY, "The number of times MPI_Testany was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_TESTSOME, "The number of times MPI_Testsome was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_WAIT, "The number of times MPI_Wait was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_WAITALL, "The number of times MPI_Waitall was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_WAITANY, "The number of times MPI_Waitany was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_WAITSOME, "The number of times MPI_Waitsome was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_BARRIER, "The number of times MPI_Barrier was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_IBARRIER, "The number of times MPI_Ibarrier was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_WTIME, "The number of times MPI_Wtime was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_CANCEL, "The number of times MPI_Cancel was called."),
+    SET_COUNTER_ARRAY(OMPI_SPC_BYTES_RECEIVED_USER, "The number of bytes received by the user through point-to-point communications. Note: Includes bytes transferred using internal RMA operations."),
+    SET_COUNTER_ARRAY(OMPI_SPC_BYTES_RECEIVED_MPI, "The number of bytes received by MPI through collective, control, or other internal communications."),
+    SET_COUNTER_ARRAY(OMPI_SPC_BYTES_SENT_USER, "The number of bytes sent by the user through point-to-point communications.  Note: Includes bytes transferred using internal RMA operations."),
+    SET_COUNTER_ARRAY(OMPI_SPC_BYTES_SENT_MPI, "The number of bytes sent by MPI through collective, control, or other internal communications."),
+    SET_COUNTER_ARRAY(OMPI_SPC_BYTES_PUT, "The number of bytes sent/received using RMA Put operations both through user-level Put functions and internal Put functions."),
+    SET_COUNTER_ARRAY(OMPI_SPC_BYTES_GET, "The number of bytes sent/received using RMA Get operations both through user-level Get functions and internal Get functions."),
+    SET_COUNTER_ARRAY(OMPI_SPC_UNEXPECTED, "The number of messages that arrived as unexpected messages."),
+    SET_COUNTER_ARRAY(OMPI_SPC_OUT_OF_SEQUENCE, "The number of messages that arrived out of the proper sequence."),
+    SET_COUNTER_ARRAY(OMPI_SPC_MATCH_TIME, "The number of microseconds spent matching unexpected messages.  Note: The timer used on the back end is in cycles, which could potentially be problematic on a system where the clock frequency can change.  On such a system, this counter could be inaccurate since we assume a fixed clock rate."),
+    SET_COUNTER_ARRAY(OMPI_SPC_UNEXPECTED_IN_QUEUE, "The number of messages that are currently in the unexpected message queue(s) of an MPI process."),
+    SET_COUNTER_ARRAY(OMPI_SPC_OOS_IN_QUEUE, "The number of messages that are currently in the out of sequence message queue(s) of an MPI process."),
+    SET_COUNTER_ARRAY(OMPI_SPC_MAX_UNEXPECTED_IN_QUEUE, "The maximum number of messages that the unexpected message queue(s) within an MPI process "
+                                                    "contained at once since the last reset of this counter. Note: This counter is reset each time it is read."),
+    SET_COUNTER_ARRAY(OMPI_SPC_MAX_OOS_IN_QUEUE, "The maximum number of messages that the out of sequence message queue(s) within an MPI process "
+                                             "contained at once since the last reset of this counter. Note: This counter is reset each time it is read.")
+};
+
+/* An array of integer values to denote whether an event is activated (1) or not (0) */
+static uint32_t ompi_spc_attached_event[OMPI_SPC_NUM_COUNTERS / sizeof(uint32_t)] = { 0 };
+/* An array of integer values to denote whether an event is timer-based (1) or not (0) */
+static uint32_t ompi_spc_timer_event[OMPI_SPC_NUM_COUNTERS / sizeof(uint32_t)] = { 0 };
+/* An array of event structures to store the event data (name and value) */
+static ompi_spc_t *ompi_spc_events = NULL;
+
+static inline void SET_SPC_BIT(uint32_t* array, int32_t pos)
+{
+    assert(pos < OMPI_SPC_NUM_COUNTERS);
+    array[pos / (8 * sizeof(uint32_t))] |= (1U << (pos % (8 * sizeof(uint32_t))));
+}
+
+static inline bool IS_SPC_BIT_SET(uint32_t* array, int32_t pos)
+{
+    assert(pos < OMPI_SPC_NUM_COUNTERS);
+    return !!(array[pos / (8 * sizeof(uint32_t))] & (1U << (pos % (8 * sizeof(uint32_t)))));
+}
+
+static inline void CLEAR_SPC_BIT(uint32_t* array, int32_t pos)
+{
+    assert(pos < OMPI_SPC_NUM_COUNTERS);
+    array[pos / (8 * sizeof(uint32_t))] &= ~(1U << (pos % (8 * sizeof(uint32_t))));
+}
+
+/* ##############################################################
+ * ################# Begin MPI_T Functions ######################
+ * ##############################################################
+ */
+static int ompi_spc_notify(mca_base_pvar_t *pvar, mca_base_pvar_event_t event, void *obj_handle, int *count)
+    __opal_attribute_unused__;
+
+static int ompi_spc_notify(mca_base_pvar_t *pvar, mca_base_pvar_event_t event, void *obj_handle, int *count)
+{
+    int index;
+
+    if(OPAL_LIKELY(!mpi_t_enabled)) {
+        return MPI_SUCCESS;
+    }
+
+    /* For this event, we need to set count to the number of long long type
+     * values for this counter.  All SPC counters are one long long, so we
+     * always set count to 1.
+     */
+    if(MCA_BASE_PVAR_HANDLE_BIND == event) {
+        *count = 1;
+    }
+    /* For this event, we need to turn on the counter */
+    else if(MCA_BASE_PVAR_HANDLE_START == event) {
+        /* Convert from MPI_T pvar index to SPC index */
+        index = pvar->pvar_index - mpi_t_offset;
+        SET_SPC_BIT(ompi_spc_attached_event, index);
+    }
+    /* For this event, we need to turn off the counter */
+    else if(MCA_BASE_PVAR_HANDLE_STOP == event) {
+        /* Convert from MPI_T pvar index to SPC index */
+        index = pvar->pvar_index - mpi_t_offset;
+        CLEAR_SPC_BIT(ompi_spc_attached_event, index);
+    }
+
+    return MPI_SUCCESS;
+}
+
+/* ##############################################################
+ * ################# Begin SPC Functions ########################
+ * ##############################################################
+ */
+
+/* This function returns the current count of an SPC counter that has been retistered
+ * as an MPI_T pvar.  The MPI_T index is not necessarily the same as the SPC index,
+ * so we need to convert from MPI_T index to SPC index and then set the 'value' argument
+ * to the correct value for this pvar.
+ */
+static int ompi_spc_get_count(const struct mca_base_pvar_t *pvar, void *value, void *obj_handle)
+    __opal_attribute_unused__;
+
+static int ompi_spc_get_count(const struct mca_base_pvar_t *pvar, void *value, void *obj_handle)
+{
+    long long *counter_value = (long long*)value;
+
+    if(OPAL_LIKELY(!mpi_t_enabled)) {
+        *counter_value = 0;
+        return MPI_SUCCESS;
+    }
+
+    /* Convert from MPI_T pvar index to SPC index */
+    int index = pvar->pvar_index - mpi_t_offset;
+    /* Set the counter value to the current SPC value */
+    *counter_value = (long long)ompi_spc_events[index].value;
+    /* If this is a timer-based counter, convert from cycles to microseconds */
+    if( IS_SPC_BIT_SET(ompi_spc_timer_event, index) ) {
+        *counter_value /= sys_clock_freq_mhz;
+    }
+    /* If this is a high watermark counter, reset it after it has been read */
+    if(index == OMPI_SPC_MAX_UNEXPECTED_IN_QUEUE || index == OMPI_SPC_MAX_OOS_IN_QUEUE) {
+        ompi_spc_events[index].value = 0;
+    }
+
+    return MPI_SUCCESS;
+}
+
+/* Initializes the events data structure and allocates memory for it if needed. */
+void ompi_spc_events_init(void)
+{
+    int i;
+
+    /* If the events data structure hasn't been allocated yet, allocate memory for it */
+    if(NULL == ompi_spc_events) {
+        ompi_spc_events = (ompi_spc_t*)malloc(OMPI_SPC_NUM_COUNTERS * sizeof(ompi_spc_t));
+        if(ompi_spc_events == NULL) {
+            opal_show_help("help-mpi-runtime.txt", "lib-call-fail", true,
+                           "malloc", __FILE__, __LINE__);
+            return;
+        }
+    }
+    /* The data structure has been allocated, so we simply initialize all of the counters
+     * with their names and an initial count of 0.
+     */
+    for(i = 0; i < OMPI_SPC_NUM_COUNTERS; i++) {
+        ompi_spc_events[i].name = (char*)ompi_spc_events_names[i].counter_name;
+        ompi_spc_events[i].value = 0;
+    }
+
+    ompi_comm_dup(&ompi_mpi_comm_world.comm, &comm);
+}
+
+/* Initializes the SPC data structures and registers all counters as MPI_T pvars.
+ * Turns on only the counters that were specified in the mpi_spc_attach MCA parameter.  
+ */
+void ompi_spc_init(void)
+{
+    int i, j, ret, found = 0, all_on = 0;
+
+    /* Initialize the clock frequency variable as the CPU's frequency in MHz */
+    sys_clock_freq_mhz = opal_timer_base_get_freq() / 1000000;
+
+    ompi_spc_events_init();
+
+    /* Get the MCA params string of counters to turn on */
+    char **arg_strings = opal_argv_split(ompi_mpi_spc_attach_string, ',');
+    int num_args       = opal_argv_count(arg_strings);
+
+    /* If there is only one argument and it is 'all', then all counters
+     * should be turned on.  If the size is 0, then no counters will be enabled.
+     */
+    if(1 == num_args) {
+        if(strcmp(arg_strings[0], "all") == 0) {
+            all_on = 1;
+        }
+    }
+
+    /* Turn on only the counters that were specified in the MCA parameter */
+    for(i = 0; i < OMPI_SPC_NUM_COUNTERS; i++) {
+        if(all_on) {
+            SET_SPC_BIT(ompi_spc_attached_event, i);
+            mpi_t_enabled = true;
+            found++;
+        } else {
+            /* Note: If no arguments were given, this will be skipped */
+            for(j = 0; j < num_args; j++) {
+                if( 0 == strcmp(ompi_spc_events_names[i].counter_name, arg_strings[j]) ) {
+                    SET_SPC_BIT(ompi_spc_attached_event, i);
+                    mpi_t_enabled = true;
+                    found++;
+                    break;
+                }
+            }
+        }
+
+        /* ########################################################################
+         * ################## Add Timer-Based Counter Enums Here ##################
+         * ########################################################################
+         */
+        CLEAR_SPC_BIT(ompi_spc_timer_event, i);
+
+        /* Registers the current counter as an MPI_T pvar regardless of whether it's been turned on or not */
+        ret = mca_base_pvar_register("ompi", "runtime", "spc", ompi_spc_events_names[i].counter_name, ompi_spc_events_names[i].counter_description,
+                                     OPAL_INFO_LVL_4, MPI_T_PVAR_CLASS_SIZE,
+                                     MCA_BASE_VAR_TYPE_UNSIGNED_LONG_LONG, NULL, MPI_T_BIND_NO_OBJECT,
+                                     MCA_BASE_PVAR_FLAG_READONLY | MCA_BASE_PVAR_FLAG_CONTINUOUS,
+                                     ompi_spc_get_count, NULL, ompi_spc_notify, NULL);
+
+        /* Check to make sure that ret is a valid index and not an error code.
+         */
+        if( ret >= 0 ) {
+            if( mpi_t_offset == -1 ) {
+                mpi_t_offset = ret;
+            }
+        }
+        if( (ret < 0) || (ret != (mpi_t_offset + found - 1)) ) {
+            mpi_t_enabled = false;
+            opal_show_help("help-mpi-runtime.txt", "spc: MPI_T disabled", true);
+            break;
+        }
+    }
+    /* If this is a timer event, sent the corresponding timer_event entry to 1 */
+    SET_SPC_BIT(ompi_spc_timer_event, OMPI_SPC_MATCH_TIME);
+    opal_argv_free(arg_strings);
+}
+
+/* Gathers all of the SPC data onto rank 0 of MPI_COMM_WORLD and prints out all
+ * of the counter values to stdout.
+ */
+static void ompi_spc_dump(void)
+{
+    int i, j, world_size, offset;
+    long long *recv_buffer = NULL, *send_buffer;
+
+    int rank = ompi_comm_rank(comm);
+    world_size = ompi_comm_size(comm);
+
+    /* Convert from cycles to usecs before sending */
+    for(i = 0; i < OMPI_SPC_NUM_COUNTERS; i++) {
+        if( IS_SPC_BIT_SET(ompi_spc_timer_event, i) ) {
+            SPC_CYCLES_TO_USECS(&ompi_spc_events[i].value);
+        }
+    }
+
+    /* Aggregate all of the information on rank 0 using MPI_Gather on MPI_COMM_WORLD */
+    send_buffer = (long long*)malloc(OMPI_SPC_NUM_COUNTERS * sizeof(long long));
+    if (NULL == send_buffer) {
+        opal_show_help("help-mpi-runtime.txt", "lib-call-fail", true,
+                       "malloc", __FILE__, __LINE__);
+        return;
+    }
+    for(i = 0; i < OMPI_SPC_NUM_COUNTERS; i++) {
+        send_buffer[i] = (long long)ompi_spc_events[i].value;
+    }
+    if( 0 == rank ) {
+        recv_buffer = (long long*)malloc(world_size * OMPI_SPC_NUM_COUNTERS * sizeof(long long));
+        if (NULL == recv_buffer) {
+            opal_show_help("help-mpi-runtime.txt", "lib-call-fail", true,
+                           "malloc", __FILE__, __LINE__);
+            return;
+        }
+    }
+    (void)comm->c_coll->coll_gather(send_buffer, OMPI_SPC_NUM_COUNTERS, MPI_LONG_LONG,
+                                    recv_buffer, OMPI_SPC_NUM_COUNTERS, MPI_LONG_LONG,
+                                    0, comm,
+                                    comm->c_coll->coll_gather_module);
+
+    /* Once rank 0 has all of the information, print the aggregated counter values for each rank in order */
+    if(rank == 0) {
+        opal_output(0, "Open MPI Software-based Performance Counters:\n");
+        offset = 0; /* Offset into the recv_buffer for each rank */
+        for(j = 0; j < world_size; j++) {
+            opal_output(0, "MPI_COMM_WORLD Rank %d:\n", j);
+            for(i = 0; i < OMPI_SPC_NUM_COUNTERS; i++) {
+                /* If this is a timer-based counter, we need to covert from cycles to usecs */
+                if( 0 == recv_buffer[offset+i] ) {
+                    continue;
+                }
+                opal_output(0, "%s -> %lld\n", ompi_spc_events[i].name, recv_buffer[offset+i]);
+            }
+            opal_output(0, "\n");
+            offset += OMPI_SPC_NUM_COUNTERS;
+        }
+        printf("###########################################################################\n");
+        printf("NOTE: Any counters not shown here were either disabled or had a value of 0.\n");
+        printf("###########################################################################\n");
+
+        free(recv_buffer);
+    }
+    free(send_buffer);
+
+    comm->c_coll->coll_barrier(comm, comm->c_coll->coll_barrier_module);
+}
+
+/* Frees any dynamically alocated OMPI SPC data structures */
+void ompi_spc_fini(void)
+{
+    if (SPC_ENABLE == 1 && ompi_mpi_spc_dump_enabled) {
+        ompi_spc_dump();
+    }
+
+    free(ompi_spc_events); ompi_spc_events = NULL;
+    ompi_comm_free(&comm);
+}
+
+/* Records an update to a counter using an atomic add operation. */
+void ompi_spc_record(unsigned int event_id, ompi_spc_value_t value)
+{
+    /* Denoted unlikely because counters will often be turned off. */
+    if( OPAL_UNLIKELY(IS_SPC_BIT_SET(ompi_spc_attached_event, event_id)) ) {
+        OPAL_THREAD_ADD_FETCH_SIZE_T(&(ompi_spc_events[event_id].value), value);
+    }
+}
+
+/* Starts cycle-precision timer and stores the start value in the 'cycles' argument.
+ * Note: This assumes that the 'cycles' argument is initialized to 0 if the timer
+ *       hasn't been started yet.
+ */
+void ompi_spc_timer_start(unsigned int event_id, opal_timer_t *cycles)
+{
+    /* Check whether cycles == 0.0 to make sure the timer hasn't started yet.
+     * This is denoted unlikely because the counters will often be turned off.
+     */
+    if( OPAL_UNLIKELY(IS_SPC_BIT_SET(ompi_spc_attached_event, event_id) && *cycles == 0) ) {
+        *cycles = opal_timer_base_get_cycles();
+    }
+}
+
+/* Stops a cycle-precision timer and calculates the total elapsed time
+ * based on the starting time in 'cycles' and stores the result in the
+ * 'cycles' argument.
+ */
+void ompi_spc_timer_stop(unsigned int event_id, opal_timer_t *cycles)
+{
+    /* This is denoted unlikely because the counters will often be turned off. */
+    if( OPAL_UNLIKELY(IS_SPC_BIT_SET(ompi_spc_attached_event, event_id)) ) {
+        *cycles = opal_timer_base_get_cycles() - *cycles;
+        OPAL_THREAD_ADD_FETCH_SIZE_T(&ompi_spc_events[event_id].value, (ompi_spc_value_t)*cycles);
+    }
+}
+
+/* Checks a tag, and records the user version of the counter if it's greater
+ * than or equal to 0 and records the mpi version of the counter otherwise.
+ */
+void ompi_spc_user_or_mpi(int tag, ompi_spc_value_t value, unsigned int user_enum, unsigned int mpi_enum)
+{
+    SPC_RECORD( (tag >= 0 ? user_enum : mpi_enum), value);
+}
+
+/* Checks whether the counter denoted by value_enum exceeds the current value of the
+ * counter denoted by watermark_enum, and if so sets the watermark_enum counter to the
+ * value of the value_enum counter.
+ */
+void ompi_spc_update_watermark(unsigned int watermark_enum, unsigned int value_enum)
+{
+    /* Denoted unlikely because counters will often be turned off. */
+    if( OPAL_UNLIKELY(IS_SPC_BIT_SET(ompi_spc_attached_event, watermark_enum) &&
+                      IS_SPC_BIT_SET(ompi_spc_attached_event, value_enum)) ) {
+        /* WARNING: This assumes that this function was called while a lock has already been taken.
+         *          This function is NOT thread safe otherwise!
+         */
+        if(ompi_spc_events[value_enum].value > ompi_spc_events[watermark_enum].value) {
+            ompi_spc_events[watermark_enum].value = ompi_spc_events[value_enum].value;
+        }
+    }
+}
+
+/* Converts a counter value that is in cycles to microseconds.
+ */
+void ompi_spc_cycles_to_usecs(ompi_spc_value_t *cycles)
+{
+    *cycles = *cycles / sys_clock_freq_mhz;
+}

ompi/runtime/ompi_spc.h

@@ -0,0 +1,212 @@
+/*
+ * Copyright (c) 2018      The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ *
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
+#ifndef OMPI_SPC
+#define OMPI_SPC
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <dlfcn.h>
+
+#include "ompi/communicator/communicator.h"
+#include "ompi/datatype/ompi_datatype.h"
+#include "ompi/runtime/params.h"
+#include "opal/mca/timer/timer.h"
+#include "opal/mca/base/mca_base_pvar.h"
+#include "opal/util/argv.h"
+#include "opal/util/show_help.h"
+#include "opal/util/output.h"
+
+#include MCA_timer_IMPLEMENTATION_HEADER
+
+/* INSTRUCTIONS FOR ADDING COUNTERS
+ * 1.) Add a new counter name in the ompi_spc_counters_t enum before
+ *     OMPI_SPC_NUM_COUNTERS below.
+ * 2.) Add corresponding counter name and descriptions to the
+ *     counter_names and counter_descriptions arrays in
+ *     ompi_spc.c  NOTE: The names and descriptions
+ *     MUST be in the same array location as where you added the
+ *     counter name in step 1.
+ * 3.) If this counter is based on a timer, add its enum name to
+ *     the logic for timer-based counters in the ompi_spc_init
+ *     function in ompi_spc.c
+ * 4.) Instrument the Open MPI code base where it makes sense for
+ *     your counter to be modified using the SPC_RECORD macro.
+ *     Note: If your counter is timer-based you should use the
+ *     SPC_TIMER_START and SPC_TIMER_STOP macros to record
+ *     the time in cycles to then be converted to microseconds later
+ *     in the ompi_spc_get_count function when requested by MPI_T
+ */
+
+/* This enumeration serves as event ids for the various events */
+typedef enum ompi_spc_counters {
+    OMPI_SPC_SEND,
+    OMPI_SPC_BSEND,
+    OMPI_SPC_RSEND,
+    OMPI_SPC_SSEND,
+    OMPI_SPC_RECV,
+    OMPI_SPC_MRECV,
+    OMPI_SPC_ISEND,
+    OMPI_SPC_IBSEND,
+    OMPI_SPC_IRSEND,
+    OMPI_SPC_ISSEND,
+    OMPI_SPC_IRECV,
+    OMPI_SPC_SENDRECV,
+    OMPI_SPC_SENDRECV_REPLACE,
+    OMPI_SPC_PUT,
+    OMPI_SPC_RPUT,
+    OMPI_SPC_GET,
+    OMPI_SPC_RGET,
+    OMPI_SPC_PROBE,
+    OMPI_SPC_IPROBE,
+    OMPI_SPC_BCAST,
+    OMPI_SPC_IBCAST,
+    OMPI_SPC_REDUCE,
+    OMPI_SPC_IREDUCE,
+    OMPI_SPC_IREDUCE_SCATTER,
+    OMPI_SPC_IREDUCE_SCATTER_BLOCK,
+    OMPI_SPC_ALLREDUCE,
+    OMPI_SPC_IALLREDUCE,
+    OMPI_SPC_SCAN,
+    OMPI_SPC_ISCAN,
+    OMPI_SPC_SCATTER,
+    OMPI_SPC_SCATTERV,
+    OMPI_SPC_ISCATTER,
+    OMPI_SPC_ISCATTERV,
+    OMPI_SPC_GATHER,
+    OMPI_SPC_GATHERV,
+    OMPI_SPC_IGATHER,
+    OMPI_SPC_ALLTOALL,
+    OMPI_SPC_IALLTOALL,
+    OMPI_SPC_IALLTOALLV,
+    OMPI_SPC_IALLTOALLW,
+    OMPI_SPC_NEIGHBOR_ALLTOALL,
+    OMPI_SPC_NEIGHBOR_ALLTOALLV,
+    OMPI_SPC_NEIGHBOR_ALLTOALLW,
+    OMPI_SPC_ALLGATHER,
+    OMPI_SPC_ALLGATHERV,
+    OMPI_SPC_IALLGATHER,
+    OMPI_SPC_IALLGATHERV,
+    OMPI_SPC_NEIGHBOR_ALLGATHER,
+    OMPI_SPC_NEIGHBOR_ALLGATHERV,
+    OMPI_SPC_TEST,
+    OMPI_SPC_TESTALL,
+    OMPI_SPC_TESTANY,
+    OMPI_SPC_TESTSOME,
+    OMPI_SPC_WAIT,
+    OMPI_SPC_WAITALL,
+    OMPI_SPC_WAITANY,
+    OMPI_SPC_WAITSOME,
+    OMPI_SPC_BARRIER,
+    OMPI_SPC_IBARRIER,
+    OMPI_SPC_WTIME,
+    OMPI_SPC_CANCEL,
+    OMPI_SPC_BYTES_RECEIVED_USER,
+    OMPI_SPC_BYTES_RECEIVED_MPI,
+    OMPI_SPC_BYTES_SENT_USER,
+    OMPI_SPC_BYTES_SENT_MPI,
+    OMPI_SPC_BYTES_PUT,
+    OMPI_SPC_BYTES_GET,
+    OMPI_SPC_UNEXPECTED,
+    OMPI_SPC_OUT_OF_SEQUENCE,
+    OMPI_SPC_MATCH_TIME,
+    OMPI_SPC_UNEXPECTED_IN_QUEUE,
+    OMPI_SPC_OOS_IN_QUEUE,
+    OMPI_SPC_MAX_UNEXPECTED_IN_QUEUE,
+    OMPI_SPC_MAX_OOS_IN_QUEUE,
+    OMPI_SPC_NUM_COUNTERS /* This serves as the number of counters.  It must be last. */
+} ompi_spc_counters_t;
+
+/* There is currently no support for atomics on long long values so we will default to
+ * size_t for now until support for such atomics is implemented.
+ */
+typedef size_t ompi_spc_value_t;
+
+/* A structure for storing the event data */
+typedef struct ompi_spc_s{
+    char *name;
+    ompi_spc_value_t value;
+} ompi_spc_t;
+
+/* Events data structure initialization function */
+void ompi_spc_events_init(void);
+
+/* OMPI SPC utility functions */
+void ompi_spc_init(void);
+void ompi_spc_fini(void);
+void ompi_spc_record(unsigned int event_id, ompi_spc_value_t value);
+void ompi_spc_timer_start(unsigned int event_id, opal_timer_t *cycles);
+void ompi_spc_timer_stop(unsigned int event_id, opal_timer_t *cycles);
+void ompi_spc_user_or_mpi(int tag, ompi_spc_value_t value, unsigned int user_enum, unsigned int mpi_enum);
+void ompi_spc_cycles_to_usecs(ompi_spc_value_t *cycles);
+void ompi_spc_update_watermark(unsigned int watermark_enum, unsigned int value_enum);
+static int ompi_spc_get_count(const struct mca_base_pvar_t *pvar, void *value, void *obj_handle);
+
+/* Macros for using the SPC utility functions throughout the codebase.
+ * If SPC_ENABLE is not 1, the macros become no-ops.
+ */
+#if SPC_ENABLE == 1
+
+#define SPC_INIT()  \
+    ompi_spc_init()
+
+#define SPC_FINI()  \
+    ompi_spc_fini()
+
+#define SPC_RECORD(event_id, value)  \
+    ompi_spc_record(event_id, value)
+
+#define SPC_TIMER_START(event_id, usec)  \
+    ompi_spc_timer_start(event_id, usec)
+
+#define SPC_TIMER_STOP(event_id, usec)  \
+    ompi_spc_timer_stop(event_id, usec)
+
+#define SPC_USER_OR_MPI(tag, value, enum_if_user, enum_if_mpi) \
+    ompi_spc_user_or_mpi(tag, value, enum_if_user, enum_if_mpi)
+
+#define SPC_CYCLES_TO_USECS(cycles) \
+    ompi_spc_cycles_to_usecs(cycles)
+
+#define SPC_UPDATE_WATERMARK(watermark_enum, value_enum) \
+    ompi_spc_update_watermark(watermark_enum, value_enum)
+
+#else /* SPCs are not enabled */
+
+#define SPC_INIT()  \
+    ((void)0)
+
+#define SPC_FINI()  \
+    ((void)0)
+
+#define SPC_RECORD(event_id, value)  \
+    ((void)0)
+
+#define SPC_TIMER_START(event_id, usec)  \
+    ((void)0)
+
+#define SPC_TIMER_STOP(event_id, usec)  \
+    ((void)0)
+
+#define SPC_USER_OR_MPI(tag, value, enum_if_user, enum_if_mpi) \
+    ((void)0)
+
+#define SPC_CYCLES_TO_USECS(cycles) \
+    ((void)0)
+
+#define SPC_UPDATE_WATERMARK(watermark_enum, value_enum) \
+    ((void)0)
+
+#endif
+
+#endif

ompi/runtime/params.h

@@ -3,7 +3,7 @@
  * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
- * Copyright (c) 2004-2005 The University of Tennessee and The University
+ * Copyright (c) 2004-2018 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
@@ -141,6 +141,19 @@ OMPI_DECLSPEC extern bool ompi_async_mpi_init;
 /* EXPERIMENTAL: do not perform an RTE barrier at the beginning of MPI_Finalize */
 OMPI_DECLSPEC extern bool ompi_async_mpi_finalize;
 
+/**
+ * A comma delimited list of SPC counters to turn on or 'attach'.  To turn
+ * all counters on, the string can be simply "all".  An empty string will
+ * keep all counters turned off.
+ */
+OMPI_DECLSPEC extern char * ompi_mpi_spc_attach_string;
+
+/**
+ * A boolean value that determines whether or not to dump the SPC counter
+ * values in MPI_Finalize.  A value of true dumps the counters and false does not.
+ */
+OMPI_DECLSPEC extern bool ompi_mpi_spc_dump_enabled;
+
 
 /**
  * Register MCA parameters used by the MPI layer.

opal/mca/base/mca_base_pvar.h

@@ -2,6 +2,9 @@
 /*
  * Copyright (c) 2013-2015 Los Alamos National Security, LLC. All rights
  *                         reserved.
+ * Copyright (c) 2018 The University of Tennessee and The University
+ *                    of Tennessee Research Foundation.  All rights
+ *                    reserved.
  *
  * Additional copyrights may follow
  *
@@ -319,7 +322,7 @@ OPAL_DECLSPEC int mca_base_pvar_register (const char *project, const char *frame
  * associated with a component.
  *
  * While quite similar to mca_base_pvar_register(), there is one key
- * difference: pvars registered this this function will automatically
+ * difference: pvars registered with this function will automatically
  * be unregistered / made unavailable when that component is closed by
  * its framework.
  */

test/Makefile.am

@@ -2,7 +2,7 @@
 # Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
 #                         University Research and Technology
 #                         Corporation.  All rights reserved.
-# Copyright (c) 2004-2015 The University of Tennessee and The University
+# Copyright (c) 2004-2018 The University of Tennessee and The University
 #                         of Tennessee Research Foundation.  All rights
 #                         reserved.
 # Copyright (c) 2004-2009 High Performance Computing Center Stuttgart,
@@ -24,6 +24,6 @@
 # support needs to be first for dependencies
 SUBDIRS = support asm class threads datatype util dss mpool
 if PROJECT_OMPI
-SUBDIRS += monitoring
+SUBDIRS += monitoring spc
 endif
 DIST_SUBDIRS = event $(SUBDIRS)

test/spc/Makefile.am

@@ -0,0 +1,24 @@
+#
+# Copyright (c) 2018 The University of Tennessee and The University
+#                    of Tennessee Research Foundation.  All rights
+#                    reserved.
+# $COPYRIGHT$
+#
+# Additional copyrights may follow
+#
+# $HEADER$
+#
+
+# This test requires multiple processes to run. Don't run it as part
+# of 'make check'
+if PROJECT_OMPI
+    noinst_PROGRAMS = spc_test
+    spc_test_SOURCES = spc_test.c
+    spc_test_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS)
+    spc_test_LDADD = \
+        $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \
+        $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la
+endif # PROJECT_OMPI
+
+distclean:
+	rm -rf *.dSYM .deps .libs *.la *.lo spc_test prof *.log *.o *.trs Makefile

test/spc/spc_test.c

@@ -0,0 +1,174 @@
+/*
+ * Copyright (c) 2018 The University of Tennessee and The University
+ *                    of Tennessee Research Foundation.  All rights
+ *                    reserved.
+ *
+ * Simple example usage of SPCs through MPI_T.
+ */
+
+#include "mpi.h"
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#define MAX_SIZE 1000000
+
+/* Sends 'num_messages' messages of 'message_size' bytes from rank 0 to rank 1.
+ * All messages are sent synchronously and with the same tag in MPI_COMM_WORLD.
+ */
+static void message_exchange(int num_messages, int message_size)
+{
+    int i, rank;
+    /* Use calloc to initialize data to 0's */
+    char *data = (char*)calloc(message_size, sizeof(char));
+    MPI_Status status;
+
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+
+    if(rank == 0) {
+        for(i = 0; i < num_messages; i++)
+            MPI_Send(data, message_size, MPI_BYTE, 1, 123, MPI_COMM_WORLD);
+    } else if(rank == 1) {
+        for(i = 0; i < num_messages; i++)
+            MPI_Recv(data, message_size, MPI_BYTE, 0, 123, MPI_COMM_WORLD, &status);
+    }
+
+    free(data);
+}
+
+int main(int argc, char **argv)
+{
+    int i, rank, size, provided, num, name_len, desc_len, verbosity, bind, var_class, readonly, continuous, atomic, count, index, MPI_result;
+    MPI_Datatype datatype;
+    MPI_T_enum enumtype;
+    char name[256], description[256];
+
+    /* Counter names to be read by ranks 0 and 1 */
+    char *counter_names[] = { "runtime_spc_OMPI_BYTES_SENT_USER",
+                              "runtime_spc_OMPI_BYTES_RECEIVED_USER" };
+
+    MPI_Init(NULL, NULL);
+    MPI_result = MPI_T_init_thread(MPI_THREAD_SINGLE, &provided);
+    if(MPI_result != MPI_SUCCESS) {
+        fprintf(stderr, "Failed to initialize MPI_T thread.\n");
+        MPI_Abort(MPI_COMM_WORLD, MPI_result);
+    }
+
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    MPI_Comm_size(MPI_COMM_WORLD, &size);
+    if(size != 2) {
+        fprintf(stderr, "ERROR: This test should be run with two MPI processes.\n");
+        MPI_Abort(MPI_COMM_WORLD, -1);
+    }
+
+    /* Determine the MPI_T pvar indices for the OMPI_BYTES_SENT/RECIEVED_USER SPCs */
+    index = -1;
+    MPI_result = MPI_T_pvar_get_num(&num);
+    if(MPI_result != MPI_SUCCESS) {
+        fprintf(stderr, "Failed to get the number of pvars.\n");
+        MPI_Abort(MPI_COMM_WORLD, MPI_result);
+    }
+
+    for(i = 0; i < num; i++) {
+        name_len = desc_len = 256;
+        MPI_T_pvar_get_info(i, name, &name_len, &verbosity,
+                             &var_class, &datatype, &enumtype, description, &desc_len, &bind,
+                             &readonly, &continuous, &atomic);
+        if(MPI_result != MPI_SUCCESS || MPI_result == MPI_T_ERR_PVAR_NO_STARTSTOP) {
+            fprintf(stderr, "Failed to get pvar info.\n");
+            MPI_Abort(MPI_COMM_WORLD, MPI_result);
+        }
+
+        if(strcmp(name, counter_names[rank]) == 0) {
+            index = i;
+            printf("[%d] %s -> %s\n", rank, name, description);
+        }
+    }
+
+    /* Make sure we found the counters */
+    if(index == -1) {
+        fprintf(stderr, "ERROR: Couldn't find the appropriate SPC counter in the MPI_T pvars.\n");
+        MPI_Abort(MPI_COMM_WORLD, -1);
+    }
+
+    long long value;
+
+    MPI_T_pvar_session session;
+    MPI_T_pvar_handle handle;
+    /* Create the MPI_T sessions/handles for the counters and start the counters */
+    MPI_result = MPI_T_pvar_session_create(&session);
+    if(MPI_result != MPI_SUCCESS) {
+        fprintf(stderr, "Failed to create MPI_T pvar session.\n");
+        MPI_Abort(MPI_COMM_WORLD, MPI_result);
+    }
+
+    MPI_result = MPI_T_pvar_handle_alloc(session, index, NULL, &handle, &count);
+    if(MPI_result != MPI_SUCCESS) {
+        fprintf(stderr, "Failed to allocate the pvar handle.\n");
+        MPI_Abort(MPI_COMM_WORLD, MPI_result);
+    }
+
+    MPI_result = MPI_T_pvar_start(session, handle);
+    if(MPI_result != MPI_SUCCESS) {
+        if(MPI_result != MPI_T_ERR_PVAR_NO_STARTSTOP) {
+            fprintf(stderr, "Failed to start the pvar session.\n");
+            MPI_Abort(MPI_COMM_WORLD, MPI_result);
+        }
+    }
+
+    int message_size = 1, expected_bytes = 0;
+    while(message_size <= MAX_SIZE) {
+        expected_bytes += message_size;
+        message_exchange(1, message_size);
+        message_size *= 10;
+    }
+
+    MPI_result = MPI_T_pvar_read(session, handle, &value);
+    if(MPI_result != MPI_SUCCESS) {
+        fprintf(stderr, "Failed to read the pvar.\n");
+        MPI_Abort(MPI_COMM_WORLD, MPI_result);
+    }
+
+    /* Print the counter values in order by rank */
+    for(i = 0; i < 2; i++) {
+        if(i == rank) {
+            printf("[%d] Value Read: %lld\n", rank, value);
+            fflush(stdout);
+            if(value != expected_bytes){
+                fprintf(stderr, "The counter value is inaccurate!  It is '%lld'.  It should be '%d'\n", value, expected_bytes);
+                MPI_Abort(MPI_COMM_WORLD, MPI_ERR_OTHER);
+            }
+        }
+        MPI_Barrier(MPI_COMM_WORLD);
+    }
+    /* Stop the MPI_T session, free the handle, and then free the session */
+    MPI_result = MPI_T_pvar_stop(session, handle);
+    if(MPI_result != MPI_SUCCESS) {
+        if(MPI_result != MPI_T_ERR_PVAR_NO_STARTSTOP) {
+            fprintf(stderr, "Failed to stop the pvar session.\n");
+            MPI_Abort(MPI_COMM_WORLD, MPI_result);
+        }
+    }
+
+    MPI_result = MPI_T_pvar_handle_free(session, &handle);
+    if(MPI_result != MPI_SUCCESS) {
+        fprintf(stderr, "Failed to free the pvar handle.\n");
+        MPI_Abort(MPI_COMM_WORLD, MPI_result);
+    }
+
+    MPI_result = MPI_T_pvar_session_free(&session);
+    if(MPI_result != MPI_SUCCESS) {
+        fprintf(stderr, "Failed to free the pvar session.\n");
+        MPI_Abort(MPI_COMM_WORLD, MPI_result);
+    }
+
+    MPI_result = MPI_T_finalize();
+    if(MPI_result != MPI_SUCCESS) {
+        fprintf(stderr, "Failed to finalize MPI_T.\n");
+        MPI_Abort(MPI_COMM_WORLD, MPI_result);
+    }
+
+    MPI_Finalize();
+
+    return EXIT_SUCCESS;
+}