From 701a1d0218274f486f0f7dbf77e134ac8f554ac3 Mon Sep 17 00:00:00 2001 From: Howard Pritchard Date: Wed, 12 Jul 2017 18:28:02 -0600 Subject: [PATCH] mtl/psm2: add pvar support for PSM2 MQ stats Add pvars for PSM2 MQ stats to help in analyzing performance of Omnipath. Tested (modestly) using modified OSU pt2pt benchmarks. Signed-off-by: Howard Pritchard --- ompi/mca/mtl/psm2/Makefile.am | 4 ++ ompi/mca/mtl/psm2/mtl_psm2.h | 3 +- ompi/mca/mtl/psm2/mtl_psm2_component.c | 5 +- ompi/mca/mtl/psm2/mtl_psm2_stats.c | 98 ++++++++++++++++++++++++++ 4 files changed, 107 insertions(+), 3 deletions(-) create mode 100644 ompi/mca/mtl/psm2/mtl_psm2_stats.c diff --git a/ompi/mca/mtl/psm2/Makefile.am b/ompi/mca/mtl/psm2/Makefile.am index fa3c5201bb..21a4c87315 100644 --- a/ompi/mca/mtl/psm2/Makefile.am +++ b/ompi/mca/mtl/psm2/Makefile.am @@ -11,6 +11,9 @@ # All rights reserved. # Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2015 Intel, Inc. All rights reserved +# Copyright (c) 2017 Los Alamos National Security, LLC. +# All rights reserved. +# # $COPYRIGHT$ # # Additional copyrights may follow @@ -35,6 +38,7 @@ mtl_psm2_sources = \ mtl_psm2_recv.c \ mtl_psm2_request.h \ mtl_psm2_send.c \ + mtl_psm2_stats.c \ mtl_psm2_types.h # Make the output library in this directory, and name it either diff --git a/ompi/mca/mtl/psm2/mtl_psm2.h b/ompi/mca/mtl/psm2/mtl_psm2.h index 44152656bf..cea7d323a0 100644 --- a/ompi/mca/mtl/psm2/mtl_psm2.h +++ b/ompi/mca/mtl/psm2/mtl_psm2.h @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2006 QLogic Corporation. All rights reserved. * Copyright (c) 2015 Intel, Inc. All rights reserved - * Copyright (c) 2015 Los Alamos National Security, LLC. All rights + * Copyright (c) 2015-2017 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ * @@ -103,6 +103,7 @@ extern int ompi_mtl_psm2_finalize(struct mca_mtl_base_module_t* mtl); int ompi_mtl_psm2_module_init(int local_rank, int num_local_procs); +extern int ompi_mtl_psm2_register_pvars(void); END_C_DECLS diff --git a/ompi/mca/mtl/psm2/mtl_psm2_component.c b/ompi/mca/mtl/psm2/mtl_psm2_component.c index e899dde4f6..a536fd6efb 100644 --- a/ompi/mca/mtl/psm2/mtl_psm2_component.c +++ b/ompi/mca/mtl/psm2/mtl_psm2_component.c @@ -115,7 +115,7 @@ static void ompi_mtl_psm2_set_shadow_env (struct ompi_mtl_psm2_shadow_variable * { mca_base_var_storage_t *storage = variable->storage; char *env_value; - int ret; + int ret = 0; switch (variable->variable_type) { case MCA_BASE_VAR_TYPE_BOOL: @@ -230,11 +230,12 @@ ompi_mtl_psm2_component_register(void) MCA_BASE_VAR_SCOPE_READONLY, ¶m_priority); - for (int i = 0 ; ompi_mtl_psm2_shadow_variables[i].variable_type >= 0 ; ++i) { ompi_mtl_psm2_register_shadow_env (ompi_mtl_psm2_shadow_variables + i); } + ompi_mtl_psm2_register_pvars(); + return OMPI_SUCCESS; } diff --git a/ompi/mca/mtl/psm2/mtl_psm2_stats.c b/ompi/mca/mtl/psm2/mtl_psm2_stats.c new file mode 100644 index 0000000000..ad3d879a3b --- /dev/null +++ b/ompi/mca/mtl/psm2/mtl_psm2_stats.c @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2010 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2006 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2006 QLogic Corporation. All rights reserved. + * Copyright (c) 2006-2017 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2013-2015 Intel, Inc. All rights reserved + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include "ompi_config.h" +#include "mtl_psm2.h" +#include "mtl_psm2_types.h" +#include "psm2.h" +#include "ompi/communicator/communicator.h" +#include "ompi/message/message.h" + +#include "opal/mca/base/mca_base_pvar.h" + +struct ompi_mtl_psm2_name_descs +{ + char *name; + char *desc; + ptrdiff_t offset; +}; + +const struct ompi_mtl_psm2_name_descs name_descs[PSM2_MQ_NUM_STATS] = +{ + { "rx_user_bytes", "Bytes received into a matched user buffer", + offsetof(struct psm2_mq_stats, rx_user_bytes) }, + { "rx_user_num", "Messages received into a matched user buffer", + offsetof(struct psm2_mq_stats, rx_user_num) }, + { "rx_sys_bytes", "Bytes received into an unmatched system buffer", + offsetof(struct psm2_mq_stats, rx_sys_bytes) }, + { "rx_sys_num", "Messages received into an unmatched system buffer", + offsetof(struct psm2_mq_stats, rx_sys_num) }, + { "tx_num", "Total Messages transmitted (shm and hfi)", + offsetof(struct psm2_mq_stats, tx_num) }, + { "tx_eager_num", "Messages transmitted eagerly", + offsetof(struct psm2_mq_stats, tx_eager_num) }, + { "tx_eager_bytes", "Bytes transmitted eagerl", + offsetof(struct psm2_mq_stats, tx_eager_bytes) }, + { "tx_rndv_num", "Messages transmitted using expected TID mechanism", + offsetof(struct psm2_mq_stats, tx_rndv_num) }, + { "tx_rndv_bytes", "Bytes transmitted using expected TID mechanism", + offsetof(struct psm2_mq_stats, tx_rndv_bytes) }, + { "tx_shm_num", "Messages transmitted (shm only)", + offsetof(struct psm2_mq_stats, tx_shm_num) }, + { "rx_shm_num", "Messages received through shm", + offsetof(struct psm2_mq_stats, rx_shm_num) }, + { "rx_sysbuf_num", "Number of system buffers allocated", + offsetof(struct psm2_mq_stats, rx_sysbuf_num) }, + { "rx_sysbuf_bytes", "Bytes allocated for system buffers", + offsetof(struct psm2_mq_stats, rx_sysbuf_bytes) }, +}; + +static int mca_mtl_psm2_get_stats(const mca_base_pvar_t *pvar, void *value, void *obj) +{ + psm2_mq_stats_t stats; + int index = (int)(intptr_t) pvar->ctx; + + psm2_mq_get_stats(ompi_mtl_psm2.mq, &stats); + + *(uint64_t *)value = *(uint64_t *)((uint8_t *)&stats + name_descs[index].offset); + + return OMPI_SUCCESS; +} + + +int ompi_mtl_psm2_register_pvars(void) +{ + int i; + + /* PSM2 MQ performance variables */ + for (i = 0 ; i < PSM2_MQ_NUM_STATS; ++i) { + (void) mca_base_component_pvar_register (&mca_mtl_psm2_component.super.mtl_version, + name_descs[i].name, name_descs[i].desc, + OPAL_INFO_LVL_4, MCA_BASE_PVAR_CLASS_COUNTER, + MCA_BASE_VAR_TYPE_UNSIGNED_LONG, + NULL, MCA_BASE_VAR_BIND_NO_OBJECT, + MCA_BASE_PVAR_FLAG_READONLY | MCA_BASE_PVAR_FLAG_CONTINUOUS, + mca_mtl_psm2_get_stats, NULL, NULL, + (void *) (intptr_t) i); + } + return OMPI_SUCCESS; +}