ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Move all usNIC stats to _stats.c|h and export them as MPI_T pvars.

Просмотр исходного кода 
This commit moves all the module stats into their own struct so that
the stats only need to appear as a single line in the module_t
definition, and then moves all the logic for reporting the stats into
btl_usnic_stats.c|h.

Further, the stats are now exported as MPI_T_BIND_NO_OBJECT entities
(i.e., not bound to any particular MPI handle), and are marked as
READONLY and CONTINUOUS.  They currently all default to verbose level
5 ("Application tuner / detailed", according to
https://svn.open-mpi.org/trac/ompi/wiki/MCAParamLevels).

Most of the statistics are counters, but a small number are high
watermark values.  Due to how counters are reported via MPI_T, none of
the counters are exported through MPI_T if the MCA param
btl_usnic_stats_relative=1 (i.e., the module resets the stats back to
zero at a given frequency).

When MPI_T_pvar_handle_alloc() is invoked on any of these pvars, it
will return a count that is equal to the number of active usnic BTL
modules.  The values returned for any given pvar (e.g.,
num_total_sends) are an array containing one value for each active
usnic BTL module.  The ordering of values in the array is both
consistent across all usnic pvars and stable throughout a single job:
array slot 0 corresponds to module X, array slot 1 corresponds to
module Y, etc.

Mapping which array slot corresponds to which underlying Linux usnic_X
device works as follows:

 * The btl_usnic_devices MPI_T state pvar is associated with a
   btl_usnic_device MPI_T enum, and be obtained via
   MPI_T_pvar_get_info().
 * If all usNIC pvars are of length N, the values [0,N) in the
   btl_usnic_device enum are associated with strings of the
   corresponding underlying Linux device.

For exampe, to look up which Linux device is reported in all usNIC
pvars' array slot 1, look up the int value 1 in the btl_usnic_devices
enum.  Its corresponding string value is underlying Linux device name
(e.g., "usnic_1").

cmr=v1.7.4:subject="usnic BTL MPI_T pvars"

This commit was SVN r29545.
Этот коммит содержится в:
Jeff Squyres 2013-10-28 22:23:08 +00:00
родитель 404cceb9c4
Коммит 6569019b06
11 изменённых файлов: 644 добавлений и 249 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

2
ompi/mca/btl/usnic/Makefile.am
Просмотреть файл

@ -44,6 +44,8 @@ sources = \
btl_usnic_recv.h \
btl_usnic_send.c \
btl_usnic_send.h \
btl_usnic_stats.h \
btl_usnic_stats.c \
btl_usnic_util.c \
btl_usnic_util.h

10
ompi/mca/btl/usnic/btl_usnic_ack.c
Просмотреть файл

@ -49,7 +49,7 @@ ompi_btl_usnic_force_retrans(
opal_list_append(&(endpoint->endpoint_module->pending_resend_segs),
&(sseg->ss_base.us_list.super));
++endpoint->endpoint_module->num_fast_retrans;
++endpoint->endpoint_module->stats.num_fast_retrans;
}
@ -76,12 +76,12 @@ ompi_btl_usnic_handle_ack(
opal_output(0, "Got OLD DUP ACK seq %"UDSEQ" < %"UDSEQ"\n",
ack_seq, endpoint->endpoint_ack_seq_rcvd);
#endif
++module->num_old_dup_acks;
++module->stats.num_old_dup_acks;
return;
/* A duplicate ACK means next seg was lost */
} else if (ack_seq == endpoint->endpoint_ack_seq_rcvd) {
++module->num_dup_acks;
++module->stats.num_dup_acks;
ompi_btl_usnic_force_retrans(endpoint, ack_seq);
return;
@ -230,7 +230,7 @@ ompi_btl_usnic_ack_send(
ompi_btl_usnic_post_segment(module, endpoint, ack);
/* Stats */
++module->num_ack_sends;
++module->stats.num_ack_sends;
return;
}
@ -282,6 +282,6 @@ ompi_btl_usnic_ack_timeout(
&(seg->ss_base.us_list.super));
/* Stats */
++module->num_timeout_retrans;
++module->stats.num_timeout_retrans;
}

7
ompi/mca/btl/usnic/btl_usnic_component.c
Просмотреть файл

@ -68,6 +68,7 @@
#include "btl_usnic_frag.h"
#include "btl_usnic_endpoint.h"
#include "btl_usnic_module.h"
#include "btl_usnic_stats.h"
#include "btl_usnic_util.h"
#include "btl_usnic_ack.h"
#include "btl_usnic_send.h"
@ -812,6 +813,10 @@ static mca_btl_base_module_t** usnic_component_init(int* num_btl_modules,
usnic_clock_timeout.tv_usec = 1000;
opal_event_add(&usnic_clock_timer_event, &usnic_clock_timeout);
/* Setup MPI_T performance variables */
ompi_btl_usnic_setup_mpit_pvars();
/* All done */
*num_btl_modules = mca_btl_usnic_component.num_modules;
opal_output_verbose(5, USNIC_OUT,
"btl:usnic: returning %d modules", *num_btl_modules);
@ -916,7 +921,7 @@ static int usnic_handle_completion(
cwc->vendor_err, cwc->byte_len));
} else {
/* silently count CRC errors */
++module->num_crc_errors;
++module->stats.num_crc_errors;
}
rseg->rs_recv_desc.next = channel->repost_recv_head;
channel->repost_recv_head = &rseg->rs_recv_desc;

202
ompi/mca/btl/usnic/btl_usnic_module.c
Просмотреть файл

@ -49,6 +49,7 @@
#include "btl_usnic_send.h"
#include "btl_usnic_ack.h"
#include "btl_usnic_hwloc.h"
#include "btl_usnic_stats.h"
static void
ompi_btl_usnic_channel_finalize(
@ -684,175 +685,6 @@ usnic_put(
return OMPI_SUCCESS;
}
static inline void usnic_stats_reset(ompi_btl_usnic_module_t *module)
{
int i;
module->num_total_sends =
module->num_resends =
module->num_chunk_sends =
module->num_frag_sends =
module->num_ack_recvs =
module->num_total_recvs =
module->num_unk_recvs =
module->num_dup_recvs =
module->num_oow_low_recvs =
module->num_oow_high_recvs =
module->num_frag_recvs =
module->num_chunk_recvs =
module->num_badfrag_recvs =
module->num_ack_sends =
module->num_recv_reposts =
module->num_crc_errors =
module->num_old_dup_acks =
module->num_dup_acks =
module->num_fast_retrans =
module->num_timeout_retrans =
module->max_sent_window_size =
module->max_rcvd_window_size =
module->pml_module_sends =
module->pml_send_callbacks =
0;
for (i=0; i<USNIC_NUM_CHANNELS; ++i) {
module->mod_channels[i].num_channel_sends = 0;
}
}
/* Prints a a few terse statistics lines via opal_output(0,...). The first
* line will be prefixed with the string "prefix". If "reset_stats" is true
* then the statistics will be reset after printing.
*
* NOTE: this routine ignores the setting of stats_enable, so it can be used
* for debugging routines even when normal stats reporting is not enabled.
*/
void ompi_btl_usnic_print_stats(
ompi_btl_usnic_module_t *module,
const char *prefix,
bool reset_stats)
{
char tmp[128], str[2048];
/* The usuals */
snprintf(str, sizeof(str), "%s:MCW:%3u, ST(P+D)/F/C/R(T+F)/A:%8lu(%8u+%8u)/%8lu/%8lu/%4lu(%4lu+%4lu)/%8lu, RcvTot/Chk/F/C/L/H/D/BF/A:%8lu/%c%c/%8lu/%8lu/%4lu+%2lu/%4lu/%4lu/%6lu OA/DA %4lu/%4lu CRC:%4lu ",
prefix,
ompi_proc_local()->proc_name.vpid,
module->num_total_sends,
module->mod_channels[USNIC_PRIORITY_CHANNEL].num_channel_sends,
module->mod_channels[USNIC_DATA_CHANNEL].num_channel_sends,
module->num_frag_sends,
module->num_chunk_sends,
module->num_resends,
module->num_timeout_retrans,
module->num_fast_retrans,
module->num_ack_sends,
module->num_total_recvs,
(module->num_total_recvs - module->num_recv_reposts) == 0 ? 'g' : 'B',
(module->num_total_recvs -
module->num_frag_recvs -
module->num_chunk_recvs -
module->num_badfrag_recvs -
module->num_oow_low_recvs -
module->num_oow_high_recvs -
module->num_dup_recvs -
module->num_ack_recvs -
module->num_unk_recvs) == 0 ? 'g' : 'B',
module->num_frag_recvs,
module->num_chunk_recvs,
module->num_oow_low_recvs,
module->num_oow_high_recvs,
module->num_dup_recvs,
module->num_badfrag_recvs,
module->num_ack_recvs,
module->num_old_dup_acks,
module->num_dup_acks,
module->num_crc_errors);
/* If our PML calls were 0, then show send and receive window
extents instead */
if (module->pml_module_sends +
module->pml_send_callbacks == 0) {
int64_t send_unacked, su_min = WINDOW_SIZE * 2, su_max = 0;
int64_t recv_depth, rd_min = WINDOW_SIZE * 2, rd_max = 0;
ompi_btl_usnic_endpoint_t *endpoint;
opal_list_item_t *item;
rd_min = su_min = WINDOW_SIZE * 2;
rd_max = su_max = 0;
item = opal_list_get_first(&module->all_endpoints);
while (item != opal_list_get_end(&(module->all_endpoints))) {
endpoint = container_of(item, mca_btl_base_endpoint_t,
endpoint_endpoint_li);
item = opal_list_get_next(item);
/* Number of un-acked sends (i.e., sends for which we're
still waiting for ACK) */
send_unacked =
endpoint->endpoint_next_seq_to_send -
endpoint->endpoint_ack_seq_rcvd - 1;
if (send_unacked > su_max) su_max = send_unacked;
if (send_unacked < su_min) su_min = send_unacked;
/* Receive window depth (i.e., difference between highest
seq received and the next message we haven't ACKed
yet) */
recv_depth =
endpoint->endpoint_highest_seq_rcvd -
endpoint->endpoint_next_contig_seq_to_recv;
if (recv_depth > rd_max) rd_max = recv_depth;
if (recv_depth < rd_min) rd_min = recv_depth;
}
snprintf(tmp, sizeof(tmp), "PML S:%1ld, Win!A/R:%4ld/%4ld %4ld/%4ld",
module->pml_module_sends,
su_min, su_max,
rd_min, rd_max);
} else {
snprintf(tmp, sizeof(tmp), "PML S/CB/Diff:%4lu/%4lu=%4ld",
module->pml_module_sends,
module->pml_send_callbacks,
module->pml_module_sends - module->pml_send_callbacks);
}
strncat(str, tmp, sizeof(str) - strlen(str) - 1);
opal_output(0, "%s", str);
if (reset_stats) {
usnic_stats_reset(module);
}
}
/* callback routine for libevent */
static void usnic_stats_callback(int fd, short flags, void *arg)
{
ompi_btl_usnic_module_t *module = (ompi_btl_usnic_module_t*) arg;
char tmp[128];
if (!mca_btl_usnic_component.stats_enabled) {
return;
}
snprintf(tmp, sizeof(tmp), "%4lu", ++module->stats_report_num);
ompi_btl_usnic_print_stats(module, tmp,
/*reset=*/mca_btl_usnic_component.stats_relative);
/* In OMPI v1.6, we have to re-add this event (because there's an
old libevent in OMPI v1.6) */
opal_event_add(&(module->stats_timer_event),
&(module->stats_timeout));
}
static int usnic_finalize(struct mca_btl_base_module_t* btl)
{
int i;
@ -870,12 +702,8 @@ static int usnic_finalize(struct mca_btl_base_module_t* btl)
ompi_btl_usnic_channel_finalize(module,
&module->mod_channels[USNIC_PRIORITY_CHANNEL]);
/* Disable the stats callback event, and then call the stats
callback manually to display the final stats */
if (mca_btl_usnic_component.stats_enabled) {
opal_event_del(&(module->stats_timer_event));
ompi_btl_usnic_print_stats(module, "final", /*reset_stats=*/false);
}
/* Shutdown the stats on this module */
ompi_btl_usnic_stats_finalize(module);
/* Empty the all_endpoints list so that we can destroy endpoints
* (i.e., so that their super/opal_list_item_t member isn't still
@ -1000,7 +828,7 @@ usnic_do_resends(
* credits more.
*/
--endpoint->endpoint_send_credits;
++module->num_resends;
++module->stats.num_resends;
}
/* restart the retrans timer */
@ -1142,7 +970,7 @@ usnic_handle_large_send(
frag->sf_base.uf_base.des_cbfunc(&module->super,
frag->sf_endpoint, &frag->sf_base.uf_base,
OMPI_SUCCESS);
++module->pml_send_callbacks;
++module->stats.pml_send_callbacks;
frag->sf_base.uf_base.des_flags &= ~MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
}
}
@ -1251,7 +1079,7 @@ ompi_btl_usnic_module_progress_sends(
frag->sf_base.uf_base.des_cbfunc(&module->super,
frag->sf_endpoint, &frag->sf_base.uf_base,
OMPI_SUCCESS);
++module->pml_send_callbacks;
++module->stats.pml_send_callbacks;
frag->sf_base.uf_base.des_flags &=
~MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
}
@ -1447,8 +1275,8 @@ usnic_send(
descriptor->des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
rc = 0;
}
++module->pml_module_sends;
++module->pml_send_callbacks; /* returning "1" is an implicit CB */
++module->stats.pml_module_sends;
++module->stats.pml_send_callbacks; /* returning "1" is an implicit CB */
return rc;
} else {
/*
@ -2089,18 +1917,8 @@ int ompi_btl_usnic_module_init(ompi_btl_usnic_module_t *module)
assert(OMPI_SUCCESS == rc);
}
if (mca_btl_usnic_component.stats_enabled) {
usnic_stats_reset(module);
module->stats_timeout.tv_sec = mca_btl_usnic_component.stats_frequency;
module->stats_timeout.tv_usec = 0;
opal_event_set(opal_event_base, &(module->stats_timer_event),
-1, EV_TIMEOUT | EV_PERSIST,
&usnic_stats_callback, module);
opal_event_add(&(module->stats_timer_event),
&(module->stats_timeout));
}
/* Initialize stats on this module */
ompi_btl_usnic_stats_init(module);
return OMPI_SUCCESS;

37
ompi/mca/btl/usnic/btl_usnic_module.h
Просмотреть файл

@ -29,6 +29,7 @@
#include "ompi/mca/common/verbs/common_verbs.h"
#include "btl_usnic_endpoint.h"
#include "btl_usnic_stats.h"
/*
* Default limits.
@ -176,40 +177,8 @@ typedef struct ompi_btl_usnic_module_t {
uint32_t qp_max_inline;
/* Debugging statistics */
bool final_stats;
uint64_t stats_report_num;
uint64_t num_total_sends;
uint64_t num_resends;
uint64_t num_timeout_retrans;
uint64_t num_fast_retrans;
uint64_t num_chunk_sends;
uint64_t num_frag_sends;
uint64_t num_ack_sends;
uint64_t num_total_recvs;
uint64_t num_unk_recvs;
uint64_t num_dup_recvs;
uint64_t num_oow_low_recvs;
uint64_t num_oow_high_recvs;
uint64_t num_frag_recvs;
uint64_t num_chunk_recvs;
uint64_t num_badfrag_recvs;
uint64_t num_ack_recvs;
uint64_t num_old_dup_acks;
uint64_t num_dup_acks;
uint64_t num_recv_reposts;
uint64_t num_crc_errors;
uint64_t max_sent_window_size;
uint64_t max_rcvd_window_size;
uint64_t pml_module_sends;
uint64_t pml_send_callbacks;
opal_event_t stats_timer_event;
struct timeval stats_timeout;
/* Performance / debugging statistics */
ompi_btl_usnic_module_stats_t stats;
} ompi_btl_usnic_module_t;
struct ompi_btl_usnic_frag_t;

14
ompi/mca/btl/usnic/btl_usnic_recv.c
Просмотреть файл

@ -66,7 +66,7 @@ void ompi_btl_usnic_recv_call(ompi_btl_usnic_module_t *module,
bseg = &seg->rs_base;
++module->num_total_recvs;
++module->stats.num_total_recvs;
/* Valgrind help */
opal_memchecker_base_mem_defined((void*)(seg->rs_recv_desc.sg_list[0].addr),
@ -95,7 +95,7 @@ void ompi_btl_usnic_recv_call(ompi_btl_usnic_module_t *module,
dest_mac,
bseg->us_btl_header->seq);
#endif
++module->num_unk_recvs;
++module->stats.num_unk_recvs;
goto repost_no_endpoint;
}
@ -245,7 +245,7 @@ void ompi_btl_usnic_recv_call(ompi_btl_usnic_module_t *module,
/* frag_id is not 0 - it must match, drop if not */
} else if (fip->rfi_frag_id != chunk_hdr->ch_frag_id) {
++module->num_badfrag_recvs;
++module->stats.num_badfrag_recvs;
goto repost;
}
#if MSGDEBUG1
@ -256,7 +256,7 @@ void ompi_btl_usnic_recv_call(ompi_btl_usnic_module_t *module,
#endif
/* Stats */
++module->num_chunk_recvs;
++module->stats.num_chunk_recvs;
/* validate offset and len to be within fragment */
assert(chunk_hdr->ch_frag_offset + chunk_hdr->ch_hdr.payload_len <=
@ -331,7 +331,7 @@ void ompi_btl_usnic_recv_call(ompi_btl_usnic_module_t *module,
ack_seq = bseg->us_btl_header->ack_seq;
/* Stats */
++module->num_ack_recvs;
++module->stats.num_ack_recvs;
#if MSGDEBUG1
opal_output(0, " Received ACK for sequence number %" UDSEQ " from %s to %s\n",
@ -345,7 +345,7 @@ void ompi_btl_usnic_recv_call(ompi_btl_usnic_module_t *module,
/***********************************************************************/
/* Have no idea what the frag is; drop it */
else {
++module->num_unk_recvs;
++module->stats.num_unk_recvs;
opal_output(0, "==========================unknown 2");
goto repost;
}
@ -358,7 +358,7 @@ void ompi_btl_usnic_recv_call(ompi_btl_usnic_module_t *module,
OBJ_RELEASE(endpoint);
}
repost_no_endpoint:
++module->num_recv_reposts;
++module->stats.num_recv_reposts;
/* Add recv to linked list for reposting */
seg->rs_recv_desc.next = channel->repost_recv_head;

12
ompi/mca/btl/usnic/btl_usnic_recv.h
Просмотреть файл

@ -166,9 +166,9 @@ ompi_btl_usnic_check_rx_seq(
/* Stats */
if (seq < endpoint->endpoint_next_contig_seq_to_recv) {
++endpoint->endpoint_module->num_oow_low_recvs;
++endpoint->endpoint_module->stats.num_oow_low_recvs;
} else {
++endpoint->endpoint_module->num_oow_high_recvs;
++endpoint->endpoint_module->stats.num_oow_high_recvs;
}
goto dup_needs_ack;
}
@ -211,7 +211,7 @@ ompi_btl_usnic_check_rx_seq(
assert (seq > endpoint->endpoint_next_contig_seq_to_recv - 1);
/* Stats */
++endpoint->endpoint_module->num_dup_recvs;
++endpoint->endpoint_module->stats.num_dup_recvs;
goto dup_needs_ack;
}
@ -314,7 +314,7 @@ ompi_btl_usnic_recv_frag_bookkeeping(
(void*)(seg->rs_recv_desc.sg_list[0].addr),
seg->rs_recv_desc.sg_list[0].length);
++module->num_total_recvs;
++module->stats.num_total_recvs;
/* Do late processing of incoming sequence # */
rc = ompi_btl_usnic_check_rx_seq(endpoint, seg, &window_index);
@ -322,7 +322,7 @@ ompi_btl_usnic_recv_frag_bookkeeping(
goto repost;
}
++module->num_frag_recvs;
++module->stats.num_frag_recvs;
ompi_btl_usnic_update_window(endpoint, window_index);
@ -332,7 +332,7 @@ repost:
OBJ_RELEASE(endpoint);
}
++module->num_recv_reposts;
++module->stats.num_recv_reposts;
/* Add recv to linked list for reposting */
seg->rs_recv_desc.next = channel->repost_recv_head;

2
ompi/mca/btl/usnic/btl_usnic_send.c
Просмотреть файл

@ -176,7 +176,7 @@ ompi_btl_usnic_send_slower(
frag->sf_base.uf_base.des_flags |= MCA_BTL_DES_SEND_ALWAYS_CALLBACK;
/* Stats */
++(((ompi_btl_usnic_module_t*)module)->pml_module_sends);
++(((ompi_btl_usnic_module_t*)module)->stats.pml_module_sends);
return rc;
}

6
ompi/mca/btl/usnic/btl_usnic_send.h
Просмотреть файл

@ -118,7 +118,7 @@ ompi_btl_usnic_post_segment(
--channel->sd_wqe;
/* Stats */
++module->num_total_sends;
++module->stats.num_total_sends;
++channel->num_channel_sends;
}
/*
@ -210,9 +210,9 @@ ompi_btl_usnic_endpoint_send_segment(
/* Stats */
if (sseg->ss_parent_frag->sf_base.uf_type
== OMPI_BTL_USNIC_FRAG_LARGE_SEND) {
++module->num_chunk_sends;
++module->stats.num_chunk_sends;
} else {
++module->num_frag_sends;
++module->stats.num_frag_sends;
}
/* If we have room in the sender's window, we also have room in

520
ompi/mca/btl/usnic/btl_usnic_stats.c Обычный файл
Просмотреть файл

@ -0,0 +1,520 @@
/*
* Copyright (c) 2013 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <unistd.h>
#include <stdlib.h>
#include "opal/util/output.h"
#include "opal/mca/base/mca_base_var.h"
#include "opal/mca/base/mca_base_pvar.h"
#include "ompi/proc/proc.h"
#include "btl_usnic.h"
#include "btl_usnic_module.h"
#include "btl_usnic_stats.h"
/*
* Local variables
*/
static mca_base_var_type_t pvar_type = MCA_BASE_VAR_TYPE_MAX;
static inline void usnic_stats_reset(ompi_btl_usnic_module_t *module)
{
int i;
module->stats.num_total_sends =
module->stats.num_resends =
module->stats.num_chunk_sends =
module->stats.num_frag_sends =
module->stats.num_ack_recvs =
module->stats.num_total_recvs =
module->stats.num_unk_recvs =
module->stats.num_dup_recvs =
module->stats.num_oow_low_recvs =
module->stats.num_oow_high_recvs =
module->stats.num_frag_recvs =
module->stats.num_chunk_recvs =
module->stats.num_badfrag_recvs =
module->stats.num_ack_sends =
module->stats.num_recv_reposts =
module->stats.num_crc_errors =
module->stats.num_old_dup_acks =
module->stats.num_dup_acks =
module->stats.num_fast_retrans =
module->stats.num_timeout_retrans =
module->stats.max_sent_window_size =
module->stats.max_rcvd_window_size =
module->stats.pml_module_sends =
module->stats.pml_send_callbacks =
0;
for (i=0; i<USNIC_NUM_CHANNELS; ++i) {
module->mod_channels[i].num_channel_sends = 0;
}
}
/* Prints a few terse statistics lines via opal_output(0,...). The first
* line will be prefixed with the string "prefix". If "reset_stats" is true
* then the statistics will be reset after printing.
*
* NOTE: this routine ignores the setting of stats_enable, so it can be used
* for debugging routines even when normal stats reporting is not enabled.
*/
void ompi_btl_usnic_print_stats(
ompi_btl_usnic_module_t *module,
const char *prefix,
bool reset_stats)
{
char tmp[128], str[2048];
/* The usuals */
snprintf(str, sizeof(str), "%s:MCW:%3u, ST(P+D)/F/C/R(T+F)/A:%8lu(%8u+%8u)/%8lu/%8lu/%4lu(%4lu+%4lu)/%8lu, RcvTot/Chk/F/C/L/H/D/BF/A:%8lu/%c%c/%8lu/%8lu/%4lu+%2lu/%4lu/%4lu/%6lu OA/DA %4lu/%4lu CRC:%4lu ",
prefix,
ompi_proc_local()->proc_name.vpid,
module->stats.num_total_sends,
module->mod_channels[USNIC_PRIORITY_CHANNEL].num_channel_sends,
module->mod_channels[USNIC_DATA_CHANNEL].num_channel_sends,
module->stats.num_frag_sends,
module->stats.num_chunk_sends,
module->stats.num_resends,
module->stats.num_timeout_retrans,
module->stats.num_fast_retrans,
module->stats.num_ack_sends,
module->stats.num_total_recvs,
(module->stats.num_total_recvs -
module->stats.num_recv_reposts) == 0 ? 'g' : 'B',
(module->stats.num_total_recvs -
module->stats.num_frag_recvs -
module->stats.num_chunk_recvs -
module->stats.num_badfrag_recvs -
module->stats.num_oow_low_recvs -
module->stats.num_oow_high_recvs -
module->stats.num_dup_recvs -
module->stats.num_ack_recvs -
module->stats.num_unk_recvs) == 0 ? 'g' : 'B',
module->stats.num_frag_recvs,
module->stats.num_chunk_recvs,
module->stats.num_oow_low_recvs,
module->stats.num_oow_high_recvs,
module->stats.num_dup_recvs,
module->stats.num_badfrag_recvs,
module->stats.num_ack_recvs,
module->stats.num_old_dup_acks,
module->stats.num_dup_acks,
module->stats.num_crc_errors);
/* If our PML calls were 0, then show send and receive window
extents instead */
if (module->stats.pml_module_sends +
module->stats.pml_send_callbacks == 0) {
int64_t send_unacked, su_min = WINDOW_SIZE * 2, su_max = 0;
int64_t recv_depth, rd_min = WINDOW_SIZE * 2, rd_max = 0;
ompi_btl_usnic_endpoint_t *endpoint;
opal_list_item_t *item;
rd_min = su_min = WINDOW_SIZE * 2;
rd_max = su_max = 0;
item = opal_list_get_first(&module->all_endpoints);
while (item != opal_list_get_end(&(module->all_endpoints))) {
endpoint = container_of(item, mca_btl_base_endpoint_t,
endpoint_endpoint_li);
item = opal_list_get_next(item);
/* Number of un-acked sends (i.e., sends for which we're
still waiting for ACK) */
send_unacked =
endpoint->endpoint_next_seq_to_send -
endpoint->endpoint_ack_seq_rcvd - 1;
if (send_unacked > su_max) su_max = send_unacked;
if (send_unacked < su_min) su_min = send_unacked;
/* Receive window depth (i.e., difference between highest
seq received and the next message we haven't ACKed
yet) */
recv_depth =
endpoint->endpoint_highest_seq_rcvd -
endpoint->endpoint_next_contig_seq_to_recv;
if (recv_depth > rd_max) rd_max = recv_depth;
if (recv_depth < rd_min) rd_min = recv_depth;
}
snprintf(tmp, sizeof(tmp), "PML S:%1ld, Win!A/R:%4ld/%4ld %4ld/%4ld",
module->stats.pml_module_sends,
su_min, su_max,
rd_min, rd_max);
} else {
snprintf(tmp, sizeof(tmp), "PML S/CB/Diff:%4lu/%4lu=%4ld",
module->stats.pml_module_sends,
module->stats.pml_send_callbacks,
module->stats.pml_module_sends -
module->stats.pml_send_callbacks);
}
strncat(str, tmp, sizeof(str) - strlen(str) - 1);
opal_output(0, "%s", str);
if (reset_stats) {
usnic_stats_reset(module);
}
}
/*
* Callback routine for libevent
*/
static void usnic_stats_callback(int fd, short flags, void *arg)
{
ompi_btl_usnic_module_t *module = (ompi_btl_usnic_module_t*) arg;
char tmp[128];
if (!mca_btl_usnic_component.stats_enabled) {
return;
}
snprintf(tmp, sizeof(tmp), "%4lu", ++module->stats.report_num);
ompi_btl_usnic_print_stats(module, tmp,
/*reset=*/mca_btl_usnic_component.stats_relative);
/* In OMPI v1.6, we have to re-add this event (because there's an
old libevent in OMPI v1.6) */
opal_event_add(&(module->stats.timer_event),
&(module->stats.timeout));
}
/*
* Initialize usnic module statistics
*/
int ompi_btl_usnic_stats_init(ompi_btl_usnic_module_t *module)
{
if (mca_btl_usnic_component.stats_enabled) {
usnic_stats_reset(module);
module->stats.timeout.tv_sec = mca_btl_usnic_component.stats_frequency;
module->stats.timeout.tv_usec = 0;
opal_event_set(opal_event_base, &(module->stats.timer_event),
-1, EV_TIMEOUT | EV_PERSIST,
&usnic_stats_callback, module);
opal_event_add(&(module->stats.timer_event),
&(module->stats.timeout));
}
return OMPI_SUCCESS;
}
/*
* Finalize usnic module statistics
*/
int ompi_btl_usnic_stats_finalize(ompi_btl_usnic_module_t *module)
{
/* Disable the stats callback event, and then call the stats
callback manually to display the final stats */
if (mca_btl_usnic_component.stats_enabled) {
opal_event_del(&(module->stats.timer_event));
ompi_btl_usnic_print_stats(module, "final", /*reset_stats=*/false);
}
return OMPI_SUCCESS;
}
/************************************************************************/
/*
* Function called by the pvar base upon MPI_T_pvar_handle_alloc,
* handle_start, and handle_stop.
*/
static int usnic_pvar_notify(struct mca_base_pvar_t *pvar,
mca_base_pvar_event_t event,
void *obj, int *count)
{
if (MCA_BASE_PVAR_HANDLE_BIND == event) {
*count = mca_btl_usnic_component.num_modules;
}
/* Don't care about the other events */
return OMPI_SUCCESS;
}
/*
* Function called by the pvar base when a user wants to read the
* value of an MPI_T performance variable.
*/
static int usnic_pvar_read(const struct mca_base_pvar_t *pvar,
void *value, void *bound_obj)
{
size_t i;
size_t offset = (size_t) pvar->ctx;
uint64_t *array = (uint64_t*) value;
for (i = 0; i < mca_btl_usnic_component.num_modules; ++i) {
char *base = (char*) &(mca_btl_usnic_component.usnic_active_modules[i]->stats);
array[i] = *((uint64_t*) (base + offset));
}
return OMPI_SUCCESS;
}
/*
* Register an MPI_T performance variable of type CLASS_HIGHWATERMARK.
*/
static void register_pvar_highwater(char *name, char *desc, size_t offset)
{
int rc __opal_attribute_unused__;
rc = mca_base_component_pvar_register(&mca_btl_usnic_component.super.btl_version,
name, desc,
OPAL_INFO_LVL_5,
MCA_BASE_PVAR_CLASS_HIGHWATERMARK,
pvar_type,
NULL, /* enumeration */
MCA_BASE_VAR_BIND_NO_OBJECT,
(MCA_BASE_PVAR_FLAG_READONLY |
MCA_BASE_PVAR_FLAG_CONTINUOUS),
usnic_pvar_read,
NULL, /* write function */
usnic_pvar_notify,
(void *) offset);
assert(rc >= 0);
}
/*
* Function called by the pvar base when a user wants to read the
* devices enum value. The array is a simple list of 0..num_modules,
* which will map to the strings in the devices_enum
* setup_mpit_pvar_type().
*/
static int usnic_pvar_enum_read(const struct mca_base_pvar_t *pvar,
void *value, void *bound_obj)
{
size_t i;
int *array = (int *) value;
for (i = 0; i < mca_btl_usnic_component.num_modules; ++i) {
array[i] = i;
}
return OMPI_SUCCESS;
}
/*
* Register an MPI_T performance variable of type CLASS_COUNTER.
*/
static void register_pvar_counter(char *name, char *desc, size_t offset)
{
int rc __opal_attribute_unused__;
rc = mca_base_component_pvar_register(&mca_btl_usnic_component.super.btl_version,
name, desc,
OPAL_INFO_LVL_5,
MCA_BASE_PVAR_CLASS_COUNTER,
pvar_type,
NULL, /* enumeration */
MCA_BASE_VAR_BIND_NO_OBJECT,
(MCA_BASE_PVAR_FLAG_READONLY |
MCA_BASE_PVAR_FLAG_CONTINUOUS),
usnic_pvar_read,
NULL, /* write function */
usnic_pvar_notify,
(void *) offset);
assert(rc >= 0);
}
/*
* Find the MPI_T type corresponding to our uint64_t counters and
* highwatermarks.
*/
static bool setup_mpit_pvar_type(void)
{
/* Our stats variables are uint64_t's, so find a pvar type that is
compatible */
if (sizeof(uint64_t) == sizeof(unsigned int)) {
pvar_type = MCA_BASE_VAR_TYPE_UNSIGNED_INT;
} else if (sizeof(uint64_t) == sizeof(unsigned long)) {
pvar_type = MCA_BASE_VAR_TYPE_UNSIGNED_LONG;
#ifdef HAVE_UNSIGNED_LONG_LONG
} else if (sizeof(uint64_t) == sizeof(unsigned long long)) {
pvar_type = MCA_BASE_VAR_TYPE_UNSIGNED_LONG_LONG;
#endif
}
/* Let the caller know if we found a compatible type or not */
if (MCA_BASE_VAR_TYPE_MAX == pvar_type) {
return false;
}
return true;
}
/*
* Setup the usnic_X device enumeration pvar
*/
static void setup_mpit_pvars_enum(void)
{
size_t i;
int rc __opal_attribute_unused__;
mca_base_var_enum_value_t *devices;
static mca_base_var_enum_t *devices_enum;
struct ibv_device *device;
devices = calloc(mca_btl_usnic_component.num_modules + 1,
sizeof(*devices));
assert(devices != NULL);
for (i = 0; i < mca_btl_usnic_component.num_modules; ++i) {
device = mca_btl_usnic_component.usnic_active_modules[i]->device;
devices[i].value = i;
devices[i].string = ibv_get_device_name(device);
}
devices[i].string = NULL;
rc = mca_base_var_enum_create("btl_usnic_devices", devices, &devices_enum);
assert(OPAL_SUCCESS == rc);
rc = mca_base_component_pvar_register(&mca_btl_usnic_component.super.btl_version,
"devices",
"Enumeration representing which slot in btl_usnic_* MPI_T pvar value arrays correspond to which usnic_X Linux device",
OPAL_INFO_LVL_5,
MCA_BASE_PVAR_CLASS_STATE,
MCA_BASE_VAR_TYPE_INT,
devices_enum,
MCA_BASE_VAR_BIND_NO_OBJECT,
(MCA_BASE_PVAR_FLAG_READONLY |
MCA_BASE_PVAR_FLAG_CONTINUOUS),
usnic_pvar_enum_read,
NULL, /* write function */
usnic_pvar_notify,
NULL /* context */);
assert(rc >= 0);
/* The devices_enum has been RETAIN'ed by the pvar, so we can
RELEASE it here, and the enum will be destroyed when the pvar
is destroyed. */
OBJ_RELEASE(devices_enum);
}
/*
* Setup high watermark MPI_T performance variables
*/
static void setup_mpit_pvars_highwatermark(void)
{
#define REGISTERHW(field, desc) \
register_pvar_highwater(#field, (desc), offsetof(ompi_btl_usnic_module_stats_t, field))
REGISTERHW(max_sent_window_size,
"Maximum number of entries in all send windows from this peer");
REGISTERHW(max_rcvd_window_size,
"Maximum number of entries in all receive windows to this peer");
}
/*
* Setup counter MPI_T performance variables
*/
static void setup_mpit_pvars_counters(void)
{
#define REGISTERC(field, desc) \
register_pvar_counter(#field, (desc), offsetof(ompi_btl_usnic_module_stats_t, field))
REGISTERC(num_total_sends,
"Total number of sends (MPI data, ACKs, retransmissions, etc.)");
REGISTERC(num_resends,
"Total number of all retransmissions");
REGISTERC(num_timeout_retrans,
"Number of times chunk retransmissions have occured because an ACK was not received within the timeout");
REGISTERC(num_fast_retrans,
"Number of times chunk retransmissions have occured because due to a repeated ACK");
REGISTERC(num_chunk_sends,
"Number of sends that were part of a larger MPI message fragment (i.e., the MPI message was so long that it had to be split into multiple MTU/network sends)");
REGISTERC(num_frag_sends,
"Number of sends where the entire MPI message fragment fit into a single MTU/network send");
REGISTERC(num_ack_sends,
"Number of ACKs sent (i.e., usNIC-BTL-to-usNIC-BTL control messages)");
REGISTERC(num_total_recvs,
"Total number of receives completed");
REGISTERC(num_unk_recvs,
"Number of receives with an unknown source or type, and therefore ignored by the usNIC BTL (this should never be >0)");
REGISTERC(num_dup_recvs,
"Number of duplicate receives");
REGISTERC(num_oow_low_recvs,
"Number of times a receive was out of the sliding window (on the low side)");
REGISTERC(num_oow_high_recvs,
"Number of times a receive was out of the sliding window (on the high side)");
REGISTERC(num_frag_recvs,
"Number of receives where the entire MPI message fragment fit into a single MTU/network send");
REGISTERC(num_chunk_recvs,
"Number of receives that were part of a larger MPI message fragment (i.e., this receive was reassembled into a larger MPI message fragment)");
REGISTERC(num_badfrag_recvs,
"Number of chunks received that had a bad fragment ID (this should never be >0)");
REGISTERC(num_ack_recvs,
"Total number of ACKs received");
REGISTERC(num_old_dup_acks,
"Number of old duplicate ACKs received (i.e., before the current expected ACK)");
REGISTERC(num_dup_acks,
"Number of duplicate ACKs received (i.e., the current expected ACK)");
REGISTERC(num_recv_reposts,
"Number of times buffers have been reposted for receives");
REGISTERC(num_crc_errors,
"Number of times receives were aborted because of a CRC error");
REGISTERC(pml_module_sends,
"Number of times the PML has called down to send a message");
REGISTERC(pml_send_callbacks,
"Number of times the usNIC BTL has called up to the PML to complete a send");
}
/*
* Initialize MPI_T performance variables
*/
int ompi_btl_usnic_setup_mpit_pvars(void)
{
/* If we cannot find a compatible pvar type, we're done (i.e.,
don't register any pvars) */
if (!setup_mpit_pvar_type()) {
return OMPI_SUCCESS;
}
/* Setup the usnic_X device enumeration pvar */
setup_mpit_pvars_enum();
/* Register watermark pvars */
setup_mpit_pvars_highwatermark();
/* If our counter stats are relative, don't report them through
MPI_T, because MPI_T expects counters to be monotonically
rising. */
if (!mca_btl_usnic_component.stats_relative) {
setup_mpit_pvars_counters();
}
/* All done */
return OMPI_SUCCESS;
}

81
ompi/mca/btl/usnic/btl_usnic_stats.h Обычный файл
Просмотреть файл

@ -0,0 +1,81 @@
/*
* Copyright (c) 2013 Cisco Systems, Inc. All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*
* Statistics for the usnic BTL component.
*/
#ifndef OMPI_BTL_USNIC_STATS_H
#define OMPI_BTL_USNIC_STATS_H
#include <sys/time.h>
#include "opal/mca/event/event.h"
/**
* Struct containing all the statistics that are trackedx
*/
typedef struct ompi_btl_usnic_module_stats_t {
bool final_stats;
uint64_t report_num;
uint64_t num_total_sends;
uint64_t num_resends;
uint64_t num_timeout_retrans;
uint64_t num_fast_retrans;
uint64_t num_chunk_sends;
uint64_t num_frag_sends;
uint64_t num_ack_sends;
uint64_t num_total_recvs;
uint64_t num_unk_recvs;
uint64_t num_dup_recvs;
uint64_t num_oow_low_recvs;
uint64_t num_oow_high_recvs;
uint64_t num_frag_recvs;
uint64_t num_chunk_recvs;
uint64_t num_badfrag_recvs;
uint64_t num_ack_recvs;
uint64_t num_old_dup_acks;
uint64_t num_dup_acks;
uint64_t num_recv_reposts;
uint64_t num_crc_errors;
uint64_t max_sent_window_size;
uint64_t max_rcvd_window_size;
uint64_t pml_module_sends;
uint64_t pml_send_callbacks;
opal_event_t timer_event;
struct timeval timeout;
} ompi_btl_usnic_module_stats_t;
/**
* Initialize the stats on a module. Must use "struct
* ompi_btl_usnic_module_t*" here to avoid an #include cycle.
*/
int ompi_btl_usnic_stats_init(struct ompi_btl_usnic_module_t *module);
/**
* Finalize the stats on a module. Must use "struct
* ompi_btl_usnic_module_t*" here to avoid an #include cycle.
*/
int ompi_btl_usnic_stats_finalize(struct ompi_btl_usnic_module_t *module);
/**
* Initialize the MPI_T performance variables (for all modules)
*/
int ompi_btl_usnic_setup_mpit_pvars(void);
#endif