379 строки
14 KiB
C
379 строки
14 KiB
C
/*
|
|
* Copyright (c) 2013-2015 The University of Tennessee and The University
|
|
* of Tennessee Research Foundation. All rights
|
|
* reserved.
|
|
* Copyright (c) 2013-2017 Inria. All rights reserved.
|
|
* Copyright (c) 2015 Cisco Systems, Inc. All rights reserved.
|
|
* Copyright (c) 2016 Intel, Inc. All rights reserved.
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
|
|
/*
|
|
pml monitoring tester.
|
|
|
|
Designed by George Bosilca <bosilca@icl.utk.edu> Emmanuel Jeannot <emmanuel.jeannot@inria.fr> and Clément Foyer <clement.foyer@inria.fr>
|
|
Contact the authors for questions.
|
|
|
|
To options are available for this test, with/without MPI_Tools, and with/without RMA operations. The default mode is without MPI_Tools, and with RMA operations.
|
|
To enable the MPI_Tools use, add "--with-mpit" as an application parameter.
|
|
To disable the RMA operations testing, add "--without-rma" as an application parameter.
|
|
|
|
To be run as (without using MPI_Tool):
|
|
|
|
mpirun -np 4 --mca pml_monitoring_enable 2 --mca pml_monitoring_enable_output 3 --mca pml_monitoring_filename prof/output ./monitoring_test
|
|
|
|
with the results being, as an example:
|
|
output.1.prof
|
|
# POINT TO POINT
|
|
E 1 2 104 bytes 26 msgs sent 0,0,0,26,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
|
|
E 1 3 208 bytes 52 msgs sent 8,0,0,65,1,5,2,0,0,0,0,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
|
|
I 1 0 140 bytes 27 msgs sent
|
|
I 1 2 2068 bytes 1 msgs sent
|
|
I 1 3 2256 bytes 31 msgs sent
|
|
# OSC
|
|
S 1 0 0 bytes 1 msgs sent
|
|
R 1 0 40960 bytes 1 msgs sent
|
|
S 1 2 40960 bytes 1 msgs sent
|
|
# COLLECTIVES
|
|
C 1 0 140 bytes 27 msgs sent
|
|
C 1 2 140 bytes 27 msgs sent
|
|
C 1 3 140 bytes 27 msgs sent
|
|
D MPI COMMUNICATOR 4 DUP FROM 0 procs: 0,1,2,3
|
|
O2A 1 0 bytes 0 msgs sent
|
|
A2O 1 0 bytes 0 msgs sent
|
|
A2A 1 276 bytes 15 msgs sent
|
|
D MPI_COMM_WORLD procs: 0,1,2,3
|
|
O2A 1 0 bytes 0 msgs sent
|
|
A2O 1 0 bytes 0 msgs sent
|
|
A2A 1 96 bytes 9 msgs sent
|
|
D MPI COMMUNICATOR 5 SPLIT_TYPE FROM 4 procs: 0,1,2,3
|
|
O2A 1 0 bytes 0 msgs sent
|
|
A2O 1 0 bytes 0 msgs sent
|
|
A2A 1 48 bytes 3 msgs sent
|
|
D MPI COMMUNICATOR 3 SPLIT FROM 0 procs: 1,3
|
|
O2A 1 0 bytes 0 msgs sent
|
|
A2O 1 0 bytes 0 msgs sent
|
|
A2A 1 0 bytes 0 msgs sent
|
|
|
|
*/
|
|
|
|
|
|
#include "mpi.h"
|
|
#include <stdio.h>
|
|
#include <string.h>
|
|
|
|
static MPI_T_pvar_handle flush_handle;
|
|
static const char flush_pvar_name[] = "pml_monitoring_flush";
|
|
static const void*nullbuf = NULL;
|
|
static int flush_pvar_idx;
|
|
static int with_mpit = 0;
|
|
static int with_rma = 1;
|
|
|
|
int main(int argc, char* argv[])
|
|
{
|
|
int rank, size, n, to, from, tagno, MPIT_result, provided, count, world_rank;
|
|
MPI_T_pvar_session session;
|
|
MPI_Comm newcomm;
|
|
char filename[1024];
|
|
|
|
for ( int arg_it = 1; argc > 1 && arg_it < argc; ++arg_it ) {
|
|
if( 0 == strcmp(argv[arg_it], "--with-mpit") ) {
|
|
with_mpit = 1;
|
|
printf("enable MPIT support\n");
|
|
} else if( 0 == strcmp(argv[arg_it], "--without-rma") ) {
|
|
with_rma = 0;
|
|
printf("disable RMA testing\n");
|
|
}
|
|
}
|
|
|
|
/* first phase : make a token circulated in MPI_COMM_WORLD */
|
|
n = -1;
|
|
MPI_Init(NULL, NULL);
|
|
MPI_Comm_rank(MPI_COMM_WORLD, &world_rank);
|
|
MPI_Comm_size(MPI_COMM_WORLD, &size);
|
|
rank = world_rank;
|
|
to = (rank + 1) % size;
|
|
from = (rank - 1) % size;
|
|
tagno = 201;
|
|
|
|
if( with_mpit ) {
|
|
MPIT_result = MPI_T_init_thread(MPI_THREAD_SINGLE, &provided);
|
|
if (MPIT_result != MPI_SUCCESS)
|
|
MPI_Abort(MPI_COMM_WORLD, MPIT_result);
|
|
|
|
MPIT_result = MPI_T_pvar_get_index(flush_pvar_name, MPI_T_PVAR_CLASS_GENERIC, &flush_pvar_idx);
|
|
if (MPIT_result != MPI_SUCCESS) {
|
|
printf("cannot find monitoring MPI_T \"%s\" pvar, check that you have monitoring pml\n",
|
|
flush_pvar_name);
|
|
MPI_Abort(MPI_COMM_WORLD, MPIT_result);
|
|
}
|
|
|
|
MPIT_result = MPI_T_pvar_session_create(&session);
|
|
if (MPIT_result != MPI_SUCCESS) {
|
|
printf("cannot create a session for \"%s\" pvar\n", flush_pvar_name);
|
|
MPI_Abort(MPI_COMM_WORLD, MPIT_result);
|
|
}
|
|
|
|
/* Allocating a new PVAR in a session will reset the counters */
|
|
MPIT_result = MPI_T_pvar_handle_alloc(session, flush_pvar_idx,
|
|
MPI_COMM_WORLD, &flush_handle, &count);
|
|
if (MPIT_result != MPI_SUCCESS) {
|
|
printf("failed to allocate handle on \"%s\" pvar, check that you have monitoring pml\n",
|
|
flush_pvar_name);
|
|
MPI_Abort(MPI_COMM_WORLD, MPIT_result);
|
|
}
|
|
|
|
MPIT_result = MPI_T_pvar_start(session, flush_handle);
|
|
if (MPIT_result != MPI_SUCCESS) {
|
|
printf("failed to start handle on \"%s\" pvar, check that you have monitoring pml\n",
|
|
flush_pvar_name);
|
|
MPI_Abort(MPI_COMM_WORLD, MPIT_result);
|
|
}
|
|
}
|
|
|
|
if (rank == 0) {
|
|
n = 25;
|
|
MPI_Send(&n,1,MPI_INT,to,tagno,MPI_COMM_WORLD);
|
|
}
|
|
while (1) {
|
|
MPI_Recv(&n, 1, MPI_INT, from, tagno, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
|
|
if (rank == 0) {n--;tagno++;}
|
|
MPI_Send(&n, 1, MPI_INT, to, tagno, MPI_COMM_WORLD);
|
|
if (rank != 0) {n--;tagno++;}
|
|
if (n<0){
|
|
break;
|
|
}
|
|
}
|
|
|
|
if( with_mpit ) {
|
|
/* Build one file per processes
|
|
Every thing that has been monitored by each
|
|
process since the last flush will be output in filename */
|
|
/*
|
|
Requires directory prof to be created.
|
|
Filename format should display the phase number
|
|
and the process rank for ease of parsing with
|
|
aggregate_profile.pl script
|
|
*/
|
|
sprintf(filename, "prof/phase_1");
|
|
|
|
if( MPI_SUCCESS != MPI_T_pvar_write(session, flush_handle, filename) ) {
|
|
fprintf(stderr, "Process %d cannot save monitoring in %s.%d.prof\n",
|
|
world_rank, filename, world_rank);
|
|
}
|
|
/* Force the writing of the monitoring data */
|
|
MPIT_result = MPI_T_pvar_stop(session, flush_handle);
|
|
if (MPIT_result != MPI_SUCCESS) {
|
|
printf("failed to stop handle on \"%s\" pvar, check that you have monitoring pml\n",
|
|
flush_pvar_name);
|
|
MPI_Abort(MPI_COMM_WORLD, MPIT_result);
|
|
}
|
|
|
|
MPIT_result = MPI_T_pvar_start(session, flush_handle);
|
|
if (MPIT_result != MPI_SUCCESS) {
|
|
printf("failed to start handle on \"%s\" pvar, check that you have monitoring pml\n",
|
|
flush_pvar_name);
|
|
MPI_Abort(MPI_COMM_WORLD, MPIT_result);
|
|
}
|
|
/* Don't set a filename. If we stop the session before setting it, then no output file
|
|
* will be generated.
|
|
*/
|
|
if( MPI_SUCCESS != MPI_T_pvar_write(session, flush_handle, (void*)&nullbuf) ) {
|
|
fprintf(stderr, "Process %d cannot save monitoring in %s\n", world_rank, filename);
|
|
}
|
|
}
|
|
|
|
/*
|
|
Second phase. Work with different communicators.
|
|
even ranks will circulate a token
|
|
while odd ranks will perform a all_to_all
|
|
*/
|
|
MPI_Comm_split(MPI_COMM_WORLD, rank%2, rank, &newcomm);
|
|
|
|
if(rank%2){ /*odd ranks (in COMM_WORD) circulate a token*/
|
|
MPI_Comm_rank(newcomm, &rank);
|
|
MPI_Comm_size(newcomm, &size);
|
|
if( size > 1 ) {
|
|
to = (rank + 1) % size;
|
|
from = (rank - 1) % size;
|
|
tagno = 201;
|
|
if (rank == 0){
|
|
n = 50;
|
|
MPI_Send(&n, 1, MPI_INT, to, tagno, newcomm);
|
|
}
|
|
while (1){
|
|
MPI_Recv(&n, 1, MPI_INT, from, tagno, newcomm, MPI_STATUS_IGNORE);
|
|
if (rank == 0) {n--; tagno++;}
|
|
MPI_Send(&n, 1, MPI_INT, to, tagno, newcomm);
|
|
if (rank != 0) {n--; tagno++;}
|
|
if (n<0){
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
} else { /*even ranks (in COMM_WORD) will perform a all_to_all and a barrier*/
|
|
int send_buff[10240];
|
|
int recv_buff[10240];
|
|
MPI_Comm newcomm2;
|
|
MPI_Comm_rank(newcomm, &rank);
|
|
MPI_Comm_size(newcomm, &size);
|
|
MPI_Alltoall(send_buff, 10240/size, MPI_INT, recv_buff, 10240/size, MPI_INT, newcomm);
|
|
MPI_Comm_split(newcomm, rank%2, rank, &newcomm2);
|
|
MPI_Barrier(newcomm2);
|
|
MPI_Comm_free(&newcomm2);
|
|
}
|
|
|
|
if( with_mpit ) {
|
|
/* Build one file per processes
|
|
Every thing that has been monitored by each
|
|
process since the last flush will be output in filename */
|
|
/*
|
|
Requires directory prof to be created.
|
|
Filename format should display the phase number
|
|
and the process rank for ease of parsing with
|
|
aggregate_profile.pl script
|
|
*/
|
|
sprintf(filename, "prof/phase_2");
|
|
|
|
if( MPI_SUCCESS != MPI_T_pvar_write(session, flush_handle, filename) ) {
|
|
fprintf(stderr, "Process %d cannot save monitoring in %s.%d.prof\n",
|
|
world_rank, filename, world_rank);
|
|
}
|
|
|
|
/* Force the writing of the monitoring data */
|
|
MPIT_result = MPI_T_pvar_stop(session, flush_handle);
|
|
if (MPIT_result != MPI_SUCCESS) {
|
|
printf("failed to stop handle on \"%s\" pvar, check that you have monitoring pml\n",
|
|
flush_pvar_name);
|
|
MPI_Abort(MPI_COMM_WORLD, MPIT_result);
|
|
}
|
|
|
|
MPIT_result = MPI_T_pvar_start(session, flush_handle);
|
|
if (MPIT_result != MPI_SUCCESS) {
|
|
printf("failed to start handle on \"%s\" pvar, check that you have monitoring pml\n",
|
|
flush_pvar_name);
|
|
MPI_Abort(MPI_COMM_WORLD, MPIT_result);
|
|
}
|
|
/* Don't set a filename. If we stop the session before setting it, then no output
|
|
* will be generated.
|
|
*/
|
|
if( MPI_SUCCESS != MPI_T_pvar_write(session, flush_handle, (void*)&nullbuf ) ) {
|
|
fprintf(stderr, "Process %d cannot save monitoring in %s\n", world_rank, filename);
|
|
}
|
|
}
|
|
|
|
if( with_rma ) {
|
|
MPI_Win win;
|
|
int rs_buff[10240];
|
|
int win_buff[10240];
|
|
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
|
|
MPI_Comm_size(MPI_COMM_WORLD, &size);
|
|
to = (rank + 1) % size;
|
|
from = (rank + size - 1) % size;
|
|
for( int v = 0; v < 10240; ++v )
|
|
rs_buff[v] = win_buff[v] = rank;
|
|
|
|
MPI_Win_create(win_buff, 10240 * sizeof(int), sizeof(int), MPI_INFO_NULL, MPI_COMM_WORLD, &win);
|
|
MPI_Win_fence(MPI_MODE_NOPRECEDE, win);
|
|
if( rank%2 ) {
|
|
MPI_Win_fence(MPI_MODE_NOSTORE | MPI_MODE_NOPUT, win);
|
|
MPI_Get(rs_buff, 10240, MPI_INT, from, 0, 10240, MPI_INT, win);
|
|
} else {
|
|
MPI_Put(rs_buff, 10240, MPI_INT, to, 0, 10240, MPI_INT, win);
|
|
MPI_Win_fence(MPI_MODE_NOSTORE | MPI_MODE_NOPUT, win);
|
|
}
|
|
MPI_Win_fence(MPI_MODE_NOSUCCEED, win);
|
|
|
|
for( int v = 0; v < 10240; ++v )
|
|
if( rs_buff[v] != win_buff[v] && ((rank%2 && rs_buff[v] != from) || (!(rank%2) && rs_buff[v] != rank)) ) {
|
|
printf("Error on checking exchanged values: %s_buff[%d] == %d instead of %d\n",
|
|
rank%2 ? "rs" : "win", v, rs_buff[v], rank%2 ? from : rank);
|
|
MPI_Abort(MPI_COMM_WORLD, -1);
|
|
}
|
|
|
|
MPI_Group world_group, newcomm_group, distant_group;
|
|
MPI_Comm_group(MPI_COMM_WORLD, &world_group);
|
|
MPI_Comm_group(newcomm, &newcomm_group);
|
|
MPI_Group_difference(world_group, newcomm_group, &distant_group);
|
|
if( rank%2 ) {
|
|
MPI_Win_post(distant_group, 0, win);
|
|
MPI_Win_wait(win);
|
|
/* Check received values */
|
|
for( int v = 0; v < 10240; ++v )
|
|
if( from != win_buff[v] ) {
|
|
printf("Error on checking exchanged values: win_buff[%d] == %d instead of %d\n",
|
|
v, win_buff[v], from);
|
|
MPI_Abort(MPI_COMM_WORLD, -1);
|
|
}
|
|
} else {
|
|
MPI_Win_start(distant_group, 0, win);
|
|
MPI_Put(rs_buff, 10240, MPI_INT, to, 0, 10240, MPI_INT, win);
|
|
MPI_Win_complete(win);
|
|
}
|
|
MPI_Group_free(&world_group);
|
|
MPI_Group_free(&newcomm_group);
|
|
MPI_Group_free(&distant_group);
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
for( int v = 0; v < 10240; ++v ) rs_buff[v] = rank;
|
|
|
|
MPI_Win_lock(MPI_LOCK_EXCLUSIVE, to, 0, win);
|
|
MPI_Put(rs_buff, 10240, MPI_INT, to, 0, 10240, MPI_INT, win);
|
|
MPI_Win_unlock(to, win);
|
|
|
|
MPI_Barrier(MPI_COMM_WORLD);
|
|
|
|
/* Check received values */
|
|
for( int v = 0; v < 10240; ++v )
|
|
if( from != win_buff[v] ) {
|
|
printf("Error on checking exchanged values: win_buff[%d] == %d instead of %d\n",
|
|
v, win_buff[v], from);
|
|
MPI_Abort(MPI_COMM_WORLD, -1);
|
|
}
|
|
|
|
MPI_Win_free(&win);
|
|
}
|
|
|
|
if( with_mpit ) {
|
|
/* the filename for flushing monitoring now uses 3 as phase number! */
|
|
sprintf(filename, "prof/phase_3");
|
|
|
|
if( MPI_SUCCESS != MPI_T_pvar_write(session, flush_handle, filename) ) {
|
|
fprintf(stderr, "Process %d cannot save monitoring in %s.%d.prof\n",
|
|
world_rank, filename, world_rank);
|
|
}
|
|
|
|
MPIT_result = MPI_T_pvar_stop(session, flush_handle);
|
|
if (MPIT_result != MPI_SUCCESS) {
|
|
printf("failed to stop handle on \"%s\" pvar, check that you have monitoring pml\n",
|
|
flush_pvar_name);
|
|
MPI_Abort(MPI_COMM_WORLD, MPIT_result);
|
|
}
|
|
|
|
MPIT_result = MPI_T_pvar_handle_free(session, &flush_handle);
|
|
if (MPIT_result != MPI_SUCCESS) {
|
|
printf("failed to free handle on \"%s\" pvar, check that you have monitoring pml\n",
|
|
flush_pvar_name);
|
|
MPI_Abort(MPI_COMM_WORLD, MPIT_result);
|
|
}
|
|
|
|
MPIT_result = MPI_T_pvar_session_free(&session);
|
|
if (MPIT_result != MPI_SUCCESS) {
|
|
printf("cannot close a session for \"%s\" pvar\n", flush_pvar_name);
|
|
MPI_Abort(MPI_COMM_WORLD, MPIT_result);
|
|
}
|
|
|
|
(void)MPI_T_finalize();
|
|
}
|
|
|
|
MPI_Comm_free(&newcomm);
|
|
/* Now, in MPI_Finalize(), the pml_monitoring library outputs, in
|
|
STDERR, the aggregated recorded monitoring of all the phases*/
|
|
MPI_Finalize();
|
|
return 0;
|
|
}
|