From e4aae6b5c82cddd0184f71a8dd5d75206c32ab46 Mon Sep 17 00:00:00 2001 From: George Bosilca Date: Tue, 4 Dec 2018 23:48:40 -0500 Subject: [PATCH] Add a test for very large data. Signed-off-by: George Bosilca --- opal/datatype/opal_convertor.h | 6 ++- opal/datatype/opal_datatype.h | 1 - test/datatype/Makefile.am | 8 ++- test/datatype/large_data.c | 93 ++++++++++++++++++++++++++++++++++ 4 files changed, 105 insertions(+), 3 deletions(-) create mode 100644 test/datatype/large_data.c diff --git a/opal/datatype/opal_convertor.h b/opal/datatype/opal_convertor.h index 50bc572675..875c111b1f 100644 --- a/opal/datatype/opal_convertor.h +++ b/opal/datatype/opal_convertor.h @@ -94,18 +94,22 @@ struct opal_convertor_t { const opal_datatype_t* pDesc; /**< the datatype description associated with the convertor */ const dt_type_desc_t* use_desc; /**< the version used by the convertor (normal or optimized) */ opal_datatype_count_t count; /**< the total number of full datatype elements */ - uint32_t stack_size; /**< size of the allocated stack */ /* --- cacheline boundary (64 bytes - if 64bits arch and !OPAL_ENABLE_DEBUG) --- */ + uint32_t stack_size; /**< size of the allocated stack */ unsigned char* pBaseBuf; /**< initial buffer as supplied by the user */ dt_stack_t* pStack; /**< the local stack for the actual conversion */ convertor_advance_fct_t fAdvance; /**< pointer to the pack/unpack functions */ + + /* --- cacheline boundary (96 bytes - if 64bits arch and !OPAL_ENABLE_DEBUG) --- */ struct opal_convertor_master_t* master; /**< the master convertor */ /* All others fields get modified for every call to pack/unpack functions */ uint32_t stack_pos; /**< the actual position on the stack */ size_t partial_length; /**< amount of data left over from the last unpack */ size_t bConverted; /**< # of bytes already converted */ + + /* --- cacheline boundary (128 bytes - if 64bits arch and !OPAL_ENABLE_DEBUG) --- */ uint32_t checksum; /**< checksum computed by pack/unpack operation */ uint32_t csum_ui1; /**< partial checksum computed by pack/unpack operation */ size_t csum_ui2; /**< partial checksum computed by pack/unpack operation */ diff --git a/opal/datatype/opal_datatype.h b/opal/datatype/opal_datatype.h index f234e8c157..a836a5aae0 100644 --- a/opal/datatype/opal_datatype.h +++ b/opal/datatype/opal_datatype.h @@ -119,7 +119,6 @@ struct opal_datatype_t { /* Attribute fields */ char name[OPAL_MAX_OBJECT_NAME]; /**< name of the datatype */ - /* --- cacheline 2 boundary (128 bytes) was 8-12 bytes ago --- */ dt_type_desc_t desc; /**< the data description */ dt_type_desc_t opt_desc; /**< short description of the data used when conversion is useless or in the send case (without conversion) */ diff --git a/test/datatype/Makefile.am b/test/datatype/Makefile.am index 8efd0344ec..4366724a52 100644 --- a/test/datatype/Makefile.am +++ b/test/datatype/Makefile.am @@ -15,7 +15,7 @@ # if PROJECT_OMPI - MPI_TESTS = checksum position position_noncontig ddt_test ddt_raw ddt_raw2 unpack_ooo ddt_pack external32 + MPI_TESTS = checksum position position_noncontig ddt_test ddt_raw ddt_raw2 unpack_ooo ddt_pack external32 large_data MPI_CHECKS = to_self endif TESTS = opal_datatype_test unpack_hetero $(MPI_TESTS) @@ -74,6 +74,12 @@ to_self_SOURCES = to_self.c to_self_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS) to_self_LDADD = $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la +large_data_SOURCES = large_data.c +large_data_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS) +large_data_LDADD = \ + $(top_builddir)/ompi/lib@OMPI_LIBMPI_NAME@.la \ + $(top_builddir)/opal/lib@OPAL_LIB_PREFIX@open-pal.la + opal_datatype_test_SOURCES = opal_datatype_test.c opal_ddt_lib.c opal_ddt_lib.h opal_datatype_test_LDFLAGS = $(OMPI_PKG_CONFIG_LDFLAGS) opal_datatype_test_LDADD = \ diff --git a/test/datatype/large_data.c b/test/datatype/large_data.c new file mode 100644 index 0000000000..8209bd94c8 --- /dev/null +++ b/test/datatype/large_data.c @@ -0,0 +1,93 @@ +#include +#include +#include +#include +#include + +#include "ompi_config.h" +#include "ompi/datatype/ompi_datatype.h" +#include "opal/runtime/opal.h" +#include "opal/datatype/opal_convertor.h" +#include "opal/datatype/opal_datatype_internal.h" + +#define MAX_IOVEC 10 +#define MAX_CHUNK (1024*1024*1024) /* 1GB */ + +static size_t +count_length_via_convertor_raw(MPI_Datatype dtype, int count) +{ + opal_convertor_t* pconv; + struct iovec iov[MAX_IOVEC]; + uint32_t iov_count = MAX_IOVEC, i; + size_t length = MAX_CHUNK, packed_iovec = 0, packed = 0; + + pconv = opal_convertor_create( opal_local_arch, 0 ); + opal_convertor_prepare_for_send(pconv, (const struct opal_datatype_t *)dtype, 1, NULL); + while( 0 == opal_convertor_raw(pconv, iov, &iov_count, &length) ) { + printf("iov_count = %d packed_iovec = %"PRIsize_t"\n", iov_count, packed_iovec); + packed += length; + for( i = 0; i < iov_count; i++ ) { + packed_iovec += iov[i].iov_len; + } + if( packed != packed_iovec ) { + printf( "Packed send amount diverges %"PRIsize_t" != %"PRIsize_t"\n", packed, packed_iovec); + exit(-1); + } + iov_count = MAX_IOVEC; /* number of available iov */ + length = MAX_CHUNK; + } + packed += length; + for( i = 0; i < iov_count; i++ ) { + packed_iovec += iov[i].iov_len; + } + if( packed != packed_iovec ) { + printf( "Packed send amount diverges %"PRIsize_t" != %"PRIsize_t"\n", packed, packed_iovec); + exit(-1); + } + return packed_iovec; +} + +int main(int argc, char * argv[]) +{ + + int const per_process = 192; + int const per_type = 20000000; + + int scounts[2] = {per_process, per_process}; + int sdispls[2] = {3*per_process, 0*per_process}; + int rcounts[2] = {per_process, per_process}; + int rdispls[2] = {1*per_process, 2*per_process}; + + MPI_Datatype ddt, stype, rtype; + + opal_init_util(&argc, &argv); + ompi_datatype_init(); + + ompi_datatype_create_contiguous( per_type, MPI_FLOAT, &ddt); + ompi_datatype_create_indexed(2, scounts, sdispls, ddt, &stype); + ompi_datatype_commit(&stype); + ompi_datatype_create_indexed(2, rcounts, rdispls, ddt, &rtype); + ompi_datatype_commit(&rtype); + + size_t packed = count_length_via_convertor_raw(stype, 1); + size_t length; + opal_datatype_type_size(&stype->super, &length); + if( length != packed ) { + printf("Mismatched length of packed data to datatype size (%"PRIsize_t" != %"PRIsize_t")\n", + packed, length); + exit(-2); + } + + packed = count_length_via_convertor_raw(rtype, 1); + opal_datatype_type_size(&rtype->super, &length); + if( length != packed ) { + printf("Mismatched length of packed data to datatype size (%"PRIsize_t" != %"PRIsize_t")\n", + packed, length); + exit(-2); + } + + ompi_datatype_destroy(&stype); + ompi_datatype_destroy(&rtype); + + return 0; +}