diff --git a/ompi/include/mpi.h.in b/ompi/include/mpi.h.in index 7d8561dafd..035abefc2b 100644 --- a/ompi/include/mpi.h.in +++ b/ompi/include/mpi.h.in @@ -306,7 +306,6 @@ typedef int (MPI_Grequest_cancel_function)(void *, int); #define MPI_SEEK_END 604 #define MPI_MAX_DATAREP_STRING 128 - #endif /* #if OMPI_PROVIDE_MPI_FILE_INTERFACE */ /* @@ -511,6 +510,15 @@ enum { #define MPI_WIN_NULL_DELETE_FN OMPI_C_MPI_WIN_NULL_DELETE_FN #define MPI_WIN_NULL_COPY_FN OMPI_C_MPI_WIN_NULL_COPY_FN #define MPI_WIN_DUP_FN OMPI_C_MPI_WIN_DUP_FN + +/* MPI_CONVERSION_FN_NULL is a sentinel value, but it has to be large + enough to be the same size as a valid function pointer. It + therefore shares many characteristics between Fortran constants and + Fortran sentinel functions. For example, it shares the problem of + having Fortran compilers have all-caps versions of the symbols that + must be able to be present, and therefore has to be in this + conditional block in mpi.h. */ +#define MPI_CONVERSION_FN_NULL ((MPI_Datarep_conversion_function*) 0) #endif OMPI_DECLSPEC int OMPI_C_MPI_TYPE_NULL_DELETE_FN( MPI_Datatype datatype, diff --git a/ompi/include/mpif.h.in b/ompi/include/mpif.h.in index 014ca932e7..2a93cf0618 100644 --- a/ompi/include/mpif.h.in +++ b/ompi/include/mpif.h.in @@ -10,7 +10,7 @@ ! University of Stuttgart. All rights reserved. ! Copyright (c) 2004-2005 The Regents of the University of California. ! All rights reserved. -! Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. +! Copyright (c) 2006-2007 Cisco Systems, Inc. All rights reserved. ! $COPYRIGHT$ ! ! Additional copyrights may follow @@ -70,6 +70,12 @@ external MPI_WIN_NULL_COPY_FN external MPI_WIN_NULL_DELETE_FN external MPI_WIN_DUP_FN +! Note that MPI_CONVERSION_FN_NULL is a "constant" (it is only ever +! checked for comparison; it is never invoked), but it is passed as +! a function pointer (to MPI_REGISTER_DATAREP) and therefore must be +! the same size/type. It is therefore external'ed here, and not +! defined with an integer value in mpif-common.h. + external MPI_CONVERSION_FN_NULL ! ! double precision functions diff --git a/ompi/mpi/cxx/file.cc b/ompi/mpi/cxx/file.cc index ad8c7ab54b..442cad967d 100644 --- a/ompi/mpi/cxx/file.cc +++ b/ompi/mpi/cxx/file.cc @@ -2,6 +2,7 @@ // // Copyright (c) 2006 Los Alamos National Security, LLC. All rights // reserved. +// Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. // $COPYRIGHT$ // // Additional copyrights may follow @@ -9,11 +10,17 @@ // $HEADER$ // -// do not include ompi_config.h because it kills the free/malloc defines +// Do not include ompi_config.h before mpi.h because it causes +// malloc/free problems due to setting OMPI_BUILDING to 1 #include "mpi.h" + +#include "ompi/constants.h" #include "ompi/mpi/cxx/mpicxx.h" #include "opal/threads/mutex.h" - +#include "opal/class/opal_list.h" +#include "ompi/file/file.h" +#include "ompi/errhandler/errhandler.h" +#include "ompi/runtime/mpiruntime.h" void MPI::File::Close() @@ -36,3 +43,188 @@ MPI::File::Set_errhandler(const MPI::Errhandler& errhandler) OPAL_THREAD_UNLOCK(MPI::mpi_map_mutex); (void)MPI_File_set_errhandler(mpi_file, errhandler); } + + +// +// Infrastructure for MPI_REGISTER_DATAREP +// +// Similar to what we have to do in the F77 bindings: call the C +// MPI_Register_datarep function with "intercept" callback functions +// that conform to the C bindings. In these intercepts, convert the +// arguments to C++ calling convertions, and then invoke the actual +// C++ callbacks. + +// Data structure passed to the intercepts (see below). It is an OPAL +// list_item_t so that we can clean this memory up during +// MPI_FINALIZE. +typedef struct intercept_extra_state { + opal_list_item_t base; + MPI::Datarep_conversion_function *read_fn_cxx; + MPI::Datarep_conversion_function *write_fn_cxx; + MPI::Datarep_extent_function *extent_fn_cxx; + void *extra_state_cxx; +} intercept_extra_state_t; + +static void intercept_extra_state_constructor(intercept_extra_state_t *obj) +{ + obj->read_fn_cxx = NULL; + obj->write_fn_cxx = NULL; + obj->extent_fn_cxx = NULL; + obj->extra_state_cxx = NULL; +} + +OBJ_CLASS_DECLARATION(intercept_extra_state_t); +OBJ_CLASS_INSTANCE(intercept_extra_state_t, + opal_list_item_t, + intercept_extra_state_constructor, NULL); + +// Intercept function for read conversions +static int read_intercept_fn(void *userbuf, MPI_Datatype type_c, int count_c, + void *filebuf, MPI_Offset position_c, + void *extra_state) +{ + MPI::Datatype type_cxx(type_c); + MPI::Offset position_cxx(position_c); + intercept_extra_state_t *intercept_data = + (intercept_extra_state_t*) extra_state; + + intercept_data->read_fn_cxx(userbuf, type_cxx, count_c, filebuf, + position_cxx, intercept_data->extra_state_cxx); + return MPI_SUCCESS; +} + +// Intercept function for write conversions +static int write_intercept_fn(void *userbuf, MPI_Datatype type_c, int count_c, + void *filebuf, MPI_Offset position_c, + void *extra_state) +{ + MPI::Datatype type_cxx(type_c); + MPI::Offset position_cxx(position_c); + intercept_extra_state_t *intercept_data = + (intercept_extra_state_t*) extra_state; + + intercept_data->write_fn_cxx(userbuf, type_cxx, count_c, filebuf, + position_cxx, intercept_data->extra_state_cxx); + return MPI_SUCCESS; +} + +// Intercept function for extent calculations +static int extent_intercept_fn(MPI_Datatype type_c, MPI_Aint *file_extent_c, + void *extra_state) +{ + MPI::Datatype type_cxx(type_c); + MPI::Aint file_extent_cxx(*file_extent_c); + intercept_extra_state_t *intercept_data = + (intercept_extra_state_t*) extra_state; + + intercept_data->extent_fn_cxx(type_cxx, file_extent_cxx, + intercept_data->extra_state_cxx); + *file_extent_c = file_extent_cxx; + return MPI_SUCCESS; +} + +// C++ bindings for MPI::Register_datarep +void +MPI::Register_datarep(const char* datarep, + Datarep_conversion_function* read_fn_cxx, + Datarep_conversion_function* write_fn_cxx, + Datarep_extent_function* extent_fn_cxx, + void* extra_state_cxx) +{ + intercept_extra_state_t *intercept; + + intercept = OBJ_NEW(intercept_extra_state_t); + if (NULL == intercept) { + OMPI_ERRHANDLER_INVOKE(MPI_FILE_NULL, OMPI_ERR_OUT_OF_RESOURCE, + "MPI::Register_datarep"); + return; + } + opal_list_append(&ompi_registered_datareps, &(intercept->base)); + intercept->read_fn_cxx = read_fn_cxx; + intercept->write_fn_cxx = write_fn_cxx; + intercept->extent_fn_cxx = extent_fn_cxx; + intercept->extra_state_cxx = extra_state_cxx; + + (void)MPI_Register_datarep(const_cast(datarep), read_intercept_fn, + write_intercept_fn, + extent_intercept_fn, intercept); +} + + +void +MPI::Register_datarep(const char* datarep, + MPI_Datarep_conversion_function* read_fn_c, + Datarep_conversion_function* write_fn_cxx, + Datarep_extent_function* extent_fn_cxx, + void* extra_state_cxx) +{ + intercept_extra_state_t *intercept; + + intercept = OBJ_NEW(intercept_extra_state_t); + if (NULL == intercept) { + OMPI_ERRHANDLER_INVOKE(MPI_FILE_NULL, OMPI_ERR_OUT_OF_RESOURCE, + "MPI::Register_datarep"); + return; + } + opal_list_append(&ompi_registered_datareps, &(intercept->base)); + intercept->write_fn_cxx = write_fn_cxx; + intercept->extent_fn_cxx = extent_fn_cxx; + intercept->extra_state_cxx = extra_state_cxx; + + (void)MPI_Register_datarep(const_cast(datarep), read_fn_c, + write_intercept_fn, + extent_intercept_fn, intercept); +} + + +void +MPI::Register_datarep(const char* datarep, + Datarep_conversion_function* read_fn_cxx, + MPI_Datarep_conversion_function* write_fn_c, + Datarep_extent_function* extent_fn_cxx, + void* extra_state_cxx) +{ + intercept_extra_state_t *intercept; + + intercept = OBJ_NEW(intercept_extra_state_t); + if (NULL == intercept) { + OMPI_ERRHANDLER_INVOKE(MPI_FILE_NULL, OMPI_ERR_OUT_OF_RESOURCE, + "MPI::Register_datarep"); + return; + } + opal_list_append(&ompi_registered_datareps, &(intercept->base)); + intercept->read_fn_cxx = read_fn_cxx; + intercept->extent_fn_cxx = extent_fn_cxx; + intercept->extra_state_cxx = extra_state_cxx; + + (void)MPI_Register_datarep(const_cast(datarep), read_intercept_fn, + write_fn_c, + extent_intercept_fn, intercept); +} + + +void +MPI::Register_datarep(const char* datarep, + MPI_Datarep_conversion_function* read_fn_c, + MPI_Datarep_conversion_function* write_fn_c, + Datarep_extent_function* extent_fn_cxx, + void* extra_state_cxx) +{ + intercept_extra_state_t *intercept; + + intercept = OBJ_NEW(intercept_extra_state_t); + if (NULL == intercept) { + OMPI_ERRHANDLER_INVOKE(MPI_FILE_NULL, OMPI_ERR_OUT_OF_RESOURCE, + "MPI::Register_datarep"); + return; + } + opal_list_append(&ompi_registered_datareps, &(intercept->base)); + intercept->extent_fn_cxx = extent_fn_cxx; + intercept->extra_state_cxx = extra_state_cxx; + + (void)MPI_Register_datarep(const_cast(datarep), read_fn_c, + write_fn_c, + extent_intercept_fn, intercept); +} + + diff --git a/ompi/mpi/cxx/file.h b/ompi/mpi/cxx/file.h index 92a31afbc6..c36850ffdd 100644 --- a/ompi/mpi/cxx/file.h +++ b/ompi/mpi/cxx/file.h @@ -10,7 +10,7 @@ // University of Stuttgart. All rights reserved. // Copyright (c) 2004-2005 The Regents of the University of California. // All rights reserved. -// Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. +// Copyright (c) 2006-2007 Cisco Systems, Inc. All rights reserved. // $COPYRIGHT$ // // Additional copyrights may follow @@ -18,6 +18,41 @@ // $HEADER$ // +// Typedefs for C++ callbacks registered via MPI::Register_datarep +typedef void Datarep_extent_function(const Datatype& datatype, + Aint& file_extent, void* extra_state); +typedef void Datarep_conversion_function(void* userbuf, Datatype& datatype, + int count, void* filebuf, + Offset position, void* extra_state); + +// Both callback functions in C++ +void Register_datarep(const char* datarep, + Datarep_conversion_function* read_conversion_fn, + Datarep_conversion_function* write_conversion_fn, + Datarep_extent_function* dtype_file_extent_fn, + void* extra_state); + +// Overload for C read callback function (MPI_CONVERSION_FN_NULL) +void Register_datarep(const char* datarep, + MPI_Datarep_conversion_function* read_conversion_fn, + Datarep_conversion_function* write_conversion_fn, + Datarep_extent_function* dtype_file_extent_fn, + void* extra_state); + +// Overload for C write callback function (MPI_CONVERSION_FN_NULL) +void Register_datarep(const char* datarep, + Datarep_conversion_function* read_conversion_fn, + MPI_Datarep_conversion_function* write_conversion_fn, + Datarep_extent_function* dtype_file_extent_fn, + void* extra_state); + +// Overload for C read and write callback functions (MPI_CONVERSION_FN_NULL) +void Register_datarep(const char* datarep, + MPI_Datarep_conversion_function* read_conversion_fn, + MPI_Datarep_conversion_function* write_conversion_fn, + Datarep_extent_function* dtype_file_extent_fn, + void* extra_state); + class File { #if 0 /* OMPI_ENABLE_MPI_PROFILING */ // friend class P; diff --git a/ompi/mpi/f77/Makefile.am b/ompi/mpi/f77/Makefile.am index d0ca45cd41..538c322aca 100644 --- a/ompi/mpi/f77/Makefile.am +++ b/ompi/mpi/f77/Makefile.am @@ -80,6 +80,7 @@ endif headers = \ bindings.h \ constants.h \ + datarep.h \ fint_2_int.h \ prototypes_mpi.h \ f77_strings.h @@ -90,6 +91,7 @@ headers = \ # libmpi_f77_base_la_SOURCES = \ attr_fn_f.c \ + conversion_fn_null_f.c \ f90_accessors.c \ strings.c \ test_constants_f.c diff --git a/ompi/mpi/f77/conversion_fn_null_f.c b/ompi/mpi/f77/conversion_fn_null_f.c new file mode 100644 index 0000000000..beb1fe8f5c --- /dev/null +++ b/ompi/mpi/f77/conversion_fn_null_f.c @@ -0,0 +1,71 @@ +/* + * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +/* + * This file defines the 5 back-end functions for Fortran's version of + * MPI_CONVERSION_FN_NULL. These functions will never be invoked, but + * are rather used as sentinel values by the Fortran + * MPI_REGISTER_DATAREP function. However, given the way Fortran + * sentinel values have to be handled (4 versions for the 4 different + * symbol conventions), and given that these symbols must be the same + * size as a function pointer (because the user could pass either + * MPI_REGISTER_DATAREP *or* a real function), it just seemed either + * to use the normal mechanisms we already have in place for Fortran + * bindings and sentinel values -- even though it's a bit overkill + * (e.g., the back-end mpi_conversion_fn_null_f function will never be + * used for anything). + * + * See also the comments in ompi/mpi/f77/datarep.h. + */ + +#include "ompi_config.h" + +#include "ompi/mpi/f77/datarep.h" +#include "ompi/mpi/f77/fint_2_int.h" + + +#if OMPI_HAVE_WEAK_SYMBOLS +#pragma weak MPI_CONVERSION_FN_NULL = mpi_conversion_fn_null_f +#pragma weak mpi_conversion_fn_null = mpi_conversion_fn_null_f +#pragma weak mpi_conversion_fn_null_ = mpi_conversion_fn_null_f +#pragma weak mpi_conversion_fn_null__ = mpi_conversion_fn_null_f + +#else +OMPI_GENERATE_F77_BINDINGS(MPI_CONVERSION_FN_NULL, + mpi_conversion_fn_null, + mpi_conversion_fn_null_, + mpi_conversion_fn_null__, + mpi_conversion_fn_null_f, + (char *userbuf, MPI_Fint *datatype, MPI_Fint *count, char *filebuf, MPI_Offset *position, MPI_Aint *extra_state, MPI_Fint *ierr), + (userbuf, datatype, count, filebuf, position, extra_state, ierr) +#endif + + +void mpi_conversion_fn_null_f(char *userbuf, MPI_Fint *datatype, + MPI_Fint *count, char *filebuf, + MPI_Offset *position, + MPI_Aint *extra_state, + MPI_Fint *ierr) +{ + /* Per MPI-2:9.5.3, this function will never be called; it's only + used as a sentinel value for comparison. But we need to put + something here so that the compiler/linker doesn't optimize it + out. */ + *ierr = OMPI_INT_2_FINT(MPI_SUCCESS); +} diff --git a/ompi/mpi/f77/datarep.h b/ompi/mpi/f77/datarep.h new file mode 100644 index 0000000000..eebb263ff3 --- /dev/null +++ b/ompi/mpi/f77/datarep.h @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2007 Cisco Systems, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +/** + * @file + * + * This file does two things: + * + * 1. Provides typedef for the Fortran versions of the callbacks + * registered by MPI_REGISTER_DATAREP. These typedefs are needed for + * the Fortran MPI API. + * + * 2. Provides the sentinel value functions/function pointers for + * Fortran's version of MPI_CONVERSION_FN_NULL, and some helpful + * macros for testing whether an argument passed to a Fortran MPI API + * function is that sentinel value or not. + */ + +#ifndef OMPI_F77_DATAREP_H +#define OMPI_F77_DATAREP_H + +#include "ompi_config.h" + +#include "mpi.h" + +BEGIN_C_DECLS + +/** + * Function typedef for the conversion function pointer in + * MPI_REGISTER_DATAREP */ +typedef void (ompi_mpi2_fortran_datarep_conversion_fn_t) + (char *userbuf, MPI_Fint *datatype, MPI_Fint *count, char *filebuf, + MPI_Offset *position, MPI_Aint *extra_state, MPI_Fint *ierr); + +/** + * Function typedef for the extent function pointer in + * MPI_REGISTER_DATAREP */ +typedef void (ompi_mpi2_fortran_datarep_extent_fn_t) + (MPI_Fint *datatype, MPI_Aint *extent, MPI_Aint *extra_state, + MPI_Fint *ierr); + +/** + * Macro for declaring each of the 5 back-end Fortran functions for + * MPI_CONVERSION_FN_NULL. We need the 4 fortran compiler convetions + * and 1 for the "real" back-end function (even though these functions + * are never invoked -- they're only used as sentinel values -- it's + * simpler to use the same kind of code structure that we use for the + * Fortran MPI API bindings and other callback functions). + */ +#define OMPI_DATAREP_FORTRAN_DECLARE(lower_name, upper_name, args) \ + OMPI_DECLSPEC void lower_name##_f args; \ + OMPI_DECLSPEC void lower_name args; \ + OMPI_DECLSPEC void lower_name##_ args; \ + OMPI_DECLSPEC void lower_name##__ args; \ + OMPI_DECLSPEC void upper_name args; + +/* + * Declare the 5 functions. + */ +OMPI_DATAREP_FORTRAN_DECLARE(mpi_conversion_fn_null, MPI_CONVERSION_FN_NULL, (char *userbuf, MPI_Fint *datatype, MPI_Fint *count, char *filebuf, MPI_Offset *position, MPI_Aint *extra_state, MPI_Fint *ierr)) + +/* Be social and remove this private macro from the global header file + space */ +#undef OMPI_DATAREP_FORTRAN_DECLARE + +/** + * Declare the test macro in all of its forms. This macro provides a + * convenient way to check whether an argument is the sentinel value + * MPI_CONVERSION_FN_NULL. + */ +#if OMPI_HAVE_WEAK_SYMBOLS +#define OMPI_IS_FORTRAN_CONVERSION_FN_NULL(addr) \ + (MPI_CONVERSION_FN_NULL == addr || \ + mpi_conversion_fn_null == addr || \ + mpi_conversion_fn_null_ == addr || \ + mpi_conversion_fn_null__ == addr) +#elif OMPI_F77_CAPS +#define OMPI_IS_FORTRAN_CONVERSION_FN_NULL(addr) \ + (MPI_CONVERSION_FN_NULL == addr) +#elif OMPI_F77_PLAIN +#define OMPI_IS_FORTRAN_CONVERSION_FN_NULL(addr) \ + (mpi_conversion_fn_null == addr) +#elif OMPI_F77_SINGLE_UNDERSCORE +#define OMPI_IS_FORTRAN_CONVERSION_FN_NULL(addr) \ + (mpi_conversion_fn_null_ == addr) +#else +#define OMPI_IS_FORTRAN_CONVERSION_FN_NULL(addr) \ + (mpi_conversion_fn_null__ == addr) +#endif + +END_C_DECLS + +#endif diff --git a/ompi/mpi/f77/prototypes_mpi.h b/ompi/mpi/f77/prototypes_mpi.h index af63dc5a63..d3428741c1 100644 --- a/ompi/mpi/f77/prototypes_mpi.h +++ b/ompi/mpi/f77/prototypes_mpi.h @@ -42,6 +42,8 @@ #ifndef OMPI_F77_PROTOTYPES_MPI_H #define OMPI_F77_PROTOTYPES_MPI_H +#include "ompi/mpi/f77/datarep.h" + /* These are the prototypes for the "real" back-end fortran functions. */ #define PN(ret, lower_name, upper_name, args) \ OMPI_DECLSPEC ret lower_name##_f args; \ @@ -250,7 +252,7 @@ PN(void, mpi_recv_init, MPI_RECV_INIT, (char *buf, MPI_Fint *count, MPI_Fint *da PN(void, mpi_recv, MPI_RECV, (char *buf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *source, MPI_Fint *tag, MPI_Fint *comm, MPI_Fint *status, MPI_Fint *ierr)); PN(void, mpi_reduce, MPI_REDUCE, (char *sendbuf, char *recvbuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *root, MPI_Fint *comm, MPI_Fint *ierr)); PN(void, mpi_reduce_scatter, MPI_REDUCE_SCATTER, (char *sendbuf, char *recvbuf, MPI_Fint *recvcounts, MPI_Fint *datatype, MPI_Fint *op, MPI_Fint *comm, MPI_Fint *ierr)); -PN(void, mpi_register_datarep, MPI_REGISTER_DATAREP, (char *datarep, void *read_conversion_fn, void *write_conversion_fn, void *dtype_file_extent_fn, MPI_Aint *extra_state, MPI_Fint *ierr, int datarep_len)); +PN(void, mpi_register_datarep, MPI_REGISTER_DATAREP, (char *datarep, ompi_mpi2_fortran_datarep_conversion_fn_t *read_conversion_fn, ompi_mpi2_fortran_datarep_conversion_fn_t *write_conversion_fn, ompi_mpi2_fortran_datarep_extent_fn_t *dtype_file_extent_fn, MPI_Aint *extra_state, MPI_Fint *ierr, int datarep_len)); PN(void, mpi_request_free, MPI_REQUEST_FREE, (MPI_Fint *request, MPI_Fint *ierr)); PN(void, mpi_request_get_status, MPI_REQUEST_GET_STATUS, (MPI_Fint *request, MPI_Flogical *flag, MPI_Fint *status, MPI_Fint *ierr)); PN(void, mpi_rsend, MPI_RSEND, (char *ibuf, MPI_Fint *count, MPI_Fint *datatype, MPI_Fint *dest, MPI_Fint *tag, MPI_Fint *comm, MPI_Fint *ierr)); diff --git a/ompi/mpi/f77/register_datarep_f.c b/ompi/mpi/f77/register_datarep_f.c index 78272433c4..7be9a94f6c 100644 --- a/ompi/mpi/f77/register_datarep_f.c +++ b/ompi/mpi/f77/register_datarep_f.c @@ -19,8 +19,14 @@ #include "ompi_config.h" +#include "opal/class/opal_object.h" + #include "ompi/mpi/f77/bindings.h" +#include "ompi/mpi/f77/constants.h" +#include "ompi/mpi/f77/datarep.h" #include "ompi/mpi/f77/f77_strings.h" +#include "ompi/mpi/f77/fint_2_int.h" +#include "ompi/runtime/mpiruntime.h" #include "ompi/file/file.h" #if OMPI_HAVE_WEAK_SYMBOLS && OMPI_PROFILE_LAYER @@ -60,38 +66,179 @@ OMPI_GENERATE_F77_BINDINGS (MPI_REGISTER_DATAREP, #include "ompi/mpi/f77/profile/defines.h" #endif -union local_type_convert { - void *voidp; - MPI_Datarep_conversion_function *convertp; - MPI_Datarep_extent_function *extentp; -}; +static const char FUNC_NAME[] = "MPI_REGISTER_DATAREP"; -void mpi_register_datarep_f(char *datarep, void *read_conversion_fn, - void *write_conversion_fn, - void *dtype_file_extent_fn, MPI_Aint *extra_state, +/* Intercept functions used below (see below for explanations in + comments) */ +static int read_intercept_fn(void *userbuf, MPI_Datatype type_c, int count_c, + void *filebuf, MPI_Offset position, + void *extra_state); +static int write_intercept_fn(void *userbuf, MPI_Datatype type_c, int count_c, + void *filebuf, MPI_Offset position, + void *extra_state); +static int extent_intercept_fn(MPI_Datatype type_c, MPI_Aint *file_extent, + void *extra_state); + +/* Data structure passed to the intercepts (see below). It is an OPAL + list_item_t so that we can clean this memory up during + MPI_FINALIZE. */ +typedef struct intercept_extra_state { + opal_list_item_t base; + ompi_mpi2_fortran_datarep_conversion_fn_t *read_fn_f77; + ompi_mpi2_fortran_datarep_conversion_fn_t *write_fn_f77; + ompi_mpi2_fortran_datarep_extent_fn_t *extent_fn_f77; + MPI_Aint *extra_state_f77; +} intercept_extra_state_t; + +static void intercept_extra_state_constructor(intercept_extra_state_t *obj) +{ + obj->read_fn_f77 = NULL; + obj->write_fn_f77 = NULL; + obj->extent_fn_f77 = NULL; + obj->extra_state_f77 = NULL; +} + +OBJ_CLASS_DECLARATION(intercept_extra_state_t); +OBJ_CLASS_INSTANCE(intercept_extra_state_t, + opal_list_item_t, + intercept_extra_state_constructor, NULL); + + +/* + * This function works by calling the C version of + * MPI_Register_datarep (like most other MPI API functions). To do + * that, however, we need to call the C MPI_Register_datarep with *C* + * callback functions -- the callback functions passed in to this + * function are Fortran functions, and expect Fortran argument passing + * conventions. + * + * So we have 3 C intercept functions that are passed to the back-end + * MPI_Register_datarep. Hence, when/if this datarep is ever used, + * the intercept function(s) are invoked, who then translate the + * arguments to Fortran and then invoke the registered callback + * function. + */ +void mpi_register_datarep_f(char *datarep, + ompi_mpi2_fortran_datarep_conversion_fn_t *read_fn_f77, + ompi_mpi2_fortran_datarep_conversion_fn_t *write_fn_f77, + ompi_mpi2_fortran_datarep_extent_fn_t *extent_fn_f77, + MPI_Aint *extra_state_f77, MPI_Fint *ierr, int datarep_len) { - char *c_datarep; - int c_err, ret; - union local_type_convert a, b, c; + char *c_datarep; + int c_err, ret; + MPI_Datarep_conversion_function *read_fn_c, *write_fn_c; + intercept_extra_state_t *intercept; + + /* Malloc space for the intercept callback data */ + intercept = OBJ_NEW(intercept_extra_state_t); + if (NULL == intercept) { + c_err = OMPI_ERRHANDLER_INVOKE(MPI_FILE_NULL, + OMPI_ERR_OUT_OF_RESOURCE, FUNC_NAME); + *ierr = OMPI_INT_2_FINT(c_err); + return; + } + /* Save the new object on a global list because per MPI-2:9.5.3, + there are no ways for the user to deregister datareps once + they've been created. Hece, this is a memory leak. So we + track these extra resources in a global list so that they can + be freed during MPI_FINALIZE (so that memory-tracking debuggers + won't show MPI as leaking memory). */ + opal_list_append(&ompi_registered_datareps, &(intercept->base)); /* Convert the fortran string */ if (OMPI_SUCCESS != (ret = ompi_fortran_string_f2c(datarep, datarep_len, &c_datarep))) { - c_err = OMPI_ERRHANDLER_INVOKE(MPI_FILE_NULL, ret, "MPI_FILE_SET_VIEW"); + c_err = OMPI_ERRHANDLER_INVOKE(MPI_FILE_NULL, ret, FUNC_NAME); *ierr = OMPI_INT_2_FINT(c_err); return; } + + /* Convert the Fortran function callbacks to C equivalents. Use + local intercepts if they're not MPI_CONVERSION_FN_NULL so that + we can just call the C MPI API MPI_Register_datarep(). If they + *are* MPI_CONVERSION_FN_NULL, then just pass that to + MPI_Register_datarep so that it becomes a no-op (i.e., no + callback is ever triggered). */ + if (OMPI_IS_FORTRAN_CONVERSION_FN_NULL(read_fn_f77)) { + /* Can't use the MPI_CONVERSION_FN_NULL macro here because it + is specifically not defined when compiling this file so + that we can prototype an all-caps Fortran function */ + read_fn_c = (MPI_Datarep_conversion_function*) 0; + } else { + intercept->read_fn_f77 = read_fn_f77; + read_fn_c = read_intercept_fn; + } + if (OMPI_IS_FORTRAN_CONVERSION_FN_NULL(write_fn_f77)) { + /* Can't use the MPI_CONVERSION_FN_NULL macro here because it + is specifically not defined when compiling this file so + that we can prototype an all-caps Fortran function */ + write_fn_c = (MPI_Datarep_conversion_function*) 0; + } else { + intercept->write_fn_f77 = write_fn_f77; + write_fn_c = write_intercept_fn; + } + intercept->extent_fn_f77 = extent_fn_f77; + intercept->extra_state_f77 = extra_state_f77; - /* Do a little type shifting so that we don't get compiler - warnings */ - a.voidp = read_conversion_fn; - b.voidp = write_conversion_fn; - c.voidp = dtype_file_extent_fn; - - *ierr = OMPI_INT_2_FINT(MPI_Register_datarep(c_datarep, a.convertp, - b.convertp, c.extentp, - extra_state)); - + /* Now that the intercept data has been setup, call the C function + with the setup intercept routines and the intercept-specific + data/extra state. */ + *ierr = OMPI_INT_2_FINT(MPI_Register_datarep(c_datarep, + read_fn_c, write_fn_c, + extent_intercept_fn, + intercept)); free(c_datarep); } + +/* + * C->Fortran intercept for the read conversion. + */ +static int read_intercept_fn(void *userbuf, MPI_Datatype type_c, int count_c, + void *filebuf, MPI_Offset position, + void *extra_state) +{ + MPI_Fint ierr, count_f77 = OMPI_FINT_2_INT(count_f77); + MPI_Fint type_f77 = MPI_Type_c2f(type_c); + intercept_extra_state_t *intercept_data = + (intercept_extra_state_t*) extra_state; + + intercept_data->read_fn_f77(userbuf, &type_f77, &count_f77, filebuf, + &position, intercept_data->extra_state_f77, + &ierr); + return OMPI_FINT_2_INT(ierr); +} + +/* + * C->Fortran intercept for the write conversion. + */ +static int write_intercept_fn(void *userbuf, MPI_Datatype type_c, int count_c, + void *filebuf, MPI_Offset position, + void *extra_state) +{ + MPI_Fint ierr, count_f77 = OMPI_FINT_2_INT(count_f77); + MPI_Fint type_f77 = MPI_Type_c2f(type_c); + intercept_extra_state_t *intercept_data = + (intercept_extra_state_t*) extra_state; + + intercept_data->write_fn_f77(userbuf, &type_f77, &count_f77, filebuf, + &position, intercept_data->extra_state_f77, + &ierr); + return OMPI_FINT_2_INT(ierr); +} + +/* + * C->Fortran intercept for the extent calculation. + */ +static int extent_intercept_fn(MPI_Datatype type_c, MPI_Aint *file_extent_f77, + void *extra_state) +{ + MPI_Fint ierr, type_f77 = MPI_Type_c2f(type_c); + intercept_extra_state_t *intercept_data = + (intercept_extra_state_t*) extra_state; + + intercept_data->extent_fn_f77(&type_f77, file_extent_f77, + intercept_data->extra_state_f77, &ierr); + return OMPI_FINT_2_INT(ierr); +} + diff --git a/ompi/mpi/f90/Makefile.am b/ompi/mpi/f90/Makefile.am index ac9416e34c..b5c28a1553 100644 --- a/ompi/mpi/f90/Makefile.am +++ b/ompi/mpi/f90/Makefile.am @@ -140,6 +140,7 @@ endif libmpi_f90_la_SOURCES = \ attr_fn-f90-interfaces.h \ + conversion_fn_null-f90-interface.h \ mpi.f90 # These files are all generated by scripts in the scripts/ directory. diff --git a/ompi/mpi/f90/conversion_fn_null-f90-interface.h b/ompi/mpi/f90/conversion_fn_null-f90-interface.h new file mode 100644 index 0000000000..8b4abad8a8 --- /dev/null +++ b/ompi/mpi/f90/conversion_fn_null-f90-interface.h @@ -0,0 +1,29 @@ +! +! Copyright (c) 2006-2007 Cisco Systems, Inc. All rights reserved. +! $COPYRIGHT$ +! +! Additional copyrights may follow +! +! $HEADER$ +! + +! Note about these declarations: these are "external" functions in +! mpif-common.h. However, if we don't declare them here, compilers will add +! them to the "mpi" module namespace, and result in linker errors if MPI +! F90 applications try to use them. because the implementations of +! these functions are not in the MPI module namespace -- they're the F77 +! functions. + +interface + + subroutine mpi_conversion_fn_null(userbuf, datatype, count, filebuf, & + position, extra_state, ierror) + include 'mpif-config.h' + character(len=*), intent(in) :: filebuf + character(len=*), intent(out) :: userbuf + integer, intent(in) :: datatype, count, ierror + integer(kind=MPI_OFFSET_KIND), intent(in) :: position + integer(kind=MPI_ADDRESS_KIND), intent(in) :: extra_state + end subroutine mpi_conversion_fn_null + +end interface diff --git a/ompi/mpi/f90/mpi.f90 b/ompi/mpi/f90/mpi.f90 index 3bc30029c0..0bb9c5525b 100644 --- a/ompi/mpi/f90/mpi.f90 +++ b/ompi/mpi/f90/mpi.f90 @@ -27,6 +27,10 @@ module mpi include "attr_fn-f90-interfaces.h" +! Similarly, we need the MPI_CONVERSION_FN_NULL function + + include "conversion_fn_null-f90-interface.h" + ! This file is generated, and is *huge*. Its size is directly related ! to the --with-f90-max-array-dim configure parameter. diff --git a/ompi/runtime/mpiruntime.h b/ompi/runtime/mpiruntime.h index 4b44e2acc6..fcfbcb8db8 100644 --- a/ompi/runtime/mpiruntime.h +++ b/ompi/runtime/mpiruntime.h @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2006-2007 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2007 Los Alamos National Security, LLC. All rights * reserved. * $COPYRIGHT$ @@ -30,6 +30,8 @@ #include "ompi_config.h" +#include "opal/class/opal_list.h" + BEGIN_C_DECLS /** forward type declaration */ @@ -57,6 +59,11 @@ OMPI_DECLSPEC extern struct opal_thread_t *ompi_mpi_main_thread; it down during MPI_FINALIZE)? */ OMPI_DECLSPEC extern bool ompi_mpi_maffinity_setup; +/** In ompi_mpi_init: a list of all memory associated with calling + MPI_REGISTER_DATAREP so that we can free it during + MPI_FINALIZE. */ +OMPI_DECLSPEC extern opal_list_t ompi_registered_datareps; + /** * Initialize the Open MPI MPI environment diff --git a/ompi/runtime/ompi_mpi_finalize.c b/ompi/runtime/ompi_mpi_finalize.c index ecaa33dbb8..fc7d6bbefc 100644 --- a/ompi/runtime/ompi_mpi_finalize.c +++ b/ompi/runtime/ompi_mpi_finalize.c @@ -87,11 +87,11 @@ #endif #include "ompi/runtime/ompi_cr.h" - int ompi_mpi_finalize(void) { int ret; static int32_t finalize_has_already_started = 0; + opal_list_item_t *item; /* Be a bit social if an erroneous program calls MPI_FINALIZE in two different threads, otherwise we may deadlock in @@ -163,8 +163,15 @@ int ompi_mpi_finalize(void) ORTE_ERROR_LOG(ret); } - /* Shut down any bindings-specific issues: C++, F77, F90 (may or - may not be necessary...?) */ + /* Shut down any bindings-specific issues: C++, F77, F90 */ + + /* Remove all memory associated by MPI_REGISTER_DATAREP (per + MPI-2:9.5.3, there is no way for an MPI application to + *un*register datareps, but we don't want the OMPI layer causing + memory leaks). */ + while (NULL != (item = opal_list_remove_first(&ompi_registered_datareps))) { + OBJ_RELEASE(item); + } /* Free communication objects */ diff --git a/ompi/runtime/ompi_mpi_init.c b/ompi/runtime/ompi_mpi_init.c index 67f8d96a18..4d0930c992 100644 --- a/ompi/runtime/ompi_mpi_init.c +++ b/ompi/runtime/ompi_mpi_init.c @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2006 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2006-2007 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2006-2007 Los Alamos National Security, LLC. All rights * reserved. * Copyright (c) 2006 University of Houston. All rights reserved. @@ -28,6 +28,7 @@ #endif /* HAVE_SYS_TIME_H */ #include "mpi.h" +#include "opal/class/opal_list.h" #include "opal/mca/base/base.h" #include "opal/mca/paffinity/base/base.h" #include "opal/mca/maffinity/base/base.h" @@ -210,6 +211,15 @@ INST(int *, MPI_FORTRAN_STATUS_IGNORE, mpi_fortran_status_ignore, INST (double, MPI_FORTRAN_STATUSES_IGNORE, mpi_fortran_statuses_ignore, mpi_fortran_statuses_ignore_, mpi_fortran_statuses_ignore__); +/* + * Per MPI-2:9.5.3, MPI_REGISTER_DATAREP is a memory leak. There is + * no way to *de*register datareps once they've been registered. So + * we have to track all registrations here so that they can be + * de-registered during MPI_FINALIZE so that memory-tracking debuggers + * don't show Open MPI as leaking memory. + */ +opal_list_t ompi_registered_datareps; + int ompi_mpi_init(int argc, char **argv, int requested, int *provided) { @@ -742,6 +752,10 @@ int ompi_mpi_init(int argc, char **argv, int requested, int *provided) return ret; } + /* Initialize the registered datarep list to be empty */ + + OBJ_CONSTRUCT(&ompi_registered_datareps, opal_list_t); + /* All done. Wasn't that simple? */ ompi_mpi_initialized = true;