482d84b6e5
The expected sequence of events for processing info during object creation
is that if there's an incoming info arg, it is opal_info_dup()ed into the obj
at obj->s_info first. Then interested components register callbacks for
keys they want to know about using opal_infosubscribe_infosubscribe().
Inside info_subscribe_subscribe() the specified callback() is called with
whatever matching k/v is in the object's info, or with the default. The
return string from the callback goes into the new k/v stored in info, and
the input k/v is saved as __IN_<key>/<val>. It's saved the same way
whether the input came from info or whether it was a default. A null return
from the callback indicates an ignored key/val, and no k/v is stored for
it, but an __IN_<key>/<val> is still kept so we still have access to the
original.
At MPI_*_set_info() time, opal_infosubscribe_change_info() is used. That
function calls the registered callbacks for each item in the provided info.
If the callback returns non-null, the info is updated with that k/v, or if
the callback returns null, that key is deleted from info. An __IN_<key>/<val>
is saved either way, and overwrites any previously saved value.
When MPI_*_get_info() is called, opal_info_dup_mpistandard() is used, which
allows relatively easy changes in interpretation of the standard, by looking
at both the <key>/<val> and __IN_<key>/<val> in info. Right now it does
1. includes system extras, eg k/v defaults not expliclty set by the user
2. omits ignored keys
3. shows input values, not callback modifications, eg not the internal values
Currently the callbacks are doing things like
return some_condition ? "true" : "false"
that is, returning static strings that are not to be freed. If the return
strings start becoming more dynamic in the future I don't see how unallocated
strings could support that, so I'd propose a change for the future that
the callback()s registered with info_subscribe_subscribe() do a strdup on
their return, and we change the callers of callback() to free the strings
it returns (there are only two callers).
Rough outline of the smaller changes spread over the less central files:
comm.c
initialize comm->super.s_info to NULL
copy into comm->super.s_info in comm creation calls that provide info
OBJ_RELEASE comm->super.s_info at free time
comm_init.c
initialize comm->super.s_info to NULL
file.c
copy into file->super.s_info if file creation provides info
OBJ_RELEASE file->super.s_info at free time
win.c
copy into win->super.s_info if win creation provides info
OBJ_RELEASE win->super.s_info at free time
comm_get_info.c
file_get_info.c
win_get_info.c
change_info() if there's no info attached (shouldn't happen if callbacks
are registered)
copy the info for the user
The other category of change is generally addressing compiler warnings where
ompi_info_t and opal_info_t were being used a little too interchangably. An
ompi_info_t* contains an opal_info_t*, at &(ompi_info->super)
Also this commit updates the copyrights.
Signed-off-by: Mark Allen <markalle@us.ibm.com>
363 строки
13 KiB
C
363 строки
13 KiB
C
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
|
|
/*
|
|
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
|
|
* University Research and Technology
|
|
* Corporation. All rights reserved.
|
|
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
|
* of Tennessee Research Foundation. All rights
|
|
* reserved.
|
|
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
|
* University of Stuttgart. All rights reserved.
|
|
* Copyright (c) 2004-2005 The Regents of the University of California.
|
|
* All rights reserved.
|
|
* Copyright (c) 2008-2017 University of Houston. All rights reserved.
|
|
* Copyright (c) 2015 Los Alamos National Security, LLC. All rights
|
|
* reserved.
|
|
* Copyright (c) 2015 Research Organization for Information Science
|
|
* and Technology (RIST). All rights reserved.
|
|
* Copyright (c) 2016-2017 IBM Corporation. All rights reserved.
|
|
* $COPYRIGHT$
|
|
*
|
|
* Additional copyrights may follow
|
|
*
|
|
* $HEADER$
|
|
*/
|
|
|
|
#include "ompi_config.h"
|
|
|
|
#include "mpi.h"
|
|
#include "opal/class/opal_list.h"
|
|
#include "opal/threads/mutex.h"
|
|
#include "opal/mca/base/base.h"
|
|
#include "ompi/mca/io/io.h"
|
|
#include "ompi/mca/fs/base/base.h"
|
|
#include "io_ompio.h"
|
|
|
|
int mca_io_ompio_cycle_buffer_size = OMPIO_DEFAULT_CYCLE_BUF_SIZE;
|
|
int mca_io_ompio_bytes_per_agg = OMPIO_PREALLOC_MAX_BUF_SIZE;
|
|
int mca_io_ompio_num_aggregators = -1;
|
|
int mca_io_ompio_record_offset_info = 0;
|
|
int mca_io_ompio_coll_timing_info = 0;
|
|
int mca_io_ompio_sharedfp_lazy_open = 0;
|
|
|
|
int mca_io_ompio_grouping_option=5;
|
|
|
|
/*
|
|
* Private functions
|
|
*/
|
|
static int register_component(void);
|
|
static int open_component(void);
|
|
static int close_component(void);
|
|
static int init_query(bool enable_progress_threads,
|
|
bool enable_mpi_threads);
|
|
static const struct mca_io_base_module_2_0_0_t *
|
|
file_query (struct ompi_file_t *file,
|
|
struct mca_io_base_file_t **private_data,
|
|
int *priority);
|
|
static int file_unquery(struct ompi_file_t *file,
|
|
struct mca_io_base_file_t *private_data);
|
|
|
|
static int delete_query(const char *filename, struct opal_info_t *info,
|
|
struct mca_io_base_delete_t **private_data,
|
|
bool *usable, int *priorty);
|
|
|
|
static int delete_select(const char *filename, struct opal_info_t *info,
|
|
struct mca_io_base_delete_t *private_data);
|
|
|
|
static int register_datarep(const char *,
|
|
MPI_Datarep_conversion_function*,
|
|
MPI_Datarep_conversion_function*,
|
|
MPI_Datarep_extent_function*,
|
|
void*);
|
|
/*
|
|
static int io_progress(void);
|
|
|
|
*/
|
|
|
|
/*
|
|
* Private variables
|
|
*/
|
|
static int priority_param = 30;
|
|
static int delete_priority_param = 30;
|
|
|
|
|
|
/*
|
|
* Global, component-wide OMPIO mutex because OMPIO is not thread safe
|
|
*/
|
|
opal_mutex_t mca_io_ompio_mutex = {{0}};
|
|
|
|
|
|
/*
|
|
* Global list of requests for this component
|
|
*/
|
|
opal_list_t mca_io_ompio_pending_requests = {{0}};
|
|
|
|
|
|
/*
|
|
* Public string showing this component's version number
|
|
*/
|
|
const char *mca_io_ompio_component_version_string =
|
|
"OMPI/MPI OMPIO io MCA component version " OMPI_VERSION;
|
|
|
|
|
|
mca_io_base_component_2_0_0_t mca_io_ompio_component = {
|
|
/* First, the mca_base_component_t struct containing meta information
|
|
about the component itself */
|
|
|
|
.io_version = {
|
|
MCA_IO_BASE_VERSION_2_0_0,
|
|
.mca_component_name = "ompio",
|
|
MCA_BASE_MAKE_VERSION(component, OMPI_MAJOR_VERSION, OMPI_MINOR_VERSION,
|
|
OMPI_RELEASE_VERSION),
|
|
.mca_open_component = open_component,
|
|
.mca_close_component = close_component,
|
|
.mca_register_component_params = register_component,
|
|
},
|
|
.io_data = {
|
|
/* The component is checkpoint ready */
|
|
MCA_BASE_METADATA_PARAM_CHECKPOINT
|
|
},
|
|
|
|
/* Initial configuration / Open a new file */
|
|
|
|
.io_init_query = init_query,
|
|
.io_file_query = file_query,
|
|
.io_file_unquery = file_unquery,
|
|
|
|
/* Delete a file */
|
|
|
|
.io_delete_query = delete_query,
|
|
.io_delete_select = delete_select,
|
|
|
|
.io_register_datarep = register_datarep,
|
|
};
|
|
|
|
static int register_component(void)
|
|
{
|
|
priority_param = 30;
|
|
(void) mca_base_component_var_register(&mca_io_ompio_component.io_version,
|
|
"priority", "Priority of the io ompio component",
|
|
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
|
|
OPAL_INFO_LVL_9,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
&priority_param);
|
|
delete_priority_param = 30;
|
|
(void) mca_base_component_var_register(&mca_io_ompio_component.io_version,
|
|
"delete_priority", "Delete priority of the io ompio component",
|
|
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
|
|
OPAL_INFO_LVL_9,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
&delete_priority_param);
|
|
|
|
mca_io_ompio_record_offset_info = 0;
|
|
(void) mca_base_component_var_register(&mca_io_ompio_component.io_version,
|
|
"record_file_offset_info",
|
|
"The information of the file offset/length",
|
|
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
|
|
OPAL_INFO_LVL_9,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
&mca_io_ompio_record_offset_info);
|
|
|
|
mca_io_ompio_coll_timing_info = 0;
|
|
(void) mca_base_component_var_register(&mca_io_ompio_component.io_version,
|
|
"coll_timing_info",
|
|
"Enable collective algorithm timing information",
|
|
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
|
|
OPAL_INFO_LVL_9,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
&mca_io_ompio_coll_timing_info);
|
|
|
|
mca_io_ompio_cycle_buffer_size = OMPIO_DEFAULT_CYCLE_BUF_SIZE;
|
|
(void) mca_base_component_var_register(&mca_io_ompio_component.io_version,
|
|
"cycle_buffer_size",
|
|
"Data size issued by individual reads/writes per call",
|
|
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
|
|
OPAL_INFO_LVL_9,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
&mca_io_ompio_cycle_buffer_size);
|
|
|
|
mca_io_ompio_bytes_per_agg = OMPIO_PREALLOC_MAX_BUF_SIZE;
|
|
(void) mca_base_component_var_register(&mca_io_ompio_component.io_version,
|
|
"bytes_per_agg",
|
|
"Size of temporary buffer for collective I/O operations",
|
|
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
|
|
OPAL_INFO_LVL_9,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
&mca_io_ompio_bytes_per_agg);
|
|
|
|
mca_io_ompio_num_aggregators = -1;
|
|
(void) mca_base_component_var_register(&mca_io_ompio_component.io_version,
|
|
"num_aggregators",
|
|
"number of aggregators for collective I/O operations",
|
|
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
|
|
OPAL_INFO_LVL_9,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
&mca_io_ompio_num_aggregators);
|
|
|
|
|
|
mca_io_ompio_sharedfp_lazy_open = 0;
|
|
(void) mca_base_component_var_register(&mca_io_ompio_component.io_version,
|
|
"sharedfp_lazy_open",
|
|
"lazy allocation of internal shared file pointer structures",
|
|
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
|
|
OPAL_INFO_LVL_9,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
&mca_io_ompio_sharedfp_lazy_open);
|
|
|
|
mca_io_ompio_grouping_option = 5;
|
|
(void) mca_base_component_var_register(&mca_io_ompio_component.io_version,
|
|
"grouping_option",
|
|
"Option for grouping of processes in the aggregator selection "
|
|
"1: Data volume based grouping 2: maximizing group size uniformity 3: maximimze "
|
|
"data contiguity 4: hybrid optimization 5: simple (default) "
|
|
"6: skip refinement step",
|
|
MCA_BASE_VAR_TYPE_INT, NULL, 0, 0,
|
|
OPAL_INFO_LVL_9,
|
|
MCA_BASE_VAR_SCOPE_READONLY,
|
|
&mca_io_ompio_grouping_option);
|
|
|
|
return OMPI_SUCCESS;
|
|
}
|
|
|
|
static int open_component(void)
|
|
{
|
|
/* Create the mutex */
|
|
OBJ_CONSTRUCT(&mca_io_ompio_mutex, opal_mutex_t);
|
|
|
|
/* Create the list of pending requests */
|
|
|
|
OBJ_CONSTRUCT(&mca_io_ompio_pending_requests, opal_list_t);
|
|
|
|
return OMPI_SUCCESS;
|
|
}
|
|
|
|
|
|
static int close_component(void)
|
|
{
|
|
/* Destroy the list of pending requests */
|
|
/* JMS: Good opprotunity here to list out all the IO requests that
|
|
were not destroyed / completed upon MPI_FINALIZE */
|
|
|
|
OBJ_DESTRUCT(&mca_io_ompio_pending_requests);
|
|
|
|
OBJ_DESTRUCT(&mca_io_ompio_mutex);
|
|
|
|
return OMPI_SUCCESS;
|
|
}
|
|
|
|
|
|
static int init_query(bool enable_progress_threads,
|
|
bool enable_mpi_threads)
|
|
{
|
|
return OMPI_SUCCESS;
|
|
}
|
|
|
|
|
|
static const struct mca_io_base_module_2_0_0_t *
|
|
file_query(struct ompi_file_t *file,
|
|
struct mca_io_base_file_t **private_data,
|
|
int *priority)
|
|
{
|
|
mca_io_ompio_data_t *data;
|
|
char *tmp;
|
|
int rank;
|
|
int is_lustre=0; //false
|
|
|
|
tmp = strchr (file->f_filename, ':');
|
|
rank = ompi_comm_rank ( file->f_comm);
|
|
if (!tmp) {
|
|
if ( 0 == rank) {
|
|
if (LUSTRE == mca_fs_base_get_fstype(file->f_filename)) {
|
|
is_lustre = 1; //true
|
|
}
|
|
}
|
|
|
|
file->f_comm->c_coll->coll_bcast (&is_lustre,
|
|
1,
|
|
MPI_INT,
|
|
0,
|
|
file->f_comm,
|
|
file->f_comm->c_coll->coll_bcast_module);
|
|
}
|
|
else {
|
|
if (!strncasecmp(file->f_filename, "lustre:", 7) ) {
|
|
is_lustre = 1;
|
|
}
|
|
}
|
|
|
|
if (is_lustre) {
|
|
*priority = 1;
|
|
}
|
|
else {
|
|
*priority = priority_param;
|
|
}
|
|
|
|
/* Allocate a space for this module to hang private data (e.g.,
|
|
the OMPIO file handle) */
|
|
|
|
data = calloc(1, sizeof(mca_io_ompio_data_t));
|
|
if (NULL == data) {
|
|
return NULL;
|
|
}
|
|
|
|
*private_data = (struct mca_io_base_file_t*) data;
|
|
|
|
/* All done */
|
|
|
|
return &mca_io_ompio_module;
|
|
}
|
|
|
|
|
|
static int file_unquery(struct ompi_file_t *file,
|
|
struct mca_io_base_file_t *private_data)
|
|
{
|
|
/* Free the ompio module-specific data that was allocated in
|
|
_file_query(), above */
|
|
|
|
if (NULL != private_data) {
|
|
free(private_data);
|
|
}
|
|
|
|
return OMPI_SUCCESS;
|
|
}
|
|
|
|
|
|
static int delete_query(const char *filename, struct opal_info_t *info,
|
|
struct mca_io_base_delete_t **private_data,
|
|
bool *usable, int *priority)
|
|
{
|
|
*priority = delete_priority_param;
|
|
*usable = true;
|
|
*private_data = NULL;
|
|
|
|
return OMPI_SUCCESS;
|
|
}
|
|
|
|
static int delete_select(const char *filename, struct opal_info_t *info,
|
|
struct mca_io_base_delete_t *private_data)
|
|
{
|
|
int ret;
|
|
|
|
OPAL_THREAD_LOCK (&mca_io_ompio_mutex);
|
|
ret = mca_io_ompio_file_delete (filename, info);
|
|
OPAL_THREAD_UNLOCK (&mca_io_ompio_mutex);
|
|
|
|
return ret;
|
|
}
|
|
|
|
static int register_datarep(const char * datarep,
|
|
MPI_Datarep_conversion_function* read_fn,
|
|
MPI_Datarep_conversion_function* write_fn,
|
|
MPI_Datarep_extent_function* extent_fn,
|
|
void* state)
|
|
{
|
|
return OMPI_ERROR;
|
|
}
|
|
|
|
/*
|
|
static int io_progress (void)
|
|
{
|
|
return OMPI_SUCCESS;
|
|
}
|
|
*/
|