commit of the OMPIO modules and frameworks.
This commit was SVN r25079.
Этот коммит содержится в:
Edgar Gabriel
родитель
5abcbfa17b
Коммит
52063267df
83
ompi/config/ompi_check_lustre.m4
Обычный файл
83
ompi/config/ompi_check_lustre.m4
Обычный файл
@ -0,0 +1,83 @@
|
||||
# -*- shell-script -*-
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2006 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
|
||||
# Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
# OMPI_CHECK_LUSTRE(prefix, [action-if-found], [action-if-not-found])
|
||||
# --------------------------------------------------------
|
||||
# check if LUSTRE support can be found. sets prefix_{CPPFLAGS,
|
||||
# LDFLAGS, LIBS} as needed and runs action-if-found if there is
|
||||
# support, otherwise executes action-if-not-found
|
||||
AC_DEFUN([OMPI_CHECK_LUSTRE],[
|
||||
|
||||
check_lustre_CPPFLAGS=
|
||||
check_lustre_LDFLAGS=
|
||||
check_lustre_LIBS=
|
||||
|
||||
check_lustre_configuration="none"
|
||||
ompi_check_lustre_happy="yes"
|
||||
|
||||
# Get some configuration information
|
||||
AC_ARG_WITH([lustre],
|
||||
[AC_HELP_STRING([--with-lustre(=DIR)],
|
||||
[Build Lustre support, optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries])])
|
||||
OMPI_CHECK_WITHDIR([lustre], [$with_lustre], [include/lustre/liblustreapi.h])
|
||||
|
||||
AC_ARG_WITH([lustre-libs],
|
||||
[AC_HELP_STRING([--with-lustre-libs=LIBS],
|
||||
[Libraries to link with for lustre])])
|
||||
|
||||
# Add correct -I and -L flags
|
||||
temp_lustre=$with_lustre
|
||||
AS_IF([test -n "$with_lustre"],
|
||||
[AS_IF([test -d "$with_lustre/include"],
|
||||
[check_lustre_CPPFLAGS="-I$with_lustre/include"
|
||||
CPPFLAGS="$CPPFLAGS $check_lustre_CPPFLAGS"], [])
|
||||
AS_IF([test -d "$with_lustre/lib"],
|
||||
[check_lustre_LDFLAGS="-L$with_lustre/lib"
|
||||
LDFLAGS="$LDFLAGS $check_lustre_LDFLAGS"], [])],
|
||||
with_lustre="/usr/local")
|
||||
|
||||
# Try to find all the lustre libraries (this is not fun!)
|
||||
if test -n "$with_lustre_libs" ; then
|
||||
check_lustre_LIBS="-llustre -llustreapi"
|
||||
for lib in $with_lustre_libs ; do
|
||||
check_lustre_LIBS="$check_lustre_LIBS -l$lib"
|
||||
done
|
||||
fi
|
||||
|
||||
# check for lustre
|
||||
LIBS="$LIBS $check_lustre_LIBS"
|
||||
AC_CHECK_HEADERS([${check_lustre_header_prefix}lustre/liblustreapi.h],
|
||||
[AC_MSG_CHECKING([if possible to link LUSTRE])
|
||||
AC_LINK_IFELSE([AC_LANG_PROGRAM([#include <${check_LUSTRE_header_prefix}lustre.h>],
|
||||
[int i;])],
|
||||
[AC_MSG_RESULT([yes])
|
||||
ompi_check_lustre_happy="yes"],
|
||||
[AC_MSG_RESULT([no])
|
||||
ompi_check_lustre_happy="no"])],
|
||||
[ompi_check_lustre_happy="no"])
|
||||
|
||||
AS_IF([test "$ompi_check_lustre_happy" = "yes"],
|
||||
[$2],
|
||||
[AS_IF([test ! -z "$with_lustre" -a "$with_lustre" != "no"],
|
||||
[echo LUSTRE support not found])
|
||||
$3])
|
||||
with_lustre="$temp_lustre"
|
||||
])
|
||||
91
ompi/config/ompi_check_pvfs2.m4
Обычный файл
91
ompi/config/ompi_check_pvfs2.m4
Обычный файл
@ -0,0 +1,91 @@
|
||||
# -*- shell-script -*-
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2006 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2009 Cisco Systems, Inc. All rights reserved.
|
||||
# Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
# OMPI_CHECK_PVFS2(prefix, [action-if-found], [action-if-not-found])
|
||||
# --------------------------------------------------------
|
||||
# check if PVFS2 support can be found. sets prefix_{CPPFLAGS,
|
||||
# LDFLAGS, LIBS} as needed and runs action-if-found if there is
|
||||
# support, otherwise executes action-if-not-found
|
||||
AC_DEFUN([OMPI_CHECK_PVFS2],[
|
||||
|
||||
check_pvfs2_CPPFLAGS=
|
||||
check_pvfs2_LDFLAGS=
|
||||
check_pvfs2_LIBS=
|
||||
|
||||
check_pvfs2_configuration="none"
|
||||
ompi_check_pvfs2_happy="yes"
|
||||
|
||||
# Get some configuration information
|
||||
AC_ARG_WITH([pvfs2],
|
||||
[AC_HELP_STRING([--with-pvfs2(=DIR)],
|
||||
[Build Pvfs2 support, optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries])])
|
||||
OMPI_CHECK_WITHDIR([pvfs2], [$with_pvfs2], [include/pvfs2.h])
|
||||
|
||||
AC_ARG_WITH([pvfs2-libs],
|
||||
[AC_HELP_STRING([--with-pvfs2-libs=LIBS],
|
||||
[Libraries to link with for pvfs2])])
|
||||
|
||||
# Add correct -I and -L flags
|
||||
temp_pvfs2=$with_pvfs2
|
||||
AS_IF([test -n "$with_pvfs2"],
|
||||
[AS_IF([test -d "$with_pvfs2/include"],
|
||||
[check_pvfs2_CPPFLAGS="-I$with_pvfs2/include"
|
||||
CPPFLAGS="$CPPFLAGS $check_pvfs2_CPPFLAGS"
|
||||
CFLAGS="$CFLAGS $check_pvfs2_CPPFLAGS"
|
||||
WRAPPER_EXTRA_CPPFLAGS="$WRAPPER_EXTRA_CPPFLAGS $check_pvfs2_CPPFLAGS"
|
||||
WRAPPER_EXTRA_CFLAGS="$WRAPPER_EXTRA_CFLAGS $check_pvfs2_CPPFLAGS"
|
||||
], [])
|
||||
AS_IF([test -d "$with_pvfs2/lib"],
|
||||
[check_pvfs2_LDFLAGS="-L$with_pvfs2/lib"
|
||||
LDFLAGS="$LDFLAGS $check_pvfs2_LDFLAGS"
|
||||
WRAPPER_EXTRA_LDFLAGS="$WRAPPER_EXTRA_LDFLAGS $check_pvfs2_LDFLAGS"
|
||||
check_pvfs2_LIBS="-lpvfs2 -lpthread"
|
||||
], [])],
|
||||
with_pvfs2="/usr/local")
|
||||
|
||||
# Try to find all the pvfs2 libraries (this is not fun!)
|
||||
if test -n "$with_pvfs2_libs" ; then
|
||||
for lib in $with_pvfs2_libs ; do
|
||||
check_pvfs2_LIBS="$check_pvfs2_LIBS -l$lib"
|
||||
done
|
||||
fi
|
||||
|
||||
LIBS="$LIBS $check_pvfs2_LIBS"
|
||||
WRAPPER_EXTRA_LIBS="$WRAPPER_EXTRA_LIBS $check_pvfs2_LIBS"
|
||||
|
||||
# check for pvfs2
|
||||
AC_CHECK_HEADERS([${check_pvfs2_header_prefix}pvfs2.h],
|
||||
[AC_MSG_CHECKING([if possible to link PVFS2])
|
||||
AC_LINK_IFELSE([AC_LANG_PROGRAM([#include <${check_PVFS2_header_prefix}pvfs2.h>],
|
||||
[int i;])],
|
||||
[AC_MSG_RESULT([yes])
|
||||
ompi_check_pvfs2_happy="yes"],
|
||||
[AC_MSG_RESULT([no])
|
||||
ompi_check_pvfs2_happy="no"])],
|
||||
[ompi_check_pvfs2_happy="no"])
|
||||
|
||||
AS_IF([test "$ompi_check_pvfs2_happy" = "yes"],
|
||||
[$2],
|
||||
[AS_IF([test ! -z "$with_pvfs2" -a "$with_pvfs2" != "no"],
|
||||
[echo PVFS2 support not found])
|
||||
$3])
|
||||
with_pvfs2="$temp_pvfs2"
|
||||
])
|
||||
40
ompi/mca/fbtl/Makefile.am
Обычный файл
40
ompi/mca/fbtl/Makefile.am
Обычный файл
@ -0,0 +1,40 @@
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
AM_CPPFLAGS = $(LTDLINCL)
|
||||
|
||||
# main library setup
|
||||
noinst_LTLIBRARIES = libmca_fbtl.la
|
||||
libmca_fbtl_la_SOURCES =
|
||||
|
||||
# local files
|
||||
headers = fbtl.h
|
||||
libmca_fbtl_la_SOURCES += $(headers)
|
||||
|
||||
# Conditionally install the header files
|
||||
if WANT_INSTALL_HEADERS
|
||||
ompidir = $(includedir)/openmpi/$(subdir)
|
||||
nobase_ompi_HEADERS = $(headers)
|
||||
endif
|
||||
|
||||
include base/Makefile.am
|
||||
|
||||
distclean-local:
|
||||
rm -f base/static-components.h
|
||||
28
ompi/mca/fbtl/base/Makefile.am
Обычный файл
28
ompi/mca/fbtl/base/Makefile.am
Обычный файл
@ -0,0 +1,28 @@
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
headers += \
|
||||
base/base.h
|
||||
|
||||
libmca_fbtl_la_SOURCES += \
|
||||
base/fbtl_base_close.c \
|
||||
base/fbtl_base_file_select.c \
|
||||
base/fbtl_base_file_unselect.c \
|
||||
base/fbtl_base_find_available.c \
|
||||
base/fbtl_base_open.c
|
||||
70
ompi/mca/fbtl/base/base.h
Обычный файл
70
ompi/mca/fbtl/base/base.h
Обычный файл
@ -0,0 +1,70 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
/**
|
||||
* @file
|
||||
*
|
||||
* MCA fbtl base framework public interface functions.
|
||||
*/
|
||||
|
||||
#ifndef MCA_FBTL_BASE_H
|
||||
#define MCA_FBTL_BASE_H
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "opal/class/opal_list.h"
|
||||
#include "ompi/mca/fbtl/fbtl.h"
|
||||
#include "opal/mca/mca.h"
|
||||
|
||||
|
||||
BEGIN_C_DECLS
|
||||
|
||||
OMPI_DECLSPEC int mca_fbtl_base_open(void);
|
||||
|
||||
OMPI_DECLSPEC int mca_fbtl_base_close(void);
|
||||
|
||||
OMPI_DECLSPEC int mca_fbtl_base_file_select(struct mca_io_ompio_file_t *file,
|
||||
mca_base_component_t *preferred);
|
||||
|
||||
OMPI_DECLSPEC int mca_fbtl_base_file_unselect(struct mca_io_ompio_file_t *file);
|
||||
|
||||
OMPI_DECLSPEC int mca_fbtl_base_find_available(bool enable_progress_threads,
|
||||
bool enable_mpi_threads);
|
||||
|
||||
OMPI_DECLSPEC int mca_fbtl_base_init_file (struct mca_io_ompio_file_t *file);
|
||||
|
||||
OMPI_DECLSPEC int mca_fbtl_base_get_param (struct mca_io_ompio_file_t *file, int keyval);
|
||||
/*
|
||||
* Globals
|
||||
*/
|
||||
|
||||
OMPI_DECLSPEC extern int mca_fbtl_base_param;
|
||||
OMPI_DECLSPEC extern int mca_fbtl_base_output;
|
||||
|
||||
OMPI_DECLSPEC extern bool mca_fbtl_base_components_opened_valid;
|
||||
OMPI_DECLSPEC extern bool mca_fbtl_base_components_available_valid;
|
||||
|
||||
OMPI_DECLSPEC extern opal_list_t mca_fbtl_base_components_opened;
|
||||
OMPI_DECLSPEC extern opal_list_t mca_fbtl_base_components_available;
|
||||
|
||||
END_C_DECLS
|
||||
|
||||
#endif /* MCA_BASE_FBTL_H */
|
||||
56
ompi/mca/fbtl/base/fbtl_base_close.c
Обычный файл
56
ompi/mca/fbtl/base/fbtl_base_close.c
Обычный файл
@ -0,0 +1,56 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHTOB$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include <stdio.h>
|
||||
|
||||
#include "ompi/constants.h"
|
||||
#include "opal/mca/mca.h"
|
||||
#include "opal/mca/base/base.h"
|
||||
#include "ompi/mca/fbtl/fbtl.h"
|
||||
#include "ompi/mca/fbtl/base/base.h"
|
||||
|
||||
int mca_fbtl_base_close(void)
|
||||
{
|
||||
/*
|
||||
Close all components that are still open. This may be the opened
|
||||
list (if we're in ompi_info), or it may be the available list (if
|
||||
we're anywhere else).
|
||||
*/
|
||||
|
||||
if (mca_fbtl_base_components_opened_valid) {
|
||||
mca_base_components_close(mca_fbtl_base_output,
|
||||
&mca_fbtl_base_components_opened, NULL);
|
||||
OBJ_DESTRUCT(&mca_fbtl_base_components_opened);
|
||||
mca_fbtl_base_components_opened_valid = false;
|
||||
} else if (mca_fbtl_base_components_available_valid) {
|
||||
mca_base_components_close(mca_fbtl_base_output,
|
||||
&mca_fbtl_base_components_available, NULL);
|
||||
OBJ_DESTRUCT(&mca_fbtl_base_components_available);
|
||||
mca_fbtl_base_components_available_valid = false;
|
||||
}
|
||||
|
||||
/* Close the output stream for this framework */
|
||||
opal_output_close (mca_fbtl_base_output);
|
||||
|
||||
/* All done */
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
357
ompi/mca/fbtl/base/fbtl_base_file_select.c
Обычный файл
357
ompi/mca/fbtl/base/fbtl_base_file_select.c
Обычный файл
@ -0,0 +1,357 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include <string.h>
|
||||
|
||||
#include "opal/class/opal_list.h"
|
||||
#include "opal/util/argv.h"
|
||||
#include "opal/mca/mca.h"
|
||||
#include "opal/mca/base/base.h"
|
||||
#include "orte/util/show_help.h"
|
||||
#include "ompi/mca/fbtl/fbtl.h"
|
||||
#include "ompi/mca/fbtl/base/base.h"
|
||||
#include "ompi/mca/io/ompio/io_ompio.h"
|
||||
|
||||
/*
|
||||
* This structure is needed so that we can close the modules
|
||||
* which are not selected but were opened. mca_base_modules_close
|
||||
* which does this job for us requires a opal_list_t which contains
|
||||
* these modules
|
||||
*/
|
||||
struct queried_module_t {
|
||||
opal_list_item_t super;
|
||||
mca_fbtl_base_component_t *om_component;
|
||||
mca_fbtl_base_module_t *om_module;
|
||||
};
|
||||
typedef struct queried_module_t queried_module_t;
|
||||
static OBJ_CLASS_INSTANCE(queried_module_t, opal_list_item_t, NULL, NULL);
|
||||
|
||||
|
||||
/*
|
||||
* Only one fbtl module can be attached to each file.
|
||||
*
|
||||
* This module calls the query funtion on all the components that were
|
||||
* detected by fbtl_base_open. This function is called on a
|
||||
* per-file basis. This function has the following function.
|
||||
*
|
||||
* 1. Iterate over the list of available_components
|
||||
* 2. Call the query function on each of these components.
|
||||
* 3. query function returns the structure containing pointers
|
||||
* to its module and its priority
|
||||
* 4. Select the module with the highest priority
|
||||
* 5. Call the init function on the selected module so that it does the
|
||||
* right setup for the file
|
||||
* 6. Call finalize on all the other modules which returned
|
||||
* their module but were unfortunate to not get selected
|
||||
*/
|
||||
|
||||
int mca_fbtl_base_file_select (struct mca_io_ompio_file_t *file,
|
||||
mca_base_component_t *preferred)
|
||||
{
|
||||
int priority;
|
||||
int best_priority;
|
||||
opal_list_item_t *item;
|
||||
opal_list_item_t *next_item;
|
||||
mca_base_component_priority_list_item_t *selectable_item;
|
||||
char *names, **name_array;
|
||||
int num_names;
|
||||
mca_base_component_priority_list_item_t *cpli;
|
||||
mca_fbtl_base_component_t *component;
|
||||
mca_fbtl_base_component_t *best_component;
|
||||
mca_fbtl_base_module_t *module;
|
||||
opal_list_t queried;
|
||||
queried_module_t *om;
|
||||
opal_list_t *selectable;
|
||||
char *str;
|
||||
int err = MPI_SUCCESS;
|
||||
int i;
|
||||
bool was_selectable_constructed = false;
|
||||
|
||||
/* Check and see if a preferred component was provided. If it was
|
||||
provided then it should be used (if possible) */
|
||||
|
||||
if (NULL != preferred) {
|
||||
|
||||
/* We have a preferred component. Check if it is available
|
||||
and if so, whether it wants to run */
|
||||
|
||||
str = &(preferred->mca_component_name[0]);
|
||||
|
||||
opal_output_verbose(10, mca_fbtl_base_output,
|
||||
"fbtl:base:file_select: Checking preferred component: %s",
|
||||
str);
|
||||
|
||||
/* query the component for its priority and get its module
|
||||
structure. This is necessary to proceed */
|
||||
|
||||
component = (mca_fbtl_base_component_t *)preferred;
|
||||
module = component->fbtlm_file_query (file, &priority);
|
||||
if (NULL != module &&
|
||||
NULL != module->fbtl_module_init) {
|
||||
|
||||
/* this query seems to have returned something legitimate
|
||||
* and we can now go ahead and initialize the
|
||||
* file with it * but first, the functions which
|
||||
* are null need to be filled in */
|
||||
|
||||
/*fill_null_pointers (module);*/
|
||||
file->f_fbtl = module;
|
||||
file->f_fbtl_component = preferred;
|
||||
|
||||
return module->fbtl_module_init(file);
|
||||
}
|
||||
/* His preferred component is present, but is unable to
|
||||
* run. This is not a good sign. We should try selecting
|
||||
* some other component We let it fall through and select
|
||||
* from the list of available components
|
||||
*/
|
||||
} /*end of selection for preferred component */
|
||||
|
||||
/*
|
||||
* We fall till here if one of the two things happened:
|
||||
* 1. The preferred component was provided but for some reason was
|
||||
* not able to be selected
|
||||
* 2. No preferred component was provided
|
||||
*
|
||||
* All we need to do is to go through the list of available
|
||||
* components and find the one which has the highest priority and
|
||||
* use that for this file
|
||||
*/
|
||||
|
||||
/* Check if anything was requested by means on the name parameters */
|
||||
names = NULL;
|
||||
mca_base_param_lookup_string (mca_fbtl_base_param, &names);
|
||||
|
||||
if (NULL != names && 0 < strlen(names)) {
|
||||
name_array = opal_argv_split (names, ',');
|
||||
num_names = opal_argv_count (name_array);
|
||||
|
||||
opal_output_verbose(10, mca_fbtl_base_output,
|
||||
"fbtl:base:file_Select: Checking all available module");
|
||||
|
||||
/* since there are somethings which the mca requested through the
|
||||
if the intersection is NULL, then we barf saying that the requested
|
||||
modules are not being available */
|
||||
|
||||
selectable = OBJ_NEW(opal_list_t);
|
||||
was_selectable_constructed = true;
|
||||
|
||||
/* go through the compoents_available list and check against the names
|
||||
* to see whether this can be added or not */
|
||||
|
||||
for (item = opal_list_get_first(&mca_fbtl_base_components_available);
|
||||
item != opal_list_get_end(&mca_fbtl_base_components_available);
|
||||
item = opal_list_get_next(item)) {
|
||||
/* convert the opal_list_item_t returned into the proper type */
|
||||
cpli = (mca_base_component_priority_list_item_t *) item;
|
||||
component = (mca_fbtl_base_component_t *) cpli->super.cli_component;
|
||||
opal_output_verbose(10, mca_fbtl_base_output,
|
||||
"select: initialising %s component %s",
|
||||
component->fbtlm_version.mca_type_name,
|
||||
component->fbtlm_version.mca_component_name);
|
||||
|
||||
/* check if this name is present in the mca_base_params */
|
||||
for (i=0; i < num_names; i++) {
|
||||
if (0 == strcmp(name_array[i], component->fbtlm_version.mca_component_name)) {
|
||||
/* this is present, and should be added o the selectable list */
|
||||
|
||||
/* We need to create a seperate object to initialise this list with
|
||||
* since we cannot have the same item in 2 lists */
|
||||
|
||||
selectable_item = OBJ_NEW (mca_base_component_priority_list_item_t);
|
||||
*selectable_item = *cpli;
|
||||
opal_list_append (selectable, (opal_list_item_t *)selectable_item);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* check for a NULL intersection between the available list and the
|
||||
* list which was asked for */
|
||||
|
||||
if (0 == opal_list_get_size(selectable)) {
|
||||
was_selectable_constructed = true;
|
||||
OBJ_RELEASE (selectable);
|
||||
opal_output_verbose (10, mca_fbtl_base_output,
|
||||
"fbtl:base:file_select: preferred modules were not available");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
} else { /* if there was no name_array, then we need to simply initialize
|
||||
selectable to mca_fbtl_base_components_available */
|
||||
selectable = &mca_fbtl_base_components_available;
|
||||
}
|
||||
|
||||
best_component = NULL;
|
||||
best_priority = -1;
|
||||
OBJ_CONSTRUCT(&queried, opal_list_t);
|
||||
|
||||
for (item = opal_list_get_first(selectable);
|
||||
item != opal_list_get_end(selectable);
|
||||
item = opal_list_get_next(item)) {
|
||||
/*
|
||||
* convert the opal_list_item_t returned into the proper type
|
||||
*/
|
||||
cpli = (mca_base_component_priority_list_item_t *) item;
|
||||
component = (mca_fbtl_base_component_t *) cpli->super.cli_component;
|
||||
opal_output_verbose(10, mca_fbtl_base_output,
|
||||
"select: initialising %s component %s",
|
||||
component->fbtlm_version.mca_type_name,
|
||||
component->fbtlm_version.mca_component_name);
|
||||
|
||||
/*
|
||||
* we can call the query function only if there is a function :-)
|
||||
*/
|
||||
if (NULL == component->fbtlm_file_query) {
|
||||
opal_output_verbose(10, mca_fbtl_base_output,
|
||||
"select: no query, ignoring the component");
|
||||
} else {
|
||||
/*
|
||||
* call the query function and see what it returns
|
||||
*/
|
||||
module = component->fbtlm_file_query (file, &priority);
|
||||
|
||||
if (NULL == module ||
|
||||
NULL == module->fbtl_module_init) {
|
||||
/*
|
||||
* query did not return any action which can be used
|
||||
*/
|
||||
opal_output_verbose(10, mca_fbtl_base_output,
|
||||
"select: query returned failure");
|
||||
} else {
|
||||
opal_output_verbose(10, mca_fbtl_base_output,
|
||||
"select: query returned priority %d",
|
||||
priority);
|
||||
/*
|
||||
* is this the best component we have found till now?
|
||||
*/
|
||||
if (priority > best_priority) {
|
||||
best_priority = priority;
|
||||
best_component = component;
|
||||
}
|
||||
|
||||
om = OBJ_NEW(queried_module_t);
|
||||
/*
|
||||
* check if we have run out of space
|
||||
*/
|
||||
if (NULL == om) {
|
||||
OBJ_DESTRUCT(&queried);
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
om->om_component = component;
|
||||
om->om_module = module;
|
||||
opal_list_append(&queried, (opal_list_item_t *)om);
|
||||
} /* end else of if (NULL == module) */
|
||||
} /* end else of if (NULL == component->fbtlm_init) */
|
||||
} /* end for ... end of traversal */
|
||||
|
||||
/* We have to remove empty out the selectable list if the selectable
|
||||
* list was constructed as a duplicate and not as a pointer to the
|
||||
* mca_base_components_available list. So, check and destroy */
|
||||
|
||||
if (was_selectable_constructed) {
|
||||
|
||||
/* remove all the items first */
|
||||
for (item = opal_list_get_first(&mca_fbtl_base_components_available);
|
||||
item != opal_list_get_end(&mca_fbtl_base_components_available);
|
||||
item = next_item) {
|
||||
next_item = opal_list_get_next(item);
|
||||
OBJ_RELEASE (item);
|
||||
}
|
||||
|
||||
/* release the list itself */
|
||||
OBJ_RELEASE (selectable);
|
||||
was_selectable_constructed = false;
|
||||
}
|
||||
|
||||
/*
|
||||
* Now we have alist of components which successfully returned
|
||||
* their module struct. One of these components has the best
|
||||
* priority. The rest have to be comm_unqueried to counter the
|
||||
* effects of file_query'ing them. Finalize happens only on
|
||||
* components which should are initialized.
|
||||
*/
|
||||
if (NULL == best_component) {
|
||||
/*
|
||||
* This typically means that there was no component which was
|
||||
* able to run properly this time. So, we need to abort
|
||||
* JMS replace with show_help
|
||||
*/
|
||||
OBJ_DESTRUCT(&queried);
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
/*
|
||||
* We now have a list of components which have successfully
|
||||
* returned their priorities from the query. We now have to
|
||||
* unquery() those components which have not been selected and
|
||||
* init() the component which was selected
|
||||
*/
|
||||
for (item = opal_list_remove_first(&queried);
|
||||
NULL != item;
|
||||
item = opal_list_remove_first(&queried)) {
|
||||
om = (queried_module_t *) item;
|
||||
if (om->om_component == best_component) {
|
||||
/*
|
||||
* this is the chosen component, we have to initialise the
|
||||
* module of this component.
|
||||
*
|
||||
* ANJU: a component might not have all the functions
|
||||
* defined. Whereever a function pointer is null in the
|
||||
* module structure we need to fill it in with the base
|
||||
* structure function pointers. This is yet to be done
|
||||
*/
|
||||
|
||||
/*
|
||||
* We don return here coz we still need to go through and
|
||||
* elease the other objects
|
||||
*/
|
||||
|
||||
/*fill_null_pointers (om->om_module);*/
|
||||
file->f_fbtl = om->om_module;
|
||||
err = om->om_module->fbtl_module_init(file);
|
||||
file->f_fbtl_component = (mca_base_component_t *)best_component;
|
||||
} else {
|
||||
/*
|
||||
* this is not the "choosen one", finalize
|
||||
*/
|
||||
if (NULL != om->om_component->fbtlm_file_unquery) {
|
||||
/* unquery the component only if they have some clean
|
||||
* up job to do. Components which are queried but do
|
||||
* not actually do anything typically do not have a
|
||||
* unquery. Hence this check is necessary
|
||||
*/
|
||||
(void) om->om_component->fbtlm_file_unquery(file);
|
||||
opal_output_verbose(10, mca_fbtl_base_output,
|
||||
"select: component %s is not selected",
|
||||
om->om_component->fbtlm_version.mca_component_name);
|
||||
} /* end if */
|
||||
} /* if not best component */
|
||||
OBJ_RELEASE(om);
|
||||
} /* traversing through the entire list */
|
||||
|
||||
opal_output_verbose(10, mca_fbtl_base_output,
|
||||
"select: component %s selected",
|
||||
best_component->fbtlm_version.mca_component_name);
|
||||
|
||||
OBJ_DESTRUCT(&queried);
|
||||
|
||||
return err;
|
||||
}
|
||||
41
ompi/mca/fbtl/base/fbtl_base_file_unselect.c
Обычный файл
41
ompi/mca/fbtl/base/fbtl_base_file_unselect.c
Обычный файл
@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/mca/io/ompio/io_ompio.h"
|
||||
#include "opal/util/show_help.h"
|
||||
#include "opal/mca/mca.h"
|
||||
#include "opal/mca/base/base.h"
|
||||
#include "ompi/mca/fbtl/fbtl.h"
|
||||
#include "ompi/mca/fbtl/base/base.h"
|
||||
|
||||
int mca_fbtl_base_file_unselect(mca_io_ompio_file_t *file)
|
||||
{
|
||||
if (NULL != file->f_fbtl && NULL != file->f_fbtl->fbtl_module_finalize) {
|
||||
return file->f_fbtl->fbtl_module_finalize(file);
|
||||
}
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
171
ompi/mca/fbtl/base/fbtl_base_find_available.c
Обычный файл
171
ompi/mca/fbtl/base/fbtl_base_find_available.c
Обычный файл
@ -0,0 +1,171 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "opal/class/opal_list.h"
|
||||
#include "orte/util/show_help.h"
|
||||
#include "opal/mca/mca.h"
|
||||
#include "opal/mca/base/base.h"
|
||||
#include "opal/mca/base/mca_base_component_repository.h"
|
||||
#include "ompi/mca/fbtl/fbtl.h"
|
||||
#include "ompi/mca/fbtl/base/base.h"
|
||||
|
||||
opal_list_t mca_fbtl_base_modules_available;
|
||||
bool mca_fbtl_base_modules_available_valid = false;
|
||||
|
||||
static int init_query(const mca_base_component_t *m,
|
||||
mca_base_component_priority_list_item_t *entry,
|
||||
bool enable_progress_threads,
|
||||
bool enable_mpi_threads);
|
||||
static int init_query_2_0_0(const mca_base_component_t *component,
|
||||
mca_base_component_priority_list_item_t *entry,
|
||||
bool enable_progress_threads,
|
||||
bool enable_mpi_threads);
|
||||
|
||||
int mca_fbtl_base_find_available(bool enable_progress_threads,
|
||||
bool enable_mpi_threads)
|
||||
{
|
||||
bool found = false;
|
||||
mca_base_component_priority_list_item_t *entry;
|
||||
opal_list_item_t *p;
|
||||
|
||||
/* Initialize the list */
|
||||
|
||||
OBJ_CONSTRUCT(&mca_fbtl_base_components_available, opal_list_t);
|
||||
mca_fbtl_base_components_available_valid = true;
|
||||
|
||||
/* The list of components which we should check is already present
|
||||
in mca_fbtl_base_components_opened, which was established in
|
||||
mca_fbtl_base_open */
|
||||
|
||||
for (found = false,
|
||||
p = opal_list_remove_first (&mca_fbtl_base_components_opened);
|
||||
NULL != p;
|
||||
p = opal_list_remove_first (&mca_fbtl_base_components_opened)) {
|
||||
entry = OBJ_NEW(mca_base_component_priority_list_item_t);
|
||||
entry->super.cli_component =
|
||||
((mca_base_component_list_item_t *)p)->cli_component;
|
||||
|
||||
/* Now for this entry, we have to determine the thread level. Call
|
||||
a subroutine to do the job for us */
|
||||
|
||||
if (OMPI_SUCCESS == init_query(entry->super.cli_component, entry,
|
||||
enable_progress_threads,
|
||||
enable_mpi_threads)) {
|
||||
/* Save the results in the list. The priority is not relvant at
|
||||
this point in time. But we save the thread arguments so that
|
||||
the initial selection algorithm can negotiate overall thread
|
||||
level for this process */
|
||||
entry->cpli_priority = 0;
|
||||
opal_list_append (&mca_fbtl_base_components_available,
|
||||
(opal_list_item_t *) entry);
|
||||
found = true;
|
||||
} else {
|
||||
/* The component does not want to run, so close it. Its close()
|
||||
has already been invoked. Close it out of the DSO repository
|
||||
(if it is there in the repository) */
|
||||
mca_base_component_repository_release(entry->super.cli_component);
|
||||
OBJ_RELEASE(entry);
|
||||
}
|
||||
/* Free entry from the "opened" list */
|
||||
OBJ_RELEASE(p);
|
||||
}
|
||||
|
||||
/* The opened list is no longer necessary, so we can free it */
|
||||
OBJ_DESTRUCT (&mca_fbtl_base_components_opened);
|
||||
mca_fbtl_base_components_opened_valid = false;
|
||||
|
||||
/* There should atleast be one fbtl component which was available */
|
||||
if (false == found) {
|
||||
/* Need to free all items in the list */
|
||||
OBJ_DESTRUCT(&mca_fbtl_base_components_available);
|
||||
mca_fbtl_base_components_available_valid = false;
|
||||
opal_output_verbose (10, mca_fbtl_base_output,
|
||||
"fbtl:find_available: no fbtl components available!");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
/* All done */
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
static int init_query(const mca_base_component_t *m,
|
||||
mca_base_component_priority_list_item_t *entry,
|
||||
bool enable_progress_threads,
|
||||
bool enable_mpi_threads)
|
||||
{
|
||||
int ret;
|
||||
|
||||
opal_output_verbose(10, mca_fbtl_base_output,
|
||||
"fbtl:find_available: querying fbtl component %s",
|
||||
m->mca_component_name);
|
||||
|
||||
/* This component has been successfully opened, now try to query it */
|
||||
if (2 == m->mca_type_major_version &&
|
||||
0 == m->mca_type_minor_version &&
|
||||
0 == m->mca_type_release_version) {
|
||||
ret = init_query_2_0_0(m, entry, enable_progress_threads,
|
||||
enable_mpi_threads);
|
||||
} else {
|
||||
/* unrecognised API version */
|
||||
opal_output_verbose(10, mca_fbtl_base_output,
|
||||
"fbtl:find_available:unrecognised fbtl API version (%d.%d.%d)",
|
||||
m->mca_type_major_version,
|
||||
m->mca_type_minor_version,
|
||||
m->mca_type_release_version);
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
/* Query done -- look at return value to see what happened */
|
||||
if (OMPI_SUCCESS != ret) {
|
||||
opal_output_verbose(10, mca_fbtl_base_output,
|
||||
"fbtl:find_available fbtl component %s is not available",
|
||||
m->mca_component_name);
|
||||
if (NULL != m->mca_close_component) {
|
||||
m->mca_close_component();
|
||||
}
|
||||
} else {
|
||||
opal_output_verbose(10, mca_fbtl_base_output,
|
||||
"fbtl:find_avalable: fbtl component %s is available",
|
||||
m->mca_component_name);
|
||||
|
||||
}
|
||||
/* All done */
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
static int init_query_2_0_0(const mca_base_component_t *component,
|
||||
mca_base_component_priority_list_item_t *entry,
|
||||
bool enable_progress_threads,
|
||||
bool enable_mpi_threads)
|
||||
{
|
||||
mca_fbtl_base_component_2_0_0_t *fbtl =
|
||||
(mca_fbtl_base_component_2_0_0_t *) component;
|
||||
|
||||
return fbtl->fbtlm_init_query(enable_progress_threads,
|
||||
enable_mpi_threads);
|
||||
}
|
||||
88
ompi/mca/fbtl/base/fbtl_base_open.c
Обычный файл
88
ompi/mca/fbtl/base/fbtl_base_open.c
Обычный файл
@ -0,0 +1,88 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include <stdio.h>
|
||||
|
||||
#include "ompi/class/ompi_free_list.h"
|
||||
#include "orte/util/show_help.h"
|
||||
#include "opal/mca/mca.h"
|
||||
#include "opal/mca/base/base.h"
|
||||
#include "opal/mca/base/mca_base_param.h"
|
||||
|
||||
#include "orte/util/show_help.h"
|
||||
|
||||
#include "ompi/mca/fbtl/fbtl.h"
|
||||
#include "ompi/mca/fbtl/base/base.h"
|
||||
|
||||
|
||||
/*
|
||||
* The following file was created by configure. It contains extern
|
||||
* statements and the definition of an array of pointers to each
|
||||
* component's public mca_base_component_t struct.
|
||||
*/
|
||||
#ifdef __WINDOWS__
|
||||
const mca_base_component_t *mca_fbtl_base_static_components[] = {NULL};
|
||||
#else
|
||||
#include "ompi/mca/fbtl/base/static-components.h"
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Global variables; most of which are loaded by back-ends of MCA
|
||||
* variables
|
||||
*/
|
||||
int mca_fbtl_base_param = -1;
|
||||
int mca_fbtl_base_output = -1;
|
||||
|
||||
opal_list_t mca_fbtl_base_components_opened;
|
||||
opal_list_t mca_fbtl_base_components_available;
|
||||
|
||||
bool mca_fbtl_base_components_available_valid = false;
|
||||
bool mca_fbtl_base_components_opened_valid = false;
|
||||
|
||||
mca_fbtl_base_component_t mca_fbtl_base_selected_component;
|
||||
mca_fbtl_base_module_t mca_fbtl;
|
||||
|
||||
/*
|
||||
* Function for finding and opening either all MCA components, or the one
|
||||
* that was specifically requested via a MCA parameter.
|
||||
*/
|
||||
int mca_fbtl_base_open(void)
|
||||
{
|
||||
/* Open an output stream for this framework */
|
||||
|
||||
mca_fbtl_base_output = opal_output_open(NULL);
|
||||
|
||||
/* Open up all available components */
|
||||
|
||||
if (OMPI_SUCCESS !=
|
||||
mca_base_components_open("fbtl", mca_fbtl_base_output,
|
||||
mca_fbtl_base_static_components,
|
||||
&mca_fbtl_base_components_opened, true)) {
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
mca_fbtl_base_components_opened_valid = true;
|
||||
|
||||
/* Find the index of the MCA "fbtl" param for selection */
|
||||
|
||||
mca_fbtl_base_param = mca_base_param_find("fbtl", "base", NULL);
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
18
ompi/mca/fbtl/base/static-components.h
Обычный файл
18
ompi/mca/fbtl/base/static-components.h
Обычный файл
@ -0,0 +1,18 @@
|
||||
/*
|
||||
* $HEADER$
|
||||
*/
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
const mca_base_component_t *mca_fbtl_base_static_components[] = {
|
||||
|
||||
NULL
|
||||
};
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
151
ompi/mca/fbtl/fbtl.h
Обычный файл
151
ompi/mca/fbtl/fbtl.h
Обычный файл
@ -0,0 +1,151 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#ifndef OMPI_MCA_FBTL_H
|
||||
#define OMPI_MCA_FBTL_H
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "mpi.h"
|
||||
#include "opal/mca/mca.h"
|
||||
#include "opal/mca/base/base.h"
|
||||
#include "ompi/request/request.h"
|
||||
#include <sys/uio.h>
|
||||
|
||||
BEGIN_C_DECLS
|
||||
|
||||
struct mca_io_ompio_file_t;
|
||||
|
||||
/*
|
||||
* Macro for use in components that are of type coll
|
||||
*/
|
||||
#define MCA_FBTL_BASE_VERSION_2_0_0 \
|
||||
MCA_BASE_VERSION_2_0_0, \
|
||||
"fbtl", 2, 0, 0
|
||||
|
||||
/*
|
||||
* These are the component function prototypes. These function pointers
|
||||
* go into the component structure. These functions (query() and finalize()
|
||||
* are called during fbtl_base_select(). Each component is query() ied
|
||||
* and subsequently, all the unselected components are finalize() 'ed
|
||||
* so that any *stuff* they did during query() can be undone. By
|
||||
* similar logic, finalize() is also called on the component which
|
||||
* was selected when the communicator is being destroyed.
|
||||
*
|
||||
* So, to sum it up, every component carries 4 functions:
|
||||
* 1. open() - called during MPI_INIT
|
||||
* 2. close() - called during MPI_FINALIZE
|
||||
* 3. query() - called to select a particular component
|
||||
* 4. finalize() - called when actions taken during query have
|
||||
* to be undone
|
||||
*/
|
||||
|
||||
/*
|
||||
* **************** component struct *******************************
|
||||
* *********** These functions go in the component struct **********
|
||||
* **************** component struct *******************************
|
||||
*/
|
||||
|
||||
typedef int (*mca_fbtl_base_component_init_query_1_0_0_fn_t)
|
||||
(bool enable_progress_threads,
|
||||
bool enable_mpi_threads);
|
||||
|
||||
typedef struct mca_fbtl_base_module_1_0_0_t *
|
||||
(*mca_fbtl_base_component_file_query_1_0_0_fn_t) (struct mca_io_ompio_file_t *file,
|
||||
int *priority);
|
||||
|
||||
typedef int (*mca_fbtl_base_component_file_unquery_1_0_0_fn_t)
|
||||
(struct mca_io_ompio_file_t *file);
|
||||
|
||||
/*
|
||||
* ****************** component struct ******************************
|
||||
* Structure for fbtl v2.0.0 components.This is chained to MCA v2.0.0
|
||||
* ****************** component struct ******************************
|
||||
*/
|
||||
struct mca_fbtl_base_component_2_0_0_t {
|
||||
mca_base_component_t fbtlm_version;
|
||||
mca_base_component_data_t fbtlm_data;
|
||||
|
||||
mca_fbtl_base_component_init_query_1_0_0_fn_t fbtlm_init_query;
|
||||
mca_fbtl_base_component_file_query_1_0_0_fn_t fbtlm_file_query;
|
||||
mca_fbtl_base_component_file_unquery_1_0_0_fn_t fbtlm_file_unquery;
|
||||
};
|
||||
typedef struct mca_fbtl_base_component_2_0_0_t mca_fbtl_base_component_2_0_0_t;
|
||||
typedef struct mca_fbtl_base_component_2_0_0_t mca_fbtl_base_component_t;
|
||||
|
||||
/*
|
||||
* ***********************************************************************
|
||||
* ************************ Interface function definitions **************
|
||||
* These are the typedefbtl for the function pointers to various fbtl
|
||||
* backend functions which will be used by the various fbtl components
|
||||
* ***********************************************************************
|
||||
*/
|
||||
|
||||
typedef int (*mca_fbtl_base_module_init_1_0_0_fn_t)
|
||||
(struct mca_io_ompio_file_t *file);
|
||||
|
||||
typedef int (*mca_fbtl_base_module_finalize_1_0_0_fn_t)
|
||||
(struct mca_io_ompio_file_t *file);
|
||||
|
||||
|
||||
typedef size_t (*mca_fbtl_base_module_preadv_fn_t)
|
||||
(struct mca_io_ompio_file_t *file,
|
||||
int *sorted);
|
||||
typedef size_t (*mca_fbtl_base_module_pwritev_fn_t)
|
||||
(struct mca_io_ompio_file_t *file,
|
||||
int *sorted);
|
||||
typedef size_t (*mca_fbtl_base_module_ipreadv_fn_t)
|
||||
(struct mca_io_ompio_file_t *file,
|
||||
int *sorted,
|
||||
ompi_request_t **request);
|
||||
typedef size_t (*mca_fbtl_base_module_ipwritev_fn_t)
|
||||
(struct mca_io_ompio_file_t *file,
|
||||
int *sorted,
|
||||
ompi_request_t **request);
|
||||
|
||||
/*
|
||||
* ***********************************************************************
|
||||
* *************************** module structure *************************
|
||||
* ***********************************************************************
|
||||
*/
|
||||
struct mca_fbtl_base_module_1_0_0_t {
|
||||
/*
|
||||
* Per-file initialization function. This is called only
|
||||
* on the module which is selected. The finalize corresponding to
|
||||
* this function is present on the component struct above
|
||||
*/
|
||||
mca_fbtl_base_module_init_1_0_0_fn_t fbtl_module_init;
|
||||
mca_fbtl_base_module_finalize_1_0_0_fn_t fbtl_module_finalize;
|
||||
|
||||
/* FBTL function pointers */
|
||||
mca_fbtl_base_module_preadv_fn_t fbtl_preadv;
|
||||
mca_fbtl_base_module_ipreadv_fn_t fbtl_ipreadv;
|
||||
mca_fbtl_base_module_pwritev_fn_t fbtl_pwritev;
|
||||
mca_fbtl_base_module_ipwritev_fn_t fbtl_ipwritev;
|
||||
/*
|
||||
mca_fbtl_base_module_test_fn_t fbtl_test;
|
||||
mca_fbtl_base_module_wait_fn_t fbtl_wait;
|
||||
mca_fbtl_base_module_progress_fn_t fbtl_progress;
|
||||
*/
|
||||
};
|
||||
typedef struct mca_fbtl_base_module_1_0_0_t mca_fbtl_base_module_1_0_0_t;
|
||||
typedef mca_fbtl_base_module_1_0_0_t mca_fbtl_base_module_t;
|
||||
|
||||
END_C_DECLS
|
||||
|
||||
#endif /* OMPI_MCA_FBTL_H */
|
||||
50
ompi/mca/fbtl/posix/Makefile.am
Обычный файл
50
ompi/mca/fbtl/posix/Makefile.am
Обычный файл
@ -0,0 +1,50 @@
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
# Make the output library in this directory, and name it either
|
||||
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
|
||||
# (for static builds).
|
||||
|
||||
if MCA_BUILD_ompi_fbtl_posix_DSO
|
||||
component_noinst =
|
||||
component_install = mca_fbtl_posix.la
|
||||
else
|
||||
component_noinst = libmca_fbtl_posix.la
|
||||
component_install =
|
||||
endif
|
||||
|
||||
mcacomponentdir = $(pkglibdir)
|
||||
mcacomponent_LTLIBRARIES = $(component_install)
|
||||
mca_fbtl_posix_la_SOURCES = $(sources)
|
||||
mca_fbtl_posix_la_LDFLAGS = -module -avoid-version
|
||||
|
||||
noinst_LTLIBRARIES = $(component_noinst)
|
||||
libmca_fbtl_posix_la_SOURCES = $(sources)
|
||||
libmca_fbtl_posix_la_LDFLAGS = -module -avoid-version
|
||||
|
||||
# Source files
|
||||
|
||||
sources = \
|
||||
fbtl_posix.h \
|
||||
fbtl_posix.c \
|
||||
fbtl_posix_component.c \
|
||||
fbtl_posix_preadv.c \
|
||||
fbtl_posix_ipreadv.c \
|
||||
fbtl_posix_pwritev.c \
|
||||
fbtl_posix_ipwritev.c
|
||||
85
ompi/mca/fbtl/posix/fbtl_posix.c
Обычный файл
85
ompi/mca/fbtl/posix/fbtl_posix.c
Обычный файл
@ -0,0 +1,85 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2006 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*
|
||||
* These symbols are in a file by themselves to provide nice linker
|
||||
* semantics. Since linkers generally pull in symbols by object fules,
|
||||
* keeping these symbols as the only symbols in this file prevents
|
||||
* utility programs such as "ompi_info" from having to import entire
|
||||
* modules just to query their version and parameters
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "mpi.h"
|
||||
#include "ompi/mca/fbtl/fbtl.h"
|
||||
#include "ompi/mca/fbtl/posix/fbtl_posix.h"
|
||||
|
||||
/*
|
||||
* *******************************************************************
|
||||
* ************************ actions structure ************************
|
||||
* *******************************************************************
|
||||
*/
|
||||
static mca_fbtl_base_module_1_0_0_t posix = {
|
||||
mca_fbtl_posix_module_init, /* initalise after being selected */
|
||||
mca_fbtl_posix_module_finalize, /* close a module on a communicator */
|
||||
mca_fbtl_posix_preadv,
|
||||
mca_fbtl_posix_ipreadv,
|
||||
mca_fbtl_posix_pwritev,
|
||||
mca_fbtl_posix_ipwritev
|
||||
};
|
||||
/*
|
||||
* *******************************************************************
|
||||
* ************************* structure ends **************************
|
||||
* *******************************************************************
|
||||
*/
|
||||
|
||||
int mca_fbtl_posix_component_init_query(bool enable_progress_threads,
|
||||
bool enable_mpi_threads) {
|
||||
/* Nothing to do */
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
struct mca_fbtl_base_module_1_0_0_t *
|
||||
mca_fbtl_posix_component_file_query (mca_io_ompio_file_t *fh, int *priority) {
|
||||
*priority = mca_fbtl_posix_priority;
|
||||
|
||||
if (UFS == fh->f_fstype) {
|
||||
if (*priority < 50) {
|
||||
*priority = 50;
|
||||
}
|
||||
}
|
||||
|
||||
return &posix;
|
||||
}
|
||||
|
||||
int mca_fbtl_posix_component_file_unquery (mca_io_ompio_file_t *file) {
|
||||
/* This function might be needed for some purposes later. for now it
|
||||
* does not have anything to do since there are no steps which need
|
||||
* to be undone if this module is not selected */
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
int mca_fbtl_posix_module_init (mca_io_ompio_file_t *file) {
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
int mca_fbtl_posix_module_finalize (mca_io_ompio_file_t *file) {
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
67
ompi/mca/fbtl/posix/fbtl_posix.h
Обычный файл
67
ompi/mca/fbtl/posix/fbtl_posix.h
Обычный файл
@ -0,0 +1,67 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2006 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#ifndef MCA_FBTL_POSIX_H
|
||||
#define MCA_FBTL_POSIX_H
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "opal/mca/mca.h"
|
||||
#include "ompi/mca/fbtl/fbtl.h"
|
||||
#include "ompi/mca/io/ompio/io_ompio.h"
|
||||
|
||||
extern int mca_fbtl_posix_priority;
|
||||
|
||||
BEGIN_C_DECLS
|
||||
|
||||
int mca_fbtl_posix_component_init_query(bool enable_progress_threads,
|
||||
bool enable_mpi_threads);
|
||||
struct mca_fbtl_base_module_1_0_0_t *
|
||||
mca_fbtl_posix_component_file_query (mca_io_ompio_file_t *file, int *priority);
|
||||
int mca_fbtl_posix_component_file_unquery (mca_io_ompio_file_t *file);
|
||||
|
||||
int mca_fbtl_posix_module_init (mca_io_ompio_file_t *file);
|
||||
int mca_fbtl_posix_module_finalize (mca_io_ompio_file_t *file);
|
||||
|
||||
OMPI_MODULE_DECLSPEC extern mca_fbtl_base_component_2_0_0_t mca_fbtl_posix_component;
|
||||
/*
|
||||
* ******************************************************************
|
||||
* ********* functions which are implemented in this module *********
|
||||
* ******************************************************************
|
||||
*/
|
||||
|
||||
size_t mca_fbtl_posix_preadv (mca_io_ompio_file_t *file,
|
||||
int *sorted);
|
||||
size_t mca_fbtl_posix_pwritev (mca_io_ompio_file_t *file,
|
||||
int *sorted);
|
||||
size_t mca_fbtl_posix_ipreadv (mca_io_ompio_file_t *file,
|
||||
int *sorted,
|
||||
ompi_request_t **request);
|
||||
size_t mca_fbtl_posix_ipwritev (mca_io_ompio_file_t *file,
|
||||
int *sorted,
|
||||
ompi_request_t **request);
|
||||
|
||||
/*
|
||||
* ******************************************************************
|
||||
* ************ functions implemented in this module end ************
|
||||
* ******************************************************************
|
||||
*/
|
||||
|
||||
END_C_DECLS
|
||||
|
||||
#endif /* MCA_FBTL_POSIX_H */
|
||||
65
ompi/mca/fbtl/posix/fbtl_posix_component.c
Обычный файл
65
ompi/mca/fbtl/posix/fbtl_posix_component.c
Обычный файл
@ -0,0 +1,65 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*
|
||||
* These symbols are in a file by themselves to provide nice linker
|
||||
* semantics. Since linkers generally pull in symbols by object
|
||||
* files, keeping these symbols as the only symbols in this file
|
||||
* prevents utility programs such as "ompi_info" from having to import
|
||||
* entire components just to query their version and parameters.
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fbtl_posix.h"
|
||||
#include "mpi.h"
|
||||
|
||||
/*
|
||||
* Public string showing the fbtl posix component version number
|
||||
*/
|
||||
const char *mca_fbtl_posix_component_version_string =
|
||||
"OMPI/MPI posix FBTL MCA component version " OMPI_VERSION;
|
||||
|
||||
int mca_fbtl_posix_priority = 10;
|
||||
|
||||
/*
|
||||
* Instantiate the public struct with all of our public information
|
||||
* and pointers to our public functions in it
|
||||
*/
|
||||
mca_fbtl_base_component_2_0_0_t mca_fbtl_posix_component = {
|
||||
|
||||
/* First, the mca_component_t struct containing meta information
|
||||
about the component itself */
|
||||
|
||||
{
|
||||
MCA_FBTL_BASE_VERSION_2_0_0,
|
||||
|
||||
/* Component name and version */
|
||||
"posix",
|
||||
OMPI_MAJOR_VERSION,
|
||||
OMPI_MINOR_VERSION,
|
||||
OMPI_RELEASE_VERSION,
|
||||
NULL,
|
||||
NULL
|
||||
},
|
||||
{
|
||||
/* This component is checkpointable */
|
||||
MCA_BASE_METADATA_PARAM_CHECKPOINT
|
||||
},
|
||||
mca_fbtl_posix_component_init_query, /* get thread level */
|
||||
mca_fbtl_posix_component_file_query, /* get priority and actions */
|
||||
mca_fbtl_posix_component_file_unquery /* undo what was done by previous function */
|
||||
};
|
||||
36
ompi/mca/fbtl/posix/fbtl_posix_ipreadv.c
Обычный файл
36
ompi/mca/fbtl/posix/fbtl_posix_ipreadv.c
Обычный файл
@ -0,0 +1,36 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fbtl_posix.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "orte/util/show_help.h"
|
||||
#include "ompi/mca/fbtl/fbtl.h"
|
||||
|
||||
size_t
|
||||
mca_fbtl_posix_ipreadv (mca_io_ompio_file_t *file,
|
||||
int *sorted,
|
||||
ompi_request_t **request)
|
||||
{
|
||||
printf ("POSIX IPREADV\n");
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
199
ompi/mca/fbtl/posix/fbtl_posix_ipwritev.c
Обычный файл
199
ompi/mca/fbtl/posix/fbtl_posix_ipwritev.c
Обычный файл
@ -0,0 +1,199 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fbtl_posix.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include <unistd.h>
|
||||
#include <sys/uio.h>
|
||||
#include <aio.h>
|
||||
#include "ompi/constants.h"
|
||||
#include "orte/util/show_help.h"
|
||||
#include "ompi/mca/fbtl/fbtl.h"
|
||||
|
||||
size_t
|
||||
mca_fbtl_posix_ipwritev (mca_io_ompio_file_t *fh,
|
||||
int *sorted,
|
||||
ompi_request_t **request)
|
||||
{
|
||||
int i;
|
||||
int num_req = 0;
|
||||
int merge = 0;
|
||||
size_t k;
|
||||
char *merge_buf = NULL;
|
||||
size_t merge_length = 0;
|
||||
OMPI_MPI_OFFSET_TYPE merge_offset = 0;
|
||||
struct aiocb *aiocbp;
|
||||
|
||||
aiocbp = (struct aiocb *) malloc (sizeof(struct aiocb) *
|
||||
fh->f_num_of_io_entries);
|
||||
if (NULL == aiocbp) {
|
||||
opal_output(1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
if (NULL != sorted) {
|
||||
for (i=0 ; i<fh->f_num_of_io_entries ; i++) {
|
||||
if (fh->f_num_of_io_entries != i+1) {
|
||||
if (((OMPI_MPI_OFFSET_TYPE)fh->f_io_array[sorted[i]].offset +
|
||||
(OPAL_PTRDIFF_TYPE)fh->f_io_array[sorted[i]].length) ==
|
||||
(OMPI_MPI_OFFSET_TYPE)fh->f_io_array[sorted[i+1]].offset) {
|
||||
if (!merge) {
|
||||
merge_offset = (OMPI_MPI_OFFSET_TYPE)
|
||||
fh->f_io_array[sorted[i]].offset;
|
||||
merge_length = fh->f_io_array[sorted[i]].length;
|
||||
}
|
||||
merge_length += fh->f_io_array[sorted[i+1]].length;
|
||||
merge++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (merge) {
|
||||
merge_buf = malloc (merge_length);
|
||||
if (NULL == merge_buf) {
|
||||
opal_output(1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
k = 0;
|
||||
while (merge >= 0) {
|
||||
memcpy (merge_buf + k,
|
||||
fh->f_io_array[sorted[i-merge]].memory_address,
|
||||
fh->f_io_array[sorted[i-merge]].length);
|
||||
k += fh->f_io_array[sorted[i-merge]].length;
|
||||
merge --;
|
||||
}
|
||||
|
||||
aiocbp[num_req].aio_offset = merge_offset;
|
||||
aiocbp[num_req].aio_buf = merge_buf;
|
||||
aiocbp[num_req].aio_nbytes = merge_length;
|
||||
aiocbp[num_req].aio_fildes = fh->fd;
|
||||
aiocbp[num_req].aio_reqprio = 0;
|
||||
aiocbp[num_req].aio_sigevent.sigev_notify = SIGEV_NONE;
|
||||
|
||||
if (-1 == aio_write(&aiocbp[num_req])) {
|
||||
perror("aio_write() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
merge = 0;
|
||||
merge_offset = 0;
|
||||
merge_length = 0;
|
||||
if (NULL != merge_buf) {
|
||||
free (merge_buf);
|
||||
merge_buf = NULL;
|
||||
}
|
||||
}
|
||||
else {
|
||||
aiocbp[num_req].aio_offset = (OMPI_MPI_OFFSET_TYPE)
|
||||
fh->f_io_array[sorted[i]].offset;
|
||||
aiocbp[num_req].aio_buf =
|
||||
fh->f_io_array[sorted[i]].memory_address;
|
||||
aiocbp[num_req].aio_nbytes = fh->f_io_array[sorted[i]].length;
|
||||
aiocbp[num_req].aio_fildes = fh->fd;
|
||||
aiocbp[num_req].aio_reqprio = 0;
|
||||
aiocbp[num_req].aio_sigevent.sigev_notify = SIGEV_NONE;
|
||||
|
||||
if (-1 == aio_write(&aiocbp[num_req])) {
|
||||
perror("aio_write() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
}
|
||||
num_req ++;
|
||||
}
|
||||
}
|
||||
|
||||
else {
|
||||
for (i=0 ; i<fh->f_num_of_io_entries ; i++) {
|
||||
if (fh->f_num_of_io_entries != i+1) {
|
||||
if (((OMPI_MPI_OFFSET_TYPE)fh->f_io_array[i].offset +
|
||||
(OPAL_PTRDIFF_TYPE)fh->f_io_array[i].length) ==
|
||||
(OMPI_MPI_OFFSET_TYPE)fh->f_io_array[i+1].offset) {
|
||||
if (!merge) {
|
||||
merge_offset = (OMPI_MPI_OFFSET_TYPE)
|
||||
fh->f_io_array[i].offset;
|
||||
merge_length = fh->f_io_array[i].length;
|
||||
}
|
||||
merge_length += fh->f_io_array[i+1].length;
|
||||
merge++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (merge) {
|
||||
merge_buf = malloc (merge_length);
|
||||
if (NULL == merge_buf) {
|
||||
opal_output(1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
k = 0;
|
||||
while (merge >= 0) {
|
||||
memcpy (merge_buf + k,
|
||||
fh->f_io_array[i-merge].memory_address,
|
||||
fh->f_io_array[i-merge].length);
|
||||
k += fh->f_io_array[i-merge].length;
|
||||
merge --;
|
||||
}
|
||||
aiocbp[num_req].aio_offset = merge_offset;
|
||||
aiocbp[num_req].aio_buf = merge_buf;
|
||||
aiocbp[num_req].aio_nbytes = merge_length;
|
||||
aiocbp[num_req].aio_fildes = fh->fd;
|
||||
aiocbp[num_req].aio_reqprio = 0;
|
||||
aiocbp[num_req].aio_sigevent.sigev_notify = SIGEV_NONE;
|
||||
|
||||
if (-1 == aio_write(&aiocbp[num_req])) {
|
||||
perror("aio_write() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
merge = 0;
|
||||
merge_offset = 0;
|
||||
merge_length = 0;
|
||||
if (NULL != merge_buf) {
|
||||
free (merge_buf);
|
||||
merge_buf = NULL;
|
||||
}
|
||||
}
|
||||
else {
|
||||
aiocbp[num_req].aio_offset = (OMPI_MPI_OFFSET_TYPE)
|
||||
fh->f_io_array[i].offset;
|
||||
aiocbp[num_req].aio_buf = fh->f_io_array[i].memory_address;
|
||||
aiocbp[num_req].aio_nbytes = fh->f_io_array[i].length;
|
||||
aiocbp[num_req].aio_fildes = fh->fd;
|
||||
aiocbp[num_req].aio_reqprio = 0;
|
||||
aiocbp[num_req].aio_sigevent.sigev_notify = SIGEV_NONE;
|
||||
|
||||
if (-1 == aio_write(&aiocbp[num_req])) {
|
||||
perror("aio_write() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
}
|
||||
num_req ++;
|
||||
}
|
||||
}
|
||||
/*
|
||||
ompi_grequest_start (mca_fbtl_aio_query_fn,
|
||||
mca_fbtl_aio_free_fn,
|
||||
mca_fbtl_aio_cancel_fn,
|
||||
mca_fbtl_aio_poll_fn,
|
||||
mca_fbtl_aio_wait_fn,
|
||||
request);
|
||||
*/
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
269
ompi/mca/fbtl/posix/fbtl_posix_preadv.c
Обычный файл
269
ompi/mca/fbtl/posix/fbtl_posix_preadv.c
Обычный файл
@ -0,0 +1,269 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fbtl_posix.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include <unistd.h>
|
||||
#include "ompi/constants.h"
|
||||
#include "orte/util/show_help.h"
|
||||
#include "ompi/mca/fbtl/fbtl.h"
|
||||
|
||||
size_t
|
||||
mca_fbtl_posix_preadv (mca_io_ompio_file_t *fh,
|
||||
int *sorted)
|
||||
{
|
||||
/*int *fp = NULL;*/
|
||||
int i, block=1;
|
||||
struct iovec *iov = NULL;
|
||||
int iov_count = 0;
|
||||
OMPI_MPI_OFFSET_TYPE iov_offset = 0;
|
||||
#if 0
|
||||
int k;
|
||||
int merge = 0;
|
||||
char *merge_buf = NULL;
|
||||
size_t merge_length = 0;
|
||||
OMPI_MPI_OFFSET_TYPE merge_offset = 0;
|
||||
#endif
|
||||
if (NULL == fh->f_io_array) {
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
/*
|
||||
fp = (int *)fh->fd;
|
||||
if (0 == *fp)
|
||||
{
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
*/
|
||||
|
||||
#if 1
|
||||
iov = (struct iovec *) malloc
|
||||
(OMPIO_IOVEC_INITIAL_SIZE * sizeof (struct iovec));
|
||||
if (NULL == iov) {
|
||||
opal_output(1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
if (NULL != sorted) {
|
||||
for (i=0 ; i<fh->f_num_of_io_entries ; i++) {
|
||||
if (0 == iov_count) {
|
||||
iov[iov_count].iov_base = fh->f_io_array[sorted[i]].memory_address;
|
||||
iov[iov_count].iov_len = fh->f_io_array[sorted[i]].length;
|
||||
iov_offset = (OMPI_MPI_OFFSET_TYPE)fh->f_io_array[sorted[i]].offset;
|
||||
iov_count ++;
|
||||
}
|
||||
|
||||
if (OMPIO_IOVEC_INITIAL_SIZE*block <= iov_count) {
|
||||
block ++;
|
||||
iov = (struct iovec *)realloc
|
||||
(iov, OMPIO_IOVEC_INITIAL_SIZE * block *
|
||||
sizeof(struct iovec));
|
||||
if (NULL == iov) {
|
||||
opal_output(1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
|
||||
if (fh->f_num_of_io_entries != i+1) {
|
||||
if (((OMPI_MPI_OFFSET_TYPE)fh->f_io_array[sorted[i]].offset +
|
||||
(OPAL_PTRDIFF_TYPE)fh->f_io_array[sorted[i]].length) ==
|
||||
(OMPI_MPI_OFFSET_TYPE)fh->f_io_array[sorted[i+1]].offset) {
|
||||
iov[iov_count].iov_base =
|
||||
fh->f_io_array[sorted[i+1]].memory_address;
|
||||
iov[iov_count].iov_len = fh->f_io_array[sorted[i+1]].length;
|
||||
iov_count ++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (-1 == lseek (fh->fd, iov_offset, SEEK_SET)) {
|
||||
perror ("fseek");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
if (-1 == readv (fh->fd, iov, iov_count)) {
|
||||
perror ("writev");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
else {
|
||||
iov_count = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
else {
|
||||
for (i=0 ; i<fh->f_num_of_io_entries ; i++) {
|
||||
if (0 == iov_count) {
|
||||
iov[iov_count].iov_base = fh->f_io_array[i].memory_address;
|
||||
iov[iov_count].iov_len = fh->f_io_array[i].length;
|
||||
iov_offset = (OMPI_MPI_OFFSET_TYPE)fh->f_io_array[i].offset;
|
||||
iov_count ++;
|
||||
}
|
||||
|
||||
if (OMPIO_IOVEC_INITIAL_SIZE*block <= iov_count) {
|
||||
block ++;
|
||||
iov = (struct iovec *)realloc
|
||||
(iov, OMPIO_IOVEC_INITIAL_SIZE * block *
|
||||
sizeof(struct iovec));
|
||||
if (NULL == iov) {
|
||||
opal_output(1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
|
||||
if (fh->f_num_of_io_entries != i+1) {
|
||||
if (((OMPI_MPI_OFFSET_TYPE)fh->f_io_array[i].offset +
|
||||
(OPAL_PTRDIFF_TYPE)fh->f_io_array[i].length) ==
|
||||
(OMPI_MPI_OFFSET_TYPE)fh->f_io_array[i+1].offset) {
|
||||
iov[iov_count].iov_base =
|
||||
fh->f_io_array[i+1].memory_address;
|
||||
iov[iov_count].iov_len = fh->f_io_array[i+1].length;
|
||||
iov_count ++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (-1 == lseek (fh->fd, iov_offset, SEEK_SET)) {
|
||||
perror ("fseek");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
if (-1 == readv (fh->fd, iov, iov_count)) {
|
||||
perror ("writev");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
else {
|
||||
iov_count = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (NULL != iov) {
|
||||
free (iov);
|
||||
iov = NULL;
|
||||
}
|
||||
#endif
|
||||
#if 0
|
||||
if (NULL != sorted) {
|
||||
for (i=0 ; i<fh->f_num_of_io_entries ; i++) {
|
||||
if (fh->f_num_of_io_entries != i+1) {
|
||||
if (((OMPI_MPI_OFFSET_TYPE)fh->f_io_array[sorted[i]].offset +
|
||||
(OPAL_PTRDIFF_TYPE)fh->f_io_array[sorted[i]].length) ==
|
||||
(OMPI_MPI_OFFSET_TYPE)fh->f_io_array[sorted[i+1]].offset) {
|
||||
if (!merge) {
|
||||
merge_offset = (OMPI_MPI_OFFSET_TYPE)
|
||||
fh->f_io_array[sorted[i]].offset;
|
||||
merge_length = fh->f_io_array[sorted[i]].length;
|
||||
}
|
||||
merge_length += fh->f_io_array[sorted[i+1]].length;
|
||||
merge++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (merge) {
|
||||
merge_buf = malloc (merge_length);
|
||||
if (-1 == pread(fh->fd,
|
||||
merge_buf,
|
||||
merge_length,
|
||||
merge_offset)) {
|
||||
perror("pread() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
k = 0;
|
||||
while (merge >= 0) {
|
||||
memcpy (fh->f_io_array[sorted[i-merge]].memory_address,
|
||||
merge_buf + k,
|
||||
fh->f_io_array[sorted[i-merge]].length);
|
||||
k += fh->f_io_array[sorted[i-merge]].length;
|
||||
merge --;
|
||||
}
|
||||
merge = 0;
|
||||
merge_offset = 0;
|
||||
merge_length = 0;
|
||||
if (NULL != merge_buf) {
|
||||
free (merge_buf);
|
||||
merge_buf = NULL;
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (-1 == pread(fh->fd,
|
||||
fh->f_io_array[sorted[i]].memory_address,
|
||||
fh->f_io_array[sorted[i]].length,
|
||||
(OMPI_MPI_OFFSET_TYPE)fh->f_io_array[sorted[i]].offset)) {
|
||||
perror("pread() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
else {
|
||||
for (i=0 ; i<fh->f_num_of_io_entries ; i++) {
|
||||
if (fh->f_num_of_io_entries != i+1) {
|
||||
if (((OMPI_MPI_OFFSET_TYPE)fh->f_io_array[i].offset +
|
||||
(OPAL_PTRDIFF_TYPE)fh->f_io_array[i].length) ==
|
||||
(OMPI_MPI_OFFSET_TYPE)fh->f_io_array[i+1].offset) {
|
||||
if (!merge) {
|
||||
merge_offset = (OMPI_MPI_OFFSET_TYPE)
|
||||
fh->f_io_array[i].offset;
|
||||
merge_length = fh->f_io_array[i].length;
|
||||
}
|
||||
merge_length += fh->f_io_array[i+1].length;
|
||||
merge++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (merge) {
|
||||
merge_buf = malloc (merge_length);
|
||||
if (-1 == pread(fh->fd,
|
||||
merge_buf,
|
||||
merge_length,
|
||||
merge_offset)) {
|
||||
perror("pread() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
k = 0;
|
||||
while (merge >= 0) {
|
||||
memcpy (fh->f_io_array[i-merge].memory_address,
|
||||
merge_buf + k,
|
||||
fh->f_io_array[i-merge].length);
|
||||
k += fh->f_io_array[i-merge].length;
|
||||
merge --;
|
||||
}
|
||||
merge = 0;
|
||||
merge_offset = 0;
|
||||
merge_length = 0;
|
||||
if (NULL != merge_buf) {
|
||||
free (merge_buf);
|
||||
merge_buf = NULL;
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (-1 == pread(fh->fd,
|
||||
fh->f_io_array[i].memory_address,
|
||||
fh->f_io_array[i].length,
|
||||
(OMPI_MPI_OFFSET_TYPE)fh->f_io_array[i].offset)) {
|
||||
perror("pread() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
299
ompi/mca/fbtl/posix/fbtl_posix_pwritev.c
Обычный файл
299
ompi/mca/fbtl/posix/fbtl_posix_pwritev.c
Обычный файл
@ -0,0 +1,299 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fbtl_posix.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include <unistd.h>
|
||||
#include <sys/uio.h>
|
||||
#include "ompi/constants.h"
|
||||
#include "orte/util/show_help.h"
|
||||
#include "ompi/mca/fbtl/fbtl.h"
|
||||
|
||||
size_t
|
||||
mca_fbtl_posix_pwritev (mca_io_ompio_file_t *fh,
|
||||
int *sorted)
|
||||
{
|
||||
/*int *fp = NULL;*/
|
||||
int i, block = 1;
|
||||
struct iovec *iov = NULL;
|
||||
int iov_count = 0;
|
||||
OMPI_MPI_OFFSET_TYPE iov_offset = 0;
|
||||
#if 0
|
||||
int merge = 0;
|
||||
size_t k;
|
||||
char *merge_buf = NULL;
|
||||
size_t merge_length = 0;
|
||||
OMPI_MPI_OFFSET_TYPE merge_offset = 0;
|
||||
#endif
|
||||
|
||||
if (NULL == fh->f_io_array) {
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
/*
|
||||
fp = (int *)fh->fd;
|
||||
if (0 == *fp)
|
||||
{
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
*/
|
||||
#if 1
|
||||
|
||||
iov = (struct iovec *) malloc
|
||||
(OMPIO_IOVEC_INITIAL_SIZE * sizeof (struct iovec));
|
||||
if (NULL == iov) {
|
||||
opal_output(1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
if (NULL != sorted) {
|
||||
for (i=0 ; i<fh->f_num_of_io_entries ; i++) {
|
||||
if (0 == iov_count) {
|
||||
iov[iov_count].iov_base = fh->f_io_array[sorted[i]].memory_address;
|
||||
iov[iov_count].iov_len = fh->f_io_array[sorted[i]].length;
|
||||
iov_offset = (OMPI_MPI_OFFSET_TYPE)fh->f_io_array[sorted[i]].offset;
|
||||
iov_count ++;
|
||||
}
|
||||
|
||||
if (OMPIO_IOVEC_INITIAL_SIZE*block <= iov_count) {
|
||||
block ++;
|
||||
iov = (struct iovec *)realloc
|
||||
(iov, OMPIO_IOVEC_INITIAL_SIZE * block *
|
||||
sizeof(struct iovec));
|
||||
if (NULL == iov) {
|
||||
opal_output(1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
|
||||
if (fh->f_num_of_io_entries != i+1) {
|
||||
if (((OMPI_MPI_OFFSET_TYPE)fh->f_io_array[sorted[i]].offset +
|
||||
(OPAL_PTRDIFF_TYPE)fh->f_io_array[sorted[i]].length) ==
|
||||
(OMPI_MPI_OFFSET_TYPE)fh->f_io_array[sorted[i+1]].offset) {
|
||||
iov[iov_count].iov_base =
|
||||
fh->f_io_array[sorted[i+1]].memory_address;
|
||||
iov[iov_count].iov_len = fh->f_io_array[sorted[i+1]].length;
|
||||
iov_count ++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
if (-1 == lseek (fh->fd, iov_offset, SEEK_SET)) {
|
||||
perror ("lseek");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
if (-1 == writev (fh->fd, iov, iov_count)) {
|
||||
perror ("writev");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
iov_count = 0;
|
||||
}
|
||||
}
|
||||
|
||||
else {
|
||||
for (i=0 ; i<fh->f_num_of_io_entries ; i++) {
|
||||
if (0 == iov_count) {
|
||||
iov[iov_count].iov_base = fh->f_io_array[i].memory_address;
|
||||
iov[iov_count].iov_len = fh->f_io_array[i].length;
|
||||
iov_offset = (OMPI_MPI_OFFSET_TYPE)fh->f_io_array[i].offset;
|
||||
iov_count ++;
|
||||
}
|
||||
|
||||
if (OMPIO_IOVEC_INITIAL_SIZE*block <= iov_count) {
|
||||
block ++;
|
||||
iov = (struct iovec *)realloc
|
||||
(iov, OMPIO_IOVEC_INITIAL_SIZE * block *
|
||||
sizeof(struct iovec));
|
||||
if (NULL == iov) {
|
||||
opal_output(1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
|
||||
if (fh->f_num_of_io_entries != i+1) {
|
||||
if (((OMPI_MPI_OFFSET_TYPE)fh->f_io_array[i].offset +
|
||||
(OPAL_PTRDIFF_TYPE)fh->f_io_array[i].length) ==
|
||||
(OMPI_MPI_OFFSET_TYPE)fh->f_io_array[i+1].offset) {
|
||||
iov[iov_count].iov_base =
|
||||
fh->f_io_array[i+1].memory_address;
|
||||
iov[iov_count].iov_len = fh->f_io_array[i+1].length;
|
||||
iov_count ++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
/*
|
||||
printf ("RANK: %d Entries: %d count: %d\n",
|
||||
fh->f_rank,
|
||||
fh->f_num_of_io_entries,
|
||||
iov_count);
|
||||
for (j=0 ; j<iov_count ; j++) {
|
||||
printf ("%p %lld\n",
|
||||
iov[j].iov_base,
|
||||
iov[j].iov_len);
|
||||
}
|
||||
|
||||
*/
|
||||
if (-1 == lseek (fh->fd, iov_offset, SEEK_SET)) {
|
||||
perror ("lseek");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
if (-1 == writev (fh->fd, iov, iov_count)) {
|
||||
perror ("writev");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
iov_count = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (NULL != iov) {
|
||||
free (iov);
|
||||
iov = NULL;
|
||||
}
|
||||
|
||||
#endif
|
||||
#if 0
|
||||
if (NULL != sorted) {
|
||||
for (i=0 ; i<fh->f_num_of_io_entries ; i++) {
|
||||
if (fh->f_num_of_io_entries != i+1) {
|
||||
if (((OMPI_MPI_OFFSET_TYPE)fh->f_io_array[sorted[i]].offset +
|
||||
(OPAL_PTRDIFF_TYPE)fh->f_io_array[sorted[i]].length) ==
|
||||
(OMPI_MPI_OFFSET_TYPE)fh->f_io_array[sorted[i+1]].offset) {
|
||||
if (!merge) {
|
||||
merge_offset = (OMPI_MPI_OFFSET_TYPE)
|
||||
fh->f_io_array[sorted[i]].offset;
|
||||
merge_length = fh->f_io_array[sorted[i]].length;
|
||||
}
|
||||
merge_length += fh->f_io_array[sorted[i+1]].length;
|
||||
merge++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
/*
|
||||
printf ("RANK: %d Entries: %d MERGE: %d \n", fh->f_rank,fh->f_num_of_io_entries,merge);
|
||||
if (merge)
|
||||
printf ("Merge size: %d\n",merge_length);
|
||||
else
|
||||
printf ("entry size: %d\n",fh->f_io_array[sorted[i]].length);
|
||||
sleep(5);
|
||||
*/
|
||||
if (merge) {
|
||||
merge_buf = malloc (merge_length);
|
||||
k = 0;
|
||||
while (merge >= 0) {
|
||||
memcpy (merge_buf + k,
|
||||
fh->f_io_array[sorted[i-merge]].memory_address,
|
||||
fh->f_io_array[sorted[i-merge]].length);
|
||||
k += fh->f_io_array[sorted[i-merge]].length;
|
||||
merge --;
|
||||
}
|
||||
if (-1 == pwrite(fh->fd,
|
||||
merge_buf,
|
||||
merge_length, merge_offset)) {
|
||||
perror("write() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
merge = 0;
|
||||
merge_offset = 0;
|
||||
merge_length = 0;
|
||||
if (NULL != merge_buf) {
|
||||
free (merge_buf);
|
||||
merge_buf = NULL;
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (-1 == pwrite(fh->fd,
|
||||
fh->f_io_array[sorted[i]].memory_address,
|
||||
fh->f_io_array[sorted[i]].length,
|
||||
(OMPI_MPI_OFFSET_TYPE)
|
||||
fh->f_io_array[sorted[i]].offset)) {
|
||||
perror("pwrite() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
else {
|
||||
for (i=0 ; i<fh->f_num_of_io_entries ; i++) {
|
||||
if (fh->f_num_of_io_entries != i+1) {
|
||||
if (((OMPI_MPI_OFFSET_TYPE)fh->f_io_array[i].offset +
|
||||
(OPAL_PTRDIFF_TYPE)fh->f_io_array[i].length) ==
|
||||
(OMPI_MPI_OFFSET_TYPE)fh->f_io_array[i+1].offset) {
|
||||
if (!merge) {
|
||||
merge_offset = (OMPI_MPI_OFFSET_TYPE)
|
||||
fh->f_io_array[i].offset;
|
||||
merge_length = fh->f_io_array[i].length;
|
||||
}
|
||||
merge_length += fh->f_io_array[i+1].length;
|
||||
merge++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
/*
|
||||
printf ("RANK: %d Entries: %d MERGE: %d \n", fh->f_rank,fh->f_num_of_io_entries,merge);
|
||||
if (merge)
|
||||
printf ("Merge size: %d\n",merge_length);
|
||||
else
|
||||
printf ("entry size: %d\n",fh->f_io_array[i].length);
|
||||
sleep(2);
|
||||
*/
|
||||
if (merge) {
|
||||
merge_buf = malloc (merge_length);
|
||||
k = 0;
|
||||
while (merge >= 0) {
|
||||
memcpy (merge_buf + k,
|
||||
fh->f_io_array[i-merge].memory_address,
|
||||
fh->f_io_array[i-merge].length);
|
||||
k += fh->f_io_array[i-merge].length;
|
||||
merge --;
|
||||
}
|
||||
if (-1 == pwrite(fh->fd,
|
||||
merge_buf,
|
||||
merge_length,
|
||||
merge_offset)) {
|
||||
perror("write() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
merge = 0;
|
||||
merge_offset = 0;
|
||||
merge_length = 0;
|
||||
if (NULL != merge_buf) {
|
||||
free (merge_buf);
|
||||
merge_buf = NULL;
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (-1 == pwrite(fh->fd,
|
||||
fh->f_io_array[i].memory_address,
|
||||
fh->f_io_array[i].length,
|
||||
(OMPI_MPI_OFFSET_TYPE)
|
||||
fh->f_io_array[i].offset)) {
|
||||
perror("pwrite() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
54
ompi/mca/fbtl/pvfs2/Makefile.am
Обычный файл
54
ompi/mca/fbtl/pvfs2/Makefile.am
Обычный файл
@ -0,0 +1,54 @@
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
# Make the output library in this directory, and name it either
|
||||
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
|
||||
# (for static builds).
|
||||
|
||||
if MCA_BUILD_ompi_fbtl_pvfs2_DSO
|
||||
component_noinst =
|
||||
component_install = mca_fbtl_pvfs2.la
|
||||
else
|
||||
component_noinst = libmca_fbtl_pvfs2.la
|
||||
component_install =
|
||||
endif
|
||||
|
||||
# Source files
|
||||
|
||||
fbtl_pvfs2_sources = \
|
||||
fbtl_pvfs2.h \
|
||||
fbtl_pvfs2.c \
|
||||
fbtl_pvfs2_component.c \
|
||||
fbtl_pvfs2_preadv.c \
|
||||
fbtl_pvfs2_ipreadv.c \
|
||||
fbtl_pvfs2_pwritev.c \
|
||||
fbtl_pvfs2_ipwritev.c
|
||||
|
||||
AM_CPPFLAGS = $(fbtl_pvfs2_CPPFLAGS)
|
||||
|
||||
mcacomponentdir = $(pkglibdir)
|
||||
mcacomponent_LTLIBRARIES = $(component_install)
|
||||
mca_fbtl_pvfs2_la_SOURCES = $(fbtl_pvfs2_sources)
|
||||
mca_fbtl_pvfs2_la_LIBADD =
|
||||
mca_fbtl_pvfs2_la_LDFLAGS = -module -avoid-version $(fbtl_pvfs2_LDFLAGS)
|
||||
|
||||
noinst_LTLIBRARIES = $(component_noinst)
|
||||
libmca_fbtl_pvfs2_la_SOURCES = $(fbtl_pvfs2_sources)
|
||||
libmca_fbtl_pvfs2_la_LIBADD =
|
||||
libmca_fbtl_pvfs2_la_LDFLAGS = -module -avoid-version $(fbtl_pvfs2_LDFLAGS)
|
||||
50
ompi/mca/fbtl/pvfs2/configure.m4
Обычный файл
50
ompi/mca/fbtl/pvfs2/configure.m4
Обычный файл
@ -0,0 +1,50 @@
|
||||
# -*- shell-script -*-
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
|
||||
# Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
|
||||
# MCA_fbtl_pvfs2_CONFIG(action-if-can-compile,
|
||||
# [action-if-cant-compile])
|
||||
# ------------------------------------------------
|
||||
AC_DEFUN([MCA_ompi_fbtl_pvfs2_CONFIG],[
|
||||
AC_CONFIG_FILES([ompi/mca/fbtl/pvfs2/Makefile])
|
||||
|
||||
OMPI_CHECK_PVFS2([fbtl_pvfs2],
|
||||
[fbtl_pvfs2_happy="yes"],
|
||||
[fbtl_pvfs2_happy="no"])
|
||||
|
||||
AS_IF([test "$fbtl_pvfs2_happy" = "yes"],
|
||||
[fbtl_pvfs2_WRAPPER_EXTRA_LDFLAGS="$fbtl_pvfs2_LDFLAGS"
|
||||
fbtl_pvfs2_WRAPPER_EXTRA_LIBS="$fbtl_pvfs2_LIBS"
|
||||
$1],
|
||||
[$2])
|
||||
|
||||
AC_CHECK_HEADERS([pvfs2.h], [],
|
||||
[AC_CHECK_HEADERS([pvfs2.h], [], [$2],
|
||||
[AC_INCLUDES_DEFAULT])],
|
||||
[AC_INCLUDES_DEFAULT])
|
||||
|
||||
|
||||
# substitute in the things needed to build pvfs2
|
||||
AC_SUBST([fbtl_pvfs2_CFLAGS])
|
||||
AC_SUBST([fbtl_pvfs2_CPPFLAGS])
|
||||
AC_SUBST([fbtl_pvfs2_LDFLAGS])
|
||||
AC_SUBST([fbtl_pvfs2_LIBS])
|
||||
])dnl
|
||||
85
ompi/mca/fbtl/pvfs2/fbtl_pvfs2.c
Обычный файл
85
ompi/mca/fbtl/pvfs2/fbtl_pvfs2.c
Обычный файл
@ -0,0 +1,85 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2006 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*
|
||||
* These symbols are in a file by themselves to provide nice linker
|
||||
* semantics. Since linkers generally pull in symbols by object fules,
|
||||
* keeping these symbols as the only symbols in this file prevents
|
||||
* utility programs such as "ompi_info" from having to import entire
|
||||
* modules just to query their version and parameters
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "mpi.h"
|
||||
#include "ompi/mca/fbtl/fbtl.h"
|
||||
#include "ompi/mca/fbtl/pvfs2/fbtl_pvfs2.h"
|
||||
|
||||
/*
|
||||
* *******************************************************************
|
||||
* ************************ actions structure ************************
|
||||
* *******************************************************************
|
||||
*/
|
||||
static mca_fbtl_base_module_1_0_0_t pvfs2 = {
|
||||
mca_fbtl_pvfs2_module_init, /* initalise after being selected */
|
||||
mca_fbtl_pvfs2_module_finalize, /* close a module on a communicator */
|
||||
mca_fbtl_pvfs2_preadv,
|
||||
mca_fbtl_pvfs2_ipreadv,
|
||||
mca_fbtl_pvfs2_pwritev,
|
||||
mca_fbtl_pvfs2_ipwritev
|
||||
};
|
||||
/*
|
||||
* *******************************************************************
|
||||
* ************************* structure ends **************************
|
||||
* *******************************************************************
|
||||
*/
|
||||
|
||||
int mca_fbtl_pvfs2_component_init_query(bool enable_progress_threads,
|
||||
bool enable_mpi_threads) {
|
||||
/* Nothing to do */
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
struct mca_fbtl_base_module_1_0_0_t *
|
||||
mca_fbtl_pvfs2_component_file_query (mca_io_ompio_file_t *fh, int *priority) {
|
||||
*priority = mca_fbtl_pvfs2_priority;
|
||||
|
||||
if (PVFS2 == fh->f_fstype) {
|
||||
if (*priority < 50) {
|
||||
*priority = 50;
|
||||
}
|
||||
}
|
||||
|
||||
return &pvfs2;
|
||||
}
|
||||
|
||||
int mca_fbtl_pvfs2_component_file_unquery (mca_io_ompio_file_t *file) {
|
||||
/* This function might be needed for some purposes later. for now it
|
||||
* does not have anything to do since there are no steps which need
|
||||
* to be undone if this module is not selected */
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
int mca_fbtl_pvfs2_module_init (mca_io_ompio_file_t *file) {
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
int mca_fbtl_pvfs2_module_finalize (mca_io_ompio_file_t *file) {
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
77
ompi/mca/fbtl/pvfs2/fbtl_pvfs2.h
Обычный файл
77
ompi/mca/fbtl/pvfs2/fbtl_pvfs2.h
Обычный файл
@ -0,0 +1,77 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2006 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#ifndef MCA_FBTL_PVFS2_H
|
||||
#define MCA_FBTL_PVFS2_H
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "opal/mca/mca.h"
|
||||
#include "ompi/mca/fbtl/fbtl.h"
|
||||
#include "ompi/mca/io/ompio/io_ompio.h"
|
||||
#include "ompi/mca/fs/pvfs2/fs_pvfs2.h"
|
||||
#include "pvfs2.h"
|
||||
#include "pvfs2-compat.h"
|
||||
|
||||
/*
|
||||
#ifdef HAVE_PVFS2_H
|
||||
#include "pvfs2.h"
|
||||
#endif
|
||||
|
||||
#ifdef PVFS2_VERSION_MAJOR
|
||||
#include "pvfs2-compat.h"
|
||||
#endif
|
||||
*/
|
||||
extern int mca_fbtl_pvfs2_priority;
|
||||
|
||||
BEGIN_C_DECLS
|
||||
|
||||
int mca_fbtl_pvfs2_component_init_query(bool enable_progress_threads,
|
||||
bool enable_mpi_threads);
|
||||
struct mca_fbtl_base_module_1_0_0_t *
|
||||
mca_fbtl_pvfs2_component_file_query (mca_io_ompio_file_t *file, int *priority);
|
||||
int mca_fbtl_pvfs2_component_file_unquery (mca_io_ompio_file_t *file);
|
||||
|
||||
int mca_fbtl_pvfs2_module_init (mca_io_ompio_file_t *file);
|
||||
int mca_fbtl_pvfs2_module_finalize (mca_io_ompio_file_t *file);
|
||||
|
||||
OMPI_MODULE_DECLSPEC extern mca_fbtl_base_component_2_0_0_t mca_fbtl_pvfs2_component;
|
||||
/*
|
||||
* ******************************************************************
|
||||
* ********* functions which are implemented in this module *********
|
||||
* ******************************************************************
|
||||
*/
|
||||
|
||||
size_t mca_fbtl_pvfs2_preadv (mca_io_ompio_file_t *file,
|
||||
int *sorted);
|
||||
size_t mca_fbtl_pvfs2_pwritev (mca_io_ompio_file_t *file,
|
||||
int *sorted);
|
||||
size_t mca_fbtl_pvfs2_ipreadv (mca_io_ompio_file_t *file,
|
||||
int *sorted, ompi_request_t **request);
|
||||
size_t mca_fbtl_pvfs2_ipwritev (mca_io_ompio_file_t *file,
|
||||
int *sorted, ompi_request_t **request);
|
||||
|
||||
/*
|
||||
* ******************************************************************
|
||||
* ************ functions implemented in this module end ************
|
||||
* ******************************************************************
|
||||
*/
|
||||
|
||||
END_C_DECLS
|
||||
|
||||
#endif /* MCA_FBTL_PVFS2_H */
|
||||
65
ompi/mca/fbtl/pvfs2/fbtl_pvfs2_component.c
Обычный файл
65
ompi/mca/fbtl/pvfs2/fbtl_pvfs2_component.c
Обычный файл
@ -0,0 +1,65 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*
|
||||
* These symbols are in a file by themselves to provide nice linker
|
||||
* semantics. Since linkers generally pull in symbols by object
|
||||
* files, keeping these symbols as the only symbols in this file
|
||||
* prevents utility programs such as "ompi_info" from having to import
|
||||
* entire components just to query their version and parameters.
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fbtl_pvfs2.h"
|
||||
#include "mpi.h"
|
||||
|
||||
/*
|
||||
* Public string showing the fbtl pvfs2 component version number
|
||||
*/
|
||||
const char *mca_fbtl_pvfs2_component_version_string =
|
||||
"OMPI/MPI pvfs2 FBTL MCA component version " OMPI_VERSION;
|
||||
|
||||
int mca_fbtl_pvfs2_priority = 0;
|
||||
|
||||
/*
|
||||
* Instantiate the public struct with all of our public information
|
||||
* and pointers to our public functions in it
|
||||
*/
|
||||
mca_fbtl_base_component_2_0_0_t mca_fbtl_pvfs2_component = {
|
||||
|
||||
/* First, the mca_component_t struct containing meta information
|
||||
about the component itself */
|
||||
|
||||
{
|
||||
MCA_FBTL_BASE_VERSION_2_0_0,
|
||||
|
||||
/* Component name and version */
|
||||
"pvfs2",
|
||||
OMPI_MAJOR_VERSION,
|
||||
OMPI_MINOR_VERSION,
|
||||
OMPI_RELEASE_VERSION,
|
||||
NULL,
|
||||
NULL
|
||||
},
|
||||
{
|
||||
/* This component is checkpointable */
|
||||
MCA_BASE_METADATA_PARAM_CHECKPOINT
|
||||
},
|
||||
mca_fbtl_pvfs2_component_init_query, /* get thread level */
|
||||
mca_fbtl_pvfs2_component_file_query, /* get priority and actions */
|
||||
mca_fbtl_pvfs2_component_file_unquery /* undo what was done by previous function */
|
||||
};
|
||||
35
ompi/mca/fbtl/pvfs2/fbtl_pvfs2_ipreadv.c
Обычный файл
35
ompi/mca/fbtl/pvfs2/fbtl_pvfs2_ipreadv.c
Обычный файл
@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fbtl_pvfs2.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "orte/util/show_help.h"
|
||||
#include "ompi/mca/fbtl/fbtl.h"
|
||||
|
||||
size_t
|
||||
mca_fbtl_pvfs2_ipreadv (mca_io_ompio_file_t *file,
|
||||
int *sorted, ompi_request_t **request)
|
||||
{
|
||||
printf ("PVFS2 IPREADV\n");
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
35
ompi/mca/fbtl/pvfs2/fbtl_pvfs2_ipwritev.c
Обычный файл
35
ompi/mca/fbtl/pvfs2/fbtl_pvfs2_ipwritev.c
Обычный файл
@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fbtl_pvfs2.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "orte/util/show_help.h"
|
||||
#include "ompi/mca/fbtl/fbtl.h"
|
||||
|
||||
size_t
|
||||
mca_fbtl_pvfs2_ipwritev (mca_io_ompio_file_t *file,
|
||||
int *sorted, ompi_request_t **request)
|
||||
{
|
||||
printf ("PVFS2 IPWRITEV\n");
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
240
ompi/mca/fbtl/pvfs2/fbtl_pvfs2_preadv.c
Обычный файл
240
ompi/mca/fbtl/pvfs2/fbtl_pvfs2_preadv.c
Обычный файл
@ -0,0 +1,240 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
/* This code is based on the PVFS2 ADIO module in ROMIO
|
||||
* Copyright (C) 1997 University of Chicago.
|
||||
* See COPYRIGHT notice in top-level directory.
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fbtl_pvfs2.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include <unistd.h>
|
||||
#include "ompi/constants.h"
|
||||
#include "orte/util/show_help.h"
|
||||
#include "ompi/mca/fbtl/fbtl.h"
|
||||
|
||||
size_t
|
||||
mca_fbtl_pvfs2_preadv (mca_io_ompio_file_t *fh,
|
||||
int *sorted)
|
||||
{
|
||||
int i;
|
||||
int ret;
|
||||
size_t k;
|
||||
int merge = 0;
|
||||
char *merge_buf = NULL;
|
||||
size_t merge_length = 0;
|
||||
OMPI_MPI_OFFSET_TYPE merge_offset = 0;
|
||||
PVFS_sysresp_io resp_io;
|
||||
PVFS_Request file_req;
|
||||
PVFS_Request mem_req;
|
||||
mca_fs_pvfs2 *pvfs2_fs;
|
||||
|
||||
pvfs2_fs = (mca_fs_pvfs2 *)fh->f_fs_ptr;
|
||||
|
||||
if (NULL == fh->f_io_array) {
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
if (NULL != sorted) {
|
||||
for (i=0 ; i<fh->f_num_of_io_entries ; i++) {
|
||||
if (fh->f_num_of_io_entries != i+1) {
|
||||
if (((OMPI_MPI_OFFSET_TYPE)fh->f_io_array[sorted[i]].offset +
|
||||
(OPAL_PTRDIFF_TYPE)fh->f_io_array[sorted[i]].length) ==
|
||||
(OMPI_MPI_OFFSET_TYPE)fh->f_io_array[sorted[i+1]].offset) {
|
||||
if (!merge) {
|
||||
merge_offset = (OMPI_MPI_OFFSET_TYPE)
|
||||
fh->f_io_array[sorted[i]].offset;
|
||||
merge_length = fh->f_io_array[sorted[i]].length;
|
||||
}
|
||||
merge_length += fh->f_io_array[sorted[i+1]].length;
|
||||
merge++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (merge) {
|
||||
merge_buf = malloc (merge_length);
|
||||
|
||||
ret = PVFS_Request_contiguous (merge_length,
|
||||
PVFS_BYTE,
|
||||
&mem_req);
|
||||
if (ret != 0) {
|
||||
perror("PVFS_Request_contiguous() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
ret = PVFS_Request_contiguous (merge_length,
|
||||
PVFS_BYTE,
|
||||
&file_req);
|
||||
if (ret != 0) {
|
||||
perror("PVFS_Request_contiguous() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
ret = PVFS_sys_read(pvfs2_fs->object_ref,
|
||||
file_req,
|
||||
merge_offset,
|
||||
merge_buf,
|
||||
mem_req,
|
||||
&(pvfs2_fs->credentials),
|
||||
&resp_io);
|
||||
if (ret != 0) {
|
||||
perror("PVFS_sys_write() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
k = 0;
|
||||
while (merge >= 0) {
|
||||
memcpy (fh->f_io_array[sorted[i-merge]].memory_address,
|
||||
merge_buf + k,
|
||||
fh->f_io_array[sorted[i-merge]].length);
|
||||
k += fh->f_io_array[sorted[i-merge]].length;
|
||||
merge --;
|
||||
}
|
||||
merge = 0;
|
||||
merge_offset = 0;
|
||||
merge_length = 0;
|
||||
if (NULL != merge_buf) {
|
||||
free (merge_buf);
|
||||
merge_buf = NULL;
|
||||
}
|
||||
}
|
||||
else {
|
||||
ret = PVFS_Request_contiguous (fh->f_io_array[sorted[i]].length,
|
||||
PVFS_BYTE,
|
||||
&mem_req);
|
||||
if (ret != 0) {
|
||||
perror("PVFS_Request_contiguous() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
ret = PVFS_Request_contiguous (fh->f_io_array[sorted[i]].length,
|
||||
PVFS_BYTE,
|
||||
&file_req);
|
||||
if (ret != 0) {
|
||||
perror("PVFS_Request_contiguous() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
ret = PVFS_sys_read(pvfs2_fs->object_ref,
|
||||
file_req,
|
||||
(OMPI_MPI_OFFSET_TYPE)
|
||||
fh->f_io_array[sorted[i]].offset,
|
||||
fh->f_io_array[sorted[i]].memory_address,
|
||||
mem_req,
|
||||
&(pvfs2_fs->credentials),
|
||||
&resp_io);
|
||||
if (ret != 0) {
|
||||
perror("PVFS_sys_write() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
else {
|
||||
for (i=0 ; i<fh->f_num_of_io_entries ; i++) {
|
||||
if (fh->f_num_of_io_entries != i+1) {
|
||||
if (((OMPI_MPI_OFFSET_TYPE)fh->f_io_array[i].offset +
|
||||
(OPAL_PTRDIFF_TYPE)fh->f_io_array[i].length) ==
|
||||
(OMPI_MPI_OFFSET_TYPE)fh->f_io_array[i+1].offset) {
|
||||
if (!merge) {
|
||||
merge_offset = (OMPI_MPI_OFFSET_TYPE)
|
||||
fh->f_io_array[i].offset;
|
||||
merge_length = fh->f_io_array[i].length;
|
||||
}
|
||||
merge_length += fh->f_io_array[i+1].length;
|
||||
merge++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
if (merge) {
|
||||
merge_buf = malloc (merge_length);
|
||||
|
||||
ret = PVFS_Request_contiguous (merge_length,
|
||||
PVFS_BYTE,
|
||||
&mem_req);
|
||||
if (ret != 0) {
|
||||
perror("PVFS_Request_contiguous() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
ret = PVFS_Request_contiguous (merge_length,
|
||||
PVFS_BYTE,
|
||||
&file_req);
|
||||
if (ret != 0) {
|
||||
perror("PVFS_Request_contiguous() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
ret = PVFS_sys_read (pvfs2_fs->object_ref,
|
||||
file_req,
|
||||
merge_offset,
|
||||
merge_buf,
|
||||
mem_req,
|
||||
&(pvfs2_fs->credentials),
|
||||
&resp_io);
|
||||
if (ret != 0) {
|
||||
perror("PVFS_sys_write() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
k = 0;
|
||||
while (merge >= 0) {
|
||||
memcpy (fh->f_io_array[i-merge].memory_address,
|
||||
merge_buf + k,
|
||||
fh->f_io_array[i-merge].length);
|
||||
k += fh->f_io_array[i-merge].length;
|
||||
merge --;
|
||||
}
|
||||
merge = 0;
|
||||
merge_offset = 0;
|
||||
merge_length = 0;
|
||||
if (NULL != merge_buf) {
|
||||
free (merge_buf);
|
||||
merge_buf = NULL;
|
||||
}
|
||||
}
|
||||
else {
|
||||
ret = PVFS_Request_contiguous (fh->f_io_array[i].length,
|
||||
PVFS_BYTE,
|
||||
&mem_req);
|
||||
if (ret != 0) {
|
||||
perror("PVFS_Request_contiguous() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
ret = PVFS_Request_contiguous (fh->f_io_array[i].length,
|
||||
PVFS_BYTE,
|
||||
&file_req);
|
||||
if (ret != 0) {
|
||||
perror("PVFS_Request_contiguous() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
ret = PVFS_sys_read (pvfs2_fs->object_ref,
|
||||
file_req,
|
||||
(OMPI_MPI_OFFSET_TYPE)
|
||||
fh ->f_io_array[i].offset,
|
||||
fh->f_io_array[i].memory_address,
|
||||
mem_req,
|
||||
&(pvfs2_fs->credentials),
|
||||
&resp_io);
|
||||
if (ret != 0) {
|
||||
perror("PVFS_sys_write() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
257
ompi/mca/fbtl/pvfs2/fbtl_pvfs2_pwritev.c
Обычный файл
257
ompi/mca/fbtl/pvfs2/fbtl_pvfs2_pwritev.c
Обычный файл
@ -0,0 +1,257 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
/* This code is based on the PVFS2 ADIO module in ROMIO
|
||||
* Copyright (C) 1997 University of Chicago.
|
||||
* See COPYRIGHT notice in top-level directory.
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fbtl_pvfs2.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include <unistd.h>
|
||||
#include <sys/uio.h>
|
||||
#include "ompi/constants.h"
|
||||
#include "orte/util/show_help.h"
|
||||
#include "ompi/mca/fbtl/fbtl.h"
|
||||
|
||||
size_t
|
||||
mca_fbtl_pvfs2_pwritev (mca_io_ompio_file_t *fh,
|
||||
int *sorted)
|
||||
{
|
||||
int i;
|
||||
int merge = 0;
|
||||
int ret;
|
||||
size_t k;
|
||||
char *merge_buf = NULL;
|
||||
size_t merge_length = 0;
|
||||
OMPI_MPI_OFFSET_TYPE merge_offset = 0;
|
||||
PVFS_sysresp_io resp_io;
|
||||
PVFS_Request file_req;
|
||||
PVFS_Request mem_req;
|
||||
mca_fs_pvfs2 *pvfs2_fs;
|
||||
|
||||
pvfs2_fs = (mca_fs_pvfs2 *)fh->f_fs_ptr;
|
||||
|
||||
if (NULL == fh->f_io_array) {
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
if (NULL != sorted) {
|
||||
for (i=0 ; i<fh->f_num_of_io_entries ; i++) {
|
||||
if (fh->f_num_of_io_entries != i+1) {
|
||||
if (((OMPI_MPI_OFFSET_TYPE)fh->f_io_array[sorted[i]].offset +
|
||||
(OPAL_PTRDIFF_TYPE)fh->f_io_array[sorted[i]].length) ==
|
||||
(OMPI_MPI_OFFSET_TYPE)fh->f_io_array[sorted[i+1]].offset) {
|
||||
if (!merge) {
|
||||
merge_offset = (OMPI_MPI_OFFSET_TYPE)
|
||||
fh->f_io_array[sorted[i]].offset;
|
||||
merge_length = fh->f_io_array[sorted[i]].length;
|
||||
}
|
||||
merge_length += fh->f_io_array[sorted[i+1]].length;
|
||||
merge++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
/*
|
||||
printf ("RANK: %d Entries: %d MERGE: %d \n", fh->f_rank,fh->f_num_of_io_entries,merge);
|
||||
if (merge)
|
||||
printf ("Merge size: %d\n",merge_length);
|
||||
else
|
||||
printf ("entry size: %d\n",fh->f_io_array[sorted[i]].length);
|
||||
sleep(5);
|
||||
*/
|
||||
if (merge) {
|
||||
merge_buf = malloc (merge_length);
|
||||
k = 0;
|
||||
while (merge >= 0) {
|
||||
memcpy (merge_buf + k,
|
||||
fh->f_io_array[sorted[i-merge]].memory_address,
|
||||
fh->f_io_array[sorted[i-merge]].length);
|
||||
k += fh->f_io_array[sorted[i-merge]].length;
|
||||
merge --;
|
||||
}
|
||||
|
||||
ret = PVFS_Request_contiguous (merge_length,
|
||||
PVFS_BYTE,
|
||||
&mem_req);
|
||||
if (ret != 0) {
|
||||
perror("PVFS_Request_contiguous() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
ret = PVFS_Request_contiguous (merge_length,
|
||||
PVFS_BYTE,
|
||||
&file_req);
|
||||
if (ret != 0) {
|
||||
perror("PVFS_Request_contiguous() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
ret = PVFS_sys_write (pvfs2_fs->object_ref,
|
||||
file_req,
|
||||
merge_offset,
|
||||
merge_buf,
|
||||
mem_req,
|
||||
&(pvfs2_fs->credentials),
|
||||
&resp_io);
|
||||
if (ret != 0) {
|
||||
perror("PVFS_sys_write() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
merge = 0;
|
||||
merge_offset = 0;
|
||||
merge_length = 0;
|
||||
if (NULL != merge_buf) {
|
||||
free (merge_buf);
|
||||
merge_buf = NULL;
|
||||
}
|
||||
}
|
||||
else {
|
||||
ret = PVFS_Request_contiguous (fh->f_io_array[sorted[i]].length,
|
||||
PVFS_BYTE,
|
||||
&mem_req);
|
||||
if (ret != 0) {
|
||||
perror("PVFS_Request_contiguous() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
ret = PVFS_Request_contiguous (fh->f_io_array[sorted[i]].length,
|
||||
PVFS_BYTE,
|
||||
&file_req);
|
||||
if (ret != 0) {
|
||||
perror("PVFS_Request_contiguous() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
ret = PVFS_sys_write (pvfs2_fs->object_ref,
|
||||
file_req,
|
||||
(OMPI_MPI_OFFSET_TYPE)
|
||||
fh->f_io_array[sorted[i]].offset,
|
||||
fh->f_io_array[sorted[i]].memory_address,
|
||||
mem_req,
|
||||
&(pvfs2_fs->credentials),
|
||||
&resp_io);
|
||||
if (ret != 0) {
|
||||
perror("PVFS_sys_write() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
else {
|
||||
for (i=0 ; i<fh->f_num_of_io_entries ; i++) {
|
||||
if (fh->f_num_of_io_entries != i+1) {
|
||||
if (((OMPI_MPI_OFFSET_TYPE)fh->f_io_array[i].offset +
|
||||
(OPAL_PTRDIFF_TYPE)fh->f_io_array[i].length) ==
|
||||
(OMPI_MPI_OFFSET_TYPE)fh->f_io_array[i+1].offset) {
|
||||
if (!merge) {
|
||||
merge_offset = (OMPI_MPI_OFFSET_TYPE)
|
||||
fh->f_io_array[i].offset;
|
||||
merge_length = fh->f_io_array[i].length;
|
||||
}
|
||||
merge_length += fh->f_io_array[i+1].length;
|
||||
merge++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
/*
|
||||
printf ("RANK: %d Entries: %d MERGE: %d \n", fh->f_rank,fh->f_num_of_io_entries,merge);
|
||||
if (merge)
|
||||
printf ("Merge size: %lld\n",merge_length);
|
||||
else
|
||||
printf ("entry size: %lld\n",fh->f_io_array[i].length);
|
||||
sleep(2);
|
||||
*/
|
||||
if (merge) {
|
||||
merge_buf = malloc (merge_length);
|
||||
k = 0;
|
||||
while (merge >= 0) {
|
||||
memcpy (merge_buf + k,
|
||||
fh->f_io_array[i-merge].memory_address,
|
||||
fh->f_io_array[i-merge].length);
|
||||
k += fh->f_io_array[i-merge].length;
|
||||
merge --;
|
||||
}
|
||||
|
||||
ret = PVFS_Request_contiguous (merge_length,
|
||||
PVFS_BYTE,
|
||||
&mem_req);
|
||||
if (ret != 0) {
|
||||
perror("PVFS_Request_contiguous() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
ret = PVFS_Request_contiguous (merge_length,
|
||||
PVFS_BYTE,
|
||||
&file_req);
|
||||
if (ret != 0) {
|
||||
perror("PVFS_Request_contiguous() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
ret = PVFS_sys_write (pvfs2_fs->object_ref,
|
||||
file_req,
|
||||
merge_offset,
|
||||
merge_buf,
|
||||
mem_req,
|
||||
&(pvfs2_fs->credentials),
|
||||
&resp_io);
|
||||
if (ret != 0) {
|
||||
perror("PVFS_sys_write() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
merge = 0;
|
||||
merge_offset = 0;
|
||||
merge_length = 0;
|
||||
if (NULL != merge_buf) {
|
||||
free (merge_buf);
|
||||
merge_buf = NULL;
|
||||
}
|
||||
}
|
||||
else {
|
||||
ret = PVFS_Request_contiguous (fh->f_io_array[i].length,
|
||||
PVFS_BYTE,
|
||||
&mem_req);
|
||||
if (ret != 0) {
|
||||
perror("PVFS_Request_contiguous() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
ret = PVFS_Request_contiguous (fh->f_io_array[i].length,
|
||||
PVFS_BYTE,
|
||||
&file_req);
|
||||
if (ret != 0) {
|
||||
perror("PVFS_Request_contiguous() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
ret = PVFS_sys_write (pvfs2_fs->object_ref,
|
||||
file_req,
|
||||
(OMPI_MPI_OFFSET_TYPE)
|
||||
fh ->f_io_array[i].offset,
|
||||
fh->f_io_array[i].memory_address,
|
||||
mem_req,
|
||||
&(pvfs2_fs->credentials),
|
||||
&resp_io);
|
||||
if (ret != 0) {
|
||||
perror("PVFS_sys_write() error");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
40
ompi/mca/fcache/Makefile.am
Обычный файл
40
ompi/mca/fcache/Makefile.am
Обычный файл
@ -0,0 +1,40 @@
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
AM_CPPFLAGS = $(LTDLINCL)
|
||||
|
||||
# main library setup
|
||||
noinst_LTLIBRARIES = libmca_fcache.la
|
||||
libmca_fcache_la_SOURCES =
|
||||
|
||||
# local files
|
||||
headers = fcache.h
|
||||
libmca_fcache_la_SOURCES += $(headers)
|
||||
|
||||
# Conditionally install the header files
|
||||
if WANT_INSTALL_HEADERS
|
||||
ompidir = $(includedir)/openmpi/$(subdir)
|
||||
nobase_ompi_HEADERS = $(headers)
|
||||
endif
|
||||
|
||||
include base/Makefile.am
|
||||
|
||||
distclean-local:
|
||||
rm -f base/static-components.h
|
||||
28
ompi/mca/fcache/base/Makefile.am
Обычный файл
28
ompi/mca/fcache/base/Makefile.am
Обычный файл
@ -0,0 +1,28 @@
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
headers += \
|
||||
base/base.h
|
||||
|
||||
libmca_fcache_la_SOURCES += \
|
||||
base/fcache_base_close.c \
|
||||
base/fcache_base_file_select.c \
|
||||
base/fcache_base_file_unselect.c \
|
||||
base/fcache_base_find_available.c \
|
||||
base/fcache_base_open.c
|
||||
70
ompi/mca/fcache/base/base.h
Обычный файл
70
ompi/mca/fcache/base/base.h
Обычный файл
@ -0,0 +1,70 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
/**
|
||||
* @file
|
||||
*
|
||||
* MCA fcache base framework public interface functions.
|
||||
*/
|
||||
|
||||
#ifndef MCA_FCACHE_BASE_H
|
||||
#define MCA_FCACHE_BASE_H
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "opal/class/opal_list.h"
|
||||
#include "ompi/mca/fcache/fcache.h"
|
||||
#include "opal/mca/mca.h"
|
||||
|
||||
|
||||
BEGIN_C_DECLS
|
||||
|
||||
OMPI_DECLSPEC int mca_fcache_base_open(void);
|
||||
|
||||
OMPI_DECLSPEC int mca_fcache_base_close(void);
|
||||
|
||||
OMPI_DECLSPEC int mca_fcache_base_file_select(struct mca_io_ompio_file_t *file,
|
||||
mca_base_component_t *preferred);
|
||||
|
||||
OMPI_DECLSPEC int mca_fcache_base_file_unselect(struct mca_io_ompio_file_t *file);
|
||||
|
||||
OMPI_DECLSPEC int mca_fcache_base_find_available(bool enable_progress_threads,
|
||||
bool enable_mpi_threads);
|
||||
|
||||
OMPI_DECLSPEC int mca_fcache_base_init_file (struct mca_io_ompio_file_t *file);
|
||||
|
||||
OMPI_DECLSPEC int mca_fcache_base_get_param (struct mca_io_ompio_file_t *file, int keyval);
|
||||
/*
|
||||
* Globals
|
||||
*/
|
||||
|
||||
OMPI_DECLSPEC extern int mca_fcache_base_param;
|
||||
OMPI_DECLSPEC extern int mca_fcache_base_output;
|
||||
|
||||
OMPI_DECLSPEC extern bool mca_fcache_base_components_opened_valid;
|
||||
OMPI_DECLSPEC extern bool mca_fcache_base_components_available_valid;
|
||||
|
||||
OMPI_DECLSPEC extern opal_list_t mca_fcache_base_components_opened;
|
||||
OMPI_DECLSPEC extern opal_list_t mca_fcache_base_components_available;
|
||||
|
||||
END_C_DECLS
|
||||
|
||||
#endif /* MCA_BASE_FCACHE_H */
|
||||
56
ompi/mca/fcache/base/fcache_base_close.c
Обычный файл
56
ompi/mca/fcache/base/fcache_base_close.c
Обычный файл
@ -0,0 +1,56 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHTOB$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include <stdio.h>
|
||||
|
||||
#include "ompi/constants.h"
|
||||
#include "opal/mca/mca.h"
|
||||
#include "opal/mca/base/base.h"
|
||||
#include "ompi/mca/fcache/fcache.h"
|
||||
#include "ompi/mca/fcache/base/base.h"
|
||||
|
||||
int mca_fcache_base_close(void)
|
||||
{
|
||||
/*
|
||||
Close all components that are still open. This may be the opened
|
||||
list (if we're in ompi_info), or it may be the available list (if
|
||||
we're anywhere else).
|
||||
*/
|
||||
|
||||
if (mca_fcache_base_components_opened_valid) {
|
||||
mca_base_components_close(mca_fcache_base_output,
|
||||
&mca_fcache_base_components_opened, NULL);
|
||||
OBJ_DESTRUCT(&mca_fcache_base_components_opened);
|
||||
mca_fcache_base_components_opened_valid = false;
|
||||
} else if (mca_fcache_base_components_available_valid) {
|
||||
mca_base_components_close(mca_fcache_base_output,
|
||||
&mca_fcache_base_components_available, NULL);
|
||||
OBJ_DESTRUCT(&mca_fcache_base_components_available);
|
||||
mca_fcache_base_components_available_valid = false;
|
||||
}
|
||||
|
||||
/* Close the output stream for this framework */
|
||||
opal_output_close (mca_fcache_base_output);
|
||||
|
||||
/* All done */
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
358
ompi/mca/fcache/base/fcache_base_file_select.c
Обычный файл
358
ompi/mca/fcache/base/fcache_base_file_select.c
Обычный файл
@ -0,0 +1,358 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include <string.h>
|
||||
|
||||
#include "opal/class/opal_list.h"
|
||||
#include "opal/util/argv.h"
|
||||
#include "opal/mca/mca.h"
|
||||
#include "opal/mca/base/base.h"
|
||||
#include "orte/util/show_help.h"
|
||||
#include "ompi/mca/fcache/fcache.h"
|
||||
#include "ompi/mca/fcache/base/base.h"
|
||||
#include "ompi/mca/io/ompio/io_ompio.h"
|
||||
|
||||
/*
|
||||
* This structure is needed so that we can close the modules
|
||||
* which are not selected but were opened. mca_base_modules_close
|
||||
* which does this job for us requires a opal_list_t which contains
|
||||
* these modules
|
||||
*/
|
||||
struct queried_module_t {
|
||||
opal_list_item_t super;
|
||||
mca_fcache_base_component_t *om_component;
|
||||
mca_fcache_base_module_t *om_module;
|
||||
};
|
||||
typedef struct queried_module_t queried_module_t;
|
||||
static OBJ_CLASS_INSTANCE(queried_module_t, opal_list_item_t, NULL, NULL);
|
||||
|
||||
|
||||
/*
|
||||
* Only one fcache module can be attached to each file.
|
||||
*
|
||||
* This module calls the query funtion on all the components that were
|
||||
* detected by fcache_base_open. This function is called on a
|
||||
* per-file basis. This function has the following function.
|
||||
*
|
||||
* 1. Iterate over the list of available_components
|
||||
* 2. Call the query function on each of these components.
|
||||
* 3. query function returns the structure containing pointers
|
||||
* to its module and its priority
|
||||
* 4. Select the module with the highest priority
|
||||
* 5. Call the init function on the selected module so that it does the
|
||||
* right setup for the file
|
||||
* 6. Call finalize on all the other modules which returned
|
||||
* their module but were unfortunate to not get selected
|
||||
*/
|
||||
|
||||
int mca_fcache_base_file_select (struct mca_io_ompio_file_t *file,
|
||||
mca_base_component_t *preferred)
|
||||
{
|
||||
int priority;
|
||||
int best_priority;
|
||||
opal_list_item_t *item;
|
||||
opal_list_item_t *next_item;
|
||||
mca_base_component_priority_list_item_t *selectable_item;
|
||||
char *names, **name_array;
|
||||
int num_names;
|
||||
mca_base_component_priority_list_item_t *cpli;
|
||||
mca_fcache_base_component_t *component;
|
||||
mca_fcache_base_component_t *best_component;
|
||||
mca_fcache_base_module_t *module;
|
||||
opal_list_t queried;
|
||||
queried_module_t *om;
|
||||
opal_list_t *selectable;
|
||||
char *str;
|
||||
int err = MPI_SUCCESS;
|
||||
int i;
|
||||
bool was_selectable_constructed = false;
|
||||
|
||||
/* Check and see if a preferred component was provided. If it was
|
||||
provided then it should be used (if possible) */
|
||||
|
||||
if (NULL != preferred) {
|
||||
|
||||
/* We have a preferred component. Check if it is available
|
||||
and if so, whether it wants to run */
|
||||
|
||||
str = &(preferred->mca_component_name[0]);
|
||||
|
||||
opal_output_verbose(10, mca_fcache_base_output,
|
||||
"fcache:base:file_select: Checking preferred component: %s",
|
||||
str);
|
||||
|
||||
/* query the component for its priority and get its module
|
||||
structure. This is necessary to proceed */
|
||||
|
||||
component = (mca_fcache_base_component_t *)preferred;
|
||||
module = component->fcachem_file_query (&priority);
|
||||
if (NULL != module &&
|
||||
NULL != module->fcache_module_init) {
|
||||
|
||||
/* this query seems to have returned something legitimate
|
||||
* and we can now go ahead and initialize the
|
||||
* file with it * but first, the functions which
|
||||
* are null need to be filled in */
|
||||
|
||||
/*fill_null_pointers (module);*/
|
||||
file->f_fcache = module;
|
||||
file->f_fcache_component = preferred;
|
||||
|
||||
return module->fcache_module_init(file);
|
||||
}
|
||||
/* His preferred component is present, but is unable to
|
||||
* run. This is not a good sign. We should try selecting
|
||||
* some other component We let it fall through and select
|
||||
* from the list of available components
|
||||
*/
|
||||
} /*end of selection for preferred component */
|
||||
|
||||
/*
|
||||
* We fall till here if one of the two things happened:
|
||||
* 1. The preferred component was provided but for some reason was
|
||||
* not able to be selected
|
||||
* 2. No preferred component was provided
|
||||
*
|
||||
* All we need to do is to go through the list of available
|
||||
* components and find the one which has the highest priority and
|
||||
* use that for this file
|
||||
*/
|
||||
|
||||
/* Check if anything was requested by means on the name parameters */
|
||||
names = NULL;
|
||||
mca_base_param_lookup_string (mca_fcache_base_param, &names);
|
||||
|
||||
if (NULL != names && 0 < strlen(names)) {
|
||||
name_array = opal_argv_split (names, ',');
|
||||
num_names = opal_argv_count (name_array);
|
||||
|
||||
opal_output_verbose(10, mca_fcache_base_output,
|
||||
"fcache:base:file_Select: Checking all available module");
|
||||
|
||||
/* since there are somethings which the mca requested through the
|
||||
if the intersection is NULL, then we barf saying that the requested
|
||||
modules are not being available */
|
||||
|
||||
selectable = OBJ_NEW(opal_list_t);
|
||||
was_selectable_constructed = true;
|
||||
|
||||
/* go through the compoents_available list and check against the names
|
||||
* to see whether this can be added or not */
|
||||
|
||||
for (item = opal_list_get_first(&mca_fcache_base_components_available);
|
||||
item != opal_list_get_end(&mca_fcache_base_components_available);
|
||||
item = opal_list_get_next(item)) {
|
||||
/* convert the opal_list_item_t returned into the proper type */
|
||||
cpli = (mca_base_component_priority_list_item_t *) item;
|
||||
component = (mca_fcache_base_component_t *) cpli->super.cli_component;
|
||||
opal_output_verbose(10, mca_fcache_base_output,
|
||||
"select: initialising %s component %s",
|
||||
component->fcachem_version.mca_type_name,
|
||||
component->fcachem_version.mca_component_name);
|
||||
|
||||
/* check if this name is present in the mca_base_params */
|
||||
for (i=0; i < num_names; i++) {
|
||||
if (0 == strcmp(name_array[i], component->fcachem_version.mca_component_name)) {
|
||||
/* this is present, and should be added o the selectable list */
|
||||
|
||||
/* We need to create a seperate object to initialise this list with
|
||||
* since we cannot have the same item in 2 lists */
|
||||
|
||||
selectable_item = OBJ_NEW (mca_base_component_priority_list_item_t);
|
||||
*selectable_item = *cpli;
|
||||
opal_list_append (selectable, (opal_list_item_t *)selectable_item);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* check for a NULL intersection between the available list and the
|
||||
* list which was asked for */
|
||||
|
||||
if (0 == opal_list_get_size(selectable)) {
|
||||
was_selectable_constructed = true;
|
||||
OBJ_RELEASE (selectable);
|
||||
opal_output_verbose (10, mca_fcache_base_output,
|
||||
"fcache:base:file_select: preferred modules were not available");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
} else { /* if there was no name_array, then we need to simply initialize
|
||||
selectable to mca_fcache_base_components_available */
|
||||
selectable = &mca_fcache_base_components_available;
|
||||
}
|
||||
|
||||
best_component = NULL;
|
||||
best_priority = -1;
|
||||
OBJ_CONSTRUCT(&queried, opal_list_t);
|
||||
|
||||
for (item = opal_list_get_first(selectable);
|
||||
item != opal_list_get_end(selectable);
|
||||
item = opal_list_get_next(item)) {
|
||||
/*
|
||||
* convert the opal_list_item_t returned into the proper type
|
||||
*/
|
||||
cpli = (mca_base_component_priority_list_item_t *) item;
|
||||
component = (mca_fcache_base_component_t *) cpli->super.cli_component;
|
||||
opal_output_verbose(10, mca_fcache_base_output,
|
||||
"select: initialising %s component %s",
|
||||
component->fcachem_version.mca_type_name,
|
||||
component->fcachem_version.mca_component_name);
|
||||
|
||||
/*
|
||||
* we can call the query function only if there is a function :-)
|
||||
*/
|
||||
if (NULL == component->fcachem_file_query) {
|
||||
opal_output_verbose(10, mca_fcache_base_output,
|
||||
"select: no query, ignoring the component");
|
||||
} else {
|
||||
/*
|
||||
* call the query function and see what it returns
|
||||
*/
|
||||
module = component->fcachem_file_query (&priority);
|
||||
|
||||
if (NULL == module ||
|
||||
NULL == module->fcache_module_init) {
|
||||
/*
|
||||
* query did not return any action which can be used
|
||||
*/
|
||||
opal_output_verbose(10, mca_fcache_base_output,
|
||||
"select: query returned failure");
|
||||
} else {
|
||||
opal_output_verbose(10, mca_fcache_base_output,
|
||||
"select: query returned priority %d",
|
||||
priority);
|
||||
/*
|
||||
* is this the best component we have found till now?
|
||||
*/
|
||||
if (priority > best_priority) {
|
||||
best_priority = priority;
|
||||
best_component = component;
|
||||
}
|
||||
|
||||
om = OBJ_NEW(queried_module_t);
|
||||
/*
|
||||
* check if we have run out of space
|
||||
*/
|
||||
if (NULL == om) {
|
||||
OBJ_DESTRUCT(&queried);
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
om->om_component = component;
|
||||
om->om_module = module;
|
||||
opal_list_append(&queried, (opal_list_item_t *)om);
|
||||
} /* end else of if (NULL == module) */
|
||||
} /* end else of if (NULL == component->fcachem_init) */
|
||||
} /* end for ... end of traversal */
|
||||
|
||||
/* We have to remove empty out the selectable list if the selectable
|
||||
* list was constructed as a duplicate and not as a pointer to the
|
||||
* mca_base_components_available list. So, check and destroy */
|
||||
|
||||
if (was_selectable_constructed) {
|
||||
|
||||
/* remove all the items first */
|
||||
for (item = opal_list_get_first(&mca_fcache_base_components_available);
|
||||
item != opal_list_get_end(&mca_fcache_base_components_available);
|
||||
item = next_item) {
|
||||
next_item = opal_list_get_next(item);
|
||||
OBJ_RELEASE (item);
|
||||
}
|
||||
|
||||
/* release the list itself */
|
||||
OBJ_RELEASE (selectable);
|
||||
was_selectable_constructed = false;
|
||||
}
|
||||
|
||||
/*
|
||||
* Now we have alist of components which successfully returned
|
||||
* their module struct. One of these components has the best
|
||||
* priority. The rest have to be comm_unqueried to counter the
|
||||
* effects of file_query'ing them. Finalize happens only on
|
||||
* components which should are initialized.
|
||||
*/
|
||||
if (NULL == best_component) {
|
||||
/*
|
||||
* This typically means that there was no component which was
|
||||
* able to run properly this time. So, we need to abort
|
||||
* JMS replace with show_help
|
||||
*/
|
||||
OBJ_DESTRUCT(&queried);
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
/*
|
||||
* We now have a list of components which have successfully
|
||||
* returned their priorities from the query. We now have to
|
||||
* unquery() those components which have not been selected and
|
||||
* init() the component which was selected
|
||||
*/
|
||||
for (item = opal_list_remove_first(&queried);
|
||||
NULL != item;
|
||||
item = opal_list_remove_first(&queried)) {
|
||||
om = (queried_module_t *) item;
|
||||
if (om->om_component == best_component) {
|
||||
/*
|
||||
* this is the chosen component, we have to initialise the
|
||||
* module of this component.
|
||||
*
|
||||
* ANJU: a component might not have all the functions
|
||||
* defined. Whereever a function pointer is null in the
|
||||
* module structure we need to fill it in with the base
|
||||
* structure function pointers. This is yet to be done
|
||||
*/
|
||||
|
||||
/*
|
||||
* We don return here coz we still need to go through and
|
||||
* elease the other objects
|
||||
*/
|
||||
|
||||
/*fill_null_pointers (om->om_module);*/
|
||||
file->f_fcache = om->om_module;
|
||||
err = om->om_module->fcache_module_init(file);
|
||||
file->f_fcache_component = (mca_base_component_t *)best_component;
|
||||
|
||||
} else {
|
||||
/*
|
||||
* this is not the "choosen one", finalize
|
||||
*/
|
||||
if (NULL != om->om_component->fcachem_file_unquery) {
|
||||
/* unquery the component only if they have some clean
|
||||
* up job to do. Components which are queried but do
|
||||
* not actually do anything typically do not have a
|
||||
* unquery. Hence this check is necessary
|
||||
*/
|
||||
(void) om->om_component->fcachem_file_unquery(file);
|
||||
opal_output_verbose(10, mca_fcache_base_output,
|
||||
"select: component %s is not selected",
|
||||
om->om_component->fcachem_version.mca_component_name);
|
||||
} /* end if */
|
||||
} /* if not best component */
|
||||
OBJ_RELEASE(om);
|
||||
} /* traversing through the entire list */
|
||||
|
||||
opal_output_verbose(10, mca_fcache_base_output,
|
||||
"select: component %s selected",
|
||||
best_component->fcachem_version.mca_component_name);
|
||||
|
||||
OBJ_DESTRUCT(&queried);
|
||||
|
||||
return err;
|
||||
}
|
||||
41
ompi/mca/fcache/base/fcache_base_file_unselect.c
Обычный файл
41
ompi/mca/fcache/base/fcache_base_file_unselect.c
Обычный файл
@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/mca/io/ompio/io_ompio.h"
|
||||
#include "opal/util/show_help.h"
|
||||
#include "opal/mca/mca.h"
|
||||
#include "opal/mca/base/base.h"
|
||||
#include "ompi/mca/fcache/fcache.h"
|
||||
#include "ompi/mca/fcache/base/base.h"
|
||||
|
||||
int mca_fcache_base_file_unselect(mca_io_ompio_file_t *file)
|
||||
{
|
||||
if (NULL != file->f_fcache && NULL != file->f_fcache->fcache_module_finalize) {
|
||||
return file->f_fcache->fcache_module_finalize(file);
|
||||
}
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
170
ompi/mca/fcache/base/fcache_base_find_available.c
Обычный файл
170
ompi/mca/fcache/base/fcache_base_find_available.c
Обычный файл
@ -0,0 +1,170 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "opal/class/opal_list.h"
|
||||
#include "orte/util/show_help.h"
|
||||
#include "opal/mca/mca.h"
|
||||
#include "opal/mca/base/base.h"
|
||||
#include "opal/mca/base/mca_base_component_repository.h"
|
||||
#include "ompi/mca/fcache/fcache.h"
|
||||
#include "ompi/mca/fcache/base/base.h"
|
||||
|
||||
opal_list_t mca_fcache_base_modules_available;
|
||||
bool mca_fcache_base_modules_available_valid = false;
|
||||
|
||||
static int init_query(const mca_base_component_t *m,
|
||||
mca_base_component_priority_list_item_t *entry,
|
||||
bool enable_progress_threads,
|
||||
bool enable_mpi_threads);
|
||||
static int init_query_2_0_0(const mca_base_component_t *component,
|
||||
mca_base_component_priority_list_item_t *entry,
|
||||
bool enable_progress_threads,
|
||||
bool enable_mpi_threads);
|
||||
|
||||
int mca_fcache_base_find_available(bool enable_progress_threads,
|
||||
bool enable_mpi_threads)
|
||||
{
|
||||
bool found = false;
|
||||
mca_base_component_priority_list_item_t *entry;
|
||||
opal_list_item_t *p;
|
||||
|
||||
/* Initialize the list */
|
||||
|
||||
OBJ_CONSTRUCT(&mca_fcache_base_components_available, opal_list_t);
|
||||
mca_fcache_base_components_available_valid = true;
|
||||
|
||||
/* The list of components which we should check is already present
|
||||
in mca_fcache_base_components_opened, which was established in
|
||||
mca_fcache_base_open */
|
||||
|
||||
for (found = false,
|
||||
p = opal_list_remove_first (&mca_fcache_base_components_opened);
|
||||
NULL != p;
|
||||
p = opal_list_remove_first (&mca_fcache_base_components_opened)) {
|
||||
entry = OBJ_NEW(mca_base_component_priority_list_item_t);
|
||||
entry->super.cli_component =
|
||||
((mca_base_component_list_item_t *)p)->cli_component;
|
||||
|
||||
/* Now for this entry, we have to determine the thread level. Call
|
||||
a subroutine to do the job for us */
|
||||
|
||||
if (OMPI_SUCCESS == init_query(entry->super.cli_component, entry,
|
||||
enable_progress_threads,
|
||||
enable_mpi_threads)) {
|
||||
/* Save the results in the list. The priority is not relvant at
|
||||
this point in time. But we save the thread arguments so that
|
||||
the initial selection algorithm can negotiate overall thread
|
||||
level for this process */
|
||||
entry->cpli_priority = 0;
|
||||
opal_list_append (&mca_fcache_base_components_available,
|
||||
(opal_list_item_t *) entry);
|
||||
found = true;
|
||||
} else {
|
||||
/* The component does not want to run, so close it. Its close()
|
||||
has already been invoked. Close it out of the DSO repository
|
||||
(if it is there in the repository) */
|
||||
mca_base_component_repository_release(entry->super.cli_component);
|
||||
OBJ_RELEASE(entry);
|
||||
}
|
||||
/* Free entry from the "opened" list */
|
||||
OBJ_RELEASE(p);
|
||||
}
|
||||
|
||||
/* The opened list is no longer necessary, so we can free it */
|
||||
OBJ_DESTRUCT (&mca_fcache_base_components_opened);
|
||||
mca_fcache_base_components_opened_valid = false;
|
||||
|
||||
/* There should atleast be one fcache component which was available */
|
||||
if (false == found) {
|
||||
/* Need to free all items in the list */
|
||||
OBJ_DESTRUCT(&mca_fcache_base_components_available);
|
||||
mca_fcache_base_components_available_valid = false;
|
||||
opal_output_verbose (10, mca_fcache_base_output,
|
||||
"fcache:find_available: no fcache components available!");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
/* All done */
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
static int init_query(const mca_base_component_t *m,
|
||||
mca_base_component_priority_list_item_t *entry,
|
||||
bool enable_progress_threads,
|
||||
bool enable_mpi_threads)
|
||||
{
|
||||
int ret;
|
||||
|
||||
opal_output_verbose(10, mca_fcache_base_output,
|
||||
"fcache:find_available: querying fcache component %s",
|
||||
m->mca_component_name);
|
||||
|
||||
/* This component has been successfully opened, now try to query it */
|
||||
if (2 == m->mca_type_major_version &&
|
||||
0 == m->mca_type_minor_version &&
|
||||
0 == m->mca_type_release_version) {
|
||||
ret = init_query_2_0_0(m, entry, enable_progress_threads,
|
||||
enable_mpi_threads);
|
||||
} else {
|
||||
/* unrecognised API version */
|
||||
opal_output_verbose(10, mca_fcache_base_output,
|
||||
"fcache:find_available:unrecognised fcache API version (%d.%d.%d)",
|
||||
m->mca_type_major_version,
|
||||
m->mca_type_minor_version,
|
||||
m->mca_type_release_version);
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
/* Query done -- look at return value to see what happened */
|
||||
if (OMPI_SUCCESS != ret) {
|
||||
opal_output_verbose(10, mca_fcache_base_output,
|
||||
"fcache:find_available fcache component %s is not available",
|
||||
m->mca_component_name);
|
||||
if (NULL != m->mca_close_component) {
|
||||
m->mca_close_component();
|
||||
}
|
||||
} else {
|
||||
opal_output_verbose(10, mca_fcache_base_output,
|
||||
"fcache:find_avalable: fcache component %s is available",
|
||||
m->mca_component_name);
|
||||
|
||||
}
|
||||
/* All done */
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
static int init_query_2_0_0(const mca_base_component_t *component,
|
||||
mca_base_component_priority_list_item_t *entry,
|
||||
bool enable_progress_threads,
|
||||
bool enable_mpi_threads)
|
||||
{
|
||||
mca_fcache_base_component_2_0_0_t *fcache = (mca_fcache_base_component_2_0_0_t *) component;
|
||||
|
||||
return fcache->fcachem_init_query(enable_progress_threads,
|
||||
enable_mpi_threads);
|
||||
}
|
||||
88
ompi/mca/fcache/base/fcache_base_open.c
Обычный файл
88
ompi/mca/fcache/base/fcache_base_open.c
Обычный файл
@ -0,0 +1,88 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include <stdio.h>
|
||||
|
||||
#include "ompi/class/ompi_free_list.h"
|
||||
#include "orte/util/show_help.h"
|
||||
#include "opal/mca/mca.h"
|
||||
#include "opal/mca/base/base.h"
|
||||
#include "opal/mca/base/mca_base_param.h"
|
||||
|
||||
#include "orte/util/show_help.h"
|
||||
|
||||
#include "ompi/mca/fcache/fcache.h"
|
||||
#include "ompi/mca/fcache/base/base.h"
|
||||
|
||||
|
||||
/*
|
||||
* The following file was created by configure. It contains extern
|
||||
* statements and the definition of an array of pointers to each
|
||||
* component's public mca_base_component_t struct.
|
||||
*/
|
||||
#ifdef __WINDOWS__
|
||||
const mca_base_component_t *mca_fcache_base_static_components[] = {NULL};
|
||||
#else
|
||||
#include "ompi/mca/fcache/base/static-components.h"
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Global variables; most of which are loaded by back-ends of MCA
|
||||
* variables
|
||||
*/
|
||||
int mca_fcache_base_param = -1;
|
||||
int mca_fcache_base_output = -1;
|
||||
|
||||
opal_list_t mca_fcache_base_components_opened;
|
||||
opal_list_t mca_fcache_base_components_available;
|
||||
|
||||
bool mca_fcache_base_components_available_valid = false;
|
||||
bool mca_fcache_base_components_opened_valid = false;
|
||||
|
||||
mca_fcache_base_component_t mca_fcache_base_selected_component;
|
||||
mca_fcache_base_module_t mca_fcache;
|
||||
|
||||
/*
|
||||
* Function for finding and opening either all MCA components, or the one
|
||||
* that was specifically requested via a MCA parameter.
|
||||
*/
|
||||
int mca_fcache_base_open(void)
|
||||
{
|
||||
/* Open an output stream for this framework */
|
||||
|
||||
mca_fcache_base_output = opal_output_open(NULL);
|
||||
|
||||
/* Open up all available components */
|
||||
|
||||
if (OMPI_SUCCESS !=
|
||||
mca_base_components_open("fcache", mca_fcache_base_output,
|
||||
mca_fcache_base_static_components,
|
||||
&mca_fcache_base_components_opened, true)) {
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
mca_fcache_base_components_opened_valid = true;
|
||||
|
||||
/* Find the index of the MCA "fcache" param for selection */
|
||||
|
||||
mca_fcache_base_param = mca_base_param_find("fcache", "base", NULL);
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
18
ompi/mca/fcache/base/static-components.h
Обычный файл
18
ompi/mca/fcache/base/static-components.h
Обычный файл
@ -0,0 +1,18 @@
|
||||
/*
|
||||
* $HEADER$
|
||||
*/
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
const mca_base_component_t *mca_fcache_base_static_components[] = {
|
||||
|
||||
NULL
|
||||
};
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
143
ompi/mca/fcache/fcache.h
Обычный файл
143
ompi/mca/fcache/fcache.h
Обычный файл
@ -0,0 +1,143 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#ifndef OMPI_MCA_FCACHE_H
|
||||
#define OMPI_MCA_FCACHE_H
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "mpi.h"
|
||||
#include "opal/mca/mca.h"
|
||||
#include "opal/mca/base/base.h"
|
||||
|
||||
BEGIN_C_DECLS
|
||||
|
||||
struct mca_io_ompio_file_t;
|
||||
struct mca_io_ompio_io_servers;
|
||||
/*
|
||||
* Macro for use in components that are of type coll
|
||||
*/
|
||||
#define MCA_FCACHE_BASE_VERSION_2_0_0 \
|
||||
MCA_BASE_VERSION_2_0_0, \
|
||||
"fcache", 2, 0, 0
|
||||
|
||||
/*
|
||||
* These are the component function prototypes. These function pointers
|
||||
* go into the component structure. These functions (query() and finalize()
|
||||
* are called during fcache_base_select(). Each component is query() ied
|
||||
* and subsequently, all the unselected components are finalize() 'ed
|
||||
* so that any *stuff* they did during query() can be undone. By
|
||||
* similar logic, finalize() is also called on the component which
|
||||
* was selected when the communicator is being destroyed.
|
||||
*
|
||||
* So, to sum it up, every component carries 4 functions:
|
||||
* 1. open() - called during MPI_INIT
|
||||
* 2. close() - called during MPI_FINALIZE
|
||||
* 3. query() - called to select a particular component
|
||||
* 4. finalize() - called when actions taken during query have
|
||||
* to be undone
|
||||
*/
|
||||
|
||||
/*
|
||||
* **************** component struct *******************************
|
||||
* *********** These functions go in the component struct **********
|
||||
* **************** component struct *******************************
|
||||
*/
|
||||
|
||||
typedef int (*mca_fcache_base_component_init_query_1_0_0_fn_t)
|
||||
(bool enable_progress_threads,
|
||||
bool enable_mpi_threads);
|
||||
|
||||
typedef struct mca_fcache_base_module_1_0_0_t *
|
||||
(*mca_fcache_base_component_file_query_1_0_0_fn_t) (int *priority);
|
||||
|
||||
typedef int (*mca_fcache_base_component_file_unquery_1_0_0_fn_t)
|
||||
(struct mca_io_ompio_file_t *file);
|
||||
|
||||
/*
|
||||
* ****************** component struct ******************************
|
||||
* Structure for fcache v2.0.0 components.This is chained to MCA v2.0.0
|
||||
* ****************** component struct ******************************
|
||||
*/
|
||||
struct mca_fcache_base_component_2_0_0_t {
|
||||
mca_base_component_t fcachem_version;
|
||||
mca_base_component_data_t fcachem_data;
|
||||
|
||||
mca_fcache_base_component_init_query_1_0_0_fn_t fcachem_init_query;
|
||||
mca_fcache_base_component_file_query_1_0_0_fn_t fcachem_file_query;
|
||||
mca_fcache_base_component_file_unquery_1_0_0_fn_t fcachem_file_unquery;
|
||||
};
|
||||
typedef struct mca_fcache_base_component_2_0_0_t mca_fcache_base_component_2_0_0_t;
|
||||
typedef struct mca_fcache_base_component_2_0_0_t mca_fcache_base_component_t;
|
||||
|
||||
/*
|
||||
* ***********************************************************************
|
||||
* ************************ Interface function definitions **************
|
||||
* These are the typedefcache for the function pointers to various fcache
|
||||
* backend functions which will be used by the various fcache components
|
||||
* ***********************************************************************
|
||||
*/
|
||||
|
||||
typedef int (*mca_fcache_base_module_init_1_0_0_fn_t)
|
||||
(struct mca_io_ompio_file_t *file);
|
||||
|
||||
typedef int (*mca_fcache_base_module_finalize_1_0_0_fn_t)
|
||||
(struct mca_io_ompio_file_t *file);
|
||||
|
||||
typedef int (*mca_fcache_base_module_get_file_layout_fn_t)(
|
||||
char* filename,
|
||||
int *num_io_servers,
|
||||
size_t *depth,
|
||||
int *file_io_servers);
|
||||
|
||||
typedef int (*mca_fcache_base_module_set_file_layout_fn_t)(
|
||||
char* filename,
|
||||
int *num_io_servers,
|
||||
size_t *depth,
|
||||
int *file_io_servers);
|
||||
|
||||
typedef int (*mca_fcache_base_module_get_io_servers_fn_t)(
|
||||
char* filename,
|
||||
struct mca_io_ompio_io_servers *io_servers,
|
||||
int num_io_servers);
|
||||
|
||||
/*
|
||||
* ***********************************************************************
|
||||
* *************************** module structure *************************
|
||||
* ***********************************************************************
|
||||
*/
|
||||
struct mca_fcache_base_module_1_0_0_t {
|
||||
/*
|
||||
* Per-file initialization function. This is called only
|
||||
* on the module which is selected. The finalize corresponding to
|
||||
* this function is present on the component struct above
|
||||
*/
|
||||
mca_fcache_base_module_init_1_0_0_fn_t fcache_module_init;
|
||||
mca_fcache_base_module_finalize_1_0_0_fn_t fcache_module_finalize;
|
||||
|
||||
/* FCACHE function pointers */
|
||||
mca_fcache_base_module_get_file_layout_fn_t fcache_get_file_layout;
|
||||
mca_fcache_base_module_set_file_layout_fn_t fcache_set_file_layout;
|
||||
mca_fcache_base_module_get_io_servers_fn_t fcache_get_io_servers;
|
||||
};
|
||||
typedef struct mca_fcache_base_module_1_0_0_t mca_fcache_base_module_1_0_0_t;
|
||||
typedef mca_fcache_base_module_1_0_0_t mca_fcache_base_module_t;
|
||||
|
||||
END_C_DECLS
|
||||
|
||||
#endif /* OMPI_MCA_FCACHE_H */
|
||||
49
ompi/mca/fcache/ux/Makefile.am
Обычный файл
49
ompi/mca/fcache/ux/Makefile.am
Обычный файл
@ -0,0 +1,49 @@
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
# Make the output library in this directory, and name it either
|
||||
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
|
||||
# (for static builds).
|
||||
|
||||
if MCA_BUILD_ompi_fcache_ux_DSO
|
||||
component_noinst =
|
||||
component_install = mca_fcache_ux.la
|
||||
else
|
||||
component_noinst = libmca_fcache_ux.la
|
||||
component_install =
|
||||
endif
|
||||
|
||||
mcacomponentdir = $(pkglibdir)
|
||||
mcacomponent_LTLIBRARIES = $(component_install)
|
||||
mca_fcache_ux_la_SOURCES = $(sources)
|
||||
mca_fcache_ux_la_LDFLAGS = -module -avoid-version
|
||||
|
||||
noinst_LTLIBRARIES = $(component_noinst)
|
||||
libmca_fcache_ux_la_SOURCES = $(sources)
|
||||
libmca_fcache_ux_la_LDFLAGS = -module -avoid-version
|
||||
|
||||
# Source files
|
||||
|
||||
sources = \
|
||||
fcache_ux.h \
|
||||
fcache_ux.c \
|
||||
fcache_ux_component.c \
|
||||
fcache_ux_get_file_layout.c \
|
||||
fcache_ux_set_file_layout.c \
|
||||
fcache_ux_get_io_servers.c
|
||||
83
ompi/mca/fcache/ux/fcache_ux.c
Обычный файл
83
ompi/mca/fcache/ux/fcache_ux.c
Обычный файл
@ -0,0 +1,83 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2006 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*
|
||||
* These symbols are in a file by themselves to provide nice linker
|
||||
* semantics. Since linkers generally pull in symbols by object fules,
|
||||
* keeping these symbols as the only symbols in this file prevents
|
||||
* utility programs such as "ompi_info" from having to import entire
|
||||
* modules just to query their version and parameters
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "mpi.h"
|
||||
#include "ompi/mca/fcache/fcache.h"
|
||||
#include "ompi/mca/fcache/ux/fcache_ux.h"
|
||||
|
||||
/*
|
||||
* *******************************************************************
|
||||
* ************************ actions structure ************************
|
||||
* *******************************************************************
|
||||
*/
|
||||
static mca_fcache_base_module_1_0_0_t ux = {
|
||||
mca_fcache_ux_module_init, /* initalise after being selected */
|
||||
mca_fcache_ux_module_finalize, /* close a module on a communicator */
|
||||
mca_fcache_ux_get_file_layout,
|
||||
mca_fcache_ux_set_file_layout,
|
||||
mca_fcache_ux_get_io_servers
|
||||
};
|
||||
/*
|
||||
* *******************************************************************
|
||||
* ************************* structure ends **************************
|
||||
* *******************************************************************
|
||||
*/
|
||||
|
||||
int mca_fcache_ux_component_init_query(bool enable_progress_threads,
|
||||
bool enable_mpi_threads)
|
||||
{
|
||||
/* Nothing to do */
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
struct mca_fcache_base_module_1_0_0_t *
|
||||
mca_fcache_ux_component_file_query (int *priority)
|
||||
{
|
||||
*priority = 20;
|
||||
|
||||
return &ux;
|
||||
}
|
||||
|
||||
int mca_fcache_ux_component_file_unquery (mca_io_ompio_file_t *file)
|
||||
{
|
||||
/* This function might be needed for some purposes later. for now it
|
||||
* does not have anything to do since there are no steps which need
|
||||
* to be undone if this module is not selected */
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
int mca_fcache_ux_module_init (mca_io_ompio_file_t *file)
|
||||
{
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
int mca_fcache_ux_module_finalize (mca_io_ompio_file_t *file)
|
||||
{
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
68
ompi/mca/fcache/ux/fcache_ux.h
Обычный файл
68
ompi/mca/fcache/ux/fcache_ux.h
Обычный файл
@ -0,0 +1,68 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2006 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#ifndef MCA_FCACHE_UX_H
|
||||
#define MCA_FCACHE_UX_H
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "opal/mca/mca.h"
|
||||
#include "ompi/mca/fcache/fcache.h"
|
||||
#include "ompi/mca/io/ompio/io_ompio.h"
|
||||
|
||||
|
||||
BEGIN_C_DECLS
|
||||
|
||||
int mca_fcache_ux_component_init_query(bool enable_progress_threads,
|
||||
bool enable_mpi_threads);
|
||||
struct mca_fcache_base_module_1_0_0_t *
|
||||
mca_fcache_ux_component_file_query (int *priority);
|
||||
int mca_fcache_ux_component_file_unquery (mca_io_ompio_file_t *file);
|
||||
|
||||
int mca_fcache_ux_module_init (mca_io_ompio_file_t *file);
|
||||
int mca_fcache_ux_module_finalize (mca_io_ompio_file_t *file);
|
||||
|
||||
OMPI_MODULE_DECLSPEC extern mca_fcache_base_component_2_0_0_t mca_fcache_ux_component;
|
||||
/*
|
||||
* ******************************************************************
|
||||
* ********* functions which are implemented in this module *********
|
||||
* ******************************************************************
|
||||
*/
|
||||
|
||||
int mca_fcache_ux_get_file_layout (char* filename,
|
||||
int *num_io_servers,
|
||||
size_t *depth,
|
||||
int *file_io_servers);
|
||||
|
||||
int mca_fcache_ux_set_file_layout (char* filename,
|
||||
int *num_io_servers,
|
||||
size_t *depth,
|
||||
int *file_io_servers);
|
||||
|
||||
int mca_fcache_ux_get_io_servers (char* filename,
|
||||
struct mca_io_ompio_io_servers *io_servers,
|
||||
int num_io_servers);
|
||||
/*
|
||||
* ******************************************************************
|
||||
* ************ functions implemented in this module end ************
|
||||
* ******************************************************************
|
||||
*/
|
||||
|
||||
END_C_DECLS
|
||||
|
||||
#endif /* MCA_FCACHE_UX_H */
|
||||
63
ompi/mca/fcache/ux/fcache_ux_component.c
Обычный файл
63
ompi/mca/fcache/ux/fcache_ux_component.c
Обычный файл
@ -0,0 +1,63 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*
|
||||
* These symbols are in a file by themselves to provide nice linker
|
||||
* semantics. Since linkers generally pull in symbols by object
|
||||
* files, keeping these symbols as the only symbols in this file
|
||||
* prevents utility programs such as "ompi_info" from having to import
|
||||
* entire components just to query their version and parameters.
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcache_ux.h"
|
||||
#include "mpi.h"
|
||||
|
||||
/*
|
||||
* Public string showing the fcache ux component version number
|
||||
*/
|
||||
const char *mca_fcache_ux_component_version_string =
|
||||
"OMPI/MPI ux FCACHE MCA component version " OMPI_VERSION;
|
||||
|
||||
/*
|
||||
* Instantiate the public struct with all of our public information
|
||||
* and pointers to our public functions in it
|
||||
*/
|
||||
mca_fcache_base_component_2_0_0_t mca_fcache_ux_component = {
|
||||
|
||||
/* First, the mca_component_t struct containing meta information
|
||||
about the component itself */
|
||||
|
||||
{
|
||||
MCA_FCACHE_BASE_VERSION_2_0_0,
|
||||
|
||||
/* Component name and version */
|
||||
"ux",
|
||||
OMPI_MAJOR_VERSION,
|
||||
OMPI_MINOR_VERSION,
|
||||
OMPI_RELEASE_VERSION,
|
||||
NULL,
|
||||
NULL
|
||||
},
|
||||
{
|
||||
/* This component is checkpointable */
|
||||
MCA_BASE_METADATA_PARAM_CHECKPOINT
|
||||
},
|
||||
mca_fcache_ux_component_init_query, /* get thread level */
|
||||
mca_fcache_ux_component_file_query, /* get priority and actions */
|
||||
mca_fcache_ux_component_file_unquery /* undo what was done by previous function */
|
||||
};
|
||||
37
ompi/mca/fcache/ux/fcache_ux_get_file_layout.c
Обычный файл
37
ompi/mca/fcache/ux/fcache_ux_get_file_layout.c
Обычный файл
@ -0,0 +1,37 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcache_ux.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "orte/util/show_help.h"
|
||||
#include "ompi/mca/fcache/fcache.h"
|
||||
|
||||
int
|
||||
mca_fcache_ux_get_file_layout (char* filename,
|
||||
int *num_io_servers,
|
||||
size_t *depth,
|
||||
int *file_io_servers)
|
||||
{
|
||||
printf ("UX GET FILE LAYOUT\n");
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
36
ompi/mca/fcache/ux/fcache_ux_get_io_servers.c
Обычный файл
36
ompi/mca/fcache/ux/fcache_ux_get_io_servers.c
Обычный файл
@ -0,0 +1,36 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcache_ux.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "orte/util/show_help.h"
|
||||
#include "ompi/mca/fcache/fcache.h"
|
||||
|
||||
int
|
||||
mca_fcache_ux_get_io_servers (char* filename,
|
||||
struct mca_io_ompio_io_servers *io_servers,
|
||||
int num_io_servers)
|
||||
{
|
||||
printf ("UX GET IO SERVERS\n");
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
37
ompi/mca/fcache/ux/fcache_ux_set_file_layout.c
Обычный файл
37
ompi/mca/fcache/ux/fcache_ux_set_file_layout.c
Обычный файл
@ -0,0 +1,37 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcache_ux.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "orte/util/show_help.h"
|
||||
#include "ompi/mca/fcache/fcache.h"
|
||||
|
||||
int
|
||||
mca_fcache_ux_set_file_layout (char* filename,
|
||||
int *num_io_servers,
|
||||
size_t *depth,
|
||||
int *file_io_servers)
|
||||
{
|
||||
printf ("UX SET FILE LAYOUT\n");
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
40
ompi/mca/fcoll/Makefile.am
Обычный файл
40
ompi/mca/fcoll/Makefile.am
Обычный файл
@ -0,0 +1,40 @@
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
AM_CPPFLAGS = $(LTDLINCL)
|
||||
|
||||
# main library setup
|
||||
noinst_LTLIBRARIES = libmca_fcoll.la
|
||||
libmca_fcoll_la_SOURCES =
|
||||
|
||||
# local files
|
||||
headers = fcoll.h
|
||||
libmca_fcoll_la_SOURCES += $(headers)
|
||||
|
||||
# Conditionally install the header files
|
||||
if WANT_INSTALL_HEADERS
|
||||
ompidir = $(includedir)/openmpi/$(subdir)
|
||||
nobase_ompi_HEADERS = $(headers)
|
||||
endif
|
||||
|
||||
include base/Makefile.am
|
||||
|
||||
distclean-local:
|
||||
rm -f base/static-components.h
|
||||
28
ompi/mca/fcoll/base/Makefile.am
Обычный файл
28
ompi/mca/fcoll/base/Makefile.am
Обычный файл
@ -0,0 +1,28 @@
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
headers += \
|
||||
base/base.h
|
||||
|
||||
libmca_fcoll_la_SOURCES += \
|
||||
base/fcoll_base_close.c \
|
||||
base/fcoll_base_file_select.c \
|
||||
base/fcoll_base_file_unselect.c \
|
||||
base/fcoll_base_find_available.c \
|
||||
base/fcoll_base_open.c
|
||||
71
ompi/mca/fcoll/base/base.h
Обычный файл
71
ompi/mca/fcoll/base/base.h
Обычный файл
@ -0,0 +1,71 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
/**
|
||||
* @file
|
||||
*
|
||||
* MCA fcoll base framework public interface functions.
|
||||
*/
|
||||
|
||||
#ifndef MCA_FCOLL_BASE_H
|
||||
#define MCA_FCOLL_BASE_H
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "opal/class/opal_list.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
#include "opal/mca/mca.h"
|
||||
|
||||
|
||||
BEGIN_C_DECLS
|
||||
|
||||
OMPI_DECLSPEC int mca_fcoll_base_open(void);
|
||||
|
||||
OMPI_DECLSPEC int mca_fcoll_base_close(void);
|
||||
|
||||
OMPI_DECLSPEC int mca_fcoll_base_file_select(struct mca_io_ompio_file_t *file,
|
||||
mca_base_component_t *preferred);
|
||||
OMPI_DECLSPEC int mca_fcoll_base_query_table (struct mca_io_ompio_file_t *file,
|
||||
char *name);
|
||||
OMPI_DECLSPEC int mca_fcoll_base_file_unselect(struct mca_io_ompio_file_t *file);
|
||||
|
||||
OMPI_DECLSPEC int mca_fcoll_base_find_available(bool enable_progress_threads,
|
||||
bool enable_mpi_threads);
|
||||
|
||||
OMPI_DECLSPEC int mca_fcoll_base_init_file (struct mca_io_ompio_file_t *file);
|
||||
|
||||
OMPI_DECLSPEC int mca_fcoll_base_get_param (struct mca_io_ompio_file_t *file, int keyval);
|
||||
/*
|
||||
* Globals
|
||||
*/
|
||||
|
||||
OMPI_DECLSPEC extern int mca_fcoll_base_param;
|
||||
OMPI_DECLSPEC extern int mca_fcoll_base_output;
|
||||
|
||||
OMPI_DECLSPEC extern bool mca_fcoll_base_components_opened_valid;
|
||||
OMPI_DECLSPEC extern bool mca_fcoll_base_components_available_valid;
|
||||
|
||||
OMPI_DECLSPEC extern opal_list_t mca_fcoll_base_components_opened;
|
||||
OMPI_DECLSPEC extern opal_list_t mca_fcoll_base_components_available;
|
||||
|
||||
END_C_DECLS
|
||||
|
||||
#endif /* MCA_BASE_FCOLL_H */
|
||||
56
ompi/mca/fcoll/base/fcoll_base_close.c
Обычный файл
56
ompi/mca/fcoll/base/fcoll_base_close.c
Обычный файл
@ -0,0 +1,56 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHTOB$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include <stdio.h>
|
||||
|
||||
#include "ompi/constants.h"
|
||||
#include "opal/mca/mca.h"
|
||||
#include "opal/mca/base/base.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
#include "ompi/mca/fcoll/base/base.h"
|
||||
|
||||
int mca_fcoll_base_close(void)
|
||||
{
|
||||
/*
|
||||
Close all components that are still open. This may be the opened
|
||||
list (if we're in ompi_info), or it may be the available list (if
|
||||
we're anywhere else).
|
||||
*/
|
||||
|
||||
if (mca_fcoll_base_components_opened_valid) {
|
||||
mca_base_components_close(mca_fcoll_base_output,
|
||||
&mca_fcoll_base_components_opened, NULL);
|
||||
OBJ_DESTRUCT(&mca_fcoll_base_components_opened);
|
||||
mca_fcoll_base_components_opened_valid = false;
|
||||
} else if (mca_fcoll_base_components_available_valid) {
|
||||
mca_base_components_close(mca_fcoll_base_output,
|
||||
&mca_fcoll_base_components_available, NULL);
|
||||
OBJ_DESTRUCT(&mca_fcoll_base_components_available);
|
||||
mca_fcoll_base_components_available_valid = false;
|
||||
}
|
||||
|
||||
/* Close the output stream for this framework */
|
||||
opal_output_close (mca_fcoll_base_output);
|
||||
|
||||
/* All done */
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
390
ompi/mca/fcoll/base/fcoll_base_file_select.c
Обычный файл
390
ompi/mca/fcoll/base/fcoll_base_file_select.c
Обычный файл
@ -0,0 +1,390 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include <string.h>
|
||||
|
||||
#include "opal/class/opal_list.h"
|
||||
#include "opal/util/argv.h"
|
||||
#include "opal/mca/mca.h"
|
||||
#include "opal/mca/base/base.h"
|
||||
#include "orte/util/show_help.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
#include "ompi/mca/fcoll/base/base.h"
|
||||
#include "ompi/mca/io/ompio/io_ompio.h"
|
||||
|
||||
/*
|
||||
* This structure is needed so that we can close the modules
|
||||
* which are not selected but were opened. mca_base_modules_close
|
||||
* which does this job for us requires a opal_list_t which contains
|
||||
* these modules
|
||||
*/
|
||||
struct queried_module_t {
|
||||
opal_list_item_t super;
|
||||
mca_fcoll_base_component_t *om_component;
|
||||
mca_fcoll_base_module_t *om_module;
|
||||
};
|
||||
typedef struct queried_module_t queried_module_t;
|
||||
static OBJ_CLASS_INSTANCE(queried_module_t, opal_list_item_t, NULL, NULL);
|
||||
|
||||
|
||||
/*
|
||||
* Only one fcoll module can be attached to each file.
|
||||
*
|
||||
* This module calls the query funtion on all the components that were
|
||||
* detected by fcoll_base_open. This function is called on a
|
||||
* per-file basis. This function has the following function.
|
||||
*
|
||||
* 1. Iterate over the list of available_components
|
||||
* 2. Call the query function on each of these components.
|
||||
* 3. query function returns the structure containing pointers
|
||||
* to its module and its priority
|
||||
* 4. Select the module with the highest priority
|
||||
* 5. Call the init function on the selected module so that it does the
|
||||
* right setup for the file
|
||||
* 6. Call finalize on all the other modules which returned
|
||||
* their module but were unfortunate to not get selected
|
||||
*/
|
||||
|
||||
int mca_fcoll_base_file_select (struct mca_io_ompio_file_t *file,
|
||||
mca_base_component_t *preferred)
|
||||
{
|
||||
int priority;
|
||||
int best_priority;
|
||||
opal_list_item_t *item;
|
||||
opal_list_item_t *next_item;
|
||||
char *names, **name_array;
|
||||
int num_names;
|
||||
mca_base_component_priority_list_item_t *cpli;
|
||||
mca_fcoll_base_component_t *component;
|
||||
mca_fcoll_base_component_t *best_component;
|
||||
mca_fcoll_base_module_t *module;
|
||||
opal_list_t queried;
|
||||
queried_module_t *om;
|
||||
opal_list_t *selectable;
|
||||
char *str;
|
||||
int err = MPI_SUCCESS;
|
||||
int i;
|
||||
bool was_selectable_constructed = false;
|
||||
|
||||
/* Check and see if a preferred component was provided. If it was
|
||||
provided then it should be used (if possible) */
|
||||
if (NULL != preferred) {
|
||||
|
||||
/* We have a preferred component. Check if it is available
|
||||
and if so, whether it wants to run */
|
||||
|
||||
str = &(preferred->mca_component_name[0]);
|
||||
|
||||
opal_output_verbose(10, mca_fcoll_base_output,
|
||||
"fcoll:base:file_select: Checking preferred component: %s",
|
||||
str);
|
||||
|
||||
/* query the component for its priority and get its module
|
||||
structure. This is necessary to proceed */
|
||||
|
||||
component = (mca_fcoll_base_component_t *)preferred;
|
||||
module = component->fcollm_file_query (file, &priority);
|
||||
if (NULL != module &&
|
||||
NULL != module->fcoll_module_init) {
|
||||
|
||||
/* this query seems to have returned something legitimate
|
||||
* and we can now go ahead and initialize the
|
||||
* file with it * but first, the functions which
|
||||
* are null need to be filled in */
|
||||
|
||||
/*fill_null_pointers (module);*/
|
||||
file->f_fcoll = module;
|
||||
file->f_fcoll_component = preferred;
|
||||
|
||||
return module->fcoll_module_init(file);
|
||||
}
|
||||
/* His preferred component is present, but is unable to
|
||||
* run. This is not a good sign. We should try selecting
|
||||
* some other component We let it fall through and select
|
||||
* from the list of available components
|
||||
*/
|
||||
} /*end of selection for preferred component */
|
||||
|
||||
/*
|
||||
* We fall till here if one of the two things happened:
|
||||
* 1. The preferred component was provided but for some reason was
|
||||
* not able to be selected
|
||||
* 2. No preferred component was provided
|
||||
*
|
||||
* All we need to do is to go through the list of available
|
||||
* components and find the one which has the highest priority and
|
||||
* use that for this file
|
||||
*/
|
||||
|
||||
/* Check if anything was requested by means on the name parameters */
|
||||
names = NULL;
|
||||
mca_base_param_lookup_string (mca_fcoll_base_param, &names);
|
||||
|
||||
if (NULL != names && 0 < strlen(names)) {
|
||||
name_array = opal_argv_split (names, ',');
|
||||
num_names = opal_argv_count (name_array);
|
||||
|
||||
opal_output_verbose(10, mca_fcoll_base_output,
|
||||
"fcoll:base:file_Select: Checking all available module");
|
||||
|
||||
/* since there are somethings which the mca requested through the
|
||||
if the intersection is NULL, then we barf saying that the requested
|
||||
modules are not being available */
|
||||
|
||||
selectable = OBJ_NEW(opal_list_t);
|
||||
was_selectable_constructed = true;
|
||||
|
||||
/* go through the compoents_available list and check against the names
|
||||
* to see whether this can be added or not */
|
||||
|
||||
for (item = opal_list_get_first(&mca_fcoll_base_components_available);
|
||||
item != opal_list_get_end(&mca_fcoll_base_components_available);
|
||||
item = opal_list_get_next(item)) {
|
||||
/* convert the opal_list_item_t returned into the proper type */
|
||||
cpli = (mca_base_component_priority_list_item_t *) item;
|
||||
component = (mca_fcoll_base_component_t *) cpli->super.cli_component;
|
||||
opal_output_verbose(10, mca_fcoll_base_output,
|
||||
"select: initialising %s component %s",
|
||||
component->fcollm_version.mca_type_name,
|
||||
component->fcollm_version.mca_component_name);
|
||||
|
||||
/* check if this name is present in the mca_base_params */
|
||||
for (i=0; i < num_names; i++) {
|
||||
if (0 == strcmp(name_array[i], component->fcollm_version.mca_component_name)) {
|
||||
/* this is present, and should be added o the selectable list */
|
||||
|
||||
/* We need to create a seperate object to initialise this list with
|
||||
* since we cannot have the same item in 2 lists */
|
||||
module = component->fcollm_file_query (file, &priority);
|
||||
if (NULL != module &&
|
||||
NULL != module->fcoll_module_init) {
|
||||
|
||||
file->f_fcoll = module;
|
||||
file->f_fcoll_component = (mca_base_component_t *)component;
|
||||
return module->fcoll_module_init(file);
|
||||
}
|
||||
|
||||
/*
|
||||
selectable_item = OBJ_NEW (mca_base_component_priority_list_item_t);
|
||||
*selectable_item = *cpli;
|
||||
opal_list_append (selectable, (opal_list_item_t *)selectable_item);
|
||||
break;*/
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* check for a NULL intersection between the available list and the
|
||||
* list which was asked for */
|
||||
|
||||
if (0 == opal_list_get_size(selectable)) {
|
||||
was_selectable_constructed = true;
|
||||
OBJ_RELEASE (selectable);
|
||||
opal_output_verbose (10, mca_fcoll_base_output,
|
||||
"fcoll:base:file_select: preferred modules were not available");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
} else { /* if there was no name_array, then we need to simply initialize
|
||||
selectable to mca_fcoll_base_components_available */
|
||||
selectable = &mca_fcoll_base_components_available;
|
||||
}
|
||||
|
||||
best_component = NULL;
|
||||
best_priority = -1;
|
||||
OBJ_CONSTRUCT(&queried, opal_list_t);
|
||||
|
||||
for (item = opal_list_get_first(selectable);
|
||||
item != opal_list_get_end(selectable);
|
||||
item = opal_list_get_next(item)) {
|
||||
/*
|
||||
* convert the opal_list_item_t returned into the proper type
|
||||
*/
|
||||
cpli = (mca_base_component_priority_list_item_t *) item;
|
||||
component = (mca_fcoll_base_component_t *) cpli->super.cli_component;
|
||||
opal_output_verbose(10, mca_fcoll_base_output,
|
||||
"select: initialising %s component %s",
|
||||
component->fcollm_version.mca_type_name,
|
||||
component->fcollm_version.mca_component_name);
|
||||
|
||||
/*
|
||||
* we can call the query function only if there is a function :-)
|
||||
*/
|
||||
if (NULL == component->fcollm_file_query) {
|
||||
opal_output_verbose(10, mca_fcoll_base_output,
|
||||
"select: no query, ignoring the component");
|
||||
} else {
|
||||
/*
|
||||
* call the query function and see what it returns
|
||||
*/
|
||||
module = component->fcollm_file_query (file, &priority);
|
||||
|
||||
if (NULL == module ||
|
||||
NULL == module->fcoll_module_init) {
|
||||
/*
|
||||
* query did not return any action which can be used
|
||||
*/
|
||||
opal_output_verbose(10, mca_fcoll_base_output,
|
||||
"select: query returned failure");
|
||||
} else {
|
||||
opal_output_verbose(10, mca_fcoll_base_output,
|
||||
"select: query returned priority %d",
|
||||
priority);
|
||||
/*
|
||||
* is this the best component we have found till now?
|
||||
*/
|
||||
if (priority > best_priority) {
|
||||
best_priority = priority;
|
||||
best_component = component;
|
||||
}
|
||||
|
||||
om = OBJ_NEW(queried_module_t);
|
||||
/*
|
||||
* check if we have run out of space
|
||||
*/
|
||||
if (NULL == om) {
|
||||
OBJ_DESTRUCT(&queried);
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
om->om_component = component;
|
||||
om->om_module = module;
|
||||
opal_list_append(&queried, (opal_list_item_t *)om);
|
||||
} /* end else of if (NULL == module) */
|
||||
} /* end else of if (NULL == component->fcollm_init) */
|
||||
} /* end for ... end of traversal */
|
||||
|
||||
/* We have to remove empty out the selectable list if the selectable
|
||||
* list was constructed as a duplicate and not as a pointer to the
|
||||
* mca_base_components_available list. So, check and destroy */
|
||||
|
||||
if (was_selectable_constructed) {
|
||||
|
||||
/* remove all the items first */
|
||||
for (item = opal_list_get_first(&mca_fcoll_base_components_available);
|
||||
item != opal_list_get_end(&mca_fcoll_base_components_available);
|
||||
item = next_item) {
|
||||
next_item = opal_list_get_next(item);
|
||||
OBJ_RELEASE (item);
|
||||
}
|
||||
|
||||
/* release the list itself */
|
||||
OBJ_RELEASE (selectable);
|
||||
was_selectable_constructed = false;
|
||||
}
|
||||
|
||||
/*
|
||||
* Now we have alist of components which successfully returned
|
||||
* their module struct. One of these components has the best
|
||||
* priority. The rest have to be comm_unqueried to counter the
|
||||
* effects of file_query'ing them. Finalize happens only on
|
||||
* components which should are initialized.
|
||||
*/
|
||||
if (NULL == best_component) {
|
||||
/*
|
||||
* This typically means that there was no component which was
|
||||
* able to run properly this time. So, we need to abort
|
||||
* JMS replace with show_help
|
||||
*/
|
||||
OBJ_DESTRUCT(&queried);
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
/*
|
||||
* We now have a list of components which have successfully
|
||||
* returned their priorities from the query. We now have to
|
||||
* unquery() those components which have not been selected and
|
||||
* init() the component which was selected
|
||||
*/
|
||||
for (item = opal_list_remove_first(&queried);
|
||||
NULL != item;
|
||||
item = opal_list_remove_first(&queried)) {
|
||||
om = (queried_module_t *) item;
|
||||
if (om->om_component == best_component) {
|
||||
/*
|
||||
* this is the chosen component, we have to initialise the
|
||||
* module of this component.
|
||||
*
|
||||
* ANJU: a component might not have all the functions
|
||||
* defined. Whereever a function pointer is null in the
|
||||
* module structure we need to fill it in with the base
|
||||
* structure function pointers. This is yet to be done
|
||||
*/
|
||||
|
||||
/*
|
||||
* We don return here coz we still need to go through and
|
||||
* elease the other objects
|
||||
*/
|
||||
|
||||
/*fill_null_pointers (om->om_module);*/
|
||||
file->f_fcoll = om->om_module;
|
||||
err = om->om_module->fcoll_module_init(file);
|
||||
file->f_fcoll_component = (mca_base_component_t *)best_component;
|
||||
/*
|
||||
printf ("SELECTED: %s\n", best_component->fcollm_version.mca_component_name);
|
||||
*/
|
||||
} else {
|
||||
/*
|
||||
* this is not the "choosen one", finalize
|
||||
*/
|
||||
if (NULL != om->om_component->fcollm_file_unquery) {
|
||||
/* unquery the component only if they have some clean
|
||||
* up job to do. Components which are queried but do
|
||||
* not actually do anything typically do not have a
|
||||
* unquery. Hence this check is necessary
|
||||
*/
|
||||
(void) om->om_component->fcollm_file_unquery(file);
|
||||
opal_output_verbose(10, mca_fcoll_base_output,
|
||||
"select: component %s is not selected",
|
||||
om->om_component->fcollm_version.mca_component_name);
|
||||
} /* end if */
|
||||
} /* if not best component */
|
||||
OBJ_RELEASE(om);
|
||||
} /* traversing through the entire list */
|
||||
|
||||
opal_output_verbose(10, mca_fcoll_base_output,
|
||||
"select: component %s selected",
|
||||
best_component->fcollm_version.mca_component_name);
|
||||
|
||||
OBJ_DESTRUCT(&queried);
|
||||
|
||||
return err;
|
||||
}
|
||||
|
||||
int mca_fcoll_base_query_table (struct mca_io_ompio_file_t *file, char *name)
|
||||
{
|
||||
if (!strcmp (name, "individual")) {
|
||||
if ((int)file->f_cc_size >= file->f_bytes_per_agg &&
|
||||
file->f_cc_size >= file->f_stripe_size) {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
if (!strcmp (name, "dynamic")) {
|
||||
if ((int)file->f_cc_size < file->f_bytes_per_agg &&
|
||||
file->f_cc_size >= file->f_stripe_size) {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
if (!strcmp (name, "two_phase")) {
|
||||
if ((int)file->f_cc_size < file->f_bytes_per_agg &&
|
||||
file->f_cc_size < file->f_stripe_size) {
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
41
ompi/mca/fcoll/base/fcoll_base_file_unselect.c
Обычный файл
41
ompi/mca/fcoll/base/fcoll_base_file_unselect.c
Обычный файл
@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/mca/io/ompio/io_ompio.h"
|
||||
#include "opal/util/show_help.h"
|
||||
#include "opal/mca/mca.h"
|
||||
#include "opal/mca/base/base.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
#include "ompi/mca/fcoll/base/base.h"
|
||||
|
||||
int mca_fcoll_base_file_unselect(mca_io_ompio_file_t *file)
|
||||
{
|
||||
if (NULL != file->f_fcoll && NULL != file->f_fcoll->fcoll_module_finalize) {
|
||||
return file->f_fcoll->fcoll_module_finalize(file);
|
||||
}
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
171
ompi/mca/fcoll/base/fcoll_base_find_available.c
Обычный файл
171
ompi/mca/fcoll/base/fcoll_base_find_available.c
Обычный файл
@ -0,0 +1,171 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "opal/class/opal_list.h"
|
||||
#include "orte/util/show_help.h"
|
||||
#include "opal/mca/mca.h"
|
||||
#include "opal/mca/base/base.h"
|
||||
#include "opal/mca/base/mca_base_component_repository.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
#include "ompi/mca/fcoll/base/base.h"
|
||||
|
||||
opal_list_t mca_fcoll_base_modules_available;
|
||||
bool mca_fcoll_base_modules_available_valid = false;
|
||||
|
||||
static int init_query(const mca_base_component_t *m,
|
||||
mca_base_component_priority_list_item_t *entry,
|
||||
bool enable_progress_threads,
|
||||
bool enable_mpi_threads);
|
||||
static int init_query_2_0_0(const mca_base_component_t *component,
|
||||
mca_base_component_priority_list_item_t *entry,
|
||||
bool enable_progress_threads,
|
||||
bool enable_mpi_threads);
|
||||
|
||||
int mca_fcoll_base_find_available(bool enable_progress_threads,
|
||||
bool enable_mpi_threads)
|
||||
{
|
||||
bool found = false;
|
||||
mca_base_component_priority_list_item_t *entry;
|
||||
opal_list_item_t *p;
|
||||
|
||||
/* Initialize the list */
|
||||
|
||||
OBJ_CONSTRUCT(&mca_fcoll_base_components_available, opal_list_t);
|
||||
mca_fcoll_base_components_available_valid = true;
|
||||
|
||||
/* The list of components which we should check is already present
|
||||
in mca_fcoll_base_components_opened, which was established in
|
||||
mca_fcoll_base_open */
|
||||
|
||||
for (found = false,
|
||||
p = opal_list_remove_first (&mca_fcoll_base_components_opened);
|
||||
NULL != p;
|
||||
p = opal_list_remove_first (&mca_fcoll_base_components_opened)) {
|
||||
entry = OBJ_NEW(mca_base_component_priority_list_item_t);
|
||||
entry->super.cli_component =
|
||||
((mca_base_component_list_item_t *)p)->cli_component;
|
||||
|
||||
/* Now for this entry, we have to determine the thread level. Call
|
||||
a subroutine to do the job for us */
|
||||
|
||||
if (OMPI_SUCCESS == init_query(entry->super.cli_component, entry,
|
||||
enable_progress_threads,
|
||||
enable_mpi_threads)) {
|
||||
/* Save the results in the list. The priority is not relvant at
|
||||
this point in time. But we save the thread arguments so that
|
||||
the initial selection algorithm can negotiate overall thread
|
||||
level for this process */
|
||||
entry->cpli_priority = 0;
|
||||
opal_list_append (&mca_fcoll_base_components_available,
|
||||
(opal_list_item_t *) entry);
|
||||
found = true;
|
||||
} else {
|
||||
/* The component does not want to run, so close it. Its close()
|
||||
has already been invoked. Close it out of the DSO repository
|
||||
(if it is there in the repository) */
|
||||
mca_base_component_repository_release(entry->super.cli_component);
|
||||
OBJ_RELEASE(entry);
|
||||
}
|
||||
/* Free entry from the "opened" list */
|
||||
OBJ_RELEASE(p);
|
||||
}
|
||||
|
||||
/* The opened list is no longer necessary, so we can free it */
|
||||
OBJ_DESTRUCT (&mca_fcoll_base_components_opened);
|
||||
mca_fcoll_base_components_opened_valid = false;
|
||||
|
||||
/* There should atleast be one fcoll component which was available */
|
||||
if (false == found) {
|
||||
/* Need to free all items in the list */
|
||||
OBJ_DESTRUCT(&mca_fcoll_base_components_available);
|
||||
mca_fcoll_base_components_available_valid = false;
|
||||
opal_output_verbose (10, mca_fcoll_base_output,
|
||||
"fcoll:find_available: no fcoll components available!");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
/* All done */
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
static int init_query(const mca_base_component_t *m,
|
||||
mca_base_component_priority_list_item_t *entry,
|
||||
bool enable_progress_threads,
|
||||
bool enable_mpi_threads)
|
||||
{
|
||||
int ret;
|
||||
|
||||
opal_output_verbose(10, mca_fcoll_base_output,
|
||||
"fcoll:find_available: querying fcoll component %s",
|
||||
m->mca_component_name);
|
||||
|
||||
/* This component has been successfully opened, now try to query it */
|
||||
if (2 == m->mca_type_major_version &&
|
||||
0 == m->mca_type_minor_version &&
|
||||
0 == m->mca_type_release_version) {
|
||||
ret = init_query_2_0_0(m, entry, enable_progress_threads,
|
||||
enable_mpi_threads);
|
||||
} else {
|
||||
/* unrecognised API version */
|
||||
opal_output_verbose(10, mca_fcoll_base_output,
|
||||
"fcoll:find_available:unrecognised fcoll API version (%d.%d.%d)",
|
||||
m->mca_type_major_version,
|
||||
m->mca_type_minor_version,
|
||||
m->mca_type_release_version);
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
/* Query done -- look at return value to see what happened */
|
||||
if (OMPI_SUCCESS != ret) {
|
||||
opal_output_verbose(10, mca_fcoll_base_output,
|
||||
"fcoll:find_available fcoll component %s is not available",
|
||||
m->mca_component_name);
|
||||
if (NULL != m->mca_close_component) {
|
||||
m->mca_close_component();
|
||||
}
|
||||
} else {
|
||||
opal_output_verbose(10, mca_fcoll_base_output,
|
||||
"fcoll:find_avalable: fcoll component %s is available",
|
||||
m->mca_component_name);
|
||||
|
||||
}
|
||||
/* All done */
|
||||
return ret;
|
||||
}
|
||||
|
||||
|
||||
static int init_query_2_0_0(const mca_base_component_t *component,
|
||||
mca_base_component_priority_list_item_t *entry,
|
||||
bool enable_progress_threads,
|
||||
bool enable_mpi_threads)
|
||||
{
|
||||
mca_fcoll_base_component_2_0_0_t *fcoll =
|
||||
(mca_fcoll_base_component_2_0_0_t *) component;
|
||||
|
||||
return fcoll->fcollm_init_query(enable_progress_threads,
|
||||
enable_mpi_threads);
|
||||
}
|
||||
88
ompi/mca/fcoll/base/fcoll_base_open.c
Обычный файл
88
ompi/mca/fcoll/base/fcoll_base_open.c
Обычный файл
@ -0,0 +1,88 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include <stdio.h>
|
||||
|
||||
#include "ompi/class/ompi_free_list.h"
|
||||
#include "orte/util/show_help.h"
|
||||
#include "opal/mca/mca.h"
|
||||
#include "opal/mca/base/base.h"
|
||||
#include "opal/mca/base/mca_base_param.h"
|
||||
|
||||
#include "orte/util/show_help.h"
|
||||
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
#include "ompi/mca/fcoll/base/base.h"
|
||||
|
||||
|
||||
/*
|
||||
* The following file was created by configure. It contains extern
|
||||
* statements and the definition of an array of pointers to each
|
||||
* component's public mca_base_component_t struct.
|
||||
*/
|
||||
#ifdef __WINDOWS__
|
||||
const mca_base_component_t *mca_fcoll_base_static_components[] = {NULL};
|
||||
#else
|
||||
#include "ompi/mca/fcoll/base/static-components.h"
|
||||
#endif
|
||||
|
||||
/*
|
||||
* Global variables; most of which are loaded by back-ends of MCA
|
||||
* variables
|
||||
*/
|
||||
int mca_fcoll_base_param = -1;
|
||||
int mca_fcoll_base_output = -1;
|
||||
|
||||
opal_list_t mca_fcoll_base_components_opened;
|
||||
opal_list_t mca_fcoll_base_components_available;
|
||||
|
||||
bool mca_fcoll_base_components_available_valid = false;
|
||||
bool mca_fcoll_base_components_opened_valid = false;
|
||||
|
||||
mca_fcoll_base_component_t mca_fcoll_base_selected_component;
|
||||
mca_fcoll_base_module_t mca_fcoll;
|
||||
|
||||
/*
|
||||
* Function for finding and opening either all MCA components, or the one
|
||||
* that was specifically requested via a MCA parameter.
|
||||
*/
|
||||
int mca_fcoll_base_open(void)
|
||||
{
|
||||
/* Open an output stream for this framework */
|
||||
|
||||
mca_fcoll_base_output = opal_output_open(NULL);
|
||||
|
||||
/* Open up all available components */
|
||||
|
||||
if (OMPI_SUCCESS !=
|
||||
mca_base_components_open("fcoll", mca_fcoll_base_output,
|
||||
mca_fcoll_base_static_components,
|
||||
&mca_fcoll_base_components_opened, true)) {
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
mca_fcoll_base_components_opened_valid = true;
|
||||
|
||||
/* Find the index of the MCA "fcoll" param for selection */
|
||||
|
||||
mca_fcoll_base_param = mca_base_param_find("fcoll", "base", NULL);
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
18
ompi/mca/fcoll/base/static-components.h
Обычный файл
18
ompi/mca/fcoll/base/static-components.h
Обычный файл
@ -0,0 +1,18 @@
|
||||
/*
|
||||
* $HEADER$
|
||||
*/
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
|
||||
|
||||
const mca_base_component_t *mca_fcoll_base_static_components[] = {
|
||||
|
||||
NULL
|
||||
};
|
||||
|
||||
#if defined(c_plusplus) || defined(__cplusplus)
|
||||
}
|
||||
#endif
|
||||
|
||||
54
ompi/mca/fcoll/dynamic/Makefile.am
Обычный файл
54
ompi/mca/fcoll/dynamic/Makefile.am
Обычный файл
@ -0,0 +1,54 @@
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
sources = \
|
||||
fcoll_dynamic.h \
|
||||
fcoll_dynamic_module.c \
|
||||
fcoll_dynamic_component.c \
|
||||
fcoll_dynamic_file_read_all.c \
|
||||
fcoll_dynamic_file_read_all_begin.c \
|
||||
fcoll_dynamic_file_read_all_end.c \
|
||||
fcoll_dynamic_file_write_all.c \
|
||||
fcoll_dynamic_file_write_all_begin.c \
|
||||
fcoll_dynamic_file_write_all_end.c
|
||||
|
||||
# Make the output library in this directory, and name it either
|
||||
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
|
||||
# (for static builds).
|
||||
|
||||
if MCA_BUILD_ompi_fcoll_dynamic_DSO
|
||||
component_noinst =
|
||||
component_install = mca_fcoll_dynamic.la
|
||||
else
|
||||
component_noinst = libmca_fcoll_dynamic.la
|
||||
component_install =
|
||||
endif
|
||||
|
||||
mcacomponentdir = $(libdir)/openmpi
|
||||
mcacomponent_LTLIBRARIES = $(component_install)
|
||||
mca_fcoll_dynamic_la_SOURCES = $(sources)
|
||||
mca_fcoll_dynamic_la_LDFLAGS = -module -avoid-version
|
||||
mca_fcoll_dynamic_la_LIBADD = \
|
||||
$(top_ompi_builddir)/ompi/libmpi.la \
|
||||
$(top_ompi_builddir)/orte/libopen-rte.la \
|
||||
$(top_ompi_builddir)/opal/libopen-pal.la
|
||||
|
||||
noinst_LTLIBRARIES = $(component_noinst)
|
||||
libmca_fcoll_dynamic_la_SOURCES =$(sources)
|
||||
libmca_fcoll_dynamic_la_LDFLAGS = -module -avoid-version
|
||||
86
ompi/mca/fcoll/dynamic/fcoll_dynamic.h
Обычный файл
86
ompi/mca/fcoll/dynamic/fcoll_dynamic.h
Обычный файл
@ -0,0 +1,86 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2006 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#ifndef MCA_FCOLL_DYNAMIC_EXPORT_H
|
||||
#define MCA_FCOLL_DYNAMIC_EXPORT_H
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "opal/mca/mca.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
#include "ompi/mca/fcoll/base/base.h"
|
||||
#include "ompi/mca/io/ompio/io_ompio.h"
|
||||
|
||||
BEGIN_C_DECLS
|
||||
|
||||
/* Globally exported variables */
|
||||
|
||||
extern int mca_fcoll_dynamic_priority;
|
||||
extern int mca_fcoll_dynamic_num_io_procs;
|
||||
extern int mca_fcoll_dynamic_constant_cbs;
|
||||
extern int mca_fcoll_dynamic_cycle_buffer_size;
|
||||
|
||||
OMPI_MODULE_DECLSPEC extern mca_fcoll_base_component_2_0_0_t mca_fcoll_dynamic_component;
|
||||
|
||||
/* API functions */
|
||||
|
||||
int mca_fcoll_dynamic_component_init_query(bool enable_progress_threads,
|
||||
bool enable_mpi_threads);
|
||||
struct mca_fcoll_base_module_1_0_0_t *
|
||||
mca_fcoll_dynamic_component_file_query (mca_io_ompio_file_t *fh, int *priority);
|
||||
|
||||
int mca_fcoll_dynamic_component_file_unquery (mca_io_ompio_file_t *file);
|
||||
|
||||
int mca_fcoll_dynamic_module_init (mca_io_ompio_file_t *file);
|
||||
int mca_fcoll_dynamic_module_finalize (mca_io_ompio_file_t *file);
|
||||
|
||||
int mca_fcoll_dynamic_file_read_all (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype,
|
||||
ompi_status_public_t * status);
|
||||
|
||||
int mca_fcoll_dynamic_file_read_all_begin (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype);
|
||||
|
||||
int mca_fcoll_dynamic_file_read_all_end (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
ompi_status_public_t * status);
|
||||
|
||||
int mca_fcoll_dynamic_file_write_all (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype,
|
||||
ompi_status_public_t * status);
|
||||
|
||||
int mca_fcoll_dynamic_file_write_all_begin (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype);
|
||||
|
||||
int mca_fcoll_dynamic_file_write_all_end (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
ompi_status_public_t * status);
|
||||
|
||||
END_C_DECLS
|
||||
|
||||
#endif /* MCA_FCOLL_DYNAMIC_EXPORT_H */
|
||||
129
ompi/mca/fcoll/dynamic/fcoll_dynamic_component.c
Обычный файл
129
ompi/mca/fcoll/dynamic/fcoll_dynamic_component.c
Обычный файл
@ -0,0 +1,129 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*
|
||||
* These symbols are in a file by themselves to provide nice linker
|
||||
* semantics. Since linkers generally pull in symbols by object
|
||||
* files, keeping these symbols as the only symbols in this file
|
||||
* prevents utility programs such as "ompi_info" from having to import
|
||||
* entire components just to query their version and parameters.
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_dynamic.h"
|
||||
#include "mpi.h"
|
||||
|
||||
/*
|
||||
* Public string showing the fcoll ompi_dynamic component version number
|
||||
*/
|
||||
const char *mca_fcoll_dynamic_component_version_string =
|
||||
"Open MPI dynamic collective MCA component version " OMPI_VERSION;
|
||||
|
||||
/*
|
||||
* Global variables
|
||||
*/
|
||||
int mca_fcoll_dynamic_priority = 10;
|
||||
int mca_fcoll_dynamic_num_io_procs = -1;
|
||||
int mca_fcoll_dynamic_constant_cbs = 0;
|
||||
int mca_fcoll_dynamic_cycle_buffer_size = OMPIO_PREALLOC_MAX_BUF_SIZE;
|
||||
|
||||
/*
|
||||
* Local function
|
||||
*/
|
||||
static int dynamic_register(void);
|
||||
|
||||
/*
|
||||
* Instantiate the public struct with all of our public information
|
||||
* and pointers to our public functions in it
|
||||
*/
|
||||
mca_fcoll_base_component_2_0_0_t mca_fcoll_dynamic_component = {
|
||||
|
||||
/* First, the mca_component_t struct containing meta information
|
||||
* about the component itself */
|
||||
|
||||
{
|
||||
MCA_FCOLL_BASE_VERSION_2_0_0,
|
||||
|
||||
/* Component name and version */
|
||||
"dynamic",
|
||||
OMPI_MAJOR_VERSION,
|
||||
OMPI_MINOR_VERSION,
|
||||
OMPI_RELEASE_VERSION,
|
||||
dynamic_register,
|
||||
NULL
|
||||
},
|
||||
{
|
||||
/* The component is checkpoint ready */
|
||||
MCA_BASE_METADATA_PARAM_CHECKPOINT
|
||||
},
|
||||
|
||||
mca_fcoll_dynamic_component_init_query,
|
||||
mca_fcoll_dynamic_component_file_query,
|
||||
mca_fcoll_dynamic_component_file_unquery
|
||||
};
|
||||
|
||||
|
||||
static int
|
||||
dynamic_register(void)
|
||||
{
|
||||
int param;
|
||||
|
||||
param = mca_base_param_find ("fcoll", NULL, "dynamic_priority");
|
||||
if (param >= 0)
|
||||
{
|
||||
mca_base_param_lookup_int (param, &mca_fcoll_dynamic_priority);
|
||||
}
|
||||
param = mca_base_param_find ("fcoll", NULL, "dynamic_num_io_procs");
|
||||
if (param >= 0)
|
||||
{
|
||||
mca_base_param_lookup_int (param, &mca_fcoll_dynamic_num_io_procs);
|
||||
}
|
||||
param = mca_base_param_find ("fcoll", NULL, "dynamic_constant_cbs");
|
||||
if (param >= 0)
|
||||
{
|
||||
mca_base_param_lookup_int (param, &mca_fcoll_dynamic_constant_cbs);
|
||||
}
|
||||
param = mca_base_param_find ("fcoll", NULL, "dynamic_cycle_buffer_size");
|
||||
if (param >= 0)
|
||||
{
|
||||
mca_base_param_lookup_int (param, &mca_fcoll_dynamic_cycle_buffer_size);
|
||||
}
|
||||
|
||||
mca_base_param_reg_int (&mca_fcoll_dynamic_component.fcollm_version,
|
||||
"priority",
|
||||
"Priority of the dynamic fcoll component",
|
||||
false, false, mca_fcoll_dynamic_priority,
|
||||
&mca_fcoll_dynamic_priority);
|
||||
mca_base_param_reg_int (&mca_fcoll_dynamic_component.fcollm_version,
|
||||
"num_io_procs",
|
||||
"Number of writers in the dynamic fcoll component",
|
||||
false, false, mca_fcoll_dynamic_num_io_procs,
|
||||
&mca_fcoll_dynamic_num_io_procs);
|
||||
mca_base_param_reg_int (&mca_fcoll_dynamic_component.fcollm_version,
|
||||
"constant_cbs",
|
||||
"wether we are using constant or scaling cycle buffer size in the dynamic fcoll component",
|
||||
false, false, mca_fcoll_dynamic_constant_cbs,
|
||||
&mca_fcoll_dynamic_constant_cbs);
|
||||
mca_base_param_reg_int (&mca_fcoll_dynamic_component.fcollm_version,
|
||||
"cycle_buffer_size",
|
||||
"Cycle Buffer Size of the dynamic fcoll component",
|
||||
false, false, mca_fcoll_dynamic_cycle_buffer_size,
|
||||
&mca_fcoll_dynamic_cycle_buffer_size);
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
622
ompi/mca/fcoll/dynamic/fcoll_dynamic_file_read_all.c
Обычный файл
622
ompi/mca/fcoll/dynamic/fcoll_dynamic_file_read_all.c
Обычный файл
@ -0,0 +1,622 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_dynamic.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
#include "ompi/mca/io/ompio/io_ompio.h"
|
||||
#include "ompi/mca/io/io.h"
|
||||
#include "math.h"
|
||||
#include "ompi/mca/pml/pml.h"
|
||||
#include <unistd.h>
|
||||
|
||||
int
|
||||
mca_fcoll_dynamic_file_read_all (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype,
|
||||
ompi_status_public_t *status)
|
||||
{
|
||||
MPI_Aint position = 0;
|
||||
MPI_Aint total_bytes = 0; /* total bytes to be read */
|
||||
MPI_Aint bytes_to_read_in_cycle = 0; /* left to be read in a cycle*/
|
||||
MPI_Aint bytes_per_cycle = 0; /* total read in each cycle by each process*/
|
||||
int index = 0;
|
||||
int cycles = 0;
|
||||
int i=0, j=0, x=0;
|
||||
int n=0; /* current position in total_bytes_per_process array */
|
||||
MPI_Aint bytes_remaining = 0; /* how many bytes have been read from the current
|
||||
value from total_bytes_per_process */
|
||||
int bytes_received = 0;
|
||||
int blocks = 0;
|
||||
/* iovec structure and count of the buffer passed in */
|
||||
uint32_t iov_count = 0;
|
||||
struct iovec *decoded_iov = NULL;
|
||||
int iov_index = 0;
|
||||
size_t current_position = 0;
|
||||
char *receive_buf = NULL;
|
||||
|
||||
/* global iovec at the readers that contain the iovecs created from
|
||||
file_set_view */
|
||||
uint32_t total_fview_count = 0;
|
||||
struct iovec *global_fview = NULL;
|
||||
int local_count = 0;
|
||||
struct iovec *iov = NULL;
|
||||
int *fview_count = NULL;
|
||||
int current_index;
|
||||
|
||||
char *global_buf = NULL;
|
||||
MPI_Aint global_count = 0;
|
||||
|
||||
/* array that contains the sorted indices of the global_iov */
|
||||
int *sorted = NULL;
|
||||
int *displs = NULL;
|
||||
size_t max_data = 0;
|
||||
int *bytes_per_process = NULL;
|
||||
MPI_Aint bytes_left = 0;
|
||||
MPI_Aint *total_bytes_per_process = NULL;
|
||||
|
||||
if (opal_datatype_is_contiguous_memory_layout(&datatype->super,1)) {
|
||||
fh->f_flags |= OMPIO_CONTIGUOUS_MEMORY;
|
||||
}
|
||||
/**************************************************************************
|
||||
** In case the data is not contigous in memory, decode it into an iovec **
|
||||
**************************************************************************/
|
||||
if (! (fh->f_flags & OMPIO_CONTIGUOUS_MEMORY)) {
|
||||
ompi_io_ompio_decode_datatype (fh,
|
||||
datatype,
|
||||
count,
|
||||
buf,
|
||||
&max_data,
|
||||
&decoded_iov,
|
||||
&iov_count);
|
||||
}
|
||||
else {
|
||||
max_data = count * datatype->super.size;
|
||||
}
|
||||
|
||||
if (! (fh->f_flags & OMPIO_AGGREGATOR_IS_SET)) {
|
||||
ompi_io_ompio_set_aggregator_props (fh,
|
||||
mca_fcoll_dynamic_num_io_procs,
|
||||
max_data);
|
||||
}
|
||||
|
||||
total_bytes_per_process = (MPI_Aint*)malloc
|
||||
(fh->f_procs_per_group*sizeof(MPI_Aint));
|
||||
if (NULL == total_bytes_per_process) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
ompi_io_ompio_allgather_array (&max_data,
|
||||
1,
|
||||
MPI_LONG,
|
||||
total_bytes_per_process,
|
||||
1,
|
||||
MPI_LONG,
|
||||
fh->f_aggregator_index,
|
||||
fh->f_procs_in_group,
|
||||
fh->f_procs_per_group,
|
||||
fh->f_comm);
|
||||
|
||||
for (i=0 ; i<fh->f_procs_per_group ; i++) {
|
||||
total_bytes += total_bytes_per_process[i];
|
||||
}
|
||||
|
||||
if (NULL != total_bytes_per_process) {
|
||||
free (total_bytes_per_process);
|
||||
total_bytes_per_process = NULL;
|
||||
}
|
||||
/*
|
||||
fh->f_comm->c_coll.coll_allreduce (&max_data,
|
||||
&total_bytes,
|
||||
1,
|
||||
MPI_DOUBLE,
|
||||
MPI_SUM,
|
||||
fh->f_comm,
|
||||
fh->f_comm->c_coll.coll_allreduce_module);
|
||||
*/
|
||||
/*********************************************************************
|
||||
*** Generate the File offsets/lengths corresponding to this write ***
|
||||
********************************************************************/
|
||||
ompi_io_ompio_generate_current_file_view (fh, max_data, &iov, &local_count);
|
||||
/*
|
||||
for (i=0 ; i<local_count ; i++) {
|
||||
printf("%d: OFFSET: %p LENGTH: %d\n",
|
||||
fh->f_rank,
|
||||
iov[i].iov_base,
|
||||
iov[i].iov_len);
|
||||
}
|
||||
*/
|
||||
/*************************************************************
|
||||
*** ALLGather the File View information at all processes ***
|
||||
*************************************************************/
|
||||
|
||||
fview_count = (int *) malloc (fh->f_procs_per_group * sizeof (int));
|
||||
if (NULL == fview_count) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
ompi_io_ompio_allgather_array (&local_count,
|
||||
1,
|
||||
MPI_INT,
|
||||
fview_count,
|
||||
1,
|
||||
MPI_INT,
|
||||
fh->f_aggregator_index,
|
||||
fh->f_procs_in_group,
|
||||
fh->f_procs_per_group,
|
||||
fh->f_comm);
|
||||
displs = (int*)malloc (fh->f_procs_per_group*sizeof(int));
|
||||
if (NULL == displs) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
displs[0] = 0;
|
||||
total_fview_count = fview_count[0];
|
||||
for (i=1 ; i<fh->f_procs_per_group ; i++) {
|
||||
total_fview_count += fview_count[i];
|
||||
displs[i] = displs[i-1] + fview_count[i-1];
|
||||
}
|
||||
/*
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
for (i=0 ; i<fh->f_procs_per_group ; i++) {
|
||||
printf ("%d: PROCESS: %d ELEMENTS: %d DISPLS: %d\n",
|
||||
fh->f_rank,
|
||||
i,
|
||||
fview_count[i],
|
||||
displs[i]);
|
||||
}
|
||||
}
|
||||
*/
|
||||
/* allocate the global iovec */
|
||||
if (0 != total_fview_count) {
|
||||
global_fview = (struct iovec*)malloc (total_fview_count *
|
||||
sizeof(struct iovec));
|
||||
if (NULL == global_fview) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
if (fh->f_flags & OMPIO_UNIFORM_FVIEW) {
|
||||
ompi_io_ompio_allgather_array (iov,
|
||||
local_count,
|
||||
fh->f_iov_type,
|
||||
global_fview,
|
||||
local_count,
|
||||
fh->f_iov_type,
|
||||
fh->f_aggregator_index,
|
||||
fh->f_procs_in_group,
|
||||
fh->f_procs_per_group,
|
||||
fh->f_comm);
|
||||
}
|
||||
else {
|
||||
ompi_io_ompio_allgatherv_array (iov,
|
||||
local_count,
|
||||
fh->f_iov_type,
|
||||
global_fview,
|
||||
fview_count,
|
||||
displs,
|
||||
fh->f_iov_type,
|
||||
fh->f_aggregator_index,
|
||||
fh->f_procs_in_group,
|
||||
fh->f_procs_per_group,
|
||||
fh->f_comm);
|
||||
}
|
||||
/* sort it */
|
||||
if (0 != total_fview_count) {
|
||||
sorted = (int *)malloc (total_fview_count * sizeof(int));
|
||||
if (NULL == sorted) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
ompi_io_ompio_sort_iovec (global_fview, total_fview_count, sorted);
|
||||
}
|
||||
if (NULL != iov) {
|
||||
free (iov);
|
||||
iov = NULL;
|
||||
}
|
||||
/*
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
for (i=0 ; i<total_fview_count ; i++) {
|
||||
printf("%d: OFFSET: %p LENGTH: %d\n",
|
||||
fh->f_rank,
|
||||
global_fview[sorted[i]].iov_base,
|
||||
global_fview[sorted[i]].iov_len);
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
bytes_per_process = (int *)malloc (fh->f_procs_per_group * sizeof (int));
|
||||
if (NULL == bytes_per_process) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Calculate how many bytes are read in each cycle
|
||||
*/
|
||||
bytes_per_cycle = mca_fcoll_dynamic_cycle_buffer_size;
|
||||
|
||||
cycles = ceil((double)total_bytes/bytes_per_cycle);
|
||||
|
||||
n = 0;
|
||||
bytes_remaining = 0;
|
||||
current_index = 0;
|
||||
|
||||
for (index = 0; index < cycles; index++) {
|
||||
int k;
|
||||
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
memset(displs, 0x0, fh->f_procs_per_group*sizeof(int));
|
||||
memset(bytes_per_process, 0x0, fh->f_procs_per_group*sizeof(int));
|
||||
}
|
||||
|
||||
if (cycles-1 == index) {
|
||||
bytes_to_read_in_cycle = total_bytes - bytes_per_cycle*index;
|
||||
}
|
||||
else {
|
||||
bytes_to_read_in_cycle = bytes_per_cycle;
|
||||
}
|
||||
/*
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
printf ("****%d: CYCLE %d Bytes %d**********\n",
|
||||
fh->f_rank,
|
||||
index,
|
||||
bytes_to_write_in_cycle);
|
||||
}
|
||||
*/
|
||||
|
||||
/* Calculate how much data will be contributed in this cycle
|
||||
by each process*/
|
||||
bytes_received = 0;
|
||||
|
||||
while (bytes_to_read_in_cycle) {
|
||||
blocks = fview_count[0];
|
||||
for (j=0 ; j<fh->f_procs_per_group ; j++) {
|
||||
if (sorted[current_index] < blocks) {
|
||||
n = j;
|
||||
break;
|
||||
}
|
||||
else {
|
||||
blocks += fview_count[j+1];
|
||||
}
|
||||
}
|
||||
if (bytes_remaining) {
|
||||
if (bytes_remaining <= bytes_to_read_in_cycle) {
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
bytes_per_process[n] += bytes_remaining;
|
||||
}
|
||||
if (fh->f_procs_in_group[n] == fh->f_rank) {
|
||||
bytes_received += bytes_remaining;
|
||||
}
|
||||
current_index ++;
|
||||
bytes_to_read_in_cycle -= bytes_remaining;
|
||||
bytes_remaining = 0;
|
||||
continue;
|
||||
}
|
||||
else {
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
bytes_per_process[n] += bytes_to_read_in_cycle;
|
||||
}
|
||||
if (fh->f_procs_in_group[n] == fh->f_rank) {
|
||||
bytes_received += bytes_to_read_in_cycle;
|
||||
}
|
||||
bytes_remaining -= bytes_to_read_in_cycle;
|
||||
bytes_to_read_in_cycle = 0;
|
||||
break;
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (bytes_to_read_in_cycle <
|
||||
global_fview[sorted[current_index]].iov_len) {
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
bytes_per_process[n] += bytes_to_read_in_cycle;
|
||||
}
|
||||
if (fh->f_procs_in_group[n] == fh->f_rank) {
|
||||
bytes_received += bytes_to_read_in_cycle;
|
||||
}
|
||||
bytes_remaining = global_fview[sorted[current_index]].iov_len -
|
||||
bytes_to_read_in_cycle;
|
||||
bytes_to_read_in_cycle = 0;
|
||||
break;
|
||||
}
|
||||
else {
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
bytes_per_process[n] +=
|
||||
global_fview[sorted[current_index]].iov_len;
|
||||
}
|
||||
if (fh->f_procs_in_group[n] == fh->f_rank) {
|
||||
bytes_received +=
|
||||
global_fview[sorted[current_index]].iov_len;
|
||||
}
|
||||
bytes_to_read_in_cycle -=
|
||||
global_fview[sorted[current_index]].iov_len;
|
||||
current_index ++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
/* Calculate the displacement on where to put the data and allocate
|
||||
the recieve buffer (global_buf) */
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
displs[0] = 0;
|
||||
global_count = bytes_per_process[0];
|
||||
for (i=1 ; i<fh->f_procs_per_group ; i++) {
|
||||
global_count += bytes_per_process[i];
|
||||
displs[i] = displs[i-1] + bytes_per_process[i-1];
|
||||
}
|
||||
/*
|
||||
for (i=0 ; i<fh->f_procs_per_group ; i++) {
|
||||
printf ("Proc %d sending %d at %d\n",
|
||||
i,
|
||||
bytes_per_process[i],
|
||||
displs[i]);
|
||||
}
|
||||
*/
|
||||
global_buf = malloc (global_count);
|
||||
if (NULL == global_buf) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
|
||||
/**********************************************************
|
||||
******* Create the io array, and pass it to fbtl *********
|
||||
*********************************************************/
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
MPI_Aint bytes_to_read = global_count;
|
||||
MPI_Aint *temp = NULL;
|
||||
int block = 1;
|
||||
k = 0;
|
||||
|
||||
temp = (MPI_Aint *)malloc (sizeof(MPI_Aint) * fh->f_procs_per_group);
|
||||
if (NULL == temp) {
|
||||
opal_output(1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
memset(temp, 0x0, fh->f_procs_per_group*sizeof(MPI_Aint));
|
||||
fh->f_io_array = (mca_io_ompio_io_array_t *) malloc
|
||||
(OMPIO_IOVEC_INITIAL_SIZE * sizeof (mca_io_ompio_io_array_t));
|
||||
if (NULL == fh->f_io_array) {
|
||||
opal_output(1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
while (bytes_to_read) {
|
||||
int start = 0;
|
||||
|
||||
if (OMPIO_IOVEC_INITIAL_SIZE*block <= k) {
|
||||
block ++;
|
||||
fh->f_io_array = (mca_io_ompio_io_array_t *)realloc
|
||||
(fh->f_io_array, OMPIO_IOVEC_INITIAL_SIZE *block *
|
||||
sizeof(mca_io_ompio_io_array_t));
|
||||
if (NULL == fh->f_io_array) {
|
||||
opal_output(1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
|
||||
blocks = fview_count[0];
|
||||
for (j=0 ; j<fh->f_procs_per_group ; j++) {
|
||||
if (sorted[x] < blocks) {
|
||||
n = j;
|
||||
break;
|
||||
}
|
||||
else {
|
||||
blocks += fview_count[j+1];
|
||||
}
|
||||
}
|
||||
for (j=0 ; j<n ; j++) {
|
||||
start += bytes_per_process[j];
|
||||
}
|
||||
|
||||
if (bytes_left) {
|
||||
if (bytes_left <= bytes_to_read) {
|
||||
fh->f_io_array[k].offset = (IOVBASE_TYPE *)
|
||||
((OPAL_PTRDIFF_TYPE)global_fview[sorted[x]].iov_base +
|
||||
(global_fview[sorted[x]].iov_len - bytes_left));
|
||||
fh->f_io_array[k].length = bytes_left;
|
||||
fh->f_io_array[k].memory_address = &global_buf[start+temp[n]];
|
||||
temp[n] += fh->f_io_array[k].length;
|
||||
bytes_to_read -= bytes_left;
|
||||
bytes_left = 0;
|
||||
k ++;
|
||||
x ++;
|
||||
continue;
|
||||
}
|
||||
else {
|
||||
fh->f_io_array[k].offset = (IOVBASE_TYPE *)
|
||||
((OPAL_PTRDIFF_TYPE)global_fview[sorted[x]].iov_base +
|
||||
(global_fview[sorted[x]].iov_len - bytes_left));
|
||||
fh->f_io_array[k].length = bytes_to_read;
|
||||
fh->f_io_array[k].memory_address = &global_buf[start+temp[n]];
|
||||
temp[n] += fh->f_io_array[k].length;
|
||||
bytes_left -= bytes_to_read;
|
||||
bytes_to_read = 0;;
|
||||
k ++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (bytes_to_read < global_fview[sorted[x]].iov_len) {
|
||||
fh->f_io_array[k].offset = global_fview[sorted[x]].iov_base;
|
||||
fh->f_io_array[k].length = bytes_to_read;
|
||||
fh->f_io_array[k].memory_address = &global_buf[start+temp[n]];
|
||||
bytes_left = global_fview[sorted[x]].iov_len - bytes_to_read;
|
||||
bytes_to_read = 0;
|
||||
k ++;
|
||||
break;
|
||||
}
|
||||
else {
|
||||
fh->f_io_array[k].offset = global_fview[sorted[x]].iov_base;
|
||||
fh->f_io_array[k].length = global_fview[sorted[x]].iov_len;
|
||||
fh->f_io_array[k].memory_address = &global_buf[start+temp[n]];
|
||||
temp[n] += fh->f_io_array[k].length;
|
||||
bytes_to_read -= global_fview[sorted[x]].iov_len;
|
||||
k ++;
|
||||
x ++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fh->f_num_of_io_entries = k;
|
||||
/*
|
||||
printf("*************************** %d\n", fh->f_num_of_io_entries);
|
||||
for (i=0 ; i<fh->f_num_of_io_entries ; i++) {
|
||||
printf(" ADDRESS: %p OFFSET: %p LENGTH: %d\n",
|
||||
fh->f_io_array[i].memory_address,
|
||||
fh->f_io_array[i].offset,
|
||||
fh->f_io_array[i].length);
|
||||
}
|
||||
*/
|
||||
if (fh->f_num_of_io_entries) {
|
||||
if (OMPI_SUCCESS != fh->f_fbtl->fbtl_preadv (fh, NULL)) {
|
||||
opal_output (1, "READ FAILED\n");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
}
|
||||
if (NULL != temp) {
|
||||
free (temp);
|
||||
temp = NULL;
|
||||
}
|
||||
}
|
||||
/**********************************************************
|
||||
******************** DONE READING ************************
|
||||
*********************************************************/
|
||||
|
||||
/**********************************************************
|
||||
********* Scatter the Data from the readers **************
|
||||
*********************************************************/
|
||||
if (fh->f_flags & OMPIO_CONTIGUOUS_MEMORY) {
|
||||
receive_buf = &((char*)buf)[position];
|
||||
}
|
||||
else if (bytes_received) {
|
||||
/* allocate a receive buffer and copy the data that needs
|
||||
to be received into it in case the data is non-contigous
|
||||
in memory */
|
||||
receive_buf = malloc (bytes_received);
|
||||
if (NULL == receive_buf) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
ompi_io_ompio_scatterv_array (global_buf,
|
||||
bytes_per_process,
|
||||
displs,
|
||||
MPI_BYTE,
|
||||
receive_buf,
|
||||
bytes_received,
|
||||
MPI_BYTE,
|
||||
fh->f_aggregator_index,
|
||||
fh->f_procs_in_group,
|
||||
fh->f_procs_per_group,
|
||||
fh->f_comm);
|
||||
position += bytes_received;
|
||||
|
||||
/* If data is not contigous in memory, copy the data from the
|
||||
receive buffer into the buffer passed in */
|
||||
if (!(fh->f_flags & OMPIO_CONTIGUOUS_MEMORY)) {
|
||||
OPAL_PTRDIFF_TYPE mem_address;
|
||||
size_t remaining = 0;
|
||||
size_t temp_position = 0;
|
||||
|
||||
remaining = bytes_received;
|
||||
|
||||
while (remaining) {
|
||||
mem_address = (OPAL_PTRDIFF_TYPE)
|
||||
(decoded_iov[iov_index].iov_base) + current_position;
|
||||
|
||||
if (remaining >=
|
||||
(decoded_iov[iov_index].iov_len - current_position)) {
|
||||
memcpy ((IOVBASE_TYPE *) mem_address,
|
||||
receive_buf+temp_position,
|
||||
decoded_iov[iov_index].iov_len - current_position);
|
||||
remaining = remaining -
|
||||
(decoded_iov[iov_index].iov_len - current_position);
|
||||
temp_position = temp_position +
|
||||
(decoded_iov[iov_index].iov_len - current_position);
|
||||
iov_index = iov_index + 1;
|
||||
current_position = 0;
|
||||
}
|
||||
else {
|
||||
memcpy ((IOVBASE_TYPE *) mem_address,
|
||||
receive_buf+temp_position,
|
||||
remaining);
|
||||
current_position = current_position + remaining;
|
||||
remaining = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (NULL != receive_buf) {
|
||||
free (receive_buf);
|
||||
receive_buf = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
/**********************************************************
|
||||
**************** DONE SCATTERING OF DATA *****************
|
||||
*********************************************************/
|
||||
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
fh->f_num_of_io_entries = 0;
|
||||
if (NULL != fh->f_io_array) {
|
||||
free (fh->f_io_array);
|
||||
fh->f_io_array = NULL;
|
||||
}
|
||||
if (NULL != global_buf) {
|
||||
free (global_buf);
|
||||
global_buf = NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (NULL != sorted) {
|
||||
free (sorted);
|
||||
sorted = NULL;
|
||||
}
|
||||
if (NULL != global_fview) {
|
||||
free (global_fview);
|
||||
global_fview = NULL;
|
||||
}
|
||||
if (NULL != fview_count) {
|
||||
free (fview_count);
|
||||
fview_count = NULL;
|
||||
}
|
||||
if (NULL != decoded_iov) {
|
||||
free (decoded_iov);
|
||||
decoded_iov = NULL;
|
||||
}
|
||||
if (NULL != bytes_per_process) {
|
||||
free (bytes_per_process);
|
||||
bytes_per_process = NULL;
|
||||
}
|
||||
if (NULL != displs) {
|
||||
free (displs);
|
||||
displs = NULL;
|
||||
}
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
36
ompi/mca/fcoll/dynamic/fcoll_dynamic_file_read_all_begin.c
Обычный файл
36
ompi/mca/fcoll/dynamic/fcoll_dynamic_file_read_all_begin.c
Обычный файл
@ -0,0 +1,36 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_dynamic.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
|
||||
|
||||
int
|
||||
mca_fcoll_dynamic_file_read_all_begin (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype)
|
||||
{
|
||||
printf ("DYNAMIC READ ALL BEGIN\n");
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
35
ompi/mca/fcoll/dynamic/fcoll_dynamic_file_read_all_end.c
Обычный файл
35
ompi/mca/fcoll/dynamic/fcoll_dynamic_file_read_all_end.c
Обычный файл
@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_dynamic.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
|
||||
|
||||
int
|
||||
mca_fcoll_dynamic_file_read_all_end (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
ompi_status_public_t *status)
|
||||
{
|
||||
printf ("DYNAMIC READ ALL END\n");
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
687
ompi/mca/fcoll/dynamic/fcoll_dynamic_file_write_all.c
Обычный файл
687
ompi/mca/fcoll/dynamic/fcoll_dynamic_file_write_all.c
Обычный файл
@ -0,0 +1,687 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_dynamic.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
#include "ompi/mca/io/ompio/io_ompio.h"
|
||||
#include "ompi/mca/io/io.h"
|
||||
#include "math.h"
|
||||
#include "ompi/mca/pml/pml.h"
|
||||
#include <unistd.h>
|
||||
|
||||
#define TIME_BREAKDOWN 0
|
||||
|
||||
int
|
||||
mca_fcoll_dynamic_file_write_all (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype,
|
||||
ompi_status_public_t *status)
|
||||
{
|
||||
MPI_Aint total_bytes_written = 0; /* total bytes that have been written*/
|
||||
MPI_Aint total_bytes = 0; /* total bytes to be written */
|
||||
MPI_Aint bytes_to_write_in_cycle = 0; /* left to be written in a cycle*/
|
||||
MPI_Aint bytes_per_cycle = 0; /* total written in each cycle by each process*/
|
||||
int index = 0;
|
||||
int cycles = 0;
|
||||
int i=0, j=0, x=0;
|
||||
int n=0; /* current position in total_bytes_per_process array */
|
||||
MPI_Aint bytes_remaining = 0; /* how many bytes have been written from the current
|
||||
value from total_bytes_per_process */
|
||||
int bytes_sent = 0;
|
||||
int blocks = 0;
|
||||
/* iovec structure and count of the buffer passed in */
|
||||
uint32_t iov_count = 0;
|
||||
struct iovec *decoded_iov = NULL;
|
||||
int iov_index = 0;
|
||||
char *send_buf = NULL;
|
||||
size_t current_position = 0;
|
||||
|
||||
/* global iovec at the writers that contain the iovecs created from
|
||||
file_set_view */
|
||||
uint32_t total_fview_count = 0;
|
||||
struct iovec *global_fview = NULL;
|
||||
int local_count = 0;
|
||||
struct iovec *iov = NULL;
|
||||
int *fview_count = NULL;
|
||||
int current_index;
|
||||
|
||||
char *global_buf = NULL;
|
||||
MPI_Aint global_count = 0;
|
||||
|
||||
/* array that contains the sorted indices of the global_iov */
|
||||
int *sorted = NULL;
|
||||
int *displs = NULL;
|
||||
size_t max_data = 0;
|
||||
int *bytes_per_process = NULL;
|
||||
MPI_Aint bytes_left = 0;
|
||||
MPI_Aint *total_bytes_per_process = NULL;
|
||||
#if TIME_BREAKDOWN
|
||||
double start_time=0, end_time=0, start_time2=0, end_time2=0;
|
||||
double total=0 , total_io=0;
|
||||
#endif
|
||||
|
||||
if (opal_datatype_is_contiguous_memory_layout(&datatype->super,1)) {
|
||||
fh->f_flags |= OMPIO_CONTIGUOUS_MEMORY;
|
||||
}
|
||||
/**************************************************************************
|
||||
** In case the data is not contigous in memory, decode it into an iovec **
|
||||
**************************************************************************/
|
||||
if (! (fh->f_flags & OMPIO_CONTIGUOUS_MEMORY)) {
|
||||
ompi_io_ompio_decode_datatype (fh,
|
||||
datatype,
|
||||
count,
|
||||
buf,
|
||||
&max_data,
|
||||
&decoded_iov,
|
||||
&iov_count);
|
||||
}
|
||||
else {
|
||||
max_data = count * datatype->super.size;
|
||||
}
|
||||
|
||||
if (! (fh->f_flags & OMPIO_AGGREGATOR_IS_SET)) {
|
||||
ompi_io_ompio_set_aggregator_props (fh,
|
||||
mca_fcoll_dynamic_num_io_procs,
|
||||
max_data);
|
||||
}
|
||||
|
||||
#if TIME_BREAKDOWN
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
start_time = MPI_Wtime();
|
||||
}
|
||||
#endif
|
||||
|
||||
total_bytes_per_process = (MPI_Aint*)malloc
|
||||
(fh->f_procs_per_group*sizeof(MPI_Aint));
|
||||
if (NULL == total_bytes_per_process) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
ompi_io_ompio_allgather_array (&max_data,
|
||||
1,
|
||||
MPI_LONG,
|
||||
total_bytes_per_process,
|
||||
1,
|
||||
MPI_LONG,
|
||||
fh->f_aggregator_index,
|
||||
fh->f_procs_in_group,
|
||||
fh->f_procs_per_group,
|
||||
fh->f_comm);
|
||||
|
||||
for (i=0 ; i<fh->f_procs_per_group ; i++) {
|
||||
total_bytes += total_bytes_per_process[i];
|
||||
}
|
||||
|
||||
if (NULL != total_bytes_per_process) {
|
||||
free (total_bytes_per_process);
|
||||
total_bytes_per_process = NULL;
|
||||
}
|
||||
|
||||
/*********************************************************************
|
||||
*** Generate the File offsets/lengths corresponding to this write ***
|
||||
********************************************************************/
|
||||
ompi_io_ompio_generate_current_file_view (fh, max_data, &iov, &local_count);
|
||||
/*
|
||||
for (i=0 ; i<local_count ; i++) {
|
||||
printf("%d: OFFSET: %p LENGTH: %lld\n",
|
||||
fh->f_rank,
|
||||
iov[i].iov_base,
|
||||
iov[i].iov_len);
|
||||
}
|
||||
*/
|
||||
/*************************************************************
|
||||
*** ALLGather the File View information at all processes ***
|
||||
*************************************************************/
|
||||
|
||||
fview_count = (int *) malloc (fh->f_procs_per_group * sizeof (int));
|
||||
if (NULL == fview_count) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
ompi_io_ompio_allgather_array (&local_count,
|
||||
1,
|
||||
MPI_INT,
|
||||
fview_count,
|
||||
1,
|
||||
MPI_INT,
|
||||
fh->f_aggregator_index,
|
||||
fh->f_procs_in_group,
|
||||
fh->f_procs_per_group,
|
||||
fh->f_comm);
|
||||
displs = (int*) malloc (fh->f_procs_per_group * sizeof (int));
|
||||
if (NULL == displs) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
displs[0] = 0;
|
||||
total_fview_count = fview_count[0];
|
||||
for (i=1 ; i<fh->f_procs_per_group ; i++) {
|
||||
total_fview_count += fview_count[i];
|
||||
displs[i] = displs[i-1] + fview_count[i-1];
|
||||
}
|
||||
/*
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
for (i=0 ; i<fh->f_procs_per_group ; i++) {
|
||||
printf ("%d: PROCESS: %d ELEMENTS: %d DISPLS: %d\n",
|
||||
fh->f_rank,
|
||||
i,
|
||||
fview_count[i],
|
||||
displs[i]);
|
||||
}
|
||||
}
|
||||
*/
|
||||
/* allocate the global iovec */
|
||||
if (0 != total_fview_count) {
|
||||
global_fview = (struct iovec*)malloc (total_fview_count *
|
||||
sizeof(struct iovec));
|
||||
if (NULL == global_fview) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
if (fh->f_flags & OMPIO_UNIFORM_FVIEW) {
|
||||
ompi_io_ompio_allgather_array (iov,
|
||||
local_count,
|
||||
fh->f_iov_type,
|
||||
global_fview,
|
||||
local_count,
|
||||
fh->f_iov_type,
|
||||
fh->f_aggregator_index,
|
||||
fh->f_procs_in_group,
|
||||
fh->f_procs_per_group,
|
||||
fh->f_comm);
|
||||
}
|
||||
else {
|
||||
ompi_io_ompio_allgatherv_array (iov,
|
||||
local_count,
|
||||
fh->f_iov_type,
|
||||
global_fview,
|
||||
fview_count,
|
||||
displs,
|
||||
fh->f_iov_type,
|
||||
fh->f_aggregator_index,
|
||||
fh->f_procs_in_group,
|
||||
fh->f_procs_per_group,
|
||||
fh->f_comm);
|
||||
}
|
||||
/* sort it */
|
||||
if (0 != total_fview_count) {
|
||||
sorted = (int *)malloc (total_fview_count * sizeof(int));
|
||||
if (NULL == sorted) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
ompi_io_ompio_sort_iovec (global_fview, total_fview_count, sorted);
|
||||
}
|
||||
|
||||
if (NULL != iov) {
|
||||
free (iov);
|
||||
iov = NULL;
|
||||
}
|
||||
/*
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
for (i=0 ; i<total_fview_count ; i++) {
|
||||
printf("%d: OFFSET: %p LENGTH: %lld\n",
|
||||
total_fview_count,
|
||||
global_fview[i].iov_base,
|
||||
global_fview[i].iov_len);
|
||||
}
|
||||
}
|
||||
*/
|
||||
/*
|
||||
fh->f_flags |= OMPIO_UNIFORM_FVIEW;
|
||||
if (ompi_ddt_is_contiguous_memory_layout (datatype, count))
|
||||
fh->f_flags |= OMPIO_CONTIGUOUS_MEMORY;
|
||||
*/
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
bytes_per_process = (int *)malloc (fh->f_procs_per_group * sizeof (int));
|
||||
if (NULL == bytes_per_process) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
|
||||
/* MSC THIS does not work at the moment
|
||||
* Calculate how many bytes are written in each cycle
|
||||
if (mca_fcoll_dynamic_constant_cbs) {
|
||||
bytes_per_cycle =
|
||||
mca_fcoll_dynamic_cycle_buffer_size/mca_fcoll_dynamic_num_io_procs;
|
||||
}
|
||||
else {
|
||||
*/
|
||||
bytes_per_cycle = mca_fcoll_dynamic_cycle_buffer_size;
|
||||
|
||||
cycles = ceil((double)total_bytes/bytes_per_cycle);
|
||||
|
||||
n = 0;
|
||||
bytes_remaining = 0;
|
||||
current_index = 0;
|
||||
|
||||
#if TIME_BREAKDOWN
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
end_time = MPI_Wtime();
|
||||
total = end_time-start_time;
|
||||
printf ("%d: Preprocessing --- %f\n", fh->f_rank, total);
|
||||
total = 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
for (index = 0; index < cycles; index++) {
|
||||
int k;
|
||||
|
||||
#if TIME_BREAKDOWN
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
start_time = MPI_Wtime();
|
||||
}
|
||||
#endif
|
||||
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
memset(displs, 0x0, fh->f_procs_per_group*sizeof(int));
|
||||
memset(bytes_per_process, 0x0, fh->f_procs_per_group*sizeof(int));
|
||||
}
|
||||
|
||||
if (cycles-1 == index) {
|
||||
bytes_to_write_in_cycle = total_bytes - bytes_per_cycle*index;
|
||||
}
|
||||
else {
|
||||
bytes_to_write_in_cycle = bytes_per_cycle;
|
||||
}
|
||||
/*
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
printf ("****%d: CYCLE %d Bytes %lld**********\n",
|
||||
fh->f_rank,
|
||||
index,
|
||||
bytes_to_write_in_cycle);
|
||||
}
|
||||
*/
|
||||
/**********************************************************
|
||||
**Gather the Data from all the processes at the writers **
|
||||
*********************************************************/
|
||||
|
||||
/* Calculate how much data will be contributed in this cycle
|
||||
by each process*/
|
||||
bytes_sent = 0;
|
||||
|
||||
while (bytes_to_write_in_cycle) {
|
||||
blocks = fview_count[0];
|
||||
for (j=0 ; j<fh->f_procs_per_group ; j++) {
|
||||
if (sorted[current_index] < blocks) {
|
||||
n = j;
|
||||
break;
|
||||
}
|
||||
else {
|
||||
blocks += fview_count[j+1];
|
||||
}
|
||||
}
|
||||
if (bytes_remaining) {
|
||||
if (bytes_remaining <= bytes_to_write_in_cycle) {
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] ==
|
||||
fh->f_rank) {
|
||||
bytes_per_process[n] += bytes_remaining;
|
||||
}
|
||||
if (fh->f_procs_in_group[n] == fh->f_rank) {
|
||||
bytes_sent += bytes_remaining;
|
||||
}
|
||||
current_index ++;
|
||||
bytes_to_write_in_cycle -= bytes_remaining;
|
||||
bytes_remaining = 0;
|
||||
continue;
|
||||
}
|
||||
else {
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] ==
|
||||
fh->f_rank) {
|
||||
bytes_per_process[n] += bytes_to_write_in_cycle;
|
||||
}
|
||||
if (fh->f_procs_in_group[n] == fh->f_rank) {
|
||||
bytes_sent += bytes_to_write_in_cycle;
|
||||
}
|
||||
bytes_remaining -= bytes_to_write_in_cycle;
|
||||
bytes_to_write_in_cycle = 0;
|
||||
break;
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (bytes_to_write_in_cycle <
|
||||
global_fview[sorted[current_index]].iov_len) {
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] ==
|
||||
fh->f_rank) {
|
||||
bytes_per_process[n] += bytes_to_write_in_cycle;
|
||||
}
|
||||
if (fh->f_procs_in_group[n] == fh->f_rank) {
|
||||
bytes_sent += bytes_to_write_in_cycle;
|
||||
}
|
||||
bytes_remaining = global_fview[sorted[current_index]].iov_len -
|
||||
bytes_to_write_in_cycle;
|
||||
bytes_to_write_in_cycle = 0;
|
||||
break;
|
||||
}
|
||||
else {
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] ==
|
||||
fh->f_rank) {
|
||||
bytes_per_process[n] +=
|
||||
global_fview[sorted[current_index]].iov_len;
|
||||
}
|
||||
if (fh->f_procs_in_group[n] == fh->f_rank) {
|
||||
bytes_sent += global_fview[sorted[current_index]].iov_len;
|
||||
}
|
||||
bytes_to_write_in_cycle -=
|
||||
global_fview[sorted[current_index]].iov_len;
|
||||
current_index ++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
/* Calculate the displacement on where to put the data and allocate
|
||||
the recieve buffer (global_buf) */
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
displs[0] = 0;
|
||||
global_count = bytes_per_process[0];
|
||||
for (i=1 ; i<fh->f_procs_per_group ; i++) {
|
||||
global_count += bytes_per_process[i];
|
||||
displs[i] = displs[i-1] + bytes_per_process[i-1];
|
||||
}
|
||||
/*
|
||||
for (i=0 ; i<fh->f_procs_per_group ; i++) {
|
||||
printf ("Proc %d sending %lld at %lld\n",
|
||||
i,
|
||||
bytes_per_process[i],
|
||||
displs[i]);
|
||||
}
|
||||
*/
|
||||
global_buf = malloc (global_count);
|
||||
if (NULL == global_buf) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
|
||||
if (fh->f_flags & OMPIO_CONTIGUOUS_MEMORY) {
|
||||
send_buf = &((char*)buf)[total_bytes_written];
|
||||
}
|
||||
else if (bytes_sent) {
|
||||
/* allocate a send buffer and copy the data that needs
|
||||
to be sent into it in case the data is non-contigous
|
||||
in memory */
|
||||
OPAL_PTRDIFF_TYPE mem_address;
|
||||
size_t remaining = 0;
|
||||
size_t temp_position = 0;
|
||||
|
||||
send_buf = malloc (bytes_sent);
|
||||
if (NULL == send_buf) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
remaining = bytes_sent;
|
||||
|
||||
while (remaining) {
|
||||
mem_address = (OPAL_PTRDIFF_TYPE)
|
||||
(decoded_iov[iov_index].iov_base) + current_position;
|
||||
|
||||
if (remaining >=
|
||||
(decoded_iov[iov_index].iov_len - current_position)) {
|
||||
memcpy (send_buf+temp_position,
|
||||
(IOVBASE_TYPE *)mem_address,
|
||||
decoded_iov[iov_index].iov_len - current_position);
|
||||
remaining = remaining -
|
||||
(decoded_iov[iov_index].iov_len - current_position);
|
||||
temp_position = temp_position +
|
||||
(decoded_iov[iov_index].iov_len - current_position);
|
||||
iov_index = iov_index + 1;
|
||||
current_position = 0;
|
||||
}
|
||||
else {
|
||||
memcpy (send_buf+temp_position,
|
||||
(IOVBASE_TYPE *) mem_address,
|
||||
remaining);
|
||||
current_position = current_position + remaining;
|
||||
remaining = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
total_bytes_written += bytes_sent;
|
||||
|
||||
/* Get the data from all processes to the writer*/
|
||||
ompi_io_ompio_gatherv_array (send_buf,
|
||||
bytes_sent,
|
||||
MPI_BYTE,
|
||||
global_buf,
|
||||
bytes_per_process,
|
||||
displs,
|
||||
MPI_BYTE,
|
||||
fh->f_aggregator_index,
|
||||
fh->f_procs_in_group,
|
||||
fh->f_procs_per_group,
|
||||
fh->f_comm);
|
||||
|
||||
if (! (fh->f_flags & OMPIO_CONTIGUOUS_MEMORY)) {
|
||||
if (NULL != send_buf) {
|
||||
free (send_buf);
|
||||
send_buf = NULL;
|
||||
}
|
||||
}
|
||||
/*
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank)
|
||||
for (i=0 ; i<global_count/4 ; i++)
|
||||
printf ("RECV %d \n",
|
||||
((int *)global_buf)[i]);
|
||||
*/
|
||||
/**********************************************************
|
||||
**************** DONE GATHERING OF DATA ******************
|
||||
*********************************************************/
|
||||
|
||||
/**********************************************************
|
||||
******* Create the io array, and pass it to fbtl *********
|
||||
*********************************************************/
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
MPI_Aint bytes_to_write = global_count;
|
||||
MPI_Aint *temp = NULL;
|
||||
int block = 1;
|
||||
k = 0;
|
||||
|
||||
temp = (MPI_Aint *)malloc (sizeof(MPI_Aint) * fh->f_procs_per_group);
|
||||
if (NULL == temp) {
|
||||
opal_output(1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
memset(temp, 0x0, fh->f_procs_per_group*sizeof(MPI_Aint));
|
||||
fh->f_io_array = (mca_io_ompio_io_array_t *) malloc
|
||||
(OMPIO_IOVEC_INITIAL_SIZE * sizeof (mca_io_ompio_io_array_t));
|
||||
if (NULL == fh->f_io_array) {
|
||||
opal_output(1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
while (bytes_to_write) {
|
||||
int start = 0;
|
||||
|
||||
if (OMPIO_IOVEC_INITIAL_SIZE*block <= k) {
|
||||
block ++;
|
||||
fh->f_io_array = (mca_io_ompio_io_array_t *)realloc
|
||||
(fh->f_io_array, OMPIO_IOVEC_INITIAL_SIZE * block *
|
||||
sizeof(mca_io_ompio_io_array_t));
|
||||
if (NULL == fh->f_io_array) {
|
||||
opal_output(1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
|
||||
blocks = fview_count[0];
|
||||
for (j=0 ; j<fh->f_procs_per_group ; j++) {
|
||||
if (sorted[x] < blocks) {
|
||||
n = j;
|
||||
break;
|
||||
}
|
||||
else {
|
||||
blocks += fview_count[j+1];
|
||||
}
|
||||
}
|
||||
for (j=0 ; j<n ; j++) {
|
||||
start += bytes_per_process[j];
|
||||
}
|
||||
|
||||
if (bytes_left) {
|
||||
if (bytes_left <= bytes_to_write) {
|
||||
fh->f_io_array[k].offset = (IOVBASE_TYPE *)
|
||||
((OPAL_PTRDIFF_TYPE)global_fview[sorted[x]].iov_base +
|
||||
(global_fview[sorted[x]].iov_len - bytes_left));
|
||||
fh->f_io_array[k].length = bytes_left;
|
||||
fh->f_io_array[k].memory_address =
|
||||
&global_buf[start+temp[n]];
|
||||
temp[n] += fh->f_io_array[k].length;
|
||||
bytes_to_write -= bytes_left;
|
||||
bytes_left = 0;
|
||||
k ++;
|
||||
x ++;
|
||||
continue;
|
||||
}
|
||||
else {
|
||||
fh->f_io_array[k].offset = (IOVBASE_TYPE *)
|
||||
((OPAL_PTRDIFF_TYPE)global_fview[sorted[x]].iov_base +
|
||||
(global_fview[sorted[x]].iov_len - bytes_left));
|
||||
fh->f_io_array[k].length = bytes_to_write;
|
||||
fh->f_io_array[k].memory_address =
|
||||
&global_buf[start+temp[n]];
|
||||
temp[n] += fh->f_io_array[k].length;
|
||||
bytes_left -= bytes_to_write;
|
||||
bytes_to_write = 0;;
|
||||
k ++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (bytes_to_write < global_fview[sorted[x]].iov_len) {
|
||||
fh->f_io_array[k].offset = global_fview[sorted[x]].iov_base;
|
||||
fh->f_io_array[k].length = bytes_to_write;
|
||||
fh->f_io_array[k].memory_address =
|
||||
&global_buf[start+temp[n]];
|
||||
bytes_left =
|
||||
global_fview[sorted[x]].iov_len - bytes_to_write;
|
||||
bytes_to_write = 0;
|
||||
k ++;
|
||||
break;
|
||||
}
|
||||
else {
|
||||
fh->f_io_array[k].offset = global_fview[sorted[x]].iov_base;
|
||||
fh->f_io_array[k].length = global_fview[sorted[x]].iov_len;
|
||||
fh->f_io_array[k].memory_address =
|
||||
&global_buf[start+temp[n]];
|
||||
temp[n] += fh->f_io_array[k].length;
|
||||
bytes_to_write -= global_fview[sorted[x]].iov_len;
|
||||
k ++;
|
||||
x ++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fh->f_num_of_io_entries = k;
|
||||
/*
|
||||
printf("*************************** %d\n", fh->f_num_of_io_entries);
|
||||
for (i=0 ; i<fh->f_num_of_io_entries ; i++) {
|
||||
printf(" ADDRESS: %p OFFSET: %p LENGTH: %d\n",
|
||||
fh->f_io_array[i].memory_address,
|
||||
fh->f_io_array[i].offset,
|
||||
fh->f_io_array[i].length);
|
||||
}
|
||||
*/
|
||||
#if TIME_BREAKDOWN
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
start_time2 = MPI_Wtime();
|
||||
}
|
||||
#endif
|
||||
if (fh->f_num_of_io_entries) {
|
||||
if (OMPI_SUCCESS != fh->f_fbtl->fbtl_pwritev (fh, NULL)) {
|
||||
opal_output (1, "WRITE FAILED\n");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
}
|
||||
#if TIME_BREAKDOWN
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
end_time2 = MPI_Wtime();
|
||||
total_io += end_time2-start_time2;
|
||||
}
|
||||
#endif
|
||||
if (NULL != temp) {
|
||||
free (temp);
|
||||
temp = NULL;
|
||||
}
|
||||
}
|
||||
/**********************************************************
|
||||
******************** DONE WRITING ************************
|
||||
*********************************************************/
|
||||
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
fh->f_num_of_io_entries = 0;
|
||||
if (NULL != fh->f_io_array) {
|
||||
free (fh->f_io_array);
|
||||
fh->f_io_array = NULL;
|
||||
}
|
||||
if (NULL != global_buf) {
|
||||
free (global_buf);
|
||||
global_buf = NULL;
|
||||
}
|
||||
}
|
||||
#if TIME_BREAKDOWN
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
end_time = MPI_Wtime();
|
||||
total += end_time-start_time;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
#if TIME_BREAKDOWN
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
printf ("%d: Total --- %f I/O ---- %f\n", fh->f_rank, total, total_io);
|
||||
}
|
||||
#endif
|
||||
|
||||
if (NULL != sorted) {
|
||||
free (sorted);
|
||||
sorted = NULL;
|
||||
}
|
||||
if (NULL != global_fview) {
|
||||
free (global_fview);
|
||||
global_fview = NULL;
|
||||
}
|
||||
if (NULL != fview_count) {
|
||||
free (fview_count);
|
||||
fview_count = NULL;
|
||||
}
|
||||
if (NULL != decoded_iov) {
|
||||
free (decoded_iov);
|
||||
decoded_iov = NULL;
|
||||
}
|
||||
if (NULL != bytes_per_process) {
|
||||
free (bytes_per_process);
|
||||
bytes_per_process = NULL;
|
||||
}
|
||||
if (NULL != displs) {
|
||||
free (displs);
|
||||
displs = NULL;
|
||||
}
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
36
ompi/mca/fcoll/dynamic/fcoll_dynamic_file_write_all_begin.c
Обычный файл
36
ompi/mca/fcoll/dynamic/fcoll_dynamic_file_write_all_begin.c
Обычный файл
@ -0,0 +1,36 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_dynamic.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
|
||||
|
||||
int
|
||||
mca_fcoll_dynamic_file_write_all_begin (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype)
|
||||
{
|
||||
printf ("DYNAMIC WRITE ALL BEGIN\n");
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
35
ompi/mca/fcoll/dynamic/fcoll_dynamic_file_write_all_end.c
Обычный файл
35
ompi/mca/fcoll/dynamic/fcoll_dynamic_file_write_all_end.c
Обычный файл
@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_dynamic.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
|
||||
|
||||
int
|
||||
mca_fcoll_dynamic_file_write_all_end (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
ompi_status_public_t *status)
|
||||
{
|
||||
printf ("DYNAMIC WRITE ALL END\n");
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
91
ompi/mca/fcoll/dynamic/fcoll_dynamic_module.c
Обычный файл
91
ompi/mca/fcoll/dynamic/fcoll_dynamic_module.c
Обычный файл
@ -0,0 +1,91 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2006 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_dynamic.h"
|
||||
|
||||
#include <stdio.h>
|
||||
|
||||
#include "mpi.h"
|
||||
#include "opal/mca/base/mca_base_param.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
#include "ompi/mca/fcoll/base/base.h"
|
||||
|
||||
|
||||
/*
|
||||
* *******************************************************************
|
||||
* ************************ actions structure ************************
|
||||
* *******************************************************************
|
||||
*/
|
||||
static mca_fcoll_base_module_1_0_0_t dynamic = {
|
||||
mca_fcoll_dynamic_module_init,
|
||||
mca_fcoll_dynamic_module_finalize,
|
||||
mca_fcoll_dynamic_file_read_all,
|
||||
mca_fcoll_dynamic_file_read_all_begin,
|
||||
mca_fcoll_dynamic_file_read_all_end,
|
||||
mca_fcoll_dynamic_file_write_all,
|
||||
mca_fcoll_dynamic_file_write_all_begin,
|
||||
mca_fcoll_dynamic_file_write_all_end
|
||||
};
|
||||
|
||||
int
|
||||
mca_fcoll_dynamic_component_init_query(bool enable_progress_threads,
|
||||
bool enable_mpi_threads)
|
||||
{
|
||||
/* Nothing to do */
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
mca_fcoll_base_module_1_0_0_t *
|
||||
mca_fcoll_dynamic_component_file_query (mca_io_ompio_file_t *fh, int *priority)
|
||||
{
|
||||
*priority = mca_fcoll_dynamic_priority;
|
||||
if (0 >= mca_fcoll_dynamic_priority) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (mca_fcoll_base_query_table (fh, "dynamic")) {
|
||||
if (*priority < 50) {
|
||||
*priority = 50;
|
||||
}
|
||||
}
|
||||
|
||||
return &dynamic;
|
||||
}
|
||||
|
||||
int mca_fcoll_dynamic_component_file_unquery (mca_io_ompio_file_t *file)
|
||||
{
|
||||
/* This function might be needed for some purposes later. for now it
|
||||
* does not have anything to do since there are no steps which need
|
||||
* to be undone if this module is not selected */
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
int mca_fcoll_dynamic_module_init (mca_io_ompio_file_t *file)
|
||||
{
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
int mca_fcoll_dynamic_module_finalize (mca_io_ompio_file_t *file)
|
||||
{
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
166
ompi/mca/fcoll/fcoll.h
Обычный файл
166
ompi/mca/fcoll/fcoll.h
Обычный файл
@ -0,0 +1,166 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#ifndef OMPI_MCA_FCOLL_H
|
||||
#define OMPI_MCA_FCOLL_H
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "mpi.h"
|
||||
#include "opal/mca/mca.h"
|
||||
#include "opal/mca/base/base.h"
|
||||
|
||||
BEGIN_C_DECLS
|
||||
|
||||
struct mca_io_ompio_file_t;
|
||||
|
||||
/*
|
||||
* Macro for use in components that are of type coll
|
||||
*/
|
||||
#define MCA_FCOLL_BASE_VERSION_2_0_0 \
|
||||
MCA_BASE_VERSION_2_0_0, \
|
||||
"fcoll", 2, 0, 0
|
||||
|
||||
/*
|
||||
* These are the component function prototypes. These function pointers
|
||||
* go into the component structure. These functions (query() and finalize()
|
||||
* are called during fcoll_base_select(). Each component is query() ied
|
||||
* and subsequently, all the unselected components are finalize() 'ed
|
||||
* so that any *stuff* they did during query() can be undone. By
|
||||
* similar logic, finalize() is also called on the component which
|
||||
* was selected when the communicator is being destroyed.
|
||||
*
|
||||
* So, to sum it up, every component carries 4 functions:
|
||||
* 1. open() - called during MPI_INIT
|
||||
* 2. close() - called during MPI_FINALIZE
|
||||
* 3. query() - called to select a particular component
|
||||
* 4. finalize() - called when actions taken during query have
|
||||
* to be undone
|
||||
*/
|
||||
|
||||
/*
|
||||
* **************** component struct *******************************
|
||||
* *********** These functions go in the component struct **********
|
||||
* **************** component struct *******************************
|
||||
*/
|
||||
|
||||
typedef int (*mca_fcoll_base_component_init_query_1_0_0_fn_t)
|
||||
(bool enable_progress_threads,
|
||||
bool enable_mpi_threads);
|
||||
|
||||
typedef struct mca_fcoll_base_module_1_0_0_t *
|
||||
(*mca_fcoll_base_component_file_query_1_0_0_fn_t) (struct mca_io_ompio_file_t *file,
|
||||
int *priority);
|
||||
|
||||
typedef int (*mca_fcoll_base_component_file_unquery_1_0_0_fn_t)
|
||||
(struct mca_io_ompio_file_t *file);
|
||||
|
||||
/*
|
||||
* ****************** component struct ******************************
|
||||
* Structure for fcoll v2.0.0 components.This is chained to MCA v2.0.0
|
||||
* ****************** component struct ******************************
|
||||
*/
|
||||
struct mca_fcoll_base_component_2_0_0_t {
|
||||
mca_base_component_t fcollm_version;
|
||||
mca_base_component_data_t fcollm_data;
|
||||
|
||||
mca_fcoll_base_component_init_query_1_0_0_fn_t fcollm_init_query;
|
||||
mca_fcoll_base_component_file_query_1_0_0_fn_t fcollm_file_query;
|
||||
mca_fcoll_base_component_file_unquery_1_0_0_fn_t fcollm_file_unquery;
|
||||
};
|
||||
typedef struct mca_fcoll_base_component_2_0_0_t mca_fcoll_base_component_2_0_0_t;
|
||||
typedef struct mca_fcoll_base_component_2_0_0_t mca_fcoll_base_component_t;
|
||||
|
||||
/*
|
||||
* ***********************************************************************
|
||||
* ************************ Interface function definitions **************
|
||||
* These are the typedefcoll for the function pointers to various fcoll
|
||||
* backend functions which will be used by the various fcoll components
|
||||
* ***********************************************************************
|
||||
*/
|
||||
|
||||
typedef int (*mca_fcoll_base_module_init_1_0_0_fn_t)
|
||||
(struct mca_io_ompio_file_t *file);
|
||||
|
||||
typedef int (*mca_fcoll_base_module_finalize_1_0_0_fn_t)
|
||||
(struct mca_io_ompio_file_t *file);
|
||||
|
||||
typedef int (*mca_fcoll_base_module_file_read_all_fn_t)
|
||||
(struct mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype,
|
||||
ompi_status_public_t *status);
|
||||
|
||||
typedef int (*mca_fcoll_base_module_file_read_all_begin_fn_t)
|
||||
(struct mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype);
|
||||
|
||||
typedef int (*mca_fcoll_base_module_file_read_all_end_fn_t)
|
||||
(struct mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
ompi_status_public_t *status);
|
||||
|
||||
typedef int (*mca_fcoll_base_module_file_write_all_fn_t)
|
||||
(struct mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype,
|
||||
ompi_status_public_t *status);
|
||||
|
||||
typedef int (*mca_fcoll_base_module_file_write_all_begin_fn_t)
|
||||
(struct mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype);
|
||||
|
||||
typedef int (*mca_fcoll_base_module_file_write_all_end_fn_t)
|
||||
(struct mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
ompi_status_public_t *status);
|
||||
|
||||
/*
|
||||
* ***********************************************************************
|
||||
* *************************** module structure *************************
|
||||
* ***********************************************************************
|
||||
*/
|
||||
struct mca_fcoll_base_module_1_0_0_t {
|
||||
/*
|
||||
* Per-file initialization function. This is called only
|
||||
* on the module which is selected. The finalize corresponding to
|
||||
* this function is present on the component struct above
|
||||
*/
|
||||
mca_fcoll_base_module_init_1_0_0_fn_t fcoll_module_init;
|
||||
mca_fcoll_base_module_finalize_1_0_0_fn_t fcoll_module_finalize;
|
||||
|
||||
/* FCOLL function pointers */
|
||||
mca_fcoll_base_module_file_read_all_fn_t fcoll_file_read_all;
|
||||
mca_fcoll_base_module_file_read_all_begin_fn_t fcoll_file_read_all_begin;
|
||||
mca_fcoll_base_module_file_read_all_end_fn_t fcoll_file_read_all_end;
|
||||
mca_fcoll_base_module_file_write_all_fn_t fcoll_file_write_all;
|
||||
mca_fcoll_base_module_file_write_all_begin_fn_t fcoll_file_write_all_begin;
|
||||
mca_fcoll_base_module_file_write_all_end_fn_t fcoll_file_write_all_end;
|
||||
};
|
||||
typedef struct mca_fcoll_base_module_1_0_0_t mca_fcoll_base_module_1_0_0_t;
|
||||
typedef mca_fcoll_base_module_1_0_0_t mca_fcoll_base_module_t;
|
||||
|
||||
END_C_DECLS
|
||||
|
||||
#endif /* OMPI_MCA_FCOLL_H */
|
||||
54
ompi/mca/fcoll/individual/Makefile.am
Обычный файл
54
ompi/mca/fcoll/individual/Makefile.am
Обычный файл
@ -0,0 +1,54 @@
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
sources = \
|
||||
fcoll_individual.h \
|
||||
fcoll_individual_module.c \
|
||||
fcoll_individual_component.c \
|
||||
fcoll_individual_file_read_all.c \
|
||||
fcoll_individual_file_read_all_begin.c \
|
||||
fcoll_individual_file_read_all_end.c \
|
||||
fcoll_individual_file_write_all.c \
|
||||
fcoll_individual_file_write_all_begin.c \
|
||||
fcoll_individual_file_write_all_end.c
|
||||
|
||||
# Make the output library in this directory, and name it either
|
||||
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
|
||||
# (for static builds).
|
||||
|
||||
if MCA_BUILD_ompi_fcoll_individual_DSO
|
||||
component_noinst =
|
||||
component_install = mca_fcoll_individual.la
|
||||
else
|
||||
component_noinst = libmca_fcoll_individual.la
|
||||
component_install =
|
||||
endif
|
||||
|
||||
mcacomponentdir = $(libdir)/openmpi
|
||||
mcacomponent_LTLIBRARIES = $(component_install)
|
||||
mca_fcoll_individual_la_SOURCES = $(sources)
|
||||
mca_fcoll_individual_la_LDFLAGS = -module -avoid-version
|
||||
mca_fcoll_individual_la_LIBADD = \
|
||||
$(top_ompi_builddir)/ompi/libmpi.la \
|
||||
$(top_ompi_builddir)/orte/libopen-rte.la \
|
||||
$(top_ompi_builddir)/opal/libopen-pal.la
|
||||
|
||||
noinst_LTLIBRARIES = $(component_noinst)
|
||||
libmca_fcoll_individual_la_SOURCES =$(sources)
|
||||
libmca_fcoll_individual_la_LDFLAGS = -module -avoid-version
|
||||
85
ompi/mca/fcoll/individual/fcoll_individual.h
Обычный файл
85
ompi/mca/fcoll/individual/fcoll_individual.h
Обычный файл
@ -0,0 +1,85 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2006 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#ifndef MCA_FCOLL_INDIVIDUAL_EXPORT_H
|
||||
#define MCA_FCOLL_INDIVIDUAL_EXPORT_H
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "opal/mca/mca.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
#include "ompi/mca/fcoll/base/base.h"
|
||||
#include "ompi/mca/io/ompio/io_ompio.h"
|
||||
|
||||
BEGIN_C_DECLS
|
||||
|
||||
/* Globally exported variables */
|
||||
|
||||
extern int mca_fcoll_individual_priority;
|
||||
extern int mca_fcoll_individual_constant_cbs;
|
||||
extern int mca_fcoll_individual_cycle_buffer_size;
|
||||
|
||||
OMPI_MODULE_DECLSPEC extern mca_fcoll_base_component_2_0_0_t mca_fcoll_individual_component;
|
||||
|
||||
/* API functions */
|
||||
|
||||
int mca_fcoll_individual_component_init_query(bool enable_progress_threads,
|
||||
bool enable_mpi_threads);
|
||||
struct mca_fcoll_base_module_1_0_0_t *
|
||||
mca_fcoll_individual_component_file_query (mca_io_ompio_file_t *fh, int *priority);
|
||||
|
||||
int mca_fcoll_individual_component_file_unquery (mca_io_ompio_file_t *file);
|
||||
|
||||
int mca_fcoll_individual_module_init (mca_io_ompio_file_t *file);
|
||||
int mca_fcoll_individual_module_finalize (mca_io_ompio_file_t *file);
|
||||
|
||||
int mca_fcoll_individual_file_read_all (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype,
|
||||
ompi_status_public_t * status);
|
||||
|
||||
int mca_fcoll_individual_file_read_all_begin (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype);
|
||||
|
||||
int mca_fcoll_individual_file_read_all_end (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
ompi_status_public_t * status);
|
||||
|
||||
int mca_fcoll_individual_file_write_all (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype,
|
||||
ompi_status_public_t * status);
|
||||
|
||||
int mca_fcoll_individual_file_write_all_begin (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype);
|
||||
|
||||
int mca_fcoll_individual_file_write_all_end (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
ompi_status_public_t * status);
|
||||
|
||||
END_C_DECLS
|
||||
|
||||
#endif /* MCA_FCOLL_INDIVIDUAL_EXPORT_H */
|
||||
121
ompi/mca/fcoll/individual/fcoll_individual_component.c
Обычный файл
121
ompi/mca/fcoll/individual/fcoll_individual_component.c
Обычный файл
@ -0,0 +1,121 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*
|
||||
* These symbols are in a file by themselves to provide nice linker
|
||||
* semantics. Since linkers generally pull in symbols by object
|
||||
* files, keeping these symbols as the only symbols in this file
|
||||
* prevents utility programs such as "ompi_info" from having to import
|
||||
* entire components just to query their version and parameters.
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_individual.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
#include "fcoll_individual.h"
|
||||
|
||||
/*
|
||||
* Public string showing the fcoll ompi_individual component version number
|
||||
*/
|
||||
const char *mca_fcoll_individual_component_version_string =
|
||||
"Open MPI individual collective MCA component version " OMPI_VERSION;
|
||||
|
||||
/*
|
||||
* Global variables
|
||||
*/
|
||||
int mca_fcoll_individual_priority = 10;
|
||||
int mca_fcoll_individual_constant_cbs = 0;
|
||||
int mca_fcoll_individual_cycle_buffer_size = OMPIO_PREALLOC_MAX_BUF_SIZE;
|
||||
|
||||
/*
|
||||
* Local function
|
||||
*/
|
||||
static int individual_register(void);
|
||||
|
||||
/*
|
||||
* Instantiate the public struct with all of our public information
|
||||
* and pointers to our public functions in it
|
||||
*/
|
||||
mca_fcoll_base_component_2_0_0_t mca_fcoll_individual_component = {
|
||||
|
||||
/* First, the mca_component_t struct containing meta information
|
||||
* about the component itself */
|
||||
|
||||
{
|
||||
MCA_FCOLL_BASE_VERSION_2_0_0,
|
||||
|
||||
/* Component name and version */
|
||||
"individual",
|
||||
OMPI_MAJOR_VERSION,
|
||||
OMPI_MINOR_VERSION,
|
||||
OMPI_RELEASE_VERSION,
|
||||
individual_register,
|
||||
NULL
|
||||
},
|
||||
{
|
||||
/* The component is checkpoint ready */
|
||||
MCA_BASE_METADATA_PARAM_CHECKPOINT
|
||||
},
|
||||
|
||||
mca_fcoll_individual_component_init_query,
|
||||
mca_fcoll_individual_component_file_query,
|
||||
mca_fcoll_individual_component_file_unquery
|
||||
};
|
||||
|
||||
|
||||
static int
|
||||
individual_register(void)
|
||||
{
|
||||
int param;
|
||||
|
||||
param = mca_base_param_find ("fcoll", NULL, "individual_priority");
|
||||
if (param >= 0)
|
||||
{
|
||||
mca_base_param_lookup_int (param, &mca_fcoll_individual_priority);
|
||||
}
|
||||
param = mca_base_param_find ("fcoll", NULL, "individual_constant_cbs");
|
||||
if (param >= 0)
|
||||
{
|
||||
mca_base_param_lookup_int (param, &mca_fcoll_individual_constant_cbs);
|
||||
}
|
||||
param = mca_base_param_find ("fcoll", NULL, "individual_cycle_buffer_size");
|
||||
if (param >= 0)
|
||||
{
|
||||
mca_base_param_lookup_int (param, &mca_fcoll_individual_cycle_buffer_size);
|
||||
}
|
||||
|
||||
mca_base_param_reg_int (&mca_fcoll_individual_component.fcollm_version,
|
||||
"priority",
|
||||
"Priority of the individual fcoll component",
|
||||
false, false, mca_fcoll_individual_priority,
|
||||
&mca_fcoll_individual_priority);
|
||||
mca_base_param_reg_int (&mca_fcoll_individual_component.fcollm_version,
|
||||
"constant_cbs",
|
||||
"wether we are using constant or scaling cycle buffer size in the individual fcoll component",
|
||||
false, false, mca_fcoll_individual_constant_cbs,
|
||||
&mca_fcoll_individual_constant_cbs);
|
||||
mca_base_param_reg_int (&mca_fcoll_individual_component.fcollm_version,
|
||||
"cycle_buffer_size",
|
||||
"Cycle Buffer Size of the individual fcoll component",
|
||||
false, false, mca_fcoll_individual_cycle_buffer_size,
|
||||
&mca_fcoll_individual_cycle_buffer_size);
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
211
ompi/mca/fcoll/individual/fcoll_individual_file_read_all.c
Обычный файл
211
ompi/mca/fcoll/individual/fcoll_individual_file_read_all.c
Обычный файл
@ -0,0 +1,211 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_individual.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
#include "ompi/mca/io/ompio/io_ompio.h"
|
||||
#include "ompi/mca/io/io.h"
|
||||
#include "math.h"
|
||||
#include <unistd.h>
|
||||
|
||||
#define TIME_BREAKDOWN 0
|
||||
|
||||
int
|
||||
mca_fcoll_individual_file_read_all (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype,
|
||||
ompi_status_public_t *status)
|
||||
{
|
||||
size_t total_bytes_read = 0; /* total bytes that have been read*/
|
||||
size_t bytes_to_read_in_cycle = 0; /* left to be read in a cycle*/
|
||||
size_t bytes_per_cycle = 0; /* total read in each cycle by each process*/
|
||||
int index = 0;
|
||||
int cycles = 0;
|
||||
|
||||
uint32_t iov_count = 0;
|
||||
struct iovec *decoded_iov = NULL;
|
||||
|
||||
size_t max_data = 0;
|
||||
int i = 0; /* index into the decoded iovec of the buffer */
|
||||
int j = 0; /* index into the file vie iovec */
|
||||
int k = 0; /* index into the io_array */
|
||||
size_t sum_previous_counts = 0;
|
||||
size_t sum_previous_length = 0;
|
||||
#if TIME_BREAKDOWN
|
||||
double start = 0, end=0, start_all=0, end_all=0, total_io=0;
|
||||
#endif
|
||||
|
||||
#if TIME_BREAKDOWN
|
||||
start_all = MPI_Wtime();
|
||||
#endif
|
||||
|
||||
ompi_io_ompio_decode_datatype (fh,
|
||||
datatype,
|
||||
count,
|
||||
buf,
|
||||
&max_data,
|
||||
&decoded_iov,
|
||||
&iov_count);
|
||||
|
||||
if (mca_fcoll_individual_constant_cbs) {
|
||||
bytes_per_cycle = mca_fcoll_individual_cycle_buffer_size/fh->f_size;
|
||||
}
|
||||
else {
|
||||
bytes_per_cycle = mca_fcoll_individual_cycle_buffer_size;
|
||||
}
|
||||
|
||||
cycles = ceil((float)max_data/bytes_per_cycle);
|
||||
|
||||
#if 0
|
||||
printf ("Bytes per Cycle: %d Cycles: %d\n",bytes_per_cycle, cycles);
|
||||
#endif
|
||||
|
||||
sum_previous_length = fh->f_position_in_file_view;
|
||||
j = fh->f_index_in_file_view;
|
||||
|
||||
for (index = 0; index < cycles; index++) {
|
||||
OPAL_PTRDIFF_TYPE disp;
|
||||
int block = 1;
|
||||
|
||||
k = 0;
|
||||
if ((index == cycles-1) && (max_data % bytes_per_cycle)) {
|
||||
bytes_to_read_in_cycle = max_data % bytes_per_cycle;
|
||||
}
|
||||
else {
|
||||
bytes_to_read_in_cycle = bytes_per_cycle;
|
||||
}
|
||||
|
||||
/*
|
||||
ompi_io_ompio_create_list (fh->f_decoded_iov, fh->f_iov_count,
|
||||
decoded_iov, iov_count,
|
||||
&total_bytes_read, &bytes_to_read_in_cycle,
|
||||
&sum_previous_counts, &sum_previous_length,
|
||||
&decoded_iov_index, &fview_iov_index,
|
||||
&fh->f_io_array, &fh->f_num_of_io_entries);
|
||||
*/
|
||||
|
||||
fh->f_io_array = (mca_io_ompio_io_array_t *)malloc
|
||||
(OMPIO_IOVEC_INITIAL_SIZE * sizeof (mca_io_ompio_io_array_t));
|
||||
if (NULL == fh->f_io_array) {
|
||||
opal_output(1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
while (bytes_to_read_in_cycle) {
|
||||
/* reallocate if needed */
|
||||
if (OMPIO_IOVEC_INITIAL_SIZE*block <= k) {
|
||||
block ++;
|
||||
fh->f_io_array = (mca_io_ompio_io_array_t *)realloc
|
||||
(fh->f_io_array, OMPIO_IOVEC_INITIAL_SIZE *
|
||||
block * sizeof (mca_io_ompio_io_array_t));
|
||||
if (NULL == fh->f_io_array) {
|
||||
opal_output(1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
|
||||
if (decoded_iov[i].iov_len -
|
||||
(total_bytes_read - sum_previous_counts) <= 0) {
|
||||
sum_previous_counts += decoded_iov[i].iov_len;
|
||||
i = i + 1;
|
||||
}
|
||||
|
||||
disp = (OPAL_PTRDIFF_TYPE)decoded_iov[i].iov_base +
|
||||
(total_bytes_read - sum_previous_counts);
|
||||
fh->f_io_array[k].memory_address = (IOVBASE_TYPE *)disp;
|
||||
|
||||
if (decoded_iov[i].iov_len -
|
||||
(total_bytes_read - sum_previous_counts) >=
|
||||
bytes_to_read_in_cycle) {
|
||||
fh->f_io_array[k].length = bytes_to_read_in_cycle;
|
||||
}
|
||||
else {
|
||||
fh->f_io_array[k].length = decoded_iov[i].iov_len -
|
||||
(total_bytes_read - sum_previous_counts);
|
||||
}
|
||||
|
||||
if (fh->f_decoded_iov[j].iov_len -
|
||||
(fh->f_total_bytes - sum_previous_length) <= 0) {
|
||||
sum_previous_length += fh->f_decoded_iov[j].iov_len;
|
||||
j = j + 1;
|
||||
if (j == (int)fh->f_iov_count) {
|
||||
j = 0;
|
||||
sum_previous_length = 0;
|
||||
fh->f_offset += fh->f_view_extent;
|
||||
fh->f_position_in_file_view = sum_previous_length;
|
||||
fh->f_index_in_file_view = j;
|
||||
fh->f_total_bytes = 0;
|
||||
}
|
||||
}
|
||||
|
||||
disp = (OPAL_PTRDIFF_TYPE)fh->f_decoded_iov[j].iov_base +
|
||||
(fh->f_total_bytes - sum_previous_length);
|
||||
fh->f_io_array[k].offset = (IOVBASE_TYPE *)(disp + fh->f_offset);
|
||||
|
||||
if (fh->f_decoded_iov[j].iov_len -
|
||||
(fh->f_total_bytes - sum_previous_length) < fh->f_io_array[k].length) {
|
||||
fh->f_io_array[k].length = fh->f_decoded_iov[j].iov_len -
|
||||
(fh->f_total_bytes - sum_previous_length);
|
||||
}
|
||||
total_bytes_read += fh->f_io_array[k].length;
|
||||
fh->f_total_bytes += fh->f_io_array[k].length;
|
||||
bytes_to_read_in_cycle -= fh->f_io_array[k].length;
|
||||
k = k + 1;
|
||||
}
|
||||
fh->f_position_in_file_view = sum_previous_length;
|
||||
fh->f_index_in_file_view = j;
|
||||
fh->f_num_of_io_entries = k;
|
||||
|
||||
#if 0
|
||||
if (fh->f_rank == 0) {
|
||||
int i;
|
||||
printf("*************************** %d\n", fh->f_num_of_io_entries);
|
||||
|
||||
for (i=0 ; i<fh->f_num_of_io_entries ; i++) {
|
||||
printf(" ADDRESS: %p OFFSET: %p LENGTH: %d\n",
|
||||
fh->f_io_array[i].memory_address,
|
||||
fh->f_io_array[i].offset,
|
||||
fh->f_io_array[i].length);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
if (fh->f_num_of_io_entries) {
|
||||
fh->f_fbtl->fbtl_preadv (fh, NULL);
|
||||
}
|
||||
|
||||
fh->f_num_of_io_entries = 0;
|
||||
if (NULL != fh->f_io_array) {
|
||||
free (fh->f_io_array);
|
||||
fh->f_io_array = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
if (NULL != decoded_iov) {
|
||||
free (decoded_iov);
|
||||
decoded_iov = NULL;
|
||||
}
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
36
ompi/mca/fcoll/individual/fcoll_individual_file_read_all_begin.c
Обычный файл
36
ompi/mca/fcoll/individual/fcoll_individual_file_read_all_begin.c
Обычный файл
@ -0,0 +1,36 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_individual.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
|
||||
|
||||
int
|
||||
mca_fcoll_individual_file_read_all_begin (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype)
|
||||
{
|
||||
printf ("INDIVIDUAL READ ALL BEGIN\n");
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
35
ompi/mca/fcoll/individual/fcoll_individual_file_read_all_end.c
Обычный файл
35
ompi/mca/fcoll/individual/fcoll_individual_file_read_all_end.c
Обычный файл
@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_individual.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
|
||||
|
||||
int
|
||||
mca_fcoll_individual_file_read_all_end (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
ompi_status_public_t *status)
|
||||
{
|
||||
printf ("INDIVIDUAL READ ALL END\n");
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
203
ompi/mca/fcoll/individual/fcoll_individual_file_write_all.c
Обычный файл
203
ompi/mca/fcoll/individual/fcoll_individual_file_write_all.c
Обычный файл
@ -0,0 +1,203 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_individual.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
#include "ompi/mca/io/ompio/io_ompio.h"
|
||||
#include "ompi/mca/io/io.h"
|
||||
#include "math.h"
|
||||
#include <unistd.h>
|
||||
|
||||
#define TIME_BREAKDOWN 0
|
||||
|
||||
int mca_fcoll_individual_file_write_all (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype,
|
||||
ompi_status_public_t *status)
|
||||
{
|
||||
size_t total_bytes_written = 0; /* total bytes that have been written*/
|
||||
size_t bytes_to_write_in_cycle = 0; /* left to be written in a cycle*/
|
||||
size_t bytes_per_cycle = 0; /* total written in each cycle by each process*/
|
||||
int index = 0;
|
||||
int cycles = 0;
|
||||
|
||||
uint32_t iov_count = 0;
|
||||
struct iovec *decoded_iov = NULL;
|
||||
|
||||
size_t max_data = 0;
|
||||
int i = 0; /* index into the decoded iovec of the buffer */
|
||||
int j = 0; /* index into the file vie iovec */
|
||||
int k = 0; /* index into the io_array */
|
||||
size_t sum_previous_counts = 0;
|
||||
size_t sum_previous_length = 0;
|
||||
#if TIME_BREAKDOWN
|
||||
double start = 0, end=0, start_all=0, end_all=0, total_io=0;
|
||||
#endif
|
||||
|
||||
#if TIME_BREAKDOWN
|
||||
start_all = MPI_Wtime();
|
||||
#endif
|
||||
|
||||
ompi_io_ompio_decode_datatype (fh,
|
||||
datatype,
|
||||
count,
|
||||
buf,
|
||||
&max_data,
|
||||
&decoded_iov,
|
||||
&iov_count);
|
||||
|
||||
if (mca_fcoll_individual_constant_cbs) {
|
||||
bytes_per_cycle = mca_fcoll_individual_cycle_buffer_size/fh->f_size;
|
||||
}
|
||||
else {
|
||||
bytes_per_cycle = mca_fcoll_individual_cycle_buffer_size;
|
||||
}
|
||||
|
||||
cycles = ceil((float)max_data/bytes_per_cycle);
|
||||
|
||||
#if 0
|
||||
printf ("MAX DATA: %d\n", max_data);
|
||||
printf ("Bytes per Cycle: %d Cycles: %d\n",bytes_per_cycle, cycles);
|
||||
#endif
|
||||
|
||||
sum_previous_length = fh->f_position_in_file_view;
|
||||
j = fh->f_index_in_file_view;
|
||||
|
||||
for (index = 0; index < cycles; index++) {
|
||||
OPAL_PTRDIFF_TYPE disp;
|
||||
int block = 1;
|
||||
|
||||
k = 0;
|
||||
if ((index == cycles-1) && (max_data % bytes_per_cycle)) {
|
||||
bytes_to_write_in_cycle = max_data % bytes_per_cycle;
|
||||
}
|
||||
else {
|
||||
bytes_to_write_in_cycle = bytes_per_cycle;
|
||||
}
|
||||
|
||||
fh->f_io_array = (mca_io_ompio_io_array_t *)malloc
|
||||
(OMPIO_IOVEC_INITIAL_SIZE * sizeof (mca_io_ompio_io_array_t));
|
||||
if (NULL == fh->f_io_array) {
|
||||
opal_output(1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
while (bytes_to_write_in_cycle) {
|
||||
/* reallocate if needed */
|
||||
if (OMPIO_IOVEC_INITIAL_SIZE*block <= k) {
|
||||
block ++;
|
||||
fh->f_io_array = (mca_io_ompio_io_array_t *)realloc
|
||||
(fh->f_io_array, OMPIO_IOVEC_INITIAL_SIZE *
|
||||
block * sizeof (mca_io_ompio_io_array_t));
|
||||
if (NULL == fh->f_io_array) {
|
||||
opal_output(1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
|
||||
if (decoded_iov[i].iov_len -
|
||||
(total_bytes_written - sum_previous_counts) <= 0) {
|
||||
sum_previous_counts += decoded_iov[i].iov_len;
|
||||
i = i + 1;
|
||||
}
|
||||
|
||||
disp = (OPAL_PTRDIFF_TYPE)decoded_iov[i].iov_base +
|
||||
(total_bytes_written - sum_previous_counts);
|
||||
fh->f_io_array[k].memory_address = (IOVBASE_TYPE *)disp;
|
||||
|
||||
if (decoded_iov[i].iov_len -
|
||||
(total_bytes_written - sum_previous_counts) >=
|
||||
bytes_to_write_in_cycle) {
|
||||
fh->f_io_array[k].length = bytes_to_write_in_cycle;
|
||||
}
|
||||
else {
|
||||
fh->f_io_array[k].length = decoded_iov[i].iov_len -
|
||||
(total_bytes_written - sum_previous_counts);
|
||||
}
|
||||
|
||||
if (fh->f_decoded_iov[j].iov_len -
|
||||
(fh->f_total_bytes - sum_previous_length) <= 0) {
|
||||
sum_previous_length += fh->f_decoded_iov[j].iov_len;
|
||||
j = j + 1;
|
||||
if (j == (int)fh->f_iov_count) {
|
||||
j = 0;
|
||||
sum_previous_length = 0;
|
||||
fh->f_offset += fh->f_view_extent;
|
||||
fh->f_position_in_file_view = sum_previous_length;
|
||||
fh->f_index_in_file_view = j;
|
||||
fh->f_total_bytes = 0;
|
||||
}
|
||||
}
|
||||
|
||||
disp = (OPAL_PTRDIFF_TYPE)fh->f_decoded_iov[j].iov_base +
|
||||
(fh->f_total_bytes - sum_previous_length);
|
||||
fh->f_io_array[k].offset = (IOVBASE_TYPE *)(disp + fh->f_offset);
|
||||
|
||||
if (fh->f_decoded_iov[j].iov_len -
|
||||
(fh->f_total_bytes - sum_previous_length)
|
||||
< fh->f_io_array[k].length) {
|
||||
fh->f_io_array[k].length = fh->f_decoded_iov[j].iov_len -
|
||||
(fh->f_total_bytes - sum_previous_length);
|
||||
}
|
||||
total_bytes_written += fh->f_io_array[k].length;
|
||||
fh->f_total_bytes += fh->f_io_array[k].length;
|
||||
bytes_to_write_in_cycle -= fh->f_io_array[k].length;
|
||||
k = k + 1;
|
||||
}
|
||||
fh->f_position_in_file_view = sum_previous_length;
|
||||
fh->f_index_in_file_view = j;
|
||||
fh->f_num_of_io_entries = k;
|
||||
|
||||
#if 0
|
||||
if (fh->f_rank == 0) {
|
||||
int d;
|
||||
printf("*************************** %d\n", fh->f_num_of_io_entries);
|
||||
|
||||
for (d=0 ; d<fh->f_num_of_io_entries ; d++) {
|
||||
printf(" ADDRESS: %p OFFSET: %p LENGTH: %d\n",
|
||||
fh->f_io_array[d].memory_address,
|
||||
fh->f_io_array[d].offset,
|
||||
fh->f_io_array[d].length);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
if (fh->f_num_of_io_entries) {
|
||||
fh->f_fbtl->fbtl_pwritev (fh, NULL);
|
||||
}
|
||||
|
||||
fh->f_num_of_io_entries = 0;
|
||||
if (NULL != fh->f_io_array) {
|
||||
free (fh->f_io_array);
|
||||
fh->f_io_array = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
if (NULL != decoded_iov) {
|
||||
free (decoded_iov);
|
||||
decoded_iov = NULL;
|
||||
}
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
@ -0,0 +1,36 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_individual.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
|
||||
|
||||
int
|
||||
mca_fcoll_individual_file_write_all_begin (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype)
|
||||
{
|
||||
printf ("INDIVIDUAL WRITE ALL BEGIN\n");
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
35
ompi/mca/fcoll/individual/fcoll_individual_file_write_all_end.c
Обычный файл
35
ompi/mca/fcoll/individual/fcoll_individual_file_write_all_end.c
Обычный файл
@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_individual.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
|
||||
|
||||
int
|
||||
mca_fcoll_individual_file_write_all_end (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
ompi_status_public_t *status)
|
||||
{
|
||||
printf ("INDIVIDUAL WRITE ALL END\n");
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
91
ompi/mca/fcoll/individual/fcoll_individual_module.c
Обычный файл
91
ompi/mca/fcoll/individual/fcoll_individual_module.c
Обычный файл
@ -0,0 +1,91 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2006 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_individual.h"
|
||||
|
||||
#include <stdio.h>
|
||||
|
||||
#include "mpi.h"
|
||||
#include "opal/mca/base/mca_base_param.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
#include "ompi/mca/fcoll/base/base.h"
|
||||
|
||||
|
||||
/*
|
||||
* *******************************************************************
|
||||
* ************************ actions structure ************************
|
||||
* *******************************************************************
|
||||
*/
|
||||
static mca_fcoll_base_module_1_0_0_t individual = {
|
||||
mca_fcoll_individual_module_init,
|
||||
mca_fcoll_individual_module_finalize,
|
||||
mca_fcoll_individual_file_read_all,
|
||||
mca_fcoll_individual_file_read_all_begin,
|
||||
mca_fcoll_individual_file_read_all_end,
|
||||
mca_fcoll_individual_file_write_all,
|
||||
mca_fcoll_individual_file_write_all_begin,
|
||||
mca_fcoll_individual_file_write_all_end
|
||||
};
|
||||
|
||||
int
|
||||
mca_fcoll_individual_component_init_query(bool enable_progress_threads,
|
||||
bool enable_mpi_threads)
|
||||
{
|
||||
/* Nothing to do */
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
mca_fcoll_base_module_1_0_0_t *
|
||||
mca_fcoll_individual_component_file_query (mca_io_ompio_file_t *fh, int *priority)
|
||||
{
|
||||
*priority = mca_fcoll_individual_priority;
|
||||
if (0 >= mca_fcoll_individual_priority) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (mca_fcoll_base_query_table (fh, "individual")) {
|
||||
if (*priority < 50) {
|
||||
*priority = 50;
|
||||
}
|
||||
}
|
||||
|
||||
return &individual;
|
||||
}
|
||||
|
||||
int mca_fcoll_individual_component_file_unquery (mca_io_ompio_file_t *file)
|
||||
{
|
||||
/* This function might be needed for some purposes later. for now it
|
||||
* does not have anything to do since there are no steps which need
|
||||
* to be undone if this module is not selected */
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
int mca_fcoll_individual_module_init (mca_io_ompio_file_t *file)
|
||||
{
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
int mca_fcoll_individual_module_finalize (mca_io_ompio_file_t *file)
|
||||
{
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
54
ompi/mca/fcoll/static/Makefile.am
Обычный файл
54
ompi/mca/fcoll/static/Makefile.am
Обычный файл
@ -0,0 +1,54 @@
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
sources = \
|
||||
fcoll_static.h \
|
||||
fcoll_static_module.c \
|
||||
fcoll_static_component.c \
|
||||
fcoll_static_file_read_all.c \
|
||||
fcoll_static_file_read_all_begin.c \
|
||||
fcoll_static_file_read_all_end.c \
|
||||
fcoll_static_file_write_all.c \
|
||||
fcoll_static_file_write_all_begin.c \
|
||||
fcoll_static_file_write_all_end.c
|
||||
|
||||
# Make the output library in this directory, and name it either
|
||||
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
|
||||
# (for static builds).
|
||||
|
||||
if MCA_BUILD_ompi_fcoll_static_DSO
|
||||
component_noinst =
|
||||
component_install = mca_fcoll_static.la
|
||||
else
|
||||
component_noinst = libmca_fcoll_static.la
|
||||
component_install =
|
||||
endif
|
||||
|
||||
mcacomponentdir = $(libdir)/openmpi
|
||||
mcacomponent_LTLIBRARIES = $(component_install)
|
||||
mca_fcoll_static_la_SOURCES = $(sources)
|
||||
mca_fcoll_static_la_LDFLAGS = -module -avoid-version
|
||||
mca_fcoll_static_la_LIBADD = \
|
||||
$(top_ompi_builddir)/ompi/libmpi.la \
|
||||
$(top_ompi_builddir)/orte/libopen-rte.la \
|
||||
$(top_ompi_builddir)/opal/libopen-pal.la
|
||||
|
||||
noinst_LTLIBRARIES = $(component_noinst)
|
||||
libmca_fcoll_static_la_SOURCES =$(sources)
|
||||
libmca_fcoll_static_la_LDFLAGS = -module -avoid-version
|
||||
85
ompi/mca/fcoll/static/fcoll_static.h
Обычный файл
85
ompi/mca/fcoll/static/fcoll_static.h
Обычный файл
@ -0,0 +1,85 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2006 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#ifndef MCA_FCOLL_STATIC_EXPORT_H
|
||||
#define MCA_FCOLL_STATIC_EXPORT_H
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "opal/mca/mca.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
#include "ompi/mca/fcoll/base/base.h"
|
||||
#include "ompi/mca/io/ompio/io_ompio.h"
|
||||
|
||||
BEGIN_C_DECLS
|
||||
|
||||
/* Globally exported variables */
|
||||
|
||||
extern int mca_fcoll_static_priority;
|
||||
extern int mca_fcoll_static_num_io_procs;
|
||||
extern int mca_fcoll_static_constant_cbs;
|
||||
extern int mca_fcoll_static_cycle_buffer_size;
|
||||
|
||||
OMPI_MODULE_DECLSPEC extern mca_fcoll_base_component_2_0_0_t mca_fcoll_static_component;
|
||||
/* API functions */
|
||||
|
||||
int mca_fcoll_static_component_init_query(bool enable_progress_threads,
|
||||
bool enable_mpi_threads);
|
||||
struct mca_fcoll_base_module_1_0_0_t *
|
||||
mca_fcoll_static_component_file_query (mca_io_ompio_file_t *fh, int *priority);
|
||||
|
||||
int mca_fcoll_static_component_file_unquery (mca_io_ompio_file_t *file);
|
||||
|
||||
int mca_fcoll_static_module_init (mca_io_ompio_file_t *file);
|
||||
int mca_fcoll_static_module_finalize (mca_io_ompio_file_t *file);
|
||||
|
||||
int mca_fcoll_static_file_read_all (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype,
|
||||
ompi_status_public_t * status);
|
||||
|
||||
int mca_fcoll_static_file_read_all_begin (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype);
|
||||
|
||||
int mca_fcoll_static_file_read_all_end (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
ompi_status_public_t * status);
|
||||
|
||||
int mca_fcoll_static_file_write_all (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype,
|
||||
ompi_status_public_t * status);
|
||||
|
||||
int mca_fcoll_static_file_write_all_begin (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype);
|
||||
|
||||
int mca_fcoll_static_file_write_all_end (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
ompi_status_public_t * status);
|
||||
|
||||
END_C_DECLS
|
||||
|
||||
#endif /* MCA_FCOLL_STATIC_EXPORT_H */
|
||||
132
ompi/mca/fcoll/static/fcoll_static_component.c
Обычный файл
132
ompi/mca/fcoll/static/fcoll_static_component.c
Обычный файл
@ -0,0 +1,132 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*
|
||||
* These symbols are in a file by themselves to provide nice linker
|
||||
* semantics. Since linkers generally pull in symbols by object
|
||||
* files, keeping these symbols as the only symbols in this file
|
||||
* prevents utility programs such as "ompi_info" from having to import
|
||||
* entire components just to query their version and parameters.
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_static.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
#include "fcoll_static.h"
|
||||
|
||||
/*
|
||||
* Public string showing the fcoll ompi_static component version number
|
||||
*/
|
||||
const char *mca_fcoll_static_component_version_string =
|
||||
"Open MPI static collective MCA component version " OMPI_VERSION;
|
||||
|
||||
/*
|
||||
* Global variables
|
||||
*/
|
||||
int mca_fcoll_static_priority = 10;
|
||||
int mca_fcoll_static_num_io_procs = -1;
|
||||
int mca_fcoll_static_constant_cbs = 1;
|
||||
int mca_fcoll_static_cycle_buffer_size = OMPIO_PREALLOC_MAX_BUF_SIZE;
|
||||
|
||||
/*
|
||||
* Local function
|
||||
*/
|
||||
static int static_register(void);
|
||||
|
||||
/*
|
||||
* Instantiate the public struct with all of our public information
|
||||
* and pointers to our public functions in it
|
||||
*/
|
||||
mca_fcoll_base_component_2_0_0_t mca_fcoll_static_component = {
|
||||
|
||||
/* First, the mca_component_t struct containing meta information
|
||||
* about the component itself */
|
||||
|
||||
{
|
||||
MCA_FCOLL_BASE_VERSION_2_0_0,
|
||||
|
||||
/* Component name and version */
|
||||
"static",
|
||||
OMPI_MAJOR_VERSION,
|
||||
OMPI_MINOR_VERSION,
|
||||
OMPI_RELEASE_VERSION,
|
||||
static_register,
|
||||
NULL
|
||||
},
|
||||
{
|
||||
/* The component is checkpoint ready */
|
||||
MCA_BASE_METADATA_PARAM_CHECKPOINT
|
||||
},
|
||||
|
||||
mca_fcoll_static_component_init_query,
|
||||
mca_fcoll_static_component_file_query,
|
||||
mca_fcoll_static_component_file_unquery
|
||||
};
|
||||
|
||||
|
||||
static int
|
||||
static_register(void)
|
||||
{
|
||||
int param;
|
||||
|
||||
param = mca_base_param_find ("fcoll", NULL, "static_priority");
|
||||
if (param >= 0)
|
||||
{
|
||||
mca_base_param_lookup_int (param, &mca_fcoll_static_priority);
|
||||
}
|
||||
param = mca_base_param_find ("fcoll", NULL, "static_num_io_procs");
|
||||
if (param >= 0)
|
||||
{
|
||||
mca_base_param_lookup_int (param, &mca_fcoll_static_num_io_procs);
|
||||
}
|
||||
param = mca_base_param_find ("fcoll", NULL, "static_constant_cbs");
|
||||
if (param >= 0)
|
||||
{
|
||||
mca_base_param_lookup_int (param, &mca_fcoll_static_constant_cbs);
|
||||
}
|
||||
param = mca_base_param_find ("fcoll", NULL, "static_cycle_buffer_size");
|
||||
if (param >= 0)
|
||||
{
|
||||
mca_base_param_lookup_int (param, &mca_fcoll_static_cycle_buffer_size);
|
||||
}
|
||||
|
||||
mca_base_param_reg_int (&mca_fcoll_static_component.fcollm_version,
|
||||
"priority",
|
||||
"Priority of the static fcoll component",
|
||||
false, false, mca_fcoll_static_priority,
|
||||
&mca_fcoll_static_priority);
|
||||
mca_base_param_reg_int (&mca_fcoll_static_component.fcollm_version,
|
||||
"num_io_procs",
|
||||
"Number of writers in the static fcoll component",
|
||||
false, false, mca_fcoll_static_num_io_procs,
|
||||
&mca_fcoll_static_num_io_procs);
|
||||
mca_base_param_reg_int (&mca_fcoll_static_component.fcollm_version,
|
||||
"constant_cbs",
|
||||
"wether we are using constant or scaling cycle buffer size in the static fcoll component",
|
||||
false, false, mca_fcoll_static_constant_cbs,
|
||||
&mca_fcoll_static_constant_cbs);
|
||||
mca_base_param_reg_int (&mca_fcoll_static_component.fcollm_version,
|
||||
"cycle_buffer_size",
|
||||
"Cycle Buffer Size of the static fcoll component",
|
||||
false, false, mca_fcoll_static_cycle_buffer_size,
|
||||
&mca_fcoll_static_cycle_buffer_size);
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
469
ompi/mca/fcoll/static/fcoll_static_file_read_all.c
Обычный файл
469
ompi/mca/fcoll/static/fcoll_static_file_read_all.c
Обычный файл
@ -0,0 +1,469 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_static.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
#include "ompi/mca/io/ompio/io_ompio.h"
|
||||
#include "ompi/mca/io/io.h"
|
||||
#include "math.h"
|
||||
#include "ompi/mca/pml/pml.h"
|
||||
#include <unistd.h>
|
||||
|
||||
#define TIME_BREAKDOWN 0
|
||||
|
||||
int
|
||||
mca_fcoll_static_file_read_all (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype,
|
||||
ompi_status_public_t *status)
|
||||
{
|
||||
size_t position = 0;
|
||||
MPI_Aint bytes_to_read_in_cycle = 0; /* left to be read in a cycle*/
|
||||
size_t bytes_per_cycle = 0; /* total read in each cycle by each process*/
|
||||
|
||||
int index = 0;
|
||||
int cycles = 0, local_cycles;
|
||||
int i=0;
|
||||
int ret;
|
||||
|
||||
/* iovec structure and count of the buffer passed in */
|
||||
uint32_t iov_count = 0;
|
||||
struct iovec *decoded_iov = NULL;
|
||||
uint32_t iov_index = 0;
|
||||
size_t current_position = 0;
|
||||
char *receive_buf = NULL;
|
||||
|
||||
/* global iovec at the readers that contain the iovecs created from
|
||||
file_set_view */
|
||||
uint32_t global_iov_count = 0;
|
||||
struct iovec *global_iov = NULL;
|
||||
|
||||
char *global_buf = NULL;
|
||||
MPI_Aint global_count = 0;
|
||||
|
||||
/* array that contains the sorted indices of the global_iov */
|
||||
int *sorted = NULL;
|
||||
|
||||
int *displs = NULL;
|
||||
|
||||
size_t max_data = 0;
|
||||
int *iovec_count_per_process = NULL;
|
||||
int *bytes_per_process = NULL;
|
||||
|
||||
#if TIME_BREAKDOWN
|
||||
double start=0, end=0, start_all=0, end_all=0;
|
||||
double total_gather=0 , total_io=0;
|
||||
#endif
|
||||
|
||||
if (opal_datatype_is_contiguous_memory_layout(&datatype->super,1)) {
|
||||
fh->f_flags |= OMPIO_CONTIGUOUS_MEMORY;
|
||||
}
|
||||
|
||||
/* In case the data is not contigous in memory, decode it into an iovec */
|
||||
if (! (fh->f_flags & OMPIO_CONTIGUOUS_MEMORY)) {
|
||||
ompi_io_ompio_decode_datatype (fh,
|
||||
datatype,
|
||||
count,
|
||||
buf,
|
||||
&max_data,
|
||||
&decoded_iov,
|
||||
&iov_count);
|
||||
}
|
||||
else {
|
||||
max_data = count * datatype->super.size;
|
||||
}
|
||||
|
||||
if (! (fh->f_flags & OMPIO_AGGREGATOR_IS_SET)) {
|
||||
ompi_io_ompio_set_aggregator_props (fh,
|
||||
mca_fcoll_static_num_io_procs,
|
||||
max_data);
|
||||
}
|
||||
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
bytes_per_process = (int *)malloc (sizeof(int)*fh->f_procs_per_group);
|
||||
if (NULL == bytes_per_process) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
iovec_count_per_process = (int *)malloc (sizeof(int)*fh->f_procs_per_group);
|
||||
if (NULL == iovec_count_per_process) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
displs = (int*)malloc (fh->f_procs_per_group*sizeof(int));
|
||||
if (NULL == displs) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Calculate how many bytes are read in each cycle
|
||||
*/
|
||||
if (mca_fcoll_static_constant_cbs) {
|
||||
bytes_per_cycle =
|
||||
mca_fcoll_static_cycle_buffer_size/fh->f_procs_per_group;
|
||||
}
|
||||
else {
|
||||
bytes_per_cycle = mca_fcoll_static_cycle_buffer_size;
|
||||
}
|
||||
|
||||
/* TODO : number of cycles has to be the same for all processes in a group,
|
||||
so need to Allreduce the cycle within a group.
|
||||
This works now if all processes are reading the same amount of data */
|
||||
cycles = ceil((double)max_data/bytes_per_cycle);
|
||||
local_cycles = cycles;
|
||||
ret = fh->f_comm->c_coll.coll_allreduce (&local_cycles,
|
||||
&cycles,
|
||||
1,
|
||||
MPI_INT,
|
||||
MPI_MAX,
|
||||
fh->f_comm,
|
||||
fh->f_comm->c_coll.coll_allreduce_module);
|
||||
#if 0
|
||||
printf ("Bytes per Process: %d Cycles: %d Procs_per_group %d\n",
|
||||
bytes_per_cycle, cycles, fh->f_procs_per_group);
|
||||
#endif
|
||||
|
||||
for (index = 0; index < cycles; index++) {
|
||||
struct iovec *iov = NULL;
|
||||
int iov_size = 0;
|
||||
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
/*
|
||||
printf ("********* READER %d PROCS %d CYCLE %d of %d************\n",
|
||||
fh->f_rank,
|
||||
fh->f_procs_per_group,
|
||||
index,
|
||||
cycles);
|
||||
*/
|
||||
memset(displs, 0x0, fh->f_procs_per_group*sizeof(int));
|
||||
memset(bytes_per_process, 0x0, fh->f_procs_per_group*sizeof(int));
|
||||
memset(iovec_count_per_process, 0x0, fh->f_procs_per_group*sizeof(int));
|
||||
}
|
||||
if (local_cycles > index) {
|
||||
if ((index == local_cycles-1) && (max_data % bytes_per_cycle)) {
|
||||
bytes_to_read_in_cycle = max_data % bytes_per_cycle;
|
||||
}
|
||||
else if (max_data <= bytes_per_cycle) {
|
||||
bytes_to_read_in_cycle = max_data;
|
||||
}
|
||||
else {
|
||||
bytes_to_read_in_cycle = bytes_per_cycle;
|
||||
}
|
||||
}
|
||||
else {
|
||||
bytes_to_read_in_cycle = 0;
|
||||
}
|
||||
|
||||
/**********************************************************
|
||||
Gather from each process iovecs to where to write the data
|
||||
*********************************************************/
|
||||
if (bytes_to_read_in_cycle) {
|
||||
ompi_io_ompio_generate_current_file_view (fh,
|
||||
bytes_to_read_in_cycle,
|
||||
&iov,
|
||||
&iov_size);
|
||||
}
|
||||
/*
|
||||
for (i=0 ; i<k ; i++) {
|
||||
printf("%d: OFFSET: %p LENGTH: %d\n",
|
||||
fh->f_rank,
|
||||
iov[i].iov_base,
|
||||
iov[i].iov_len);
|
||||
}
|
||||
*/
|
||||
ompi_io_ompio_gather_array (&iov_size,
|
||||
1,
|
||||
MPI_INT,
|
||||
iovec_count_per_process,
|
||||
1,
|
||||
MPI_INT,
|
||||
fh->f_aggregator_index,
|
||||
fh->f_procs_in_group,
|
||||
fh->f_procs_per_group,
|
||||
fh->f_comm);
|
||||
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
displs[0] = 0;
|
||||
global_iov_count = iovec_count_per_process[0];
|
||||
for (i=1 ; i<fh->f_procs_per_group ; i++) {
|
||||
global_iov_count += iovec_count_per_process[i];
|
||||
displs[i] = displs[i-1] + iovec_count_per_process[i-1];
|
||||
}
|
||||
}
|
||||
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
global_iov = (struct iovec*)malloc (global_iov_count *
|
||||
sizeof(struct iovec));
|
||||
if (NULL == global_iov) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
|
||||
if (fh->f_flags & OMPIO_UNIFORM_FVIEW) {
|
||||
ompi_io_ompio_gather_array (iov,
|
||||
iov_size,
|
||||
fh->f_iov_type,
|
||||
global_iov,
|
||||
iov_size,
|
||||
fh->f_iov_type,
|
||||
fh->f_aggregator_index,
|
||||
fh->f_procs_in_group,
|
||||
fh->f_procs_per_group,
|
||||
fh->f_comm);
|
||||
}
|
||||
else {
|
||||
ompi_io_ompio_gatherv_array (iov,
|
||||
iov_size,
|
||||
fh->f_iov_type,
|
||||
global_iov,
|
||||
iovec_count_per_process,
|
||||
displs,
|
||||
fh->f_iov_type,
|
||||
fh->f_aggregator_index,
|
||||
fh->f_procs_in_group,
|
||||
fh->f_procs_per_group,
|
||||
fh->f_comm);
|
||||
}
|
||||
/* sort it */
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
sorted = (int *)malloc (global_iov_count * sizeof(int));
|
||||
if (NULL == sorted) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
ompi_io_ompio_sort_iovec (global_iov, global_iov_count, sorted);
|
||||
}
|
||||
|
||||
/**********************************************************
|
||||
**************** DONE GATHERING OF IOVECS ****************
|
||||
*********************************************************/
|
||||
|
||||
/* gather from each process how many bytes each will be recieving */
|
||||
ompi_io_ompio_gather_array (&bytes_to_read_in_cycle,
|
||||
1,
|
||||
MPI_INT,
|
||||
bytes_per_process,
|
||||
1,
|
||||
MPI_INT,
|
||||
fh->f_aggregator_index,
|
||||
fh->f_procs_in_group,
|
||||
fh->f_procs_per_group,
|
||||
fh->f_comm);
|
||||
|
||||
/* Calculate the displacement on where to put the data and allocate
|
||||
the recieve buffer (global_buf) */
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
displs[0] = 0;
|
||||
global_count = bytes_per_process[0];
|
||||
for (i=1 ; i<fh->f_procs_per_group ; i++) {
|
||||
global_count += bytes_per_process[i];
|
||||
displs[i] = displs[i-1] + bytes_per_process[i-1];
|
||||
}
|
||||
global_buf = malloc (global_count);
|
||||
if (NULL == global_buf) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
|
||||
/**********************************************************
|
||||
** Create the io array, sort it, and pass it to fbtl ****
|
||||
*********************************************************/
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
MPI_Aint temp = 0;
|
||||
int x = 0, k = 0;
|
||||
|
||||
fh->f_io_array = (mca_io_ompio_io_array_t *) malloc
|
||||
(global_iov_count * sizeof (mca_io_ompio_io_array_t));
|
||||
if (NULL == fh->f_io_array) {
|
||||
opal_output(1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
for (i=0 ; i<fh->f_procs_per_group ; i++) {
|
||||
for (x=0 ; x<iovec_count_per_process[i] ; x++) {
|
||||
fh->f_io_array[k].offset = global_iov[sorted[k]].iov_base;
|
||||
fh->f_io_array[k].length = global_iov[sorted[k]].iov_len;
|
||||
fh->f_io_array[k].memory_address = &global_buf[temp];
|
||||
temp += fh->f_io_array[k].length;
|
||||
k ++;
|
||||
}
|
||||
}
|
||||
fh->f_num_of_io_entries = k;
|
||||
|
||||
/*
|
||||
printf("%d *************************** %d\n",fh->f_rank, fh->f_num_of_io_entries);
|
||||
for (i=0 ; i<fh->f_num_of_io_entries ; i++)
|
||||
{
|
||||
printf(" ADDRESS: %p OFFSET: %p LENGTH: %d\n",
|
||||
fh->f_io_array[i].memory_address,
|
||||
fh->f_io_array[i].offset,
|
||||
fh->f_io_array[i].length);
|
||||
}
|
||||
|
||||
printf("******* SORTED ************ %d\n", fh->f_num_of_io_entries);
|
||||
for (i=0 ; i<fh->f_num_of_io_entries ; i++)
|
||||
{
|
||||
printf(" ADDRESS: %p OFFSET: %p LENGTH: %d\n",
|
||||
fh->f_io_array[sorted[i]].memory_address,
|
||||
fh->f_io_array[sorted[i]].offset,
|
||||
fh->f_io_array[sorted[i]].length);
|
||||
}
|
||||
*/
|
||||
if (fh->f_num_of_io_entries) {
|
||||
fh->f_fbtl->fbtl_preadv (fh, NULL);
|
||||
}
|
||||
/*
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank)
|
||||
for (i=0 ; i<global_count/4 ; i++)
|
||||
printf ("READ %d \n",
|
||||
((int *)global_buf)[i]);
|
||||
*/
|
||||
}
|
||||
/**********************************************************
|
||||
******************** DONE READING ************************
|
||||
*********************************************************/
|
||||
|
||||
|
||||
/**********************************************************
|
||||
********* Scatter the Data from the readers **************
|
||||
*********************************************************/
|
||||
if (fh->f_flags & OMPIO_CONTIGUOUS_MEMORY) {
|
||||
receive_buf = &((char*)buf)[position];
|
||||
}
|
||||
else if (bytes_to_read_in_cycle) {
|
||||
/* allocate a receive buffer and copy the data that needs
|
||||
to be received into it in case the data is non-contigous
|
||||
in memory */
|
||||
receive_buf = malloc (bytes_to_read_in_cycle);
|
||||
if (NULL == receive_buf) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
|
||||
ompi_io_ompio_scatterv_array (global_buf,
|
||||
bytes_per_process,
|
||||
displs,
|
||||
MPI_BYTE,
|
||||
receive_buf,
|
||||
bytes_to_read_in_cycle,
|
||||
MPI_BYTE,
|
||||
fh->f_aggregator_index,
|
||||
fh->f_procs_in_group,
|
||||
fh->f_procs_per_group,
|
||||
fh->f_comm);
|
||||
|
||||
position += bytes_to_read_in_cycle;
|
||||
|
||||
/* If data is not contigous in memory, copy the data from the
|
||||
receive buffer into the buffer passed in */
|
||||
if (!(fh->f_flags & OMPIO_CONTIGUOUS_MEMORY)) {
|
||||
OPAL_PTRDIFF_TYPE mem_address;
|
||||
size_t remaining = 0;
|
||||
size_t temp_position = 0;
|
||||
|
||||
remaining = bytes_to_read_in_cycle;
|
||||
|
||||
while (remaining && (iov_count > iov_index)) {
|
||||
mem_address = (OPAL_PTRDIFF_TYPE)
|
||||
(decoded_iov[iov_index].iov_base) + current_position;
|
||||
|
||||
if (remaining >=
|
||||
(decoded_iov[iov_index].iov_len - current_position)) {
|
||||
memcpy ((IOVBASE_TYPE *) mem_address,
|
||||
receive_buf+temp_position,
|
||||
decoded_iov[iov_index].iov_len - current_position);
|
||||
remaining = remaining -
|
||||
(decoded_iov[iov_index].iov_len - current_position);
|
||||
temp_position = temp_position +
|
||||
(decoded_iov[iov_index].iov_len - current_position);
|
||||
iov_index = iov_index + 1;
|
||||
current_position = 0;
|
||||
}
|
||||
else {
|
||||
memcpy ((IOVBASE_TYPE *) mem_address,
|
||||
receive_buf+temp_position,
|
||||
remaining);
|
||||
current_position = current_position + remaining;
|
||||
remaining = 0;
|
||||
}
|
||||
}
|
||||
|
||||
if (NULL != receive_buf) {
|
||||
free (receive_buf);
|
||||
receive_buf = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
/**********************************************************
|
||||
**************** DONE SCATTERING OF DATA *****************
|
||||
*********************************************************/
|
||||
|
||||
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
fh->f_num_of_io_entries = 0;
|
||||
if (NULL != fh->f_io_array) {
|
||||
free (fh->f_io_array);
|
||||
fh->f_io_array = NULL;
|
||||
}
|
||||
if (NULL != global_iov) {
|
||||
free (global_iov);
|
||||
global_iov = NULL;
|
||||
}
|
||||
if (NULL != global_buf) {
|
||||
free (global_buf);
|
||||
global_buf = NULL;
|
||||
}
|
||||
if (NULL != sorted) {
|
||||
free (sorted);
|
||||
sorted = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
if (NULL != iov) {
|
||||
free (iov);
|
||||
iov = NULL;
|
||||
}
|
||||
}
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
if (NULL != iovec_count_per_process) {
|
||||
free (iovec_count_per_process);
|
||||
iovec_count_per_process = NULL;
|
||||
}
|
||||
if (NULL != bytes_per_process) {
|
||||
free (bytes_per_process);
|
||||
bytes_per_process = NULL;
|
||||
}
|
||||
if (NULL != displs) {
|
||||
free (displs);
|
||||
displs = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
36
ompi/mca/fcoll/static/fcoll_static_file_read_all_begin.c
Обычный файл
36
ompi/mca/fcoll/static/fcoll_static_file_read_all_begin.c
Обычный файл
@ -0,0 +1,36 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_static.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
|
||||
|
||||
int
|
||||
mca_fcoll_static_file_read_all_begin (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype)
|
||||
{
|
||||
printf ("STATIC READ ALL BEGIN\n");
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
35
ompi/mca/fcoll/static/fcoll_static_file_read_all_end.c
Обычный файл
35
ompi/mca/fcoll/static/fcoll_static_file_read_all_end.c
Обычный файл
@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_static.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
|
||||
|
||||
int
|
||||
mca_fcoll_static_file_read_all_end (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
ompi_status_public_t *status)
|
||||
{
|
||||
printf ("STATIC READ ALL END\n");
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
457
ompi/mca/fcoll/static/fcoll_static_file_write_all.c
Обычный файл
457
ompi/mca/fcoll/static/fcoll_static_file_write_all.c
Обычный файл
@ -0,0 +1,457 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_static.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
#include "ompi/mca/io/ompio/io_ompio.h"
|
||||
#include "ompi/mca/io/io.h"
|
||||
#include "math.h"
|
||||
#include "ompi/mca/pml/pml.h"
|
||||
#include <unistd.h>
|
||||
|
||||
int
|
||||
mca_fcoll_static_file_write_all (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype,
|
||||
ompi_status_public_t *status)
|
||||
{
|
||||
MPI_Aint total_bytes_written = 0; /* total bytes that have been written*/
|
||||
MPI_Aint bytes_to_write_in_cycle = 0; /* left to be written in a cycle*/
|
||||
size_t bytes_per_cycle = 0; /* total written in each cycle by each process*/
|
||||
|
||||
int index = 0;
|
||||
int cycles = 0, local_cycles = 0;
|
||||
int i=0;
|
||||
int ret;
|
||||
|
||||
/* iovec structure and count of the buffer passed in */
|
||||
uint32_t iov_count = 0;
|
||||
struct iovec *decoded_iov = NULL;
|
||||
uint32_t iov_index = 0;
|
||||
size_t current_position = 0;
|
||||
char *send_buf = NULL;
|
||||
|
||||
/* global iovec at the writers that contain the iovecs created from
|
||||
file_set_view */
|
||||
uint32_t global_iov_count = 0;
|
||||
struct iovec *global_iov = NULL;
|
||||
int *sorted;
|
||||
|
||||
char *global_buf = NULL;
|
||||
MPI_Aint global_count = 0;
|
||||
|
||||
int *displs = NULL;
|
||||
|
||||
size_t max_data = 0;
|
||||
int *iovec_count_per_process = NULL;
|
||||
int *bytes_per_process = NULL;
|
||||
|
||||
if (opal_datatype_is_contiguous_memory_layout(&datatype->super,1)) {
|
||||
fh->f_flags |= OMPIO_CONTIGUOUS_MEMORY;
|
||||
}
|
||||
|
||||
/* In case the data is not contigous in memory, decode it into an iovec */
|
||||
if (! (fh->f_flags & OMPIO_CONTIGUOUS_MEMORY)) {
|
||||
ompi_io_ompio_decode_datatype (fh,
|
||||
datatype,
|
||||
count,
|
||||
buf,
|
||||
&max_data,
|
||||
&decoded_iov,
|
||||
&iov_count);
|
||||
}
|
||||
else {
|
||||
max_data = count * datatype->super.size;
|
||||
}
|
||||
|
||||
if (! (fh->f_flags & OMPIO_AGGREGATOR_IS_SET)) {
|
||||
ompi_io_ompio_set_aggregator_props (fh,
|
||||
mca_fcoll_static_num_io_procs,
|
||||
max_data);
|
||||
}
|
||||
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
bytes_per_process = (int *)malloc (sizeof(int)*fh->f_procs_per_group);
|
||||
if (NULL == bytes_per_process) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
iovec_count_per_process = (int *)malloc (sizeof(int)*fh->f_procs_per_group);
|
||||
if (NULL == iovec_count_per_process) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
displs = (int*)malloc (fh->f_procs_per_group*sizeof(int));
|
||||
if (NULL == displs) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Calculate how many bytes are written in each cycle
|
||||
*/
|
||||
if (mca_fcoll_static_constant_cbs) {
|
||||
bytes_per_cycle =
|
||||
mca_fcoll_static_cycle_buffer_size/fh->f_procs_per_group;
|
||||
}
|
||||
else {
|
||||
bytes_per_cycle = mca_fcoll_static_cycle_buffer_size;
|
||||
}
|
||||
|
||||
/* TODO : number of cycles has to be the same for all processes in a group,
|
||||
so need to Allreduce the cycle within a group.
|
||||
This works now if all processes are writing the same amount of data */
|
||||
|
||||
cycles = ceil((double)max_data/bytes_per_cycle);
|
||||
local_cycles = cycles;
|
||||
ret = fh->f_comm->c_coll.coll_allreduce (&local_cycles,
|
||||
&cycles,
|
||||
1,
|
||||
MPI_INT,
|
||||
MPI_MAX,
|
||||
fh->f_comm,
|
||||
fh->f_comm->c_coll.coll_allreduce_module);
|
||||
#if 0
|
||||
printf ("Max Data: %d Bytes per Process: %d Cycles: %d Procs_per_group %d\n",
|
||||
max_data, bytes_per_cycle, cycles, fh->f_procs_per_group);
|
||||
#endif
|
||||
for (index = 0; index < cycles; index++) {
|
||||
struct iovec *iov = NULL;
|
||||
int iov_size = 0;
|
||||
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
/*printf ("********** CYCLE %d **************\n",index);*/
|
||||
memset(displs, 0x0, fh->f_procs_per_group*sizeof(int));
|
||||
memset(bytes_per_process, 0x0, fh->f_procs_per_group*sizeof(int));
|
||||
memset(iovec_count_per_process, 0x0, fh->f_procs_per_group*sizeof(int));
|
||||
}
|
||||
|
||||
if (local_cycles > index) {
|
||||
if ((index == local_cycles-1) && (max_data % bytes_per_cycle)) {
|
||||
bytes_to_write_in_cycle = max_data % bytes_per_cycle;
|
||||
}
|
||||
else if (max_data <= bytes_per_cycle) {
|
||||
bytes_to_write_in_cycle = max_data;
|
||||
}
|
||||
else {
|
||||
bytes_to_write_in_cycle = bytes_per_cycle;
|
||||
}
|
||||
}
|
||||
else {
|
||||
bytes_to_write_in_cycle = 0;
|
||||
}
|
||||
/*
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
printf ("****%d: CYCLE %d Bytes %d**********\n",
|
||||
fh->f_rank,
|
||||
index,
|
||||
bytes_to_write_in_cycle);
|
||||
}
|
||||
*/
|
||||
|
||||
/**********************************************************
|
||||
**Gather the Data from all the processes at the writers **
|
||||
*********************************************************/
|
||||
|
||||
/* gather from each process how many bytes each will be sending */
|
||||
ompi_io_ompio_gather_array (&bytes_to_write_in_cycle,
|
||||
1,
|
||||
MPI_INT,
|
||||
bytes_per_process,
|
||||
1,
|
||||
MPI_INT,
|
||||
fh->f_aggregator_index,
|
||||
fh->f_procs_in_group,
|
||||
fh->f_procs_per_group,
|
||||
fh->f_comm);
|
||||
|
||||
/* Calculate the displacement on where to put the data and allocate
|
||||
the recieve buffer (global_buf) */
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
displs[0] = 0;
|
||||
global_count = bytes_per_process[0];
|
||||
for (i=1 ; i<fh->f_procs_per_group ; i++) {
|
||||
global_count += bytes_per_process[i];
|
||||
displs[i] = displs[i-1] + bytes_per_process[i-1];
|
||||
}
|
||||
|
||||
global_buf = malloc (global_count);
|
||||
if (NULL == global_buf) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
|
||||
if (fh->f_flags & OMPIO_CONTIGUOUS_MEMORY) {
|
||||
send_buf = &((char*)buf)[total_bytes_written];
|
||||
}
|
||||
else if (bytes_to_write_in_cycle) {
|
||||
/* allocate a send buffer and copy the data that needs
|
||||
to be sent into it in case the data is non-contigous
|
||||
in memory */
|
||||
OPAL_PTRDIFF_TYPE mem_address;
|
||||
size_t remaining = 0;
|
||||
size_t temp_position = 0;
|
||||
|
||||
send_buf = malloc (bytes_to_write_in_cycle);
|
||||
if (NULL == send_buf) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
remaining = bytes_to_write_in_cycle;
|
||||
|
||||
while (remaining && (iov_count > iov_index)) {
|
||||
mem_address = (OPAL_PTRDIFF_TYPE)
|
||||
(decoded_iov[iov_index].iov_base) + current_position;
|
||||
|
||||
if (remaining >=
|
||||
(decoded_iov[iov_index].iov_len - current_position)) {
|
||||
memcpy (send_buf+temp_position,
|
||||
(IOVBASE_TYPE *)mem_address,
|
||||
decoded_iov[iov_index].iov_len - current_position);
|
||||
remaining = remaining -
|
||||
(decoded_iov[iov_index].iov_len - current_position);
|
||||
temp_position = temp_position +
|
||||
(decoded_iov[iov_index].iov_len - current_position);
|
||||
iov_index = iov_index + 1;
|
||||
current_position = 0;
|
||||
}
|
||||
else {
|
||||
memcpy (send_buf+temp_position,
|
||||
(IOVBASE_TYPE *)mem_address,
|
||||
remaining);
|
||||
current_position = current_position + remaining;
|
||||
remaining = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
total_bytes_written += bytes_to_write_in_cycle;
|
||||
|
||||
/* Get the data from all processes to the writer*/
|
||||
ompi_io_ompio_gatherv_array (send_buf,
|
||||
bytes_to_write_in_cycle,
|
||||
MPI_BYTE,
|
||||
global_buf,
|
||||
bytes_per_process,
|
||||
displs,
|
||||
MPI_BYTE,
|
||||
fh->f_aggregator_index,
|
||||
fh->f_procs_in_group,
|
||||
fh->f_procs_per_group,
|
||||
fh->f_comm);
|
||||
|
||||
if (!(fh->f_flags & OMPIO_CONTIGUOUS_MEMORY)) {
|
||||
if (NULL != send_buf) {
|
||||
free (send_buf);
|
||||
send_buf = NULL;
|
||||
}
|
||||
}
|
||||
/*
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank)
|
||||
for (i=0 ; i<global_count/4 ; i++)
|
||||
printf ("RECV %d \n",
|
||||
((int *)global_buf)[i]);
|
||||
*/
|
||||
/**********************************************************
|
||||
**************** DONE GATHERING OF DATA ******************
|
||||
*********************************************************/
|
||||
|
||||
/**********************************************************
|
||||
Gather from each process iovecs to where to write the data
|
||||
*********************************************************/
|
||||
iov_size = 0;
|
||||
ompi_io_ompio_generate_current_file_view (fh,
|
||||
bytes_to_write_in_cycle,
|
||||
&iov,
|
||||
&iov_size);
|
||||
/*
|
||||
for (i=0 ; i<iov_size ; i++) {
|
||||
printf("%d: OFFSET: %p LENGTH: %d\n",
|
||||
fh->f_rank,
|
||||
iov[i].iov_base,
|
||||
iov[i].iov_len);
|
||||
}
|
||||
*/
|
||||
ompi_io_ompio_gather_array (&iov_size,
|
||||
1,
|
||||
MPI_INT,
|
||||
iovec_count_per_process,
|
||||
1,
|
||||
MPI_INT,
|
||||
fh->f_aggregator_index,
|
||||
fh->f_procs_in_group,
|
||||
fh->f_procs_per_group,
|
||||
fh->f_comm);
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
displs[0] = 0;
|
||||
global_iov_count = iovec_count_per_process[0];
|
||||
for (i=1 ; i<fh->f_procs_per_group ; i++) {
|
||||
global_iov_count += iovec_count_per_process[i];
|
||||
displs[i] = displs[i-1] + iovec_count_per_process[i-1];
|
||||
}
|
||||
}
|
||||
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
global_iov = (struct iovec*)malloc (global_iov_count *
|
||||
sizeof(struct iovec));
|
||||
if (NULL == global_iov) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
|
||||
if (fh->f_flags & OMPIO_UNIFORM_FVIEW) {
|
||||
ompi_io_ompio_gather_array (iov,
|
||||
iov_size,
|
||||
fh->f_iov_type,
|
||||
global_iov,
|
||||
iov_size,
|
||||
fh->f_iov_type,
|
||||
fh->f_aggregator_index,
|
||||
fh->f_procs_in_group,
|
||||
fh->f_procs_per_group,
|
||||
fh->f_comm);
|
||||
}
|
||||
else {
|
||||
ompi_io_ompio_gatherv_array (iov,
|
||||
iov_size,
|
||||
fh->f_iov_type,
|
||||
global_iov,
|
||||
iovec_count_per_process,
|
||||
displs,
|
||||
fh->f_iov_type,
|
||||
fh->f_aggregator_index,
|
||||
fh->f_procs_in_group,
|
||||
fh->f_procs_per_group,
|
||||
fh->f_comm);
|
||||
}
|
||||
|
||||
/* sort it */
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
sorted = (int *)malloc (global_iov_count * sizeof(int));
|
||||
if (NULL == sorted) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
ompi_io_ompio_sort_iovec (global_iov, global_iov_count, sorted);
|
||||
}
|
||||
|
||||
/**********************************************************
|
||||
**************** DONE GATHERING OF IOVECS ****************
|
||||
*********************************************************/
|
||||
|
||||
/**********************************************************
|
||||
** Create the io array, sort it, and pass it to fbtl ****
|
||||
*********************************************************/
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
MPI_Aint temp = 0;
|
||||
int x = 0, k = 0;
|
||||
|
||||
fh->f_io_array = (mca_io_ompio_io_array_t *) malloc
|
||||
(global_iov_count * sizeof (mca_io_ompio_io_array_t));
|
||||
if (NULL == fh->f_io_array) {
|
||||
opal_output(1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
for (i=0 ; i<fh->f_procs_per_group ; i++) {
|
||||
for (x=0 ; x<iovec_count_per_process[i] ; x++) {
|
||||
fh->f_io_array[k].offset = global_iov[sorted[k]].iov_base;
|
||||
fh->f_io_array[k].length = global_iov[sorted[k]].iov_len;
|
||||
fh->f_io_array[k].memory_address = &global_buf[temp];
|
||||
temp += fh->f_io_array[k].length;
|
||||
k ++;
|
||||
}
|
||||
}
|
||||
fh->f_num_of_io_entries = k;
|
||||
|
||||
/*
|
||||
printf("*************************** %d\n", fh->f_num_of_io_entries);
|
||||
for (i=0 ; i<fh->f_num_of_io_entries ; i++)
|
||||
{
|
||||
printf(" ADDRESS: %p OFFSET: %p LENGTH: %d\n",
|
||||
fh->f_io_array[i].memory_address,
|
||||
fh->f_io_array[i].offset,
|
||||
fh->f_io_array[i].length);
|
||||
}
|
||||
|
||||
printf("******* SORTED ************ %d\n", fh->f_num_of_io_entries);
|
||||
for (i=0 ; i<fh->f_num_of_io_entries ; i++)
|
||||
{
|
||||
printf(" ADDRESS: %p OFFSET: %p LENGTH: %d\n",
|
||||
fh->f_io_array[sorted[i]].memory_address,
|
||||
fh->f_io_array[sorted[i]].offset,
|
||||
fh->f_io_array[sorted[i]].length);
|
||||
}
|
||||
*/
|
||||
if (fh->f_num_of_io_entries) {
|
||||
fh->f_fbtl->fbtl_pwritev (fh, NULL);
|
||||
}
|
||||
}
|
||||
/**********************************************************
|
||||
******************** DONE WRITING ************************
|
||||
*********************************************************/
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
fh->f_num_of_io_entries = 0;
|
||||
if (NULL != fh->f_io_array) {
|
||||
free (fh->f_io_array);
|
||||
fh->f_io_array = NULL;
|
||||
}
|
||||
if (NULL != global_iov) {
|
||||
free (global_iov);
|
||||
global_iov = NULL;
|
||||
}
|
||||
if (NULL != global_buf) {
|
||||
free (global_buf);
|
||||
global_buf = NULL;
|
||||
}
|
||||
if (NULL != sorted) {
|
||||
free (sorted);
|
||||
sorted = NULL;
|
||||
}
|
||||
}
|
||||
if (NULL != iov) {
|
||||
free (iov);
|
||||
iov = NULL;
|
||||
}
|
||||
}
|
||||
if (fh->f_procs_in_group[fh->f_aggregator_index] == fh->f_rank) {
|
||||
if (NULL != iovec_count_per_process) {
|
||||
free (iovec_count_per_process);
|
||||
iovec_count_per_process = NULL;
|
||||
}
|
||||
if (NULL != bytes_per_process) {
|
||||
free (bytes_per_process);
|
||||
bytes_per_process = NULL;
|
||||
}
|
||||
if (NULL != displs) {
|
||||
free (displs);
|
||||
displs = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
36
ompi/mca/fcoll/static/fcoll_static_file_write_all_begin.c
Обычный файл
36
ompi/mca/fcoll/static/fcoll_static_file_write_all_begin.c
Обычный файл
@ -0,0 +1,36 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_static.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
|
||||
|
||||
int
|
||||
mca_fcoll_static_file_write_all_begin (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype)
|
||||
{
|
||||
printf ("STATIC WRITE ALL BEGIN\n");
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
35
ompi/mca/fcoll/static/fcoll_static_file_write_all_end.c
Обычный файл
35
ompi/mca/fcoll/static/fcoll_static_file_write_all_end.c
Обычный файл
@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_static.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
|
||||
|
||||
int
|
||||
mca_fcoll_static_file_write_all_end (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
ompi_status_public_t *status)
|
||||
{
|
||||
printf ("STATIC WRITE ALL END\n");
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
91
ompi/mca/fcoll/static/fcoll_static_module.c
Обычный файл
91
ompi/mca/fcoll/static/fcoll_static_module.c
Обычный файл
@ -0,0 +1,91 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2006 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_static.h"
|
||||
|
||||
#include <stdio.h>
|
||||
|
||||
#include "mpi.h"
|
||||
#include "opal/mca/base/mca_base_param.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
#include "ompi/mca/fcoll/base/base.h"
|
||||
|
||||
|
||||
/*
|
||||
* *******************************************************************
|
||||
* ************************ actions structure ************************
|
||||
* *******************************************************************
|
||||
*/
|
||||
static mca_fcoll_base_module_1_0_0_t static_t = {
|
||||
mca_fcoll_static_module_init,
|
||||
mca_fcoll_static_module_finalize,
|
||||
mca_fcoll_static_file_read_all,
|
||||
mca_fcoll_static_file_read_all_begin,
|
||||
mca_fcoll_static_file_read_all_end,
|
||||
mca_fcoll_static_file_write_all,
|
||||
mca_fcoll_static_file_write_all_begin,
|
||||
mca_fcoll_static_file_write_all_end
|
||||
};
|
||||
|
||||
int
|
||||
mca_fcoll_static_component_init_query(bool enable_progress_threads,
|
||||
bool enable_mpi_threads)
|
||||
{
|
||||
/* Nothing to do */
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
mca_fcoll_base_module_1_0_0_t *
|
||||
mca_fcoll_static_component_file_query (mca_io_ompio_file_t *fh, int *priority)
|
||||
{
|
||||
*priority = mca_fcoll_static_priority;
|
||||
if (0 >= mca_fcoll_static_priority) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (mca_fcoll_base_query_table (fh, "static")) {
|
||||
if (*priority < 50) {
|
||||
*priority = 50;
|
||||
}
|
||||
}
|
||||
|
||||
return &static_t;
|
||||
}
|
||||
|
||||
int mca_fcoll_static_component_file_unquery (mca_io_ompio_file_t *file)
|
||||
{
|
||||
/* This function might be needed for some purposes later. for now it
|
||||
* does not have anything to do since there are no steps which need
|
||||
* to be undone if this module is not selected */
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
int mca_fcoll_static_module_init (mca_io_ompio_file_t *file)
|
||||
{
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
int mca_fcoll_static_module_finalize (mca_io_ompio_file_t *file)
|
||||
{
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
54
ompi/mca/fcoll/two_phase/Makefile.am
Обычный файл
54
ompi/mca/fcoll/two_phase/Makefile.am
Обычный файл
@ -0,0 +1,54 @@
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
sources = \
|
||||
fcoll_two_phase.h \
|
||||
fcoll_two_phase_module.c \
|
||||
fcoll_two_phase_component.c \
|
||||
fcoll_two_phase_file_read_all.c \
|
||||
fcoll_two_phase_file_read_all_begin.c \
|
||||
fcoll_two_phase_file_read_all_end.c \
|
||||
fcoll_two_phase_file_write_all.c \
|
||||
fcoll_two_phase_file_write_all_begin.c \
|
||||
fcoll_two_phase_file_write_all_end.c
|
||||
|
||||
# Make the output library in this directory, and name it either
|
||||
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
|
||||
# (for static builds).
|
||||
|
||||
if MCA_BUILD_ompi_fcoll_two_phase_DSO
|
||||
component_noinst =
|
||||
component_install = mca_fcoll_two_phase.la
|
||||
else
|
||||
component_noinst = libmca_fcoll_two_phase.la
|
||||
component_install =
|
||||
endif
|
||||
|
||||
mcacomponentdir = $(libdir)/openmpi
|
||||
mcacomponent_LTLIBRARIES = $(component_install)
|
||||
mca_fcoll_two_phase_la_SOURCES = $(sources)
|
||||
mca_fcoll_two_phase_la_LDFLAGS = -module -avoid-version
|
||||
mca_fcoll_two_phase_la_LIBADD = \
|
||||
$(top_ompi_builddir)/ompi/libmpi.la \
|
||||
$(top_ompi_builddir)/orte/libopen-rte.la \
|
||||
$(top_ompi_builddir)/opal/libopen-pal.la
|
||||
|
||||
noinst_LTLIBRARIES = $(component_noinst)
|
||||
libmca_fcoll_two_phase_la_SOURCES =$(sources)
|
||||
libmca_fcoll_two_phase_la_LDFLAGS = -module -avoid-version
|
||||
86
ompi/mca/fcoll/two_phase/fcoll_two_phase.h
Обычный файл
86
ompi/mca/fcoll/two_phase/fcoll_two_phase.h
Обычный файл
@ -0,0 +1,86 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2006 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#ifndef MCA_FCOLL_TWO_PHASE_EXPORT_H
|
||||
#define MCA_FCOLL_TWO_PHASE_EXPORT_H
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "opal/mca/mca.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
#include "ompi/mca/fcoll/base/base.h"
|
||||
#include "ompi/mca/io/ompio/io_ompio.h"
|
||||
|
||||
BEGIN_C_DECLS
|
||||
|
||||
/* Globally exported variables */
|
||||
|
||||
extern int mca_fcoll_two_phase_priority;
|
||||
extern int mca_fcoll_two_phase_num_io_procs;
|
||||
extern int mca_fcoll_two_phase_constant_cbs;
|
||||
extern int mca_fcoll_two_phase_cycle_buffer_size;
|
||||
|
||||
OMPI_MODULE_DECLSPEC extern mca_fcoll_base_component_2_0_0_t mca_fcoll_two_phase_component;
|
||||
|
||||
/* API functions */
|
||||
|
||||
int mca_fcoll_two_phase_component_init_query(bool enable_progress_threads,
|
||||
bool enable_mpi_threads);
|
||||
struct mca_fcoll_base_module_1_0_0_t *
|
||||
mca_fcoll_two_phase_component_file_query (mca_io_ompio_file_t *fh, int *priority);
|
||||
|
||||
int mca_fcoll_two_phase_component_file_unquery (mca_io_ompio_file_t *file);
|
||||
|
||||
int mca_fcoll_two_phase_module_init (mca_io_ompio_file_t *file);
|
||||
int mca_fcoll_two_phase_module_finalize (mca_io_ompio_file_t *file);
|
||||
|
||||
int mca_fcoll_two_phase_file_read_all (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype,
|
||||
ompi_status_public_t * status);
|
||||
|
||||
int mca_fcoll_two_phase_file_read_all_begin (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype);
|
||||
|
||||
int mca_fcoll_two_phase_file_read_all_end (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
ompi_status_public_t * status);
|
||||
|
||||
int mca_fcoll_two_phase_file_write_all (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype,
|
||||
ompi_status_public_t * status);
|
||||
|
||||
int mca_fcoll_two_phase_file_write_all_begin (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype);
|
||||
|
||||
int mca_fcoll_two_phase_file_write_all_end (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
ompi_status_public_t * status);
|
||||
|
||||
END_C_DECLS
|
||||
|
||||
#endif /* MCA_FCOLL_TWO_PHASE_EXPORT_H */
|
||||
132
ompi/mca/fcoll/two_phase/fcoll_two_phase_component.c
Обычный файл
132
ompi/mca/fcoll/two_phase/fcoll_two_phase_component.c
Обычный файл
@ -0,0 +1,132 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*
|
||||
* These symbols are in a file by themselves to provide nice linker
|
||||
* semantics. Since linkers generally pull in symbols by object
|
||||
* files, keeping these symbols as the only symbols in this file
|
||||
* prevents utility programs such as "ompi_info" from having to import
|
||||
* entire components just to query their version and parameters.
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_two_phase.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
#include "fcoll_two_phase.h"
|
||||
|
||||
/*
|
||||
* Public string showing the fcoll ompi_two_phase component version number
|
||||
*/
|
||||
const char *mca_fcoll_two_phase_component_version_string =
|
||||
"Open MPI two_phase collective MCA component version " OMPI_VERSION;
|
||||
|
||||
/*
|
||||
* Global variables
|
||||
*/
|
||||
int mca_fcoll_two_phase_priority = 10;
|
||||
int mca_fcoll_two_phase_num_io_procs = -1;
|
||||
int mca_fcoll_two_phase_constant_cbs = 0;
|
||||
int mca_fcoll_two_phase_cycle_buffer_size = OMPIO_PREALLOC_MAX_BUF_SIZE;
|
||||
|
||||
/*
|
||||
* Local function
|
||||
*/
|
||||
static int two_phase_register(void);
|
||||
|
||||
/*
|
||||
* Instantiate the public struct with all of our public information
|
||||
* and pointers to our public functions in it
|
||||
*/
|
||||
mca_fcoll_base_component_2_0_0_t mca_fcoll_two_phase_component = {
|
||||
|
||||
/* First, the mca_component_t struct containing meta information
|
||||
* about the component itself */
|
||||
|
||||
{
|
||||
MCA_FCOLL_BASE_VERSION_2_0_0,
|
||||
|
||||
/* Component name and version */
|
||||
"two_phase",
|
||||
OMPI_MAJOR_VERSION,
|
||||
OMPI_MINOR_VERSION,
|
||||
OMPI_RELEASE_VERSION,
|
||||
two_phase_register,
|
||||
NULL
|
||||
},
|
||||
{
|
||||
/* The component is checkpoint ready */
|
||||
MCA_BASE_METADATA_PARAM_CHECKPOINT
|
||||
},
|
||||
|
||||
mca_fcoll_two_phase_component_init_query,
|
||||
mca_fcoll_two_phase_component_file_query,
|
||||
mca_fcoll_two_phase_component_file_unquery
|
||||
};
|
||||
|
||||
|
||||
static int
|
||||
two_phase_register(void)
|
||||
{
|
||||
int param;
|
||||
|
||||
param = mca_base_param_find ("fcoll", NULL, "two_phase_priority");
|
||||
if (param >= 0)
|
||||
{
|
||||
mca_base_param_lookup_int (param, &mca_fcoll_two_phase_priority);
|
||||
}
|
||||
param = mca_base_param_find ("fcoll", NULL, "two_phase_num_io_procs");
|
||||
if (param >= 0)
|
||||
{
|
||||
mca_base_param_lookup_int (param, &mca_fcoll_two_phase_num_io_procs);
|
||||
}
|
||||
param = mca_base_param_find ("fcoll", NULL, "two_phase_constant_cbs");
|
||||
if (param >= 0)
|
||||
{
|
||||
mca_base_param_lookup_int (param, &mca_fcoll_two_phase_constant_cbs);
|
||||
}
|
||||
param = mca_base_param_find ("fcoll", NULL, "two_phase_cycle_buffer_size");
|
||||
if (param >= 0)
|
||||
{
|
||||
mca_base_param_lookup_int (param, &mca_fcoll_two_phase_cycle_buffer_size);
|
||||
}
|
||||
|
||||
mca_base_param_reg_int (&mca_fcoll_two_phase_component.fcollm_version,
|
||||
"priority",
|
||||
"Priority of the two_phase fcoll component",
|
||||
false, false, mca_fcoll_two_phase_priority,
|
||||
&mca_fcoll_two_phase_priority);
|
||||
mca_base_param_reg_int (&mca_fcoll_two_phase_component.fcollm_version,
|
||||
"num_io_procs",
|
||||
"Number of writers in the two_phase fcoll component",
|
||||
false, false, mca_fcoll_two_phase_num_io_procs,
|
||||
&mca_fcoll_two_phase_num_io_procs);
|
||||
mca_base_param_reg_int (&mca_fcoll_two_phase_component.fcollm_version,
|
||||
"constant_cbs",
|
||||
"wether we are using constant or scaling cycle buffer size in the two_phase fcoll component",
|
||||
false, false, mca_fcoll_two_phase_constant_cbs,
|
||||
&mca_fcoll_two_phase_constant_cbs);
|
||||
mca_base_param_reg_int (&mca_fcoll_two_phase_component.fcollm_version,
|
||||
"cycle_buffer_size",
|
||||
"Cycle Buffer Size of the two_phase fcoll component",
|
||||
false, false, mca_fcoll_two_phase_cycle_buffer_size,
|
||||
&mca_fcoll_two_phase_cycle_buffer_size);
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
713
ompi/mca/fcoll/two_phase/fcoll_two_phase_file_read_all.c
Обычный файл
713
ompi/mca/fcoll/two_phase/fcoll_two_phase_file_read_all.c
Обычный файл
@ -0,0 +1,713 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_two_phase.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
#include "ompi/mca/io/ompio/io_ompio.h"
|
||||
#include "ompi/mca/io/io.h"
|
||||
#include "math.h"
|
||||
#include "ompi/mca/pml/pml.h"
|
||||
#include <unistd.h>
|
||||
|
||||
int
|
||||
mca_fcoll_two_phase_file_read_all (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype,
|
||||
ompi_status_public_t *status)
|
||||
{
|
||||
size_t total_bytes_read = 0; /* total bytes that have been read*/
|
||||
size_t total_bytes = 0; /* total bytes to be read */
|
||||
size_t total_bytes_global = 0;
|
||||
size_t bytes_to_read_in_cycle = 0; /* left to be read in a cycle*/
|
||||
size_t max_data = 0;
|
||||
size_t bytes_remaining = 0;
|
||||
size_t *bytes_rem = NULL;
|
||||
size_t *prev_bytes_rem = NULL;
|
||||
size_t stripe_size =0;
|
||||
|
||||
int index = 0;
|
||||
int current_index = 0;
|
||||
int *current = NULL;
|
||||
int *previous = NULL;
|
||||
int cycles = 0;
|
||||
int i=0, j=0, x=0, n=0;
|
||||
int blocks = 0;
|
||||
int bytes_left = 0;
|
||||
int two_phase_num_io_procs = 0;
|
||||
|
||||
/* array that contains the sorted indices of the global_iov */
|
||||
int *sorted = NULL;
|
||||
int *displs = NULL;
|
||||
int *bytes_per_process = NULL;
|
||||
int *bytes_received = NULL;
|
||||
|
||||
/* iovec structure and count of the buffer passed in */
|
||||
uint32_t iov_count = 0;
|
||||
struct iovec *decoded_iov = NULL;
|
||||
|
||||
int global_fview_count = 0;
|
||||
struct iovec *global_fview = NULL;
|
||||
|
||||
int local_count = 0;
|
||||
struct iovec *iov = NULL;
|
||||
|
||||
int broken_count = 0;
|
||||
struct iovec *broken_iovec = NULL;
|
||||
|
||||
int *fview_count = NULL;
|
||||
|
||||
int global_count = 0;
|
||||
char *global_buf = NULL;
|
||||
|
||||
if (opal_datatype_is_contiguous_memory_layout(&datatype->super,1)) {
|
||||
fh->f_flags |= OMPIO_CONTIGUOUS_MEMORY;
|
||||
}
|
||||
|
||||
/**************************************************************************
|
||||
** In case the data is not contigous in memory, decode it into an iovec **
|
||||
**************************************************************************/
|
||||
if (!(fh->f_flags & OMPIO_CONTIGUOUS_MEMORY)) {
|
||||
ompi_io_ompio_decode_datatype (fh,
|
||||
datatype,
|
||||
count,
|
||||
buf,
|
||||
&max_data,
|
||||
&decoded_iov,
|
||||
&iov_count);
|
||||
}
|
||||
else {
|
||||
max_data = count * datatype->super.size;
|
||||
}
|
||||
|
||||
if (! (fh->f_flags & OMPIO_AGGREGATOR_IS_SET)) {
|
||||
if (-1 == mca_fcoll_two_phase_num_io_procs) {
|
||||
ompi_io_ompio_set_aggregator_props (fh,
|
||||
mca_fcoll_two_phase_num_io_procs,
|
||||
max_data);
|
||||
two_phase_num_io_procs =
|
||||
ceil((float)fh->f_size/fh->f_procs_per_group);
|
||||
}
|
||||
fh->f_aggregator_index =
|
||||
ceil((float)fh->f_size/two_phase_num_io_procs);
|
||||
if (fh->f_aggregator_index * two_phase_num_io_procs > fh->f_size) {
|
||||
two_phase_num_io_procs =
|
||||
ceil((float)fh->f_size/fh->f_aggregator_index);
|
||||
}
|
||||
}
|
||||
|
||||
/*********************************************************************
|
||||
*** Generate the File offsets/lengths corresponding to this read ***
|
||||
********************************************************************/
|
||||
ompi_io_ompio_generate_current_file_view (fh,
|
||||
max_data,
|
||||
&iov,
|
||||
&local_count);
|
||||
/*
|
||||
for (i=0 ; i<local_count ; i++) {
|
||||
printf("%d: OFFSET: %d LENGTH: %d\n",
|
||||
fh->f_rank,
|
||||
iov[i].iov_base,
|
||||
iov[i].iov_len);
|
||||
}
|
||||
*/
|
||||
|
||||
/*************************************************************
|
||||
* Breakdown the file view at each process per OST then send *
|
||||
* each portion of the file view t0 the corresp aggregator *
|
||||
*************************************************************/
|
||||
|
||||
fh->f_comm->c_coll.coll_allreduce (&max_data,
|
||||
&total_bytes,
|
||||
1,
|
||||
MPI_DOUBLE,
|
||||
MPI_SUM,
|
||||
fh->f_comm,
|
||||
fh->f_comm->c_coll.coll_allreduce_module);
|
||||
|
||||
stripe_size = ceil((float)total_bytes/two_phase_num_io_procs);
|
||||
|
||||
ompi_io_ompio_break_file_view (fh,
|
||||
iov,
|
||||
local_count,
|
||||
two_phase_num_io_procs,
|
||||
stripe_size,
|
||||
&broken_iovec,
|
||||
&broken_count);
|
||||
/*
|
||||
for (i=0 ; i<broken_count ; i++) {
|
||||
printf("%d: OFFSET: %d LENGTH: %d\n",
|
||||
fh->f_rank,
|
||||
broken_iovec[i].iov_base,
|
||||
broken_iovec[i].iov_len);
|
||||
}
|
||||
*/
|
||||
|
||||
if (NULL != iov) {
|
||||
free (iov);
|
||||
iov = NULL;
|
||||
}
|
||||
|
||||
ompi_io_ompio_distribute_file_view (fh,
|
||||
broken_iovec,
|
||||
broken_count,
|
||||
two_phase_num_io_procs,
|
||||
stripe_size,
|
||||
&fview_count,
|
||||
&global_fview,
|
||||
&global_fview_count);
|
||||
/*
|
||||
for (i=0 ; i<global_fview_count ; i++) {
|
||||
printf("%d: OFFSET: %d LENGTH: %d\n",
|
||||
fh->f_rank,
|
||||
global_fview[i].iov_base,
|
||||
global_fview[i].iov_len);
|
||||
}
|
||||
*/
|
||||
|
||||
total_bytes = 0;
|
||||
if (0 == fh->f_rank%fh->f_aggregator_index) {
|
||||
if (global_fview_count) {
|
||||
for (i=0 ; i<global_fview_count ; i++) {
|
||||
total_bytes += global_fview[i].iov_len;
|
||||
}
|
||||
/* sort it */
|
||||
sorted = (int *)malloc (global_fview_count * sizeof(int));
|
||||
if (NULL == sorted) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
ompi_io_ompio_sort_iovec (global_fview, global_fview_count, sorted);
|
||||
}
|
||||
bytes_per_process = (int *) malloc (fh->f_size * sizeof (int));
|
||||
if (NULL == bytes_per_process) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
displs = (int *) malloc (fh->f_size * sizeof (int));
|
||||
if (NULL == displs) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
fh->f_comm->c_coll.coll_allreduce (&total_bytes,
|
||||
&total_bytes_global,
|
||||
1,
|
||||
MPI_DOUBLE,
|
||||
MPI_MAX,
|
||||
fh->f_comm,
|
||||
fh->f_comm->c_coll.coll_allreduce_module);
|
||||
|
||||
bytes_received = (int *)malloc (two_phase_num_io_procs * sizeof (int));
|
||||
if (NULL == bytes_received) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
current = (int *)malloc (two_phase_num_io_procs * sizeof (int));
|
||||
if (NULL == current) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
previous = (int *)malloc (two_phase_num_io_procs * sizeof (int));
|
||||
if (NULL == previous) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
bytes_rem = (size_t *)malloc (two_phase_num_io_procs * sizeof (size_t));
|
||||
if (NULL == bytes_rem) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
prev_bytes_rem = (size_t *)malloc (two_phase_num_io_procs * sizeof (size_t));
|
||||
if (NULL == prev_bytes_rem) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
memset(current, 0x0, two_phase_num_io_procs*sizeof(int));
|
||||
memset(previous, 0x0, two_phase_num_io_procs*sizeof(int));
|
||||
memset(bytes_rem, 0x0, two_phase_num_io_procs*sizeof(size_t));
|
||||
memset(prev_bytes_rem, 0x0, two_phase_num_io_procs*sizeof(size_t));
|
||||
|
||||
cycles = ceil ((float)total_bytes_global/
|
||||
mca_fcoll_two_phase_cycle_buffer_size);
|
||||
|
||||
for (index = 0; index < cycles; index++) {
|
||||
int k = 0;
|
||||
size_t total_bytes_recv = 0;
|
||||
size_t temp = 0;
|
||||
global_count = 0;
|
||||
|
||||
memset(bytes_received, 0x0, two_phase_num_io_procs*sizeof(int));
|
||||
if (0 == fh->f_rank%fh->f_aggregator_index) {
|
||||
memset(displs, 0x0, fh->f_size*sizeof(int));
|
||||
memset(bytes_per_process, 0x0, fh->f_size*sizeof(int));
|
||||
|
||||
if ((int)total_bytes > mca_fcoll_two_phase_cycle_buffer_size) {
|
||||
bytes_to_read_in_cycle = mca_fcoll_two_phase_cycle_buffer_size;
|
||||
}
|
||||
else {
|
||||
bytes_to_read_in_cycle = total_bytes;
|
||||
}
|
||||
}
|
||||
/*
|
||||
printf ("****%d: Total_bytes: %d CYCLE %d Bytes %d OFFSET %d******\n",
|
||||
fh->f_rank,
|
||||
total_bytes,
|
||||
index,
|
||||
bytes_to_read_in_cycle,
|
||||
fh->f_offset);
|
||||
sleep(1);
|
||||
*/
|
||||
/**********************************************************
|
||||
**Gather the Data from all the processes at the readers **
|
||||
*********************************************************/
|
||||
|
||||
/* Calculate how much data will be contributed in this cycle
|
||||
by each process*/
|
||||
for (k=0 ; k<two_phase_num_io_procs ; k++) {
|
||||
previous[k] = current[k];
|
||||
prev_bytes_rem[k] = bytes_rem[k];
|
||||
temp = (k * stripe_size) +
|
||||
((size_t)mca_fcoll_two_phase_cycle_buffer_size * (index+1)) + fh->f_offset;
|
||||
|
||||
while (current[k] < broken_count) {
|
||||
if (k*stripe_size+fh->f_offset >
|
||||
(size_t)broken_iovec[current[k]].iov_base ||
|
||||
(k+1)*stripe_size+fh->f_offset <=
|
||||
(size_t)broken_iovec[current[k]].iov_base) {
|
||||
if ((k+1)*stripe_size+fh->f_offset <=
|
||||
(size_t)broken_iovec[current[k]].iov_base) {
|
||||
break;
|
||||
}
|
||||
current[k] ++;
|
||||
previous[k] = current[k];
|
||||
continue;
|
||||
}
|
||||
if (temp >=
|
||||
(size_t)((OPAL_PTRDIFF_TYPE)broken_iovec[current[k]].iov_base +
|
||||
broken_iovec[current[k]].iov_len)) {
|
||||
if (bytes_rem[k]) {
|
||||
bytes_received[k] += bytes_rem[k];
|
||||
total_bytes_recv += bytes_rem[k];
|
||||
bytes_rem[k] = 0;
|
||||
}
|
||||
else {
|
||||
bytes_received[k] += broken_iovec[current[k]].iov_len;
|
||||
total_bytes_recv += broken_iovec[current[k]].iov_len;
|
||||
}
|
||||
current[k] ++;
|
||||
}
|
||||
else {
|
||||
if (bytes_rem[k]) {
|
||||
bytes_received[k] += temp -
|
||||
((broken_iovec[current[k]].iov_len - bytes_rem[k]) +
|
||||
(OPAL_PTRDIFF_TYPE)broken_iovec[current[k]].iov_base);
|
||||
total_bytes_recv += temp -
|
||||
((broken_iovec[current[k]].iov_len - bytes_rem[k]) +
|
||||
(OPAL_PTRDIFF_TYPE)broken_iovec[current[k]].iov_base);
|
||||
bytes_rem[k] -= temp -
|
||||
((broken_iovec[current[k]].iov_len - bytes_rem[k]) +
|
||||
(OPAL_PTRDIFF_TYPE)broken_iovec[current[k]].iov_base);
|
||||
break;
|
||||
}
|
||||
else {
|
||||
if (temp > (size_t)broken_iovec[current[k]].iov_base) {
|
||||
bytes_received[k] += temp -
|
||||
(OPAL_PTRDIFF_TYPE)broken_iovec[current[k]].iov_base;
|
||||
total_bytes_recv += temp -
|
||||
(OPAL_PTRDIFF_TYPE)broken_iovec[current[k]].iov_base;
|
||||
bytes_rem[k] = broken_iovec[current[k]].iov_len -
|
||||
(temp -
|
||||
(OPAL_PTRDIFF_TYPE)broken_iovec[current[k]].iov_base);
|
||||
break;
|
||||
}
|
||||
else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
/*
|
||||
for (i=0 ; i<two_phase_num_io_procs ; i++) {
|
||||
printf ("%d sending %d to %d\n",fh->f_rank,
|
||||
bytes_received[i], i);
|
||||
}
|
||||
sleep(1);
|
||||
*/
|
||||
if (0 == fh->f_rank%fh->f_aggregator_index && bytes_to_read_in_cycle) {
|
||||
/* Calculate how much data will be recieved this cycle
|
||||
by each aggregator*/
|
||||
while (bytes_to_read_in_cycle) {
|
||||
blocks = fview_count[0];
|
||||
for (j=0 ; j<fh->f_size ; j++) {
|
||||
if (sorted[current_index] < blocks) {
|
||||
n = j;
|
||||
break;
|
||||
}
|
||||
else {
|
||||
blocks += fview_count[j+1];
|
||||
}
|
||||
}
|
||||
if (bytes_remaining) {
|
||||
if (bytes_remaining <= bytes_to_read_in_cycle) {
|
||||
bytes_per_process[n] += bytes_remaining;
|
||||
current_index ++;
|
||||
bytes_to_read_in_cycle -= bytes_remaining;
|
||||
bytes_remaining = 0;
|
||||
continue;
|
||||
}
|
||||
else {
|
||||
bytes_per_process[n] += bytes_to_read_in_cycle;
|
||||
bytes_remaining -= bytes_to_read_in_cycle;
|
||||
bytes_to_read_in_cycle = 0;
|
||||
break;
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (bytes_to_read_in_cycle <
|
||||
global_fview[sorted[current_index]].iov_len) {
|
||||
bytes_per_process[n] += bytes_to_read_in_cycle;
|
||||
bytes_remaining =
|
||||
global_fview[sorted[current_index]].iov_len -
|
||||
bytes_to_read_in_cycle;
|
||||
bytes_to_read_in_cycle = 0;
|
||||
break;
|
||||
}
|
||||
else {
|
||||
bytes_per_process[n] +=
|
||||
global_fview[sorted[current_index]].iov_len;
|
||||
bytes_to_read_in_cycle -=
|
||||
global_fview[sorted[current_index]].iov_len;
|
||||
current_index ++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
/*
|
||||
for (i=0 ; i<fh->f_size ; i++) {
|
||||
printf ("%d --> expecting %d from %d\n",fh->f_rank,
|
||||
bytes_per_process[i], i);
|
||||
}
|
||||
*/
|
||||
/* Calculate the displacement on where to put the data and allocate
|
||||
the recieve buffer (global_buf) */
|
||||
displs[0] = 0;
|
||||
global_count = bytes_per_process[0];
|
||||
for (i=1 ; i<fh->f_size ; i++) {
|
||||
global_count += bytes_per_process[i];
|
||||
displs[i] = displs[i-1] + bytes_per_process[i-1];
|
||||
}
|
||||
/*
|
||||
for (i=0 ; i<fh->f_size ; i++) {
|
||||
printf ("Proc %d sending %d at %d\n",
|
||||
i,
|
||||
bytes_per_process[i],
|
||||
displs[i]);
|
||||
}
|
||||
*/
|
||||
if (0 != global_count) {
|
||||
global_buf = malloc (global_count);
|
||||
if (NULL == global_buf) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
}
|
||||
/**********************************************************
|
||||
******* Create the io array, and pass it to fbtl *********
|
||||
*********************************************************/
|
||||
if (0 == fh->f_rank%fh->f_aggregator_index && global_count) {
|
||||
int bytes_to_read = global_count;
|
||||
int *temp = NULL;
|
||||
int block = 1;
|
||||
k = 0;
|
||||
|
||||
temp = (int *)malloc (sizeof(int) * fh->f_size);
|
||||
if (NULL == temp) {
|
||||
opal_output(1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
memset(temp, 0x0, fh->f_size*sizeof(int));
|
||||
fh->f_io_array = (mca_io_ompio_io_array_t *) malloc
|
||||
(OMPIO_IOVEC_INITIAL_SIZE * sizeof (mca_io_ompio_io_array_t));
|
||||
if (NULL == fh->f_io_array) {
|
||||
opal_output(1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
while (bytes_to_read) {
|
||||
int start = 0;
|
||||
|
||||
if (OMPIO_IOVEC_INITIAL_SIZE*block <= k) {
|
||||
block ++;
|
||||
fh->f_io_array = (mca_io_ompio_io_array_t *)realloc
|
||||
(fh->f_io_array, OMPIO_IOVEC_INITIAL_SIZE * block *
|
||||
sizeof(mca_io_ompio_io_array_t));
|
||||
if (NULL == fh->f_io_array) {
|
||||
opal_output(1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
|
||||
blocks = fview_count[0];
|
||||
for (j=0 ; j<fh->f_size ; j++) {
|
||||
if (sorted[x] < blocks) {
|
||||
n = j;
|
||||
break;
|
||||
}
|
||||
else {
|
||||
blocks += fview_count[j+1];
|
||||
}
|
||||
}
|
||||
for (j=0 ; j<n ; j++) {
|
||||
start += bytes_per_process[j];
|
||||
}
|
||||
|
||||
if (bytes_left) {
|
||||
if (bytes_left <= bytes_to_read) {
|
||||
fh->f_io_array[k].offset = (IOVBASE_TYPE *)
|
||||
((OPAL_PTRDIFF_TYPE)global_fview[sorted[x]].iov_base +
|
||||
(global_fview[sorted[x]].iov_len - bytes_left));
|
||||
fh->f_io_array[k].length = bytes_left;
|
||||
fh->f_io_array[k].memory_address =
|
||||
&global_buf[start+temp[n]];
|
||||
temp[n] += (int)fh->f_io_array[k].length;
|
||||
bytes_to_read -= bytes_left;
|
||||
bytes_left = 0;
|
||||
k ++;
|
||||
x ++;
|
||||
continue;
|
||||
}
|
||||
else {
|
||||
fh->f_io_array[k].offset = (IOVBASE_TYPE *)
|
||||
((OPAL_PTRDIFF_TYPE)global_fview[sorted[x]].iov_base +
|
||||
(global_fview[sorted[x]].iov_len - bytes_left));
|
||||
fh->f_io_array[k].length = bytes_to_read;
|
||||
fh->f_io_array[k].memory_address =
|
||||
&global_buf[start+temp[n]];
|
||||
temp[n] += (int)fh->f_io_array[k].length;
|
||||
bytes_left -= bytes_to_read;
|
||||
bytes_to_read = 0;;
|
||||
k ++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (bytes_to_read < (int)global_fview[sorted[x]].iov_len) {
|
||||
fh->f_io_array[k].offset = global_fview[sorted[x]].iov_base;
|
||||
fh->f_io_array[k].length = bytes_to_read;
|
||||
fh->f_io_array[k].memory_address =
|
||||
&global_buf[start+temp[n]];
|
||||
bytes_left =
|
||||
global_fview[sorted[x]].iov_len - bytes_to_read;
|
||||
bytes_to_read = 0;
|
||||
k ++;
|
||||
break;
|
||||
}
|
||||
else {
|
||||
fh->f_io_array[k].offset = global_fview[sorted[x]].iov_base;
|
||||
fh->f_io_array[k].length = global_fview[sorted[x]].iov_len;
|
||||
fh->f_io_array[k].memory_address =
|
||||
&global_buf[start+temp[n]];
|
||||
temp[n] += (int)fh->f_io_array[k].length;
|
||||
bytes_to_read -= global_fview[sorted[x]].iov_len;
|
||||
k ++;
|
||||
x ++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fh->f_num_of_io_entries = k;
|
||||
/*
|
||||
printf("*************************** %d\n", fh->f_num_of_io_entries);
|
||||
for (i=0 ; i<fh->f_num_of_io_entries ; i++) {
|
||||
printf(" ADDRESS: %p OFFSET: %p LENGTH: %d\n",
|
||||
fh->f_io_array[i].memory_address,
|
||||
fh->f_io_array[i].offset,
|
||||
fh->f_io_array[i].length);
|
||||
}
|
||||
*/
|
||||
if (fh->f_num_of_io_entries) {
|
||||
if (OMPI_SUCCESS != fh->f_fbtl->fbtl_preadv (fh, NULL)) {
|
||||
opal_output (1, "READ FAILED\n");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
}
|
||||
if (NULL != temp) {
|
||||
free (temp);
|
||||
temp = NULL;
|
||||
}
|
||||
}
|
||||
/**********************************************************
|
||||
******************** DONE READING ************************
|
||||
*********************************************************/
|
||||
|
||||
|
||||
/**********************************************************
|
||||
********* Scatter the Data from the readers **************
|
||||
*********************************************************/
|
||||
#if 0
|
||||
if (fh->f_flags & OMPIO_CONTIGUOUS_MEMORY) {
|
||||
receive_buf = &((char*)buf)[total_bytes_read];
|
||||
}
|
||||
else if (total_bytes_recv) {
|
||||
/* allocate a send buffer and copy the data that needs
|
||||
to be sent into it in case the data is non-contigous
|
||||
in memory */
|
||||
receive_buf = malloc (total_bytes_recv);
|
||||
if (NULL == receive_buf) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
/* distribute the data to its corresponding processes */
|
||||
ompi_io_ompio_receive_data (fh,
|
||||
buf,
|
||||
total_bytes_recv,
|
||||
decoded_iov,
|
||||
iov_count,
|
||||
bytes_received,
|
||||
broken_iovec,
|
||||
previous,
|
||||
prev_bytes_rem,
|
||||
global_buf,
|
||||
bytes_per_process,
|
||||
displs,
|
||||
two_phase_num_io_procs,
|
||||
stripe_size);
|
||||
|
||||
/*
|
||||
if (0 == fh->f_rank%fh->f_aggregator_index) {
|
||||
for (k=0 ; k<global_count/4 ; k++) {
|
||||
printf ("%d: RECV %d \n",fh->f_rank,
|
||||
((int *)global_buf)[k]);
|
||||
}
|
||||
}
|
||||
*/
|
||||
#if 0
|
||||
/* If data is not contigous in memory, copy the data from the
|
||||
receive buffer into the buffer passed in */
|
||||
if (!(fh->f_flags & OMPIO_CONTIGUOUS_MEMORY)) {
|
||||
OPAL_PTRDIFF_TYPE mem_address;
|
||||
size_t remaining = 0;
|
||||
size_t temp_position = 0;
|
||||
|
||||
remaining = total_bytes_recv;
|
||||
|
||||
while (remaining) {
|
||||
mem_address = (OPAL_PTRDIFF_TYPE)
|
||||
(decoded_iov[iov_index].iov_base) + current_position;
|
||||
|
||||
if (remaining >=
|
||||
(decoded_iov[iov_index].iov_len - current_position)) {
|
||||
memcpy ((IOVBASE_TYPE *) mem_address,
|
||||
receive_buf+temp_position,
|
||||
decoded_iov[iov_index].iov_len - current_position);
|
||||
remaining = remaining -
|
||||
(decoded_iov[iov_index].iov_len - current_position);
|
||||
temp_position = temp_position +
|
||||
(decoded_iov[iov_index].iov_len - current_position);
|
||||
iov_index = iov_index + 1;
|
||||
current_position = 0;
|
||||
}
|
||||
else {
|
||||
memcpy ((IOVBASE_TYPE *) mem_address,
|
||||
receive_buf+temp_position,
|
||||
remaining);
|
||||
current_position = current_position + remaining;
|
||||
remaining = 0;
|
||||
}
|
||||
}
|
||||
if (NULL != receive_buf) {
|
||||
free (receive_buf);
|
||||
receive_buf = NULL;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
total_bytes_read += total_bytes_recv;
|
||||
total_bytes -= global_count;
|
||||
|
||||
/**********************************************************
|
||||
**************** DONE GATHERING OF DATA ******************
|
||||
*********************************************************/
|
||||
|
||||
|
||||
|
||||
if (0 == fh->f_rank%fh->f_aggregator_index) {
|
||||
fh->f_num_of_io_entries = 0;
|
||||
if (NULL != fh->f_io_array) {
|
||||
free (fh->f_io_array);
|
||||
fh->f_io_array = NULL;
|
||||
}
|
||||
if (NULL != global_buf) {
|
||||
free (global_buf);
|
||||
global_buf = NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (NULL != sorted) {
|
||||
free (sorted);
|
||||
sorted = NULL;
|
||||
}
|
||||
if (NULL != broken_iovec) {
|
||||
free (broken_iovec);
|
||||
broken_iovec = NULL;
|
||||
}
|
||||
if (NULL != global_fview) {
|
||||
free (global_fview);
|
||||
global_fview = NULL;
|
||||
}
|
||||
if (NULL != fview_count) {
|
||||
free (fview_count);
|
||||
fview_count = NULL;
|
||||
}
|
||||
if (NULL != decoded_iov) {
|
||||
free (decoded_iov);
|
||||
decoded_iov = NULL;
|
||||
}
|
||||
if (NULL != bytes_per_process) {
|
||||
free (bytes_per_process);
|
||||
bytes_per_process = NULL;
|
||||
}
|
||||
if (NULL != bytes_received) {
|
||||
free (bytes_received);
|
||||
bytes_received = NULL;
|
||||
}
|
||||
if (NULL != displs) {
|
||||
free (displs);
|
||||
displs = NULL;
|
||||
}
|
||||
/*
|
||||
if (NULL != total_bytes_per_process) {
|
||||
free (total_bytes_per_process);
|
||||
total_bytes_per_process = NULL;
|
||||
}
|
||||
*/
|
||||
|
||||
fh->f_flags ^= OMPIO_AGGREGATOR_IS_SET;
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
36
ompi/mca/fcoll/two_phase/fcoll_two_phase_file_read_all_begin.c
Обычный файл
36
ompi/mca/fcoll/two_phase/fcoll_two_phase_file_read_all_begin.c
Обычный файл
@ -0,0 +1,36 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_two_phase.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
|
||||
|
||||
int
|
||||
mca_fcoll_two_phase_file_read_all_begin (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype)
|
||||
{
|
||||
printf ("DYNAMIC READ ALL BEGIN\n");
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
35
ompi/mca/fcoll/two_phase/fcoll_two_phase_file_read_all_end.c
Обычный файл
35
ompi/mca/fcoll/two_phase/fcoll_two_phase_file_read_all_end.c
Обычный файл
@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_two_phase.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
|
||||
|
||||
int
|
||||
mca_fcoll_two_phase_file_read_all_end (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
ompi_status_public_t *status)
|
||||
{
|
||||
printf ("DYNAMIC READ ALL END\n");
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
736
ompi/mca/fcoll/two_phase/fcoll_two_phase_file_write_all.c
Обычный файл
736
ompi/mca/fcoll/two_phase/fcoll_two_phase_file_write_all.c
Обычный файл
@ -0,0 +1,736 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_two_phase.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
#include "ompi/mca/io/ompio/io_ompio.h"
|
||||
#include "ompi/mca/io/io.h"
|
||||
#include "math.h"
|
||||
#include "ompi/mca/pml/pml.h"
|
||||
#include <unistd.h>
|
||||
|
||||
#define TIME_BREAKDOWN 0
|
||||
int
|
||||
mca_fcoll_two_phase_file_write_all (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype,
|
||||
ompi_status_public_t *status)
|
||||
{
|
||||
size_t total_bytes_written = 0; /* total bytes that have been written*/
|
||||
size_t total_bytes = 0; /* total bytes to be written */
|
||||
size_t total_bytes_global = 0;
|
||||
size_t bytes_to_write_in_cycle = 0; /* left to be written in a cycle*/
|
||||
size_t max_data = 0;
|
||||
size_t bytes_remaining = 0;
|
||||
size_t *bytes_rem = 0;
|
||||
size_t *prev_bytes_rem = 0;
|
||||
size_t stripe_size =0;
|
||||
size_t bytes_left = 0;
|
||||
|
||||
int index = 0;
|
||||
int current_index = 0;
|
||||
int *current = NULL;
|
||||
int *previous = NULL;
|
||||
int cycles = 0;
|
||||
int i=0, j=0, x=0, n=0;
|
||||
int blocks = 0;
|
||||
int two_phase_num_io_procs = 0;
|
||||
|
||||
/* array that contains the sorted indices of the global_iov */
|
||||
int *sorted = NULL;
|
||||
int *displs = NULL;
|
||||
int *bytes_per_process = NULL;
|
||||
int *bytes_sent = NULL;
|
||||
|
||||
/* iovec structure and count of the buffer passed in */
|
||||
uint32_t iov_count = 0;
|
||||
struct iovec *decoded_iov = NULL;
|
||||
|
||||
int global_fview_count = 0;
|
||||
struct iovec *global_fview = NULL;
|
||||
|
||||
int local_count = 0;
|
||||
struct iovec *iov = NULL;
|
||||
|
||||
int broken_count = 0;
|
||||
struct iovec *broken_iovec = NULL;
|
||||
|
||||
int *fview_count = NULL;
|
||||
|
||||
int global_count = 0;
|
||||
char *global_buf = NULL;
|
||||
|
||||
#if TIME_BREAKDOWN
|
||||
double start_time=0, end_time=0, start_time2=0, end_time2=0;
|
||||
double total=0 , total_io=0;
|
||||
#endif
|
||||
|
||||
if (opal_datatype_is_contiguous_memory_layout(&datatype->super,1)) {
|
||||
fh->f_flags = fh->f_flags | OMPIO_CONTIGUOUS_MEMORY;
|
||||
}
|
||||
|
||||
|
||||
/**************************************************************************
|
||||
** In case the data is not contigous in memory, decode it into an iovec **
|
||||
**************************************************************************/
|
||||
if (!(fh->f_flags & OMPIO_CONTIGUOUS_MEMORY)) {
|
||||
ompi_io_ompio_decode_datatype (fh,
|
||||
datatype,
|
||||
count,
|
||||
buf,
|
||||
&max_data,
|
||||
&decoded_iov,
|
||||
&iov_count);
|
||||
}
|
||||
else {
|
||||
max_data = count * datatype->super.size;
|
||||
}
|
||||
|
||||
if (! (fh->f_flags & OMPIO_AGGREGATOR_IS_SET)) {
|
||||
|
||||
if (-1 == mca_fcoll_two_phase_num_io_procs) {
|
||||
ompi_io_ompio_set_aggregator_props (fh,
|
||||
mca_fcoll_two_phase_num_io_procs,
|
||||
max_data);
|
||||
two_phase_num_io_procs =
|
||||
ceil((float)fh->f_size/fh->f_procs_per_group);
|
||||
|
||||
}
|
||||
fh->f_aggregator_index =
|
||||
ceil((float)fh->f_size/two_phase_num_io_procs);
|
||||
if (fh->f_aggregator_index * two_phase_num_io_procs > fh->f_size) {
|
||||
two_phase_num_io_procs =
|
||||
ceil((float)fh->f_size/fh->f_aggregator_index);
|
||||
}
|
||||
}
|
||||
|
||||
/* printf("two_phase_num_io_procs : %ld \n", two_phase_num_io_procs);*/
|
||||
|
||||
#if TIME_BREAKDOWN
|
||||
if (0 == fh->f_rank%fh->f_aggregator_index) {
|
||||
start_time = MPI_Wtime();
|
||||
}
|
||||
#endif
|
||||
/*********************************************************************
|
||||
*** Generate the File offsets/lengths corresponding to this write ***
|
||||
********************************************************************/
|
||||
ompi_io_ompio_generate_current_file_view (fh,
|
||||
max_data,
|
||||
&iov,
|
||||
&local_count);
|
||||
/*
|
||||
for (i=0 ; i<local_count ; i++) {
|
||||
printf("%d: OFFSET: %d LENGTH: %d\n",
|
||||
fh->f_rank,
|
||||
iov[i].iov_base,
|
||||
iov[i].iov_len);
|
||||
}
|
||||
*/
|
||||
/**************************************************************
|
||||
* Breakdown the file view at each process for each aggregator*
|
||||
* then send each portion of the file view to the corresp agg.*
|
||||
**************************************************************/
|
||||
|
||||
|
||||
|
||||
fh->f_comm->c_coll.coll_allreduce (&max_data,
|
||||
&total_bytes,
|
||||
1,
|
||||
MPI_DOUBLE,
|
||||
MPI_SUM,
|
||||
fh->f_comm,
|
||||
fh->f_comm->c_coll.coll_allreduce_module);
|
||||
|
||||
|
||||
stripe_size = ceil((float)total_bytes/two_phase_num_io_procs);
|
||||
|
||||
|
||||
|
||||
ompi_io_ompio_break_file_view (fh,
|
||||
iov,
|
||||
local_count,
|
||||
two_phase_num_io_procs,
|
||||
stripe_size,
|
||||
&broken_iovec,
|
||||
&broken_count);
|
||||
/*
|
||||
for (i=0 ; i<broken_count ; i++) {
|
||||
printf("%d: OFFSET: %d LENGTH: %d\n",
|
||||
fh->f_rank,
|
||||
broken_iovec[i].iov_base,
|
||||
broken_iovec[i].iov_len);
|
||||
}
|
||||
*/
|
||||
if (NULL != iov) {
|
||||
free (iov);
|
||||
iov = NULL;
|
||||
}
|
||||
|
||||
ompi_io_ompio_distribute_file_view (fh,
|
||||
broken_iovec,
|
||||
broken_count,
|
||||
two_phase_num_io_procs,
|
||||
stripe_size,
|
||||
&fview_count,
|
||||
&global_fview,
|
||||
&global_fview_count);
|
||||
/*
|
||||
if (0 == fh->f_rank%fh->f_aggregator_index) {
|
||||
for (i=0; i<fh->f_size ; i++) {
|
||||
printf("%d: fview_count[%d] = %d:\n",
|
||||
fh->f_rank,
|
||||
i,
|
||||
fview_count[i]);
|
||||
}
|
||||
}
|
||||
for (i=0 ; i<global_fview_count ; i++) {
|
||||
printf("%d: OFFSET: %d LENGTH: %d\n",
|
||||
fh->f_rank,
|
||||
global_fview[i].iov_base,
|
||||
global_fview[i].iov_len);
|
||||
}
|
||||
*/
|
||||
total_bytes = 0;
|
||||
if (0 == fh->f_rank%fh->f_aggregator_index) {
|
||||
if (global_fview_count) {
|
||||
for (i=0 ; i<global_fview_count ; i++) {
|
||||
total_bytes += global_fview[i].iov_len;
|
||||
}
|
||||
/* sort it */
|
||||
sorted = (int *)malloc (global_fview_count * sizeof(int));
|
||||
if (NULL == sorted) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
ompi_io_ompio_sort_iovec (global_fview, global_fview_count, sorted);
|
||||
}
|
||||
bytes_per_process = (int *) malloc (fh->f_size * sizeof (int));
|
||||
if (NULL == bytes_per_process) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
displs = (int *) malloc (fh->f_size * sizeof (int));
|
||||
if (NULL == displs) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
fh->f_comm->c_coll.coll_allreduce (&total_bytes,
|
||||
&total_bytes_global,
|
||||
1,
|
||||
MPI_DOUBLE,
|
||||
MPI_MAX,
|
||||
fh->f_comm,
|
||||
fh->f_comm->c_coll.coll_allreduce_module);
|
||||
|
||||
bytes_sent = (int *)malloc (two_phase_num_io_procs * sizeof (int));
|
||||
if (NULL == bytes_sent) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
current = (int *)malloc (two_phase_num_io_procs * sizeof (int));
|
||||
if (NULL == current) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
previous = (int *)malloc (two_phase_num_io_procs * sizeof (int));
|
||||
if (NULL == previous) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
bytes_rem = (size_t *)malloc (two_phase_num_io_procs * sizeof (size_t));
|
||||
if (NULL == bytes_rem) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
prev_bytes_rem = (size_t *)malloc (two_phase_num_io_procs * sizeof (size_t));
|
||||
if (NULL == prev_bytes_rem) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
memset(current, 0x0, two_phase_num_io_procs*sizeof(int));
|
||||
memset(previous, 0x0, two_phase_num_io_procs*sizeof(int));
|
||||
memset(bytes_rem, 0x0, two_phase_num_io_procs*sizeof(size_t));
|
||||
memset(prev_bytes_rem, 0x0, two_phase_num_io_procs*sizeof(size_t));
|
||||
|
||||
cycles = ceil ((float)total_bytes_global/
|
||||
mca_fcoll_two_phase_cycle_buffer_size);
|
||||
/*
|
||||
printf ("%d: Cycles: %d Total Bytes: %lld Global: %lld\n",
|
||||
fh->f_rank,
|
||||
cycles,
|
||||
total_bytes,
|
||||
total_bytes_global);
|
||||
*/
|
||||
#if TIME_BREAKDOWN
|
||||
if (0 == fh->f_rank%fh->f_aggregator_index) {
|
||||
end_time = MPI_Wtime();
|
||||
total = end_time-start_time;
|
||||
printf ("%d: Preprocessing --- %f\n", fh->f_rank, total);
|
||||
total = 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
for (index = 0; index < cycles; index++) {
|
||||
int k = 0;
|
||||
size_t total_bytes_sent = 0;
|
||||
size_t temp = 0;
|
||||
global_count = 0;
|
||||
|
||||
#if TIME_BREAKDOWN
|
||||
if (0 == fh->f_rank%fh->f_aggregator_index) {
|
||||
start_time = MPI_Wtime();
|
||||
}
|
||||
#endif
|
||||
|
||||
memset(bytes_sent, 0x0, two_phase_num_io_procs*sizeof(int));
|
||||
if (0 == fh->f_rank%fh->f_aggregator_index) {
|
||||
memset(displs, 0x0, fh->f_size*sizeof(int));
|
||||
memset(bytes_per_process, 0x0, fh->f_size*sizeof(int));
|
||||
|
||||
if (total_bytes > (size_t)mca_fcoll_two_phase_cycle_buffer_size) {
|
||||
bytes_to_write_in_cycle = mca_fcoll_two_phase_cycle_buffer_size;
|
||||
}
|
||||
else {
|
||||
bytes_to_write_in_cycle = total_bytes;
|
||||
}
|
||||
}
|
||||
/*
|
||||
printf ("****%d: Total_bytes: %lld CYCLE %d Bytes %lld OFFSET %d******\n",
|
||||
fh->f_rank,
|
||||
total_bytes,
|
||||
index,
|
||||
bytes_to_write_in_cycle,
|
||||
fh->f_offset);
|
||||
*/
|
||||
/**********************************************************
|
||||
**Gather the Data from all the processes at the writers **
|
||||
*********************************************************/
|
||||
/* Calculate how much data will be contributed in this cycle
|
||||
by each process*/
|
||||
for (k=0 ; k<two_phase_num_io_procs ; k++) {
|
||||
previous[k] = current[k];
|
||||
prev_bytes_rem[k] = bytes_rem[k];
|
||||
temp = (k * stripe_size) +
|
||||
((size_t)mca_fcoll_two_phase_cycle_buffer_size * (index+1)) +
|
||||
fh->f_offset;
|
||||
while (current[k] < broken_count) {
|
||||
if (k*stripe_size+fh->f_offset >
|
||||
(size_t)broken_iovec[current[k]].iov_base ||
|
||||
(k+1)*stripe_size+fh->f_offset <=
|
||||
(size_t)broken_iovec[current[k]].iov_base) {
|
||||
if ((k+1)*stripe_size+fh->f_offset <=
|
||||
(size_t)broken_iovec[current[k]].iov_base) {
|
||||
break;
|
||||
}
|
||||
current[k] ++;
|
||||
previous[k] = current[k];
|
||||
continue;
|
||||
}
|
||||
if (temp >=
|
||||
(size_t)((OPAL_PTRDIFF_TYPE)broken_iovec[current[k]].iov_base +
|
||||
broken_iovec[current[k]].iov_len)) {
|
||||
if (bytes_rem[k]) {
|
||||
bytes_sent[k] += bytes_rem[k];
|
||||
total_bytes_sent += bytes_rem[k];
|
||||
bytes_rem[k] = 0;
|
||||
}
|
||||
else {
|
||||
bytes_sent[k] += broken_iovec[current[k]].iov_len;
|
||||
total_bytes_sent += broken_iovec[current[k]].iov_len;
|
||||
}
|
||||
current[k] ++;
|
||||
}
|
||||
else {
|
||||
if (bytes_rem[k]) {
|
||||
bytes_sent[k] += temp -
|
||||
((broken_iovec[current[k]].iov_len - bytes_rem[k]) +
|
||||
(OPAL_PTRDIFF_TYPE)broken_iovec[current[k]].iov_base);
|
||||
total_bytes_sent += temp -
|
||||
((broken_iovec[current[k]].iov_len - bytes_rem[k]) +
|
||||
(OPAL_PTRDIFF_TYPE)broken_iovec[current[k]].iov_base);
|
||||
bytes_rem[k] -= temp -
|
||||
((broken_iovec[current[k]].iov_len - bytes_rem[k]) +
|
||||
(OPAL_PTRDIFF_TYPE)broken_iovec[current[k]].iov_base);
|
||||
break;
|
||||
}
|
||||
else {
|
||||
if (temp > (size_t)broken_iovec[current[k]].iov_base) {
|
||||
bytes_sent[k] += temp -
|
||||
(OPAL_PTRDIFF_TYPE)broken_iovec[current[k]].iov_base;
|
||||
total_bytes_sent += temp -
|
||||
(OPAL_PTRDIFF_TYPE)broken_iovec[current[k]].iov_base;
|
||||
bytes_rem[k] = broken_iovec[current[k]].iov_len -
|
||||
(temp -
|
||||
(OPAL_PTRDIFF_TYPE)broken_iovec[current[k]].iov_base);
|
||||
break;
|
||||
}
|
||||
else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
/*
|
||||
if (total_bytes_sent) {
|
||||
printf ("%d ---> %d\n", fh->f_rank, total_bytes_sent);
|
||||
}
|
||||
for (i=0 ; i<two_phase_num_io_procs ; i++) {
|
||||
if (bytes_sent[i]) {
|
||||
printf ("%d sending %d to %d\n",fh->f_rank,
|
||||
bytes_sent[i], i);
|
||||
}
|
||||
}
|
||||
*/
|
||||
if (0 == fh->f_rank%fh->f_aggregator_index && bytes_to_write_in_cycle) {
|
||||
/* Calculate how much data will be recieved this cycle
|
||||
by each aggregator*/
|
||||
while (bytes_to_write_in_cycle) {
|
||||
blocks = fview_count[0];
|
||||
for (j=0 ; j<fh->f_size ; j++) {
|
||||
if (sorted[current_index] < blocks) {
|
||||
n = j;
|
||||
break;
|
||||
}
|
||||
else {
|
||||
blocks += fview_count[j+1];
|
||||
}
|
||||
}
|
||||
if (bytes_remaining) {
|
||||
if (bytes_remaining <= bytes_to_write_in_cycle) {
|
||||
bytes_per_process[n] += bytes_remaining;
|
||||
current_index ++;
|
||||
bytes_to_write_in_cycle -= bytes_remaining;
|
||||
bytes_remaining = 0;
|
||||
continue;
|
||||
}
|
||||
else {
|
||||
bytes_per_process[n] += bytes_to_write_in_cycle;
|
||||
bytes_remaining -= bytes_to_write_in_cycle;
|
||||
bytes_to_write_in_cycle = 0;
|
||||
break;
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (bytes_to_write_in_cycle <
|
||||
global_fview[sorted[current_index]].iov_len) {
|
||||
bytes_per_process[n] += bytes_to_write_in_cycle;
|
||||
bytes_remaining =
|
||||
global_fview[sorted[current_index]].iov_len -
|
||||
bytes_to_write_in_cycle;
|
||||
bytes_to_write_in_cycle = 0;
|
||||
break;
|
||||
}
|
||||
else {
|
||||
bytes_per_process[n] +=
|
||||
global_fview[sorted[current_index]].iov_len;
|
||||
bytes_to_write_in_cycle -=
|
||||
global_fview[sorted[current_index]].iov_len;
|
||||
current_index ++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
/*
|
||||
for (i=0 ; i<fh->f_size ; i++) {
|
||||
if (bytes_per_process[i]) {
|
||||
printf ("%d --> expecting %d from %d\n",fh->f_rank,
|
||||
bytes_per_process[i], i);
|
||||
}
|
||||
}
|
||||
*/
|
||||
/* Calculate the displacement on where to put the data and allocate
|
||||
the recieve buffer (global_buf) */
|
||||
displs[0] = 0;
|
||||
global_count = bytes_per_process[0];
|
||||
for (i=1 ; i<fh->f_size ; i++) {
|
||||
global_count += bytes_per_process[i];
|
||||
displs[i] = displs[i-1] + bytes_per_process[i-1];
|
||||
}
|
||||
|
||||
if (0 != global_count) {
|
||||
global_buf = malloc (global_count);
|
||||
if (NULL == global_buf) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* Send the data to the corresponding aggregator */
|
||||
if ( OMPI_SUCCESS != ompi_io_ompio_send_data (fh,
|
||||
buf,
|
||||
total_bytes_sent,
|
||||
decoded_iov,
|
||||
iov_count,
|
||||
bytes_sent,
|
||||
broken_iovec,
|
||||
previous,
|
||||
prev_bytes_rem,
|
||||
global_buf,
|
||||
bytes_per_process,
|
||||
displs,
|
||||
two_phase_num_io_procs,
|
||||
stripe_size)) {
|
||||
opal_output (1, "ERROR IN SENDING DATA\n");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
/*
|
||||
if (0 == fh->f_rank%fh->f_aggregator_index) {
|
||||
for (k=0 ; k<global_count/4 ; k++) {
|
||||
printf ("%d: RECV %d \n",fh->f_rank,
|
||||
((int *)global_buf)[k]);
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
total_bytes_written += total_bytes_sent;
|
||||
total_bytes -= global_count;
|
||||
|
||||
/**********************************************************
|
||||
**************** DONE GATHERING OF DATA ******************
|
||||
*********************************************************/
|
||||
|
||||
/**********************************************************
|
||||
******* Create the io array, and pass it to fbtl *********
|
||||
*********************************************************/
|
||||
if (0 == fh->f_rank%fh->f_aggregator_index && global_count) {
|
||||
size_t bytes_to_write = global_count;
|
||||
int *temp = NULL;
|
||||
int block = 1;
|
||||
k = 0;
|
||||
|
||||
temp = (int *)malloc (sizeof(int) * fh->f_size);
|
||||
if (NULL == temp) {
|
||||
opal_output(1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
memset(temp, 0x0, fh->f_size*sizeof(int));
|
||||
fh->f_io_array = (mca_io_ompio_io_array_t *) malloc
|
||||
(OMPIO_IOVEC_INITIAL_SIZE * sizeof (mca_io_ompio_io_array_t));
|
||||
if (NULL == fh->f_io_array) {
|
||||
opal_output(1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
while (bytes_to_write) {
|
||||
int start = 0;
|
||||
|
||||
if (OMPIO_IOVEC_INITIAL_SIZE*block <= k) {
|
||||
block ++;
|
||||
fh->f_io_array = (mca_io_ompio_io_array_t *)realloc
|
||||
(fh->f_io_array, OMPIO_IOVEC_INITIAL_SIZE * block *
|
||||
sizeof(mca_io_ompio_io_array_t));
|
||||
if (NULL == fh->f_io_array) {
|
||||
opal_output(1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
|
||||
blocks = fview_count[0];
|
||||
for (j=0 ; j<fh->f_size ; j++) {
|
||||
if (sorted[x] < blocks) {
|
||||
n = j;
|
||||
break;
|
||||
}
|
||||
else {
|
||||
blocks += fview_count[j+1];
|
||||
}
|
||||
}
|
||||
for (j=0 ; j<n ; j++) {
|
||||
start += bytes_per_process[j];
|
||||
}
|
||||
|
||||
if (bytes_left) {
|
||||
if (bytes_left <= bytes_to_write) {
|
||||
fh->f_io_array[k].offset = (IOVBASE_TYPE *)
|
||||
((OPAL_PTRDIFF_TYPE)global_fview[sorted[x]].iov_base +
|
||||
(global_fview[sorted[x]].iov_len - bytes_left));
|
||||
fh->f_io_array[k].length = bytes_left;
|
||||
fh->f_io_array[k].memory_address =
|
||||
&global_buf[start+temp[n]];
|
||||
temp[n] += (int)fh->f_io_array[k].length;
|
||||
bytes_to_write -= bytes_left;
|
||||
bytes_left = 0;
|
||||
k ++;
|
||||
x ++;
|
||||
continue;
|
||||
}
|
||||
else {
|
||||
fh->f_io_array[k].offset = (IOVBASE_TYPE *)
|
||||
((OPAL_PTRDIFF_TYPE)global_fview[sorted[x]].iov_base +
|
||||
(global_fview[sorted[x]].iov_len - bytes_left));
|
||||
fh->f_io_array[k].length = bytes_to_write;
|
||||
fh->f_io_array[k].memory_address =
|
||||
&global_buf[start+temp[n]];
|
||||
temp[n] += (int)fh->f_io_array[k].length;
|
||||
bytes_left -= bytes_to_write;
|
||||
bytes_to_write = 0;;
|
||||
k ++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (bytes_to_write < global_fview[sorted[x]].iov_len) {
|
||||
fh->f_io_array[k].offset = global_fview[sorted[x]].iov_base;
|
||||
fh->f_io_array[k].length = bytes_to_write;
|
||||
fh->f_io_array[k].memory_address =
|
||||
&global_buf[start+temp[n]];
|
||||
bytes_left =
|
||||
global_fview[sorted[x]].iov_len - bytes_to_write;
|
||||
bytes_to_write = 0;
|
||||
k ++;
|
||||
break;
|
||||
}
|
||||
else {
|
||||
fh->f_io_array[k].offset = global_fview[sorted[x]].iov_base;
|
||||
fh->f_io_array[k].length = global_fview[sorted[x]].iov_len;
|
||||
fh->f_io_array[k].memory_address =
|
||||
&global_buf[start+temp[n]];
|
||||
temp[n] += (int)fh->f_io_array[k].length;
|
||||
bytes_to_write -= global_fview[sorted[x]].iov_len;
|
||||
k ++;
|
||||
x ++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fh->f_num_of_io_entries = k;
|
||||
/*
|
||||
printf("%d: *************************** %d\n", fh->f_rank, fh->f_num_of_io_entries);
|
||||
for (i=0 ; i<fh->f_num_of_io_entries ; i++) {
|
||||
printf(" ADDRESS: %p OFFSET: %d LENGTH: %d\n",
|
||||
fh->f_io_array[i].memory_address,
|
||||
fh->f_io_array[i].offset,
|
||||
fh->f_io_array[i].length);
|
||||
}
|
||||
*/
|
||||
#if TIME_BREAKDOWN
|
||||
if (0 == fh->f_rank%fh->f_aggregator_index) {
|
||||
start_time2 = MPI_Wtime();
|
||||
}
|
||||
#endif
|
||||
if (fh->f_num_of_io_entries) {
|
||||
if (OMPI_SUCCESS != fh->f_fbtl->fbtl_pwritev (fh, NULL)) {
|
||||
opal_output (1, "WRITE FAILED\n");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
}
|
||||
#if TIME_BREAKDOWN
|
||||
if (0 == fh->f_rank%fh->f_aggregator_index) {
|
||||
end_time2 = MPI_Wtime();
|
||||
total_io += end_time2-start_time2;
|
||||
}
|
||||
#endif
|
||||
if (NULL != temp) {
|
||||
free (temp);
|
||||
temp = NULL;
|
||||
}
|
||||
}
|
||||
/**********************************************************
|
||||
******************** DONE WRITING ************************
|
||||
*********************************************************/
|
||||
|
||||
if (0 == fh->f_rank%fh->f_aggregator_index) {
|
||||
fh->f_num_of_io_entries = 0;
|
||||
if (NULL != fh->f_io_array) {
|
||||
free (fh->f_io_array);
|
||||
fh->f_io_array = NULL;
|
||||
}
|
||||
if (NULL != global_buf) {
|
||||
free (global_buf);
|
||||
global_buf = NULL;
|
||||
}
|
||||
}
|
||||
#if TIME_BREAKDOWN
|
||||
if (0 == fh->f_rank%fh->f_aggregator_index) {
|
||||
end_time = MPI_Wtime();
|
||||
total += end_time-start_time;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
#if TIME_BREAKDOWN
|
||||
if (0 == fh->f_rank%fh->f_aggregator_index) {
|
||||
printf ("%d: Total --- %f I/O ---- %f\n", fh->f_rank, total, total_io);
|
||||
}
|
||||
#endif
|
||||
|
||||
if (NULL != sorted) {
|
||||
free (sorted);
|
||||
sorted = NULL;
|
||||
}
|
||||
if (NULL != broken_iovec) {
|
||||
free (broken_iovec);
|
||||
broken_iovec = NULL;
|
||||
}
|
||||
if (NULL != global_fview) {
|
||||
free (global_fview);
|
||||
global_fview = NULL;
|
||||
}
|
||||
if (NULL != fview_count) {
|
||||
free (fview_count);
|
||||
fview_count = NULL;
|
||||
}
|
||||
if (NULL != decoded_iov) {
|
||||
free (decoded_iov);
|
||||
decoded_iov = NULL;
|
||||
}
|
||||
if (NULL != bytes_per_process) {
|
||||
free (bytes_per_process);
|
||||
bytes_per_process = NULL;
|
||||
}
|
||||
if (NULL != bytes_sent) {
|
||||
free (bytes_sent);
|
||||
bytes_sent = NULL;
|
||||
}
|
||||
if (NULL != current) {
|
||||
free (current);
|
||||
current = NULL;
|
||||
}
|
||||
if (NULL != previous) {
|
||||
free (previous);
|
||||
previous = NULL;
|
||||
}
|
||||
if (NULL != bytes_rem) {
|
||||
free (bytes_rem);
|
||||
bytes_rem = NULL;
|
||||
}
|
||||
if (NULL != prev_bytes_rem) {
|
||||
free (prev_bytes_rem);
|
||||
prev_bytes_rem = NULL;
|
||||
}
|
||||
if (NULL != displs) {
|
||||
free (displs);
|
||||
displs = NULL;
|
||||
}
|
||||
|
||||
fh->f_flags ^= OMPIO_AGGREGATOR_IS_SET;
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
36
ompi/mca/fcoll/two_phase/fcoll_two_phase_file_write_all_begin.c
Обычный файл
36
ompi/mca/fcoll/two_phase/fcoll_two_phase_file_write_all_begin.c
Обычный файл
@ -0,0 +1,36 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_two_phase.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
|
||||
|
||||
int
|
||||
mca_fcoll_two_phase_file_write_all_begin (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype)
|
||||
{
|
||||
printf ("DYNAMIC WRITE ALL BEGIN\n");
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
35
ompi/mca/fcoll/two_phase/fcoll_two_phase_file_write_all_end.c
Обычный файл
35
ompi/mca/fcoll/two_phase/fcoll_two_phase_file_write_all_end.c
Обычный файл
@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_two_phase.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
|
||||
|
||||
int
|
||||
mca_fcoll_two_phase_file_write_all_end (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
ompi_status_public_t *status)
|
||||
{
|
||||
printf ("DYNAMIC WRITE ALL END\n");
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
91
ompi/mca/fcoll/two_phase/fcoll_two_phase_module.c
Обычный файл
91
ompi/mca/fcoll/two_phase/fcoll_two_phase_module.c
Обычный файл
@ -0,0 +1,91 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2006 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_two_phase.h"
|
||||
|
||||
#include <stdio.h>
|
||||
|
||||
#include "mpi.h"
|
||||
#include "opal/mca/base/mca_base_param.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
#include "ompi/mca/fcoll/base/base.h"
|
||||
|
||||
|
||||
/*
|
||||
* *******************************************************************
|
||||
* ************************ actions structure ************************
|
||||
* *******************************************************************
|
||||
*/
|
||||
static mca_fcoll_base_module_1_0_0_t two_phase = {
|
||||
mca_fcoll_two_phase_module_init,
|
||||
mca_fcoll_two_phase_module_finalize,
|
||||
mca_fcoll_two_phase_file_read_all,
|
||||
mca_fcoll_two_phase_file_read_all_begin,
|
||||
mca_fcoll_two_phase_file_read_all_end,
|
||||
mca_fcoll_two_phase_file_write_all,
|
||||
mca_fcoll_two_phase_file_write_all_begin,
|
||||
mca_fcoll_two_phase_file_write_all_end
|
||||
};
|
||||
|
||||
int
|
||||
mca_fcoll_two_phase_component_init_query(bool enable_progress_threads,
|
||||
bool enable_mpi_threads)
|
||||
{
|
||||
/* Nothing to do */
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
mca_fcoll_base_module_1_0_0_t *
|
||||
mca_fcoll_two_phase_component_file_query (mca_io_ompio_file_t *fh, int *priority)
|
||||
{
|
||||
*priority = mca_fcoll_two_phase_priority;
|
||||
if (0 >= mca_fcoll_two_phase_priority) {
|
||||
return NULL;
|
||||
}
|
||||
|
||||
if (mca_fcoll_base_query_table (fh, "two_phase")) {
|
||||
if (*priority < 50) {
|
||||
*priority = 50;
|
||||
}
|
||||
}
|
||||
|
||||
return &two_phase;
|
||||
}
|
||||
|
||||
int mca_fcoll_two_phase_component_file_unquery (mca_io_ompio_file_t *file)
|
||||
{
|
||||
/* This function might be needed for some purposes later. for now it
|
||||
* does not have anything to do since there are no steps which need
|
||||
* to be undone if this module is not selected */
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
int mca_fcoll_two_phase_module_init (mca_io_ompio_file_t *file)
|
||||
{
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
int mca_fcoll_two_phase_module_finalize (mca_io_ompio_file_t *file)
|
||||
{
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
54
ompi/mca/fcoll/ylib/Makefile.am
Обычный файл
54
ompi/mca/fcoll/ylib/Makefile.am
Обычный файл
@ -0,0 +1,54 @@
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
# University Research and Technology
|
||||
# Corporation. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
# of Tennessee Research Foundation. All rights
|
||||
# reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
sources = \
|
||||
fcoll_ylib.h \
|
||||
fcoll_ylib_module.c \
|
||||
fcoll_ylib_component.c \
|
||||
fcoll_ylib_file_read_all.c \
|
||||
fcoll_ylib_file_read_all_begin.c \
|
||||
fcoll_ylib_file_read_all_end.c \
|
||||
fcoll_ylib_file_write_all.c \
|
||||
fcoll_ylib_file_write_all_begin.c \
|
||||
fcoll_ylib_file_write_all_end.c
|
||||
|
||||
# Make the output library in this directory, and name it either
|
||||
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
|
||||
# (for static builds).
|
||||
|
||||
if MCA_BUILD_ompi_fcoll_ylib_DSO
|
||||
component_noinst =
|
||||
component_install = mca_fcoll_ylib.la
|
||||
else
|
||||
component_noinst = libmca_fcoll_ylib.la
|
||||
component_install =
|
||||
endif
|
||||
|
||||
mcacomponentdir = $(libdir)/openmpi
|
||||
mcacomponent_LTLIBRARIES = $(component_install)
|
||||
mca_fcoll_ylib_la_SOURCES = $(sources)
|
||||
mca_fcoll_ylib_la_LDFLAGS = -module -avoid-version
|
||||
mca_fcoll_ylib_la_LIBADD = \
|
||||
$(top_ompi_builddir)/ompi/libmpi.la \
|
||||
$(top_ompi_builddir)/orte/libopen-rte.la \
|
||||
$(top_ompi_builddir)/opal/libopen-pal.la
|
||||
|
||||
noinst_LTLIBRARIES = $(component_noinst)
|
||||
libmca_fcoll_ylib_la_SOURCES =$(sources)
|
||||
libmca_fcoll_ylib_la_LDFLAGS = -module -avoid-version
|
||||
86
ompi/mca/fcoll/ylib/fcoll_ylib.h
Обычный файл
86
ompi/mca/fcoll/ylib/fcoll_ylib.h
Обычный файл
@ -0,0 +1,86 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2006 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#ifndef MCA_FCOLL_YLIB_EXPORT_H
|
||||
#define MCA_FCOLL_YLIB_EXPORT_H
|
||||
|
||||
#include "ompi_config.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "opal/mca/mca.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
#include "ompi/mca/fcoll/base/base.h"
|
||||
#include "ompi/mca/io/ompio/io_ompio.h"
|
||||
|
||||
BEGIN_C_DECLS
|
||||
|
||||
/* Globally exported variables */
|
||||
|
||||
extern int mca_fcoll_ylib_priority;
|
||||
extern int mca_fcoll_ylib_num_io_procs;
|
||||
extern int mca_fcoll_ylib_stripe_size;
|
||||
extern int mca_fcoll_ylib_blocks_per_cycle;
|
||||
|
||||
OMPI_MODULE_DECLSPEC extern mca_fcoll_base_component_2_0_0_t mca_fcoll_ylib_component;
|
||||
|
||||
/* API functions */
|
||||
|
||||
int mca_fcoll_ylib_component_init_query(bool enable_progress_threads,
|
||||
bool enable_mpi_threads);
|
||||
struct mca_fcoll_base_module_1_0_0_t *
|
||||
mca_fcoll_ylib_component_file_query (mca_io_ompio_file_t *fh, int *priority);
|
||||
|
||||
int mca_fcoll_ylib_component_file_unquery (mca_io_ompio_file_t *file);
|
||||
|
||||
int mca_fcoll_ylib_module_init (mca_io_ompio_file_t *file);
|
||||
int mca_fcoll_ylib_module_finalize (mca_io_ompio_file_t *file);
|
||||
|
||||
int mca_fcoll_ylib_file_read_all (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype,
|
||||
ompi_status_public_t * status);
|
||||
|
||||
int mca_fcoll_ylib_file_read_all_begin (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype);
|
||||
|
||||
int mca_fcoll_ylib_file_read_all_end (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
ompi_status_public_t * status);
|
||||
|
||||
int mca_fcoll_ylib_file_write_all (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype,
|
||||
ompi_status_public_t * status);
|
||||
|
||||
int mca_fcoll_ylib_file_write_all_begin (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype);
|
||||
|
||||
int mca_fcoll_ylib_file_write_all_end (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
ompi_status_public_t * status);
|
||||
|
||||
END_C_DECLS
|
||||
|
||||
#endif /* MCA_FCOLL_YLIB_EXPORT_H */
|
||||
132
ompi/mca/fcoll/ylib/fcoll_ylib_component.c
Обычный файл
132
ompi/mca/fcoll/ylib/fcoll_ylib_component.c
Обычный файл
@ -0,0 +1,132 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008 Cisco Systems, Inc. All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*
|
||||
* These symbols are in a file by themselves to provide nice linker
|
||||
* semantics. Since linkers generally pull in symbols by object
|
||||
* files, keeping these symbols as the only symbols in this file
|
||||
* prevents utility programs such as "ompi_info" from having to import
|
||||
* entire components just to query their version and parameters.
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_ylib.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
#include "fcoll_ylib.h"
|
||||
|
||||
/*
|
||||
* Public string showing the fcoll ompi_ylib component version number
|
||||
*/
|
||||
const char *mca_fcoll_ylib_component_version_string =
|
||||
"Open MPI ylib collective MCA component version " OMPI_VERSION;
|
||||
|
||||
/*
|
||||
* Global variables
|
||||
*/
|
||||
int mca_fcoll_ylib_priority = 0;
|
||||
int mca_fcoll_ylib_num_io_procs = 1;
|
||||
int mca_fcoll_ylib_stripe_size = 1048576;
|
||||
int mca_fcoll_ylib_blocks_per_cycle = 20;
|
||||
|
||||
/*
|
||||
* Local function
|
||||
*/
|
||||
static int ylib_register(void);
|
||||
|
||||
/*
|
||||
* Instantiate the public struct with all of our public information
|
||||
* and pointers to our public functions in it
|
||||
*/
|
||||
mca_fcoll_base_component_2_0_0_t mca_fcoll_ylib_component = {
|
||||
|
||||
/* First, the mca_component_t struct containing meta information
|
||||
* about the component itself */
|
||||
|
||||
{
|
||||
MCA_FCOLL_BASE_VERSION_2_0_0,
|
||||
|
||||
/* Component name and version */
|
||||
"ylib",
|
||||
OMPI_MAJOR_VERSION,
|
||||
OMPI_MINOR_VERSION,
|
||||
OMPI_RELEASE_VERSION,
|
||||
ylib_register,
|
||||
NULL
|
||||
},
|
||||
{
|
||||
/* The component is checkpoint ready */
|
||||
MCA_BASE_METADATA_PARAM_CHECKPOINT
|
||||
},
|
||||
|
||||
mca_fcoll_ylib_component_init_query,
|
||||
mca_fcoll_ylib_component_file_query,
|
||||
mca_fcoll_ylib_component_file_unquery
|
||||
};
|
||||
|
||||
|
||||
static int
|
||||
ylib_register(void)
|
||||
{
|
||||
int param;
|
||||
|
||||
param = mca_base_param_find ("fcoll", NULL, "ylib_priority");
|
||||
if (param >= 0)
|
||||
{
|
||||
mca_base_param_lookup_int (param, &mca_fcoll_ylib_priority);
|
||||
}
|
||||
param = mca_base_param_find ("fcoll", NULL, "ylib_num_io_procs");
|
||||
if (param >= 0)
|
||||
{
|
||||
mca_base_param_lookup_int (param, &mca_fcoll_ylib_num_io_procs);
|
||||
}
|
||||
param = mca_base_param_find ("fcoll", NULL, "ylib_stripe_size");
|
||||
if (param >= 0)
|
||||
{
|
||||
mca_base_param_lookup_int (param, &mca_fcoll_ylib_stripe_size);
|
||||
}
|
||||
param = mca_base_param_find ("fcoll", NULL, "ylib_blocks_per_cycle");
|
||||
if (param >= 0)
|
||||
{
|
||||
mca_base_param_lookup_int (param, &mca_fcoll_ylib_blocks_per_cycle);
|
||||
}
|
||||
|
||||
mca_base_param_reg_int (&mca_fcoll_ylib_component.fcollm_version,
|
||||
"priority",
|
||||
"Priority of the ylib fcoll component",
|
||||
false, false, mca_fcoll_ylib_priority,
|
||||
&mca_fcoll_ylib_priority);
|
||||
mca_base_param_reg_int (&mca_fcoll_ylib_component.fcollm_version,
|
||||
"num_io_procs",
|
||||
"Number of writers in the ylib fcoll component",
|
||||
false, false, mca_fcoll_ylib_num_io_procs,
|
||||
&mca_fcoll_ylib_num_io_procs);
|
||||
mca_base_param_reg_int (&mca_fcoll_ylib_component.fcollm_version,
|
||||
"stripe_size",
|
||||
"Stripe Size of the ylib fcoll component",
|
||||
false, false, mca_fcoll_ylib_stripe_size,
|
||||
&mca_fcoll_ylib_stripe_size);
|
||||
mca_base_param_reg_int (&mca_fcoll_ylib_component.fcollm_version,
|
||||
"blocks_per_cycle",
|
||||
"Blocks to write per cycle of the ylib fcoll component",
|
||||
false, false, mca_fcoll_ylib_blocks_per_cycle,
|
||||
&mca_fcoll_ylib_blocks_per_cycle);
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
662
ompi/mca/fcoll/ylib/fcoll_ylib_file_read_all.c
Обычный файл
662
ompi/mca/fcoll/ylib/fcoll_ylib_file_read_all.c
Обычный файл
@ -0,0 +1,662 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
|
||||
* University Research and Technology
|
||||
* Corporation. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The University of Tennessee and The University
|
||||
* of Tennessee Research Foundation. All rights
|
||||
* reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2008-2011 University of Houston. All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "fcoll_ylib.h"
|
||||
|
||||
#include "mpi.h"
|
||||
#include "ompi/constants.h"
|
||||
#include "ompi/mca/fcoll/fcoll.h"
|
||||
#include "ompi/mca/io/ompio/io_ompio.h"
|
||||
#include "ompi/mca/io/io.h"
|
||||
#include "math.h"
|
||||
#include "ompi/mca/pml/pml.h"
|
||||
#include <unistd.h>
|
||||
|
||||
int
|
||||
mca_fcoll_ylib_file_read_all (mca_io_ompio_file_t *fh,
|
||||
void *buf,
|
||||
int count,
|
||||
struct ompi_datatype_t *datatype,
|
||||
ompi_status_public_t *status)
|
||||
{
|
||||
size_t total_bytes_read = 0; /* total bytes that have been read*/
|
||||
size_t total_bytes = 0; /* total bytes to be read */
|
||||
size_t total_bytes_global = 0;
|
||||
size_t bytes_per_cycle = 0; /* total read in each cycle by each process*/
|
||||
size_t bytes_to_read_in_cycle = 0; /* left to be read in a cycle*/
|
||||
size_t current_position = 0;
|
||||
size_t max_data = 0;
|
||||
size_t bytes_remaining = 0;
|
||||
size_t bytes_rem = 0;
|
||||
size_t prev_bytes_rem = 0;
|
||||
|
||||
int index = 0;
|
||||
int current_index = 0;
|
||||
int current = 0;
|
||||
int previous = 0;
|
||||
int cycles = 0;
|
||||
int i=0, j=0, x=0, n=0;
|
||||
int blocks = 0;
|
||||
int bytes_left = 0;
|
||||
|
||||
/* array that contains the sorted indices of the global_iov */
|
||||
int *sorted = NULL;
|
||||
int *displs = NULL;
|
||||
int *bytes_per_process = NULL;
|
||||
int *bytes_received = NULL;
|
||||
|
||||
/* iovec structure and count of the buffer passed in */
|
||||
uint32_t iov_count = 0;
|
||||
struct iovec *decoded_iov = NULL;
|
||||
int iov_index = 0;
|
||||
|
||||
char *receive_buf = NULL;
|
||||
|
||||
int global_fview_count = 0;
|
||||
struct iovec *global_fview = NULL;
|
||||
|
||||
int local_count = 0;
|
||||
struct iovec *iov = NULL;
|
||||
|
||||
int broken_count = 0;
|
||||
struct iovec *broken_iovec = NULL;
|
||||
|
||||
int *fview_count = NULL;
|
||||
|
||||
int global_count = 0;
|
||||
char *global_buf = NULL;
|
||||
|
||||
if (opal_datatype_is_contiguous_memory_layout(&datatype->super,1)) {
|
||||
fh->f_flags |= OMPIO_CONTIGUOUS_MEMORY;
|
||||
}
|
||||
|
||||
/**************************************************************************
|
||||
** In case the data is not contigous in memory, decode it into an iovec **
|
||||
**************************************************************************/
|
||||
if (! (fh->f_flags & OMPIO_CONTIGUOUS_MEMORY)) {
|
||||
ompi_io_ompio_decode_datatype (fh,
|
||||
datatype,
|
||||
count,
|
||||
buf,
|
||||
&max_data,
|
||||
&decoded_iov,
|
||||
&iov_count);
|
||||
}
|
||||
else {
|
||||
max_data = count * datatype->super.size;
|
||||
}
|
||||
|
||||
if (! (fh->f_flags & OMPIO_AGGREGATOR_IS_SET)) {
|
||||
ompi_io_ompio_set_aggregator_props (fh,
|
||||
mca_fcoll_ylib_num_io_procs,
|
||||
max_data);
|
||||
mca_fcoll_ylib_num_io_procs =
|
||||
ceil((float)fh->f_size/fh->f_procs_per_group);
|
||||
fh->f_aggregator_index =
|
||||
ceil((float)fh->f_size/mca_fcoll_ylib_num_io_procs);
|
||||
}
|
||||
|
||||
/*********************************************************************
|
||||
*** Generate the File offsets/lengths corresponding to this read ***
|
||||
********************************************************************/
|
||||
ompi_io_ompio_generate_current_file_view (fh,
|
||||
max_data,
|
||||
&iov,
|
||||
&local_count);
|
||||
/*
|
||||
for (i=0 ; i<local_count ; i++) {
|
||||
printf("%d: OFFSET: %d LENGTH: %d\n",
|
||||
fh->f_rank,
|
||||
iov[i].iov_base,
|
||||
iov[i].iov_len);
|
||||
}
|
||||
*/
|
||||
|
||||
/*************************************************************
|
||||
* Breakdown the file view at each process per OST then send *
|
||||
* each portion of the file view t0 the corresp aggregator *
|
||||
*************************************************************/
|
||||
ompi_io_ompio_break_file_view (fh,
|
||||
iov,
|
||||
local_count,
|
||||
mca_fcoll_ylib_num_io_procs,
|
||||
mca_fcoll_ylib_stripe_size,
|
||||
&broken_iovec,
|
||||
&broken_count);
|
||||
/*
|
||||
for (i=0 ; i<broken_count ; i++) {
|
||||
printf("%d: OFFSET: %d LENGTH: %d\n",
|
||||
fh->f_rank,
|
||||
broken_iovec[i].iov_base,
|
||||
broken_iovec[i].iov_len);
|
||||
}
|
||||
*/
|
||||
|
||||
if (NULL != iov) {
|
||||
free (iov);
|
||||
iov = NULL;
|
||||
}
|
||||
|
||||
ompi_io_ompio_distribute_file_view (fh,
|
||||
broken_iovec,
|
||||
broken_count,
|
||||
mca_fcoll_ylib_num_io_procs,
|
||||
mca_fcoll_ylib_stripe_size,
|
||||
&fview_count,
|
||||
&global_fview,
|
||||
&global_fview_count);
|
||||
/*
|
||||
for (i=0 ; i<global_fview_count ; i++) {
|
||||
printf("%d: OFFSET: %d LENGTH: %d\n",
|
||||
fh->f_rank,
|
||||
global_fview[i].iov_base,
|
||||
global_fview[i].iov_len);
|
||||
}
|
||||
*/
|
||||
if (0 == fh->f_rank%fh->f_aggregator_index) {
|
||||
if (global_fview_count) {
|
||||
for (i=0 ; i<global_fview_count ; i++) {
|
||||
total_bytes += global_fview[i].iov_len;
|
||||
}
|
||||
/* sort it */
|
||||
sorted = (int *)malloc (global_fview_count * sizeof(int));
|
||||
if (NULL == sorted) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
ompi_io_ompio_sort_iovec (global_fview, global_fview_count, sorted);
|
||||
}
|
||||
bytes_per_process = (int *) malloc (fh->f_size * sizeof (int));
|
||||
if (NULL == bytes_per_process) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
displs = (int *) malloc (fh->f_size * sizeof (int));
|
||||
if (NULL == displs) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
fh->f_comm->c_coll.coll_allreduce (&total_bytes,
|
||||
&total_bytes_global,
|
||||
1,
|
||||
MPI_DOUBLE,
|
||||
MPI_MAX,
|
||||
fh->f_comm,
|
||||
fh->f_comm->c_coll.coll_allreduce_module);
|
||||
|
||||
bytes_received = (int *)malloc (mca_fcoll_ylib_num_io_procs * sizeof (int));
|
||||
if (NULL == bytes_received) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
bytes_per_cycle = mca_fcoll_ylib_stripe_size * mca_fcoll_ylib_blocks_per_cycle;
|
||||
cycles = ceil ((float)total_bytes_global/bytes_per_cycle);
|
||||
|
||||
for (index = 0; index < cycles; index++) {
|
||||
int k = 0;
|
||||
size_t total_bytes_recv = 0;
|
||||
size_t temp = 0;
|
||||
global_count = 0;
|
||||
|
||||
memset(bytes_received, 0x0, mca_fcoll_ylib_num_io_procs*sizeof(int));
|
||||
if (0 == fh->f_rank%fh->f_aggregator_index) {
|
||||
memset(displs, 0x0, fh->f_size*sizeof(int));
|
||||
memset(bytes_per_process, 0x0, fh->f_size*sizeof(int));
|
||||
}
|
||||
|
||||
if (0 == fh->f_rank%fh->f_aggregator_index) {
|
||||
if (total_bytes > bytes_per_cycle) {
|
||||
bytes_to_read_in_cycle = bytes_per_cycle;
|
||||
}
|
||||
else {
|
||||
bytes_to_read_in_cycle = total_bytes;
|
||||
}
|
||||
}
|
||||
/*
|
||||
printf ("****%d: Total_bytes: %d CYCLE %d Bytes %d OFFSET %d******\n",
|
||||
fh->f_rank,
|
||||
total_bytes,
|
||||
index,
|
||||
bytes_to_read_in_cycle,
|
||||
fh->f_offset);
|
||||
sleep(1);
|
||||
*/
|
||||
/**********************************************************
|
||||
**Gather the Data from all the processes at the readers **
|
||||
*********************************************************/
|
||||
|
||||
/* Calculate how much data will be contributed in this cycle
|
||||
by each process*/
|
||||
previous = current;
|
||||
prev_bytes_rem = bytes_rem;
|
||||
temp = bytes_per_cycle * mca_fcoll_ylib_num_io_procs * (index+1)
|
||||
+ fh->f_offset;
|
||||
|
||||
while (current < broken_count) {
|
||||
if (temp >=
|
||||
(size_t)((OPAL_PTRDIFF_TYPE)broken_iovec[current].iov_base +
|
||||
broken_iovec[current].iov_len)) {
|
||||
k = ((OPAL_PTRDIFF_TYPE)broken_iovec[current].iov_base /
|
||||
mca_fcoll_ylib_stripe_size) % mca_fcoll_ylib_num_io_procs;
|
||||
|
||||
if (bytes_rem) {
|
||||
bytes_received[k] += bytes_rem;
|
||||
total_bytes_recv += bytes_rem;
|
||||
bytes_rem = 0;
|
||||
}
|
||||
else {
|
||||
bytes_received[k] += broken_iovec[current].iov_len;
|
||||
total_bytes_recv += broken_iovec[current].iov_len;
|
||||
}
|
||||
current ++;
|
||||
}
|
||||
else {
|
||||
k = ((OPAL_PTRDIFF_TYPE)broken_iovec[current].iov_base /
|
||||
mca_fcoll_ylib_stripe_size) % mca_fcoll_ylib_num_io_procs;
|
||||
if (bytes_rem) {
|
||||
bytes_received[k] += temp -
|
||||
((broken_iovec[current].iov_len - bytes_rem) +
|
||||
(OPAL_PTRDIFF_TYPE)broken_iovec[current].iov_base);
|
||||
total_bytes_recv += temp -
|
||||
((broken_iovec[current].iov_len - bytes_rem) +
|
||||
(OPAL_PTRDIFF_TYPE)broken_iovec[current].iov_base);
|
||||
bytes_rem -= temp -
|
||||
((broken_iovec[current].iov_len - bytes_rem) +
|
||||
(OPAL_PTRDIFF_TYPE)broken_iovec[current].iov_base);
|
||||
break;
|
||||
}
|
||||
else {
|
||||
if (temp > (size_t)broken_iovec[current].iov_base) {
|
||||
bytes_received[k] += temp -
|
||||
(OPAL_PTRDIFF_TYPE)broken_iovec[current].iov_base;
|
||||
total_bytes_recv += temp -
|
||||
(OPAL_PTRDIFF_TYPE)broken_iovec[current].iov_base;
|
||||
bytes_rem = broken_iovec[current].iov_len -
|
||||
(temp -
|
||||
(OPAL_PTRDIFF_TYPE)broken_iovec[current].iov_base);
|
||||
break;
|
||||
}
|
||||
else {
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
/*
|
||||
for (i=0 ; i<mca_fcoll_ylib_num_io_procs ; i++) {
|
||||
printf ("%d sending %d to %d\n",fh->f_rank,
|
||||
bytes_received[i], i);
|
||||
}
|
||||
sleep(1);
|
||||
*/
|
||||
if (0 == fh->f_rank%fh->f_aggregator_index && bytes_to_read_in_cycle) {
|
||||
/* Calculate how much data will be recieved this cycle
|
||||
by each aggregator*/
|
||||
while (bytes_to_read_in_cycle) {
|
||||
blocks = fview_count[0];
|
||||
for (j=0 ; j<fh->f_size ; j++) {
|
||||
if (sorted[current_index] < blocks) {
|
||||
n = j;
|
||||
break;
|
||||
}
|
||||
else {
|
||||
blocks += fview_count[j+1];
|
||||
}
|
||||
}
|
||||
if (bytes_remaining) {
|
||||
if (bytes_remaining <= bytes_to_read_in_cycle) {
|
||||
bytes_per_process[n] += bytes_remaining;
|
||||
current_index ++;
|
||||
bytes_to_read_in_cycle -= bytes_remaining;
|
||||
bytes_remaining = 0;
|
||||
continue;
|
||||
}
|
||||
else {
|
||||
bytes_per_process[n] += bytes_to_read_in_cycle;
|
||||
bytes_remaining -= bytes_to_read_in_cycle;
|
||||
bytes_to_read_in_cycle = 0;
|
||||
break;
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (bytes_to_read_in_cycle <
|
||||
global_fview[sorted[current_index]].iov_len) {
|
||||
bytes_per_process[n] += bytes_to_read_in_cycle;
|
||||
bytes_remaining =
|
||||
global_fview[sorted[current_index]].iov_len -
|
||||
bytes_to_read_in_cycle;
|
||||
bytes_to_read_in_cycle = 0;
|
||||
break;
|
||||
}
|
||||
else {
|
||||
bytes_per_process[n] +=
|
||||
global_fview[sorted[current_index]].iov_len;
|
||||
bytes_to_read_in_cycle -=
|
||||
global_fview[sorted[current_index]].iov_len;
|
||||
current_index ++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
/*
|
||||
for (i=0 ; i<fh->f_size ; i++) {
|
||||
printf ("%d --> expecting %d from %d\n",fh->f_rank,
|
||||
bytes_per_process[i], i);
|
||||
}
|
||||
*/
|
||||
/* Calculate the displacement on where to put the data and allocate
|
||||
the recieve buffer (global_buf) */
|
||||
displs[0] = 0;
|
||||
global_count = bytes_per_process[0];
|
||||
for (i=1 ; i<fh->f_size ; i++) {
|
||||
global_count += bytes_per_process[i];
|
||||
displs[i] = displs[i-1] + bytes_per_process[i-1];
|
||||
}
|
||||
/*
|
||||
for (i=0 ; i<fh->f_size ; i++) {
|
||||
printf ("Proc %d sending %d at %d\n",
|
||||
i,
|
||||
bytes_per_process[i],
|
||||
displs[i]);
|
||||
}
|
||||
*/
|
||||
if (0 != global_count) {
|
||||
global_buf = malloc (global_count);
|
||||
if (NULL == global_buf) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
}
|
||||
/**********************************************************
|
||||
******* Create the io array, and pass it to fbtl *********
|
||||
*********************************************************/
|
||||
if (0 == fh->f_rank%fh->f_aggregator_index && global_count) {
|
||||
int bytes_to_read = global_count;
|
||||
int *temp = NULL;
|
||||
int block = 1;
|
||||
k = 0;
|
||||
|
||||
temp = (int *)malloc (sizeof(int) * fh->f_size);
|
||||
if (NULL == temp) {
|
||||
opal_output(1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
memset(temp, 0x0, fh->f_size*sizeof(int));
|
||||
fh->f_io_array = (mca_io_ompio_io_array_t *) malloc
|
||||
(OMPIO_IOVEC_INITIAL_SIZE * sizeof (mca_io_ompio_io_array_t));
|
||||
if (NULL == fh->f_io_array) {
|
||||
opal_output(1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
while (bytes_to_read) {
|
||||
int start = 0;
|
||||
|
||||
if (OMPIO_IOVEC_INITIAL_SIZE*block <= k) {
|
||||
block ++;
|
||||
fh->f_io_array = (mca_io_ompio_io_array_t *)realloc
|
||||
(fh->f_io_array, OMPIO_IOVEC_INITIAL_SIZE * block *
|
||||
sizeof(mca_io_ompio_io_array_t));
|
||||
if (NULL == fh->f_io_array) {
|
||||
opal_output(1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
|
||||
blocks = fview_count[0];
|
||||
for (j=0 ; j<fh->f_size ; j++) {
|
||||
if (sorted[x] < blocks) {
|
||||
n = j;
|
||||
break;
|
||||
}
|
||||
else {
|
||||
blocks += fview_count[j+1];
|
||||
}
|
||||
}
|
||||
for (j=0 ; j<n ; j++) {
|
||||
start += bytes_per_process[j];
|
||||
}
|
||||
|
||||
if (bytes_left) {
|
||||
if (bytes_left <= bytes_to_read) {
|
||||
fh->f_io_array[k].offset = (IOVBASE_TYPE *)
|
||||
((OPAL_PTRDIFF_TYPE)global_fview[sorted[x]].iov_base +
|
||||
(global_fview[sorted[x]].iov_len - bytes_left));
|
||||
fh->f_io_array[k].length = bytes_left;
|
||||
fh->f_io_array[k].memory_address =
|
||||
&global_buf[start+temp[n]];
|
||||
temp[n] += (int)fh->f_io_array[k].length;
|
||||
bytes_to_read -= bytes_left;
|
||||
bytes_left = 0;
|
||||
k ++;
|
||||
x ++;
|
||||
continue;
|
||||
}
|
||||
else {
|
||||
fh->f_io_array[k].offset = (IOVBASE_TYPE *)
|
||||
((OPAL_PTRDIFF_TYPE)global_fview[sorted[x]].iov_base +
|
||||
(global_fview[sorted[x]].iov_len - bytes_left));
|
||||
fh->f_io_array[k].length = bytes_to_read;
|
||||
fh->f_io_array[k].memory_address =
|
||||
&global_buf[start+temp[n]];
|
||||
temp[n] += (int)fh->f_io_array[k].length;
|
||||
bytes_left -= bytes_to_read;
|
||||
bytes_to_read = 0;;
|
||||
k ++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
else {
|
||||
if (bytes_to_read < (int)global_fview[sorted[x]].iov_len) {
|
||||
fh->f_io_array[k].offset = global_fview[sorted[x]].iov_base;
|
||||
fh->f_io_array[k].length = bytes_to_read;
|
||||
fh->f_io_array[k].memory_address =
|
||||
&global_buf[start+temp[n]];
|
||||
bytes_left =
|
||||
global_fview[sorted[x]].iov_len - bytes_to_read;
|
||||
bytes_to_read = 0;
|
||||
k ++;
|
||||
break;
|
||||
}
|
||||
else {
|
||||
fh->f_io_array[k].offset = global_fview[sorted[x]].iov_base;
|
||||
fh->f_io_array[k].length = global_fview[sorted[x]].iov_len;
|
||||
fh->f_io_array[k].memory_address =
|
||||
&global_buf[start+temp[n]];
|
||||
temp[n] += (int)fh->f_io_array[k].length;
|
||||
bytes_to_read -= global_fview[sorted[x]].iov_len;
|
||||
k ++;
|
||||
x ++;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fh->f_num_of_io_entries = k;
|
||||
/*
|
||||
printf("*************************** %d\n", fh->f_num_of_io_entries);
|
||||
for (i=0 ; i<fh->f_num_of_io_entries ; i++) {
|
||||
printf(" ADDRESS: %p OFFSET: %p LENGTH: %d\n",
|
||||
fh->f_io_array[i].memory_address,
|
||||
fh->f_io_array[i].offset,
|
||||
fh->f_io_array[i].length);
|
||||
}
|
||||
*/
|
||||
if (fh->f_num_of_io_entries) {
|
||||
if (OMPI_SUCCESS != fh->f_fbtl->fbtl_preadv (fh, NULL)) {
|
||||
opal_output (1, "READ FAILED\n");
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
}
|
||||
if (NULL != temp) {
|
||||
free (temp);
|
||||
temp = NULL;
|
||||
}
|
||||
}
|
||||
/**********************************************************
|
||||
******************** DONE READING ************************
|
||||
*********************************************************/
|
||||
|
||||
|
||||
/**********************************************************
|
||||
********* Scatter the Data from the readers **************
|
||||
*********************************************************/
|
||||
|
||||
if (fh->f_flags & OMPIO_CONTIGUOUS_MEMORY) {
|
||||
receive_buf = &((char*)buf)[total_bytes_read];
|
||||
}
|
||||
else if (total_bytes_recv) {
|
||||
/* allocate a send buffer and copy the data that needs
|
||||
to be sent into it in case the data is non-contigous
|
||||
in memory */
|
||||
receive_buf = malloc (total_bytes_recv);
|
||||
if (NULL == receive_buf) {
|
||||
opal_output (1, "OUT OF MEMORY\n");
|
||||
return OMPI_ERR_OUT_OF_RESOURCE;
|
||||
}
|
||||
}
|
||||
|
||||
/* distribute the data to its corresponding processes */
|
||||
ompi_io_ompio_scatter_data (fh,
|
||||
receive_buf,
|
||||
total_bytes_recv,
|
||||
bytes_received,
|
||||
broken_iovec,
|
||||
previous,
|
||||
prev_bytes_rem,
|
||||
global_buf,
|
||||
bytes_per_process,
|
||||
displs,
|
||||
mca_fcoll_ylib_num_io_procs,
|
||||
mca_fcoll_ylib_stripe_size);
|
||||
|
||||
/*
|
||||
if (0 == fh->f_rank%fh->f_aggregator_index) {
|
||||
for (k=0 ; k<global_count/4 ; k++) {
|
||||
printf ("%d: RECV %d \n",fh->f_rank,
|
||||
((int *)global_buf)[k]);
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
/* If data is not contigous in memory, copy the data from the
|
||||
receive buffer into the buffer passed in */
|
||||
if (!(fh->f_flags & OMPIO_CONTIGUOUS_MEMORY)) {
|
||||
OPAL_PTRDIFF_TYPE mem_address;
|
||||
size_t remaining = 0;
|
||||
size_t temp_position = 0;
|
||||
|
||||
remaining = total_bytes_recv;
|
||||
|
||||
while (remaining) {
|
||||
mem_address = (OPAL_PTRDIFF_TYPE)
|
||||
(decoded_iov[iov_index].iov_base) + current_position;
|
||||
|
||||
if (remaining >=
|
||||
(decoded_iov[iov_index].iov_len - current_position)) {
|
||||
memcpy ((IOVBASE_TYPE *) mem_address,
|
||||
receive_buf+temp_position,
|
||||
decoded_iov[iov_index].iov_len - current_position);
|
||||
remaining = remaining -
|
||||
(decoded_iov[iov_index].iov_len - current_position);
|
||||
temp_position = temp_position +
|
||||
(decoded_iov[iov_index].iov_len - current_position);
|
||||
iov_index = iov_index + 1;
|
||||
current_position = 0;
|
||||
}
|
||||
else {
|
||||
memcpy ((IOVBASE_TYPE *) mem_address,
|
||||
receive_buf+temp_position,
|
||||
remaining);
|
||||
current_position = current_position + remaining;
|
||||
remaining = 0;
|
||||
}
|
||||
}
|
||||
if (NULL != receive_buf) {
|
||||
free (receive_buf);
|
||||
receive_buf = NULL;
|
||||
}
|
||||
}
|
||||
|
||||
total_bytes_read += total_bytes_recv;
|
||||
total_bytes -= global_count;
|
||||
|
||||
/**********************************************************
|
||||
**************** DONE GATHERING OF DATA ******************
|
||||
*********************************************************/
|
||||
|
||||
|
||||
|
||||
if (0 == fh->f_rank%fh->f_aggregator_index) {
|
||||
fh->f_num_of_io_entries = 0;
|
||||
if (NULL != fh->f_io_array) {
|
||||
free (fh->f_io_array);
|
||||
fh->f_io_array = NULL;
|
||||
}
|
||||
if (NULL != global_buf) {
|
||||
free (global_buf);
|
||||
global_buf = NULL;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (NULL != sorted) {
|
||||
free (sorted);
|
||||
sorted = NULL;
|
||||
}
|
||||
if (NULL != broken_iovec) {
|
||||
free (broken_iovec);
|
||||
broken_iovec = NULL;
|
||||
}
|
||||
if (NULL != global_fview) {
|
||||
free (global_fview);
|
||||
global_fview = NULL;
|
||||
}
|
||||
if (NULL != fview_count) {
|
||||
free (fview_count);
|
||||
fview_count = NULL;
|
||||
}
|
||||
if (NULL != decoded_iov) {
|
||||
free (decoded_iov);
|
||||
decoded_iov = NULL;
|
||||
}
|
||||
if (NULL != bytes_per_process) {
|
||||
free (bytes_per_process);
|
||||
bytes_per_process = NULL;
|
||||
}
|
||||
if (NULL != bytes_received) {
|
||||
free (bytes_received);
|
||||
bytes_received = NULL;
|
||||
}
|
||||
if (NULL != displs) {
|
||||
free (displs);
|
||||
displs = NULL;
|
||||
}
|
||||
/*
|
||||
if (NULL != total_bytes_per_process) {
|
||||
free (total_bytes_per_process);
|
||||
total_bytes_per_process = NULL;
|
||||
}
|
||||
*/
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
Некоторые файлы не были показаны из-за слишком большого количества измененных файлов Показать больше
Загрузка…
x
Ссылка в новой задаче
Block a user