* start of the Portals bmi. Does not work, and is .ompi_ignored for everyone
else. This commit was SVN r6194.
Этот коммит содержится в:
Brian Barrett
родитель
225c20162e
Коммит
146389a13e
0
src/mca/bmi/portals/.ompi_ignore
Обычный файл
0
src/mca/bmi/portals/.ompi_ignore
Обычный файл
2
src/mca/bmi/portals/.ompi_unignore
Обычный файл
2
src/mca/bmi/portals/.ompi_unignore
Обычный файл
@ -0,0 +1,2 @@
|
||||
brbarret
|
||||
bbarrett
|
||||
55
src/mca/bmi/portals/Makefile.am
Обычный файл
55
src/mca/bmi/portals/Makefile.am
Обычный файл
@ -0,0 +1,55 @@
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
# Use the top-level Makefile.options
|
||||
|
||||
include $(top_ompi_srcdir)/config/Makefile.options
|
||||
|
||||
# Make the output library in this directory, and name it either
|
||||
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
|
||||
# (for static builds).
|
||||
|
||||
if OMPI_BUILD_bmi_portals_DSO
|
||||
component_noinst =
|
||||
component_install = mca_bmi_portals.la
|
||||
else
|
||||
component_noinst = libmca_bmi_portals.la
|
||||
component_install =
|
||||
endif
|
||||
|
||||
EXTRA_DIST = \
|
||||
src/bmi_portals_compat_utcp.c \
|
||||
src/bmi_portals_compat_redstorm.c
|
||||
|
||||
portals_SOURCES = \
|
||||
src/bmi_portals.h \
|
||||
src/bmi_portals_compat.h \
|
||||
src/bmi_portals.c \
|
||||
src/bmi_portals_component.c \
|
||||
src/bmi_portals_stubs.c \
|
||||
src/bmi_portals_compat_utcp.c
|
||||
|
||||
|
||||
mcacomponentdir = $(libdir)/openmpi
|
||||
mcacomponent_LTLIBRARIES = $(component_install)
|
||||
mca_bmi_portals_la_SOURCES = $(portals_SOURCES)
|
||||
mca_bmi_portals_la_LIBADD =
|
||||
mca_bmi_portals_la_LDFLAGS = -module -avoid-version
|
||||
|
||||
noinst_LTLIBRARIES = $(component_noinst)
|
||||
libmca_bmi_portals_la_SOURCES = $(portals_SOURCES)
|
||||
libmca_bmi_portals_la_LIBADD =
|
||||
libmca_bmi_portals_la_LDFLAGS = -module -avoid-version
|
||||
22
src/mca/bmi/portals/configure.params
Обычный файл
22
src/mca/bmi/portals/configure.params
Обычный файл
@ -0,0 +1,22 @@
|
||||
# -*- shell-script -*-
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
# Specific to this module
|
||||
|
||||
PARAM_INIT_FILE=src/bmi_portals.h
|
||||
PARAM_CONFIG_HEADER_FILE="portals_config.h"
|
||||
PARAM_CONFIG_FILES="Makefile"
|
||||
215
src/mca/bmi/portals/configure.stub
Обычный файл
215
src/mca/bmi/portals/configure.stub
Обычный файл
@ -0,0 +1,215 @@
|
||||
# -*- shell-script -*-
|
||||
#
|
||||
# Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
# All rights reserved.
|
||||
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
# University of Stuttgart. All rights reserved.
|
||||
# Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
# All rights reserved.
|
||||
# $COPYRIGHT$
|
||||
#
|
||||
# Additional copyrights may follow
|
||||
#
|
||||
# $HEADER$
|
||||
#
|
||||
|
||||
#
|
||||
# quicky function to set #defines based on argument values
|
||||
#
|
||||
# ARGUMENTS:
|
||||
# 1 configure name (first argument to ARG_WITH, minus the bmi-portals-)
|
||||
# 2 define name
|
||||
# 3 default value
|
||||
# 4 description (used for both ARG_WITH and DEFINE)
|
||||
AC_DEFUN([MCA_BMI_PORTALS_CONFIG_VAL],
|
||||
[
|
||||
AC_ARG_WITH([bmi-portals-$1], AC_HELP_STRING([--with-bmi-portals-$1],
|
||||
[$4 (default: $3)]))
|
||||
AC_MSG_CHECKING([for $1 value])
|
||||
case "[$with_]m4_bpatsubst([bmi-portals-$1], -, _)" in
|
||||
"")
|
||||
$2=$3
|
||||
AC_MSG_RESULT([[$]$2 (default)])
|
||||
;;
|
||||
"no")
|
||||
AC_MSG_RESULT([error])
|
||||
AC_MSG_ERROR([--without-bmi-portals-$1 is invalid argument])
|
||||
;;
|
||||
*)
|
||||
$2="$with_m4_bpatsubst([bmi-portals-$1], -, _)"
|
||||
AC_MSG_RESULT([[$]$2])
|
||||
;;
|
||||
esac
|
||||
AC_DEFINE_UNQUOTED([$2], [[$]$2], [$4])
|
||||
])dnl
|
||||
|
||||
|
||||
#
|
||||
# Main function. This will be invoked in the middle of the templated
|
||||
# configure script.
|
||||
#
|
||||
AC_DEFUN([MCA_CONFIGURE_STUB],[
|
||||
|
||||
# Additional --with flags that can be specified
|
||||
|
||||
AC_ARG_WITH(bmi-portals,
|
||||
AC_HELP_STRING([--with-bmi-portals=DIR],
|
||||
[Specify the installation directory of PORTALS]))
|
||||
|
||||
# Add to CPPFLAGS if necessary
|
||||
EXTRA_CPPFLAGS=
|
||||
if test -n "$with_bmi_portals"; then
|
||||
if test -d "$with_bmi_portals/include"; then
|
||||
EXTRA_CPPFLAGS="-I$with_bmi_portals/include"
|
||||
else
|
||||
AC_MSG_WARN([*** Warning: cannot find $with_bmi_portals/include])
|
||||
AC_MSG_WARN([*** Will still try to configure portals bmi anyway...])
|
||||
fi
|
||||
fi
|
||||
|
||||
# See if we can find portals.h
|
||||
CPPFLAGS="$CPPFLAGS $EXTRA_CPPFLAGS"
|
||||
AC_CHECK_HEADERS(portals3.h,,
|
||||
AC_MSG_ERROR([*** Cannot find working portals3.h.]))
|
||||
|
||||
# Add to LDFLAGS if necessary
|
||||
EXTRA_LDFLAGS=
|
||||
if test -n "$with_bmi_portals"; then
|
||||
if test -d "$with_bmi_portals/lib"; then
|
||||
EXTRA_LDFLAGS="-L$with_bmi_portals/lib"
|
||||
else
|
||||
AC_MSG_WARN([*** Warning: cannot find $with_bmi_portals/lib])
|
||||
AC_MSG_WARN([*** Will still try to configure portals bmi anyway...])
|
||||
fi
|
||||
fi
|
||||
|
||||
|
||||
#
|
||||
# Configure Portals for our local environment
|
||||
#
|
||||
BMI_PORTALS_UTCP=0
|
||||
BMI_PORTALS_REDSTORM=0
|
||||
BMI_PORTALS_COMPAT=""
|
||||
BMI_PORTALS_HAVE_EVENT_UNLINK=0
|
||||
|
||||
AC_ARG_WITH([bmi-portals-config],
|
||||
AC_HELP_STRING([--with-bmi-portals-config],
|
||||
[configuration to use for Portals support.
|
||||
One of "utcp", "redstorm". (default: utcp)]))
|
||||
AC_MSG_CHECKING([for Portals configuration])
|
||||
if test "$with_bmi_portals_config" = "" ; then
|
||||
with_bmi_portals_config="utcp"
|
||||
fi
|
||||
case "$with_bmi_portals_config" in
|
||||
"utcp")
|
||||
BMI_PORTALS_UTCP=1
|
||||
PORTALS_LIBS="-lutcpapi -lutcplib -lp3api -lp3lib -lp3rt"
|
||||
BMI_PORTALS_HAVE_EVENT_UNLINK=1
|
||||
AC_MSG_RESULT([utcp])
|
||||
;;
|
||||
"redstorm")
|
||||
BMI_PORTALS_REDSTORM=1
|
||||
PORTALS_LIBS="-lp3api -lp3lib -lp3rt"
|
||||
BMI_PORTALS_HAVE_EVENT_UNLINK=0
|
||||
AC_MSG_RESULT([red storm])
|
||||
;;
|
||||
*)
|
||||
AC_MSG_ERROR([unknown Portals configuration. Can not continue])
|
||||
;;
|
||||
esac
|
||||
|
||||
# Try to find all the portals libraries (this is not fun!)
|
||||
AC_ARG_WITH(bmi-portals-libs,
|
||||
AC_HELP_STRING([--with-bmi-portals-libs=LIBS],
|
||||
[Libraries to link with for portals]))
|
||||
if test -n "$with_bmi_portals_libs" ; then
|
||||
PORTALS_LIBS=""
|
||||
for lib in $with_bmi_portals_libs ; do
|
||||
PORTALS_LIBS="$PORTALS_LIBS -l$lib"
|
||||
done
|
||||
fi
|
||||
|
||||
AC_MSG_CHECKING([if possible to link Portals application])
|
||||
LIBS="$LIBS $PORTALS_LIBS"
|
||||
LDFLAGS="$LDFLAGS $EXTRA_LDFLAGS"
|
||||
AC_LINK_IFELSE([AC_LANG_PROGRAM([#include <portals3.h>], [int i; PtlInit(&i);])],
|
||||
[AC_MSG_RESULT([yes])],
|
||||
[AC_MSG_RESULT([no])
|
||||
AC_MSG_ERROR([Can not link with Portals libraries])])
|
||||
|
||||
AC_DEFINE_UNQUOTED([BMI_PORTALS_HAVE_EVENT_UNLINK],
|
||||
[$BMI_PORTALS_HAVE_EVENT_UNLINK],
|
||||
[Does Portals send a BMI_EVENT_UNLINK event])
|
||||
|
||||
AC_DEFINE_UNQUOTED([BMI_PORTALS_UTCP], [$BMI_PORTALS_UTCP],
|
||||
[Use the UTCP reference implementation or Portals])
|
||||
AM_CONDITIONAL([BMI_PORTALS_UTCP], [test "$BMI_PORTALS_UTCP" = "1"])
|
||||
|
||||
AC_DEFINE_UNQUOTED([BMI_PORTALS_REDSTORM], [$BMI_PORTALS_REDSTORM],
|
||||
[Use the Red Storm implementation or Portals])
|
||||
AM_CONDITIONAL([BMI_PORTALS_REDSTORM], [test "$BMI_PORTALS_REDSTORM" = "1"])
|
||||
|
||||
MCA_BMI_PORTALS_CONFIG_VAL([frag-table-id],
|
||||
[BMI_PORTALS_FRAG_TABLE_ID], [1],
|
||||
[Portals table id to use for fragment receive queue])
|
||||
|
||||
MCA_BMI_PORTALS_CONFIG_VAL([retrans-table-id],
|
||||
[BMI_PORTALS_RETRANS_TABLE_ID], [2],
|
||||
[Portals table id to use for retransmit request queue])
|
||||
|
||||
MCA_BMI_PORTALS_CONFIG_VAL([debug-level],
|
||||
[BMI_PORTALS_DEFAULT_DEBUG_LEVEL], [100],
|
||||
[debugging level for portals bmi])
|
||||
|
||||
MCA_BMI_PORTALS_CONFIG_VAL([request-cache-size],
|
||||
[BMI_PORTALS_DEFAULT_REQUEST_CACHE_SIZE], [1],
|
||||
[request cache size for portals bmi])
|
||||
|
||||
MCA_BMI_PORTALS_CONFIG_VAL([first-frag-size],
|
||||
[BMI_PORTALS_DEFAULT_FIRST_FRAG_SIZE], [16384],
|
||||
[first frag size for portals bmi])
|
||||
|
||||
MCA_BMI_PORTALS_CONFIG_VAL([first-frag-num-entries],
|
||||
[BMI_PORTALS_DEFAULT_FIRST_FRAG_NUM_ENTRIES], [3],
|
||||
[number of memory descriptors for first fragments])
|
||||
|
||||
MCA_BMI_PORTALS_CONFIG_VAL([first-frag-entry-size],
|
||||
[BMI_PORTALS_DEFAULT_FIRST_FRAG_ENTRY_SIZE], [2098152],
|
||||
[size of memory associeted with first fag md])
|
||||
|
||||
MCA_BMI_PORTALS_CONFIG_VAL([recv-queue-size],
|
||||
[BMI_PORTALS_DEFAULT_RECV_QUEUE_SIZE], [512],
|
||||
[size of event queue for receiving frags])
|
||||
|
||||
MCA_BMI_PORTALS_CONFIG_VAL([send-queue-size],
|
||||
[BMI_PORTALS_DEFAULT_SEND_QUEUE_SIZE], [128],
|
||||
[Max number of send fragmenst pending])
|
||||
|
||||
MCA_BMI_PORTALS_CONFIG_VAL([rndv-frag-min-size],
|
||||
[BMI_PORTALS_DEFAULT_RNDV_FRAG_MIN_SIZE], [0],
|
||||
[minimum size of rndv fragments])
|
||||
|
||||
MCA_BMI_PORTALS_CONFIG_VAL([rndv-frag-max-size],
|
||||
[BMI_PORTALS_DEFAULT_RNDV_FRAG_MAX_SIZE], [16384],
|
||||
[maximum size of rndv fragments])
|
||||
|
||||
MCA_BMI_PORTALS_CONFIG_VAL([free-list-init-num],
|
||||
[BMI_PORTALS_DEFAULT_FREE_LIST_INIT_NUM], [256],
|
||||
[starting size of free lists])
|
||||
MCA_BMI_PORTALS_CONFIG_VAL([free-list-max-num],
|
||||
[BMI_PORTALS_DEFAULT_FREE_LIST_MAX_NUM], [-1],
|
||||
[maximum size of free lists])
|
||||
MCA_BMI_PORTALS_CONFIG_VAL([free-list-inc-num],
|
||||
[BMI_PORTALS_DEFAULT_FREE_LIST_inc_NUM], [256],
|
||||
[grow size for freelists])
|
||||
|
||||
#
|
||||
# Save extra compiler/linker flags so that they can be added in
|
||||
# the wrapper compilers, if necessary
|
||||
#
|
||||
|
||||
WRAPPER_EXTRA_LDFLAGS="$EXTRA_LDFLAGS"
|
||||
WRAPPER_EXTRA_LIBS="$PORTALS_LIBS"
|
||||
])dnl
|
||||
143
src/mca/bmi/portals/src/bmi_portals.c
Обычный файл
143
src/mca/bmi/portals/src/bmi_portals.c
Обычный файл
@ -0,0 +1,143 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "portals_config.h"
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <unistd.h>
|
||||
#include <stdio.h>
|
||||
#include <errno.h>
|
||||
|
||||
#include "include/constants.h"
|
||||
#include "util/output.h"
|
||||
#include "mca/pml/pml.h"
|
||||
#include "mca/bmi/bmi.h"
|
||||
|
||||
#include "bmi_portals.h"
|
||||
#include "bmi_portals_compat.h"
|
||||
|
||||
mca_bmi_portals_module_t mca_bmi_portals_module = {
|
||||
{
|
||||
&mca_bmi_portals_component.super,
|
||||
0, /* max size of first frag */
|
||||
0, /* min send size */
|
||||
0, /* max send size */
|
||||
0, /* min rdma size */
|
||||
0, /* max rdma size */
|
||||
60, /* exclusivity - higher than sm, lower than self */
|
||||
0, /* latency */
|
||||
0, /* bandwidth */
|
||||
0, /* bmi flags */
|
||||
|
||||
mca_bmi_portals_add_procs,
|
||||
mca_bmi_portals_del_procs,
|
||||
mca_bmi_portals_register,
|
||||
mca_bmi_portals_finalize,
|
||||
|
||||
mca_bmi_portals_alloc,
|
||||
mca_bmi_portals_free,
|
||||
mca_bmi_portals_prepare_src,
|
||||
mca_bmi_portals_prepare_dst,
|
||||
mca_bmi_portals_send,
|
||||
mca_bmi_portals_put,
|
||||
mca_bmi_portals_get,
|
||||
},
|
||||
};
|
||||
|
||||
|
||||
|
||||
int
|
||||
mca_bmi_portals_add_procs(struct mca_bmi_base_module_t* bmi,
|
||||
size_t nprocs, struct ompi_proc_t **procs,
|
||||
struct mca_bmi_base_endpoint_t** peers,
|
||||
ompi_bitmap_t* reachable)
|
||||
{
|
||||
int ret;
|
||||
struct ompi_proc_t *local_proc = ompi_proc_local();
|
||||
struct ompi_proc_t *curr_proc;
|
||||
ptl_process_id_t *portals_procs;
|
||||
size_t i;
|
||||
unsigned long distance;
|
||||
struct mca_bmi_portals_module_t *mybmi =
|
||||
(struct mca_bmi_portals_module_t*) bmi;
|
||||
|
||||
/* make sure our environment is fully initialized. At end of this
|
||||
call, we have a working network handle on our module and
|
||||
portals_procs will have the portals process identifier for each
|
||||
proc (ordered, in theory) */
|
||||
ret = mca_bmi_portals_add_procs_compat(mybmi, nprocs, procs,
|
||||
&portals_procs);
|
||||
if (OMPI_SUCCESS != ret) return ret;
|
||||
|
||||
/* loop through all procs, setting our reachable flag */
|
||||
for (i= 0; i < nprocs ; ++i) {
|
||||
curr_proc = procs[i];
|
||||
/* BWB - do we want to send to self? No for now */
|
||||
if (curr_proc == local_proc) continue;
|
||||
|
||||
/* make sure we can reach the process */
|
||||
ret = PtlNIDist(mybmi->ni_handle,
|
||||
portals_procs[i],
|
||||
&distance);
|
||||
if (ret != PTL_OK) {
|
||||
ompi_output_verbose(10, mca_bmi_portals_component.portals_output,
|
||||
"Could not find distance to process %d", i);
|
||||
continue;
|
||||
}
|
||||
|
||||
/* set the peer as a pointer to the address */
|
||||
peers[i] = (struct mca_bmi_base_endpoint_t*) &(portals_procs[i]);
|
||||
|
||||
/* and here we can reach */
|
||||
ompi_bitmap_set_bit(reachable, i);
|
||||
}
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
int
|
||||
mca_bmi_portals_del_procs(struct mca_bmi_base_module_t *bmi,
|
||||
size_t nprocs,
|
||||
struct ompi_proc_t **procs,
|
||||
struct mca_bmi_base_endpoint_t **peers)
|
||||
{
|
||||
/* yeah, I have no idea what to do here */
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
|
||||
int
|
||||
mca_bmi_portals_finalize(struct mca_bmi_base_module_t *bmi_base)
|
||||
{
|
||||
struct mca_bmi_portals_module_t *bmi =
|
||||
(struct mca_bmi_portals_module_t *) bmi_base;
|
||||
int ret;
|
||||
|
||||
ret = PtlNIFini(bmi->ni_handle);
|
||||
if (PTL_OK != ret) {
|
||||
ompi_output_verbose(20, mca_bmi_portals_component.portals_output,
|
||||
"PtlNIFini returned %d", ret);
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
ompi_output_verbose(20, mca_bmi_portals_component.portals_output,
|
||||
"successfully finalized module");
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
200
src/mca/bmi/portals/src/bmi_portals.h
Обычный файл
200
src/mca/bmi/portals/src/bmi_portals.h
Обычный файл
@ -0,0 +1,200 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
/*
|
||||
* @file
|
||||
*/
|
||||
#ifndef MCA_BMI_PORTALS_H
|
||||
#define MCA_BMI_PORTALS_H
|
||||
|
||||
#include <portals3.h>
|
||||
|
||||
#include "mca/pml/pml.h"
|
||||
#include "mca/bmi/bmi.h"
|
||||
#include "class/ompi_bitmap.h"
|
||||
#include "class/ompi_free_list.h"
|
||||
#include "class/ompi_proc_table.h"
|
||||
|
||||
|
||||
/*
|
||||
* Portals BMI component.
|
||||
*/
|
||||
struct mca_bmi_portals_component_t {
|
||||
/* base BMI component */
|
||||
mca_bmi_base_component_1_0_0_t super;
|
||||
|
||||
/* output channel for debugging. Value settings when using
|
||||
* output_verbose:
|
||||
*
|
||||
* - 0 : critical user information
|
||||
* - 10: general execution diagnostic information
|
||||
* - 20: initialization / shutdown diagnostic information
|
||||
* - 30: basic debugging information
|
||||
* - 90: useful only to developers
|
||||
* - 100: lots and lots of performance impacting output
|
||||
*/
|
||||
int portals_output;
|
||||
|
||||
#if BMI_PORTALS_UTCP
|
||||
/* ethernet interface to use - only has meaning with utcp
|
||||
reference */
|
||||
char *portals_ifname;
|
||||
#endif
|
||||
|
||||
/* Number of currently active portals modules. We assume these
|
||||
never change between init and finalize, so these aren't thread
|
||||
locked */
|
||||
uint32_t portals_num_modules;
|
||||
/* List of currently available modules */
|
||||
struct mca_bmi_portals_module_t **portals_modules;
|
||||
|
||||
/* initial size of free lists */
|
||||
int portals_free_list_init_num;
|
||||
/* max size of free lists */
|
||||
int portals_free_list_max_num;
|
||||
/* numer of elements to grow free lists */
|
||||
int portals_free_list_inc_num;
|
||||
|
||||
/* lock for accessing component */
|
||||
ompi_mutex_t portals_lock;
|
||||
};
|
||||
typedef struct mca_bmi_portals_component_t mca_bmi_portals_component_t;
|
||||
|
||||
|
||||
#define MCA_BMI_PORTALS_EQ_RECV 0
|
||||
#define MCA_BMI_PORTALS_EQ_SEND 1
|
||||
#define MCA_BMI_PORTALS_EQ_SIZE 2
|
||||
|
||||
struct mca_bmi_portals_module_t {
|
||||
/* base BMI module interface */
|
||||
mca_bmi_base_module_t super;
|
||||
|
||||
/* number of mds for first frags */
|
||||
int first_frag_num_entries;
|
||||
/* size of each md for first frags */
|
||||
int first_frag_entry_size;
|
||||
|
||||
/* size for event queue */
|
||||
int eq_sizes[MCA_BMI_PORTALS_EQ_SIZE];
|
||||
/* frag receive event queue */
|
||||
ptl_handle_eq_t eq_handles[MCA_BMI_PORTALS_EQ_SIZE];
|
||||
|
||||
/* our portals network interface */
|
||||
ptl_handle_ni_t ni_handle;
|
||||
/* the limits returned from PtlNIInit for interface */
|
||||
ptl_ni_limits_t limits;
|
||||
|
||||
/* number of dropped messages */
|
||||
ptl_sr_value_t dropped;
|
||||
};
|
||||
typedef struct mca_bmi_portals_module_t mca_bmi_portals_module_t;
|
||||
|
||||
struct mca_bmi_portals_recv_frag_t;
|
||||
struct mca_bmi_portals_send_frag_t;
|
||||
|
||||
/*
|
||||
* Component functions (bmi_portals_component.c)
|
||||
*/
|
||||
int mca_bmi_portals_component_open(void);
|
||||
int mca_bmi_portals_component_close(void);
|
||||
|
||||
|
||||
mca_bmi_base_module_t** mca_bmi_portals_component_init(int *num_bmis,
|
||||
bool has_progress_threads,
|
||||
bool has_mpi_threads);
|
||||
|
||||
int mca_bmi_portals_component_progress(void);
|
||||
|
||||
/*
|
||||
* Compatibility functions (bmi_portals_compat_{}.c)
|
||||
*
|
||||
* Not part of the BMI interface. Need to be implemented for every
|
||||
* version of Portals
|
||||
*/
|
||||
int mca_bmi_portals_init(mca_bmi_portals_component_t *comp);
|
||||
|
||||
int mca_bmi_portals_add_procs_compat(mca_bmi_portals_module_t* bmi,
|
||||
size_t nprocs, struct ompi_proc_t **procs,
|
||||
ptl_process_id_t **portals_procs);
|
||||
|
||||
/*
|
||||
* Module configuration functions (bmi_portals.c)
|
||||
*/
|
||||
int mca_bmi_portals_finalize(struct mca_bmi_base_module_t* bmi);
|
||||
|
||||
int mca_bmi_portals_add_procs(struct mca_bmi_base_module_t* bmi,
|
||||
size_t nprocs,
|
||||
struct ompi_proc_t **procs,
|
||||
struct mca_bmi_base_endpoint_t** peers,
|
||||
ompi_bitmap_t* reachable);
|
||||
|
||||
int mca_bmi_portals_del_procs(struct mca_bmi_base_module_t* bmi,
|
||||
size_t nprocs,
|
||||
struct ompi_proc_t **procs,
|
||||
struct mca_bmi_base_endpoint_t** peers);
|
||||
/*
|
||||
* stubbed functions
|
||||
*/
|
||||
int mca_bmi_portals_register(struct mca_bmi_base_module_t* bmi,
|
||||
mca_bmi_base_tag_t tag,
|
||||
mca_bmi_base_module_recv_cb_fn_t cbfunc,
|
||||
void* cbdata);
|
||||
|
||||
mca_bmi_base_descriptor_t*
|
||||
mca_bmi_portals_alloc(struct mca_bmi_base_module_t* bmi,
|
||||
size_t size);
|
||||
|
||||
int mca_bmi_portals_free(struct mca_bmi_base_module_t* bmi,
|
||||
mca_bmi_base_descriptor_t* des);
|
||||
|
||||
mca_bmi_base_descriptor_t*
|
||||
mca_bmi_portals_prepare_src(struct mca_bmi_base_module_t* bmi,
|
||||
struct mca_bmi_base_endpoint_t* peer,
|
||||
mca_mpool_base_registration_t* registration,
|
||||
struct ompi_convertor_t* convertor,
|
||||
size_t reserve,
|
||||
size_t* size);
|
||||
|
||||
mca_bmi_base_descriptor_t*
|
||||
mca_bmi_portals_prepare_dst(struct mca_bmi_base_module_t* bmi,
|
||||
struct mca_bmi_base_endpoint_t* peer,
|
||||
mca_mpool_base_registration_t* registration,
|
||||
struct ompi_convertor_t* convertor,
|
||||
size_t reserve,
|
||||
size_t* size);
|
||||
|
||||
int mca_bmi_portals_send(struct mca_bmi_base_module_t* bmi,
|
||||
struct mca_bmi_base_endpoint_t* bmi_peer,
|
||||
struct mca_bmi_base_descriptor_t* descriptor,
|
||||
mca_bmi_base_tag_t tag);
|
||||
|
||||
int mca_bmi_portals_put(struct mca_bmi_base_module_t* bmi,
|
||||
struct mca_bmi_base_endpoint_t* bmi_peer,
|
||||
struct mca_bmi_base_descriptor_t* decriptor);
|
||||
|
||||
|
||||
int mca_bmi_portals_get(struct mca_bmi_base_module_t* bmi,
|
||||
struct mca_bmi_base_endpoint_t* bmi_peer,
|
||||
struct mca_bmi_base_descriptor_t* decriptor);
|
||||
|
||||
|
||||
/*
|
||||
* global structures
|
||||
*/
|
||||
extern mca_bmi_portals_component_t mca_bmi_portals_component;
|
||||
/* don't use, except as base for creating module instances */
|
||||
extern mca_bmi_portals_module_t mca_bmi_portals_module;
|
||||
|
||||
#endif
|
||||
37
src/mca/bmi/portals/src/bmi_portals_compat.h
Обычный файл
37
src/mca/bmi/portals/src/bmi_portals_compat.h
Обычный файл
@ -0,0 +1,37 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
|
||||
#ifndef BMI_PORTALS_COMPAT_H
|
||||
#define BMI_PORTALS_COMPAT_H
|
||||
|
||||
#if BMI_PORTALS_UTCP
|
||||
|
||||
#include <p3nal_utcp.h>
|
||||
#include <p3rt/p3rt.h>
|
||||
#include <p3api/debug.h>
|
||||
|
||||
#elif BMI_PORTALS_REDSTORM
|
||||
|
||||
#error "Red Storm Compatibility not implemented"
|
||||
|
||||
#else
|
||||
|
||||
#error "Unknown Portals library configuration"
|
||||
|
||||
#endif
|
||||
|
||||
#endif /* BMI_PORTALS_NAL_H */
|
||||
110
src/mca/bmi/portals/src/bmi_portals_compat_redstorm.c
Обычный файл
110
src/mca/bmi/portals/src/bmi_portals_compat_redstorm.c
Обычный файл
@ -0,0 +1,110 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "portals_config.h"
|
||||
|
||||
#include "include/constants.h"
|
||||
#include "util/output.h"
|
||||
|
||||
#include "bmi_portals.h"
|
||||
#include "bmi_portals_compat.h"
|
||||
|
||||
|
||||
int
|
||||
mca_bmi_portals_init(mca_bmi_portals_component_t *comp)
|
||||
{
|
||||
int ret, max_interfaces;
|
||||
struct mca_bmi_portals_module_t *bmi;
|
||||
|
||||
/*
|
||||
* Initialize Portals interface
|
||||
*/
|
||||
ret = PtlInit(&max_interfaces);
|
||||
if (PTL_OK != ret) {
|
||||
ompi_output_verbose(10, mca_bmi_portals_component.portals_output,
|
||||
"PtlInit failed, returning %d\n", ret);
|
||||
return OMPI_ERR_FATAL;
|
||||
}
|
||||
|
||||
/*
|
||||
* create module - only ever one "NIC" on red storm
|
||||
*/
|
||||
comp->portals_num_modules = 1;
|
||||
comp->portals_modules = calloc(comp->portals_num_modules,
|
||||
sizeof(mca_bmi_portals_module_t *));
|
||||
if (NULL == comp->portals_modules) {
|
||||
ompi_output_verbose(10, mca_bmi_portals_component.portals_output,
|
||||
"malloc failed in mca_bmi_portals_init");
|
||||
return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
|
||||
}
|
||||
comp->portals_modules[0] = malloc(sizeof(mca_bmi_portals_module_t));
|
||||
if (NULL == comp->portals_modules) {
|
||||
ompi_output_verbose(10, mca_bmi_portals_component.portals_output,
|
||||
"malloc failed in mca_bmi_portals_init");
|
||||
return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
|
||||
}
|
||||
bmi = comp->portals_modules[0];
|
||||
|
||||
*bmi = = mca_bmi_portals_module;
|
||||
|
||||
/*
|
||||
* Initialize a network device
|
||||
*/
|
||||
ret = PtlNIInit(PTL_IFACE_DEFAULT, /* interface to initialize */
|
||||
PTL_PID_ANY, /* let library assign our pid */
|
||||
NULL, /* no desired limits */
|
||||
&(bmi->limits), /* save our limits somewhere */
|
||||
&(bmi->ni_handle) /* our interface handle */
|
||||
);
|
||||
if (PTL_OK != ret) {
|
||||
ompi_output_verbose(10, mca_ptl_portals_component.portals_output,
|
||||
"PtlNIInit failed, returning %d\n", ret);
|
||||
return OMPI_ERR_FATAL;
|
||||
}
|
||||
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
int
|
||||
mca_bmi_portals_add_procs_compat(struct mca_bmi_portals_module_t* bmi,
|
||||
size_t nprocs, struct ompi_proc_t **procs,
|
||||
bmi_process_id_t **portals_procs)
|
||||
{
|
||||
int nptl_procs = 0;
|
||||
|
||||
/*
|
||||
* FIXME - XXX - FIXME
|
||||
* BWB - implicit assumption that cnos procs list will match our
|
||||
* procs list. Don't know what to do about that...
|
||||
*/
|
||||
|
||||
nptl_procs = cnos_get_nidpid_map(portals_procs);
|
||||
if (nptl_procs <= 0) {
|
||||
ompi_output_verbose(10, mca_bmi_portals_component.portals_output,
|
||||
"cnos_get_nidpid_map() returned %d", nptl_procs);
|
||||
return OMPI_ERR_FATAL;
|
||||
} else if (nptl_procs != nprocs) {
|
||||
ompi_output_verbose(10, mca_bmi_portals_component.portals_output,
|
||||
"nptl_procs != nprocs (%d, %d)", nptl_procs,
|
||||
nprocs);
|
||||
return OMPI_ERR_FATAL;
|
||||
}
|
||||
|
||||
return OMPI_ERR_NOT_IMPLEMENTED;
|
||||
}
|
||||
219
src/mca/bmi/portals/src/bmi_portals_compat_utcp.c
Обычный файл
219
src/mca/bmi/portals/src/bmi_portals_compat_utcp.c
Обычный файл
@ -0,0 +1,219 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "portals_config.h"
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <unistd.h>
|
||||
#include <stdio.h>
|
||||
#include <errno.h>
|
||||
#include <netinet/in.h>
|
||||
|
||||
#include "include/constants.h"
|
||||
#include "util/output.h"
|
||||
|
||||
#include "bmi_portals.h"
|
||||
#include "bmi_portals_compat.h"
|
||||
|
||||
#include <p3api/debug.h>
|
||||
|
||||
/* how's this for source code diving? - find private method for
|
||||
getting interface */
|
||||
extern unsigned int utcp_my_nid(const char *if_str);
|
||||
|
||||
/* these need to be defined, or things get "unhappy" */
|
||||
FILE* utcp_api_out;
|
||||
FILE* utcp_lib_out;
|
||||
|
||||
int
|
||||
mca_bmi_portals_init(mca_bmi_portals_component_t *comp)
|
||||
{
|
||||
ptl_process_id_t info;
|
||||
int ret;
|
||||
#if 0
|
||||
FILE *output;
|
||||
char *tmp;
|
||||
|
||||
asprintf(&tmp, "portals.%d", getpid());
|
||||
output = fopen(tmp, "w");
|
||||
free(tmp);
|
||||
|
||||
utcp_lib_out = output;
|
||||
utcp_api_out = output;
|
||||
#else
|
||||
utcp_lib_out = stderr;
|
||||
utcp_api_out = stderr;
|
||||
#endif
|
||||
|
||||
info.nid = htonl(utcp_my_nid(mca_bmi_portals_component.portals_ifname));
|
||||
info.pid = htonl((ptl_pid_t) getpid());
|
||||
ompi_output_verbose(100, mca_bmi_portals_component.portals_output,
|
||||
"contact info: %u, %u", ntohl(info.nid),
|
||||
ntohl(info.pid));
|
||||
|
||||
ret = mca_base_modex_send(&mca_bmi_portals_component.super.bmi_version,
|
||||
&info, sizeof(ptl_process_id_t));
|
||||
if (OMPI_SUCCESS != ret) {
|
||||
ompi_output_verbose(10, mca_bmi_portals_component.portals_output,
|
||||
"mca_base_modex_send failed: %d", ret);
|
||||
return ret;
|
||||
}
|
||||
|
||||
/* with the utcp interface, only ever one "NIC" */
|
||||
comp->portals_num_modules = 1;
|
||||
comp->portals_modules = calloc(comp->portals_num_modules,
|
||||
sizeof(mca_bmi_portals_module_t *));
|
||||
if (NULL == comp->portals_modules) {
|
||||
ompi_output_verbose(10, mca_bmi_portals_component.portals_output,
|
||||
"malloc failed in mca_bmi_portals_init");
|
||||
return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
|
||||
}
|
||||
comp->portals_modules[0] = malloc(sizeof(mca_bmi_portals_module_t));
|
||||
if (NULL == comp->portals_modules) {
|
||||
ompi_output_verbose(10, mca_bmi_portals_component.portals_output,
|
||||
"malloc failed in mca_bmi_portals_init");
|
||||
return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
|
||||
}
|
||||
*(comp->portals_modules[0]) = mca_bmi_portals_module;
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
int
|
||||
mca_bmi_portals_add_procs_compat(struct mca_bmi_portals_module_t* bmi,
|
||||
size_t nprocs, struct ompi_proc_t **procs,
|
||||
ptl_process_id_t **portals_procs)
|
||||
{
|
||||
int ret, my_rid;
|
||||
ptl_process_id_t *info;
|
||||
char *nidmap = NULL;
|
||||
char *pidmap = NULL;
|
||||
char *nid_str;
|
||||
char *pid_str;
|
||||
const size_t map_size = nprocs * 12 + 1; /* 12 is max length of long in decimal */
|
||||
size_t size, i;
|
||||
char *tmp;
|
||||
ompi_proc_t* proc_self = ompi_proc_local();
|
||||
int max_interfaces;
|
||||
|
||||
/*
|
||||
* Do all the NID/PID map setup
|
||||
*/
|
||||
/* each nid is a int, so need 10 there, plus the : */
|
||||
nidmap = malloc(map_size);
|
||||
pidmap = malloc(map_size);
|
||||
nid_str = malloc(12 + 1);
|
||||
pid_str = malloc(12 + 1);
|
||||
if (NULL == nidmap || NULL == pidmap || NULL == nid_str || NULL == pid_str)
|
||||
return OMPI_ERROR;
|
||||
|
||||
/* get space for the portals procs list */
|
||||
*portals_procs = calloc(nprocs, sizeof(ptl_process_id_t));
|
||||
if (NULL == *portals_procs) {
|
||||
ompi_output_verbose(10, mca_bmi_portals_component.portals_output,
|
||||
"calloc(nprocs, sizeof(ptl_process_id_t)) failed");
|
||||
return OMPI_ERR_TEMP_OUT_OF_RESOURCE;
|
||||
}
|
||||
|
||||
for (i = 0 ; i < nprocs ; ++i) {
|
||||
if (proc_self == procs[i]) my_rid = i;
|
||||
|
||||
ret = mca_base_modex_recv(&mca_bmi_portals_component.super.bmi_version,
|
||||
procs[i], (void**) &info, &size);
|
||||
if (OMPI_SUCCESS != ret) {
|
||||
ompi_output_verbose(10, mca_bmi_portals_component.portals_output,
|
||||
"mca_base_modex_recv failed: %d", ret);
|
||||
return ret;
|
||||
} else if (sizeof(ptl_process_id_t) != size) {
|
||||
ompi_output_verbose(10, mca_bmi_portals_component.portals_output,
|
||||
"mca_base_modex_recv returned size %d, expected %d",
|
||||
size, sizeof(ptl_process_id_t));
|
||||
return OMPI_ERROR;
|
||||
}
|
||||
|
||||
if (i == 0) {
|
||||
snprintf(nidmap, map_size, "%u", ntohl(info->nid));
|
||||
snprintf(pidmap, map_size, "%u", ntohl(info->pid));
|
||||
} else {
|
||||
snprintf(nid_str, 12 + 1, ":%u", ntohl(info->nid));
|
||||
snprintf(pid_str, 12 + 1, ":%u", ntohl(info->pid));
|
||||
strncat(nidmap, nid_str, 12);
|
||||
strncat(pidmap, pid_str, 12);
|
||||
}
|
||||
|
||||
/* update my local array of proc structs */
|
||||
(*portals_procs)[i].nid = ntohl(info->nid);
|
||||
(*portals_procs)[i].pid = ntohl(info->pid);
|
||||
|
||||
free(info);
|
||||
}
|
||||
|
||||
ompi_output_verbose(100, mca_bmi_portals_component.portals_output,
|
||||
"my rid: %u", my_rid);
|
||||
ompi_output_verbose(100, mca_bmi_portals_component.portals_output,
|
||||
"nid map: %s", nidmap);
|
||||
ompi_output_verbose(100, mca_bmi_portals_component.portals_output,
|
||||
"pid map: %s", pidmap);
|
||||
ompi_output_verbose(100, mca_bmi_portals_component.portals_output,
|
||||
"iface: %s",
|
||||
mca_bmi_portals_component.portals_ifname);
|
||||
|
||||
asprintf(&tmp, "PTL_MY_RID=%u", my_rid);
|
||||
putenv(tmp);
|
||||
asprintf(&tmp, "PTL_NIDMAP=%s", nidmap);
|
||||
putenv(tmp);
|
||||
asprintf(&tmp, "PTL_PIDMAP=%s", pidmap);
|
||||
putenv(tmp);
|
||||
asprintf(&tmp, "PTL_IFACE=%s", mca_bmi_portals_component.portals_ifname);
|
||||
putenv(tmp);
|
||||
|
||||
free(pidmap);
|
||||
free(nidmap);
|
||||
free(pid_str);
|
||||
free(nid_str);
|
||||
|
||||
/*
|
||||
* Initialize Portals
|
||||
*/
|
||||
ret = PtlInit(&max_interfaces);
|
||||
if (PTL_OK != ret) {
|
||||
ompi_output_verbose(10, mca_bmi_portals_component.portals_output,
|
||||
"PtlInit failed, returning %d\n", ret);
|
||||
return OMPI_ERR_FATAL;
|
||||
}
|
||||
|
||||
ret = PtlNIInit(PTL_IFACE_DEFAULT, /* interface to initialize */
|
||||
PTL_PID_ANY, /* let library assign our pid */
|
||||
NULL, /* no desired limits */
|
||||
&(bmi->limits), /* save our limits somewhere */
|
||||
&(bmi->ni_handle) /* our interface handle */
|
||||
);
|
||||
if (PTL_OK != ret) {
|
||||
ompi_output_verbose(10, mca_bmi_portals_component.portals_output,
|
||||
"PtlNIInit failed, returning %d\n", ret);
|
||||
return OMPI_ERR_FATAL;
|
||||
}
|
||||
|
||||
#if 0
|
||||
PtlNIDebug(bmi->ni_handle, PTL_DBG_ALL | PTL_DBG_NI_ALL);
|
||||
#endif
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
345
src/mca/bmi/portals/src/bmi_portals_component.c
Обычный файл
345
src/mca/bmi/portals/src/bmi_portals_component.c
Обычный файл
@ -0,0 +1,345 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "portals_config.h"
|
||||
|
||||
#include <sys/types.h>
|
||||
#include <unistd.h>
|
||||
|
||||
#include "include/constants.h"
|
||||
|
||||
#include "util/output.h"
|
||||
#include "threads/thread.h"
|
||||
|
||||
#include "bmi_portals.h"
|
||||
#include "bmi_portals_compat.h"
|
||||
|
||||
|
||||
mca_bmi_portals_component_t mca_bmi_portals_component = {
|
||||
{
|
||||
/* First, the mca_base_module_t struct containing meta
|
||||
information about the module itself */
|
||||
{
|
||||
/* Indicate that we are a pml v1.0.0 module (which also
|
||||
implies a specific MCA version) */
|
||||
|
||||
MCA_BMI_BASE_VERSION_1_0_0,
|
||||
|
||||
"portals", /* MCA module name */
|
||||
1, /* MCA module major version */
|
||||
0, /* MCA module minor version */
|
||||
0, /* MCA module release version */
|
||||
mca_bmi_portals_component_open, /* module open */
|
||||
mca_bmi_portals_component_close /* module close */
|
||||
},
|
||||
|
||||
/* Next the MCA v1.0.0 module meta data */
|
||||
|
||||
{
|
||||
/* Whether the module is checkpointable or not */
|
||||
|
||||
false
|
||||
},
|
||||
|
||||
mca_bmi_portals_component_init,
|
||||
mca_bmi_portals_component_progress,
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
static ompi_output_stream_t portals_output_stream = {
|
||||
true, /* is debugging */
|
||||
0, /* verbose level */
|
||||
0, /* want syslog */
|
||||
0, /* syslog priority */
|
||||
NULL, /* syslog ident */
|
||||
NULL, /* prefix */
|
||||
true, /* want stdout */
|
||||
false, /* want stderr */
|
||||
false, /* want file */
|
||||
false, /* file append */
|
||||
"bmi-portals" /* file suffix */
|
||||
};
|
||||
|
||||
|
||||
static inline char*
|
||||
param_register_string(const char* param_name,
|
||||
const char* default_value)
|
||||
{
|
||||
char *param_value;
|
||||
int id = mca_base_param_register_string("bmi", "portals",
|
||||
param_name, NULL,
|
||||
default_value);
|
||||
mca_base_param_lookup_string(id, ¶m_value);
|
||||
return param_value;
|
||||
}
|
||||
|
||||
|
||||
static inline int
|
||||
param_register_int(const char* param_name,
|
||||
int default_value)
|
||||
{
|
||||
int id = mca_base_param_register_int("bmi", "portals", param_name,
|
||||
NULL, default_value);
|
||||
int param_value = default_value;
|
||||
mca_base_param_lookup_int(id, ¶m_value);
|
||||
return param_value;
|
||||
}
|
||||
|
||||
|
||||
|
||||
int
|
||||
mca_bmi_portals_component_open(void)
|
||||
{
|
||||
int i;
|
||||
|
||||
/* initialize state */
|
||||
mca_bmi_portals_component.portals_num_modules = 0;
|
||||
mca_bmi_portals_component.portals_modules = NULL;
|
||||
|
||||
/* initialize objects */
|
||||
#if 0
|
||||
OBJ_CONSTRUCT(&mca_bmi_portals_component.portals_send_frags,
|
||||
ompi_free_list_t);
|
||||
OBJ_CONSTRUCT(&mca_bmi_portals_component.portals_recv_frags,
|
||||
ompi_free_list_t);
|
||||
OBJ_CONSTRUCT(&mca_bmi_portals_component.portals_pending_acks,
|
||||
ompi_list_t);
|
||||
#endif
|
||||
OBJ_CONSTRUCT(&mca_bmi_portals_component.portals_lock,
|
||||
ompi_mutex_t);
|
||||
|
||||
/* register portals module parameters */
|
||||
#if BMI_PORTALS_UTCP
|
||||
mca_bmi_portals_component.portals_ifname =
|
||||
param_register_string("ifname", "eth0");
|
||||
#endif
|
||||
portals_output_stream.lds_verbose_level =
|
||||
param_register_int("debug_level",
|
||||
BMI_PORTALS_DEFAULT_DEBUG_LEVEL);
|
||||
|
||||
mca_bmi_portals_component.portals_free_list_init_num =
|
||||
param_register_int("free_list_init_num",
|
||||
BMI_PORTALS_DEFAULT_FREE_LIST_INIT_NUM);
|
||||
mca_bmi_portals_component.portals_free_list_max_num =
|
||||
param_register_int("free_list_max_num",
|
||||
BMI_PORTALS_DEFAULT_FREE_LIST_MAX_NUM);
|
||||
mca_bmi_portals_component.portals_free_list_inc_num =
|
||||
param_register_int("free_list_inc_num",
|
||||
BMI_PORTALS_DEFAULT_FREE_LIST_inc_NUM);
|
||||
|
||||
#if 0
|
||||
mca_bmi_portals_module.super.bmi_cache_size =
|
||||
param_register_int("request_cache_size",
|
||||
BMI_PORTALS_DEFAULT_REQUEST_CACHE_SIZE);
|
||||
mca_bmi_portals_module.super.bmi_first_frag_size =
|
||||
param_register_int("first_frag_size",
|
||||
BMI_PORTALS_DEFAULT_FIRST_FRAG_SIZE);
|
||||
mca_bmi_portals_module.super.bmi_min_frag_size =
|
||||
param_register_int("rndv_frag_min_size",
|
||||
BMI_PORTALS_DEFAULT_RNDV_FRAG_MIN_SIZE);
|
||||
mca_bmi_portals_module.super.bmi_max_frag_size =
|
||||
param_register_int("rndv_frag_max_size",
|
||||
BMI_PORTALS_DEFAULT_RNDV_FRAG_MAX_SIZE);
|
||||
#endif
|
||||
|
||||
mca_bmi_portals_module.first_frag_num_entries =
|
||||
param_register_int("first_frag_num_entries",
|
||||
BMI_PORTALS_DEFAULT_FIRST_FRAG_NUM_ENTRIES);
|
||||
mca_bmi_portals_module.first_frag_entry_size =
|
||||
param_register_int("first_frag_entry_size",
|
||||
BMI_PORTALS_DEFAULT_FIRST_FRAG_ENTRY_SIZE);
|
||||
|
||||
mca_bmi_portals_module.eq_sizes[MCA_BMI_PORTALS_EQ_RECV] =
|
||||
param_register_int("recv_queue_size",
|
||||
BMI_PORTALS_DEFAULT_RECV_QUEUE_SIZE);
|
||||
mca_bmi_portals_module.eq_sizes[MCA_BMI_PORTALS_EQ_SEND] =
|
||||
(param_register_int("send_queue_size",
|
||||
BMI_PORTALS_DEFAULT_SEND_QUEUE_SIZE)) * 3;
|
||||
|
||||
/* finish with objects */
|
||||
asprintf(&(portals_output_stream.lds_prefix),
|
||||
"bmi: portals (%5d): ", getpid());
|
||||
|
||||
mca_bmi_portals_component.portals_output =
|
||||
ompi_output_open(&portals_output_stream);
|
||||
|
||||
/* fill in remaining defaults for module data */
|
||||
for (i = 0 ; i < MCA_BMI_PORTALS_EQ_SIZE ; ++i) {
|
||||
mca_bmi_portals_module.eq_handles[i] = PTL_EQ_NONE;
|
||||
}
|
||||
|
||||
mca_bmi_portals_module.ni_handle = PTL_INVALID_HANDLE;
|
||||
mca_bmi_portals_module.dropped = 0;
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
int
|
||||
mca_bmi_portals_component_close(void)
|
||||
{
|
||||
/* print out debugging if anything is pending */
|
||||
/* BWB - implement me, if possible */
|
||||
|
||||
/* release resources */
|
||||
OBJ_DESTRUCT(&mca_bmi_portals_component.portals_lock);
|
||||
#if 0
|
||||
OBJ_DESTRUCT(&mca_bmi_portals_component.portals_recv_frags);
|
||||
OBJ_DESTRUCT(&mca_bmi_portals_component.portals_pending_acks);
|
||||
OBJ_DESTRUCT(&mca_bmi_portals_component.portals_lock);
|
||||
#endif
|
||||
|
||||
if (NULL != mca_bmi_portals_component.portals_ifname) {
|
||||
free(mca_bmi_portals_component.portals_ifname);
|
||||
}
|
||||
|
||||
if (NULL != portals_output_stream.lds_prefix) {
|
||||
free(portals_output_stream.lds_prefix);
|
||||
}
|
||||
|
||||
ompi_output_close(mca_bmi_portals_component.portals_output);
|
||||
mca_bmi_portals_component.portals_output = -1;
|
||||
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
mca_bmi_base_module_t**
|
||||
mca_bmi_portals_component_init(int *num_bmis,
|
||||
bool enable_progress_threads,
|
||||
bool enable_mpi_threads)
|
||||
{
|
||||
mca_bmi_base_module_t** bmis;
|
||||
*num_bmis = 0;
|
||||
|
||||
if (enable_progress_threads) {
|
||||
ompi_output_verbose(20, mca_bmi_portals_component.portals_output,
|
||||
"disabled because progress threads enabled");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
#if 0
|
||||
ompi_free_list_init(&mca_bmi_portals_component.portals_send_frags,
|
||||
sizeof(mca_bmi_portals_send_frag_t),
|
||||
OBJ_CLASS(mca_bmi_portals_send_frag_t),
|
||||
mca_bmi_portals_component.portals_free_list_init_num,
|
||||
mca_bmi_portals_component.portals_free_list_max_num,
|
||||
mca_bmi_portals_component.portals_free_list_inc_num,
|
||||
NULL); /* use default allocator */
|
||||
|
||||
ompi_free_list_init(&mca_bmi_portals_component.portals_recv_frags,
|
||||
sizeof(mca_bmi_portals_recv_frag_t),
|
||||
OBJ_CLASS(mca_bmi_portals_recv_frag_t),
|
||||
mca_bmi_portals_component.portals_free_list_init_num,
|
||||
mca_bmi_portals_component.portals_free_list_max_num,
|
||||
mca_bmi_portals_component.portals_free_list_inc_num,
|
||||
NULL); /* use default allocator */
|
||||
#endif
|
||||
|
||||
/* initialize portals bmi. note that this is in the compat code because
|
||||
it's fairly non-portable between implementations */
|
||||
if (OMPI_SUCCESS != mca_bmi_portals_init(&mca_bmi_portals_component)) {
|
||||
ompi_output_verbose(20, mca_bmi_portals_component.portals_output,
|
||||
"disabled because compatibility init failed");
|
||||
return NULL;
|
||||
}
|
||||
|
||||
/* return array of bmis */
|
||||
bmis = malloc(mca_bmi_portals_component.portals_num_modules *
|
||||
sizeof(mca_bmi_base_module_t*));
|
||||
if (NULL == bmis) return NULL;
|
||||
|
||||
memcpy(bmis,
|
||||
mca_bmi_portals_component.portals_modules,
|
||||
mca_bmi_portals_component.portals_num_modules *
|
||||
sizeof(mca_bmi_base_module_t*));
|
||||
*num_bmis = mca_bmi_portals_component.portals_num_modules;
|
||||
|
||||
ompi_output_verbose(20, mca_bmi_portals_component.portals_output,
|
||||
"initialized %d modules",
|
||||
*num_bmis);
|
||||
|
||||
return bmis;
|
||||
}
|
||||
|
||||
|
||||
int
|
||||
mca_bmi_portals_component_progress(void)
|
||||
{
|
||||
int num_progressed = 0;
|
||||
size_t i;
|
||||
|
||||
for (i = 0 ; i < mca_bmi_portals_component.portals_num_modules ; ++i) {
|
||||
struct mca_bmi_portals_module_t *module =
|
||||
mca_bmi_portals_component.portals_modules[i];
|
||||
ptl_event_t ev;
|
||||
ptl_sr_value_t numdropped;
|
||||
int which;
|
||||
int ret;
|
||||
|
||||
if (module->eq_handles[MCA_BMI_PORTALS_EQ_SIZE - 1] ==
|
||||
PTL_EQ_NONE) continue; /* they are all initialized at once */
|
||||
|
||||
#if OMPI_ENABLE_DEBUG
|
||||
/* BWB - this is going to kill performance */
|
||||
PtlNIStatus(module->ni_handle,
|
||||
PTL_SR_DROP_COUNT,
|
||||
&numdropped);
|
||||
if (numdropped != module->dropped) {
|
||||
ompi_output_verbose(30, mca_bmi_portals_component.portals_output,
|
||||
"*** Dropped message count changed. %lld, %lld",
|
||||
module->dropped, numdropped);
|
||||
module->dropped = numdropped;
|
||||
}
|
||||
#endif
|
||||
|
||||
ret = PtlEQPoll(module->eq_handles,
|
||||
MCA_BMI_PORTALS_EQ_SIZE, /* number of eq handles */
|
||||
10,
|
||||
&ev,
|
||||
&which);
|
||||
if (PTL_EQ_EMPTY == ret) {
|
||||
/* nothing to see here - move along */
|
||||
continue;
|
||||
} else if (!(PTL_OK == ret || PTL_EQ_DROPPED == ret)) {
|
||||
/* BWB - how can we report errors? */
|
||||
ompi_output(mca_bmi_portals_component.portals_output,
|
||||
"Error calling PtlEQGet: %d", ret);
|
||||
continue;
|
||||
} else if (PTL_EQ_DROPPED == ret) {
|
||||
ompi_output_verbose(10, mca_bmi_portals_component.portals_output,
|
||||
"*** Event queue entries were dropped");
|
||||
}
|
||||
|
||||
#if BMI_PORTALS_HAVE_EVENT_UNLINK
|
||||
/* not everyone has UNLINK. Use it only to print the event,
|
||||
so we can make sure we properly re-initialize the ones that
|
||||
need to be re-initialized */
|
||||
if (PTL_EVENT_UNLINK == ev.type) {
|
||||
OMPI_OUTPUT_VERBOSE((100, mca_bmi_portals_component.portals_output,
|
||||
"unlink event occurred"));
|
||||
continue;
|
||||
}
|
||||
#endif
|
||||
|
||||
num_progressed++;
|
||||
}
|
||||
|
||||
return num_progressed;
|
||||
}
|
||||
|
||||
107
src/mca/bmi/portals/src/bmi_portals_stubs.c
Обычный файл
107
src/mca/bmi/portals/src/bmi_portals_stubs.c
Обычный файл
@ -0,0 +1,107 @@
|
||||
/*
|
||||
* Copyright (c) 2004-2005 The Trustees of Indiana University.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
|
||||
* All rights reserved.
|
||||
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
|
||||
* University of Stuttgart. All rights reserved.
|
||||
* Copyright (c) 2004-2005 The Regents of the University of California.
|
||||
* All rights reserved.
|
||||
* $COPYRIGHT$
|
||||
*
|
||||
* Additional copyrights may follow
|
||||
*
|
||||
* $HEADER$
|
||||
*/
|
||||
|
||||
|
||||
#include "ompi_config.h"
|
||||
#include "portals_config.h"
|
||||
|
||||
#include "bmi_portals.h"
|
||||
|
||||
/* BWB - README - BWB - README - BWB - README - BWB - README - BWB
|
||||
*
|
||||
* These are stub functions that return error so that the
|
||||
* initialization code can be developed and the whole thing will
|
||||
* link. This file will disappear once all functions are
|
||||
* implemented. Do not implement any functions in this file.
|
||||
*
|
||||
* BWB - README - BWB - README - BWB - README - BWB - README - BWB */
|
||||
|
||||
|
||||
int
|
||||
mca_bmi_portals_register(struct mca_bmi_base_module_t* bmi,
|
||||
mca_bmi_base_tag_t tag,
|
||||
mca_bmi_base_module_recv_cb_fn_t cbfunc,
|
||||
void* cbdata)
|
||||
{
|
||||
return OMPI_SUCCESS;
|
||||
}
|
||||
|
||||
|
||||
mca_bmi_base_descriptor_t*
|
||||
mca_bmi_portals_alloc(struct mca_bmi_base_module_t* bmi,
|
||||
size_t size)
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
|
||||
int
|
||||
mca_bmi_portals_free(struct mca_bmi_base_module_t* bmi,
|
||||
mca_bmi_base_descriptor_t* des)
|
||||
{
|
||||
return OMPI_ERR_NOT_IMPLEMENTED;
|
||||
}
|
||||
|
||||
|
||||
mca_bmi_base_descriptor_t*
|
||||
mca_bmi_portals_prepare_src(struct mca_bmi_base_module_t* bmi,
|
||||
struct mca_bmi_base_endpoint_t* peer,
|
||||
mca_mpool_base_registration_t* registration,
|
||||
struct ompi_convertor_t* convertor,
|
||||
size_t reserve,
|
||||
size_t* size)
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
||||
mca_bmi_base_descriptor_t*
|
||||
mca_bmi_portals_prepare_dst(struct mca_bmi_base_module_t* bmi,
|
||||
struct mca_bmi_base_endpoint_t* peer,
|
||||
mca_mpool_base_registration_t* registration,
|
||||
struct ompi_convertor_t* convertor,
|
||||
size_t reserve,
|
||||
size_t* size)
|
||||
{
|
||||
return NULL;
|
||||
}
|
||||
|
||||
|
||||
int
|
||||
mca_bmi_portals_send(struct mca_bmi_base_module_t* bmi,
|
||||
struct mca_bmi_base_endpoint_t* bmi_peer,
|
||||
struct mca_bmi_base_descriptor_t* descriptor,
|
||||
mca_bmi_base_tag_t tag)
|
||||
{
|
||||
return OMPI_ERR_NOT_IMPLEMENTED;
|
||||
}
|
||||
|
||||
|
||||
int
|
||||
mca_bmi_portals_put(struct mca_bmi_base_module_t* bmi,
|
||||
struct mca_bmi_base_endpoint_t* bmi_peer,
|
||||
struct mca_bmi_base_descriptor_t* decriptor)
|
||||
{
|
||||
return OMPI_ERR_NOT_IMPLEMENTED;
|
||||
}
|
||||
|
||||
|
||||
int
|
||||
mca_bmi_portals_get(struct mca_bmi_base_module_t* bmi,
|
||||
struct mca_bmi_base_endpoint_t* bmi_peer,
|
||||
struct mca_bmi_base_descriptor_t* decriptor)
|
||||
{
|
||||
return OMPI_ERR_NOT_IMPLEMENTED;
|
||||
}
|
||||
Загрузка…
x
Ссылка в новой задаче
Block a user