ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

This PTL is long out of date and doesn't even compile. If we ever

Просмотр исходного кода 
need it back, it can be pulled out of the SVN history.

This commit was SVN r6477.
Этот коммит содержится в:
Jeff Squyres 2005-07-14 02:24:07 +00:00
родитель 44ace2f64e
Коммит 3d69646177
29 изменённых файлов: 0 добавлений и 6247 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

0
ompi/mca/ptl/elan/.ompi_ignore
Просмотреть файл

53
ompi/mca/ptl/elan/Makefile.am
Просмотреть файл

@ -1,53 +0,0 @@
#
# Copyright (c) 2004-2005 The Trustees of Indiana University.
# All rights reserved.
# Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
# All rights reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# Use the top-level Makefile.options
include $(top_ompi_srcdir)/config/Makefile.options
AM_CPPFLAGS = -I$(top_ompi_builddir)/src/include \
-I$(top_ompi_srcdir)/src -I$(top_ompi_srcdir)/src/include \
-I/usr/lib/qsnet/elan4/include
SUBDIRS = tests
sources =
include src/Makefile.extra
# Make the output library in this directory, and name it either
# mca_<type>_<name>.la (for DSO builds) or libmca_<type>_<name>.la
# (for static builds).
if OMPI_BUILD_ptl_elan_DSO
lib =
lib_sources =
component = mca_ptl_elan.la
component_sources = $(sources)
else
lib = libmca_ptl_elan.la
lib_sources = $(sources)
component =
component_sources =
endif
mcacomponentdir = $(libdir)/openmpi
mcacomponent_LTLIBRARIES = $(component)
mca_ptl_elan_la_SOURCES = $(component_sources)
mca_ptl_elan_la_LDFLAGS = -module -avoid-version
noinst_LTLIBRARIES = $(lib)
libmca_ptl_elan_la_SOURCES = $(lib_sources)
libmca_ptl_elan_la_LDFLAGS = -module -avoid-version

23
ompi/mca/ptl/elan/configure.params
Просмотреть файл

@ -1,23 +0,0 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University.
# All rights reserved.
# Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
# All rights reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
# Specific to this module
PARAM_INIT_FILE=src/ptl_elan.c
PARAM_CONFIG_HEADER_FILE="src/elan_config.h"
PARAM_CONFIG_FILES="Makefile tests/Makefile"
PARAM_WANT_CXX=no

93
ompi/mca/ptl/elan/configure.stub
Просмотреть файл

@ -1,93 +0,0 @@
# -*- shell-script -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University.
# All rights reserved.
# Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
# All rights reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
#
# Main function. This will be invoked in the middle of the templated
# configure script.
#
AC_DEFUN([MCA_CONFIGURE_STUB],[
AC_ARG_WITH(ptl-elan-qsnet-headers,
AC_HELP_STRING([--with-ptl-elan-qsnet-headers=DIR],
[The path to qsnet header files, default /usr ]))
EXTRA_CPPFLAGS=
if test -n "$with_ptl_elan_qsnet_headers"; then
EXTRA_CPPFLAGS="-I$with_ptl_elan_qsnet_headers/include/"
else
AC_MSG_WARN([*** Warning: using default qsnet header path /usr/include])
fi
# See if we can find a sample kernel header
CPPFLAGS="$EXTRA_CPPFLAGS $CPPFLAGS "
AC_CHECK_HEADERS([qsnet/config.h],,
[AC_MSG_ERROR([*** Cannot find qsnet header files, e.g., config.h])])
# Look for a bunch of libraries; abort if we don't have them
PTL_ELAN_LIBS="$LIBS"
pairs="rmscall:rms_getcap elan:elan_init elan4:elan4_init"
for pair in $pairs; do
lib="`echo $pair | cut -d: -f1`"
func="`echo $pair | cut -d: -f2`"
AC_CHECK_LIB([$lib], [$func], [happy=yes], [happy=])
if test -z "$happy"; then
AC_MSG_WARN([*** Cannot find lib$lib])
AC_MSG_ERROR([Cannot continue])
fi
PTL_ELAN_LIBS="$PTL_ELAN_LIBS -l$lib"
done
# We only need one of elanctrl or elan3
AC_CHECK_LIB([elanctrl], [elanctrl_open],
[happy=yes PTL_ELAN_LIBS="$PTL_ELAN_LIBS -lelanctrl"], [happy=])
if test -z "$happy"; then
AC_CHECK_LIB([elan3], [elan3_create],
[happy=yes PTL_ELAN_LIBS="$PTL_ELAN_LIBS -lelan3"], [happy=])
fi
if test -z "$happy"; then
AC_MSG_WARN([*** Cannot find libelan3 or libelanctrl])
AC_MSG_ERROR([Cannot continue])
fi
# Some versions of AC seem to automatically append LIBS with the
# result of a successful AC_CHECK_LIB. So ensure to override this
# here.
LIBS="$PTL_ELAN_LIBS"
# Need the elan qs2netlib source as well
AC_ARG_WITH(ptl_elan_qsnet2libsrc,
AC_HELP_STRING([--with-ptl-elan-qsnet2libsrc],
[provide the path to qsnet2lib source]))
QSNET2SRC="$with_ptl_elan_qsnet2libsrc"
if test -z "$QSNET2SRC"; then
AC_MSG_WARN([*** Need path to qsnet2 library source; please use --with-ptl-elan-qs2netlibsrc=/path/to/qsnet2/source])
AC_MSG_ERROR([Cannot continue])
else
CPPFLAGS="$CPPFLAGS -I${QSNET2SRC}/include -I${QSNET2SRC}/elan4lib/include -I${QSNET2SRC}/elan4lib/elan4 -I${QSNET2SRC}/elan4lib/common"
fi
AC_SUBST(LIBS)
AC_SUBST(CPPFLAGS)
AC_SUBST(QSNET2SRC)
WRAPPER_EXTRA_LDFLAGS="$LDFLAGS"
WRAPPER_EXTRA_LIBS="$PTL_ELAN_LIBS"
])dnl

37
ompi/mca/ptl/elan/src/Makefile.extra
Просмотреть файл

@ -1,37 +0,0 @@
# -*- makefile -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University.
# All rights reserved.
# Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
# All rights reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
AM_CPPFLAGS = -fPIC -D_ELAN4 -D_REENTRANT -O3 -march=i686
AM_CPPFLAGS += -I$(top_ompi_builddir)/src/include \
-I$(top_ompi_srcdir)/src -I$(top_ompi_srcdir)/src/include \
-I/usr/lib/qsnet/elan4/include
sources += \
src/elan_config.h \
src/ptl_elan.h \
src/ptl_elan_frag.h \
src/ptl_elan_proc.h \
src/ptl_elan_peer.h \
src/ptl_elan_priv.h \
src/ptl_elan_frag.c \
src/ptl_elan_proc.c \
src/ptl_elan_priv.c \
src/ptl_elan_peer.c \
src/ptl_elan_init.c \
src/ptl_elan_comm_init.c \
src/ptl_elan_component.c \
src/ptl_elan.c

490
ompi/mca/ptl/elan/src/ptl_elan.c
Просмотреть файл

@ -1,490 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <string.h>
#include "opal/util/output.h"
#include "opal/util/if.h"
#include "mca/pml/pml.h"
#include "mca/ptl/ptl.h"
#include "mca/ptl/base/ptl_base_header.h"
#include "mca/ptl/base/ptl_base_sendfrag.h"
#include "mca/pml/base/pml_base_sendreq.h"
#include "mca/pml/base/pml_base_recvreq.h"
#include "mca/pml/teg/src/pml_teg_proc.h"
#include "mca/ptl/base/ptl_base_recvfrag.h"
#include "mca/pml/base/pml_base_module_exchange.h"
#include "ptl_elan.h"
#include "ptl_elan_peer.h"
#include "ptl_elan_proc.h"
#include "ptl_elan_frag.h"
#include "ptl_elan_priv.h"
/* XXX: There must be multiple PTL's. This could be the template */
mca_ptl_elan_module_t mca_ptl_elan_module = {
{
&mca_ptl_elan_component.super,
1,
sizeof(mca_ptl_elan_send_frag_t),
0, /* ptl_exclusivity */
0, /* ptl_latency */
0, /* ptl_bandwidth */
0, /* ptl_frag_first_size */
0, /* ptl_frag_min_size */
0, /* ptl_frag_max_size */
MCA_PTL_PUT, /* ptl flags */
/* collection of interfaces */
mca_ptl_elan_add_procs,
mca_ptl_elan_del_procs,
mca_ptl_elan_finalize,
mca_ptl_elan_isend,
mca_ptl_elan_put,
mca_ptl_elan_get,
mca_ptl_elan_matched,
mca_ptl_elan_req_init,
mca_ptl_elan_req_fini,
NULL,
NULL,
NULL
}
};
int
mca_ptl_elan_add_procs (struct mca_ptl_base_module_t *ptl,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_ptl_base_peer_t **peers,
ompi_bitmap_t * reachable)
{
struct ompi_proc_t *ompi_proc;
mca_ptl_elan_proc_t *ptl_proc;
mca_ptl_elan_peer_t *ptl_peer;
int i;
/* Here nprocs is the number of peer processes */
for (i = 0; i < nprocs; i++) {
ompi_proc = procs[i];
ptl_proc = mca_ptl_elan_proc_create (ompi_proc);
if (NULL == ptl_proc) {
opal_output (0,
"[%s:%d] could not create a ptl proc\n",
__FILE__, __LINE__);
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* Check to make sure that the peer has at least as many
* interface addresses exported as we are trying to use.
* If not, then don't bind this PTL instance to the proc.
*/
OPAL_THREAD_LOCK (&ptl_proc->proc_lock);
if (ptl_proc->proc_addr_count == ptl_proc->proc_peer_count) {
OPAL_THREAD_UNLOCK (&ptl_proc->proc_lock);
opal_output (0, "all peers are taken already\n");
return OMPI_ERR_UNREACH;
}
/* The ptl_proc datastructure is shared by all PTL
* instances that are trying to reach this destination.
* Cache the peer instance on the ptl_proc.
*/
ptl_peer = OBJ_NEW (mca_ptl_elan_peer_t);
if (NULL == ptl_peer) {
OPAL_THREAD_UNLOCK (&ptl_proc->proc_lock);
opal_output (0, "[%s:%d] unabled to allocate ptl_peer \n",
__FILE__, __LINE__);
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* TODO: Make the add_procs function cleaner and simpler
* 1) Since elan_proc_t will only have one vp to address the
* remote processes, there will be only one ptl_peer_t per proc.
* The information to be stored there should be just
* the addressing information, which is the elan_vp.
* If needed, the information can be expanded with memory-handle,
* queue-handle, put/get-handle, memory-statics.
* 2) XXX: Consider matching elan_vp and ompi_proc_name_t.
*/
ptl_peer->peer_ptl = (mca_ptl_elan_module_t *) ptl;
ptl_peer->peer_proc = ptl_proc;
/* There is only one peer per elan_peer_proc_t. */
ptl_proc->proc_peers[0] = ptl_peer;
if (ptl_proc == mca_ptl_elan_component.elan_local) {
ptl_peer->peer_vp = ((mca_ptl_elan_module_t *)ptl)->elan_vp;
} else {
ptl_peer->peer_vp = ptl_proc->proc_addrs->elan_vp;
ptl_proc->proc_addrs->inuse = 1;
}
ptl_peer->peer_rails = ((mca_ptl_elan_module_t *)ptl)->ptl_ni_total;
/* There is only one peer per elan_peer_proc_t */
ptl_proc->proc_peer_count = 1;
ompi_bitmap_set_bit (reachable, i);
OPAL_THREAD_UNLOCK (&ptl_proc->proc_lock);
peers[i] = (struct mca_ptl_base_peer_t *) ptl_peer;
}
return OMPI_SUCCESS;
}
int
mca_ptl_elan_del_procs (struct mca_ptl_base_module_t *ptl,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_ptl_base_peer_t **peers)
{
int i;
for (i = 0; i < nprocs; i++) {
OBJ_RELEASE (peers[i]);
}
return OMPI_SUCCESS;
}
int
mca_ptl_elan_finalize (struct mca_ptl_base_module_t *ptl)
{
int rail_index;
struct mca_ptl_elan_module_t *elan_ptl;
/* TODO: move from mca_ptl_elan_component_close()
all the ptl_elan_finalize related code here */
#if 0
elan_ptl = (struct mca_ptl_elan_module_t *) ptl;
/* XXX: Free all the lists, etc, hanged over PTL
* before freeing the PTLs */
rail_index = elan_ptl->ptl_ni_local;
free (elan_ptl);
/* Record the missing of this entry */
mca_ptl_elan_component.modules[rail_index] = NULL;
mca_ptl_elan_component.num_modules--;
#endif
return OMPI_SUCCESS;
}
int
mca_ptl_elan_req_init (struct mca_ptl_base_module_t *ptl,
struct mca_pml_base_send_request_t *request)
{
mca_ptl_elan_send_frag_t *desc;
desc = mca_ptl_elan_alloc_desc(ptl,
(struct mca_pml_base_request_t *) request,
MCA_PTL_ELAN_DESC_QDMA);
if (NULL == desc) {
opal_output(0,
"[%s:%d] Unable to allocate an elan send descriptors \n",
__FILE__, __LINE__);
return OMPI_ERR_OUT_OF_RESOURCE;
} else {
/* XXX: Hope PML never writes into the fragment */
((mca_ptl_elan_send_request_t *)request)->req_frag = desc;
}
desc->desc->desc_status = MCA_PTL_ELAN_DESC_CACHED;
return OMPI_SUCCESS;
}
void
mca_ptl_elan_req_fini (struct mca_ptl_base_module_t *ptl,
struct mca_pml_base_send_request_t *request)
{
/* XXX: Lock to be added */
ompi_ptl_elan_queue_ctrl_t *queue;
mca_ptl_elan_send_frag_t *desc;
/* return the fragment and update the status */
queue = ((struct mca_ptl_elan_module_t * )ptl)->queue;
desc = ((mca_ptl_elan_send_request_t *) request)->req_frag;
OMPI_FREE_LIST_RETURN (&queue->tx_desc_free, (opal_list_item_t *) desc);
desc->desc->desc_status = MCA_PTL_ELAN_DESC_LOCAL;
return;
}
void
mca_ptl_elan_recv_frag_return (struct mca_ptl_base_module_t *ptl,
struct mca_ptl_elan_recv_frag_t *frag)
{
OMPI_FREE_LIST_RETURN(&mca_ptl_elan_component.elan_recv_frags_free,
(opal_list_item_t*)frag);
return;
}
void
mca_ptl_elan_send_frag_return (struct mca_ptl_base_module_t *ptl,
struct mca_ptl_elan_send_frag_t *frag)
{
return;
}
/*
* Initiate an isend operation
*/
int
mca_ptl_elan_isend (struct mca_ptl_base_module_t *ptl,
struct mca_ptl_base_peer_t *ptl_peer,
struct mca_pml_base_send_request_t *sendreq,
size_t offset,
size_t size,
int flags)
{
int rc = OMPI_SUCCESS;
mca_ptl_elan_send_frag_t *desc;
/* XXX:
* PML extract an request from PTL component and then use this
* a request to ask for a fragment
* Is it too deep across stacks to get a request and
* correspondingly multiple LOCKS to go through
*/
if (offset == 0 && sendreq->req_cached) {
/* The first fragment uses a cached desc */
desc = ((mca_ptl_elan_send_request_t*)sendreq)->req_frag;
} else {
desc = mca_ptl_elan_alloc_desc(ptl,
(struct mca_pml_base_request_t *) sendreq,
MCA_PTL_ELAN_DESC_QDMA);
if (NULL == desc) {
opal_output(0,
"[%s:%d] Unable to allocate an elan send descriptors \n",
__FILE__, __LINE__);
}
}
#if OMPI_PTL_ELAN_ZERO_FFRAG
if (sendreq->req_bytes_packed >
(OMPI_PTL_ELAN_MAX_QSIZE - sizeof(mca_ptl_base_header_t)))
size = 0;
#endif
((struct mca_ptl_elan_send_request_t *)sendreq)->req_frag = desc;
rc = mca_ptl_elan_start_desc(desc,
(struct mca_ptl_elan_peer_t *)ptl_peer,
sendreq, offset, &size, flags);
/* Update offset */
sendreq->req_offset += size;
return rc;
}
/*
* Initiate a put operation.
*/
int
mca_ptl_elan_put (struct mca_ptl_base_module_t *ptl,
struct mca_ptl_base_peer_t *ptl_peer,
struct mca_pml_base_send_request_t *sendreq,
size_t offset,
size_t size,
int flags)
{
int rc = OMPI_SUCCESS;
mca_ptl_elan_send_frag_t *desc;
/* PML still utilize this interface the same as a send option.
* So we need to generate a QDMA to the remote side for completion
* notification */
/* XXX:
* Since the address passed down from PML does not provide
* elan information, so there needs to be a change
*/
desc = mca_ptl_elan_alloc_desc(ptl,
(struct mca_pml_base_request_t *) sendreq,
MCA_PTL_ELAN_DESC_PUT);
if (NULL == desc) {
opal_output(0,
"[%s:%d] Unable to allocate an elan send descriptors \n",
__FILE__, __LINE__);
}
rc = mca_ptl_elan_start_desc(desc,
(struct mca_ptl_elan_peer_t *)ptl_peer,
sendreq, offset, &size, flags);
/* XXX: It is very important to update offset,
* otherwise, you stuck */
sendreq->req_offset += size;
/* Update all the sends until the put is done */
return rc;
}
/*
* Get routine. We need an interface that provides one more argument,
* describing the source of the data.
*/
int
mca_ptl_elan_get (struct mca_ptl_base_module_t *ptl,
struct mca_ptl_base_peer_t *ptl_peer,
struct mca_pml_base_recv_request_t *req,
size_t offset,
size_t size,
int flags)
{
int rc = OMPI_SUCCESS;
#if OMPI_PTL_ELAN_ENABLE_GET && defined (HAVE_GET_INTERFACE)
mca_ptl_elan_send_frag_t *desc;
/* TODO:
* There is no interface support in PML.
* So a walkround is to twist the way ack works.
* Step 1: Have elan_isend send E4_addr over
* Step 2: Have elan_matched trigger elan_get,
* with an ack chained, which generate events to both sides.
* Step 3: Done at both the send side and the recv side.
*/
/* XXX:
* Since the address passed down from PML does not provide
* elan information, so there needs to be a change
*/
desc = mca_ptl_elan_alloc_desc(ptl,
(struct mca_pml_base_request_t *) req,
MCA_PTL_ELAN_DESC_GET);
if (NULL == desc) {
opal_output(0,
"[%s:%d] Unable to allocate an elan send descriptors \n",
__FILE__, __LINE__);
}
rc = mca_ptl_elan_start_get(desc, (struct mca_ptl_elan_peer_t *)ptl_peer,
req, offset, &size, flags);
/* XXX: It is very important to update offset,
* otherwise, you stuck */
/*req->req_offset += size;*/
/* Update all the sends until the put is done */
#endif
return rc;
}
/* A posted receive has been matched
* + Copy the data into user buffer
* + Return an ack if need to
*/
void
mca_ptl_elan_matched (mca_ptl_base_module_t * ptl,
mca_ptl_base_recv_frag_t * frag)
{
mca_pml_base_recv_request_t *request;
mca_ptl_base_header_t *header;
mca_ptl_elan_recv_frag_t * recv_frag;
int set = 0;
header = &frag->frag_base.frag_header;
request = frag->frag_request;
recv_frag = (mca_ptl_elan_recv_frag_t * ) frag;
if (header->hdr_common.hdr_flags & MCA_PTL_FLAGS_ACK) {
int desc_type ;
/* Basic ACK scheme following TCP cases */
mca_ptl_elan_send_frag_t *desc;
#if OMPI_PTL_ELAN_ENABLE_GET
desc_type = MCA_PTL_ELAN_DESC_GET;
#else
desc_type = MCA_PTL_ELAN_DESC_QDMA;
#endif
/* Get a frag desc and allocate a send desc */
desc = mca_ptl_elan_alloc_desc(ptl, NULL, desc_type);
if (NULL == desc) {
opal_output(0,
"[%s:%d] Unable to allocate an elan send descriptors \n",
__FILE__, __LINE__);
OPAL_THREAD_LOCK(&mca_ptl_elan_component.elan_lock);
recv_frag->frag_ack_pending = true;
opal_list_append(&((mca_ptl_elan_module_t * )ptl)->pending_acks,
(opal_list_item_t*)frag);
OPAL_THREAD_UNLOCK(&mca_ptl_elan_component.elan_lock);
} else {
/* XXX: recv_frag is released a few lines below,
* pay more attention to timing of the release */
#if OMPI_PTL_ELAN_ENABLE_GET
mca_ptl_elan_get_with_ack (ptl, desc, recv_frag);
LOG_PRINT(PTL_ELAN_DEBUG_GET, "Get desc %p type %d\n",
desc, desc->desc->desc_type);
#else
mca_ptl_elan_start_ack (ptl, desc, recv_frag);
#endif
}
}
set = opal_atomic_fetch_and_set_int (
&((mca_ptl_elan_recv_frag_t *)frag)->frag_progressed, 1);
if (!set) {
/* IN TCP case, IO_VEC is first allocated.
* then recv the data, and copy if needed,
*
* But in ELAN cases, we save the data into an unex buffer
* if the recv descriptor is not posted (for too long) (TODO).
* We then need to copy from unex_buffer to application buffer */
if (header->hdr_frag.hdr_frag_length > 0) {
#if !OMPI_PTL_ELAN_USE_DTP
memcpy(request->req_base.req_addr,
frag->frag_base.frag_addr, frag->frag_base.frag_size);
#else
int iov_count = 1, max_data, freeAfter;
struct iovec iov;
ompi_proc_t *proc;
iov.iov_base = frag->frag_base.frag_addr;
iov.iov_len = frag->frag_base.frag_size;
max_data = iov.iov_len;
proc = ompi_comm_peer_lookup(request->req_base.req_comm,
request->req_base.req_peer);
ompi_convertor_copy(proc->proc_convertor,
&frag->frag_base.frag_convertor);
ompi_convertor_init_for_recv(
&frag->frag_base.frag_convertor,
0,
request->req_base.req_datatype,
request->req_base.req_count,
request->req_base.req_addr,
header->hdr_frag.hdr_frag_offset,
NULL);
ompi_convertor_unpack(&frag->frag_base.frag_convertor,
&iov, &iov_count, &max_data, &freeAfter);
#endif
}
/* XXX: progress the request based on the status of this recv frag
* It is possible to employ a scheduling logic here.
* Then Done with this fragment, i.e., data */
mca_ptl_elan_recv_frag_done (header,
(mca_ptl_elan_recv_frag_t *)frag, request);
}
}

295
ompi/mca/ptl/elan/src/ptl_elan.h
Просмотреть файл

@ -1,295 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PTL_ELAN_H
#define MCA_PTL_ELAN_H
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include "class/ompi_free_list.h"
#include "opal/event/event.h"
#include "mca/pml/pml.h"
#include "mca/ptl/ptl.h"
#include "elan.h"
#include "init.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
struct mca_ptl_elan_state_t;
struct ompi_ptl_elan_queue_ctrl_t;
extern struct mca_ptl_elan_state_t mca_ptl_elan_global_state;
extern struct ompi_ptl_elan_cmdq_space_t ptl_elan_cmdq_space;
/**
* ELAN PTL Interface
*/
struct mca_ptl_elan_module_t {
/**< The elan progress related interface */
mca_ptl_base_module_t super; /**< base PTL interface */
/**< The following are elan-related control structures */
ELAN_RAIL *ptl_elan_rail; /**< Pointer to this Rail */
ELAN_CTX *ptl_elan_ctx; /**< Elan ctx of this rail */
int ptl_ni_local; /**< PTL NI local rank */
int ptl_ni_total; /**< PTL NI total */
/* common elan structures, each ptl keeps a copy */
unsigned int elan_vp; /**< elan vpid, not ompi vpid */
unsigned int elan_nvp; /**< total # of elan vpid */
opal_list_t send_frags; /**< outstanding send/put/get */
opal_list_t recv_frags; /**< outstanding recv's */
opal_list_t pending_acks;
struct ompi_ptl_elan_comp_queue_t *comp; /**< completion queue */
struct ompi_ptl_elan_queue_ctrl_t *queue; /**< Queue ctrl struct*/
struct ompi_ptl_elan_putget_ctrl_t *putget;/**< putget ctrl struct */
};
typedef struct mca_ptl_elan_module_t mca_ptl_elan_module_t;
extern mca_ptl_elan_module_t mca_ptl_elan_module;
/**
* ELAN PTL module.
*/
struct mca_ptl_elan_component_t {
mca_ptl_base_component_t super; /**< base PTL component */
size_t num_modules; /**< number of ptls activated */
size_t free_list_num; /**< min number of list items */
size_t free_list_max; /**< max number of list items*/
size_t free_list_inc; /**< inc for each grow */
/* We create our own simplified structure for managing elan state
* although libelan already provides one. We do not need
* all those tport, group, atomic, shmem and NIC threads support.
*/
struct mca_ptl_elan_state_t *elan_ctrl;
struct mca_ptl_elan_proc_t *elan_local;
struct mca_ptl_elan_module_t **modules; /**< available PTL modules */
struct ompi_ptl_elan_thread_t **recv_threads; /**< send-related threads */
struct ompi_ptl_elan_thread_t **send_threads; /**< recv-related threads*/
opal_mutex_t elan_lock; /**< lock for module state */
opal_list_t elan_procs; /**< elan proc's */
ompi_free_list_t elan_recv_frags_free;
};
typedef struct mca_ptl_elan_component_t mca_ptl_elan_component_t;
struct mca_ptl_elan_send_frag_t;
struct mca_ptl_elan_recv_frag_t;
extern mca_ptl_elan_component_t mca_ptl_elan_component;
/**
* Register ELAN module parameters with the MCA framework
*/
extern int mca_ptl_elan_component_open (void);
/**
* Any final cleanup before being unloaded.
*/
extern int mca_ptl_elan_component_close (void);
/**
* ELAN module initialization.
*
* @param num_ptl_modules (OUT)
* Number of PTLs returned in PTL array.
* @param allow_multi_user_threads (OUT)
* Flag indicating wether PTL supports user threads (TRUE)
* @param have_hidden_threads (OUT)
* Flag indicating whether PTL uses threads (TRUE)
*
*/
extern mca_ptl_base_module_t **
mca_ptl_elan_component_init (int *num_ptl_modules,
bool * multi_user_threads,
bool * have_hidden_threads);
/**
* ELAN module control.
*/
extern int mca_ptl_elan_component_control (int param,
void *value,
size_t size);
/**
* ELAN module progress.
*/
extern int mca_ptl_elan_component_progress (mca_ptl_tstamp_t tstamp);
/**
* Cleanup any resources held by the PTL.
*
* @param ptl PTL instance.
* @return OMPI_SUCCESS or error status on failure.
*/
extern int mca_ptl_elan_finalize (struct mca_ptl_base_module_t *ptl);
/**
* PML->PTL notification of change in the process list.
*
* @param ptl (IN)
* @param proc (IN)
* @param peer (OUT)
* @return OMPI_SUCCESS or error status on failure.
*/
extern int
mca_ptl_elan_add_procs (struct mca_ptl_base_module_t *ptl,
size_t nprocs,
struct ompi_proc_t **procs,
struct mca_ptl_base_peer_t **peers,
ompi_bitmap_t* reachable);
/**
* PML->PTL notification of change in the process list.
*
* @param ptl (IN) PTL instance
* @param proc (IN) Peer process
* @param peer (IN) Peer addressing information.
* @return Status indicating if cleanup was successful
*/
extern int
mca_ptl_elan_del_procs (struct mca_ptl_base_module_t *ptl,
size_t nprocs,
struct ompi_proc_t ** procs,
struct mca_ptl_base_peer_t **peers);
/**
* PML->PTL acquire and initialize a send desc
*
* @param ptl (IN) PTL instance
* @param request (OUT) Pointer to allocated request.
* @return Status indicating if allocation was successful.
*/
extern int
mca_ptl_elan_req_init (struct mca_ptl_base_module_t *ptl,
struct mca_pml_base_send_request_t *req);
/**
* PML->PTL free the cached desc
*
* @param ptl (IN) PTL instance
* @param request (IN) Pointer to allocated request.
*/
extern void mca_ptl_elan_req_fini (struct mca_ptl_base_module_t *ptl,
struct mca_pml_base_send_request_t *);
/**
* PML->PTL Notification that a receive fragment has been matched.
*
* @param ptl (IN) PTL instance
* @param recv_frag (IN) Receive fragment
*/
extern void mca_ptl_elan_matched (struct mca_ptl_base_module_t *ptl,
struct mca_ptl_base_recv_frag_t *frag);
/**
* PML->PTL Initiate an isend operation
*
* @param ptl (IN) PTL instance
* @param ptl_base_peer (IN) PTL peer addressing
* @param send_request (IN/OUT) Send request (allocated by PML via
* mca_ptl_base_request_alloc_fn_t)
* @param size (IN)
* Number of bytes PML is requesting PTL to deliver
* @param flags (IN)
* Flags that should be passed to the peer via the message header.
* @param request (OUT)
* OMPI_SUCCESS if the PTL was able to queue one or more fragments
*/
extern int
mca_ptl_elan_isend (struct mca_ptl_base_module_t* ptl,
struct mca_ptl_base_peer_t* ptl_base_peer,
struct mca_pml_base_send_request_t* request,
size_t offset,
size_t size,
int flags);
/**
* PML->PTL Initiate a put of the specified size.
*
* @param ptl (IN) PTL instance
* @param ptl_base_peer (IN) PTL peer addressing
* @param send_request (IN/OUT) Send request (allocated by PML via
* mca_ptl_base_request_alloc_fn_t)
* @param size (IN)
* Number of bytes PML is requesting PTL to deliver
* @param flags (IN)
* Flags that should be passed to the peer via the message header.
* @param request (OUT)
* OMPI_SUCCESS if the PTL was able to queue one or more fragments
*/
extern int
mca_ptl_elan_put (struct mca_ptl_base_module_t* ptl,
struct mca_ptl_base_peer_t* ptl_base_peer,
struct mca_pml_base_send_request_t* request,
size_t offset,
size_t size,
int flags);
/**
* PML->PTL Initiate a get of the specified size.
*
* @param ptl (IN) PTL instance
* @param ptl_base_peer (IN) PTL peer addressing
* @param send_request (IN/OUT) Send request (allocated by PML via
* mca_ptl_base_request_alloc_fn_t)
* @param size (IN)
* Number of bytes PML is requesting PTL to deliver
* @param flags (IN)
* Flags that should be passed to the peer via the message header.
* @param request (OUT)
* OMPI_SUCCESS if the PTL was able to queue one or more fragments
*/
extern int
mca_ptl_elan_get (struct mca_ptl_base_module_t* ptl,
struct mca_ptl_base_peer_t* ptl_base_peer,
struct mca_pml_base_recv_request_t* request,
size_t offset,
size_t size,
int flags);
/**
* Return a recv fragment to the modules free list.
*
* @param ptl (IN) PTL instance
* @param frag (IN) ELAN receive fragment
*/
extern void mca_ptl_elan_recv_frag_return (struct mca_ptl_base_module_t *ptl,
struct mca_ptl_elan_recv_frag_t
*frag);
/**
* Return a send fragment to the modules free list.
*
* @param ptl (IN) PTL instance
* @param frag (IN) ELAN send fragment
*/
extern void mca_ptl_elan_send_frag_return (struct mca_ptl_base_module_t *ptl,
struct mca_ptl_elan_send_frag_t
*frag);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

652
ompi/mca/ptl/elan/src/ptl_elan_comm_init.c
Просмотреть файл

@ -1,652 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <signal.h>
#include <unistd.h>
#include <stdio.h>
#include "ptl_elan.h"
#include "ptl_elan_priv.h"
#define OMPI_PTL_ELAN_CTRL_LIST(flist, init_num, inc_num, max_num) \
do { \
OBJ_CONSTRUCT (flist, ompi_free_list_t); \
OBJ_CONSTRUCT(&flist->fl_lock, opal_mutex_t); \
flist->fl_elem_size = flist->fl_max_to_alloc = max_num; \
flist->fl_num_allocated = init_num; \
flist->fl_num_per_alloc = inc_num; \
flist->fl_elem_class = NULL; /* leave it null */ \
flist->fl_mpool = NULL; /* leave it null */ \
} while (0)
static int
ompi_init_elan_queue_events (mca_ptl_elan_module_t * ptl,
ompi_ptl_elan_queue_ctrl_t * queue)
{
int i;
int count;
int main_align, main_size;
int elan_align, elan_size;
mca_ptl_elan_send_frag_t *frag;
RAIL *rail;
ELAN4_CTX *ctx;
ompi_free_list_t *flist;
ompi_ptl_elan_qdma_desc_t *desc;
E4_Event32 *elan_ptr;
rail = (RAIL *) ptl->ptl_elan_rail;
ctx = (ELAN4_CTX *) ptl->ptl_elan_ctx;
#if OMPI_PTL_ELAN_CMQ_REUSE
#define OMPI_PTL_ELAN_CMQ_ENTRIES 1024
{
ptl_elan_cmdq_space.total = OMPI_PTL_ELAN_CMQ_ENTRIES;
ptl_elan_cmdq_space.free = OMPI_PTL_ELAN_CMQ_ENTRIES;
ptl_elan_cmdq_space.space = elan4_alloccq_space(ctx,
8*OMPI_PTL_ELAN_CMQ_ENTRIES, CQ_Size8K);
}
#endif
/* initialize list */
OBJ_CONSTRUCT (&queue->tx_desc_free, ompi_free_list_t);
flist = &queue->tx_desc_free;
main_align = OMPI_PTL_ELAN_GET_MAX (sizeof (void *), 8);
elan_align = OMPI_PTL_ELAN_GET_MAX (sizeof (int *), ELAN_BLOCK_ALIGN);
main_size = OMPI_PTL_ELAN_ALIGNUP (sizeof (ompi_ptl_elan_qdma_desc_t),
main_align);
#if OMPI_PTL_ELAN_COMP_QUEUE
elan_size = OMPI_PTL_ELAN_ALIGNUP (
(2*sizeof (E4_Event32) + ELAN_BLOCK_SIZE), elan_align);
#else
elan_size = OMPI_PTL_ELAN_ALIGNUP (sizeof (E4_Event32), elan_align);
#endif
OBJ_CONSTRUCT(&flist->fl_lock, opal_mutex_t);
flist->fl_elem_size = flist->fl_max_to_alloc = OMPI_PTL_ELAN_MAX_QDESCS;
flist->fl_num_allocated = 0;
flist->fl_num_per_alloc = count = OMPI_PTL_ELAN_NUM_QDESCS;
flist->fl_elem_class = NULL; /* leave it null */
flist->fl_mpool = NULL; /* leave it null */
/* Allocate the elements */
frag = (mca_ptl_elan_send_frag_t *)
malloc(sizeof(mca_ptl_elan_send_frag_t) * count);
OMPI_PTL_ELAN_CHECK_UNEX (frag, NULL, OMPI_ERROR, 0);
desc = (ompi_ptl_elan_qdma_desc_t *) elan4_allocMain (rail->r_alloc,
main_align,
main_size * count);
OMPI_PTL_ELAN_CHECK_UNEX (desc, NULL, OMPI_ERROR, 0);
/* Allocating elan related structures */
elan_ptr = (E4_Event32 *) elan4_allocElan (rail->r_alloc,
elan_align, elan_size * count);
OMPI_PTL_ELAN_CHECK_UNEX (elan_ptr, NULL, OMPI_ERROR, 0);
for (i = 0; i < flist->fl_num_per_alloc; i++) {
opal_list_item_t *item;
desc->ptl = ptl;
desc->elan_event = (E4_Event *) elan_ptr;
frag->desc = (ompi_ptl_elan_base_desc_t *)desc;
#if OMPI_PTL_ELAN_COMP_QUEUE
/* XXX: provide a DMA structure for each chained event */
desc->comp_dma.dma_typeSize = E4_DMA_TYPE_SIZE (
sizeof(mca_ptl_base_header_t),
DMA_DataTypeByte, DMA_QueueWrite, 8);
desc->comp_dma.dma_cookie = elan4_local_cookie(ptl->queue->tx_cpool,
E4_COOKIE_TYPE_LOCAL_DMA, ptl->elan_vp);
desc->comp_dma.dma_vproc = ptl->elan_vp;
desc->comp_dma.dma_srcAddr = 0x0ULL; /* To be filled in */
desc->comp_dma.dma_dstAddr = 0x0ULL; /* To be filled in */
/* XXX: If completion is to be detected from the Queue
* there is no need to trigger a local event */
#if OMPI_PTL_ELAN_ONE_QUEUE
desc->comp_dma.dma_dstEvent = elan4_main2elan (ctx,
(void *) ptl->queue->input);
#else
desc->comp_dma.dma_dstEvent = elan4_main2elan (ctx,
(void *) ptl->comp->input);
#endif
desc->comp_dma.dma_srcEvent = 0x0ULL;
desc->comp_dma.dma_typeSize |= RUN_DMA_CMD;
desc->comp_dma.dma_pad = NOP_CMD;
desc->comp_event = (E4_Event *) (elan_ptr + 1);
desc->comp_buff = (E4_Addr *) (elan_ptr + 2);
desc->comp_event->ev_CountAndType = E4_EVENT_INIT_VALUE(-32,
E4_EVENT_COPY, E4_EVENT_DTYPE_LONG, 8);
desc->comp_event->ev_Params[0] = elan4_main2elan (ctx,
(void *)desc->comp_buff);
desc->comp_event->ev_Params[1] = 0x0ULL;
/* Initialize some of the dma structures */
desc->main_dma.dma_dstAddr = 0;
desc->main_dma.dma_srcEvent= elan4_main2elan(ctx,
(E4_Event *)desc->comp_event);
desc->main_dma.dma_dstEvent= SDRAM2ELAN (ctx, queue->input);
LOG_PRINT(PTL_ELAN_DEBUG_DESC,
"desc %p comp_buff %p elan_event %p comp_event %p \n",
desc, desc->comp_buff, desc->elan_event, desc->comp_event);
#else
/* Initialize some of the dma structures */
desc->main_dma.dma_dstAddr = 0;
desc->main_dma.dma_srcEvent = SDRAM2ELAN (ctx, desc->elan_event);
desc->main_dma.dma_dstEvent = SDRAM2ELAN (ctx, queue->input);
INITEVENT_WORD (ctx, desc->elan_event, &desc->main_doneWord);
RESETEVENT_WORD (&desc->main_doneWord);
PRIMEEVENT_WORD (ctx, desc->elan_event, 1);
#endif
item = (opal_list_item_t *) frag;
opal_list_append (&flist->super, item);
/* Progress to the next element */
desc = (ompi_ptl_elan_qdma_desc_t *) ((char *) desc + main_size);
elan_ptr = (E4_Event32 *) ((char *) elan_ptr + elan_size);
frag ++;
}
#if OMPI_PTL_ELAN_THREADING
/* Allocating a DMA:event pair for threads notification */
queue->last = (ompi_ptl_elan_ctrl_desc_t *) elan4_allocMain (
rail->r_alloc, main_align, sizeof(ompi_ptl_elan_ctrl_desc_t));
OMPI_PTL_ELAN_CHECK_UNEX (queue->last, NULL, OMPI_ERROR, 0);
queue->last->elan_event = (E4_Event *) elan4_allocElan (
rail->r_alloc, elan_align, sizeof(E4_Event32));
OMPI_PTL_ELAN_CHECK_UNEX (queue->last->elan_event, NULL, OMPI_ERROR, 0);
queue->last->mesg = malloc(sizeof(mca_ptl_base_header_t));
OMPI_PTL_ELAN_CHECK_UNEX (queue->last->mesg, NULL, OMPI_ERROR, 0);
#endif
flist->fl_num_allocated += flist->fl_num_per_alloc;
return OMPI_SUCCESS;
}
static void
mca_ptl_elan_putget_desc_construct (
mca_ptl_elan_module_t * ptl,
ompi_ptl_elan_putget_desc_t *desc,
EVENT *elan_event,
E4_Addr src_elan4_addr,
E4_Addr dst_elan4_addr,
int local /* dma_src is local */ )
{
ELAN4_CTX *ctx;
ctx = (ELAN4_CTX *)ptl->ptl_elan_ctx;
memset(desc, 0, sizeof(desc));
desc->ptl = ptl;
desc->req = NULL;
#if OMPI_PTL_ELAN_COMP_QUEUE
/* Allocate elan memory for chained event and buff */
desc->chain_buff = (E4_Addr *) ((char *)elan_event);
desc->comp_buff = (E4_Addr *) ((char *)elan_event + ELAN_BLOCK_SIZE );
desc->elan_event = (E4_Event *) ((char *)elan_event + 2*ELAN_BLOCK_SIZE );
desc->chain_event= (E4_Event *) ((char *)elan_event
+ 2 * ELAN_BLOCK_SIZE + sizeof (E4_Event32));
desc->comp_event= (E4_Event *) ((char *)elan_event
+ 2 * ELAN_BLOCK_SIZE + 2 * sizeof (E4_Event32));
LOG_PRINT(PTL_ELAN_DEBUG_DESC,
"desc %p chain_buff %p comp_buff %p elan_event %p "
" chain_event %p comp_event %p \n",
desc, desc->chain_buff, desc->comp_buff, desc->elan_event,
desc->chain_event, desc->comp_event);
/* XXX: provide a DMA structure for each chained event */
desc->comp_dma.dma_typeSize = E4_DMA_TYPE_SIZE (
sizeof(mca_ptl_base_header_t),
DMA_DataTypeByte, DMA_QueueWrite, 8);
desc->comp_dma.dma_vproc = ptl->elan_vp;
desc->comp_dma.dma_srcAddr = 0x0ULL; /* To be filled in */
desc->comp_dma.dma_dstAddr = 0x0ULL; /* To be filled in */
/* XXX: If completion is to be detected from the Queue
* there is no need to trigger a local event */
#if OMPI_PTL_ELAN_ONE_QUEUE
desc->comp_dma.dma_dstEvent = elan4_main2elan (ctx,
(void *) ptl->queue->input);
#else
desc->comp_dma.dma_dstEvent = elan4_main2elan (ctx,
(void *) ptl->comp->input);
#endif
desc->comp_dma.dma_srcEvent = 0x0ULL;
desc->comp_dma.dma_typeSize |= RUN_DMA_CMD;
desc->comp_dma.dma_pad = NOP_CMD;
desc->comp_event->ev_CountAndType = E4_EVENT_INIT_VALUE(-32,
E4_EVENT_COPY, E4_EVENT_DTYPE_LONG, 8);
desc->comp_event->ev_Params[0] = elan4_main2elan (ctx,
(void *)desc->comp_buff);
desc->comp_event->ev_Params[1] = 0x0ULL;
/* Initialize some of the dma structures */
desc->main_dma.dma_srcEvent= elan4_main2elan(ctx,
(E4_Event *)desc->chain_event);
desc->main_dma.dma_dstEvent= 0x0ULL;
#else
desc->elan_event = elan_event;
desc->chain_event= (E4_Event32 *)
((char *)elan_event + sizeof (E4_Event32));
desc->chain_buff = (E4_Addr *)
((char *)elan_event + 2*sizeof (E4_Event32));
if (local) {
desc->main_dma.dma_srcEvent = elan4_main2elan(ctx, elan_event);
} else {
desc->main_dma.dma_dstEvent = elan4_main2elan(ctx, elan_event);
}
/* XXX: Remember to reset all event and doneWord */
INITEVENT_WORD (ctx, elan_event, &desc->main_doneWord);
RESETEVENT_WORD (&desc->main_doneWord);
PRIMEEVENT_WORD (ctx, elan_event, 1);
#endif
/* Make PCI write visable */
mb();
}
#define OMPI_ELAN_PUTGET_GROW(ptl, flist, frag, dp, eptr, msize, esize, local)\
do { \
int i; \
for (i = 0; i < flist->fl_num_per_alloc; i++) { \
opal_list_item_t *item; \
\
frag->desc = (ompi_ptl_elan_base_desc_t *)dp; \
\
/* Initialize some of the dma structures */ \
mca_ptl_elan_putget_desc_construct (ptl, dp, \
eptr, 0, 0, local); \
\
item = (opal_list_item_t *) frag; \
opal_list_append (&flist->super, item); \
\
/* Progress to the next element */ \
dp= (ompi_ptl_elan_putget_desc_t *) ((char *)dp + msize); \
eptr = (E4_Event *) ((char *) eptr + esize); \
frag ++; \
} \
flist->fl_num_allocated += flist->fl_num_per_alloc; \
} while (0)
static int
ompi_ptl_elan_init_putget_ctrl (mca_ptl_elan_module_t * ptl,
RAIL *rail,
ompi_ptl_elan_putget_ctrl_t * putget,
int init_num, int inc_num, int max_num)
{
int main_size;
int main_align;
int elan_size;
int elan_align;
ELAN4_CTX *ctx;
E4_Event *elan_ptr;
mca_ptl_elan_send_frag_t *frag;
ompi_free_list_t *put_list, *get_list;
ompi_ptl_elan_putget_desc_t *put_desc, *get_desc;
main_align = OMPI_PTL_ELAN_GET_MAX (sizeof (void *), ELAN_ALIGN);
elan_align = OMPI_PTL_ELAN_GET_MAX (sizeof (int *), ELAN_BLOCK_ALIGN);
main_size = OMPI_PTL_ELAN_ALIGNUP(sizeof(ompi_ptl_elan_putget_desc_t),
main_align);
#if OMPI_PTL_ELAN_COMP_QUEUE
elan_size = OMPI_PTL_ELAN_ALIGNUP(
(ELAN_BLOCK_SIZE * 2 + sizeof(E4_Event32)*3 ), elan_align);
#else
elan_size = OMPI_PTL_ELAN_ALIGNUP(
(ELAN_BLOCK_SIZE + sizeof(E4_Event32)*2 ), elan_align);
#endif
rail = (RAIL *) ptl->ptl_elan_rail;
ctx = (ELAN4_CTX *) ptl->ptl_elan_ctx;
/* initialize list */
OBJ_CONSTRUCT (&putget->put_desc_free, ompi_free_list_t);
put_list = &putget->put_desc_free;
OMPI_PTL_ELAN_CTRL_LIST(put_list, 0, inc_num, max_num);
/* Allocate the elements */
frag = (mca_ptl_elan_send_frag_t *)
malloc(sizeof(mca_ptl_elan_send_frag_t) * inc_num);
OMPI_PTL_ELAN_CHECK_UNEX (frag, NULL, OMPI_ERROR, 0);
/* Allocating elan related structures */
elan_ptr = (E4_Event *) elan4_allocElan (rail->r_alloc,
elan_align, elan_size * inc_num);
OMPI_PTL_ELAN_CHECK_UNEX (elan_ptr, NULL, OMPI_ERROR, 0);
put_desc = (ompi_ptl_elan_putget_desc_t *) elan4_allocMain (
rail->r_alloc, main_align, main_size * inc_num);
OMPI_PTL_ELAN_CHECK_UNEX (put_desc, NULL, OMPI_ERROR, 0);
OMPI_ELAN_PUTGET_GROW(ptl, put_list, frag, put_desc, elan_ptr,
main_size, elan_size, 1);
OBJ_CONSTRUCT (&putget->get_desc_free, ompi_free_list_t);
get_list = &putget->get_desc_free;
OMPI_PTL_ELAN_CTRL_LIST(get_list, 0, inc_num, max_num);
/* Allocate the elements */
frag = (mca_ptl_elan_send_frag_t *)
malloc(sizeof(mca_ptl_elan_send_frag_t) * inc_num);
OMPI_PTL_ELAN_CHECK_UNEX (frag, NULL, OMPI_ERROR, 0);
/* Allocating elan related structures */
elan_ptr = (E4_Event *) elan4_allocElan (rail->r_alloc,
elan_align, elan_size * inc_num);
OMPI_PTL_ELAN_CHECK_UNEX (elan_ptr, NULL, OMPI_ERROR, 0);
get_desc = (ompi_ptl_elan_putget_desc_t *) elan4_allocMain (
rail->r_alloc, main_align, main_size * inc_num);
OMPI_PTL_ELAN_CHECK_UNEX (get_desc, NULL, OMPI_ERROR, 0);
OMPI_ELAN_PUTGET_GROW(ptl, get_list, frag, get_desc, elan_ptr,
main_size, elan_size, 0);
return OMPI_SUCCESS;
}
int
ompi_init_elan_stat (mca_ptl_elan_component_t * emp,
int num_rails)
{
return (OMPI_SUCCESS);
}
int
ompi_init_elan_qdma (mca_ptl_elan_component_t * emp,
int num_rails)
{
int i;
int nslots;
int slotsize;
RAIL *rail;
ELAN4_CTX *ctx;
struct mca_ptl_elan_module_t *ptl;
nslots = OMPI_PTL_ELAN_MAX_QSLOTS;
#if OMPI_PTL_ELAN_COMP_QUEUE
/* Create a complete queue here, later use the queue above directly */
/* Init the Transmit Queue structure */
for (i = 0; i < num_rails; i++) {
ompi_ptl_elan_recv_queue_t *rxq;
ompi_ptl_elan_comp_queue_t *comp;
ptl = emp->modules[i];
rail = (RAIL *) ptl->ptl_elan_rail;
ctx = (ELAN4_CTX *) ptl->ptl_elan_ctx;
comp = ptl->comp = (ompi_ptl_elan_comp_queue_t *)
malloc (sizeof (ompi_ptl_elan_comp_queue_t));
OMPI_PTL_ELAN_CHECK_UNEX (comp, NULL, OMPI_ERROR, 0);
memset (comp, 0, sizeof (ompi_ptl_elan_comp_queue_t));
/* Allocate input queue */
comp->input = (E4_InputQueue *) elan4_allocElan (rail->r_alloc,
INPUT_QUEUE_ALIGN,
INPUT_QUEUE_SIZE);
OMPI_PTL_ELAN_CHECK_UNEX (comp->input, NULL, OMPI_ERROR, 0);
/* Init the Receive Queue structure */
comp->rx_nslots = nslots;
nslots += OMPI_PTL_ELAN_LOST_QSLOTS;
slotsize = sizeof(mca_ptl_base_header_t);
comp->rx_slotsize = ELAN_ALIGNUP (slotsize, OMPI_PTL_ELAN_SLOT_ALIGN);
comp->rx_buffsize = (slotsize > INPUT_QUEUE_MAX) ?
INPUT_QUEUE_MAX : slotsize;
rxq = comp->rxq = (ompi_ptl_elan_recv_queue_t *)
elan4_allocMain (rail->r_alloc, 64,
sizeof (ompi_ptl_elan_recv_queue_t));
OMPI_PTL_ELAN_CHECK_UNEX (rxq, NULL, OMPI_ERROR, 0);
memset (rxq, 0, sizeof (ompi_ptl_elan_recv_queue_t));
rxq->qr_rail = rail;
rxq->qr_fptr = elan4_allocMain (rail->r_alloc,
128, nslots * comp->rx_slotsize);
OMPI_PTL_ELAN_CHECK_UNEX (rxq->qr_fptr, NULL, OMPI_ERROR, 0);
memset (rxq->qr_fptr, 0xeb, nslots * comp->rx_slotsize);
rxq->qr_elanDone = ALLOC_ELAN (rail,
OMPI_PTL_ELAN_SLOT_ALIGN, sizeof (EVENT32));
OMPI_PTL_ELAN_CHECK_UNEX (rxq->qr_elanDone, NULL, OMPI_ERROR, 0);
/* Set the top et al */
rxq->qr_efitem = (E4_uint64) elan4_main2elan (ctx, rxq->qr_fptr);
assert(rxq->qr_efitem != ELAN_BAD_ADDR);
rxq->qr_base = rxq->qr_fptr;
rxq->qr_top = (void *) ((uintptr_t) rxq->qr_base +
(comp->rx_slotsize * (nslots - OMPI_PTL_ELAN_LOST_QSLOTS)));
rxq->qr_efptr = rxq->qr_efitem;
rxq->qr_elitem = rxq->qr_efitem +
(comp->rx_slotsize * (nslots - OMPI_PTL_ELAN_LOST_QSLOTS));
/* Event to wait/block on, Bug here for the event */
rxq->qr_qEvent = rxq->qr_elanDone;
comp->input->q_event= SDRAM2ELAN (ctx, (void *) rxq->qr_elanDone);
comp->input->q_fptr = rxq->qr_efitem;
comp->input->q_bptr = rxq->qr_efitem;
comp->input->q_control =
E4_InputQueueControl (rxq->qr_efitem, rxq->qr_elitem,
comp->rx_slotsize);
/* The event */
INITEVENT_WORD (ctx, (EVENT *) rxq->qr_elanDone,
&rxq->qr_doneWord);
RESETEVENT_WORD (&rxq->qr_doneWord);
PRIMEEVENT_WORD (ctx, (EVENT *) rxq->qr_elanDone, 1);
rxq->qr_cmdq = OMPI_PTL_ELAN_ALLOC_CMDQ (ctx,
rail->r_alloc,
CQ_Size1K,
CQ_WriteEnableBit |
CQ_WaitEventEnableBit, NULL);
OMPI_PTL_ELAN_CHECK_UNEX (rxq->qr_cmdq, NULL, OMPI_ERROR, 0);
/* Allocate a sleepDesc for threads to block on */
rxq->qr_es = ompi_init_elan_sleepdesc (&mca_ptl_elan_global_state,
rxq->qr_rail);
OMPI_PTL_ELAN_CHECK_UNEX (rxq->qr_es, NULL, OMPI_ERROR, 0);
OBJ_CONSTRUCT (&comp->rx_lock, opal_mutex_t);
}
#endif
/* Init the Transmit Queue structure */
for (i = 0; i < num_rails; i++) {
ompi_ptl_elan_recv_queue_t *rxq;
ompi_ptl_elan_queue_ctrl_t *queue;
ptl = emp->modules[i];
rail = (RAIL *) ptl->ptl_elan_rail;
ctx = (ELAN4_CTX *) ptl->ptl_elan_ctx;
queue = ptl->queue = (ompi_ptl_elan_queue_ctrl_t *)
malloc (sizeof (ompi_ptl_elan_queue_ctrl_t));
OMPI_PTL_ELAN_CHECK_UNEX (queue, NULL, OMPI_ERROR, 0);
memset (queue, 0, sizeof (ompi_ptl_elan_queue_ctrl_t));
/* TODO: move the input queue into ptl->comp */
queue->input = (E4_InputQueue *) elan4_allocElan (rail->r_alloc,
INPUT_QUEUE_ALIGN,
INPUT_QUEUE_SIZE);
OMPI_PTL_ELAN_CHECK_UNEX (queue->input, NULL, OMPI_ERROR, 0);
queue->tx_cmdq = OMPI_PTL_ELAN_ALLOC_CMDQ (ctx,
rail->r_alloc,
CQ_Size8K,
CQ_WriteEnableBit |
CQ_DmaStartEnableBit |
CQ_STENEnableBit, NULL);
OMPI_PTL_ELAN_CHECK_UNEX (queue->tx_cmdq, NULL, OMPI_ERROR, 0);
/* Allocate a cookie pool */
queue->tx_cpool = elan4_allocCookiePool (ctx, ptl->elan_vp);
ompi_init_elan_queue_events (ptl, queue);
/* Init the Receive Queue structure */
queue->rx_nslots = nslots;
nslots += OMPI_PTL_ELAN_LOST_QSLOTS;
slotsize = OMPI_PTL_ELAN_MAX_QSIZE;
queue->rx_slotsize = ELAN_ALIGNUP (slotsize, OMPI_PTL_ELAN_SLOT_ALIGN);
queue->rx_buffsize = (slotsize > INPUT_QUEUE_MAX) ?
INPUT_QUEUE_MAX : slotsize;
rxq = queue->rxq = (ompi_ptl_elan_recv_queue_t *)
elan4_allocMain (rail->r_alloc, 64,
sizeof (ompi_ptl_elan_recv_queue_t));
OMPI_PTL_ELAN_CHECK_UNEX (rxq, NULL, OMPI_ERROR, 0);
memset (rxq, 0, sizeof (ompi_ptl_elan_recv_queue_t));
rxq->qr_rail = rail;
rxq->qr_fptr = elan4_allocMain (rail->r_alloc,
128, nslots * queue->rx_slotsize);
OMPI_PTL_ELAN_CHECK_UNEX (rxq->qr_fptr, NULL, OMPI_ERROR, 0);
memset (rxq->qr_fptr, 0xeb, nslots * queue->rx_slotsize);
rxq->qr_elanDone = ALLOC_ELAN (rail,
OMPI_PTL_ELAN_SLOT_ALIGN, sizeof (EVENT32));
OMPI_PTL_ELAN_CHECK_UNEX (rxq->qr_elanDone, NULL, OMPI_ERROR, 0);
/* Set the top etc */
rxq->qr_efitem = (E4_uint64) elan4_main2elan (ctx, rxq->qr_fptr);
assert(rxq->qr_efitem != ELAN_BAD_ADDR);
rxq->qr_base = rxq->qr_fptr;
rxq->qr_top = (void *) ((uintptr_t) rxq->qr_base +
(queue->rx_slotsize * (nslots - OMPI_PTL_ELAN_LOST_QSLOTS)));
rxq->qr_efptr = rxq->qr_efitem;
rxq->qr_elitem = rxq->qr_efitem +
(queue->rx_slotsize * (nslots - OMPI_PTL_ELAN_LOST_QSLOTS));
/* XXX: Event to wait/block on, was buggy here for the event */
rxq->qr_qEvent = rxq->qr_elanDone;
queue->input->q_event= SDRAM2ELAN (ctx, (void *) rxq->qr_elanDone);
queue->input->q_fptr = rxq->qr_efitem;
queue->input->q_bptr = rxq->qr_efitem;
queue->input->q_control =
E4_InputQueueControl (rxq->qr_efitem, rxq->qr_elitem,
queue->rx_slotsize);
/* The event */
INITEVENT_WORD (ctx, (EVENT *) rxq->qr_elanDone,
&rxq->qr_doneWord);
RESETEVENT_WORD (&rxq->qr_doneWord);
PRIMEEVENT_WORD (ctx, (EVENT *) rxq->qr_elanDone, 1);
rxq->qr_cmdq = OMPI_PTL_ELAN_ALLOC_CMDQ (ctx,
rail->r_alloc,
CQ_Size1K,
CQ_WriteEnableBit |
CQ_WaitEventEnableBit, NULL);
OMPI_PTL_ELAN_CHECK_UNEX (rxq->qr_cmdq, NULL, OMPI_ERROR, 0);
/* Allocate a sleepDesc for threads to block on */
rxq->qr_es = ompi_init_elan_sleepdesc (&mca_ptl_elan_global_state,
rxq->qr_rail);
OMPI_PTL_ELAN_CHECK_UNEX (rxq->qr_es, NULL, OMPI_ERROR, 0);
OBJ_CONSTRUCT (&queue->rx_lock, opal_mutex_t);
}
return (OMPI_SUCCESS);
}
int
ompi_init_elan_putget (mca_ptl_elan_component_t * emp,
int num_rails)
{
int i;
RAIL *rail;
ELAN4_CTX *ctx;
struct mca_ptl_elan_module_t *ptl;
/* Init the Transmit Queue structure */
for (i = 0; i < num_rails; i++) {
E4_CmdQParams *cqp;
ompi_ptl_elan_putget_ctrl_t *putget;
ptl = emp->modules[i];
rail = (RAIL *) ptl->ptl_elan_rail;
ctx = (ELAN4_CTX *) ptl->ptl_elan_ctx;
putget = ptl->putget = (ompi_ptl_elan_putget_ctrl_t *)
malloc (sizeof (ompi_ptl_elan_putget_ctrl_t));
OMPI_PTL_ELAN_CHECK_UNEX (putget, NULL, OMPI_ERROR, 0);
memset (putget, 0, sizeof (ompi_ptl_elan_putget_ctrl_t));
putget->pg_throttle = OMPI_PTL_ELAN_MAX_PGDESC;
putget->pg_flags = OMPI_PTL_ELAN_FASTPATH;
putget->pg_retryCount = 16;
putget->pg_evictCache = TRUE;
putget->pg_waitType = ELAN_POLL_EVENT;
/* construct the lock variable */
OBJ_CONSTRUCT (&putget->pg_lock, opal_mutex_t);
cqp = OMPI_PTL_ELAN_PROBE_CMDQ (ctx, rail->r_alloc,
0x10, CQ_AutoCtrlFlowOn);
putget->put_cmdq = OMPI_PTL_ELAN_ALLOC_CMDQ(ctx,
rail->r_alloc,
CQ_Size8K,
CQ_WriteEnableBit |
CQ_DmaStartEnableBit |
CQ_SetEventEnableBit |
CQ_STENEnableBit, cqp);
OMPI_PTL_ELAN_CHECK_UNEX (putget->put_cmdq, NULL, OMPI_ERROR, 0);
putget->get_cmdq = OMPI_PTL_ELAN_ALLOC_CMDQ(ctx,
rail->r_alloc,
CQ_Size8K,
CQ_WriteEnableBit |
CQ_STENEnableBit |
CQ_SetEventEnableBit, cqp);
OMPI_PTL_ELAN_CHECK_UNEX (putget->get_cmdq, NULL, OMPI_ERROR, 0);
putget->pg_cmdStream = malloc(PAGESIZE);
OMPI_PTL_ELAN_CHECK_UNEX (putget->pg_cmdStream, NULL, OMPI_ERROR, 0);
/* Allocate a per vp counter to throttle outstanding get DMAs */
putget->pg_pendingGetCount = malloc(sizeof(uint32_t)*ptl->elan_nvp);
OMPI_PTL_ELAN_CHECK_UNEX (putget->pg_pendingGetCount,
NULL, OMPI_ERROR, 0);
memset(putget->pg_pendingGetCount, 0, sizeof(uint32_t)*ptl->elan_nvp);
putget->pg_cpool = elan4_allocCookiePool(ctx, ptl->elan_vp);
ompi_ptl_elan_init_putget_ctrl (ptl, rail, putget,
0, OMPI_PTL_ELAN_NUM_PUTGET, OMPI_PTL_ELAN_MAX_PUTGET);
}
return (OMPI_SUCCESS);
}

304
ompi/mca/ptl/elan/src/ptl_elan_component.c
Просмотреть файл

@ -1,304 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <errno.h>
#include <unistd.h>
#include <string.h>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include "constants.h"
#include "opal/event/event.h"
#include "opal/util/if.h"
#include "opal/util/argv.h"
#include "opal/util/output.h"
#include "mca/pml/pml.h"
#include "mca/ptl/ptl.h"
#include "mca/pml/base/pml_base_sendreq.h"
#include "mca/base/mca_base_param.h"
#include "mca/pml/base/pml_base_module_exchange.h"
#include "ptl_elan.h"
#include "ptl_elan_proc.h"
#include "ptl_elan_frag.h"
#include "ptl_elan_priv.h"
extern ompi_proc_t *ompi_proc_local_proc;
mca_ptl_elan_component_t mca_ptl_elan_component = {
{
/* Base module information about itself */
{
/* Indicate that we are a pml v1.0.0 module
* (which also implies a specific MCA version) */
MCA_PTL_BASE_VERSION_1_0_0,
"elan", /* MCA module name */
OMPI_MAJOR_VERSION, /* MCA module major version */
OMPI_MINOR_VERSION, /* MCA module minor version */
OMPI_RELEASE_VERSION, /* MCA module release version */
mca_ptl_elan_component_open, /* module open */
mca_ptl_elan_component_close /* module close */
},
/* Next the MCA v1.0.0 module meta data */
{
/* Whether the module is checkpointable or not */
false
},
/* The management related interfaces */
mca_ptl_elan_component_init,
mca_ptl_elan_component_control,
mca_ptl_elan_component_progress
}
};
static mca_ptl_elan_component_t *elan_mp = &mca_ptl_elan_component;
static bool mca_ptl_elan_component_initialized = false;
/*
* XXX: Leave it as a routine for possible extension
* some elan vp information to the the global registery
*/
static int mca_ptl_elan_addr_put (mca_ptl_elan_component_t *emp)
{
int rc;
size_t i;
size_t size;
mca_ptl_elan_addr_t *addrs;
size = emp->num_modules * sizeof(mca_ptl_elan_addr_t);
addrs = (mca_ptl_elan_addr_t *) malloc(size);
for(i=0; i< emp->num_modules; i++) {
mca_ptl_elan_module_t * ptl = emp->modules[i];
addrs[i].elan_vp = ptl->elan_vp;
addrs[i].inuse = 0;
addrs[i].gid = ompi_proc_local_proc->proc_name;
}
rc = mca_base_modex_send(&emp->super.ptlm_version, addrs, size);
free(addrs);
return rc;
}
/*
* Called by MCA framework to open the module, registers
* module parameters.
*/
int
mca_ptl_elan_component_open (void)
{
int length;
int param1, param2, param3;
mca_ptl_elan_module.super.ptl_exclusivity =
mca_ptl_elan_param_register_int ("exclusivity", 0);
length = OMPI_PTL_ELAN_MAX_QSIZE - sizeof(mca_ptl_base_header_t);
param1 = mca_ptl_elan_param_register_int ("first_frag_size", length);
param2 = mca_ptl_elan_param_register_int ("min_frag_size", length);
param3 = mca_ptl_elan_param_register_int ("max_frag_size", (1<<31));
/* Correct these if user give violent parameters */
mca_ptl_elan_module.super.ptl_first_frag_size =
OMPI_PTL_ELAN_GET_MIN(param1, length);
mca_ptl_elan_module.super.ptl_min_frag_size =
OMPI_PTL_ELAN_GET_MAX(param2, length);
mca_ptl_elan_module.super.ptl_max_frag_size =
OMPI_PTL_ELAN_GET_MIN(param3, (1<<31));
/* initialize state */
elan_mp->elan_local = NULL;
elan_mp->num_modules = 0;
elan_mp->modules = NULL;
elan_mp->free_list_max = 128;
elan_mp->free_list_num = 32;
elan_mp->free_list_inc = 32;
/* initialize objects*/
OBJ_CONSTRUCT (&elan_mp->elan_procs, opal_list_t);
OBJ_CONSTRUCT (&elan_mp->elan_recv_frags_free, ompi_free_list_t);
OBJ_CONSTRUCT (&elan_mp->elan_lock, opal_mutex_t);
return OMPI_SUCCESS;
}
int
mca_ptl_elan_component_close (void)
{
if (mca_ptl_elan_component_initialized) {
#if OMPI_PTL_ELAN_THREADING
if ((mca_ptl_elan_thread_close(elan_mp)) != OMPI_SUCCESS) {
opal_output(0, "unable to close asynchronous threads\n");
}
#endif
/* cleanup the proc, ptl, and the module */
mca_ptl_elan_state_finalize(&mca_ptl_elan_component);
if (elan_mp->elan_local) {
free (elan_mp->elan_local);
}
if (NULL != elan_mp->modules) {
int i;
for (i = elan_mp->num_modules; i > 0; i--) {
free (elan_mp->modules[i - 1]);
}
free (elan_mp->modules);
}
}
/* Free the empty list holders */
OBJ_DESTRUCT (&(elan_mp->elan_procs));
/* FIXME:
* We need free all the memory allocated for this list
* before desctructing this free_list */
if (elan_mp->elan_recv_frags_free.fl_num_allocated !=
elan_mp->elan_recv_frags_free.super.opal_list_length) {
opal_output (0,
"[%s:%d] recv_frags : %d allocated %d returned\n",
__FILE__, __LINE__,
elan_mp->elan_recv_frags_free.fl_num_allocated,
elan_mp->elan_recv_frags_free.super.opal_list_length);
}
OBJ_DESTRUCT (&(elan_mp->elan_recv_frags_free));
/* Destruct other structures */
OBJ_DESTRUCT (&elan_mp->elan_lock);
return OMPI_SUCCESS;
}
/*
* ELAN module initialization:
* (1) elan4_init() to initialize the basic support and mapping etc.
* (2) set up STEN, RDMA and QDMA structures.
* (3) register the list of PTL parameters with the MCA
*/
mca_ptl_base_module_t **
mca_ptl_elan_component_init (int *num_ptls,
bool enable_progress_threads,
bool enable_mpi_threads)
{
mca_ptl_base_module_t **ptls;
*num_ptls = 0;
ompi_free_list_init (&(elan_mp->elan_recv_frags_free),
sizeof (mca_ptl_elan_recv_frag_t),
OBJ_CLASS (mca_ptl_elan_recv_frag_t),
/*32, 128, 32, */
elan_mp->free_list_num,
elan_mp->free_list_max,
elan_mp->free_list_inc, NULL);
/* open basic elan device */
if (OMPI_SUCCESS != mca_ptl_elan_state_init(&mca_ptl_elan_component)) {
opal_output(0,
"[%s:%d] error in initializing elan state and PTL's.\n",
__FILE__, __LINE__);
return NULL;
}
if (OMPI_SUCCESS != mca_ptl_elan_addr_put(&mca_ptl_elan_component)) {
opal_output(0,
"[%s:%d] error in registering with Runtime/OOB \n",
__FILE__, __LINE__);
return NULL;
}
ptls = (mca_ptl_base_module_t **) malloc (elan_mp->num_modules *
sizeof (mca_ptl_elan_module_t *));
if (NULL == ptls) {
opal_output(0,
"[%s:%d] error in allocating memory \n",
__FILE__, __LINE__);
return NULL;
}
memcpy (ptls, elan_mp->modules,
elan_mp->num_modules * sizeof (mca_ptl_elan_module_t *));
*num_ptls = elan_mp->num_modules;
/* XXX + TODO:
* have threads listening on send/recv completion from this point on.
* Points to be aware of.
* a) Main thread trigger the send
* b) Asynchronous thread detects the completion of send/recv, put/get
* then update fragment descriptors and requests if needed.
* c) Main thread progresses with the status of requests being detected.
* d) Asynchronous thread handles the retransmission and recv side
* data copying; Main thread does nothing more than initiating
* the messaging.
* e) Asynchronous thread should also detect the completion of the
* program and exit accordingly, via a signal/interrupt.
*/
#if OMPI_PTL_ELAN_THREADING
if ((mca_ptl_elan_thread_init(elan_mp)) != OMPI_SUCCESS) {
opal_output(0,
"unable to initialize %d asynchronous threads\n",
elan_mp->num_modules);
}
#endif
mca_ptl_elan_component_initialized = true;
return ptls;
}
/* support ELAN module control */
int
mca_ptl_elan_component_control (int param,
void *value,
size_t size)
{
switch (param) {
case MCA_PTL_ENABLE:
default:
break;
}
return OMPI_SUCCESS;
}
/* TODO: to support event-based module progress later. */
int
mca_ptl_elan_component_progress (mca_ptl_tstamp_t tstamp)
{
int i, no_ptls;
no_ptls = elan_mp->num_modules;
/* Iterate over all the PTL input Queues */
for (i = 0; i < no_ptls; i++) {
#if OMPI_PTL_ELAN_ONE_QUEUE
mca_ptl_elan_lookup(elan_mp->modules[i]);
#else
mca_ptl_elan_update_desc(elan_mp->modules[i]);
mca_ptl_elan_drain_recv(elan_mp->modules[i]);
#endif
}
return OMPI_SUCCESS;
}

256
ompi/mca/ptl/elan/src/ptl_elan_frag.c
Просмотреть файл

@ -1,256 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <unistd.h>
#include <sys/types.h>
#include <sys/errno.h>
#include "types.h"
#include "datatype/datatype.h"
#include "mca/pml/base/pml_base_sendreq.h"
#include "mca/pml/base/pml_base_recvreq.h"
#include "ptl_elan.h"
#include "ptl_elan_peer.h"
#include "ptl_elan_proc.h"
#include "ptl_elan_frag.h"
#include "ptl_elan_priv.h"
#include "mca/pml/base/pml_base_sendreq.h"
#include "mca/pml/base/pml_base_recvreq.h"
#include "mca/ptl/base/ptl_base_sendfrag.h"
#include "mca/ptl/base/ptl_base_recvfrag.h"
#include "ptl_elan.h"
static void
mca_ptl_elan_send_frag_construct (mca_ptl_elan_send_frag_t * frag)
{
frag->frag_progressed = 0;
frag->desc = 0;
}
static void
mca_ptl_elan_send_frag_destruct (mca_ptl_elan_send_frag_t * frag)
{
/* Nothing to do then */
}
opal_class_t mca_ptl_elan_send_frag_t_class = {
"mca_ptl_elan_send_frag_t",
OBJ_CLASS (mca_ptl_base_frag_t),
(opal_construct_t) mca_ptl_elan_send_frag_construct,
(opal_destruct_t) mca_ptl_elan_send_frag_destruct
};
static void
mca_ptl_elan_recv_frag_construct (mca_ptl_elan_recv_frag_t * frag)
{
frag->frag_hdr_cnt = 0;
frag->frag_msg_cnt = 0;
frag->frag_progressed = 0;
/*frag->frag.qdma = NULL;*/
frag->alloc_buff = (char *) malloc (sizeof (char) * 2048 + 32);
if (NULL == frag->alloc_buff) {
opal_output (0,
"[%s:%d] Fatal error, unable to allocate recv buff \n",
__FILE__, __LINE__);
}
frag->unex_buff = (char *) (((int) frag->alloc_buff + 32) >> 5 << 5);
}
static void
mca_ptl_elan_recv_frag_destruct (mca_ptl_elan_recv_frag_t * frag)
{
frag->frag_hdr_cnt = 0;
frag->frag_msg_cnt = 0;
frag->frag_progressed = 0;
/*frag->frag.qdma = NULL;*/
free (frag->alloc_buff);
frag->alloc_buff = NULL;
frag->unex_buff = NULL;
}
opal_class_t mca_ptl_elan_recv_frag_t_class = {
"mca_ptl_elan_recv_frag_t",
OBJ_CLASS (mca_ptl_base_recv_frag_t),
(opal_construct_t) mca_ptl_elan_recv_frag_construct,
(opal_destruct_t) mca_ptl_elan_recv_frag_destruct
};
extern mca_ptl_elan_state_t mca_ptl_elan_global_state;
mca_ptl_elan_send_frag_t *
mca_ptl_elan_alloc_desc (struct mca_ptl_base_module_t *ptl_ptr,
struct mca_pml_base_request_t *req, int desc_type)
{
ompi_free_list_t *flist;
opal_list_item_t *item = NULL;
mca_ptl_elan_send_frag_t *desc;
/* TODO: Dynamically bind a base request to PUT/GET/QDMA/STEN */
if (MCA_PTL_ELAN_DESC_QDMA == desc_type) {
flist = &(((mca_ptl_elan_module_t *) ptl_ptr)->queue)->tx_desc_free;
} else if (MCA_PTL_ELAN_DESC_PUT == desc_type) {
flist = &(((mca_ptl_elan_module_t *) ptl_ptr)->putget)->put_desc_free;
} else if (MCA_PTL_ELAN_DESC_GET == desc_type) {
flist = &(((mca_ptl_elan_module_t *) ptl_ptr)->putget)->get_desc_free;
} else {
opal_output (0,
"[%s:%d] Error: unknown to descriptor desc type\n",
__FILE__, __LINE__);
return NULL;
}
LOG_PRINT(PTL_ELAN_DEBUG_SEND, "flist %p length %d type %d\n",
flist, flist->super.opal_list_length, desc_type);
if (opal_using_threads ()) {
opal_mutex_lock(&flist->fl_lock);
item = opal_list_remove_first (&((flist)->super));
while (NULL == item) {
mca_ptl_tstamp_t tstamp = 0;
ptl_ptr->ptl_component->ptlm_progress (tstamp);
item = opal_list_remove_first (&((flist)->super));
}
opal_mutex_unlock(&flist->fl_lock);
} else {
item = opal_list_remove_first (&((flist)->super));
/* XXX:
* Ouch..., this still does not trigger the progress on
* PTL's from other modules. Wait for PML to change.
* Otherwise have to trigger PML progress from PTL. */
while (NULL == item) {
mca_ptl_tstamp_t tstamp = 0;
ptl_ptr->ptl_component->ptlm_progress (tstamp);
item = opal_list_remove_first (&((flist)->super));
}
}
desc = (mca_ptl_elan_send_frag_t *) item;
desc->desc->req = req;
desc->desc->desc_type = desc_type;
LOG_PRINT(PTL_ELAN_DEBUG_SEND, "Got frag %p desc %d type %d\n",
desc, desc->desc, desc_type);
return desc;
}
mca_ptl_elan_recv_frag_t *
mca_ptl_elan_alloc_recv_desc (struct mca_pml_base_recv_request_t * req)
{
return NULL;
}
void
mca_ptl_elan_send_desc_done (
mca_ptl_elan_send_frag_t *frag,
mca_pml_base_send_request_t *req)
{
mca_ptl_elan_module_t *ptl;
mca_ptl_base_header_t *header;
int dtype;
dtype = frag->desc->desc_type;
ptl = ((ompi_ptl_elan_qdma_desc_t *)frag->desc)->ptl;
header = &frag->frag_base.frag_header;
#if OMPI_PTL_ELAN_ENABLE_GET
if (frag->desc->desc_type == MCA_PTL_ELAN_DESC_GET) {
if(opal_atomic_fetch_and_set_int (&frag->frag_progressed, 1) == 0) {
ptl->super.ptl_recv_progress(ptl,
(mca_pml_base_recv_request_t *) req,
frag->frag_base.frag_size,
frag->frag_base.frag_size);
}
PTL_ELAN4_FREE_QBUFF (ptl->ptl_elan_ctx,
((ompi_ptl_elan_putget_desc_t *) frag->desc)
->chain_event->ev_Params[1], 8);
OMPI_FREE_LIST_RETURN (&ptl->putget->get_desc_free,
(opal_list_item_t *) frag);
return;
}
#endif
LOG_PRINT(PTL_ELAN_DEBUG_SEND,
"req %p done frag %p desc %p desc_type %d length %d\n",
req, frag, frag->desc,
frag->desc->desc_type,
header->hdr_frag.hdr_frag_length);
if(NULL == req) { /* An ack descriptor */
OMPI_FREE_LIST_RETURN (&ptl->queue->tx_desc_free,
(opal_list_item_t *) frag);
} else if (0 == (header->hdr_common.hdr_flags & MCA_PTL_FLAGS_ACK)
|| mca_pml_base_send_request_matched(req)) {
if(opal_atomic_fetch_and_set_int (&frag->frag_progressed, 1) == 0)
{
ptl->super.ptl_send_progress(
(struct mca_ptl_base_module_t*) ptl,
req, header->hdr_frag.hdr_frag_length);
}
LOG_PRINT(PTL_ELAN_DEBUG_SEND, "return frag %p desc %p type %d\n",
frag, frag->desc, frag->desc->desc_type);
/* Return a frag or if not cached, or it is a follow up */
if ( (frag->desc->desc_status != MCA_PTL_ELAN_DESC_CACHED)){
ompi_free_list_t *flist;
if (frag->desc->desc_type == MCA_PTL_ELAN_DESC_PUT) {
flist = &ptl->putget->put_desc_free;
PTL_ELAN4_FREE_QBUFF (ptl->ptl_elan_ctx,
((ompi_ptl_elan_putget_desc_t *) frag->desc)
->chain_event->ev_Params[1], 8);
} else {
flist = &ptl->queue->tx_desc_free;
}
OMPI_FREE_LIST_RETURN (flist, (opal_list_item_t *) frag);
} else {
LOG_PRINT(PTL_ELAN_DEBUG_ACK,
"PML will return frag to list %p, length %d\n",
&ptl->queue->tx_desc_free,
ptl->queue->tx_desc_free.super.opal_list_length);
}
}
}
void
mca_ptl_elan_recv_frag_done (
mca_ptl_base_header_t *header,
mca_ptl_elan_recv_frag_t* frag,
mca_pml_base_recv_request_t *request)
{
frag->frag_recv.frag_base.frag_owner->ptl_recv_progress (
frag->frag_recv.frag_base.frag_owner,
request,
frag->frag_recv.frag_base.frag_size,
frag->frag_recv.frag_base.frag_size);
/* FIXME:
* To support the required ACK, do not return
* until the ack is out */
if (frag->frag_ack_pending == false) {
mca_ptl_elan_recv_frag_return (
frag->frag_recv.frag_base.frag_owner, frag);
} else {
/* XXX: Chaining it into the list of completion pending recv_frag,
* Until the ack frag is sent out, they will stay in the list */
}
}

101
ompi/mca/ptl/elan/src/ptl_elan_frag.h
Просмотреть файл

@ -1,101 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef _MCA_PTL_ELAN_FRAG_H
#define _MCA_PTL_ELAN_FRAG_H
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include "ompi_config.h"
#include "mca/pml/base/pml_base_sendreq.h"
#include "mca/pml/base/pml_base_recvreq.h"
#include "mca/ptl/base/ptl_base_sendfrag.h"
#include "mca/ptl/base/ptl_base_recvfrag.h"
#include "ptl_elan.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
struct mca_ptl_elan_peer_t;
struct ompi_ptl_elan_base_desc_t;
struct mca_ptl_elan_send_frag_t {
mca_ptl_base_frag_t frag_base;
struct ompi_ptl_elan_base_desc_t *desc;
volatile int frag_progressed;
bool frag_ack_pending; /* Is there an ack to recv */
};
typedef struct mca_ptl_elan_send_frag_t mca_ptl_elan_send_frag_t;
/* XXX: Extend the header a bit with an pointer to frag */
struct mca_ptl_elan_ack_header_t {
struct mca_ptl_base_ack_header_t base_ack; /* 32 bytes */
struct mca_ptl_elan_send_frag_t *frag;
};
typedef struct mca_ptl_elan_ack_header_t mca_ptl_elan_ack_header_t;
/**
* ELAN received fragment derived type.
*/
struct mca_ptl_elan_recv_frag_t {
mca_ptl_base_recv_frag_t frag_recv;
size_t frag_hdr_cnt;
size_t frag_msg_cnt;
volatile int frag_progressed; /* Is it record to request */
bool frag_ack_pending; /* Is there an ack to send */
char *alloc_buff;
char *unex_buff;
};
typedef struct mca_ptl_elan_recv_frag_t mca_ptl_elan_recv_frag_t;
extern opal_class_t mca_ptl_elan_send_frag_t_class;
extern opal_class_t mca_ptl_elan_recv_frag_t_class;
mca_ptl_elan_send_frag_t *
mca_ptl_elan_alloc_desc(struct mca_ptl_base_module_t *ptl,
struct mca_pml_base_request_t *req,
int desc_type);
mca_ptl_elan_recv_frag_t *
mca_ptl_elan_alloc_recv_desc(struct mca_pml_base_recv_request_t *req);
/**
* TODO: Change frag to be a struct
* ELAN send request derived type. The send request contains
* the base send request and a point to the elan fragment descriptor
*/
struct mca_ptl_elan_send_request_t {
mca_pml_base_send_request_t super;
mca_ptl_elan_send_frag_t *req_frag;
};
typedef struct mca_ptl_elan_send_request_t mca_ptl_elan_send_request_t;
void
mca_ptl_elan_send_desc_done (
mca_ptl_elan_send_frag_t *desc,
mca_pml_base_send_request_t *req);
void
mca_ptl_elan_recv_frag_done (
mca_ptl_base_header_t *header,
mca_ptl_elan_recv_frag_t* frag,
mca_pml_base_recv_request_t *request);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

779
ompi/mca/ptl/elan/src/ptl_elan_init.c
Просмотреть файл

@ -1,779 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <signal.h>
#include <unistd.h>
#include <stdio.h>
#include "ptl_elan.h"
#include "ptl_elan_priv.h"
#define _ELAN4
mca_ptl_elan_state_t mca_ptl_elan_global_state;
struct ompi_ptl_elan_cmdq_space_t ptl_elan_cmdq_space;
static int
ompi_mca_ptl_elan_setup (mca_ptl_elan_state_t * ems)
{
mca_ptl_elan_component_t *emp;
int rail_count;
rail_count = ems->elan_nrails;
emp = ems->elan_component;
emp->num_modules = 0;
emp->modules = malloc (rail_count * sizeof (mca_ptl_elan_module_t *));
if (NULL == emp->modules) {
opal_output (0,
"[%s:%d] error in malloc for ptl references \n",
__FILE__, __LINE__);
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* Initialiaze PTL's */
do {
char param[256];
mca_ptl_elan_module_t *ptl;
ptl = malloc (sizeof (mca_ptl_elan_module_t));
if (NULL == ptl) {
opal_output (0,
"[%s:%d] error in malloc for ptl structures \n",
__FILE__, __LINE__);
return OMPI_ERR_OUT_OF_RESOURCE;
}
memcpy (ptl, &mca_ptl_elan_module, sizeof (mca_ptl_elan_module));
emp->modules[emp->num_modules] = ptl;
/* MCA related structures */
ptl->ptl_ni_local = emp->num_modules;
ptl->ptl_ni_total = rail_count;
/* allow user to specify per rail bandwidth and latency */
sprintf (param, "bandwidth_elanrail%d", emp->num_modules);
ptl->super.ptl_bandwidth =
mca_ptl_elan_param_register_int (param, 1000);
sprintf (param, "latency_elanrail%d", emp->num_modules);
ptl->super.ptl_latency =
mca_ptl_elan_param_register_int (param, 1);
/* Setup elan related structures such as ctx, rail */
ptl->ptl_elan_rail = ems->elan_rail[emp->num_modules];
ptl->ptl_elan_ctx = ems->elan_rail[emp->num_modules]->rail_ctx;
ptl->elan_vp = ems->elan_vp;
ptl->elan_nvp = ems->elan_nvp;
OBJ_CONSTRUCT (&ptl->recv_frags, opal_list_t);
OBJ_CONSTRUCT (&ptl->send_frags, opal_list_t);
OBJ_CONSTRUCT (&ptl->pending_acks, opal_list_t);
emp->num_modules++;
} while (emp->num_modules < rail_count);
/* Allocating all the communication strcutures for PTL's, */
if (OMPI_SUCCESS != ompi_init_elan_qdma (emp, rail_count)) {
return OMPI_ERROR;
}
if (OMPI_SUCCESS != ompi_init_elan_putget (emp, rail_count)) {
return OMPI_ERROR;
}
/* TODO: initialize STAT (including SYNC) structures. */
if (OMPI_SUCCESS != ompi_init_elan_stat (emp, rail_count)) {
return OMPI_ERROR;
}
return (OMPI_SUCCESS);
}
/* Attach to the network */
static int elan_attached = 0;
static int
ompi_elan_attach_network (mca_ptl_elan_state_t * ems)
{
int i;
int vp;
int *vps;
int num_rails;
ELAN_LOCATION loc;
ELAN_CAPABILITY *cap;
cap = ems->elan_cap;
num_rails = ems->elan_nrails;
if (elan_attached) {
/* already successfully attached */
return OMPI_SUCCESS;
} else {
elan_attached = 1;
}
for (i = 0; i < num_rails; i++) {
RAIL *rail = ems->all_rails[i];
ELAN_LOCATION loc;
/* Add all virtual process from 0 to (nvp-1) */
if (elan4_add_p2pvp (rail->r_ctx, 0, cap) < 0) {
opal_output (0,
"[%s:%d] error in adding vp to elan capability \n",
__FILE__, __LINE__);
return OMPI_ERROR;
}
/* block the inputter until we are really ready */
elan4_block_inputter (rail->r_ctx, 1);
if (elan4_attach (rail->r_ctx, cap)) {
perror("can not attach to elan network");
opal_output (0,
"[%s:%d] error in attaching to the network \n",
__FILE__, __LINE__);
return OMPI_ERROR;
}
/* NB: We should have same vp for all capabilities */
loc.loc_node = ((ELAN4_CTX *) rail->r_ctx)->ctx_position.pos_nodeid
- cap->cap_lownode;
loc.loc_context = cap->cap_mycontext - cap->cap_lowcontext;
/* TODO: debug code for loc */
if (cap->cap_type == ELAN_CAP_TYPE_HWTEST)
ems->elan_vp = 0;
else
ems->elan_vp = elan_location2vp (loc, ems->elan_cap);
/* Initialise the Elan version of this data */
((ELAN_ESTATE *) (rail->r_estate))->vp = ems->elan_vp;
/* Allocate a cookie pool for the thread processor
* and copy to sdram */
rail->r_cpool = elan4_allocCookiePool (rail->r_ctx, ems->elan_vp);
}
loc = elan_vp2location (ems->elan_vp, ems->elan_cap);
/* update THREAD elan_dbg info of debugfile */
if (ems->elan_debugfile && fileno (ems->elan_debugfile) != -1) {
for (i = 0; i < num_rails; i++) {
ELAN4_CTX *ctx = ems->elan_rail[i]->rail_ctx;
/* Convert FILE * stream to fd for THRD output */
((ELAN_ESTATE *) ems->all_estates[i])->debugFd =
fileno (ems->elan_debugfile);
/* Also re-direct libelan4 output to this file */
elan4_set_debugfd (ctx, fileno (ems->elan_debugfile));
}
}
/* Determine the number of processes described by the capability */
ems->elan_nvp = elan_nvps (ems->elan_cap);
if (ems->elan_vp >= ems->elan_nvp) {
opal_output (0,
"[%s:%d] error getting vp and nvp from capability \n",
__FILE__, __LINE__);
return OMPI_ERROR;
}
/* Allocate more than we need to keep the heap in sync */
ems->elan_localvps = (int *) malloc (sizeof (int) * (ems->elan_nvp + 1));
if (NULL == ems->elan_localvps) {
opal_output (0,
"[%s:%d] error in malloc for elan_localvps \n",
__FILE__, __LINE__);
return OMPI_ERROR;
}
/* Set all to non local initially */
for (vp = 0; vp < ems->elan_nvp; vp++)
ems->elan_localvps[vp] = -1;
/* Stash our own process location */
ems->elan_myloc = elan_vp2location (ems->elan_vp, ems->elan_cap);
ems->elan_maxlocals = elan_maxlocal (ems->elan_cap);
ems->elan_numlocals =
elan_nlocal (ems->elan_myloc.loc_node, ems->elan_cap);
/* Allocate more than we need to keep the heap in sync */
if (NULL == (vps = (int *) malloc (sizeof (int) * ems->elan_nvp))) {
opal_output (0,
"[%s:%d] error in malloc for vps \n", __FILE__,
__LINE__);
return OMPI_ERROR;
}
/* Fill out the local vp array */
elan_localvps (ems->elan_myloc.loc_node,
ems->elan_cap, vps, ems->elan_numlocals);
for (i = 0; i < ems->elan_numlocals; i++) {
int localvp = vps[i];
/* This is a local process */
ems->elan_localvps[localvp] = i;
if (localvp == ems->elan_vp) {
ems->elan_localid = i;
}
}
/* Done with vps array now */
free (vps);
return (OMPI_SUCCESS);
}
static void
ompi_module_elan_close_ptls (mca_ptl_elan_component_t * emp,
int num_rails)
{
int i;
struct mca_ptl_elan_module_t *ptl;
for (i = 0; i < num_rails; i ++ ) {
ptl = emp->modules[i];
if (NULL == ptl) continue;
/* TODO: Deconstruct the module: ptl->super; */
OBJ_DESTRUCT (&(ptl->recv_frags));
OBJ_DESTRUCT (&(ptl->send_frags));
OBJ_DESTRUCT (&(ptl->pending_acks));
/* TODO: Free send/recv/comp queues */
/*ptl->comp; ptl->queue; ptl->putget;*/
}
}
static void
ompi_module_elan_close_procs (mca_ptl_elan_component_t * emp,
int num_rails)
{
/* FIXME: find left procs and free them */
}
static int
ompi_elan_device_init (mca_ptl_elan_component_t * emp,
mca_ptl_elan_state_t *ems)
{
int i;
int *rails;
int num_rails;
int alloc_mainsize;
int alloc_mainbase;
int alloc_elansize;
int alloc_elanbase;
mca_ptl_elan_state_t *ems;
ems = &mca_ptl_elan_global_state;
/* Hook two of them togther */
ems->elan_component = emp;
emp->elan_ctrl = ems;
/* Initialise enough of state so we can call elan_exception() */
ems->elan_version = ELAN_VERSION;
ems->elan_ctx = NULL;
ems->elan_rail = NULL;
ems->elan_vp = ELAN_INVALID_PROCESS;
ems->elan_nvp = 0;
ems->elan_debug = 0;
ems->elan_traced = 0;
ems->elan_pagesize = sysconf (_SC_PAGESIZE);
ems->elan_pid = getpid ();
/* Default allocator parameters */
ems->elan_flags = 0;
ems->elan_waittype = ELAN_POLL_EVENT; /* or ELAN_WAIT_EVENT */
ems->main_size = ELAN_ALLOC_SIZE;
ems->elan_size = ELAN_ALLOCELAN_SIZE;
ems->elan_flags |= (EXCEPTIONCORE | EXCEPTIONTRACE | EXCEPTIONDBGDUMP);
ems->elan_debugfile = (FILE *) NULL;
ems->elan_signalnum = SIGABRT;
#ifdef ELAN_VERSION
if (!elan_checkVersion (ELAN_VERSION)) {
opal_output (0,
"Elan version is not compatible with %s \n",
ELAN_VERSION);
return OMPI_ERROR;
}
#endif
/* Allocate elan capability from the heap */
ems->elan_cap = (ELAN_CAPABILITY *) malloc (sizeof (ELAN_CAPABILITY));
OMPI_PTL_ELAN_CHECK_UNEX (ems->elan_cap, NULL, OMPI_ERROR, 0);
memset (ems->elan_cap, 0, sizeof (ELAN_CAPABILITY));
/* Process the capability info supplied by RMS */
if (getenv ("ELAN_AUTO") || getenv ("RMS_NPROCS")) {
/* RMS generated capabilities */
if (rms_getcap (0, ems->elan_cap)) {
opal_output (0,
"[%s:%d] error in gettting elan capability \n",
__FILE__, __LINE__);
return OMPI_ERROR;
}
} else if ( elan_getenvCap (ems->elan_cap, 0) < 0 ) {
/* Grab the capability from the user environment */
opal_output (0,
"[%s:%d] Can't get capability from environment \n",
__FILE__, __LINE__);
return OMPI_ERROR;
}
if ((num_rails = ems->elan_nrails = elan_nrails (ems->elan_cap)) <= 0) {
opal_output (0,
"[%s:%d] error in gettting number of rails \n",
__FILE__, __LINE__);
return OMPI_ERROR;
}
ems->all_rails = (RAIL **) malloc (sizeof (RAIL *) * num_rails);
OMPI_PTL_ELAN_CHECK_UNEX (ems->all_rails, NULL,
OMPI_ERR_OUT_OF_RESOURCE, 0);
ems->all_estates = (ADDR_SDRAM *)
malloc (sizeof (ELAN_ESTATE *) * num_rails);
OMPI_PTL_ELAN_CHECK_UNEX (ems->all_estates, NULL,
OMPI_ERR_OUT_OF_RESOURCE, 0);
rails = (int *) malloc (sizeof (int) * num_rails);
OMPI_PTL_ELAN_CHECK_UNEX (rails, NULL, OMPI_ERR_OUT_OF_RESOURCE, 0);
(void) elan_rails (ems->elan_cap, rails);
ems->elan_rail = (ELAN_RAIL **) malloc (sizeof (ELAN_RAIL **)
* (num_rails + 1));
OMPI_PTL_ELAN_CHECK_UNEX (ems->elan_rail, NULL,
OMPI_ERR_OUT_OF_RESOURCE, 0);
ems->elan_rail[num_rails] = NULL;
alloc_mainsize = ELAN_ALIGNUP (ems->main_size, ems->elan_pagesize);
alloc_mainbase = (ADDR_ELAN) ((uintptr_t) ems->main_base);
alloc_elansize = ELAN_ALIGNUP (ems->elan_size, ems->elan_pagesize);
alloc_elanbase = (ADDR_ELAN) ((uintptr_t) ems->elan_base);
/* XXX: Magic Quadrics number for the starting cookie value */
ems->intcookie = 42;
ems->rail_intcookie = (int *) malloc (sizeof (int) * (num_rails + 1));
OMPI_PTL_ELAN_CHECK_UNEX (ems->rail_intcookie, NULL,
OMPI_ERR_OUT_OF_RESOURCE, 0);
memset ((void*)ems->rail_intcookie, 0, (num_rails + 1) * sizeof (int));
ems->rail_intcookie[num_rails] = 0;
for (i = 0; i < num_rails; i++) {
RAIL *rail;
ELAN_ESTATE *estate;
ELAN_EPRIVSTATE *priv_estate;
ELAN_SLEEP *es;
/* Allocate the Main memory control structure for this rail */
rail = ems->all_rails[i] = (RAIL *) malloc (sizeof (RAIL));
OMPI_PTL_ELAN_CHECK_UNEX (rail, NULL, OMPI_ERROR, 0);
memset (rail, 0, sizeof (RAIL));
rail->r_ctx = elan4_init (rails[i]);
OMPI_PTL_ELAN_CHECK_UNEX (rail->r_ctx, NULL, OMPI_ERROR, 0);
rail->r_sdram = elan4_open_sdram (rails[i], 0, alloc_elansize);
OMPI_PTL_ELAN_CHECK_UNEX (rail->r_sdram, NULL, OMPI_ERROR, 0);
rail->r_alloc = elan4_createAllocator (ems->main_size,
rail->r_sdram, 0,
ems->elan_size);
OMPI_PTL_ELAN_CHECK_UNEX (rail->r_alloc, NULL, OMPI_ERROR, 0);
if (elan4_set_standard_mappings (rail->r_ctx) < 0
|| elan4_set_required_mappings (rail->r_ctx) < 0) {
opal_output (0,
"[%s:%d] error setting memory mapping for rail %d \n",
__FILE__, __LINE__, rails[i]);
return OMPI_ERROR;
}
/* Now allocate the SDRAM Elan control structure for this rail */
estate = ems->all_estates[i] = elan4_allocElan (rail->r_alloc,
ELAN_ALIGN,
sizeof
(ELAN_EPRIVSTATE));
OMPI_PTL_ELAN_CHECK_UNEX (estate, NULL, OMPI_ERROR, 0);
priv_estate = (ELAN_EPRIVSTATE *) estate;
memset (priv_estate, 0, sizeof (ELAN_EPRIVSTATE));
/* Allocate a command port for non sten functions etc */
rail->r_cmdq = OMPI_PTL_ELAN_ALLOC_CMDQ (rail->r_ctx,
rail->r_alloc,
CQ_Size8K,
(CQ_ModifyEnableBit |
CQ_WriteEnableBit |
CQ_WaitEventEnableBit |
CQ_SetEventEnableBit |
CQ_ThreadStartEnableBit), NULL);
OMPI_PTL_ELAN_CHECK_UNEX (rail->r_cmdq, NULL, OMPI_ERROR, 0);
/* Allocate a command port for thread rescheduling etc */
rail->r_ecmdq = OMPI_PTL_ELAN_ALLOC_CMDQ (rail->r_ctx,
rail->r_alloc,
CQ_Size8K,
CQ_EnableAllBits,
NULL);
OMPI_PTL_ELAN_CHECK_UNEX (rail->r_ecmdq, NULL, OMPI_ERROR, 0);
priv_estate->cport = elan4_main2elan (rail->r_ctx,
rail->r_ecmdq->cmdq_mapping);
/* Save the rail pointers */
ems->elan_rail[i] = (ELAN_RAIL *) rail;
ems->rail_intcookie[i] = ems->intcookie;
/* Allocate a Sleep Desc */
es = ompi_init_elan_sleepdesc (ems, rail);
OPAL_LOCK(&mca_ptl_elan_component.elan_lock);
es->es_next = rail->r_sleepDescs;
rail->r_sleepDescs = es;
OPAL_UNLOCK(&mca_ptl_elan_component.elan_lock);
estate->alloc = rail->r_alloc;
estate->vp = ems->elan_vp;
estate->debugFlags = ems->elan_flags;
estate->debugFd = 1;
priv_estate->pageSize = ems->elan_pagesize;
rail->r_estate = estate;
rail->r_railNo = rails[i];
{
struct railtable *rt;
rt = (struct railtable *) malloc (sizeof (struct railtable));
OMPI_PTL_ELAN_CHECK_UNEX (rt, NULL, OMPI_ERROR, 0);
memset (rt, 0, sizeof (struct railtable));
rt->rt_nrails = 1;
rt->rt_rail = 0;
rt->rt_railReal = i;
rt->rt_allRails = (RAIL **) & (ems->all_rails[i]);
rail->r_railTable = rt;
}
} /* for each rail */
/* Free the local variable */
free (rails);
return OMPI_SUCCESS;
}
ELAN_SLEEP *
ompi_init_elan_sleepdesc (mca_ptl_elan_state_t * ems,
RAIL * rail)
{
ELAN_SLEEP *es;
/* XXX: asking the caller to hold the lock */
es = MALLOC (sizeof (ELAN_SLEEP));
OMPI_PTL_ELAN_CHECK_UNEX (es, NULL, NULL, 0);
memset (es, 0, sizeof (ELAN_SLEEP));
/* Assign next interrupt cookie value */
es->es_cookie = ems->intcookie++;
/* XXX, rail[0] is choosen instead this rail */
if (elan4_alloc_intcookie (ems->elan_rail[0]->rail_ctx,
es->es_cookie) < 0) {
opal_output (0,
"[%s:%d] Failed to allocate IRQ cookie \n",
__FILE__, __LINE__);
}
es->es_cmdBlk = ALLOC_ELAN (rail, E4_EVENTBLOCK_SIZE,
E4_EVENTBLOCK_SIZE);
OMPI_PTL_ELAN_CHECK_UNEX (es->es_cmdBlk, 0, NULL, 0);
/*Allocate a pair of command queues for blocking waits with */
es->es_cmdq = OMPI_PTL_ELAN_ALLOC_CMDQ(rail->r_ctx,
rail->r_alloc,
CQ_Size1K,
(CQ_WriteEnableBit |
CQ_WaitEventEnableBit), NULL);
OMPI_PTL_ELAN_CHECK_UNEX (es->es_cmdq, NULL, NULL, 0);
/* This command queue used to fire the IRQ via
a cmd port copy event */
es->es_ecmdq = OMPI_PTL_ELAN_ALLOC_CMDQ (rail->r_ctx,
rail->r_alloc,
CQ_Size1K, /* CQ_EnableAllBits, */
(CQ_WriteEnableBit
| CQ_InterruptEnableBit),
NULL);
OMPI_PTL_ELAN_CHECK_UNEX (es->es_ecmdq, NULL, NULL, 0);
es->es_next = NULL;
/* XXX: asking the caller to release the lock */
return es;
}
int
mca_ptl_elan_state_init (mca_ptl_elan_component_t * emp)
{
int i;
mca_ptl_elan_state_t * ems = &mca_ptl_elan_global_state;
#if 1 || defined(PTL_ELAN_DEV_INIT)
if (OMPI_SUCCESS != ompi_elan_device_init (emp, ems))
return OMPI_ERROR;
#else
#error Not implemented yet
/* Consider using elan_int() later */
ELAN_STATE *elan_state;
elan_state = elan_init(0);
#endif
ems->elan_ctx = ems->elan_rail[0]->rail_ctx;
ems->elan_estate = (void *) ems->all_estates[0];
/* Attach to the device and open to the network */
if(OMPI_SUCCESS != ompi_elan_attach_network (ems))
return OMPI_ERROR;
/* Set the rms_resourceId */
if (rms_getprgid (getpid (), &ems->elan_rmsid) < 0) {
ems->elan_rmsid = -1;
}
/* Now open ourselves to the network */
for (i = 0; ems->elan_rail[i]; i++) {
elan4_block_inputter (ems->elan_rail[i]->rail_ctx, 0);
}
/* Setup communication infrastructure and construct PTL's */
if (OMPI_SUCCESS != ompi_mca_ptl_elan_setup (ems)) {
opal_output (0,
"[%s:%d] error in setting up elan "
"communication state machines for elan PTL's.\n",
__FILE__, __LINE__);
return OMPI_ERROR;
}
return (OMPI_SUCCESS);
}
int
mca_ptl_elan_state_finalize (mca_ptl_elan_component_t * emp)
{
int i;
int num_rails;
mca_ptl_elan_state_t *ems;
ems = &mca_ptl_elan_global_state;
num_rails = ems->elan_nrails;
ompi_module_elan_close_ptls (emp, num_rails);
ompi_module_elan_close_procs (emp, num_rails);
for (i = 0; i < num_rails; i++) {
RAIL *rail;
rail = ems->all_rails[i];
free (rail->r_railTable);
/* Free the memory from the rail allocator */
#if (QSNETLIBS_VERSION_CODE <= QSNETLIBS_VERSION(1,6,6))
elan4_freeMain (rail->r_alloc, rail->r_ecmdq);
elan4_freeMain (rail->r_alloc, rail->r_cmdq);
#else
elan4_free_cmdq (rail->r_ctx, rail->r_ecmdq);
elan4_free_cmdq (rail->r_ctx, rail->r_cmdq);
#endif
elan4_freeElan (rail->r_alloc, ems->all_estates[i]);
/* Since the cookie allocated from rail[0], be consistent here */
elan4_free_intcookie (ems->all_rails[0]->r_ctx,
ems->rail_intcookie[i]);
elan4_destroyAllocator (rail->r_alloc);
elan4_close_sdram (rail->r_sdram);
/*elan4_fini (rail->r_ctx); Not working yet from libelan */
/* Free the rail structure used one the array of pointers
* to the RAILs, either all_rails for elan_rails */
free (ems->all_rails[i]);
}
free (ems->elan_rail);
free (ems->all_estates);
free (ems->all_rails);
free (ems->elan_cap);
return (OMPI_SUCCESS);
}
#if OMPI_PTL_ELAN_THREADING
/* XXX:
* Create threads per PTL and have them up running,
* Blocking on the completion queues */
int
mca_ptl_elan_thread_init (mca_ptl_elan_component_t * emp)
{
int i, num_rails;
num_rails = emp->num_modules;
#if OMPI_PTL_ELAN_ONE_QUEUE
emp->recv_threads = (struct ompi_ptl_elan_thread_t **)
malloc (num_rails * sizeof(struct ompi_ptl_elan_thread_t*));
for (i = 0; i < num_rails; i ++) {
ompi_ptl_elan_thread_t * t;
t = (struct ompi_ptl_elan_thread_t *)
malloc (sizeof(struct ompi_ptl_elan_thread_t));
OBJ_CONSTRUCT(&t->thread, opal_thread_t);
t->thread.t_run = (opal_thread_fn_t) mca_ptl_elan_lookup;
t->ptl = emp->modules[i];
pthread_create(&t->thread.t_handle, NULL,
(void *)t->thread.t_run, (void*)t->ptl);
emp->recv_threads[i] = t;
}
#else
/*struct ompi_ptl_elan_thread_t **threads; */
emp->send_threads = (struct ompi_ptl_elan_thread_t **)
malloc (num_rails * sizeof(struct ompi_ptl_elan_thread_t*));
emp->recv_threads = (struct ompi_ptl_elan_thread_t **)
malloc (num_rails * sizeof(struct ompi_ptl_elan_thread_t*));
for (i = 0; i < num_rails; i ++) {
ompi_ptl_elan_thread_t * t;
/* XXX:
* For now, there is a thread for send also.
* Later to be combined with the receiving thread */
t = (struct ompi_ptl_elan_thread_t *)
malloc (sizeof(struct ompi_ptl_elan_thread_t));
OBJ_CONSTRUCT(&t->thread, opal_thread_t);
t->thread.t_run = (opal_thread_fn_t) mca_ptl_elan_update_desc;
t->ptl = emp->modules[i];
pthread_create(&t->thread.t_handle, NULL,
(void *)t->thread.t_run, (void*)t->ptl);
emp->send_threads[i] = t;
t = (struct ompi_ptl_elan_thread_t *)
malloc (sizeof(struct ompi_ptl_elan_thread_t));
OBJ_CONSTRUCT(&t->thread, opal_thread_t);
t->thread.t_run = (opal_thread_fn_t) mca_ptl_elan_drain_recv;
t->ptl = emp->modules[i];
pthread_create(&t->thread.t_handle, NULL,
(void *)t->thread.t_run, (void*)t->ptl);
emp->recv_threads[i] = t;
}
#endif
return (OMPI_SUCCESS);
}
/* XXX:
* Send signal/interrupt(s) to threads and wait for them to join */
int
mca_ptl_elan_thread_close (mca_ptl_elan_component_t * emp)
{
int i, num_rails;
num_rails = emp->num_modules;
for (i = 0; i < num_rails; i ++) {
int header_length;
int destvp;
mca_ptl_elan_module_t *ptl;
mca_ptl_base_header_t *hdr;
struct ompi_ptl_elan_ctrl_desc_t * desc;
ELAN4_CTX *ctx;
ptl = mca_ptl_elan_component.modules[i];
header_length = sizeof(mca_ptl_base_header_t);
destvp = ptl->elan_vp;
ctx = ptl->ptl_elan_ctx,
desc = ptl->queue->last;
hdr = (mca_ptl_base_header_t *) desc->mesg;
#if OMPI_PTL_ELAN_ONE_QUEUE
hdr->hdr_common.hdr_type = MCA_PTL_HDR_TYPE_STOP + 8;
#else
hdr->hdr_common.hdr_type = MCA_PTL_HDR_TYPE_STOP;
#endif
hdr->hdr_common.hdr_flags = 0; /* XXX: to change if needed */
hdr->hdr_common.hdr_size = sizeof (mca_ptl_base_header_t);
INITEVENT_WORD (ctx, desc->elan_event, &desc->main_doneWord);
RESETEVENT_WORD (&desc->main_doneWord);
PRIMEEVENT_WORD (ctx, desc->elan_event, 2);
/* Initialize some of the dma structures */
desc->main_dma.dma_srcEvent = 0; //SDRAM2ELAN (ctx, desc->elan_event);
desc->main_dma.dma_dstEvent = SDRAM2ELAN (ctx, ptl->queue->input);
desc->main_dma.dma_dstAddr = 0x0ULL;
desc->main_dma.dma_srcAddr = MAIN2ELAN (ctx, hdr);
desc->main_dma.dma_typeSize = E4_DMA_TYPE_SIZE (header_length,
DMA_DataTypeByte,
DMA_QueueWrite, 16);
desc->main_dma.dma_cookie = elan4_local_cookie (ptl->queue->tx_cpool,
E4_COOKIE_TYPE_LOCAL_DMA, destvp);
desc->main_dma.dma_vproc = destvp;
#if defined(OMPI_PTL_PCM_RMS)
/* finish the recv thread */
MEMBAR_VISIBLE ();
elan4_run_dma_cmd (ptl->queue->tx_cmdq, (DMA *) & desc->main_dma);
elan4_flush_cmdq_reorder (ptl->queue->tx_cmdq);
#endif
#if !OMPI_PTL_ELAN_ONE_QUEUE
/* finish the send thread */
desc->main_dma.dma_dstEvent = SDRAM2ELAN (ctx, ptl->comp->input);
MEMBAR_VISIBLE ();
elan4_run_dma_cmd (ptl->queue->tx_cmdq, (DMA *) & desc->main_dma);
elan4_flush_cmdq_reorder (ptl->queue->tx_cmdq);
#endif
}
/* Join all threads */
for (i = 0; i < num_rails; i ++) {
ompi_ptl_elan_thread_t * tsend, *trecv;
int *ptr = (int *)malloc(sizeof(int));
trecv = emp->recv_threads[i];
#if !OMPI_PTL_ELAN_ONE_QUEUE
tsend = emp->send_threads[i];
opal_thread_join(&tsend->thread, &ptr);
#endif
opal_thread_join(&trecv->thread, &ptr);
}
return (OMPI_SUCCESS);
}
#endif /* End of OMPI_PTL_ELAN_THREADING */

52
ompi/mca/ptl/elan/src/ptl_elan_peer.c
Просмотреть файл

@ -1,52 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include "include/types.h"
#include "mca/pml/base/pml_base_sendreq.h"
#include "ptl_elan.h"
#include "ptl_elan_peer.h"
#include "ptl_elan_proc.h"
#include "ptl_elan_frag.h"
static void
mca_ptl_elan_peer_construct (mca_ptl_elan_peer_t * ptl_peer)
{
ptl_peer->peer_ptl = NULL;
ptl_peer->peer_proc = NULL;
ptl_peer->peer_vp = -1;
ptl_peer->peer_rails = 0;
ptl_peer->num_credits = 0; /* Number of credits for the local PTL */
ptl_peer->max_credits = 0; /* Number of credits for the local PTL */
ptl_peer->resending = 0; /* A resending stage, no more new dma's */
ptl_peer->num_resends = 0; /* How many times I have retried */
}
/* Cleanup any resources held by the peer. */
static void
mca_ptl_elan_peer_destruct (mca_ptl_elan_peer_t * ptl_peer)
{
mca_ptl_elan_proc_remove (ptl_peer->peer_proc, ptl_peer);
/*mca_ptl_elan_peer_close(ptl_peer); */
}
opal_class_t mca_ptl_elan_peer_t_class = {
"mca_elan_ptl_peer_t",
OBJ_CLASS (opal_list_item_t),
(opal_construct_t) mca_ptl_elan_peer_construct,
(opal_destruct_t) mca_ptl_elan_peer_destruct
};

58
ompi/mca/ptl/elan/src/ptl_elan_peer.h
Просмотреть файл

@ -1,58 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PTL_ELAN_PEER_H
#define MCA_PTL_ELAN_PEER_H
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include "opal/class/opal_list.h"
#include "opal/event/event.h"
#include "mca/pml/pml.h"
#include "mca/ptl/ptl.h"
#include "ptl_elan_frag.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
/**
* An abstraction that represents a connection to a peer process.
*/
struct mca_ptl_elan_peer_t {
opal_list_item_t super;
struct mca_ptl_elan_module_t *peer_ptl;
struct mca_ptl_elan_proc_t *peer_proc;
int peer_vp;
int peer_rails;
int num_credits; /* Got to be an arry for rails */
int max_credits; /* Got to be an arry for rails */
int resending;
int num_resends;
};
typedef struct mca_ptl_elan_peer_t mca_ptl_elan_peer_t;
OBJ_CLASS_DECLARATION(mca_ptl_elan_peer_t);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

1404
ompi/mca/ptl/elan/src/ptl_elan_priv.c
Просмотреть файл

Разница между файлами не показана из-за своего большого размера Загрузить разницу

484
ompi/mca/ptl/elan/src/ptl_elan_priv.h
Просмотреть файл

@ -1,484 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#ifndef MCA_PTL_ELAN_PRIV_H
#define MCA_PTL_ELAN_PRIV_H
#include <errno.h>
#include <unistd.h>
#include <string.h>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include "constants.h"
#include "opal/event/event.h"
#include "opal/util/if.h"
#include "opal/util/argv.h"
#include "opal/util/output.h"
#include "mca/pml/pml.h"
#include "mca/ptl/ptl.h"
#include "mca/pml/base/pml_base_sendreq.h"
#include "mca/pml/base/pml_base_recvreq.h"
#include "mca/base/mca_base_param.h"
#include "mca/pml/base/pml_base_module_exchange.h"
#include "ptl_elan.h"
#include "ptl_elan_proc.h"
#include "ptl_elan_frag.h"
#define _TRACK_MALLOC 0
#include <elan/elan.h>
#include <elan/init.h>
#include <rms/rmscall.h>
#include "misc_sys.h"
#include "init_sys.h"
#include "elan4/events.h"
#define PTL_ELAN_DEBUG_NONE (0x000)
#define PTL_ELAN_DEBUG_INIT (0x001)
#define PTL_ELAN_DEBUG_FIN (0x002)
#define PTL_ELAN_DEBUG_DESC (0x004)
#define PTL_ELAN_DEBUG_THREAD (0x008)
#define PTL_ELAN_DEBUG_SEND (0x010)
#define PTL_ELAN_DEBUG_RECV (0x020)
#define PTL_ELAN_DEBUG_ACK (0x040)
#define PTL_ELAN_DEBUG_MAC (0x080)
#define PTL_ELAN_DEBUG_QDMA (0x100)
#define PTL_ELAN_DEBUG_PUT (0x200)
#define PTL_ELAN_DEBUG_GET (0x400)
#define PTL_ELAN_DEBUG_CHAIN (0x800)
#define PTL_ELAN_DEBUG_FLAG (PTL_ELAN_DEBUG_NONE)
#define LOG_PRINT(flag, args...) \
do { \
if (PTL_ELAN_DEBUG_FLAG & flag) { \
char *rms_rank = getenv("RMS_RANK"); \
fprintf(stderr, "[proc%s:%s:%d] ", \
rms_rank, __FUNCTION__, __LINE__); \
fprintf(stderr, args); \
} \
} while (0)
#define OMPI_PTL_ELAN_CMQ_REUSE (1)
#define OMPI_PTL_ELAN_MAX_QSIZE (2048)
#define OMPI_PTL_ELAN_MAX_QSLOTS (128)
#define OMPI_PTL_ELAN_LOST_QSLOTS (1)
#define OMPI_PTL_ELAN_MAX_QDESCS (128)
#define OMPI_PTL_ELAN_MAX_PUTGET (32)
#define OMPI_PTL_ELAN_QDMA_RETRY (16)
#define OMPI_PTL_ELAN_MAX_PGDESC (8)
#define OMPI_PTL_ELAN_FASTPATH (0x1)
#define OMPI_PTL_ELAN_SLOT_ALIGN (128)
#define OMPI_PTL_ELAN_GET_MAX(a,b) ((a>b)? a:b)
#define OMPI_PTL_ELAN_GET_MIN(a,b) ((a<b)? a:b)
#define OMPI_PTL_ELAN_ALIGNUP(x,a) (((unsigned int)(x) + ((a)-1)) & (-(a)))
/* XXX: Potentially configurable parameters */
#define OMPI_PTL_ELAN_NUM_QDESCS (16)
#define OMPI_PTL_ELAN_NUM_PUTGET (16)
#define OMPI_PTL_ELAN_ZERO_FFRAG (1)
#define OMPI_PTL_ELAN_USE_DTP (0)
#define OMPI_PTL_ELAN_ENABLE_GET (0)
#define OMPI_PTL_ELAN_COMP_QUEUE (1)
#define OMPI_PTL_ELAN_ONE_QUEUE (OMPI_PTL_ELAN_COMP_QUEUE && 1)
#define OMPI_PTL_ELAN_THREADING \
(OMPI_PTL_ELAN_COMP_QUEUE && OMPI_HAVE_POSIX_THREADS)
#if (QSNETLIBS_VERSION_CODE <= QSNETLIBS_VERSION(1,6,6))
#define OMPI_PTL_ELAN_ALLOC_CMDQ elan4_alloc_cmdq
#define OMPI_PTL_ELAN_PROBE_CMDQ elan4_probe_cmdq
#else
#define OMPI_PTL_ELAN_ALLOC_CMDQ(ctx, alloc, size, bits, params) \
elan4_alloc_cmdq (ctx, size, bits, params)
#define OMPI_PTL_ELAN_PROBE_CMDQ(ctx, alloc, maxcmds, flags) \
elan4_probe_cmdq (ctx, maxcmds, flags)
#endif
#define OMPI_PTL_ELAN_CHECK_UNEX(value, unexp, errno, output) \
do { \
if (value == unexp) { \
opal_output(output, \
"[%s:%d] alloc received unexpect value \n", \
__FILE__, __LINE__); \
return errno; \
} \
} while (0)
#if OMPI_PTL_ELAN_CMQ_REUSE
#define PTL_ELAN4_GET_QBUFF(dspace, ctx, bsize, csize) \
do { \
if (ptl_elan_cmdq_space.free == 0) { \
opal_output(0, \
"[%s:%d] error acquiring cmdq space \n", \
__FILE__, __LINE__); \
} else { \
ptl_elan_cmdq_space.free --; \
dspace = ptl_elan_cmdq_space.space; \
} \
} while (0)
#define PTL_ELAN4_FREE_QBUFF(ctx, buff, bsize) \
do { \
if (ptl_elan_cmdq_space.free >= ptl_elan_cmdq_space.total || \
ptl_elan_cmdq_space.space != buff ) { \
opal_output(0, \
"[%s:%d] error releasing cmdq space \n", \
__FILE__, __LINE__); \
} else { \
ptl_elan_cmdq_space.free ++; \
} \
} while (0)
#else
#define PTL_ELAN4_GET_QBUFF(dspace, ctx, bsize, csize) \
dspace = elan4_alloccq_space(ctx, bsize, csize);
#define PTL_ELAN4_FREE_QBUFF elan4_freecq_space
#endif
enum {
/* the first four bits for type */
MCA_PTL_ELAN_DESC_NULL = 0x00,
MCA_PTL_ELAN_DESC_QDMA = 0x01,
MCA_PTL_ELAN_DESC_PUT = 0x02, /* QDMA + PUT */
MCA_PTL_ELAN_DESC_GET = 0x04, /* QDMA + GET */
/* next first four bits for status */
MCA_PTL_ELAN_DESC_LOCAL = 0x10,
MCA_PTL_ELAN_DESC_CACHED = 0x20
};
/* XXX: Temporarily a type of header to stop threads */
enum {
MCA_PTL_HDR_TYPE_STOP = 0x3F /* Only a character */
};
/* To set up a component-wise list of free cmdq space */
struct ompi_ptl_elan_cmdq_space_t {
int total;
int free;
E4_Addr space;
};
typedef struct ompi_ptl_elan_cmdq_space_t ompi_ptl_elan_cmdq_space_t;
struct ompi_ptl_elan_thread_t
{
opal_thread_t thread;
mca_ptl_elan_module_t *ptl;
};
typedef struct ompi_ptl_elan_thread_t ompi_ptl_elan_thread_t;
/**
* Structure used to publish elan information to peers.
*/
struct mca_ptl_elan_addr_t {
int elan_vp;
int inuse;
ompi_process_name_t gid;
};
typedef struct mca_ptl_elan_addr_t mca_ptl_elan_addr_t;
struct ompi_ptl_elan_recv_queue_t {
/* Events needs to be aligned */
EVENT_WORD qr_doneWord;
ADDR_SDRAM qr_qEvent;
EVENT32 *qr_elanDone;
/* The one don't care */
E4_uint64 qr_efitem;
E4_uint64 qr_efptr;
E4_uint64 qr_elitem;
void *qr_base;
void *qr_fptr;
void *qr_top;
E4_CmdQ *qr_cmdq;
ELAN_SLEEP *qr_es;
RAIL *qr_rail;
};
typedef struct ompi_ptl_elan_recv_queue_t ompi_ptl_elan_recv_queue_t;
struct ompi_ptl_elan_comp_queue_t {
/** <Elan located INPUT_QUEUE_ALIGN'ed with INPUT_QUEUE_SIZE */
E4_InputQueue *input;
opal_mutex_t rx_lock;
int rx_buffsize;
int rx_slotsize;
int rx_nslots;
/* Recv Queue has to be well-aligned */
ompi_ptl_elan_recv_queue_t *rxq;
};
typedef struct ompi_ptl_elan_comp_queue_t ompi_ptl_elan_comp_queue_t;
/**
* ELAN descriptor for send
*/
#define ELAN_BASE_DESC_FIELDS \
E4_DMA64 main_dma; /**< 8-byte aligned */ \
/* 8 byte aligned */ \
volatile E4_uint64 main_doneWord; \
/* 8 byte aligned */ \
E4_Event *elan_event; \
void *desc_buff; \
/* 8 byte aligned */ \
mca_pml_base_request_t *req; \
mca_ptl_elan_module_t *ptl; \
/* 8 byte aligned */ \
int desc_type; \
int desc_status; \
/* 8 byte aligned */ \
E4_DMA64 comp_dma; \
/* 8 byte aligned */ \
volatile E4_uint64 comp_doneWord; \
/* 8 byte aligned */ \
E4_Event *comp_event; /* E4_Event plus pad */ \
/* 8 byte aligned */ \
E4_Addr *comp_buff; \
E4_Addr *comp_pad; \
E4_Addr comp_srcAddr; \
E4_Addr comp_dstAddr; \
/* 8 byte aligned */
struct ompi_ptl_elan_ctrl_desc_t {
E4_DMA64 main_dma;
/* 8 byte aligned */
volatile E4_uint64 main_doneWord;
/* 8 byte aligned */
E4_Event *elan_event;
void *mesg;
/* 8 byte aligned */
};
typedef struct ompi_ptl_elan_ctrl_desc_t ompi_ptl_elan_ctrl_desc_t;
struct ompi_ptl_elan_base_desc_t {
ELAN_BASE_DESC_FIELDS
/* 8 byte aligned */
};
typedef struct ompi_ptl_elan_base_desc_t ompi_ptl_elan_base_desc_t;
struct ompi_ptl_elan_qdma_desc_t {
ELAN_BASE_DESC_FIELDS
/* 8 byte aligned */
uint8_t buff[INPUT_QUEUE_MAX]; /**< queue data */
/* 8 byte aligned */
};
typedef struct ompi_ptl_elan_qdma_desc_t ompi_ptl_elan_qdma_desc_t;
struct ompi_ptl_elan_queue_ctrl_t {
/** <Elan located INPUT_QUEUE_ALIGN'ed with INPUT_QUEUE_SIZE */
E4_InputQueue *input;
/** <transmit queue structures */
void *tx_q;
E4_CmdQ *tx_cmdq;
ELAN4_COOKIEPOOL *tx_cpool;
ompi_free_list_t tx_desc_free;
/* User progression */
opal_mutex_t rx_lock;
int rx_buffsize;
int rx_slotsize;
int rx_nslots;
/* Recv Queue has to be well-aligned */
ompi_ptl_elan_recv_queue_t *rxq;
ompi_ptl_elan_ctrl_desc_t *last;
};
typedef struct ompi_ptl_elan_queue_ctrl_t ompi_ptl_elan_queue_ctrl_t;
struct ompi_ptl_elan_putget_desc_t {
ELAN_BASE_DESC_FIELDS
/* 8 byte aligned */
E4_DMA64 chain_dma; /**< Must be 8-byte aligned */
/* 8 byte aligned */
volatile E4_uint64 chain_doneWord;
/* 8 byte aligned */
E4_Event *chain_event; /* E4_Event plus pad */
E4_Addr *chain_buff;
E4_Addr src_elan_addr;
E4_Addr dst_elan_addr;
/* 8 byte aligned */
uint8_t buff[sizeof(mca_ptl_base_header_t)];
};
typedef struct ompi_ptl_elan_putget_desc_t ompi_ptl_elan_putget_desc_t;
struct ompi_ptl_elan_putget_ctrl_t {
/** <transmit queue structures */
uint32_t pg_throttle;
int pg_retryCount;
int pg_evictCache;
int32_t pg_waitType;
ELAN_FLAGS pg_flags;
opal_mutex_t pg_lock;
E4_CmdQ *put_cmdq;
E4_CmdQ *get_cmdq;
E4_uint64 *pg_cmdStream;
ELAN4_COOKIEPOOL *pg_cpool;
E4_CmdQParams *pg_cmdPar;
uint32_t *pg_pendingGetCount;
opal_list_t put_desc;
opal_list_t get_desc;
ompi_free_list_t put_desc_free;
ompi_free_list_t get_desc_free;
};
typedef struct ompi_ptl_elan_putget_ctrl_t ompi_ptl_elan_putget_ctrl_t;
struct mca_ptl_elan_state_t {
/* User configurable parameters */
int initialized;
char *elan_version; /**< Version of the elan library */
uint64_t elan_debug; /**< elan debug tracing output */
uint64_t elan_traced; /**< elan TRACE output */
uint64_t elan_flags;
FILE *elan_debugfile; /* Debug output file handle */
int elan_signalnum;
long elan_waittype; /**< how to wait for events */
size_t main_size; /**< size of Main memory allocator heap */
size_t elan_size; /**< size of Elan memory allocator heap */
void *main_base; /**< Main memory allocator heap base */
void *elan_base; /**< Elan memory allocator heap base */
/* other state parameters */
unsigned int elan_vp; /**< elan vpid, not ompi vpid */
unsigned int elan_nvp; /**< total # of elan vpid */
int *elan_localvps; /**< mapping of localId to elan vp */
int elan_localid; /**< # of local elan vpids */
int elan_numlocals; /**< # of local elan vpids */
int elan_maxlocals; /**< maximum # of local elan vpids */
int elan_nrails; /**< # of rails */
int elan_rmsid; /**< rms resource id */
int intcookie;
long elan_pagesize;
pid_t elan_pid;
/* TODO:
* Even though the elan threads are not utilized for now.
* We provide memory/state control structures for later extensions.
* A simple type casting of ELAN_ESTATE can bring
* the complete structure of the ELAN_EPRIVSATE.
*/
ELAN_LOCATION elan_myloc;
void *elan_cap; /**< job capability */
ELAN_CTX *elan_ctx; /**< Elan ctx of the 0th rail */
void *elan_estate; /**< Elan state of the 0th rail */
ELAN_RAIL **elan_rail; /**< pointers to Rail control struct for all rails */
RAIL **all_rails; /**< all rails */
int *rail_intcookie; /**< record the cookies for the rail */
ADDR_SDRAM *all_estates;
mca_ptl_elan_component_t *elan_component;
};
typedef struct mca_ptl_elan_state_t mca_ptl_elan_state_t;
/* Util functions, consider moving into a file ptl_elan_util.h */
ELAN_SLEEP *ompi_init_elan_sleepdesc (mca_ptl_elan_state_t * ems,
RAIL * rail);
/* Initialization and finalization routines */
int mca_ptl_elan_state_init(mca_ptl_elan_component_t * emp);
int mca_ptl_elan_state_finalize(mca_ptl_elan_component_t * emp);
int mca_ptl_elan_thread_init(mca_ptl_elan_component_t * emp);
int mca_ptl_elan_thread_close(mca_ptl_elan_component_t * emp);
/* communication initialization prototypes */
int ompi_init_elan_qdma (mca_ptl_elan_component_t * emp,
int num_rails);
int ompi_init_elan_putget (mca_ptl_elan_component_t * emp,
int num_rails);
int ompi_init_elan_stat (mca_ptl_elan_component_t * emp,
int num_rails);
/* communication prototypes */
int mca_ptl_elan_start_desc(mca_ptl_elan_send_frag_t *desc,
mca_ptl_elan_peer_t *ptl_peer,
mca_pml_base_send_request_t *sendreq,
size_t offset,
size_t *size,
int flags);
int mca_ptl_elan_start_ack (mca_ptl_base_module_t * ptl,
mca_ptl_elan_send_frag_t * desc,
mca_ptl_elan_recv_frag_t * recv_frag);
int mca_ptl_elan_get_with_ack (mca_ptl_base_module_t * ptl,
mca_ptl_elan_send_frag_t * frag,
mca_ptl_elan_recv_frag_t * recv_frag);
int mca_ptl_elan_poll_queue(ompi_ptl_elan_recv_queue_t *rxq);
int mca_ptl_elan_wait_queue(mca_ptl_elan_module_t * ptl,
ompi_ptl_elan_recv_queue_t *rxq,
long usecs);
/* control, synchronization and state prototypes */
int mca_ptl_elan_drain_recv(mca_ptl_elan_module_t * ptl);
int mca_ptl_elan_update_desc(mca_ptl_elan_module_t * ptl);
int mca_ptl_elan_lookup(mca_ptl_elan_module_t * ptl);
int
mca_ptl_elan_start_get (mca_ptl_elan_send_frag_t * frag,
struct mca_ptl_elan_peer_t *ptl_peer,
struct mca_pml_base_recv_request_t *req,
size_t offset,
size_t *size,
int flags);
/**
* utility routines for parameter registration
*/
static inline char *
mca_ptl_elan_param_register_string (const char *param_name,
const char *default_value)
{
int id;
char *param_value;
id = mca_base_param_register_string ("ptl", "elan", param_name, NULL,
default_value);
mca_base_param_lookup_string (id, &param_value);
return param_value;
}
static inline int
mca_ptl_elan_param_register_int (const char *param_name,
int default_value)
{
int id;
int param_value;
param_value = default_value;
id = mca_base_param_register_int ("ptl", "elan", param_name, NULL,
default_value);
mca_base_param_lookup_int (id, &param_value);
return param_value;
}
#endif

215
ompi/mca/ptl/elan/src/ptl_elan_proc.c
Просмотреть файл

@ -1,215 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <string.h>
#include "include/sys/atomic.h"
#include "class/opal_hash_table.h"
#include "mca/pml/base/pml_base_module_exchange.h"
#include "ptl_elan.h"
#include "ptl_elan_peer.h"
#include "ptl_elan_proc.h"
#include "ptl_elan_priv.h"
static void mca_ptl_elan_proc_construct (mca_ptl_elan_proc_t * proc);
static void mca_ptl_elan_proc_destruct (mca_ptl_elan_proc_t * proc);
static mca_ptl_elan_proc_t *mca_ptl_elan_proc_lookup_ompi (ompi_proc_t *
ompi_proc);
opal_class_t mca_ptl_elan_proc_t_class = {
"mca_ptl_elan_proc_t",
OBJ_CLASS (opal_list_item_t),
(opal_construct_t) mca_ptl_elan_proc_construct,
(opal_destruct_t) mca_ptl_elan_proc_destruct
};
/**
* Initialize elan proc instance
*/
void
mca_ptl_elan_proc_construct (mca_ptl_elan_proc_t * proc)
{
proc->proc_ompi = NULL;
proc->proc_addrs = NULL;
proc->proc_addr_count = 0;
proc->proc_peers = NULL;
proc->proc_peer_count = 0;
proc->proc_guid.cellid = 0;
proc->proc_guid.jobid = 0;
proc->proc_guid.vpid = 0;
OBJ_CONSTRUCT (&proc->proc_lock, opal_mutex_t);
/* add to list of all proc instance */
OPAL_THREAD_LOCK (&mca_ptl_elan_component.elan_lock);
opal_list_append (&mca_ptl_elan_component.elan_procs, &proc->super);
OPAL_THREAD_UNLOCK (&mca_ptl_elan_component.elan_lock);
return;
}
/*
* Cleanup elan proc instance
*/
void
mca_ptl_elan_proc_destruct (mca_ptl_elan_proc_t * proc)
{
/* remove from list of all proc instances */
OPAL_THREAD_LOCK (&mca_ptl_elan_component.elan_lock);
opal_list_remove_item (&mca_ptl_elan_component.elan_procs, &proc->super);
OPAL_THREAD_UNLOCK (&mca_ptl_elan_component.elan_lock);
/* release resources */
if (NULL != proc->proc_peers)
free (proc->proc_peers);
return;
}
/*
* Create a ELAN process structure. There is a one-to-one correspondence
* between a ompi_proc_t and a mca_ptl_elan_proc_t instance.
* We cache additional data (specifically the list
* of mca_ptl_elan_peer_t instances, and publiched
* addresses) associated w/ a given destination on this datastructure.
*/
mca_ptl_elan_proc_t *
mca_ptl_elan_proc_create (ompi_proc_t * ompi_proc)
{
int rc;
size_t size;
mca_ptl_elan_proc_t *ptl_proc;
ptl_proc = mca_ptl_elan_proc_lookup_ompi (ompi_proc);
if (ptl_proc != NULL) {
return ptl_proc;
}
ptl_proc = OBJ_NEW (mca_ptl_elan_proc_t);
ptl_proc->proc_ompi = ompi_proc;
ptl_proc->proc_guid = ompi_proc->proc_name;
/* Extract exposed addresses from remote proc */
rc = mca_base_modex_recv (&mca_ptl_elan_component.super.ptlm_version,
ompi_proc, (void **) &ptl_proc->proc_addrs,
&size);
if (rc != OMPI_SUCCESS) {
opal_output (0,
"[%s:%d] mca_base_modex_recv failed to recv data \n",
__FILE__, __LINE__);
OBJ_RELEASE (ptl_proc);
return NULL;
}
if (0 != (size % sizeof (mca_ptl_elan_addr_t))) {
opal_output (0, "[%s:%d] invalid received data size %d\n",
__FILE__, __LINE__, size);
return NULL;
}
ptl_proc->proc_addr_count = size / sizeof (mca_ptl_elan_addr_t);
/* allocate space for peer array - one for each exported address */
ptl_proc->proc_peers = (mca_ptl_elan_peer_t **)
malloc (ptl_proc->proc_addr_count *
sizeof (mca_ptl_elan_peer_t *));
if (NULL == ptl_proc->proc_peers) {
OBJ_RELEASE (ptl_proc);
opal_output (0, "[%s:%d] unable to allocate peer procs \n"
__FILE__, __LINE__);
return NULL;
}
if (NULL == mca_ptl_elan_component.elan_local
&& ompi_proc == ompi_proc_local ()) {
mca_ptl_elan_component.elan_local = ptl_proc;
}
return ptl_proc;
}
/*
* Look for an existing ELAN process instances based on the associated
* ompi_proc_t instance.
*/
static mca_ptl_elan_proc_t *
mca_ptl_elan_proc_lookup_ompi (ompi_proc_t * ompi_proc)
{
mca_ptl_elan_proc_t *elan_proc;
OPAL_THREAD_LOCK (&mca_ptl_elan_component.elan_lock);
elan_proc = (mca_ptl_elan_proc_t *)
opal_list_get_first (&mca_ptl_elan_component.elan_procs);
for (; elan_proc != (mca_ptl_elan_proc_t *)
opal_list_get_end (&mca_ptl_elan_component.elan_procs);
elan_proc =
(mca_ptl_elan_proc_t *) opal_list_get_next (elan_proc)) {
if (elan_proc->proc_ompi == ompi_proc) {
OPAL_THREAD_UNLOCK (&mca_ptl_elan_component.elan_lock);
return elan_proc;
}
}
OPAL_THREAD_UNLOCK (&mca_ptl_elan_component.elan_lock);
return NULL;
}
/*
* Look for an existing ELAN process instance based on the globally unique
* process identifier.
*/
mca_ptl_elan_proc_t *
mca_ptl_elan_proc_lookup (void *guid,
size_t size)
{
return NULL;
}
/*
* Remove a peer from the proc array and indicate the address is
* no longer in use.
*/
int
mca_ptl_elan_proc_remove (mca_ptl_elan_proc_t * ptl_proc,
mca_ptl_elan_peer_t * ptl_peer)
{
return OMPI_SUCCESS;
}
/*
* loop through all available PTLs for one matching the source address
* of the request.
*/
bool
mca_ptl_elan_proc_accept (mca_ptl_elan_proc_t * ptl_proc,
struct sockaddr_in * addr,
int sd)
{
return false;
}

69
ompi/mca/ptl/elan/src/ptl_elan_proc.h
Просмотреть файл

@ -1,69 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PTL_ELAN_PROC_H
#define MCA_PTL_ELAN_PROC_H
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include "opal/class/opal_object.h"
#include "proc/proc.h"
#include "ptl_elan.h"
#include "ptl_elan_peer.h"
#if defined(c_plusplus) || defined(__cplusplus)
extern "C" {
#endif
extern opal_class_t mca_ptl_elan_proc_t_class;
/**
* Represents the state of a remote process. Also cache an instance
* of mca_ptl_base_peer_t for each
* PTL instance that attempts to open a connection to the process.
*/
struct mca_ptl_elan_proc_t {
opal_list_item_t super; /**< allow proc to be placed on a list */
ompi_proc_t *proc_ompi; /**< pointer to corresponding ompi_proc_t */
ompi_process_name_t proc_guid; /**< globally unique identifier
for the process */
struct mca_ptl_elan_addr_t *proc_addrs; /**< array of addresses published
by peer */
size_t proc_addr_count;
struct mca_ptl_elan_peer_t **proc_peers; /**< array of peers */
size_t proc_peer_count; /**< number of peers */
opal_mutex_t proc_lock; /**< lock to for proc state */
};
typedef struct mca_ptl_elan_proc_t mca_ptl_elan_proc_t;
mca_ptl_elan_proc_t* mca_ptl_elan_proc_create(ompi_proc_t* ompi_proc);
mca_ptl_elan_proc_t* mca_ptl_elan_proc_lookup(void *guid, size_t size);
static inline mca_ptl_elan_proc_t* mca_ptl_elan_proc_local(void)
{
return NULL;
}
int mca_ptl_elan_proc_remove(mca_ptl_elan_proc_t *, mca_ptl_elan_peer_t *);
bool mca_ptl_elan_proc_accept(mca_ptl_elan_proc_t *,
struct sockaddr_in *, int sd);
#if defined(c_plusplus) || defined(__cplusplus)
}
#endif
#endif

59
ompi/mca/ptl/elan/tests/Makefile.am
Просмотреть файл

@ -1,59 +0,0 @@
# -*- makefile -*-
#
# Copyright (c) 2004-2005 The Trustees of Indiana University.
# All rights reserved.
# Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
# All rights reserved.
# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
# University of Stuttgart. All rights reserved.
# Copyright (c) 2004-2005 The Regents of the University of California.
# All rights reserved.
# $COPYRIGHT$
#
# Additional copyrights may follow
#
# $HEADER$
#
include $(top_ompi_srcdir)/config/Makefile.options
AM_CPPFLAGS = -I$(top_ompi_builddir)/src/include \
-I$(top_ompi_srcdir)/src -I$(top_ompi_srcdir)/src/include \
-I../src -I../.. -I../../tcp/base \
-I/usr/lib/qsnet/elan4/include
LDFLAGS += -L$(prefix)/lib -L../src/.libs
EXECS = qsnet_init qsnet_qdma qsnet_rdma mpitest lat check init_elan bw check_bw
units: $(EXECS)
qsnet_init: qsnet_init.c
$(CC) -g $(AM_CPPFLAGS) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) -o $@ $< $(LIBS) -lmpi -pthread -lm -lmca_ptl_elan
qsnet_qdma: qsnet_qdma.c
$(CC) -g $(AM_CPPFLAGS) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) -o $@ $< $(LIBS) -lmpi -pthread -lm -lmca_ptl_elan
qsnet_rdma: qsnet_rdma.c
$(CC) -g $(AM_CPPFLAGS) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) -o $@ $< $(LIBS) -lmpi -pthread -lm -lmca_ptl_elan
mpitest: mpi_test.c
${HOME}/installs/openmpi/bin/mpicc -g -o mpitest mpi_test.c
check_bw: %: %.c
${HOME}/installs/openmpi/bin/mpicc -g -o $@ check_bw.c
check: %: %.c
${HOME}/installs/openmpi/bin/mpicc -g -o $@ check.c
bw: %: %.c
${HOME}/installs/openmpi/bin/mpicc -g -o bw bw.c
lat: %: %.c
${HOME}/installs/openmpi/bin/mpicc -g -o lat lat.c
init_elan: %: %.c
gcc -g $< -o $@ -lelan4 -lelan -lrmscall -lelan3
clean:
-rm -rf $(EXECS)

106
ompi/mca/ptl/elan/tests/bw.c
Просмотреть файл

@ -1,106 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "mpi.h"
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <math.h>
#include <assert.h>
#include <sys/time.h>
#include "test_util.h"
#define MYBUFSIZE (1<<22)
#define MAX_REQ_NUM 1024
MPI_Request request[MAX_REQ_NUM];
MPI_Status tmp_stat[MAX_REQ_NUM];
int skip = 40;
int
main (int argc,
char *argv[])
{
int myid, numprocs, i, j;
int min=0, max=0, size=0, loop, win, page_size;
struct timeval t_start, t_end;
char *s_buf, *r_buf;
s_buf = (char *) malloc (sizeof(char)*MYBUFSIZE);
r_buf = (char *) malloc (sizeof(char)*MYBUFSIZE);
/* Get some environmental variables set for Open MPI, OOB */
/*env_init_for_elan();*/
MPI_Init (&argc, &argv);
MPI_Comm_size (MPI_COMM_WORLD, &numprocs);
MPI_Comm_rank (MPI_COMM_WORLD, &myid);
if (argc < 5) {
fprintf (stderr, "Usage: %s loop win min max \n", argv[0]);
MPI_Finalize ();
return 0;
}
max = atoi (argv[4]);
min = atoi (argv[3]);
win = atoi (argv[2]);
loop= atoi (argv[1]);
page_size = getpagesize ();
for (i = 0; i < max ; i++) {
s_buf[i] = 'a';
r_buf[i] = 'b';
}
for (size = min; size <= max ; size = (size==0) ? 1 : 2*size) {
for (i = 0; i < loop + skip; i++) {
if (myid == 0) {
/* Start time */
if ( i == skip )
gettimeofday (&t_start, 0);
for (j = 0; j < win; j++) {
MPI_Isend (s_buf, size, MPI_CHAR, 1, 100, MPI_COMM_WORLD,
request + j);
}
MPI_Waitall (win, request, tmp_stat);
MPI_Recv (r_buf, 4, MPI_CHAR, 1, 101, MPI_COMM_WORLD, &tmp_stat[0]);
} else {
for (j = 0; j < win; j++) {
MPI_Irecv (r_buf, size, MPI_CHAR, 0, 100, MPI_COMM_WORLD,
request + j);
}
MPI_Waitall (win, request, tmp_stat);
MPI_Send (s_buf, 4, MPI_CHAR, 0, 101, MPI_COMM_WORLD);
}
}
if (myid == 0) {
double latency;
gettimeofday (&t_end, 0);
latency = ((1.0e6 * t_end.tv_sec + t_end.tv_usec)
- (1.0e6 * t_start.tv_sec + t_start.tv_usec))
/ (loop * win);
fprintf (stdout, "%8d %8.2f\n", size, size / latency);
fflush (stdout);
}
MPI_Barrier (MPI_COMM_WORLD);
}
MPI_Finalize ();
return 0;
}

119
ompi/mca/ptl/elan/tests/check.c
Просмотреть файл

@ -1,119 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <stdio.h>
#include <sys/types.h>
#include <unistd.h>
#include <math.h>
#include <sys/time.h>
#include "mpi.h"
#include "test_util.h"
#define MYBUFSIZE (4*1024*1024)
#define CHECK 1
#define PONG 0
char s_buf[MYBUFSIZE];
char r_buf[MYBUFSIZE];
int skip = 0;
int
main (int argc, char *argv[])
{
int myid, numprocs, i, j;
double startwtime = 0.0, endwtime;
int namelen;
int size;
int loop;
MPI_Status stat;
int sleep = 1;
/*while (sleep > 0);*/
struct timeval t_start, t_end;
loop = 2;
if (argc < 2) {
fprintf (stderr, "Usage: %s msg_size\n", argv[0]);
return 0;
} else {
size = atoi (argv[1]);
if (argc > 2)
loop = atoi (argv[2]);
}
/* Get some environmental variables set for Open MPI, OOB */
env_init_for_elan();
MPI_Init (&argc, &argv);
MPI_Comm_size (MPI_COMM_WORLD, &numprocs);
MPI_Comm_rank (MPI_COMM_WORLD, &myid);
/* touch the data */
for (i = 0; i < size; i++) {
s_buf[i] = 'A' + i % 26;
}
fprintf(stderr, "[proc%d:%s:%d] done with init, to loop %d \n",
myid, __FUNCTION__, __LINE__, loop);
fflush(stderr);
for (i = 0; i < loop + skip; i++) {
if (i == skip)
gettimeofday (&t_start, 0);
if (myid == 0) {
MPI_Send (s_buf, size, MPI_CHAR, 1, i, MPI_COMM_WORLD);
if (PONG)
MPI_Recv (r_buf, size, MPI_CHAR, 1, i, MPI_COMM_WORLD, &stat);
} else {
MPI_Recv (r_buf, size, MPI_CHAR, 0, i, MPI_COMM_WORLD, &stat);
if (PONG)
MPI_Send (s_buf, size, MPI_CHAR, 0, i, MPI_COMM_WORLD);
}
if (CHECK && myid != 0) {
for (j=0; j < size; j ++) {
if (r_buf[j] != 'A' + j % 26 ) {
fprintf(stderr, "[proc%d:%s] byte %d error, %2x \n",
myid, __FUNCTION__, j, r_buf[j]);
break;
} else {
r_buf[j] = '0';
}
}
}
}
gettimeofday (&t_end, 0);
fprintf(stderr, "[proc%d:%s:%d] pingpong\n",
myid, __FUNCTION__, __LINE__);
fflush(stderr);
if (myid == 0) {
double latency;
latency = ((1.0e6 * t_end.tv_sec + t_end.tv_usec)
- (1.0e6 * t_start.tv_sec + t_start.tv_usec)) / (2.0 * loop);
fprintf(stdout, "length %d latency %8f\n",
size, latency);
fflush(stdout);
}
MPI_Finalize ();
return 0;
}

98
ompi/mca/ptl/elan/tests/check_bw.c
Просмотреть файл

@ -1,98 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "mpi.h"
#include <stdio.h>
#include <unistd.h>
#include <stdlib.h>
#include <math.h>
#include <assert.h>
#include <sys/time.h>
#include "test_util.h"
#define MYBUFSIZE (1<<22)
#define MAX_REQ_NUM 1024
MPI_Request request[MAX_REQ_NUM];
MPI_Status tmp_stat[MAX_REQ_NUM];
int skip = 0;
int
main (int argc,
char *argv[])
{
int myid, numprocs, i, j;
int size=0, loop, page_size;
struct timeval t_start, t_end;
char *s_buf, *r_buf;
s_buf = (char *) malloc (sizeof(char)*MYBUFSIZE);
r_buf = (char *) malloc (sizeof(char)*MYBUFSIZE);
/* Get some environmental variables set for Open MPI, OOB */
env_init_for_elan();
if (argc < 3) {
fprintf (stderr, "Usage: %s loop size \n", argv[0]);
MPI_Finalize ();
return 0;
}
MPI_Init (&argc, &argv);
MPI_Comm_size (MPI_COMM_WORLD, &numprocs);
MPI_Comm_rank (MPI_COMM_WORLD, &myid);
size = atoi (argv[2]);
loop= atoi (argv[1]);
page_size = getpagesize ();
for (i = 0; i < size ; i++) {
s_buf[i] = 'a';
r_buf[i] = 'b';
}
if (myid == 0) {
/* Start time */
if ( i == skip )
gettimeofday (&t_start, 0);
for (j = 0; j < loop; j++) {
MPI_Isend (s_buf, size, MPI_CHAR, 1, 100, MPI_COMM_WORLD,
request + j);
}
MPI_Waitall (loop, request, tmp_stat);
} else {
for (j = 0; j < loop; j++) {
MPI_Irecv (r_buf, size, MPI_CHAR, 0, 100, MPI_COMM_WORLD,
request + j);
/*MPI_Wait(request, tmp_stat);*/
}
MPI_Waitall (loop, request, tmp_stat);
}
if (myid == 0) {
double latency;
gettimeofday (&t_end, 0);
latency = ((1.0e6 * t_end.tv_sec + t_end.tv_usec)
- (1.0e6 * t_start.tv_sec + t_start.tv_usec))
/ (loop);
fprintf (stdout, "%8d %8.2f\n", size, size / latency);
fflush (stdout);
}
MPI_Finalize ();
return 0;
}

87
ompi/mca/ptl/elan/tests/init_elan.c
Просмотреть файл

@ -1,87 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <unistd.h>
#include <stdio.h>
#include <stdlib.h>
#include <stdarg.h>
#include <string.h>
#include <sys/param.h>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/time.h>
#include <elan/elan.h>
#include <elan/capability.h>
#include <qsnet/fence.h>
#include <elan4/library.h>
#define USE_BASEINIT 0
#define USE_ELANINIT (1-USE_BASEINIT)
ELAN_LOCATION location;
ELAN_CAPABILITY Cap;
ELAN4_CTX *ctx = NULL;
ELAN4_SDRAM *sdram = NULL;
ELAN4_ALLOC *alloc = NULL;
ELAN4_COOKIEPOOL *cpool = NULL;
ELAN_BASE *elan_base;
int i, self;
int nproc;
int
main (int argc, char *argv[])
{
#if USE_BASEINIT
if (!(elan_base = elan_baseInit(0))) {
perror( "elan_baseInit() failed" );
exit(1);
}
nproc = elan_base->state->nvp;
self = elan_base->state->vp;
elan_gsync(elan_base->allGroup);
#elif USE_ELANINIT
elan_base = (ELAN_BASE*) malloc(sizeof(ELAN_BASE));
if (!(elan_base->state = elan_init(0))) {
perror( "elan_init() failed" );
exit(1);
}
for (i = 0; i < 1; i++) {
ELAN_STATE *elan_state;
ELAN_CAPABILITY *cap;
ELAN_LOCATION loc;
elan_state = elan_base->state;
ctx = elan_state->ctx;
cap = elan_state->cap;
if (elan4_add_p2pvp (ctx, i, cap) < 0)
fprintf(stderr, "elan_init() failed" );
elan4_block_inputter (ctx, 1);
if (elan4_attach (ctx, cap))
fprintf(stderr, "elan4_attach() failed" );
}
nproc = elan_base->state->nvp;
self = elan_base->state->vp;
#endif
return 0;
}

88
ompi/mca/ptl/elan/tests/lat.c
Просмотреть файл

@ -1,88 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <stdio.h>
#include <sys/types.h>
#include <unistd.h>
#include <math.h>
#include <sys/time.h>
#include "mpi.h"
#include "test_util.h"
#define MYBUFSIZE (4*1024*1024)
char s_buf[MYBUFSIZE];
char r_buf[MYBUFSIZE];
int skip = 40;
int
main (int argc, char *argv[])
{
MPI_Status stat;
int myid, numprocs, i, j;
double startwtime = 0.0, endwtime;
int namelen;
int min, max, size;
int loop = 1;
struct timeval t_start, t_end;
if (argc < 4) {
fprintf (stderr, "Usage: %s min max loop_size\n", argv[0]);
return 0;
} else {
min = atoi (argv[1]);
max = atoi (argv[2]);
loop = atoi (argv[3]);
}
/* Get some environmental variables set for Open MPI, OOB */
/*env_init_for_elan();*/
MPI_Init (&argc, &argv);
MPI_Comm_size (MPI_COMM_WORLD, &numprocs);
MPI_Comm_rank (MPI_COMM_WORLD, &myid);
if (myid == 0)
fprintf(stdout, "%10s %10s \n", "# length", "latency(us)");
for (size = min; size <= max; size = (size == 0) ? 1 : 2*size) {
for (i = 0; i < loop + skip; i++) {
if (i == skip)
gettimeofday (&t_start, 0);
if (myid == 0) {
MPI_Send (s_buf, size, MPI_CHAR, 1, i, MPI_COMM_WORLD);
MPI_Recv (r_buf, size, MPI_CHAR, 1, i, MPI_COMM_WORLD, &stat);
} else {
MPI_Recv (r_buf, size, MPI_CHAR, 0, i, MPI_COMM_WORLD, &stat);
MPI_Send (s_buf, size, MPI_CHAR, 0, i, MPI_COMM_WORLD);
}
}
gettimeofday (&t_end, 0);
if (myid == 0) {
double latency;
latency = ((1.0e6 * t_end.tv_sec + t_end.tv_usec)
- (1.0e6 * t_start.tv_sec + t_start.tv_usec))
/ (2.0 * loop);
fprintf(stdout, " %8d %10.2f \n", size, latency);
}
fflush(stdout);
}
MPI_Finalize ();
return 0;
}

39
ompi/mca/ptl/elan/tests/mpi_test.c
Просмотреть файл

@ -1,39 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <stdio.h>
#include "mpi.h"
#include "test_util.h"
int main (int argc, char ** argv)
{
char hostname[32];
int proc, nproc;
/* Get some environmental variables set for Open MPI, OOB */
env_init_for_elan();
gethostname(hostname, 32);
MPI_Init(&argc, &argv);
MPI_Comm_rank(MPI_COMM_WORLD, &proc);
MPI_Comm_size(MPI_COMM_WORLD, &nproc);
fprintf(stdout, "[%s:%s:%d] done with init \n",
hostname, __FUNCTION__, __LINE__);
fflush(stdout);
MPI_Finalize();
return 0;
}

46
ompi/mca/ptl/elan/tests/qsnet_init.c
Просмотреть файл

@ -1,46 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include <stdio.h>
#include "qsnet/fence.h"
#include "ptl_elan.h"
#include "ptl_elan_priv.h"
#include "test_util.h"
extern mca_ptl_elan_module_1_0_0_t mca_ptl_elan_module;
int main (int argc, char ** argv)
{
bool allow_threads;
bool have_hidden_threads;
int input;
int num;
/* Get some environmental variables set for Open MPI, OOB */
env_init_for_elan();
/* Initialization test */
mca_ptl_elan_module_open();
mca_ptl_elan_module_init(&num, &allow_threads, &have_hidden_threads);
mca_ptl_elan_module_control(1, &input, 4);
mca_ptl_elan_module_close();
/* Tell alive */
fprintf(stdout, "I am still alive\n");
fflush(stdout);
return 0;
}

101
ompi/mca/ptl/elan/tests/qsnet_qdma.c
Просмотреть файл

@ -1,101 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "qsnet/fence.h"
#include "ptl_elan.h"
#include "ptl_elan_priv.h"
#include "test_util.h"
extern mca_ptl_elan_module_1_0_0_t mca_ptl_elan_module;
int ptl_elan_queue_send(); /* Initialize a Tx event */
int ptl_elan_queue_recv(); /* Wait for a recv event */
int test_qdma(mca_ptl_elan_module_1_0_0_t *emp, int reps);
int main (int argc, char ** argv)
{
bool allow_threads;
bool have_hidden_threads;
int input;
int num;
/* Get some environmental variables set for Open MPI, OOB */
env_init_for_elan();
/* Initialization */
mca_ptl_elan_module_open();
mca_ptl_elan_module_init(&num, &allow_threads, &have_hidden_threads);
mca_ptl_elan_module_control(1, &input, 4);
/* Testing QDMA */
test_qdma(&mca_ptl_elan_module, 1);
mca_ptl_elan_module_close();
return 0;
}
int test_qdma(mca_ptl_elan_module_1_0_0_t *emp, int reps)
{
uint64_t start0, end0;
double t;
int r;
double nsec;
ELAN4_CTX *ctx;
RAIL *rail;
mca_ptl_elan_t *ptl;
r = reps;
ptl = emp->elan_ptls[0];
ctx = emp->elan_ctrl->elan_ctx;
rail = (RAIL *) emp->elan_ctrl->elan_rail[0];
start0 = elan4_clock(ctx);
if (0 != ptl->elan_vp) {
r--;
ptl_elan_queue_recv();
}
while (--r >= 0) {
/* Trigger a send event */
ptl_elan_queue_send();
ptl_elan_queue_recv();
}
if (0 != ptl->elan_vp) {
/* Trigger one more send */
ptl_elan_queue_send();
}
end0 = elan4_clock(ctx);
nsec = ((end0 - start0) / reps);
t = ((double) nsec)/(2*1000.0);
return(0);
}
int ptl_elan_queue_send() {
return OMPI_SUCCESS;
}
/* This function needs no event related knowledge */
int ptl_elan_queue_recv ()
{
return OMPI_SUCCESS;
}

101
ompi/mca/ptl/elan/tests/qsnet_rdma.c
Просмотреть файл

@ -1,101 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include "ompi_config.h"
#include "qsnet/fence.h"
#include "ptl_elan.h"
#include "ptl_elan_priv.h"
#include "test_util.h"
extern mca_ptl_elan_module_1_0_0_t mca_ptl_elan_module;
int ptl_elan_rdma_put(); /* Initialize a Tx event */
int ptl_elan_rdma_get(); /* Wait for a recv event */
int test_rdma(mca_ptl_elan_module_1_0_0_t *emp, int reps);
int main (int argc, char ** argv)
{
bool allow_threads;
bool have_hidden_threads;
int input;
int num;
/* Get some environmental variables set for Open MPI, OOB */
env_init_for_elan();
/* Initialization */
mca_ptl_elan_module_open();
mca_ptl_elan_module_init(&num, &allow_threads, &have_hidden_threads);
mca_ptl_elan_module_control(1, &input, 4);
/* Testing QDMA */
test_rdma(&mca_ptl_elan_module, 1);
mca_ptl_elan_module_close();
return 0;
}
int test_rdma(mca_ptl_elan_module_1_0_0_t *emp, int reps)
{
uint64_t start0, end0;
double t;
int r;
double nsec;
ELAN4_CTX *ctx;
RAIL *rail;
mca_ptl_elan_t *ptl;
r = reps;
ptl = emp->elan_ptls[0];
ctx = emp->elan_ctrl->elan_ctx;
rail = (RAIL *) emp->elan_ctrl->elan_rail[0];
start0 = elan4_clock(ctx);
if (0 != ptl->elan_vp) {
r--;
ptl_elan_rdma_get();
}
while (--r >= 0) {
/* Trigger a send event */
ptl_elan_rdma_put();
ptl_elan_rdma_get();
}
if (0 != ptl->elan_vp) {
/* Trigger one more send */
ptl_elan_rdma_put();
}
end0 = elan4_clock(ctx);
nsec = ((end0 - start0) / reps);
t = ((double) nsec)/(2*1000.0);
return(0);
}
int ptl_elan_rdma_put() {
return OMPI_SUCCESS;
}
/* This function needs no event related knowledge */
int ptl_elan_rdma_get ()
{
return OMPI_SUCCESS;
}

38
ompi/mca/ptl/elan/tests/test_util.h
Просмотреть файл

@ -1,38 +0,0 @@
/*
* Copyright (c) 2004-2005 The Trustees of Indiana University.
* All rights reserved.
* Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
* All rights reserved.
* Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
* University of Stuttgart. All rights reserved.
* Copyright (c) 2004-2005 The Regents of the University of California.
* All rights reserved.
* $COPYRIGHT$
*
* Additional copyrights may follow
*
* $HEADER$
*/
#include <stdlib.h>
#include <string.h>
static void env_init_for_elan()
{
char *rms_rank;
setenv("OMPI_MCA_oob_cofs_dir", "/home/1/yuw/tmp", 1);
setenv("OMPI_MCA_pcm_cofs_cellid", "1", 1);
setenv("OMPI_MCA_pcm_cofs_jobid", "1", 1);
setenv("OMPI_MCA_pcm_cofs_num_procs", "2", 1);
setenv("OMPI_MCA_ptl_base_exclude", "tcp", 1);
setenv("OMPI_MCA_oob_base_include", "cofs", 1);
/*setenv("OMPI_MCA_oob_base_exclude", "tcp", 1);*/
if (NULL != (rms_rank = getenv("RMS_RANK"))) {
/* RMS_JOBID:RMS_NNODES:RMS_NPROCS:RMS_NODEID:RMS_RESOURCEID */
setenv("OMPI_MCA_pcm_cofs_procid", rms_rank, 1);
} else {
fprintf(stderr, "Hi, please test elan4 from RMS for now\n");
}
}