ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Just to save some codes in elan

Просмотр исходного кода 
This commit was SVN r1252.
Этот коммит содержится в:
Weikuan Yu 2004-06-14 22:39:14 +00:00
родитель 8f7b128941
Коммит 0710f299e1
16 изменённых файлов: 1793 добавлений и 0 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

0
src/mca/ptl/elan/.ompi_ignore Обычный файл
Просмотреть файл

36
src/mca/ptl/elan/Makefile.am Обычный файл
Просмотреть файл

@ -0,0 +1,36 @@
#
# $HEADER$
#
# Use the top-level OMPI Makefile.options
include $(top_ompi_srcdir)/config/Makefile.options
SUBDIRS = src
EXTRA_DIST = VERSION
# According to the MCA spec, we have to make the output library here
# in the top-level directory, and it has to be named
# libompi_ssi_coll_ompi_basic.la
if OMPI_BUILD_ptl_elan_LOADABLE_MODULE
module_noinst =
module_install = mca_ptl_elan.la
else
module_noinst = libmca_ptl_elan.la
module_install =
endif
mcamoduledir = $(libdir)/ompi
mcamodule_LTLIBRARIES = $(module_install)
mca_ptl_elan_la_SOURCES =
mca_ptl_elan_la_LIBADD = \
src/libmca_ptl_elan.la \
$(LIBOMPI_LA)
mca_ptl_elan_la_LDFLAGS = -module -avoid-version
noinst_LTLIBRARIES = $(module_noinst)
libmca_ptl_elan_la_SOURCES =
libmca_ptl_elan_la_LIBADD = src/libmca_ptl_elan.la
libmca_ptl_elan_la_LDFLAGS = -module -avoid-version

6
src/mca/ptl/elan/VERSION Обычный файл
Просмотреть файл

@ -0,0 +1,6 @@
major=1
minor=0
release=0
alpha=0
beta=0
svn=1

10
src/mca/ptl/elan/configure.params Обычный файл
Просмотреть файл

@ -0,0 +1,10 @@
# -*- shell-script -*-
#
# $HEADER$
#
# Specific to this module
PARAM_INIT_FILE=src/ptl_elan.c
PARAM_CONFIG_HEADER_FILE="src/elan_config.h"
PARAM_CONFIG_FILES="Makefile src/Makefile"

28
src/mca/ptl/elan/src/Makefile.am Обычный файл
Просмотреть файл

@ -0,0 +1,28 @@
# -*- makefile -*-
#
# $HEADER$
#
include $(top_ompi_srcdir)/config/Makefile.options
AM_CPPFLAGS = \
-I$(top_ompi_builddir)/src/include \
-I$(top_ompi_builddir)/src/ompi/event \
-I$(top_ompi_srcdir)/src \
-I$(top_ompi_srcdir)/src/include
noinst_LTLIBRARIES = libmca_ptl_elan.la
libmca_ptl_elan_la_SOURCES = \
ptl_elan.h \
ptl_elan_frag.h \
ptl_elan_req.h \
ptl_elan_proc.h \
ptl_elan_addr.h \
ptl_elan_peer.h \
ptl_elan_proc.c \
ptl_elan_module.c \
ptl_elan_priv.c \
ptl_elan_init.c \
ptl_elan.c

120
src/mca/ptl/elan/src/ptl_elan.c Обычный файл
Просмотреть файл

@ -0,0 +1,120 @@
/*
* $HEADER$
*/
#include <string.h>
#include "util/output.h"
#include "util/if.h"
#include "mca/pml/pml.h"
#include "mca/ptl/ptl.h"
#include "mca/ptl/base/ptl_base_header.h"
#include "mca/ptl/base/ptl_base_sendreq.h"
#include "mca/ptl/base/ptl_base_sendfrag.h"
#include "mca/ptl/base/ptl_base_recvreq.h"
#include "mca/ptl/base/ptl_base_recvfrag.h"
#include "mca/base/mca_base_module_exchange.h"
#include "ptl_elan.h"
#include "ptl_elan_peer.h"
#include "ptl_elan_proc.h"
#include "ptl_elan_req.h"
#include "ptl_elan_frag.h"
mca_ptl_elan_t mca_ptl_elan = {
{
&mca_ptl_elan_module.super,
0, /* ptl_exclusivity */
0, /* ptl_latency */
0, /* ptl_bandwidth */
0, /* ptl_frag_first_size */
0, /* ptl_frag_min_size */
0, /* ptl_frag_max_size */
MCA_PTL_PUT, /* ptl flags */
/* collection of interfaces */
mca_ptl_elan_add_proc,
mca_ptl_elan_del_proc,
mca_ptl_elan_finalize,
mca_ptl_elan_isend,
mca_ptl_elan_irecv,
mca_ptl_elan_put,
mca_ptl_elan_get,
mca_ptl_elan_matched,
mca_ptl_elan_req_alloc,
mca_ptl_elan_req_return
}
};
int mca_ptl_elan_add_proc (struct mca_ptl_t *ptl,
struct ompi_proc_t *ompi_proc,
struct mca_ptl_base_peer_t **peer_ret)
{
return OMPI_SUCCESS;
}
int mca_ptl_elan_del_proc (struct mca_ptl_t *ptl, struct ompi_proc_t *proc,
struct mca_ptl_base_peer_t *ptl_peer)
{
return OMPI_SUCCESS;
}
int mca_ptl_elan_finalize (struct mca_ptl_t *ptl)
{
return OMPI_SUCCESS;
}
int mca_ptl_elan_req_alloc (struct mca_ptl_t *ptl,
struct mca_ptl_base_send_request_t **request)
{
int rc;
return rc;
}
void mca_ptl_elan_req_return (struct mca_ptl_t *ptl,
struct mca_ptl_base_send_request_t *request)
{
return;
}
void mca_ptl_elan_recv_frag_return (struct mca_ptl_t *ptl,
struct mca_ptl_elan_recv_frag_t *frag)
{
return;
}
void mca_ptl_elan_send_frag_return (struct mca_ptl_t *ptl,
struct mca_ptl_elan_send_frag_t *frag)
{
return;
}
/*
* Initiate a send. If this is the first fragment, use the fragment
* descriptor allocated with the send requests, otherwise obtain
* one from the free list. Initialize the fragment and foward
* on to the peer.
*/
int mca_ptl_elan_isend (struct mca_ptl_t *ptl,
struct mca_ptl_base_peer_t *ptl_peer,
struct mca_ptl_base_send_request_t *sendreq,
size_t offset, size_t * size, int flags)
{
return OMPI_SUCCESS;
}
/*
* A posted receive has been matched - if required send an
* ack back to the peer and process the fragment.
*/
void mca_ptl_elan_matched (mca_ptl_t * ptl,
mca_ptl_base_recv_frag_t * frag)
{
return;
}

371
src/mca/ptl/elan/src/ptl_elan.h Обычный файл
Просмотреть файл

@ -0,0 +1,371 @@
/*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PTL_ELAN_H
#define MCA_PTL_ELAN_H
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include "mem/free_list.h"
#include "event/event.h"
#include "mca/pml/pml.h"
#include "mca/ptl/ptl.h"
/*#include "elan/sys/init_sys.h"*/
/*#include "elan/elan.h"*/
#define MCA_PTL_ELAN_STATISTICS 0
/**
* ELAN PTL module.
*/
struct mca_ptl_elan_state_t {
/* User configurable parameters */
char *elan_version; /**< Version of the elan library */
uint64_t elan_debug; /**< elan debug tracing output */
uint64_t elan_traced; /**< elan TRACE output */
uint64_t elan_flags;
FILE elan_debugfile;
int elan_signalnum;
#ifdef ENABLE_ELAN_MEMORY_ALLOCATOR
size_t main_size; /**< size of Main memory allocator heap */
size_t elan_size; /**< size of Elan memory allocator heap */
void *main_base; /**< Main memory allocator heap base */
void *elan_base; /**< Elan memory allocator heap base */
#endif
/* other state parameters */
int elan_attached; /**< 0 until elan_attach() called */
unsigned int elan_vp; /**< elan vpid, not ompi vpid */
unsigned int elan_nvp; /**< total # of elan vpid */
int *elan_localvps; /**< mapping of localId to elan vp */
int elan_localid; /**< # of local elan vpids */
int elan_numlocals; /**< # of local elan vpids */
int elan_maxlocals; /**< maximum # of local elan vpids */
int elan_nrails; /**< # of rails elan vpids */
int elan_rmsid; /**< rms resource id */
long elan_pagesize;
pid_t elan_pid;
/* TODO:
* Even though the elan threads are not utilized for now.
* We provide memory/state control structures for later extensions.
* A simple type casting of ELAN_ESTATE can bring
* the complete structure of the ELAN_EPRIVSATE.
*/
void *elan_cap; /**< job capability */
void *elan_estate; /**< Elan state of the 0th rail */
#ifdef ELAN_COMP
ELAN_CTX *elan_ctx; /**< Elan ctx of the 0th rail */
ELAN_RAIL **elan_rail; /**< Rail control struct for all rails */
RAIL **all_rails; /**< all rails */
ELAN_ESTATE **all_estates; /**< elan (priv)states of all rails */
#endif
ompi_mutex_t state_lock; /**< lock for elan state */
};
typedef struct mca_ptl_elan_state_t mca_ptl_elan_state_t;
struct mca_ptl_elan_module_1_0_0_t {
mca_ptl_base_module_1_0_0_t super; /**< base PTL module */
/* These parameters does not provided good freedom,
* It does not hurt to skip them */
#if 1
int elan_free_list_num; /**< initial size of free lists */
int elan_free_list_max; /**< maximum size of free lists */
int elan_free_list_inc; /**< # to alloc when growing lists */
#endif
/*
* We create our own simplified structure for managing elan state
* although libelan already provides one. We do not need
* all that tport, group structures.
*/
struct mca_ptl_elan_state_t *elan_state; /**< elan state */
struct mca_ptl_elan_t **elan_ptls; /**< array of available PTLs */
size_t elan_num_ptls; /**< number of ptls activated */
ompi_list_t elan_reqs; /**< all elan requests */
ompi_list_t elan_prog_events; /**< events in progress */
ompi_list_t elan_comp_events; /**< events completed, but to reclaim */
ompi_free_list_t elan_events_free; /**< free events */
ompi_free_list_t elan_reqs_free; /**< all elan requests */
ompi_event_t elan_send_event; /**< event structure for sends */
ompi_event_t elan_recv_event; /**< event structure for recvs */
ompi_list_t elan_procs; /**< elan proc's */
ompi_mutex_t elan_lock; /**< lock for module state */
};
typedef struct mca_ptl_elan_module_1_0_0_t mca_ptl_elan_module_1_0_0_t;
struct mca_ptl_elan_send_frag_t;
struct mca_ptl_elan_recv_frag_t;
extern mca_ptl_elan_module_1_0_0_t mca_ptl_elan_module;
/**
* Register ELAN module parameters with the MCA framework
*/
extern int mca_ptl_elan_module_open (void);
/**
* Any final cleanup before being unloaded.
*/
extern int mca_ptl_elan_module_close (void);
/**
* ELAN module initialization.
*
* @param num_ptls (OUT)
* Number of PTLs returned in PTL array.
* @param allow_multi_user_threads (OUT)
* Flag indicating wether PTL supports user threads (TRUE)
* @param have_hidden_threads (OUT)
* Flag indicating whether PTL uses threads (TRUE)
*
*/
extern mca_ptl_t **mca_ptl_elan_module_init (int *num_ptls,
bool * multi_user_threads,
bool * have_hidden_threads);
/**
* ELAN module control.
*/
extern int mca_ptl_elan_module_control (int param,
void *value,
size_t size);
/**
* ELAN module progress.
*/
extern int mca_ptl_elan_module_progress (mca_ptl_tstamp_t tstamp);
/**
* ELAN PTL Interface
*/
struct mca_ptl_elan_t {
/**< The elan progress related interface */
mca_ptl_t super; /**< base PTL interface */
/**< The following are elan-related control structures */
#ifdef ELAN_COMP
ELAN_RAIL *elan_rail; /**< Pointer to this Rail */
ELAN_CTX *elan_ctx; /**< Elan ctx of this rail */
#endif
int ptl_ni_local; /**< PTL NI local rank */
int ptl_ni_total; /**< PTL NI total */
int elan_sten_size; /**< sten packet len */
int elan_rdma_size; /**< qdma packet length */
int elan_qdma_size; /**< qdma packet length */
int sten_total; /**< total sten descriptors */
int rdma_total; /**< total rdma descriptors */
int qdma_total; /**< total rdma descriptors */
int sten_num; /**< num of outstanding sten packets */
int rdma_num; /**< num of outstanding rdma packets */
int qdma_num; /**< num of outstanding rdma packets */
int max_num_dmas; /**< total rdma descriptors */
ompi_list_t elan_stens; /**< used elan sten descriptors*/
ompi_list_t elan_dmas; /**< used elan dma descriptors*/
ompi_list_t elan_rdmas; /**< used elan rdma descriptors */
ompi_list_t elan_frags; /**< used elan fragments */
ompi_free_list_t elan_dmas_free; /**< free elan dma descriptors*/
ompi_free_list_t elan_stens_free; /**< free elan sten descriptors*/
ompi_free_list_t elan_rdmas_free; /**< free elan rdma descriptors */
ompi_free_list_t elan_frags_free; /**< free elan rdma fragments */
#if MCA_PTL_ELAN_STATISTICS /* some statistics */
size_t ptl_bytes_sent;
size_t ptl_bytes_recv;
#endif
};
typedef struct mca_ptl_elan_t mca_ptl_elan_t;
extern mca_ptl_elan_t mca_ptl_elan;
/**
* Cleanup any resources held by the PTL.
*
* @param ptl PTL instance.
* @return OMPI_SUCCESS or error status on failure.
*/
extern int mca_ptl_elan_finalize (struct mca_ptl_t *ptl);
/**
* PML->PTL notification of change in the process list.
*
* @param ptl (IN)
* @param proc (IN)
* @param peer (OUT)
* @return OMPI_SUCCESS or error status on failure.
*/
extern int mca_ptl_elan_add_proc (struct mca_ptl_t *ptl,
struct ompi_proc_t *proc,
struct mca_ptl_base_peer_t **peer);
/**
* PML->PTL notification of change in the process list.
*
* @param ptl (IN) PTL instance
* @param proc (IN) Peer process
* @param peer (IN) Peer addressing information.
* @return Status indicating if cleanup was successful
*/
extern int mca_ptl_elan_del_proc (struct mca_ptl_t *ptl,
struct ompi_proc_t *procs,
struct mca_ptl_base_peer_t *addr);
/**
* PML->PTL Allocate a send request from the PTL modules free list.
*
* @param ptl (IN) PTL instance
* @param request (OUT) Pointer to allocated request.
* @return Status indicating if allocation was successful.
*/
extern int mca_ptl_elan_req_alloc (struct mca_ptl_t *ptl,
struct mca_ptl_base_send_request_t **);
/**
* PML->PTL Return a send request to the PTL modules free list.
*
* @param ptl (IN) PTL instance
* @param request (IN) Pointer to allocated request.
*/
extern void mca_ptl_elan_req_return (struct mca_ptl_t *ptl,
struct mca_ptl_base_send_request_t *);
/**
* PML->PTL Notification that a receive fragment has been matched.
*
* @param ptl (IN) PTL instance
* @param recv_frag (IN) Receive fragment
*/
extern void mca_ptl_elan_matched (struct mca_ptl_t *ptl,
struct mca_ptl_base_recv_frag_t *frag);
/**
* PML->PTL Initiate an isend of the specified size.
*
* @param ptl (IN) PTL instance
* @param ptl_base_peer (IN) PTL peer addressing
* @param send_request (IN/OUT) Send request (allocated by PML via
* mca_ptl_base_request_alloc_fn_t)
* @param size (IN)
* Number of bytes PML is requesting PTL to deliver
* @param flags (IN)
* Flags that should be passed to the peer via the message header.
* @param request (OUT)
* OMPI_SUCCESS if the PTL was able to queue one or more fragments
*/
extern int mca_ptl_elan_isend (struct mca_ptl_t *ptl,
struct mca_ptl_base_peer_t *ptl_peer,
struct mca_ptl_base_send_request_t *,
size_t offset,
size_t * size,
int flags);
/**
* PML->PTL Initiate an irecv of the specified size.
*
* @param ptl (IN) PTL instance
* @param ptl_base_peer (IN) PTL peer addressing
* @param send_request (IN/OUT) Send request (allocated by PML via
* mca_ptl_base_request_alloc_fn_t)
* @param size (IN)
* Number of bytes PML is requesting PTL to deliver
* @param flags (IN)
* Flags that should be passed to the peer via the message header.
* @param request (OUT)
* OMPI_SUCCESS if the PTL was able to queue one or more fragments
*/
extern int mca_ptl_elan_irecv (struct mca_ptl_t *ptl,
struct mca_ptl_base_peer_t *ptl_peer,
struct mca_ptl_base_send_request_t *,
size_t offset,
size_t * size,
int flags);
/**
* PML->PTL Initiate a put of the specified size.
*
* @param ptl (IN) PTL instance
* @param ptl_base_peer (IN) PTL peer addressing
* @param send_request (IN/OUT) Send request (allocated by PML via
* mca_ptl_base_request_alloc_fn_t)
* @param size (IN)
* Number of bytes PML is requesting PTL to deliver
* @param flags (IN)
* Flags that should be passed to the peer via the message header.
* @param request (OUT)
* OMPI_SUCCESS if the PTL was able to queue one or more fragments
*/
extern int mca_ptl_elan_put (struct mca_ptl_t *ptl,
struct mca_ptl_base_peer_t *ptl_peer,
struct mca_ptl_base_send_request_t *,
size_t offset,
size_t * size,
int flags);
/**
* PML->PTL Initiate a get of the specified size.
*
* @param ptl (IN) PTL instance
* @param ptl_base_peer (IN) PTL peer addressing
* @param send_request (IN/OUT) Send request (allocated by PML via
* mca_ptl_base_request_alloc_fn_t)
* @param size (IN)
* Number of bytes PML is requesting PTL to deliver
* @param flags (IN)
* Flags that should be passed to the peer via the message header.
* @param request (OUT)
* OMPI_SUCCESS if the PTL was able to queue one or more fragments
*/
extern int mca_ptl_elan_get (struct mca_ptl_t *ptl,
struct mca_ptl_base_peer_t *ptl_peer,
struct mca_ptl_base_send_request_t *,
size_t offset,
size_t * size,
int flags);
/**
* Return a recv fragment to the modules free list.
*
* @param ptl (IN) PTL instance
* @param frag (IN) ELAN receive fragment
*/
extern void mca_ptl_elan_recv_frag_return (struct mca_ptl_t *ptl,
struct mca_ptl_elan_recv_frag_t
*frag);
/**
* Return a send fragment to the modules free list.
*
* @param ptl (IN) PTL instance
* @param frag (IN) ELAN send fragment
*/
extern void mca_ptl_elan_send_frag_return (struct mca_ptl_t *ptl,
struct mca_ptl_elan_send_frag_t
*frag);
#endif

146
src/mca/ptl/elan/src/ptl_elan_frag.h Обычный файл
Просмотреть файл

@ -0,0 +1,146 @@
/*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PTL_ELAN_FRAG_H
#define MCA_PTL_ELAN_FRAG_H
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include "ompi_config.h"
#include "mca/ptl/base/ptl_base_sendreq.h"
#include "mca/ptl/base/ptl_base_recvreq.h"
#include "mca/ptl/base/ptl_base_sendfrag.h"
#include "mca/ptl/base/ptl_base_recvfrag.h"
#include "ptl_elan.h"
extern ompi_class_t mca_ptl_elan_send_frag_t_class;
struct mca_ptl_base_peer_t;
/**
* ELAN send fragment derived type.
*/
struct mca_ptl_elan_send_frag_t {
mca_ptl_base_send_frag_t super; /**< base send fragment descriptor */
size_t frag_vec_cnt;
volatile int frag_progressed;
};
typedef struct mca_ptl_elan_send_frag_t mca_ptl_elan_send_frag_t;
#define MCA_PTL_ELAN_SEND_FRAG_ALLOC(item, rc) \
OMPI_FREE_LIST_GET(&mca_ptl_elan_module.elan_send_frags, item, rc);
#define MCA_PTL_ELAN_RECV_FRAG_ALLOC(frag, rc) \
{ \
ompi_list_item_t* item; \
OMPI_FREE_LIST_GET(&mca_ptl_elan_module.elan_recv_frags, item, rc); \
frag = (mca_ptl_elan_recv_frag_t*)item; \
}
bool
mca_ptl_elan_send_frag_handler (mca_ptl_elan_send_frag_t *, int sd);
int
mca_ptl_elan_send_frag_init (mca_ptl_elan_send_frag_t *,
struct mca_ptl_base_peer_t *,
struct mca_ptl_base_send_request_t *,
size_t offset, size_t * size, int flags);
extern ompi_class_t mca_ptl_elan_recv_frag_t_class;
/**
* ELAN received fragment derived type.
*/
struct mca_ptl_elan_recv_frag_t {
mca_ptl_base_recv_frag_t super; /**< base receive fragment descriptor */
size_t frag_hdr_cnt; /**< number of header bytes received */
size_t frag_msg_cnt; /**< number of message bytes received */
bool frag_ack_pending; /**< an ack pending for this fragment */
volatile int frag_progressed; /**< flag to atomically progress */
};
typedef struct mca_ptl_elan_recv_frag_t mca_ptl_elan_recv_frag_t;
bool
mca_ptl_elan_recv_frag_handler (mca_ptl_elan_recv_frag_t *, int sd);
void
mca_ptl_elan_recv_frag_init (mca_ptl_elan_recv_frag_t * frag,
struct mca_ptl_base_peer_t *peer);
bool
mca_ptl_elan_recv_frag_send_ack (mca_ptl_elan_recv_frag_t * frag);
/*
* For fragments that require an acknowledgment, this routine will be called
* twice, once when the send completes, and again when the acknowledgment is
* returned. Only the last caller should update the request status, so we
* add a lock w/ the frag_progressed flag.
*/
static inline void
mca_ptl_elan_send_frag_progress (mca_ptl_elan_send_frag_t * frag)
{
mca_ptl_base_send_request_t *request = frag->super.frag_request;
/* if this is an ack - simply return to pool */
if (request == NULL) {
mca_ptl_elan_send_frag_return (frag->super.super.frag_owner, frag);
} else if (frag->frag_vec_cnt == 0 &&
((frag->super.super.frag_header.hdr_common.hdr_flags
& MCA_PTL_FLAGS_ACK_MATCHED) == 0 ||
mca_ptl_base_send_request_matched (request))) {
/* otherwise, if the message has been sent, and an ack has
* already been received, go ahead and update the request status
*/
/* make sure this only happens once in threaded case */
if (ompi_using_threads ()
&& fetchNset (&frag->frag_progressed, 1) == 1) {
return;
} else {
frag->frag_progressed = 1;
}
/* update request status */
frag->super.super.frag_owner->ptl_send_progress (request,
&frag->super);
/* the first fragment is allocated with the request,
* all others need to be returned to free list
*/
if (frag->super.super.frag_header.hdr_frag.hdr_frag_offset != 0)
mca_ptl_elan_send_frag_return (frag->super.super.frag_owner,
frag);
}
}
static inline void
mca_ptl_elan_send_frag_init_ack (mca_ptl_elan_send_frag_t * ack,
struct mca_ptl_t *ptl,
struct mca_ptl_base_peer_t *ptl_peer,
mca_ptl_elan_recv_frag_t * frag)
{
return;
}
static inline void
mca_ptl_elan_recv_frag_matched (mca_ptl_elan_recv_frag_t * frag)
{
mca_ptl_base_recv_request_t *request = frag->super.frag_request;
return;
}
static inline void
mca_ptl_elan_recv_frag_progress (mca_ptl_elan_recv_frag_t * frag)
{
return;
}
#endif

509
src/mca/ptl/elan/src/ptl_elan_init.c Обычный файл
Просмотреть файл

@ -0,0 +1,509 @@
#include "ptl_elan.h"
static int ompi_init_elan_qdma (mca_ptl_elan_module_1_0_0_t * mp);
static int ompi_init_elan_sten (mca_ptl_elan_module_1_0_0_t * mp);
static int ompi_init_elan_rdma (mca_ptl_elan_module_1_0_0_t * mp);
static int ompi_init_elan_stat (mca_ptl_elan_module_1_0_0_t * mp);
static int ompi_elan_attach_network (mca_ptl_elan_state_t * state);
int
ompi_mca_ptl_elan_init (mca_ptl_elan_module_1_0_0_t * mp)
{
#if ELAN_COMP
int i;
int *rails;
int num_rails;
int max_backoff;
int max_fastbackoff;
int alloc_mainsize;
int alloc_mainbase;
int alloc_elansize;
int alloc_elanbase;
mca_ptl_elan_state_t *state;
max_backoff = elan_rup2 (ELAN_MAXBACKOFF) - 1;
max_fastbackoff = elan_rup2 (ELAN_MAXFASTBACKOFF) - 1;
/* Allocate the elan (priv)state structure off the heap.
* it is not available to the Elan thread. Another structure
* for the Elan thread is to be used that instead
*/
mp->elan_state = (mca_ptl_elan_state_t *)
malloc (sizeof (mca_ptl_elan_state_t));
if (NULL == mp->elan_state)
return (NULL);
memset ((void *) mp->elan_state, 0, sizeof (mca_ptl_elan_state_t));
state = mp->elan_state;
/* Initialise enough of state so we can call elan_exception() */
state->elan_version = ELAN_VERSION;
state->elan_ctx = NULL;
state->elan_rail = NULL;
state->elan_vp = ELAN_INVALID_PROCESS;
state->elan_nvp = 0;
state->elan_debug = 0;
state->elan_traced = 0;
state->elan_pagesize = sysconf (_SC_PAGESIZE);
state->elan_pid = getpid ();
/* Default allocator parameters */
state->main_size = ELAN_ALLOC_SIZE;
state->elan_size = ELAN_ALLOCELAN_SIZE;
state->elan_flags |=
(EXCEPTIONCORE | EXCEPTIONTRACE | EXCEPTIONDBGDUMP);
state->elan_debugfile = NULL;
state->elan_signalnum = SIGABRT;
#ifdef ELAN_VERSION
if (!elan_checkVersion (ELAN_VERSION)) {
return OMPI_ERROR;
}
#endif
/* FIXME: when to initialize it */
OBJ_CONSTRUCT (&state->state_lock, ompi_mutex_t);
/* Allocate elan capability from the heap */
state->elan_cap =
(ELAN_CAPABILITY *) malloc (sizeof (ELAN_CAPABILITY));
if (NULL == state->elan_cap) {
return OMPI_ERROR;
} else {
memset (state->elan_cap, 0, sizeof (ELAN_CAPABILITY));
}
/* Process the capability info supplied by RMS */
if (getenv ("ELAN_AUTO") || getenv ("RMS_NPROCS")) {
/* RMS generated capabilities */
if (rms_getcap (0, state->elan_cap)) {
return OMPI_ERROR;
}
}
if ((num_rails = state->elan_nrails =
elan_nrails (state->elan_cap)) <= 0) {
return OMPI_ERROR;
}
/* MULTI-RAIL:
* Allocate storage space for each Elan rail info (ptrs)
* Allocate storage space for each Elan SDRAM state (ptrs) */
state->all_rails = (RAIL **) malloc (sizeof (RAIL *) * num_rails);
state->all_estates = (ELAN_ESTATE **)
malloc (sizeof (ELAN_ESTATE *) * num_rails);
if (state->all_estates == NULL || state->all_rails == NULL) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
/* MULTI-RAIL:
* initialise each elan rail, using the physical rail info gleaned from
* the capability */
if (NULL == (rails = (int *) malloc (sizeof (int) * num_rails))) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
(void) elan_rails (state->elan_cap, rails);
if (NULL == (state->elan_rail = (ELAN_RAIL **)
malloc (sizeof (ELAN_RAIL **) * (num_rails + 1)))) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
state->elan_rail[num_rails] = NULL;
alloc_mainsize = ELAN_ALIGNUP (state->main_size, state->elan_pagesize);
alloc_mainbase = (ADDR_ELAN) ((uintptr_t) state->main_base);
alloc_elansize = ELAN_ALIGNUP (state->elan_size, state->elan_pagesize);
alloc_elanbase = (ADDR_ELAN) ((uintptr_t) state->elan_base);
for (i = 0; i < num_rails; i++) {
RAIL *rail;
ELAN_ESTATE *estate;
ELAN_EPRIVSTATE *priv_estate;
/* Allocate the Main memory control structure for this rail */
if (NULL == (rail = state->all_rails[i] =
(RAIL *) malloc (sizeof (RAIL)))) {
return OMPI_ERROR;
}
memset (rail, 0, sizeof (RAIL));
/* TODO: Debug the state of rails */
if (NULL == (rail->r_ctx = elan4_init (rails[i]))) {
return OMPI_ERROR;
}
if (NULL == (rail->r_sdram = elan4_open_sdram (rails[i],
0,
alloc_elansize))) {
return OMPI_ERROR;
}
if (NULL == (rail->r_alloc =
elan4_createAllocator (state->alloc_mainsize,
rail->r_sdram, 0,
state->alloc_elansize))) {
return OMPI_ERROR;
}
if (elan4_set_standard_mappings (rail->r_ctx) < 0
|| elan4_set_required_mappings (rail->r_ctx) < 0) {
return OMPI_ERROR;
}
if (elan4_register_trap_handler (rail->r_ctx,
UTS_UNIMP_INSTR, UTS_TPROC,
elan_unimp_handler, NULL) < 0) {
return OMPI_ERROR;
}
/* Now allocate the SDRAM Elan control structure for this rail */
if (NULL == (estate = mp->all_estates[i] =
ALLOC_ELAN (rail, ELAN_ALIGN,
sizeof (ELAN_EPRIVSTATE)))) {
return OMPI_ERROR;
}
priv_estate = (ELAN_EPRIVSTATE *) estate;
memset (priv_estate, 0, sizeof (ELAN_EPRIVSTATE));
/* Allocate a command port for non sten functions etc */
if (NULL == (rail->r_cmdq = elan4_alloc_cmdq (rail->r_ctx,
rail->r_alloc, CQ_Size8K,
CQ_ModifyEnableBit |
CQ_WriteEnableBit |
CQ_WaitEventEnableBit |
CQ_SetEventEnableBit |
CQ_ThreadStartEnableBit,
NULL))) {
return OMPI_ERROR;
}
/* Allocate a command port for thread rescheduling etc */
if (NULL == (rail->r_ecmdq = elan4_alloc_cmdq (rail->r_ctx,
rail->r_alloc, CQ_Size8K,
CQ_EnableAllBits,
NULL))) {
return OMPI_ERROR;
}
priv_estate->cport =
MAIN2ELAN (rail->r_ctx, rail->r_ecmdq->cmdq_mapping);
/* Make the rail pointers 'external' */
state->elan_rail[i] = (ELAN_RAIL *) rail;
/* TODO: Find out how the following code work. */
{
ELAN_SLEEP *es;
/* Allocate and then free an ELAN_SLEEP DESC to
* keep heap in sync */
es = _elan_allocSleepDesc (state, rail);
_elan_freeSleepDesc (state, rail, es);
}
estate->alloc = rail->r_alloc;
estate->vp = state->vp; /* still ELAN_INVALID_PROCESS */
estate->debugFlags = state->debugFlags;
priv_estate->debugFd = 1; /* stdout */
priv_estate->maxBackOff = max_backoff;
priv_estate->maxFastBackoff = max_fastbackoff;
priv_estate->pageSize = state->elan_pagesize;
rail->r_estate = estate;
rail->r_railNo = rails[i];
{
RAILTABLE *rt;
if (NULL == (rt = (RAILTABLE *) malloc (sizeof (RAILTABLE)))) {
return OMPI_ERROR;
}
memset (rt, 0, sizeof (RAILTABLE));
rt->rt_nrails = 1;
rt->rt_rail = 0;
rt->rt_railReal = rail;
rt->rt_allRails = &(state->all_rails[rail]);
/* Debug the state, see _elan_railtable_createSingle */
rail->r_railTable = rt;
}
} /* for each rail */
free (rails);
state->elan_ctx = state->rail[0]->rail_ctx;
state->elan_estate = (void *) state->all_estates[0];
/*_elan_eventInit(state);*/
/*atexit(_elan_atExitCallBack); */
ompi_elan_attach_network (state);
/* Set the rms_resourceId */
if (rms_getprgid (getpid (), &state->elan_rmsid) < 0) {
state->elan_rmsid = -1;
}
/* Now open ourselves to the network */
for (i = 0; state->elan_rail[i]; i++) {
elan4_block_inputter (state->elan_rail[i]->rail_ctx, 0);
}
#endif
return (OMPI_SUCCESS);
}
int
ompi_mca_ptl_elan_setup (mca_ptl_elan_module_1_0_0_t * mp)
{
/* TODO:
* a) init transport structure for all rails
* including STEN, QDMA and RDMA
* b) For each possible transport mechanism, allocate:
* send/recv descriptors;
* system buffer;
* event structure for transport control
* c) initialize STAT (including SYNC) structures.
*/
if (OMPI_SUCCESS != ompi_init_elan_qdma (mp)) {
return OMPI_ERROR;
}
#if MCA_PTL_ELAN_STAGE_TWO
if (OMPI_SUCCESS != ompi_init_elan_rdma (mp)) {
return OMPI_ERROR;
}
if (OMPI_SUCCESS != ompi_init_elan_sten (mp)) {
return OMPI_ERROR;
}
if (OMPI_SUCCESS != ompi_init_elan_stat (mp)) {
return OMPI_ERROR;
}
#endif
return (OMPI_SUCCESS);
}
int
ompi_mca_ptl_elan_fin (mca_ptl_elan_module_1_0_0_t * mp)
{
return (OMPI_SUCCESS);
}
/* Attach to the network:
* a) First add myself into the capability
* b) For each rail,
* Block the inputter for any incoming traffic
* attach to the device
* Find out the location
* Fill out the vp in main and elan structures.
* create a cookiePool for threading control
* c) Allocate a cookiePool for export Oth Rail
* d) Find out total vpids, localVpids, localId and number of locals
* e) Allocate a bcastVp (not used for now) and a railTable
*/
static int
ompi_elan_attach_network (mca_ptl_elan_state_t * state)
{
#if ELAN_COMP
int i, vp, *vps, num_rails;
ELAN_LOCATION loc;
ELAN_CAPABILITY *cap = state->elan_cap;
num_rails = state->elan_nrails;
if (state->attached) {
/* already successfully attached */
return;
} else {
state->attached = 1;
}
for (i = 0; i < num_rails; i++) {
RAIL *rail = state->all_rails[i];
ELAN_LOCATION loc;
/* Add all virtual process from 0 to (nvp-1) */
if (elan4_add_p2pvp (rail->r_ctx, 0, cap) < 0) {
/* FIXME: exception code */
return OMPI_ERROR;
}
/* block the inputter until we are really ready */
elan4_block_inputter (rail->r_ctx, 1);
if (elan4_attach (rail->r_ctx, cap)) {
/* FIXME: exception code */
return OMPI_ERROR;
}
/* NB: We should have same vp for all capabilities */
loc.loc_node = ((CTX_ELAN *) rail->r_ctx)->ctx_position.pos_nodeid
- cap->cap_lownode;
loc.loc_context = cap->cap_mycontext - cap->cap_lowcontext;
/* TODO: debug code for loc */
if (cap->cap_type == ELAN_CAP_TYPE_HWTEST)
state->elan_vp = 0;
else
state->elan_vp = elan_location2vp (loc, state->elan_cap);
/* Initialise the Elan version of this data */
((ELAN_ESTATE *) rail->r_estate)->vp = state->elan_vp;
/* Allocate a cookie pool for the thread processor
* and copy to sdram */
rail->r_cpool =
elan4_allocCookiePool (rail->r_ctx, state->elan_vp);
((ELAN_EPRIVSTATE *) rail->r_estate)->tcookie =
rail->r_cpool->cp_cookie;
}
/* Allocate a cookie pool for the exported API (0th rail) */
state->cpool = elan4_allocCookiePool (state->elan_ctx, state->elan_vp);
loc = elan_vp2location (state->elan_vp, state->elan_cap);
/* update THREAD elan_dbg info of debugfile */
if (state->elan_debugfile && fileno (state->elan_debugfile) != -1) {
for (i = 0; i < num_rails; i++) {
CTX_ELAN *ctx = state->elan_rail[i]->rail_ctx;
/* Convert FILE * stream to fd for THRD output */
((ELAN_ESTATE *) state->all_estates[i])->debugFd =
fileno (state->elan_debugfile);
/* Also re-direct libelan4 output to this file */
elan4_set_debugfd (ctx, fileno (state->elan_debugfile));
}
}
/* Determine the number of processes described by the capability */
state->elan_nvp = elan_nvps (state->elan_cap);
/* Start allocating bcastVps a little way after the pt2pt ones */
state->bcastVp = state->baseBcastVp = state->elan_nvp + 16;
if (state->elan_vp >= elan_state->elan_nvp) {
/* FIXME: exception code */
return OMPI_ERROR;
}
/* FIXME: fix the following rail table stuff */
state->railTable = _elan_railtable_create (elan_state);
state->rail = _elan_railtable_getRailSelf (elan_state,
elan_state->railTable);
if (NULL == (state->elan_localIds = (int *)
MALLOC (elan_state->nvp * sizeof (int)))) {
/* FIXME: exception code */
return OMPI_ERROR;
}
/* Set all to non local initially */
for (vp = 0; vp < state->elan_nvp; vp++)
state->elan_localvps[vp] = -1;
/* Stash our own process location */
state->myloc = elan_vp2location (state->elan_vp, state->elan_cap);
state->elan_maxlocals = elan_maxlocal (elan_state->cap);
state->elan_numlocals = elan_nlocal (state->myloc.loc_node,
state->elan_cap);
/* Allocate more than we need to keep the heap in sync */
if (NULL == (vps = (int *) malloc (sizeof (int) * elan_state->nvp))) {
/* FIXME: exception code */
return OMPI_ERROR;
}
/* Fill out the local vp array */
elan_localvps (state->myloc.loc_node,
state->elan_cap, vps, state->elan_numlocals);
for (i = 0; i < state->elan_numlocals; i++) {
int localvp = vps[i];
/* This is a local process */
state->elan_localvps[localvp] = i;
if (localvp == state->elan_vp) {
state->elan_localid = i;
}
}
/* TODO: Debug code here for elan_state */
/*
* We need to create one of these maps in each rail estate ctx too
*/
for (i = 0; i < num_rails; i++) {
RAIL *rail = state->all_rails[i];
ELAN_ESTATE *estate = state->all_estates[i];
int *localIdsElan;
/* Allocate a copy of this in the SDRAM of each rail */
if (0 == (localIdsElan = ALLOC_ELAN (rail, ELAN_ALIGN,
elan_state->nvp *
sizeof (int)))) {
/* FIXME: exception code */
return OMPI_ERROR;
}
estate->localIds = SDRAM2ELAN (rail->r_ctx, localIdsElan);
for (vp = 0; vp < state->elan_nvp; vp++) {
localIdsElan[vp] = state->elan_localvps[vp];
}
estate->nLocalIds = state->elan_numlocals;
estate->localId = state->elan_localid;
estate->nvp = state->elan_nvp;
/* Update thread info on primary rail too */
((ELAN_EPRIVSTATE *) estate)->rail = state->elan_rail;
}
/* Done with vps array now */
free (vps);
#endif
return (OMPI_SUCCESS);
}
static int
ompi_init_elan_qdma (mca_ptl_elan_module_1_0_0_t * mp)
{
return (OMPI_SUCCESS);
}
static int
ompi_init_elan_rdma (mca_ptl_elan_module_1_0_0_t * mp)
{
return (OMPI_SUCCESS);
}
static int
ompi_init_elan_sten (mca_ptl_elan_module_1_0_0_t * mp)
{
return (OMPI_SUCCESS);
}
static int
ompi_init_elan_stat (mca_ptl_elan_module_1_0_0_t * mp)
{
return (OMPI_SUCCESS);
}

280
src/mca/ptl/elan/src/ptl_elan_module.c Обычный файл
Просмотреть файл

@ -0,0 +1,280 @@
/*
* $HEADER$
*/
#include <errno.h>
#include <unistd.h>
#include <string.h>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include "constants.h"
#include "event/event.h"
#include "util/if.h"
#include "util/argv.h"
#include "util/output.h"
#include "mca/pml/pml.h"
#include "mca/ptl/ptl.h"
#include "mca/ptl/base/ptl_base_sendreq.h"
#include "mca/base/mca_base_param.h"
#include "mca/base/mca_base_module_exchange.h"
#include "ptl_elan.h"
#include "ptl_elan_proc.h"
#include "ptl_elan_frag.h"
#include "ptl_elan_req.h"
#include "ptl_elan_priv.h"
mca_ptl_elan_module_1_0_0_t mca_ptl_elan_module = {
{
/* Base module information about itself */
{
/* Indicate that we are a pml v1.0.0 module
* (which also implies a specific MCA version) */
MCA_PTL_BASE_VERSION_1_0_0,
"elan", /* MCA module name */
1, /* MCA module major version */
0, /* MCA module minor version */
0, /* MCA module release version */
mca_ptl_elan_module_open, /* module open */
mca_ptl_elan_module_close /* module close */
},
/* Next the MCA v1.0.0 module meta data */
{
/* Whether the module is checkpointable or not */
false
},
/* The management related interfaces */
mca_ptl_elan_module_init,
mca_ptl_elan_module_control,
mca_ptl_elan_module_progress
}
};
static mca_ptl_elan_module_1_0_0_t *mp = &mca_ptl_elan_module;
/**
* utility routines for parameter registration
*/
static inline char *
mca_ptl_elan_param_register_string (const char *param_name,
const char *default_value)
{
int id;
char *param_value;
id = mca_base_param_register_string ("ptl", "elan", param_name, NULL,
default_value);
mca_base_param_lookup_string (id, &param_value);
return param_value;
}
static inline int
mca_ptl_elan_param_register_int (const char *param_name,
int default_value)
{
int id;
int param_value;
param_value = default_value;
id = mca_base_param_register_int ("ptl", "elan", param_name, NULL,
default_value);
mca_base_param_lookup_int (id, &param_value);
return param_value;
}
/*
* Called by MCA framework to open the module, registers
* module parameters.
*/
int
mca_ptl_elan_module_open (void)
{
/* register super module parameters */
mca_ptl_elan.super.ptl_exclusivity =
mca_ptl_elan_param_register_int ("exclusivity", 0);
mca_ptl_elan.super.ptl_first_frag_size =
mca_ptl_elan_param_register_int ("first_frag_size", 1984/*magic*/);
mca_ptl_elan.super.ptl_min_frag_size =
mca_ptl_elan_param_register_int ("min_frag_size", 320);
mca_ptl_elan.super.ptl_max_frag_size =
mca_ptl_elan_param_register_int ("max_frag_size", -1);
#if 0 /* These parameters not very useful */
/* register ELAN module parameters */
mca_ptl_elan_module.elan_free_list_num =
mca_ptl_elan_param_register_int ("free_list_num", 64);
mca_ptl_elan_module.elan_free_list_max =
mca_ptl_elan_param_register_int ("free_list_max", -1);
mca_ptl_elan_module.elan_free_list_inc =
mca_ptl_elan_param_register_int ("free_list_inc", 64);
/* initialize state */
mca_ptl_elan_module.elan_state = NULL;
mca_ptl_elan_module.elan_ptls = NULL;
mca_ptl_elan_module.elan_num_ptls = 0;
/* initialize objects */
OBJ_CONSTRUCT (&mca_ptl_elan_module.elan_lock, ompi_mutex_t);
OBJ_CONSTRUCT (&mca_ptl_elan_module.elan_procs, ompi_list_t);
#endif
return OMPI_SUCCESS;
}
int
mca_ptl_elan_module_close (void)
{
if (mca_ptl_elan_module.elan_reqs_free.fl_num_allocated !=
mca_ptl_elan_module.elan_reqs_free.super.ompi_list_length) {
ompi_output (0, "elan requests: %d allocated %d returned\n",
mca_ptl_elan_module.elan_reqs_free.fl_num_allocated,
mca_ptl_elan_module.elan_reqs_free.super.ompi_list_length);
}
if (mca_ptl_elan_module.elan_events_free.fl_num_allocated !=
mca_ptl_elan_module.elan_events_free.super.ompi_list_length) {
}
if (NULL != mca_ptl_elan_module.elan_ptls) {
int i;
for (i = mca_ptl_elan_module.elan_num_ptls; i > 0; i--) {
free (mca_ptl_elan_module.elan_ptls[i - 1]);
}
free (mca_ptl_elan_module.elan_ptls);
}
/* Free the empty list holders */
OBJ_DESTRUCT (&(mca_ptl_elan_module.elan_reqs));
OBJ_DESTRUCT (&(mca_ptl_elan_module.elan_prog_events));
OBJ_DESTRUCT (&(mca_ptl_elan_module.elan_comp_events));
OBJ_DESTRUCT (&(mca_ptl_elan_module.elan_procs));
OBJ_DESTRUCT (&(mca_ptl_elan_module.elan_lock));
/* TODO: check if this is needed.
* Free the free lists */
OBJ_DESTRUCT (&(mca_ptl_elan_module.elan_reqs_free));
OBJ_DESTRUCT (&(mca_ptl_elan_module.elan_events_free));
return OMPI_SUCCESS;
}
/*
* ELAN module initialization:
* (1) elan4_init() to initialize the basic support and mapping etc.
* (2) set up STEN, RDMA and QDMA structures.
* (3) register the list of PTL parameters with the MCA
*/
mca_ptl_t **
mca_ptl_elan_module_init (int *num_ptls,
bool * allow_multi_user_threads,
bool * have_hidden_threads)
{
mca_ptl_t **ptls;
int rc;
*num_ptls = 0;
*allow_multi_user_threads = true;
*have_hidden_threads = OMPI_HAVE_THREADS;
if ((rc = ompi_event_init ()) != OMPI_SUCCESS) {
ompi_output (0,
"mca_ptl_elan_module_init: "
"unable to initialize event dispatch thread: %d\n",
rc);
return NULL;
}
#if ELAN_COMP
/* initialize free lists */
ompi_free_list_init (&(mca_ptl_elan_module.elan_reqs_free),
sizeof (mca_ptl_elan_send_request_t),
OBJ_CLASS (mca_ptl_elan_send_request_t),
mca_ptl_elan_module.elan_free_list_num,
mca_ptl_elan_module.elan_free_list_max,
mca_ptl_elan_module.elan_free_list_inc, NULL);
/* use default allocator */
ompi_free_list_init (&mca_ptl_elan_module.elan_events_free,
sizeof (mca_ptl_elan_send_frag_t),
OBJ_CLASS (mca_ptl_elan_send_frag_t),
mca_ptl_elan_module.elan_free_list_num,
mca_ptl_elan_module.elan_free_list_max,
mca_ptl_elan_module.elan_free_list_inc, NULL);
#endif
/* open basic elan device */
if (OMPI_SUCCESS != ompi_mca_ptl_elan_init(&mca_ptl_elan_module)) {
return NULL;
}
/* setup communication infrastructure */
if (OMPI_SUCCESS != ompi_mca_ptl_elan_setup (&mca_ptl_elan_module)) {
return NULL;
}
/*
* (mca_ptl_elan_module_exchange () != OMPI_SUCCESS)
*
* No need to publish parameters with the MCA framework
*
* This is called only by those processes who have elan.
* So it does not qualify to be a global call.
* Since the processes has elan support can already communicate
* over elan, there is no need for a oob_based exchange.
*/
ptls = (mca_ptl_elan_t **) malloc (mca_ptl_elan_module.elan_num_ptls *
sizeof (mca_ptl_t *));
if (NULL == ptls) {
return NULL;
}
memcpy (ptls, mca_ptl_elan_module.elan_ptls,
mca_ptl_elan_module.elan_num_ptls * sizeof (mca_ptl_elan_t *));
*num_ptls = mca_ptl_elan_module.elan_num_ptls;
return ptls;
}
/*
* FIXME: to support ELAN module control
*/
int
mca_ptl_elan_module_control (int param,
void *value,
size_t size)
{
switch (param) {
case MCA_PTL_ENABLE:
if (*(int *) value) {
ompi_event_add (&mca_ptl_elan_module.elan_recv_event, 0);
} else {
ompi_event_del (&mca_ptl_elan_module.elan_recv_event);
}
break;
default:
break;
}
return OMPI_SUCCESS;
}
/*
* FIXME: to support event-based module progress.
*/
int
mca_ptl_elan_module_progress (mca_ptl_tstamp_t tstamp)
{
ompi_event_loop (OMPI_EVLOOP_ONCE);
return OMPI_SUCCESS;
}

47
src/mca/ptl/elan/src/ptl_elan_peer.h Обычный файл
Просмотреть файл

@ -0,0 +1,47 @@
/*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PTL_ELAN_PEER_H
#define MCA_PTL_ELAN_PEER_H
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include "class/ompi_list.h"
#include "event/event.h"
#include "mca/pml/pml.h"
#include "mca/ptl/ptl.h"
#include "ptl_elan_frag.h"
typedef enum {
MCA_PTL_ELAN_CLOSED,
MCA_PTL_ELAN_CONNECTED,
MCA_PTL_ELAN_FAILED,
NUM_MCA_PTL_ELAN_STAT
} mca_ptl_elan_status_t;
/**
* An abstraction that represents a connection to a peer process.
* Peers are always connected unless they are in different LAN or died.
*/
struct mca_ptl_elan_peer_t {
ompi_list_item_t super;
struct mca_ptl_elan_t* peer_ptl;
struct mca_ptl_elan_proc_t* peer_proc;
int resending; /* A resending stage, no more new dma's */
int num_resend; /* How many times I have retried */
double open_time;
double close_time;
double known_alive_time;
};
typedef struct mca_ptl_elan_peer_t mca_ptl_elan_peer_t;
extern ompi_class_t mca_ptl_elan_peer_t_class;
#endif

0
src/mca/ptl/elan/src/ptl_elan_priv.c Обычный файл
Просмотреть файл

45
src/mca/ptl/elan/src/ptl_elan_priv.h Обычный файл
Просмотреть файл

@ -0,0 +1,45 @@
/*
* $HEADER$
*/
#ifndef MCA_PTL_ELAN_PRIV_H
#define MCA_PTL_ELAN_PRIV_H
#include <errno.h>
#include <unistd.h>
#include <string.h>
#include <fcntl.h>
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include <arpa/inet.h>
#include "constants.h"
#include "event/event.h"
#include "util/if.h"
#include "util/argv.h"
#include "util/output.h"
#include "mca/pml/pml.h"
#include "mca/ptl/ptl.h"
#include "mca/ptl/base/ptl_base_sendreq.h"
#include "mca/base/mca_base_param.h"
#include "mca/base/mca_base_module_exchange.h"
#include "ptl_elan.h"
#include "ptl_elan_proc.h"
#include "ptl_elan_frag.h"
#include "ptl_elan_req.h"
/*#include "elan/init.h"*/
/*#include "elan/sys/misc_sys.h"*/
/*#include "elan/sys/init_sys.h"*/
/*#include "elan/elan.h"*/
/* Initialization and finalization routines */
int ompi_mca_ptl_elan_init (mca_ptl_elan_module_1_0_0_t *mp);
int ompi_mca_ptl_elan_setup (mca_ptl_elan_module_1_0_0_t *mp);
int ompi_mca_ptl_elan_fin (mca_ptl_elan_module_1_0_0_t *mp);
/* communication prototypes */
/* control, synchronization and state prototypes */
#endif

110
src/mca/ptl/elan/src/ptl_elan_proc.c Обычный файл
Просмотреть файл

@ -0,0 +1,110 @@
/*
* $HEADER$
*/
#include <string.h>
#include "atomic.h"
#include "class/ompi_hash_table.h"
#include "mca/base/mca_base_module_exchange.h"
#include "ptl_elan.h"
#include "ptl_elan_peer.h"
#include "ptl_elan_proc.h"
static void mca_ptl_elan_proc_construct (mca_ptl_elan_proc_t * proc);
static void mca_ptl_elan_proc_destruct (mca_ptl_elan_proc_t * proc);
static mca_ptl_elan_proc_t *mca_ptl_elan_proc_lookup_ompi (ompi_proc_t *
ompi_proc);
ompi_class_t mca_ptl_elan_proc_t_class = {
"mca_ptl_elan_proc_t",
OBJ_CLASS (ompi_list_item_t),
(ompi_construct_t) mca_ptl_elan_proc_construct,
(ompi_destruct_t) mca_ptl_elan_proc_destruct
};
/**
* Initialize elan proc instance
*/
void mca_ptl_elan_proc_construct (mca_ptl_elan_proc_t * proc)
{
return;
}
/*
* Cleanup elan proc instance
*/
void mca_ptl_elan_proc_destruct (mca_ptl_elan_proc_t * proc)
{
return;
}
/*
* Create a ELAN process structure. There is a one-to-one correspondence
* between a ompi_proc_t and a mca_ptl_elan_proc_t instance.
* We cache additional data (specifically the list
* of mca_ptl_elan_peer_t instances, and publiched
* addresses) associated w/ a given destination on this datastructure.
*/
mca_ptl_elan_proc_t *mca_ptl_elan_proc_create (ompi_proc_t * ompi_proc)
{
return NULL;
}
/*
* Look for an existing ELAN process instances based on the associated
* ompi_proc_t instance.
*/
static mca_ptl_elan_proc_t *
mca_ptl_elan_proc_lookup_ompi (ompi_proc_t *ompi_proc)
{
return NULL;
}
/*
* Look for an existing ELAN process instance based on the globally unique
* process identifier.
*/
mca_ptl_elan_proc_t *mca_ptl_elan_proc_lookup (void *guid, size_t size)
{
return NULL;
}
/*
* Note that this routine must be called with the lock on the process already
* held. Insert a ptl instance into the proc array and assign it an address.
*/
int mca_ptl_elan_proc_insert (mca_ptl_elan_proc_t * ptl_proc,
mca_ptl_elan_peer_t * ptl_peer)
{
return OMPI_SUCCESS;
}
/*
* Remove a peer from the proc array and indicate the address is
* no longer in use.
*/
int mca_ptl_elan_proc_remove (mca_ptl_elan_proc_t * ptl_proc,
mca_ptl_elan_peer_t * ptl_peer)
{
return OMPI_SUCCESS;
}
/*
* loop through all available PTLs for one matching the source address
* of the request.
*/
bool mca_ptl_elan_proc_accept (mca_ptl_elan_proc_t * ptl_proc,
struct sockaddr_in * addr, int sd)
{
return false;
}

50
src/mca/ptl/elan/src/ptl_elan_proc.h Обычный файл
Просмотреть файл

@ -0,0 +1,50 @@
/*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PTL_ELAN_PROC_H
#define MCA_PTL_ELAN_PROC_H
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include "class/ompi_object.h"
#include "proc/proc.h"
#include "ptl_elan.h"
#include "ptl_elan_peer.h"
extern ompi_class_t mca_ptl_elan_proc_t_class;
/**
* Represents the state of a remote process. Also cache an instance
* of mca_ptl_base_peer_t for each
* PTL instance that attempts to open a connection to the process.
*/
struct mca_ptl_elan_proc_t {
ompi_list_item_t super; /**< allow proc to be placed on a list */
ompi_proc_t *proc_ompi; /**< pointer to corresponding ompi_proc_t */
struct mca_ptl_base_peer_t **proc_peers; /**< array of peers */
size_t proc_peer_count; /**< number of peers */
void* proc_uuid; /**< univeral unique identifier the process */
size_t univ_size; /**< size of the universe */
ompi_mutex_t proc_lock; /**< lock to for proc state */
};
typedef struct mca_ptl_elan_proc_t mca_ptl_elan_proc_t;
mca_ptl_elan_proc_t* mca_ptl_elan_proc_create(ompi_proc_t* ompi_proc);
mca_ptl_elan_proc_t* mca_ptl_elan_proc_lookup(void *guid, size_t size);
static inline mca_ptl_elan_proc_t* mca_ptl_elan_proc_local(void)
{
return NULL;
}
int mca_ptl_elan_proc_insert(mca_ptl_elan_proc_t *, mca_ptl_elan_peer_t *);
int mca_ptl_elan_proc_remove(mca_ptl_elan_proc_t *, mca_ptl_elan_peer_t *);
bool mca_ptl_elan_proc_accept(mca_ptl_elan_proc_t *,
struct sockaddr_in *, int sd);
#endif

35
src/mca/ptl/elan/src/ptl_elan_req.h Обычный файл
Просмотреть файл

@ -0,0 +1,35 @@
/*
* $HEADER$
*/
/**
* @file
*/
#ifndef MCA_PTL_ELAN_REQ_H
#define MCA_PTL_ELAN_REQ_H
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include "ompi_config.h"
#include "mca/ptl/base/ptl_base_sendreq.h"
#include "mca/ptl/base/ptl_base_recvreq.h"
#include "ptl_elan_frag.h"
/* Again, what is this! */
/* extern ompi_class_t mca_ptl_elan_send_request_t_class;*/
/**
* ELAN send request derived type. The send request contains both the
* base send request, and space for the first ELAN send fragment descriptor.
* This avoids the overhead of a second allocation for the initial send
* fragment on every send request.
*/
struct mca_ptl_elan_send_request_t {
mca_ptl_base_send_request_t super;
mca_ptl_elan_send_frag_t req_frag; /* first fragment */
};
typedef struct mca_ptl_elan_send_request_t mca_ptl_elan_send_request_t;
#endif