From 0710f299e1f41457c38a5fa4cf39ba2f269d9871 Mon Sep 17 00:00:00 2001 From: Weikuan Yu Date: Mon, 14 Jun 2004 22:39:14 +0000 Subject: [PATCH] Just to save some codes in elan This commit was SVN r1252. --- src/mca/ptl/elan/.ompi_ignore | 0 src/mca/ptl/elan/Makefile.am | 36 ++ src/mca/ptl/elan/VERSION | 6 + src/mca/ptl/elan/configure.params | 10 + src/mca/ptl/elan/src/Makefile.am | 28 ++ src/mca/ptl/elan/src/ptl_elan.c | 120 ++++++ src/mca/ptl/elan/src/ptl_elan.h | 371 ++++++++++++++++++ src/mca/ptl/elan/src/ptl_elan_frag.h | 146 +++++++ src/mca/ptl/elan/src/ptl_elan_init.c | 509 +++++++++++++++++++++++++ src/mca/ptl/elan/src/ptl_elan_module.c | 280 ++++++++++++++ src/mca/ptl/elan/src/ptl_elan_peer.h | 47 +++ src/mca/ptl/elan/src/ptl_elan_priv.c | 0 src/mca/ptl/elan/src/ptl_elan_priv.h | 45 +++ src/mca/ptl/elan/src/ptl_elan_proc.c | 110 ++++++ src/mca/ptl/elan/src/ptl_elan_proc.h | 50 +++ src/mca/ptl/elan/src/ptl_elan_req.h | 35 ++ 16 files changed, 1793 insertions(+) create mode 100644 src/mca/ptl/elan/.ompi_ignore create mode 100644 src/mca/ptl/elan/Makefile.am create mode 100644 src/mca/ptl/elan/VERSION create mode 100644 src/mca/ptl/elan/configure.params create mode 100644 src/mca/ptl/elan/src/Makefile.am create mode 100644 src/mca/ptl/elan/src/ptl_elan.c create mode 100644 src/mca/ptl/elan/src/ptl_elan.h create mode 100644 src/mca/ptl/elan/src/ptl_elan_frag.h create mode 100644 src/mca/ptl/elan/src/ptl_elan_init.c create mode 100644 src/mca/ptl/elan/src/ptl_elan_module.c create mode 100644 src/mca/ptl/elan/src/ptl_elan_peer.h create mode 100644 src/mca/ptl/elan/src/ptl_elan_priv.c create mode 100644 src/mca/ptl/elan/src/ptl_elan_priv.h create mode 100644 src/mca/ptl/elan/src/ptl_elan_proc.c create mode 100644 src/mca/ptl/elan/src/ptl_elan_proc.h create mode 100644 src/mca/ptl/elan/src/ptl_elan_req.h diff --git a/src/mca/ptl/elan/.ompi_ignore b/src/mca/ptl/elan/.ompi_ignore new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/mca/ptl/elan/Makefile.am b/src/mca/ptl/elan/Makefile.am new file mode 100644 index 0000000000..5c3fee8556 --- /dev/null +++ b/src/mca/ptl/elan/Makefile.am @@ -0,0 +1,36 @@ +# +# $HEADER$ +# + +# Use the top-level OMPI Makefile.options + +include $(top_ompi_srcdir)/config/Makefile.options + +SUBDIRS = src + +EXTRA_DIST = VERSION + +# According to the MCA spec, we have to make the output library here +# in the top-level directory, and it has to be named +# libompi_ssi_coll_ompi_basic.la + +if OMPI_BUILD_ptl_elan_LOADABLE_MODULE +module_noinst = +module_install = mca_ptl_elan.la +else +module_noinst = libmca_ptl_elan.la +module_install = +endif + +mcamoduledir = $(libdir)/ompi +mcamodule_LTLIBRARIES = $(module_install) +mca_ptl_elan_la_SOURCES = +mca_ptl_elan_la_LIBADD = \ + src/libmca_ptl_elan.la \ + $(LIBOMPI_LA) +mca_ptl_elan_la_LDFLAGS = -module -avoid-version + +noinst_LTLIBRARIES = $(module_noinst) +libmca_ptl_elan_la_SOURCES = +libmca_ptl_elan_la_LIBADD = src/libmca_ptl_elan.la +libmca_ptl_elan_la_LDFLAGS = -module -avoid-version diff --git a/src/mca/ptl/elan/VERSION b/src/mca/ptl/elan/VERSION new file mode 100644 index 0000000000..dadc1f42ab --- /dev/null +++ b/src/mca/ptl/elan/VERSION @@ -0,0 +1,6 @@ +major=1 +minor=0 +release=0 +alpha=0 +beta=0 +svn=1 diff --git a/src/mca/ptl/elan/configure.params b/src/mca/ptl/elan/configure.params new file mode 100644 index 0000000000..65c097b1a4 --- /dev/null +++ b/src/mca/ptl/elan/configure.params @@ -0,0 +1,10 @@ +# -*- shell-script -*- +# +# $HEADER$ +# + +# Specific to this module + +PARAM_INIT_FILE=src/ptl_elan.c +PARAM_CONFIG_HEADER_FILE="src/elan_config.h" +PARAM_CONFIG_FILES="Makefile src/Makefile" diff --git a/src/mca/ptl/elan/src/Makefile.am b/src/mca/ptl/elan/src/Makefile.am new file mode 100644 index 0000000000..21fd64a96e --- /dev/null +++ b/src/mca/ptl/elan/src/Makefile.am @@ -0,0 +1,28 @@ +# -*- makefile -*- +# +# $HEADER$ +# + +include $(top_ompi_srcdir)/config/Makefile.options + +AM_CPPFLAGS = \ + -I$(top_ompi_builddir)/src/include \ + -I$(top_ompi_builddir)/src/ompi/event \ + -I$(top_ompi_srcdir)/src \ + -I$(top_ompi_srcdir)/src/include + +noinst_LTLIBRARIES = libmca_ptl_elan.la +libmca_ptl_elan_la_SOURCES = \ + ptl_elan.h \ + ptl_elan_frag.h \ + ptl_elan_req.h \ + ptl_elan_proc.h \ + ptl_elan_addr.h \ + ptl_elan_peer.h \ + ptl_elan_proc.c \ + ptl_elan_module.c \ + ptl_elan_priv.c \ + ptl_elan_init.c \ + ptl_elan.c + + diff --git a/src/mca/ptl/elan/src/ptl_elan.c b/src/mca/ptl/elan/src/ptl_elan.c new file mode 100644 index 0000000000..c519515bce --- /dev/null +++ b/src/mca/ptl/elan/src/ptl_elan.c @@ -0,0 +1,120 @@ +/* + * $HEADER$ + */ + +#include +#include "util/output.h" +#include "util/if.h" +#include "mca/pml/pml.h" +#include "mca/ptl/ptl.h" +#include "mca/ptl/base/ptl_base_header.h" +#include "mca/ptl/base/ptl_base_sendreq.h" +#include "mca/ptl/base/ptl_base_sendfrag.h" +#include "mca/ptl/base/ptl_base_recvreq.h" +#include "mca/ptl/base/ptl_base_recvfrag.h" +#include "mca/base/mca_base_module_exchange.h" +#include "ptl_elan.h" +#include "ptl_elan_peer.h" +#include "ptl_elan_proc.h" +#include "ptl_elan_req.h" +#include "ptl_elan_frag.h" + + +mca_ptl_elan_t mca_ptl_elan = { + { + &mca_ptl_elan_module.super, + 0, /* ptl_exclusivity */ + 0, /* ptl_latency */ + 0, /* ptl_bandwidth */ + 0, /* ptl_frag_first_size */ + 0, /* ptl_frag_min_size */ + 0, /* ptl_frag_max_size */ + MCA_PTL_PUT, /* ptl flags */ + + /* collection of interfaces */ + mca_ptl_elan_add_proc, + mca_ptl_elan_del_proc, + mca_ptl_elan_finalize, + mca_ptl_elan_isend, + mca_ptl_elan_irecv, + mca_ptl_elan_put, + mca_ptl_elan_get, + mca_ptl_elan_matched, + mca_ptl_elan_req_alloc, + mca_ptl_elan_req_return + } +}; + +int mca_ptl_elan_add_proc (struct mca_ptl_t *ptl, + struct ompi_proc_t *ompi_proc, + struct mca_ptl_base_peer_t **peer_ret) +{ + return OMPI_SUCCESS; +} + + +int mca_ptl_elan_del_proc (struct mca_ptl_t *ptl, struct ompi_proc_t *proc, + struct mca_ptl_base_peer_t *ptl_peer) +{ + return OMPI_SUCCESS; +} + +int mca_ptl_elan_finalize (struct mca_ptl_t *ptl) +{ + return OMPI_SUCCESS; +} + +int mca_ptl_elan_req_alloc (struct mca_ptl_t *ptl, + struct mca_ptl_base_send_request_t **request) +{ + int rc; + return rc; +} + + +void mca_ptl_elan_req_return (struct mca_ptl_t *ptl, + struct mca_ptl_base_send_request_t *request) +{ + return; +} + + +void mca_ptl_elan_recv_frag_return (struct mca_ptl_t *ptl, + struct mca_ptl_elan_recv_frag_t *frag) +{ + return; +} + + +void mca_ptl_elan_send_frag_return (struct mca_ptl_t *ptl, + struct mca_ptl_elan_send_frag_t *frag) +{ + return; +} + +/* + * Initiate a send. If this is the first fragment, use the fragment + * descriptor allocated with the send requests, otherwise obtain + * one from the free list. Initialize the fragment and foward + * on to the peer. + */ + +int mca_ptl_elan_isend (struct mca_ptl_t *ptl, + struct mca_ptl_base_peer_t *ptl_peer, + struct mca_ptl_base_send_request_t *sendreq, + size_t offset, size_t * size, int flags) +{ + return OMPI_SUCCESS; +} + + +/* + * A posted receive has been matched - if required send an + * ack back to the peer and process the fragment. + */ + +void mca_ptl_elan_matched (mca_ptl_t * ptl, + mca_ptl_base_recv_frag_t * frag) +{ + return; +} diff --git a/src/mca/ptl/elan/src/ptl_elan.h b/src/mca/ptl/elan/src/ptl_elan.h new file mode 100644 index 0000000000..fb153ddfe6 --- /dev/null +++ b/src/mca/ptl/elan/src/ptl_elan.h @@ -0,0 +1,371 @@ +/* + * $HEADER$ + */ +/** + * @file + */ +#ifndef MCA_PTL_ELAN_H +#define MCA_PTL_ELAN_H + +#include +#include +#include +#include "mem/free_list.h" +#include "event/event.h" +#include "mca/pml/pml.h" +#include "mca/ptl/ptl.h" + +/*#include "elan/sys/init_sys.h"*/ +/*#include "elan/elan.h"*/ + +#define MCA_PTL_ELAN_STATISTICS 0 + +/** + * ELAN PTL module. + */ + +struct mca_ptl_elan_state_t { + + /* User configurable parameters */ + + char *elan_version; /**< Version of the elan library */ + uint64_t elan_debug; /**< elan debug tracing output */ + uint64_t elan_traced; /**< elan TRACE output */ + uint64_t elan_flags; + FILE elan_debugfile; + int elan_signalnum; + +#ifdef ENABLE_ELAN_MEMORY_ALLOCATOR + size_t main_size; /**< size of Main memory allocator heap */ + size_t elan_size; /**< size of Elan memory allocator heap */ + void *main_base; /**< Main memory allocator heap base */ + void *elan_base; /**< Elan memory allocator heap base */ +#endif + + /* other state parameters */ + + int elan_attached; /**< 0 until elan_attach() called */ + unsigned int elan_vp; /**< elan vpid, not ompi vpid */ + unsigned int elan_nvp; /**< total # of elan vpid */ + int *elan_localvps; /**< mapping of localId to elan vp */ + int elan_localid; /**< # of local elan vpids */ + int elan_numlocals; /**< # of local elan vpids */ + int elan_maxlocals; /**< maximum # of local elan vpids */ + int elan_nrails; /**< # of rails elan vpids */ + int elan_rmsid; /**< rms resource id */ + long elan_pagesize; + pid_t elan_pid; + + /* TODO: + * Even though the elan threads are not utilized for now. + * We provide memory/state control structures for later extensions. + * A simple type casting of ELAN_ESTATE can bring + * the complete structure of the ELAN_EPRIVSATE. + */ + void *elan_cap; /**< job capability */ + void *elan_estate; /**< Elan state of the 0th rail */ + +#ifdef ELAN_COMP + ELAN_CTX *elan_ctx; /**< Elan ctx of the 0th rail */ + ELAN_RAIL **elan_rail; /**< Rail control struct for all rails */ + RAIL **all_rails; /**< all rails */ + ELAN_ESTATE **all_estates; /**< elan (priv)states of all rails */ +#endif + ompi_mutex_t state_lock; /**< lock for elan state */ +}; +typedef struct mca_ptl_elan_state_t mca_ptl_elan_state_t; + +struct mca_ptl_elan_module_1_0_0_t { + + mca_ptl_base_module_1_0_0_t super; /**< base PTL module */ + + /* These parameters does not provided good freedom, + * It does not hurt to skip them */ +#if 1 + int elan_free_list_num; /**< initial size of free lists */ + int elan_free_list_max; /**< maximum size of free lists */ + int elan_free_list_inc; /**< # to alloc when growing lists */ +#endif + + /* + * We create our own simplified structure for managing elan state + * although libelan already provides one. We do not need + * all that tport, group structures. + */ + struct mca_ptl_elan_state_t *elan_state; /**< elan state */ + struct mca_ptl_elan_t **elan_ptls; /**< array of available PTLs */ + size_t elan_num_ptls; /**< number of ptls activated */ + + ompi_list_t elan_reqs; /**< all elan requests */ + ompi_list_t elan_prog_events; /**< events in progress */ + ompi_list_t elan_comp_events; /**< events completed, but to reclaim */ + + ompi_free_list_t elan_events_free; /**< free events */ + ompi_free_list_t elan_reqs_free; /**< all elan requests */ + + ompi_event_t elan_send_event; /**< event structure for sends */ + ompi_event_t elan_recv_event; /**< event structure for recvs */ + ompi_list_t elan_procs; /**< elan proc's */ + ompi_mutex_t elan_lock; /**< lock for module state */ +}; +typedef struct mca_ptl_elan_module_1_0_0_t mca_ptl_elan_module_1_0_0_t; + +struct mca_ptl_elan_send_frag_t; +struct mca_ptl_elan_recv_frag_t; + +extern mca_ptl_elan_module_1_0_0_t mca_ptl_elan_module; + +/** + * Register ELAN module parameters with the MCA framework + */ +extern int mca_ptl_elan_module_open (void); + +/** + * Any final cleanup before being unloaded. + */ +extern int mca_ptl_elan_module_close (void); + +/** + * ELAN module initialization. + * + * @param num_ptls (OUT) + * Number of PTLs returned in PTL array. + * @param allow_multi_user_threads (OUT) + * Flag indicating wether PTL supports user threads (TRUE) + * @param have_hidden_threads (OUT) + * Flag indicating whether PTL uses threads (TRUE) + * + */ +extern mca_ptl_t **mca_ptl_elan_module_init (int *num_ptls, + bool * multi_user_threads, + bool * have_hidden_threads); + +/** + * ELAN module control. + */ +extern int mca_ptl_elan_module_control (int param, + void *value, + size_t size); + +/** + * ELAN module progress. + */ +extern int mca_ptl_elan_module_progress (mca_ptl_tstamp_t tstamp); + +/** + * ELAN PTL Interface + */ +struct mca_ptl_elan_t { + + /**< The elan progress related interface */ + + mca_ptl_t super; /**< base PTL interface */ + + /**< The following are elan-related control structures */ + +#ifdef ELAN_COMP + ELAN_RAIL *elan_rail; /**< Pointer to this Rail */ + ELAN_CTX *elan_ctx; /**< Elan ctx of this rail */ +#endif + int ptl_ni_local; /**< PTL NI local rank */ + int ptl_ni_total; /**< PTL NI total */ + + int elan_sten_size; /**< sten packet len */ + int elan_rdma_size; /**< qdma packet length */ + int elan_qdma_size; /**< qdma packet length */ + + int sten_total; /**< total sten descriptors */ + int rdma_total; /**< total rdma descriptors */ + int qdma_total; /**< total rdma descriptors */ + + int sten_num; /**< num of outstanding sten packets */ + int rdma_num; /**< num of outstanding rdma packets */ + int qdma_num; /**< num of outstanding rdma packets */ + + int max_num_dmas; /**< total rdma descriptors */ + + ompi_list_t elan_stens; /**< used elan sten descriptors*/ + ompi_list_t elan_dmas; /**< used elan dma descriptors*/ + ompi_list_t elan_rdmas; /**< used elan rdma descriptors */ + ompi_list_t elan_frags; /**< used elan fragments */ + + ompi_free_list_t elan_dmas_free; /**< free elan dma descriptors*/ + ompi_free_list_t elan_stens_free; /**< free elan sten descriptors*/ + ompi_free_list_t elan_rdmas_free; /**< free elan rdma descriptors */ + ompi_free_list_t elan_frags_free; /**< free elan rdma fragments */ + +#if MCA_PTL_ELAN_STATISTICS /* some statistics */ + size_t ptl_bytes_sent; + size_t ptl_bytes_recv; +#endif + +}; +typedef struct mca_ptl_elan_t mca_ptl_elan_t; +extern mca_ptl_elan_t mca_ptl_elan; + +/** + * Cleanup any resources held by the PTL. + * + * @param ptl PTL instance. + * @return OMPI_SUCCESS or error status on failure. + */ +extern int mca_ptl_elan_finalize (struct mca_ptl_t *ptl); + + +/** + * PML->PTL notification of change in the process list. + * + * @param ptl (IN) + * @param proc (IN) + * @param peer (OUT) + * @return OMPI_SUCCESS or error status on failure. + */ +extern int mca_ptl_elan_add_proc (struct mca_ptl_t *ptl, + struct ompi_proc_t *proc, + struct mca_ptl_base_peer_t **peer); + +/** + * PML->PTL notification of change in the process list. + * + * @param ptl (IN) PTL instance + * @param proc (IN) Peer process + * @param peer (IN) Peer addressing information. + * @return Status indicating if cleanup was successful + */ +extern int mca_ptl_elan_del_proc (struct mca_ptl_t *ptl, + struct ompi_proc_t *procs, + struct mca_ptl_base_peer_t *addr); + +/** + * PML->PTL Allocate a send request from the PTL modules free list. + * + * @param ptl (IN) PTL instance + * @param request (OUT) Pointer to allocated request. + * @return Status indicating if allocation was successful. + */ +extern int mca_ptl_elan_req_alloc (struct mca_ptl_t *ptl, + struct mca_ptl_base_send_request_t **); + +/** + * PML->PTL Return a send request to the PTL modules free list. + * + * @param ptl (IN) PTL instance + * @param request (IN) Pointer to allocated request. + */ +extern void mca_ptl_elan_req_return (struct mca_ptl_t *ptl, + struct mca_ptl_base_send_request_t *); + +/** + * PML->PTL Notification that a receive fragment has been matched. + * + * @param ptl (IN) PTL instance + * @param recv_frag (IN) Receive fragment + */ +extern void mca_ptl_elan_matched (struct mca_ptl_t *ptl, + struct mca_ptl_base_recv_frag_t *frag); + +/** + * PML->PTL Initiate an isend of the specified size. + * + * @param ptl (IN) PTL instance + * @param ptl_base_peer (IN) PTL peer addressing + * @param send_request (IN/OUT) Send request (allocated by PML via + * mca_ptl_base_request_alloc_fn_t) + * @param size (IN) + * Number of bytes PML is requesting PTL to deliver + * @param flags (IN) + * Flags that should be passed to the peer via the message header. + * @param request (OUT) + * OMPI_SUCCESS if the PTL was able to queue one or more fragments + */ +extern int mca_ptl_elan_isend (struct mca_ptl_t *ptl, + struct mca_ptl_base_peer_t *ptl_peer, + struct mca_ptl_base_send_request_t *, + size_t offset, + size_t * size, + int flags); + +/** + * PML->PTL Initiate an irecv of the specified size. + * + * @param ptl (IN) PTL instance + * @param ptl_base_peer (IN) PTL peer addressing + * @param send_request (IN/OUT) Send request (allocated by PML via + * mca_ptl_base_request_alloc_fn_t) + * @param size (IN) + * Number of bytes PML is requesting PTL to deliver + * @param flags (IN) + * Flags that should be passed to the peer via the message header. + * @param request (OUT) + * OMPI_SUCCESS if the PTL was able to queue one or more fragments + */ +extern int mca_ptl_elan_irecv (struct mca_ptl_t *ptl, + struct mca_ptl_base_peer_t *ptl_peer, + struct mca_ptl_base_send_request_t *, + size_t offset, + size_t * size, + int flags); + +/** + * PML->PTL Initiate a put of the specified size. + * + * @param ptl (IN) PTL instance + * @param ptl_base_peer (IN) PTL peer addressing + * @param send_request (IN/OUT) Send request (allocated by PML via + * mca_ptl_base_request_alloc_fn_t) + * @param size (IN) + * Number of bytes PML is requesting PTL to deliver + * @param flags (IN) + * Flags that should be passed to the peer via the message header. + * @param request (OUT) + * OMPI_SUCCESS if the PTL was able to queue one or more fragments + */ +extern int mca_ptl_elan_put (struct mca_ptl_t *ptl, + struct mca_ptl_base_peer_t *ptl_peer, + struct mca_ptl_base_send_request_t *, + size_t offset, + size_t * size, + int flags); + +/** + * PML->PTL Initiate a get of the specified size. + * + * @param ptl (IN) PTL instance + * @param ptl_base_peer (IN) PTL peer addressing + * @param send_request (IN/OUT) Send request (allocated by PML via + * mca_ptl_base_request_alloc_fn_t) + * @param size (IN) + * Number of bytes PML is requesting PTL to deliver + * @param flags (IN) + * Flags that should be passed to the peer via the message header. + * @param request (OUT) + * OMPI_SUCCESS if the PTL was able to queue one or more fragments + */ +extern int mca_ptl_elan_get (struct mca_ptl_t *ptl, + struct mca_ptl_base_peer_t *ptl_peer, + struct mca_ptl_base_send_request_t *, + size_t offset, + size_t * size, + int flags); + +/** + * Return a recv fragment to the modules free list. + * + * @param ptl (IN) PTL instance + * @param frag (IN) ELAN receive fragment + */ +extern void mca_ptl_elan_recv_frag_return (struct mca_ptl_t *ptl, + struct mca_ptl_elan_recv_frag_t + *frag); + +/** + * Return a send fragment to the modules free list. + * + * @param ptl (IN) PTL instance + * @param frag (IN) ELAN send fragment + */ +extern void mca_ptl_elan_send_frag_return (struct mca_ptl_t *ptl, + struct mca_ptl_elan_send_frag_t + *frag); + +#endif diff --git a/src/mca/ptl/elan/src/ptl_elan_frag.h b/src/mca/ptl/elan/src/ptl_elan_frag.h new file mode 100644 index 0000000000..97548bb99e --- /dev/null +++ b/src/mca/ptl/elan/src/ptl_elan_frag.h @@ -0,0 +1,146 @@ +/* + * $HEADER$ + */ +/** + * @file + */ +#ifndef MCA_PTL_ELAN_FRAG_H +#define MCA_PTL_ELAN_FRAG_H + +#include +#include +#include +#include "ompi_config.h" +#include "mca/ptl/base/ptl_base_sendreq.h" +#include "mca/ptl/base/ptl_base_recvreq.h" +#include "mca/ptl/base/ptl_base_sendfrag.h" +#include "mca/ptl/base/ptl_base_recvfrag.h" +#include "ptl_elan.h" + +extern ompi_class_t mca_ptl_elan_send_frag_t_class; +struct mca_ptl_base_peer_t; + +/** + * ELAN send fragment derived type. + */ +struct mca_ptl_elan_send_frag_t { + mca_ptl_base_send_frag_t super; /**< base send fragment descriptor */ + size_t frag_vec_cnt; + volatile int frag_progressed; +}; +typedef struct mca_ptl_elan_send_frag_t mca_ptl_elan_send_frag_t; + + +#define MCA_PTL_ELAN_SEND_FRAG_ALLOC(item, rc) \ + OMPI_FREE_LIST_GET(&mca_ptl_elan_module.elan_send_frags, item, rc); + + +#define MCA_PTL_ELAN_RECV_FRAG_ALLOC(frag, rc) \ +{ \ + ompi_list_item_t* item; \ + OMPI_FREE_LIST_GET(&mca_ptl_elan_module.elan_recv_frags, item, rc); \ + frag = (mca_ptl_elan_recv_frag_t*)item; \ +} + +bool +mca_ptl_elan_send_frag_handler (mca_ptl_elan_send_frag_t *, int sd); + + +int +mca_ptl_elan_send_frag_init (mca_ptl_elan_send_frag_t *, + struct mca_ptl_base_peer_t *, + struct mca_ptl_base_send_request_t *, + size_t offset, size_t * size, int flags); + +extern ompi_class_t mca_ptl_elan_recv_frag_t_class; + +/** + * ELAN received fragment derived type. + */ +struct mca_ptl_elan_recv_frag_t { + mca_ptl_base_recv_frag_t super; /**< base receive fragment descriptor */ + size_t frag_hdr_cnt; /**< number of header bytes received */ + size_t frag_msg_cnt; /**< number of message bytes received */ + bool frag_ack_pending; /**< an ack pending for this fragment */ + volatile int frag_progressed; /**< flag to atomically progress */ +}; +typedef struct mca_ptl_elan_recv_frag_t mca_ptl_elan_recv_frag_t; + +bool +mca_ptl_elan_recv_frag_handler (mca_ptl_elan_recv_frag_t *, int sd); + +void +mca_ptl_elan_recv_frag_init (mca_ptl_elan_recv_frag_t * frag, + struct mca_ptl_base_peer_t *peer); + +bool +mca_ptl_elan_recv_frag_send_ack (mca_ptl_elan_recv_frag_t * frag); + +/* + * For fragments that require an acknowledgment, this routine will be called + * twice, once when the send completes, and again when the acknowledgment is + * returned. Only the last caller should update the request status, so we + * add a lock w/ the frag_progressed flag. + */ +static inline void +mca_ptl_elan_send_frag_progress (mca_ptl_elan_send_frag_t * frag) +{ + mca_ptl_base_send_request_t *request = frag->super.frag_request; + + /* if this is an ack - simply return to pool */ + if (request == NULL) { + mca_ptl_elan_send_frag_return (frag->super.super.frag_owner, frag); + + } else if (frag->frag_vec_cnt == 0 && + ((frag->super.super.frag_header.hdr_common.hdr_flags + & MCA_PTL_FLAGS_ACK_MATCHED) == 0 || + mca_ptl_base_send_request_matched (request))) { + + /* otherwise, if the message has been sent, and an ack has + * already been received, go ahead and update the request status + */ + + /* make sure this only happens once in threaded case */ + if (ompi_using_threads () + && fetchNset (&frag->frag_progressed, 1) == 1) { + return; + } else { + frag->frag_progressed = 1; + } + + /* update request status */ + frag->super.super.frag_owner->ptl_send_progress (request, + &frag->super); + + /* the first fragment is allocated with the request, + * all others need to be returned to free list + */ + if (frag->super.super.frag_header.hdr_frag.hdr_frag_offset != 0) + mca_ptl_elan_send_frag_return (frag->super.super.frag_owner, + frag); + } +} + +static inline void +mca_ptl_elan_send_frag_init_ack (mca_ptl_elan_send_frag_t * ack, + struct mca_ptl_t *ptl, + struct mca_ptl_base_peer_t *ptl_peer, + mca_ptl_elan_recv_frag_t * frag) +{ + return; +} + +static inline void +mca_ptl_elan_recv_frag_matched (mca_ptl_elan_recv_frag_t * frag) +{ + mca_ptl_base_recv_request_t *request = frag->super.frag_request; + return; +} + +static inline void +mca_ptl_elan_recv_frag_progress (mca_ptl_elan_recv_frag_t * frag) +{ + return; +} + +#endif diff --git a/src/mca/ptl/elan/src/ptl_elan_init.c b/src/mca/ptl/elan/src/ptl_elan_init.c new file mode 100644 index 0000000000..cb16545067 --- /dev/null +++ b/src/mca/ptl/elan/src/ptl_elan_init.c @@ -0,0 +1,509 @@ + +#include "ptl_elan.h" + +static int ompi_init_elan_qdma (mca_ptl_elan_module_1_0_0_t * mp); +static int ompi_init_elan_sten (mca_ptl_elan_module_1_0_0_t * mp); +static int ompi_init_elan_rdma (mca_ptl_elan_module_1_0_0_t * mp); +static int ompi_init_elan_stat (mca_ptl_elan_module_1_0_0_t * mp); +static int ompi_elan_attach_network (mca_ptl_elan_state_t * state); + +int +ompi_mca_ptl_elan_init (mca_ptl_elan_module_1_0_0_t * mp) +{ +#if ELAN_COMP + int i; + int *rails; + int num_rails; + int max_backoff; + int max_fastbackoff; + + int alloc_mainsize; + int alloc_mainbase; + int alloc_elansize; + int alloc_elanbase; + + mca_ptl_elan_state_t *state; + + max_backoff = elan_rup2 (ELAN_MAXBACKOFF) - 1; + max_fastbackoff = elan_rup2 (ELAN_MAXFASTBACKOFF) - 1; + + /* Allocate the elan (priv)state structure off the heap. + * it is not available to the Elan thread. Another structure + * for the Elan thread is to be used that instead + */ + mp->elan_state = (mca_ptl_elan_state_t *) + malloc (sizeof (mca_ptl_elan_state_t)); + + if (NULL == mp->elan_state) + return (NULL); + + memset ((void *) mp->elan_state, 0, sizeof (mca_ptl_elan_state_t)); + state = mp->elan_state; + + /* Initialise enough of state so we can call elan_exception() */ + state->elan_version = ELAN_VERSION; + state->elan_ctx = NULL; + state->elan_rail = NULL; + state->elan_vp = ELAN_INVALID_PROCESS; + state->elan_nvp = 0; + state->elan_debug = 0; + state->elan_traced = 0; + + state->elan_pagesize = sysconf (_SC_PAGESIZE); + state->elan_pid = getpid (); + + /* Default allocator parameters */ + state->main_size = ELAN_ALLOC_SIZE; + state->elan_size = ELAN_ALLOCELAN_SIZE; + state->elan_flags |= + (EXCEPTIONCORE | EXCEPTIONTRACE | EXCEPTIONDBGDUMP); + state->elan_debugfile = NULL; + state->elan_signalnum = SIGABRT; + +#ifdef ELAN_VERSION + if (!elan_checkVersion (ELAN_VERSION)) { + return OMPI_ERROR; + } +#endif + + /* FIXME: when to initialize it */ + OBJ_CONSTRUCT (&state->state_lock, ompi_mutex_t); + + /* Allocate elan capability from the heap */ + state->elan_cap = + (ELAN_CAPABILITY *) malloc (sizeof (ELAN_CAPABILITY)); + + if (NULL == state->elan_cap) { + return OMPI_ERROR; + } else { + memset (state->elan_cap, 0, sizeof (ELAN_CAPABILITY)); + } + + /* Process the capability info supplied by RMS */ + if (getenv ("ELAN_AUTO") || getenv ("RMS_NPROCS")) { + /* RMS generated capabilities */ + if (rms_getcap (0, state->elan_cap)) { + return OMPI_ERROR; + } + } + + if ((num_rails = state->elan_nrails = + elan_nrails (state->elan_cap)) <= 0) { + return OMPI_ERROR; + } + + /* MULTI-RAIL: + * Allocate storage space for each Elan rail info (ptrs) + * Allocate storage space for each Elan SDRAM state (ptrs) */ + + state->all_rails = (RAIL **) malloc (sizeof (RAIL *) * num_rails); + state->all_estates = (ELAN_ESTATE **) + malloc (sizeof (ELAN_ESTATE *) * num_rails); + + if (state->all_estates == NULL || state->all_rails == NULL) { + return OMPI_ERR_OUT_OF_RESOURCE; + } + + /* MULTI-RAIL: + * initialise each elan rail, using the physical rail info gleaned from + * the capability */ + if (NULL == (rails = (int *) malloc (sizeof (int) * num_rails))) { + return OMPI_ERR_OUT_OF_RESOURCE; + } + + (void) elan_rails (state->elan_cap, rails); + + if (NULL == (state->elan_rail = (ELAN_RAIL **) + malloc (sizeof (ELAN_RAIL **) * (num_rails + 1)))) { + return OMPI_ERR_OUT_OF_RESOURCE; + } + state->elan_rail[num_rails] = NULL; + + alloc_mainsize = ELAN_ALIGNUP (state->main_size, state->elan_pagesize); + alloc_mainbase = (ADDR_ELAN) ((uintptr_t) state->main_base); + alloc_elansize = ELAN_ALIGNUP (state->elan_size, state->elan_pagesize); + alloc_elanbase = (ADDR_ELAN) ((uintptr_t) state->elan_base); + + for (i = 0; i < num_rails; i++) { + RAIL *rail; + ELAN_ESTATE *estate; + ELAN_EPRIVSTATE *priv_estate; + + /* Allocate the Main memory control structure for this rail */ + if (NULL == (rail = state->all_rails[i] = + (RAIL *) malloc (sizeof (RAIL)))) { + return OMPI_ERROR; + } + memset (rail, 0, sizeof (RAIL)); + + /* TODO: Debug the state of rails */ + + if (NULL == (rail->r_ctx = elan4_init (rails[i]))) { + return OMPI_ERROR; + } + + if (NULL == (rail->r_sdram = elan4_open_sdram (rails[i], + 0, + alloc_elansize))) { + return OMPI_ERROR; + } + + if (NULL == (rail->r_alloc = + elan4_createAllocator (state->alloc_mainsize, + rail->r_sdram, 0, + state->alloc_elansize))) { + return OMPI_ERROR; + } + + if (elan4_set_standard_mappings (rail->r_ctx) < 0 + || elan4_set_required_mappings (rail->r_ctx) < 0) { + return OMPI_ERROR; + } + + if (elan4_register_trap_handler (rail->r_ctx, + UTS_UNIMP_INSTR, UTS_TPROC, + elan_unimp_handler, NULL) < 0) { + return OMPI_ERROR; + } + + /* Now allocate the SDRAM Elan control structure for this rail */ + if (NULL == (estate = mp->all_estates[i] = + ALLOC_ELAN (rail, ELAN_ALIGN, + sizeof (ELAN_EPRIVSTATE)))) { + return OMPI_ERROR; + } + + priv_estate = (ELAN_EPRIVSTATE *) estate; + memset (priv_estate, 0, sizeof (ELAN_EPRIVSTATE)); + + /* Allocate a command port for non sten functions etc */ + if (NULL == (rail->r_cmdq = elan4_alloc_cmdq (rail->r_ctx, + rail->r_alloc, CQ_Size8K, + CQ_ModifyEnableBit | + CQ_WriteEnableBit | + CQ_WaitEventEnableBit | + CQ_SetEventEnableBit | + CQ_ThreadStartEnableBit, + NULL))) { + return OMPI_ERROR; + } + + /* Allocate a command port for thread rescheduling etc */ + if (NULL == (rail->r_ecmdq = elan4_alloc_cmdq (rail->r_ctx, + rail->r_alloc, CQ_Size8K, + CQ_EnableAllBits, + NULL))) { + return OMPI_ERROR; + } + + priv_estate->cport = + MAIN2ELAN (rail->r_ctx, rail->r_ecmdq->cmdq_mapping); + + /* Make the rail pointers 'external' */ + state->elan_rail[i] = (ELAN_RAIL *) rail; + + /* TODO: Find out how the following code work. */ + { + ELAN_SLEEP *es; + + /* Allocate and then free an ELAN_SLEEP DESC to + * keep heap in sync */ + es = _elan_allocSleepDesc (state, rail); + _elan_freeSleepDesc (state, rail, es); + } + + estate->alloc = rail->r_alloc; + estate->vp = state->vp; /* still ELAN_INVALID_PROCESS */ + estate->debugFlags = state->debugFlags; + + priv_estate->debugFd = 1; /* stdout */ + priv_estate->maxBackOff = max_backoff; + priv_estate->maxFastBackoff = max_fastbackoff; + priv_estate->pageSize = state->elan_pagesize; + + rail->r_estate = estate; + rail->r_railNo = rails[i]; + + { + RAILTABLE *rt; + + if (NULL == (rt = (RAILTABLE *) malloc (sizeof (RAILTABLE)))) { + return OMPI_ERROR; + } + memset (rt, 0, sizeof (RAILTABLE)); + + rt->rt_nrails = 1; + rt->rt_rail = 0; + rt->rt_railReal = rail; + rt->rt_allRails = &(state->all_rails[rail]); + /* Debug the state, see _elan_railtable_createSingle */ + + rail->r_railTable = rt; + } + + } /* for each rail */ + + free (rails); + + state->elan_ctx = state->rail[0]->rail_ctx; + state->elan_estate = (void *) state->all_estates[0]; + + /*_elan_eventInit(state);*/ + /*atexit(_elan_atExitCallBack); */ + + ompi_elan_attach_network (state); + + /* Set the rms_resourceId */ + if (rms_getprgid (getpid (), &state->elan_rmsid) < 0) { + state->elan_rmsid = -1; + } + + /* Now open ourselves to the network */ + for (i = 0; state->elan_rail[i]; i++) { + elan4_block_inputter (state->elan_rail[i]->rail_ctx, 0); + } +#endif + + return (OMPI_SUCCESS); +} + +int +ompi_mca_ptl_elan_setup (mca_ptl_elan_module_1_0_0_t * mp) +{ + /* TODO: + * a) init transport structure for all rails + * including STEN, QDMA and RDMA + * b) For each possible transport mechanism, allocate: + * send/recv descriptors; + * system buffer; + * event structure for transport control + * c) initialize STAT (including SYNC) structures. + */ + if (OMPI_SUCCESS != ompi_init_elan_qdma (mp)) { + return OMPI_ERROR; + } +#if MCA_PTL_ELAN_STAGE_TWO + if (OMPI_SUCCESS != ompi_init_elan_rdma (mp)) { + return OMPI_ERROR; + } + + if (OMPI_SUCCESS != ompi_init_elan_sten (mp)) { + return OMPI_ERROR; + } + + if (OMPI_SUCCESS != ompi_init_elan_stat (mp)) { + return OMPI_ERROR; + } +#endif + + return (OMPI_SUCCESS); +} + +int +ompi_mca_ptl_elan_fin (mca_ptl_elan_module_1_0_0_t * mp) +{ + return (OMPI_SUCCESS); +} + +/* Attach to the network: + * a) First add myself into the capability + * b) For each rail, + * Block the inputter for any incoming traffic + * attach to the device + * Find out the location + * Fill out the vp in main and elan structures. + * create a cookiePool for threading control + * c) Allocate a cookiePool for export Oth Rail + * d) Find out total vpids, localVpids, localId and number of locals + * e) Allocate a bcastVp (not used for now) and a railTable + */ +static int +ompi_elan_attach_network (mca_ptl_elan_state_t * state) +{ +#if ELAN_COMP + int i, vp, *vps, num_rails; + ELAN_LOCATION loc; + ELAN_CAPABILITY *cap = state->elan_cap; + + num_rails = state->elan_nrails; + + if (state->attached) { + /* already successfully attached */ + return; + } else { + state->attached = 1; + } + + for (i = 0; i < num_rails; i++) { + RAIL *rail = state->all_rails[i]; + ELAN_LOCATION loc; + + /* Add all virtual process from 0 to (nvp-1) */ + if (elan4_add_p2pvp (rail->r_ctx, 0, cap) < 0) { + /* FIXME: exception code */ + return OMPI_ERROR; + } + + /* block the inputter until we are really ready */ + elan4_block_inputter (rail->r_ctx, 1); + + if (elan4_attach (rail->r_ctx, cap)) { + /* FIXME: exception code */ + return OMPI_ERROR; + } + + /* NB: We should have same vp for all capabilities */ + loc.loc_node = ((CTX_ELAN *) rail->r_ctx)->ctx_position.pos_nodeid + - cap->cap_lownode; + loc.loc_context = cap->cap_mycontext - cap->cap_lowcontext; + + /* TODO: debug code for loc */ + if (cap->cap_type == ELAN_CAP_TYPE_HWTEST) + state->elan_vp = 0; + else + state->elan_vp = elan_location2vp (loc, state->elan_cap); + + /* Initialise the Elan version of this data */ + ((ELAN_ESTATE *) rail->r_estate)->vp = state->elan_vp; + + /* Allocate a cookie pool for the thread processor + * and copy to sdram */ + rail->r_cpool = + elan4_allocCookiePool (rail->r_ctx, state->elan_vp); + ((ELAN_EPRIVSTATE *) rail->r_estate)->tcookie = + rail->r_cpool->cp_cookie; + } + + /* Allocate a cookie pool for the exported API (0th rail) */ + state->cpool = elan4_allocCookiePool (state->elan_ctx, state->elan_vp); + + loc = elan_vp2location (state->elan_vp, state->elan_cap); + + /* update THREAD elan_dbg info of debugfile */ + if (state->elan_debugfile && fileno (state->elan_debugfile) != -1) { + for (i = 0; i < num_rails; i++) { + CTX_ELAN *ctx = state->elan_rail[i]->rail_ctx; + + /* Convert FILE * stream to fd for THRD output */ + ((ELAN_ESTATE *) state->all_estates[i])->debugFd = + fileno (state->elan_debugfile); + + /* Also re-direct libelan4 output to this file */ + elan4_set_debugfd (ctx, fileno (state->elan_debugfile)); + } + } + + /* Determine the number of processes described by the capability */ + state->elan_nvp = elan_nvps (state->elan_cap); + + /* Start allocating bcastVps a little way after the pt2pt ones */ + state->bcastVp = state->baseBcastVp = state->elan_nvp + 16; + if (state->elan_vp >= elan_state->elan_nvp) { + /* FIXME: exception code */ + return OMPI_ERROR; + } + + /* FIXME: fix the following rail table stuff */ + state->railTable = _elan_railtable_create (elan_state); + state->rail = _elan_railtable_getRailSelf (elan_state, + elan_state->railTable); + + if (NULL == (state->elan_localIds = (int *) + MALLOC (elan_state->nvp * sizeof (int)))) { + /* FIXME: exception code */ + return OMPI_ERROR; + } + + /* Set all to non local initially */ + for (vp = 0; vp < state->elan_nvp; vp++) + state->elan_localvps[vp] = -1; + + /* Stash our own process location */ + state->myloc = elan_vp2location (state->elan_vp, state->elan_cap); + state->elan_maxlocals = elan_maxlocal (elan_state->cap); + state->elan_numlocals = elan_nlocal (state->myloc.loc_node, + state->elan_cap); + + /* Allocate more than we need to keep the heap in sync */ + if (NULL == (vps = (int *) malloc (sizeof (int) * elan_state->nvp))) { + /* FIXME: exception code */ + return OMPI_ERROR; + } + + /* Fill out the local vp array */ + elan_localvps (state->myloc.loc_node, + state->elan_cap, vps, state->elan_numlocals); + + for (i = 0; i < state->elan_numlocals; i++) { + int localvp = vps[i]; + + /* This is a local process */ + state->elan_localvps[localvp] = i; + + if (localvp == state->elan_vp) { + state->elan_localid = i; + } + } + + /* TODO: Debug code here for elan_state */ + + /* + * We need to create one of these maps in each rail estate ctx too + */ + for (i = 0; i < num_rails; i++) { + RAIL *rail = state->all_rails[i]; + ELAN_ESTATE *estate = state->all_estates[i]; + int *localIdsElan; + + /* Allocate a copy of this in the SDRAM of each rail */ + if (0 == (localIdsElan = ALLOC_ELAN (rail, ELAN_ALIGN, + elan_state->nvp * + sizeof (int)))) { + /* FIXME: exception code */ + return OMPI_ERROR; + } + + estate->localIds = SDRAM2ELAN (rail->r_ctx, localIdsElan); + + for (vp = 0; vp < state->elan_nvp; vp++) { + localIdsElan[vp] = state->elan_localvps[vp]; + } + + estate->nLocalIds = state->elan_numlocals; + estate->localId = state->elan_localid; + estate->nvp = state->elan_nvp; + + /* Update thread info on primary rail too */ + ((ELAN_EPRIVSTATE *) estate)->rail = state->elan_rail; + } + + /* Done with vps array now */ + free (vps); +#endif + + return (OMPI_SUCCESS); +} + +static int +ompi_init_elan_qdma (mca_ptl_elan_module_1_0_0_t * mp) +{ + return (OMPI_SUCCESS); +} + +static int +ompi_init_elan_rdma (mca_ptl_elan_module_1_0_0_t * mp) +{ + return (OMPI_SUCCESS); +} + +static int +ompi_init_elan_sten (mca_ptl_elan_module_1_0_0_t * mp) +{ + return (OMPI_SUCCESS); +} + +static int +ompi_init_elan_stat (mca_ptl_elan_module_1_0_0_t * mp) +{ + return (OMPI_SUCCESS); +} diff --git a/src/mca/ptl/elan/src/ptl_elan_module.c b/src/mca/ptl/elan/src/ptl_elan_module.c new file mode 100644 index 0000000000..60c7f0e866 --- /dev/null +++ b/src/mca/ptl/elan/src/ptl_elan_module.c @@ -0,0 +1,280 @@ +/* + * $HEADER$ + */ +#include +#include +#include +#include +#include +#include +#include +#include + +#include "constants.h" +#include "event/event.h" +#include "util/if.h" +#include "util/argv.h" +#include "util/output.h" +#include "mca/pml/pml.h" +#include "mca/ptl/ptl.h" +#include "mca/ptl/base/ptl_base_sendreq.h" +#include "mca/base/mca_base_param.h" +#include "mca/base/mca_base_module_exchange.h" +#include "ptl_elan.h" +#include "ptl_elan_proc.h" +#include "ptl_elan_frag.h" +#include "ptl_elan_req.h" +#include "ptl_elan_priv.h" + +mca_ptl_elan_module_1_0_0_t mca_ptl_elan_module = { + { + /* Base module information about itself */ + { + /* Indicate that we are a pml v1.0.0 module + * (which also implies a specific MCA version) */ + MCA_PTL_BASE_VERSION_1_0_0, + "elan", /* MCA module name */ + 1, /* MCA module major version */ + 0, /* MCA module minor version */ + 0, /* MCA module release version */ + mca_ptl_elan_module_open, /* module open */ + mca_ptl_elan_module_close /* module close */ + }, + + /* Next the MCA v1.0.0 module meta data */ + { + /* Whether the module is checkpointable or not */ + false + }, + + /* The management related interfaces */ + mca_ptl_elan_module_init, + mca_ptl_elan_module_control, + mca_ptl_elan_module_progress + } +}; + +static mca_ptl_elan_module_1_0_0_t *mp = &mca_ptl_elan_module; + +/** + * utility routines for parameter registration + */ +static inline char * +mca_ptl_elan_param_register_string (const char *param_name, + const char *default_value) +{ + int id; + char *param_value; + + id = mca_base_param_register_string ("ptl", "elan", param_name, NULL, + default_value); + mca_base_param_lookup_string (id, ¶m_value); + return param_value; +} + +static inline int +mca_ptl_elan_param_register_int (const char *param_name, + int default_value) +{ + int id; + int param_value; + + param_value = default_value; + id = mca_base_param_register_int ("ptl", "elan", param_name, NULL, + default_value); + mca_base_param_lookup_int (id, ¶m_value); + return param_value; +} + +/* + * Called by MCA framework to open the module, registers + * module parameters. + */ + +int +mca_ptl_elan_module_open (void) +{ + /* register super module parameters */ + mca_ptl_elan.super.ptl_exclusivity = + mca_ptl_elan_param_register_int ("exclusivity", 0); + mca_ptl_elan.super.ptl_first_frag_size = + mca_ptl_elan_param_register_int ("first_frag_size", 1984/*magic*/); + mca_ptl_elan.super.ptl_min_frag_size = + mca_ptl_elan_param_register_int ("min_frag_size", 320); + mca_ptl_elan.super.ptl_max_frag_size = + mca_ptl_elan_param_register_int ("max_frag_size", -1); + +#if 0 /* These parameters not very useful */ + /* register ELAN module parameters */ + mca_ptl_elan_module.elan_free_list_num = + mca_ptl_elan_param_register_int ("free_list_num", 64); + mca_ptl_elan_module.elan_free_list_max = + mca_ptl_elan_param_register_int ("free_list_max", -1); + mca_ptl_elan_module.elan_free_list_inc = + mca_ptl_elan_param_register_int ("free_list_inc", 64); + + /* initialize state */ + mca_ptl_elan_module.elan_state = NULL; + mca_ptl_elan_module.elan_ptls = NULL; + mca_ptl_elan_module.elan_num_ptls = 0; + + /* initialize objects */ + OBJ_CONSTRUCT (&mca_ptl_elan_module.elan_lock, ompi_mutex_t); + OBJ_CONSTRUCT (&mca_ptl_elan_module.elan_procs, ompi_list_t); +#endif + + return OMPI_SUCCESS; +} + + +int +mca_ptl_elan_module_close (void) +{ + if (mca_ptl_elan_module.elan_reqs_free.fl_num_allocated != + mca_ptl_elan_module.elan_reqs_free.super.ompi_list_length) { + ompi_output (0, "elan requests: %d allocated %d returned\n", + mca_ptl_elan_module.elan_reqs_free.fl_num_allocated, + mca_ptl_elan_module.elan_reqs_free.super.ompi_list_length); + } + + if (mca_ptl_elan_module.elan_events_free.fl_num_allocated != + mca_ptl_elan_module.elan_events_free.super.ompi_list_length) { + } + + if (NULL != mca_ptl_elan_module.elan_ptls) { + int i; + for (i = mca_ptl_elan_module.elan_num_ptls; i > 0; i--) { + free (mca_ptl_elan_module.elan_ptls[i - 1]); + } + free (mca_ptl_elan_module.elan_ptls); + } + + /* Free the empty list holders */ + OBJ_DESTRUCT (&(mca_ptl_elan_module.elan_reqs)); + OBJ_DESTRUCT (&(mca_ptl_elan_module.elan_prog_events)); + OBJ_DESTRUCT (&(mca_ptl_elan_module.elan_comp_events)); + OBJ_DESTRUCT (&(mca_ptl_elan_module.elan_procs)); + OBJ_DESTRUCT (&(mca_ptl_elan_module.elan_lock)); + + /* TODO: check if this is needed. + * Free the free lists */ + OBJ_DESTRUCT (&(mca_ptl_elan_module.elan_reqs_free)); + OBJ_DESTRUCT (&(mca_ptl_elan_module.elan_events_free)); + + return OMPI_SUCCESS; +} + +/* + * ELAN module initialization: + * (1) elan4_init() to initialize the basic support and mapping etc. + * (2) set up STEN, RDMA and QDMA structures. + * (3) register the list of PTL parameters with the MCA + */ +mca_ptl_t ** +mca_ptl_elan_module_init (int *num_ptls, + bool * allow_multi_user_threads, + bool * have_hidden_threads) +{ + mca_ptl_t **ptls; + int rc; + + *num_ptls = 0; + *allow_multi_user_threads = true; + *have_hidden_threads = OMPI_HAVE_THREADS; + + if ((rc = ompi_event_init ()) != OMPI_SUCCESS) { + ompi_output (0, + "mca_ptl_elan_module_init: " + "unable to initialize event dispatch thread: %d\n", + rc); + return NULL; + } + +#if ELAN_COMP + /* initialize free lists */ + ompi_free_list_init (&(mca_ptl_elan_module.elan_reqs_free), + sizeof (mca_ptl_elan_send_request_t), + OBJ_CLASS (mca_ptl_elan_send_request_t), + mca_ptl_elan_module.elan_free_list_num, + mca_ptl_elan_module.elan_free_list_max, + mca_ptl_elan_module.elan_free_list_inc, NULL); + + /* use default allocator */ + ompi_free_list_init (&mca_ptl_elan_module.elan_events_free, + sizeof (mca_ptl_elan_send_frag_t), + OBJ_CLASS (mca_ptl_elan_send_frag_t), + mca_ptl_elan_module.elan_free_list_num, + mca_ptl_elan_module.elan_free_list_max, + mca_ptl_elan_module.elan_free_list_inc, NULL); +#endif + + /* open basic elan device */ + if (OMPI_SUCCESS != ompi_mca_ptl_elan_init(&mca_ptl_elan_module)) { + return NULL; + } + + /* setup communication infrastructure */ + if (OMPI_SUCCESS != ompi_mca_ptl_elan_setup (&mca_ptl_elan_module)) { + return NULL; + } + + /* + * (mca_ptl_elan_module_exchange () != OMPI_SUCCESS) + * + * No need to publish parameters with the MCA framework + * + * This is called only by those processes who have elan. + * So it does not qualify to be a global call. + * Since the processes has elan support can already communicate + * over elan, there is no need for a oob_based exchange. + */ + + ptls = (mca_ptl_elan_t **) malloc (mca_ptl_elan_module.elan_num_ptls * + sizeof (mca_ptl_t *)); + + if (NULL == ptls) { + return NULL; + } + + memcpy (ptls, mca_ptl_elan_module.elan_ptls, + mca_ptl_elan_module.elan_num_ptls * sizeof (mca_ptl_elan_t *)); + *num_ptls = mca_ptl_elan_module.elan_num_ptls; + + return ptls; +} + +/* + * FIXME: to support ELAN module control + */ + +int +mca_ptl_elan_module_control (int param, + void *value, + size_t size) +{ + switch (param) { + case MCA_PTL_ENABLE: + if (*(int *) value) { + ompi_event_add (&mca_ptl_elan_module.elan_recv_event, 0); + } else { + ompi_event_del (&mca_ptl_elan_module.elan_recv_event); + } + break; + default: + break; + } + return OMPI_SUCCESS; +} + + +/* + * FIXME: to support event-based module progress. + */ + +int +mca_ptl_elan_module_progress (mca_ptl_tstamp_t tstamp) +{ + ompi_event_loop (OMPI_EVLOOP_ONCE); + return OMPI_SUCCESS; +} + diff --git a/src/mca/ptl/elan/src/ptl_elan_peer.h b/src/mca/ptl/elan/src/ptl_elan_peer.h new file mode 100644 index 0000000000..3c4f96fb99 --- /dev/null +++ b/src/mca/ptl/elan/src/ptl_elan_peer.h @@ -0,0 +1,47 @@ +/* + * $HEADER$ + */ +/** + * @file + */ +#ifndef MCA_PTL_ELAN_PEER_H +#define MCA_PTL_ELAN_PEER_H + +#include +#include +#include +#include "class/ompi_list.h" +#include "event/event.h" +#include "mca/pml/pml.h" +#include "mca/ptl/ptl.h" +#include "ptl_elan_frag.h" + +typedef enum { + MCA_PTL_ELAN_CLOSED, + MCA_PTL_ELAN_CONNECTED, + MCA_PTL_ELAN_FAILED, + NUM_MCA_PTL_ELAN_STAT +} mca_ptl_elan_status_t; + +/** + * An abstraction that represents a connection to a peer process. + * Peers are always connected unless they are in different LAN or died. + */ +struct mca_ptl_elan_peer_t { + ompi_list_item_t super; + + struct mca_ptl_elan_t* peer_ptl; + struct mca_ptl_elan_proc_t* peer_proc; + + int resending; /* A resending stage, no more new dma's */ + int num_resend; /* How many times I have retried */ + double open_time; + double close_time; + double known_alive_time; +}; +typedef struct mca_ptl_elan_peer_t mca_ptl_elan_peer_t; + +extern ompi_class_t mca_ptl_elan_peer_t_class; + +#endif + diff --git a/src/mca/ptl/elan/src/ptl_elan_priv.c b/src/mca/ptl/elan/src/ptl_elan_priv.c new file mode 100644 index 0000000000..e69de29bb2 diff --git a/src/mca/ptl/elan/src/ptl_elan_priv.h b/src/mca/ptl/elan/src/ptl_elan_priv.h new file mode 100644 index 0000000000..b94571a11e --- /dev/null +++ b/src/mca/ptl/elan/src/ptl_elan_priv.h @@ -0,0 +1,45 @@ +/* + * $HEADER$ + */ +#ifndef MCA_PTL_ELAN_PRIV_H +#define MCA_PTL_ELAN_PRIV_H + +#include +#include +#include +#include +#include +#include +#include +#include + +#include "constants.h" +#include "event/event.h" +#include "util/if.h" +#include "util/argv.h" +#include "util/output.h" +#include "mca/pml/pml.h" +#include "mca/ptl/ptl.h" +#include "mca/ptl/base/ptl_base_sendreq.h" +#include "mca/base/mca_base_param.h" +#include "mca/base/mca_base_module_exchange.h" +#include "ptl_elan.h" +#include "ptl_elan_proc.h" +#include "ptl_elan_frag.h" +#include "ptl_elan_req.h" + +/*#include "elan/init.h"*/ +/*#include "elan/sys/misc_sys.h"*/ +/*#include "elan/sys/init_sys.h"*/ +/*#include "elan/elan.h"*/ + +/* Initialization and finalization routines */ +int ompi_mca_ptl_elan_init (mca_ptl_elan_module_1_0_0_t *mp); +int ompi_mca_ptl_elan_setup (mca_ptl_elan_module_1_0_0_t *mp); +int ompi_mca_ptl_elan_fin (mca_ptl_elan_module_1_0_0_t *mp); + +/* communication prototypes */ + +/* control, synchronization and state prototypes */ + +#endif diff --git a/src/mca/ptl/elan/src/ptl_elan_proc.c b/src/mca/ptl/elan/src/ptl_elan_proc.c new file mode 100644 index 0000000000..1e405ea621 --- /dev/null +++ b/src/mca/ptl/elan/src/ptl_elan_proc.c @@ -0,0 +1,110 @@ +/* + * $HEADER$ + */ +#include +#include "atomic.h" +#include "class/ompi_hash_table.h" +#include "mca/base/mca_base_module_exchange.h" +#include "ptl_elan.h" +#include "ptl_elan_peer.h" +#include "ptl_elan_proc.h" + + +static void mca_ptl_elan_proc_construct (mca_ptl_elan_proc_t * proc); +static void mca_ptl_elan_proc_destruct (mca_ptl_elan_proc_t * proc); +static mca_ptl_elan_proc_t *mca_ptl_elan_proc_lookup_ompi (ompi_proc_t * + ompi_proc); + +ompi_class_t mca_ptl_elan_proc_t_class = { + "mca_ptl_elan_proc_t", + OBJ_CLASS (ompi_list_item_t), + (ompi_construct_t) mca_ptl_elan_proc_construct, + (ompi_destruct_t) mca_ptl_elan_proc_destruct +}; + + +/** + * Initialize elan proc instance + */ + +void mca_ptl_elan_proc_construct (mca_ptl_elan_proc_t * proc) +{ + return; +} + + +/* + * Cleanup elan proc instance + */ + +void mca_ptl_elan_proc_destruct (mca_ptl_elan_proc_t * proc) +{ + return; +} + + +/* + * Create a ELAN process structure. There is a one-to-one correspondence + * between a ompi_proc_t and a mca_ptl_elan_proc_t instance. + * We cache additional data (specifically the list + * of mca_ptl_elan_peer_t instances, and publiched + * addresses) associated w/ a given destination on this datastructure. + */ + +mca_ptl_elan_proc_t *mca_ptl_elan_proc_create (ompi_proc_t * ompi_proc) +{ + return NULL; +} + +/* + * Look for an existing ELAN process instances based on the associated + * ompi_proc_t instance. + */ +static mca_ptl_elan_proc_t * +mca_ptl_elan_proc_lookup_ompi (ompi_proc_t *ompi_proc) +{ + return NULL; +} + + +/* + * Look for an existing ELAN process instance based on the globally unique + * process identifier. + */ +mca_ptl_elan_proc_t *mca_ptl_elan_proc_lookup (void *guid, size_t size) +{ + return NULL; +} + + +/* + * Note that this routine must be called with the lock on the process already + * held. Insert a ptl instance into the proc array and assign it an address. + */ +int mca_ptl_elan_proc_insert (mca_ptl_elan_proc_t * ptl_proc, + mca_ptl_elan_peer_t * ptl_peer) +{ + return OMPI_SUCCESS; +} + +/* + * Remove a peer from the proc array and indicate the address is + * no longer in use. + */ + +int mca_ptl_elan_proc_remove (mca_ptl_elan_proc_t * ptl_proc, + mca_ptl_elan_peer_t * ptl_peer) +{ + return OMPI_SUCCESS; +} + + +/* + * loop through all available PTLs for one matching the source address + * of the request. + */ +bool mca_ptl_elan_proc_accept (mca_ptl_elan_proc_t * ptl_proc, + struct sockaddr_in * addr, int sd) +{ + return false; +} diff --git a/src/mca/ptl/elan/src/ptl_elan_proc.h b/src/mca/ptl/elan/src/ptl_elan_proc.h new file mode 100644 index 0000000000..e478b66201 --- /dev/null +++ b/src/mca/ptl/elan/src/ptl_elan_proc.h @@ -0,0 +1,50 @@ +/* + * $HEADER$ + */ +/** + * @file + */ +#ifndef MCA_PTL_ELAN_PROC_H +#define MCA_PTL_ELAN_PROC_H + +#include +#include +#include +#include "class/ompi_object.h" +#include "proc/proc.h" +#include "ptl_elan.h" +#include "ptl_elan_peer.h" + +extern ompi_class_t mca_ptl_elan_proc_t_class; + +/** + * Represents the state of a remote process. Also cache an instance + * of mca_ptl_base_peer_t for each + * PTL instance that attempts to open a connection to the process. + */ +struct mca_ptl_elan_proc_t { + ompi_list_item_t super; /**< allow proc to be placed on a list */ + ompi_proc_t *proc_ompi; /**< pointer to corresponding ompi_proc_t */ + struct mca_ptl_base_peer_t **proc_peers; /**< array of peers */ + size_t proc_peer_count; /**< number of peers */ + void* proc_uuid; /**< univeral unique identifier the process */ + size_t univ_size; /**< size of the universe */ + ompi_mutex_t proc_lock; /**< lock to for proc state */ +}; +typedef struct mca_ptl_elan_proc_t mca_ptl_elan_proc_t; + +mca_ptl_elan_proc_t* mca_ptl_elan_proc_create(ompi_proc_t* ompi_proc); +mca_ptl_elan_proc_t* mca_ptl_elan_proc_lookup(void *guid, size_t size); + +static inline mca_ptl_elan_proc_t* mca_ptl_elan_proc_local(void) +{ + return NULL; +} + +int mca_ptl_elan_proc_insert(mca_ptl_elan_proc_t *, mca_ptl_elan_peer_t *); +int mca_ptl_elan_proc_remove(mca_ptl_elan_proc_t *, mca_ptl_elan_peer_t *); +bool mca_ptl_elan_proc_accept(mca_ptl_elan_proc_t *, + struct sockaddr_in *, int sd); + + +#endif diff --git a/src/mca/ptl/elan/src/ptl_elan_req.h b/src/mca/ptl/elan/src/ptl_elan_req.h new file mode 100644 index 0000000000..ab28df3a46 --- /dev/null +++ b/src/mca/ptl/elan/src/ptl_elan_req.h @@ -0,0 +1,35 @@ +/* + * $HEADER$ + */ +/** + * @file + */ + +#ifndef MCA_PTL_ELAN_REQ_H +#define MCA_PTL_ELAN_REQ_H + +#include +#include +#include +#include "ompi_config.h" +#include "mca/ptl/base/ptl_base_sendreq.h" +#include "mca/ptl/base/ptl_base_recvreq.h" +#include "ptl_elan_frag.h" + +/* Again, what is this! */ +/* extern ompi_class_t mca_ptl_elan_send_request_t_class;*/ + +/** + * ELAN send request derived type. The send request contains both the + * base send request, and space for the first ELAN send fragment descriptor. + * This avoids the overhead of a second allocation for the initial send + * fragment on every send request. + */ +struct mca_ptl_elan_send_request_t { + mca_ptl_base_send_request_t super; + mca_ptl_elan_send_frag_t req_frag; /* first fragment */ +}; +typedef struct mca_ptl_elan_send_request_t mca_ptl_elan_send_request_t; + +#endif +