ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

add more files

Просмотр исходного кода 
This commit was SVN r1338.
Этот коммит содержится в:
Weikuan Yu 2004-06-16 22:59:01 +00:00
родитель 3f42ad3475
Коммит 0b1e67b4bf
7 изменённых файлов: 210 добавлений и 224 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

44
src/mca/ptl/elan/src/ptl_elan.c
Просмотреть файл

@ -35,8 +35,6 @@ mca_ptl_elan_t mca_ptl_elan = {
mca_ptl_elan_add_proc,
mca_ptl_elan_del_proc,
mca_ptl_elan_finalize,
mca_ptl_elan_isend,
mca_ptl_elan_irecv,
mca_ptl_elan_put,
mca_ptl_elan_get,
mca_ptl_elan_matched,
@ -46,15 +44,18 @@ mca_ptl_elan_t mca_ptl_elan = {
};
int mca_ptl_elan_add_proc (struct mca_ptl_t *ptl,
struct ompi_proc_t *ompi_proc,
struct mca_ptl_base_peer_t **peer_ret)
size_t nprocs,
struct ompi_proc_t **ompi_proc,
struct mca_ptl_base_peer_t **peer_ret,
ompi_bitmap_t* reachable)
{
return OMPI_SUCCESS;
}
int mca_ptl_elan_del_proc (struct mca_ptl_t *ptl, struct ompi_proc_t *proc,
struct mca_ptl_base_peer_t *ptl_peer)
int mca_ptl_elan_del_proc (struct mca_ptl_t *ptl,
size_t nprocs,
struct ompi_proc_t ** procs,
struct mca_ptl_base_peer_t **ptl_peer)
{
return OMPI_SUCCESS;
}
@ -93,16 +94,29 @@ void mca_ptl_elan_send_frag_return (struct mca_ptl_t *ptl,
}
/*
* Initiate a send. If this is the first fragment, use the fragment
* descriptor allocated with the send requests, otherwise obtain
* one from the free list. Initialize the fragment and foward
* on to the peer.
* Initiate a put operation.
*/
int mca_ptl_elan_isend (struct mca_ptl_t *ptl,
struct mca_ptl_base_peer_t *ptl_peer,
struct mca_ptl_base_send_request_t *sendreq,
size_t offset, size_t * size, int flags)
int mca_ptl_elan_put (struct mca_ptl_t* ptl,
struct mca_ptl_base_peer_t* ptl_base_peer,
struct mca_ptl_base_send_request_t* request,
size_t offset,
size_t size,
int flags)
{
return OMPI_SUCCESS;
}
/*
* Initiate a get.
*/
int mca_ptl_elan_get (struct mca_ptl_t* ptl,
struct mca_ptl_base_peer_t* ptl_base_peer,
struct mca_ptl_base_recv_request_t* request,
size_t offset,
size_t size,
int flags)
{
return OMPI_SUCCESS;
}

123
src/mca/ptl/elan/src/ptl_elan.h
Просмотреть файл

@ -10,16 +10,23 @@
#include <sys/types.h>
#include <sys/socket.h>
#include <netinet/in.h>
#include "class/ompi_free_list.h"
#include "mem/free_list.h"
#include "event/event.h"
#include "mca/pml/pml.h"
#include "mca/ptl/ptl.h"
/*#include "elan/sys/init_sys.h"*/
/*#include "elan/elan.h"*/
#define MCA_PTL_ELAN_STATISTICS 0
/*#define __ELAN__*/
/*#define __elan4__*/
/*#include "elan/sys/misc_sys.h"*/
/*#include "elan/sys/init_sys.h"*/
#include "elan/elan.h"
#include "elan/init.h"
#include "rms/rmscall.h"
#include "elan4/library.h"
/**
* ELAN PTL module.
*/
@ -32,15 +39,13 @@ struct mca_ptl_elan_state_t {
uint64_t elan_debug; /**< elan debug tracing output */
uint64_t elan_traced; /**< elan TRACE output */
uint64_t elan_flags;
FILE elan_debugfile;
FILE *elan_debugfile; /* Debug output file handle */
int elan_signalnum;
#ifdef ENABLE_ELAN_MEMORY_ALLOCATOR
size_t main_size; /**< size of Main memory allocator heap */
size_t elan_size; /**< size of Elan memory allocator heap */
void *main_base; /**< Main memory allocator heap base */
void *elan_base; /**< Elan memory allocator heap base */
#endif
/* other state parameters */
@ -62,16 +67,15 @@ struct mca_ptl_elan_state_t {
* A simple type casting of ELAN_ESTATE can bring
* the complete structure of the ELAN_EPRIVSATE.
*/
ELAN_LOCATION elan_myloc;
void *elan_cap; /**< job capability */
void *elan_estate; /**< Elan state of the 0th rail */
#ifdef ELAN_COMP
ELAN_CTX *elan_ctx; /**< Elan ctx of the 0th rail */
ELAN_RAIL **elan_rail; /**< Rail control struct for all rails */
RAIL **all_rails; /**< all rails */
struct ompi_elan_rail_t ** all_rails; /**< all rails */
ELAN4_COOKIEPOOL *elan_cpool;
ELAN_ESTATE **all_estates; /**< elan (priv)states of all rails */
#endif
ompi_mutex_t state_lock; /**< lock for elan state */
};
typedef struct mca_ptl_elan_state_t mca_ptl_elan_state_t;
@ -81,11 +85,9 @@ struct mca_ptl_elan_module_1_0_0_t {
/* These parameters does not provided good freedom,
* It does not hurt to skip them */
#if 1
int elan_free_list_num; /**< initial size of free lists */
int elan_free_list_max; /**< maximum size of free lists */
int elan_free_list_inc; /**< # to alloc when growing lists */
#endif
/*
* We create our own simplified structure for managing elan state
@ -100,7 +102,7 @@ struct mca_ptl_elan_module_1_0_0_t {
ompi_list_t elan_prog_events; /**< events in progress */
ompi_list_t elan_comp_events; /**< events completed, but to reclaim */
ompi_free_list_t elan_events_free; /**< free events */
ompi_free_list_t elan_events_free;/**< free events */
ompi_free_list_t elan_reqs_free; /**< all elan requests */
ompi_event_t elan_send_event; /**< event structure for sends */
@ -163,10 +165,9 @@ struct mca_ptl_elan_t {
/**< The following are elan-related control structures */
#ifdef ELAN_COMP
ELAN_RAIL *elan_rail; /**< Pointer to this Rail */
ELAN_CTX *elan_ctx; /**< Elan ctx of this rail */
#endif
int ptl_ni_local; /**< PTL NI local rank */
int ptl_ni_total; /**< PTL NI total */
@ -220,9 +221,12 @@ extern int mca_ptl_elan_finalize (struct mca_ptl_t *ptl);
* @param peer (OUT)
* @return OMPI_SUCCESS or error status on failure.
*/
extern int mca_ptl_elan_add_proc (struct mca_ptl_t *ptl,
struct ompi_proc_t *proc,
struct mca_ptl_base_peer_t **peer);
extern int
mca_ptl_elan_add_proc (struct mca_ptl_t *ptl,
size_t nprocs,
struct ompi_proc_t **ompi_proc,
struct mca_ptl_base_peer_t **peer_ret,
ompi_bitmap_t* reachable);
/**
* PML->PTL notification of change in the process list.
@ -232,9 +236,11 @@ extern int mca_ptl_elan_add_proc (struct mca_ptl_t *ptl,
* @param peer (IN) Peer addressing information.
* @return Status indicating if cleanup was successful
*/
extern int mca_ptl_elan_del_proc (struct mca_ptl_t *ptl,
struct ompi_proc_t *procs,
struct mca_ptl_base_peer_t *addr);
extern int
mca_ptl_elan_del_proc (struct mca_ptl_t *ptl,
size_t nprocs,
struct ompi_proc_t ** procs,
struct mca_ptl_base_peer_t **ptl_peer);
/**
* PML->PTL Allocate a send request from the PTL modules free list.
@ -243,8 +249,9 @@ extern int mca_ptl_elan_del_proc (struct mca_ptl_t *ptl,
* @param request (OUT) Pointer to allocated request.
* @return Status indicating if allocation was successful.
*/
extern int mca_ptl_elan_req_alloc (struct mca_ptl_t *ptl,
struct mca_ptl_base_send_request_t **);
extern int
mca_ptl_elan_req_alloc (struct mca_ptl_t *ptl,
struct mca_ptl_base_send_request_t **);
/**
* PML->PTL Return a send request to the PTL modules free list.
@ -264,48 +271,6 @@ extern void mca_ptl_elan_req_return (struct mca_ptl_t *ptl,
extern void mca_ptl_elan_matched (struct mca_ptl_t *ptl,
struct mca_ptl_base_recv_frag_t *frag);
/**
* PML->PTL Initiate an isend of the specified size.
*
* @param ptl (IN) PTL instance
* @param ptl_base_peer (IN) PTL peer addressing
* @param send_request (IN/OUT) Send request (allocated by PML via
* mca_ptl_base_request_alloc_fn_t)
* @param size (IN)
* Number of bytes PML is requesting PTL to deliver
* @param flags (IN)
* Flags that should be passed to the peer via the message header.
* @param request (OUT)
* OMPI_SUCCESS if the PTL was able to queue one or more fragments
*/
extern int mca_ptl_elan_isend (struct mca_ptl_t *ptl,
struct mca_ptl_base_peer_t *ptl_peer,
struct mca_ptl_base_send_request_t *,
size_t offset,
size_t * size,
int flags);
/**
* PML->PTL Initiate an irecv of the specified size.
*
* @param ptl (IN) PTL instance
* @param ptl_base_peer (IN) PTL peer addressing
* @param send_request (IN/OUT) Send request (allocated by PML via
* mca_ptl_base_request_alloc_fn_t)
* @param size (IN)
* Number of bytes PML is requesting PTL to deliver
* @param flags (IN)
* Flags that should be passed to the peer via the message header.
* @param request (OUT)
* OMPI_SUCCESS if the PTL was able to queue one or more fragments
*/
extern int mca_ptl_elan_irecv (struct mca_ptl_t *ptl,
struct mca_ptl_base_peer_t *ptl_peer,
struct mca_ptl_base_send_request_t *,
size_t offset,
size_t * size,
int flags);
/**
* PML->PTL Initiate a put of the specified size.
*
@ -320,12 +285,13 @@ extern int mca_ptl_elan_irecv (struct mca_ptl_t *ptl,
* @param request (OUT)
* OMPI_SUCCESS if the PTL was able to queue one or more fragments
*/
extern int mca_ptl_elan_put (struct mca_ptl_t *ptl,
struct mca_ptl_base_peer_t *ptl_peer,
struct mca_ptl_base_send_request_t *,
size_t offset,
size_t * size,
int flags);
extern int
mca_ptl_elan_put (struct mca_ptl_t* ptl,
struct mca_ptl_base_peer_t* ptl_base_peer,
struct mca_ptl_base_send_request_t* request,
size_t offset,
size_t size,
int flags);
/**
* PML->PTL Initiate a get of the specified size.
@ -341,12 +307,13 @@ extern int mca_ptl_elan_put (struct mca_ptl_t *ptl,
* @param request (OUT)
* OMPI_SUCCESS if the PTL was able to queue one or more fragments
*/
extern int mca_ptl_elan_get (struct mca_ptl_t *ptl,
struct mca_ptl_base_peer_t *ptl_peer,
struct mca_ptl_base_send_request_t *,
size_t offset,
size_t * size,
int flags);
extern int
mca_ptl_elan_get (struct mca_ptl_t* ptl,
struct mca_ptl_base_peer_t* ptl_base_peer,
struct mca_ptl_base_recv_request_t* request,
size_t offset,
size_t size,
int flags);
/**
* Return a recv fragment to the modules free list.

1
src/mca/ptl/elan/src/ptl_elan_frag.h
Просмотреть файл

@ -133,7 +133,6 @@ mca_ptl_elan_send_frag_init_ack (mca_ptl_elan_send_frag_t * ack,
static inline void
mca_ptl_elan_recv_frag_matched (mca_ptl_elan_recv_frag_t * frag)
{
mca_ptl_base_recv_request_t *request = frag->super.frag_request;
return;
}

207
src/mca/ptl/elan/src/ptl_elan_init.c
Просмотреть файл

@ -1,5 +1,16 @@
#include <signal.h>
#include <unistd.h>
#include <stdio.h>
#include "ptl_elan.h"
#include "elan/sys/misc_sys.h"
#include "elan/sys/init_sys.h"
#include "elan/elan.h"
#include "elan/init.h"
#include "rms/rmscall.h"
#include "elan4/library.h"
#include "ptl_elan_priv.h"
static int ompi_init_elan_qdma (mca_ptl_elan_module_1_0_0_t * mp);
static int ompi_init_elan_sten (mca_ptl_elan_module_1_0_0_t * mp);
@ -10,12 +21,9 @@ static int ompi_elan_attach_network (mca_ptl_elan_state_t * state);
int
ompi_mca_ptl_elan_init (mca_ptl_elan_module_1_0_0_t * mp)
{
#if ELAN_COMP
int i;
int *rails;
int num_rails;
int max_backoff;
int max_fastbackoff;
int alloc_mainsize;
int alloc_mainbase;
@ -24,9 +32,6 @@ ompi_mca_ptl_elan_init (mca_ptl_elan_module_1_0_0_t * mp)
mca_ptl_elan_state_t *state;
max_backoff = elan_rup2 (ELAN_MAXBACKOFF) - 1;
max_fastbackoff = elan_rup2 (ELAN_MAXFASTBACKOFF) - 1;
/* Allocate the elan (priv)state structure off the heap.
* it is not available to the Elan thread. Another structure
* for the Elan thread is to be used that instead
@ -35,7 +40,7 @@ ompi_mca_ptl_elan_init (mca_ptl_elan_module_1_0_0_t * mp)
malloc (sizeof (mca_ptl_elan_state_t));
if (NULL == mp->elan_state)
return (NULL);
return (OMPI_ERROR);
memset ((void *) mp->elan_state, 0, sizeof (mca_ptl_elan_state_t));
state = mp->elan_state;
@ -48,16 +53,16 @@ ompi_mca_ptl_elan_init (mca_ptl_elan_module_1_0_0_t * mp)
state->elan_nvp = 0;
state->elan_debug = 0;
state->elan_traced = 0;
state->elan_pagesize = sysconf (_SC_PAGESIZE);
state->elan_pid = getpid ();
/* Default allocator parameters */
state->elan_flags = 0;
state->main_size = ELAN_ALLOC_SIZE;
state->elan_size = ELAN_ALLOCELAN_SIZE;
state->elan_flags |=
(EXCEPTIONCORE | EXCEPTIONTRACE | EXCEPTIONDBGDUMP);
state->elan_debugfile = NULL;
state->elan_flags |=
(EXCEPTIONCORE | EXCEPTIONTRACE | EXCEPTIONDBGDUMP);
state->elan_debugfile = (FILE*)NULL;
state->elan_signalnum = SIGABRT;
#ifdef ELAN_VERSION
@ -66,9 +71,6 @@ ompi_mca_ptl_elan_init (mca_ptl_elan_module_1_0_0_t * mp)
}
#endif
/* FIXME: when to initialize it */
OBJ_CONSTRUCT (&state->state_lock, ompi_mutex_t);
/* Allocate elan capability from the heap */
state->elan_cap =
(ELAN_CAPABILITY *) malloc (sizeof (ELAN_CAPABILITY));
@ -96,10 +98,10 @@ ompi_mca_ptl_elan_init (mca_ptl_elan_module_1_0_0_t * mp)
* Allocate storage space for each Elan rail info (ptrs)
* Allocate storage space for each Elan SDRAM state (ptrs) */
state->all_rails = (RAIL **) malloc (sizeof (RAIL *) * num_rails);
state->all_rails = (ompi_elan_rail_t**)
malloc (sizeof (ompi_elan_rail_t *) * num_rails);
state->all_estates = (ELAN_ESTATE **)
malloc (sizeof (ELAN_ESTATE *) * num_rails);
if (state->all_estates == NULL || state->all_rails == NULL) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
@ -112,63 +114,56 @@ ompi_mca_ptl_elan_init (mca_ptl_elan_module_1_0_0_t * mp)
}
(void) elan_rails (state->elan_cap, rails);
if (NULL == (state->elan_rail = (ELAN_RAIL **)
malloc (sizeof (ELAN_RAIL **) * (num_rails + 1)))) {
return OMPI_ERR_OUT_OF_RESOURCE;
}
state->elan_rail[num_rails] = NULL;
alloc_mainsize = ELAN_ALIGNUP (state->main_size, state->elan_pagesize);
alloc_mainbase = (ADDR_ELAN) ((uintptr_t) state->main_base);
alloc_elansize = ELAN_ALIGNUP (state->elan_size, state->elan_pagesize);
alloc_elanbase = (ADDR_ELAN) ((uintptr_t) state->elan_base);
for (i = 0; i < num_rails; i++) {
RAIL *rail;
ompi_elan_rail_t *rail;
ELAN_ESTATE *estate;
ELAN_EPRIVSTATE *priv_estate;
/* Allocate the Main memory control structure for this rail */
if (NULL == (rail = state->all_rails[i] =
(RAIL *) malloc (sizeof (RAIL)))) {
return OMPI_ERROR;
}
memset (rail, 0, sizeof (RAIL));
/* TODO: Debug the state of rails */
if (NULL == (rail->r_ctx = elan4_init (rails[i]))) {
if (NULL == (rail = state->all_rails[i] =
(ompi_elan_rail_t *)
malloc (sizeof (ompi_elan_rail_t)))) {
return OMPI_ERROR;
}
if (NULL == (rail->r_sdram = elan4_open_sdram (rails[i],
memset (rail, 0, sizeof (ompi_elan_rail_t));
if (NULL == (rail->rail_ctx = elan4_init (rails[i]))) {
return OMPI_ERROR;
}
if (NULL == (rail->rail_sdram = elan4_open_sdram (rails[i],
0,
alloc_elansize))) {
return OMPI_ERROR;
}
if (NULL == (rail->r_alloc =
elan4_createAllocator (state->alloc_mainsize,
rail->r_sdram, 0,
state->alloc_elansize))) {
if (NULL == (rail->rail_alloc =
elan4_createAllocator (state->main_size,
rail->rail_sdram, 0,
state->elan_size))) {
return OMPI_ERROR;
}
if (elan4_set_standard_mappings (rail->r_ctx) < 0
|| elan4_set_required_mappings (rail->r_ctx) < 0) {
return OMPI_ERROR;
}
if (elan4_register_trap_handler (rail->r_ctx,
UTS_UNIMP_INSTR, UTS_TPROC,
elan_unimp_handler, NULL) < 0) {
if (elan4_set_standard_mappings (rail->rail_ctx) < 0
|| elan4_set_required_mappings (rail->rail_ctx) < 0) {
return OMPI_ERROR;
}
/* Now allocate the SDRAM Elan control structure for this rail */
if (NULL == (estate = mp->all_estates[i] =
ALLOC_ELAN (rail, ELAN_ALIGN,
if (NULL == (estate = state->all_estates[i] =
elan4_allocElan(rail->rail_alloc, ELAN_ALIGN,
sizeof (ELAN_EPRIVSTATE)))) {
return OMPI_ERROR;
}
@ -177,8 +172,8 @@ ompi_mca_ptl_elan_init (mca_ptl_elan_module_1_0_0_t * mp)
memset (priv_estate, 0, sizeof (ELAN_EPRIVSTATE));
/* Allocate a command port for non sten functions etc */
if (NULL == (rail->r_cmdq = elan4_alloc_cmdq (rail->r_ctx,
rail->r_alloc, CQ_Size8K,
if (NULL == (rail->rail_cmdq = elan4_alloc_cmdq (rail->rail_ctx,
rail->rail_alloc, CQ_Size8K,
CQ_ModifyEnableBit |
CQ_WriteEnableBit |
CQ_WaitEventEnableBit |
@ -189,68 +184,43 @@ ompi_mca_ptl_elan_init (mca_ptl_elan_module_1_0_0_t * mp)
}
/* Allocate a command port for thread rescheduling etc */
if (NULL == (rail->r_ecmdq = elan4_alloc_cmdq (rail->r_ctx,
rail->r_alloc, CQ_Size8K,
if (NULL == (rail->rail_ecmdq = elan4_alloc_cmdq (rail->rail_ctx,
rail->rail_alloc, CQ_Size8K,
CQ_EnableAllBits,
NULL))) {
return OMPI_ERROR;
}
priv_estate->cport =
MAIN2ELAN (rail->r_ctx, rail->r_ecmdq->cmdq_mapping);
/* Make the rail pointers 'external' */
state->elan_rail[i] = (ELAN_RAIL *) rail;
/* TODO: Find out how the following code work. */
{
ELAN_SLEEP *es;
/* Allocate and then free an ELAN_SLEEP DESC to
* keep heap in sync */
es = _elan_allocSleepDesc (state, rail);
_elan_freeSleepDesc (state, rail, es);
}
estate->alloc = rail->r_alloc;
estate->vp = state->vp; /* still ELAN_INVALID_PROCESS */
estate->debugFlags = state->debugFlags;
priv_estate->debugFd = 1; /* stdout */
priv_estate->maxBackOff = max_backoff;
priv_estate->maxFastBackoff = max_fastbackoff;
priv_estate->pageSize = state->elan_pagesize;
rail->r_estate = estate;
rail->r_railNo = rails[i];
rail->rail_estate = estate;
rail->rail_railNo = rails[i];
{
RAILTABLE *rt;
ompi_elan_railtable_t *rt;
if (NULL == (rt = (RAILTABLE *) malloc (sizeof (RAILTABLE)))) {
if (NULL == (rt = (ompi_elan_railtable_t *) malloc (
sizeof (ompi_elan_railtable_t)))) {
return OMPI_ERROR;
}
memset (rt, 0, sizeof (RAILTABLE));
memset (rt, 0, sizeof (ompi_elan_railtable_t));
rt->rt_nrails = 1;
rt->rt_rail = 0;
rt->rt_railReal = rail;
rt->rt_allRails = &(state->all_rails[rail]);
/* Debug the state, see _elan_railtable_createSingle */
rt->rt_railReal = i;
rt->rt_allRails = (struct ompi_elan_rail_t *)
&(state->all_rails[i]);
rail->r_railTable = rt;
rail->rail_railTable = rt;
}
} /* for each rail */
free (rails);
state->elan_ctx = state->rail[0]->rail_ctx;
state->elan_ctx = state->elan_rail[0]->rail_ctx;
state->elan_estate = (void *) state->all_estates[0];
/*_elan_eventInit(state);*/
/*atexit(_elan_atExitCallBack); */
ompi_elan_attach_network (state);
/* Set the rms_resourceId */
@ -262,7 +232,6 @@ ompi_mca_ptl_elan_init (mca_ptl_elan_module_1_0_0_t * mp)
for (i = 0; state->elan_rail[i]; i++) {
elan4_block_inputter (state->elan_rail[i]->rail_ctx, 0);
}
#endif
return (OMPI_SUCCESS);
}
@ -282,7 +251,7 @@ ompi_mca_ptl_elan_setup (mca_ptl_elan_module_1_0_0_t * mp)
if (OMPI_SUCCESS != ompi_init_elan_qdma (mp)) {
return OMPI_ERROR;
}
#if MCA_PTL_ELAN_STAGE_TWO
if (OMPI_SUCCESS != ompi_init_elan_rdma (mp)) {
return OMPI_ERROR;
}
@ -294,7 +263,6 @@ ompi_mca_ptl_elan_setup (mca_ptl_elan_module_1_0_0_t * mp)
if (OMPI_SUCCESS != ompi_init_elan_stat (mp)) {
return OMPI_ERROR;
}
#endif
return (OMPI_SUCCESS);
}
@ -320,40 +288,39 @@ ompi_mca_ptl_elan_fin (mca_ptl_elan_module_1_0_0_t * mp)
static int
ompi_elan_attach_network (mca_ptl_elan_state_t * state)
{
#if ELAN_COMP
int i, vp, *vps, num_rails;
ELAN_LOCATION loc;
ELAN_CAPABILITY *cap = state->elan_cap;
num_rails = state->elan_nrails;
if (state->attached) {
if (state->elan_attached) {
/* already successfully attached */
return;
return OMPI_SUCCESS;
} else {
state->attached = 1;
state->elan_attached = 1;
}
for (i = 0; i < num_rails; i++) {
RAIL *rail = state->all_rails[i];
ompi_elan_rail_t *rail = state->all_rails[i];
ELAN_LOCATION loc;
/* Add all virtual process from 0 to (nvp-1) */
if (elan4_add_p2pvp (rail->r_ctx, 0, cap) < 0) {
if (elan4_add_p2pvp (rail->rail_ctx, 0, cap) < 0) {
/* FIXME: exception code */
return OMPI_ERROR;
}
/* block the inputter until we are really ready */
elan4_block_inputter (rail->r_ctx, 1);
elan4_block_inputter (rail->rail_ctx, 1);
if (elan4_attach (rail->r_ctx, cap)) {
if (elan4_attach (rail->rail_ctx, cap)) {
/* FIXME: exception code */
return OMPI_ERROR;
}
/* NB: We should have same vp for all capabilities */
loc.loc_node = ((CTX_ELAN *) rail->r_ctx)->ctx_position.pos_nodeid
loc.loc_node = ((ELAN4_CTX *) rail->rail_ctx)->ctx_position.pos_nodeid
- cap->cap_lownode;
loc.loc_context = cap->cap_mycontext - cap->cap_lowcontext;
@ -364,25 +331,26 @@ ompi_elan_attach_network (mca_ptl_elan_state_t * state)
state->elan_vp = elan_location2vp (loc, state->elan_cap);
/* Initialise the Elan version of this data */
((ELAN_ESTATE *) rail->r_estate)->vp = state->elan_vp;
((ELAN_ESTATE *) rail->rail_estate)->vp = state->elan_vp;
/* Allocate a cookie pool for the thread processor
* and copy to sdram */
rail->r_cpool =
elan4_allocCookiePool (rail->r_ctx, state->elan_vp);
((ELAN_EPRIVSTATE *) rail->r_estate)->tcookie =
rail->r_cpool->cp_cookie;
rail->rail_cpool =
elan4_allocCookiePool (rail->rail_ctx, state->elan_vp);
((ELAN_EPRIVSTATE *) rail->rail_estate)->tcookie =
rail->rail_cpool->cp_cookie;
}
/* Allocate a cookie pool for the exported API (0th rail) */
state->cpool = elan4_allocCookiePool (state->elan_ctx, state->elan_vp);
state->elan_cpool = elan4_allocCookiePool (
state->elan_ctx, state->elan_vp);
loc = elan_vp2location (state->elan_vp, state->elan_cap);
/* update THREAD elan_dbg info of debugfile */
if (state->elan_debugfile && fileno (state->elan_debugfile) != -1) {
for (i = 0; i < num_rails; i++) {
CTX_ELAN *ctx = state->elan_rail[i]->rail_ctx;
ELAN4_CTX *ctx = state->elan_rail[i]->rail_ctx;
/* Convert FILE * stream to fd for THRD output */
((ELAN_ESTATE *) state->all_estates[i])->debugFd =
@ -396,16 +364,10 @@ ompi_elan_attach_network (mca_ptl_elan_state_t * state)
/* Determine the number of processes described by the capability */
state->elan_nvp = elan_nvps (state->elan_cap);
/* Start allocating bcastVps a little way after the pt2pt ones */
state->bcastVp = state->baseBcastVp = state->elan_nvp + 16;
if (state->elan_vp >= elan_state->elan_nvp) {
/* FIXME: exception code */
return OMPI_ERROR;
}
#if 0
/* FIXME: fix the following rail table stuff */
state->railTable = _elan_railtable_create (elan_state);
state->rail = _elan_railtable_getRailSelf (elan_state,
state->elan_rail = _elan_railtable_getRailSelf (elan_state,
elan_state->railTable);
if (NULL == (state->elan_localIds = (int *)
@ -413,25 +375,26 @@ ompi_elan_attach_network (mca_ptl_elan_state_t * state)
/* FIXME: exception code */
return OMPI_ERROR;
}
#endif
/* Set all to non local initially */
for (vp = 0; vp < state->elan_nvp; vp++)
state->elan_localvps[vp] = -1;
/* Stash our own process location */
state->myloc = elan_vp2location (state->elan_vp, state->elan_cap);
state->elan_maxlocals = elan_maxlocal (elan_state->cap);
state->elan_numlocals = elan_nlocal (state->myloc.loc_node,
state->elan_myloc = elan_vp2location (state->elan_vp, state->elan_cap);
state->elan_maxlocals = elan_maxlocal (state->elan_cap);
state->elan_numlocals = elan_nlocal (state->elan_myloc.loc_node,
state->elan_cap);
/* Allocate more than we need to keep the heap in sync */
if (NULL == (vps = (int *) malloc (sizeof (int) * elan_state->nvp))) {
if (NULL == (vps = (int *) malloc (sizeof (int) * state->elan_nvp))) {
/* FIXME: exception code */
return OMPI_ERROR;
}
/* Fill out the local vp array */
elan_localvps (state->myloc.loc_node,
elan_localvps (state->elan_myloc.loc_node,
state->elan_cap, vps, state->elan_numlocals);
for (i = 0; i < state->elan_numlocals; i++) {
@ -447,23 +410,25 @@ ompi_elan_attach_network (mca_ptl_elan_state_t * state)
/* TODO: Debug code here for elan_state */
#if 1
/*
* We need to create one of these maps in each rail estate ctx too
*/
for (i = 0; i < num_rails; i++) {
RAIL *rail = state->all_rails[i];
ompi_elan_rail_t *rail = state->all_rails[i];
ELAN_ESTATE *estate = state->all_estates[i];
int *localIdsElan;
/* Allocate a copy of this in the SDRAM of each rail */
if (0 == (localIdsElan = ALLOC_ELAN (rail, ELAN_ALIGN,
elan_state->nvp *
if (0 == (localIdsElan = elan4_allocElan(rail->rail_alloc, ELAN_ALIGN,
state->elan_nvp *
sizeof (int)))) {
/* FIXME: exception code */
return OMPI_ERROR;
}
estate->localIds = SDRAM2ELAN (rail->r_ctx, localIdsElan);
estate->localIds = elan4_main2elan( (ELAN4_CTX*)rail->rail_ctx,
(void*)localIdsElan);
for (vp = 0; vp < state->elan_nvp; vp++) {
localIdsElan[vp] = state->elan_localvps[vp];
@ -474,12 +439,12 @@ ompi_elan_attach_network (mca_ptl_elan_state_t * state)
estate->nvp = state->elan_nvp;
/* Update thread info on primary rail too */
((ELAN_EPRIVSTATE *) estate)->rail = state->elan_rail;
((ELAN_EPRIVSTATE *) estate)->rail = (ELAN_RAIL **) state->elan_rail;
}
#endif
/* Done with vps array now */
free (vps);
#endif
return (OMPI_SUCCESS);
}

8
src/mca/ptl/elan/src/ptl_elan_module.c
Просмотреть файл

@ -54,7 +54,7 @@ mca_ptl_elan_module_1_0_0_t mca_ptl_elan_module = {
}
};
static mca_ptl_elan_module_1_0_0_t *mp = &mca_ptl_elan_module;
/*static mca_ptl_elan_module_1_0_0_t *mp = &mca_ptl_elan_module;*/
/**
* utility routines for parameter registration
@ -190,14 +190,15 @@ mca_ptl_elan_module_init (int *num_ptls,
return NULL;
}
#if ELAN_COMP
/* initialize free lists */
#if 1
ompi_free_list_init (&(mca_ptl_elan_module.elan_reqs_free),
sizeof (mca_ptl_elan_send_request_t),
OBJ_CLASS (mca_ptl_elan_send_request_t),
mca_ptl_elan_module.elan_free_list_num,
mca_ptl_elan_module.elan_free_list_max,
mca_ptl_elan_module.elan_free_list_inc, NULL);
#endif
/* use default allocator */
ompi_free_list_init (&mca_ptl_elan_module.elan_events_free,
@ -206,7 +207,6 @@ mca_ptl_elan_module_init (int *num_ptls,
mca_ptl_elan_module.elan_free_list_num,
mca_ptl_elan_module.elan_free_list_max,
mca_ptl_elan_module.elan_free_list_inc, NULL);
#endif
/* open basic elan device */
if (OMPI_SUCCESS != ompi_mca_ptl_elan_init(&mca_ptl_elan_module)) {
@ -229,7 +229,7 @@ mca_ptl_elan_module_init (int *num_ptls,
* over elan, there is no need for a oob_based exchange.
*/
ptls = (mca_ptl_elan_t **) malloc (mca_ptl_elan_module.elan_num_ptls *
ptls = (mca_ptl_t **) malloc (mca_ptl_elan_module.elan_num_ptls *
sizeof (mca_ptl_t *));
if (NULL == ptls) {

49
src/mca/ptl/elan/src/ptl_elan_priv.h
Просмотреть файл

@ -28,10 +28,51 @@
#include "ptl_elan_frag.h"
#include "ptl_elan_req.h"
/*#include "elan/init.h"*/
/*#include "elan/sys/misc_sys.h"*/
/*#include "elan/sys/init_sys.h"*/
/*#include "elan/elan.h"*/
#define __elan4__
#include <elan/elan.h>
#include <elan/init.h>
#include <rms/rmscall.h>
#include <elan4/library.h>
#include <elan4/types.h>
#include <elan4/intcookie.h>
/*#include "./elan.h"*/
/*#include "./misc_sys.h"*/
/*#include "./init_sys.h"*/
#include "elan/sys/misc_sys.h"
#include "elan/sys/init_sys.h"
int elan4_block_inputter (ELAN4_CTX *ctx, unsigned blocked);
struct ompi_elan_railtable_t {
int rt_nrails;
int rt_rail;
int rt_railReal;
int *rt_table;
int *rt_realRail;
struct ompi_elan_rail_t *rt_allRails;
};
typedef struct ompi_elan_railtable_t ompi_elan_railtable_t;
struct ompi_elan_rail_t {
struct elan4_ctx *rail_ctx;
ELAN4_SDRAM *rail_sdram;
E4_CmdQ *rail_cmdq;
E4_CmdQ *rail_ecmdq;
ELAN4_ALLOC *rail_alloc;
struct elan_sleep *rail_sleepDescs;
ELAN4_COOKIEPOOL *rail_cpool;
u_int rail_index;
ELAN_TPORT *rail_tport;
ompi_elan_railtable_t *rail_railTable;
void *rail_estate; /* ADDR_SDRAM r_estate; */
int rail_railNo;
E4_Addr rail_tportrecvSym;
E4_Addr rail_atomicSym;
};
typedef struct ompi_elan_rail_t ompi_elan_rail_t;
/* Initialization and finalization routines */
int ompi_mca_ptl_elan_init (mca_ptl_elan_module_1_0_0_t *mp);

2
src/mca/ptl/elan/src/ptl_elan_req.h
Просмотреть файл

@ -17,7 +17,7 @@
#include "ptl_elan_frag.h"
/* Again, what is this! */
/* extern ompi_class_t mca_ptl_elan_send_request_t_class;*/
extern ompi_class_t mca_ptl_elan_send_request_t_class;
/**
* ELAN send request derived type. The send request contains both the