ports/openmpi
ports/openmpi
1
1
Форкнуть 0
Код Релизы Активность

Fill up more cleanup code

Просмотр исходного кода 
This commit was SVN r2103.
Этот коммит содержится в:
Weikuan Yu 2004-08-12 23:39:55 +00:00
родитель 07e838570d
Коммит 8852ada41d
6 изменённых файлов: 140 добавлений и 166 удалений
Показать статистику Скачать Patch файл Скачать Diff файл Показать все файлы Свернуть все файлы

34
src/mca/ptl/elan/src/ptl_elan.c
Просмотреть файл

@ -20,10 +20,6 @@
#include "ptl_elan_frag.h" #include "ptl_elan_frag.h"
#include "ptl_elan_priv.h" #include "ptl_elan_priv.h"
#if 0
#elif defined(ABC)
#endif
/* XXX: There must be multiple PTL's. This could be the template */ /* XXX: There must be multiple PTL's. This could be the template */
mca_ptl_elan_module_t mca_ptl_elan_module = { mca_ptl_elan_module_t mca_ptl_elan_module = {
{ {
@ -288,7 +284,6 @@ mca_ptl_elan_put (struct mca_ptl_base_module_t *ptl,
int rc = OMPI_SUCCESS; int rc = OMPI_SUCCESS;
mca_ptl_elan_send_frag_t *desc; mca_ptl_elan_send_frag_t *desc;
/* PML still utilize this interface the same as a send option. /* PML still utilize this interface the same as a send option.
* So we need to generate a QDMA to the remote side for completion * So we need to generate a QDMA to the remote side for completion
* notification */ * notification */
@ -312,7 +307,7 @@ mca_ptl_elan_put (struct mca_ptl_base_module_t *ptl,
sendreq, offset, &size, flags); sendreq, offset, &size, flags);
/* Update all the sends until the put is done */ /* Update all the sends until the put is done */
END_FUNC(PTL_ELAN_DEBUG_NONE); END_FUNC(PTL_ELAN_DEBUG_PUT);
return rc; return rc;
} }
@ -330,30 +325,6 @@ mca_ptl_elan_get (struct mca_ptl_base_module_t *ptl,
int flags) int flags)
{ {
int rc = OMPI_SUCCESS; int rc = OMPI_SUCCESS;
#if 0
mca_ptl_elan_send_frag_t *desc;
/* XXX:
* Since the address passed down from PML does not provide
* elan information, so there needs to be a change
*/
START_FUNC(PTL_ELAN_DEBUG_NONE);
desc = mca_ptl_elan_alloc_send_desc(ptl, sendreq, MCA_PTL_ELAN_DESC_GET);
if (NULL == desc) {
ompi_output(0,
"[%s:%d] Unable to allocate an elan send descriptors \n",
__FILE__, __LINE__);
}
rc = mca_ptl_elan_start_desc(desc,
(struct mca_ptl_elan_peer_t *)ptl_peer,
sendreq, offset, &size, flags);
/* Update all the sends until the put is done */
END_FUNC(PTL_ELAN_DEBUG_NONE);
#endif
return rc; return rc;
} }
@ -399,8 +370,9 @@ mca_ptl_elan_matched (mca_ptl_base_module_t * ptl,
#endif #endif
if (header->hdr_common.hdr_flags & MCA_PTL_FLAGS_ACK_MATCHED) if (header->hdr_common.hdr_flags & MCA_PTL_FLAGS_ACK_MATCHED)
#if 1 /* Basic ACK scheme following TCP cases */ #if 1
{ {
/* Basic ACK scheme following TCP cases */
mca_ptl_elan_send_frag_t *desc; mca_ptl_elan_send_frag_t *desc;
/* Get a frag desc and allocate a send desc */ /* Get a frag desc and allocate a send desc */

34
src/mca/ptl/elan/src/ptl_elan_comm_init.c
Просмотреть файл

@ -9,14 +9,6 @@
#include "ptl_elan.h" #include "ptl_elan.h"
#include "ptl_elan_priv.h" #include "ptl_elan_priv.h"
#define PUTGET_THROTTLE (32)
#define ELAN_PTL_FASTPATH (0x1)
#define ELAN_QUEUE_MAX (INPUT_QUEUE_MAX)
#define ELAN_QUEUE_LOST_SLOTS (1)
#define SLOT_ALIGN (128)
#define MAX(a,b) ((a>b)? a:b)
#define ALIGNUP(x,a) (((unsigned int)(x) + ((a)-1)) & (-(a)))
#define OMPI_PTL_ELAN_CTRL_LIST(flist, init_num, inc_num, max_num) \ #define OMPI_PTL_ELAN_CTRL_LIST(flist, init_num, inc_num, max_num) \
do { \ do { \
OBJ_CONSTRUCT (flist, ompi_free_list_t); \ OBJ_CONSTRUCT (flist, ompi_free_list_t); \
@ -47,7 +39,7 @@ ompi_init_elan_queue_events (mca_ptl_elan_module_t * ptl,
ompi_ptl_elan_qdma_desc_t *desc; ompi_ptl_elan_qdma_desc_t *desc;
E4_Event *elan_ptr; E4_Event *elan_ptr;
START_FUNC(PTL_ELAN_DEBUG_NONE); START_FUNC(PTL_ELAN_DEBUG_INIT);
rail = (RAIL *) ptl->ptl_elan_rail; rail = (RAIL *) ptl->ptl_elan_rail;
ctx = (ELAN4_CTX *) ptl->ptl_elan_ctx; ctx = (ELAN4_CTX *) ptl->ptl_elan_ctx;
@ -58,8 +50,8 @@ ompi_init_elan_queue_events (mca_ptl_elan_module_t * ptl,
flist = &queue->tx_desc_free; flist = &queue->tx_desc_free;
main_align = MAX (sizeof (void *), 8); main_align = GET_MAX (sizeof (void *), 8);
elan_align = MAX (sizeof (int *), ELAN_BLOCK_ALIGN); elan_align = GET_MAX (sizeof (int *), ELAN_BLOCK_ALIGN);
main_size = ALIGNUP (sizeof (ompi_ptl_elan_qdma_desc_t), main_align); main_size = ALIGNUP (sizeof (ompi_ptl_elan_qdma_desc_t), main_align);
elan_size = ALIGNUP (sizeof (E4_Event), elan_align); elan_size = ALIGNUP (sizeof (E4_Event), elan_align);
@ -113,7 +105,7 @@ ompi_init_elan_queue_events (mca_ptl_elan_module_t * ptl,
} }
flist->fl_num_allocated += flist->fl_num_per_alloc; flist->fl_num_allocated += flist->fl_num_per_alloc;
END_FUNC(PTL_ELAN_DEBUG_NONE); END_FUNC(PTL_ELAN_DEBUG_INIT);
return OMPI_SUCCESS; return OMPI_SUCCESS;
} }
@ -199,10 +191,10 @@ ompi_ptl_elan_init_putget_ctrl (mca_ptl_elan_module_t * ptl,
ompi_free_list_t *put_list, *get_list; ompi_free_list_t *put_list, *get_list;
ompi_ptl_elan_putget_desc_t *put_desc, *get_desc; ompi_ptl_elan_putget_desc_t *put_desc, *get_desc;
START_FUNC(PTL_ELAN_DEBUG_NONE); START_FUNC(PTL_ELAN_DEBUG_INIT);
main_align = MAX (sizeof (void *), ELAN_ALIGN); main_align = GET_MAX (sizeof (void *), ELAN_ALIGN);
elan_align = MAX (sizeof (int *), ELAN_BLOCK_ALIGN); elan_align = GET_MAX (sizeof (int *), ELAN_BLOCK_ALIGN);
main_size = ALIGNUP(sizeof(ompi_ptl_elan_putget_desc_t), main_align); main_size = ALIGNUP(sizeof(ompi_ptl_elan_putget_desc_t), main_align);
/* Contain elan_event, chain_event and a chain_buff */ /* Contain elan_event, chain_event and a chain_buff */
@ -254,7 +246,7 @@ ompi_ptl_elan_init_putget_ctrl (mca_ptl_elan_module_t * ptl,
OMPI_ELAN_PUTGET_GROW(ctx, get_list, frag, get_desc, elan_ptr, OMPI_ELAN_PUTGET_GROW(ctx, get_list, frag, get_desc, elan_ptr,
main_size, elan_size, 0); main_size, elan_size, 0);
END_FUNC(PTL_ELAN_DEBUG_NONE); END_FUNC (PTL_ELAN_DEBUG_INIT);
return OMPI_SUCCESS; return OMPI_SUCCESS;
} }
@ -278,7 +270,7 @@ ompi_init_elan_qdma (mca_ptl_elan_component_t * emp,
ELAN4_CTX *ctx; ELAN4_CTX *ctx;
struct mca_ptl_elan_module_t *ptl; struct mca_ptl_elan_module_t *ptl;
START_FUNC(PTL_ELAN_DEBUG_NONE); START_FUNC(PTL_ELAN_DEBUG_INIT);
/* Init the Transmit Queue structure */ /* Init the Transmit Queue structure */
for (i = 0; i < num_rails; i++) { for (i = 0; i < num_rails; i++) {
@ -390,13 +382,13 @@ ompi_init_elan_qdma (mca_ptl_elan_component_t * emp,
OBJ_CONSTRUCT (&queue->rx_lock, ompi_mutex_t); OBJ_CONSTRUCT (&queue->rx_lock, ompi_mutex_t);
} }
END_FUNC(PTL_ELAN_DEBUG_NONE); END_FUNC(PTL_ELAN_DEBUG_INIT);
return (OMPI_SUCCESS); return (OMPI_SUCCESS);
} }
int int
ompi_init_elan_putget (mca_ptl_elan_component_t * emp, ompi_init_elan_putget (mca_ptl_elan_component_t * emp,
int num_rails) int num_rails)
{ {
int i; int i;
int nslots = 128; int nslots = 128;
@ -405,7 +397,7 @@ ompi_init_elan_putget (mca_ptl_elan_component_t * emp,
ELAN4_CTX *ctx; ELAN4_CTX *ctx;
struct mca_ptl_elan_module_t *ptl; struct mca_ptl_elan_module_t *ptl;
START_FUNC(PTL_ELAN_DEBUG_NONE); START_FUNC(PTL_ELAN_DEBUG_INIT);
/* Init the Transmit Queue structure */ /* Init the Transmit Queue structure */
for (i = 0; i < num_rails; i++) { for (i = 0; i < num_rails; i++) {
@ -469,7 +461,7 @@ ompi_init_elan_putget (mca_ptl_elan_component_t * emp,
ompi_ptl_elan_init_putget_ctrl (ptl, rail, putget, 0, 2, 32); ompi_ptl_elan_init_putget_ctrl (ptl, rail, putget, 0, 2, 32);
} }
END_FUNC(PTL_ELAN_DEBUG_NONE); END_FUNC(PTL_ELAN_DEBUG_INIT);
return (OMPI_SUCCESS); return (OMPI_SUCCESS);
} }

98
src/mca/ptl/elan/src/ptl_elan_component.c
Просмотреть файл

@ -60,9 +60,10 @@ static mca_ptl_elan_component_t *elan_mp = &mca_ptl_elan_component;
static bool mca_ptl_elan_component_initialized = false; static bool mca_ptl_elan_component_initialized = false;
/* /*
* some elan vp information to the the global registery * XXX: Leave it as a routine for possible extension
* some elan vp information to the the global registery
*/ */
static int mca_ptl_elan_component_register (mca_ptl_elan_component_t *emp) static int mca_ptl_elan_addr_put (mca_ptl_elan_component_t *emp)
{ {
int rc; int rc;
size_t i; size_t i;
@ -94,27 +95,28 @@ static int mca_ptl_elan_component_register (mca_ptl_elan_component_t *emp)
int int
mca_ptl_elan_component_open (void) mca_ptl_elan_component_open (void)
{ {
/* FIXME: register the default super module parameters, int length;
* Some sanity checking is needed to ensure that user int param1, param2, param3;
* would not provide unrealistic parameters.*/
mca_ptl_elan_module.super.ptl_exclusivity = mca_ptl_elan_module.super.ptl_exclusivity =
mca_ptl_elan_param_register_int ("exclusivity", 0); mca_ptl_elan_param_register_int ("exclusivity", 0);
mca_ptl_elan_module.super.ptl_first_frag_size =
mca_ptl_elan_param_register_int ("first_frag_size",
(PTL_ELAN_INPUT_QUEUE_MAX - sizeof(mca_ptl_base_header_t)));
mca_ptl_elan_module.super.ptl_min_frag_size =
mca_ptl_elan_param_register_int ("min_frag_size",
(PTL_ELAN_INPUT_QUEUE_MAX - sizeof(mca_ptl_base_header_t)));
mca_ptl_elan_module.super.ptl_max_frag_size =
mca_ptl_elan_param_register_int ("max_frag_size", 2<<31);
/* register ELAN module parameters */ length = PTL_ELAN_INPUT_QUEUE_MAX - sizeof(mca_ptl_base_header_t);
elan_mp->elan_free_list_num = param1 = mca_ptl_elan_param_register_int ("first_frag_size", length);
mca_ptl_elan_param_register_int ("free_list_num", 32); param2 = mca_ptl_elan_param_register_int ("min_frag_size", length);
elan_mp->elan_free_list_max = param3 = mca_ptl_elan_param_register_int ("max_frag_size", 2<<31);
mca_ptl_elan_param_register_int ("free_list_max", 128);
elan_mp->elan_free_list_inc = /* Correct these if user give violent parameters */
mca_ptl_elan_param_register_int ("free_list_inc", 32); mca_ptl_elan_module.super.ptl_first_frag_size = GET_MIN(param1, length);
mca_ptl_elan_module.super.ptl_min_frag_size = GET_MAX(param2, length);
mca_ptl_elan_module.super.ptl_max_frag_size = GET_MIN(param3, 2<<31);
/* XXX: Do not want to make these configuraable, since they are
* very much related to number of outstanding operations in elan4
* and therefore flow control */
elan_mp->elan_free_list_num = 32;
elan_mp->elan_free_list_max = 128;
elan_mp->elan_free_list_inc = 32;
/* initialize state */ /* initialize state */
elan_mp->elan_ptl_modules = NULL; elan_mp->elan_ptl_modules = NULL;
@ -142,8 +144,6 @@ mca_ptl_elan_component_close (void)
/* cleanup the proc, ptl, and the module */ /* cleanup the proc, ptl, and the module */
ompi_mca_ptl_elan_finalize (&mca_ptl_elan_component); ompi_mca_ptl_elan_finalize (&mca_ptl_elan_component);
/* XXX: Make sure this is not just an alias pointer */
if (elan_mp->elan_local) { if (elan_mp->elan_local) {
free (elan_mp->elan_local); free (elan_mp->elan_local);
} }
@ -157,25 +157,23 @@ mca_ptl_elan_component_close (void)
} }
} }
if (elan_mp->elan_recv_frags_free.fl_num_allocated !=
elan_mp->elan_recv_frags_free.super.ompi_list_length) {
ompi_output (0, "[%s:%d] recv_frags : %d allocated %d returned\n",
__FILE__, __LINE__,
elan_mp->elan_recv_frags_free.fl_num_allocated,
elan_mp->elan_recv_frags_free.super.ompi_list_length);
}
/* FIXME: free free_list entries before destructing lists */
/* Free the empty list holders */ /* Free the empty list holders */
OBJ_DESTRUCT (&(elan_mp->elan_procs)); OBJ_DESTRUCT (&(elan_mp->elan_procs));
OBJ_DESTRUCT (&(elan_mp->elan_pending_acks)); OBJ_DESTRUCT (&(elan_mp->elan_pending_acks));
OBJ_DESTRUCT (&(elan_mp->elan_send_frags)); OBJ_DESTRUCT (&(elan_mp->elan_send_frags));
OBJ_DESTRUCT (&(elan_mp->elan_recv_frags)); OBJ_DESTRUCT (&(elan_mp->elan_recv_frags));
/* TODO: /* FIXME:
* We need free all the memory allocated for this list * We need free all the memory allocated for this list
* before desctructing this free_list */ * before desctructing this free_list */
if (elan_mp->elan_recv_frags_free.fl_num_allocated !=
elan_mp->elan_recv_frags_free.super.ompi_list_length) {
ompi_output (0,
"[%s:%d] recv_frags : %d allocated %d returned\n",
__FILE__, __LINE__,
elan_mp->elan_recv_frags_free.fl_num_allocated,
elan_mp->elan_recv_frags_free.super.ompi_list_length);
}
OBJ_DESTRUCT (&(elan_mp->elan_recv_frags_free)); OBJ_DESTRUCT (&(elan_mp->elan_recv_frags_free));
/* Destruct other structures */ /* Destruct other structures */
@ -192,23 +190,15 @@ mca_ptl_elan_component_close (void)
*/ */
mca_ptl_base_module_t ** mca_ptl_base_module_t **
mca_ptl_elan_component_init (int *num_ptl_modules, mca_ptl_elan_component_init (int *num_ptl_modules,
bool * allow_multi_user_threads, bool * allow_multi_user_threads,
bool * have_hidden_threads) bool * have_hidden_threads)
{ {
mca_ptl_base_module_t **ptls; mca_ptl_base_module_t **ptls;
*num_ptl_modules = 0; START_FUNC(PTL_ELAN_DEBUG_INIT);
START_FUNC(PTL_ELAN_DEBUG_NONE);
if (PTL_ELAN_DEBUG_FLAG & PTL_ELAN_DEBUG_INIT) {
char hostname[32]; gethostname(hostname, 32);
fprintf(stderr, "[%s:%s:%d] debugging ...\n",
hostname, __FUNCTION__, __LINE__);
}
/* TODO: support multiple threads */ /* TODO: support multiple threads */
*num_ptl_modules = 0;
*allow_multi_user_threads = true; *allow_multi_user_threads = true;
*have_hidden_threads = OMPI_HAVE_THREADS; *have_hidden_threads = OMPI_HAVE_THREADS;
@ -227,8 +217,7 @@ mca_ptl_elan_component_init (int *num_ptl_modules,
return NULL; return NULL;
} }
if (OMPI_SUCCESS != if (OMPI_SUCCESS != mca_ptl_elan_addr_put(&mca_ptl_elan_component)) {
mca_ptl_elan_component_register(&mca_ptl_elan_component)) {
ompi_output(0, ompi_output(0,
"[%s:%d] error in registering with Runtime/OOB \n", "[%s:%d] error in registering with Runtime/OOB \n",
__FILE__, __LINE__); __FILE__, __LINE__);
@ -238,9 +227,9 @@ mca_ptl_elan_component_init (int *num_ptl_modules,
ptls = (mca_ptl_base_module_t **) malloc (elan_mp->elan_num_ptl_modules * ptls = (mca_ptl_base_module_t **) malloc (elan_mp->elan_num_ptl_modules *
sizeof (mca_ptl_elan_module_t *)); sizeof (mca_ptl_elan_module_t *));
if (NULL == ptls) { if (NULL == ptls) {
ompi_output(0, ompi_output(0,
"[%s:%d] error in malloc for elan PTL references\n", "[%s:%d] error in allocating memory \n",
__FILE__, __LINE__); __FILE__, __LINE__);
return NULL; return NULL;
} }
@ -249,7 +238,7 @@ mca_ptl_elan_component_init (int *num_ptl_modules,
*num_ptl_modules = elan_mp->elan_num_ptl_modules; *num_ptl_modules = elan_mp->elan_num_ptl_modules;
mca_ptl_elan_component_initialized = true; mca_ptl_elan_component_initialized = true;
END_FUNC(PTL_ELAN_DEBUG_NONE); END_FUNC(PTL_ELAN_DEBUG_INIT);
return ptls; return ptls;
} }
@ -269,15 +258,12 @@ mca_ptl_elan_component_control (int param,
/* TODO: to support event-based module progress later. */ /* TODO: to support event-based module progress later. */
static int times = 0;
int int
mca_ptl_elan_component_progress (mca_ptl_tstamp_t tstamp) mca_ptl_elan_component_progress (mca_ptl_tstamp_t tstamp)
{ {
START_FUNC(PTL_ELAN_DEBUG_NONE); START_FUNC (PTL_ELAN_DEBUG_NONE);
mca_ptl_elan_drain_recv(elan_mp); mca_ptl_elan_drain_recv(elan_mp);
mca_ptl_elan_update_desc(elan_mp); mca_ptl_elan_update_desc(elan_mp);
mca_ptl_elan_update_putget(elan_mp); END_FUNC (PTL_ELAN_DEBUG_NONE);
END_FUNC(PTL_ELAN_DEBUG_NONE);
return OMPI_SUCCESS; return OMPI_SUCCESS;
} }

63
src/mca/ptl/elan/src/ptl_elan_init.c
Просмотреть файл

@ -6,15 +6,10 @@
#include <unistd.h> #include <unistd.h>
#include <stdio.h> #include <stdio.h>
#define _ELAN4
#define __elan4
#include "ptl_elan.h" #include "ptl_elan.h"
#include "ptl_elan_priv.h" #include "ptl_elan_priv.h"
#define ELAN_QUEUE_MAX INPUT_QUEUE_MAX #define _ELAN4
#define ELAN_QUEUE_LOST_SLOTS 1
#define SLOT_ALIGN 128
mca_ptl_elan_state_t mca_ptl_elan_global_state; mca_ptl_elan_state_t mca_ptl_elan_global_state;
@ -24,7 +19,7 @@ ompi_mca_ptl_elan_setup (mca_ptl_elan_state_t * ems)
mca_ptl_elan_component_t *emp; mca_ptl_elan_component_t *emp;
int rail_count; int rail_count;
START_FUNC(PTL_ELAN_DEBUG_NONE); START_FUNC(PTL_ELAN_DEBUG_INIT);
rail_count = ems->elan_nrails; rail_count = ems->elan_nrails;
emp = ems->elan_component; emp = ems->elan_component;
@ -94,19 +89,24 @@ ompi_mca_ptl_elan_setup (mca_ptl_elan_state_t * ems)
return OMPI_ERROR; return OMPI_ERROR;
} }
END_FUNC(PTL_ELAN_DEBUG_NONE); END_FUNC(PTL_ELAN_DEBUG_INIT);
return (OMPI_SUCCESS); return (OMPI_SUCCESS);
} }
/* Attach to the network */ /* Attach to the network */
static int elan_attached = 0;
static int static int
ompi_elan_attach_network (mca_ptl_elan_state_t * ems) ompi_elan_attach_network (mca_ptl_elan_state_t * ems)
{ {
static int elan_attached = 0; int i;
int i, vp, *vps, num_rails; int vp;
ELAN_LOCATION loc; int *vps;
ELAN_CAPABILITY *cap = ems->elan_cap; int num_rails;
ELAN_LOCATION loc;
ELAN_CAPABILITY *cap;
cap = ems->elan_cap;
num_rails = ems->elan_nrails; num_rails = ems->elan_nrails;
if (elan_attached) { if (elan_attached) {
@ -238,14 +238,14 @@ static void
ompi_module_elan_close_ptls (mca_ptl_elan_component_t * emp, ompi_module_elan_close_ptls (mca_ptl_elan_component_t * emp,
int num_rails) int num_rails)
{ {
/* TODO: find the ones that are still there and free them */ /* FIXME: find the ones that are still there and free them */
} }
static void static void
ompi_module_elan_close_procs (mca_ptl_elan_component_t * emp, ompi_module_elan_close_procs (mca_ptl_elan_component_t * emp,
int num_rails) int num_rails)
{ {
/* TODO: find the ones that are still there and free them */ /* FIXME: find the ones that are still there and free them */
} }
ELAN_SLEEP * ELAN_SLEEP *
@ -255,7 +255,6 @@ ompi_init_elan_sleepdesc (mca_ptl_elan_state_t * ems,
ELAN_SLEEP *es; ELAN_SLEEP *es;
/* XXX: asking the caller to hold the lock */ /* XXX: asking the caller to hold the lock */
es = MALLOC (sizeof (ELAN_SLEEP)); es = MALLOC (sizeof (ELAN_SLEEP));
OMPI_PTL_ELAN_CHECK_UNEX (es, NULL, NULL, 0); OMPI_PTL_ELAN_CHECK_UNEX (es, NULL, NULL, 0);
memset (es, 0, sizeof (ELAN_SLEEP)); memset (es, 0, sizeof (ELAN_SLEEP));
@ -273,7 +272,6 @@ ompi_init_elan_sleepdesc (mca_ptl_elan_state_t * ems,
es->es_cmdBlk = ALLOC_ELAN (rail, E4_EVENTBLOCK_SIZE, es->es_cmdBlk = ALLOC_ELAN (rail, E4_EVENTBLOCK_SIZE,
E4_EVENTBLOCK_SIZE); E4_EVENTBLOCK_SIZE);
OMPI_PTL_ELAN_CHECK_UNEX (es->es_cmdBlk, 0, NULL, 0); OMPI_PTL_ELAN_CHECK_UNEX (es->es_cmdBlk, 0, NULL, 0);
/*Allocate a pair of command queues for blocking waits with */ /*Allocate a pair of command queues for blocking waits with */
@ -285,14 +283,13 @@ ompi_init_elan_sleepdesc (mca_ptl_elan_state_t * ems,
/* This command queue used to fire the IRQ via /* This command queue used to fire the IRQ via
a cmd port copy event */ a cmd port copy event */
es->es_ecmdq = elan4_alloc_cmdq (rail->r_ctx, rail->r_alloc, CQ_Size1K, /* CQ_EnableAllBits, */ es->es_ecmdq = elan4_alloc_cmdq (rail->r_ctx,
CQ_WriteEnableBit | rail->r_alloc, CQ_Size1K, /* CQ_EnableAllBits, */
CQ_InterruptEnableBit, NULL); CQ_WriteEnableBit | CQ_InterruptEnableBit, NULL);
OMPI_PTL_ELAN_CHECK_UNEX (es->es_ecmdq, NULL, NULL, 0); OMPI_PTL_ELAN_CHECK_UNEX (es->es_ecmdq, NULL, NULL, 0);
es->es_next = NULL; es->es_next = NULL;
/* XXX: asking the caller to release the lock */ /* XXX: asking the caller to release the lock */
return es; return es;
} }
@ -310,7 +307,7 @@ ompi_mca_ptl_elan_init (mca_ptl_elan_component_t * emp)
mca_ptl_elan_state_t *ems; mca_ptl_elan_state_t *ems;
START_FUNC(PTL_ELAN_DEBUG_NONE); START_FUNC(PTL_ELAN_DEBUG_INIT);
ems = &mca_ptl_elan_global_state; ems = &mca_ptl_elan_global_state;
@ -374,8 +371,8 @@ ompi_mca_ptl_elan_init (mca_ptl_elan_component_t * emp)
OMPI_PTL_ELAN_CHECK_UNEX (ems->all_rails, NULL, OMPI_PTL_ELAN_CHECK_UNEX (ems->all_rails, NULL,
OMPI_ERR_OUT_OF_RESOURCE, 0); OMPI_ERR_OUT_OF_RESOURCE, 0);
ems->all_estates = (ADDR_SDRAM *) ems->all_estates = (ADDR_SDRAM *)
malloc (sizeof (ELAN_ESTATE *) * num_rails); malloc (sizeof (ELAN_ESTATE *) * num_rails);
OMPI_PTL_ELAN_CHECK_UNEX (ems->all_estates, NULL, OMPI_PTL_ELAN_CHECK_UNEX (ems->all_estates, NULL,
OMPI_ERR_OUT_OF_RESOURCE, 0); OMPI_ERR_OUT_OF_RESOURCE, 0);
@ -394,7 +391,7 @@ ompi_mca_ptl_elan_init (mca_ptl_elan_component_t * emp)
alloc_elansize = ELAN_ALIGNUP (ems->elan_size, ems->elan_pagesize); alloc_elansize = ELAN_ALIGNUP (ems->elan_size, ems->elan_pagesize);
alloc_elanbase = (ADDR_ELAN) ((uintptr_t) ems->elan_base); alloc_elanbase = (ADDR_ELAN) ((uintptr_t) ems->elan_base);
/* Magic quadrics number for the starting cookie value */ /* XXX: Magic Quadrics number for the starting cookie value */
ems->intcookie = 42; ems->intcookie = 42;
ems->rail_intcookie = (int *) malloc (sizeof (int) * (num_rails + 1)); ems->rail_intcookie = (int *) malloc (sizeof (int) * (num_rails + 1));
OMPI_PTL_ELAN_CHECK_UNEX (ems->rail_intcookie, NULL, OMPI_PTL_ELAN_CHECK_UNEX (ems->rail_intcookie, NULL,
@ -460,8 +457,8 @@ ompi_mca_ptl_elan_init (mca_ptl_elan_component_t * emp)
NULL); NULL);
OMPI_PTL_ELAN_CHECK_UNEX (rail->r_ecmdq, NULL, OMPI_ERROR, 0); OMPI_PTL_ELAN_CHECK_UNEX (rail->r_ecmdq, NULL, OMPI_ERROR, 0);
priv_estate->cport = MAIN2ELAN (rail->r_ctx, priv_estate->cport = elan4_main2elan (rail->r_ctx,
rail->r_ecmdq->cmdq_mapping); rail->r_ecmdq->cmdq_mapping);
/* Save the rail pointers */ /* Save the rail pointers */
ems->elan_rail[i] = (ELAN_RAIL *) rail; ems->elan_rail[i] = (ELAN_RAIL *) rail;
@ -484,7 +481,6 @@ ompi_mca_ptl_elan_init (mca_ptl_elan_component_t * emp)
rail->r_railNo = rails[i]; rail->r_railNo = rails[i];
{ {
/*ompi_elan_railtable_t *rt; */
struct railtable *rt; struct railtable *rt;
rt = (struct railtable *) malloc (sizeof (struct railtable)); rt = (struct railtable *) malloc (sizeof (struct railtable));
OMPI_PTL_ELAN_CHECK_UNEX (rt, NULL, OMPI_ERROR, 0); OMPI_PTL_ELAN_CHECK_UNEX (rt, NULL, OMPI_ERROR, 0);
@ -496,7 +492,7 @@ ompi_mca_ptl_elan_init (mca_ptl_elan_component_t * emp)
rt->rt_allRails = (RAIL **) & (ems->all_rails[i]); rt->rt_allRails = (RAIL **) & (ems->all_rails[i]);
rail->r_railTable = rt; rail->r_railTable = rt;
} }
} /* for each rail */ } /* for each rail */
/* Free the local variable */ /* Free the local variable */
free (rails); free (rails);
@ -504,8 +500,6 @@ ompi_mca_ptl_elan_init (mca_ptl_elan_component_t * emp)
ems->elan_ctx = ems->elan_rail[0]->rail_ctx; ems->elan_ctx = ems->elan_rail[0]->rail_ctx;
ems->elan_estate = (void *) ems->all_estates[0]; ems->elan_estate = (void *) ems->all_estates[0];
/* XXX: Initialize a list of null events here */
/* Attach to the device and open to the network */ /* Attach to the device and open to the network */
ompi_elan_attach_network (ems); ompi_elan_attach_network (ems);
@ -519,7 +513,7 @@ ompi_mca_ptl_elan_init (mca_ptl_elan_component_t * emp)
elan4_block_inputter (ems->elan_rail[i]->rail_ctx, 0); elan4_block_inputter (ems->elan_rail[i]->rail_ctx, 0);
} }
/* setup communication infrastructure and construct PTL's */ /* Setup communication infrastructure and construct PTL's */
if (OMPI_SUCCESS != ompi_mca_ptl_elan_setup (ems)) { if (OMPI_SUCCESS != ompi_mca_ptl_elan_setup (ems)) {
ompi_output (0, ompi_output (0,
"[%s:%d] error in setting up elan " "[%s:%d] error in setting up elan "
@ -528,7 +522,7 @@ ompi_mca_ptl_elan_init (mca_ptl_elan_component_t * emp)
return OMPI_ERROR; return OMPI_ERROR;
} }
END_FUNC(PTL_ELAN_DEBUG_NONE); END_FUNC(PTL_ELAN_DEBUG_INIT);
return (OMPI_SUCCESS); return (OMPI_SUCCESS);
} }
@ -539,6 +533,8 @@ ompi_mca_ptl_elan_finalize (mca_ptl_elan_component_t * emp)
int num_rails; int num_rails;
mca_ptl_elan_state_t *ems; mca_ptl_elan_state_t *ems;
START_FUNC(PTL_ELAN_DEBUG_FIN);
ems = &mca_ptl_elan_global_state; ems = &mca_ptl_elan_global_state;
num_rails = ems->elan_nrails; num_rails = ems->elan_nrails;
@ -564,7 +560,7 @@ ompi_mca_ptl_elan_finalize (mca_ptl_elan_component_t * emp)
elan4_destroyAllocator (rail->r_alloc); elan4_destroyAllocator (rail->r_alloc);
elan4_close_sdram (rail->r_sdram); elan4_close_sdram (rail->r_sdram);
/*elan4_fini (rail->r_ctx); Not working yet */ /*elan4_fini (rail->r_ctx); Not working yet from libelan */
/* Free the rail structure used one the array of pointers /* Free the rail structure used one the array of pointers
* to the RAILs, either all_rails for elan_rails */ * to the RAILs, either all_rails for elan_rails */
@ -576,6 +572,7 @@ ompi_mca_ptl_elan_finalize (mca_ptl_elan_component_t * emp)
free (ems->all_rails); free (ems->all_rails);
free (ems->elan_cap); free (ems->elan_cap);
END_FUNC(PTL_ELAN_DEBUG_FIN);
return (OMPI_SUCCESS); return (OMPI_SUCCESS);
} }

4
src/mca/ptl/elan/src/ptl_elan_priv.c
Просмотреть файл

@ -715,6 +715,7 @@ mca_ptl_elan_drain_recv (mca_ptl_elan_component_t * emp)
case MCA_PTL_HDR_TYPE_LAST: case MCA_PTL_HDR_TYPE_LAST:
/* a control fragment for a message */ /* a control fragment for a message */
mca_ptl_elan_last_frag (ptl, header); mca_ptl_elan_last_frag (ptl, header);
break;
default: default:
fprintf(stdout, "[%s:%d] unknow fragment type %d\n", fprintf(stdout, "[%s:%d] unknow fragment type %d\n",
__FILE__, __LINE__, __FILE__, __LINE__,
@ -832,6 +833,9 @@ mca_ptl_elan_update_desc (mca_ptl_elan_component_t * emp)
} /* end of the while loop */ } /* end of the while loop */
} /* end of the for loop */ } /* end of the for loop */
/* Have the putget list checking to be in the same function */
mca_ptl_elan_update_putget(emp);
END_FUNC(PTL_ELAN_DEBUG_NONE); END_FUNC(PTL_ELAN_DEBUG_NONE);
return OMPI_SUCCESS; return OMPI_SUCCESS;
} }

73
src/mca/ptl/elan/src/ptl_elan_priv.h
Просмотреть файл

@ -55,39 +55,62 @@
#define PTL_ELAN_DEBUG_CHAIN (0x800) #define PTL_ELAN_DEBUG_CHAIN (0x800)
/* For now only debug send's */ /* For now only debug send's */
#if 1
#define PTL_ELAN_DEBUG_FLAG PTL_ELAN_DEBUG_NONE
#else
#define PTL_ELAN_DEBUG_FLAG (PTL_ELAN_DEBUG_ACK \ #define PTL_ELAN_DEBUG_FLAG (PTL_ELAN_DEBUG_ACK \
| PTL_ELAN_DEBUG_SEND | PTL_ELAN_DEBUG_PUT | PTL_ELAN_DEBUG_RECV) | PTL_ELAN_DEBUG_SEND | PTL_ELAN_DEBUG_PUT | PTL_ELAN_DEBUG_RECV)
#endif
#define OMPI_PTL_ELAN_CHECK_UNEX(value, unexp, errno, output) \ #define LOG_PRINT(flag, args...) \
do { \ do { \
if (value == unexp) { \ if (PTL_ELAN_DEBUG_FLAG & flag) { \
ompi_output(output, \ char hostname[32]; gethostname(hostname, 32); \
"[%s:%d] received unexpect allocated value \n",\ fprintf(stderr, "[%s:%s:%d] ", \
__FILE__, __LINE__); \ hostname, __FUNCTION__, __LINE__); \
return errno; \ fprintf(stderr, args); \
} \ } \
} while (0) } while (0)
#define START_FUNC(flag) \ #define OMPI_PTL_ELAN_CHECK_UNEX(value, unexp, errno, output) \
do { \ do { \
if (PTL_ELAN_DEBUG_FLAG & flag) { \ if (value == unexp) { \
char hostname[32]; gethostname(hostname, 32); \ ompi_output(output, \
fprintf(stderr, "[%s:%s:%d] Entering ...\n", \ "[%s:%d] received unexpect allocated value \n",\
hostname, __FUNCTION__, __LINE__); \ __FILE__, __LINE__); \
} \ return errno; \
} while (0) } \
} while (0)
#define START_FUNC(flag) \
do { \
if (PTL_ELAN_DEBUG_FLAG & flag) { \
char hostname[32]; gethostname(hostname, 32); \
fprintf(stderr, "[%s:%s:%d] Entering ...\n", \
hostname, __FUNCTION__, __LINE__); \
} \
} while (0)
#define END_FUNC(flag) \
do { \
if (PTL_ELAN_DEBUG_FLAG & flag) { \
char hostname[32]; gethostname(hostname, 32); \
fprintf(stderr, "[%s:%s:%d] Completes ...\n", \
hostname, __FUNCTION__, __LINE__); \
} \
} while (0)
#define END_FUNC(flag) \
do { \
if (PTL_ELAN_DEBUG_FLAG & flag) { \
char hostname[32]; gethostname(hostname, 32); \
fprintf(stderr, "[%s:%s:%d] Completes ...\n", \
hostname, __FUNCTION__, __LINE__); \
} \
} while (0)
#define PTL_ELAN_INPUT_QUEUE_MAX (2048) #define PTL_ELAN_INPUT_QUEUE_MAX (2048)
#define PUTGET_THROTTLE (32)
#define ELAN_PTL_FASTPATH (0x1)
#define ELAN_QUEUE_LOST_SLOTS (1)
#define SLOT_ALIGN (128)
#define GET_MAX(a,b) ((a>b)? a:b)
#define GET_MIN(a,b) ((a<b)? a:b)
#define ALIGNUP(x,a) (((unsigned int)(x) + ((a)-1)) & (-(a)))
enum { enum {
/* the first four bits for type */ /* the first four bits for type */
MCA_PTL_ELAN_DESC_NULL = 0x00, MCA_PTL_ELAN_DESC_NULL = 0x00,