diff --git a/src/mca/ptl/elan/src/ptl_elan.c b/src/mca/ptl/elan/src/ptl_elan.c index 56c9679c09..98f0498a41 100644 --- a/src/mca/ptl/elan/src/ptl_elan.c +++ b/src/mca/ptl/elan/src/ptl_elan.c @@ -20,10 +20,6 @@ #include "ptl_elan_frag.h" #include "ptl_elan_priv.h" -#if 0 -#elif defined(ABC) -#endif - /* XXX: There must be multiple PTL's. This could be the template */ mca_ptl_elan_module_t mca_ptl_elan_module = { { @@ -288,7 +284,6 @@ mca_ptl_elan_put (struct mca_ptl_base_module_t *ptl, int rc = OMPI_SUCCESS; mca_ptl_elan_send_frag_t *desc; - /* PML still utilize this interface the same as a send option. * So we need to generate a QDMA to the remote side for completion * notification */ @@ -312,7 +307,7 @@ mca_ptl_elan_put (struct mca_ptl_base_module_t *ptl, sendreq, offset, &size, flags); /* Update all the sends until the put is done */ - END_FUNC(PTL_ELAN_DEBUG_NONE); + END_FUNC(PTL_ELAN_DEBUG_PUT); return rc; } @@ -330,30 +325,6 @@ mca_ptl_elan_get (struct mca_ptl_base_module_t *ptl, int flags) { int rc = OMPI_SUCCESS; -#if 0 - mca_ptl_elan_send_frag_t *desc; - - /* XXX: - * Since the address passed down from PML does not provide - * elan information, so there needs to be a change - */ - - START_FUNC(PTL_ELAN_DEBUG_NONE); - - desc = mca_ptl_elan_alloc_send_desc(ptl, sendreq, MCA_PTL_ELAN_DESC_GET); - if (NULL == desc) { - ompi_output(0, - "[%s:%d] Unable to allocate an elan send descriptors \n", - __FILE__, __LINE__); - } - - rc = mca_ptl_elan_start_desc(desc, - (struct mca_ptl_elan_peer_t *)ptl_peer, - sendreq, offset, &size, flags); - - /* Update all the sends until the put is done */ - END_FUNC(PTL_ELAN_DEBUG_NONE); -#endif return rc; } @@ -399,8 +370,9 @@ mca_ptl_elan_matched (mca_ptl_base_module_t * ptl, #endif if (header->hdr_common.hdr_flags & MCA_PTL_FLAGS_ACK_MATCHED) -#if 1 /* Basic ACK scheme following TCP cases */ +#if 1 { + /* Basic ACK scheme following TCP cases */ mca_ptl_elan_send_frag_t *desc; /* Get a frag desc and allocate a send desc */ diff --git a/src/mca/ptl/elan/src/ptl_elan_comm_init.c b/src/mca/ptl/elan/src/ptl_elan_comm_init.c index 1469d99c0e..b8ce494491 100644 --- a/src/mca/ptl/elan/src/ptl_elan_comm_init.c +++ b/src/mca/ptl/elan/src/ptl_elan_comm_init.c @@ -9,14 +9,6 @@ #include "ptl_elan.h" #include "ptl_elan_priv.h" -#define PUTGET_THROTTLE (32) -#define ELAN_PTL_FASTPATH (0x1) -#define ELAN_QUEUE_MAX (INPUT_QUEUE_MAX) -#define ELAN_QUEUE_LOST_SLOTS (1) -#define SLOT_ALIGN (128) -#define MAX(a,b) ((a>b)? a:b) -#define ALIGNUP(x,a) (((unsigned int)(x) + ((a)-1)) & (-(a))) - #define OMPI_PTL_ELAN_CTRL_LIST(flist, init_num, inc_num, max_num) \ do { \ OBJ_CONSTRUCT (flist, ompi_free_list_t); \ @@ -47,7 +39,7 @@ ompi_init_elan_queue_events (mca_ptl_elan_module_t * ptl, ompi_ptl_elan_qdma_desc_t *desc; E4_Event *elan_ptr; - START_FUNC(PTL_ELAN_DEBUG_NONE); + START_FUNC(PTL_ELAN_DEBUG_INIT); rail = (RAIL *) ptl->ptl_elan_rail; ctx = (ELAN4_CTX *) ptl->ptl_elan_ctx; @@ -58,8 +50,8 @@ ompi_init_elan_queue_events (mca_ptl_elan_module_t * ptl, flist = &queue->tx_desc_free; - main_align = MAX (sizeof (void *), 8); - elan_align = MAX (sizeof (int *), ELAN_BLOCK_ALIGN); + main_align = GET_MAX (sizeof (void *), 8); + elan_align = GET_MAX (sizeof (int *), ELAN_BLOCK_ALIGN); main_size = ALIGNUP (sizeof (ompi_ptl_elan_qdma_desc_t), main_align); elan_size = ALIGNUP (sizeof (E4_Event), elan_align); @@ -113,7 +105,7 @@ ompi_init_elan_queue_events (mca_ptl_elan_module_t * ptl, } flist->fl_num_allocated += flist->fl_num_per_alloc; - END_FUNC(PTL_ELAN_DEBUG_NONE); + END_FUNC(PTL_ELAN_DEBUG_INIT); return OMPI_SUCCESS; } @@ -199,10 +191,10 @@ ompi_ptl_elan_init_putget_ctrl (mca_ptl_elan_module_t * ptl, ompi_free_list_t *put_list, *get_list; ompi_ptl_elan_putget_desc_t *put_desc, *get_desc; - START_FUNC(PTL_ELAN_DEBUG_NONE); + START_FUNC(PTL_ELAN_DEBUG_INIT); - main_align = MAX (sizeof (void *), ELAN_ALIGN); - elan_align = MAX (sizeof (int *), ELAN_BLOCK_ALIGN); + main_align = GET_MAX (sizeof (void *), ELAN_ALIGN); + elan_align = GET_MAX (sizeof (int *), ELAN_BLOCK_ALIGN); main_size = ALIGNUP(sizeof(ompi_ptl_elan_putget_desc_t), main_align); /* Contain elan_event, chain_event and a chain_buff */ @@ -254,7 +246,7 @@ ompi_ptl_elan_init_putget_ctrl (mca_ptl_elan_module_t * ptl, OMPI_ELAN_PUTGET_GROW(ctx, get_list, frag, get_desc, elan_ptr, main_size, elan_size, 0); - END_FUNC(PTL_ELAN_DEBUG_NONE); + END_FUNC (PTL_ELAN_DEBUG_INIT); return OMPI_SUCCESS; } @@ -278,7 +270,7 @@ ompi_init_elan_qdma (mca_ptl_elan_component_t * emp, ELAN4_CTX *ctx; struct mca_ptl_elan_module_t *ptl; - START_FUNC(PTL_ELAN_DEBUG_NONE); + START_FUNC(PTL_ELAN_DEBUG_INIT); /* Init the Transmit Queue structure */ for (i = 0; i < num_rails; i++) { @@ -390,13 +382,13 @@ ompi_init_elan_qdma (mca_ptl_elan_component_t * emp, OBJ_CONSTRUCT (&queue->rx_lock, ompi_mutex_t); } - END_FUNC(PTL_ELAN_DEBUG_NONE); + END_FUNC(PTL_ELAN_DEBUG_INIT); return (OMPI_SUCCESS); } int ompi_init_elan_putget (mca_ptl_elan_component_t * emp, - int num_rails) + int num_rails) { int i; int nslots = 128; @@ -405,7 +397,7 @@ ompi_init_elan_putget (mca_ptl_elan_component_t * emp, ELAN4_CTX *ctx; struct mca_ptl_elan_module_t *ptl; - START_FUNC(PTL_ELAN_DEBUG_NONE); + START_FUNC(PTL_ELAN_DEBUG_INIT); /* Init the Transmit Queue structure */ for (i = 0; i < num_rails; i++) { @@ -469,7 +461,7 @@ ompi_init_elan_putget (mca_ptl_elan_component_t * emp, ompi_ptl_elan_init_putget_ctrl (ptl, rail, putget, 0, 2, 32); } - END_FUNC(PTL_ELAN_DEBUG_NONE); + END_FUNC(PTL_ELAN_DEBUG_INIT); return (OMPI_SUCCESS); } diff --git a/src/mca/ptl/elan/src/ptl_elan_component.c b/src/mca/ptl/elan/src/ptl_elan_component.c index 8c030c86bb..14a705cbc3 100644 --- a/src/mca/ptl/elan/src/ptl_elan_component.c +++ b/src/mca/ptl/elan/src/ptl_elan_component.c @@ -60,9 +60,10 @@ static mca_ptl_elan_component_t *elan_mp = &mca_ptl_elan_component; static bool mca_ptl_elan_component_initialized = false; /* - * some elan vp information to the the global registery + * XXX: Leave it as a routine for possible extension + * some elan vp information to the the global registery */ -static int mca_ptl_elan_component_register (mca_ptl_elan_component_t *emp) +static int mca_ptl_elan_addr_put (mca_ptl_elan_component_t *emp) { int rc; size_t i; @@ -94,27 +95,28 @@ static int mca_ptl_elan_component_register (mca_ptl_elan_component_t *emp) int mca_ptl_elan_component_open (void) { - /* FIXME: register the default super module parameters, - * Some sanity checking is needed to ensure that user - * would not provide unrealistic parameters.*/ + int length; + int param1, param2, param3; + mca_ptl_elan_module.super.ptl_exclusivity = mca_ptl_elan_param_register_int ("exclusivity", 0); - mca_ptl_elan_module.super.ptl_first_frag_size = - mca_ptl_elan_param_register_int ("first_frag_size", - (PTL_ELAN_INPUT_QUEUE_MAX - sizeof(mca_ptl_base_header_t))); - mca_ptl_elan_module.super.ptl_min_frag_size = - mca_ptl_elan_param_register_int ("min_frag_size", - (PTL_ELAN_INPUT_QUEUE_MAX - sizeof(mca_ptl_base_header_t))); - mca_ptl_elan_module.super.ptl_max_frag_size = - mca_ptl_elan_param_register_int ("max_frag_size", 2<<31); - /* register ELAN module parameters */ - elan_mp->elan_free_list_num = - mca_ptl_elan_param_register_int ("free_list_num", 32); - elan_mp->elan_free_list_max = - mca_ptl_elan_param_register_int ("free_list_max", 128); - elan_mp->elan_free_list_inc = - mca_ptl_elan_param_register_int ("free_list_inc", 32); + length = PTL_ELAN_INPUT_QUEUE_MAX - sizeof(mca_ptl_base_header_t); + param1 = mca_ptl_elan_param_register_int ("first_frag_size", length); + param2 = mca_ptl_elan_param_register_int ("min_frag_size", length); + param3 = mca_ptl_elan_param_register_int ("max_frag_size", 2<<31); + + /* Correct these if user give violent parameters */ + mca_ptl_elan_module.super.ptl_first_frag_size = GET_MIN(param1, length); + mca_ptl_elan_module.super.ptl_min_frag_size = GET_MAX(param2, length); + mca_ptl_elan_module.super.ptl_max_frag_size = GET_MIN(param3, 2<<31); + + /* XXX: Do not want to make these configuraable, since they are + * very much related to number of outstanding operations in elan4 + * and therefore flow control */ + elan_mp->elan_free_list_num = 32; + elan_mp->elan_free_list_max = 128; + elan_mp->elan_free_list_inc = 32; /* initialize state */ elan_mp->elan_ptl_modules = NULL; @@ -142,8 +144,6 @@ mca_ptl_elan_component_close (void) /* cleanup the proc, ptl, and the module */ ompi_mca_ptl_elan_finalize (&mca_ptl_elan_component); - - /* XXX: Make sure this is not just an alias pointer */ if (elan_mp->elan_local) { free (elan_mp->elan_local); } @@ -157,25 +157,23 @@ mca_ptl_elan_component_close (void) } } - if (elan_mp->elan_recv_frags_free.fl_num_allocated != - elan_mp->elan_recv_frags_free.super.ompi_list_length) { - ompi_output (0, "[%s:%d] recv_frags : %d allocated %d returned\n", - __FILE__, __LINE__, - elan_mp->elan_recv_frags_free.fl_num_allocated, - elan_mp->elan_recv_frags_free.super.ompi_list_length); - } - - /* FIXME: free free_list entries before destructing lists */ - /* Free the empty list holders */ OBJ_DESTRUCT (&(elan_mp->elan_procs)); OBJ_DESTRUCT (&(elan_mp->elan_pending_acks)); OBJ_DESTRUCT (&(elan_mp->elan_send_frags)); OBJ_DESTRUCT (&(elan_mp->elan_recv_frags)); - /* TODO: + /* FIXME: * We need free all the memory allocated for this list * before desctructing this free_list */ + if (elan_mp->elan_recv_frags_free.fl_num_allocated != + elan_mp->elan_recv_frags_free.super.ompi_list_length) { + ompi_output (0, + "[%s:%d] recv_frags : %d allocated %d returned\n", + __FILE__, __LINE__, + elan_mp->elan_recv_frags_free.fl_num_allocated, + elan_mp->elan_recv_frags_free.super.ompi_list_length); + } OBJ_DESTRUCT (&(elan_mp->elan_recv_frags_free)); /* Destruct other structures */ @@ -192,23 +190,15 @@ mca_ptl_elan_component_close (void) */ mca_ptl_base_module_t ** mca_ptl_elan_component_init (int *num_ptl_modules, - bool * allow_multi_user_threads, - bool * have_hidden_threads) + bool * allow_multi_user_threads, + bool * have_hidden_threads) { mca_ptl_base_module_t **ptls; - *num_ptl_modules = 0; - - START_FUNC(PTL_ELAN_DEBUG_NONE); - - if (PTL_ELAN_DEBUG_FLAG & PTL_ELAN_DEBUG_INIT) { - char hostname[32]; gethostname(hostname, 32); - fprintf(stderr, "[%s:%s:%d] debugging ...\n", - hostname, __FUNCTION__, __LINE__); - } + START_FUNC(PTL_ELAN_DEBUG_INIT); /* TODO: support multiple threads */ - + *num_ptl_modules = 0; *allow_multi_user_threads = true; *have_hidden_threads = OMPI_HAVE_THREADS; @@ -227,8 +217,7 @@ mca_ptl_elan_component_init (int *num_ptl_modules, return NULL; } - if (OMPI_SUCCESS != - mca_ptl_elan_component_register(&mca_ptl_elan_component)) { + if (OMPI_SUCCESS != mca_ptl_elan_addr_put(&mca_ptl_elan_component)) { ompi_output(0, "[%s:%d] error in registering with Runtime/OOB \n", __FILE__, __LINE__); @@ -238,9 +227,9 @@ mca_ptl_elan_component_init (int *num_ptl_modules, ptls = (mca_ptl_base_module_t **) malloc (elan_mp->elan_num_ptl_modules * sizeof (mca_ptl_elan_module_t *)); if (NULL == ptls) { - ompi_output(0, - "[%s:%d] error in malloc for elan PTL references\n", - __FILE__, __LINE__); + ompi_output(0, + "[%s:%d] error in allocating memory \n", + __FILE__, __LINE__); return NULL; } @@ -249,7 +238,7 @@ mca_ptl_elan_component_init (int *num_ptl_modules, *num_ptl_modules = elan_mp->elan_num_ptl_modules; mca_ptl_elan_component_initialized = true; - END_FUNC(PTL_ELAN_DEBUG_NONE); + END_FUNC(PTL_ELAN_DEBUG_INIT); return ptls; } @@ -269,15 +258,12 @@ mca_ptl_elan_component_control (int param, /* TODO: to support event-based module progress later. */ - -static int times = 0; int mca_ptl_elan_component_progress (mca_ptl_tstamp_t tstamp) { - START_FUNC(PTL_ELAN_DEBUG_NONE); + START_FUNC (PTL_ELAN_DEBUG_NONE); mca_ptl_elan_drain_recv(elan_mp); mca_ptl_elan_update_desc(elan_mp); - mca_ptl_elan_update_putget(elan_mp); - END_FUNC(PTL_ELAN_DEBUG_NONE); + END_FUNC (PTL_ELAN_DEBUG_NONE); return OMPI_SUCCESS; } diff --git a/src/mca/ptl/elan/src/ptl_elan_init.c b/src/mca/ptl/elan/src/ptl_elan_init.c index 5692432628..c37f947743 100644 --- a/src/mca/ptl/elan/src/ptl_elan_init.c +++ b/src/mca/ptl/elan/src/ptl_elan_init.c @@ -6,15 +6,10 @@ #include #include -#define _ELAN4 -#define __elan4 - #include "ptl_elan.h" #include "ptl_elan_priv.h" -#define ELAN_QUEUE_MAX INPUT_QUEUE_MAX -#define ELAN_QUEUE_LOST_SLOTS 1 -#define SLOT_ALIGN 128 +#define _ELAN4 mca_ptl_elan_state_t mca_ptl_elan_global_state; @@ -24,7 +19,7 @@ ompi_mca_ptl_elan_setup (mca_ptl_elan_state_t * ems) mca_ptl_elan_component_t *emp; int rail_count; - START_FUNC(PTL_ELAN_DEBUG_NONE); + START_FUNC(PTL_ELAN_DEBUG_INIT); rail_count = ems->elan_nrails; emp = ems->elan_component; @@ -94,19 +89,24 @@ ompi_mca_ptl_elan_setup (mca_ptl_elan_state_t * ems) return OMPI_ERROR; } - END_FUNC(PTL_ELAN_DEBUG_NONE); + END_FUNC(PTL_ELAN_DEBUG_INIT); return (OMPI_SUCCESS); } /* Attach to the network */ +static int elan_attached = 0; static int ompi_elan_attach_network (mca_ptl_elan_state_t * ems) { - static int elan_attached = 0; - int i, vp, *vps, num_rails; - ELAN_LOCATION loc; - ELAN_CAPABILITY *cap = ems->elan_cap; + int i; + int vp; + int *vps; + int num_rails; + ELAN_LOCATION loc; + ELAN_CAPABILITY *cap; + + cap = ems->elan_cap; num_rails = ems->elan_nrails; if (elan_attached) { @@ -238,14 +238,14 @@ static void ompi_module_elan_close_ptls (mca_ptl_elan_component_t * emp, int num_rails) { - /* TODO: find the ones that are still there and free them */ + /* FIXME: find the ones that are still there and free them */ } static void ompi_module_elan_close_procs (mca_ptl_elan_component_t * emp, int num_rails) { - /* TODO: find the ones that are still there and free them */ + /* FIXME: find the ones that are still there and free them */ } ELAN_SLEEP * @@ -255,7 +255,6 @@ ompi_init_elan_sleepdesc (mca_ptl_elan_state_t * ems, ELAN_SLEEP *es; /* XXX: asking the caller to hold the lock */ - es = MALLOC (sizeof (ELAN_SLEEP)); OMPI_PTL_ELAN_CHECK_UNEX (es, NULL, NULL, 0); memset (es, 0, sizeof (ELAN_SLEEP)); @@ -273,7 +272,6 @@ ompi_init_elan_sleepdesc (mca_ptl_elan_state_t * ems, es->es_cmdBlk = ALLOC_ELAN (rail, E4_EVENTBLOCK_SIZE, E4_EVENTBLOCK_SIZE); - OMPI_PTL_ELAN_CHECK_UNEX (es->es_cmdBlk, 0, NULL, 0); /*Allocate a pair of command queues for blocking waits with */ @@ -285,14 +283,13 @@ ompi_init_elan_sleepdesc (mca_ptl_elan_state_t * ems, /* This command queue used to fire the IRQ via a cmd port copy event */ - es->es_ecmdq = elan4_alloc_cmdq (rail->r_ctx, rail->r_alloc, CQ_Size1K, /* CQ_EnableAllBits, */ - CQ_WriteEnableBit | - CQ_InterruptEnableBit, NULL); + es->es_ecmdq = elan4_alloc_cmdq (rail->r_ctx, + rail->r_alloc, CQ_Size1K, /* CQ_EnableAllBits, */ + CQ_WriteEnableBit | CQ_InterruptEnableBit, NULL); OMPI_PTL_ELAN_CHECK_UNEX (es->es_ecmdq, NULL, NULL, 0); es->es_next = NULL; /* XXX: asking the caller to release the lock */ - return es; } @@ -310,7 +307,7 @@ ompi_mca_ptl_elan_init (mca_ptl_elan_component_t * emp) mca_ptl_elan_state_t *ems; - START_FUNC(PTL_ELAN_DEBUG_NONE); + START_FUNC(PTL_ELAN_DEBUG_INIT); ems = &mca_ptl_elan_global_state; @@ -374,8 +371,8 @@ ompi_mca_ptl_elan_init (mca_ptl_elan_component_t * emp) OMPI_PTL_ELAN_CHECK_UNEX (ems->all_rails, NULL, OMPI_ERR_OUT_OF_RESOURCE, 0); - ems->all_estates = (ADDR_SDRAM *) - malloc (sizeof (ELAN_ESTATE *) * num_rails); + ems->all_estates = (ADDR_SDRAM *) + malloc (sizeof (ELAN_ESTATE *) * num_rails); OMPI_PTL_ELAN_CHECK_UNEX (ems->all_estates, NULL, OMPI_ERR_OUT_OF_RESOURCE, 0); @@ -394,7 +391,7 @@ ompi_mca_ptl_elan_init (mca_ptl_elan_component_t * emp) alloc_elansize = ELAN_ALIGNUP (ems->elan_size, ems->elan_pagesize); alloc_elanbase = (ADDR_ELAN) ((uintptr_t) ems->elan_base); - /* Magic quadrics number for the starting cookie value */ + /* XXX: Magic Quadrics number for the starting cookie value */ ems->intcookie = 42; ems->rail_intcookie = (int *) malloc (sizeof (int) * (num_rails + 1)); OMPI_PTL_ELAN_CHECK_UNEX (ems->rail_intcookie, NULL, @@ -460,8 +457,8 @@ ompi_mca_ptl_elan_init (mca_ptl_elan_component_t * emp) NULL); OMPI_PTL_ELAN_CHECK_UNEX (rail->r_ecmdq, NULL, OMPI_ERROR, 0); - priv_estate->cport = MAIN2ELAN (rail->r_ctx, - rail->r_ecmdq->cmdq_mapping); + priv_estate->cport = elan4_main2elan (rail->r_ctx, + rail->r_ecmdq->cmdq_mapping); /* Save the rail pointers */ ems->elan_rail[i] = (ELAN_RAIL *) rail; @@ -484,7 +481,6 @@ ompi_mca_ptl_elan_init (mca_ptl_elan_component_t * emp) rail->r_railNo = rails[i]; { - /*ompi_elan_railtable_t *rt; */ struct railtable *rt; rt = (struct railtable *) malloc (sizeof (struct railtable)); OMPI_PTL_ELAN_CHECK_UNEX (rt, NULL, OMPI_ERROR, 0); @@ -496,7 +492,7 @@ ompi_mca_ptl_elan_init (mca_ptl_elan_component_t * emp) rt->rt_allRails = (RAIL **) & (ems->all_rails[i]); rail->r_railTable = rt; } - } /* for each rail */ + } /* for each rail */ /* Free the local variable */ free (rails); @@ -504,8 +500,6 @@ ompi_mca_ptl_elan_init (mca_ptl_elan_component_t * emp) ems->elan_ctx = ems->elan_rail[0]->rail_ctx; ems->elan_estate = (void *) ems->all_estates[0]; - /* XXX: Initialize a list of null events here */ - /* Attach to the device and open to the network */ ompi_elan_attach_network (ems); @@ -519,7 +513,7 @@ ompi_mca_ptl_elan_init (mca_ptl_elan_component_t * emp) elan4_block_inputter (ems->elan_rail[i]->rail_ctx, 0); } - /* setup communication infrastructure and construct PTL's */ + /* Setup communication infrastructure and construct PTL's */ if (OMPI_SUCCESS != ompi_mca_ptl_elan_setup (ems)) { ompi_output (0, "[%s:%d] error in setting up elan " @@ -528,7 +522,7 @@ ompi_mca_ptl_elan_init (mca_ptl_elan_component_t * emp) return OMPI_ERROR; } - END_FUNC(PTL_ELAN_DEBUG_NONE); + END_FUNC(PTL_ELAN_DEBUG_INIT); return (OMPI_SUCCESS); } @@ -539,6 +533,8 @@ ompi_mca_ptl_elan_finalize (mca_ptl_elan_component_t * emp) int num_rails; mca_ptl_elan_state_t *ems; + START_FUNC(PTL_ELAN_DEBUG_FIN); + ems = &mca_ptl_elan_global_state; num_rails = ems->elan_nrails; @@ -564,7 +560,7 @@ ompi_mca_ptl_elan_finalize (mca_ptl_elan_component_t * emp) elan4_destroyAllocator (rail->r_alloc); elan4_close_sdram (rail->r_sdram); - /*elan4_fini (rail->r_ctx); Not working yet */ + /*elan4_fini (rail->r_ctx); Not working yet from libelan */ /* Free the rail structure used one the array of pointers * to the RAILs, either all_rails for elan_rails */ @@ -576,6 +572,7 @@ ompi_mca_ptl_elan_finalize (mca_ptl_elan_component_t * emp) free (ems->all_rails); free (ems->elan_cap); + END_FUNC(PTL_ELAN_DEBUG_FIN); return (OMPI_SUCCESS); } diff --git a/src/mca/ptl/elan/src/ptl_elan_priv.c b/src/mca/ptl/elan/src/ptl_elan_priv.c index 192e6b8672..2ff8e62b21 100644 --- a/src/mca/ptl/elan/src/ptl_elan_priv.c +++ b/src/mca/ptl/elan/src/ptl_elan_priv.c @@ -715,6 +715,7 @@ mca_ptl_elan_drain_recv (mca_ptl_elan_component_t * emp) case MCA_PTL_HDR_TYPE_LAST: /* a control fragment for a message */ mca_ptl_elan_last_frag (ptl, header); + break; default: fprintf(stdout, "[%s:%d] unknow fragment type %d\n", __FILE__, __LINE__, @@ -832,6 +833,9 @@ mca_ptl_elan_update_desc (mca_ptl_elan_component_t * emp) } /* end of the while loop */ } /* end of the for loop */ + /* Have the putget list checking to be in the same function */ + mca_ptl_elan_update_putget(emp); + END_FUNC(PTL_ELAN_DEBUG_NONE); return OMPI_SUCCESS; } diff --git a/src/mca/ptl/elan/src/ptl_elan_priv.h b/src/mca/ptl/elan/src/ptl_elan_priv.h index d2bb28b41d..9a5e355290 100644 --- a/src/mca/ptl/elan/src/ptl_elan_priv.h +++ b/src/mca/ptl/elan/src/ptl_elan_priv.h @@ -55,39 +55,62 @@ #define PTL_ELAN_DEBUG_CHAIN (0x800) /* For now only debug send's */ +#if 1 +#define PTL_ELAN_DEBUG_FLAG PTL_ELAN_DEBUG_NONE +#else #define PTL_ELAN_DEBUG_FLAG (PTL_ELAN_DEBUG_ACK \ | PTL_ELAN_DEBUG_SEND | PTL_ELAN_DEBUG_PUT | PTL_ELAN_DEBUG_RECV) +#endif -#define OMPI_PTL_ELAN_CHECK_UNEX(value, unexp, errno, output) \ - do { \ - if (value == unexp) { \ - ompi_output(output, \ - "[%s:%d] received unexpect allocated value \n",\ - __FILE__, __LINE__); \ - return errno; \ - } \ - } while (0) +#define LOG_PRINT(flag, args...) \ +do { \ + if (PTL_ELAN_DEBUG_FLAG & flag) { \ + char hostname[32]; gethostname(hostname, 32); \ + fprintf(stderr, "[%s:%s:%d] ", \ + hostname, __FUNCTION__, __LINE__); \ + fprintf(stderr, args); \ + } \ +} while (0) -#define START_FUNC(flag) \ - do { \ - if (PTL_ELAN_DEBUG_FLAG & flag) { \ - char hostname[32]; gethostname(hostname, 32); \ - fprintf(stderr, "[%s:%s:%d] Entering ...\n", \ - hostname, __FUNCTION__, __LINE__); \ - } \ - } while (0) +#define OMPI_PTL_ELAN_CHECK_UNEX(value, unexp, errno, output) \ +do { \ + if (value == unexp) { \ + ompi_output(output, \ + "[%s:%d] received unexpect allocated value \n",\ + __FILE__, __LINE__); \ + return errno; \ + } \ +} while (0) + +#define START_FUNC(flag) \ +do { \ + if (PTL_ELAN_DEBUG_FLAG & flag) { \ + char hostname[32]; gethostname(hostname, 32); \ + fprintf(stderr, "[%s:%s:%d] Entering ...\n", \ + hostname, __FUNCTION__, __LINE__); \ + } \ +} while (0) + +#define END_FUNC(flag) \ +do { \ + if (PTL_ELAN_DEBUG_FLAG & flag) { \ + char hostname[32]; gethostname(hostname, 32); \ + fprintf(stderr, "[%s:%s:%d] Completes ...\n", \ + hostname, __FUNCTION__, __LINE__); \ + } \ +} while (0) -#define END_FUNC(flag) \ - do { \ - if (PTL_ELAN_DEBUG_FLAG & flag) { \ - char hostname[32]; gethostname(hostname, 32); \ - fprintf(stderr, "[%s:%s:%d] Completes ...\n", \ - hostname, __FUNCTION__, __LINE__); \ - } \ - } while (0) #define PTL_ELAN_INPUT_QUEUE_MAX (2048) +#define PUTGET_THROTTLE (32) +#define ELAN_PTL_FASTPATH (0x1) +#define ELAN_QUEUE_LOST_SLOTS (1) +#define SLOT_ALIGN (128) +#define GET_MAX(a,b) ((a>b)? a:b) +#define GET_MIN(a,b) ((a