diff --git a/src/mca/ptl/elan/src/ptl_elan.c b/src/mca/ptl/elan/src/ptl_elan.c index 8ae3f8a82e..b0861e038f 100644 --- a/src/mca/ptl/elan/src/ptl_elan.c +++ b/src/mca/ptl/elan/src/ptl_elan.c @@ -149,7 +149,7 @@ mca_ptl_elan_finalize (struct mca_ptl_base_module_t *ptl) START_FUNC(PTL_ELAN_DEBUG_FIN); - /* FIXME: move from ptl_elan_component_close() + /* TODO: move from mca_ptl_elan_component_close() all the ptl_elan_finalize related code here */ #if 0 elan_ptl = (struct mca_ptl_elan_module_t *) ptl; diff --git a/src/mca/ptl/elan/src/ptl_elan_component.c b/src/mca/ptl/elan/src/ptl_elan_component.c index 4325b1b915..d1ad04b64e 100644 --- a/src/mca/ptl/elan/src/ptl_elan_component.c +++ b/src/mca/ptl/elan/src/ptl_elan_component.c @@ -167,8 +167,8 @@ mca_ptl_elan_component_close (void) if (elan_mp->elan_recv_frags_free.fl_num_allocated != elan_mp->elan_recv_frags_free.super.ompi_list_length) { ompi_output (0, - "[proc%s:%s:%d] recv_frags : %d allocated %d returned\n", - getenv("RMS_RANK"), __FILE__, __LINE__, + "[%s:%d] recv_frags : %d allocated %d returned\n", + __FILE__, __LINE__, elan_mp->elan_recv_frags_free.fl_num_allocated, elan_mp->elan_recv_frags_free.super.ompi_list_length); } diff --git a/src/mca/ptl/elan/src/ptl_elan_frag.c b/src/mca/ptl/elan/src/ptl_elan_frag.c index c6ee1f92f4..f1c654c059 100644 --- a/src/mca/ptl/elan/src/ptl_elan_frag.c +++ b/src/mca/ptl/elan/src/ptl_elan_frag.c @@ -86,8 +86,6 @@ ompi_class_t mca_ptl_elan_recv_frag_t_class = { extern mca_ptl_elan_state_t mca_ptl_elan_global_state; -/* FIXME: Even though we have also get implemened here, temporarily - * mca_ptl_elan_send_frag_t is still used as the return type */ mca_ptl_elan_send_frag_t * mca_ptl_elan_alloc_desc (struct mca_ptl_base_module_t *ptl_ptr, struct mca_pml_base_request_t *req, int desc_type) @@ -165,6 +163,7 @@ mca_ptl_elan_send_desc_done ( ptl = ((ompi_ptl_elan_qdma_desc_t *)frag->desc)->ptl; header = &frag->frag_base.frag_header; +#if OMPI_PTL_ELAN_ENABLE_GET if (frag->desc->desc_type == MCA_PTL_ELAN_DESC_GET) { if(ompi_atomic_fetch_and_set_int (&frag->frag_progressed, 1) == 0) { ptl->super.ptl_recv_progress(ptl, @@ -180,6 +179,7 @@ mca_ptl_elan_send_desc_done ( END_FUNC(PTL_ELAN_DEBUG_SEND); return; } +#endif LOG_PRINT(PTL_ELAN_DEBUG_SEND, "req %p done frag %p desc %p desc_type %d length %d\n", @@ -205,8 +205,7 @@ mca_ptl_elan_send_desc_done ( frag, frag->desc, frag->desc->desc_type); /* Return a frag or if not cached, or it is a follow up */ - if ( /*(header->hdr_frag.hdr_frag_offset != 0) || */ - (frag->desc->desc_status != MCA_PTL_ELAN_DESC_CACHED)){ + if ( (frag->desc->desc_status != MCA_PTL_ELAN_DESC_CACHED)){ ompi_free_list_t *flist; if (frag->desc->desc_type == MCA_PTL_ELAN_DESC_PUT) { flist = &ptl->putget->put_desc_free; @@ -226,9 +225,7 @@ mca_ptl_elan_send_desc_done ( } #else else { - - /* XXX: need to discuss this with TSW. - * There is a little confusion here. + /* XXX: * Why the release of this send fragment is dependent * on the receiving of an acknowledgement * There are two drawbacks, @@ -256,7 +253,6 @@ mca_ptl_elan_send_desc_done ( (ompi_list_item_t *) frag); } #endif - END_FUNC(PTL_ELAN_DEBUG_SEND); } diff --git a/src/mca/ptl/elan/src/ptl_elan_frag.h b/src/mca/ptl/elan/src/ptl_elan_frag.h index 11cc9211f3..086e65e35f 100644 --- a/src/mca/ptl/elan/src/ptl_elan_frag.h +++ b/src/mca/ptl/elan/src/ptl_elan_frag.h @@ -61,7 +61,7 @@ mca_ptl_elan_recv_frag_t * mca_ptl_elan_alloc_recv_desc(struct mca_pml_base_recv_request_t *req); /** - * FIXME: Change frag to be a struct + * TODO: Change frag to be a struct * ELAN send request derived type. The send request contains * the base send request and a point to the elan fragment descriptor */ diff --git a/src/mca/ptl/elan/src/ptl_elan_init.c b/src/mca/ptl/elan/src/ptl_elan_init.c index baf089931e..7ccd17420c 100644 --- a/src/mca/ptl/elan/src/ptl_elan_init.c +++ b/src/mca/ptl/elan/src/ptl_elan_init.c @@ -78,16 +78,11 @@ ompi_mca_ptl_elan_setup (mca_ptl_elan_state_t * ems) return OMPI_ERROR; } - /* - * XXX: Leave the following later after testing of QDMA is done - */ if (OMPI_SUCCESS != ompi_init_elan_putget (emp, rail_count)) { return OMPI_ERROR; } - /* - * XXX: initialize STAT (including SYNC) structures. - */ + /* TODO: initialize STAT (including SYNC) structures. */ if (OMPI_SUCCESS != ompi_init_elan_stat (emp, rail_count)) { return OMPI_ERROR; } @@ -243,13 +238,17 @@ ompi_module_elan_close_ptls (mca_ptl_elan_component_t * emp, int i; struct mca_ptl_elan_module_t *ptl; - /* FIXME: find the ones that are still there and free them */ for (i = 0; i < num_rails; i ++ ) { ptl = emp->modules[i]; if (NULL == ptl) continue; + + /* TODO: Deconstruct the module: ptl->super; */ OBJ_DESTRUCT (&(ptl->recv_frags)); OBJ_DESTRUCT (&(ptl->send_frags)); OBJ_DESTRUCT (&(ptl->pending_acks)); + + /* TODO: Free send/recv/comp queues */ + /*ptl->comp; ptl->queue; ptl->putget;*/ } } @@ -257,7 +256,7 @@ static void ompi_module_elan_close_procs (mca_ptl_elan_component_t * emp, int num_rails) { - /* FIXME: find the ones that are still there and free them */ + /* FIXME: find left procs and free them */ } static int diff --git a/src/mca/ptl/elan/src/ptl_elan_priv.c b/src/mca/ptl/elan/src/ptl_elan_priv.c index 8d6af06e55..6572dcf91b 100644 --- a/src/mca/ptl/elan/src/ptl_elan_priv.c +++ b/src/mca/ptl/elan/src/ptl_elan_priv.c @@ -33,7 +33,7 @@ mca_ptl_elan_data_frag (struct mca_ptl_elan_module_t *ptl, item, rc); while (OMPI_SUCCESS != rc) { - /* TODO: progress the recv state machine */ + /* FIXME: progress the recv state machine */ ompi_output (0, "[%s:%d] Retry to allocate a recv fragment", __FILE__, __LINE__); @@ -203,8 +203,7 @@ mca_ptl_elan_ctrl_frag (struct mca_ptl_elan_module_t *ptl, req->req_peer_addr.pval, req->req_peer_size); - /* FIXME: - * This sort of synchronized fragment release will lead + /* XXX: This sort of synchronized fragment release will lead * to race conditions, also see the note insize the follwoing routine */ mca_ptl_elan_send_desc_done (desc, req); END_FUNC(PTL_ELAN_DEBUG_ACK); @@ -315,7 +314,7 @@ mca_ptl_elan_init_qdma_desc (struct mca_ptl_elan_send_frag_t *frag, *size = size_out; hdr->hdr_frag.hdr_frag_length = size_out; - /* FIXME: change this ugly fix for get */ + /* TODO: change this ugly fix for get */ hdr->hdr_frag.hdr_dst_ptr.lval = elan4_main2elan( ctx, (char *)pml_req->req_base.req_addr + size_out); @@ -325,10 +324,7 @@ mca_ptl_elan_init_qdma_desc (struct mca_ptl_elan_send_frag_t *frag, frag->frag_base.frag_header = *hdr; #if OMPI_PTL_ELAN_COMP_QUEUE - /* XXX: Chain a QDMA to each queue and - * Have all the srcEvent fired to the Queue - * - * XXX: The chain dma will go directly into a command stream + /* XXX: The chain dma will go directly into a command stream * so we need addend the command queue control bits. * Allocate space from command queues hanged off the CTX. */ @@ -449,8 +445,7 @@ mca_ptl_elan_init_put_desc (struct mca_ptl_elan_send_frag_t *frag, offset); } - /* - * TODO: + /* TODO: * For now, eager sends are always packed into the descriptor * Inline up to 256 bytes (including the header), then * do a chained send for mesg < first_frag_size */ @@ -546,8 +541,6 @@ mca_ptl_elan_init_put_desc (struct mca_ptl_elan_send_frag_t *frag, /* Chain an event */ desc->main_dma.dma_srcEvent= elan4_main2elan(ctx, desc->chain_event); - /* FIXME: no additional flags for the DMA, remote, shmem, qwrite, - * broadcast, etc. Be sure to correctly setup a chained DMA. */ flags = 0; desc->main_dma.dma_typeSize = E4_DMA_TYPE_SIZE (size_out, DMA_DataTypeByte, flags, ptl->putget->pg_retryCount); @@ -605,13 +598,12 @@ mca_ptl_elan_init_get_desc (mca_ptl_elan_module_t *ptl, desc->dst_elan_addr = hdr->hdr_ack.hdr_dst_addr.lval; desc->desc_buff = hdr; - /* FIXME: - * initialize convertor and get the fragment copied out - * use the convertor hanged over request */ + /* FIXME: initialize convertor and get the fragment + * copied with the convertor hanged over request */ size_out = size_in; *size = size_out; - /* FIXME: hdr_frag and hdr_ack overlap partially, please be aware */ + /* XXX: hdr_frag and hdr_ack overlap partially, please be aware */ desc->chain_dma.dma_typeSize = E4_DMA_TYPE_SIZE ( sizeof(mca_ptl_base_frag_header_t), DMA_DataTypeByte, DMA_QueueWrite, 8); @@ -687,8 +679,6 @@ mca_ptl_elan_init_get_desc (mca_ptl_elan_module_t *ptl, desc->main_dma.dma_dstEvent= elan4_main2elan(ctx, (E4_Event *)desc->chain_event); - /* FIXME: no additional flags for the DMA, remote, shmem, qwrite, - * broadcast, etc. Be sure to correctly setup a chained DMA. */ flags = 0; desc->main_dma.dma_typeSize = E4_DMA_TYPE_SIZE (size_out, DMA_DataTypeByte, flags, ptl->putget->pg_retryCount); @@ -725,7 +715,6 @@ mca_ptl_elan_wait_queue(mca_ptl_elan_module_t * ptl, ready = rxq->qr_qEvent; readyWord = &rxq->qr_doneWord; - /* FIXME: Make sure the event and doneWord are correctly initialized */ LOG_PRINT(PTL_ELAN_DEBUG_NONE, "rail %p ctx %p ready %p readyWord %p\n", rail, ctx, ready, readyWord); @@ -986,10 +975,6 @@ mca_ptl_elan_start_ack ( mca_ptl_base_module_t * ptl, hdr->hdr_ack.hdr_dst_match.lval = 0; hdr->hdr_ack.hdr_dst_match.pval = request; - /* FIXME: this needs to be some offsete from the base addr - * posted buffer size is the leftover. Length is - * recv_frag->frag_recv.frag_base.frag_header.hdr_frag.hdr_frag_length; - */ hdr->hdr_ack.hdr_dst_addr.pval = 0; hdr->hdr_ack.hdr_dst_addr.lval = elan4_main2elan(ctx, (char *)request->req_base.req_addr + recv_len); @@ -999,10 +984,9 @@ mca_ptl_elan_start_ack ( mca_ptl_base_module_t * ptl, desc, hdr, request->req_bytes_packed, recv_len); #if OMPI_PTL_ELAN_COMP_QUEUE - /* FIXME: save frag descriptor somewhere in the header */ - ((mca_ptl_elan_ack_header_t *) hdr)->frag = desc; /* XXX: Need to have a way to differentiate different frag */ + ((mca_ptl_elan_ack_header_t *) hdr)->frag = desc; qdma->comp_event->ev_Params[1] = elan4_alloccq_space (ctx, 8, CQ_Size8K); qdma->comp_event->ev_CountAndType = E4_EVENT_INIT_VALUE(-32, E4_EVENT_COPY, E4_EVENT_DTYPE_LONG, 8); diff --git a/src/mca/ptl/elan/src/ptl_elan_priv.h b/src/mca/ptl/elan/src/ptl_elan_priv.h index 0d45b84469..8eeeb56c26 100644 --- a/src/mca/ptl/elan/src/ptl_elan_priv.h +++ b/src/mca/ptl/elan/src/ptl_elan_priv.h @@ -99,7 +99,7 @@ do { \ #define OMPI_PTL_ELAN_ALIGNUP(x,a) (((unsigned int)(x) + ((a)-1)) & (-(a))) /* XXX: Potentially configurable parameters */ -#define OMPI_PTL_ELAN_NUM_QDESCS (4) +#define OMPI_PTL_ELAN_NUM_QDESCS (16) #define OMPI_PTL_ELAN_NUM_PUTGET (8) #define OMPI_PTL_ELAN_ZERO_FFRAG (0) diff --git a/src/mca/ptl/elan/tests/bw.c b/src/mca/ptl/elan/tests/bw.c index 38e6601221..ba24d43a46 100644 --- a/src/mca/ptl/elan/tests/bw.c +++ b/src/mca/ptl/elan/tests/bw.c @@ -13,7 +13,7 @@ MPI_Request request[MAX_REQ_NUM]; MPI_Status tmp_stat[MAX_REQ_NUM]; -int skip = 0; +int skip = 40; int main (int argc, @@ -28,7 +28,7 @@ main (int argc, r_buf = (char *) malloc (sizeof(char)*MYBUFSIZE); /* Get some environmental variables set for Open MPI, OOB */ - env_init_for_elan(); + /*env_init_for_elan();*/ MPI_Init (&argc, &argv); MPI_Comm_size (MPI_COMM_WORLD, &numprocs); diff --git a/src/mca/ptl/elan/tests/lat.c b/src/mca/ptl/elan/tests/lat.c index fb7b8a1ffc..9f05891df1 100644 --- a/src/mca/ptl/elan/tests/lat.c +++ b/src/mca/ptl/elan/tests/lat.c @@ -35,7 +35,7 @@ main (int argc, char *argv[]) } /* Get some environmental variables set for Open MPI, OOB */ - env_init_for_elan(); + /*env_init_for_elan();*/ MPI_Init (&argc, &argv); MPI_Comm_size (MPI_COMM_WORLD, &numprocs); diff --git a/src/mca/ptl/elan/tests/test_util.h b/src/mca/ptl/elan/tests/test_util.h index 90c9f6a6f4..962446cd67 100644 --- a/src/mca/ptl/elan/tests/test_util.h +++ b/src/mca/ptl/elan/tests/test_util.h @@ -5,12 +5,12 @@ static void env_init_for_elan() { char *rms_rank; - setenv("OMPI_MCA_oob_cofs_dir", "/u/yuw/tmp", 1); + setenv("OMPI_MCA_oob_cofs_dir", "/home/1/yuw/tmp", 1); setenv("OMPI_MCA_pcm_cofs_cellid", "1", 1); setenv("OMPI_MCA_pcm_cofs_jobid", "1", 1); setenv("OMPI_MCA_pcm_cofs_num_procs", "2", 1); setenv("OMPI_MCA_ptl_base_exclude", "tcp", 1); - /*setenv("OMPI_MCA_oob_base_include", "cofs", 1)*/ + setenv("OMPI_MCA_oob_base_include", "cofs", 1); /*setenv("OMPI_MCA_oob_base_exclude", "tcp", 1);*/ if (NULL != (rms_rank = getenv("RMS_RANK"))) {