-- Varios cleanup

This commit was SVN r2605.

src/mca/ptl/elan/src/ptl_elan.c

@@ -149,7 +149,7 @@ mca_ptl_elan_finalize (struct mca_ptl_base_module_t *ptl)
 
     START_FUNC(PTL_ELAN_DEBUG_FIN);
 
-    /* FIXME: move from ptl_elan_component_close()
+    /* TODO: move from mca_ptl_elan_component_close()
        all the ptl_elan_finalize related code here */
 #if 0  
     elan_ptl = (struct mca_ptl_elan_module_t *) ptl;

src/mca/ptl/elan/src/ptl_elan_component.c

@@ -167,8 +167,8 @@ mca_ptl_elan_component_close (void)
     if (elan_mp->elan_recv_frags_free.fl_num_allocated !=
         elan_mp->elan_recv_frags_free.super.ompi_list_length) {
         ompi_output (0, 
-		     "[proc%s:%s:%d] recv_frags : %d allocated %d returned\n",
+		     "[%s:%d] recv_frags : %d allocated %d returned\n",
-		     getenv("RMS_RANK"), __FILE__, __LINE__,
+		     __FILE__, __LINE__,
                      elan_mp->elan_recv_frags_free.fl_num_allocated,
                      elan_mp->elan_recv_frags_free.super.ompi_list_length);
     }

src/mca/ptl/elan/src/ptl_elan_frag.c

@@ -86,8 +86,6 @@ ompi_class_t mca_ptl_elan_recv_frag_t_class = {
 
 extern mca_ptl_elan_state_t mca_ptl_elan_global_state;
 
-/* FIXME: Even though we have also get implemened here, temporarily 
- * mca_ptl_elan_send_frag_t is still used as the return type */
 mca_ptl_elan_send_frag_t *
 mca_ptl_elan_alloc_desc (struct mca_ptl_base_module_t *ptl_ptr,
        	struct mca_pml_base_request_t *req, int desc_type)
@@ -165,6 +163,7 @@ mca_ptl_elan_send_desc_done (
     ptl = ((ompi_ptl_elan_qdma_desc_t *)frag->desc)->ptl;
     header = &frag->frag_base.frag_header;
 
+#if OMPI_PTL_ELAN_ENABLE_GET
     if (frag->desc->desc_type == MCA_PTL_ELAN_DESC_GET) {
 	if(ompi_atomic_fetch_and_set_int (&frag->frag_progressed, 1) == 0) {
 	    ptl->super.ptl_recv_progress(ptl, 
@@ -180,6 +179,7 @@ mca_ptl_elan_send_desc_done (
 	END_FUNC(PTL_ELAN_DEBUG_SEND);
 	return;
     }
+#endif
 
     LOG_PRINT(PTL_ELAN_DEBUG_SEND, 
 	    "req %p done frag %p desc %p desc_type %d length %d\n", 
@@ -205,8 +205,7 @@ mca_ptl_elan_send_desc_done (
 		frag, frag->desc, frag->desc->desc_type);
 
 	/* Return a frag or if not cached, or it is a follow up */ 
-	if ( /*(header->hdr_frag.hdr_frag_offset != 0) || */
+	if ( (frag->desc->desc_status != MCA_PTL_ELAN_DESC_CACHED)){
-		(frag->desc->desc_status != MCA_PTL_ELAN_DESC_CACHED)){
 	    ompi_free_list_t  *flist;
 	    if (frag->desc->desc_type == MCA_PTL_ELAN_DESC_PUT) {
 		flist = &ptl->putget->put_desc_free;
@@ -226,9 +225,7 @@ mca_ptl_elan_send_desc_done (
      }
 #else
     else  {
-
+	/* XXX: 
-	/* XXX: need to discuss this with TSW.
-	 * There is a little confusion here. 
 	 * Why the release of this send fragment is dependent 
 	 * on the receiving of an acknowledgement 
 	 * There are two drawbacks,
@@ -256,7 +253,6 @@ mca_ptl_elan_send_desc_done (
 		    (ompi_list_item_t *) frag);
     } 
 #endif
-
     END_FUNC(PTL_ELAN_DEBUG_SEND);
 }
  

src/mca/ptl/elan/src/ptl_elan_frag.h

@@ -61,7 +61,7 @@ mca_ptl_elan_recv_frag_t *
 mca_ptl_elan_alloc_recv_desc(struct mca_pml_base_recv_request_t *req);
 
 /**
- * FIXME: Change frag to be a struct 
+ * TODO: Change frag to be a struct 
  * ELAN send request derived type. The send request contains 
  * the base send request and a point to the elan fragment descriptor
  */

src/mca/ptl/elan/src/ptl_elan_init.c

@@ -78,16 +78,11 @@ ompi_mca_ptl_elan_setup (mca_ptl_elan_state_t * ems)
         return OMPI_ERROR;
     }
 
-    /* 
-     * XXX: Leave the following later after testing of QDMA is done
-     */
     if (OMPI_SUCCESS != ompi_init_elan_putget (emp, rail_count)) {
         return OMPI_ERROR;
     }
 
-    /* 
+    /* TODO: initialize STAT (including SYNC) structures. */
-     * XXX: initialize STAT (including SYNC) structures.
-     */
     if (OMPI_SUCCESS != ompi_init_elan_stat (emp, rail_count)) {
         return OMPI_ERROR;
     }
@@ -243,13 +238,17 @@ ompi_module_elan_close_ptls (mca_ptl_elan_component_t * emp,
     int i;
     struct mca_ptl_elan_module_t *ptl; 
 
-    /* FIXME: find the ones that are still there and free them */
     for (i = 0; i < num_rails; i ++ ) {
 	ptl = emp->modules[i];
 	if (NULL == ptl) continue;
+
+	/* TODO: Deconstruct the module: ptl->super; */
 	OBJ_DESTRUCT (&(ptl->recv_frags));
 	OBJ_DESTRUCT (&(ptl->send_frags));
 	OBJ_DESTRUCT (&(ptl->pending_acks));
+
+	/* TODO: Free send/recv/comp queues */
+	/*ptl->comp;  ptl->queue; ptl->putget;*/
     }
 }
 
@@ -257,7 +256,7 @@ static void
 ompi_module_elan_close_procs (mca_ptl_elan_component_t * emp,
                               int num_rails)
 {
-    /* FIXME: find the ones that are still there and free them */
+    /* FIXME: find left procs and free them */
 }
 
 static int

src/mca/ptl/elan/src/ptl_elan_priv.c

@@ -33,7 +33,7 @@ mca_ptl_elan_data_frag (struct mca_ptl_elan_module_t *ptl,
                         item, rc);
 
     while (OMPI_SUCCESS != rc) {
-	/* TODO: progress the recv state machine */
+	/* FIXME: progress the recv state machine */
         ompi_output (0,
                      "[%s:%d] Retry to allocate a recv fragment",
                      __FILE__, __LINE__);
@@ -203,8 +203,7 @@ mca_ptl_elan_ctrl_frag (struct mca_ptl_elan_module_t *ptl,
 	    req->req_peer_addr.pval, 
 	    req->req_peer_size);
 
-     /* FIXME: 
+     /* XXX: This sort of synchronized fragment release will lead
-      * This sort of synchronized fragment release will lead
       * to race conditions, also see the note insize the follwoing routine */
      mca_ptl_elan_send_desc_done (desc, req); 
      END_FUNC(PTL_ELAN_DEBUG_ACK);
@@ -315,7 +314,7 @@ mca_ptl_elan_init_qdma_desc (struct mca_ptl_elan_send_frag_t *frag,
     *size = size_out;
     hdr->hdr_frag.hdr_frag_length = size_out;
 
-    /* FIXME: change this ugly fix for get */
+    /* TODO: change this ugly fix for get */
     hdr->hdr_frag.hdr_dst_ptr.lval = elan4_main2elan(
 	    ctx, (char *)pml_req->req_base.req_addr + size_out);
 
@@ -325,10 +324,7 @@ mca_ptl_elan_init_qdma_desc (struct mca_ptl_elan_send_frag_t *frag,
     frag->frag_base.frag_header = *hdr; 
 
 #if OMPI_PTL_ELAN_COMP_QUEUE
-    /* XXX: Chain a QDMA to each queue and 
+    /* XXX: The chain dma will go directly into a command stream
-     * Have all the srcEvent fired to the Queue 
-     *
-     * XXX: The chain dma will go directly into a command stream
      * so we need addend the command queue control bits.
      * Allocate space from command queues hanged off the CTX.
      */
@@ -449,8 +445,7 @@ mca_ptl_elan_init_put_desc (struct mca_ptl_elan_send_frag_t *frag,
                 offset);
         }
 
-	/* 
+	/* TODO: 
-         * TODO: 
 	 *   For now, eager sends are always packed into the descriptor
 	 *   Inline up to 256 bytes (including the header), then
 	 *   do a chained send for mesg < first_frag_size */
@@ -546,8 +541,6 @@ mca_ptl_elan_init_put_desc (struct mca_ptl_elan_send_frag_t *frag,
     /* Chain an event */
     desc->main_dma.dma_srcEvent= elan4_main2elan(ctx, desc->chain_event);
 
-    /* FIXME: no additional flags for the DMA, remote, shmem, qwrite, 
-     *        broadcast, etc. Be sure to correctly setup a chained DMA. */
     flags = 0;
     desc->main_dma.dma_typeSize = E4_DMA_TYPE_SIZE (size_out, 
 	    DMA_DataTypeByte, flags, ptl->putget->pg_retryCount);
@@ -605,13 +598,12 @@ mca_ptl_elan_init_get_desc (mca_ptl_elan_module_t *ptl,
     desc->dst_elan_addr = hdr->hdr_ack.hdr_dst_addr.lval;
     desc->desc_buff = hdr;
 
-    /* FIXME: 
+    /* FIXME: initialize convertor and get the fragment 
-     *   initialize convertor and get the fragment copied out 
+     * copied with the convertor hanged over request */
-     *   use the convertor hanged over request */
     size_out = size_in;
     *size = size_out;
 
-    /* FIXME: hdr_frag and hdr_ack overlap partially, please be aware */
+    /* XXX: hdr_frag and hdr_ack overlap partially, please be aware */
     desc->chain_dma.dma_typeSize = E4_DMA_TYPE_SIZE (
 	    sizeof(mca_ptl_base_frag_header_t), 
 	    DMA_DataTypeByte, DMA_QueueWrite, 8);
@@ -687,8 +679,6 @@ mca_ptl_elan_init_get_desc (mca_ptl_elan_module_t *ptl,
     desc->main_dma.dma_dstEvent= elan4_main2elan(ctx, 
 	    (E4_Event *)desc->chain_event);
 
-    /* FIXME: no additional flags for the DMA, remote, shmem, qwrite, 
-     *        broadcast, etc. Be sure to correctly setup a chained DMA. */
     flags = 0;
     desc->main_dma.dma_typeSize = E4_DMA_TYPE_SIZE (size_out, 
 	    DMA_DataTypeByte, flags, ptl->putget->pg_retryCount);
@@ -725,7 +715,6 @@ mca_ptl_elan_wait_queue(mca_ptl_elan_module_t * ptl,
     ready = rxq->qr_qEvent;
     readyWord = &rxq->qr_doneWord;
 
-    /* FIXME: Make sure the event and doneWord are correctly initialized */
     LOG_PRINT(PTL_ELAN_DEBUG_NONE,
 	    "rail %p ctx %p ready %p readyWord %p\n",
 	    rail, ctx, ready, readyWord);
@@ -986,10 +975,6 @@ mca_ptl_elan_start_ack ( mca_ptl_base_module_t * ptl,
     hdr->hdr_ack.hdr_dst_match.lval = 0; 
     hdr->hdr_ack.hdr_dst_match.pval = request;
 
-    /* FIXME: this needs to be some offsete from the base addr 
-     *        posted buffer size is the leftover. Length is 
-     * recv_frag->frag_recv.frag_base.frag_header.hdr_frag.hdr_frag_length;
-     */
     hdr->hdr_ack.hdr_dst_addr.pval = 0; 
     hdr->hdr_ack.hdr_dst_addr.lval = elan4_main2elan(ctx, 
 	    (char *)request->req_base.req_addr + recv_len);
@@ -999,10 +984,9 @@ mca_ptl_elan_start_ack ( mca_ptl_base_module_t * ptl,
 	    desc, hdr, request->req_bytes_packed, recv_len);
 
 #if OMPI_PTL_ELAN_COMP_QUEUE
-    /* FIXME: save frag descriptor somewhere in the header */
-    ((mca_ptl_elan_ack_header_t *) hdr)->frag = desc; 
 
     /* XXX: Need to have a way to differentiate different frag */
+    ((mca_ptl_elan_ack_header_t *) hdr)->frag = desc; 
     qdma->comp_event->ev_Params[1] = elan4_alloccq_space (ctx, 8, CQ_Size8K);
     qdma->comp_event->ev_CountAndType = E4_EVENT_INIT_VALUE(-32,
 	    E4_EVENT_COPY, E4_EVENT_DTYPE_LONG, 8);

src/mca/ptl/elan/src/ptl_elan_priv.h

@@ -99,7 +99,7 @@ do {                                                           \
 #define  OMPI_PTL_ELAN_ALIGNUP(x,a)  (((unsigned int)(x) + ((a)-1)) & (-(a)))
 
 /* XXX: Potentially configurable parameters */
-#define  OMPI_PTL_ELAN_NUM_QDESCS    (4)
+#define  OMPI_PTL_ELAN_NUM_QDESCS    (16)
 #define  OMPI_PTL_ELAN_NUM_PUTGET    (8)
 #define  OMPI_PTL_ELAN_ZERO_FFRAG    (0)
 

src/mca/ptl/elan/tests/bw.c

@@ -13,7 +13,7 @@
 MPI_Request request[MAX_REQ_NUM];
 MPI_Status  tmp_stat[MAX_REQ_NUM];
 
-int         skip = 0;
+int         skip = 40;
 
 int
 main (int argc,
@@ -28,7 +28,7 @@ main (int argc,
     r_buf = (char *) malloc (sizeof(char)*MYBUFSIZE);
 
     /* Get some environmental variables set for Open MPI, OOB */
-    env_init_for_elan();
+    /*env_init_for_elan();*/
 
     MPI_Init (&argc, &argv);
     MPI_Comm_size (MPI_COMM_WORLD, &numprocs);

src/mca/ptl/elan/tests/lat.c

@@ -35,7 +35,7 @@ main (int argc, char *argv[])
     }
 
     /* Get some environmental variables set for Open MPI, OOB */
-    env_init_for_elan();
+    /*env_init_for_elan();*/
 
     MPI_Init (&argc, &argv);
     MPI_Comm_size (MPI_COMM_WORLD, &numprocs);

src/mca/ptl/elan/tests/test_util.h

@@ -5,12 +5,12 @@ static void env_init_for_elan()
 {
     char  *rms_rank;
 
-    setenv("OMPI_MCA_oob_cofs_dir", "/u/yuw/tmp", 1);
+    setenv("OMPI_MCA_oob_cofs_dir", "/home/1/yuw/tmp", 1);
     setenv("OMPI_MCA_pcm_cofs_cellid", "1", 1);
     setenv("OMPI_MCA_pcm_cofs_jobid", "1", 1);
     setenv("OMPI_MCA_pcm_cofs_num_procs", "2", 1);
     setenv("OMPI_MCA_ptl_base_exclude", "tcp", 1);
-    /*setenv("OMPI_MCA_oob_base_include", "cofs", 1)*/
+    setenv("OMPI_MCA_oob_base_include", "cofs", 1);
     /*setenv("OMPI_MCA_oob_base_exclude", "tcp", 1);*/
 
     if (NULL != (rms_rank = getenv("RMS_RANK"))) {