Update the elan BTL. Now we support the following protocols: send, put

and partially get.

This commit was SVN r15564.

ompi/mca/btl/elan/btl_elan.c

@@ -14,18 +14,24 @@
 #include "opal/util/if.h"
 #include "ompi/mca/pml/pml.h"
 #include "ompi/mca/btl/btl.h"
-
+#include "ompi/communicator/communicator.h"
 #include "btl_elan.h"
 #include "btl_elan_frag.h" 
 #include "btl_elan_proc.h"
 #include "btl_elan_endpoint.h"
 
 #include "ompi/datatype/convertor.h" 
-#include "ompi/mca/mpool/base/base.h" 
+#include "ompi/mca/btl/base/base.h" 
 #include "ompi/mca/mpool/mpool.h" 
+#include "ompi/runtime/ompi_module_exchange.h"
+#include "orte/class/orte_proc_table.h" 
+#include "opal/class/opal_hash_table.h"   
 
+#include "stdio.h"
+#include "elan/elan.h"
 #include "opal/util/os_path.h"
 #include "opal/util/opal_environ.h"
+#include "orte/util/proc_info.h"
 
 mca_btl_elan_module_t mca_btl_elan_module = {
     {
@@ -50,7 +56,7 @@ mca_btl_elan_module_t mca_btl_elan_module = {
         mca_btl_elan_prepare_dst,
         mca_btl_elan_send,
         mca_btl_elan_put,
-        NULL, /*mca_btl_elan_get,*/
+        mca_btl_elan_get,
 	NULL, /*mca_btl_elan_dump,*/
         NULL, /* mpool */
         NULL, /* register error cb */
@@ -71,11 +77,58 @@ int mca_btl_elan_add_procs( struct mca_btl_base_module_t* btl,
 {
     mca_btl_elan_module_t* elan_btl = (mca_btl_elan_module_t*)btl;
     int i, rc;
+    FILE* file;
+    char* filename;
+    ELAN_BASE    * base;  
+    ELAN_STATE   * state;  
+    ELAN_QUEUE   * q= NULL;  
+    ELAN_TPORT   * p= NULL;  
+    filename = opal_os_path( false, orte_process_info.proc_session_dir, "ELAN_ID", NULL );
+    file = fopen( filename, "w" );
+    for( i = 0; i < (int)nprocs; i++ ) {
+        struct ompi_proc_t* ompi_proc = ompi_procs[i];
+        fprintf( file, "%s %d\n", ompi_proc->proc_hostname, i );
+    }
+    fclose( file );
+    opal_setenv( "LIBELAN_MACHINES_FILE", filename, true, &environ );
+    /* opal_setenv( "LIBELAN_MACHINES_FILE", "/home/tma/machinefile", false, &environ );*/ 
+    opal_setenv( "MPIRUN_ELANIDMAP_FILE", "/etc/elanidmap", false, &environ ); 
+    base = elan_baseInit(0);  
+    if (base == NULL)  
+        return NULL;  
+    state = base->state;   
+    if( NULL == state ) {  
+        mca_btl_base_error_no_nics( "ELAN", "Quadrics" ); 
+        return NULL;  
+    }
+    elan_gsync(base->allGroup);  
+    if ((q = elan_allocQueue(base->state)) == NULL) {  
+        return NULL;  
+    }  
+    if (!(p = elan_tportInit(base->state,  
+                             (ELAN_QUEUE *)q, 
+                             base->tport_nslots,  
+                             base->tport_smallmsg,   
+                             base->tport_bigmsg, 
+                             base->tport_stripemsg, 
+                             ELAN_POLL_EVENT,  
+                             base->retryCount, 
+                             &base->shm_key,  
+                             base->shm_fifodepth, 
+                             base->shm_fragsize,  
+                             0))) { 
+        return NULL; 
+    }
+    elan_btl->base  = base; 
+    elan_btl->state = state; 
+    elan_btl->queue = q;  
+    elan_btl->tport = p;  
+    elan_btl->elan_vp = state->vp;  
+    elan_btl->elan_nvp = state->nvp; 
     for(i = 0; i < (int) nprocs; i++) {
         struct ompi_proc_t* ompi_proc = ompi_procs[i];
         mca_btl_elan_proc_t* elan_proc;
         mca_btl_base_endpoint_t* elan_endpoint;
-
         if(NULL == (elan_proc = mca_btl_elan_proc_create(ompi_proc))) {
             return OMPI_ERR_OUT_OF_RESOURCE;
         }
@@ -85,7 +138,6 @@ int mca_btl_elan_add_procs( struct mca_btl_base_module_t* btl,
             OPAL_THREAD_UNLOCK(&elan_proc->proc_lock);
             return OMPI_ERR_OUT_OF_RESOURCE;
         }
-
         elan_endpoint->endpoint_btl = elan_btl;
         rc = mca_btl_elan_proc_insert(elan_proc, elan_endpoint);
         if(rc != OMPI_SUCCESS) {
@@ -94,13 +146,10 @@ int mca_btl_elan_add_procs( struct mca_btl_base_module_t* btl,
             OPAL_THREAD_UNLOCK(&elan_proc->proc_lock);
             continue;
         }
-
         ompi_bitmap_set_bit(reachable, i);
         OPAL_THREAD_UNLOCK(&elan_proc->proc_lock);
         peers[i] = elan_endpoint;
-
     }
-
     return OMPI_SUCCESS;
 }
 
@@ -124,10 +173,9 @@ int mca_btl_elan_register( struct mca_btl_base_module_t* btl,
 {
     mca_btl_elan_module_t* elan_btl = (mca_btl_elan_module_t*) btl; 
     void * tbuf = NULL;
-    int rc;
+    int send_len, rc;
     bufdesc_t *desc;
     mca_btl_elan_frag_t* frag;
-
     elan_btl->elan_reg[tag].cbfunc = cbfunc; 
     elan_btl->elan_reg[tag].cbdata = cbdata; 
     if (NULL != cbfunc) {
@@ -143,13 +191,12 @@ int mca_btl_elan_register( struct mca_btl_base_module_t* btl,
             frag->tag              = tag;
             frag->type             = MCA_BTL_ELAN_HDR_TYPE_RECV;
             tbuf   = (void*)(frag+1);
+            send_len  = elan_btl->super.btl_eager_limit;
             desc = (bufdesc_t * )malloc (sizeof(struct bufdesc_t));
-        desc->eve = elan_tportRxStart( elan_btl->tport, 0, 0,  0, BTL_ELAN_RECV_MASK,
+            desc->eve = elan_tportRxStart (elan_btl->tport,0 , 0,  0, 0xffffffff, frag->tag, tbuf,send_len) ;
-                                       frag->tag, tbuf, elan_btl->super.btl_eager_limit );
             desc->frag = frag;
             desc->next  = NULL;
-
+            BTL_ELAN_ADD_TO_FIFO(elan_btl, desc);
-        BTL_ELAN_ADD_TO_FIFO( elan_btl, desc );
     }
     return OMPI_SUCCESS;
 }
@@ -166,24 +213,19 @@ mca_btl_base_descriptor_t* mca_btl_elan_alloc(struct mca_btl_base_module_t* btl,
                                               uint8_t order,
 					      size_t size )
 {
-
     mca_btl_elan_frag_t* frag;
     int rc;
-    
-
     if(size <= btl->btl_eager_limit){ 
         MCA_BTL_TEMPLATE_FRAG_ALLOC_EAGER(frag, rc); 
         if( OPAL_UNLIKELY(NULL == frag) ) {
             return NULL;
         }
-
         frag->segment.seg_len = size;
     } else if (size <= btl->btl_max_send_size){ 
         MCA_BTL_TEMPLATE_FRAG_ALLOC_MAX(frag, rc); 
         if( OPAL_UNLIKELY(NULL == frag) ) {
             return NULL;
         }
-
         frag->segment.seg_len = size;
     } else {
         return NULL;
@@ -197,7 +239,6 @@ mca_btl_base_descriptor_t* mca_btl_elan_alloc(struct mca_btl_base_module_t* btl,
     frag->btl = (mca_btl_elan_module_t*)btl;
     frag->base.order = MCA_BTL_NO_ORDER;
     return (mca_btl_base_descriptor_t*)frag;
-
 }
 
 
@@ -251,11 +292,10 @@ mca_btl_base_descriptor_t* mca_btl_elan_prepare_src( struct mca_btl_base_module_
             MCA_BTL_TEMPLATE_FRAG_RETURN(frag);
             return NULL;
         }
+        frag->segment.seg_addr.pval = frag+1;
         frag->segment.seg_len = max_data + reserve;
     }
-
+    else if(max_data+reserve <= btl->btl_max_send_size){
-
-    else {
 		
         MCA_BTL_TEMPLATE_FRAG_ALLOC_MAX(frag, rc);
         if(NULL == frag) {
@@ -273,9 +313,22 @@ mca_btl_base_descriptor_t* mca_btl_elan_prepare_src( struct mca_btl_base_module_
             MCA_BTL_TEMPLATE_FRAG_RETURN(frag);
             return NULL;
         }
+        frag->segment.seg_addr.pval = frag+1;
 	*size  = max_data;
-    }
         frag->segment.seg_len = max_data + reserve;
+	}else{
+		MCA_BTL_TEMPLATE_FRAG_ALLOC_USER(frag, rc);
+		if(NULL == frag) {
+			return NULL;
+		}
+		frag->type = MCA_BTL_ELAN_HDR_TYPE_PUT;
+		iov.iov_len = max_data;
+		iov.iov_base = NULL;
+		ompi_convertor_pack(convertor, &iov, &iov_count, &max_data);
+		*size = max_data;
+        frag->segment.seg_addr.pval = iov.iov_base;
+        frag->segment.seg_len = max_data;
+	}
     frag->base.des_src = &(frag->segment);
     frag->base.des_src_cnt = 1;
     frag->base.order = MCA_BTL_NO_ORDER;
@@ -283,6 +336,7 @@ mca_btl_base_descriptor_t* mca_btl_elan_prepare_src( struct mca_btl_base_module_
     frag->base.des_dst_cnt = 0;
     frag->base.des_flags = 0;
     return &frag->base;
+
 }
 
 /**
@@ -309,30 +363,27 @@ mca_btl_base_descriptor_t* mca_btl_elan_prepare_dst( struct mca_btl_base_module_
 {
 
     mca_btl_elan_frag_t* frag;
+    mca_mpool_base_module_t* mpool = btl->btl_mpool;
     int rc;
     if( OPAL_UNLIKELY((*size) > UINT32_MAX) ) {  
         *size = (size_t)UINT32_MAX;
     }
-    
     MCA_BTL_TEMPLATE_FRAG_ALLOC_USER(frag, rc);
     if(NULL == frag) {
         return NULL;
     }
-
-    frag->segment.seg_len = *size;
     ompi_convertor_get_current_pointer( convertor, (void**)&(frag->segment.seg_addr.pval) );
+    frag->segment.seg_len = *size;
+    frag->segment.seg_key.key64 = (uint64_t)(intptr_t)convertor;
     /*frag->segment.seg_addr.pval = convertor->pBaseBuf + convertor->bConverted;*/
     frag->type         = MCA_BTL_ELAN_HDR_TYPE_PUT;
     frag->base.des_src = NULL;
     frag->base.des_src_cnt = 0;
     frag->base.des_flags = 0;	
-	
     frag->base.des_dst = &(frag->segment);
     frag->base.des_dst_cnt = 1;
-
     frag->base.order = MCA_BTL_NO_ORDER;
     return &frag->base;
-
 }
 
 
@@ -358,24 +409,20 @@ int mca_btl_elan_send( struct mca_btl_base_module_t* btl,
     bufdesc_t * desc;
 
     /* TODO */
-    /*opal_output(0, "I am send,tag is %d\n", tag);*/
     frag->btl = elan_btl;
     frag->endpoint = endpoint;
     frag->tag = tag;
     frag->type = MCA_BTL_ELAN_HDR_TYPE_SEND;
-
     peer = endpoint->elan_vp;
     proc = elan_btl->elan_vp;
-
     sbuf       = (void *)frag->base.des_src->seg_addr.pval;
     send_len    = frag->base.des_src->seg_len; 
     desc = (bufdesc_t * )malloc (sizeof(struct bufdesc_t));
     desc->eve = elan_tportTxStart (elan_btl->tport, 0, peer, proc,frag->tag, sbuf, send_len) ;
-    /*opal_output( 0, "send message startoing from %p with ;length %d\n", sbuf, send_len );*/
+    /*opal_output( 0, "send message startoing from %d to %d\n", proc, peer );*/
-
     desc->frag = frag;
     desc->next  = NULL;
-    BTL_ELAN_ADD_TO_FIFO( elan_btl, desc );
+    BTL_ELAN_ADD_TO_FIFO(elan_btl, desc);
     return OMPI_SUCCESS;
 }
 
@@ -400,17 +447,15 @@ int mca_btl_elan_put( mca_btl_base_module_t* btl,
     unsigned char* src_addr = (unsigned char*)src->seg_addr.pval;
     size_t src_len = src->seg_len;
     unsigned char* dst_addr = (unsigned char*)ompi_ptr_ltop(dst->seg_addr.lval);
-    /*size_t dst_len = dst->seg_len;*/
     bufdesc_t * desc = (bufdesc_t * )malloc (sizeof(struct bufdesc_t));
     frag->endpoint = endpoint;
     frag->btl = elan_btl;
     frag->type = MCA_BTL_ELAN_HDR_TYPE_PUT;
+    /* opal_output(0, "put from %p to %d peer , %d\n", src_addr, peer, src_len); */
     desc->eve = elan_put(elan_btl->state, src_addr, dst_addr, src_len, peer);
     desc->frag = frag;
     desc->next  = NULL;
-
+    BTL_ELAN_ADD_TO_FIFO(elan_btl, desc);
-    BTL_ELAN_ADD_TO_FIFO( elan_btl, desc );
-
     return OMPI_SUCCESS;
 }
 
@@ -426,13 +471,27 @@ int mca_btl_elan_put( mca_btl_base_module_t* btl,
 
 int mca_btl_elan_get( mca_btl_base_module_t* btl,
 		       mca_btl_base_endpoint_t* endpoint,
-                      mca_btl_base_descriptor_t* descriptor )
+		       mca_btl_base_descriptor_t* des )
 {
-    /* mca_btl_elan_module_t* elan_btl = (mca_btl_elan_module_t*) btl; */
+    mca_btl_elan_module_t* elan_btl = (mca_btl_elan_module_t*) btl;
-    mca_btl_elan_frag_t* frag = (mca_btl_elan_frag_t*) descriptor; 
+    mca_btl_elan_frag_t* frag = (mca_btl_elan_frag_t*) des; 
+    int     peer = endpoint->elan_vp;
+    mca_btl_base_segment_t* src = des->des_src;
+    mca_btl_base_segment_t* dst = des->des_dst;
+    unsigned char* src_addr = (unsigned char*)src->seg_addr.pval;
+    size_t src_len = src->seg_len;
+    unsigned char* dst_addr = (unsigned char*)ompi_ptr_ltop(dst->seg_addr.lval);
+    /*size_t dst_len = dst->seg_len;*/
+    bufdesc_t * desc = (bufdesc_t * )malloc (sizeof(struct bufdesc_t));
     frag->endpoint = endpoint;
-    /* TODO */
+    frag->btl = elan_btl;
-    return OMPI_ERR_NOT_IMPLEMENTED; 
+    frag->type = MCA_BTL_ELAN_HDR_TYPE_GET;
+    /*opal_output(0, "get from %p to %d peer , %d\n", src_addr, peer, src_len); */
+    desc->eve = elan_get(elan_btl->state, src_addr, dst_addr, src_len, peer);
+    desc->frag = frag;
+    desc->next  = NULL;
+    BTL_ELAN_ADD_TO_FIFO(elan_btl, desc);
+    return OMPI_SUCCESS;
 }
 
 
@@ -443,20 +502,17 @@ int mca_btl_elan_get( mca_btl_base_module_t* btl,
 void cancel_elanRx(mca_btl_elan_module_t* elan_btl)
 {
     bufdesc_t * index = elan_btl->tportFIFOHead;
-    int i=0;
     while(index!= NULL)
     {
 	if(index->frag->type == MCA_BTL_ELAN_HDR_TYPE_RECV)
             {
+
                 if(elan_tportRxCancel(index->eve))
-                        {
-                            i++;
                     MCA_BTL_TEMPLATE_FRAG_RETURN(index->frag);
-                        }
+
             }
         index = index->next;
     }
-    opal_output(0, "cancel no. is %d\n", i);
 }
 
 int mca_btl_elan_finalize( struct mca_btl_base_module_t* btl )
@@ -492,20 +548,20 @@ int mca_btl_elan_ft_event(int state) {
 bufdesc_t * elan_ipeek(mca_btl_elan_module_t* elan_btl)
 {
 	bufdesc_t * desc =  elan_btl->tportFIFOHead;
-
 	if( NULL == desc )
 		return NULL;
-    if( MCA_BTL_ELAN_HDR_TYPE_RECV == desc->frag->type ) {	
+	if(MCA_BTL_ELAN_HDR_TYPE_RECV == desc->frag->type) {	
 	      if( !elan_tportRxDone(desc->eve) )
                   return NULL;
-    } else if( MCA_BTL_ELAN_HDR_TYPE_SEND == desc->frag->type ) {
+	}else if(MCA_BTL_ELAN_HDR_TYPE_SEND == desc->frag->type) {
 	      if( !elan_tportTxDone(desc->eve) )
                   return NULL;
-    } else {
+	}else if(MCA_BTL_ELAN_HDR_TYPE_PUT == desc->frag->type || MCA_BTL_ELAN_HDR_TYPE_GET == desc->frag->type) {
-	if( !elan_done(desc->eve,0) )
+              if( !elan_done(desc->eve,0))
+                  return NULL;	
+	}else {
             return NULL;
         }
-
 	elan_btl->tportFIFOHead = elan_btl->tportFIFOHead->next;
 	if( NULL == elan_btl->tportFIFOHead )
             elan_btl->tportFIFOTail = NULL;

ompi/mca/btl/elan/btl_elan.h

@@ -116,9 +116,7 @@ struct mca_btl_elan_module_t {
     opal_mutex_t elan_lock;
     struct bufdesc_t *    tportFIFOHead;
     struct bufdesc_t *    tportFIFOTail;
-#if defined MCA_BTL_HAS_MPOOL
     struct mca_mpool_base_module_t* elan_mpool;
-#endif
 }; 
 typedef struct mca_btl_elan_module_t mca_btl_elan_module_t;
 extern mca_btl_elan_module_t mca_btl_elan_module;
@@ -349,7 +347,7 @@ extern bufdesc_t * elan_ipeek(mca_btl_elan_module_t* elan_btl);
 
 #define BTL_ELAN_ADD_TO_FIFO(BTL, DESC)                         \
     do {                                                        \
-        OPAL_THREAD_LOCK(&elan_btl->elan_lock);                 \
+        OPAL_THREAD_LOCK(&((BTL)->elan_lock));                 \
         if( (BTL)->tportFIFOTail ) {                            \
             (BTL)->tportFIFOTail->next = (DESC);                \
             (BTL)->tportFIFOTail = (DESC);                      \
@@ -357,7 +355,7 @@ extern bufdesc_t * elan_ipeek(mca_btl_elan_module_t* elan_btl);
             (BTL)->tportFIFOHead = (DESC);                      \
             (BTL)->tportFIFOTail = (DESC);                      \
         }                                                       \
-        OPAL_THREAD_UNLOCK(&elan_btl->elan_lock);               \
+        OPAL_THREAD_UNLOCK(&((BTL)->elan_lock));               \
     } while(0)
 
 #endif

ompi/mca/btl/elan/btl_elan_component.c

@@ -21,6 +21,7 @@
 
 #include "opal/mca/base/mca_base_param.h"
 #include "ompi/runtime/ompi_module_exchange.h"
+
 #include "orte/mca/errmgr/errmgr.h"
 #include "ompi/mca/mpool/base/base.h" 
 #include "btl_elan.h"
@@ -35,7 +36,7 @@
 
 #include "opal/util/os_path.h"
 #include "opal/util/opal_environ.h"
-
+#include "ompi/communicator/communicator.h" 
 mca_btl_elan_component_t mca_btl_elan_component = {
     {
         /* First, the mca_base_component_t struct containing meta information
@@ -59,7 +60,6 @@ mca_btl_elan_component_t mca_btl_elan_component = {
             /* The component is not checkpoint ready */
             MCA_BASE_METADATA_PARAM_NONE
         },
-
         mca_btl_elan_component_init,  
         mca_btl_elan_component_progress,
     }
@@ -113,11 +113,12 @@ int mca_btl_elan_component_open(void)
     mca_btl_elan_module.super.btl_exclusivity = 0;
     mca_btl_elan_module.super.btl_eager_limit =  32*1024;
     mca_btl_elan_module.super.btl_min_send_size = 32*1024;
-    mca_btl_elan_module.super.btl_max_send_size = 32*1024; /*64*1024;*/
+    mca_btl_elan_module.super.btl_max_send_size = 64*1024; /*64*1024;*/
-    mca_btl_elan_module.super.btl_rdma_pipeline_send_length = 512 *1024;
+    mca_btl_elan_module.super.btl_rdma_pipeline_send_length = 512 * 1024;
-    mca_btl_elan_module.super.btl_rdma_pipeline_frag_size = 128 *1024;
+    mca_btl_elan_module.super.btl_rdma_pipeline_frag_size = 128 * 1024;
-    mca_btl_elan_module.super.btl_min_rdma_pipeline_size = 128* 1024;
+    mca_btl_elan_module.super.btl_min_rdma_pipeline_size = 128 * 1024;
-    mca_btl_elan_module.super.btl_flags = MCA_BTL_FLAGS_SEND_INPLACE | /*MCA_BTL_FLAGS_PUT | */MCA_BTL_FLAGS_SEND;
+    mca_btl_elan_module.super.btl_flags = MCA_BTL_FLAGS_SEND_INPLACE /*| MCA_BTL_FLAGS_GET */| MCA_BTL_FLAGS_PUT | MCA_BTL_FLAGS_SEND ;
+    /* mca_btl_elan_module.super.btl_flags = MCA_BTL_FLAGS_SEND_INPLACE|MCA_BTL_FLAGS_RDMA | MCA_BTL_FLAGS_SEND ;*/
     mca_btl_elan_module.super.btl_bandwidth = 2000;
     mca_btl_elan_module.super.btl_latency = 5;
     mca_btl_base_param_register(&mca_btl_elan_component.super.btl_version,
@@ -132,7 +133,7 @@ int mca_btl_elan_component_open(void)
 
 int mca_btl_elan_component_close(void)
 {
-    if( NULL != mca_btl_elan_component.elan_btls )
+    if( NULL != mca_btl_elan_component.elan_btls ) {
         free( mca_btl_elan_component.elan_btls );
         /* release resources */
 
@@ -141,7 +142,7 @@ int mca_btl_elan_component_close(void)
         OBJ_DESTRUCT(&mca_btl_elan_component.elan_frag_user);
         OBJ_DESTRUCT(&mca_btl_elan_component.elan_frag_max);
         OBJ_DESTRUCT(&mca_btl_elan_component.elan_lock);
-    
+    }
     return OMPI_SUCCESS;
 }
 
@@ -158,7 +159,7 @@ mca_btl_base_module_t** mca_btl_elan_component_init( int *num_btl_modules,
 {
 
     mca_btl_base_module_t** btls;
-    size_t rails, i , count;
+    size_t rails, i , count, vpid;
     ELAN_BASE    * base;
     ELAN_STATE   * state;
     ELAN_QUEUE   * q= NULL;
@@ -198,37 +199,12 @@ mca_btl_base_module_t** mca_btl_elan_component_init( int *num_btl_modules,
                          mca_btl_elan_component.elan_free_list_inc,
                          NULL ); /* use default allocator */
     
-    opal_setenv( "LIBELAN_MACHINES_FILE", "/home/tma/machinefile", false, &environ );
+
     opal_setenv( "MPIRUN_ELANIDMAP_FILE", "/etc/elanidmap", false, &environ );
-    base = elan_baseInit(0);
+    vpid = orte_process_info.my_name->vpid;
-    if (base == NULL)
+   
-	return NULL;
+    ompi_modex_send( &mca_btl_elan_component.super.btl_version, &vpid, sizeof(vpid));
-    state = base->state; 
+    mca_btl_elan_component.elan_num_btls = 1;
-    if( NULL == state ) {
-	mca_btl_base_error_no_nics( "ELAN", "Quadrics" );
-	return NULL;
-    }
-    elan_gsync(base->allGroup);
-    ompi_modex_send( &mca_btl_elan_component.super.btl_version, &state->vp, sizeof(state->vp));
-    rails = elan_nRails(state);
-    mca_btl_elan_component.elan_num_btls = rails;
-    if ((q = elan_allocQueue(base->state)) == NULL) {
-        return NULL;
-    }
-    if (!(p = elan_tportInit(base->state, 
-		            (ELAN_QUEUE *)q,
-		             base->tport_nslots,
-		             base->tport_smallmsg, 
-                     base->tport_bigmsg,
-                     base->tport_stripemsg,
-		             ELAN_POLL_EVENT, 
-                     base->retryCount,
-		             &base->shm_key, 
-                     base->shm_fifodepth,
-		             base->shm_fragsize, 
-                     0))) {
-	    return NULL;
-    }
     mca_btl_elan_component.elan_btls = malloc( (mca_btl_elan_component.elan_num_btls) * sizeof(mca_btl_base_module_t*) );
     for( i = count = 0; i < mca_btl_elan_component.elan_num_btls; i++ ) {
         mca_btl_elan_module_t* btl =  malloc (sizeof (mca_btl_elan_module_t));	
@@ -238,12 +214,6 @@ mca_btl_base_module_t** mca_btl_elan_component_init( int *num_btl_modules,
         OBJ_CONSTRUCT (&btl->elan_lock, opal_mutex_t);
     	btl->tportFIFOHead=NULL;
     	btl->tportFIFOTail=NULL;
-    	btl->base  = base;
-    	btl->state = state;
-    	btl->queue = q;
-    	btl->tport = p;
-	    btl->elan_vp = state->vp;
-	    btl->elan_nvp = state->nvp;
         mca_btl_elan_component.elan_btls[count++] = btl;
     }
     mca_btl_elan_component.elan_num_btls = count ;
@@ -294,8 +264,11 @@ int mca_btl_elan_component_progress()
                     {
                         /* it's a put*/
                         /* call the completion callback */
+                        //opal_output(0, "I am a RDMA\n");            
                         elan_wait(desc->eve,ELAN_WAIT_EVENT);
+                        //opal_output(0, "I am a RDMA_done\n");
                         frag->base.des_cbfunc( &(elan_btl->super), frag->endpoint, &(frag->base), OMPI_SUCCESS );
+                        //opal_output(0, "I am a RDMA_cb_done\n");
                         free(desc);
                     }
                 else{
@@ -315,7 +288,17 @@ int mca_btl_elan_component_progress()
 
                     desc->frag = frag;
                     desc->next  = NULL;
-                    BTL_ELAN_ADD_TO_FIFO( elan_btl, desc );
+                    OPAL_THREAD_LOCK(&elan_btl->elan_lock);
+                    if(elan_btl->tportFIFOTail)
+                        {
+                            elan_btl->tportFIFOTail->next = desc;
+                            elan_btl->tportFIFOTail=desc;
+                        }
+                    else{
+                        elan_btl->tportFIFOHead = desc;
+                        elan_btl->tportFIFOTail = desc;   
+                    }
+                    OPAL_THREAD_UNLOCK(&elan_btl->elan_lock);
                 }
             } else {
                 opal_output( 0, "Something bad happened the frag == NULL\n" );
@@ -326,5 +309,3 @@ int mca_btl_elan_component_progress()
 
     return num_progressed;
 }
-
-

ompi/mca/btl/elan/btl_elan_endpoint.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004-2007 The University of Tennessee and The University
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * $COPYRIGHT$

ompi/mca/btl/elan/btl_elan_frag.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004-2007 The University of Tennessee and The University
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * $COPYRIGHT$

ompi/mca/btl/elan/btl_elan_frag.h

@@ -45,9 +45,7 @@ struct mca_btl_elan_frag_t {
     ompi_free_list_t* my_list;
     mca_btl_base_tag_t tag;
     size_t size; 
-#if defined MCA_BTL_HAS_MPOOL
     struct mca_mpool_base_registration_t* registration;
-#endif
 }; 
 typedef struct mca_btl_elan_frag_t mca_btl_elan_frag_t; 
 OBJ_CLASS_DECLARATION(mca_btl_elan_frag_t); 

ompi/mca/btl/elan/btl_elan_proc.c

@@ -1,5 +1,6 @@
+
 /*
- * Copyright (c) 2004-2007 The University of Tennessee and The University
+ * Copyright (c) 2004-2006 The University of Tennessee and The University
  *                         of Tennessee Research Foundation.  All rights
  *                         reserved.
  * $COPYRIGHT$
@@ -13,15 +14,16 @@
 #include "orte/class/orte_proc_table.h"
 #include "opal/class/opal_hash_table.h" 
 #include "ompi/runtime/ompi_module_exchange.h"
-
 #include "btl_elan.h"
 #include "btl_elan_proc.h"
+#include "orte/util/proc_info.h"
+#include "ompi/communicator/communicator.h"
 
 static void mca_btl_elan_proc_construct(mca_btl_elan_proc_t* proc);
 static void mca_btl_elan_proc_destruct(mca_btl_elan_proc_t* proc);
 
 OBJ_CLASS_INSTANCE(mca_btl_elan_proc_t, 
-                   opal_list_item_t, mca_btl_elan_proc_construct, 
+                   opal_list_item_t,mca_btl_elan_proc_construct, 
                    mca_btl_elan_proc_destruct);
 
 void mca_btl_elan_proc_construct(mca_btl_elan_proc_t* proc)
@@ -47,7 +49,6 @@ void mca_btl_elan_proc_destruct(mca_btl_elan_proc_t* proc)
     OPAL_THREAD_LOCK(&mca_btl_elan_component.elan_lock);
     opal_list_remove_item(&mca_btl_elan_component.elan_procs, &proc->super);
     OPAL_THREAD_UNLOCK(&mca_btl_elan_component.elan_lock);
-
     /* release resources */
     if(NULL != proc->proc_endpoints) {
         free(proc->proc_endpoints);
@@ -62,24 +63,18 @@ void mca_btl_elan_proc_destruct(mca_btl_elan_proc_t* proc)
 static mca_btl_elan_proc_t* mca_btl_elan_proc_lookup_ompi(ompi_proc_t* ompi_proc)
 {
     mca_btl_elan_proc_t* elan_proc;
-
     OPAL_THREAD_LOCK(&mca_btl_elan_component.elan_lock);
-
     for(elan_proc = (mca_btl_elan_proc_t*)
         opal_list_get_first(&mca_btl_elan_component.elan_procs);
         elan_proc != (mca_btl_elan_proc_t*)
         opal_list_get_end(&mca_btl_elan_component.elan_procs);
         elan_proc  = (mca_btl_elan_proc_t*)opal_list_get_next(elan_proc)) {
-
         if(elan_proc->proc_ompi == ompi_proc) {
             OPAL_THREAD_UNLOCK(&mca_btl_elan_component.elan_lock);
             return elan_proc;
         }
-
     }
-
     OPAL_THREAD_UNLOCK(&mca_btl_elan_component.elan_lock);
-
     return NULL;
 }
 
@@ -96,19 +91,15 @@ mca_btl_elan_proc_t* mca_btl_elan_proc_create(ompi_proc_t* ompi_proc)
     int rc;
     size_t size;
     mca_btl_elan_proc_t* module_proc = NULL;
-
     /* Check if we have already created a Elan proc
      * structure for this ompi process */
     module_proc = mca_btl_elan_proc_lookup_ompi(ompi_proc);
-
     if(module_proc != NULL) {
         /* Gotcha! */
         return module_proc;
     }
-
     /* Oops! First time, gotta create a new Elan proc
      * out of the ompi_proc ... */
-
     module_proc = OBJ_NEW(mca_btl_elan_proc_t);
 	if(NULL == module_proc)
             return NULL;
@@ -119,15 +110,6 @@ mca_btl_elan_proc_t* mca_btl_elan_proc_create(ompi_proc_t* ompi_proc)
     /* build a unique identifier (of arbitrary
      * size) to represent the proc */
     module_proc->proc_guid = ompi_proc->proc_name;
-
-    /* Elan module doesn't have addresses exported at
-     * initialization, so the addr_count is set to one. */
-   
-    /**
-       orte_hash_table_set_proc( &mca_btl_elan_component.elan_procs,
-       &module_proc->proc_guid,
-       module_proc );*/
-
     rc = ompi_modex_recv( &mca_btl_elan_component.super.btl_version,
                           ompi_proc,
                           (void**)&module_proc->elan_vp_array,
@@ -141,11 +123,11 @@ mca_btl_elan_proc_t* mca_btl_elan_proc_create(ompi_proc_t* ompi_proc)
     /* XXX: Right now, there can be only 1 peer associated
      * with a proc. Needs a little bit change in 
      * mca_btl_elan_proc_t to allow on demand increasing of
-     * number of endpoints for this proc */
+     * number of endpoints for this proc 
+    */
 
     module_proc->proc_endpoints = (mca_btl_base_endpoint_t**)
     malloc((1+module_proc->proc_addr_count )* sizeof(mca_btl_base_endpoint_t*));
-
     if(NULL == module_proc->proc_endpoints) {
         OBJ_RELEASE(module_proc);
         return NULL;
@@ -164,12 +146,11 @@ int mca_btl_elan_proc_insert( mca_btl_elan_proc_t* module_proc,
 {
     /* insert into endpoint array */
     size_t i;
-	
     module_endpoint->endpoint_proc = module_proc;
     module_proc->proc_endpoints[module_proc->proc_endpoint_count++] = module_endpoint;
-
     for( i = 0; i < module_proc->proc_addr_count; i++ ) {
         module_endpoint->elan_vp = module_proc->elan_vp_array[i];
+        /* module_endpoint->elan_vp = ompi_comm_rank(&ompi_mpi_comm_world);*/
         return OMPI_SUCCESS;
     }
     return OMPI_SUCCESS;