Many different things, the big ones:

- Start filling in the progress function, focusing on connection establishment.
 - Initialize udapl mpool and free lists
 - Create/destroy a protection zone with each IA
 - Misc organization as I learn how things work

This commit was SVN r8969.

ompi/mca/btl/udapl/Makefile.am

@@ -20,6 +20,7 @@
 
 
 
+CFLAGS = $(btl_udapl_CFLAGS)
 AM_CPPFLAGS = $(btl_udapl_CPPFLAGS)
 
 udapl_sources = \

ompi/mca/btl/udapl/btl_udapl.c

@@ -30,7 +30,8 @@
 #include "datatype/convertor.h" 
 #include "datatype/datatype.h" 
 #include "mca/mpool/base/base.h" 
-#include "mca/mpool/mpool.h" 
+/*#include "mca/mpool/mpool.h"*/
+#include "mca/mpool/udapl/mpool_udapl.h"
 #include "ompi/proc/proc.h"
 
 mca_btl_udapl_module_t mca_btl_udapl_module = {
@@ -60,32 +61,6 @@ mca_btl_udapl_module_t mca_btl_udapl_module = {
 };
 
 
-/**
-  * Report a uDAPL error - for debugging
-  */
-
-static void
-mca_btl_udapl_error(DAT_RETURN ret, char* str)
-{
-    char* major;
-    char* minor;
-
-    /* don't output anything if debug is not set */
-    if(0 == mca_btl_udapl_component.udapl_debug) {
-        return;
-    }
-
-    if(DAT_SUCCESS != dat_strerror(ret,
-            (const char**)&major, (const char**)&minor))
-    {
-        printf("dat_strerror failed! ret is %d\n", ret);
-        exit(-1);
-    }
-
-    opal_output(0, "ERROR: %s %s %s\n", str, major, minor);
-}
-
-
 /**
  * Initialize module module resources.
  */
@@ -93,21 +68,29 @@ mca_btl_udapl_error(DAT_RETURN ret, char* str)
 int
 mca_btl_udapl_init(DAT_NAME_PTR ia_name, mca_btl_udapl_module_t * btl)
 {
+    mca_mpool_base_resources_t res;
     DAT_IA_ATTR attr;
     DAT_RETURN rc;
 
     /* open the uDAPL interface */
-    btl->udapl_evd_dflt = DAT_HANDLE_NULL;
+    btl->udapl_evd_async = DAT_HANDLE_NULL;
     rc = dat_ia_open(ia_name, mca_btl_udapl_component.udapl_evd_qlen,
-            &btl->udapl_evd_dflt, &btl->udapl_ia);
+            &btl->udapl_evd_async, &btl->udapl_ia);
     if(DAT_SUCCESS != rc) {
         mca_btl_udapl_error(rc, "dat_ia_open");
         return OMPI_ERROR;
     }
 
+    /* create a protection zone */
+    rc = dat_pz_create(btl->udapl_ia, &btl->udapl_pz);
+    if(DAT_SUCCESS != rc) {
+        mca_btl_udapl_error(rc, "dat_pz_create");
+        return OMPI_ERROR;
+    }
+
     /* query to get address information */
     /* TODO - we only get the address, but there's other useful stuff here */
-    rc = dat_ia_query(btl->udapl_ia, &btl->udapl_evd_dflt,
+    rc = dat_ia_query(btl->udapl_ia, &btl->udapl_evd_async,
             DAT_IA_FIELD_IA_ADDRESS_PTR, &attr, DAT_IA_FIELD_NONE, NULL);
     if(DAT_SUCCESS != rc) {
         mca_btl_udapl_error(rc, "dat_ia_query");
@@ -129,7 +112,7 @@ mca_btl_udapl_init(DAT_NAME_PTR ia_name, mca_btl_udapl_module_t * btl)
 
     rc = dat_evd_create(btl->udapl_ia,
             mca_btl_udapl_component.udapl_evd_qlen, DAT_HANDLE_NULL,
-            DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG, &btl->udapl_evd_conn);
+            DAT_EVD_CR_FLAG | DAT_EVD_CONNECTION_FLAG, &btl->udapl_evd_conn);
     if(DAT_SUCCESS != rc) {
         mca_btl_udapl_error(rc, "dat_evd_create (conn)");
         dat_evd_free(btl->udapl_evd_dto);
@@ -137,17 +120,65 @@ mca_btl_udapl_init(DAT_NAME_PTR ia_name, mca_btl_udapl_module_t * btl)
         return OMPI_ERROR;
     }
 
-    /* TODO - post some receives - involves setting up ep's, psp's, and LMR's */
+    /* initialize the memory pool */
+    res.udapl_ia = btl->udapl_ia;
+    res.udapl_pz = btl->udapl_pz;
+
+    btl->super.btl_mpool = mca_mpool_base_module_create(
+            mca_btl_udapl_component.udapl_mpool_name, &btl->super, &res);
 
     /* initialize objects */
     OBJ_CONSTRUCT(&btl->udapl_frag_eager, ompi_free_list_t);
     OBJ_CONSTRUCT(&btl->udapl_frag_max, ompi_free_list_t);
     OBJ_CONSTRUCT(&btl->udapl_frag_user, ompi_free_list_t);
+    OBJ_CONSTRUCT(&btl->udapl_frag_recv, ompi_free_list_t);
     OBJ_CONSTRUCT(&btl->udapl_pending, opal_list_t);
     OBJ_CONSTRUCT(&btl->udapl_repost, opal_list_t);
     OBJ_CONSTRUCT(&btl->udapl_mru_reg, opal_list_t);
     OBJ_CONSTRUCT(&btl->udapl_lock, opal_mutex_t);
     
+    /* initialize free lists */
+    ompi_free_list_init(&btl->udapl_frag_eager,
+          sizeof(mca_btl_udapl_frag_eager_t) +
+                 mca_btl_udapl_module.super.btl_eager_limit,
+          OBJ_CLASS(mca_btl_udapl_frag_eager_t),
+          mca_btl_udapl_component.udapl_free_list_num,
+          mca_btl_udapl_component.udapl_free_list_max,
+          mca_btl_udapl_component.udapl_free_list_inc,
+          btl->super.btl_mpool);
+
+    ompi_free_list_init(&btl->udapl_frag_max,
+          sizeof(mca_btl_udapl_frag_max_t) +
+                 mca_btl_udapl_module.super.btl_max_send_size,
+          OBJ_CLASS(mca_btl_udapl_frag_max_t),
+          mca_btl_udapl_component.udapl_free_list_num,
+          mca_btl_udapl_component.udapl_free_list_max,
+          mca_btl_udapl_component.udapl_free_list_inc,
+          btl->super.btl_mpool);
+
+    ompi_free_list_init(&btl->udapl_frag_user,
+          sizeof(mca_btl_udapl_frag_user_t),
+          OBJ_CLASS(mca_btl_udapl_frag_user_t),
+          mca_btl_udapl_component.udapl_free_list_num,
+          mca_btl_udapl_component.udapl_free_list_max,
+          mca_btl_udapl_component.udapl_free_list_inc,
+          NULL);
+
+    ompi_free_list_init(&btl->udapl_frag_recv,
+          sizeof(mca_btl_udapl_frag_recv_t),
+          OBJ_CLASS(mca_btl_udapl_frag_recv_t),
+          mca_btl_udapl_component.udapl_free_list_num,
+          mca_btl_udapl_component.udapl_free_list_max,
+          mca_btl_udapl_component.udapl_free_list_inc,
+          btl->super.btl_mpool);
+
+    /* Connections are done lazily - the process doing the send acts as a client
+       when initiating the connect.  progress should always be checking for
+       incoming connections, and establishing them when they arrive.  When
+       connection is established, recv's are posted. */
+
+    /* TODO - post receives */
+    /* TODO - can I always use SRQ, or just on new enough uDAPLs? */
     return OMPI_SUCCESS;
 }
 
@@ -160,11 +191,12 @@ int mca_btl_udapl_finalize(struct mca_btl_base_module_t* base_btl)
 {
     mca_btl_udapl_module_t* udapl_btl = (mca_btl_udapl_module_t*) base_btl; 
 
-    opal_output(0, "udapl_finalize\n");
+    OPAL_OUTPUT((0, "udapl_finalize\n"));
 
     /* release uDAPL resources */
     dat_evd_free(udapl_btl->udapl_evd_dto);
     dat_evd_free(udapl_btl->udapl_evd_conn);
+    dat_pz_free(udapl_btl->udapl_pz);
     dat_ia_close(udapl_btl->udapl_ia, DAT_CLOSE_GRACEFUL_FLAG);
 
     /* destroy objects */
@@ -192,7 +224,7 @@ int mca_btl_udapl_add_procs(
     mca_btl_udapl_module_t* udapl_btl = (mca_btl_udapl_module_t*)btl;
     int i, rc;
 
-    opal_output(0, "udapl_add_procs\n");
+    OPAL_OUTPUT((0, "udapl_add_procs\n"));
 
     for(i = 0; i < (int) nprocs; i++) {
 
@@ -244,7 +276,7 @@ int mca_btl_udapl_del_procs(struct mca_btl_base_module_t* btl,
         struct ompi_proc_t **procs, 
         struct mca_btl_base_endpoint_t ** peers)
 {
-    opal_output(0, "udapl_del_procs\n");
+    OPAL_OUTPUT((0, "udapl_del_procs\n"));
     /* TODO */
     return OMPI_SUCCESS;
 }
@@ -264,7 +296,7 @@ int mca_btl_udapl_register(
     udapl_btl->udapl_reg[tag].cbfunc = cbfunc; 
     udapl_btl->udapl_reg[tag].cbdata = cbdata; 
 
-    opal_output(0, "udapl_register\n");
+    OPAL_OUTPUT((0, "udapl_register\n"));
     return OMPI_SUCCESS;
 }
 
@@ -284,7 +316,7 @@ mca_btl_base_descriptor_t* mca_btl_udapl_alloc(
     mca_btl_udapl_frag_t* frag;
     int rc;
 
-    opal_output(0, "udapl_alloc\n");
+    OPAL_OUTPUT((0, "udapl_alloc\n"));
     
     if(size <= btl->btl_eager_limit) { 
         MCA_BTL_UDAPL_FRAG_ALLOC_EAGER(udapl_btl, frag, rc); 
@@ -317,7 +349,7 @@ int mca_btl_udapl_free(
 {
     mca_btl_udapl_frag_t* frag = (mca_btl_udapl_frag_t*)des;
 
-    opal_output(0, "udapl_free\n");
+    OPAL_OUTPUT((0, "udapl_free\n"));
 
     if(frag->size == 0) {
         btl->btl_mpool->mpool_release(btl->btl_mpool, frag->registration);
@@ -327,7 +359,7 @@ int mca_btl_udapl_free(
     } else if(frag->size == mca_btl_udapl_component.udapl_max_frag_size) {
         MCA_BTL_UDAPL_FRAG_RETURN_MAX(btl, frag); 
     }  else {
-        opal_output(0, "[%s:%d] mca_btl_udapl_free: invalid descriptor\n", __FILE__,__LINE__);
+        OPAL_OUTPUT((0, "[%s:%d] mca_btl_udapl_free: invalid descriptor\n", __FILE__,__LINE__));
         return OMPI_ERR_BAD_PARAM;
     }
     return OMPI_SUCCESS; 
@@ -356,7 +388,7 @@ mca_btl_base_descriptor_t* mca_btl_udapl_prepare_src(
     int32_t free_after;
     int rc;
 
-    opal_output(0, "udapl_prepare_src\n");
+    OPAL_OUTPUT((0, "udapl_prepare_src\n"));
 
     /*
      * If the data has already been pinned and is contigous than we can
@@ -511,7 +543,7 @@ mca_btl_base_descriptor_t* mca_btl_udapl_prepare_dst(
     long lb;
     int rc;
 
-    opal_output(0, "udapl_prepare_dst\n");
+    OPAL_OUTPUT((0, "udapl_prepare_dst\n"));
 
     MCA_BTL_UDAPL_FRAG_ALLOC_USER(btl, frag, rc);
     if(NULL == frag) {
@@ -572,8 +604,26 @@ int mca_btl_udapl_send(
     mca_btl_base_tag_t tag)
    
 {
-    opal_output(0, "udapl_send\n");
+    mca_btl_udapl_module_t* udapl_btl = (mca_btl_udapl_module_t*)btl;
-    return OMPI_ERR_NOT_IMPLEMENTED; 
+    mca_btl_udapl_frag_t* frag = (mca_btl_udapl_frag_t*)des;
+
+    OPAL_OUTPUT((0, "udapl_send\n"));
+    
+    frag->btl = udapl_btl;
+    frag->endpoint = endpoint;
+    frag->hdr->tag = tag;
+    frag->type = MCA_BTL_UDAPL_SEND;
+
+    /* Check if we are connected to this peer.
+       Should be three states we care about -
+        connected, connecting, disconnected.
+       If no connection exists, request the connection and queue the send.
+       If a connection is pending, queue the send
+       If the connection is established, fire off the send.
+       need to consider locking around the connection state and queue.
+     */
+
+    return OMPI_SUCCESS; 
 }
 
 
@@ -591,7 +641,7 @@ int mca_btl_udapl_put(
     mca_btl_base_endpoint_t* endpoint,
     mca_btl_base_descriptor_t* des)
 {
-    opal_output(0, "udapl_put\n");
+    OPAL_OUTPUT((0, "udapl_put\n"));
     return OMPI_ERR_NOT_IMPLEMENTED; 
 }
 
@@ -611,7 +661,7 @@ int mca_btl_udapl_get(
     mca_btl_base_endpoint_t* endpoint,
     mca_btl_base_descriptor_t* des)
 {
-    opal_output(0, "udapl_get\n");
+    OPAL_OUTPUT((0, "udapl_get\n"));
     return OMPI_ERR_NOT_IMPLEMENTED;
 }
 

ompi/mca/btl/udapl/btl_udapl.h

@@ -55,16 +55,16 @@ struct mca_btl_udapl_component_t {
     struct  mca_btl_udapl_module_t **udapl_btls; /**< array of available BTL modules */
     size_t  udapl_num_mru;
     size_t  udapl_evd_qlen;
+    int32_t udapl_num_repost;
+    int     udapl_debug;        /**< turn on debug output */
+
     size_t udapl_eager_frag_size;
     size_t udapl_max_frag_size;
-    char*   udapl_port_name; 
-    int32_t udapl_num_repost;
-    int32_t udapl_num_high_priority; /**< number of receive descriptors at high priority */
-    int     udapl_debug;        /**< turn on debug output */
 
     int udapl_free_list_num;   /**< initial size of free lists */
     int udapl_free_list_max;   /**< maximum size of free lists */
-    int udapl_free_list_inc;   /**< number of elements to alloc when growing free lists */
+    int udapl_free_list_inc;   /**< number of elements to alloc when growing */
+
     opal_list_t udapl_procs;   /**< list of udapl proc structures */
     opal_mutex_t udapl_lock;   /**< lock for accessing module state */
     char* udapl_mpool_name;    /**< name of memory pool */ 
@@ -84,11 +84,12 @@ struct mca_btl_udapl_module_t {
     mca_btl_base_recv_reg_t udapl_reg[256]; 
     mca_btl_udapl_addr_t udapl_addr;
 
-    /* interface handle */
+    /* interface handle and protection zone */
     DAT_IA_HANDLE udapl_ia;
+    DAT_PZ_HANDLE udapl_pz;
 
     /* event dispatchers - default, data transfer, connection negotiation */
-    DAT_EVD_HANDLE udapl_evd_dflt;
+    DAT_EVD_HANDLE udapl_evd_async;
     DAT_EVD_HANDLE udapl_evd_dto;
     DAT_EVD_HANDLE udapl_evd_conn;
 
@@ -96,26 +97,23 @@ struct mca_btl_udapl_module_t {
     ompi_free_list_t udapl_frag_eager;
     ompi_free_list_t udapl_frag_max;
     ompi_free_list_t udapl_frag_user;
+    ompi_free_list_t udapl_frag_recv;
 
-    /* number of send/recv tokens */
-#if 0
-    int32_t udapl_num_send_tokens;
-    int32_t udapl_max_send_tokens;
-    int32_t udapl_num_recv_tokens;
-    int32_t udapl_max_recv_tokens;
-    int32_t udapl_num_repost;
-#endif
-
-    /* lock for accessing module state */
     opal_list_t udapl_pending;  /**< list of pending send descriptors */
     opal_list_t udapl_repost;   /**< list of pending fragments */
     opal_list_t udapl_mru_reg;  /**< list of most recently used registrations */
-    opal_mutex_t udapl_lock;
+    opal_mutex_t udapl_lock;    /* lock for accessing module state */
 }; 
 typedef struct mca_btl_udapl_module_t mca_btl_udapl_module_t;
 extern mca_btl_udapl_module_t mca_btl_udapl_module;
 
 
+/**
+  * Report a uDAPL error - for debugging
+  */
+
+extern void mca_btl_udapl_error(DAT_RETURN ret, char* str);
+
 /**
  * Register uDAPL component parameters with the MCA framework
  */

ompi/mca/btl/udapl/btl_udapl_component.c

@@ -72,8 +72,34 @@ mca_btl_udapl_component_t mca_btl_udapl_component = {
 };
 
 
+/**
+  * Report a uDAPL error - for debugging
+  */
+
+void
+mca_btl_udapl_error(DAT_RETURN ret, char* str)
+{
+    char* major;
+    char* minor;
+
+    /* don't output anything if debug is not set */
+    if(0 == mca_btl_udapl_component.udapl_debug) {
+        return;
+    }
+
+    if(DAT_SUCCESS != dat_strerror(ret,
+            (const char**)&major, (const char**)&minor))
+    {
+        printf("dat_strerror failed! ret is %d\n", ret);
+        exit(-1);
+    }
+
+    OPAL_OUTPUT((0, "ERROR: %s %s %s\n", str, major, minor));
+}
+
+
 /*
- * utility routines for parameter registration
+ * Utility routines for parameter registration
  */
 
 static inline char* mca_btl_udapl_param_register_string(
@@ -105,7 +131,7 @@ int mca_btl_udapl_component_open(void)
 {  
     int param, value;
 
-    opal_output(0, "udapl_component_open\n");
+    OPAL_OUTPUT((0, "udapl_component_open\n"));
 
     /* initialize state */
     mca_btl_udapl_component.udapl_num_btls=0;
@@ -130,14 +156,10 @@ int mca_btl_udapl_component_open(void)
         mca_btl_udapl_param_register_int("max_modules", 4);
     mca_btl_udapl_component.udapl_evd_qlen =
         mca_btl_udapl_param_register_int("evd_qlen", 8);
-    mca_btl_udapl_component.udapl_num_high_priority = 
-        mca_btl_udapl_param_register_int("num_high_priority", 8); 
     mca_btl_udapl_component.udapl_num_repost = 
         mca_btl_udapl_param_register_int("num_repost", 4);
     mca_btl_udapl_component.udapl_num_mru = 
         mca_btl_udapl_param_register_int("num_mru", 64);
-    mca_btl_udapl_component.udapl_port_name=
-        mca_btl_udapl_param_register_string("port_name", "OMPI"); 
 
     /* register uDAPL module parameters */
     mca_btl_udapl_module.super.btl_exclusivity =
@@ -155,34 +177,11 @@ int mca_btl_udapl_component_open(void)
     mca_btl_udapl_module.super.btl_bandwidth  = 
         mca_btl_udapl_param_register_int("bandwidth", 225); 
 
-    /* compute the eager and max frag sizes */
+    /* TODO - computer udapl_eager_frag_size and udapl_max_frag_size */
     mca_btl_udapl_component.udapl_eager_frag_size =
             mca_btl_udapl_module.super.btl_eager_limit;
-    /*mca_btl_udapl_component.udapl_eager_limit =
-            mca_btl_udapl_module.super.btl_eager_limit -
-            sizeof(mca_btl_base_header_t);*/
-
     mca_btl_udapl_component.udapl_max_frag_size =
             mca_btl_udapl_module.super.btl_max_send_size;
-    mca_btl_udapl_module.super.btl_max_send_size =
-        mca_btl_udapl_module.super.btl_max_send_size -
-        sizeof(mca_btl_base_header_t);
-#if 0
-    mca_btl_udapl_component.udapl_eager_frag_size =
-        udapl_min_size_for_length(mca_btl_udapl_module.super.btl_eager_limit) - 1;
-    mca_btl_udapl_module.super.btl_eager_limit = 
-        udapl_max_length_for_size(mca_btl_udapl_component.udapl_eager_frag_size) -
-        sizeof(mca_btl_base_header_t);
-#endif
-
-    /* compute the max frag size */
-#if 0
-    mca_btl_udapl_component.udapl_max_frag_size = 
-        udapl_min_size_for_length(mca_btl_udapl_module.super.btl_max_send_size) - 1;
-    mca_btl_udapl_module.super.btl_max_send_size = 
-        udapl_max_length_for_size(mca_btl_udapl_component.udapl_max_frag_size) -
-        sizeof(mca_btl_base_header_t);
-#endif
 
     /* leave pinned option */
     value = 0;
@@ -199,7 +198,7 @@ int mca_btl_udapl_component_open(void)
 
 int mca_btl_udapl_component_close(void)
 {
-    opal_output(0, "udapl_component_close\n");
+    OPAL_OUTPUT((0, "udapl_component_close\n"));
 
     /* TODO - what needs to be done here? */
     return OMPI_SUCCESS;
@@ -222,10 +221,8 @@ mca_btl_udapl_modex_send(void)
     size = sizeof(mca_btl_udapl_addr_t) *
             mca_btl_udapl_component.udapl_num_btls;
 
-    if(mca_btl_udapl_component.udapl_debug) {
+    OPAL_OUTPUT((0, "udapl_modex_send %d addrs %d bytes\n",
-        opal_output(0, "udapl_modex_send %d addrs %d bytes\n",
+            mca_btl_udapl_component.udapl_num_btls, size));
-                mca_btl_udapl_component.udapl_num_btls, size);
-    }
 
     if (0 != size) {
         addrs = (mca_btl_udapl_addr_t *)malloc(size);
@@ -262,7 +259,7 @@ mca_btl_udapl_component_init (int *num_btl_modules,
     DAT_COUNT num_ias;
     int32_t i;
 
-    opal_output(0, "udapl_component_init\n");
+    OPAL_OUTPUT((0, "udapl_component_init\n"));
 
     /* enumerate uDAPL interfaces */
     datinfo = malloc(sizeof(DAT_PROVIDER_INFO) *
@@ -278,7 +275,7 @@ mca_btl_udapl_component_init (int *num_btl_modules,
     }
 
     /* allocate space for the each possible BTL */
-    mca_btl_udapl_component.udapl_btls = (mca_btl_udapl_module_t *)
+    mca_btl_udapl_component.udapl_btls = (mca_btl_udapl_module_t **)
             malloc(num_ias * sizeof(mca_btl_udapl_module_t *));
     if(NULL == mca_btl_udapl_component.udapl_btls) {
         free(datinfo);
@@ -287,7 +284,7 @@ mca_btl_udapl_component_init (int *num_btl_modules,
 
     /* create a BTL module for each interface */
     for(mca_btl_udapl_component.udapl_num_btls = i = 0; i < num_ias; i++) {
-        opal_output(0, "udapl creating btl for %s\n", datinfo[i].ia_name);
+        OPAL_OUTPUT((0, "udapl creating btl for %s\n", datinfo[i].ia_name));
 
         btl = malloc(sizeof(mca_btl_udapl_module_t));
         if(NULL == btl) {
@@ -355,9 +352,12 @@ mca_btl_udapl_component_init (int *num_btl_modules,
 
 int mca_btl_udapl_component_progress()
 {
+    mca_btl_udapl_module_t* btl;
     static int32_t inprogress = 0;
+    DAT_EVENT event;
     int count = 0;
     size_t i;
+    int rc;
 
     /* prevent deadlock - only one thread should be 'progressing' at a time */
     if(OPAL_THREAD_ADD32(&inprogress, 1) > 1) {
@@ -365,14 +365,76 @@ int mca_btl_udapl_component_progress()
         return OMPI_SUCCESS;
     }
 
-    opal_output(0, "udapl_component_progress\n");
+    OPAL_OUTPUT((0, "udapl_component_progress\n"));
     
     /* check for work to do on each uDAPL btl */
-    for( i = 0; i < mca_btl_udapl_component.udapl_num_btls; ) {
+    for(i = 0; i < mca_btl_udapl_component.udapl_num_btls; i++) {
-        mca_btl_udapl_module_t *btl = mca_btl_udapl_component.udapl_btls[i];
+        btl = mca_btl_udapl_component.udapl_btls[i];
+        /* TODO - lock this properly */
 
-        /* TODO - check the DTO EVD for events */
+        /* Check DTO EVD */
-        i++;
+        while(DAT_SUCCESS ==
+                dat_evd_dequeue(btl->udapl_evd_dto, &event)) {
+            switch(event.event_number) {
+                case DAT_DTO_COMPLETION_EVENT:
+                    count++;
+                    break;
+                default:
+                    OPAL_OUTPUT((0, "WARNING unknown dto event: %d\n",
+                            event.event_number));
+            }
+        }
+
+        /* Check connection EVD */
+        while(DAT_SUCCESS ==
+                dat_evd_dequeue(btl->udapl_evd_conn, &event)) {
+            switch(event.event_number) {
+                case DAT_CONNECTION_REQUEST_EVENT:
+                    /* Accept a new connection */
+                    rc = dat_cr_accept(
+                            event.event_data.cr_arrival_event_data.cr_handle,
+                            DAT_HANDLE_NULL, 0, NULL);
+                    if(DAT_SUCCESS != rc) {
+                        mca_btl_udapl_error(rc, "dat_cr_accept");
+                    }
+
+                    count++;
+                    break;
+                case DAT_CONNECTION_EVENT_ESTABLISHED:
+                    /* TODO - at this point we have a uDPAL enpoint in
+                       event.event_data.connect_event_data.ep_handle,
+                       need to figure out how to tie back into the BTL */
+                    count++;
+                    break;
+                case DAT_CONNECTION_EVENT_PEER_REJECTED:
+                case DAT_CONNECTION_EVENT_NON_PEER_REJECTED:
+                case DAT_CONNECTION_EVENT_ACCEPT_COMPLETION_ERROR:
+                case DAT_CONNECTION_EVENT_DISCONNECTED:
+                case DAT_CONNECTION_EVENT_BROKEN:
+                case DAT_CONNECTION_EVENT_TIMED_OUT:
+                case DAT_CONNECTION_EVENT_UNREACHABLE:
+                    break;
+                default:
+                    OPAL_OUTPUT((0, "WARNING unknown conn event: %d\n",
+                            event.event_number));
+            }
+        }
+
+        /* Check async EVD */
+        while(DAT_SUCCESS ==
+                dat_evd_dequeue(btl->udapl_evd_async, &event)) {
+            switch(event.event_number) {
+                case DAT_ASYNC_ERROR_EVD_OVERFLOW:
+                case DAT_ASYNC_ERROR_IA_CATASTROPHIC:
+                case DAT_ASYNC_ERROR_EP_BROKEN:
+                case DAT_ASYNC_ERROR_TIMED_OUT:
+                case DAT_ASYNC_ERROR_PROVIDER_INTERNAL_ERROR:
+                    break;
+                default:
+                    OPAL_OUTPUT((0, "WARNING unknown async event: %d\n",
+                            event.event_number));
+            }
+        }
     }
 
     /* unlock and return */

ompi/mca/btl/udapl/btl_udapl_endpoint.c

@@ -21,11 +21,11 @@
 #include <sys/time.h>
 #include <time.h>
 #include "include/types.h"
-#include "mca/ns/base/base.h"
+#include "orte/mca/ns/base/base.h"
-#include "mca/oob/base/base.h"
+#include "orte/mca/oob/base/base.h"
-#include "mca/rml/rml.h"
+#include "orte/mca/rml/rml.h"
-#include "mca/errmgr/errmgr.h"
+#include "orte/mca/errmgr/errmgr.h"
-#include "dps/dps.h"
+#include "orte/dss/dss.h"
 #include "btl_udapl.h"
 #include "btl_udapl_endpoint.h" 
 #include "btl_udapl_proc.h"

ompi/mca/btl/udapl/btl_udapl_endpoint.h

@@ -39,6 +39,18 @@ struct mca_btl_udapl_addr_t {
 typedef struct mca_btl_udapl_addr_t mca_btl_udapl_addr_t;
 
 
+/**
+ * State of uDAPL endpoint connection.
+ */
+
+typedef enum {
+    MCA_BTL_UDAPL_CONNECTING,
+    MCA_BTL_UDAPL_CONNECTED,
+    MCA_BTL_UDAPL_CLOSED,
+    MCA_BTL_UDAPL_FAILED
+} mca_btl_udapl_endpoint_state_t;
+
+
 /**
  * An abstraction that represents a connection to a endpoint process.
  * An instance of mca_btl_base_endpoint_t is associated w/ each process
@@ -55,6 +67,12 @@ struct mca_btl_base_endpoint_t {
     struct mca_btl_udapl_proc_t*   endpoint_proc;
     /**< proc structure corresponding to endpoint */
 
+    mca_btl_udapl_endpoint_state_t endpoint_state;
+    /**< current state of the endpoint connection */
+
+    opal_list_t pending_frags;
+    /**< pending send frags on this endpoint */
+
     mca_btl_udapl_addr_t endpoint_addr;
 };
 

ompi/mca/btl/udapl/btl_udapl_frag.h

@@ -68,6 +68,10 @@ typedef struct mca_btl_udapl_frag_t mca_btl_udapl_frag_user_t;
     
 OBJ_CLASS_DECLARATION(mca_btl_udapl_frag_user_t); 
 
+typedef struct mca_btl_udapl_frag_t mca_btl_udapl_frag_recv_t; 
+    
+OBJ_CLASS_DECLARATION(mca_btl_udapl_frag_recv_t); 
+
 
 /*
  * Macros to allocate/return descriptors from module specific