Checkpoint:

- Borrow configure.m4 from the mvapi btl.  One of the uDAPL headers emits a
   warning when -pedantic is enabled, so strip it out.
 - Change function check in ompi_check_dapl.m4 from dat_ia_open to
   dat_registry_list_providers.. dat_ia_open wasn't working right
 - Make the references to prepare_dst, put, and get NULL for now
 - Add opal_output() calls in all the udapl interface functions for debugging
 - Add evd_qlen component parameter to control event dispatcher queue length
 - First stab at component_init and module_init
 - Misc cleanups - whitespace, dead code removal
 - Update copyrights to 2006

This commit was SVN r8701.

config/ompi_check_udapl.m4

@@ -49,7 +49,7 @@ AC_DEFUN([OMPI_CHECK_UDAPL],[
            OMPI_CHECK_PACKAGE([$1],
 	          [dat/udat.h],
                   [dapl],
-                  [dat_ia_open],
+                  [dat_registry_list_providers],
                   [-ldat],
                   [$ompi_check_udapl_dir],
                   [$ompi_check_udapl_libdir],

ompi/mca/btl/udapl/btl_udapl.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
  * Copyright (c) 2004-2005 The University of Tennessee and The University
@@ -52,9 +52,9 @@ mca_btl_udapl_module_t mca_btl_udapl_module = {
         mca_btl_udapl_alloc, 
         mca_btl_udapl_free, 
         mca_btl_udapl_prepare_src,
-        mca_btl_udapl_prepare_dst,
+        NULL, /* prepare_dst */
         mca_btl_udapl_send,
-        mca_btl_udapl_put,
+        NULL, /* put */
         NULL /* get */ 
     }
 };
@@ -74,6 +74,8 @@ int mca_btl_udapl_add_procs(
     mca_btl_udapl_module_t* udapl_btl = (mca_btl_udapl_module_t*)btl;
     int i, rc;
 
+	opal_output(0, "udapl_add_procs\n");
+
     for(i = 0; i < (int) nprocs; i++) {
 
         struct ompi_proc_t* ompi_proc = ompi_procs[i];
@@ -124,6 +126,7 @@ int mca_btl_udapl_del_procs(struct mca_btl_base_module_t* btl,
         struct ompi_proc_t **procs, 
         struct mca_btl_base_endpoint_t ** peers)
 {
+    opal_output(0, "udapl_del_procs\n");
     /* TODO */
     return OMPI_SUCCESS;
 }
@@ -142,6 +145,8 @@ int mca_btl_udapl_register(
     mca_btl_udapl_module_t* udapl_btl = (mca_btl_udapl_module_t*) btl; 
     udapl_btl->udapl_reg[tag].cbfunc = cbfunc; 
     udapl_btl->udapl_reg[tag].cbdata = cbdata; 
+
+    opal_output(0, "udapl_register\n");
     return OMPI_SUCCESS;
 }
 
@@ -160,6 +165,8 @@ mca_btl_base_descriptor_t* mca_btl_udapl_alloc(
     mca_btl_udapl_module_t* udapl_btl = (mca_btl_udapl_module_t*) btl; 
     mca_btl_udapl_frag_t* frag;
     int rc;
+
+    opal_output(0, "udapl_alloc\n");
     
     if(size <= btl->btl_eager_limit) { 
         MCA_BTL_UDAPL_FRAG_ALLOC_EAGER(udapl_btl, frag, rc); 
@@ -190,7 +197,10 @@ int mca_btl_udapl_free(
     struct mca_btl_base_module_t* btl, 
     mca_btl_base_descriptor_t* des) 
 {
-    mca_btl_udapl_frag_t* frag = (mca_btl_udapl_frag_t*)des; 
+    mca_btl_udapl_frag_t* frag = (mca_btl_udapl_frag_t*)des;
+
+    opal_output(0, "udapl_free\n");
+
     if(frag->size == 0) {
         btl->btl_mpool->mpool_release(btl->btl_mpool, frag->registration);
         MCA_BTL_UDAPL_FRAG_RETURN_USER(btl, frag); 
@@ -228,6 +238,8 @@ mca_btl_base_descriptor_t* mca_btl_udapl_prepare_src(
     int32_t free_after;
     int rc;
 
+    opal_output(0, "udapl_prepare_src\n");
+
     /*
      * If the data has already been pinned and is contigous than we can
      * use it in place.
@@ -378,6 +390,8 @@ mca_btl_base_descriptor_t* mca_btl_udapl_prepare_dst(
     long lb;
     int rc;
 
+    opal_output(0, "udapl_prepare_dst\n");
+
     MCA_BTL_UDAPL_FRAG_ALLOC_USER(btl, frag, rc);
     if(NULL == frag) {
         return NULL;
@@ -437,6 +451,7 @@ int mca_btl_udapl_send(
     mca_btl_base_tag_t tag)
    
 {
+    opal_output(0, "udapl_send\n");
     return OMPI_ERR_NOT_IMPLEMENTED; 
 }
 
@@ -455,6 +470,7 @@ int mca_btl_udapl_put(
     mca_btl_base_endpoint_t* endpoint,
     mca_btl_base_descriptor_t* des)
 {
+    opal_output(0, "udapl_put\n");
     return OMPI_ERR_NOT_IMPLEMENTED; 
 }
 
@@ -474,6 +490,7 @@ int mca_btl_udapl_get(
     mca_btl_base_endpoint_t* endpoint,
     mca_btl_base_descriptor_t* des)
 {
+    opal_output(0, "udapl_get\n");
     return OMPI_ERR_NOT_IMPLEMENTED;
 }
 
@@ -486,6 +503,7 @@ int mca_btl_udapl_finalize(struct mca_btl_base_module_t* btl)
 {
     mca_btl_udapl_module_t* udapl_btl = (mca_btl_udapl_module_t*) btl; 
 
+    opal_output(0, "udapl_finalize\n");
     OBJ_DESTRUCT(&udapl_btl->udapl_lock);
     OBJ_DESTRUCT(&udapl_btl->udapl_frag_eager);
     OBJ_DESTRUCT(&udapl_btl->udapl_frag_max);

ompi/mca/btl/udapl/btl_udapl.h

@@ -1,6 +1,5 @@
-
 /*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
  * Copyright (c) 2004-2005 The University of Tennessee and The University
@@ -43,10 +42,6 @@
 extern "C" {
 #endif
 
-/*
-#define GM_BUFFER_SIZE   7
-#define GM_BUFFER_LENGTH gm_max_length_for_size(GM_BUFFER_SIZE)
-*/
 
 /**
  * uDAPL BTL component.
@@ -59,6 +54,7 @@ struct mca_btl_udapl_component_t {
     size_t  udapl_max_btls; /**< maximum number of supported hcas */
     struct  mca_btl_udapl_module_t **udapl_btls; /**< array of available BTL modules */
     size_t  udapl_num_mru;
+    size_t  udapl_evd_qlen;
     size_t  udapl_eager_frag_size;
     size_t  udapl_max_frag_size;
     char*   udapl_port_name; 
@@ -86,11 +82,16 @@ extern mca_btl_udapl_component_t mca_btl_udapl_component;
 struct mca_btl_udapl_module_t {
     mca_btl_base_module_t  super;  /**< base BTL interface */
     mca_btl_base_recv_reg_t udapl_reg[256]; 
-
-    /* local port handle/address */
-    /* struct gm_port *port; */
     mca_btl_udapl_addr_t udapl_addr;
 
+    /* interface handle */
+    DAT_IA_HANDLE udapl_ia;
+
+    /* event dispatchers - default, data transfer, connection negotiation */
+    DAT_EVD_HANDLE udapl_evd_dflt;
+    DAT_EVD_HANDLE udapl_evd_dto;
+    DAT_EVD_HANDLE udapl_evd_conn;
+
     /* free list of fragment descriptors */
     ompi_free_list_t udapl_frag_eager;
     ompi_free_list_t udapl_frag_max;

ompi/mca/btl/udapl/btl_udapl_component.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
  * Copyright (c) 2004-2005 The University of Tennessee and The University
@@ -105,6 +105,8 @@ int mca_btl_udapl_component_open(void)
 {  
     int param, value;
 
+    opal_output(0, "udapl_component_open\n");
+
     /* initialize state */
     mca_btl_udapl_component.udapl_num_btls=0;
     mca_btl_udapl_component.udapl_btls=NULL;
@@ -121,11 +123,13 @@ int mca_btl_udapl_component_open(void)
     mca_btl_udapl_component.udapl_free_list_inc =
         mca_btl_udapl_param_register_int ("free_list_inc", 8);
     mca_btl_udapl_component.udapl_debug = 
-        mca_btl_udapl_param_register_int("debug", 0); 
+        mca_btl_udapl_param_register_int("debug", 1); 
     mca_btl_udapl_component.udapl_mpool_name = 
         mca_btl_udapl_param_register_string("mpool", "udapl"); 
     mca_btl_udapl_component.udapl_max_btls = 
-        mca_btl_udapl_param_register_int("max_modules", 4); 
+        mca_btl_udapl_param_register_int("max_modules", 4);
+    mca_btl_udapl_component.udapl_evd_qlen =
+        mca_btl_udapl_param_register_int("evd_qlen", 8);
     mca_btl_udapl_component.udapl_num_high_priority = 
         mca_btl_udapl_param_register_int("num_high_priority", 8); 
     mca_btl_udapl_component.udapl_num_repost = 
@@ -184,22 +188,73 @@ int mca_btl_udapl_component_open(void)
 
 int mca_btl_udapl_component_close(void)
 {
+    opal_output(0, "udapl_component_close\n");
+
+    /* TODO - clean up each btl module */
     return OMPI_SUCCESS;
 }
 
 
+/**
+  * Report a uDAPL error - for debugging
+  */
+
+static void
+mca_btl_udapl_error(DAT_RETURN ret, char* str)
+{
+	char* major;
+	char* minor;
+
+    /* don't output anything if debug is not set */
+    if(0 == mca_btl_udapl_component.udapl_debug) {
+        return;
+    }
+
+	if(DAT_SUCCESS != dat_strerror(ret,
+			(const char**)&major, (const char**)&minor))
+	{
+		printf("dat_strerror failed! ret is %d\n", ret);
+		exit(-1);
+	}
+
+	opal_output(0, "ERROR: %s %s %s\n", str, major, minor);
+}
+
+
 /**
  * Initialize module instance
  */
 
-#if 0
 static int
-mca_btl_udapl_module_init (mca_btl_udapl_module_t * btl)
+mca_btl_udapl_module_init (DAT_NAME_PTR ia_name,
+                        mca_btl_udapl_module_t * btl)
 {
-    /*mca_mpool_base_resources_t resources;*/
+    DAT_RETURN rc;
-    /*int32_t num_high_priority;
+
-    int32_t i;
+    /* open the uDAPL interface */
-    int rc;*/
+    rc = dat_ia_open(ia_name, mca_btl_udapl_component.udapl_evd_qlen,
+            &btl->udapl_evd_dflt, &btl->udapl_ia);
+    if(DAT_SUCCESS != rc) {
+        mca_btl_udapl_error(rc, "dat_ia_open");
+        return OMPI_ERROR;
+    }
+
+    /* set up evd's */
+    rc = dat_evd_create(btl->udapl_ia,
+            mca_btl_udapl_component.udapl_evd_qlen, DAT_HANDLE_NULL,
+            DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG, &btl->udapl_evd_dto);
+    if(DAT_SUCCESS != rc) {
+        mca_btl_udapl_error(rc, "dat_evd_create (dto)");
+        return OMPI_ERROR;
+    }
+
+    rc = dat_evd_create(btl->udapl_ia,
+            mca_btl_udapl_component.udapl_evd_qlen, DAT_HANDLE_NULL,
+            DAT_EVD_DTO_FLAG | DAT_EVD_RMR_BIND_FLAG, &btl->udapl_evd_conn);
+    if(DAT_SUCCESS != rc) {
+        mca_btl_udapl_error(rc, "dat_evd_create (conn)");
+        return OMPI_ERROR;
+    }
 
     /* initialize objects */
     OBJ_CONSTRUCT(&btl->udapl_frag_eager, ompi_free_list_t);
@@ -209,20 +264,9 @@ mca_btl_udapl_module_init (mca_btl_udapl_module_t * btl)
     OBJ_CONSTRUCT(&btl->udapl_repost, opal_list_t);
     OBJ_CONSTRUCT(&btl->udapl_mru_reg, opal_list_t);
     OBJ_CONSTRUCT(&btl->udapl_lock, opal_mutex_t);
-                                                                                                  
-    /* query nic tokens */
-
-    /* initialize memory pool */
-
-    /* initialize free lists */
-
-    /* post receive buffers */
-
-    /* enable rdma */
     
     return OMPI_SUCCESS;
 }
-#endif
 
 /*
  *  Register uDAPL component addressing information. The MCA framework
@@ -237,6 +281,8 @@ mca_btl_udapl_modex_send(void)
     size_t      size;
     mca_btl_udapl_addr_t *addrs = NULL;
 
+    opal_output(0, "udapl_modex_send\n");
+
     size = mca_btl_udapl_component.udapl_num_btls * sizeof (mca_btl_udapl_addr_t);
     if (0 != size) {
         addrs = (mca_btl_udapl_addr_t *)malloc (size);
@@ -256,7 +302,6 @@ mca_btl_udapl_modex_send(void)
     return rc;
 }
                                                                                                                 
-
 /*
  * Initialize the uDAPL component,
  * check how many interfaces are available and create a btl module for each.
@@ -269,17 +314,61 @@ mca_btl_udapl_component_init (int *num_btl_modules,
 {
     DAT_PROVIDER_INFO* datinfo;
     mca_btl_base_module_t **btls;
-    *num_btl_modules = 0;
+    mca_btl_udapl_module_t *btl;
+    size_t i;
+
+    opal_output(0, "udapl_component_init\n");
 
     /* enumerate uDAPL interfaces */
     datinfo = malloc(mca_btl_udapl_component.udapl_max_btls * sizeof(DAT_PROVIDER_INFO));
+    if(NULL == datinfo) {
+        return NULL;
+    }
     if(DAT_SUCCESS != dat_registry_list_providers(mca_btl_udapl_component.udapl_max_btls,
             (DAT_COUNT*)&mca_btl_udapl_component.udapl_num_btls, &datinfo)) {
         free(datinfo);
         return NULL;
     }
- 
+
+    /* Make sure we have some interfaces */
+    if(0 == mca_btl_udapl_component.udapl_num_btls) {
+        mca_btl_base_error_no_nics("uDAPL", "NIC");
+        free(datinfo);
+        return NULL;
+    }
+
     /* create a BTL module for each interface */
+    mca_btl_udapl_component.udapl_btls =
+            malloc(mca_btl_udapl_component.udapl_num_btls *
+            sizeof(mca_btl_udapl_module_t *));
+    if(NULL == mca_btl_udapl_component.udapl_btls) {
+        free(datinfo);
+        return NULL;
+    }
+
+    for(i = 0; i < mca_btl_udapl_component.udapl_num_btls; i++) {
+        opal_output(0, "udapl creating btl for %s\n", datinfo[i].ia_name);
+
+        btl = malloc(sizeof(mca_btl_udapl_module_t));
+        if(NULL == btl) {
+            free(datinfo);
+            free(mca_btl_udapl_component.udapl_btls);
+            return NULL;
+        }
+
+        /* copy default values into the new BTL */
+        memcpy(btl, &mca_btl_udapl_module, sizeof(mca_btl_udapl_module_t));
+
+        if(OMPI_SUCCESS != mca_btl_udapl_module_init(datinfo[i].ia_name, btl)) {
+            opal_output(0, "udapl module init for %s failed\n",
+                    datinfo[i].ia_name);
+            /*TODO - how do i correctly handle an error here? */
+            free(btl);
+        }
+
+        /* successful btl creation */
+        mca_btl_udapl_component.udapl_btls[i] = btl;
+    }
 
     /* finished with datinfo */
     free(datinfo);
@@ -304,7 +393,7 @@ mca_btl_udapl_component_init (int *num_btl_modules,
                                                                                                                 
 
 /*
- *  GM component progress.
+ *  uDAPL component progress.
  */
 
 
@@ -313,6 +402,8 @@ int mca_btl_udapl_component_progress()
     static int32_t inprogress = 0;
     int count = 0;
     size_t i;
+
+    opal_output(0, "udapl_component_progress\n");
     
     /* could get into deadlock in this case as we post recvs after callback completes */
     if(OPAL_THREAD_ADD32(&inprogress, 1) > 1) {

ompi/mca/btl/udapl/btl_udapl_endpoint.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana
+ * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
  *                         Corporation.  All rights reserved.
  * Copyright (c) 2004-2005 The University of Tennessee and The University
@@ -34,15 +34,7 @@ extern "C" {
  * Structure used to publish uDAPL id information to peers.
  */
 struct mca_btl_udapl_addr_t {
-#if 0
+    int foo;    /* placeholder to prevent division by 0 */
-#if GM_API_VERSION > 0x200
-    unsigned int global_id;
-#else
-    char global_id[GM_MAX_HOST_NAME_LEN];
-#endif  /* GM_API_VERSION > 0x200 */
-#endif
-    unsigned int node_id;
-    unsigned int port_id;
 };
 typedef struct mca_btl_udapl_addr_t mca_btl_udapl_addr_t;
                                                                                                                 

ompi/mca/btl/udapl/configure.m4

@@ -0,0 +1,47 @@
+# -*- shell-script -*-
+#
+# Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
+#                         University Research and Technology
+#                         Corporation.  All rights reserved.
+# Copyright (c) 2004-2005 The University of Tennessee and The University
+#                         of Tennessee Research Foundation.  All rights
+#                         reserved.
+# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+#                         University of Stuttgart.  All rights reserved.
+# Copyright (c) 2004-2005 The Regents of the University of California.
+#                         All rights reserved.
+# $COPYRIGHT$
+# 
+# Additional copyrights may follow
+# 
+# $HEADER$
+#
+
+
+# MCA_btl_udapl_CONFIG([action-if-can-compile], 
+#                      [action-if-cant-compile])
+# ------------------------------------------------
+AC_DEFUN([MCA_btl_udapl_CONFIG],[
+    OMPI_CHECK_UDAPL([btl_udapl],
+                     [btl_udapl_happy="yes"],
+                     [btl_udapl_happy="no"])
+
+    AS_IF([test "$btl_udapl_happy" = "yes"],
+          [btl_udapl_WRAPPER_EXTRA_LDFLAGS="$btl_udapl_LDFLAGS"
+           btl_udapl_WRAPPER_EXTRA_LIBS="$btl_udapl_LIBS"
+           $1],
+          [$2])
+
+    # Borrowed from MVAPI BTL - a data structure in the uDAPL headers
+    # is not fully ISO C.  Remove -pedantic to silence a warning.
+    btl_udapl_CFLAGS="`echo $CFLAGS | sed 's/-pedantic//g'`"
+    AS_IF([test "$btl_udapl_CFLAGS" != "$CFLAGS" -a "$btl_udapl_happy" = "yes"],
+          [AC_MSG_WARN([Removed -pedantic from CFLAGS for
+uDAPL component because the uDAPL headers are not fully ISO C: $btl_udapl_CFLAGS])])
+
+    # substitute in the things needed to build udapl
+    AC_SUBST([btl_udapl_CFLAGS])
+    AC_SUBST([btl_udapl_CPPFLAGS])
+    AC_SUBST([btl_udapl_LDFLAGS])
+    AC_SUBST([btl_udapl_LIBS])
+])dnl