udapl btl fix for #1725; replace WAIT with GET

This commit was SVN r20227.

ompi/mca/btl/udapl/btl_udapl.c

@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2006      Sandia National Laboratories. All rights
  *                         reserved.
- * Copyright (c) 2007-2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2007-2009 Sun Microsystems, Inc.  All rights reserved.
  *
  * $COPYRIGHT$
  * 
@@ -359,7 +359,9 @@ mca_btl_udapl_init(DAT_NAME_PTR ia_name, mca_btl_udapl_module_t* btl)
  
     /* initialize objects */
     OBJ_CONSTRUCT(&btl->udapl_frag_eager, ompi_free_list_t);
+    OBJ_CONSTRUCT(&btl->udapl_frag_eager_recv, ompi_free_list_t);
     OBJ_CONSTRUCT(&btl->udapl_frag_max, ompi_free_list_t);
+    OBJ_CONSTRUCT(&btl->udapl_frag_max_recv, ompi_free_list_t);
     OBJ_CONSTRUCT(&btl->udapl_frag_user, ompi_free_list_t);
     OBJ_CONSTRUCT(&btl->udapl_frag_control, ompi_free_list_t);
     OBJ_CONSTRUCT(&btl->udapl_lock, opal_mutex_t);
@@ -391,6 +393,20 @@ mca_btl_udapl_init(DAT_NAME_PTR ia_name, mca_btl_udapl_module_t* btl)
                            NULL,
                            NULL);
 
+    ompi_free_list_init_ex_new(&btl->udapl_frag_eager_recv,
+        sizeof(mca_btl_udapl_frag_eager_t) +
+            mca_btl_udapl_component.udapl_eager_frag_size,
+        mca_btl_udapl_component.udapl_buffer_alignment,
+        OBJ_CLASS(mca_btl_udapl_frag_eager_t),
+        mca_btl_udapl_component.udapl_eager_frag_size,
+        mca_btl_udapl_component.udapl_buffer_alignment,
+        mca_btl_udapl_component.udapl_free_list_num,
+        mca_btl_udapl_component.udapl_free_list_max,
+        mca_btl_udapl_component.udapl_free_list_inc,
+                           btl->super.btl_mpool,
+                           NULL,
+                           NULL);
+
     ompi_free_list_init_ex_new(&btl->udapl_frag_max,
         sizeof(mca_btl_udapl_frag_max_t) +
             mca_btl_udapl_component.udapl_max_frag_size,
@@ -405,6 +421,20 @@ mca_btl_udapl_init(DAT_NAME_PTR ia_name, mca_btl_udapl_module_t* btl)
                            NULL,
                            NULL);
 
+    ompi_free_list_init_ex_new(&btl->udapl_frag_max_recv,
+        sizeof(mca_btl_udapl_frag_max_t) +
+            mca_btl_udapl_component.udapl_max_frag_size,
+        mca_btl_udapl_component.udapl_buffer_alignment,
+        OBJ_CLASS(mca_btl_udapl_frag_max_t),
+        mca_btl_udapl_component.udapl_max_frag_size,
+        mca_btl_udapl_component.udapl_buffer_alignment,
+        mca_btl_udapl_component.udapl_free_list_num,
+        mca_btl_udapl_component.udapl_free_list_max,
+        mca_btl_udapl_component.udapl_free_list_inc,
+                           btl->super.btl_mpool,
+                           NULL,
+                           NULL);
+
     ompi_free_list_init_ex_new(&btl->udapl_frag_user,
         sizeof(mca_btl_udapl_frag_user_t),
         mca_btl_udapl_component.udapl_buffer_alignment,
@@ -485,7 +515,9 @@ int mca_btl_udapl_finalize(struct mca_btl_base_module_t* base_btl)
     /* destroy objects */
     OBJ_DESTRUCT(&udapl_btl->udapl_lock);
     OBJ_DESTRUCT(&udapl_btl->udapl_frag_eager);
+    OBJ_DESTRUCT(&udapl_btl->udapl_frag_eager_recv);
     OBJ_DESTRUCT(&udapl_btl->udapl_frag_max);
+    OBJ_DESTRUCT(&udapl_btl->udapl_frag_max_recv);
     OBJ_DESTRUCT(&udapl_btl->udapl_frag_user);
     OBJ_DESTRUCT(&udapl_btl->udapl_frag_control);
     OBJ_DESTRUCT(&udapl_btl->udapl_eager_rdma_lock);
@@ -912,6 +944,10 @@ mca_btl_base_descriptor_t* mca_btl_udapl_alloc(
         return NULL;
     }
 
+    if (NULL == frag) {
+        return NULL;
+    }
+
     frag->segment.seg_len = size;
 
     /* Set up the LMR triplet from the frag segment.

ompi/mca/btl/udapl/btl_udapl.h

@@ -10,7 +10,7 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
- * Copyright (c) 2007-2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2007-2009 Sun Microsystems, Inc.  All rights reserved.
  * Copyright (c) 2007      Cisco Systems, Inc.  All rights reserved.
  *
  * $COPYRIGHT$
@@ -123,7 +123,9 @@ struct mca_btl_udapl_module_t {
 
     /* free list of fragment descriptors */
     ompi_free_list_t udapl_frag_eager;
+    ompi_free_list_t udapl_frag_eager_recv;
     ompi_free_list_t udapl_frag_max;
+    ompi_free_list_t udapl_frag_max_recv;
     ompi_free_list_t udapl_frag_user;
     ompi_free_list_t udapl_frag_control;
     

ompi/mca/btl/udapl/btl_udapl_endpoint.c

@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2006      Sandia National Laboratories. All rights
  *                         reserved.
- * Copyright (c) 2007-2008 Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2007-2009 Sun Microsystems, Inc.  All rights reserved.
  *
  * $COPYRIGHT$
  * 
@@ -790,10 +790,17 @@ static int mca_btl_udapl_endpoint_finish_max(mca_btl_udapl_endpoint_t* endpoint)
     OPAL_THREAD_ADD32(&(endpoint->endpoint_btl->udapl_connect_inprogress), -1);
 
     /* post eager/max recv buffers */
-    mca_btl_udapl_endpoint_post_recv(endpoint,
+    ret = mca_btl_udapl_endpoint_post_recv(endpoint,
             mca_btl_udapl_component.udapl_eager_frag_size);
-    mca_btl_udapl_endpoint_post_recv(endpoint,
+    if (OMPI_SUCCESS != ret) {
+        return ret;
+    }
+
+    ret = mca_btl_udapl_endpoint_post_recv(endpoint,
             mca_btl_udapl_component.udapl_max_frag_size);
+    if (OMPI_SUCCESS != ret) {
+        return ret;
+    }
 
     /* progress eager frag queue as allowed */
     queue_len = opal_list_get_size(&(endpoint->endpoint_eager_frags));
@@ -849,14 +856,20 @@ static int mca_btl_udapl_endpoint_post_recv(mca_btl_udapl_endpoint_t* endpoint,
     
     for(i = 0; i < mca_btl_udapl_component.udapl_num_recvs; i++) {
         if(size == mca_btl_udapl_component.udapl_eager_frag_size) {
-            MCA_BTL_UDAPL_FRAG_ALLOC_EAGER(endpoint->endpoint_btl, frag, rc);
+            MCA_BTL_UDAPL_FRAG_ALLOC_EAGER_RECV(endpoint->endpoint_btl, frag, rc);
             ep = endpoint->endpoint_eager;
         } else {
             assert(size == mca_btl_udapl_component.udapl_max_frag_size);
-            MCA_BTL_UDAPL_FRAG_ALLOC_MAX(endpoint->endpoint_btl, frag, rc);
+            MCA_BTL_UDAPL_FRAG_ALLOC_MAX_RECV(endpoint->endpoint_btl, frag, rc);
             ep = endpoint->endpoint_max;
         } 
     
+        if (NULL == frag) {
+            BTL_ERROR(("ERROR: %s posting recv, out of resources\n",
+                "MCA_BTL_UDAPL_ALLOC"));
+            return rc;
+        }
+
         assert(size == frag->size);
         /* Set up the LMR triplet from the frag segment */
         /* Note that this triplet defines a sub-region of a registered LMR */

ompi/mca/btl/udapl/btl_udapl_frag.h

@@ -11,7 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2006      Sandia National Laboratories. All rights
  *                         reserved.
- * Copyright (c) 2007      Sun Microsystems, Inc.  All rights reserved.
+ * Copyright (c) 2007-2009 Sun Microsystems, Inc.  All rights reserved.
  *
  * $COPYRIGHT$
  * 
@@ -124,7 +124,7 @@ OBJ_CLASS_DECLARATION(mca_btl_udapl_frag_eager_rdma_t);
 #define MCA_BTL_UDAPL_FRAG_ALLOC_EAGER(btl, frag, rc)              \
 {                                                                  \
     ompi_free_list_item_t *item;                                   \
-    OMPI_FREE_LIST_WAIT(&((mca_btl_udapl_module_t*)btl)->udapl_frag_eager, item, rc); \
+    OMPI_FREE_LIST_GET(&((mca_btl_udapl_module_t*)btl)->udapl_frag_eager, item, rc); \
     frag = (mca_btl_udapl_frag_t*) item;                           \
 }
 
@@ -134,10 +134,17 @@ OBJ_CLASS_DECLARATION(mca_btl_udapl_frag_eager_rdma_t);
         (ompi_free_list_item_t*)(frag));                           \
 }
 
+#define MCA_BTL_UDAPL_FRAG_ALLOC_EAGER_RECV(btl, frag, rc)              \
+{                                                                  \
+    ompi_free_list_item_t *item;                                   \
+    OMPI_FREE_LIST_GET(&((mca_btl_udapl_module_t*)btl)->udapl_frag_eager_recv, item, rc); \
+    frag = (mca_btl_udapl_frag_t*) item;                           \
+}
+
 #define MCA_BTL_UDAPL_FRAG_ALLOC_MAX(btl, frag, rc)                \
 {                                                                  \
     ompi_free_list_item_t *item;                                   \
-    OMPI_FREE_LIST_WAIT(&((mca_btl_udapl_module_t*)btl)->udapl_frag_max, item, rc); \
+    OMPI_FREE_LIST_GET(&((mca_btl_udapl_module_t*)btl)->udapl_frag_max, item, rc); \
     frag = (mca_btl_udapl_frag_t*) item;                           \
 }
 
@@ -147,10 +154,17 @@ OBJ_CLASS_DECLARATION(mca_btl_udapl_frag_eager_rdma_t);
         (ompi_free_list_item_t*)(frag));                           \
 }
 
+#define MCA_BTL_UDAPL_FRAG_ALLOC_MAX_RECV(btl, frag, rc)                \
+{                                                                  \
+    ompi_free_list_item_t *item;                                   \
+    OMPI_FREE_LIST_GET(&((mca_btl_udapl_module_t*)btl)->udapl_frag_max_recv, item, rc); \
+    frag = (mca_btl_udapl_frag_t*) item;                           \
+}
+
 #define MCA_BTL_UDAPL_FRAG_ALLOC_USER(btl, frag, rc)               \
 {                                                                  \
     ompi_free_list_item_t *item;                                   \
-    OMPI_FREE_LIST_WAIT(&((mca_btl_udapl_module_t*)btl)->udapl_frag_user, item, rc); \
+    OMPI_FREE_LIST_GET(&((mca_btl_udapl_module_t*)btl)->udapl_frag_user, item, rc); \
     frag = (mca_btl_udapl_frag_t*) item;                           \
 }
 
@@ -163,7 +177,7 @@ OBJ_CLASS_DECLARATION(mca_btl_udapl_frag_eager_rdma_t);
 #define MCA_BTL_UDAPL_FRAG_ALLOC_CONTROL(btl, frag, rc)              \
 {                                                                  \
     ompi_free_list_item_t *item;                                   \
-    OMPI_FREE_LIST_WAIT(&((mca_btl_udapl_module_t*)btl)->udapl_frag_control, item, rc); \
+    OMPI_FREE_LIST_GET(&((mca_btl_udapl_module_t*)btl)->udapl_frag_control, item, rc); \
     frag = (mca_btl_udapl_frag_t*) item;                           \
 }