Update opal_free_list_t usage to reflect new class interface.

Please verify your components have been updated correctly. Keep in
mind that in terms of threading:

OPAL_FREE_LIST_GET -> opal_free_list_get_st
OPAL_FREE_LIST_RETURN -> opal_free_list_return_st

I used the opal_using_threads() variant anytime it appeared multiple
threads could be operating on the free list. If this is not the case
update to _st. If multiple threads are always in use change to _mt.

ompi/communicator/comm_request.c

@@ -33,7 +33,8 @@ void ompi_comm_request_init (void)
 {
     OBJ_CONSTRUCT(&ompi_comm_requests, opal_free_list_t);
     (void) opal_free_list_init (&ompi_comm_requests, sizeof (ompi_comm_request_t),
-                                OBJ_CLASS(ompi_comm_request_t), 0, -1, 8);
+                                OBJ_CLASS(ompi_comm_request_t), 0, 0, 0, -1, 8,
+                                NULL, 0, NULL, NULL, NULL);
 
     OBJ_CONSTRUCT(&ompi_comm_requests_active, opal_list_t);
     ompi_comm_request_progress_active = false;
@@ -237,10 +238,11 @@ OBJ_CLASS_INSTANCE(ompi_comm_request_item_t, opal_list_item_t, NULL, NULL);
 ompi_comm_request_t *ompi_comm_request_get (void)
 {
     opal_free_list_item_t *item;
-    int rc;
 
-    OPAL_FREE_LIST_GET(&ompi_comm_requests, item, rc);
+    item = opal_free_list_get (&ompi_comm_requests);
-    (void) rc;
+    if (OPAL_UNLIKELY(NULL == item)) {
+        return NULL;
+    }
 
     OMPI_REQUEST_INIT((ompi_request_t *) item, false);
 
@@ -254,6 +256,6 @@ void ompi_comm_request_return (ompi_comm_request_t *request)
         request->context = NULL;
     }
 
-    OPAL_FREE_LIST_RETURN(&ompi_comm_requests, (opal_free_list_item_t *) request);
+    opal_free_list_return (&ompi_comm_requests, (opal_free_list_item_t *) request);
 }
 

ompi/mca/mtl/ofi/mtl_ofi_component.c

@@ -93,8 +93,8 @@ ompi_mtl_ofi_component_open(void)
     OBJ_CONSTRUCT(&ompi_mtl_ofi.free_messages, opal_free_list_t);
     opal_free_list_init(&ompi_mtl_ofi.free_messages,
                         sizeof(ompi_mtl_ofi_message_t),
-                        OBJ_CLASS(ompi_mtl_ofi_message_t),
+                        OBJ_CLASS(ompi_mtl_ofi_message_t), 0, 0,
-                        1, -1, 1);
+                        1, -1, 1, NULL, 0, NULL, NULL, NULL);
 
     ompi_mtl_ofi.domain =  NULL;
     ompi_mtl_ofi.av     =  NULL;

ompi/mca/mtl/ofi/mtl_ofi_message.h

@@ -1,6 +1,9 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2013-2014 Intel, Inc. All rights reserved
  * Copyright (c) 2014      Cisco Systems, Inc. All rights reserved
+ * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
  *
  * $COPYRIGHT$
  *
@@ -26,13 +29,10 @@ OBJ_CLASS_DECLARATION(ompi_mtl_ofi_message_t);
 static inline ompi_mtl_ofi_message_t*
 ompi_mtl_ofi_message_alloc(const struct fi_cq_tagged_entry *wc)
 {
-    int rc __opal_attribute_unused__;
     opal_free_list_item_t *tmp;
     ompi_mtl_ofi_message_t *message;
 
-    OPAL_FREE_LIST_GET(&ompi_mtl_ofi.free_messages,
+    tmp = opal_free_list_get (&ompi_mtl_ofi.free_messages);
-                       tmp,
-                       rc);
     if (NULL == tmp) return NULL;
 
     message = (ompi_mtl_ofi_message_t*) tmp;
@@ -45,8 +45,8 @@ ompi_mtl_ofi_message_alloc(const struct fi_cq_tagged_entry *wc)
 static inline void
 ompi_mtl_ofi_message_free(ompi_mtl_ofi_message_t *message)
 {
-    OPAL_FREE_LIST_RETURN(&ompi_mtl_ofi.free_messages,
+    opal_free_list_return (&ompi_mtl_ofi.free_messages,
-                          &message->super);
+                           &message->super);
 }
 
 #endif

ompi/mca/mtl/portals4/mtl_portals4_component.c

@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -204,7 +205,7 @@ ompi_mtl_portals4_component_open(void)
                         sizeof(ompi_mtl_portals4_message_t) + 
                         ompi_mtl_portals4.eager_limit,
                         OBJ_CLASS(ompi_mtl_portals4_message_t),
-                        1, -1, 1);
+                        0, 0, 1, -1, 1, NULL, 0, NULL, NULL, NULL);
 
     ompi_mtl_portals4.ni_h = PTL_INVALID_HANDLE;
     ompi_mtl_portals4.send_eq_h = PTL_INVALID_HANDLE;

ompi/mca/mtl/portals4/mtl_portals4_flowctl.c

@@ -1,5 +1,8 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2012      Sandia National Laboratories.  All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -41,7 +44,7 @@ ompi_mtl_portals4_flowctl_init(void)
     opal_free_list_init(&ompi_mtl_portals4.flowctl.pending_fl,
                         sizeof(ompi_mtl_portals4_pending_request_t),
                         OBJ_CLASS(ompi_mtl_portals4_pending_request_t),
-                        1, -1, 1);
+                        0, 0, 1, -1, 1, NULL, 0, NULL, NULL, NULL);
 
     ompi_mtl_portals4.flowctl.max_send_slots = (ompi_mtl_portals4.send_queue_size - 3) / 3;
     ompi_mtl_portals4.flowctl.send_slots = ompi_mtl_portals4.flowctl.max_send_slots;

ompi/mca/mtl/portals4/mtl_portals4_message.h

@@ -1,5 +1,8 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2012      Sandia National Laboratories.  All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC. All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -22,13 +25,10 @@ OBJ_CLASS_DECLARATION(ompi_mtl_portals4_message_t);
 static inline ompi_mtl_portals4_message_t*
 ompi_mtl_portals4_message_alloc(const ptl_event_t *ev)
 {
-    int rc __opal_attribute_unused__;
     opal_free_list_item_t *tmp;
     ompi_mtl_portals4_message_t* message;
 
-    OPAL_FREE_LIST_GET(&ompi_mtl_portals4.fl_message,
+    tmp = opal_free_list_get (&ompi_mtl_portals4.fl_message);
-                       tmp,
-                       rc);
     if (NULL == tmp) return NULL;
 
     message = (ompi_mtl_portals4_message_t*) tmp;
@@ -51,8 +51,8 @@ ompi_mtl_portals4_message_alloc(const ptl_event_t *ev)
 static inline void
 ompi_mtl_portals4_message_free(ompi_mtl_portals4_message_t *message)
 {
-    OPAL_FREE_LIST_RETURN(&ompi_mtl_portals4.fl_message,
+    opal_free_list_return (&ompi_mtl_portals4.fl_message,
-                          &message->super);
+                           &message->super);
 }
 
 #endif

ompi/mca/mtl/portals4/mtl_portals4_send.c

@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -10,6 +11,8 @@
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
  * Copyright (c) 2010      Sandia National Laboratories.  All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -118,8 +121,8 @@ ompi_mtl_portals4_callback(ptl_event_t *ev,
         *complete = true;
 #if OMPI_MTL_PORTALS4_FLOW_CONTROL
         OPAL_THREAD_ADD32(&ompi_mtl_portals4.flowctl.send_slots, 1);
-        OPAL_FREE_LIST_RETURN(&ompi_mtl_portals4.flowctl.pending_fl,
+        opal_free_list_return (&ompi_mtl_portals4.flowctl.pending_fl,
-                              &ptl_request->pending->super);
+                               &ptl_request->pending->super);
 
         if (OPAL_UNLIKELY(0 != opal_list_get_size(&ompi_mtl_portals4.flowctl.pending_sends))) {
             ompi_mtl_portals4_pending_list_progress();
@@ -423,7 +426,7 @@ ompi_mtl_portals4_send_start(struct mca_mtl_base_module_t* mtl,
                          (int)length));
 
 #if OMPI_MTL_PORTALS4_FLOW_CONTROL
-    OPAL_FREE_LIST_GET(&ompi_mtl_portals4.flowctl.pending_fl, item, ret);
+    item = opal_free_list_get (&ompi_mtl_portals4.flowctl.pending_fl);
     if (NULL == item) return OMPI_ERR_OUT_OF_RESOURCE;
 
     pending = (ompi_mtl_portals4_pending_request_t*) item;

ompi/mca/pml/yalla/pml_yalla_freelist.h

@@ -1,5 +1,8 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (C) Mellanox Technologies Ltd. 2001-2011.  ALL RIGHTS RESERVED.
+ * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -17,20 +20,15 @@
 #define mca_pml_yalla_freelist_t opal_free_list_t
 
 #define PML_YALLA_FREELIST_GET(_freelist) \
-    ({ \
+        opal_free_list_get (_freelist);\
-        opal_free_list_item_t *item; \
-        int rc; \
-        OPAL_FREE_LIST_GET(_freelist, item, rc); \
-        (void*)(item); \
-    })
 
 #define PML_YALLA_FREELIST_RETURN(_freelist, _item) \
     { \
-        OPAL_FREE_LIST_RETURN(_freelist, _item); \
+        opal_free_list_return (_freelist, _item); \
     }
 
 #define PML_YALLA_FREELIST_INIT(_fl, _type, _initial, _max, _batch) \
     opal_free_list_init(_fl, sizeof(_type), OBJ_CLASS(_type), \
-                        _initial, _max, _batch);
+                        0, 0, _initial, _max, _batch, NULL, 0, NULL, NULL, NULL);
 
 #endif /* PML_YALLA_FREELIST_H_ */

ompi/message/message.c

@@ -1,6 +1,9 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2011      Sandia National Laboratories. All rights reserved.
  * Copyright (c) 2012 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -45,7 +48,7 @@ ompi_message_init(void)
     rc = opal_free_list_init(&ompi_message_free_list,
                              sizeof(ompi_message_t),
                              OBJ_CLASS(ompi_message_t),
-                             8, -1, 8);
+                             0, 0, 8, -1, 8, NULL, 0, NULL, NULL, NULL);
 
     OBJ_CONSTRUCT(&ompi_message_f_to_c_table, opal_pointer_array_t);
 

ompi/message/message.h

@@ -1,6 +1,9 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2011-2012 Sandia National Laboratories. All rights reserved.
  * Copyright (c) 2012      Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -56,13 +59,7 @@ static inline
 ompi_message_t*
 ompi_message_alloc(void)
 {
-    int rc;
+    return (ompi_message_t *) opal_free_list_get (&ompi_message_free_list);
-    opal_free_list_item_t *tmp;
-    OPAL_FREE_LIST_GET(&ompi_message_free_list,
-                       tmp,
-                       rc);
-    (void)rc;  /* prevent "set but not used" compiler complaints */
-    return (ompi_message_t*) tmp;
 }
 
 static inline
@@ -75,8 +72,8 @@ ompi_message_return(ompi_message_t* msg)
         msg->m_f_to_c_index = MPI_UNDEFINED;
     }
 
-    OPAL_FREE_LIST_RETURN(&ompi_message_free_list, 
+    opal_free_list_return (&ompi_message_free_list,
-                          &msg->super);
+                           &msg->super);
 }
 
 

ompi/mpi/java/c/mpi_MPI.c

@@ -1,3 +1,4 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
  * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
  *                         University Research and Technology
@@ -9,6 +10,8 @@
  *                         University of Stuttgart.  All rights reserved.
  * Copyright (c) 2004-2005 The Regents of the University of California.
  *                         All rights reserved.
+ * Copyright (c) 2015      Los Alamos National Security, LLC.  All rights
+ *                         reserved.
  * $COPYRIGHT$
  * 
  * Additional copyrights may follow
@@ -149,7 +152,8 @@ static void initFreeList(void)
     OBJ_CONSTRUCT(&ompi_java_buffers, opal_free_list_t);
 
     int r = opal_free_list_init(&ompi_java_buffers, sizeof(ompi_java_buffer_t),
-                                OBJ_CLASS(ompi_java_buffer_t), 2, -1, 2);
+                                OBJ_CLASS(ompi_java_buffer_t), 0, 0, 2, -1, 2,
+                                NULL, 0, NULL, NULL, NULL);
     if(r != OPAL_SUCCESS)
     {
         fprintf(stderr, "Unable to initialize ompi_java_buffers.\n");
@@ -511,12 +515,11 @@ static void* getBuffer(JNIEnv *env, ompi_java_buffer_t **item, int size)
     }
     else
     {
-        int rc;
         opal_free_list_item_t *freeListItem;
-        OPAL_FREE_LIST_GET(&ompi_java_buffers, freeListItem, rc);
+        freeListItem = opal_free_list_get (&ompi_java_buffers);
 
-        ompi_java_exceptionCheck(env,
+        ompi_java_exceptionCheck(env, NULL == freeListItem ? OMPI_ERROR :
-                rc==OPAL_SUCCESS ? OMPI_SUCCESS : OMPI_ERROR);
+                                 OMPI_SUCCESS);
 
         *item = (ompi_java_buffer_t*)freeListItem;
         return (*item)->buffer;
@@ -532,7 +535,7 @@ static void releaseBuffer(void *ptr, ompi_java_buffer_t *item)
     else
     {
         assert(item->buffer == ptr);
-        OPAL_FREE_LIST_RETURN(&ompi_java_buffers, (opal_free_list_item_t*)item);
+        opal_free_list_return (&ompi_java_buffers, (opal_free_list_item_t*)item);
     }
 }
 

opal/mca/btl/sm/btl_sm.c

@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2006-2007 Voltaire. All rights reserved.
  * Copyright (c) 2009-2012 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2014 Los Alamos National Security, LLC.
+ * Copyright (c) 2010-2015 Los Alamos National Security, LLC.
  *                         All rights reserved. 
  * Copyright (c) 2010-2012 IBM Corporation.  All rights reserved.
  * Copyright (c) 2012      Oracle and/or its affiliates.  All rights reserved.
@@ -413,7 +413,8 @@ sm_btl_first_time_init(mca_btl_sm_t *sm_btl,
     i = opal_free_list_init(&mca_btl_sm_component.pending_send_fl,
                             sizeof(btl_sm_pending_send_item_t),
                             OBJ_CLASS(opal_free_list_item_t),
-                            16, -1, 32);
+                            0, 0, 16, -1, 32, NULL, 0, NULL, NULL
+                            NULL);
     if ( OPAL_SUCCESS != i )
         return i;
 

opal/mca/btl/sm/btl_sm_component.c

@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2006-2007 Voltaire. All rights reserved.
  * Copyright (c) 2009-2014 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2014 Los Alamos National Security, LLC.
+ * Copyright (c) 2010-2015 Los Alamos National Security, LLC.
  *                         All rights reserved.
  * Copyright (c) 2011-2014 NVIDIA Corporation.  All rights reserved.
  * Copyright (c) 2010-2012 IBM Corporation.  All rights reserved.
@@ -1054,7 +1054,7 @@ void btl_sm_process_pending_sends(struct mca_btl_base_endpoint_t *ep)
         MCA_BTL_SM_FIFO_WRITE(ep, ep->my_smp_rank, ep->peer_smp_rank, si->data,
                           true, false, rc);
 
-        OPAL_FREE_LIST_RETURN(&mca_btl_sm_component.pending_send_fl, (opal_list_item_t*)si);
+        opal_free_list_return (&mca_btl_sm_component.pending_send_fl, (opal_list_item_t*)si);
 
         if ( OPAL_SUCCESS != rc )
             return;

opal/mca/btl/sm/btl_sm_fifo.h

@@ -1,3 +1,27 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2012 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 Voltaire. All rights reserved.
+ * Copyright (c) 2009-2010 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2015 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2012 IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
+
 #ifndef MCA_BTL_SM_FIFO_H
 #define MCA_BTL_SM_FIFO_H
 
@@ -7,13 +31,12 @@
 static void
 add_pending(struct mca_btl_base_endpoint_t *ep, void *data, bool resend)
 {
-    int rc;
     btl_sm_pending_send_item_t *si;
     opal_free_list_item_t *i;
-    OPAL_FREE_LIST_GET(&mca_btl_sm_component.pending_send_fl, i, rc);
+    i = opal_free_list_get (&mca_btl_sm_component.pending_send_fl);
 
     /* don't handle error for now */
-    assert(i != NULL && rc == OPAL_SUCCESS);
+    assert(i != NULL);
 
     si = (btl_sm_pending_send_item_t*)i;
     si->data = data;

opal/mca/btl/smcuda/btl_smcuda.c

@@ -441,7 +441,7 @@ smcuda_btl_first_time_init(mca_btl_smcuda_t *smcuda_btl,
     i = opal_free_list_init(&mca_btl_smcuda_component.pending_send_fl,
                             sizeof(btl_smcuda_pending_send_item_t),
                             OBJ_CLASS(opal_free_list_item_t),
-                            16, -1, 32);
+                            0, 0, 16, -1, 32, NULL, 0, NULL, NULL, NULL);
     if ( OPAL_SUCCESS != i )
         return i;
 

opal/mca/btl/smcuda/btl_smcuda_component.c

@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2006-2007 Voltaire. All rights reserved.
  * Copyright (c) 2009-2010 Cisco Systems, Inc.  All rights reserved.
- * Copyright (c) 2010-2014 Los Alamos National Security, LLC.
+ * Copyright (c) 2010-2015 Los Alamos National Security, LLC.
  *                         All rights reserved.
  * Copyright (c) 2011-2014 NVIDIA Corporation.  All rights reserved.
  * Copyright (c) 2014      Intel, Inc. All rights reserved.
@@ -990,7 +990,7 @@ void btl_smcuda_process_pending_sends(struct mca_btl_base_endpoint_t *ep)
         MCA_BTL_SMCUDA_FIFO_WRITE(ep, ep->my_smp_rank, ep->peer_smp_rank, si->data,
                           true, false, rc);
 
-        OPAL_FREE_LIST_RETURN(&mca_btl_smcuda_component.pending_send_fl, (opal_list_item_t*)si);
+        opal_free_list_return (&mca_btl_smcuda_component.pending_send_fl, (opal_list_item_t*)si);
 
         if ( OPAL_SUCCESS != rc )
             return;

opal/mca/btl/smcuda/btl_smcuda_fifo.h

@@ -1,3 +1,26 @@
+/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
+/*
+ * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana
+ *                         University Research and Technology
+ *                         Corporation.  All rights reserved.
+ * Copyright (c) 2004-2012 The University of Tennessee and The University
+ *                         of Tennessee Research Foundation.  All rights
+ *                         reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart,
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * Copyright (c) 2006-2007 Voltaire. All rights reserved.
+ * Copyright (c) 2009-2010 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2010-2015 Los Alamos National Security, LLC.
+ *                         All rights reserved.
+ * Copyright (c) 2010-2012 IBM Corporation.  All rights reserved.
+ * $COPYRIGHT$
+ *
+ * Additional copyrights may follow
+ *
+ * $HEADER$
+ */
 #ifndef MCA_BTL_SMCUDA_FIFO_H
 #define MCA_BTL_SMCUDA_FIFO_H
 
@@ -7,13 +30,12 @@
 static void
 add_pending(struct mca_btl_base_endpoint_t *ep, void *data, bool resend)
 {
-    int rc;
     btl_smcuda_pending_send_item_t *si;
     opal_free_list_item_t *i;
-    OPAL_FREE_LIST_GET(&mca_btl_smcuda_component.pending_send_fl, i, rc);
+    i = opal_free_list_get (&mca_btl_smcuda_component.pending_send_fl);
 
     /* don't handle error for now */
-    assert(i != NULL && rc == OPAL_SUCCESS);
+    assert(i != NULL);
 
     si = (btl_smcuda_pending_send_item_t*)i;
     si->data = data;

orte/mca/oob/ud/oob_ud.h

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2011-2015 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2014      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -136,6 +136,8 @@ struct mca_oob_ud_port_t {
     int                  mtu;
     uint16_t             lid;
     uint8_t              port_num;
+    /** current send buffer index. used by init function for free_msgs member */
+    int                  send_buffer_index;
 
     mca_oob_ud_reg_mem_t grh_buf;
     mca_oob_ud_reg_mem_t msg_buf;

orte/mca/oob/ud/oob_ud_component.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2011-2015 Los Alamos National Security, LLC. All rights
  *                         reserved.
  * Copyright (c) 2014      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -355,11 +355,11 @@ static int mca_oob_ud_component_startup(void)
             }
 
             rc = opal_free_list_init (&port->data_qps,
-                                      sizeof (mca_oob_ud_qp_t),
+                                      sizeof (mca_oob_ud_qp_t), 8,
-                                      OBJ_CLASS(mca_oob_ud_qp_t),
+                                      OBJ_CLASS(mca_oob_ud_qp_t), 0, 0,
                                       mca_oob_ud_component.ud_min_qp,
                                       mca_oob_ud_component.ud_max_qp,
-                                      2);
+                                      2, NULL, 0, NULL, NULL, NULL);
             if (OPAL_SUCCESS != rc) {
                 mca_oob_ud_listen_destroy (port);
                 continue;
@@ -506,8 +506,7 @@ static int   mca_oob_ud_component_ft_event(int state) {
 static int mca_oob_ud_port_alloc_buffers (mca_oob_ud_port_t *port) {
     int total_buffer_count = mca_oob_ud_component.ud_recv_buffer_count +
         mca_oob_ud_component.ud_send_buffer_count;
-    opal_list_item_t *item;
+    int rc;
-    int rc, i;
 
     rc = mca_oob_ud_alloc_reg_mem (port->device->ib_pd, &port->grh_buf,
                                    mca_oob_ud_component.ud_recv_buffer_count * sizeof (struct ibv_grh));
@@ -522,23 +521,15 @@ static int mca_oob_ud_port_alloc_buffers (mca_oob_ud_port_t *port) {
         return rc;
     }
 
-    rc = opal_free_list_init (&port->free_msgs, sizeof (mca_oob_ud_msg_t),
+    port->send_buffer_index = 0;
-                              OBJ_CLASS(mca_oob_ud_msg_t), mca_oob_ud_component.ud_send_buffer_count,
+    rc = opal_free_list_init (&port->free_msgs, sizeof (mca_oob_ud_msg_t), 8,
-                              mca_oob_ud_component.ud_send_buffer_count, 0);
+                              OBJ_CLASS(mca_oob_ud_msg_t), 0, 0, mca_oob_ud_component.ud_send_buffer_count,
+                              mca_oob_ud_component.ud_send_buffer_count, 0, NULL, 0, NULL, mca_oob_ud_msg_init,
+                              port);
     if (ORTE_SUCCESS != rc) {
         return rc;
     }
 
-    for (i = 0, item = opal_list_get_first (&port->free_msgs.super) ;
-         item != opal_list_get_end (&port->free_msgs.super) ;
-         item = opal_list_get_next (item), ++i) {
-        char *ptr = port->msg_buf.ptr + (i + mca_oob_ud_component.ud_recv_buffer_count) *
-            port->mtu;
-
-        mca_oob_ud_msg_init ((mca_oob_ud_msg_t *) item, port,
-                             ptr, port->msg_buf.mr);
-    }
-
     return rc;
 }
 

orte/mca/oob/ud/oob_ud_qp.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2011-2015 Los Alamos National Security, LLC. All rights
  *                         reserved.
  *               2014      Mellanox Technologies, Inc.
  *                         All rights reserved.
@@ -275,11 +275,11 @@ int mca_oob_ud_qp_data_aquire (struct mca_oob_ud_port_t *port, mca_oob_ud_qp_t *
     opal_free_list_item_t *item;
 
     do {
-        OPAL_FREE_LIST_GET(&port->data_qps, item, rc);
+        item = opal_free_list_get_st (&port->data_qps);
-        if (OPAL_SUCCESS != rc) {
+        if (NULL == item) {
             opal_output_verbose(5, orte_oob_base_framework.framework_output,
-                                 "%s oob:ud:qp_data_aquire error allocating new data qp. error = %d",
+                                 "%s oob:ud:qp_data_aquire error allocating new data qp",
-                                 ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), rc);
+                                 ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
             break;
         }
 
@@ -305,7 +305,7 @@ int mca_oob_ud_qp_data_release (mca_oob_ud_qp_t *qp) {
         return rc;
     }
 
-    OPAL_FREE_LIST_RETURN(&qp->port->data_qps, qp);
+    opal_free_list_return_st (&qp->port->data_qps, &qp->super);
 
     return ORTE_SUCCESS;
 }

orte/mca/oob/ud/oob_ud_req.c

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2011-2015 Los Alamos National Security, LLC. All rights
  *                         reserved.
  *               2014      Mellanox Technologies, Inc.
  *                         All rights reserved.
@@ -13,6 +13,7 @@
  */
 
 #include "oob_ud_component.h"
+#include "oob_ud_req.h"
 
 #include "orte/util/name_fns.h"
 #include "orte/runtime/orte_globals.h"
@@ -77,13 +78,12 @@ int mca_oob_ud_msg_get (struct mca_oob_ud_port_t *port, mca_oob_ud_req_t *req,
 {
     opal_free_list_item_t *item;
     opal_free_list_t *list = &port->free_msgs;
-    int rc;
 
-    OPAL_FREE_LIST_WAIT(list, item, rc);
+    item = opal_free_list_wait_st (list);
-    if (OPAL_SUCCESS != rc) {
+    if (NULL == item) {
         opal_output_verbose(5, orte_oob_base_framework.framework_output,
-                             "%s oob:ud:msg_get error getting message buffer. rc = %d",
+                            "%s oob:ud:msg_get error getting message buffer",
-                             ORTE_NAME_PRINT(ORTE_PROC_MY_NAME), rc);
+                            ORTE_NAME_PRINT(ORTE_PROC_MY_NAME));
         return ORTE_ERROR;
     }
 
@@ -111,12 +111,15 @@ int mca_oob_ud_msg_get (struct mca_oob_ud_port_t *port, mca_oob_ud_req_t *req,
     return ORTE_SUCCESS;
 }
 
-int mca_oob_ud_msg_init (mca_oob_ud_msg_t *msg, struct mca_oob_ud_port_t *port,
+int mca_oob_ud_msg_init (opal_free_list_item_t *item, void *context) {
-                         char *buf, struct ibv_mr *mr)
+    mca_oob_ud_port_t *port = (mca_oob_ud_port_t *) context;
-{
+    int buffer_id = port->send_buffer_index++ + mca_oob_ud_component.ud_recv_buffer_count;
+    char *buf = port->msg_buf.ptr + buffer_id * port->mtu;
+    mca_oob_ud_msg_t *msg = (mca_oob_ud_msg_t *) item;
+
     msg->port = port;
     msg->hdr  = (mca_oob_ud_msg_hdr_t *) buf;
-    msg->mr   = mr;
+    msg->mr   = port->msg_buf.mr;
 
     return ORTE_SUCCESS;
 }
@@ -134,7 +137,7 @@ void mca_oob_ud_msg_return (mca_oob_ud_msg_t *msg)
     msg->qp     = NULL;
     msg->req    = NULL;
 
-    OPAL_FREE_LIST_RETURN(list, msg);
+    opal_free_list_return_st (list, msg);
 }
 
 static void mca_oob_ud_msg_construct (mca_oob_ud_msg_t *msg)

orte/mca/oob/ud/oob_ud_req.h

@@ -1,6 +1,6 @@
 /* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */
 /*
- * Copyright (c) 2011-2012 Los Alamos National Security, LLC. All rights
+ * Copyright (c) 2011-2015 Los Alamos National Security, LLC. All rights
  *                         reserved.
  *               2014      Mellanox Technologies, Inc.
  *                         All rights reserved.
@@ -15,10 +15,13 @@
 #if !defined(MCA_OOB_UD_REQ_H)
 #define MCA_OOB_UD_REQ_H
 
+#include "oob_ud_peer.h"
+
 #include "orte_config.h"
 #include "orte/types.h"
 #include "opal/threads/condition.h"
 #include "opal/mca/event/event.h"
+#include "opal/class/opal_free_list.h"
 #include "orte/mca/rml/rml.h"
 
 #include <infiniband/verbs.h>
@@ -252,8 +255,7 @@ static inline int mca_oob_ud_recv_alloc (mca_oob_ud_req_t *recv_req)
 int mca_oob_ud_msg_get (struct mca_oob_ud_port_t *port, mca_oob_ud_req_t *req,
                         mca_oob_ud_qp_t *qp, mca_oob_ud_peer_t *peer, bool persist,
                         mca_oob_ud_msg_t **msgp);
-int mca_oob_ud_msg_init (mca_oob_ud_msg_t *msg, struct mca_oob_ud_port_t *port,
+int mca_oob_ud_msg_init (opal_free_list_item_t *item, void *context);
-                         char *buf, struct ibv_mr *mr);
 void mca_oob_ud_msg_return (mca_oob_ud_msg_t *msg);