usnic: properly differentiate data pool vs. malloc

usnic_fls() can actually return 0, leading us to incorrectly free() a
buffer instead of OMPI_FREE_LIST_RETURN_MT'ing it.

So add an explicit bool in the struct that tracks whether the buffer
came from malloc or a freelist.

This was CID 1269660.

opal/mca/btl/usnic/btl_usnic_endpoint.h

@@ -11,7 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2006      Sandia National Laboratories. All rights
  *                         reserved.
- * Copyright (c) 2013-2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013-2015 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -101,8 +101,9 @@ typedef struct opal_btl_usnic_rx_frag_info_t {
     uint32_t    rfi_frag_id;    /* ID for this fragment */
     uint32_t    rfi_frag_size;  /* bytes in this fragment */
     uint32_t    rfi_bytes_left; /* bytes remaining to RX in fragment */
+    bool        rfi_data_in_pool; /* data in data_pool if true, else malloced */
+    int         rfi_data_pool;  /* if <0, data malloced, else rx buf pool */
     char       *rfi_data;       /* pointer to assembly area */
-    int         rfi_data_pool;  /* if 0, data malloced, else rx buf pool */
     ompi_free_list_item_t *rfi_fl_elt; /* free list elemement from buf pool
                                           when rfi_data_pool is nonzero */
 } opal_btl_usnic_rx_frag_info_t;

opal/mca/btl/usnic/btl_usnic_recv.c

@@ -11,7 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2006      Sandia National Laboratories. All rights
  *                         reserved.
- * Copyright (c) 2008-2014 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2008-2015 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2012      Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * $COPYRIGHT$
@@ -224,11 +224,12 @@ void opal_btl_usnic_recv_call(opal_btl_usnic_module_t *module,
                         fip->rfi_fl_elt = item;
                         fip->rfi_data = rx_buf->buf;
                         fip->rfi_data_pool = pool;
+                        fip->rfi_data_in_pool = true;
                     }
                 }
                 if (fip->rfi_data == NULL) {
                     fip->rfi_data = malloc(chunk_hdr->ch_frag_size);
-                    fip->rfi_data_pool = 0;
+                    fip->rfi_data_in_pool = false;
                 }
                 if (fip->rfi_data == NULL) {
                     abort();
@@ -302,12 +303,12 @@ void opal_btl_usnic_recv_call(opal_btl_usnic_module_t *module,
                         &desc, reg->cbdata);
 
                 /* free temp buffer for non-put */
-                if (0 == fip->rfi_data_pool) {
+                if (fip->rfi_data_in_pool) {
-                    free(fip->rfi_data);
-                } else {
                     OMPI_FREE_LIST_RETURN_MT(
                             &module->module_recv_buffers[fip->rfi_data_pool],
                             fip->rfi_fl_elt);
+                } else {
+                    free(fip->rfi_data);
                 }
 
 #if MSGDEBUG1