usnic: ensure to check send credits for ACKs

Don't just blindly send ACKs; ensure that we have send credits before
doing so.  If we don't have any send credits, just don't send the ACK
(it'll come again soon enough; it's not a tragedy if we don't send it
now).

Signed-off-by: Jeff Squyres <jsquyres@cisco.com>

opal/mca/btl/usnic/btl_usnic_ack.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013-2016 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013-2017 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -198,19 +198,27 @@ opal_btl_usnic_handle_ack(
 /*
  * Send an ACK
  */
-void
+int
 opal_btl_usnic_ack_send(
     opal_btl_usnic_module_t *module,
     opal_btl_usnic_endpoint_t *endpoint)
 {
     opal_btl_usnic_ack_segment_t *ack;
 
+    /* If we don't have any send credits in the priority channel,
+       don't send it */
+    if (module->mod_channels[USNIC_PRIORITY_CHANNEL].credits < 1) {
+        return OPAL_ERR_OUT_OF_RESOURCE;
+    }
+
     /* Get an ACK frag.  If we don't get one, just discard this ACK. */
     ack = opal_btl_usnic_ack_segment_alloc(module);
     if (OPAL_UNLIKELY(NULL == ack)) {
-        return;
+        return OPAL_ERR_OUT_OF_RESOURCE;
     }
 
+    --module->mod_channels[USNIC_PRIORITY_CHANNEL].credits;
+
     /* send the seq of the lowest item in the window that
        we've received */
     ack->ss_base.us_btl_header->ack_seq =
@@ -246,7 +254,7 @@ opal_btl_usnic_ack_send(
     /* Stats */
     ++module->stats.num_ack_sends;
 
-    return;
+    return OPAL_SUCCESS;
 }
 
 /*
@@ -256,6 +264,7 @@ void
 opal_btl_usnic_ack_complete(opal_btl_usnic_module_t *module,
                                    opal_btl_usnic_ack_segment_t *ack)
 {
+    ++module->mod_channels[USNIC_PRIORITY_CHANNEL].credits;
     opal_btl_usnic_ack_segment_return(module, ack);
     ++module->mod_channels[ack->ss_channel].credits;
 }

opal/mca/btl/usnic/btl_usnic_ack.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013-2016 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013-2017 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -67,7 +67,7 @@ void opal_btl_usnic_ack_complete(opal_btl_usnic_module_t *module,
 /*
  * Send an ACK
  */
-void opal_btl_usnic_ack_send(opal_btl_usnic_module_t *module,
+int opal_btl_usnic_ack_send(opal_btl_usnic_module_t *module,
                                opal_btl_usnic_endpoint_t *endpoint);
 
 /*

opal/mca/btl/usnic/btl_usnic_module.c

@@ -1237,8 +1237,13 @@ opal_btl_usnic_module_progress_sends(
         /* Is it time to send ACK? */
         if (endpoint->endpoint_acktime == 0 ||
             endpoint->endpoint_acktime <= get_nsec()) {
-            opal_btl_usnic_ack_send(module, endpoint);
-            opal_btl_usnic_remove_from_endpoints_needing_ack(endpoint);
+            if (OPAL_LIKELY(opal_btl_usnic_ack_send(module, endpoint) == OPAL_SUCCESS)) {
+                opal_btl_usnic_remove_from_endpoints_needing_ack(endpoint);
+            } else {
+                // If we fail, it means we're out of send credits on
+                // the ACK channel
+                break;
+            }
         }
 
         endpoint = next_endpoint;