usnic: trivial updates (no code/logic changes)

- Add more explanatory comments
- Trivial whitespace / style updates
- Rename opal_btl_usnic_force_retrans() -> opal_btl_usnic_fast_retrans()

Signed-off-by: Jeff Squyres <jsquyres@cisco.com>

opal/mca/btl/usnic/btl_usnic_ack.c

@@ -26,18 +26,24 @@
 #include "btl_usnic_connectivity.h"
 
 /*
- * Force a retrans of a segment
+ * Special case: we know exactly which segment is missing at the
+ * receive; explicitly force retrans of that segment.
  */
 static void
-opal_btl_usnic_force_retrans(
+opal_btl_usnic_fast_retrans(
     opal_btl_usnic_endpoint_t *endpoint,
     opal_btl_usnic_seq_t ack_seq)
 {
     opal_btl_usnic_send_segment_t *sseg;
     int is;
 
-    is = WINDOW_SIZE_MOD(ack_seq+1);
+    is = WINDOW_SIZE_MOD(ack_seq + 1);
     sseg = endpoint->endpoint_sent_segs[is];
+
+    // If the sseg is NULL, then there's nothing to retransmit.  If
+    // the hotel room is -1, the segment has already been queued up
+    // for retransmit and there's nothing additional we need to do
+    // here.
     if (sseg == NULL || sseg->ss_hotel_room == -1) {
         return;
     }
@@ -79,12 +85,14 @@ opal_btl_usnic_handle_ack(
 #endif
         ++module->stats.num_old_dup_acks;
         return;
+    }
 
-    /* A duplicate ACK means next seg was lost */
+    /* A duplicate ACK means the sender did not receive the next
-    } else if (ack_seq == endpoint->endpoint_ack_seq_rcvd) {
+       seg that we sent */
+    else if (ack_seq == endpoint->endpoint_ack_seq_rcvd) {
         ++module->stats.num_dup_acks;
 
-        opal_btl_usnic_force_retrans(endpoint, ack_seq);
+        opal_btl_usnic_fast_retrans(endpoint, ack_seq);
         return;
     }
 
@@ -114,12 +122,11 @@ opal_btl_usnic_handle_ack(
            already been evicted and queued for resend.
            If it's not in the hotel, don't check it out! */
         if (OPAL_LIKELY(sseg->ss_hotel_room != -1)) {
-
             opal_hotel_checkout(&endpoint->endpoint_hotel, sseg->ss_hotel_room);
             sseg->ss_hotel_room = -1;
-
+        }
         /* hotel_room == -1 means queued for resend, remove it */
-        } else {
+        else {
             opal_list_remove_item((&module->pending_resend_segs),
                     &sseg->ss_base.us_list.super);
         }
@@ -291,4 +298,3 @@ opal_btl_usnic_ack_timeout(
     /* Stats */
     ++module->stats.num_timeout_retrans;
 }
-

opal/mca/btl/usnic/btl_usnic_component.c

@@ -12,7 +12,7 @@
  *                         All rights reserved.
  * Copyright (c) 2006      Sandia National Laboratories. All rights
  *                         reserved.
- * Copyright (c) 2008-2016 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2008-2017 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2012-2014 Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * Copyright (c) 2014      Intel, Inc. All rights reserved.
@@ -1202,13 +1202,15 @@ static int usnic_handle_completion(
                 (opal_btl_usnic_ack_segment_t *)seg);
         break;
 
-    /**** Send of frag segment completion ****/
+    /**** Send of frag segment completion (i.e., the MPI message's
+          one-and-only segment has completed sending) ****/
     case OPAL_BTL_USNIC_SEG_FRAG:
         opal_btl_usnic_frag_send_complete(module,
                 (opal_btl_usnic_frag_segment_t*)seg);
         break;
 
-    /**** Send of chunk segment completion ****/
+    /**** Send of chunk segment completion (i.e., part of a large MPI
+          message is done sending) ****/
     case OPAL_BTL_USNIC_SEG_CHUNK:
         opal_btl_usnic_chunk_send_complete(module,
                 (opal_btl_usnic_chunk_segment_t*)seg);

opal/mca/btl/usnic/btl_usnic_endpoint.h

@@ -11,7 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2006      Sandia National Laboratories. All rights
  *                         reserved.
- * Copyright (c) 2013-2016 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013-2017 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -160,6 +160,8 @@ typedef struct mca_btl_base_endpoint_t {
     opal_btl_usnic_seq_t endpoint_next_seq_to_send; /* n_t */
     opal_btl_usnic_seq_t endpoint_ack_seq_rcvd; /* n_a */
 
+    /* Table where sent segments sit while waiting for their ACKs.
+       When a segment is ACKed, it is removed from this table. */
     struct opal_btl_usnic_send_segment_t *endpoint_sent_segs[WINDOW_SIZE];
 
     /* Values for the current proc to receive from this endpoint on

opal/mca/btl/usnic/btl_usnic_module.c

@@ -987,7 +987,7 @@ usnic_do_resends(
             /* resends are always standard segments */
             sseg->ss_channel = USNIC_DATA_CHANNEL;
 
-            /* re-send the segment */
+            /* re-send the segment (we have a send credit available) */
             opal_btl_usnic_post_segment(module, endpoint, sseg);
 
             /* consume a send credit for this endpoint.  May send us
@@ -1017,6 +1017,9 @@ usnic_do_resends(
  * endpoint_send_segment() it.  Takes care of subsequent frag
  * cleanup/bookkeeping (dequeue, descriptor callback, etc.) if this frag was
  * completed by this segment.
+ *
+ * ASSUMES THAT THE CALLER HAS ALREADY CHECKED TO SEE IF WE HAVE
+ * A SEND CREDIT!
  */
 static void
 usnic_handle_large_send(
@@ -1070,7 +1073,8 @@ usnic_handle_large_send(
     /* payload length into the header*/
     sseg->ss_base.us_btl_header->payload_len = payload_len;
 
-    /* do the send */
+    // We assume that the caller has checked to see that we have a
+    // send credit, so do the send.
     opal_btl_usnic_endpoint_send_segment(module, sseg);
 
     /* do fragment bookkeeping */
@@ -1182,7 +1186,7 @@ opal_btl_usnic_module_progress_sends(
                     sseg->ss_base.us_btl_header->tag);
 #endif
 
-            /* post the send */
+            /* post the send (we have a send credit available) */
             opal_btl_usnic_endpoint_send_segment(module, sseg);
 
             /* don't do callback yet if this is a put */
@@ -1342,7 +1346,7 @@ usnic_send(
         opal_output(0, "INLINE send, sseg=%p", (void *)sseg);
 #endif
 
-        /* post the segment now */
+        /* post the segment now (we have a send credit available) */
         opal_btl_usnic_endpoint_send_segment(module, sseg);
 
         /* If we own the frag and callback was requested, callback now,

opal/mca/btl/usnic/btl_usnic_module.h

@@ -71,7 +71,7 @@ typedef struct opal_btl_usnic_channel_t {
     int chan_rd_num;
     int chan_sd_num;
 
-    int credits;  /* RFXXX until libfab credits fixed */
+    int credits;
     uint32_t rx_post_cnt;
 
     /* fastsend enabled if num_credits_available >= fastsend_wqe_thresh */

opal/mca/btl/usnic/btl_usnic_send.c

@@ -11,7 +11,7 @@
  *                         All rights reserved.
  * Copyright (c) 2006      Sandia National Laboratories. All rights
  *                         reserved.
- * Copyright (c) 2008-2015 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2008-2017 Cisco Systems, Inc.  All rights reserved.
  * Copyright (c) 2012      Los Alamos National Security, LLC.  All rights
  *                         reserved.
  * $COPYRIGHT$
@@ -43,8 +43,9 @@
 
 
 /*
- * This function is called when a send of a full-fragment segment completes
+ * This function is called when a send of a segment completes that is
- * Return the WQE and also return the segment if no ACK pending
+ * the one-and-only segment of an MPI message.  Return the WQE and
+ * also return the segment if no ACK pending.
  */
 void
 opal_btl_usnic_frag_send_complete(opal_btl_usnic_module_t *module,
@@ -71,8 +72,10 @@ opal_btl_usnic_frag_send_complete(opal_btl_usnic_module_t *module,
 }
 
 /*
- * This function is called when a send segment completes
+ * This function is called when a send segment completes that is part
- * Return the WQE and also return the segment if no ACK pending
+ * of a larger MPI message (ie., there may still be other chunk
+ * segments that have not yet completed sending).  Return the WQE and
+ * also return the segment if no ACK pending.
  */
 void
 opal_btl_usnic_chunk_send_complete(opal_btl_usnic_module_t *module,

opal/mca/btl/usnic/btl_usnic_send.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013-2016 Cisco Systems, Inc.  All rights reserved.
+ * Copyright (c) 2013-2017 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
  *
  * Additional copyrights may follow
@@ -52,7 +52,10 @@ opal_btl_usnic_check_rts(
 }
 
 /*
- * Common point for posting a segment
+ * Common point for posting a segment.
+ *
+ * ASSUMES THAT THE CALLER HAS ALREADY CHECKED TO SEE IF WE HAVE
+ * A SEND CREDIT!
  */
 static inline void
 opal_btl_usnic_post_segment(
@@ -105,6 +108,9 @@ opal_btl_usnic_post_segment(
 
 /*
  * Common point for posting an ACK
+ *
+ * ASSUMES THAT THE CALLER HAS ALREADY CHECKED TO SEE IF WE HAVE
+ * A SEND CREDIT!
  */
 static inline void
 opal_btl_usnic_post_ack(
@@ -230,10 +236,10 @@ opal_btl_usnic_endpoint_send_segment(
     /* do the actual send */
     opal_btl_usnic_post_segment(module, endpoint, sseg);
 
-    /* Track this header by stashing in an array on the endpoint that
+    /* Stash this segment in an array on the endpoint that is the same
-       is the same length as the sender's window (i.e., WINDOW_SIZE).
+       length as the sender's window (i.e., WINDOW_SIZE) until it
-       To find a unique slot in this array, use (seq % WINDOW_SIZE).
+       receives its ACK.  To find a unique slot in this array, use
-     */
+       (seq % WINDOW_SIZE). */
     sfi = WINDOW_SIZE_MOD(sseg->ss_base.us_btl_header->pkt_seq);
     endpoint->endpoint_sent_segs[sfi] = sseg;
     sseg->ss_ack_pending = true;