Correct faulty max payload and MTU computations (and update some

debugging that helped us find those).

This commit was SVN r28942.

ompi/mca/btl/usnic/btl_usnic_ack.c

@@ -249,7 +249,7 @@ ompi_btl_usnic_ack_timeout(
     endpoint = seg->ss_parent_frag->sf_endpoint;
     module = endpoint->endpoint_module;
 
-#if MSGDEBUG1
+#if MSGDEBUG2
     {
         static int num_timeouts = 0;
         opal_output(0, "Send timeout!  seg %p, room %d, seq %" UDSEQ "\n",

ompi/mca/btl/usnic/btl_usnic_component.c

@@ -616,6 +616,7 @@ static mca_btl_base_module_t** usnic_component_init(int* num_btl_modules,
         /* Find the max payload this port can handle */
         module->max_frag_payload =
             module->if_mtu - /* start with the MTU */
+            sizeof(ompi_btl_usnic_protocol_header_t) -
             sizeof(ompi_btl_usnic_btl_header_t); /* subtract size of
                                                     the BTL header */
         /* same, but use chunk header */
@@ -873,10 +874,11 @@ static int usnic_component_progress(void)
                         if (cwc->byte_len <
                              (sizeof(ompi_btl_usnic_protocol_header_t)+
                               sizeof(ompi_btl_usnic_btl_header_t))) {
-                        BTL_ERROR(("RX error polling CQ with status %d for wr_id %" PRIx64 " vend_err %d, byte_len %d (%d of %d)",
+                        BTL_ERROR(("RX error polling CQ[%d] with status %d for wr_id %" PRIx64 " vend_err %d, byte_len %d (%d of %d)",
-                               cwc->status, cwc->wr_id,
+                               c, cwc->status, cwc->wr_id,
                                cwc->vendor_err,
                                j, num_events, cwc->byte_len));
+abort();
                         } else {
                             /* silently count CRC errors */
                             ++module->num_crc_errors;

ompi/mca/btl/usnic/btl_usnic_send.c

@@ -179,7 +179,7 @@ ompi_btl_usnic_send_slower(
             (int)frag->sf_base.uf_src_seg[0].seg_len,
             frag->sf_base.uf_src_seg[1].seg_addr.pval,
             (int)frag->sf_base.uf_src_seg[1].seg_len);
-    opal_output(0, "  inline seg  %d segs %p(%d) + %p(%d)\n",
+    opal_output(0, "  small seg  %d segs %p(%d) + %p(%d)\n",
             sseg->ss_send_desc.num_sge,
             (void *)sseg->ss_send_desc.sg_list[0].addr,
             sseg->ss_send_desc.sg_list[0].length,

ompi/mca/btl/usnic/btl_usnic_util.c

@@ -127,10 +127,7 @@ int ompi_btl_usnic_find_ip(ompi_btl_usnic_module_t *module, uint8_t mac[6])
             /* Since verbs doesn't offer a way to get standard
                Ethernet MTUs (as of libibverbs 1.1.5, the MTUs are
                enums, and don't inlcude values for 1500 or 9000), look
-               up the MTU in the corresponding enic interface.
+               up the MTU in the corresponding enic interface. */
-               Subtract 40 off the MTU value so that we provide enough
-               space for the GRH on the remote side. */
-            module->if_mtu -= 40;
             module->local_addr.mtu = module->if_mtu;
 
             inet_ntop(AF_INET, &(module->if_ipv4_addr),