Prevent iWARP qp flush errors.

For iWARP, the TCP connection is tied to the QP once the QP is in RTS.  
And destroying the QP is thus tied to connection teardown for iWARP.  
This is a key distinction from IB, I think.   Anyway, to destroy the 
connection in iWARP you must move the QP out of RTS, either into CLOSING 
for a nice graceful close, or to ERROR if you want to be rude.  In both 
cases, all pending non-completed SQ and RQ WRs must be flushed.

This patch ignores all flush errors reaped by the cq and removes an
earlier attempt to work around this in the rdmacm cpc.

This commit was SVN r18388.

ompi/mca/btl/openib/btl_openib_component.c

@@ -2062,6 +2062,16 @@ error:
     if(endpoint && endpoint->endpoint_proc && endpoint->endpoint_proc->proc_ompi)
         remote_proc = endpoint->endpoint_proc->proc_ompi;
 
+    /* For iWARP, the TCP connection is tied to the QP once the QP is in RTS.  
+     * And destroying the QP is thus tied to connection teardown for iWARP.  
+     * To destroy the connection in iWARP you must move the QP out of RTS, either
+     * into CLOSING for a nice graceful close, or to ERROR if you want to be rude.
+     * In both cases, all pending non-completed SQ and RQ WRs must be flushed. 
+     */
+    if (wc->status == IBV_WC_WR_FLUSH_ERR && IBV_TRANSPORT_IWARP == hca->ib_dev->transport_type) {
+        return;
+    }
+
     if(wc->status != IBV_WC_WR_FLUSH_ERR || !flush_err_printed[cq]++) {
         BTL_PEER_ERROR(remote_proc, ("error polling %s with status %s "
                     "status number %d for wr_id %llu opcode %d qp_idx %d",

ompi/mca/btl/openib/connect/btl_openib_connect_rdmacm.c

@@ -574,12 +574,6 @@ static int rdmacm_connection_shutdown(struct mca_btl_base_endpoint_t *endpoint)
         return 0;
     }
 
-    if (IBV_TRANSPORT_IWARP == endpoint->endpoint_btl->hca->ib_dev->transport_type) {
-        int i;
-        for (i = 0; i < mca_btl_openib_component.num_qps; i++)
-            endpoint->qps[i].qp->lcl_qp = NULL;
-    }
-
     for (temp = client_list_head; NULL != temp; temp = temp->next) {
         if (endpoint == temp->item->endpoint) {
             int i;