usnic: unify teardown between trunk and v1.8 branches

Make the del_procs, module finalize, and endpoint destructors be the
same between trunk and v1.8, with one exception: the very beginning of
v1.8 module_finalize calls del_procs for each proc to simulate/pretend
the trunk/v1.9 PML behavior of calling del_procs before module_finalize.

This commit was SVN r32437.

opal/mca/btl/usnic/btl_usnic_endpoint.c

@@ -53,6 +53,7 @@ static void endpoint_construct(mca_btl_base_endpoint_t* endpoint)
     endpoint->endpoint_proc_index = -1;
     endpoint->endpoint_exiting = false;
     endpoint->endpoint_connectivity_checked = false;
+    endpoint->endpoint_on_all_endpoints = false;
 
     for (i=0; i<USNIC_NUM_CHANNELS; ++i) {
         endpoint->endpoint_remote_addr.qp_num[i] = 0;
@@ -116,6 +117,15 @@ static void endpoint_destruct(mca_btl_base_endpoint_t* endpoint)
        so it should be safe to unconditionally destruct the ack_li */
     OBJ_DESTRUCT(&(endpoint->endpoint_ack_li));
 
+    /* Remove the endpoint from the all_endpoints list */
+    opal_btl_usnic_module_t *module = endpoint->endpoint_module;
+    opal_mutex_lock(&module->all_endpoints_lock);
+    if (endpoint->endpoint_on_all_endpoints) {
+        opal_list_remove_item(&module->all_endpoints,
+                              &endpoint->endpoint_endpoint_li);
+        endpoint->endpoint_on_all_endpoints = false;
+    }
+    opal_mutex_unlock(&module->all_endpoints_lock);
     OBJ_DESTRUCT(&(endpoint->endpoint_endpoint_li));
 
     if (endpoint->endpoint_hotel.rooms != NULL) {
@@ -173,12 +183,3 @@ opal_btl_usnic_flush_endpoint(
     /* Now, ACK everything that is pending */
     opal_btl_usnic_handle_ack(endpoint, endpoint->endpoint_next_seq_to_send-1);
 }
-
-void opal_btl_usnic_release_endpoint(opal_btl_usnic_module_t *module,
-                                     opal_btl_usnic_endpoint_t *endpoint)
-{
-    opal_mutex_lock(&module->all_endpoints_lock);
-    opal_list_remove_item(&module->all_endpoints, &endpoint->endpoint_endpoint_li);
-    opal_mutex_unlock(&module->all_endpoints_lock);
-    OBJ_RELEASE(endpoint);
-}

opal/mca/btl/usnic/btl_usnic_endpoint.h

@@ -174,6 +174,7 @@ typedef struct mca_btl_base_endpoint_t {
     uint32_t                        endpoint_rfstart;
 
     bool                            endpoint_connectivity_checked;
+    bool                            endpoint_on_all_endpoints;
 } mca_btl_base_endpoint_t;
 
 typedef mca_btl_base_endpoint_t opal_btl_usnic_endpoint_t;
@@ -186,11 +187,5 @@ void
 opal_btl_usnic_flush_endpoint(
     opal_btl_usnic_endpoint_t *endpoint);
 
-/* Release the given endpoint and remove it from the all_endpoints list.  The
- * reference that is released was logically held by the all_endpoints list
- * (this is not a generic release function). */
-void opal_btl_usnic_release_endpoint(struct opal_btl_usnic_module_t *module,
-                                     opal_btl_usnic_endpoint_t *endpoint);
-
 END_C_DECLS
 #endif

opal/mca/btl/usnic/btl_usnic_module.c

@@ -263,7 +263,7 @@ static int add_procs_create_ahs(opal_btl_usnic_module_t *module,
                          EHOSTUNREACH == errno) {
                     add_procs_warn_ah_fail(module, endpoints[i]);
 
-                    opal_btl_usnic_release_endpoint(module, endpoints[i]);
+                    OBJ_RELEASE(endpoints[i]);
                     endpoints[i] = NULL;
                     --num_ah_left;
                 }
@@ -317,7 +317,7 @@ static int add_procs_create_ahs(opal_btl_usnic_module_t *module,
         if (NULL != endpoints[i]) {
             if (OPAL_SUCCESS != ret ||
                 NULL == endpoints[i]->endpoint_remote_ah) {
-                opal_btl_usnic_release_endpoint(module, endpoints[i]);
+                OBJ_RELEASE(endpoints[i]);
                 endpoints[i] = NULL;
             } else {
                 ++num_created;
@@ -402,7 +402,7 @@ static int usnic_add_procs(struct mca_btl_base_module_t* base_module,
        reachable. */
     for (size_t i = 0; i < nprocs; ++i) {
         if (NULL != endpoints[i]) {
-            opal_btl_usnic_release_endpoint(module, endpoints[i]);
+            OBJ_RELEASE(endpoints[i]);
             endpoints[i] = NULL;
         }
     }
@@ -451,7 +451,7 @@ static int usnic_del_procs(struct mca_btl_base_module_t *base_module,
                     }
 
                     /* We're all done with this endpoint */
-                    opal_btl_usnic_release_endpoint(module, endpoint);
+                    OBJ_RELEASE(endpoint);
 
                     break;  /* done once we found match */
                 }

opal/mca/btl/usnic/btl_usnic_proc.c

@@ -704,10 +704,13 @@ opal_btl_usnic_create_endpoint(opal_btl_usnic_module_t *module,
     ++proc->proc_endpoint_count;
     OBJ_RETAIN(proc);
 
-    /* also add endpoint to module's list of endpoints */
+    /* also add endpoint to module's list of endpoints (done here and
+       not in the endpoint constructor because we aren't able to pass
+       the module as a constructor argument -- doh!). */
     opal_mutex_lock(&module->all_endpoints_lock);
     opal_list_append(&(module->all_endpoints),
             &(endpoint->endpoint_endpoint_li));
+    endpoint->endpoint_on_all_endpoints = true;
     opal_mutex_unlock(&module->all_endpoints_lock);
 
     *endpoint_o = endpoint;

opal/mca/btl/usnic/btl_usnic_recv.c

@@ -362,7 +362,7 @@ void opal_btl_usnic_recv_call(opal_btl_usnic_module_t *module,
 
     /* if endpoint exiting, and all ACKs received, release the endpoint */
     if (endpoint->endpoint_exiting && ENDPOINT_DRAINED(endpoint)) {
-        opal_btl_usnic_release_endpoint(module, endpoint);
+        OBJ_RELEASE(endpoint);
     }
  repost_no_endpoint:
     ++module->stats.num_recv_reposts;

opal/mca/btl/usnic/btl_usnic_recv.h

@@ -334,7 +334,7 @@ opal_btl_usnic_recv_frag_bookkeeping(
 repost:
     /* if endpoint exiting, and all ACKs received, release the endpoint */
     if (endpoint->endpoint_exiting && ENDPOINT_DRAINED(endpoint)) {
-        opal_btl_usnic_release_endpoint(module, endpoint);
+        OBJ_RELEASE(endpoint);
     }
 
     ++module->stats.num_recv_reposts;