Add code for mem_free hooks, commented out right now as there are performance

issues. 
In PML-OB1 - fix for sync-sends of zero length. 

This commit was SVN r7362.

ompi/mca/mpool/base/Makefile.am

@@ -21,7 +21,8 @@ noinst_LTLIBRARIES = libmca_mpool_base.la
 # Source code files
 
 headers = \
-        base.h 
+        base.h \
+	mpool_base_mem_cb.h
 
 libmca_mpool_base_la_SOURCES = \
         $(headers) \

ompi/mca/mpool/base/base.h

@@ -124,3 +124,6 @@ OMPI_DECLSPEC extern opal_mutex_t mca_mpool_base_tree_lock;
 #endif
 
 #endif /* MCA_MEM_BASE_H */
+
+
+

ompi/mca/mpool/base/mpool_base_close.c

@@ -23,7 +23,7 @@
 #include "mca/base/base.h"
 #include "mca/mpool/mpool.h"
 #include "mca/mpool/base/base.h"
-
+#include "mpool_base_mem_cb.h"
 
 int mca_mpool_base_close(void)
 {
@@ -54,6 +54,10 @@ int mca_mpool_base_close(void)
   mca_base_components_close(mca_mpool_base_output, 
                             &mca_mpool_base_components, NULL);
 
+  /* deregister memory free callback */
+  /* if(opal_mem_free_is_supported()) {  */
+/*       opal_mem_free_unregister_handler(mca_mpool_base_mem_cb);  */
+/*   } */
   /* All done */
 
   return OMPI_SUCCESS;

ompi/mca/mpool/base/mpool_base_lookup.c

@@ -23,6 +23,7 @@
 #include "mca/base/base.h"
 #include "mca/mpool/mpool.h"
 #include "mca/mpool/base/base.h"
+#include "mpool_base_mem_cb.h"
 
 
 mca_mpool_base_component_t* mca_mpool_base_component_lookup(const char* name)
@@ -77,6 +78,11 @@ mca_mpool_base_module_t* mca_mpool_base_module_create(
     sm->user_data = user_data;
     sm->mpool_resources = resources;
     opal_list_append(&mca_mpool_base_modules, (opal_list_item_t*) sm); 
+    /* on the very first creation of a module we init the memory callback*/ 
+    /* if(opal_list_get_size(&mca_mpool_base_modules) == 1 && */
+/*        opal_mem_free_is_supported()) {  */
+/*         opal_mem_free_register_handler(mca_mpool_base_mem_cb, NULL);  */
+/*     } */
     return module; 
 }
 

ompi/mca/mpool/base/mpool_base_mem_cb.h

@@ -0,0 +1,87 @@
+/*
+ * Copyright (c) 2004-2005 The Trustees of Indiana University.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 The Trustees of the University of Tennessee.
+ *                         All rights reserved.
+ * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, 
+ *                         University of Stuttgart.  All rights reserved.
+ * Copyright (c) 2004-2005 The Regents of the University of California.
+ *                         All rights reserved.
+ * $COPYRIGHT$
+ * 
+ * Additional copyrights may follow
+ * 
+ * $HEADER$
+ */
+/**
+ * @file
+ */
+#ifndef MCA_MPOOL_BASE_MEM_CB_H
+#define MCA_MPOOL_BASE_MEM_CB_H
+
+#include "ompi_config.h"
+
+#include "opal/class/opal_list.h"
+#include "class/ompi_rb_tree.h"
+#include "mca/mca.h"
+#include "mca/mpool/mpool.h"
+#include "opal/threads/mutex.h" 
+#include "opal/memory/memory.h"
+
+#if defined(c_plusplus) || defined(__cplusplus)
+extern "C" {
+#endif
+
+/*
+ *  memory hook callback, called when memory is free'd out from under us
+ */
+static void mca_mpool_base_mem_cb(void* base, size_t size, void* cbdata){ 
+    uint32_t i, cnt;
+    ompi_pointer_array_t regs;
+    mca_mpool_base_registration_t* reg;
+    mca_mpool_base_selected_module_t* current;
+    int rc;
+    opal_list_item_t* item;
+
+    for(item = opal_list_get_first(&mca_mpool_base_modules);
+        item != opal_list_get_end(&mca_mpool_base_modules);
+        item = opal_list_get_next(item)) {
+        
+        current = (mca_mpool_base_selected_module_t*) item; 
+        if(NULL != current->mpool_module->mpool_find) { 
+            rc = current->mpool_module->mpool_find(
+                                                   current->mpool_module, 
+                                                   base,
+                                                   size,
+                                                   &regs, 
+                                                   &cnt 
+                                                   ); 
+            if(OMPI_SUCCESS != rc) { 
+                return;
+            }
+            if(0 < cnt) { 
+                for(i = 0; i < cnt; i++) { 
+                    reg = (mca_mpool_base_registration_t*) 
+                        ompi_pointer_array_get_item(&regs, i);
+                    
+                    rc = current->mpool_module->mpool_deregister(
+                                                                 current->mpool_module, 
+                                                                 reg
+                                                                 ); 
+                    if(OMPI_SUCCESS != rc) { 
+                        return; 
+                    }
+                }
+            }
+        }    
+    }    
+}
+
+#if defined(c_plusplus) || defined(__cplusplus)
+}
+#endif
+
+#endif /* MCA_MPOOL_BASE_MEM_CB_H */
+
+
+

ompi/mca/mpool/base/mpool_base_open.c

@@ -70,3 +70,4 @@ int mca_mpool_base_open(void)
 
   return OMPI_SUCCESS;
 }
+

ompi/mca/mpool/mvapi/mpool_mvapi_module.c

@@ -17,6 +17,7 @@
 #include "ompi_config.h"
 #include <string.h>
 #include "opal/util/output.h"
+#include "opal/memory/memory.h"
 #include "mca/mpool/mvapi/mpool_mvapi.h"
 #include <vapi.h>
 #include <vapi_types.h> 
@@ -24,6 +25,8 @@
 #include "mca/rcache/rcache.h" 
 #include "mca/rcache/base/base.h"
 
+
+
 /*
  *  Initializes the mpool module.
  */ 
@@ -242,5 +245,6 @@ int mca_mpool_mvapi_retain(struct mca_mpool_base_module_t* mpool,
 
 
 
+
 
 

ompi/mca/pml/ob1/pml_ob1_sendreq.c

@@ -348,13 +348,21 @@ static int mca_pml_ob1_send_request_start_rndv(
     int rc;
 
     /* prepare descriptor */
-    mca_bml_base_prepare_src(
+    if(size == 0) {
-        bml_btl, 
+        mca_bml_base_alloc(
-        NULL,
+                           bml_btl, 
-        &sendreq->req_send.req_convertor,
+                           &des, 
-        sizeof(mca_pml_ob1_rendezvous_hdr_t),
+                           sizeof(mca_pml_ob1_rendezvous_hdr_t)
-        &size,
+                           ); 
-        &des);
+    } else {
+        mca_bml_base_prepare_src(
+                                 bml_btl, 
+                                 NULL,
+                                 &sendreq->req_send.req_convertor,
+                                 sizeof(mca_pml_ob1_rendezvous_hdr_t),
+                                 &size,
+                                 &des);
+    }
 
     if(NULL == des) {
         return OMPI_ERR_OUT_OF_RESOURCE;
@@ -477,7 +485,8 @@ int mca_pml_ob1_send_request_start_copy(
         /* if the data is contiguous and registered then we attempt to do 
          * an rdma of the entire message.
          */
-        if(ompi_convertor_need_buffers(&sendreq->req_send.req_convertor) == false &&
+        if(size > 0 &&
+           ompi_convertor_need_buffers(&sendreq->req_send.req_convertor) == false &&
            0 != (sendreq->req_rdma_cnt = mca_pml_ob1_rdma_btls(
                sendreq->req_endpoint,
                sendreq->req_send.req_addr,