diff --git a/ompi/mca/mpool/base/Makefile.am b/ompi/mca/mpool/base/Makefile.am index 1ce2bd7f3f..a12b2d5f28 100644 --- a/ompi/mca/mpool/base/Makefile.am +++ b/ompi/mca/mpool/base/Makefile.am @@ -21,7 +21,8 @@ noinst_LTLIBRARIES = libmca_mpool_base.la # Source code files headers = \ - base.h + base.h \ + mpool_base_mem_cb.h libmca_mpool_base_la_SOURCES = \ $(headers) \ diff --git a/ompi/mca/mpool/base/base.h b/ompi/mca/mpool/base/base.h index 25e06c384f..75d782dcb1 100644 --- a/ompi/mca/mpool/base/base.h +++ b/ompi/mca/mpool/base/base.h @@ -124,3 +124,6 @@ OMPI_DECLSPEC extern opal_mutex_t mca_mpool_base_tree_lock; #endif #endif /* MCA_MEM_BASE_H */ + + + diff --git a/ompi/mca/mpool/base/mpool_base_close.c b/ompi/mca/mpool/base/mpool_base_close.c index ce84662c7a..44373fd573 100644 --- a/ompi/mca/mpool/base/mpool_base_close.c +++ b/ompi/mca/mpool/base/mpool_base_close.c @@ -23,7 +23,7 @@ #include "mca/base/base.h" #include "mca/mpool/mpool.h" #include "mca/mpool/base/base.h" - +#include "mpool_base_mem_cb.h" int mca_mpool_base_close(void) { @@ -54,6 +54,10 @@ int mca_mpool_base_close(void) mca_base_components_close(mca_mpool_base_output, &mca_mpool_base_components, NULL); + /* deregister memory free callback */ + /* if(opal_mem_free_is_supported()) { */ +/* opal_mem_free_unregister_handler(mca_mpool_base_mem_cb); */ +/* } */ /* All done */ return OMPI_SUCCESS; diff --git a/ompi/mca/mpool/base/mpool_base_lookup.c b/ompi/mca/mpool/base/mpool_base_lookup.c index 7c79174885..f9810dd615 100644 --- a/ompi/mca/mpool/base/mpool_base_lookup.c +++ b/ompi/mca/mpool/base/mpool_base_lookup.c @@ -23,6 +23,7 @@ #include "mca/base/base.h" #include "mca/mpool/mpool.h" #include "mca/mpool/base/base.h" +#include "mpool_base_mem_cb.h" mca_mpool_base_component_t* mca_mpool_base_component_lookup(const char* name) @@ -77,6 +78,11 @@ mca_mpool_base_module_t* mca_mpool_base_module_create( sm->user_data = user_data; sm->mpool_resources = resources; opal_list_append(&mca_mpool_base_modules, (opal_list_item_t*) sm); + /* on the very first creation of a module we init the memory callback*/ + /* if(opal_list_get_size(&mca_mpool_base_modules) == 1 && */ +/* opal_mem_free_is_supported()) { */ +/* opal_mem_free_register_handler(mca_mpool_base_mem_cb, NULL); */ +/* } */ return module; } diff --git a/ompi/mca/mpool/base/mpool_base_mem_cb.h b/ompi/mca/mpool/base/mpool_base_mem_cb.h new file mode 100644 index 0000000000..1718914b0a --- /dev/null +++ b/ompi/mca/mpool/base/mpool_base_mem_cb.h @@ -0,0 +1,87 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University. + * All rights reserved. + * Copyright (c) 2004-2005 The Trustees of the University of Tennessee. + * All rights reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +/** + * @file + */ +#ifndef MCA_MPOOL_BASE_MEM_CB_H +#define MCA_MPOOL_BASE_MEM_CB_H + +#include "ompi_config.h" + +#include "opal/class/opal_list.h" +#include "class/ompi_rb_tree.h" +#include "mca/mca.h" +#include "mca/mpool/mpool.h" +#include "opal/threads/mutex.h" +#include "opal/memory/memory.h" + +#if defined(c_plusplus) || defined(__cplusplus) +extern "C" { +#endif + +/* + * memory hook callback, called when memory is free'd out from under us + */ +static void mca_mpool_base_mem_cb(void* base, size_t size, void* cbdata){ + uint32_t i, cnt; + ompi_pointer_array_t regs; + mca_mpool_base_registration_t* reg; + mca_mpool_base_selected_module_t* current; + int rc; + opal_list_item_t* item; + + for(item = opal_list_get_first(&mca_mpool_base_modules); + item != opal_list_get_end(&mca_mpool_base_modules); + item = opal_list_get_next(item)) { + + current = (mca_mpool_base_selected_module_t*) item; + if(NULL != current->mpool_module->mpool_find) { + rc = current->mpool_module->mpool_find( + current->mpool_module, + base, + size, + ®s, + &cnt + ); + if(OMPI_SUCCESS != rc) { + return; + } + if(0 < cnt) { + for(i = 0; i < cnt; i++) { + reg = (mca_mpool_base_registration_t*) + ompi_pointer_array_get_item(®s, i); + + rc = current->mpool_module->mpool_deregister( + current->mpool_module, + reg + ); + if(OMPI_SUCCESS != rc) { + return; + } + } + } + } + } +} + +#if defined(c_plusplus) || defined(__cplusplus) +} +#endif + +#endif /* MCA_MPOOL_BASE_MEM_CB_H */ + + + diff --git a/ompi/mca/mpool/base/mpool_base_open.c b/ompi/mca/mpool/base/mpool_base_open.c index b760bcd870..e16cedfc12 100644 --- a/ompi/mca/mpool/base/mpool_base_open.c +++ b/ompi/mca/mpool/base/mpool_base_open.c @@ -70,3 +70,4 @@ int mca_mpool_base_open(void) return OMPI_SUCCESS; } + diff --git a/ompi/mca/mpool/mvapi/mpool_mvapi_module.c b/ompi/mca/mpool/mvapi/mpool_mvapi_module.c index 29b98919a7..abed581d64 100644 --- a/ompi/mca/mpool/mvapi/mpool_mvapi_module.c +++ b/ompi/mca/mpool/mvapi/mpool_mvapi_module.c @@ -17,6 +17,7 @@ #include "ompi_config.h" #include #include "opal/util/output.h" +#include "opal/memory/memory.h" #include "mca/mpool/mvapi/mpool_mvapi.h" #include #include @@ -24,6 +25,8 @@ #include "mca/rcache/rcache.h" #include "mca/rcache/base/base.h" + + /* * Initializes the mpool module. */ @@ -242,5 +245,6 @@ int mca_mpool_mvapi_retain(struct mca_mpool_base_module_t* mpool, + diff --git a/ompi/mca/pml/ob1/pml_ob1_sendreq.c b/ompi/mca/pml/ob1/pml_ob1_sendreq.c index f14b000551..1939cf8e11 100644 --- a/ompi/mca/pml/ob1/pml_ob1_sendreq.c +++ b/ompi/mca/pml/ob1/pml_ob1_sendreq.c @@ -348,13 +348,21 @@ static int mca_pml_ob1_send_request_start_rndv( int rc; /* prepare descriptor */ - mca_bml_base_prepare_src( - bml_btl, - NULL, - &sendreq->req_send.req_convertor, - sizeof(mca_pml_ob1_rendezvous_hdr_t), - &size, - &des); + if(size == 0) { + mca_bml_base_alloc( + bml_btl, + &des, + sizeof(mca_pml_ob1_rendezvous_hdr_t) + ); + } else { + mca_bml_base_prepare_src( + bml_btl, + NULL, + &sendreq->req_send.req_convertor, + sizeof(mca_pml_ob1_rendezvous_hdr_t), + &size, + &des); + } if(NULL == des) { return OMPI_ERR_OUT_OF_RESOURCE; @@ -477,7 +485,8 @@ int mca_pml_ob1_send_request_start_copy( /* if the data is contiguous and registered then we attempt to do * an rdma of the entire message. */ - if(ompi_convertor_need_buffers(&sendreq->req_send.req_convertor) == false && + if(size > 0 && + ompi_convertor_need_buffers(&sendreq->req_send.req_convertor) == false && 0 != (sendreq->req_rdma_cnt = mca_pml_ob1_rdma_btls( sendreq->req_endpoint, sendreq->req_send.req_addr,