From 54998e5745424a683fec8653831c85a5dc8ac9b5 Mon Sep 17 00:00:00 2001 From: Nathan Hjelm Date: Mon, 24 Aug 2015 13:26:47 -0600 Subject: [PATCH 1/2] opal: add recursive mutex This new class is the same as the opal_mutex_t class but has a different constructor. This constructor adds the recursive flag to the mutex attributes for the lock. This class can be used where there may be re-enty into the lock from within the same thread. Signed-off-by: Nathan Hjelm --- opal/threads/mutex.c | 26 ++++++++++++++++++++++++++ opal/threads/mutex.h | 2 +- opal/threads/mutex_unix.h | 1 + 3 files changed, 28 insertions(+), 1 deletion(-) diff --git a/opal/threads/mutex.c b/opal/threads/mutex.c index d7b7805308..23153ea843 100644 --- a/opal/threads/mutex.c +++ b/opal/threads/mutex.c @@ -72,3 +72,29 @@ OBJ_CLASS_INSTANCE(opal_mutex_t, opal_object_t, opal_mutex_construct, opal_mutex_destruct); + +static void opal_recursive_mutex_construct(opal_recursive_mutex_t *m) +{ + pthread_mutexattr_t attr; + pthread_mutexattr_init(&attr); + +#if OPAL_ENABLE_DEBUG + m->m_lock_debug = 0; + m->m_lock_file = NULL; + m->m_lock_line = 0; +#endif + + pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE); + + pthread_mutex_init(&m->m_lock_pthread, &attr); + pthread_mutexattr_destroy(&attr); + +#if OPAL_HAVE_ATOMIC_SPINLOCKS + opal_atomic_init( &m->m_lock_atomic, OPAL_ATOMIC_UNLOCKED ); +#endif +} + +OBJ_CLASS_INSTANCE(opal_recursive_mutex_t, + opal_object_t, + opal_recursive_mutex_construct, + opal_mutex_destruct); diff --git a/opal/threads/mutex.h b/opal/threads/mutex.h index 542cd2a952..1e8682ec2b 100644 --- a/opal/threads/mutex.h +++ b/opal/threads/mutex.h @@ -50,7 +50,7 @@ OPAL_DECLSPEC extern bool opal_uses_threads; * Opaque mutex object */ typedef struct opal_mutex_t opal_mutex_t; - +typedef struct opal_mutex_t opal_recursive_mutex_t; /** * Try to acquire a mutex. diff --git a/opal/threads/mutex_unix.h b/opal/threads/mutex_unix.h index c8cad4975f..d159285d8e 100644 --- a/opal/threads/mutex_unix.h +++ b/opal/threads/mutex_unix.h @@ -61,6 +61,7 @@ struct opal_mutex_t { opal_atomic_lock_t m_lock_atomic; }; OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_mutex_t); +OPAL_DECLSPEC OBJ_CLASS_DECLARATION(opal_recursive_mutex_t); /************************************************************************ * From 1d56007ab107d5dfe646a08d1cd18bceef2f6275 Mon Sep 17 00:00:00 2001 From: Nathan Hjelm Date: Mon, 24 Aug 2015 13:29:35 -0600 Subject: [PATCH 2/2] rcache/vma: make rcache lock recursive There is currently a path through the grdma mpool and vma rcache that leads to deadlock. It happens during the rcache insert. Before the insert the rcache mutex is locked. During the call a new vma item is allocated and then inserted into the rcache tree. The allocation currently goes through the malloc hooks which may (and does) call back into the mpool if the ptmalloc heap needs to be reallocated. This callback tries to lock the rcache mutex which leads to the deadlock. This has been observed with multi-threaded tests and the openib btl. This change may lead to some minor slowdown in the rcache vma when threading is enabled. This will only affect larger message paths in some of the btls. Signed-off-by: Nathan Hjelm --- opal/mca/rcache/vma/rcache_vma.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/opal/mca/rcache/vma/rcache_vma.c b/opal/mca/rcache/vma/rcache_vma.c index 7910e671f0..8c9bd5e6f0 100644 --- a/opal/mca/rcache/vma/rcache_vma.c +++ b/opal/mca/rcache/vma/rcache_vma.c @@ -42,7 +42,7 @@ void mca_rcache_vma_module_init( mca_rcache_vma_module_t* rcache ) { rcache->base.rcache_clean = mca_rcache_vma_clean; rcache->base.rcache_finalize = mca_rcache_vma_finalize; rcache->base.rcache_dump_range = mca_rcache_vma_dump_range; - OBJ_CONSTRUCT(&rcache->base.lock, opal_mutex_t); + OBJ_CONSTRUCT(&rcache->base.lock, opal_recursive_mutex_t); mca_rcache_vma_tree_init(rcache); }