From d7fba60de8e7ce7d677f46b7ab0be635bdf1250b Mon Sep 17 00:00:00 2001 From: Shintaro Iwasaki Date: Thu, 26 Mar 2020 11:30:31 -0500 Subject: [PATCH] mca/threads: remove libevent hack Argobots/Qthreads-aware libevent should be used instead. Signed-off-by: Shintaro Iwasaki --- opal/mca/event/event.h | 2 - opal/mca/event/external/external.h | 3 + opal/mca/event/libevent2022/libevent2022.h | 3 + opal/mca/threads/README.md | 4 +- opal/mca/threads/argobots/Makefile.am | 1 - .../threads/argobots/threads_argobots_event.c | 143 ------------------ .../pthreads/threads_pthreads_module.c | 5 - opal/mca/threads/threads.h | 1 - 8 files changed, 7 insertions(+), 155 deletions(-) delete mode 100644 opal/mca/threads/argobots/threads_argobots_event.c diff --git a/opal/mca/event/event.h b/opal/mca/event/event.h index 65f8f69e05..b0958b64e0 100644 --- a/opal/mca/event/event.h +++ b/opal/mca/event/event.h @@ -48,8 +48,6 @@ BEGIN_C_DECLS #define OPAL_TIMEOUT_DEFAULT {1, 0} -OPAL_DECLSPEC void opal_event_use_threads(void); - /** * Structure for event components. */ diff --git a/opal/mca/event/external/external.h b/opal/mca/event/external/external.h index 6fe67feeea..00378f684f 100644 --- a/opal/mca/event/external/external.h +++ b/opal/mca/event/external/external.h @@ -71,6 +71,9 @@ OPAL_DECLSPEC int opal_event_finalize(void); #define opal_event_set_priority(x, n) event_priority_set((x), (n)) +/* thread support APIs */ +#define opal_event_use_threads() evthread_use_pthreads() + /* Basic event APIs */ #define opal_event_enable_debug_mode() event_enable_debug_mode() diff --git a/opal/mca/event/libevent2022/libevent2022.h b/opal/mca/event/libevent2022/libevent2022.h index 3b8834d0fb..5989a48610 100644 --- a/opal/mca/event/libevent2022/libevent2022.h +++ b/opal/mca/event/libevent2022/libevent2022.h @@ -102,6 +102,9 @@ OPAL_DECLSPEC int opal_event_finalize(void); #define opal_event_set_priority(x, n) event_priority_set((x), (n)) +/* thread support APIs */ +#define opal_event_use_threads() evthread_use_pthreads() + /* Basic event APIs */ #define opal_event_enable_debug_mode() event_enable_debug_mode() diff --git a/opal/mca/threads/README.md b/opal/mca/threads/README.md index aaf7911d4a..8c600ccb01 100644 --- a/opal/mca/threads/README.md +++ b/opal/mca/threads/README.md @@ -26,8 +26,6 @@ The rest of the threading implementation follows the normal MCA model: * `threads__condition.c` defines an instance of `opal_condition_t` which is used by `condition.h` to define Open MPI specific condition variables. -* `threads__event.c` defines an interface to Open MPI's libevent hooks. It allows the threading module to use threading model specific memory allocation and synchronization structures with Open MPI's libevent integration. - * `threads__module.c` defines the interface to opal's thread handle. It provides ways of comparing threads, getting the value of a thread via its handle and the implementation of thread local storage. * `threads__mutex.c` provides a slow path interface to creating and destroying mutices dynamically via mca allocation. They can also be defined statically using the `.h` fast path interface. @@ -36,6 +34,6 @@ The rest of the threading implementation follows the normal MCA model: ## TODO -Libevent integration with lightweight threading models is a work in progress. The current Open MPI libevent library assumes preemption and does not yield by default. Lightweight threading libraries typically require tasks to be cooperative and to voluntarily yield after some time. +Some components in the current Open MPI runtime assume preemption and does not yield by default. Lightweight threading libraries typically require tasks to be cooperative and to voluntarily yield after some time. Open MPI itself needs to be altered to use a common yielding model instead of usleep(3). diff --git a/opal/mca/threads/argobots/Makefile.am b/opal/mca/threads/argobots/Makefile.am index cf8a941fd9..ba74c728d4 100644 --- a/opal/mca/threads/argobots/Makefile.am +++ b/opal/mca/threads/argobots/Makefile.am @@ -24,7 +24,6 @@ libmca_threads_argobots_la_SOURCES = \ threads_argobots.h \ threads_argobots_component.c \ threads_argobots_condition.c \ - threads_argobots_event.c \ threads_argobots_module.c \ threads_argobots_mutex.c \ threads_argobots_mutex.h \ diff --git a/opal/mca/threads/argobots/threads_argobots_event.c b/opal/mca/threads/argobots/threads_argobots_event.c deleted file mode 100644 index c4c14e1aef..0000000000 --- a/opal/mca/threads/argobots/threads_argobots_event.c +++ /dev/null @@ -1,143 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ -/* - * Copyright (c) 2019 Sandia National Laboratories. All rights reserved. - * - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include "opal/mca/threads/threads.h" -#include "opal/mca/threads/argobots/threads_argobots.h" -#include "opal/mca/event/libevent2022/libevent/include/event2/thread.h" -#include "opal/mca/event/libevent2022/libevent/include/event2/event-config.h" -#include "opal/mca/event/libevent2022/libevent/include/event2/util.h" - -#include - -static void *evthread_argobots_lock_alloc(unsigned locktype) -{ - ABT_mutex lock; - if (locktype & EVTHREAD_LOCKTYPE_RECURSIVE) { - ABT_mutex_attr abt_mutex_attr; - ABT_mutex_attr_create(&abt_mutex_attr); - ABT_mutex_attr_set_recursive(abt_mutex_attr, ABT_TRUE); - ABT_mutex_create_with_attr(abt_mutex_attr, &lock); - ABT_mutex_attr_free(&abt_mutex_attr); - } else { - ABT_mutex_create(&lock); - } - return lock; -} - -static void evthread_argobots_lock_free(void *_lock, unsigned locktype) -{ - ABT_mutex lock = _lock; - ABT_mutex_free(&lock); -} - -static int evthread_argobots_lock(unsigned mode, void *_lock) -{ - int ret; - ABT_mutex lock = _lock; - if (mode & EVTHREAD_TRY) { - ret = ABT_mutex_trylock(lock); - } else { - ret = ABT_mutex_lock(lock); - } - return ABT_SUCCESS == ret ? 0 : -1; -} - -static int evthread_argobots_unlock(unsigned mode, void *_lock) -{ - ABT_mutex lock = _lock; - int ret = ABT_mutex_unlock(lock); - /* This yield is necessary to avoid taking a lock consecutively. */ - ABT_thread_yield(); - return ABT_SUCCESS == ret ? 0 : -1; -} - -static unsigned long evthread_argobots_get_id(void) -{ - ABT_thread thr; - ABT_thread_self(&thr); - return (unsigned long)((intptr_t)thr); -} - -static void *evthread_argobots_cond_alloc(unsigned condflags) -{ - ABT_cond cond; - ABT_cond_create(&cond); - return cond; -} - -static void evthread_argobots_cond_free(void *_cond) -{ - ABT_cond cond = _cond; - ABT_cond_free(&cond); -} - -static int evthread_argobots_cond_signal(void *_cond, int broadcast) -{ - ABT_cond cond = _cond; - int ret; - if (broadcast) { - ret = ABT_cond_broadcast(cond); - } else { - ret = ABT_cond_signal(cond); - } - return ABT_SUCCESS == ret ? 0 : -1; -} - -static int evthread_argobots_cond_wait(void *_cond, void *_lock, - const struct timeval *tv) -{ - int ret; - ABT_cond cond = _cond; - ABT_mutex lock = _lock; - - if (tv) { - struct timeval now, abstime; - struct timespec ts; - evutil_gettimeofday(&now, NULL); - evutil_timeradd(&now, tv, &abstime); - ts.tv_sec = abstime.tv_sec; - ts.tv_nsec = abstime.tv_usec * 1000; - ret = ABT_cond_timedwait(cond, lock, &ts); - if (ABT_ERR_COND_TIMEDOUT == ret) { - return 1; - } else if (ABT_SUCCESS != ret) { - return -1; - } else { - return 0; - } - } else { - ret = ABT_cond_wait(cond, lock); - return ABT_SUCCESS == ret ? 0 : -1; - } -} - -void opal_event_use_threads(void) -{ - struct evthread_lock_callbacks cbs = { - EVTHREAD_LOCK_API_VERSION, - EVTHREAD_LOCKTYPE_RECURSIVE, - evthread_argobots_lock_alloc, - evthread_argobots_lock_free, - evthread_argobots_lock, - evthread_argobots_unlock - }; - struct evthread_condition_callbacks cond_cbs = { - EVTHREAD_CONDITION_API_VERSION, - evthread_argobots_cond_alloc, - evthread_argobots_cond_free, - evthread_argobots_cond_signal, - evthread_argobots_cond_wait - }; - opal_threads_argobots_ensure_init(); - evthread_set_lock_callbacks(&cbs); - evthread_set_condition_callbacks(&cond_cbs); - evthread_set_id_callback(evthread_argobots_get_id); -} diff --git a/opal/mca/threads/pthreads/threads_pthreads_module.c b/opal/mca/threads/pthreads/threads_pthreads_module.c index 5a08836224..aac8a62c86 100644 --- a/opal/mca/threads/pthreads/threads_pthreads_module.c +++ b/opal/mca/threads/pthreads/threads_pthreads_module.c @@ -137,8 +137,3 @@ int opal_tsd_keys_destruct(void) void opal_thread_set_main(void) { } - -void opal_event_use_threads(void) -{ - evthread_use_pthreads(); -} diff --git a/opal/mca/threads/threads.h b/opal/mca/threads/threads.h index bcbfa88197..21a25db6ac 100644 --- a/opal/mca/threads/threads.h +++ b/opal/mca/threads/threads.h @@ -132,7 +132,6 @@ OPAL_DECLSPEC bool opal_thread_self_compare(opal_thread_t *); OPAL_DECLSPEC opal_thread_t *opal_thread_get_self(void); OPAL_DECLSPEC void opal_thread_kill(opal_thread_t *, int sig); OPAL_DECLSPEC void opal_thread_set_main(void); -OPAL_DECLSPEC void opal_event_use_threads(void); END_C_DECLS