opal progress: Added MCA for multithreaded opal_progress.

This commit added MCA param `opal_max_thread_in_progress` to set the
number of threads allowed to do opal_progress concurrently. The default
value is 1.

Component with multithreaded design can benefit from this change to
parallelize their component progress function.

Signed-off-by: Thananon Patinyasakdikul <tpatinya@utk.edu>

opal/runtime/opal_params.c

@@ -76,6 +76,8 @@ bool opal_leave_pinned_pipeline = false;
 bool opal_abort_print_stack = false;
 int opal_abort_delay = 0;
 
+int opal_max_thread_in_progress = 1;
+
 static bool opal_register_done = false;
 
 int opal_register_params(void)
@@ -360,6 +362,12 @@ int opal_register_params(void)
             MCA_BASE_VAR_TYPE_STRING, NULL, 0, MCA_BASE_VAR_FLAG_INTERNAL, OPAL_INFO_LVL_3,
             MCA_BASE_VAR_SCOPE_READONLY, &mca_base_env_list_internal);
 
+    /* Number of threads allowed in opal_progress. This might increase multithreaded performance. */
+    (void)mca_base_var_register ("opal", "opal", NULL, "max_thread_in_progress",
+            "Number of thread allowed in opal_progress. Default: 1",
+            MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, OPAL_INFO_LVL_8,
+            MCA_BASE_VAR_SCOPE_READONLY, &opal_max_thread_in_progress);
+
     /* The ddt engine has a few parameters */
     ret = opal_datatype_register_params();
     if (OPAL_SUCCESS != ret) {

opal/threads/wait_sync.c

@@ -17,6 +17,8 @@
 static opal_mutex_t wait_sync_lock = OPAL_MUTEX_STATIC_INIT;
 static ompi_wait_sync_t* wait_sync_list = NULL;
 
+static opal_atomic_int32_t num_thread_in_progress = 0;
+
 #define WAIT_SYNC_PASS_OWNERSHIP(who)                  \
     do {                                               \
         pthread_mutex_lock( &(who)->lock);             \
@@ -63,7 +65,7 @@ int ompi_sync_wait_mt(ompi_wait_sync_t *sync)
      *  - our sync has been triggered.
      */
  check_status:
-    if( sync != wait_sync_list ) {
+    if( sync != wait_sync_list && num_thread_in_progress >= opal_max_thread_in_progress) {
         pthread_cond_wait(&sync->condition, &sync->lock);
 
         /**
@@ -79,11 +81,14 @@ int ompi_sync_wait_mt(ompi_wait_sync_t *sync)
         /* either promoted, or spurious wakeup ! */
         goto check_status;
     }
-
     pthread_mutex_unlock(&sync->lock);
+
+    OPAL_THREAD_ADD_FETCH32(&num_thread_in_progress, 1);
     while(sync->count > 0) {  /* progress till completion */
         opal_progress();  /* don't progress with the sync lock locked or you'll deadlock */
     }
+    OPAL_THREAD_ADD_FETCH32(&num_thread_in_progress, -1);
+
     assert(sync == wait_sync_list);
 
  i_am_done:

opal/threads/wait_sync.h

@@ -25,6 +25,8 @@
 
 BEGIN_C_DECLS
 
+extern int opal_max_thread_in_progress;
+
 typedef struct ompi_wait_sync_t {
     opal_atomic_int32_t count;
     int32_t status;