oshmem: refactor group cache

- Use opal hash table instead of list for group lookup.
- Code cleanup/refactoring. Group cache is now a part
  of the proc_group.

Signed-off-by: Alex Mikheev <alexm@mellanox.com>

oshmem/proc/proc.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013      Mellanox Technologies, Inc.
+ * Copyright (c) 2013-2018 Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2014-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -17,6 +17,7 @@
 #include "oshmem/constants.h"
 #include "oshmem/runtime/runtime.h"
 #include "oshmem/mca/scoll/base/base.h"
+#include "oshmem/proc/proc_group_cache.h"
 
 #ifdef HAVE_STRINGS_H
 #include <strings.h>
@@ -65,40 +66,67 @@ oshmem_group_t* oshmem_group_null = NULL;
 
 OBJ_CLASS_INSTANCE(oshmem_group_t, opal_object_t, NULL, NULL);
 
+static void oshmem_proc_group_destroy_internal(oshmem_group_t* group,
+                                               int scoll_unselect);
+
 int oshmem_proc_group_init(void)
 {
+    int rc;
+
+    rc = oshmem_group_cache_init();
+    if (OSHMEM_SUCCESS != rc) {
+        return rc;
+    }
+
     /* Setup communicator array */
     OBJ_CONSTRUCT(&oshmem_group_array, opal_pointer_array_t);
-    if (OPAL_SUCCESS
+
-            != opal_pointer_array_init(&oshmem_group_array,
+    rc = opal_pointer_array_init(&oshmem_group_array, 0,
-                                       0,
+                                 ORTE_GLOBAL_ARRAY_MAX_SIZE, 1);
-                                       ORTE_GLOBAL_ARRAY_MAX_SIZE,
+    if (OPAL_SUCCESS != rc) {
-                                       1)) {
+        goto err1;
-        return OSHMEM_ERROR;
     }
 
     /* Setup SHMEM_GROUP_ALL */
-    if (NULL
+    oshmem_group_all = oshmem_proc_group_create(0, 1, ompi_comm_size(oshmem_comm_world));
-            == (oshmem_group_all =
+    if (NULL == oshmem_group_all) {
-                    oshmem_proc_group_create(0,
+        goto err2;
-                                             1,
-					     ompi_comm_size(oshmem_comm_world)))) {
-        return OSHMEM_ERROR;
     }
 
     /* Setup SHMEM_GROUP_SELF */
-    if (NULL
+    oshmem_group_self = oshmem_proc_group_create(oshmem_proc_pe(oshmem_proc_local()), 0, 1);
-            == (oshmem_group_self = oshmem_proc_group_create(oshmem_proc_pe(oshmem_proc_local()),
+    if (NULL == oshmem_group_self) {
-                                                             0,
+        goto err3;
-                                                             1))) {
-        oshmem_proc_group_destroy(oshmem_group_self);
-        return OSHMEM_ERROR;
     }
 
     /* Setup SHMEM_GROUP_NULL */
     oshmem_group_null = NULL;
 
     return OSHMEM_SUCCESS;
+
+err3:
+    oshmem_proc_group_destroy_internal(oshmem_group_all, 1);
+err2:
+    OBJ_DESTRUCT(&oshmem_group_array);
+err1:
+    oshmem_group_cache_destroy();
+    return OSHMEM_ERROR;
+}
+
+void oshmem_proc_group_finalize_scoll(void)
+{
+    int max, i;
+    oshmem_group_t *group;
+
+    /* Check whether we have some left */
+    max = opal_pointer_array_get_size(&oshmem_group_array);
+    for (i = 0; i < max; i++) {
+        group = (oshmem_group_t *) opal_pointer_array_get_item(&oshmem_group_array,
+                                                               i);
+        if (NULL != group) {
+            mca_scoll_base_group_unselect(group);
+        }
+    }
 }
 
 int oshmem_proc_group_finalize(void)
@@ -114,18 +142,17 @@ int oshmem_proc_group_finalize(void)
                                                                i);
         if (NULL != group) {
             /* Group has not been freed before finalize */
-            oshmem_proc_group_destroy(group);
+            oshmem_proc_group_destroy_internal(group, 0);
         }
     }
 
     OBJ_DESTRUCT(&oshmem_group_array);
 
+    oshmem_group_cache_destroy();
     return OSHMEM_SUCCESS;
 }
 
-oshmem_group_t* oshmem_proc_group_create(int pe_start,
+oshmem_group_t* oshmem_proc_group_create(int pe_start, int pe_stride, int pe_size)
-                                         int pe_stride,
-                                         size_t pe_size)
 {
     int cur_pe, count_pe;
     int i;
@@ -135,9 +162,16 @@ oshmem_group_t* oshmem_proc_group_create(int pe_start,
 
     assert(oshmem_proc_local());
 
-    group = OBJ_NEW(oshmem_group_t);
+    group = oshmem_group_cache_find(pe_start, pe_stride, pe_size);
+    if (NULL != group) {
+        return group;
+    }
+
+    group = OBJ_NEW(oshmem_group_t);
+    if (NULL == group) {
+        return NULL;
+    }
 
-    if (group) {
     cur_pe = 0;
     count_pe = 0;
 
@@ -197,20 +231,32 @@ oshmem_group_t* oshmem_proc_group_create(int pe_start,
     if (OSHMEM_SUCCESS != mca_scoll_base_select(group)) {
         opal_output(0,
                 "Error: No collective modules are available: group is not created, returning NULL");
-            oshmem_proc_group_destroy(group);
+        oshmem_proc_group_destroy_internal(group, 0);
         OPAL_THREAD_UNLOCK(&oshmem_proc_lock);
         return NULL;
     }
+
+    if (OSHMEM_SUCCESS != oshmem_group_cache_insert(group, pe_start,
+                                                    pe_stride, pe_size)) {
+        oshmem_proc_group_destroy_internal(group, 1);
         OPAL_THREAD_UNLOCK(&oshmem_proc_lock);
+        return NULL;
     }
 
+    OPAL_THREAD_UNLOCK(&oshmem_proc_lock);
     return group;
 }
 
-void oshmem_proc_group_destroy(oshmem_group_t* group)
+static void
+oshmem_proc_group_destroy_internal(oshmem_group_t* group, int scoll_unselect)
 {
-    if (group) {
+    if (NULL == group) {
+        return;
+    }
+
+    if (scoll_unselect) {
         mca_scoll_base_group_unselect(group);
+    }
 
     /* Destroy proc array */
     if (group->proc_array) {
@@ -238,4 +284,11 @@ void oshmem_proc_group_destroy(oshmem_group_t* group)
 
     OBJ_RELEASE(group);
 }
+
+void oshmem_proc_group_destroy(oshmem_group_t* group)
+{
+    if (oshmem_group_cache_enabled()) {
+        return;
+    }
+    oshmem_proc_group_destroy_internal(group, 1);
 }

oshmem/proc/proc.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013      Mellanox Technologies, Inc.
+ * Copyright (c) 2013-2018 Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2016      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -17,8 +17,6 @@
 #include "oshmem/types.h"
 #include "oshmem/constants.h"
 
-#include "oshmem/mca/scoll/scoll.h"
-
 #include "opal/class/opal_list.h"
 #include "opal/util/proc.h"
 #include "opal/dss/dss_types.h"
@@ -30,6 +28,10 @@
 #include "ompi/proc/proc.h"
 #include "ompi/communicator/communicator.h"
 
+#include "oshmem/mca/scoll/scoll.h"
+#include "oshmem/runtime/runtime.h"
+#include "oshmem/shmem/shmem_api_logger.h"
+
 BEGIN_C_DECLS
 
 /* ******************************************************************** */
@@ -177,19 +179,17 @@ OSHMEM_DECLSPEC int oshmem_proc_group_init(void);
 /**
  * Finalize the OSHMEM process predefined groups
  *
- * Initialize the Open SHMEM process predefined groups.  This function will
- * query the run-time environment and build a list of the proc
- * instances in the current pe set.  The local information not
- * easily determined by the run-time ahead of time (architecture and
- * hostname) will be published during this call.
- *
- * @note This is primarily used once during SHMEM setup.
- *
  * @retval OSHMEM_SUCESS  System successfully initialized
  * @retval OSHMEM_ERROR   Initialization failed due to unspecified error
  */
 OSHMEM_DECLSPEC int oshmem_proc_group_finalize(void);
 
+/**
+ * Release collectives used by the groups. The function
+ * must be called prior to the oshmem_proc_group_finalize()
+ */
+OSHMEM_DECLSPEC void oshmem_proc_group_finalize_scoll(void);
+
 /**
  * Create processes group.
  *
@@ -205,7 +205,29 @@ OSHMEM_DECLSPEC int oshmem_proc_group_finalize(void);
  */
 OSHMEM_DECLSPEC oshmem_group_t *oshmem_proc_group_create(int pe_start,
                                                          int pe_stride,
-                                                         size_t pe_size);
+                                                         int pe_size);
+
+/**
+ * same as above but abort on failure
+ */
+static inline oshmem_group_t *
+oshmem_proc_group_create_nofail(int pe_start, int pe_stride, int pe_size)
+{
+    oshmem_group_t *group;
+
+    group = oshmem_proc_group_create(pe_start, pe_stride, pe_size);
+    if (NULL == group) {
+        goto fatal;
+    }
+    return group;
+
+fatal:
+    SHMEM_API_ERROR("Failed to create group (%d,%d,%d)",
+                    pe_start, pe_stride, pe_size);
+    oshmem_shmem_abort(-1);
+    return NULL;
+}
+
 
 /**
  * Destroy processes group.

oshmem/proc/proc_group_cache.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013      Mellanox Technologies, Inc.
+ * Copyright (c) 2013-2018 Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -13,104 +13,73 @@
 #include "oshmem/constants.h"
 #include "oshmem/runtime/runtime.h"
 
-OBJ_CLASS_INSTANCE(oshmem_group_cache_t, opal_object_t, NULL, NULL);
+#define OSHMEM_GROUP_CACHE_SIZE 1024
-opal_list_t oshmem_group_cache_list = {{0}};
+
-unsigned int oshmem_group_cache_size = 0;
+static opal_hash_table_t group_cache;
-oshmem_group_t* find_group_in_cache(int PE_start, int logPE_stride, int PE_size)
+
+typedef struct {
+    int pe_start;
+    int pe_size;
+    int pe_stride;
+} oshmem_group_key_t;
+
+static int group_cache_n_hits;
+static int group_cache_n_lookups;
+
+int oshmem_group_cache_init(void)
 {
-    int cache_look_up_id[3] = { PE_start, logPE_stride, PE_size };
+    OBJ_CONSTRUCT(&group_cache, opal_hash_table_t);
-    opal_list_item_t *item;
+    if (OPAL_SUCCESS != opal_hash_table_init(&group_cache, OSHMEM_GROUP_CACHE_SIZE)) {
-    if (opal_list_is_empty(&oshmem_group_cache_list)) {
+        return OSHMEM_ERROR;
+    }
+    return OSHMEM_SUCCESS;
+}
+
+void oshmem_group_cache_destroy(void)
+{
+    OBJ_DESTRUCT(&group_cache);
+}
+
+oshmem_group_t *oshmem_group_cache_find(int pe_start, int pe_stride, int pe_size)
+{
+    oshmem_group_key_t key;
+    oshmem_group_t *group;
+
+    if (!oshmem_group_cache_enabled()) {
         return NULL;
     }
 
-    for (item = opal_list_get_first(&oshmem_group_cache_list);
+    key.pe_start  = pe_start;
-            item && (item != opal_list_get_end(&oshmem_group_cache_list));
+    key.pe_size   = pe_size;
-            item = opal_list_get_next(item)) {
+    key.pe_stride = pe_stride;
-        if (!memcmp(((oshmem_group_cache_t *) item)->cache_id,
+
-                    cache_look_up_id,
+    group_cache_n_lookups++;
-                    3 * sizeof(int))) {
+
-            return ((oshmem_group_cache_t *) item)->group;
+    if (OPAL_SUCCESS != opal_hash_table_get_value_ptr(&group_cache, &key,
-        }
+                                                      sizeof(key), (void **)&group)) {
-    }
         return NULL;
     }
 
-int cache_group(oshmem_group_t *group,
+    group_cache_n_hits++;
-                int PE_start,
+    return group;
-                int logPE_stride,
+}
-                int PE_size)
+
+int oshmem_group_cache_insert(oshmem_group_t *group, int pe_start,
+                              int pe_stride, int pe_size)
 {
-    oshmem_group_cache_t *cached_group = NULL;
+    oshmem_group_key_t key;
-    cached_group = OBJ_NEW(oshmem_group_cache_t);
+
-#if OPAL_ENABLE_DEBUG
+    if (!oshmem_group_cache_enabled()) {
-    cached_group->item.opal_list_item_belong_to = NULL;
+        return OSHMEM_SUCCESS;
-    cached_group->item.opal_list_item_refcount = 0;
+    }
-#endif
+
-    cached_group->group = group;
+    key.pe_start  = pe_start;
-    cached_group->cache_id[0] = PE_start;
+    key.pe_size   = pe_size;
-    cached_group->cache_id[1] = logPE_stride;
+    key.pe_stride = pe_stride;
-    cached_group->cache_id[2] = PE_size;
+
-    if (opal_list_get_size(&oshmem_group_cache_list)
+    if (OPAL_SUCCESS != opal_hash_table_set_value_ptr(&group_cache, &key,
-            < oshmem_group_cache_size) {
+                                                      sizeof(key), group)) {
-        opal_list_append(&oshmem_group_cache_list,
+        return OSHMEM_ERROR;
-                         (opal_list_item_t *)cached_group);
-    } else {
-#if ABORT_ON_CACHE_OVERFLOW
-        opal_output(0,
-                    "error: group cache overflow on rank %i: cache_size = %u: try increasing oshmem_proc_group_cache_size mca parameter",
-                    group->my_pe,
-                    oshmem_group_cache_size);
-        oshmem_shmem_abort(-1);
-#else
-        /*This part of code makes FIFO group cache management. Define ABORT_ON_CACHE_OVERFLOW as 0 to enable this.*/
-        oshmem_group_cache_t *cached_group_to_remove = (oshmem_group_cache_t *)opal_list_remove_first(&oshmem_group_cache_list);
-        oshmem_proc_group_destroy(cached_group_to_remove->group);
-        OBJ_RELEASE(cached_group_to_remove);
-        opal_list_append(&oshmem_group_cache_list,(opal_list_item_t *)cached_group);
-#endif
-    }
-    return OSHMEM_SUCCESS;
-}
-
-int oshmem_group_cache_list_init(void)
-{
-    int mca_value;
-    int cache_size_default = 1000;
-    OBJ_CONSTRUCT(&oshmem_group_cache_list, opal_list_t);
-
-    mca_value = cache_size_default;
-    (void) mca_base_var_register("oshmem",
-                                 "proc",
-                                 NULL,
-                                 "group_cache_size",
-                                 "The depth of the oshmem_group cache list used to speed up collective operations",
-                                 MCA_BASE_VAR_TYPE_INT,
-                                 NULL,
-                                 0,
-                                 MCA_BASE_VAR_FLAG_SETTABLE,
-                                 OPAL_INFO_LVL_9,
-                                 MCA_BASE_VAR_SCOPE_READONLY,
-                                 &mca_value);
-    if (mca_value < 0) {
-        opal_output(0,
-                    "error: oshmem_proc_group_cache_size mca parameter was set to %i while it has to be positive value. Default value %i will be used.",
-                    mca_value,
-                    cache_size_default);
-        mca_value = cache_size_default;
-    }
-    oshmem_group_cache_size = (unsigned int) mca_value;
-    return OSHMEM_SUCCESS;
-}
-
-int oshmem_group_cache_list_free(void)
-{
-    oshmem_group_cache_t *cached_group = NULL;
-    opal_list_item_t *item;
-    while (NULL != (item = opal_list_remove_first(&oshmem_group_cache_list))) {
-        cached_group = (oshmem_group_cache_t *) item;
-        oshmem_proc_group_destroy(cached_group->group);
-        OBJ_RELEASE(cached_group);
     }
     return OSHMEM_SUCCESS;
 }

oshmem/proc/proc_group_cache.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013      Mellanox Technologies, Inc.
+ * Copyright (c) 2013-2018 Mellanox Technologies, Inc.
  *                         All rights reserved.
  * $COPYRIGHT$
  *
@@ -14,28 +14,32 @@
 #include "proc.h"
 
 #define OSHMEM_GROUP_CACHE_ENABLED 1
-#define ABORT_ON_CACHE_OVERFLOW 1
+
 BEGIN_C_DECLS
-struct oshmem_group_cache_t {
-    opal_list_item_t item;
-    oshmem_group_t *group;
-    int cache_id[3];
-};
 
-typedef struct oshmem_group_cache_t oshmem_group_cache_t;
+/**
-OSHMEM_DECLSPEC OBJ_CLASS_DECLARATION(oshmem_group_cache_t);
+ * A group cache.
-OSHMEM_DECLSPEC extern opal_list_t oshmem_group_cache_list;
+ *
+ * Deletion of a group is not implemented because it
+ * requires a synchronization between PEs
+ *
+ * If cache enabled every group is kept until the
+ * shmem_finalize() is called
+ */
 
-oshmem_group_t* find_group_in_cache(int PE_start, int logPE_stride, int PE_size);
+int oshmem_group_cache_init(void);
+void oshmem_group_cache_destroy(void);
 
-int cache_group(oshmem_group_t *group,
+oshmem_group_t* oshmem_group_cache_find(int pe_start, int pe_stride, int pe_size);
-                int PE_start,
+
-                int logPE_stride,
+int oshmem_group_cache_insert(oshmem_group_t *group, int pe_start,
-                int PE_size);
+                              int pe_stride, int pe_size);
-int oshmem_group_cache_list_init(void);
+
-int oshmem_group_cache_list_free(void);
+static inline int oshmem_group_cache_enabled(void)
+{
+    return OSHMEM_GROUP_CACHE_ENABLED;
+}
 
-extern unsigned int oshmem_group_cache_size;
 END_C_DECLS
 
 #endif

oshmem/runtime/oshmem_shmem_finalize.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013      Mellanox Technologies, Inc.
+ * Copyright (c) 2013-2018 Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2015      Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -102,17 +102,10 @@ static int _shmem_finalize(void)
     if (OSHMEM_SUCCESS != (ret = oshmem_request_finalize())) {
         return ret;
     }
-    /* must free cached groups before we kill collectives */
+
-    if (OSHMEM_SUCCESS != (ret = oshmem_group_cache_list_free())) {
+    oshmem_proc_group_finalize_scoll();
-        return ret;
-    }
-    /* We need to call mca_scoll_base_group_unselect explicitly for each group
-     * that are not freed by oshmem_group_cache_list_free. We can only release its collectives at this point */
-    mca_scoll_base_group_unselect(oshmem_group_all);
-    mca_scoll_base_group_unselect(oshmem_group_self);
 
     /* Close down MCA modules */
-
     if (OSHMEM_SUCCESS != (ret = mca_base_framework_close(&oshmem_atomic_base_framework) ) ) {
         return ret;
     }

oshmem/runtime/oshmem_shmem_init.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013      Mellanox Technologies, Inc.
+ * Copyright (c) 2013-2018 Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2015-2016 Research Organization for Information Science
  *                         and Technology (RIST). All rights reserved.
@@ -259,11 +259,6 @@ static int _shmem_init(int argc, char **argv, int requested, int *provided)
         goto error;
     }
 
-    if (OSHMEM_SUCCESS != (ret = oshmem_group_cache_list_init())) {
-        error = "oshmem_group_cache_list_init() failed";
-        goto error;
-    }
-
     if (OSHMEM_SUCCESS != (ret = oshmem_op_init())) {
         error = "oshmem_op_init() failed";
         goto error;

oshmem/shmem/c/shmem_alltoall.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2016      Mellanox Technologies, Inc.
+ * Copyright (c) 2016-2018 Mellanox Technologies, Inc.
  *                         All rights reserved.
  * $COPYRIGHT$
  *
@@ -19,7 +19,6 @@
 #include "oshmem/mca/scoll/scoll.h"
 
 #include "oshmem/proc/proc.h"
-#include "oshmem/proc/proc_group_cache.h"
 
 static void _shmem_alltoall(void *target,
                             const void *source,
@@ -78,31 +77,11 @@ static void _shmem_alltoall(void *target,
                             int PE_size,
                             long *pSync)
 {
-    int rc = OSHMEM_SUCCESS;
+    int rc;
-    oshmem_group_t* group = NULL;
+    oshmem_group_t* group;
 
-    if ((0 <= PE_start) && (0 <= logPE_stride)) {
     /* Create group basing PE_start, logPE_stride and PE_size */
-#if OSHMEM_GROUP_CACHE_ENABLED == 0
+    group = oshmem_proc_group_create_nofail(PE_start, 1<<logPE_stride, PE_size);
-        group = oshmem_proc_group_create(PE_start, (1 << logPE_stride), PE_size);
-        if (!group)
-        rc = OSHMEM_ERROR;
-#else
-        group = find_group_in_cache(PE_start, logPE_stride, PE_size);
-        if (!group) {
-            group = oshmem_proc_group_create(PE_start,
-                                             (1 << logPE_stride),
-                                             PE_size);
-            if (!group) {
-                rc = OSHMEM_ERROR;
-            } else {
-                cache_group(group, PE_start, logPE_stride, PE_size);
-            }
-        }
-#endif /* OSHMEM_GROUP_CACHE_ENABLED */
-
-        /* Collective operation call */
-        if (rc == OSHMEM_SUCCESS) {
     /* Call collective alltoall operation */
     rc = group->g_scoll.scoll_alltoall(group,
                                        target,
@@ -113,13 +92,8 @@ static void _shmem_alltoall(void *target,
                                        element_size,
                                        pSync,
                                        SCOLL_DEFAULT_ALG);
-        }
-#if OSHMEM_GROUP_CACHE_ENABLED == 0
-        if ( rc == OSHMEM_SUCCESS ) {
     oshmem_proc_group_destroy(group);
-        }
+    RUNTIME_CHECK_RC(rc);
-#endif /* OSHMEM_GROUP_CACHE_ENABLED */
-    }
 }
 
 #if OSHMEM_PROFILING

oshmem/shmem/c/shmem_barrier.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013      Mellanox Technologies, Inc.
+ * Copyright (c) 2013-2018 Mellanox Technologies, Inc.
  *                         All rights reserved.
  * $COPYRIGHT$
  *
@@ -18,7 +18,6 @@
 #include "oshmem/mca/scoll/base/base.h"
 
 #include "oshmem/proc/proc.h"
-#include "oshmem/proc/proc_group_cache.h"
 
 
 #if OSHMEM_PROFILING
@@ -30,8 +29,8 @@
 
 void shmem_barrier(int PE_start, int logPE_stride, int PE_size, long *pSync)
 {
-    int rc = OSHMEM_SUCCESS;
+    int rc;
-    oshmem_group_t* group = NULL;
+    oshmem_group_t* group;
 
     RUNTIME_CHECK_INIT();
 
@@ -40,38 +39,12 @@ void shmem_barrier(int PE_start, int logPE_stride, int PE_size, long *pSync)
     shmem_fence();
 #endif
 
-    if ((0 <= PE_start) && (0 <= logPE_stride)) {
     /* Create group basing PE_start, logPE_stride and PE_size */
-#if OSHMEM_GROUP_CACHE_ENABLED == 0
+    group = oshmem_proc_group_create_nofail(PE_start, 1<<logPE_stride, PE_size);
-        group = oshmem_proc_group_create(PE_start, (1 << logPE_stride), PE_size);
-        if (!group)
-        rc = OSHMEM_ERROR;
-#else
-        group = find_group_in_cache(PE_start, logPE_stride, PE_size);
-        if (!group) {
-            group = oshmem_proc_group_create(PE_start,
-                                             (1 << logPE_stride),
-                                             PE_size);
-            if (!group) {
-                rc = OSHMEM_ERROR;
-            } else {
-                cache_group(group, PE_start, logPE_stride, PE_size);
-            }
-        }
-#endif /* OSHMEM_GROUP_CACHE_ENABLED */
-        /* Collective operation call */
-        if (rc == OSHMEM_SUCCESS) {
     /* Call barrier operation */
     rc = group->g_scoll.scoll_barrier(group, pSync, SCOLL_DEFAULT_ALG);
-        }
 
-#if OSHMEM_GROUP_CACHE_ENABLED == 0
-        if ( rc == OSHMEM_SUCCESS )
-        {
     oshmem_proc_group_destroy(group);
-        }
-#endif /* OSHMEM_GROUP_CACHE_ENABLED */
-    }
     RUNTIME_CHECK_RC(rc);
 }
 

oshmem/shmem/c/shmem_broadcast.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013-2017 Mellanox Technologies, Inc.
+ * Copyright (c) 2013-2018 Mellanox Technologies, Inc.
  *                         All rights reserved.
  * $COPYRIGHT$
  *
@@ -19,7 +19,6 @@
 #include "oshmem/mca/scoll/scoll.h"
 
 #include "oshmem/proc/proc.h"
-#include "oshmem/proc/proc_group_cache.h"
 
 static void _shmem_broadcast(void *target,
                               const void *source,
@@ -58,33 +57,17 @@ static void _shmem_broadcast(void *target,
                               int PE_size,
                               long *pSync)
 {
-    int rc = OSHMEM_SUCCESS;
+    int rc;
-    oshmem_group_t* group = NULL;
+    oshmem_group_t *group;
 
     if ((0 <= PE_root) && (PE_root < PE_size)) {
         /* Create group basing PE_start, logPE_stride and PE_size */
-#if OSHMEM_GROUP_CACHE_ENABLED == 0
+        group = oshmem_proc_group_create_nofail(PE_start, 1 << logPE_stride, PE_size);
-        group = oshmem_proc_group_create(PE_start, (1 << logPE_stride), PE_size);
+        if (PE_root >= group->proc_count) {
-        if (!group || (PE_root >= group->proc_count))
-        {
             rc = OSHMEM_ERROR;
+            goto out;
         }
-#else
-        group = find_group_in_cache(PE_start, logPE_stride, PE_size);
-        if (!group) {
-            group = oshmem_proc_group_create(PE_start,
-                                             (1 << logPE_stride),
-                                             PE_size);
-            if (!group || (PE_root >= group->proc_count)) {
-                rc = OSHMEM_ERROR;
-            } else {
-                cache_group(group, PE_start, logPE_stride, PE_size);
-            }
-        }
-#endif /* OSHMEM_GROUP_CACHE_ENABLED */
 
-        /* Collective operation call */
-        if (rc == OSHMEM_SUCCESS) {
         /* Define actual PE using relative in active set */
         PE_root = oshmem_proc_pe(group->proc_array[PE_root]);
 
@@ -96,13 +79,9 @@ static void _shmem_broadcast(void *target,
                                             nbytes,
                                             pSync,
                                             SCOLL_DEFAULT_ALG);
-        }
+out:
-#if OSHMEM_GROUP_CACHE_ENABLED == 0
-        if ( rc == OSHMEM_SUCCESS )
-        {
         oshmem_proc_group_destroy(group);
-        }
+        RUNTIME_CHECK_RC(rc);
-#endif /* OSHMEM_GROUP_CACHE_ENABLED */
     }
 }
 

oshmem/shmem/c/shmem_collect.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013      Mellanox Technologies, Inc.
+ * Copyright (c) 2013-2018 Mellanox Technologies, Inc.
  *                         All rights reserved.
  * $COPYRIGHT$
  *
@@ -19,7 +19,6 @@
 #include "oshmem/mca/scoll/scoll.h"
 
 #include "oshmem/proc/proc.h"
-#include "oshmem/proc/proc_group_cache.h"
 
 static void _shmem_collect(void *target,
                             const void *source,
@@ -58,31 +57,11 @@ static void _shmem_collect(void *target,
                             long *pSync,
                             bool array_type)
 {
-    int rc = OSHMEM_SUCCESS;
+    int rc;
-    oshmem_group_t* group = NULL;
+    oshmem_group_t *group;
 
-    {
     /* Create group basing PE_start, logPE_stride and PE_size */
-#if OSHMEM_GROUP_CACHE_ENABLED == 0
+    group = oshmem_proc_group_create_nofail(PE_start, 1<<logPE_stride, PE_size);
-        group = oshmem_proc_group_create(PE_start, (1 << logPE_stride), PE_size);
-        if (!group)
-        rc = OSHMEM_ERROR;
-#else
-        group = find_group_in_cache(PE_start, logPE_stride, PE_size);
-        if (!group) {
-            group = oshmem_proc_group_create(PE_start,
-                                             (1 << logPE_stride),
-                                             PE_size);
-            if (!group) {
-                rc = OSHMEM_ERROR;
-            } else {
-                cache_group(group, PE_start, logPE_stride, PE_size);
-            }
-        }
-#endif /* OSHMEM_GROUP_CACHE_ENABLED */
-
-        /* Collective operation call */
-        if (rc == OSHMEM_SUCCESS) {
     /* Call collective broadcast operation */
     rc = group->g_scoll.scoll_collect(group,
                                       target,
@@ -91,14 +70,8 @@ static void _shmem_collect(void *target,
                                       pSync,
                                       array_type,
                                       SCOLL_DEFAULT_ALG);
-        }
-#if OSHMEM_GROUP_CACHE_ENABLED == 0
-        if ( rc == OSHMEM_SUCCESS )
-        {
     oshmem_proc_group_destroy(group);
-        }
+    RUNTIME_CHECK_RC(rc);
-#endif /* OSHMEM_GROUP_CACHE_ENABLED */
-    }
 }
 
 #if OSHMEM_PROFILING

oshmem/shmem/c/shmem_reduce.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013      Mellanox Technologies, Inc.
+ * Copyright (c) 2013-2018 Mellanox Technologies, Inc.
  *                         All rights reserved.
  * $COPYRIGHT$
  *
@@ -16,15 +16,8 @@
 
 #include "oshmem/mca/scoll/scoll.h"
 #include "oshmem/proc/proc.h"
-#include "oshmem/proc/proc_group_cache.h"
 #include "oshmem/op/op.h"
 
-#if OSHMEM_GROUP_CACHE_ENABLED == 0
-static bool __group_cache_enabled = false;
-#else
-static bool __group_cache_enabled = true;
-#endif /* OSHMEM_GROUP_CACHE_ENABLED */
-
 /*
  * The shared memory (SHMEM) reduction routines perform an associative binary operation
  * across symmetric arrays on multiple virtual PEs.
@@ -50,28 +43,7 @@ static bool __group_cache_enabled = true;
     RUNTIME_CHECK_ADDR(source);                                                             \
                                                                                             \
     {                                                                                       \
-        /* Create group basing PE_start, logPE_stride and PE_size */                        \
+        group = oshmem_proc_group_create_nofail(PE_start, 1<<logPE_stride, PE_size);        \
-        if (!__group_cache_enabled)                                                         \
-        {                                                                                   \
-            group = oshmem_proc_group_create(PE_start, (1 << logPE_stride), PE_size);       \
-            if (!group)                                                                     \
-                rc = OSHMEM_ERROR;                                                          \
-        }                                                                                   \
-        else                                                                                \
-        {                                                                                   \
-            group = find_group_in_cache(PE_start,logPE_stride,PE_size);                     \
-            if (!group)                                                                     \
-            {                                                                               \
-                group = oshmem_proc_group_create(PE_start, (1 << logPE_stride), PE_size);   \
-                if (!group)                                                                 \
-                    rc = OSHMEM_ERROR;                                                      \
-                cache_group(group,PE_start,logPE_stride,PE_size);                           \
-            }                                                                               \
-        }                                                                                   \
-                                                                                            \
-        /* Collective operation call */                                                     \
-        if ( rc == OSHMEM_SUCCESS )                                                         \
-        {                                                                                   \
         oshmem_op_t* op = oshmem_op_##name##type_name;                                      \
         size_t size = nreduce * op->dt_size;                                                \
                                                                                             \
@@ -85,13 +57,9 @@ static bool __group_cache_enabled = true;
                 pSync,                                                                      \
                 (void*)pWrk,                                                                \
                 SCOLL_DEFAULT_ALG );                                                        \
-        }                                                                                   \
                                                                                             \
-        if ( !__group_cache_enabled && (rc == OSHMEM_SUCCESS ) )                            \
-        {                                                                                   \
         oshmem_proc_group_destroy(group);                                                   \
     }                                                                                       \
-    }                                                                                       \
     RUNTIME_CHECK_RC(rc);                                                                   \
 }
 

oshmem/shmem/fortran/shmem_alltoall_f.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013-2016 Mellanox Technologies, Inc.
+ * Copyright (c) 2013-2018 Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2013 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
@@ -15,7 +15,6 @@
 #include "oshmem/constants.h"
 #include "oshmem/mca/scoll/scoll.h"
 #include "oshmem/proc/proc.h"
-#include "oshmem/proc/proc_group_cache.h"
 #include "oshmem/op/op.h"
 
 #if OSHMEM_PROFILING
@@ -59,7 +58,7 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
         (FORTRAN_POINTER_T target, FORTRAN_POINTER_T source, MPI_Fint *dst, MPI_Fint *sst, MPI_Fint *nlong, MPI_Fint *PE_start, MPI_Fint * logPE_stride, MPI_Fint *PE_size, FORTRAN_POINTER_T pSync),
         (target, source, dst, sst, nlong, PE_start, logPE_stride, PE_size, pSync))
 
-#define SHMEM_ALLTOALL(F_NAME, T_NAME, OSHMEM_GROUP_CACHE_ENABLED) void F_NAME(FORTRAN_POINTER_T target, \
+#define SHMEM_ALLTOALL(F_NAME, T_NAME) void F_NAME(FORTRAN_POINTER_T target, \
     FORTRAN_POINTER_T source, \
     MPI_Fint *nlong,\
     MPI_Fint *PE_start, \
@@ -67,38 +66,12 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
     MPI_Fint *PE_size, \
     FORTRAN_POINTER_T pSync)\
 {\
-    int rc = OSHMEM_SUCCESS;\
+    int rc;\
-    oshmem_group_t*  group = NULL;\
+    oshmem_group_t *group;\
-    {\
     /* Create group basing PE_start, logPE_stride and PE_size */\
-        if (OSHMEM_GROUP_CACHE_ENABLED == 0)\
+    group = oshmem_proc_group_create_nofail(OMPI_FINT_2_INT(*PE_start), \
-        {\
-            group = oshmem_proc_group_create(OMPI_FINT_2_INT(*PE_start), \
                                             (1 << OMPI_FINT_2_INT(*logPE_stride)), \
                                             OMPI_FINT_2_INT(*PE_size));\
-            if (!group)\
-                rc = OSHMEM_ERROR;\
-        }\
-        else\
-        {\
-            group = find_group_in_cache(OMPI_FINT_2_INT(*PE_start),\
-                OMPI_FINT_2_INT(*logPE_stride),\
-                OMPI_FINT_2_INT(*PE_size));\
-            if (!group)\
-            {\
-                group = oshmem_proc_group_create(OMPI_FINT_2_INT(*PE_start), \
-                    (1 << OMPI_FINT_2_INT(*logPE_stride)), \
-                    OMPI_FINT_2_INT(*PE_size));\
-                if (!group)\
-                    rc = OSHMEM_ERROR;\
-                cache_group(group,OMPI_FINT_2_INT(*PE_start),\
-                    OMPI_FINT_2_INT(*logPE_stride),\
-                    OMPI_FINT_2_INT(*PE_size));\
-            }\
-        } /* OSHMEM_GROUP_CACHE_ENABLED */\
-        /* Collective operation call */\
-        if ( rc == OSHMEM_SUCCESS )\
-        {\
     oshmem_op_t* op = T_NAME;\
     \
     /* Call collective broadcast operation */\
@@ -110,18 +83,11 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
             OMPI_FINT_2_INT(*nlong), \
             op->dt_size, \
             FPTR_2_VOID_PTR(pSync), SCOLL_DEFAULT_ALG );\
-        }\
-        if (OSHMEM_GROUP_CACHE_ENABLED == 0) \
-        {\
-            if ( group )\
-            {\
     oshmem_proc_group_destroy(group);\
-            }\
+    RUNTIME_CHECK_RC(rc);\
-        } /* OSHMEM_GROUP_CACHE_ENABLED */\
-    }\
 }
 
-#define SHMEM_ALLTOALLS(F_NAME, T_NAME, OSHMEM_GROUP_CACHE_ENABLED) void F_NAME(FORTRAN_POINTER_T target, \
+#define SHMEM_ALLTOALLS(F_NAME, T_NAME) void F_NAME(FORTRAN_POINTER_T target, \
     FORTRAN_POINTER_T source, \
     MPI_Fint *dst,\
     MPI_Fint *sst,\
@@ -131,38 +97,12 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
     MPI_Fint *PE_size, \
     FORTRAN_POINTER_T pSync)\
 {\
-    int rc = OSHMEM_SUCCESS;\
+    int rc;\
-    oshmem_group_t*  group = NULL;\
+    oshmem_group_t *group;\
-    {\
     /* Create group basing PE_start, logPE_stride and PE_size */\
-        if (OSHMEM_GROUP_CACHE_ENABLED == 0)\
+    group = oshmem_proc_group_create_nofail(OMPI_FINT_2_INT(*PE_start), \
-        {\
-            group = oshmem_proc_group_create(OMPI_FINT_2_INT(*PE_start), \
                                             (1 << OMPI_FINT_2_INT(*logPE_stride)), \
                                             OMPI_FINT_2_INT(*PE_size));\
-            if (!group)\
-                rc = OSHMEM_ERROR;\
-        }\
-        else\
-        {\
-            group = find_group_in_cache(OMPI_FINT_2_INT(*PE_start),\
-                OMPI_FINT_2_INT(*logPE_stride),\
-                OMPI_FINT_2_INT(*PE_size));\
-            if (!group)\
-            {\
-                group = oshmem_proc_group_create(OMPI_FINT_2_INT(*PE_start), \
-                    (1 << OMPI_FINT_2_INT(*logPE_stride)), \
-                    OMPI_FINT_2_INT(*PE_size));\
-                if (!group)\
-                    rc = OSHMEM_ERROR;\
-                cache_group(group,OMPI_FINT_2_INT(*PE_start),\
-                    OMPI_FINT_2_INT(*logPE_stride),\
-                    OMPI_FINT_2_INT(*PE_size));\
-            }\
-        } /* OSHMEM_GROUP_CACHE_ENABLED */\
-        /* Collective operation call */\
-        if ( rc == OSHMEM_SUCCESS )\
-        {\
     oshmem_op_t* op = T_NAME;\
     \
     /* Call collective broadcast operation */\
@@ -174,18 +114,11 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
             OMPI_FINT_2_INT(*nlong), \
             op->dt_size, \
             FPTR_2_VOID_PTR(pSync), SCOLL_DEFAULT_ALG );\
-        }\
-        if (OSHMEM_GROUP_CACHE_ENABLED == 0) \
-        {\
-            if ( group )\
-            {\
     oshmem_proc_group_destroy(group);\
-            }\
+    RUNTIME_CHECK_RC(rc);\
-        } /* OSHMEM_GROUP_CACHE_ENABLED */\
-    }\
 }
 
-SHMEM_ALLTOALL(shmem_alltoall32_f, oshmem_op_prod_fint4, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_ALLTOALL(shmem_alltoall32_f, oshmem_op_prod_fint4)
-SHMEM_ALLTOALL(shmem_alltoall64_f, oshmem_op_prod_fint8, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_ALLTOALL(shmem_alltoall64_f, oshmem_op_prod_fint8)
-SHMEM_ALLTOALLS(shmem_alltoalls32_f, oshmem_op_prod_fint4, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_ALLTOALLS(shmem_alltoalls32_f, oshmem_op_prod_fint4)
-SHMEM_ALLTOALLS(shmem_alltoalls64_f, oshmem_op_prod_fint8, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_ALLTOALLS(shmem_alltoalls64_f, oshmem_op_prod_fint8)

oshmem/shmem/fortran/shmem_and_to_all_f.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013      Mellanox Technologies, Inc.
+ * Copyright (c) 2013-2018 Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2013 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
@@ -15,7 +15,6 @@
 #include "oshmem/constants.h"
 #include "oshmem/mca/scoll/scoll.h"
 #include "oshmem/proc/proc.h"
-#include "oshmem/proc/proc_group_cache.h"
 #include "oshmem/op/op.h"
 
 #if OSHMEM_PROFILING
@@ -50,7 +49,7 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
         (FORTRAN_POINTER_T target, FORTRAN_POINTER_T source, MPI_Fint *nreduce, MPI_Fint *PE_start, MPI_Fint * logPE_stride, MPI_Fint *PE_size, FORTRAN_POINTER_T *pWrk, FORTRAN_POINTER_T pSync),
         (target,source,nreduce,PE_start,logPE_stride,PE_size,pWrk,pSync) )
 
-#define SHMEM_AND_TO_ALL(F_NAME, T_NAME, OSHMEM_GROUP_CACHE_ENABLED) void F_NAME(FORTRAN_POINTER_T target, \
+#define SHMEM_AND_TO_ALL(F_NAME, T_NAME) void F_NAME(FORTRAN_POINTER_T target, \
     FORTRAN_POINTER_T source, \
     MPI_Fint *nreduce, \
     MPI_Fint *PE_start, \
@@ -59,39 +58,12 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
     FORTRAN_POINTER_T *pWrk, \
     FORTRAN_POINTER_T pSync)\
 {\
-    int rc = OSHMEM_SUCCESS;\
+    int rc;\
-    oshmem_group_t*  group = NULL;\
+    oshmem_group_t *group;\
-    {\
     /* Create group basing PE_start, logPE_stride and PE_size */\
-        if (0 == OSHMEM_GROUP_CACHE_ENABLED)\
+    group = oshmem_proc_group_create_nofail(OMPI_FINT_2_INT(*PE_start), \
-        {\
-            group = oshmem_proc_group_create(OMPI_FINT_2_INT(*PE_start), \
                                             (1 << OMPI_FINT_2_INT(*logPE_stride)), \
                                             OMPI_FINT_2_INT(*PE_size));\
-            if (!group) \
-                rc = OSHMEM_ERROR;\
-        }\
-        else \
-        {\
-            group = find_group_in_cache(OMPI_FINT_2_INT(*PE_start),\
-                OMPI_FINT_2_INT(*logPE_stride),\
-                OMPI_FINT_2_INT(*PE_size));\
-            if (!group)\
-            {\
-                group = oshmem_proc_group_create(OMPI_FINT_2_INT(*PE_start), \
-                    (1 << OMPI_FINT_2_INT(*logPE_stride)), \
-                    OMPI_FINT_2_INT(*PE_size));\
-                if (!group)\
-                    rc = OSHMEM_ERROR;\
-                cache_group(group,\
-                    OMPI_FINT_2_INT(*PE_start),\
-                    OMPI_FINT_2_INT(*logPE_stride),\
-                    OMPI_FINT_2_INT(*PE_size));\
-            }\
-        }\
-        /* Collective operation call */\
-        if ( rc == OSHMEM_SUCCESS )\
-        {\
     oshmem_op_t* op = T_NAME;\
     size_t size = OMPI_FINT_2_INT(*nreduce) * op->dt_size;\
     \
@@ -103,17 +75,10 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
             size, \
             FPTR_2_VOID_PTR(pSync), \
             FPTR_2_VOID_PTR(*pWrk), SCOLL_DEFAULT_ALG);\
-        }\
-        if (0 == OSHMEM_GROUP_CACHE_ENABLED)\
-        {\
-            if ( rc == OSHMEM_SUCCESS )\
-            {\
     oshmem_proc_group_destroy(group);\
-            }\
+    RUNTIME_CHECK_RC(rc); \
-        }\
-    }\
 }
 
-SHMEM_AND_TO_ALL(shmem_int2_and_to_all_f, oshmem_op_and_fint2, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_AND_TO_ALL(shmem_int2_and_to_all_f, oshmem_op_and_fint2)
-SHMEM_AND_TO_ALL(shmem_int4_and_to_all_f, oshmem_op_and_fint4, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_AND_TO_ALL(shmem_int4_and_to_all_f, oshmem_op_and_fint4)
-SHMEM_AND_TO_ALL(shmem_int8_and_to_all_f, oshmem_op_and_fint8, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_AND_TO_ALL(shmem_int8_and_to_all_f, oshmem_op_and_fint8)

oshmem/shmem/fortran/shmem_broadcast_f.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013      Mellanox Technologies, Inc.
+ * Copyright (c) 2013-2018 Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2013 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
@@ -15,7 +15,6 @@
 #include "oshmem/constants.h"
 #include "oshmem/mca/scoll/scoll.h"
 #include "oshmem/proc/proc.h"
-#include "oshmem/proc/proc_group_cache.h"
 #include "oshmem/op/op.h"
 
 #if OSHMEM_PROFILING
@@ -59,7 +58,7 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
         (FORTRAN_POINTER_T target, FORTRAN_POINTER_T source, MPI_Fint *nlong, MPI_Fint *PE_root, MPI_Fint *PE_start, MPI_Fint * logPE_stride, MPI_Fint *PE_size, FORTRAN_POINTER_T pSync),
         (target, source, nlong, PE_root, PE_start, logPE_stride, PE_size, pSync))
 
-#define SHMEM_BROADCAST(F_NAME, T_NAME, OSHMEM_GROUP_CACHE_ENABLED) void F_NAME(FORTRAN_POINTER_T target, \
+#define SHMEM_BROADCAST(F_NAME, T_NAME) void F_NAME(FORTRAN_POINTER_T target, \
     FORTRAN_POINTER_T source, \
     MPI_Fint *nlong,\
     MPI_Fint *PE_root, \
@@ -68,47 +67,22 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
     MPI_Fint *PE_size, \
     FORTRAN_POINTER_T pSync)\
 {\
-    int rc = OSHMEM_SUCCESS;\
+    int rc;\
-    oshmem_group_t*  group = NULL;\
+    oshmem_group_t *group;\
+    int rel_PE_root = 0;\
+    oshmem_op_t* op = T_NAME;\
 \
     if ((0 <= OMPI_FINT_2_INT(*PE_root)) && \
             (OMPI_FINT_2_INT(*PE_root) < OMPI_FINT_2_INT(*PE_size)))\
     {\
-        /* Create group basing PE_start, logPE_stride and PE_size */\
+        group = oshmem_proc_group_create_nofail(OMPI_FINT_2_INT(*PE_start), \
-        if (OSHMEM_GROUP_CACHE_ENABLED == 0)\
-        {\
-            group = oshmem_proc_group_create(OMPI_FINT_2_INT(*PE_start), \
                 (1 << OMPI_FINT_2_INT(*logPE_stride)), \
                 OMPI_FINT_2_INT(*PE_size));\
-            if (!group || (OMPI_FINT_2_INT(*PE_root) >= group->proc_count))\
+        if (OMPI_FINT_2_INT(*PE_root) >= group->proc_count)\
         {\
             rc = OSHMEM_ERROR;\
+            goto out;\
         }\
-        }\
-        else\
-        {\
-             group = find_group_in_cache(OMPI_FINT_2_INT(*PE_start),\
-                OMPI_FINT_2_INT(*logPE_stride),\
-                OMPI_FINT_2_INT(*PE_size));\
-            if (!group)\
-            {\
-                group = oshmem_proc_group_create(OMPI_FINT_2_INT(*PE_start),\
-                    (1 << OMPI_FINT_2_INT(*logPE_stride)),\
-                    OMPI_FINT_2_INT(*PE_size));\
-                if (!group || (OMPI_FINT_2_INT(*PE_root) >= group->proc_count))\
-                {\
-                    rc = OSHMEM_ERROR;\
-                }\
-                cache_group(group,OMPI_FINT_2_INT(*PE_start),\
-                    OMPI_FINT_2_INT(*logPE_stride),\
-                    OMPI_FINT_2_INT(*PE_size));\
-            }\
-        } /* OSHMEM_GROUP_CACHE_ENABLED */\
-        /* Collective operation call */\
-        if ( rc == OSHMEM_SUCCESS )\
-        {\
-            int rel_PE_root = 0;\
-            oshmem_op_t* op = T_NAME;\
         \
         /* Define actual PE using relative in active set */\
         rel_PE_root = oshmem_proc_pe(group->proc_array[OMPI_FINT_2_INT(*PE_root)]);\
@@ -120,18 +94,13 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
                 FPTR_2_VOID_PTR(source), \
                 OMPI_FINT_2_INT(*nlong) * op->dt_size, \
                 FPTR_2_VOID_PTR(pSync), SCOLL_DEFAULT_ALG );\
-        }\
+    out: \
-        if (OSHMEM_GROUP_CACHE_ENABLED == 0) \
-        {\
-            if ( group )\
-            {\
         oshmem_proc_group_destroy(group);\
-            }\
+        RUNTIME_CHECK_RC(rc); \
-        } /* OSHMEM_GROUP_CACHE_ENABLED */\
   }\
 }
 
-SHMEM_BROADCAST(shmem_broadcast4_f, oshmem_op_prod_fint4, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_BROADCAST(shmem_broadcast4_f, oshmem_op_prod_fint4)
-SHMEM_BROADCAST(shmem_broadcast8_f, oshmem_op_prod_fint8, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_BROADCAST(shmem_broadcast8_f, oshmem_op_prod_fint8)
-SHMEM_BROADCAST(shmem_broadcast32_f, oshmem_op_prod_fint4, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_BROADCAST(shmem_broadcast32_f, oshmem_op_prod_fint4)
-SHMEM_BROADCAST(shmem_broadcast64_f, oshmem_op_prod_fint8, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_BROADCAST(shmem_broadcast64_f, oshmem_op_prod_fint8)

oshmem/shmem/fortran/shmem_collect_f.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013      Mellanox Technologies, Inc.
+ * Copyright (c) 2013-2018 Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2013 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
@@ -15,7 +15,6 @@
 #include "oshmem/constants.h"
 #include "oshmem/mca/scoll/scoll.h"
 #include "oshmem/proc/proc.h"
-#include "oshmem/proc/proc_group_cache.h"
 #include "oshmem/op/op.h"
 
 #if OSHMEM_PROFILING
@@ -95,7 +94,7 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
         (FORTRAN_POINTER_T target, FORTRAN_POINTER_T source, MPI_Fint *nlong, MPI_Fint *PE_start, MPI_Fint * logPE_stride, MPI_Fint *PE_size, FORTRAN_POINTER_T pSync),
         (target,source,nlong,PE_start,logPE_stride,PE_size,pSync) )
 
-#define SHMEM_COLLECT(F_NAME, T_NAME, OSHMEM_GROUP_CACHE_ENABLED) void F_NAME(FORTRAN_POINTER_T target, \
+#define SHMEM_COLLECT(F_NAME, T_NAME) void F_NAME(FORTRAN_POINTER_T target, \
     FORTRAN_POINTER_T source, \
     MPI_Fint *nlong, \
     MPI_Fint *PE_start, \
@@ -103,38 +102,12 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
     MPI_Fint *PE_size, \
     FORTRAN_POINTER_T pSync)\
 {\
-    int rc = OSHMEM_SUCCESS;\
+    int rc;\
-    oshmem_group_t*  group = NULL;\
+    oshmem_group_t *group;\
-    {\
     /* Create group basing PE_start, logPE_stride and PE_size */\
-        if (OSHMEM_GROUP_CACHE_ENABLED == 0)\
+    group = oshmem_proc_group_create_nofail(OMPI_FINT_2_INT(*PE_start), \
-        {\
-            group = oshmem_proc_group_create(OMPI_FINT_2_INT(*PE_start), \
                                             (1 << OMPI_FINT_2_INT(*logPE_stride)), \
                                             OMPI_FINT_2_INT(*PE_size));\
-            if (!group)\
-                rc = OSHMEM_ERROR;\
-        }\
-        else\
-        {\
-            group = find_group_in_cache(OMPI_FINT_2_INT(*PE_start),\
-                OMPI_FINT_2_INT(*logPE_stride),\
-                OMPI_FINT_2_INT(*PE_size));\
-            if (!group)\
-            {\
-                group = oshmem_proc_group_create(OMPI_FINT_2_INT(*PE_start), \
-                    (1 << OMPI_FINT_2_INT(*logPE_stride)), \
-                    OMPI_FINT_2_INT(*PE_size));\
-                if (!group)\
-                    rc = OSHMEM_ERROR;\
-                cache_group(group,OMPI_FINT_2_INT(*PE_start),\
-                    OMPI_FINT_2_INT(*logPE_stride),\
-                    OMPI_FINT_2_INT(*PE_size));\
-            }\
-        } /* OSHMEM_GROUP_CACHE_ENABLED */\
-        /* Collective operation call */\
-        if ( rc == OSHMEM_SUCCESS )\
-        {\
     oshmem_op_t* op = T_NAME;\
     /* Call collective broadcast operation */\
     rc = group->g_scoll.scoll_collect( group, \
@@ -143,22 +116,15 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
             OMPI_FINT_2_INT(*nlong) * op->dt_size, \
             FPTR_2_VOID_PTR(pSync), \
             false, SCOLL_DEFAULT_ALG);\
-        }\
-        if (OSHMEM_GROUP_CACHE_ENABLED == 0)\
-        {\
-            if ( rc == OSHMEM_SUCCESS )\
-            {\
     oshmem_proc_group_destroy(group);\
-            }\
+    RUNTIME_CHECK_RC(rc);\
-        }/* OSHMEM_GROUP_CACHE_ENABLED */\
-    }\
 }
 
-SHMEM_COLLECT(shmem_collect4_f, oshmem_op_prod_fint4, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_COLLECT(shmem_collect4_f, oshmem_op_prod_fint4)
-SHMEM_COLLECT(shmem_collect8_f, oshmem_op_prod_fint8, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_COLLECT(shmem_collect8_f, oshmem_op_prod_fint8)
-SHMEM_COLLECT(shmem_collect32_f, oshmem_op_prod_fint4, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_COLLECT(shmem_collect32_f, oshmem_op_prod_fint4)
-SHMEM_COLLECT(shmem_collect64_f, oshmem_op_prod_fint8, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_COLLECT(shmem_collect64_f, oshmem_op_prod_fint8)
-SHMEM_COLLECT(shmem_fcollect4_f, oshmem_op_prod_freal4, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_COLLECT(shmem_fcollect4_f, oshmem_op_prod_freal4)
-SHMEM_COLLECT(shmem_fcollect8_f, oshmem_op_prod_freal8, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_COLLECT(shmem_fcollect8_f, oshmem_op_prod_freal8)
-SHMEM_COLLECT(shmem_fcollect32_f, oshmem_op_prod_freal4, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_COLLECT(shmem_fcollect32_f, oshmem_op_prod_freal4)
-SHMEM_COLLECT(shmem_fcollect64_f, oshmem_op_prod_freal8, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_COLLECT(shmem_fcollect64_f, oshmem_op_prod_freal8)

oshmem/shmem/fortran/shmem_max_to_all_f.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013      Mellanox Technologies, Inc.
+ * Copyright (c) 2013-2018 Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2013 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
@@ -15,7 +15,6 @@
 #include "oshmem/constants.h"
 #include "oshmem/mca/scoll/scoll.h"
 #include "oshmem/proc/proc.h"
-#include "oshmem/proc/proc_group_cache.h"
 #include "oshmem/op/op.h"
 
 #if OSHMEM_PROFILING
@@ -77,7 +76,7 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
         (FORTRAN_POINTER_T target, FORTRAN_POINTER_T source, MPI_Fint *nreduce, MPI_Fint *PE_start, MPI_Fint * logPE_stride, MPI_Fint *PE_size, FORTRAN_POINTER_T *pWrk, FORTRAN_POINTER_T pSync),
         (target,source,nreduce,PE_start,logPE_stride,PE_size,pWrk,pSync) )
 
-#define SHMEM_MAX_TO_ALL(F_NAME, T_NAME, OSHMEM_GROUP_CACHE_ENABLED) void F_NAME(FORTRAN_POINTER_T target, \
+#define SHMEM_MAX_TO_ALL(F_NAME, T_NAME) void F_NAME(FORTRAN_POINTER_T target, \
     FORTRAN_POINTER_T source, \
     MPI_Fint *nreduce,\
     MPI_Fint *PE_start,\
@@ -86,37 +85,12 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
     FORTRAN_POINTER_T *pWrk,\
     FORTRAN_POINTER_T pSync)\
 {\
-    int rc = OSHMEM_SUCCESS;\
+    int rc;\
-    oshmem_group_t*  group = NULL;\
+    oshmem_group_t *group;\
-    {\
     /* Create group basing PE_start, logPE_stride and PE_size */\
-        if (OSHMEM_GROUP_CACHE_ENABLED == 0) {\
+    group = oshmem_proc_group_create_nofail(OMPI_FINT_2_INT(*PE_start),\
-            group = oshmem_proc_group_create(OMPI_FINT_2_INT(*PE_start),\
                                             (1 << OMPI_FINT_2_INT(*logPE_stride)),\
                                             OMPI_FINT_2_INT(*PE_size));\
-            if (!group)\
-                rc = OSHMEM_ERROR;\
-            }\
-        else\
-        {\
-            group = find_group_in_cache(OMPI_FINT_2_INT(*PE_start),\
-                OMPI_FINT_2_INT(*logPE_stride),\
-                OMPI_FINT_2_INT(*PE_size));\
-            if (!group)\
-            {\
-                group = oshmem_proc_group_create(OMPI_FINT_2_INT(*PE_start),\
-                    (1 << OMPI_FINT_2_INT(*logPE_stride)),\
-                    OMPI_FINT_2_INT(*PE_size));\
-                if (!group)\
-                    rc = OSHMEM_ERROR;\
-                cache_group(group,OMPI_FINT_2_INT(*PE_start),\
-                    OMPI_FINT_2_INT(*logPE_stride),\
-                    OMPI_FINT_2_INT(*PE_size));\
-            }\
-        }\
-        /* Collective operation call */\
-        if ( rc == OSHMEM_SUCCESS )\
-        {\
     oshmem_op_t* op = T_NAME;\
     size_t size = OMPI_FINT_2_INT(*nreduce) * op->dt_size;\
     /* Call collective reduce operation */\
@@ -127,20 +101,13 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
             size,\
             FPTR_2_VOID_PTR(pSync),\
             FPTR_2_VOID_PTR(*pWrk), SCOLL_DEFAULT_ALG);\
-        }\
-        if (OSHMEM_GROUP_CACHE_ENABLED == 0)\
-        {\
-            if ( rc == OSHMEM_SUCCESS )\
-            {\
     oshmem_proc_group_destroy(group);\
-            }\
+    RUNTIME_CHECK_RC(rc);\
-        }\
-    }\
 }
 
-SHMEM_MAX_TO_ALL(shmem_int2_max_to_all_f, oshmem_op_max_fint2, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_MAX_TO_ALL(shmem_int2_max_to_all_f, oshmem_op_max_fint2)
-SHMEM_MAX_TO_ALL(shmem_int4_max_to_all_f, oshmem_op_max_fint4, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_MAX_TO_ALL(shmem_int4_max_to_all_f, oshmem_op_max_fint4)
-SHMEM_MAX_TO_ALL(shmem_int8_max_to_all_f, oshmem_op_max_fint8, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_MAX_TO_ALL(shmem_int8_max_to_all_f, oshmem_op_max_fint8)
-SHMEM_MAX_TO_ALL(shmem_real4_max_to_all_f, oshmem_op_max_freal4, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_MAX_TO_ALL(shmem_real4_max_to_all_f, oshmem_op_max_freal4)
-SHMEM_MAX_TO_ALL(shmem_real8_max_to_all_f, oshmem_op_max_freal8, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_MAX_TO_ALL(shmem_real8_max_to_all_f, oshmem_op_max_freal8)
-SHMEM_MAX_TO_ALL(shmem_real16_max_to_all_f, oshmem_op_max_freal16, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_MAX_TO_ALL(shmem_real16_max_to_all_f, oshmem_op_max_freal16)

oshmem/shmem/fortran/shmem_min_to_all_f.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013      Mellanox Technologies, Inc.
+ * Copyright (c) 2013-2018 Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2013 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
@@ -15,7 +15,6 @@
 #include "oshmem/constants.h"
 #include "oshmem/mca/scoll/scoll.h"
 #include "oshmem/proc/proc.h"
-#include "oshmem/proc/proc_group_cache.h"
 #include "oshmem/op/op.h"
 
 #if OSHMEM_PROFILING
@@ -78,7 +77,7 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
         (target,source,nreduce,PE_start,logPE_stride,PE_size,pWrk,pSync) )
 
 
-#define SHMEM_MIN_TO_ALL(F_NAME, T_NAME, OSHMEM_GROUP_CACHE_ENABLED) void F_NAME(FORTRAN_POINTER_T target, \
+#define SHMEM_MIN_TO_ALL(F_NAME, T_NAME) void F_NAME(FORTRAN_POINTER_T target, \
     FORTRAN_POINTER_T source, \
     MPI_Fint *nreduce,\
     MPI_Fint *PE_start,\
@@ -87,37 +86,13 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
     FORTRAN_POINTER_T *pWrk,\
     FORTRAN_POINTER_T pSync)\
 {\
-    int rc = OSHMEM_SUCCESS;\
+    int rc;\
-    oshmem_group_t*  group = NULL;\
+    oshmem_group_t *group;\
-    {\
     /* Create group basing PE_start, logPE_stride and PE_size */\
-        if (OSHMEM_GROUP_CACHE_ENABLED == 0) {\
+    group = oshmem_proc_group_create_nofail(OMPI_FINT_2_INT(*PE_start),\
-            group = oshmem_proc_group_create(OMPI_FINT_2_INT(*PE_start),\
                                             (1 << OMPI_FINT_2_INT(*logPE_stride)),\
                                             OMPI_FINT_2_INT(*PE_size));\
-            if (!group)\
-                rc = OSHMEM_ERROR;\
-            }\
-        else\
-        {\
-            group = find_group_in_cache(OMPI_FINT_2_INT(*PE_start),\
-                OMPI_FINT_2_INT(*logPE_stride),\
-                OMPI_FINT_2_INT(*PE_size));\
-            if (!group)\
-            {\
-                group = oshmem_proc_group_create(OMPI_FINT_2_INT(*PE_start),\
-                    (1 << OMPI_FINT_2_INT(*logPE_stride)),\
-                    OMPI_FINT_2_INT(*PE_size));\
-                if (!group)\
-                    rc = OSHMEM_ERROR;\
-                cache_group(group,OMPI_FINT_2_INT(*PE_start),\
-                    OMPI_FINT_2_INT(*logPE_stride),\
-                    OMPI_FINT_2_INT(*PE_size));\
-            }\
-        }\
     /* Collective operation call */\
-        if ( rc == OSHMEM_SUCCESS )\
-        {\
     oshmem_op_t* op = T_NAME;\
     size_t size = OMPI_FINT_2_INT(*nreduce) * op->dt_size;\
     /* Call collective reduce operation */\
@@ -128,20 +103,13 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
             size,\
             FPTR_2_VOID_PTR(pSync),\
             FPTR_2_VOID_PTR(*pWrk), SCOLL_DEFAULT_ALG);\
-        }\
-        if (OSHMEM_GROUP_CACHE_ENABLED == 0)\
-        {\
-            if ( rc == OSHMEM_SUCCESS )\
-            {\
     oshmem_proc_group_destroy(group);\
-            }\
+    RUNTIME_CHECK_RC(rc);\
-        }\
-    }\
 }
 
-SHMEM_MIN_TO_ALL(shmem_int2_min_to_all_f, oshmem_op_min_fint2, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_MIN_TO_ALL(shmem_int2_min_to_all_f, oshmem_op_min_fint2)
-SHMEM_MIN_TO_ALL(shmem_int4_min_to_all_f, oshmem_op_min_fint4, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_MIN_TO_ALL(shmem_int4_min_to_all_f, oshmem_op_min_fint4)
-SHMEM_MIN_TO_ALL(shmem_int8_min_to_all_f, oshmem_op_min_fint8, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_MIN_TO_ALL(shmem_int8_min_to_all_f, oshmem_op_min_fint8)
-SHMEM_MIN_TO_ALL(shmem_real4_min_to_all_f, oshmem_op_min_freal4, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_MIN_TO_ALL(shmem_real4_min_to_all_f, oshmem_op_min_freal4)
-SHMEM_MIN_TO_ALL(shmem_real8_min_to_all_f, oshmem_op_min_freal8, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_MIN_TO_ALL(shmem_real8_min_to_all_f, oshmem_op_min_freal8)
-SHMEM_MIN_TO_ALL(shmem_real16_min_to_all_f, oshmem_op_min_freal16, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_MIN_TO_ALL(shmem_real16_min_to_all_f, oshmem_op_min_freal16)

oshmem/shmem/fortran/shmem_or_to_all_f.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013      Mellanox Technologies, Inc.
+ * Copyright (c) 2013-2018 Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2013 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
@@ -15,7 +15,6 @@
 #include "oshmem/constants.h"
 #include "oshmem/mca/scoll/scoll.h"
 #include "oshmem/proc/proc.h"
-#include "oshmem/proc/proc_group_cache.h"
 #include "oshmem/op/op.h"
 
 #if OSHMEM_PROFILING
@@ -50,7 +49,7 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
         (FORTRAN_POINTER_T target, FORTRAN_POINTER_T source, MPI_Fint *nreduce, MPI_Fint *PE_start, MPI_Fint * logPE_stride, MPI_Fint *PE_size, FORTRAN_POINTER_T *pWrk, FORTRAN_POINTER_T pSync),
         (target,source,nreduce,PE_start,logPE_stride,PE_size,pWrk,pSync) )
 
-#define SHMEM_OR_TO_ALL(F_NAME, T_NAME, OSHMEM_GROUP_CACHE_ENABLED) void F_NAME(FORTRAN_POINTER_T target, \
+#define SHMEM_OR_TO_ALL(F_NAME, T_NAME)  void F_NAME(FORTRAN_POINTER_T target, \
     FORTRAN_POINTER_T source, \
     MPI_Fint *nreduce,\
     MPI_Fint *PE_start,\
@@ -59,37 +58,12 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
     FORTRAN_POINTER_T *pWrk,\
     FORTRAN_POINTER_T pSync)\
 {\
-    int rc = OSHMEM_SUCCESS;\
+    int rc;\
-    oshmem_group_t*  group = NULL;\
+    oshmem_group_t *group;\
-    {\
     /* Create group basing PE_start, logPE_stride and PE_size */\
-        if (OSHMEM_GROUP_CACHE_ENABLED == 0) {\
+    group = oshmem_proc_group_create_nofail(OMPI_FINT_2_INT(*PE_start),\
-            group = oshmem_proc_group_create(OMPI_FINT_2_INT(*PE_start),\
                                             (1 << OMPI_FINT_2_INT(*logPE_stride)),\
                                             OMPI_FINT_2_INT(*PE_size));\
-            if (!group)\
-                rc = OSHMEM_ERROR;\
-            }\
-        else\
-        {\
-            group = find_group_in_cache(OMPI_FINT_2_INT(*PE_start),\
-                OMPI_FINT_2_INT(*logPE_stride),\
-                OMPI_FINT_2_INT(*PE_size));\
-            if (!group)\
-            {\
-                group = oshmem_proc_group_create(OMPI_FINT_2_INT(*PE_start),\
-                    (1 << OMPI_FINT_2_INT(*logPE_stride)),\
-                    OMPI_FINT_2_INT(*PE_size));\
-                if (!group)\
-                    rc = OSHMEM_ERROR;\
-                cache_group(group,OMPI_FINT_2_INT(*PE_start),\
-                    OMPI_FINT_2_INT(*logPE_stride),\
-                    OMPI_FINT_2_INT(*PE_size));\
-            }\
-        }\
-        /* Collective operation call */\
-        if ( rc == OSHMEM_SUCCESS )\
-        {\
     oshmem_op_t* op = T_NAME;\
     size_t size = OMPI_FINT_2_INT(*nreduce) * op->dt_size;\
     /* Call collective reduce operation */\
@@ -100,17 +74,10 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
             size,\
             FPTR_2_VOID_PTR(pSync),\
             FPTR_2_VOID_PTR(*pWrk), SCOLL_DEFAULT_ALG);\
-        }\
-        if (OSHMEM_GROUP_CACHE_ENABLED == 0)\
-        {\
-            if ( rc == OSHMEM_SUCCESS )\
-            {\
     oshmem_proc_group_destroy(group);\
-            }\
+    RUNTIME_CHECK_RC(rc); \
-        }\
-    }\
 }
 
-SHMEM_OR_TO_ALL(shmem_int2_or_to_all_f, oshmem_op_or_fint2, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_OR_TO_ALL(shmem_int2_or_to_all_f, oshmem_op_or_fint2)
-SHMEM_OR_TO_ALL(shmem_int4_or_to_all_f, oshmem_op_or_fint4, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_OR_TO_ALL(shmem_int4_or_to_all_f, oshmem_op_or_fint4)
-SHMEM_OR_TO_ALL(shmem_int8_or_to_all_f, oshmem_op_or_fint8, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_OR_TO_ALL(shmem_int8_or_to_all_f, oshmem_op_or_fint8)

oshmem/shmem/fortran/shmem_prod_to_all_f.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013      Mellanox Technologies, Inc.
+ * Copyright (c) 2013-2018 Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2013 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
@@ -15,7 +15,6 @@
 #include "oshmem/constants.h"
 #include "oshmem/mca/scoll/scoll.h"
 #include "oshmem/proc/proc.h"
-#include "oshmem/proc/proc_group_cache.h"
 #include "oshmem/op/op.h"
 
 #if OSHMEM_PROFILING
@@ -96,7 +95,7 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
         (FORTRAN_POINTER_T target, FORTRAN_POINTER_T source, MPI_Fint *nreduce, MPI_Fint *PE_start, MPI_Fint * logPE_stride, MPI_Fint *PE_size, FORTRAN_POINTER_T *pWrk, FORTRAN_POINTER_T pSync),
         (target,source,nreduce,PE_start,logPE_stride,PE_size,pWrk,pSync) )
 
-#define SHMEM_PROD_TO_ALL(F_NAME, T_NAME, OSHMEM_GROUP_CACHE_ENABLED) void F_NAME(FORTRAN_POINTER_T target, \
+#define SHMEM_PROD_TO_ALL(F_NAME, T_NAME) void F_NAME(FORTRAN_POINTER_T target, \
     FORTRAN_POINTER_T source, \
     MPI_Fint *nreduce,\
     MPI_Fint *PE_start,\
@@ -105,37 +104,12 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
     FORTRAN_POINTER_T *pWrk,\
     FORTRAN_POINTER_T pSync)\
 {\
-    int rc = OSHMEM_SUCCESS;\
+    int rc;\
-    oshmem_group_t*  group = NULL;\
+    oshmem_group_t *group;\
-    {\
     /* Create group basing PE_start, logPE_stride and PE_size */\
-        if (OSHMEM_GROUP_CACHE_ENABLED == 0) {\
+    group = oshmem_proc_group_create_nofail(OMPI_FINT_2_INT(*PE_start),\
-            group = oshmem_proc_group_create(OMPI_FINT_2_INT(*PE_start),\
                                             (1 << OMPI_FINT_2_INT(*logPE_stride)),\
                                             OMPI_FINT_2_INT(*PE_size));\
-            if (!group)\
-                rc = OSHMEM_ERROR;\
-            }\
-        else\
-        {\
-            group = find_group_in_cache(OMPI_FINT_2_INT(*PE_start),\
-                OMPI_FINT_2_INT(*logPE_stride),\
-                OMPI_FINT_2_INT(*PE_size));\
-            if (!group)\
-            {\
-                group = oshmem_proc_group_create(OMPI_FINT_2_INT(*PE_start),\
-                    (1 << OMPI_FINT_2_INT(*logPE_stride)),\
-                    OMPI_FINT_2_INT(*PE_size));\
-                if (!group)\
-                    rc = OSHMEM_ERROR;\
-                cache_group(group,OMPI_FINT_2_INT(*PE_start),\
-                    OMPI_FINT_2_INT(*logPE_stride),\
-                    OMPI_FINT_2_INT(*PE_size));\
-            }\
-        }\
-        /* Collective operation call */\
-        if ( rc == OSHMEM_SUCCESS )\
-        {\
     oshmem_op_t* op = T_NAME;\
     size_t size = OMPI_FINT_2_INT(*nreduce) * op->dt_size;\
     /* Call collective reduce operation */\
@@ -146,22 +120,15 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
             size,\
             FPTR_2_VOID_PTR(pSync),\
             FPTR_2_VOID_PTR(*pWrk), SCOLL_DEFAULT_ALG);\
-        }\
-        if (OSHMEM_GROUP_CACHE_ENABLED == 0)\
-        {\
-            if ( rc == OSHMEM_SUCCESS )\
-            {\
     oshmem_proc_group_destroy(group);\
-            }\
+    RUNTIME_CHECK_RC(rc);\
-        }\
-    }\
 }
 
-SHMEM_PROD_TO_ALL(shmem_int2_prod_to_all_f, oshmem_op_prod_fint2, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_PROD_TO_ALL(shmem_int2_prod_to_all_f, oshmem_op_prod_fint2)
-SHMEM_PROD_TO_ALL(shmem_int4_prod_to_all_f, oshmem_op_prod_fint4, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_PROD_TO_ALL(shmem_int4_prod_to_all_f, oshmem_op_prod_fint4)
-SHMEM_PROD_TO_ALL(shmem_int8_prod_to_all_f, oshmem_op_prod_fint8, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_PROD_TO_ALL(shmem_int8_prod_to_all_f, oshmem_op_prod_fint8)
-SHMEM_PROD_TO_ALL(shmem_comp4_prod_to_all_f, oshmem_op_prod_complexf, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_PROD_TO_ALL(shmem_comp4_prod_to_all_f, oshmem_op_prod_complexf)
-SHMEM_PROD_TO_ALL(shmem_comp8_prod_to_all_f, oshmem_op_prod_complexd, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_PROD_TO_ALL(shmem_comp8_prod_to_all_f, oshmem_op_prod_complexd)
-SHMEM_PROD_TO_ALL(shmem_real4_prod_to_all_f, oshmem_op_prod_freal4, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_PROD_TO_ALL(shmem_real4_prod_to_all_f, oshmem_op_prod_freal4)
-SHMEM_PROD_TO_ALL(shmem_real8_prod_to_all_f, oshmem_op_prod_freal8, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_PROD_TO_ALL(shmem_real8_prod_to_all_f, oshmem_op_prod_freal8)
-SHMEM_PROD_TO_ALL(shmem_real16_prod_to_all_f, oshmem_op_prod_freal16, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_PROD_TO_ALL(shmem_real16_prod_to_all_f, oshmem_op_prod_freal16)

oshmem/shmem/fortran/shmem_sum_to_all_f.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013      Mellanox Technologies, Inc.
+ * Copyright (c) 2013-2018 Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2013 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
@@ -15,7 +15,6 @@
 #include "oshmem/constants.h"
 #include "oshmem/mca/scoll/scoll.h"
 #include "oshmem/proc/proc.h"
-#include "oshmem/proc/proc_group_cache.h"
 #include "oshmem/op/op.h"
 
 #if OSHMEM_PROFILING
@@ -95,7 +94,7 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
         (FORTRAN_POINTER_T target, FORTRAN_POINTER_T source, MPI_Fint *nreduce, MPI_Fint *PE_start, MPI_Fint * logPE_stride, MPI_Fint *PE_size, FORTRAN_POINTER_T *pWrk, FORTRAN_POINTER_T pSync),
         (target,source,nreduce,PE_start,logPE_stride,PE_size,pWrk,pSync) )
 
-#define SHMEM_SUM_TO_ALL(F_NAME, T_NAME, OSHMEM_GROUP_CACHE_ENABLED) void F_NAME(FORTRAN_POINTER_T target, \
+#define SHMEM_SUM_TO_ALL(F_NAME, T_NAME) void F_NAME(FORTRAN_POINTER_T target, \
     FORTRAN_POINTER_T source, \
     MPI_Fint *nreduce,\
     MPI_Fint *PE_start,\
@@ -104,37 +103,12 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
     FORTRAN_POINTER_T *pWrk,\
     FORTRAN_POINTER_T pSync)\
 {\
-    int rc = OSHMEM_SUCCESS;\
+    int rc;\
-    oshmem_group_t*  group = NULL;\
+    oshmem_group_t *group;\
-    {\
     /* Create group basing PE_start, logPE_stride and PE_size */\
-        if (OSHMEM_GROUP_CACHE_ENABLED == 0) {\
+    group = oshmem_proc_group_create_nofail(OMPI_FINT_2_INT(*PE_start),\
-            group = oshmem_proc_group_create(OMPI_FINT_2_INT(*PE_start),\
                                             (1 << OMPI_FINT_2_INT(*logPE_stride)),\
                                             OMPI_FINT_2_INT(*PE_size));\
-            if (!group)\
-                rc = OSHMEM_ERROR;\
-            }\
-        else\
-        {\
-            group = find_group_in_cache(OMPI_FINT_2_INT(*PE_start),\
-                OMPI_FINT_2_INT(*logPE_stride),\
-                OMPI_FINT_2_INT(*PE_size));\
-            if (!group)\
-            {\
-                group = oshmem_proc_group_create(OMPI_FINT_2_INT(*PE_start),\
-                    (1 << OMPI_FINT_2_INT(*logPE_stride)),\
-                    OMPI_FINT_2_INT(*PE_size));\
-                if (!group)\
-                    rc = OSHMEM_ERROR;\
-                cache_group(group,OMPI_FINT_2_INT(*PE_start),\
-                    OMPI_FINT_2_INT(*logPE_stride),\
-                    OMPI_FINT_2_INT(*PE_size));\
-            }\
-        }\
-        /* Collective operation call */\
-        if ( rc == OSHMEM_SUCCESS )\
-        {\
     oshmem_op_t* op = T_NAME;\
     size_t size = OMPI_FINT_2_INT(*nreduce) * op->dt_size;\
     /* Call collective reduce operation */\
@@ -145,22 +119,15 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
             size,\
             FPTR_2_VOID_PTR(pSync),\
             FPTR_2_VOID_PTR(*pWrk), SCOLL_DEFAULT_ALG);\
-        }\
-        if (OSHMEM_GROUP_CACHE_ENABLED == 0)\
-        {\
-            if ( rc == OSHMEM_SUCCESS )\
-            {\
     oshmem_proc_group_destroy(group);\
-            }\
+    RUNTIME_CHECK_RC(rc);\
-        }\
-    }\
 }
 
-SHMEM_SUM_TO_ALL(shmem_int2_sum_to_all_f, oshmem_op_sum_fint2, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_SUM_TO_ALL(shmem_int2_sum_to_all_f, oshmem_op_sum_fint2)
-SHMEM_SUM_TO_ALL(shmem_int4_sum_to_all_f, oshmem_op_sum_fint4, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_SUM_TO_ALL(shmem_int4_sum_to_all_f, oshmem_op_sum_fint4)
-SHMEM_SUM_TO_ALL(shmem_int8_sum_to_all_f, oshmem_op_sum_fint8, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_SUM_TO_ALL(shmem_int8_sum_to_all_f, oshmem_op_sum_fint8)
-SHMEM_SUM_TO_ALL(shmem_comp4_sum_to_all_f, oshmem_op_sum_complexf, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_SUM_TO_ALL(shmem_comp4_sum_to_all_f, oshmem_op_sum_complexf)
-SHMEM_SUM_TO_ALL(shmem_comp8_sum_to_all_f, oshmem_op_sum_complexd, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_SUM_TO_ALL(shmem_comp8_sum_to_all_f, oshmem_op_sum_complexd)
-SHMEM_SUM_TO_ALL(shmem_real4_sum_to_all_f, oshmem_op_sum_freal4, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_SUM_TO_ALL(shmem_real4_sum_to_all_f, oshmem_op_sum_freal4)
-SHMEM_SUM_TO_ALL(shmem_real8_sum_to_all_f, oshmem_op_sum_freal8, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_SUM_TO_ALL(shmem_real8_sum_to_all_f, oshmem_op_sum_freal8)
-SHMEM_SUM_TO_ALL(shmem_real16_sum_to_all_f, oshmem_op_sum_freal16, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_SUM_TO_ALL(shmem_real16_sum_to_all_f, oshmem_op_sum_freal16)

oshmem/shmem/fortran/shmem_xor_to_all_f.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013      Mellanox Technologies, Inc.
+ * Copyright (c) 2013-2018 Mellanox Technologies, Inc.
  *                         All rights reserved.
  * Copyright (c) 2013 Cisco Systems, Inc.  All rights reserved.
  * $COPYRIGHT$
@@ -15,7 +15,6 @@
 #include "oshmem/constants.h"
 #include "oshmem/mca/scoll/scoll.h"
 #include "oshmem/proc/proc.h"
-#include "oshmem/proc/proc_group_cache.h"
 #include "oshmem/op/op.h"
 
 #if OSHMEM_PROFILING
@@ -68,7 +67,7 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
         (FORTRAN_POINTER_T target, FORTRAN_POINTER_T source, MPI_Fint *nreduce, MPI_Fint *PE_start, MPI_Fint * logPE_stride, MPI_Fint *PE_size, FORTRAN_POINTER_T *pWrk, FORTRAN_POINTER_T pSync),
         (target,source,nreduce,PE_start,logPE_stride,PE_size,pWrk,pSync) )
 
-#define SHMEM_XOR_TO_ALL(F_NAME, T_NAME, OSHMEM_GROUP_CACHE_ENABLED) void F_NAME(FORTRAN_POINTER_T target, \
+#define SHMEM_XOR_TO_ALL(F_NAME, T_NAME) void F_NAME(FORTRAN_POINTER_T target, \
     FORTRAN_POINTER_T source, \
     MPI_Fint *nreduce,\
     MPI_Fint *PE_start,\
@@ -77,38 +76,12 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
     FORTRAN_POINTER_T *pWrk,\
     FORTRAN_POINTER_T pSync)\
 {\
-    int rc = OSHMEM_SUCCESS;\
+    int rc;\
-    oshmem_group_t*  group = NULL;\
+    oshmem_group_t *group;\
-    {\
     /* Create group basing PE_start, logPE_stride and PE_size */\
-        if (OSHMEM_GROUP_CACHE_ENABLED == 0)\
+    group = oshmem_proc_group_create_nofail(OMPI_FINT_2_INT(*PE_start), \
-        {\
-            group = oshmem_proc_group_create(OMPI_FINT_2_INT(*PE_start), \
                                             (1 << OMPI_FINT_2_INT(*logPE_stride)), \
                                             OMPI_FINT_2_INT(*PE_size));\
-            if (!group)\
-                rc = OSHMEM_ERROR;\
-        }\
-        else\
-        {\
-            group = find_group_in_cache(OMPI_FINT_2_INT(*PE_start),\
-                OMPI_FINT_2_INT(*logPE_stride),\
-                OMPI_FINT_2_INT(*PE_size));\
-            if (!group)\
-            {\
-                group = oshmem_proc_group_create(OMPI_FINT_2_INT(*PE_start), \
-                    (1 << OMPI_FINT_2_INT(*logPE_stride)), \
-                    OMPI_FINT_2_INT(*PE_size));\
-                if (!group)\
-                    rc = OSHMEM_ERROR;\
-                cache_group(group,OMPI_FINT_2_INT(*PE_start),\
-                    OMPI_FINT_2_INT(*logPE_stride),\
-                    OMPI_FINT_2_INT(*PE_size));\
-            }\
-        }\
-        /* Collective operation call */\
-        if ( rc == OSHMEM_SUCCESS )\
-        {\
     oshmem_op_t* op = T_NAME;\
     size_t size = OMPI_FINT_2_INT(*nreduce) * op->dt_size;\
     \
@@ -120,19 +93,12 @@ SHMEM_GENERATE_FORTRAN_BINDINGS_SUB (void,
             size,\
             FPTR_2_VOID_PTR(pSync),\
             FPTR_2_VOID_PTR(*pWrk), SCOLL_DEFAULT_ALG);\
-        }\
-        if (OSHMEM_GROUP_CACHE_ENABLED == 0)\
-        {\
-            if ( rc == OSHMEM_SUCCESS )\
-            {\
     oshmem_proc_group_destroy(group);\
-            }\
+    RUNTIME_CHECK_RC(rc); \
-        }\
-    }\
 }
 
-SHMEM_XOR_TO_ALL(shmem_int2_xor_to_all_f, oshmem_op_xor_fint2, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_XOR_TO_ALL(shmem_int2_xor_to_all_f, oshmem_op_xor_fint2)
-SHMEM_XOR_TO_ALL(shmem_int4_xor_to_all_f, oshmem_op_xor_fint4, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_XOR_TO_ALL(shmem_int4_xor_to_all_f, oshmem_op_xor_fint4)
-SHMEM_XOR_TO_ALL(shmem_int8_xor_to_all_f, oshmem_op_xor_fint8, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_XOR_TO_ALL(shmem_int8_xor_to_all_f, oshmem_op_xor_fint8)
-SHMEM_XOR_TO_ALL(shmem_comp4_xor_to_all_f, oshmem_op_xor_fint4, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_XOR_TO_ALL(shmem_comp4_xor_to_all_f, oshmem_op_xor_fint4)
-SHMEM_XOR_TO_ALL(shmem_comp8_xor_to_all_f, oshmem_op_xor_fint8, OSHMEM_GROUP_CACHE_ENABLED)
+SHMEM_XOR_TO_ALL(shmem_comp8_xor_to_all_f, oshmem_op_xor_fint8)