From edcfdf236505ead897c288e648e3de2d0fd8726d Mon Sep 17 00:00:00 2001 From: Ralph Castain Date: Tue, 31 Jan 2017 08:01:37 -0800 Subject: [PATCH] Update to latest PMIx master Signed-off-by: Ralph Castain --- opal/mca/pmix/pmix2x/pmix/VERSION | 6 +- opal/mca/pmix/pmix2x/pmix/config/Makefile.am | 3 +- opal/mca/pmix/pmix2x/pmix/config/pmix.m4 | 28 ++- .../pmix2x/pmix/config/pmix_check_lock.m4 | 58 +++++ .../pmix/pmix2x/pmix/include/pmix_common.h | 1 + .../pmix/pmix2x/pmix/src/dstore/pmix_esh.c | 227 ++++++++++++++---- .../pmix/pmix2x/pmix/src/dstore/pmix_esh.h | 12 + .../pmix2x/pmix/src/include/pmix_globals.h | 4 +- .../pmix/src/mca/base/pmix_mca_base_var.c | 2 +- .../pmix/src/mca/base/pmix_mca_base_var.h | 4 +- .../pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp.c | 16 +- opal/mca/pmix/pmix2x/pmix/src/sm/pmix_mmap.c | 16 ++ opal/mca/pmix/pmix2x/pmix/test/Makefile.am | 9 +- opal/mca/pmix/pmix2x/pmix/test/pmi2_client.c | 20 +- opal/mca/pmix/pmix2x/pmix/test/pmi_client.c | 9 +- opal/mca/pmix/pmix2x/pmix/test/pmix_client.c | 25 +- opal/mca/pmix/pmix2x/pmix/test/test_common.c | 91 ++++++- opal/mca/pmix/pmix2x/pmix/test/test_common.h | 139 ++++++++++- opal/mca/pmix/pmix2x/pmix/test/test_fence.c | 136 +---------- .../mca/pmix/pmix2x/pmix/test/test_internal.c | 88 +++++++ .../mca/pmix/pmix2x/pmix/test/test_internal.h | 18 ++ opal/mca/pmix/pmix2x/pmix/test/test_replace.c | 136 +++++++++++ opal/mca/pmix/pmix2x/pmix/test/test_replace.h | 18 ++ opal/mca/pmix/pmix2x/pmix/test/utils.c | 12 +- 24 files changed, 865 insertions(+), 213 deletions(-) create mode 100644 opal/mca/pmix/pmix2x/pmix/config/pmix_check_lock.m4 create mode 100644 opal/mca/pmix/pmix2x/pmix/test/test_internal.c create mode 100644 opal/mca/pmix/pmix2x/pmix/test/test_internal.h create mode 100644 opal/mca/pmix/pmix2x/pmix/test/test_replace.c create mode 100644 opal/mca/pmix/pmix2x/pmix/test/test_replace.h diff --git a/opal/mca/pmix/pmix2x/pmix/VERSION b/opal/mca/pmix/pmix2x/pmix/VERSION index 4efc0a1610..f9413cd265 100644 --- a/opal/mca/pmix/pmix2x/pmix/VERSION +++ b/opal/mca/pmix/pmix2x/pmix/VERSION @@ -23,14 +23,14 @@ release=0 # The only requirement is that it must be entirely printable ASCII # characters and have no white space. -greek= +greek=a1 # If repo_rev is empty, then the repository version number will be # obtained during "make dist" via the "git describe --tags --always" # command, or with the date (if "git describe" fails) in the form of # "date". -repo_rev=git89ec3bc +repo_rev=git972f79a # If tarball_version is not empty, it is used as the version string in # the tarball filename, regardless of all other versions listed in @@ -44,7 +44,7 @@ tarball_version= # The date when this release was created -date="Jan 23, 2017" +date="Jan 31, 2017" # The shared library version of each of PMIx's public libraries. # These versions are maintained in accordance with the "Library diff --git a/opal/mca/pmix/pmix2x/pmix/config/Makefile.am b/opal/mca/pmix/pmix2x/pmix/config/Makefile.am index 0c0556379a..4edc92afef 100644 --- a/opal/mca/pmix/pmix2x/pmix/config/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/config/Makefile.am @@ -1,4 +1,4 @@ -# Copyright (c) 2013-2016 Intel, Inc. All rights reserved +# Copyright (c) 2013-2017 Intel, Inc. All rights reserved. # Copyright (c) 2016 Research Organization for Information Science # and Technology (RIST). All rights reserved. # Copyright (c) 2006-2016 Cisco Systems, Inc. All rights reserved. @@ -35,6 +35,7 @@ EXTRA_DIST = \ pmix_check_package.m4 \ pmix_check_vendor.m4 \ pmix_check_visibility.m4 \ + pmix_check_lock.m4 \ pmix_config_subdir.m4 \ pmix_ensure_contains_optflags.m4 \ pmix_functions.m4 \ diff --git a/opal/mca/pmix/pmix2x/pmix/config/pmix.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix.m4 index 4432ef708a..0bd8dcfc93 100644 --- a/opal/mca/pmix/pmix2x/pmix/config/pmix.m4 +++ b/opal/mca/pmix/pmix2x/pmix/config/pmix.m4 @@ -18,7 +18,7 @@ dnl reserved. dnl Copyright (c) 2009-2011 Oak Ridge National Labs. All rights reserved. dnl Copyright (c) 2011-2013 NVIDIA Corporation. All rights reserved. dnl Copyright (c) 2013-2017 Intel, Inc. All rights reserved. -dnl Copyright (c) 2015-2016 Research Organization for Information Science +dnl Copyright (c) 2015-2017 Research Organization for Information Science dnl and Technology (RIST). All rights reserved. dnl Copyright (c) 2016 Mellanox Technologies, Inc. dnl All rights reserved. @@ -559,7 +559,7 @@ AC_DEFUN([PMIX_SETUP_CORE],[ # Darwin doesn't need -lm, as it's a symlink to libSystem.dylib PMIX_SEARCH_LIBS_CORE([ceil], [m]) - AC_CHECK_FUNCS([asprintf snprintf vasprintf vsnprintf strsignal socketpair strncpy_s usleep statfs statvfs getpeereid getpeerucred strnlen]) + AC_CHECK_FUNCS([asprintf snprintf vasprintf vsnprintf strsignal socketpair strncpy_s usleep statfs statvfs getpeereid getpeerucred strnlen posix_fallocate]) # On some hosts, htonl is a define, so the AC_CHECK_FUNC will get # confused. On others, it's in the standard library, but stubbed with @@ -655,6 +655,14 @@ AC_DEFUN([PMIX_SETUP_CORE],[ PMIX_MCA + ################################## + # Dstore Locking + ################################## + + pmix_show_title "Dstore Locking" + + PMIX_CHECK_DSTOR_LOCK + ############################################################################ # final compiler config ############################################################################ @@ -867,6 +875,7 @@ AC_DEFINE_UNQUOTED([PMIX_WANT_PRETTY_PRINT_STACKTRACE], [$WANT_PRETTY_PRINT_STACKTRACE], [if want pretty-print stack trace feature]) +# # Do we want the shared memory datastore usage? # @@ -886,7 +895,22 @@ AC_DEFINE_UNQUOTED([PMIX_ENABLE_DSTORE], [if want shared memory dstore feature]) # +# Use pthread-based locking +# +DSTORE_PTHREAD_LOCK="1" +AC_MSG_CHECKING([if want dstore pthread-based locking]) +AC_ARG_ENABLE([dstore-pthlck], + [AC_HELP_STRING([--disable-dstore-pthlck], + [Disable pthread-based lockig in dstor (default: enabled)])]) +if test "$enable_dstore_pthlck" == "no" ; then + AC_MSG_RESULT([no]) + DSTORE_PTHREAD_LOCK="0" +else + AC_MSG_RESULT([yes]) + DSTORE_PTHREAD_LOCK="1" +fi +# # Ident string # AC_MSG_CHECKING([if want ident string]) diff --git a/opal/mca/pmix/pmix2x/pmix/config/pmix_check_lock.m4 b/opal/mca/pmix/pmix2x/pmix/config/pmix_check_lock.m4 new file mode 100644 index 0000000000..097e286e2f --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/config/pmix_check_lock.m4 @@ -0,0 +1,58 @@ +dnl -*- shell-script -*- +dnl +dnl Copyright (c) 2017 Mellanox Technologies, Inc. +dnl All rights reserved. +dnl Copyright (c) 2017 Intel, Inc. All rights reserved. +dnl $COPYRIGHT$ +dnl +dnl Additional copyrights may follow +dnl +dnl $HEADER$ +dnl + +AC_DEFUN([PMIX_CHECK_DSTOR_LOCK],[ + orig_libs=$LIBS + LIBS="-lpthread $LIBS" + + _x_ac_pthread_lock_found="0" + _x_ac_fcntl_lock_found="0" + + AC_CHECK_MEMBERS([struct flock.l_type], + [ + AC_DEFINE([HAVE_FCNTL_FLOCK], [1], + [Define to 1 if you have the locking by fcntl.]) + _x_ac_fcntl_lock_found="1" + ], [], [#include ]) + + if test "$ESH_PTHREAD_LOCK" == "1"; then + AC_CHECK_FUNC([pthread_rwlockattr_setkind_np], + [AC_EGREP_HEADER([PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP], + [pthread.h],[ + AC_DEFINE([HAVE_PTHREAD_SETKIND], [1], + [Define to 1 if you have the `pthread_rwlockattr_setkind_np` function.])])]) + + AC_CHECK_FUNC([pthread_rwlockattr_setpshared], + [AC_EGREP_HEADER([PTHREAD_PROCESS_SHARED], + [pthread.h],[ + AC_DEFINE([HAVE_PTHREAD_SHARED], [1], + [Define to 1 if you have the `PTHREAD_PROCESS_SHARED` definition. + ]) + _x_ac_pthread_lock_found="1" + ]) + ]) + + if test "$_x_ac_pthread_lock_found" == "0"; then + if test "$_x_ac_fcntl_lock_found" == "1"; then + AC_MSG_WARN([dstore: pthread-based locking not found, will use fcntl-based locking.]) + else + AC_MSG_ERROR([dstore: no available locking mechanisms was found. Can not continue. Try disabling dstore]) + fi + fi + else + if test "$_x_ac_fcntl_lock_found" == "0"; then + AC_MSG_ERROR([dstore: no available locking mechanisms was found. Can not continue. Try disabling dstore]) + fi + fi + + LIBS="$orig_libs" +]) diff --git a/opal/mca/pmix/pmix2x/pmix/include/pmix_common.h b/opal/mca/pmix/pmix2x/pmix/include/pmix_common.h index 27e1c21e40..10ea9dc143 100644 --- a/opal/mca/pmix/pmix2x/pmix/include/pmix_common.h +++ b/opal/mca/pmix/pmix2x/pmix/include/pmix_common.h @@ -172,6 +172,7 @@ typedef uint32_t pmix_rank_t; #define PMIX_LOCALLDR "pmix.lldr" // (pmix_rank_t) lowest rank on this node within this job #define PMIX_APPLDR "pmix.aldr" // (pmix_rank_t) lowest rank in this app within this job #define PMIX_PROC_PID "pmix.ppid" // (pid_t) pid of specified proc +#define PMIX_SESSION_ID "pmix.session.id" // (uint32_t) session identifier #define PMIX_NODE_LIST "pmix.nlist" // (char*) comma-delimited list of nodes running procs for the specified nspace #define PMIX_ALLOCATED_NODELIST "pmix.alist" // (char*) comma-delimited list of all nodes in this allocation regardless of diff --git a/opal/mca/pmix/pmix2x/pmix/src/dstore/pmix_esh.c b/opal/mca/pmix/pmix2x/pmix/src/dstore/pmix_esh.c index ee57cb083c..151fb58150 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/dstore/pmix_esh.c +++ b/opal/mca/pmix/pmix2x/pmix/src/dstore/pmix_esh.c @@ -3,7 +3,7 @@ * All rights reserved. * Copyright (c) 2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2016 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2017 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -15,7 +15,6 @@ #include #include #include -#include #include #include #include @@ -37,6 +36,14 @@ #include "pmix_dstore.h" #include "pmix_esh.h" +#ifdef ESH_FCNTL_LOCK +#include +#endif + +#ifdef ESH_PTHREAD_LOCK +#include +#endif + static int _esh_init(pmix_info_t info[], size_t ninfo); static int _esh_finalize(void); static int _esh_store(const char *nspace, pmix_rank_t rank, pmix_kval_t *kv); @@ -122,6 +129,43 @@ __extension__ ({ \ buffer, size); \ }) +#ifdef ESH_PTHREAD_LOCK +#define _ESH_LOCK(rwlock, operation) \ +__extension__ ({ \ + pmix_status_t ret = PMIX_SUCCESS; \ + int rc; \ + switch (operation) { \ + case F_WRLCK: \ + rc = pthread_rwlock_wrlock(rwlock); \ + break; \ + case F_RDLCK: \ + rc = pthread_rwlock_rdlock(rwlock); \ + break; \ + case F_UNLCK: \ + rc = pthread_rwlock_unlock(rwlock); \ + break; \ + default: \ + rc = PMIX_ERR_BAD_PARAM; \ + } \ + if (0 != rc) { \ + switch (errno) { \ + case EINVAL: \ + ret = PMIX_ERR_INIT; \ + break; \ + case EPERM: \ + ret = PMIX_ERR_NO_PERMISSIONS; \ + break; \ + } \ + } \ + if (ret) { \ + pmix_output(0, "%s %d:%s lock failed: %s", \ + __FILE__, __LINE__, __func__, strerror(errno)); \ + } \ + ret; \ +}) +#endif + +#ifdef ESH_FCNTL_LOCK #define _ESH_LOCK(lockfd, operation) \ __extension__ ({ \ pmix_status_t ret = PMIX_SUCCESS; \ @@ -159,10 +203,11 @@ __extension__ ({ \ } \ ret; \ }) +#endif -#define _ESH_WRLOCK(lockfd) _ESH_LOCK(lockfd, F_WRLCK) -#define _ESH_RDLOCK(lockfd) _ESH_LOCK(lockfd, F_RDLCK) -#define _ESH_UNLOCK(lockfd) _ESH_LOCK(lockfd, F_UNLCK) +#define _ESH_WRLOCK(lock) _ESH_LOCK(lock, F_WRLCK) +#define _ESH_RDLOCK(lock) _ESH_LOCK(lock, F_RDLCK) +#define _ESH_UNLOCK(lock) _ESH_LOCK(lock, F_UNLCK) #define ESH_INIT_SESSION_TBL_SIZE 2 #define ESH_INIT_NS_MAP_TBL_SIZE 2 @@ -201,6 +246,7 @@ static size_t _max_ns_num; static size_t _meta_segment_size = 0; static size_t _max_meta_elems; static size_t _data_segment_size = 0; +static size_t _lock_segment_size = 0; static uid_t _jobuid; static char _setjobuid = 0; @@ -213,11 +259,21 @@ ns_map_data_t * (*_esh_session_map_search)(const char *nspace) = NULL; #define _ESH_SESSION_path(tbl_idx) (PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t)[tbl_idx].nspace_path) #define _ESH_SESSION_lockfile(tbl_idx) (PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t)[tbl_idx].lockfile) #define _ESH_SESSION_jobuid(tbl_idx) (PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t)[tbl_idx].jobuid) -#define _ESH_SESSION_lockfd(tbl_idx) (PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t)[tbl_idx].lockfd) #define _ESH_SESSION_sm_seg_first(tbl_idx) (PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t)[tbl_idx].sm_seg_first) #define _ESH_SESSION_sm_seg_last(tbl_idx) (PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t)[tbl_idx].sm_seg_last) #define _ESH_SESSION_ns_info(tbl_idx) (PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t)[tbl_idx].ns_info) +#ifdef ESH_PTHREAD_LOCK +#define _ESH_SESSION_pthread_rwlock(tbl_idx) (PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t)[tbl_idx].rwlock) +#define _ESH_SESSION_pthread_seg(tbl_idx) (PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t)[tbl_idx].rwlock_seg) +#define _ESH_SESSION_lock(tbl_idx) _ESH_SESSION_pthread_rwlock(tbl_idx) +#endif + +#ifdef ESH_FCNTL_LOCK +#define _ESH_SESSION_lockfd(tbl_idx) (PMIX_VALUE_ARRAY_GET_BASE(_session_array, session_t)[tbl_idx].lockfd) +#define _ESH_SESSION_lock(tbl_idx) _ESH_SESSION_lockfd(tbl_idx) +#endif + /* If _direct_mode is set, it means that we use linear search * along the array of rank meta info objects inside a meta segment * to find the requested rank. Otherwise, we do a fast lookup @@ -250,6 +306,93 @@ static inline void _esh_session_map_clean(ns_map_t *m) { m->data.track_idx = -1; } +#ifdef ESH_PTHREAD_LOCK +static inline int _rwlock_init(size_t idx, char *lockfile) { + pmix_status_t rc = PMIX_SUCCESS; + size_t size = _lock_segment_size; + pthread_rwlockattr_t attr; + + if ((NULL != _ESH_SESSION_pthread_seg(idx)) || (NULL != _ESH_SESSION_pthread_rwlock(idx))) { + rc = PMIX_ERR_INIT; + return rc; + } + _ESH_SESSION_pthread_seg(idx) = (pmix_sm_seg_t *)malloc(sizeof(pmix_sm_seg_t)); + if (NULL == _ESH_SESSION_pthread_seg(idx)) { + rc = PMIX_ERR_OUT_OF_RESOURCE; + return rc; + } + + if (PMIX_PROC_SERVER == pmix_globals.proc_type) { + if (PMIX_SUCCESS != (rc = pmix_sm_segment_create(_ESH_SESSION_pthread_seg(idx), lockfile, size))) { + return rc; + } + memset(_ESH_SESSION_pthread_seg(idx)->seg_base_addr, 0, size); + _ESH_SESSION_pthread_rwlock(idx) = (pthread_rwlock_t *)_ESH_SESSION_pthread_seg(idx)->seg_base_addr; + + if (0 != pthread_rwlockattr_init(&attr)) { + rc = PMIX_ERR_INIT; + pmix_sm_segment_detach(_ESH_SESSION_pthread_seg(idx)); + return rc; + } + if (0 != pthread_rwlockattr_setpshared(&attr, PTHREAD_PROCESS_SHARED)) { + rc = PMIX_ERR_INIT; + pmix_sm_segment_detach(_ESH_SESSION_pthread_seg(idx)); + pthread_rwlockattr_destroy(&attr); + return rc; + } +#ifdef HAVE_PTHREAD_SETKIND + if (0 != pthread_rwlockattr_setkind_np(&attr, PTHREAD_RWLOCK_PREFER_WRITER_NONRECURSIVE_NP)) { + rc = PMIX_ERR_INIT; + pmix_sm_segment_detach(_ESH_SESSION_pthread_seg(idx)); + pthread_rwlockattr_destroy(&attr); + return rc; + } +#endif + if (0 != pthread_rwlock_init(_ESH_SESSION_pthread_rwlock(idx), &attr)) { + rc = PMIX_ERR_INIT; + pmix_sm_segment_detach(_ESH_SESSION_pthread_seg(idx)); + pthread_rwlockattr_destroy(&attr); + return rc; + } + if (0 != pthread_rwlockattr_destroy(&attr)) { + rc = PMIX_ERR_INIT; + return rc; + } + + } + else { + _ESH_SESSION_pthread_seg(idx)->seg_size = size; + snprintf(_ESH_SESSION_pthread_seg(idx)->seg_name, PMIX_PATH_MAX, "%s", lockfile); + if (PMIX_SUCCESS != (rc = pmix_sm_segment_attach(_ESH_SESSION_pthread_seg(idx), PMIX_SM_RW))) { + return rc; + } + _ESH_SESSION_pthread_rwlock(idx) = (pthread_rwlock_t *)_ESH_SESSION_pthread_seg(idx)->seg_base_addr; + } + + return rc; +} + +static inline void _rwlock_release(session_t *s) { + pmix_status_t rc; + + if (0 != pthread_rwlock_destroy(s->rwlock)) { + rc = PMIX_ERROR; + PMIX_ERROR_LOG(rc); + return; + } + + /* detach & unlink from current desc */ + if (s->rwlock_seg->seg_cpid == getpid()) { + pmix_sm_segment_unlink(s->rwlock_seg); + } + pmix_sm_segment_detach(s->rwlock_seg); + + free(s->rwlock_seg); + s->rwlock_seg = NULL; + s->rwlock = NULL; +} +#endif + static inline const char *_unique_id(void) { static const char *str = NULL; @@ -574,12 +717,6 @@ static inline int _esh_session_init(size_t idx, ns_map_data_t *m, size_t jobuid, session_t *s = &(PMIX_VALUE_ARRAY_GET_ITEM(_session_array, session_t, idx)); pmix_status_t rc = PMIX_SUCCESS; - if (NULL == s) { - rc = PMIX_ERR_BAD_PARAM; - PMIX_ERROR_LOG(rc); - return rc; - } - s->jobuid = jobuid; s->nspace_path = strdup(_base_path); @@ -654,6 +791,12 @@ static inline int _esh_session_init(size_t idx, ns_map_data_t *m, size_t jobuid, } } +#ifdef ESH_PTHREAD_LOCK + if ( PMIX_SUCCESS != (rc = _rwlock_init(m->tbl_idx, s->lockfile))) { + PMIX_ERROR_LOG(rc); + return rc; + } +#endif s->sm_seg_first = seg; s->sm_seg_last = s->sm_seg_first; return PMIX_SUCCESS; @@ -680,6 +823,9 @@ static inline void _esh_session_release(session_t *s) } free(s->nspace_path); } +#ifdef ESH_PTHREAD_LOCK + _rwlock_release(s); +#endif memset ((char *) s, 0, sizeof(*s)); } @@ -866,7 +1012,7 @@ int _esh_store(const char *nspace, pmix_rank_t rank, pmix_kval_t *kv) { pmix_status_t rc = PMIX_SUCCESS, tmp_rc; ns_track_elem_t *elem; - pmix_buffer_t pbkt, xfer; + pmix_buffer_t xfer; ns_seg_info_t ns_info; ns_map_data_t *ns_map = NULL; @@ -885,7 +1031,7 @@ int _esh_store(const char *nspace, pmix_rank_t rank, pmix_kval_t *kv) } /* set exclusive lock */ - if (PMIX_SUCCESS != (rc = _ESH_WRLOCK(_ESH_SESSION_lockfd(ns_map->tbl_idx)))) { + if (PMIX_SUCCESS != (rc = _ESH_WRLOCK(_ESH_SESSION_lock(ns_map->tbl_idx)))) { PMIX_ERROR_LOG(rc); return rc; } @@ -935,17 +1081,14 @@ int _esh_store(const char *nspace, pmix_rank_t rank, pmix_kval_t *kv) /* Now we know info about meta segment for this namespace. If meta segment * is not empty, then we look for data for the target rank. If they present, replace it. */ - PMIX_CONSTRUCT(&pbkt, pmix_buffer_t); PMIX_CONSTRUCT(&xfer, pmix_buffer_t); PMIX_LOAD_BUFFER(&xfer, kv->value->data.bo.bytes, kv->value->data.bo.size); - pmix_buffer_t *pxfer = &xfer; - pmix_bfrop.pack(&pbkt, &pxfer, 1, PMIX_BUFFER); + + rc = _store_data_for_rank(elem, rank, &xfer); + xfer.base_ptr = NULL; xfer.bytes_used = 0; - - rc = _store_data_for_rank(elem, rank, &pbkt); PMIX_DESTRUCT(&xfer); - PMIX_DESTRUCT(&pbkt); if (PMIX_SUCCESS != rc) { PMIX_ERROR_LOG(rc); @@ -953,14 +1096,14 @@ int _esh_store(const char *nspace, pmix_rank_t rank, pmix_kval_t *kv) } /* unset lock */ - if (PMIX_SUCCESS != (rc = _ESH_UNLOCK(_ESH_SESSION_lockfd(ns_map->tbl_idx)))) { + if (PMIX_SUCCESS != (rc = _ESH_UNLOCK(_ESH_SESSION_lock(ns_map->tbl_idx)))) { PMIX_ERROR_LOG(rc); } return rc; err_exit: /* unset lock */ - if (PMIX_SUCCESS != (tmp_rc = _ESH_UNLOCK(_ESH_SESSION_lockfd(ns_map->tbl_idx)))) { + if (PMIX_SUCCESS != (tmp_rc = _ESH_UNLOCK(_ESH_SESSION_lock(ns_map->tbl_idx)))) { PMIX_ERROR_LOG(tmp_rc); } return rc; @@ -1021,7 +1164,7 @@ int _esh_fetch(const char *nspace, pmix_rank_t rank, const char *key, pmix_value } /* grab shared lock */ - if (PMIX_SUCCESS != (lock_rc = _ESH_RDLOCK(_ESH_SESSION_lockfd(ns_map->tbl_idx)))) { + if (PMIX_SUCCESS != (lock_rc = _ESH_RDLOCK(_ESH_SESSION_lock(ns_map->tbl_idx)))) { /* Something wrong with the lock. The error is fatal */ rc = PMIX_ERR_FATAL; PMIX_ERROR_LOG(lock_rc); @@ -1180,7 +1323,7 @@ int _esh_fetch(const char *nspace, pmix_rank_t rank, const char *key, pmix_value done: /* unset lock */ - if (PMIX_SUCCESS != (lock_rc = _ESH_UNLOCK(_ESH_SESSION_lockfd(ns_map->tbl_idx)))) { + if (PMIX_SUCCESS != (lock_rc = _ESH_UNLOCK(_ESH_SESSION_lock(ns_map->tbl_idx)))) { PMIX_ERROR_LOG(lock_rc); } @@ -1379,6 +1522,7 @@ static void _set_constants_from_env() } } + _lock_segment_size = page_size; _max_ns_num = (_initial_segment_size - sizeof(size_t) * 2) / sizeof(ns_seg_info_t); _max_meta_elems = (_meta_segment_size - sizeof(size_t)) / sizeof(rank_meta_info); @@ -2208,9 +2352,7 @@ static int pmix_sm_store(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_kval_t static int _store_data_for_rank(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix_buffer_t *buf) { pmix_status_t rc; - int32_t cnt; - pmix_buffer_t *bptr; pmix_kval_t *kp; seg_desc_t *metadesc, *datadesc; @@ -2245,30 +2387,23 @@ static int _store_data_for_rank(ns_track_elem_t *ns_info, pmix_rank_t rank, pmix * so unpack these buffers, and then unpack kvals from each modex buffer, * storing them in the shared memory dstore. */ - cnt = 1; free_offset = get_free_offset(datadesc); - while (PMIX_SUCCESS == (rc = pmix_bfrop.unpack(buf, &bptr, &cnt, PMIX_BUFFER))) { - cnt = 1; - kp = PMIX_NEW(pmix_kval_t); - while (PMIX_SUCCESS == (rc = pmix_bfrop.unpack(bptr, kp, &cnt, PMIX_KVAL))) { - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix: unpacked key %s", kp->key); - if (PMIX_SUCCESS != (rc = pmix_sm_store(ns_info, rank, kp, &rinfo, data_exist))) { - PMIX_ERROR_LOG(rc); - return rc; - } - PMIX_RELEASE(kp); // maintain acctg - hash_store does a retain - cnt = 1; - kp = PMIX_NEW(pmix_kval_t); - } - cnt = 1; - PMIX_RELEASE(kp); - PMIX_RELEASE(bptr); // free's the data region - if (PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) { + kp = PMIX_NEW(pmix_kval_t); + while (PMIX_SUCCESS == (rc = pmix_bfrop.unpack(buf, kp, &(int){1}, PMIX_KVAL))) { + pmix_output_verbose(2, pmix_globals.debug_output, + "pmix: unpacked key %s", kp->key); + if (PMIX_SUCCESS != (rc = pmix_sm_store(ns_info, rank, kp, &rinfo, data_exist))) { PMIX_ERROR_LOG(rc); - break; + if (NULL != rinfo) { + free(rinfo); + } + return rc; } + PMIX_RELEASE(kp); // maintain acctg - hash_store does a retain + kp = PMIX_NEW(pmix_kval_t); } + PMIX_RELEASE(kp); + if (PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER != rc) { PMIX_ERROR_LOG(rc); /* TODO: should we error-exit here? */ diff --git a/opal/mca/pmix/pmix2x/pmix/src/dstore/pmix_esh.h b/opal/mca/pmix/pmix2x/pmix/src/dstore/pmix_esh.h index 1f0fb4d561..3e533817fc 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/dstore/pmix_esh.h +++ b/opal/mca/pmix/pmix2x/pmix/src/dstore/pmix_esh.h @@ -27,6 +27,14 @@ BEGIN_C_DECLS #define PMIX_DSTORE_ESH_BASE_PATH "PMIX_DSTORE_ESH_BASE_PATH" +#ifdef HAVE_PTHREAD_SHARED +#define ESH_PTHREAD_LOCK +#elif defined HAVE_FCNTL_FLOCK +#define ESH_FCNTL_LOCK +#else +#error No locking mechanism was found +#endif + /* this structs are used to store information about * shared segments addresses locally at each process, * so they are common for different types of segments @@ -56,6 +64,10 @@ struct session_s { uid_t jobuid; char *nspace_path; char *lockfile; +#ifdef ESH_PTHREAD_LOCK + pmix_sm_seg_t *rwlock_seg; + pthread_rwlock_t *rwlock; +#endif int lockfd; seg_desc_t *sm_seg_first; seg_desc_t *sm_seg_last; diff --git a/opal/mca/pmix/pmix2x/pmix/src/include/pmix_globals.h b/opal/mca/pmix/pmix2x/pmix/src/include/pmix_globals.h index 9709f4a880..5dd46cab06 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/include/pmix_globals.h +++ b/opal/mca/pmix/pmix2x/pmix/src/include/pmix_globals.h @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2014-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -368,7 +368,7 @@ typedef struct { } pmix_globals_t; -extern pmix_globals_t pmix_globals; +PMIX_EXPORT extern pmix_globals_t pmix_globals; END_C_DECLS diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_var.c b/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_var.c index bbf159454b..858ca1a91a 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_var.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_var.c @@ -563,7 +563,7 @@ int pmix_mca_base_var_cache_files(bool rel_path_search) /* * Look up an integer MCA parameter. */ -int pmix_mca_base_var_get_value (int vari, const void *value, +int pmix_mca_base_var_get_value (int vari, void *value, pmix_mca_base_var_source_t *source, const char **source_file) { diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_var.h b/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_var.h index 1a26a4ba6b..328cdb1f52 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_var.h +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/base/pmix_mca_base_var.h @@ -13,7 +13,7 @@ * Copyright (c) 2008-2011 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2015 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2016 Intel, Inc. All rights reserved + * Copyright (c) 2016-2017 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -512,7 +512,7 @@ int pmix_mca_base_var_deregister(int vari); * Note: The value can be changed by the registering code without using * the pmix_mca_base_var_* interface so the source may be incorrect. */ -int pmix_mca_base_var_get_value (int vari, const void *value, +int pmix_mca_base_var_get_value (int vari, void *value, pmix_mca_base_var_source_t *source, const char **source_file); diff --git a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp.c b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp.c index 9f3eee1816..0f7032a1a6 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp.c +++ b/opal/mca/pmix/pmix2x/pmix/src/mca/ptl/tcp/ptl_tcp.c @@ -142,7 +142,11 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, /* set the server nspace */ p = uri[0]; - p2 = strchr(p, '.'); + if (NULL == (p2 = strchr(p, '.'))) { + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + pmix_argv_free(uri); + return PMIX_ERR_NOT_SUPPORTED; + } *p2 = '\0'; ++p2; pmix_client_globals.myserver.info = PMIX_NEW(pmix_rank_info_t); @@ -542,11 +546,6 @@ static pmix_status_t recv_connect_ack(int sd) return rc; } reply = ntohl(u32); - /* if the status indicates an error, then we are done */ - if (PMIX_SUCCESS != reply) { - PMIX_ERROR_LOG(reply); - return reply; - } if (PMIX_PROC_IS_CLIENT) { /* see if they want us to do the handshake */ @@ -568,6 +567,11 @@ static pmix_status_t recv_connect_ack(int sd) } pmix_globals.pindex = ntohl(u32); } else { + /* if the status indicates an error, then we are done */ + if (PMIX_SUCCESS != reply) { + PMIX_ERROR_LOG(reply); + return reply; + } /* recv our nspace */ rc = pmix_ptl_base_recv_blocking(sd, (char*)&pmix_globals.myid.nspace, PMIX_MAX_NSLEN+1); if (PMIX_SUCCESS != rc) { diff --git a/opal/mca/pmix/pmix2x/pmix/src/sm/pmix_mmap.c b/opal/mca/pmix/pmix2x/pmix/src/sm/pmix_mmap.c index 8dac3d5a82..bc5ec511c6 100644 --- a/opal/mca/pmix/pmix2x/pmix/src/sm/pmix_mmap.c +++ b/opal/mca/pmix/pmix2x/pmix/src/sm/pmix_mmap.c @@ -1,6 +1,8 @@ /* * Copyright (c) 2015-2016 Mellanox Technologies, Inc. * All rights reserved. + * Copyright (c) 2017 Research Organization for Information Science + * and Technology (RIST). All rights reserved. * Copyright (c) 2017 Intel, Inc. All rights reserved. * $COPYRIGHT$ * @@ -17,6 +19,7 @@ #include #include #include +#include #include #include @@ -59,12 +62,25 @@ int _mmap_segment_create(pmix_sm_seg_t *sm_seg, const char *file_name, size_t si goto out; } /* size backing file - note the use of real_size here */ +#ifdef HAVE_POSIX_FALLOCATE + if (0 != posix_fallocate(sm_seg->seg_id, 0, size)) { + pmix_output_verbose(2, pmix_globals.debug_output, + "sys call posix_fallocate(2) fail\n"); + if (ENOSPC == errno) { + rc = PMIX_ERR_OUT_OF_RESOURCE; + } else { + rc = PMIX_ERROR; + } + goto out; + } +#else if (0 != ftruncate(sm_seg->seg_id, size)) { pmix_output_verbose(2, pmix_globals.debug_output, "sys call ftruncate(2) fail\n"); rc = PMIX_ERROR; goto out; } +#endif if (MAP_FAILED == (seg_addr = mmap(NULL, size, PROT_READ | PROT_WRITE, MAP_SHARED, sm_seg->seg_id, 0))) { diff --git a/opal/mca/pmix/pmix2x/pmix/test/Makefile.am b/opal/mca/pmix/pmix2x/pmix/test/Makefile.am index c8af0a653a..651d3cfd4e 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/Makefile.am +++ b/opal/mca/pmix/pmix2x/pmix/test/Makefile.am @@ -11,7 +11,7 @@ # All rights reserved. # Copyright (c) 2006-2010 Cisco Systems, Inc. All rights reserved. # Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. -# Copyright (c) 2013-2016 Intel, Inc. All rights reserved +# Copyright (c) 2013-2017 Intel, Inc. All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -25,7 +25,9 @@ if !WANT_HIDDEN SUBDIRS = simple endif -headers = test_common.h cli_stages.h server_callbacks.h utils.h test_fence.h test_publish.h test_spawn.h test_cd.h test_resolve_peers.h test_error.h +headers = test_common.h cli_stages.h server_callbacks.h utils.h test_fence.h \ + test_publish.h test_spawn.h test_cd.h test_resolve_peers.h test_error.h \ + test_replace.h test_internal.h AM_CPPFLAGS = -I$(top_builddir)/src -I$(top_builddir)/src/include -I$(top_builddir)/src/api @@ -54,7 +56,8 @@ pmi2_client_LDADD = \ $(top_builddir)/src/libpmix.la pmix_client_SOURCES = $(headers) \ - pmix_client.c test_fence.c test_common.c test_publish.c test_spawn.c test_cd.c test_resolve_peers.c test_error.c + pmix_client.c test_fence.c test_common.c test_publish.c test_spawn.c \ + test_cd.c test_resolve_peers.c test_error.c test_replace.c test_internal.c pmix_client_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) pmix_client_LDADD = \ $(top_builddir)/src/libpmix.la diff --git a/opal/mca/pmix/pmix2x/pmix/test/pmi2_client.c b/opal/mca/pmix/pmix2x/pmix/test/pmi2_client.c index ee48e6be8a..a7a944e557 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/pmi2_client.c +++ b/opal/mca/pmix/pmix2x/pmix/test/pmi2_client.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. * All rights reserved. * $COPYRIGHT$ @@ -31,13 +31,15 @@ static void log_fatal(const char *format, ...) va_start(arglist, format); if (_verbose > 0) { - if (0 > vasprintf(output, format, arglist)) { + if (0 > vasprintf(output, format, arglist) || + NULL == output || NULL == *output) { + va_end(arglist); return; } fprintf(stderr, "FATAL: %s", *output); - va_end(arglist); free(*output); } + va_end(arglist); } static void log_error(const char *format, ...) @@ -47,13 +49,15 @@ static void log_error(const char *format, ...) va_start(arglist, format); if (_verbose > 0) { - if (0 > vasprintf(output, format, arglist)) { + if (0 > vasprintf(output, format, arglist) || + NULL == output || NULL == *output) { + va_end(arglist); return; } fprintf(stderr, "ERROR: %s", *output); - va_end(arglist); free(*output); } + va_end(arglist); } static void log_info(const char *format, ...) @@ -63,13 +67,15 @@ static void log_info(const char *format, ...) va_start(arglist, format); if (_verbose > 0) { - if (0 > vasprintf(output, format, arglist)) { + if (0 > vasprintf(output, format, arglist) || + NULL == output || NULL == *output) { + va_end(arglist); return; } fprintf(stderr, "INFO: %s", *output); - va_end(arglist); free(*output); } + va_end(arglist); } #define log_assert(e, msg) \ diff --git a/opal/mca/pmix/pmix2x/pmix/test/pmi_client.c b/opal/mca/pmix/pmix2x/pmix/test/pmi_client.c index 22cf1a86dc..cfc9d9d6b8 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/pmi_client.c +++ b/opal/mca/pmix/pmix2x/pmix/test/pmi_client.c @@ -32,13 +32,14 @@ static void log_fatal(const char *format, ...) va_start(arglist, format); if (_verbose > 0) { if (0 > vasprintf(output, format, arglist) || - NULL == *output) { + NULL == output || NULL == *output) { + va_end(arglist); return; } fprintf(stderr, "FATAL: %s", *output); - va_end(arglist); free(*output); } + va_end(arglist); } static void log_error(const char *format, ...) @@ -49,7 +50,7 @@ static void log_error(const char *format, ...) va_start(arglist, format); if (_verbose > 0) { if (0 > vasprintf(output, format, arglist) || - NULL == *output) { + NULL == output || NULL == *output) { va_end(arglist); return; } @@ -67,7 +68,7 @@ static void log_info(const char *format, ...) va_start(arglist, format); if (_verbose > 0) { if (0 > vasprintf(output, format, arglist) || - NULL == *output) { + NULL == output || NULL == *output) { va_end(arglist); return; } diff --git a/opal/mca/pmix/pmix2x/pmix/test/pmix_client.c b/opal/mca/pmix/pmix2x/pmix/test/pmix_client.c index 8235b0914b..87e13c2678 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/pmix_client.c +++ b/opal/mca/pmix/pmix2x/pmix/test/pmix_client.c @@ -13,8 +13,8 @@ * All rights reserved. * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. - * Copyright (c) 2013-2016 Intel, Inc. All rights reserved. - * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. + * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2017 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -39,7 +39,8 @@ #include "test_cd.h" #include "test_resolve_peers.h" #include "test_error.h" - +#include "test_replace.h" +#include "test_internal.h" static void errhandler(size_t evhdlr_registration_id, pmix_status_t status, @@ -194,6 +195,24 @@ int main(int argc, char **argv) } } + if (NULL != params.key_replace) { + rc = test_replace(myproc.nspace, myproc.rank, params); + if (PMIX_SUCCESS != rc) { + FREE_TEST_PARAMS(params); + TEST_ERROR(("%s:%d error key replace test failed: %d", myproc.nspace, myproc.rank, rc)); + exit(0); + } + } + + if (params.test_internal) { + rc = test_internal(myproc.nspace, myproc.rank, params); + if (PMIX_SUCCESS != rc) { + FREE_TEST_PARAMS(params); + TEST_ERROR(("%s:%d error key store internal test failed: %d", myproc.nspace, myproc.rank, rc)); + exit(0); + } + } + TEST_VERBOSE(("Client ns %s rank %d: PASSED", myproc.nspace, myproc.rank)); PMIx_Deregister_event_handler(1, op_callbk, NULL); diff --git a/opal/mca/pmix/pmix2x/pmix/test/test_common.c b/opal/mca/pmix/pmix2x/pmix/test/test_common.c index 59c9598620..e3de6bde18 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/test_common.c +++ b/opal/mca/pmix/pmix2x/pmix/test/test_common.c @@ -2,7 +2,7 @@ * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. * Copyright (c) 2015 Artem Y. Polyakov . * All rights reserved. - * Copyright (c) 2015 Mellanox Technologies, Inc. + * Copyright (c) 2015-2017 Mellanox Technologies, Inc. * All rights reserved. * $COPYRIGHT$ * @@ -18,6 +18,7 @@ #include "test_common.h" #include #include +#include int pmix_test_verbose = 0; @@ -76,7 +77,9 @@ void parse_cmd(int argc, char **argv, test_params *params) fprintf(stderr, "\t--test-spawn test spawn api.\n"); fprintf(stderr, "\t--test-connect test connect/disconnect api.\n"); fprintf(stderr, "\t--test-resolve-peers test resolve_peers api.\n"); - fprintf(stderr, "t--test-error test error handling api.\n"); + fprintf(stderr, "\t--test-error test error handling api.\n"); + fprintf(stderr, "\t--test-replace N:k0,k1,...,k(N-1) test key replace for N keys, k0,k1,k(N-1) - key indexes to replace \n"); + fprintf(stderr, "\t--test-internal N test store internal key, N - number of internal keys\n"); exit(0); } else if (0 == strcmp(argv[i], "--exec") || 0 == strcmp(argv[i], "-e")) { i++; @@ -169,6 +172,24 @@ void parse_cmd(int argc, char **argv, test_params *params) params->test_resolve_peers = 1; } else if( 0 == strcmp(argv[i], "--test-error") ){ params->test_error = 1; + } else if(0 == strcmp(argv[i], "--test-replace") ) { + i++; + if (NULL != argv[i] && (*argv[i] != '-')) { + params->key_replace = strdup(argv[i]); + if (0 != parse_replace(params->key_replace, 0, NULL)) { + fprintf(stderr, "Incorrect --test-replace option format: %s\n", params->key_replace); + exit(1); + } + } else { + params->key_replace = strdup(TEST_REPLACE_DEFAULT); + } + } else if(0 == strcmp(argv[i], "--test-internal")) { + i++; + if ((NULL != argv[i]) && (*argv[i] != '-')) { + params->test_internal = strtol(argv[i], NULL, 10); + } else { + params->test_internal = 1; + } } else { @@ -256,11 +277,16 @@ PMIX_CLASS_INSTANCE(participant_t, pmix_list_item_t, NULL, NULL); +PMIX_CLASS_INSTANCE(key_replace_t, + pmix_list_item_t, + NULL, NULL); + static int ns_id = -1; static fence_desc_t *fdesc = NULL; pmix_list_t *participants = NULL; pmix_list_t test_fences; pmix_list_t *noise_range = NULL; +pmix_list_t key_replace; #define CHECK_STRTOL_VAL(val, str, store) do { \ if (0 == val) { \ @@ -485,6 +511,67 @@ int parse_noise(char *noise_param, int store) return ret; } +static int is_digit(const char *str) +{ + if (NULL == str) + return 0; + + while (0 != *str) { + if (!isdigit(*str)) { + return 0; + } + else { + str++; + } + } + return 1; +} + +int parse_replace(char *replace_param, int store, int *key_num) { + int ret = 0; + char *tmp = strdup(replace_param); + char tmp_str[32]; + char * pch, *ech; + key_replace_t *item; + int cnt = 0; + + if (NULL == replace_param) { + free(tmp); + return 1; + } + + pch = strchr(tmp, ':'); + snprintf(tmp_str, pch - tmp + 1, "%s", tmp); + cnt = atol(tmp_str); + + if (NULL != key_num) { + *key_num = cnt; + } + + while(NULL != pch) { + pch++; + ech = strchr(pch, ','); + if (NULL != ech || (strlen(pch) > 0)) { + snprintf(tmp_str, ech - pch + 1, "%s", pch); + if ((0 == is_digit(tmp_str)) || ((atoi(tmp_str) + 1) > cnt)) { + ret = 1; + break; + } + pch = ech; + if (store) { + item = PMIX_NEW(key_replace_t); + item->key_idx = atoi(tmp_str); + pmix_list_append(&key_replace, &item->super); + } + } else { + ret = 1; + break; + } + } + free(tmp); + return ret; +} + int get_total_ns_number(test_params params) { int num = 0; diff --git a/opal/mca/pmix/pmix2x/pmix/test/test_common.h b/opal/mca/pmix/pmix2x/pmix/test/test_common.h index ca571c46f9..3593eca781 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/test_common.h +++ b/opal/mca/pmix/pmix2x/pmix/test/test_common.h @@ -1,10 +1,10 @@ /* - * Copyright (c) 2013-2016 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2017 Intel, Inc. All rights reserved. * Copyright (c) 2015 Artem Y. Polyakov . * All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2015 Mellanox Technologies, Inc. + * Copyright (c) 2015-2017 Mellanox Technologies, Inc. * All rights reserved. * $COPYRIGHT$ * @@ -72,6 +72,7 @@ extern FILE *file; #define TEST_DEFAULT_TIMEOUT 10 #define MAX_DIGIT_LEN 10 +#define TEST_REPLACE_DEFAULT "3:1" #define TEST_SET_FILE(prefix, ns_id, rank) { \ char *fname = malloc( strlen(prefix) + MAX_DIGIT_LEN + 2 ); \ @@ -116,6 +117,8 @@ typedef struct { int test_connect; int test_resolve_peers; int test_error; + char *key_replace; + int test_internal; } test_params; #define INIT_TEST_PARAMS(params) do { \ @@ -144,6 +147,8 @@ typedef struct { params.noise = NULL; \ params.ns_dist = NULL; \ params.test_error = 0; \ + params.key_replace = NULL; \ + params.test_internal = 0; \ } while (0) #define FREE_TEST_PARAMS(params) do { \ @@ -173,6 +178,7 @@ typedef struct { void parse_cmd(int argc, char **argv, test_params *params); int parse_fence(char *fence_param, int store); int parse_noise(char *noise_param, int store); +int parse_replace(char *replace_param, int store, int *key_num); typedef struct { pmix_list_item_t super; @@ -188,11 +194,140 @@ typedef struct { } participant_t; PMIX_CLASS_DECLARATION(participant_t); +typedef struct { + pmix_list_item_t super; + int key_idx; +} key_replace_t; +PMIX_CLASS_DECLARATION(key_replace_t); + extern pmix_list_t test_fences; extern pmix_list_t *noise_range; +extern pmix_list_t key_replace; #define NODE_NAME "node1" int get_total_ns_number(test_params params); int get_all_ranks_from_namespace(test_params params, char *nspace, pmix_proc_t **ranks, size_t *nranks); +typedef struct { + int in_progress; + pmix_value_t *kv; + int status; +} get_cbdata; + +#define SET_KEY(key, fence_num, ind, use_same_keys) do { \ + if (use_same_keys) { \ + (void)snprintf(key, sizeof(key)-1, "key-%d", ind); \ + } else { \ + (void)snprintf(key, sizeof(key)-1, "key-f%d:%d", fence_num, ind); \ + } \ +} while (0) + +#define PUT(dtype, data, flag, fence_num, ind, use_same_keys) do { \ + char key[50]; \ + pmix_value_t value; \ + SET_KEY(key, fence_num, ind, use_same_keys); \ + PMIX_VAL_SET(&value, dtype, data); \ + TEST_VERBOSE(("%s:%d put key %s", my_nspace, my_rank, key)); \ + if (PMIX_SUCCESS != (rc = PMIx_Put(flag, key, &value))) { \ + TEST_ERROR(("%s:%d: PMIx_Put key %s failed: %d", my_nspace, my_rank, key, rc)); \ + rc = PMIX_ERROR; \ + } \ + PMIX_VALUE_DESTRUCT(&value); \ +} while (0) + +#define GET(dtype, data, ns, r, fence_num, ind, use_same_keys, blocking, ok_notfnd) do { \ + char key[50]; \ + pmix_value_t *val; \ + get_cbdata cbdata; \ + cbdata.status = PMIX_SUCCESS; \ + pmix_proc_t foobar; \ + SET_KEY(key, fence_num, ind, use_same_keys); \ + (void)strncpy(foobar.nspace, ns, PMIX_MAX_NSLEN); \ + foobar.rank = r; \ + TEST_VERBOSE(("%s:%d want to get from %s:%d key %s", my_nspace, my_rank, ns, r, key)); \ + if (blocking) { \ + if (PMIX_SUCCESS != (rc = PMIx_Get(&foobar, key, NULL, 0, &val))) { \ + if( !( rc == PMIX_ERR_NOT_FOUND && ok_notfnd ) ){ \ + TEST_ERROR(("%s:%d: PMIx_Get failed: %d from %s:%d, key %s", my_nspace, my_rank, rc, ns, r, key)); \ + } \ + rc = PMIX_ERROR; \ + } \ + } else { \ + int count; \ + cbdata.in_progress = 1; \ + PMIX_VALUE_CREATE(val, 1); \ + cbdata.kv = val; \ + if (PMIX_SUCCESS != (rc = PMIx_Get_nb(&foobar, key, NULL, 0, get_cb, (void*)&cbdata))) { \ + TEST_VERBOSE(("%s:%d: PMIx_Get_nb failed: %d from %s:%d, key=%s", my_nspace, my_rank, rc, ns, r, key)); \ + rc = PMIX_ERROR; \ + } else { \ + count = 0; \ + while(cbdata.in_progress){ \ + struct timespec ts; \ + ts.tv_sec = 0; \ + ts.tv_nsec = 100; \ + nanosleep(&ts,NULL); \ + count++; \ + } \ + } \ + } \ + if (PMIX_SUCCESS == rc) { \ + if( PMIX_SUCCESS != cbdata.status ){ \ + if( !( rc == PMIX_ERR_NOT_FOUND && ok_notfnd ) ){ \ + TEST_VERBOSE(("%s:%d: PMIx_Get_nb failed: %d from %s:%d, key=%s", \ + my_nspace, my_rank, rc, my_nspace, r)); \ + } \ + rc = PMIX_ERROR; \ + } else if (NULL == val) { \ + TEST_VERBOSE(("%s:%d: PMIx_Get returned NULL value", my_nspace, my_rank)); \ + rc = PMIX_ERROR; \ + } \ + else if (val->type != PMIX_VAL_TYPE_ ## dtype || PMIX_VAL_CMP(dtype, PMIX_VAL_FIELD_ ## dtype((val)), data)) { \ + TEST_VERBOSE(("%s:%u: from %s:%d Key %s value or type mismatch," \ + " want type %d get type %d", \ + my_nspace, my_rank, ns, r, key, PMIX_VAL_TYPE_ ## dtype, val->type)); \ + rc = PMIX_ERROR; \ + } \ + } \ + if (PMIX_SUCCESS == rc) { \ + TEST_VERBOSE(("%s:%d: GET OF %s from %s:%d SUCCEEDED", my_nspace, my_rank, key, ns, r)); \ + PMIX_VALUE_RELEASE(val); \ + } \ +} while (0) + +#define FENCE(blocking, data_ex, pcs, nprocs) do { \ + if( blocking ){ \ + pmix_info_t *info = NULL; \ + size_t ninfo = 0; \ + if (data_ex) { \ + bool value = 1; \ + PMIX_INFO_CREATE(info, 1); \ + (void)strncpy(info->key, PMIX_COLLECT_DATA, PMIX_MAX_KEYLEN); \ + pmix_value_load(&info->value, &value, PMIX_BOOL); \ + ninfo = 1; \ + } \ + rc = PMIx_Fence(pcs, nprocs, info, ninfo); \ + PMIX_INFO_FREE(info, ninfo); \ + } else { \ + int in_progress = 1, count; \ + rc = PMIx_Fence_nb(pcs, nprocs, NULL, 0, release_cb, &in_progress); \ + if ( PMIX_SUCCESS == rc ) { \ + count = 0; \ + while( in_progress ){ \ + struct timespec ts; \ + ts.tv_sec = 0; \ + ts.tv_nsec = 100; \ + nanosleep(&ts,NULL); \ + count++; \ + } \ + TEST_VERBOSE(("PMIx_Fence_nb(barrier,collect): free time: %lfs", \ + count*100*1E-9)); \ + } \ + } \ + if (PMIX_SUCCESS == rc) { \ + TEST_VERBOSE(("%s:%d: Fence successfully completed", \ + my_nspace, my_rank)); \ + } \ +} while (0) + #endif // TEST_COMMON_H diff --git a/opal/mca/pmix/pmix2x/pmix/test/test_fence.c b/opal/mca/pmix/pmix2x/pmix/test/test_fence.c index b3f059aab3..e7c0a086be 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/test_fence.c +++ b/opal/mca/pmix/pmix2x/pmix/test/test_fence.c @@ -1,6 +1,6 @@ /* * Copyright (c) 2016-2017 Intel, Inc. All rights reserved. - * Copyright (c) 2015 Mellanox Technologies, Inc. + * Copyright (c) 2015-2017 Mellanox Technologies, Inc. * All rights reserved. * $COPYRIGHT$ * @@ -13,18 +13,6 @@ #include "test_fence.h" #include "src/buffer_ops/buffer_ops.h" -static void release_cb(pmix_status_t status, void *cbdata) -{ - int *ptr = (int*)cbdata; - *ptr = 0; -} - -typedef struct { - int in_progress; - pmix_value_t *kv; - int status; -} get_cbdata; - static void get_cb(pmix_status_t status, pmix_value_t *kv, void *cbdata) { get_cbdata *cb = (get_cbdata*)cbdata; @@ -58,121 +46,11 @@ static void add_noise(char *noise_param, char *my_nspace, pmix_rank_t my_rank) } } -#define SET_KEY(key, fence_num, ind, use_same_keys) do { \ - if (use_same_keys) { \ - (void)snprintf(key, sizeof(key)-1, "key-%d", ind); \ - } else { \ - (void)snprintf(key, sizeof(key)-1, "key-f%d:%d", fence_num, ind); \ - } \ -} while (0) - -#define PUT(dtype, data, flag, fence_num, ind, use_same_keys) do { \ - char key[50]; \ - pmix_value_t value; \ - SET_KEY(key, fence_num, ind, use_same_keys); \ - PMIX_VAL_SET(&value, dtype, data); \ - TEST_VERBOSE(("%s:%d put key %s", my_nspace, my_rank, key)); \ - if (PMIX_SUCCESS != (rc = PMIx_Put(flag, key, &value))) { \ - TEST_ERROR(("%s:%d: PMIx_Put key %s failed: %d", my_nspace, my_rank, key, rc)); \ - rc = PMIX_ERROR; \ - } \ - PMIX_VALUE_DESTRUCT(&value); \ -} while (0) - -#define GET(dtype, data, ns, r, fence_num, ind, use_same_keys, blocking, ok_notfnd) do { \ - char key[50]; \ - pmix_value_t *val; \ - get_cbdata cbdata; \ - cbdata.status = PMIX_SUCCESS; \ - pmix_proc_t foobar; \ - SET_KEY(key, fence_num, ind, use_same_keys); \ - (void)strncpy(foobar.nspace, ns, PMIX_MAX_NSLEN); \ - foobar.rank = r; \ - TEST_VERBOSE(("%s:%d want to get from %s:%d key %s", my_nspace, my_rank, ns, r, key)); \ - if (blocking) { \ - if (PMIX_SUCCESS != (rc = PMIx_Get(&foobar, key, NULL, 0, &val))) { \ - if( !( rc == PMIX_ERR_NOT_FOUND && ok_notfnd ) ){ \ - TEST_ERROR(("%s:%d: PMIx_Get failed: %d from %s:%d, key %s", my_nspace, my_rank, rc, ns, r, key)); \ - } \ - rc = PMIX_ERROR; \ - } \ - } else { \ - int count; \ - cbdata.in_progress = 1; \ - PMIX_VALUE_CREATE(val, 1); \ - cbdata.kv = val; \ - if (PMIX_SUCCESS != (rc = PMIx_Get_nb(&foobar, key, NULL, 0, get_cb, (void*)&cbdata))) { \ - TEST_VERBOSE(("%s:%d: PMIx_Get_nb failed: %d from %s:%d, key=%s", my_nspace, my_rank, rc, ns, r, key)); \ - rc = PMIX_ERROR; \ - } else { \ - count = 0; \ - while(cbdata.in_progress){ \ - struct timespec ts; \ - ts.tv_sec = 0; \ - ts.tv_nsec = 100; \ - nanosleep(&ts,NULL); \ - count++; \ - } \ - } \ - } \ - if (PMIX_SUCCESS == rc) { \ - if( PMIX_SUCCESS != cbdata.status ){ \ - if( !( rc == PMIX_ERR_NOT_FOUND && ok_notfnd ) ){ \ - TEST_VERBOSE(("%s:%d: PMIx_Get_nb failed: %d from %s:%d, key=%s", \ - my_nspace, my_rank, rc, my_nspace, r)); \ - } \ - rc = PMIX_ERROR; \ - } else if (NULL == val) { \ - TEST_VERBOSE(("%s:%d: PMIx_Get returned NULL value", my_nspace, my_rank)); \ - rc = PMIX_ERROR; \ - } \ - else if (val->type != PMIX_VAL_TYPE_ ## dtype || PMIX_VAL_CMP(dtype, PMIX_VAL_FIELD_ ## dtype((val)), data)) { \ - TEST_VERBOSE(("%s:%u: from %s:%d Key %s value or type mismatch," \ - " want type %d get type %d", \ - my_nspace, my_rank, ns, r, key, PMIX_VAL_TYPE_ ## dtype, val->type)); \ - rc = PMIX_ERROR; \ - } \ - } \ - if (PMIX_SUCCESS == rc) { \ - TEST_VERBOSE(("%s:%d: GET OF %s from %s:%d SUCCEEDED", my_nspace, my_rank, key, ns, r)); \ - PMIX_VALUE_RELEASE(val); \ - } \ -} while (0) - -#define FENCE(blocking, data_ex, pcs, nprocs) do { \ - if( blocking ){ \ - pmix_info_t *info = NULL; \ - size_t ninfo = 0; \ - if (data_ex) { \ - bool value = 1; \ - PMIX_INFO_CREATE(info, 1); \ - (void)strncpy(info->key, PMIX_COLLECT_DATA, PMIX_MAX_KEYLEN); \ - pmix_value_load(&info->value, &value, PMIX_BOOL); \ - ninfo = 1; \ - } \ - rc = PMIx_Fence(pcs, nprocs, info, ninfo); \ - PMIX_INFO_FREE(info, ninfo); \ - } else { \ - int in_progress = 1, count; \ - rc = PMIx_Fence_nb(pcs, nprocs, NULL, 0, release_cb, &in_progress); \ - if ( PMIX_SUCCESS == rc ) { \ - count = 0; \ - while( in_progress ){ \ - struct timespec ts; \ - ts.tv_sec = 0; \ - ts.tv_nsec = 100; \ - nanosleep(&ts,NULL); \ - count++; \ - } \ - TEST_VERBOSE(("PMIx_Fence_nb(barrier,collect): free time: %lfs", \ - count*100*1E-9)); \ - } \ - } \ - if (PMIX_SUCCESS == rc) { \ - TEST_VERBOSE(("%s:%d: Fence successfully completed", \ - my_nspace, my_rank)); \ - } \ -} while (0) +static void release_cb(pmix_status_t status, void *cbdata) +{ + int *ptr = (int*)cbdata; + *ptr = 0; +} int test_fence(test_params params, char *my_nspace, pmix_rank_t my_rank) { @@ -194,6 +72,8 @@ int test_fence(test_params params, char *my_nspace, pmix_rank_t my_rank) PMIX_CONSTRUCT(&test_fences, pmix_list_t); parse_fence(params.fences, 1); + TEST_VERBOSE(("fences %s\n", params.fences)); + /* cycle thru all the test fence descriptors to find * those that include my nspace/rank */ PMIX_LIST_FOREACH(desc, &test_fences, fence_desc_t) { diff --git a/opal/mca/pmix/pmix2x/pmix/test/test_internal.c b/opal/mca/pmix/pmix2x/pmix/test/test_internal.c new file mode 100644 index 0000000000..f9e2238dd2 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/test/test_internal.c @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2017 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2017 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + */ + +#include "test_internal.h" + +static void release_cb(pmix_status_t status, void *cbdata) +{ + int *ptr = (int*)cbdata; + *ptr = 0; +} + +static void get_cb(pmix_status_t status, pmix_value_t *kv, void *cbdata) +{ + get_cbdata *cb = (get_cbdata*)cbdata; + if (PMIX_SUCCESS == status) { + pmix_value_xfer(cb->kv, kv); + } + cb->in_progress = 0; + cb->status = status; +} + +int test_internal(char *my_nspace, pmix_rank_t my_rank, test_params params) { + int idx; + char sval[PMIX_MAX_NSLEN]; + char key[PMIX_MAX_KEYLEN]; + pmix_value_t value; + pmix_proc_t proc; + pmix_status_t rc; + + PMIX_PROC_CONSTRUCT(&proc); + (void)strncpy(proc.nspace, my_nspace, PMIX_MAX_NSLEN); + proc.rank = my_rank; + + for (idx = 0; idx < params.test_internal; idx++) { + memset(sval, 0, PMIX_MAX_NSLEN); + sprintf(sval, "test_internal:%s:%d:%d", my_nspace, my_rank, idx); + + SET_KEY(key, 0, idx, 1); + value.type = PMIX_STRING; + value.data.string = sval; + if (PMIX_SUCCESS != (rc = PMIx_Store_internal(&proc, key, &value))) { + TEST_ERROR(("%s:%d: PMIx_Store_internal failed: %d", my_nspace, my_rank, rc)); + PMIX_PROC_DESTRUCT(&proc); + return PMIX_ERROR; + } + } + + /* Submit the data */ + if (PMIX_SUCCESS != (rc = PMIx_Commit())) { + TEST_ERROR(("%s:%d: PMIx_Commit failed: %d", my_nspace, my_rank, rc)); + PMIX_LIST_DESTRUCT(&key_replace); + PMIX_PROC_DESTRUCT(&proc); + return PMIX_ERROR; + } + + proc.rank = PMIX_RANK_WILDCARD; + FENCE(1, 1, (&proc), 1); + if (PMIX_SUCCESS != rc) { + TEST_ERROR(("%s:%d: PMIx_Fence failed: %d", my_nspace, my_rank, rc)); + PMIX_LIST_DESTRUCT(&key_replace); + PMIX_PROC_DESTRUCT(&proc); + return rc; + } + + for (idx = 0; idx < params.test_internal; idx++) { + memset(sval, 0, PMIX_MAX_NSLEN); + sprintf(sval, "test_internal:%s:%d:%d", my_nspace, my_rank, idx); + + GET(string, sval, my_nspace, my_rank, 0, idx, 1, 1, 0); + if (PMIX_SUCCESS != rc) { + TEST_ERROR(("%s:%d: PMIx_Get of remote key on local proc", my_nspace, my_rank)); + PMIX_PROC_DESTRUCT(&proc); + return PMIX_ERROR; + } + } + + PMIX_PROC_DESTRUCT(&proc); + return PMIX_SUCCESS; +} diff --git a/opal/mca/pmix/pmix2x/pmix/test/test_internal.h b/opal/mca/pmix/pmix2x/pmix/test/test_internal.h new file mode 100644 index 0000000000..3ed71db5d8 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/test/test_internal.h @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2017 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2017 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + */ + +#include +#include + +#include "test_common.h" + +int test_internal(char *my_nspace, pmix_rank_t my_rank, test_params params); diff --git a/opal/mca/pmix/pmix2x/pmix/test/test_replace.c b/opal/mca/pmix/pmix2x/pmix/test/test_replace.c new file mode 100644 index 0000000000..523c96445d --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/test/test_replace.c @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2017 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2017 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + */ + +#include "test_replace.h" + +static void release_cb(pmix_status_t status, void *cbdata) +{ + int *ptr = (int*)cbdata; + *ptr = 0; +} + +static void get_cb(pmix_status_t status, pmix_value_t *kv, void *cbdata) +{ + get_cbdata *cb = (get_cbdata*)cbdata; + if (PMIX_SUCCESS == status) { + pmix_value_xfer(cb->kv, kv); + } + cb->in_progress = 0; + cb->status = status; +} + +static int key_is_replace(int key_idx) { + key_replace_t *item; + + PMIX_LIST_FOREACH(item, &key_replace, key_replace_t) { + if (item->key_idx == key_idx) + return 1; + } + return 0; +} + +int test_replace(char *my_nspace, pmix_rank_t my_rank, test_params params) { + int key_idx = 0; + int key_cnt = 0; + char sval[PMIX_MAX_NSLEN]; + pmix_proc_t proc; + pmix_status_t rc; + key_replace_t *item; + + PMIX_CONSTRUCT(&key_replace, pmix_list_t); + parse_replace(params.key_replace, 1, &key_cnt); + + for (key_idx = 0; key_idx < key_cnt; key_idx++) { + memset(sval, 0, PMIX_MAX_NSLEN); + sprintf(sval, "test_replace:%s:%d:%d", my_nspace, my_rank, key_idx); + + PUT(string, sval, PMIX_GLOBAL, 0, key_idx, 1); + if (PMIX_SUCCESS != rc) { + TEST_ERROR(("%s:%d: PMIx_Put failed: %d", my_nspace, my_rank, rc)); + PMIX_LIST_DESTRUCT(&key_replace); + return rc; + } + } + + PMIX_PROC_CONSTRUCT(&proc); + (void)strncpy(proc.nspace, my_nspace, PMIX_MAX_NSLEN); + proc.rank = PMIX_RANK_WILDCARD; + + /* Submit the data */ + if (PMIX_SUCCESS != (rc = PMIx_Commit())) { + TEST_ERROR(("%s:%d: PMIx_Commit failed: %d", my_nspace, my_rank, rc)); + PMIX_LIST_DESTRUCT(&key_replace); + PMIX_PROC_DESTRUCT(&proc); + return PMIX_ERROR; + } + + FENCE(1, 1, (&proc), 1); + if (PMIX_SUCCESS != rc) { + TEST_ERROR(("%s:%d: PMIx_Fence failed: %d", my_nspace, my_rank, rc)); + PMIX_LIST_DESTRUCT(&key_replace); + PMIX_PROC_DESTRUCT(&proc); + return rc; + } + + PMIX_LIST_FOREACH(item, &key_replace, key_replace_t) { + memset(sval, 0, PMIX_MAX_NSLEN); + sprintf(sval, "test_replace:%s:%d:%d: replaced key", my_nspace, my_rank, item->key_idx); + + PUT(string, sval, PMIX_GLOBAL, 0, item->key_idx, 1); + if (PMIX_SUCCESS != rc) { + TEST_ERROR(("%s:%d: PMIx_Put failed: %d", my_nspace, my_rank, rc)); + PMIX_LIST_DESTRUCT(&key_replace); + PMIX_PROC_DESTRUCT(&proc); + return rc; + } + } + + + /* Submit the data */ + if (PMIX_SUCCESS != (rc = PMIx_Commit())) { + TEST_ERROR(("%s:%d: PMIx_Commit failed: %d", my_nspace, my_rank, rc)); + PMIX_LIST_DESTRUCT(&key_replace); + PMIX_PROC_DESTRUCT(&proc); + return PMIX_ERROR; + } + + FENCE(1, 1, (&proc), 1); + if (PMIX_SUCCESS != rc) { + TEST_ERROR(("%s:%d: PMIx_Fence failed: %d", my_nspace, my_rank, rc)); + PMIX_LIST_DESTRUCT(&key_replace); + PMIX_PROC_DESTRUCT(&proc); + return rc; + } + + for (key_idx = 0; key_idx < key_cnt; key_idx++) { + memset(sval, 0, PMIX_MAX_NSLEN); + + if (key_is_replace(key_idx)) { + sprintf(sval, "test_replace:%s:%d:%d: replaced key", my_nspace, my_rank, key_idx); + } else { + sprintf(sval, "test_replace:%s:%d:%d", my_nspace, my_rank, key_idx); + } + + + GET(string, sval, my_nspace, my_rank, 0, key_idx, 1, 1, 0); + if (PMIX_SUCCESS != rc) { + TEST_ERROR(("%s:%d: PMIx_Get of remote key on local proc", my_nspace, my_rank)); + PMIX_LIST_DESTRUCT(&key_replace); + PMIX_PROC_DESTRUCT(&proc); + return PMIX_ERROR; + } + } + + PMIX_LIST_DESTRUCT(&key_replace); + PMIX_PROC_DESTRUCT(&proc); + return PMIX_SUCCESS; +} diff --git a/opal/mca/pmix/pmix2x/pmix/test/test_replace.h b/opal/mca/pmix/pmix2x/pmix/test/test_replace.h new file mode 100644 index 0000000000..e91f13ca61 --- /dev/null +++ b/opal/mca/pmix/pmix2x/pmix/test/test_replace.h @@ -0,0 +1,18 @@ +/* + * Copyright (c) 2017 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2017 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + */ + +#include +#include + +#include "test_common.h" + +int test_replace(char *my_nspace, pmix_rank_t my_rank, test_params params); diff --git a/opal/mca/pmix/pmix2x/pmix/test/utils.c b/opal/mca/pmix/pmix2x/pmix/test/utils.c index afd75182af..895025e6e3 100644 --- a/opal/mca/pmix/pmix2x/pmix/test/utils.c +++ b/opal/mca/pmix/pmix2x/pmix/test/utils.c @@ -1,6 +1,6 @@ /* * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. - * Copyright (c) 2015 Mellanox Technologies, Inc. + * Copyright (c) 2015-2017 Mellanox Technologies, Inc. * All rights reserved. * Copyright (c) 2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -155,6 +155,16 @@ void set_client_argv(test_params *params, char ***argv) if (params->test_error) { pmix_argv_append_nosize(argv, "--test-error"); } + if (params->key_replace) { + pmix_argv_append_nosize(argv, "--test-replace"); + pmix_argv_append_nosize(argv, params->key_replace); + } + if (params->test_internal) { + char tmp[32]; + sprintf(tmp, "%d", params->test_internal); + pmix_argv_append_nosize(argv, "--test-internal"); + pmix_argv_append_nosize(argv, tmp); + } } int launch_clients(int num_procs, char *binary, char *** client_env, char ***base_argv)