diff --git a/opal/mca/pmix/pmix4x/pmix/VERSION b/opal/mca/pmix/pmix4x/pmix/VERSION index 863b4fed9c..99dd004a35 100644 --- a/opal/mca/pmix/pmix4x/pmix/VERSION +++ b/opal/mca/pmix/pmix4x/pmix/VERSION @@ -30,7 +30,7 @@ greek=a1 # command, or with the date (if "git describe" fails) in the form of # "date". -repo_rev=gitbde4a8a5 +repo_rev=git186dca19 # If tarball_version is not empty, it is used as the version string in # the tarball filename, regardless of all other versions listed in @@ -44,7 +44,7 @@ tarball_version= # The date when this release was created -date="Apr 23, 2019" +date="Jun 10, 2019" # The shared library version of each of PMIx's public libraries. # These versions are maintained in accordance with the "Library diff --git a/opal/mca/pmix/pmix4x/pmix/bindings/python/Makefile.am b/opal/mca/pmix/pmix4x/pmix/bindings/python/Makefile.am index 01339d34cf..6679463be4 100644 --- a/opal/mca/pmix/pmix4x/pmix/bindings/python/Makefile.am +++ b/opal/mca/pmix/pmix4x/pmix/bindings/python/Makefile.am @@ -21,7 +21,7 @@ # $HEADER$ # -helpers = setup.py client.py server.py pmix.pyx +helpers = setup.py pmix.pyx if WANT_PYTHON_BINDINGS diff --git a/opal/mca/pmix/pmix4x/pmix/bindings/python/client.py.in b/opal/mca/pmix/pmix4x/pmix/bindings/python/client.py.in new file mode 100755 index 0000000000..80112edbc4 --- /dev/null +++ b/opal/mca/pmix/pmix4x/pmix/bindings/python/client.py.in @@ -0,0 +1,33 @@ +@PMIX_PYTHON_PATH@ + +from pmix import * + +def main(): + foo = PMIxClient() + print("Testing PMIx ", foo.get_version()) + info = {PMIX_PROGRAMMING_MODEL: ('TEST', PMIX_STRING), PMIX_MODEL_LIBRARY_NAME: ("PMIX", PMIX_STRING)} + my_result = foo.init(info) + print("Init result ", my_result) + if 0 != my_result: + print("FAILED TO INIT") + exit(1) + # try putting something + print("PUT") + rc = foo.put(PMIX_GLOBAL, "mykey", (1, PMIX_INT32)) + print("Put result ", rc); + # commit it + print("COMMIT") + rc = foo.commit() + print ("Commit result ", rc) + # execute fence + print("FENCE") + procs = [] + info = {} + rc = foo.fence(procs, info) + print("Fence result ", rc) + # finalize + info = {} + foo.finalize(info) + print("Client finalize complete") +if __name__ == '__main__': + main() diff --git a/opal/mca/pmix/pmix4x/pmix/bindings/python/server.py.in b/opal/mca/pmix/pmix4x/pmix/bindings/python/server.py.in new file mode 100755 index 0000000000..8fdd0845bf --- /dev/null +++ b/opal/mca/pmix/pmix4x/pmix/bindings/python/server.py.in @@ -0,0 +1,107 @@ +@PMIX_PYTHON_PATH@ + +from pmix import * +import signal, time +import os +import select +import subprocess + +global killer + +class GracefulKiller: + kill_now = False + def __init__(self): + signal.signal(signal.SIGINT, self.exit_gracefully) + signal.signal(signal.SIGTERM, self.exit_gracefully) + + def exit_gracefully(self,signum, frame): + self.kill_now = True + +def clientconnected(proc:tuple is not None): + print("CLIENT CONNECTED", proc) + return PMIX_SUCCESS + +def clientfinalized(proc:tuple is not None): + print("CLIENT FINALIZED", proc) + return PMIX_SUCCESS + +def clientfence(args:dict is not None): + print("SERVER FENCE", args) + return PMIX_SUCCESS + +def main(): + try: + foo = PMIxServer() + except: + print("FAILED TO CREATE SERVER") + exit(1) + print("Testing server version ", foo.get_version()) + args = {'FOOBAR': ('VAR', PMIX_STRING), 'BLAST': (7, PMIX_INT32)} + map = {'clientconnected': clientconnected, + 'clientfinalized': clientfinalized, + 'fencenb': clientfence} + my_result = foo.init(args, map) + print("Testing PMIx_Initialized") + rc = foo.initialized() + print("Initialized: ", rc) + vers = foo.get_version() + print("Version: ", vers) + # get our environment as a base + env = os.environ.copy() + # register an nspace for the client app + darray = (PMIX_SIZE, [1, 2, 3, 4, 5]) + kvals = {'testkey': (darray, PMIX_DATA_ARRAY)} + print("REGISTERING NSPACE") + rc = foo.register_nspace("testnspace", 1, kvals) + print("RegNspace ", rc) + # register a client + uid = os.getuid() + gid = os.getgid() + rc = foo.register_client(("testnspace", 0), uid, gid) + print("RegClient ", rc) + # setup the fork + rc = foo.setup_fork(("testnspace", 0), env) + print("SetupFrk", rc) + # setup the client argv + args = ["./client.py"] + # open a subprocess with stdout and stderr + # as distinct pipes so we can capture their + # output as the process runs + p = subprocess.Popen(args, env=env, + stdout=subprocess.PIPE, stderr=subprocess.PIPE) + # define storage to catch the output + stdout = [] + stderr = [] + # loop until the pipes close + while True: + reads = [p.stdout.fileno(), p.stderr.fileno()] + ret = select.select(reads, [], []) + + stdout_done = True + stderr_done = True + + for fd in ret[0]: + # if the data + if fd == p.stdout.fileno(): + read = p.stdout.readline() + if read: + read = read.decode('utf-8').rstrip() + print('stdout: ' + read) + stdout_done = False + elif fd == p.stderr.fileno(): + read = p.stderr.readline() + if read: + read = read.decode('utf-8').rstrip() + print('stderr: ' + read) + stderr_done = False + + if stdout_done and stderr_done: + break + + print("FINALIZING") + foo.finalize() + +if __name__ == '__main__': + global killer + killer = GracefulKiller() + main() diff --git a/opal/mca/pmix/pmix4x/pmix/config/pmix.m4 b/opal/mca/pmix/pmix4x/pmix/config/pmix.m4 index 9f9f382000..ad9aed6660 100644 --- a/opal/mca/pmix/pmix4x/pmix/config/pmix.m4 +++ b/opal/mca/pmix/pmix4x/pmix/config/pmix.m4 @@ -418,7 +418,8 @@ AC_DEFUN([PMIX_SETUP_CORE],[ ioLib.h sockLib.h hostLib.h limits.h \ sys/fcntl.h sys/statfs.h sys/statvfs.h \ netdb.h ucred.h zlib.h sys/auxv.h \ - sys/sysctl.h]) + sys/sysctl.h termio.h termios.h pty.h \ + libutil.h util.h grp.h sys/cdefs.h utmp.h stropts.h]) AC_CHECK_HEADERS([sys/mount.h], [], [], [AC_INCLUDES_DEFAULT @@ -663,7 +664,7 @@ AC_DEFUN([PMIX_SETUP_CORE],[ # -lrt might be needed for clock_gettime PMIX_SEARCH_LIBS_CORE([clock_gettime], [rt]) - AC_CHECK_FUNCS([asprintf snprintf vasprintf vsnprintf strsignal socketpair strncpy_s usleep statfs statvfs getpeereid getpeerucred strnlen posix_fallocate tcgetpgrp]) + AC_CHECK_FUNCS([asprintf snprintf vasprintf vsnprintf strsignal socketpair strncpy_s usleep statfs statvfs getpeereid getpeerucred strnlen posix_fallocate tcgetpgrp setpgid ptsname openpty]) # On some hosts, htonl is a define, so the AC_CHECK_FUNC will get # confused. On others, it's in the standard library, but stubbed with @@ -909,6 +910,10 @@ AC_DEFUN([PMIX_SETUP_CORE],[ pmix_config_prefix[src/tools/pps/Makefile] pmix_config_prefix[src/tools/pattrs/Makefile] ) + if test "$WANT_PYTHON_BINDINGS" = "1"; then + AC_CONFIG_FILES(pmix_config_prefix[bindings/python/server.py], [chmod +x bindings/python/server.py]) + AC_CONFIG_FILES(pmix_config_prefix[bindings/python/client.py], [chmod +x bindings/python/client.py]) + fi # publish any embedded flags so external wrappers can use them AC_SUBST(PMIX_EMBEDDED_LIBS) @@ -1215,6 +1220,7 @@ if test "$WANT_PYTHON_BINDINGS" = "1"; then fi python_version=`python --version 2>&1` PMIX_SUMMARY_ADD([[Bindings]],[[Python]], [pmix_python], [yes ($python_version)]) + AC_SUBST([PMIX_PYTHON_PATH], [#!"$PYTHON"], "Full Python executable path") AC_MSG_CHECKING([if Cython package installed]) have_cython=`$srcdir/config/pmix_check_cython.py 2> /dev/null` @@ -1251,6 +1257,40 @@ fi AS_IF([test -z "$enable_nonglobal_dlopen" && test "x$pmix_mode" = "xembedded" && test $WANT_INSTALL_HEADERS -eq 0 && test $pmix_need_libpmix -eq 1], [pmix_need_libpmix=0]) +# +# Do we want PTY support? +# + +AC_MSG_CHECKING([if want pty support]) +AC_ARG_ENABLE(pty-support, + AC_HELP_STRING([--enable-pty-support], + [Enable/disable PTY support for STDIO forwarding. (default: enabled)])) +if test "$enable_pty_support" = "no" ; then + AC_MSG_RESULT([no]) + PMIX_ENABLE_PTY_SUPPORT=0 +else + AC_MSG_RESULT([yes]) + PMIX_ENABLE_PTY_SUPPORT=1 +fi +AC_DEFINE_UNQUOTED([PMIX_ENABLE_PTY_SUPPORT], [$PMIX_ENABLE_PTY_SUPPORT], + [Whether user wants PTY support or not]) + +# +# psec/dummy_handshake +# + +AC_MSG_CHECKING([if want build psec/dummy_handshake]) +AC_ARG_ENABLE(dummy-handshake, + AC_HELP_STRING([--enable-dummy-handshake], + [Enables psec dummy component intended to check the PTL handshake scenario (default: disabled)])) +if test "$enable_dummy_handshake" != "yes"; then + AC_MSG_RESULT([no]) + eval "DISABLE_psec_dummy_handshake=1" +else + AC_MSG_RESULT([yes]) + eval "DISABLE_psec_dummy_handshake=0" +fi +AM_CONDITIONAL(MCA_BUILD_PSEC_DUMMY_HANDSHAKE, test "$DISABLE_psec_dummy_handshake" = "0") ])dnl # This must be a standalone routine so that it can be called both by diff --git a/opal/mca/pmix/pmix4x/pmix/config/pmix_check_psm2.m4 b/opal/mca/pmix/pmix4x/pmix/config/pmix_check_psm2.m4 deleted file mode 100644 index b2c291fae4..0000000000 --- a/opal/mca/pmix/pmix4x/pmix/config/pmix_check_psm2.m4 +++ /dev/null @@ -1,89 +0,0 @@ -# -*- shell-script -*- -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2006 The Regents of the University of California. -# All rights reserved. -# Copyright (c) 2006 QLogic Corp. All rights reserved. -# Copyright (c) 2009-2016 Cisco Systems, Inc. All rights reserved. -# Copyright (c) 2016-2017 Intel, Inc. All rights reserved. -# Copyright (c) 2015 Research Organization for Information Science -# and Technology (RIST). All rights reserved. -# Copyright (c) 2016 Los Alamos National Security, LLC. All rights -# reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -# PMIX_CHECK_PSM2(prefix, [action-if-found], [action-if-not-found]) -# -------------------------------------------------------- -# check if PSM2 support can be found. sets prefix_{CPPFLAGS, -# LDFLAGS, LIBS} as needed and runs action-if-found if there is -# support, otherwise executes action-if-not-found -AC_DEFUN([PMIX_CHECK_PSM2],[ - if test -z "$pmix_check_psm2_happy" ; then - AC_ARG_WITH([psm2], - [AC_HELP_STRING([--with-psm2(=DIR)], - [Build PSM2 (Intel PSM2) support, optionally adding DIR/include, DIR/lib, and DIR/lib64 to the search path for headers and libraries])]) - PMIX_CHECK_WITHDIR([psm2], [$with_psm2], [include/psm2.h]) - AC_ARG_WITH([psm2-libdir], - [AC_HELP_STRING([--with-psm2-libdir=DIR], - [Search for PSM (Intel PSM2) libraries in DIR])]) - PMIX_CHECK_WITHDIR([psm2-libdir], [$with_psm2_libdir], [libpsm2.*]) - - pmix_check_psm2_$1_save_CPPFLAGS="$CPPFLAGS" - pmix_check_psm2_$1_save_LDFLAGS="$LDFLAGS" - pmix_check_psm2_$1_save_LIBS="$LIBS" - - AS_IF([test "$with_psm2" != "no"], - [AS_IF([test ! -z "$with_psm2" && test "$with_psm2" != "yes"], - [pmix_check_psm2_dir="$with_psm2"]) - AS_IF([test ! -z "$with_psm2_libdir" && test "$with_psm2_libdir" != "yes"], - [pmix_check_psm2_libdir="$with_psm2_libdir"]) - - PMIX_CHECK_PACKAGE([pmix_check_psm2], - [psm2.h], - [psm2], - [psm2_mq_irecv2], - [], - [$pmix_check_psm2_dir], - [$pmix_check_psm2_libdir], - [pmix_check_psm2_happy="yes"], - [pmix_check_psm2_happy="no"])], - [pmix_check_psm2_happy="no"]) - - CPPFLAGS="$pmix_check_psm2_$1_save_CPPFLAGS" - LDFLAGS="$pmix_check_psm2_$1_save_LDFLAGS" - LIBS="$pmix_check_psm2_$1_save_LIBS" - - AS_IF([test "$pmix_check_psm2_happy" = "yes" && test "$enable_progress_threads" = "yes"], - [AC_MSG_WARN([PSM2 driver does not currently support progress threads. Disabling MTL.]) - pmix_check_psm2_happy="no"]) - - AS_IF([test "$pmix_check_psm2_happy" = "yes"], - [AC_CHECK_HEADERS( - glob.h, - [], - [AC_MSG_WARN([glob.h not found. Can not build component.]) - pmix_check_psm2_happy="no"])]) - - fi - - AS_IF([test "$pmix_check_psm2_happy" = "yes"], - [$1_LDFLAGS="[$]$1_LDFLAGS $pmix_check_psm2_LDFLAGS" - $1_CPPFLAGS="[$]$1_CPPFLAGS $pmix_check_psm2_CPPFLAGS" - $1_LIBS="[$]$1_LIBS $pmix_check_psm2_LIBS" - $2], - [AS_IF([test ! -z "$with_psm2" && test "$with_psm2" != "no"], - [AC_MSG_ERROR([PSM2 support requested but not found. Aborting])]) - $3]) -]) diff --git a/opal/mca/pmix/pmix4x/pmix/configure.ac b/opal/mca/pmix/pmix4x/pmix/configure.ac index 1adb2877e3..5d24d82aaf 100644 --- a/opal/mca/pmix/pmix4x/pmix/configure.ac +++ b/opal/mca/pmix/pmix4x/pmix/configure.ac @@ -19,7 +19,7 @@ # Copyright (c) 2012 Oracle and/or its affiliates. All rights reserved. # Copyright (c) 2013 Mellanox Technologies, Inc. # All rights reserved. -# Copyright (c) 2014-2018 Intel, Inc. All rights reserved. +# Copyright (c) 2014-2019 Intel, Inc. All rights reserved. # Copyright (c) 2016 IBM Corporation. All rights reserved. # Copyright (c) 2016-2018 Research Organization for Information Science # and Technology (RIST). All rights reserved. @@ -51,6 +51,11 @@ AC_CONFIG_AUX_DIR(./config) # -I in ACLOCAL_AMFLAGS in the top-level Makefile.am. AC_CONFIG_MACRO_DIR(./config) +# autotools expects to perform tests without interference +# from user-provided CFLAGS, so preserve them here +PMIX_CFLAGS_user=$CFLAGS +CFLAGS= + PMIX_CAPTURE_CONFIGURE_CLI([PMIX_CONFIGURE_CLI]) # Get our platform support file. This has to be done very, very early @@ -205,7 +210,17 @@ AS_IF([test -z "$CC_FOR_BUILD"],[ AC_SUBST([CC_FOR_BUILD], [$CC]) ]) +# restore any user-provided flags +AS_IF([test ! -z "$PMIX_CFLAGS_user"], [CFLAGS="$CFLAGS $PMIX_CFLAGS_user"]) + +# Delay setting pickyness until here so we +# don't break configure code tests +#if test "$WANT_PICKY_COMPILER" = "1"; then +# CFLAGS="$CFLAGS -Wall -Wextra -Werror" +#fi + # Cleanup duplicate flags +PMIX_FLAGS_UNIQ(CFLAGS) PMIX_FLAGS_UNIQ(CPPFLAGS) PMIX_FLAGS_UNIQ(LDFLAGS) PMIX_FLAGS_UNIQ(LIBS) @@ -232,6 +247,17 @@ AC_MSG_RESULT([$LDFLAGS]) AC_MSG_CHECKING([final LIBS]) AC_MSG_RESULT([$LIBS]) +#################################################################### +# -Werror for CI scripts +#################################################################### + +AC_ARG_ENABLE(werror, + AC_HELP_STRING([--enable-werror], + [Treat compiler warnings as errors]), +[ + CFLAGS="$CFLAGS -Werror" +]) + #################################################################### # Version information #################################################################### diff --git a/opal/mca/pmix/pmix4x/pmix/contrib/pmix.spec b/opal/mca/pmix/pmix4x/pmix/contrib/pmix.spec index 0139e85997..1b53681a31 100644 --- a/opal/mca/pmix/pmix4x/pmix/contrib/pmix.spec +++ b/opal/mca/pmix/pmix4x/pmix/contrib/pmix.spec @@ -231,6 +231,22 @@ scalability. This RPM contains all the tools necessary to compile and link against PMIx. +# if build_all_in_one_rpm = 0, build split packages +%if !%{build_all_in_one_rpm} +%package libpmi +Summary: PMI-1 and PMI-2 compatibility libraries +Requires: %{name}%{?_isa} = %{version}-%{release} +Conflicts: slurm-libpmi + +%description libpmi +The %{name}-libpmi package contains libpmi and libpmi2 libraries that provide +the respective APIs and a copy of the PMIx library – each API is translated +into its PMIx equivalent. This is especially targeted at apps/libs that are +hardcoded to dlopen “libpmi” or “libpmi2”. +This package conflicts sith slurm-libpmi, which provides its own, incompatible +versions of libpmi.so and libpmi2.so. +%endif + ############################################################################# # # Prepatory Section @@ -347,6 +363,10 @@ export CFLAGS CXXFLAGS FCFLAGS # We don't need that in an RPM. find $RPM_BUILD_ROOT -name config.log -exec rm -f {} \; +# If we build separate RPMs, then move the libpmi.* and libpmi2.* compat libs +# out of the way +find $RPM_BUILD_ROOT -name 'libpmi.' | xargs rm -f + # First, the [optional] modulefile %if %{install_modulefile} @@ -491,6 +511,19 @@ test "x$RPM_BUILD_ROOT" != "x" && rm -rf $RPM_BUILD_ROOT %endif %doc README INSTALL LICENSE +# if building separate RPMs, split the compatibility libs +%if !%{build_all_in_one_rpm} +%exclude %{_libdir}/libpmi.* +%exclude %{_libdir}/libpmi2.* +%exclude %{_includedir}/pmi.* +%exclude %{_includedir}/pmi2.* + +%files libpmi +%{_libdir}/libpmi.* +%{_libdir}/libpmi2.* +%{_includedir}/pmi.* +%{_includedir}/pmi2.* +%endif ############################################################################# # @@ -498,6 +531,11 @@ test "x$RPM_BUILD_ROOT" != "x" && rm -rf $RPM_BUILD_ROOT # ############################################################################# %changelog +* Tue Apr 30 2019 Kilian Cavalotti +- Enable multiple RPMs build to allow backward compatibility PMI-1 and PMI-2 + libs to be built separate. "rpmbuild --define 'build_all_in_one_rpm 0' ..." + will build separate pmix and pmix-libpmi RPMs. + * Tue Oct 17 2017 Ralph Castain - Add PMIx bin directory diff --git a/opal/mca/pmix/pmix4x/pmix/examples/asyncgroup.c b/opal/mca/pmix/pmix4x/pmix/examples/asyncgroup.c index 2fb12da9c2..449ddf0c9f 100644 --- a/opal/mca/pmix/pmix4x/pmix/examples/asyncgroup.c +++ b/opal/mca/pmix/pmix4x/pmix/examples/asyncgroup.c @@ -15,6 +15,8 @@ * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. + * Copyright (c) 2019 Triad National Security, LLC. All rights + * reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -126,7 +128,7 @@ static void invitefn(size_t evhdlr_registration_id, void *cbdata) { size_t n; - char *grp; + char *grp = NULL; pmix_status_t rc; /* if I am the leader, I can ignore this event */ diff --git a/opal/mca/pmix/pmix4x/pmix/examples/dmodex.c b/opal/mca/pmix/pmix4x/pmix/examples/dmodex.c index 76a1ac8ca0..c7b906b158 100644 --- a/opal/mca/pmix/pmix4x/pmix/examples/dmodex.c +++ b/opal/mca/pmix/pmix4x/pmix/examples/dmodex.c @@ -178,7 +178,6 @@ int main(int argc, char **argv) if (0 > asprintf(&tmp, "%s-%d-local", myproc.nspace, n)) { exit(1); } - (void)strncpy(proc.nspace, tmp, PMIX_MAX_NSLEN); proc.rank = n; if (PMIX_SUCCESS != (rc = PMIx_Get_nb(&proc, tmp, NULL, 0, valcbfunc, tmp))) { @@ -189,7 +188,6 @@ int main(int argc, char **argv) if (0 > asprintf(&tmp, "%s-%d-remote", myproc.nspace, n)) { exit(1); } - (void)strncpy(proc.nspace, tmp, PMIX_MAX_NSLEN); if (PMIX_SUCCESS != (rc = PMIx_Get_nb(&proc, tmp, NULL, 0, valcbfunc, tmp))) { fprintf(stderr, "Client ns %s rank %d: PMIx_Get %s failed: %d\n", myproc.nspace, n, tmp, rc); diff --git a/opal/mca/pmix/pmix4x/pmix/examples/fault.c b/opal/mca/pmix/pmix4x/pmix/examples/fault.c index abab388681..e01021f913 100644 --- a/opal/mca/pmix/pmix4x/pmix/examples/fault.c +++ b/opal/mca/pmix/pmix4x/pmix/examples/fault.c @@ -15,6 +15,8 @@ * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. + * Copyright (c) 2019 Triad National Security, LLC. All rights + * reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -45,7 +47,7 @@ static void notification_fn(size_t evhdlr_registration_id, { myrel_t *lock; bool found; - int exit_code; + int exit_code = 0; size_t n; pmix_proc_t *affected = NULL; diff --git a/opal/mca/pmix/pmix4x/pmix/examples/tool.c b/opal/mca/pmix/pmix4x/pmix/examples/tool.c index e10699309e..36b161f98e 100644 --- a/opal/mca/pmix/pmix4x/pmix/examples/tool.c +++ b/opal/mca/pmix/pmix4x/pmix/examples/tool.c @@ -15,6 +15,8 @@ * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Mellanox Technologies, Inc. All rights reserved. + * Copyright (c) 2019 Triad National Security, LLC. All rights + * reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -31,6 +33,8 @@ #include #include "examples.h" +static pmix_proc_t myproc = {"UNDEF", PMIX_RANK_UNDEF}; + static void cbfunc(pmix_status_t status, pmix_info_t *info, size_t ninfo, void *cbdata, @@ -61,10 +65,94 @@ static void cbfunc(pmix_status_t status, DEBUG_WAKEUP_THREAD(&mq->lock); } +/* this is an event notification function that we explicitly request + * be called when the PMIX_ERR_JOB_TERMINATED notification is issued. + * We could catch it in the general event notification function and test + * the status to see if it was "job terminated", but it often is simpler + * to declare a use-specific notification callback point. In this case, + * we are asking to know whenever a job terminates, and we will then + * know we can exit */ +static void release_fn(size_t evhdlr_registration_id, + pmix_status_t status, + const pmix_proc_t *source, + pmix_info_t info[], size_t ninfo, + pmix_info_t results[], size_t nresults, + pmix_event_notification_cbfunc_fn_t cbfunc, + void *cbdata) +{ + myrel_t *lock; + bool found; + int exit_code = 0; + size_t n; + pmix_proc_t *affected = NULL; + + /* find the return object */ + lock = NULL; + found = false; + for (n=0; n < ninfo; n++) { + if (0 == strncmp(info[n].key, PMIX_EVENT_RETURN_OBJECT, PMIX_MAX_KEYLEN)) { + lock = (myrel_t*)info[n].value.data.ptr; + /* not every RM will provide an exit code, but check if one was given */ + } else if (0 == strncmp(info[n].key, PMIX_EXIT_CODE, PMIX_MAX_KEYLEN)) { + exit_code = info[n].value.data.integer; + found = true; + } else if (0 == strncmp(info[n].key, PMIX_EVENT_AFFECTED_PROC, PMIX_MAX_KEYLEN)) { + affected = info[n].value.data.proc; + } + } + /* if the object wasn't returned, then that is an error */ + if (NULL == lock) { + fprintf(stderr, "LOCK WASN'T RETURNED IN RELEASE CALLBACK\n"); + /* let the event handler progress */ + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, NULL, 0, NULL, NULL, cbdata); + } + return; + } + + fprintf(stderr, "TOOL NOTIFIED THAT ALL CHILD PROCS %s TERMINATED \n", + (NULL == affected) ? "NULL" : affected->nspace); + if (found) { + if (!lock->exit_code_given) { + lock->exit_code = exit_code; + lock->exit_code_given = true; + } + } + DEBUG_WAKEUP_THREAD(&lock->lock); + + /* tell the event handler state machine that we are the last step */ + if (NULL != cbfunc) { + cbfunc(PMIX_EVENT_ACTION_COMPLETE, NULL, 0, NULL, NULL, cbdata); + } + return; +} + +/* event handler registration is done asynchronously because it + * may involve the PMIx server registering with the host RM for + * external events. So we provide a callback function that returns + * the status of the request (success or an error), plus a numerical index + * to the registered event. The index is used later on to deregister + * an event handler - if we don't explicitly deregister it, then the + * PMIx server will do so when it see us exit */ +static void evhandler_reg_callbk(pmix_status_t status, + size_t evhandler_ref, + void *cbdata) +{ + mylock_t *lock = (mylock_t*)cbdata; + + if (PMIX_SUCCESS != status) { + fprintf(stderr, "Client %s:%d EVENT HANDLER REGISTRATION FAILED WITH STATUS %d, ref=%lu\n", + myproc.nspace, myproc.rank, status, (unsigned long)evhandler_ref); + } + lock->status = status; + DEBUG_WAKEUP_THREAD(lock); +} + + int main(int argc, char **argv) { pmix_status_t rc; - pmix_proc_t myproc; + pmix_proc_t proc; pmix_query_t *query; size_t nq, ninfo = 0, n, m; myquery_data_t mydata; @@ -75,6 +163,12 @@ int main(int argc, char **argv) pmix_data_array_t *darray, *dptr; bool geturi = false; char hostname[1024]; + char *spawn = NULL; + pmix_app_t app; + pmix_nspace_t child; + pmix_status_t code = PMIX_ERR_JOB_TERMINATED; + myrel_t myrel; + mylock_t mylock; gethostname(hostname, 1024); for (n=1; n < (size_t)argc; n++) { @@ -84,24 +178,43 @@ int main(int argc, char **argv) exit(1); } server_uri = argv[n+1]; + ++n; + ++ninfo; } else if (0 == strcmp("-nspace", argv[n]) || 0 == strcmp("--nspace", argv[n])) { if (NULL == argv[n+1]) { fprintf(stderr, "Must provide nspace argument to %s option\n", argv[n]); exit(1); } nspace = argv[n+1]; + ++n; } else if (0 == strcmp("-uri", argv[n]) || 0 == strcmp("--uri", argv[n])) { /* retrieve the PMIx server's uri from the indicated node */ nodename = argv[n+1]; geturi = true; + ++n; + } else if (0 == strcmp("-spawn", argv[n]) || 0 == strcmp("--spawn", argv[n])) { + if (NULL == argv[n+1]) { + fprintf(stderr, "Must provide executable argument to %s option\n", argv[n]); + exit(1); + } + spawn = argv[n+1]; + ++n; + ninfo += 2; } } + PMIX_INFO_CREATE(info, ninfo); + n = 0; if (NULL != server_uri) { - ninfo = 1; - PMIX_INFO_CREATE(info, ninfo); - PMIX_INFO_LOAD(&info[0], PMIX_SERVER_URI, server_uri, PMIX_STRING); + PMIX_INFO_LOAD(&info[n], PMIX_SERVER_URI, server_uri, PMIX_STRING); fprintf(stderr, "Connecting to %s\n", server_uri); + ++n; + } + if (NULL != spawn) { + PMIX_INFO_LOAD(&info[n], PMIX_TOOL_CONNECT_OPTIONAL, NULL, PMIX_BOOL); + ++n; + PMIX_INFO_LOAD(&info[n], PMIX_LAUNCHER, NULL, PMIX_BOOL); + ++n; } /* init us */ @@ -144,6 +257,39 @@ int main(int argc, char **argv) goto done; } + /* if we want to spawn a proc, then do so */ + if (NULL != spawn) { + DEBUG_CONSTRUCT_LOCK(&myrel.lock); + /* setup notification so we know when the child has terminated */ + PMIX_INFO_CREATE(info, 2); + PMIX_INFO_LOAD(&info[0], PMIX_EVENT_RETURN_OBJECT, &myrel, PMIX_POINTER); + /* only call me back when this specific job terminates - the + * children will have our nspace */ + PMIX_LOAD_PROCID(&proc, myproc.nspace, PMIX_RANK_WILDCARD); + PMIX_INFO_LOAD(&info[1], PMIX_EVENT_AFFECTED_PROC, &proc, PMIX_PROC); + + DEBUG_CONSTRUCT_LOCK(&mylock); + PMIx_Register_event_handler(&code, 1, info, 2, + release_fn, evhandler_reg_callbk, (void*)&mylock); + DEBUG_WAIT_THREAD(&mylock); + rc = mylock.status; + DEBUG_DESTRUCT_LOCK(&mylock); + PMIX_INFO_FREE(info, 2); + + PMIX_APP_CONSTRUCT(&app); + PMIX_ARGV_SPLIT(app.argv, spawn, ' '); + app.cmd = strdup(app.argv[0]); + app.maxprocs = 1; + rc = PMIx_Spawn(NULL, 0, &app, 1, child); + if (PMIX_SUCCESS != rc && PMIX_OPERATION_SUCCEEDED != rc) { + fprintf(stderr, "Failed to spawn %s\n", PMIx_Error_string(rc)); + goto done; + } + /* wait here */ + DEBUG_WAIT_THREAD(&myrel.lock); + goto done; + } + if (NULL == nspace) { /* query the list of active nspaces */ nq = 1; @@ -218,6 +364,6 @@ int main(int argc, char **argv) done: /* finalize us */ - PMIx_Finalize(NULL, 0); + PMIx_tool_finalize(); return(rc); } diff --git a/opal/mca/pmix/pmix4x/pmix/include/Makefile.am b/opal/mca/pmix/pmix4x/pmix/include/Makefile.am index 1776131bbf..0f118ceadb 100644 --- a/opal/mca/pmix/pmix4x/pmix/include/Makefile.am +++ b/opal/mca/pmix/pmix4x/pmix/include/Makefile.am @@ -1,5 +1,5 @@ # -# Copyright (c) 2015-2018 Intel, Inc. All rights reserved. +# Copyright (c) 2015-2019 Intel, Inc. All rights reserved. # # $COPYRIGHT$ # @@ -15,7 +15,8 @@ include_HEADERS = \ pmix.h \ pmix_server.h \ pmix_tool.h \ - pmix_extend.h + pmix_extend.h \ + pmix_sched.h if WANT_PMI_BACKWARD include_HEADERS += \ diff --git a/opal/mca/pmix/pmix4x/pmix/include/pmix_common.h.in b/opal/mca/pmix/pmix4x/pmix/include/pmix_common.h.in index 42994df52b..e53d83d5cd 100644 --- a/opal/mca/pmix/pmix4x/pmix/include/pmix_common.h.in +++ b/opal/mca/pmix/pmix4x/pmix/include/pmix_common.h.in @@ -157,6 +157,7 @@ typedef uint32_t pmix_rank_t; #define PMIX_SERVER_GATEWAY "pmix.srv.gway" // (bool) Server is acting as a gateway for PMIx requests // that cannot be serviced on backend nodes // (e.g., logging to email) +#define PMIX_SERVER_SCHEDULER "pmix.src.sched" // (bool) Server supports system scheduler /* tool-related attributes */ #define PMIX_TOOL_NSPACE "pmix.tool.nspace" // (char*) Name of the nspace to use for this tool @@ -172,6 +173,8 @@ typedef uint32_t pmix_rank_t; #define PMIX_TOOL_DO_NOT_CONNECT "pmix.tool.nocon" // (bool) the tool wants to use internal PMIx support, but does // not want to connect to a PMIx server // from the specified processes to this tool +#define PMIX_TOOL_CONNECT_OPTIONAL "pmix.tool.conopt" // (bool) tool shall connect to a server if available, but otherwise + // continue to operate unconnected #define PMIX_RECONNECT_SERVER "pmix.cnct.recon" // (bool) tool is requesting to change server connections #define PMIX_LAUNCHER "pmix.tool.launcher" // (bool) tool is a launcher and needs rendezvous files created #define PMIX_LAUNCHER_RENDEZVOUS_FILE "pmix.tool.lncrnd" // (char*) Pathname of file where connection info is to be stored @@ -268,7 +271,11 @@ typedef uint32_t pmix_rank_t; #define PMIX_NETWORK_COORDINATE "pmix.net.coord" // (pmix_coord_t*) Network coordinate of the specified process #define PMIX_NETWORK_COORD_SYSTEM "pmix.net.coordsys" // (pmix_coord_system_t) Network coordinate system being employed to // describe the specified network plane - +#define PMIX_NETWORK_PLANE "pmix.net.plane" // (char*) string ID of a network plane +#define PMIX_NETWORK_SWITCH "pmix.net.switch" // (char*) string ID of a network switch +#define PMIX_NETWORK_NIC "pmix.net.nic" // (char*) string ID of a NIC +#define PMIX_NETWORK_ENDPT "pmix.net.endpt" // (assigned) network endpt for process - type assigned by + // fabric provider /* size info */ #define PMIX_UNIV_SIZE "pmix.univ.size" // (uint32_t) #procs in this nspace @@ -426,6 +433,9 @@ typedef uint32_t pmix_rank_t; // from the spawned processes to this process (typically used by a tool) #define PMIX_SPAWN_TOOL "pmix.spwn.tool" // (bool) job being spawned is a tool #define PMIX_CMD_LINE "pmix.cmd.line" // (char*) command line executing in the specified nspace +#define PMIX_FORK_EXEC_AGENT "pmix.fe.agnt" // (char*) command line of fork/exec agent to be used for starting + // local processes + /* query attributes */ #define PMIX_QUERY_REFRESH_CACHE "pmix.qry.rfsh" // (bool) retrieve updated information from server @@ -891,6 +901,7 @@ typedef int pmix_status_t; #define PMIX_ERR_IOF_FAILURE -171 #define PMIX_ERR_IOF_COMPLETE -172 #define PMIX_LAUNCH_COMPLETE -173 // include nspace of the launched job with notification +#define PMIX_FABRIC_UPDATED -174 /* system failures */ #define PMIX_ERR_NODE_DOWN -231 diff --git a/opal/mca/pmix/pmix4x/pmix/include/pmix_rename.h.in b/opal/mca/pmix/pmix4x/pmix/include/pmix_rename.h.in index e5a74b5c2e..a06bbfdfde 100644 --- a/opal/mca/pmix/pmix4x/pmix/include/pmix_rename.h.in +++ b/opal/mca/pmix/pmix4x/pmix/include/pmix_rename.h.in @@ -444,6 +444,7 @@ #define pmix_output_close @PMIX_RENAME@pmix_output_close #define pmix_output_finalize @PMIX_RENAME@pmix_output_finalize #define pmix_output_get_verbosity @PMIX_RENAME@pmix_output_get_verbosity +#define pmix_output_check_verbosity @PMIX_RENAME@pmix_output_check_verbosity #define pmix_output_hexdump @PMIX_RENAME@pmix_output_hexdump #define pmix_output_init @PMIX_RENAME@pmix_output_init #define pmix_output_open @PMIX_RENAME@pmix_output_open @@ -452,7 +453,6 @@ #define pmix_output_set_output_file_info @PMIX_RENAME@pmix_output_set_output_file_info #define pmix_output_set_verbosity @PMIX_RENAME@pmix_output_set_verbosity #define pmix_output_switch @PMIX_RENAME@pmix_output_switch -#define pmix_output_verbose @PMIX_RENAME@pmix_output_verbose #define pmix_output_vverbose @PMIX_RENAME@pmix_output_vverbose #define pmix_path_access @PMIX_RENAME@pmix_path_access #define pmix_path_df @PMIX_RENAME@pmix_path_df diff --git a/opal/mca/pmix/pmix4x/pmix/include/pmix_sched.h b/opal/mca/pmix/pmix4x/pmix/include/pmix_sched.h new file mode 100644 index 0000000000..561ef6366a --- /dev/null +++ b/opal/mca/pmix/pmix4x/pmix/include/pmix_sched.h @@ -0,0 +1,224 @@ +/* + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2015 Artem Y. Polyakov . + * All rights reserved. + * Copyright (c) 2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * $COPYRIGHT$ + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * - Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * - Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer listed + * in this license in the documentation and/or other materials + * provided with the distribution. + * + * - Neither the name of the copyright holders nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * The copyright holders provide no reassurances that the source code + * provided does not infringe any patent, copyright, or any other + * intellectual property rights of third parties. The copyright holders + * disclaim any liability to any recipient for claims brought against + * recipient by any third party for infringement of that parties + * intellectual property rights. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + * $HEADER$ + * + * PMIx interfaces for support of Workload Managers (Schedulers) + */ + +#ifndef PMIx_SCHED_API_H +#define PMIx_SCHED_API_H + +/* Structure and constant definitions */ +#include +#include + +#if defined(c_plusplus) || defined(__cplusplus) +extern "C" { +#endif + +/****** FABRIC-SCHEDULER INTERACTIONS ******/ + +/* Define a pmix_fabric_t struct that host RMs can use to + * interact with fabric-related interfaces */ +typedef struct pmix_fabric_s { + /* user-supplied name for this fabric */ + char *name; + /* revision - tracks how many times the + * fabric info has been updated. Used to detect + * that a change has occurred since the last + * time the data was accessed. Restricted to + * PMIx-internal use */ + uint64_t revision; + /* PMIx server-defined object for internal use */ + void *module; +} pmix_fabric_t; + +/* Register for access to fabric-related information, including + * communication cost matrix. This call must be made prior to + * requesting information from a fabric. + * + * fabric - address of a pmix_fabric_t (backed by storage). User + * may populate the "name" field at will - PMIx does not + * utilize this field + * + * directives - an optional array of values indicating desired + * behaviors and/or fabric to be accessed via + * the returned struct. If NULL, then the highest + * priority available fabric will return the struct + * + * ndirs - number of elements in the directives array + * + * Return values include: + * + * PMIX_SUCCESS - indicates success + */ +PMIX_EXPORT pmix_status_t PMIx_server_register_fabric(pmix_fabric_t *fabric, + const pmix_info_t directives[], + size_t ndirs); + +/* Deregister a fabric object, providing an opportunity for + * the PMIx server library to cleanup any information + * (e.g., cost matrix) associated with it + * + * fabric - pointer to the pmix_fabric_t struct provided + * to the registration function + */ +PMIX_EXPORT pmix_status_t PMIx_server_deregister_fabric(pmix_fabric_t *fabric); + +/* Get the number of vertices in the provided fabric. + * To avoid blocking the caller, this function will + * always return immediately, returning a PMIX_ERR_RESOURCE_BUSY + * status if the matrix is in the process of being updated. + * + * fabric - pointer to the pmix_fabric_t struct provided + * to the registration function + * + * nverts - pointer to the location where the number of + * vertices is to be returned + * + * Return values include: + * + * PMIX_SUCCESS - indicates return of a valid value + * PMIX_ERR_RESOURCE_BUSY - matrix is being updated + * PMIX_ERR_FABRIC_UPDATED - fabric info has been updated since + * last call involving this pmix_fabric_t + */ +PMIX_EXPORT pmix_status_t PMIx_server_get_num_vertices(pmix_fabric_t *fabric, + uint32_t *nverts); + +/* Obtain communication cost for messages transmitted from indicated + * source to destination across the provided fabric - i.e., + * the value of the (src,dest) entry of that fabric's communication + * cost matrix. To avoid blocking the caller, this function will + * always return immediately, returning a PMIX_ERR_RESOURCE_BUSY + * status if the matrix is in the process of being updated. + * + * fabric - pointer to the pmix_fabric_t struct provided to + * the registration function + * + * src - the index of the originating vertex for the communication + * + * dest - the index of the destination vertex for the communication + * + * cost - pointer to the location where the cost is to be returned + * + * Return values include: + * + * PMIX_SUCCESS - indicates return of a valid value + * PMIX_ERR_BAD_PARAM - src and/or dest is out of bounds + * PMIX_ERR_RESOURCE_BUSY - matrix is being updated + * PMIX_ERR_FABRIC_UPDATED - fabric info has been updated since + * last call involving this pmix_fabric_t + */ +PMIX_EXPORT pmix_status_t PMIx_server_get_comm_cost(pmix_fabric_t *fabric, + uint32_t src, uint32_t dest, + uint16_t *cost); + +/* Given a communication cost matrix index, return the corresponding + * vertex info in the provided fabric and the name of the node upon + * which it resides. + * If the PMIX_ERR_RESOURCE_BUSY or PMIX_ERR_FABRIC_UPDATED status is + * returned, then the caller should update their cost information + * before re-issuing this request to ensure accurate correlation + * between cost and LID + * + * fabric - pointer to the pmix_fabric_t struct provided to + * the registration function + * + * i - communication cost matrix index + * + * vertex - pointer to the pmix_value_t where the vertex info is to + * be returned (backed by storage) + * + * nodename - pointer to the location where the string nodename + * is to be returned. The caller is responsible for + * releasing the string when done + * + * Return values include: + * + * PMIX_SUCCESS - indicates return of a valid value + * PMIX_ERR_BAD_PARAM - provided index is out of bounds + * PMIX_ERR_RESOURCE_BUSY - matrix is being updated + * PMIX_ERR_FABRIC_UPDATED - fabric info has been updated since + * last call involving this pmix_fabric_t + */ +PMIX_EXPORT pmix_status_t PMIx_server_get_vertex_info(pmix_fabric_t *fabric, + uint32_t i, pmix_value_t *vertex, + char **nodename); + +/* Given vertex info, return the corresponding communication cost matrix + * index and the name of the node upon which it resides. + * If the PMIX_ERR_RESOURCE_BUSY or PMIX_ERR_FABRIC_UPDATED status is + * returned, then the caller should update their cost information + * before re-issuing this request to ensure accurate correlation + * between cost and LID + * + * fabric - pointer to the pmix_fabric_t struct provided to + * the registration function + * + * vertex - pointer to the vertex info whose index is being requested + * + * i - pointer to the location where the index is to be returned + * + * nodename - pointer to the location where the string nodename + * is to be returned. The caller is responsible for + * releasing the string when done + * + * Return values include: + * + * PMIX_SUCCESS - indicates return of a valid value + * PMIX_ERR_NOT_FOUND - provided vertex description is not found + * PMIX_ERR_RESOURCE_BUSY - matrix is being updated + * PMIX_ERR_FABRIC_UPDATED - fabric info has been updated since + * last call involving this pmix_fabric_t + */ +PMIX_EXPORT pmix_status_t PMIx_server_get_index(pmix_fabric_t *fabric, + pmix_value_t *vertex, uint32_t *i, + char **nodename); + +#if defined(c_plusplus) || defined(__cplusplus) +} +#endif + +#endif diff --git a/opal/mca/pmix/pmix4x/pmix/src/atomics/sys/atomic_stdc.h b/opal/mca/pmix/pmix4x/pmix/src/atomics/sys/atomic_stdc.h index 7c4a608909..8bd5e5eac6 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/atomics/sys/atomic_stdc.h +++ b/opal/mca/pmix/pmix4x/pmix/src/atomics/sys/atomic_stdc.h @@ -2,7 +2,7 @@ /* * Copyright (c) 2018 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -72,7 +72,14 @@ static inline void pmix_atomic_wmb (void) static inline void pmix_atomic_rmb (void) { +#if PMIX_ASSEMBLY_ARCH == PMIX_X86_64 + /* work around a bug in older gcc versions (observed in gcc 6.x) + * where acquire seems to get treated as a no-op instead of being + * equivalent to __asm__ __volatile__("": : :"memory") on x86_64 */ + pmix_atomic_mb (); +#else atomic_thread_fence (memory_order_acquire); +#endif } #define pmix_atomic_compare_exchange_strong_32(addr, compare, value) atomic_compare_exchange_strong_explicit (addr, compare, value, memory_order_relaxed, memory_order_relaxed) @@ -208,6 +215,7 @@ typedef atomic_flag pmix_atomic_lock_t; */ static inline void pmix_atomic_lock_init (pmix_atomic_lock_t *lock, bool value) { + (void)value; atomic_flag_clear (lock); } diff --git a/opal/mca/pmix/pmix4x/pmix/src/class/pmix_hotel.c b/opal/mca/pmix/pmix4x/pmix/src/class/pmix_hotel.c index fd114a77aa..7b8057ea98 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/class/pmix_hotel.c +++ b/opal/mca/pmix/pmix4x/pmix/src/class/pmix_hotel.c @@ -21,6 +21,8 @@ static void local_eviction_callback(int fd, short flags, void *arg) { + (void)fd; + (void)flags; pmix_hotel_room_eviction_callback_arg_t *eargs = (pmix_hotel_room_eviction_callback_arg_t*) arg; void *occupant = eargs->hotel->rooms[eargs->room_num].occupant; diff --git a/opal/mca/pmix/pmix4x/pmix/src/class/pmix_list.h b/opal/mca/pmix/pmix4x/pmix/src/class/pmix_list.h index f29bea22a9..c3e9589f03 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/class/pmix_list.h +++ b/opal/mca/pmix/pmix4x/pmix/src/class/pmix_list.h @@ -13,7 +13,7 @@ * Copyright (c) 2007 Voltaire All rights reserved. * Copyright (c) 2013 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2013-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -119,6 +119,14 @@ struct pmix_list_item_t */ typedef struct pmix_list_item_t pmix_list_item_t; +/* static initializer for pmix_list_t */ +#define PMIX_LIST_ITEM_STATIC_INIT \ + { \ + .super = PMIX_OBJ_STATIC_INIT(pmix_object_t), \ + .pmix_list_next = NULL, \ + .pmix_list_prev = NULL, \ + .item_free = 0 \ + } /** * Get the next item in a list. @@ -160,6 +168,15 @@ struct pmix_list_t */ typedef struct pmix_list_t pmix_list_t; +/* static initializer for pmix_list_t */ +#define PMIX_LIST_STATIC_INIT \ + { \ + .super = PMIX_OBJ_STATIC_INIT(pmix_object_t), \ + .pmix_list_sentinel = PMIX_LIST_ITEM_STATIC_INIT, \ + .pmix_list_length = 0 \ + } + + /** Cleanly destruct a list * * The pmix_list_t destructor doesn't release the items on the diff --git a/opal/mca/pmix/pmix4x/pmix/src/client/pmix_client_spawn.c b/opal/mca/pmix/pmix4x/pmix/src/client/pmix_client_spawn.c index 667d2e84b3..4e3e70cafa 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/client/pmix_client_spawn.c +++ b/opal/mca/pmix/pmix4x/pmix/src/client/pmix_client_spawn.c @@ -53,6 +53,7 @@ #include "src/util/error.h" #include "src/util/output.h" #include "src/mca/gds/gds.h" +#include "src/mca/pfexec/pfexec.h" #include "src/mca/ptl/ptl.h" #include "pmix_client_ops.h" @@ -63,15 +64,15 @@ static void wait_cbfunc(struct pmix_peer_t *pr, static void spawn_cbfunc(pmix_status_t status, char nspace[], void *cbdata); PMIX_EXPORT pmix_status_t PMIx_Spawn(const pmix_info_t job_info[], size_t ninfo, - const pmix_app_t apps[], size_t napps, - pmix_nspace_t nspace) + const pmix_app_t apps[], size_t napps, + pmix_nspace_t nspace) { pmix_status_t rc; pmix_cb_t *cb; PMIX_ACQUIRE_THREAD(&pmix_global_lock); - pmix_output_verbose(2, pmix_globals.debug_output, + pmix_output_verbose(2, pmix_client_globals.spawn_output, "pmix: spawn called"); if (pmix_globals.init_cntr <= 0) { @@ -80,7 +81,7 @@ PMIX_EXPORT pmix_status_t PMIx_Spawn(const pmix_info_t job_info[], size_t ninfo, } /* if we aren't connected, don't attempt to send */ - if (!pmix_globals.connected) { + if (!pmix_globals.connected && !PMIX_PROC_IS_TOOL(pmix_globals.mypeer)) { PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_UNREACH; } @@ -96,6 +97,8 @@ PMIX_EXPORT pmix_status_t PMIx_Spawn(const pmix_info_t job_info[], size_t ninfo, cb = PMIX_NEW(pmix_cb_t); if (PMIX_SUCCESS != (rc = PMIx_Spawn_nb(job_info, ninfo, apps, napps, spawn_cbfunc, cb))) { + /* note: the call may have return PMIX_OPERATION_SUCCEEDED thus indicating + * that the spawn was atomically completed */ PMIX_RELEASE(cb); return rc; } @@ -122,14 +125,15 @@ PMIX_EXPORT pmix_status_t PMIx_Spawn_nb(const pmix_info_t job_info[], size_t nin size_t n, m; pmix_app_t *aptr; bool jobenvars = false; + bool forkexec = false; char *harvest[2] = {"PMIX_MCA_", NULL}; pmix_kval_t *kv; pmix_list_t ilist; PMIX_ACQUIRE_THREAD(&pmix_global_lock); - pmix_output_verbose(2, pmix_globals.debug_output, - "pmix: spawn called"); + pmix_output_verbose(2, pmix_client_globals.spawn_output, + "pmix: spawn_nb called"); if (pmix_globals.init_cntr <= 0) { PMIX_RELEASE_THREAD(&pmix_global_lock); @@ -138,8 +142,13 @@ PMIX_EXPORT pmix_status_t PMIx_Spawn_nb(const pmix_info_t job_info[], size_t nin /* if we aren't connected, don't attempt to send */ if (!pmix_globals.connected) { - PMIX_RELEASE_THREAD(&pmix_global_lock); - return PMIX_ERR_UNREACH; + /* if I am a tool, we default to local fork/exec */ + if (PMIX_PROC_IS_TOOL(pmix_globals.mypeer)) { + forkexec = true; + } else { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return PMIX_ERR_UNREACH; + } } PMIX_RELEASE_THREAD(&pmix_global_lock); @@ -207,6 +216,14 @@ PMIX_EXPORT pmix_status_t PMIx_Spawn_nb(const pmix_info_t job_info[], size_t nin } } + /* if I am a tool and not connected, then just fork/exec + * the specified application */ + if (forkexec) { + rc = pmix_pfexec.spawn_proc(job_info, ninfo, + apps, napps); + return rc; + } + msg = PMIX_NEW(pmix_buffer_t); /* pack the cmd */ PMIX_BFROPS_PACK(rc, pmix_client_globals.myserver, diff --git a/opal/mca/pmix/pmix4x/pmix/src/common/pmix_iof.c b/opal/mca/pmix/pmix4x/pmix/src/common/pmix_iof.c index be2d6b11a2..680800f120 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/common/pmix_iof.c +++ b/opal/mca/pmix/pmix4x/pmix/src/common/pmix_iof.c @@ -36,6 +36,7 @@ #include "src/util/name_fns.h" #include "src/util/output.h" #include "src/mca/bfrops/bfrops.h" +#include "src/mca/pfexec/base/base.h" #include "src/mca/ptl/ptl.h" #include "src/client/pmix_client_ops.h" @@ -689,7 +690,7 @@ void pmix_iof_static_dump_output(pmix_iof_sink_t *sink) dump = false; /* make one last attempt to write this out */ while (NULL != (output = (pmix_iof_write_output_t*)pmix_list_remove_first(&wev->outputs))) { - if (!dump) { + if (!dump && 0 < output->numbytes) { num_written = write(wev->fd, output->data, output->numbytes); if (num_written < output->numbytes) { /* don't retry - just cleanout the list and dump it */ @@ -720,7 +721,7 @@ void pmix_iof_write_handler(int _fd, short event, void *cbdata) output = (pmix_iof_write_output_t*)item; if (0 == output->numbytes) { /* indicates we are to close this stream */ - PMIX_RELEASE(sink); + PMIX_DESTRUCT(sink); return; } num_written = write(wev->fd, output->data, output->numbytes); @@ -851,19 +852,20 @@ void pmix_iof_read_local_handler(int unusedfd, short event, void *cbdata) pmix_iof_read_event_t *rev = (pmix_iof_read_event_t*)cbdata; unsigned char data[PMIX_IOF_BASE_MSG_MAX]; int32_t numbytes; - int fd; pmix_status_t rc; pmix_buffer_t *msg; pmix_cmd_t cmd = PMIX_IOF_PUSH_CMD; pmix_byte_object_t bo; + int fd; + pmix_pfexec_child_t *child = (pmix_pfexec_child_t*)rev->childproc; PMIX_ACQUIRE_OBJECT(rev); - /* As we may use timer events, fd can be bogus (-1) - * use the right one here - */ - fd = fileno(stdin); - + if (0 > rev->fd) { + fd = fileno(stdin); + } else { + fd = rev->fd; + } /* read up to the fragment size */ memset(data, 0, PMIX_IOF_BASE_MSG_MAX); numbytes = read(fd, data, sizeof(data)); @@ -891,6 +893,19 @@ void pmix_iof_read_local_handler(int unusedfd, short event, void *cbdata) re-add it */ rev->active = false; + /* if this is from our own child proc, then + * just push it to the corresponding sink */ + if (NULL != child) { + bo.bytes = (char*)data; + bo.size = numbytes; + pmix_iof_write_output(&rev->name, rev->channel, &bo, NULL); + if (0 == numbytes) { + PMIX_PFEXEC_CHK_COMPLETE(child); + return; + } + goto reactivate; + } + /* pass the data to our PMIx server so it can relay it * to the host RM for distribution */ msg = PMIX_NEW(pmix_buffer_t); @@ -960,6 +975,12 @@ void pmix_iof_read_local_handler(int unusedfd, short event, void *cbdata) PMIX_ERROR_LOG(rc); PMIX_RELEASE(msg); } + + reactivate: + if (0 < numbytes) { + PMIX_IOF_READ_ACTIVATE(rev); + } + /* nothing more to do */ return; } @@ -992,6 +1013,7 @@ static void iof_read_event_construct(pmix_iof_read_event_t* rev) { rev->fd = -1; rev->active = false; + rev->childproc = NULL; rev->tv.tv_sec = 0; rev->tv.tv_usec = 0; rev->targets = NULL; diff --git a/opal/mca/pmix/pmix4x/pmix/src/common/pmix_iof.h b/opal/mca/pmix/pmix4x/pmix/src/common/pmix_iof.h index b4cf8c20c4..23fcf4c177 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/common/pmix_iof.h +++ b/opal/mca/pmix/pmix4x/pmix/src/common/pmix_iof.h @@ -97,7 +97,10 @@ typedef struct { struct timeval tv; int fd; bool active; + void *childproc; bool always_readable; + pmix_proc_t name; + pmix_iof_channel_t channel; pmix_proc_t *targets; size_t ntargets; pmix_info_t *directives; @@ -198,13 +201,15 @@ pmix_iof_fd_always_ready(int fd) "defining read event at: %s %d", \ __FILE__, __LINE__)); \ rev = PMIX_NEW(pmix_iof_read_event_t); \ - (rev)->ntargets = (np); \ - PMIX_PROC_CREATE((rev)->targets, (rev)->ntargets); \ - memcpy((rev)->targets, (p), (np) * sizeof(pmix_proc_t)); \ - if (NULL != (d)) { \ + if (NULL != (p)) { \ + (rev)->ntargets = (np); \ + PMIX_PROC_CREATE((rev)->targets, (rev)->ntargets); \ + memcpy((rev)->targets, (p), (np) * sizeof(pmix_proc_t)); \ + } \ + if (NULL != (d) && 0 < (nd)) { \ PMIX_INFO_CREATE((rev)->directives, (nd)); \ (rev)->ndirs = (nd); \ - for (_ii=0; _ii < (nd); _ii++) { \ + for (_ii=0; _ii < (size_t)nd; _ii++) { \ PMIX_INFO_XFER(&((rev)->directives[_ii]), &((d)[_ii])); \ } \ } \ diff --git a/opal/mca/pmix/pmix4x/pmix/src/event/pmix_event_notification.c b/opal/mca/pmix/pmix4x/pmix/src/event/pmix_event_notification.c index 8444fdac87..b7ba1e6eef 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/event/pmix_event_notification.c +++ b/opal/mca/pmix/pmix4x/pmix/src/event/pmix_event_notification.c @@ -71,7 +71,7 @@ PMIX_EXPORT pmix_status_t PMIx_Notify_event(pmix_status_t status, } /* if we aren't connected, don't attempt to send */ - if (!pmix_globals.connected) { + if (!pmix_globals.connected && PMIX_RANGE_PROC_LOCAL != range) { PMIX_RELEASE_THREAD(&pmix_global_lock); return PMIX_ERR_UNREACH; } @@ -93,6 +93,7 @@ PMIX_EXPORT pmix_status_t PMIx_Notify_event(pmix_status_t status, static void notify_event_cbfunc(struct pmix_peer_t *pr, pmix_ptl_hdr_t *hdr, pmix_buffer_t *buf, void *cbdata) { + (void)hdr; pmix_status_t rc, ret; int32_t cnt = 1; pmix_cb_t *cb = (pmix_cb_t*)cbdata; @@ -806,6 +807,8 @@ static void local_cbfunc(pmix_status_t status, void *cbdata) static void _notify_client_event(int sd, short args, void *cbdata) { + (void)sd; + (void)args; pmix_notify_caddy_t *cd = (pmix_notify_caddy_t*)cbdata; pmix_regevents_info_t *reginfoptr; pmix_peer_events_info_t *pr; @@ -1255,6 +1258,8 @@ bool pmix_notify_check_affected(pmix_proc_t *interested, size_t ninterested, void pmix_event_timeout_cb(int fd, short flags, void *arg) { + (void)fd; + (void)flags; pmix_event_chain_t *ch = (pmix_event_chain_t*)arg; /* need to acquire the object from its originating thread */ diff --git a/opal/mca/pmix/pmix4x/pmix/src/include/pmix_globals.h b/opal/mca/pmix/pmix4x/pmix/src/include/pmix_globals.h index 39d641a4e4..ec803880ad 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/include/pmix_globals.h +++ b/opal/mca/pmix/pmix4x/pmix/src/include/pmix_globals.h @@ -473,6 +473,7 @@ typedef struct { uid_t uid; // my effective uid gid_t gid; // my effective gid char *hostname; // my hostname + pid_t pid; // my local pid uint32_t nodeid; // my nodeid, if given int pindex; pmix_event_base_t *evbase; diff --git a/opal/mca/pmix/pmix4x/pmix/src/include/types.h b/opal/mca/pmix/pmix4x/pmix/src/include/types.h index 821c788e6d..d8297807d3 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/include/types.h +++ b/opal/mca/pmix/pmix4x/pmix/src/include/types.h @@ -144,6 +144,18 @@ static inline uint64_t pmix_ntoh64(uint64_t val) #endif } +/* Convert size_t value from host to network byte order and back */ +#if SIZEOF_SIZE_T == 4 + +#define pmix_htonsizet(x) htonl(x) +#define pmix_ntohsizet(x) ntohl(x) + +#elif SIZEOF_SIZE_T == 8 + +#define pmix_htonsizet(x) pmix_hton64(x) +#define pmix_ntohsizet(x) pmix_ntoh64(x) + +#endif /** * Convert between a local representation of pointer and a 64 bits value. @@ -232,6 +244,8 @@ static inline uint64_t pmix_swap_bytes8(uint64_t val) #define PMIX_EVLOOP_ONCE EVLOOP_ONCE /**< Block at most once. */ #define PMIX_EVLOOP_NONBLOCK EVLOOP_NONBLOCK /**< Do not block. */ +#define PMIX_EVENT_SIGNAL(ev) pmix_event_get_signal(ev) + typedef struct event_base pmix_event_base_t; typedef struct event pmix_event_t; diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/base/pmix_mca_base_component_repository.c b/opal/mca/pmix/pmix4x/pmix/src/mca/base/pmix_mca_base_component_repository.c index 062b1cb75d..2a96d7ed81 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/base/pmix_mca_base_component_repository.c +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/base/pmix_mca_base_component_repository.c @@ -94,6 +94,7 @@ static pmix_hash_table_t pmix_mca_base_component_repository; static int process_repository_item (const char *filename, void *data) { + (void)data; char name[PMIX_MCA_BASE_MAX_COMPONENT_NAME_LEN + 1]; char type[PMIX_MCA_BASE_MAX_TYPE_NAME_LEN + 1]; pmix_mca_base_component_repository_item_t *ri; diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/base/pmix_mca_base_var.c b/opal/mca/pmix/pmix4x/pmix/src/mca/base/pmix_mca_base_var.c index d6e7232ed0..e48eb4c965 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/base/pmix_mca_base_var.c +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/base/pmix_mca_base_var.c @@ -792,6 +792,7 @@ static int var_set_from_string (pmix_mca_base_var_t *var, char *src) int pmix_mca_base_var_set_value (int vari, const void *value, size_t size, pmix_mca_base_var_source_t source, const char *source_file) { + (void)size; pmix_mca_base_var_t *var; int ret; @@ -952,6 +953,7 @@ static int var_find (const char *project_name, const char *framework_name, const char *component_name, const char *variable_name, bool invalidok) { + (void)project_name; char *full_name; int ret, vari; @@ -1608,6 +1610,7 @@ int pmix_mca_base_var_register_synonym (int synonym_for, const char *project_nam static int var_get_env (pmix_mca_base_var_t *var, const char *name, char **source, char **value) { + (void)var; char *source_env, *value_env; int ret; diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/base/pmix_mca_base_var_enum.c b/opal/mca/pmix/pmix4x/pmix/src/mca/base/pmix_mca_base_var_enum.c index b5bb281b68..767725df5b 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/base/pmix_mca_base_var_enum.c +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/base/pmix_mca_base_var_enum.c @@ -48,6 +48,7 @@ static int enum_get_value (pmix_mca_base_var_enum_t *self, int index, int *value static int pmix_mca_base_var_enum_bool_get_count (pmix_mca_base_var_enum_t *enumerator, int *count) { + (void)enumerator; *count = 2; return PMIX_SUCCESS; } @@ -55,6 +56,7 @@ static int pmix_mca_base_var_enum_bool_get_count (pmix_mca_base_var_enum_t *enum static int pmix_mca_base_var_enum_bool_get_value (pmix_mca_base_var_enum_t *self, int index, int *value, const char **string_value) { + (void)self; if (1 < index) { return PMIX_ERR_VALUE_OUT_OF_BOUNDS; } @@ -68,6 +70,7 @@ static int pmix_mca_base_var_enum_bool_get_value (pmix_mca_base_var_enum_t *self static int pmix_mca_base_var_enum_bool_vfs (pmix_mca_base_var_enum_t *self, const char *string_value, int *value) { + (void)self; char *tmp; int v; @@ -95,6 +98,7 @@ static int pmix_mca_base_var_enum_bool_vfs (pmix_mca_base_var_enum_t *self, cons static int pmix_mca_base_var_enum_bool_sfv (pmix_mca_base_var_enum_t *self, const int value, char **string_value) { + (void)self; if (string_value) { *string_value = strdup (value ? "true" : "false"); } @@ -104,6 +108,7 @@ static int pmix_mca_base_var_enum_bool_sfv (pmix_mca_base_var_enum_t *self, cons static int pmix_mca_base_var_enum_bool_dump (pmix_mca_base_var_enum_t *self, char **out) { + (void)self; *out = strdup ("0: f|false|disabled|no, 1: t|true|enabled|yes"); return *out ? PMIX_SUCCESS : PMIX_ERR_OUT_OF_RESOURCE; } @@ -135,6 +140,7 @@ static pmix_mca_base_var_enum_value_t verbose_values[] = { static int pmix_mca_base_var_enum_verbose_vfs (pmix_mca_base_var_enum_t *self, const char *string_value, int *value) { + (void)self; char *tmp; int v; @@ -165,6 +171,7 @@ static int pmix_mca_base_var_enum_verbose_vfs (pmix_mca_base_var_enum_t *self, c static int pmix_mca_base_var_enum_verbose_sfv (pmix_mca_base_var_enum_t *self, const int value, char **string_value) { + (void)self; int ret; if (value < 0 || value > 100) { diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/bfrops/base/bfrop_base_copy.c b/opal/mca/pmix/pmix4x/pmix/src/mca/bfrops/base/bfrop_base_copy.c index 9675c15078..3bba47b3ca 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/bfrops/base/bfrop_base_copy.c +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/bfrops/base/bfrop_base_copy.c @@ -202,6 +202,9 @@ pmix_status_t pmix_bfrops_base_std_copy(void **dest, void *src, pmix_status_t pmix_bfrops_base_copy_string(char **dest, char *src, pmix_data_type_t type) { + if (PMIX_STRING != type) { + return PMIX_ERR_BAD_PARAM; + } if (NULL == src) { /* got zero-length string/NULL pointer - store NULL */ *dest = NULL; } else { @@ -218,6 +221,9 @@ pmix_status_t pmix_bfrops_base_copy_value(pmix_value_t **dest, { pmix_value_t *p; + if (PMIX_VALUE != type) { + return PMIX_ERR_BAD_PARAM; + } /* create the new object */ *dest = (pmix_value_t*)malloc(sizeof(pmix_value_t)); if (NULL == *dest) { @@ -235,6 +241,9 @@ pmix_status_t pmix_bfrops_base_copy_info(pmix_info_t **dest, pmix_info_t *src, pmix_data_type_t type) { + if (PMIX_VALUE != type) { + return PMIX_ERR_BAD_PARAM; + } *dest = (pmix_info_t*)malloc(sizeof(pmix_info_t)); pmix_strncpy((*dest)->key, src->key, PMIX_MAX_KEYLEN); (*dest)->flags = src->flags; @@ -245,6 +254,9 @@ pmix_status_t pmix_bfrops_base_copy_buf(pmix_buffer_t **dest, pmix_buffer_t *src, pmix_data_type_t type) { + if (PMIX_BUFFER != type) { + return PMIX_ERR_BAD_PARAM; + } *dest = PMIX_NEW(pmix_buffer_t); pmix_bfrops_base_copy_payload(*dest, src); return PMIX_SUCCESS; @@ -256,6 +268,9 @@ pmix_status_t pmix_bfrops_base_copy_app(pmix_app_t **dest, { size_t j; + if (PMIX_APP != type) { + return PMIX_ERR_BAD_PARAM; + } *dest = (pmix_app_t*)malloc(sizeof(pmix_app_t)); (*dest)->cmd = strdup(src->cmd); (*dest)->argv = pmix_argv_copy(src->argv); @@ -279,6 +294,9 @@ pmix_status_t pmix_bfrops_base_copy_kval(pmix_kval_t **dest, { pmix_kval_t *p; + if (PMIX_KVAL != type) { + return PMIX_ERR_BAD_PARAM; + } /* create the new object */ *dest = PMIX_NEW(pmix_kval_t); if (NULL == *dest) { @@ -296,6 +314,9 @@ pmix_status_t pmix_bfrops_base_copy_proc(pmix_proc_t **dest, pmix_proc_t *src, pmix_data_type_t type) { + if (PMIX_PROC != type) { + return PMIX_ERR_BAD_PARAM; + } *dest = (pmix_proc_t*)malloc(sizeof(pmix_proc_t)); if (NULL == *dest) { return PMIX_ERR_OUT_OF_RESOURCE; @@ -309,6 +330,9 @@ pmix_status_t pmix_bfrop_base_copy_persist(pmix_persistence_t **dest, pmix_persistence_t *src, pmix_data_type_t type) { + if (PMIX_PERSIST != type) { + return PMIX_ERR_BAD_PARAM; + } *dest = (pmix_persistence_t*)malloc(sizeof(pmix_persistence_t)); if (NULL == *dest) { return PMIX_ERR_OUT_OF_RESOURCE; @@ -321,6 +345,9 @@ pmix_status_t pmix_bfrops_base_copy_bo(pmix_byte_object_t **dest, pmix_byte_object_t *src, pmix_data_type_t type) { + if (PMIX_BYTE_OBJECT != type) { + return PMIX_ERR_BAD_PARAM; + } *dest = (pmix_byte_object_t*)malloc(sizeof(pmix_byte_object_t)); if (NULL == *dest) { return PMIX_ERR_OUT_OF_RESOURCE; @@ -335,6 +362,9 @@ pmix_status_t pmix_bfrops_base_copy_pdata(pmix_pdata_t **dest, pmix_pdata_t *src, pmix_data_type_t type) { + if (PMIX_PDATA != type) { + return PMIX_ERR_BAD_PARAM; + } *dest = (pmix_pdata_t*)malloc(sizeof(pmix_pdata_t)); pmix_strncpy((*dest)->proc.nspace, src->proc.nspace, PMIX_MAX_NSLEN); (*dest)->proc.rank = src->proc.rank; @@ -348,6 +378,9 @@ pmix_status_t pmix_bfrops_base_copy_pinfo(pmix_proc_info_t **dest, { pmix_proc_info_t *p; + if (PMIX_INFO != type) { + return PMIX_ERR_BAD_PARAM; + } PMIX_PROC_INFO_CREATE(p, 1); if (NULL == p) { return PMIX_ERR_NOMEM; @@ -389,6 +422,10 @@ pmix_status_t pmix_bfrops_base_copy_darray(pmix_data_array_t **dest, pmix_envar_t *pe, *se; pmix_regattr_t *pr, *sr; + if (PMIX_DATA_ARRAY != type) { + return PMIX_ERR_BAD_PARAM; + } + p = (pmix_data_array_t*)calloc(1, sizeof(pmix_data_array_t)); if (NULL == p) { return PMIX_ERR_NOMEM; @@ -845,6 +882,9 @@ pmix_status_t pmix_bfrops_base_copy_query(pmix_query_t **dest, { pmix_status_t rc; + if (PMIX_QUERY != type) { + return PMIX_ERR_BAD_PARAM; + } *dest = (pmix_query_t*)malloc(sizeof(pmix_query_t)); if (NULL != src->keys) { (*dest)->keys = pmix_argv_copy(src->keys); @@ -863,6 +903,9 @@ pmix_status_t pmix_bfrops_base_copy_envar(pmix_envar_t **dest, pmix_envar_t *src, pmix_data_type_t type) { + if (PMIX_ENVAR != type) { + return PMIX_ERR_BAD_PARAM; + } PMIX_ENVAR_CREATE(*dest, 1); if (NULL == (*dest)) { return PMIX_ERR_NOMEM; @@ -881,6 +924,9 @@ pmix_status_t pmix_bfrops_base_copy_coord(pmix_coord_t **dest, pmix_coord_t *src, pmix_data_type_t type) { + if (PMIX_COORD != type) { + return PMIX_ERR_BAD_PARAM; + } *dest = (pmix_coord_t*)malloc(sizeof(pmix_coord_t)); memcpy(*dest, src, sizeof(pmix_coord_t)); return PMIX_SUCCESS; @@ -890,6 +936,9 @@ pmix_status_t pmix_bfrops_base_copy_regattr(pmix_regattr_t **dest, pmix_regattr_t *src, pmix_data_type_t type) { + if (PMIX_REGATTR != type) { + return PMIX_ERR_BAD_PARAM; + } PMIX_REGATTR_CREATE(*dest, 1); if (NULL == (*dest)) { return PMIX_ERR_NOMEM; diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/bfrops/base/bfrop_base_frame.c b/opal/mca/pmix/pmix4x/pmix/src/mca/bfrops/base/bfrop_base_frame.c index 952ca015bb..721c2f482c 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/bfrops/base/bfrop_base_frame.c +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/bfrops/base/bfrop_base_frame.c @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ @@ -47,11 +47,26 @@ #include "src/mca/bfrops/base/static-components.h" /* Instantiate the global vars */ -pmix_bfrops_globals_t pmix_bfrops_globals = {{{0}}}; +pmix_bfrops_globals_t pmix_bfrops_globals = { + .actives = PMIX_LIST_STATIC_INIT, + .initialized = false, + .initial_size = 0, + .threshold_size = 0, +#if PMIX_ENABLE_DEBUG + .default_type = PMIX_BFROP_BUFFER_FULLY_DESC +#else + .default_type = PMIX_BFROP_BUFFER_NON_DESC +#endif +}; int pmix_bfrops_base_output = 0; static int pmix_bfrop_register(pmix_mca_base_register_flag_t flags) { + if (PMIX_MCA_BASE_REGISTER_DEFAULT == flags) { + /* do something to silence warning */ + int count=0; + ++count; + } pmix_bfrops_globals.initial_size = PMIX_BFROP_DEFAULT_INITIAL_SIZE; pmix_mca_base_var_register("pmix", "bfrops", "base", "initial_size", "Initial size of a buffer", diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/bfrops/base/bfrop_base_pack.c b/opal/mca/pmix/pmix4x/pmix/src/mca/bfrops/base/bfrop_base_pack.c index acb0492df4..38301eaf3c 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/bfrops/base/bfrop_base_pack.c +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/bfrops/base/bfrop_base_pack.c @@ -101,6 +101,12 @@ pmix_status_t pmix_bfrops_base_pack_buffer(pmix_pointer_array_t *regtypes, pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, "pmix_bfrops_base_pack_bool * %d\n", num_vals); + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_BOOL != type) { + return PMIX_ERR_BAD_PARAM; + } /* check to see if buffer needs extending */ if (NULL == (dst = (uint8_t*)pmix_bfrop_buffer_extend(buffer, num_vals))) { return PMIX_ERR_OUT_OF_RESOURCE; @@ -131,6 +137,9 @@ pmix_status_t pmix_bfrops_base_pack_int(pmix_pointer_array_t *regtypes, { pmix_status_t ret; + if (PMIX_INT != type && PMIX_UINT != type) { + return PMIX_ERR_BAD_PARAM; + } /* System types need to always be described so we can properly unpack them */ if (PMIX_SUCCESS != (ret = pmix_bfrop_store_data_type(regtypes, buffer, BFROP_TYPE_INT))) { @@ -151,6 +160,9 @@ pmix_status_t pmix_bfrops_base_pack_sizet(pmix_pointer_array_t *regtypes, { int ret; + if (PMIX_SIZE != type) { + return PMIX_ERR_BAD_PARAM; + } /* System types need to always be described so we can properly unpack them. */ if (PMIX_SUCCESS != (ret = pmix_bfrop_store_data_type(regtypes, buffer, BFROP_TYPE_SIZE_T))) { @@ -170,6 +182,9 @@ pmix_status_t pmix_bfrops_base_pack_pid(pmix_pointer_array_t *regtypes, { int ret; + if (PMIX_PID != type) { + return PMIX_ERR_BAD_PARAM; + } /* System types need to always be described so we can properly unpack them. */ if (PMIX_SUCCESS != (ret = pmix_bfrop_store_data_type(regtypes, buffer, BFROP_TYPE_PID_T))) { @@ -193,6 +208,12 @@ pmix_status_t pmix_bfrops_base_pack_byte(pmix_pointer_array_t *regtypes, pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, "pmix_bfrops_base_pack_byte * %d\n", num_vals); + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_BYTE != type && PMIX_UINT8 != type && PMIX_INT8 != type) { + return PMIX_ERR_BAD_PARAM; + } /* check to see if buffer needs extending */ if (NULL == (dst = pmix_bfrop_buffer_extend(buffer, num_vals))) { return PMIX_ERR_OUT_OF_RESOURCE; @@ -222,6 +243,12 @@ pmix_status_t pmix_bfrops_base_pack_int16(pmix_pointer_array_t *regtypes, pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, "pmix_bfrops_base_pack_int16 * %d\n", num_vals); + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_INT16 != type && PMIX_UINT16 != type) { + return PMIX_ERR_BAD_PARAM; + } /* check to see if buffer needs extending */ if (NULL == (dst = pmix_bfrop_buffer_extend(buffer, num_vals*sizeof(tmp)))) { return PMIX_ERR_OUT_OF_RESOURCE; @@ -252,6 +279,12 @@ pmix_status_t pmix_bfrops_base_pack_int32(pmix_pointer_array_t *regtypes, pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, "pmix_bfrops_base_pack_int32 * %d\n", num_vals); + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_INT32 != type && PMIX_UINT32 != type) { + return PMIX_ERR_BAD_PARAM; + } /* check to see if buffer needs extending */ if (NULL == (dst = pmix_bfrop_buffer_extend(buffer, num_vals*sizeof(tmp)))) { return PMIX_ERR_OUT_OF_RESOURCE; @@ -282,6 +315,12 @@ pmix_status_t pmix_bfrops_base_pack_int64(pmix_pointer_array_t *regtypes, pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, "pmix_bfrops_base_pack_int64 * %d\n", num_vals); + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_INT64 != type && PMIX_UINT64 != type) { + return PMIX_ERR_BAD_PARAM; + } /* check to see if buffer needs extending */ if (NULL == (dst = pmix_bfrop_buffer_extend(buffer, bytes_packed))) { return PMIX_ERR_OUT_OF_RESOURCE; @@ -310,6 +349,12 @@ pmix_status_t pmix_bfrops_base_pack_string(pmix_pointer_array_t *regtypes, int32_t i, len; char **ssrc = (char**) src; + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_STRING != type) { + return PMIX_ERR_BAD_PARAM; + } for (i = 0; i < num_vals; ++i) { if (NULL == ssrc[i]) { /* got zero-length string/NULL pointer - store NULL */ len = 0; @@ -342,6 +387,12 @@ pmix_status_t pmix_bfrops_base_pack_float(pmix_pointer_array_t *regtypes, float *ssrc = (float*)src; char *convert; + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_FLOAT != type) { + return PMIX_ERR_BAD_PARAM; + } for (i = 0; i < num_vals; ++i) { ret = asprintf(&convert, "%f", ssrc[i]); if (0 > ret) { @@ -367,6 +418,12 @@ pmix_status_t pmix_bfrops_base_pack_double(pmix_pointer_array_t *regtypes, double *ssrc = (double*)src; char *convert; + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_DOUBLE != type) { + return PMIX_ERR_BAD_PARAM; + } for (i = 0; i < num_vals; ++i) { ret = asprintf(&convert, "%f", ssrc[i]); if (0 > ret) { @@ -392,6 +449,12 @@ pmix_status_t pmix_bfrops_base_pack_timeval(pmix_pointer_array_t *regtypes, int32_t i; struct timeval *ssrc = (struct timeval *)src; + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_TIMEVAL != type) { + return PMIX_ERR_BAD_PARAM; + } for (i = 0; i < num_vals; ++i) { tmp[0] = (int64_t)ssrc[i].tv_sec; tmp[1] = (int64_t)ssrc[i].tv_usec; @@ -413,6 +476,12 @@ pmix_status_t pmix_bfrops_base_pack_time(pmix_pointer_array_t *regtypes, time_t *ssrc = (time_t *)src; uint64_t ui64; + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_TIME != type) { + return PMIX_ERR_BAD_PARAM; + } /* time_t is a system-dependent size, so cast it * to uint64_t as a generic safe size */ @@ -436,6 +505,12 @@ pmix_status_t pmix_bfrops_base_pack_status(pmix_pointer_array_t *regtypes, pmix_status_t *ssrc = (pmix_status_t *)src; int32_t status; + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_STATUS != type) { + return PMIX_ERR_BAD_PARAM; + } for (i = 0; i < num_vals; ++i) { status = (int32_t)ssrc[i]; PMIX_BFROPS_PACK_TYPE(ret, buffer, &status, 1, PMIX_INT32, regtypes); @@ -455,6 +530,12 @@ pmix_status_t pmix_bfrops_base_pack_buf(pmix_pointer_array_t *regtypes, int32_t i; int ret; + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_BUFFER != type) { + return PMIX_ERR_BAD_PARAM; + } ptr = (pmix_buffer_t *) src; for (i = 0; i < num_vals; ++i) { @@ -490,6 +571,12 @@ pmix_status_t pmix_bfrops_base_pack_bo(pmix_pointer_array_t *regtypes, int i; pmix_byte_object_t *bo; + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_BYTE_OBJECT != type) { + return PMIX_ERR_BAD_PARAM; + } bo = (pmix_byte_object_t*)src; for (i=0; i < num_vals; i++) { PMIX_BFROPS_PACK_TYPE(ret, buffer, &bo[i].size, 1, PMIX_SIZE, regtypes); @@ -515,6 +602,12 @@ pmix_status_t pmix_bfrops_base_pack_proc(pmix_pointer_array_t *regtypes, int32_t i; int ret; + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_PROC != type) { + return PMIX_ERR_BAD_PARAM; + } proc = (pmix_proc_t *) src; for (i = 0; i < num_vals; ++i) { @@ -542,6 +635,12 @@ pmix_status_t pmix_bfrops_base_pack_value(pmix_pointer_array_t *regtypes, int32_t i; int ret; + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_VALUE != type) { + return PMIX_ERR_BAD_PARAM; + } ptr = (pmix_value_t *) src; for (i = 0; i < num_vals; ++i) { @@ -567,6 +666,12 @@ pmix_status_t pmix_bfrops_base_pack_info(pmix_pointer_array_t *regtypes, int ret; char *foo; + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_INFO != type) { + return PMIX_ERR_BAD_PARAM; + } info = (pmix_info_t *) src; for (i = 0; i < num_vals; ++i) { @@ -603,6 +708,12 @@ pmix_status_t pmix_bfrops_base_pack_pdata(pmix_pointer_array_t *regtypes, int ret; char *foo; + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_PDATA != type) { + return PMIX_ERR_BAD_PARAM; + } pdata = (pmix_pdata_t *) src; for (i = 0; i < num_vals; ++i) { @@ -641,6 +752,12 @@ pmix_status_t pmix_bfrops_base_pack_app(pmix_pointer_array_t *regtypes, int32_t i, j, nvals; int ret; + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_APP != type) { + return PMIX_ERR_BAD_PARAM; + } app = (pmix_app_t *) src; for (i = 0; i < num_vals; ++i) { @@ -709,6 +826,12 @@ pmix_status_t pmix_bfrops_base_pack_kval(pmix_pointer_array_t *regtypes, int32_t i; int ret; + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_KVAL != type) { + return PMIX_ERR_BAD_PARAM; + } ptr = (pmix_kval_t *) src; for (i = 0; i < num_vals; ++i) { @@ -731,6 +854,12 @@ pmix_status_t pmix_bfrops_base_pack_persist(pmix_pointer_array_t *regtypes, int32_t num_vals, pmix_data_type_t type) { pmix_status_t ret; + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_PERSIST != type) { + return PMIX_ERR_BAD_PARAM; + } PMIX_BFROPS_PACK_TYPE(ret, buffer, src, num_vals, PMIX_BYTE, regtypes); return ret; } @@ -740,6 +869,12 @@ pmix_status_t pmix_bfrops_base_pack_datatype(pmix_pointer_array_t *regtypes, int32_t num_vals, pmix_data_type_t type) { pmix_status_t ret; + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_DATA_TYPE != type) { + return PMIX_ERR_BAD_PARAM; + } PMIX_BFROPS_PACK_TYPE(ret, buffer, src, num_vals, PMIX_UINT16, regtypes); return ret; } @@ -751,6 +886,13 @@ pmix_status_t pmix_bfrops_base_pack_ptr(pmix_pointer_array_t *regtypes, { pmix_status_t ret; uint8_t foo=1; + + if (NULL == regtypes || NULL != src || 0 == num_vals) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_POINTER != type) { + return PMIX_ERR_BAD_PARAM; + } /* it obviously makes no sense to pack a pointer and * send it somewhere else, so we just pack a sentinel */ PMIX_BFROPS_PACK_TYPE(ret, buffer, &foo, 1, PMIX_UINT8, regtypes); @@ -762,6 +904,13 @@ pmix_status_t pmix_bfrops_base_pack_scope(pmix_pointer_array_t *regtypes, int32_t num_vals, pmix_data_type_t type) { pmix_status_t ret; + + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_SCOPE != type) { + return PMIX_ERR_BAD_PARAM; + } PMIX_BFROPS_PACK_TYPE(ret, buffer, src, num_vals, PMIX_UINT8, regtypes); return ret; } @@ -771,6 +920,13 @@ pmix_status_t pmix_bfrops_base_pack_range(pmix_pointer_array_t *regtypes, int32_t num_vals, pmix_data_type_t type) { pmix_status_t ret; + + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_DATA_RANGE != type) { + return PMIX_ERR_BAD_PARAM; + } PMIX_BFROPS_PACK_TYPE(ret, buffer, src, num_vals, PMIX_UINT8, regtypes); return ret; } @@ -780,6 +936,13 @@ pmix_status_t pmix_bfrops_base_pack_cmd(pmix_pointer_array_t *regtypes, int32_t num_vals, pmix_data_type_t type) { pmix_status_t ret; + + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_COMMAND != type) { + return PMIX_ERR_BAD_PARAM; + } PMIX_BFROPS_PACK_TYPE(ret, buffer, src, num_vals, PMIX_UINT8, regtypes); return ret; } @@ -789,6 +952,13 @@ pmix_status_t pmix_bfrops_base_pack_info_directives(pmix_pointer_array_t *regtyp int32_t num_vals, pmix_data_type_t type) { pmix_status_t ret; + + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_INFO_DIRECTIVES != type) { + return PMIX_ERR_BAD_PARAM; + } PMIX_BFROPS_PACK_TYPE(ret, buffer, src, num_vals, PMIX_UINT32, regtypes); return ret; } @@ -798,6 +968,13 @@ pmix_status_t pmix_bfrops_base_pack_pstate(pmix_pointer_array_t *regtypes, int32_t num_vals, pmix_data_type_t type) { pmix_status_t ret; + + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_PROC_STATE != type) { + return PMIX_ERR_BAD_PARAM; + } PMIX_BFROPS_PACK_TYPE(ret, buffer, src, num_vals, PMIX_UINT8, regtypes); return ret; } @@ -810,6 +987,12 @@ pmix_status_t pmix_bfrops_base_pack_pinfo(pmix_pointer_array_t *regtypes, pmix_status_t ret; int32_t i; + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_PROC_INFO != type) { + return PMIX_ERR_BAD_PARAM; + } for (i=0; i < num_vals; i++) { /* pack the proc identifier */ PMIX_BFROPS_PACK_TYPE(ret, buffer, &pinfo[i].proc, 1, PMIX_PROC, regtypes); @@ -846,6 +1029,12 @@ pmix_status_t pmix_bfrops_base_pack_darray(pmix_pointer_array_t *regtypes, pmix_status_t ret; int32_t i; + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_DATA_ARRAY != type) { + return PMIX_ERR_BAD_PARAM; + } for (i=0; i < num_vals; i++) { /* pack the actual type in the array */ if (PMIX_SUCCESS != (ret = pmix_bfrop_store_data_type(regtypes, buffer, @@ -879,6 +1068,13 @@ pmix_status_t pmix_bfrops_base_pack_rank(pmix_pointer_array_t *regtypes, int32_t num_vals, pmix_data_type_t type) { pmix_status_t ret; + + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_PROC_RANK != type) { + return PMIX_ERR_BAD_PARAM; + } PMIX_BFROPS_PACK_TYPE(ret, buffer, src, num_vals, PMIX_UINT32, regtypes); return ret; } @@ -892,6 +1088,12 @@ pmix_status_t pmix_bfrops_base_pack_query(pmix_pointer_array_t *regtypes, int32_t i; int32_t nkeys; + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_QUERY != type) { + return PMIX_ERR_BAD_PARAM; + } for (i=0; i < num_vals; i++) { /* pack the number of keys */ nkeys = pmix_argv_count(pq[i].keys); @@ -963,6 +1165,13 @@ pmix_status_t pmix_bfrops_base_pack_alloc_directive(pmix_pointer_array_t *regtyp int32_t num_vals, pmix_data_type_t type) { pmix_status_t ret; + + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_ALLOC_DIRECTIVE != type) { + return PMIX_ERR_BAD_PARAM; + } PMIX_BFROPS_PACK_TYPE(ret, buffer, src, num_vals, PMIX_UINT8, regtypes); return ret; } @@ -972,6 +1181,13 @@ pmix_status_t pmix_bfrops_base_pack_iof_channel(pmix_pointer_array_t *regtypes, int32_t num_vals, pmix_data_type_t type) { pmix_status_t ret; + + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_IOF_CHANNEL != type) { + return PMIX_ERR_BAD_PARAM; + } PMIX_BFROPS_PACK_TYPE(ret, buffer, src, num_vals, PMIX_UINT16, regtypes); return ret; } @@ -984,6 +1200,12 @@ pmix_status_t pmix_bfrops_base_pack_envar(pmix_pointer_array_t *regtypes, int32_t i; pmix_status_t ret; + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_ENVAR != type) { + return PMIX_ERR_BAD_PARAM; + } for (i=0; i < num_vals; ++i) { /* pack the name */ PMIX_BFROPS_PACK_TYPE(ret, buffer, &ptr[i].envar, 1, PMIX_STRING, regtypes); @@ -1012,6 +1234,12 @@ pmix_status_t pmix_bfrops_base_pack_coord(pmix_pointer_array_t *regtypes, int32_t i; pmix_status_t ret; + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_COORD != type) { + return PMIX_ERR_BAD_PARAM; + } for (i=0; i < num_vals; ++i) { /* pack the x-coord */ PMIX_BFROPS_PACK_TYPE(ret, buffer, &ptr[i].x, 1, PMIX_INT, regtypes); @@ -1041,6 +1269,12 @@ pmix_status_t pmix_bfrops_base_pack_regattr(pmix_pointer_array_t *regtypes, pmix_status_t ret; char *foo; + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_REGATTR != type) { + return PMIX_ERR_BAD_PARAM; + } for (i=0; i < num_vals; ++i) { /* pack the name */ PMIX_BFROPS_PACK_TYPE(ret, buffer, &ptr[i].name, 1, PMIX_STRING, regtypes); diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/bfrops/base/bfrop_base_print.c b/opal/mca/pmix/pmix4x/pmix/src/mca/bfrops/base/bfrop_base_print.c index b1554299bc..d6effe7073 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/bfrops/base/bfrop_base_print.c +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/bfrops/base/bfrop_base_print.c @@ -64,6 +64,9 @@ int pmix_bfrops_base_print_bool(char **output, char *prefix, char *prefx; int ret; + if (PMIX_BOOL != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -105,6 +108,9 @@ int pmix_bfrops_base_print_byte(char **output, char *prefix, char *prefx; int ret; + if (PMIX_BYTE != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -145,6 +151,9 @@ int pmix_bfrops_base_print_string(char **output, char *prefix, char *prefx; int ret; + if (PMIX_STRING != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -185,6 +194,9 @@ int pmix_bfrops_base_print_size(char **output, char *prefix, char *prefx; int ret; + if (PMIX_SIZE != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -225,6 +237,9 @@ int pmix_bfrops_base_print_pid(char **output, char *prefix, char *prefx; int ret; + if (PMIX_PID != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -264,6 +279,9 @@ int pmix_bfrops_base_print_int(char **output, char *prefix, char *prefx; int ret; + if (PMIX_INT != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -304,6 +322,9 @@ int pmix_bfrops_base_print_uint(char **output, char *prefix, char *prefx; int ret; + if (PMIX_UINT != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -344,6 +365,9 @@ int pmix_bfrops_base_print_uint8(char **output, char *prefix, char *prefx; int ret; + if (PMIX_UINT8 != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -384,6 +408,9 @@ int pmix_bfrops_base_print_uint16(char **output, char *prefix, char *prefx; int ret; + if (PMIX_UINT16 != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -424,6 +451,9 @@ int pmix_bfrops_base_print_uint32(char **output, char *prefix, char *prefx; int ret; + if (PMIX_UINT32 != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -464,6 +494,9 @@ int pmix_bfrops_base_print_int8(char **output, char *prefix, char *prefx; int ret; + if (PMIX_INT8 != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -504,6 +537,9 @@ int pmix_bfrops_base_print_int16(char **output, char *prefix, char *prefx; int ret; + if (PMIX_INT16 != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -544,6 +580,9 @@ int pmix_bfrops_base_print_int32(char **output, char *prefix, char *prefx; int ret; + if (PMIX_INT32 != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -584,6 +623,9 @@ int pmix_bfrops_base_print_uint64(char **output, char *prefix, char *prefx; int ret; + if (PMIX_UINT64 != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -625,6 +667,9 @@ int pmix_bfrops_base_print_int64(char **output, char *prefix, char *prefx; int ret; + if (PMIX_INT64 != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -665,6 +710,9 @@ int pmix_bfrops_base_print_float(char **output, char *prefix, char *prefx; int ret; + if (PMIX_FLOAT != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -705,6 +753,9 @@ int pmix_bfrops_base_print_double(char **output, char *prefix, char *prefx; int ret; + if (PMIX_DOUBLE != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -746,6 +797,9 @@ int pmix_bfrops_base_print_time(char **output, char *prefix, char *t; int ret; + if (PMIX_TIME != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -789,6 +843,9 @@ int pmix_bfrops_base_print_timeval(char **output, char *prefix, char *prefx; int ret; + if (PMIX_TIMEVAL != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -830,6 +887,9 @@ int pmix_bfrops_base_print_status(char **output, char *prefix, char *prefx; int ret; + if (PMIX_STATUS != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -877,6 +937,9 @@ int pmix_bfrops_base_print_status(char **output, char *prefix, int rc; pmix_regattr_t *r; + if (PMIX_VALUE != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -1054,6 +1117,9 @@ int pmix_bfrops_base_print_info(char **output, char *prefix, char *tmp=NULL, *tmp2=NULL; int ret; + if (PMIX_INFO != type) { + return PMIX_ERR_BAD_PARAM; + } pmix_bfrops_base_print_value(&tmp, NULL, &src->value, PMIX_VALUE); pmix_bfrops_base_print_info_directives(&tmp2, NULL, &src->flags, PMIX_INFO_DIRECTIVES); ret = asprintf(output, "%sKEY: %s\n%s\t%s\n%s\t%s", prefix, src->key, @@ -1073,6 +1139,9 @@ int pmix_bfrops_base_print_pdata(char **output, char *prefix, char *tmp1, *tmp2; int ret; + if (PMIX_PDATA != type) { + return PMIX_ERR_BAD_PARAM; + } pmix_bfrops_base_print_proc(&tmp1, NULL, &src->proc, PMIX_PROC); pmix_bfrops_base_print_value(&tmp2, NULL, &src->value, PMIX_VALUE); ret = asprintf(output, "%s %s KEY: %s %s", prefix, tmp1, src->key, @@ -1093,12 +1162,20 @@ int pmix_bfrops_base_print_pdata(char **output, char *prefix, int pmix_bfrops_base_print_buf(char **output, char *prefix, pmix_buffer_t *src, pmix_data_type_t type) { + if (NULL == output || NULL == prefix || + NULL == src || PMIX_BUFFER != type) { + return PMIX_ERR_BAD_PARAM; + } return PMIX_SUCCESS; } int pmix_bfrops_base_print_app(char **output, char *prefix, pmix_app_t *src, pmix_data_type_t type) { + if (NULL == output || NULL == prefix || + NULL == src || PMIX_APP != type) { + return PMIX_ERR_BAD_PARAM; + } return PMIX_SUCCESS; } @@ -1108,6 +1185,9 @@ int pmix_bfrops_base_print_proc(char **output, char *prefix, char *prefx; int rc; + if (PMIX_PROC != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -1147,6 +1227,10 @@ int pmix_bfrops_base_print_proc(char **output, char *prefix, int pmix_bfrops_base_print_kval(char **output, char *prefix, pmix_kval_t *src, pmix_data_type_t type) { + if (NULL == output || NULL == prefix || + NULL == src || PMIX_KVAL != type) { + return PMIX_ERR_BAD_PARAM; + } return PMIX_SUCCESS; } @@ -1155,6 +1239,9 @@ int pmix_bfrops_base_print_persist(char **output, char *prefix, { char *prefx; + if (PMIX_PERSIST != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -1191,6 +1278,9 @@ pmix_status_t pmix_bfrops_base_print_scope(char **output, char *prefix, { char *prefx; + if (PMIX_SCOPE != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -1217,6 +1307,9 @@ pmix_status_t pmix_bfrops_base_print_range(char **output, char *prefix, { char *prefx; + if (PMIX_DATA_RANGE != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -1242,6 +1335,9 @@ pmix_status_t pmix_bfrops_base_print_cmd(char **output, char *prefix, { char *prefx; + if (PMIX_COMMAND != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -1268,6 +1364,9 @@ pmix_status_t pmix_bfrops_base_print_info_directives(char **output, char *prefix { char *prefx; + if (PMIX_INFO_DIRECTIVES != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -1295,6 +1394,9 @@ pmix_status_t pmix_bfrops_base_print_datatype(char **output, char *prefix, char *prefx; int ret; + if (PMIX_DATA_TYPE != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -1335,6 +1437,9 @@ int pmix_bfrops_base_print_bo(char **output, char *prefix, char *prefx; int ret; + if (PMIX_BYTE_OBJECT != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -1375,6 +1480,9 @@ int pmix_bfrops_base_print_ptr(char **output, char *prefix, char *prefx; int ret; + if (PMIX_POINTER != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -1403,6 +1511,9 @@ pmix_status_t pmix_bfrops_base_print_pstate(char **output, char *prefix, char *prefx; int ret; + if (PMIX_PROC_STATE != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -1433,6 +1544,9 @@ pmix_status_t pmix_bfrops_base_print_pinfo(char **output, char *prefix, pmix_status_t rc = PMIX_SUCCESS; char *p2, *tmp; + if (PMIX_PROC_INFO != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -1475,6 +1589,9 @@ pmix_status_t pmix_bfrops_base_print_darray(char **output, char *prefix, char *prefx; int ret; + if (PMIX_DATA_ARRAY != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -1506,6 +1623,9 @@ pmix_status_t pmix_bfrops_base_print_query(char **output, char *prefix, char *tmp, *t2, *t3; size_t n; + if (PMIX_QUERY != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -1577,6 +1697,9 @@ pmix_status_t pmix_bfrops_base_print_rank(char **output, char *prefix, char *prefx; int rc; + if (PMIX_PROC_RANK != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -1622,6 +1745,9 @@ pmix_status_t pmix_bfrops_base_print_alloc_directive(char **output, char *prefix char *prefx; int ret; + if (PMIX_ALLOC_DIRECTIVE != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -1651,6 +1777,9 @@ pmix_status_t pmix_bfrops_base_print_iof_channel(char **output, char *prefix, char *prefx; int ret; + if (PMIX_IOF_CHANNEL != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -1680,6 +1809,9 @@ pmix_status_t pmix_bfrops_base_print_envar(char **output, char *prefix, char *prefx; int ret; + if (PMIX_ENVAR != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -1711,6 +1843,9 @@ pmix_status_t pmix_bfrops_base_print_coord(char **output, char *prefix, char *prefx; int ret; + if (PMIX_COORD != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { @@ -1740,6 +1875,9 @@ pmix_status_t pmix_bfrops_base_print_regattr(char **output, char *prefix, char *prefx; int ret; + if (PMIX_REGATTR != type) { + return PMIX_ERR_BAD_PARAM; + } /* deal with NULL prefix */ if (NULL == prefix) { if (0 > asprintf(&prefx, " ")) { diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/bfrops/base/bfrop_base_unpack.c b/opal/mca/pmix/pmix4x/pmix/src/mca/bfrops/base/bfrop_base_unpack.c index dfe3d59261..43afa63e62 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/bfrops/base/bfrop_base_unpack.c +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/bfrops/base/bfrop_base_unpack.c @@ -160,6 +160,13 @@ pmix_status_t pmix_bfrops_base_unpack(pmix_pointer_array_t *regtypes, pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, "pmix_bfrop_unpack_bool * %d\n", (int)*num_vals); + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_BOOL != type) { + return PMIX_ERR_BAD_PARAM; + } + /* check to see if there's enough data in buffer */ if (pmix_bfrop_too_small(buffer, *num_vals)) { return PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER; @@ -193,6 +200,10 @@ pmix_status_t pmix_bfrops_base_unpack_int(pmix_pointer_array_t *regtypes, pmix_status_t ret; pmix_data_type_t remote_type; + if (PMIX_INT != type && PMIX_UINT != type) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_SUCCESS != (ret = pmix_bfrop_get_data_type(regtypes, buffer, &remote_type))) { return ret; } @@ -219,6 +230,10 @@ pmix_status_t pmix_bfrops_base_unpack_sizet(pmix_pointer_array_t *regtypes, pmix_status_t ret; pmix_data_type_t remote_type; + if (PMIX_SIZE != type) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_SUCCESS != (ret = pmix_bfrop_get_data_type(regtypes, buffer, &remote_type))) { PMIX_ERROR_LOG(ret); @@ -250,6 +265,10 @@ pmix_status_t pmix_bfrops_base_unpack_pid(pmix_pointer_array_t *regtypes, pmix_status_t ret; pmix_data_type_t remote_type; + if (PMIX_PID != type) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_SUCCESS != (ret = pmix_bfrop_get_data_type(regtypes, buffer, &remote_type))) { return ret; } @@ -279,6 +298,13 @@ pmix_status_t pmix_bfrops_base_unpack_byte(pmix_pointer_array_t *regtypes, pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, "pmix_bfrop_unpack_byte * %d\n", (int)*num_vals); + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_BYTE != type && PMIX_UINT8 != type && PMIX_INT8 != type) { + return PMIX_ERR_BAD_PARAM; + } + /* check to see if there's enough data in buffer */ if (pmix_bfrop_too_small(buffer, *num_vals)) { return PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER; @@ -303,6 +329,13 @@ pmix_status_t pmix_bfrops_base_unpack_int16(pmix_pointer_array_t *regtypes, pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, "pmix_bfrop_unpack_int16 * %d\n", (int)*num_vals); + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_INT16 != type && PMIX_UINT16 != type) { + return PMIX_ERR_BAD_PARAM; + } + /* check to see if there's enough data in buffer */ if (pmix_bfrop_too_small(buffer, (*num_vals)*sizeof(tmp))) { return PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER; @@ -329,6 +362,13 @@ pmix_status_t pmix_bfrops_base_unpack_int32(pmix_pointer_array_t *regtypes, pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, "pmix_bfrop_unpack_int32 * %d\n", (int)*num_vals); + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_INT32 != type && PMIX_UINT32 != type) { + return PMIX_ERR_BAD_PARAM; + } + /* check to see if there's enough data in buffer */ if (pmix_bfrop_too_small(buffer, (*num_vals)*sizeof(tmp))) { return PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER; @@ -351,6 +391,9 @@ pmix_status_t pmix_bfrops_base_unpack_datatype(pmix_pointer_array_t *regtypes, { pmix_status_t ret; + if (PMIX_DATA_TYPE != type) { + return PMIX_ERR_BAD_PARAM; + } PMIX_BFROPS_UNPACK_TYPE(ret, buffer, dest, num_vals, PMIX_INT16, regtypes); return ret; } @@ -365,6 +408,13 @@ pmix_status_t pmix_bfrops_base_unpack_int64(pmix_pointer_array_t *regtypes, pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, "pmix_bfrop_unpack_int64 * %d\n", (int)*num_vals); + if (NULL == regtypes) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_INT64 != type && PMIX_UINT64 != type) { + return PMIX_ERR_BAD_PARAM; + } + /* check to see if there's enough data in buffer */ if (pmix_bfrop_too_small(buffer, (*num_vals)*sizeof(tmp))) { return PMIX_ERR_UNPACK_READ_PAST_END_OF_BUFFER; @@ -389,6 +439,10 @@ pmix_status_t pmix_bfrops_base_unpack_string(pmix_pointer_array_t *regtypes, int32_t i, len, n=1; char **sdest = (char**) dest; + if (PMIX_STRING != type) { + return PMIX_ERR_BAD_PARAM; + } + for (i = 0; i < (*num_vals); ++i) { PMIX_BFROPS_UNPACK_TYPE(ret, buffer, &len, &n, PMIX_INT32, regtypes); if (PMIX_SUCCESS != ret) { @@ -423,6 +477,10 @@ pmix_status_t pmix_bfrops_base_unpack_float(pmix_pointer_array_t *regtypes, pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, "pmix_bfrop_unpack_float * %d\n", (int)*num_vals); + if (PMIX_FLOAT != type) { + return PMIX_ERR_BAD_PARAM; + } + /* unpack the data */ for (i = 0; i < (*num_vals); ++i) { n=1; @@ -452,6 +510,10 @@ pmix_status_t pmix_bfrops_base_unpack_double(pmix_pointer_array_t *regtypes, pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, "pmix_bfrop_unpack_double * %d\n", (int)*num_vals); + if (PMIX_DOUBLE != type) { + return PMIX_ERR_BAD_PARAM; + } + /* unpack the data */ for (i = 0; i < (*num_vals); ++i) { n=1; @@ -481,6 +543,10 @@ pmix_status_t pmix_bfrops_base_unpack_timeval(pmix_pointer_array_t *regtypes, pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, "pmix_bfrop_unpack_timeval * %d\n", (int)*num_vals); + if (PMIX_TIMEVAL != type) { + return PMIX_ERR_BAD_PARAM; + } + /* unpack the data */ for (i = 0; i < (*num_vals); ++i) { n=2; @@ -511,6 +577,10 @@ pmix_status_t pmix_bfrops_base_unpack_time(pmix_pointer_array_t *regtypes, pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, "pmix_bfrop_unpack_time * %d\n", (int)*num_vals); + if (PMIX_TIME != type) { + return PMIX_ERR_BAD_PARAM; + } + /* unpack the data */ for (i = 0; i < (*num_vals); ++i) { n=1; @@ -534,6 +604,10 @@ pmix_status_t pmix_bfrops_base_unpack_status(pmix_pointer_array_t *regtypes, pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, "pmix_bfrop_unpack_status * %d\n", (int)*num_vals); + if (PMIX_STATUS != type) { + return PMIX_ERR_BAD_PARAM; + } + /* unpack the data */ PMIX_BFROPS_UNPACK_TYPE(ret, buffer, dest, num_vals, PMIX_INT32, regtypes); return ret; @@ -608,6 +682,10 @@ pmix_status_t pmix_bfrops_base_unpack_value(pmix_pointer_array_t *regtypes, ptr = (pmix_value_t *) dest; n = *num_vals; + if (PMIX_VALUE != type) { + return PMIX_ERR_BAD_PARAM; + } + for (i = 0; i < n; ++i) { /* unpack the type */ if (PMIX_SUCCESS != (ret = pmix_bfrop_get_data_type(regtypes, buffer, &ptr[i].type))) { @@ -635,6 +713,10 @@ pmix_status_t pmix_bfrops_base_unpack_info(pmix_pointer_array_t *regtypes, pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, "pmix_bfrop_unpack: %d info", *num_vals); + if (PMIX_INFO != type) { + return PMIX_ERR_BAD_PARAM; + } + ptr = (pmix_info_t *) dest; n = *num_vals; @@ -688,6 +770,10 @@ pmix_status_t pmix_bfrops_base_unpack_pdata(pmix_pointer_array_t *regtypes, pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, "pmix_bfrop_unpack: %d pdata", *num_vals); + if (PMIX_PDATA != type) { + return PMIX_ERR_BAD_PARAM; + } + ptr = (pmix_pdata_t *) dest; n = *num_vals; @@ -742,6 +828,10 @@ pmix_status_t pmix_bfrops_base_unpack_buf(pmix_pointer_array_t *regtypes, ptr = (pmix_buffer_t *) dest; n = *num_vals; + if (PMIX_BUFFER != type) { + return PMIX_ERR_BAD_PARAM; + } + for (i = 0; i < n; ++i) { PMIX_CONSTRUCT(&ptr[i], pmix_buffer_t); /* unpack the type of buffer */ @@ -789,6 +879,10 @@ pmix_status_t pmix_bfrops_base_unpack_proc(pmix_pointer_array_t *regtypes, pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, "pmix_bfrop_unpack: %d procs", *num_vals); + if (PMIX_PROC != type) { + return PMIX_ERR_BAD_PARAM; + } + ptr = (pmix_proc_t *) dest; n = *num_vals; @@ -832,6 +926,10 @@ pmix_status_t pmix_bfrops_base_unpack_app(pmix_pointer_array_t *regtypes, pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, "pmix_bfrop_unpack: %d apps", *num_vals); + if (PMIX_APP != type) { + return PMIX_ERR_BAD_PARAM; + } + ptr = (pmix_app_t *) dest; n = *num_vals; @@ -924,6 +1022,10 @@ pmix_status_t pmix_bfrops_base_unpack_kval(pmix_pointer_array_t *regtypes, pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, "pmix_bfrop_unpack: %d kvals", *num_vals); + if (PMIX_KVAL != type) { + return PMIX_ERR_BAD_PARAM; + } + ptr = (pmix_kval_t*) dest; n = *num_vals; @@ -952,6 +1054,11 @@ pmix_status_t pmix_bfrops_base_unpack_persist(pmix_pointer_array_t *regtypes, int32_t *num_vals, pmix_data_type_t type) { pmix_status_t ret; + + if (PMIX_PERSIST != type) { + return PMIX_ERR_BAD_PARAM; + } + PMIX_BFROPS_UNPACK_TYPE(ret, buffer, dest, num_vals, PMIX_BYTE, regtypes); return ret; } @@ -967,6 +1074,10 @@ pmix_status_t pmix_bfrops_base_unpack_bo(pmix_pointer_array_t *regtypes, pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, "pmix_bfrop_unpack: %d byte_object", *num_vals); + if (PMIX_BYTE_OBJECT != type) { + return PMIX_ERR_BAD_PARAM; + } + ptr = (pmix_byte_object_t *) dest; n = *num_vals; @@ -998,6 +1109,16 @@ pmix_status_t pmix_bfrops_base_unpack_ptr(pmix_pointer_array_t *regtypes, int32_t cnt=1; pmix_status_t ret; + if (NULL == dest) { + return PMIX_ERR_BAD_PARAM; + } + if (NULL == num_vals) { + return PMIX_ERR_BAD_PARAM; + } + if (PMIX_POINTER != type) { + return PMIX_ERR_BAD_PARAM; + } + /* it obviously makes no sense to pack a pointer and * send it somewhere else, so we just unpack the sentinel */ PMIX_BFROPS_UNPACK_TYPE(ret, buffer, &foo, &cnt, PMIX_UINT8, regtypes); @@ -1009,6 +1130,11 @@ pmix_status_t pmix_bfrops_base_unpack_scope(pmix_pointer_array_t *regtypes, int32_t *num_vals, pmix_data_type_t type) { pmix_status_t ret; + + if (PMIX_SCOPE != type) { + return PMIX_ERR_BAD_PARAM; + } + PMIX_BFROPS_UNPACK_TYPE(ret, buffer, dest, num_vals, PMIX_UINT8, regtypes); return ret; } @@ -1018,6 +1144,11 @@ pmix_status_t pmix_bfrops_base_unpack_range(pmix_pointer_array_t *regtypes, int32_t *num_vals, pmix_data_type_t type) { pmix_status_t ret; + + if (PMIX_DATA_RANGE != type) { + return PMIX_ERR_BAD_PARAM; + } + PMIX_BFROPS_UNPACK_TYPE(ret, buffer, dest, num_vals, PMIX_UINT8, regtypes); return ret; } @@ -1027,6 +1158,11 @@ pmix_status_t pmix_bfrops_base_unpack_cmd(pmix_pointer_array_t *regtypes, int32_t *num_vals, pmix_data_type_t type) { pmix_status_t ret; + + if (PMIX_COMMAND != type) { + return PMIX_ERR_BAD_PARAM; + } + PMIX_BFROPS_UNPACK_TYPE(ret, buffer, dest, num_vals, PMIX_UINT8, regtypes); return ret; } @@ -1036,6 +1172,11 @@ pmix_status_t pmix_bfrops_base_unpack_info_directives(pmix_pointer_array_t *regt int32_t *num_vals, pmix_data_type_t type) { pmix_status_t ret; + + if (PMIX_INFO_DIRECTIVES != type) { + return PMIX_ERR_BAD_PARAM; + } + PMIX_BFROPS_UNPACK_TYPE(ret, buffer, dest, num_vals, PMIX_UINT32, regtypes); return ret; } @@ -1045,6 +1186,11 @@ pmix_status_t pmix_bfrops_base_unpack_pstate(pmix_pointer_array_t *regtypes, int32_t *num_vals, pmix_data_type_t type) { pmix_status_t ret; + + if (PMIX_PROC_STATE != type) { + return PMIX_ERR_BAD_PARAM; + } + PMIX_BFROPS_UNPACK_TYPE(ret, buffer, dest, num_vals, PMIX_UINT8, regtypes); return ret; } @@ -1061,6 +1207,10 @@ pmix_status_t pmix_bfrops_base_unpack_pinfo(pmix_pointer_array_t *regtypes, pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, "pmix_bfrop_unpack: %d pinfo", *num_vals); + if (PMIX_PROC_INFO != type) { + return PMIX_ERR_BAD_PARAM; + } + ptr = (pmix_proc_info_t *) dest; n = *num_vals; @@ -1112,6 +1262,10 @@ pmix_status_t pmix_bfrops_base_unpack_darray(pmix_pointer_array_t *regtypes, pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, "pmix_bfrop_unpack: %d data arrays", *num_vals); + if (PMIX_DATA_ARRAY != type) { + return PMIX_ERR_BAD_PARAM; + } + ptr = (pmix_data_array_t *) dest; n = *num_vals; @@ -1153,6 +1307,11 @@ pmix_status_t pmix_bfrops_base_unpack_rank(pmix_pointer_array_t *regtypes, int32_t *num_vals, pmix_data_type_t type) { pmix_status_t ret; + + if (PMIX_PROC_RANK != type) { + return PMIX_ERR_BAD_PARAM; + } + PMIX_BFROPS_UNPACK_TYPE(ret, buffer, dest, num_vals, PMIX_UINT32, regtypes); return ret; } @@ -1169,6 +1328,10 @@ pmix_status_t pmix_bfrops_base_unpack_query(pmix_pointer_array_t *regtypes, pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, "pmix_bfrop_unpack: %d queries", *num_vals); + if (PMIX_QUERY != type) { + return PMIX_ERR_BAD_PARAM; + } + ptr = (pmix_query_t *) dest; n = *num_vals; @@ -1216,6 +1379,11 @@ pmix_status_t pmix_bfrops_base_unpack_alloc_directive(pmix_pointer_array_t *regt int32_t *num_vals, pmix_data_type_t type) { pmix_status_t ret; + + if (PMIX_ALLOC_DIRECTIVE != type) { + return PMIX_ERR_BAD_PARAM; + } + PMIX_BFROPS_UNPACK_TYPE(ret, buffer, dest, num_vals, PMIX_UINT8, regtypes); return ret; } @@ -1225,6 +1393,11 @@ pmix_status_t pmix_bfrops_base_unpack_iof_channel(pmix_pointer_array_t *regtypes int32_t *num_vals, pmix_data_type_t type) { pmix_status_t ret; + + if (PMIX_IOF_CHANNEL != type) { + return PMIX_ERR_BAD_PARAM; + } + PMIX_BFROPS_UNPACK_TYPE(ret, buffer, dest, num_vals, PMIX_UINT16, regtypes); return ret; } @@ -1240,6 +1413,10 @@ pmix_status_t pmix_bfrops_base_unpack_envar(pmix_pointer_array_t *regtypes, pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, "pmix_bfrop_unpack: %d envars", *num_vals); + if (PMIX_ENVAR != type) { + return PMIX_ERR_BAD_PARAM; + } + ptr = (pmix_envar_t *) dest; n = *num_vals; @@ -1278,6 +1455,10 @@ pmix_status_t pmix_bfrops_base_unpack_coord(pmix_pointer_array_t *regtypes, pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, "pmix_bfrop_unpack: %d coordinates", *num_vals); + if (PMIX_COORD != type) { + return PMIX_ERR_BAD_PARAM; + } + ptr = (pmix_coord_t *) dest; n = *num_vals; @@ -1316,6 +1497,10 @@ pmix_status_t pmix_bfrops_base_unpack_regattr(pmix_pointer_array_t *regtypes, pmix_output_verbose(20, pmix_bfrops_base_framework.framework_output, "pmix_bfrop_unpack: %d regattrs", *num_vals); + if (PMIX_REGATTR != type) { + return PMIX_ERR_BAD_PARAM; + } + ptr = (pmix_regattr_t *) dest; n = *num_vals; diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/gds/base/gds_base_fns.c b/opal/mca/pmix/pmix4x/pmix/src/mca/gds/base/gds_base_fns.c index 4c3a494abe..54ad969322 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/gds/base/gds_base_fns.c +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/gds/base/gds_base_fns.c @@ -80,7 +80,8 @@ pmix_status_t pmix_gds_base_setup_fork(const pmix_proc_t *proc, if (NULL == active->module->setup_fork) { continue; } - if (PMIX_SUCCESS != (rc = active->module->setup_fork(proc, env))) { + rc = active->module->setup_fork(proc, env); + if (PMIX_SUCCESS != rc && PMIX_ERR_NOT_AVAILABLE != rc) { return rc; } } @@ -94,6 +95,7 @@ pmix_status_t pmix_gds_base_store_modex(struct pmix_namespace_t *nspace, pmix_gds_base_store_modex_cb_fn_t cb_fn, void *cbdata) { + (void)nspace; pmix_status_t rc = PMIX_SUCCESS; pmix_buffer_t bkt; pmix_byte_object_t bo, bo2; diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/gds/hash/gds_hash.c b/opal/mca/pmix/pmix4x/pmix/src/mca/gds/hash/gds_hash.c index 1f3c1d0d69..89d1d3bcf1 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/gds/hash/gds_hash.c +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/gds/hash/gds_hash.c @@ -202,8 +202,16 @@ static pmix_status_t hash_assign_module(pmix_info_t *info, size_t ninfo, return PMIX_SUCCESS; } +/* Define a bitmask to track what information may not have + * been provided but is computable from other info */ +#define PMIX_HASH_PROC_DATA 0x00000001 +#define PMIX_HASH_JOB_SIZE 0x00000002 +#define PMIX_HASH_MAX_PROCS 0x00000004 +#define PMIX_HASH_NUM_NODES 0x00000008 + static pmix_status_t store_map(pmix_hash_table_t *ht, - char **nodes, char **ppn) + char **nodes, char **ppn, + uint32_t flags) { pmix_status_t rc; pmix_value_t *val; @@ -213,6 +221,8 @@ static pmix_status_t store_map(pmix_hash_table_t *ht, bool updated; pmix_kval_t *kp2; char **procs; + uint32_t totalprocs=0; + bool localldr; pmix_output_verbose(2, pmix_gds_base_framework.framework_output, "[%s:%d] gds:hash:store_map", @@ -224,6 +234,22 @@ static pmix_status_t store_map(pmix_hash_table_t *ht, return PMIX_ERR_BAD_PARAM; } + /* if they didn't provide the number of nodes, then + * compute it from the list of nodes */ + if (!(PMIX_HASH_NUM_NODES & flags)) { + kp2 = PMIX_NEW(pmix_kval_t); + kp2->key = strdup(PMIX_NUM_NODES); + kp2->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + kp2->value->type = PMIX_UINT32; + kp2->value->data.uint32 = pmix_argv_count(nodes); + if (PMIX_SUCCESS != (rc = pmix_hash_store(ht, PMIX_RANK_WILDCARD, kp2))) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(kp2); + return rc; + } + PMIX_RELEASE(kp2); // maintain acctg + } + for (n=0; NULL != nodes[n]; n++) { /* check and see if we already have data for this node */ val = NULL; @@ -241,18 +267,22 @@ static pmix_status_t store_map(pmix_hash_table_t *ht, } iptr = (pmix_info_t*)val->data.darray->array; updated = false; + localldr = false; for (m=0; m < val->data.darray->size; m++) { - if (0 == strncmp(iptr[m].key, PMIX_LOCAL_PEERS, PMIX_MAX_KEYLEN)) { + if (PMIX_CHECK_KEY(&iptr[m], PMIX_LOCAL_PEERS)) { /* we will update this entry */ if (NULL != iptr[m].value.data.string) { free(iptr[m].value.data.string); } iptr[m].value.data.string = strdup(ppn[n]); - updated = true; - break; + updated = true; // no need to add the local_peers to the array + } else if (PMIX_CHECK_KEY(&iptr[m], PMIX_LOCALLDR)) { + rank = strtoul(ppn[n], NULL, 10); + iptr[m].value.data.rank = rank; + localldr = true; // no need to add localldr to the array } } - if (!updated) { + if (!updated || !localldr) { /* append this entry to the current data */ kp2 = PMIX_NEW(pmix_kval_t); if (NULL == kp2) { @@ -271,7 +301,18 @@ static pmix_status_t store_map(pmix_hash_table_t *ht, return PMIX_ERR_NOMEM; } kp2->value->data.darray->type = PMIX_INFO; - kp2->value->data.darray->size = val->data.darray->size + 1; + /* if we didn't update the local leader, then we will + * add it here */ + m = 0; + if (!localldr) { + kp2->value->data.darray->size = val->data.darray->size + 1; + ++m; + } + /* if they didn't update the local peers, then we add it here */ + if (!updated) { + kp2->value->data.darray->size = val->data.darray->size + 1; + ++m; + } PMIX_INFO_CREATE(info, kp2->value->data.darray->size); if (NULL == info) { PMIX_RELEASE(kp2); @@ -281,7 +322,15 @@ static pmix_status_t store_map(pmix_hash_table_t *ht, for (m=0; m < val->data.darray->size; m++) { PMIX_INFO_XFER(&info[m], &iptr[m]); } - PMIX_INFO_LOAD(&info[kp2->value->data.darray->size-1], PMIX_LOCAL_PEERS, ppn[n], PMIX_STRING); + if (!updated) { + PMIX_INFO_LOAD(&info[kp2->value->data.darray->size-m], PMIX_LOCAL_PEERS, ppn[n], PMIX_STRING); + --m; + } + if (!localldr) { + rank = strtoul(ppn[n], NULL, 10); + PMIX_INFO_LOAD(&info[kp2->value->data.darray->size-m], PMIX_LOCALLDR, &rank, PMIX_PROC_RANK); + --m; + } kp2->value->data.darray->array = info; if (PMIX_SUCCESS != (rc = pmix_hash_store(ht, PMIX_RANK_WILDCARD, kp2))) { PMIX_ERROR_LOG(rc); @@ -309,14 +358,16 @@ static pmix_status_t store_map(pmix_hash_table_t *ht, return PMIX_ERR_NOMEM; } kp2->value->data.darray->type = PMIX_INFO; - PMIX_INFO_CREATE(info, 1); + PMIX_INFO_CREATE(info, 2); if (NULL == info) { PMIX_RELEASE(kp2); return PMIX_ERR_NOMEM; } PMIX_INFO_LOAD(&info[0], PMIX_LOCAL_PEERS, ppn[n], PMIX_STRING); + rank = strtoul(ppn[n], NULL, 10); + PMIX_INFO_LOAD(&info[1], PMIX_LOCALLDR, &rank, PMIX_PROC_RANK); kp2->value->data.darray->array = info; - kp2->value->data.darray->size = 1; + kp2->value->data.darray->size = 2; if (PMIX_SUCCESS != (rc = pmix_hash_store(ht, PMIX_RANK_WILDCARD, kp2))) { PMIX_ERROR_LOG(rc); PMIX_RELEASE(kp2); @@ -327,6 +378,7 @@ static pmix_status_t store_map(pmix_hash_table_t *ht, /* split the list of procs so we can store their * individual location data */ procs = pmix_argv_split(ppn[n], ','); + totalprocs += pmix_argv_count(procs); for (m=0; NULL != procs[m]; m++) { /* store the hostname for each proc */ kp2 = PMIX_NEW(pmix_kval_t); @@ -342,6 +394,48 @@ static pmix_status_t store_map(pmix_hash_table_t *ht, return rc; } PMIX_RELEASE(kp2); // maintain acctg + if (!(PMIX_HASH_PROC_DATA & flags)) { + /* add an entry for the nodeid */ + kp2 = PMIX_NEW(pmix_kval_t); + kp2->key = strdup(PMIX_NODEID); + kp2->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + kp2->value->type = PMIX_UINT32; + kp2->value->data.uint32 = n; + if (PMIX_SUCCESS != (rc = pmix_hash_store(ht, rank, kp2))) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(kp2); + pmix_argv_free(procs); + return rc; + } + PMIX_RELEASE(kp2); // maintain acctg + /* add an entry for the local rank */ + kp2 = PMIX_NEW(pmix_kval_t); + kp2->key = strdup(PMIX_LOCAL_RANK); + kp2->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + kp2->value->type = PMIX_UINT16; + kp2->value->data.uint16 = m; + if (PMIX_SUCCESS != (rc = pmix_hash_store(ht, rank, kp2))) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(kp2); + pmix_argv_free(procs); + return rc; + } + PMIX_RELEASE(kp2); // maintain acctg + /* add an entry for the node rank - for now, we assume + * only the one job is running */ + kp2 = PMIX_NEW(pmix_kval_t); + kp2->key = strdup(PMIX_NODE_RANK); + kp2->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + kp2->value->type = PMIX_UINT16; + kp2->value->data.uint16 = m; + if (PMIX_SUCCESS != (rc = pmix_hash_store(ht, rank, kp2))) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(kp2); + pmix_argv_free(procs); + return rc; + } + PMIX_RELEASE(kp2); // maintain acctg + } } pmix_argv_free(procs); } @@ -361,6 +455,41 @@ static pmix_status_t store_map(pmix_hash_table_t *ht, } PMIX_RELEASE(kp2); // maintain acctg + /* if they didn't provide the job size, compute it as + * being the number of provided procs (i.e., size of + * ppn list) */ + if (!(PMIX_HASH_JOB_SIZE & flags)) { + kp2 = PMIX_NEW(pmix_kval_t); + kp2->key = strdup(PMIX_JOB_SIZE); + kp2->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + kp2->value->type = PMIX_UINT32; + kp2->value->data.uint32 = totalprocs; + if (PMIX_SUCCESS != (rc = pmix_hash_store(ht, PMIX_RANK_WILDCARD, kp2))) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(kp2); + return rc; + } + PMIX_RELEASE(kp2); // maintain acctg + } + + /* if they didn't provide a value for max procs, just + * assume it is the same as the number of procs in the + * job and store it */ + if (!(PMIX_HASH_MAX_PROCS & flags)) { + kp2 = PMIX_NEW(pmix_kval_t); + kp2->key = strdup(PMIX_MAX_PROCS); + kp2->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + kp2->value->type = PMIX_UINT32; + kp2->value->data.uint32 = totalprocs; + if (PMIX_SUCCESS != (rc = pmix_hash_store(ht, PMIX_RANK_WILDCARD, kp2))) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(kp2); + return rc; + } + PMIX_RELEASE(kp2); // maintain acctg + } + + return PMIX_SUCCESS; } @@ -377,6 +506,7 @@ pmix_status_t hash_cache_job_info(struct pmix_namespace_t *ns, pmix_rank_t rank; pmix_status_t rc=PMIX_SUCCESS; size_t n, j, size, len; + uint32_t flags = 0; pmix_output_verbose(2, pmix_gds_base_framework.framework_output, "[%s:%d] gds:hash:cache_job_info for nspace %s", @@ -432,29 +562,14 @@ pmix_status_t hash_cache_job_info(struct pmix_namespace_t *ns, PMIX_ERROR_LOG(rc); goto release; } - /* if we have already found the proc map, then parse - * and store the detailed map */ - if (NULL != procs) { - if (PMIX_SUCCESS != (rc = store_map(ht, nodes, procs))) { - PMIX_ERROR_LOG(rc); - goto release; - } - } } else if (0 == strcmp(info[n].key, PMIX_PROC_MAP)) { /* parse the regex to get the argv array containing proc ranks on each node */ if (PMIX_SUCCESS != (rc = pmix_preg.parse_procs(info[n].value.data.string, &procs))) { PMIX_ERROR_LOG(rc); goto release; } - /* if we have already recv'd the node map, then parse - * and store the detailed map */ - if (NULL != nodes) { - if (PMIX_SUCCESS != (rc = store_map(ht, nodes, procs))) { - PMIX_ERROR_LOG(rc); - goto release; - } - } } else if (0 == strcmp(info[n].key, PMIX_PROC_DATA)) { + flags |= PMIX_HASH_PROC_DATA; /* an array of data pertaining to a specific proc */ if (PMIX_DATA_ARRAY != info[n].value.type) { PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); @@ -500,6 +615,10 @@ pmix_status_t hash_cache_job_info(struct pmix_namespace_t *ns, kp2->value->data.bo.size = len; } } + pmix_output_verbose(2, pmix_gds_base_framework.framework_output, + "[%s:%d] gds:hash:cache_job_info data for rank %u: key %s", + pmix_globals.myid.nspace, pmix_globals.myid.rank, + rank, kp2->key); /* store it in the hash_table */ if (PMIX_SUCCESS != (rc = pmix_hash_store(ht, rank, kp2))) { PMIX_ERROR_LOG(rc); @@ -544,9 +663,15 @@ pmix_status_t hash_cache_job_info(struct pmix_namespace_t *ns, goto release; } PMIX_RELEASE(kp2); // maintain acctg - /* if this is the job size, then store it */ - if (0 == strncmp(info[n].key, PMIX_JOB_SIZE, PMIX_MAX_KEYLEN)) { + /* if this is the job size, then store it in + * the nptr tracker and flag that we were given it */ + if (PMIX_CHECK_KEY(&info[n], PMIX_JOB_SIZE)) { nptr->nprocs = info[n].value.data.uint32; + flags |= PMIX_HASH_JOB_SIZE; + } else if (PMIX_CHECK_KEY(&info[n], PMIX_NUM_NODES)) { + flags |= PMIX_HASH_NUM_NODES; + } else if (PMIX_CHECK_KEY(&info[n], PMIX_MAX_PROCS)) { + flags |= PMIX_HASH_MAX_PROCS; } } } @@ -578,6 +703,17 @@ pmix_status_t hash_cache_job_info(struct pmix_namespace_t *ns, trk->gdata_added = true; } + /* we must have the proc AND node maps */ + if (NULL == procs || NULL == nodes) { + rc = PMIX_ERR_NOT_FOUND; + goto release; + } + + if (PMIX_SUCCESS != (rc = store_map(ht, nodes, procs, flags))) { + PMIX_ERROR_LOG(rc); + goto release; + } + release: if (NULL != nodes) { pmix_argv_free(nodes); @@ -645,25 +781,24 @@ static pmix_status_t register_info(pmix_peer_t *peer, for (rank=0; rank < ns->nprocs; rank++) { val = NULL; rc = pmix_hash_fetch(ht, rank, NULL, &val); - if (PMIX_SUCCESS != rc) { + if (PMIX_SUCCESS != rc && PMIX_ERR_PROC_ENTRY_NOT_FOUND != rc) { PMIX_ERROR_LOG(rc); if (NULL != val) { PMIX_VALUE_RELEASE(val); } return rc; } - if (NULL == val) { - return PMIX_ERR_NOT_FOUND; - } PMIX_CONSTRUCT(&buf, pmix_buffer_t); PMIX_BFROPS_PACK(rc, peer, &buf, &rank, 1, PMIX_PROC_RANK); - info = (pmix_info_t*)val->data.darray->array; - ninfo = val->data.darray->size; - for (n=0; n < ninfo; n++) { - kv.key = info[n].key; - kv.value = &info[n].value; - PMIX_BFROPS_PACK(rc, peer, &buf, &kv, 1, PMIX_KVAL); + if (NULL != val) { + info = (pmix_info_t*)val->data.darray->array; + ninfo = val->data.darray->size; + for (n=0; n < ninfo; n++) { + kv.key = info[n].key; + kv.value = &info[n].value; + PMIX_BFROPS_PACK(rc, peer, &buf, &kv, 1, PMIX_KVAL); + } } kv.key = PMIX_PROC_BLOB; kv.value = &blob; @@ -708,6 +843,9 @@ static pmix_status_t hash_register_job_info(struct pmix_peer_t *pr, * for another peer in this nspace so we don't waste * time doing it again */ if (NULL != ns->jobbkt) { + pmix_output_verbose(2, pmix_gds_base_framework.framework_output, + "[%s:%d] gds:hash:register_job_info copying prepacked payload", + pmix_globals.myid.nspace, pmix_globals.myid.rank); /* we have packed this before - can just deliver it */ PMIX_BFROPS_COPY_PAYLOAD(rc, peer, reply, ns->jobbkt); if (PMIX_SUCCESS != rc) { @@ -715,7 +853,7 @@ static pmix_status_t hash_register_job_info(struct pmix_peer_t *pr, } /* now see if we have delivered it to all our local * clients for this nspace */ - if (ns->ndelivered == ns->nlocalprocs) { + if (!PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer) && ns->ndelivered == ns->nlocalprocs) { /* we have, so let's get rid of the packed * copy of the data */ PMIX_RELEASE(ns->jobbkt); @@ -734,6 +872,9 @@ static pmix_status_t hash_register_job_info(struct pmix_peer_t *pr, trk->ns = strdup(ns->nspace); } break; + } else if (0 == strcmp(ns->nspace, t2->ns)) { + trk = t2; + break; } } if (NULL == trk) { @@ -748,6 +889,9 @@ static pmix_status_t hash_register_job_info(struct pmix_peer_t *pr, * been given to us in the info array - pack * them for delivery */ /* pack the name of the nspace */ + pmix_output_verbose(2, pmix_gds_base_framework.framework_output, + "[%s:%d] gds:hash:register_job_info packing new payload", + pmix_globals.myid.nspace, pmix_globals.myid.rank); msg = ns->nspace; PMIX_BFROPS_PACK(rc, peer, reply, &msg, 1, PMIX_STRING); if (PMIX_SUCCESS != rc) { @@ -759,7 +903,7 @@ static pmix_status_t hash_register_job_info(struct pmix_peer_t *pr, if (PMIX_SUCCESS == rc) { /* if we have more than one local client for this nspace, * save this packed object so we don't do this again */ - if (1 < ns->nlocalprocs) { + if (PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer) || 1 < ns->nlocalprocs) { PMIX_RETAIN(reply); ns->jobbkt = reply; } diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/pcompress/base/pcompress_base_frame.c b/opal/mca/pmix/pmix4x/pmix/src/mca/pcompress/base/pcompress_base_frame.c index fdf3f01ab2..9d9aec70c2 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/pcompress/base/pcompress_base_frame.c +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/pcompress/base/pcompress_base_frame.c @@ -28,12 +28,18 @@ static bool compress_block(char *instring, uint8_t **outbytes, size_t *nbytes) { + (void)instring; + (void)outbytes; + (void)nbytes; return false; } static bool decompress_block(char **outstring, uint8_t *inbytes, size_t len) { + (void)outstring; + (void)inbytes; + (void)len; return false; } @@ -53,6 +59,7 @@ pmix_compress_base_component_t pmix_compress_base_selected_component = {{0}}; static int pmix_compress_base_register(pmix_mca_base_register_flag_t flags) { + (void)flags; pmix_compress_base.compress_limit = 4096; (void) pmix_mca_base_var_register("pmix", "compress", "base", "limit", "Threshold beyond which data will be compressed", diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/Makefile.am b/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/Makefile.am new file mode 100644 index 0000000000..497185d85d --- /dev/null +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/Makefile.am @@ -0,0 +1,41 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2019 Intel, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# main library setup +noinst_LTLIBRARIES = libmca_pfexec.la +libmca_pfexec_la_SOURCES = + +# pkgdata setup +dist_pmixdata_DATA = + +# local files +headers = pfexec.h +libmca_pfexec_la_SOURCES += $(headers) + +# Conditionally install the header files +if WANT_INSTALL_HEADERS +pmixdir = $(pmixincludedir)/$(subdir) +nobase_pmix_HEADERS = $(headers) +endif + +include base/Makefile.am + +distclean-local: + rm -f base/static-components.h diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/base/Makefile.am b/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/base/Makefile.am new file mode 100644 index 0000000000..b86ad88963 --- /dev/null +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/base/Makefile.am @@ -0,0 +1,30 @@ +# +# Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2012-2013 Los Alamos National Security, LLC. +# All rights reserved +# Copyright (c) 2019 Intel, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +headers += \ + base/base.h + +libmca_pfexec_la_SOURCES += \ + base/pfexec_base_frame.c \ + base/pfexec_base_select.c \ + base/pfexec_base_default_fns.c + +dist_pmixdata_DATA += base/help-pfexec-base.txt diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/base/base.h b/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/base/base.h new file mode 100644 index 0000000000..6fc23dd98d --- /dev/null +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/base/base.h @@ -0,0 +1,205 @@ +/* + * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2011 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2011 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2013 Los Alamos National Security, LLC. All rights reserved. + * Copyright (c) 2017-2019 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +/** @file: + */ + +#ifndef MCA_PFEXEC_BASE_H +#define MCA_PFEXEC_BASE_H + +/* + * includes + */ +#include "pmix_config.h" + +#include "src/class/pmix_list.h" +#include "src/mca/mca.h" +#include "src/common/pmix_iof.h" +#include "src/mca/pfexec/pfexec.h" + + +BEGIN_C_DECLS + +/* + * MCA framework + */ +PMIX_EXPORT extern pmix_mca_base_framework_t pmix_pfexec_base_framework; +/* + * Select an available component. + */ +PMIX_EXPORT pmix_status_t pmix_pfexec_base_select(void); + +typedef struct { + int usepty; + bool connect_stdin; + + /* private - callers should not modify these fields */ + int p_stdin[2]; + int p_stdout[2]; + int p_stderr[2]; +} pmix_pfexec_base_io_conf_t; + +typedef struct { + pmix_list_item_t super; + pmix_event_t ev; + pmix_rank_t rank; + pid_t pid; + bool completed; + int exitcode; + pmix_pfexec_base_io_conf_t opts; + pmix_iof_read_event_t *stdoutev; + pmix_iof_read_event_t *stderrev; +} pmix_pfexec_child_t; +PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_pfexec_child_t); + +typedef struct { + pmix_event_t handler; + bool active; + pmix_list_t children; + int timeout_before_sigkill; + size_t next; +} pmix_pfexec_globals_t; + +PMIX_EXPORT extern pmix_pfexec_globals_t pmix_pfexec_globals; + +/* define a function that will fork/exec a local proc */ +typedef pmix_status_t (*pmix_pfexec_base_fork_proc_fn_t)(pmix_app_t *app, + pmix_pfexec_child_t *child, + char **env); + +/* define a function type for signaling a local proc */ +typedef pmix_status_t (*pmix_pfexec_base_signal_local_fn_t)(pid_t pd, int signum); + +typedef struct { + pmix_object_t super; + pmix_event_t ev; + const pmix_info_t *jobinfo; + size_t njinfo; + const pmix_app_t *apps; + size_t napps; + pmix_pfexec_base_fork_proc_fn_t frkfn; + pmix_lock_t *lock; +} pmix_pfexec_fork_caddy_t; +PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_pfexec_fork_caddy_t); + +typedef struct { + pmix_object_t super; + pmix_event_t ev; + pmix_rank_t rank; + int signal; + pmix_pfexec_base_signal_local_fn_t sigfn; + pmix_lock_t *lock; +} pmix_pfexec_signal_caddy_t; +PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_pfexec_signal_caddy_t); + + +PMIX_EXPORT void pmix_pfexec_base_spawn_proc(int sd, short args, void *cbdata); + +PMIX_EXPORT void pmix_pfexec_base_kill_proc(int sd, short args, void *cbdata); + +PMIX_EXPORT void pmix_pfexec_base_signal_proc(int sd, short args, void *cbdata); + +PMIX_EXPORT void pmix_pfexec_check_complete(int sd, short args, void *cbdata); + +#define PMIX_PFEXEC_SPAWN(fcd, j, nj, a, na, fn, lk) \ + do { \ + (fcd) = PMIX_NEW(pmix_pfexec_fork_caddy_t); \ + (fcd)->jobinfo = (j); \ + (fcd)->njinfo = (nj); \ + (fcd)->apps = (a); \ + (fcd)->napps = (na); \ + (fcd)->frkfn = (fn); \ + (fcd)->lock = (lk); \ + pmix_event_assign(&((fcd)->ev), pmix_globals.evbase, -1, \ + EV_WRITE, pmix_pfexec_base_spawn_proc, (fcd)); \ + PMIX_POST_OBJECT((fcd)); \ + pmix_event_active(&((fcd)->ev), EV_WRITE, 1); \ + } while(0) + +#define PMIX_PFEXEC_KILL(scd, r, fn, lk) \ + do { \ + (scd) = PMIX_NEW(pmix_pfexec_signal_caddy_t); \ + (scd)->rank = (r); \ + (scd)->sigfn = (fn); \ + (scd)->lock = (lk); \ + pmix_event_assign(&((scd)->ev), pmix_globals.evbase, -1, \ + EV_WRITE, pmix_pfexec_base_kill_proc, (scd)); \ + PMIX_POST_OBJECT((scd)); \ + pmix_event_active(&((scd)->ev), EV_WRITE, 1); \ + } while(0) + +#define PMIX_PFEXEC_SIGNAL(scd, r, nm, fn, lk) \ + do { \ + (scd) = PMIX_NEW(pmix_pfexec_signal_caddy_t); \ + (scd)->rank = (r); \ + (scd)->signal = (nm); \ + (scd)->sigfn = (fn); \ + (scd)->lock = (lk); \ + pmix_event_assign(&((scd)->ev), pmix_globals.evbase, -1, \ + EV_WRITE, pmix_pfexec_base_signal_proc, (scd)); \ + PMIX_POST_OBJECT((scd)); \ + pmix_event_active(&((scd)->ev), EV_WRITE, 1); \ + } while(0) + +typedef struct { + pmix_object_t super; + pmix_event_t ev; + pmix_pfexec_child_t *child; +} pmix_pfexec_cmpl_caddy_t; +PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_pfexec_cmpl_caddy_t); + +#define PMIX_PFEXEC_CHK_COMPLETE(c) \ + do { \ + pmix_pfexec_cmpl_caddy_t *pc = PMIX_NEW(pmix_pfexec_cmpl_caddy_t); \ + pc->child = (c); \ + pmix_event_assign(&((pc)->ev), pmix_globals.evbase, -1, \ + EV_WRITE, pmix_pfexec_check_complete, (pc)); \ + PMIX_POST_OBJECT((pc)); \ + pmix_event_active(&((pc)->ev), EV_WRITE, 1); \ + } while(0) + +/* + * Struct written up the pipe from the child to the parent. + */ +typedef struct { + /* True if the child has died; false if this is just a warning to + be printed. */ + bool fatal; + /* Relevant only if fatal==true */ + int exit_status; + + /* Length of the strings that are written up the pipe after this + struct */ + int file_str_len; + int topic_str_len; + int msg_str_len; +} pmix_pfexec_pipe_err_msg_t; + +PMIX_EXPORT pmix_status_t pmix_pfexec_base_setup_child(pmix_pfexec_child_t *child); + +/* + * Max length of strings from the pmix_pfexec_pipe_err_msg_t + */ +#define PMIX_PFEXEC_MAX_FILE_LEN 511 +#define PMIX_PFEXEC_MAX_TOPIC_LEN PMIX_PFEXEC_MAX_FILE_LEN + + +END_C_DECLS +#endif diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/base/help-pfexec-base.txt b/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/base/help-pfexec-base.txt new file mode 100644 index 0000000000..e9806448bc --- /dev/null +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/base/help-pfexec-base.txt @@ -0,0 +1,103 @@ +# -*- text -*- +# +# Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2014 Research Organization for Information Science +# and Technology (RIST). All rights reserved. +# Copyright (c) 2017-2019 Intel, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# +# This is the US/English general help file for Open RTE's ODLS Framework +# +[orte-odls-base:could-not-kill] +WARNING: A process refused to die despite all the efforts! +This process may still be running and/or consuming resources. + +Host: %s +PID: %d + +[orte-odls-base:could-not-preload-binary] +WARNING: Could not preload the binary file. + +Binary: %s + +Will continue attempting to launch the process. +[orte-odls-base:could-not-preload-files] +WARNING: Could not preload the files specified. + +Fileset: %s + +Will continue attempting to launch the process. +[orte-odls-base:could-not-preload] +WARNING: Could not preload the requested files and directories. + +Binary : %s +Fileset: %s + +Will continue attempting to launch the process. + +# +[orte-odls-base:xterm-rank-out-of-bounds] +The xterm option was asked to display a rank that is larger +than the number of procs in the job: + +Node: %s +Rank: %d +Num procs: %d + +Note that ranks start with 0, not 1, and must be specified +accordingly. +# +[orte-odls-base:xterm-neg-rank] +The xterm option was asked to display a rank that is negative: + +Rank: %d +Num procs: %d + +Note that ranks start with 0, not 1, and must be specified +accordingly. +# +[orte-odls-base:show-bindings] +System has detected external process binding to cores %04lx. +# +[warn not bound] +A request to bind the processes to a %s was made, but the operation +resulted in the processes being unbound. This was most likely caused +by the following: + + %s + +This is only a warning that can be suppressed in the future by +setting the odls_warn_if_not_bound MCA parameter to 0. Execution +will continue. + + Local host: %s + Application name: %s + Action requested: %s %s +# +[error not bound] +A request to bind the processes to a %s was made, but the operation +resulted in the processes being unbound. This was most likely caused +by the following: + + %s + +This is an error; your job will now abort. + + Local host: %s + Application name: %s + Action requested: %s %s +# +[orte-odls-base:fork-agent-not-found] +The specified fork agent was not found: + + Node: %s + Fork agent: %s + +The application cannot be launched. diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/base/pfexec_base_default_fns.c b/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/base/pfexec_base_default_fns.c new file mode 100644 index 0000000000..cd60797193 --- /dev/null +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/base/pfexec_base_default_fns.c @@ -0,0 +1,522 @@ +/* + * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2011 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2007-2011 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. + * Copyright (c) 2011-2013 Los Alamos National Security, LLC. + * All rights reserved. + * Copyright (c) 2011-2017 Cisco Systems, Inc. All rights reserved + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2017 Mellanox Technologies Ltd. All rights reserved. + * Copyright (c) 2017 IBM Corporation. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "pmix_config.h" + +#include +#ifdef HAVE_UNISTD_H +#include +#endif +#include +#include +#ifdef HAVE_SYS_WAIT_H +#include +#endif +#include +#ifdef HAVE_UTIL_H +#include +#endif +#ifdef HAVE_PTY_H +#include +#endif +#ifdef HAVE_FCNTL_H +#include +#endif +#ifdef HAVE_TERMIOS_H +#include +# ifdef HAVE_TERMIO_H +# include +# endif +#endif +#ifdef HAVE_LIBUTIL_H +#include +#endif + +#include +#include +#include "pmix_common.h" + +#include "src/include/pmix_stdint.h" +#include "src/include/pmix_globals.h" +#include "src/threads/threads.h" +#include "src/util/argv.h" +#include "src/util/context_fns.h" +#include "src/util/error.h" +#include "src/util/name_fns.h" +#include "src/util/os_dirpath.h" +#include "src/util/os_path.h" +#include "src/util/path.h" +#include "src/util/pmix_environ.h" +#include "src/util/pmix_pty.h" +#include "src/util/printf.h" +#include "src/util/show_help.h" + +#include "src/client/pmix_client_ops.h" +#include "src/server/pmix_server_ops.h" +#include "src/mca/pfexec/base/base.h" + +static pmix_status_t setup_prefork(pmix_pfexec_child_t *child); + + +static pmix_status_t setup_path(pmix_app_t *app) +{ + pmix_status_t rc; + char dir[MAXPATHLEN]; + + /* see if the app specifies a working dir */ + if (NULL != app->cwd) { + /* Try to change to the app's cwd and check that the app + exists and is executable The function will + take care of outputting a pretty error message, if required + */ + if (PMIX_SUCCESS != (rc = pmix_util_check_context_cwd(app))) { + /* do not ERROR_LOG - it will be reported elsewhere */ + return rc; + } + + /* The prior function will have done a chdir() to jump us to + * wherever the app is to be executed. It seems that chdir doesn't + * adjust the $PWD enviro variable when it changes the directory. This + * can cause a user to get a different response when doing getcwd vs + * looking at the enviro variable. To keep this consistent, we explicitly + * ensure that the PWD enviro variable matches the CWD we moved to. + * + * NOTE: if a user's program does a chdir(), then $PWD will once + * again not match getcwd! This is beyond our control - we are only + * ensuring they start out matching. + */ + if (NULL == getcwd(dir, sizeof(dir))) { + return PMIX_ERR_OUT_OF_RESOURCE; + } + pmix_setenv("PWD", dir, true, &app->env); + } + + /* ensure the app is pointing to a full path */ + rc = pmix_util_check_context_app(app, app->env); + + return rc; +} + + +void pmix_pfexec_base_spawn_proc(int sd, short args, void *cbdata) +{ + (void)sd; + (void)args; + pmix_pfexec_fork_caddy_t *fcd = (pmix_pfexec_fork_caddy_t*)cbdata; + pmix_app_t *app; + int i, n; + size_t m, k; + pmix_status_t rc; + char **argv = NULL, **env = NULL; + char basedir[MAXPATHLEN]; + pmix_pfexec_child_t *child; + pmix_proc_t proc; + pmix_rank_info_t *info; + pmix_namespace_t *nptr, *n2; + + pmix_output_verbose(5, pmix_pfexec_base_framework.framework_output, + "%s pfexec:base spawn proc", + PMIX_NAME_PRINT(&pmix_globals.myid)); + + /* establish our baseline working directory - we will be potentially + * bouncing around as we execute various apps, but we will always return + * to this place as our default directory + */ + if (NULL == getcwd(basedir, sizeof(basedir))) { + rc = PMIX_ERROR; + goto complete; + } + + /* ensure our nspace is on the server global list */ + nptr = NULL; + PMIX_LIST_FOREACH(n2, &pmix_server_globals.nspaces, pmix_namespace_t) { + if (0 == strcmp(n2->nspace, pmix_globals.myid.nspace)) { + nptr = n2; + break; + } + } + if (NULL == nptr) { + /* add it */ + nptr = PMIX_NEW(pmix_namespace_t); + nptr->nspace = strdup(pmix_globals.myid.nspace); + pmix_list_append(&pmix_server_globals.nspaces, &nptr->super); + } + /* mark all children as "registered" so collectives don't falter */ + nptr->all_registered = true; + + PMIX_LOAD_NSPACE(proc.nspace, pmix_globals.myid.nspace); + for (m=0; m < fcd->napps; m++) { + app = (pmix_app_t*)&fcd->apps[m]; + /* merge our launch environment into the proc */ + for (i=0; NULL != environ[i]; i++) { + pmix_argv_append_unique_nosize(&app->env, environ[i]); + } + + /* check for a fork/exec agent we should use */ + if (NULL != app->info) { + for (k=0; k < app->ninfo; k++) { + if (PMIX_CHECK_KEY(&app->info[k], PMIX_FORK_EXEC_AGENT)) { + /* we were given a fork agent - use it. We have to put its + * argv at the beginning of the app argv array */ + argv = pmix_argv_split(app->info[k].value.data.string, ' '); + /* add in the argv from the app */ + for (i=0; NULL != argv[i]; i++) { + pmix_argv_prepend_nosize(&app->argv, argv[i]); + } + if (NULL != app->cmd) { + free(app->cmd); + } + app->cmd = pmix_path_findv(argv[0], X_OK, app->env, NULL); + if (NULL == app->cmd) { + pmix_show_help("help-pfexec-base.txt", + "fork-agent-not-found", + true, pmix_globals.hostname, argv[0]); + rc = PMIX_ERR_NOT_FOUND; + pmix_argv_free(argv); + goto complete; + } + pmix_argv_free(argv); + } + } + } + + /* setup the path */ + if (PMIX_SUCCESS != (rc = setup_path(app))) { + goto complete; + } + + for (n=0; n < app->maxprocs; n++) { + /* create a tracker for this child */ + child = PMIX_NEW(pmix_pfexec_child_t); + pmix_list_append(&pmix_pfexec_globals.children, &child->super); + + /* setup any IOF */ + child->opts.usepty = PMIX_ENABLE_PTY_SUPPORT; + if (PMIX_SUCCESS != (rc = setup_prefork(child))) { + PMIX_ERROR_LOG(rc); + pmix_list_remove_item(&pmix_pfexec_globals.children, &child->super); + PMIX_RELEASE(child); + goto complete; + } + + /* register this client in case they callback to us */ + info = PMIX_NEW(pmix_rank_info_t); + if (NULL == info) { + rc = PMIX_ERR_NOMEM; + pmix_list_remove_item(&pmix_pfexec_globals.children, &child->super); + PMIX_RELEASE(child); + goto complete; + } + info->pname.nspace = strdup(pmix_globals.myid.nspace); + info->pname.rank = child->rank; + info->uid = pmix_globals.uid; + info->gid = pmix_globals.gid; + pmix_list_append(&nptr->ranks, &info->super); + + /* setup the PMIx environment */ + env = pmix_argv_copy(app->env); + proc.rank = child->rank; + rc = PMIx_server_setup_fork(&proc, &env); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + pmix_list_remove_item(&pmix_pfexec_globals.children, &child->super); + PMIX_RELEASE(child); + pmix_argv_free(env); + goto complete; + } + pmix_output_verbose(5, pmix_pfexec_base_framework.framework_output, + "%s pfexec:base spawning child %s", + PMIX_NAME_PRINT(&pmix_globals.myid), app->cmd); + + rc = fcd->frkfn(app, child, env); + pmix_argv_free(env); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + pmix_list_remove_item(&pmix_pfexec_globals.children, &child->super); + PMIX_RELEASE(child); + goto complete; + } + PMIX_IOF_READ_ACTIVATE(child->stdoutev); + PMIX_IOF_READ_ACTIVATE(child->stderrev); + } + } + + /* ensure we reset our working directory back to our default location */ + if (0 != chdir(basedir)) { + PMIX_ERROR_LOG(PMIX_ERROR); + } + + complete: + fcd->lock->status = rc; + PMIX_WAKEUP_THREAD(fcd->lock); + return; +} + +void pmix_pfexec_base_kill_proc(int sd, short args, void *cbdata) +{ + (void)sd; + (void)args; + pmix_pfexec_signal_caddy_t *scd = (pmix_pfexec_signal_caddy_t*)cbdata; + pmix_pfexec_child_t *child, *cd; + + /* find the process */ + child = NULL; + PMIX_LIST_FOREACH(cd, &pmix_pfexec_globals.children, pmix_pfexec_child_t) { + if (scd->rank == cd->rank) { + child = cd; + break; + } + } + if (NULL == child) { + scd->lock->status = PMIX_SUCCESS; + PMIX_WAKEUP_THREAD(scd->lock); + return; + } + +#if 0 + /* if we opened the stdin IOF channel, be sure + * we close it */ + if (NULL != orte_iof.close) { + orte_iof.close(&child->name, ORTE_IOF_STDIN); + } +#endif + + /* remove the child from the list so waitpid callback won't + * find it as this induces unmanageable race + * conditions when we are deliberately killing the process + */ + pmix_list_remove_item(&pmix_pfexec_globals.children, &child->super); + + /* First send a SIGCONT in case the process is in stopped state. + If it is in a stopped state and we do not first change it to + running, then SIGTERM will not get delivered. Ignore return + value. */ + PMIX_OUTPUT_VERBOSE((5, pmix_pfexec_base_framework.framework_output, + "%s SENDING SIGCONT", + PMIX_NAME_PRINT(&pmix_globals.myid))); + scd->sigfn(child->pid, SIGCONT); + + /* wait a little to give the proc a chance to wakeup */ + sleep(pmix_pfexec_globals.timeout_before_sigkill); + /* issue a SIGTERM */ + PMIX_OUTPUT_VERBOSE((5, pmix_pfexec_base_framework.framework_output, + "%s SENDING SIGTERM", + PMIX_NAME_PRINT(&pmix_globals.myid))); + scd->lock->status = scd->sigfn(child->pid, SIGTERM); + + if (0 != scd->lock->status) { + /* wait a little again */ + sleep(pmix_pfexec_globals.timeout_before_sigkill); + /* issue a SIGKILL */ + PMIX_OUTPUT_VERBOSE((5, pmix_pfexec_base_framework.framework_output, + "%s SENDING SIGKILL", + PMIX_NAME_PRINT(&pmix_globals.myid))); + scd->lock->status = scd->sigfn(child->pid, SIGKILL); + } + + /* cleanup */ + PMIX_RELEASE(child); + PMIX_WAKEUP_THREAD(scd->lock); + +#if 0 + /* ensure the child's session directory is cleaned up */ + orte_session_dir_finalize(&child->name); +#endif + + return; +} + +void pmix_pfexec_base_signal_proc(int sd, short args, void *cbdata) +{ + (void)sd; + (void)args; + pmix_pfexec_signal_caddy_t *scd = (pmix_pfexec_signal_caddy_t*)cbdata; + pmix_pfexec_child_t *child, *cd; + + /* find the process */ + child = NULL; + PMIX_LIST_FOREACH(cd, &pmix_pfexec_globals.children, pmix_pfexec_child_t) { + if (scd->rank == cd->rank) { + child = cd; + break; + } + } + if (NULL == child) { + scd->lock->status = PMIX_SUCCESS; + PMIX_WAKEUP_THREAD(scd->lock); + return; + } + + PMIX_OUTPUT_VERBOSE((5, pmix_pfexec_base_framework.framework_output, + "%s SIGNALING %d", + PMIX_NAME_PRINT(&pmix_globals.myid), scd->signal)); + scd->lock->status = scd->sigfn(child->pid, scd->signal); + + PMIX_WAKEUP_THREAD(scd->lock); +} + +static pmix_status_t setup_prefork(pmix_pfexec_child_t *child) +{ + int ret = -1; + pmix_pfexec_base_io_conf_t *opts = &child->opts; + pmix_proc_t *targets = NULL; + pmix_info_t *directives = NULL; + + fflush(stdout); + + /* first check to make sure we can do ptys */ +#if PMIX_ENABLE_PTY_SUPPORT + if (opts->usepty) { + ret = pmix_openpty(&(opts->p_stdout[0]), &(opts->p_stdout[1]), + (char*)NULL, (struct termios*)NULL, (struct winsize*)NULL); + } +#else + opts->usepty = 0; +#endif + + if (ret < 0) { + opts->usepty = 0; + if (pipe(opts->p_stdout) < 0) { + PMIX_ERROR_LOG(PMIX_ERR_SYS_OTHER); + return PMIX_ERR_SYS_OTHER; + } + } + if (opts->connect_stdin) { + if (pipe(opts->p_stdin) < 0) { + PMIX_ERROR_LOG(PMIX_ERR_SYS_OTHER); + return PMIX_ERR_SYS_OTHER; + } + } + if (pipe(opts->p_stderr) < 0) { + PMIX_ERROR_LOG(PMIX_ERR_SYS_OTHER); + return PMIX_ERR_SYS_OTHER; + } + +#if 0 + /* connect stdin endpoint */ + if (opts->connect_stdin) { + /* and connect the pty to stdin */ + ret = orte_iof.pull(name, ORTE_IOF_STDIN, opts->p_stdin[1]); + if(ORTE_SUCCESS != ret) { + ORTE_ERROR_LOG(ret); + return ret; + } + } +#endif + /* connect read ends to IOF */ + PMIX_IOF_READ_EVENT(&child->stdoutev, + targets, 0, directives, 0, opts->p_stdout[0], + pmix_iof_read_local_handler, false); + PMIX_LOAD_PROCID(&child->stdoutev->name, pmix_globals.myid.nspace, child->rank); + child->stdoutev->childproc = (void*)child; + child->stdoutev->channel = PMIX_FWD_STDOUT_CHANNEL; + PMIX_IOF_READ_EVENT(&child->stderrev, + targets, 0, directives, 0, opts->p_stderr[0], + pmix_iof_read_local_handler, false); + PMIX_LOAD_PROCID(&child->stderrev->name, pmix_globals.myid.nspace, child->rank); + child->stderrev->childproc = (void*)child; + child->stderrev->channel = PMIX_FWD_STDERR_CHANNEL; + + return PMIX_SUCCESS; +} + + +pmix_status_t pmix_pfexec_base_setup_child(pmix_pfexec_child_t *child) +{ + int ret; + pmix_pfexec_base_io_conf_t *opts = &child->opts; + + if (opts->connect_stdin) { + close(opts->p_stdin[1]); + } + close(opts->p_stdout[0]); + close(opts->p_stderr[0]); + + if (opts->usepty) { + /* disable echo */ + struct termios term_attrs; + if (tcgetattr(opts->p_stdout[1], &term_attrs) < 0) { + return PMIX_ERR_SYS_OTHER; + } + term_attrs.c_lflag &= ~ (ECHO | ECHOE | ECHOK | + ECHOCTL | ECHOKE | ECHONL); + term_attrs.c_iflag &= ~ (ICRNL | INLCR | ISTRIP | INPCK | IXON); + term_attrs.c_oflag &= ~ (OCRNL | ONLCR); + if (tcsetattr(opts->p_stdout[1], TCSANOW, &term_attrs) == -1) { + return PMIX_ERR_SYS_OTHER; + } + ret = dup2(opts->p_stdout[1], fileno(stdout)); + if (ret < 0) { + return PMIX_ERR_SYS_OTHER; + } + close(opts->p_stdout[1]); + } else { + if(opts->p_stdout[1] != fileno(stdout)) { + ret = dup2(opts->p_stdout[1], fileno(stdout)); + if (ret < 0) { + return PMIX_ERR_SYS_OTHER; + } + close(opts->p_stdout[1]); + } + } + if (opts->connect_stdin) { + if(opts->p_stdin[0] != fileno(stdin)) { + ret = dup2(opts->p_stdin[0], fileno(stdin)); + if (ret < 0) { + return PMIX_ERR_SYS_OTHER; + } + close(opts->p_stdin[0]); + } + } else { + int fd; + + /* connect input to /dev/null */ + fd = open("/dev/null", O_RDONLY, 0); + if (0 > fd) { + return PMIX_ERROR; + } + if (fd != fileno(stdin)) { + ret = dup2(fd, fileno(stdin)); + if (ret < 0) { + return PMIX_ERR_SYS_OTHER; + } + } + close(fd); + } + + if (opts->p_stderr[1] != fileno(stderr)) { + ret = dup2(opts->p_stderr[1], fileno(stderr)); + if (ret < 0) { + return PMIX_ERR_SYS_OTHER; + } + close(opts->p_stderr[1]); + } + + return PMIX_SUCCESS; +} diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/base/pfexec_base_frame.c b/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/base/pfexec_base_frame.c new file mode 100644 index 0000000000..706aa95407 --- /dev/null +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/base/pfexec_base_frame.c @@ -0,0 +1,259 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2011 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2010-2011 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011-2017 Cisco Systems, Inc. All rights reserved + * Copyright (c) 2011-2013 Los Alamos National Security, LLC. + * All rights reserved. + * Copyright (c) 2014-2018 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2017-2019 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "pmix_config.h" +#include "pmix_common.h" +#include "src/include/types.h" + +#include +#include +#ifdef HAVE_SYS_WAIT_H +#include +#endif + +#include "src/mca/mca.h" +#include "src/mca/base/base.h" +#include "src/threads/threads.h" +#include "src/include/pmix_globals.h" +#include "src/common/pmix_iof.h" +#include "src/client/pmix_client_ops.h" +#include "src/util/error.h" + +#include "src/mca/pfexec/base/base.h" + + +/* + * The following file was created by configure. It contains extern + * statements and the definition of an array of pointers to each + * component's public mca_base_component_t struct. + */ + +#include "src/mca/pfexec/base/static-components.h" + +/* + * Instantiate globals + */ +pmix_pfexec_base_module_t pmix_pfexec = {0}; + +/* + * Framework global variables + */ +pmix_pfexec_globals_t pmix_pfexec_globals = {0}; + +static int pmix_pfexec_base_close(void) +{ + PMIX_LIST_DESTRUCT(&pmix_pfexec_globals.children); + if (pmix_pfexec_globals.active) { + pmix_event_del(&pmix_pfexec_globals.handler); + } + pmix_pfexec_globals.active = false; + + return pmix_mca_base_framework_components_close(&pmix_pfexec_base_framework, NULL); +} + +/* callback from the event library whenever a SIGCHLD is received */ +static void wait_signal_callback(int fd, short event, void *arg) +{ + (void)fd; + (void)event; + pmix_event_t *signal = (pmix_event_t*) arg; + int status; + pid_t pid; + pmix_pfexec_child_t *child; + + PMIX_ACQUIRE_OBJECT(signal); + + if (SIGCHLD != PMIX_EVENT_SIGNAL(signal)) { + return; + } + + /* if we haven't spawned anyone, then ignore this */ + if (0 == pmix_list_get_size(&pmix_pfexec_globals.children)) { + return; + } + + /* reap all queued waitpids until we + * don't get anything valid back */ + while (1) { + pid = waitpid(-1, &status, WNOHANG); + if (-1 == pid && EINTR == errno) { + /* try it again */ + continue; + } + /* if we got garbage, then nothing we can do */ + if (pid <= 0) { + return; + } + + /* we are already in an event, so it is safe to access globals */ + PMIX_LIST_FOREACH(child, &pmix_pfexec_globals.children, pmix_pfexec_child_t) { + if (pid == child->pid) { + /* record the exit status */ + if (WIFEXITED(status)) { + child->exitcode = WEXITSTATUS(status); + } else { + if (WIFSIGNALED(status)) { + child->exitcode = WTERMSIG(status) + 128; + } + } + /* mark the child as complete */ + child->completed = true; + PMIX_PFEXEC_CHK_COMPLETE(child); + break; + } + } + } +} + +void pmix_pfexec_check_complete(int sd, short args, void *cbdata) +{ + (void)sd; + (void)args; + pmix_pfexec_cmpl_caddy_t *cd = (pmix_pfexec_cmpl_caddy_t*)cbdata; + pmix_info_t info[2]; + pmix_status_t rc; + + /* if the waitpid fired and the sink is empty, then that means + * it terminated and all output has been written, so remove + * it from the list of children */ + if (cd->child->completed && + (NULL == cd->child->stdoutev || !cd->child->stdoutev->active) && + (NULL == cd->child->stderrev || !cd->child->stderrev->active)) { + pmix_list_remove_item(&pmix_pfexec_globals.children, &cd->child->super); + PMIX_RELEASE(cd->child); + if (0 == pmix_list_get_size(&pmix_pfexec_globals.children)) { + /* generate a local event indicating job terminated */ + PMIX_INFO_LOAD(&info[0], PMIX_EVENT_NON_DEFAULT, NULL, PMIX_BOOL); + PMIX_INFO_LOAD(&info[1], PMIX_EVENT_AFFECTED_PROC, &pmix_globals.myid, PMIX_PROC); + rc = PMIx_Notify_event(PMIX_ERR_JOB_TERMINATED, + &pmix_globals.myid, PMIX_RANGE_PROC_LOCAL, + info, 2, NULL, NULL); + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + } + } + } + PMIX_RELEASE(cd); +} + +static int pmix_pfexec_register(pmix_mca_base_register_flag_t flags) +{ + (void)flags; + pmix_pfexec_globals.timeout_before_sigkill = 1; + pmix_mca_base_var_register("pmix", "pfexec", "base", "sigkill_timeout", + "Time to wait for a process to die after issuing a kill signal to it", + PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, + PMIX_INFO_LVL_2, + PMIX_MCA_BASE_VAR_SCOPE_READONLY, + &pmix_pfexec_globals.timeout_before_sigkill); + return PMIX_SUCCESS; +} + +/** + * Function for finding and opening either all MCA components, or the one + * that was specifically requested via a MCA parameter. + */ +static int pmix_pfexec_base_open(pmix_mca_base_open_flag_t flags) +{ + sigset_t unblock; + + memset(&pmix_pfexec_globals, 0, sizeof(pmix_pfexec_globals_t)); + + /* setup the list of children */ + PMIX_CONSTRUCT(&pmix_pfexec_globals.children, pmix_list_t); + pmix_pfexec_globals.next = 1; + + /* ensure that SIGCHLD is unblocked as we need to capture it */ + if (0 != sigemptyset(&unblock)) { + return PMIX_ERROR; + } + if (0 != sigaddset(&unblock, SIGCHLD)) { + return PMIX_ERROR; + } + if (0 != sigprocmask(SIG_UNBLOCK, &unblock, NULL)) { + return PMIX_ERR_NOT_SUPPORTED; + } + + /* set to catch SIGCHLD events */ + pmix_event_set(pmix_globals.evbase, + &pmix_pfexec_globals.handler, + SIGCHLD, PMIX_EV_SIGNAL|PMIX_EV_PERSIST, + wait_signal_callback, + &pmix_pfexec_globals.handler); + pmix_pfexec_globals.active = true; + pmix_event_add(&pmix_pfexec_globals.handler, NULL); + + /* Open up all available components */ + return pmix_mca_base_framework_components_open(&pmix_pfexec_base_framework, flags); +} + +PMIX_MCA_BASE_FRAMEWORK_DECLARE(pmix, pfexec, "PMIx fork/exec Subsystem", + pmix_pfexec_register, pmix_pfexec_base_open, pmix_pfexec_base_close, + mca_pfexec_base_static_components, 0); + + +/**** FRAMEWORK CLASS INSTANTIATIONS ****/ + +static void chcon(pmix_pfexec_child_t *p) +{ + pmix_proc_t proc; + + p->rank = pmix_pfexec_globals.next; + PMIX_LOAD_PROCID(&proc, pmix_globals.myid.nspace, p->rank); + pmix_pfexec_globals.next++; + p->pid = 0; +} +static void chdes(pmix_pfexec_child_t *p) +{ + if (NULL != p->stdoutev) { + PMIX_RELEASE(p->stdoutev); + } + if (NULL != p->stderrev) { + PMIX_RELEASE(p->stderrev); + } +} +PMIX_CLASS_INSTANCE(pmix_pfexec_child_t, + pmix_list_item_t, + chcon, chdes); + +static void fccon(pmix_pfexec_fork_caddy_t *p) +{ + p->jobinfo = NULL; + p->njinfo = 0; + p->apps = NULL; + p->napps = 0; +} +PMIX_CLASS_INSTANCE(pmix_pfexec_fork_caddy_t, + pmix_object_t, + fccon, NULL); + +PMIX_CLASS_INSTANCE(pmix_pfexec_signal_caddy_t, + pmix_object_t, + NULL, NULL); + +PMIX_CLASS_INSTANCE(pmix_pfexec_cmpl_caddy_t, + pmix_object_t, + NULL, NULL); diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/base/pfexec_base_select.c b/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/base/pfexec_base_select.c new file mode 100644 index 0000000000..084c042de8 --- /dev/null +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/base/pfexec_base_select.c @@ -0,0 +1,57 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2015 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2019 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + + +#include "pmix_config.h" +#include "pmix_common.h" + +#include "src/mca/mca.h" +#include "src/mca/base/base.h" + +#include "src/mca/pfexec/base/base.h" + + +/** + * Function for selecting one component from all those that are + * available. + */ +int pmix_pfexec_base_select(void) +{ + pmix_pfexec_base_component_t *best_component = NULL; + pmix_pfexec_base_module_t *best_module = NULL; + + /* + * Select the best component + */ + if (PMIX_SUCCESS != pmix_mca_base_select("pfexec", pmix_pfexec_base_framework.framework_output, + &pmix_pfexec_base_framework.framework_components, + (pmix_mca_base_module_t **) &best_module, + (pmix_mca_base_component_t **) &best_component, NULL) ) { + /* This will only happen if no component was selected */ + return PMIX_ERR_NOT_FOUND; + } + + /* Save the winner */ + pmix_pfexec = *best_module; + + return PMIX_SUCCESS; +} diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/linux/Makefile.am b/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/linux/Makefile.am new file mode 100644 index 0000000000..0ae7713208 --- /dev/null +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/linux/Makefile.am @@ -0,0 +1,48 @@ +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2017 IBM Corporation. All rights reserved. +# Copyright (c) 2017-2019 Intel, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +dist_pmixdata_DATA = help-pfexec-linux.txt + +sources = \ + pfexec_linux.h \ + pfexec_linux_component.c \ + pfexec_linux.c + +# Make the output library in this directory, and name it either +# mca__.la (for DSO builds) or libmca__.la +# (for static builds). + +if MCA_BUILD_pmix_pfexec_linux_DSO +component_noinst = +component_install = mca_pfexec_linux.la +else +component_noinst = libmca_pfexec_linux.la +component_install = +endif + +mcacomponentdir = $(pmixlibdir) +mcacomponent_LTLIBRARIES = $(component_install) +mca_pfexec_linux_la_SOURCES = $(sources) +mca_pfexec_linux_la_LDFLAGS = -module -avoid-version + +noinst_LTLIBRARIES = $(component_noinst) +libmca_pfexec_linux_la_SOURCES =$(sources) +libmca_pfexec_linux_la_LDFLAGS = -module -avoid-version diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/linux/configure.m4 b/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/linux/configure.m4 new file mode 100644 index 0000000000..64fefcbc16 --- /dev/null +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/linux/configure.m4 @@ -0,0 +1,34 @@ +# -*- shell-script -*- +# +# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2011 Los Alamos National Security, LLC. +# All rights reserved. +# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2019 Intel, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# + +# MCA_pfexec_linux_CONFIG([action-if-found], [action-if-not-found]) +# ----------------------------------------------------------- +AC_DEFUN([MCA_pmix_pfexec_linux_CONFIG],[ + AC_CONFIG_FILES([src/mca/pfexec/linux/Makefile]) + + AC_CHECK_FUNC([fork], [pfexec_linux_happy="yes"], [pfexec_linux_happy="no"]) + + AS_IF([test "$pfexec_linux_happy" = "yes"], [$1], [$2]) + +])dnl + diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/linux/help-pfexec-linux.txt b/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/linux/help-pfexec-linux.txt new file mode 100644 index 0000000000..d07788b14e --- /dev/null +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/linux/help-pfexec-linux.txt @@ -0,0 +1,147 @@ +# -*- text -*- +# +# Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana +# University Research and Technology +# Corporation. All rights reserved. +# Copyright (c) 2004-2005 The University of Tennessee and The University +# of Tennessee Research Foundation. All rights +# reserved. +# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, +# University of Stuttgart. All rights reserved. +# Copyright (c) 2004-2005 The Regents of the University of California. +# All rights reserved. +# Copyright (c) 2009 Sun Microsystems, Inc. All rights reserved. +# Copyright (c) 2010-2011 Cisco Systems, Inc. All rights reserved. +# Copyright (c) 2017-2019 Intel, Inc. All rights reserved. +# $COPYRIGHT$ +# +# Additional copyrights may follow +# +# $HEADER$ +# +# This is a US/English help file. +# +[execve error] +PMIx tried to fork a new process via the "execve" system call but +failed. PMIx checks many things before attempting to launch a +child process, but nothing is perfect. This error may be indicative +of another problem on the target host, or even something as silly as +having specified a directory for your application. Your job will now +abort. + + Local host: %s + Working dir: %s + Application name: %s + Error: %s +# +[binding not supported] +PMIx tried to bind a new process, but process binding is not +supported on the host where it was launched. The process was killed +without launching the target application. Your job will now abort. + + Local host: %s + Application name: %s +# +[binding generic error] +PMIx tried to bind a new process, but something went wrong. The +process was killed without launching the target application. Your job +will now abort. + + Local host: %s + Application name: %s + Error message: %s + Location: %s:%d +# +[bound to everything] +PMIx tried to bind a new process to a specific set of processors, +but ended up binding it to *all* processors. This means that the new +process is effectively unbound. + +This is only a warning -- your job will continue. You can suppress +this warning in the future by setting the odls_warn_if_not_bound MCA +parameter to 0. + + Local host: %s + Application name: %s + Location: %s:%d +# +[slot list and paffinity_alone] +PMIx detected that both a slot list was specified and the MCA +parameter "paffinity_alone" was set to true. Only one of these can be +used at a time. Your job will now abort. + + Local host: %s + Application name: %s +# +[iof setup failed] +PMIx tried to launch a child process but the "IOF child setup" +failed. This should not happen. Your job will now abort. + + Local host: %s + Application name: %s +# +[not bound] +WARNING: PMIx tried to bind a process but failed. This is a +warning only; your job will continue. + + Local host: %s + Application name: %s + Error message: %s + Location: %s:%d +# +[syscall fail] +A system call failed that should not have. In this particular case, +a warning or error message was not displayed that should have been. +Your job may behave unpredictably after this, or abort. + + Local host: %s + Application name: %s + Function: %s + Location: %s:%d +# +[memory not bound] +WARNING: PMIx tried to bind a process but failed. This is a +warning only; your job will continue, though performance may +be degraded. + + Local host: %s + Application name: %s + Error message: %s + Location: %s:%d + +# +[memory binding error] +PMIx tried to bind memory for a new process but something went +wrong. The process was killed without launching the target +application. Your job will now abort. + + Local host: %s + Application name: %s + Error message: %s + Location: %s:%d +# +[set limit] +Error message received from: + + Local host: %s + Application name: %s + Location: %s:%d + +Message: + +%s +# +[incorrectly-bound] +WARNING: PMIx incorrectly bound a process to the daemon's cores. +This is a warning only; your job will continue. + + Local host: %s + Application name: %s + Location: %s:%d +# +[wdir-not-found] +PMIx was unable to launch the specified application as it could not +change to the specified working directory: + +Working directory: %s +Node: %s diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/linux/pfexec_linux.c b/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/linux/pfexec_linux.c new file mode 100644 index 0000000000..ab01ae5b1e --- /dev/null +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/linux/pfexec_linux.c @@ -0,0 +1,608 @@ +/* + * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2008 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2007-2010 Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2007 Evergrid, Inc. All rights reserved. + * Copyright (c) 2008-2017 Cisco Systems, Inc. All rights reserved + * Copyright (c) 2010 IBM Corporation. All rights reserved. + * Copyright (c) 2011-2013 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2017 Rutgers, The State University of New Jersey. + * All rights reserved. + * Copyright (c) 2017 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +/* + * There is a complicated sequence of events that occurs when the + * parent forks a child process that is intended to launch the target + * executable. + * + * Before the child process exec's the target executable, it might tri + * to set the affinity of that new child process according to a + * complex series of rules. This binding may fail in a myriad of + * different ways. A lot of this code deals with reporting that error + * occurately to the end user. This is a complex task in itself + * because the child process is not "really" an PMIX process -- all + * error reporting must be proxied up to the parent who can use normal + * PMIX error reporting mechanisms. + * + * Here's a high-level description of what is occurring in this file: + * + * - parent opens a pipe + * - parent forks a child + * - parent blocks reading on the pipe: the pipe will either close + * (indicating that the child successfully exec'ed) or the child will + * write some proxied error data up the pipe + * + * - the child tries to set affinity and do other housekeeping in + * preparation of exec'ing the target executable + * - if the child fails anywhere along the way, it sends a message up + * the pipe to the parent indicating what happened -- including a + * rendered error message detailing the problem (i.e., human-readable). + * - it is important that the child renders the error message: there + * are so many errors that are possible that the child is really the + * only entity that has enough information to make an accuate error string + * to report back to the user. + * - the parent reads this message + rendered string in and uses PMIX + * reporting mechanisms to display it to the user + * - if the problem was only a warning, the child continues processing + * (potentially eventually exec'ing the target executable). + * - if the problem was an error, the child exits and the parent + * handles the death of the child as appropriate (i.e., this PFEXEC + * simply reports the error -- other things decide what to do). + */ + +#include "pmix_config.h" +#include "pmix.h" +#include "src/include/types.h" + +#include +#include +#ifdef HAVE_UNISTD_H +#include +#endif +#include +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_WAIT_H +#include +#endif +#include +#ifdef HAVE_FCNTL_H +#include +#endif +#ifdef HAVE_SYS_TIME_H +#include +#endif +#ifdef HAVE_SYS_PARAM_H +#include +#endif +#ifdef HAVE_NETDB_H +#include +#endif +#include +#ifdef HAVE_SYS_STAT_H +#include +#endif /* HAVE_SYS_STAT_H */ +#include +#ifdef HAVE_SYS_SELECT_H +#include +#endif +#ifdef HAVE_DIRENT_H +#include +#endif +#include + +#include "src/hwloc/hwloc-internal.h" +#include "src/class/pmix_pointer_array.h" +#include "src/util/pmix_environ.h" +#include "src/util/show_help.h" +#include "src/util/fd.h" +#include "src/util/error.h" + +#include "src/include/pmix_globals.h" +#include "src/util/name_fns.h" +#include "src/threads/threads.h" + +#include "src/mca/pfexec/base/base.h" +#include "src/mca/pfexec/linux/pfexec_linux.h" + +/* + * Module functions (function pointers used in a struct) + */ +static pmix_status_t spawn_proc(const pmix_info_t job_info[], size_t ninfo, + const pmix_app_t apps[], size_t napps); +static pmix_status_t kill_proc(pmix_rank_t rank); +static pmix_status_t signal_proc(pmix_rank_t rank, int32_t signal); + +/* + * Explicitly declared functions so that we can get the noreturn + * attribute registered with the compiler. + */ +static void send_error_show_help(int fd, int exit_status, + const char *file, const char *topic, ...) + __pmix_attribute_noreturn__; + +static void do_child(pmix_app_t *cd, char **env, pmix_pfexec_child_t *child, int write_fd) + __pmix_attribute_noreturn__; + + +/* + * Module + */ +pmix_pfexec_base_module_t pmix_pfexec_linux_module = { + .spawn_proc = spawn_proc, + .kill_proc = kill_proc, + .signal_proc = signal_proc, +}; + + +/* deliver a signal to a specified pid. */ +static pmix_status_t sigproc(pid_t pd, int signum) +{ + pid_t pgrp; + pid_t pid; + + pid = pd; + +#if HAVE_SETPGID + pgrp = getpgid(pd); + if (-1 != pgrp) { + /* target the lead process of the process + * group so we ensure that the signal is + * seen by all members of that group. This + * ensures that the signal is seen by any + * child processes our child may have + * started + */ + pid = -pgrp; + } +#endif + + if (0 != kill(pid, signum)) { + if (ESRCH != errno) { + PMIX_OUTPUT_VERBOSE((2, pmix_pfexec_base_framework.framework_output, + "%s pfexec:linux:SENT SIGNAL %d TO PID %d GOT ERRNO %d", + PMIX_NAME_PRINT(&pmix_globals.myid), signum, (int)pid, errno)); + return errno; + } + } + PMIX_OUTPUT_VERBOSE((2, pmix_pfexec_base_framework.framework_output, + "%s pfexec:linux:SENT SIGNAL %d TO PID %d SUCCESS", + PMIX_NAME_PRINT(&pmix_globals.myid), signum, (int)pid)); + return 0; +} + +static pmix_status_t kill_proc(pmix_rank_t rank) +{ + pmix_status_t rc; + pmix_lock_t mylock; + pmix_pfexec_signal_caddy_t *kcd; + + PMIX_CONSTRUCT_LOCK(&mylock); + PMIX_PFEXEC_KILL(kcd, rank, sigproc, &mylock); + PMIX_WAIT_THREAD(&mylock); + rc = mylock.status; + PMIX_DESTRUCT_LOCK(&mylock); + PMIX_RELEASE(kcd); + + return rc; +} + +static pmix_status_t signal_proc(pmix_rank_t rank, int32_t signal) +{ + pmix_status_t rc; + pmix_lock_t mylock; + pmix_pfexec_signal_caddy_t *scd; + + PMIX_CONSTRUCT_LOCK(&mylock); + PMIX_PFEXEC_SIGNAL(scd, rank, signal, sigproc, &mylock); + PMIX_WAIT_THREAD(&mylock); + rc = mylock.status; + PMIX_DESTRUCT_LOCK(&mylock); + PMIX_RELEASE(scd); + + return rc; +} + +static void set_handler_linux(int sig) +{ + struct sigaction act; + + act.sa_handler = SIG_DFL; + act.sa_flags = 0; + sigemptyset(&act.sa_mask); + + sigaction(sig, &act, (struct sigaction *)0); +} + +/* + * Internal function to write a rendered show_help message back up the + * pipe to the waiting parent. + */ +static int write_help_msg(int fd, pmix_pfexec_pipe_err_msg_t *msg, const char *file, + const char *topic, va_list ap) +{ + int ret; + char *str; + + if (NULL == file || NULL == topic) { + return PMIX_ERR_BAD_PARAM; + } + + str = pmix_show_help_vstring(file, topic, true, ap); + + msg->file_str_len = (int) strlen(file); + if (msg->file_str_len > PMIX_PFEXEC_MAX_FILE_LEN) { + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + return PMIX_ERR_BAD_PARAM; + } + msg->topic_str_len = (int) strlen(topic); + if (msg->topic_str_len > PMIX_PFEXEC_MAX_TOPIC_LEN) { + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + return PMIX_ERR_BAD_PARAM; + } + msg->msg_str_len = (int) strlen(str); + + /* Only keep writing if each write() succeeds */ + if (PMIX_SUCCESS != (ret = pmix_fd_write(fd, sizeof(*msg), msg))) { + goto out; + } + if (msg->file_str_len > 0 && + PMIX_SUCCESS != (ret = pmix_fd_write(fd, msg->file_str_len, file))) { + goto out; + } + if (msg->topic_str_len > 0 && + PMIX_SUCCESS != (ret = pmix_fd_write(fd, msg->topic_str_len, topic))) { + goto out; + } + if (msg->msg_str_len > 0 && + PMIX_SUCCESS != (ret = pmix_fd_write(fd, msg->msg_str_len, str))) { + goto out; + } + + out: + free(str); + return ret; +} + + +/* Called from the child to send an error message up the pipe to the + waiting parent. */ +static void send_error_show_help(int fd, int exit_status, + const char *file, const char *topic, ...) +{ + va_list ap; + pmix_pfexec_pipe_err_msg_t msg; + + msg.fatal = true; + msg.exit_status = exit_status; + + /* Send it */ + va_start(ap, topic); + write_help_msg(fd, &msg, file, topic, ap); + va_end(ap); + + exit(exit_status); +} + +/* close all open file descriptors w/ exception of stdin/stdout/stderr + and the pipe up to the parent. */ +static int close_open_file_descriptors(int write_fd) { + DIR *dir = opendir("/proc/self/fd"); + if (NULL == dir) { + return PMIX_ERR_FILE_OPEN_FAILURE; + } + struct dirent *files; + + /* grab the fd of the opendir above so we don't close in the + * middle of the scan. */ + int dir_scan_fd = dirfd(dir); + if(dir_scan_fd < 0 ) { + return PMIX_ERR_FILE_OPEN_FAILURE; + } + + + while (NULL != (files = readdir(dir))) { + if (!isdigit(files->d_name[0])) { + continue; + } + int fd = strtol(files->d_name, NULL, 10); + if (errno == EINVAL || errno == ERANGE) { + closedir(dir); + return PMIX_ERR_TYPE_MISMATCH; + } + if (fd >=3 && + fd != write_fd && + fd != dir_scan_fd) { + close(fd); + } + } + closedir(dir); + return PMIX_SUCCESS; +} + +static void do_child(pmix_app_t *app, char **env, + pmix_pfexec_child_t *child, int write_fd) +{ + int i, errval; + sigset_t sigs; + long fd, fdmax = sysconf(_SC_OPEN_MAX); + char dir[MAXPATHLEN]; + +#if HAVE_SETPGID + /* Set a new process group for this child, so that any + * signals we send to it will reach any children it spawns */ + setpgid(0, 0); +#endif + + /* Setup the pipe to be close-on-exec */ + pmix_fd_set_cloexec(write_fd); + + /* setup stdout/stderr so that any error messages that we + may print out will get displayed back at pmixrun. + + NOTE: Definitely do this AFTER we check contexts so + that any error message from those two functions doesn't + come out to the user. IF we didn't do it in this order, + THEN a user who gives us a bad executable name or + working directory would get N error messages, where + N=num_procs. This would be very annoying for large + jobs, so instead we set things up so that pmixrun + always outputs a nice, single message indicating what + happened + */ + if (PMIX_SUCCESS != (i = pmix_pfexec_base_setup_child(child))) { + PMIX_ERROR_LOG(i); + send_error_show_help(write_fd, 1, + "help-pfexec-linux.txt", + "iof setup failed", + pmix_globals.hostname, app->cmd); + /* Does not return */ + } + + /* close all open file descriptors w/ exception of stdin/stdout/stderr, + the pipe used for the IOF INTERNAL messages, and the pipe up to + the parent. */ + if (PMIX_SUCCESS != close_open_file_descriptors(write_fd)) { + // close *all* file descriptors -- slow + for(fd=3; fdcwd) { + if (0 != chdir(app->cwd)) { + send_error_show_help(write_fd, 1, + "help-pfexec-linux.txt", + "wdir-not-found", + "pmixd", + app->cwd, + pmix_globals.hostname); + /* Does not return */ + } + } + + /* Exec the new executable */ + execve(app->cmd, app->argv, env); + errval = errno; + getcwd(dir, sizeof(dir)); + send_error_show_help(write_fd, 1, + "help-pfexec-linux.txt", "execve error", + pmix_globals.hostname, dir, app->cmd, strerror(errval)); + /* Does not return */ +} + + +static pmix_status_t do_parent(pmix_app_t *app, pmix_pfexec_child_t *child, int read_fd) +{ + pmix_status_t rc; + pmix_pfexec_pipe_err_msg_t msg; + char file[PMIX_PFEXEC_MAX_FILE_LEN + 1], topic[PMIX_PFEXEC_MAX_TOPIC_LEN + 1], *str = NULL; + + if (child->opts.connect_stdin) { + close(child->opts.p_stdin[0]); + } + close(child->opts.p_stdout[1]); + close(child->opts.p_stderr[1]); + + /* Block reading a message from the pipe */ + while (1) { + rc = pmix_fd_read(read_fd, sizeof(msg), &msg); + + /* If the pipe closed, then the child successfully launched */ + if (PMIX_ERR_TIMEOUT == rc) { + break; + } + + /* If Something Bad happened in the read, error out */ + if (PMIX_SUCCESS != rc) { + PMIX_ERROR_LOG(rc); + close(read_fd); + return rc; + } + + /* Read in the strings; ensure to terminate them with \0 */ + if (msg.file_str_len > 0) { + rc = pmix_fd_read(read_fd, msg.file_str_len, file); + if (PMIX_SUCCESS != rc) { + pmix_show_help("help-pfexec-linux.txt", "syscall fail", + true, + pmix_globals.hostname, app->cmd, + "pmix_fd_read", __FILE__, __LINE__); + return rc; + } + file[msg.file_str_len] = '\0'; + } + if (msg.topic_str_len > 0) { + rc = pmix_fd_read(read_fd, msg.topic_str_len, topic); + if (PMIX_SUCCESS != rc) { + pmix_show_help("help-pfexec-linux.txt", "syscall fail", + true, + pmix_globals.hostname, app->cmd, + "pmix_fd_read", __FILE__, __LINE__); + return rc; + } + topic[msg.topic_str_len] = '\0'; + } + if (msg.msg_str_len > 0) { + str = calloc(1, msg.msg_str_len + 1); + if (NULL == str) { + pmix_show_help("help-pfexec-linux.txt", "syscall fail", + true, + pmix_globals.hostname, app->cmd, + "calloc", __FILE__, __LINE__); + return PMIX_ERR_NOMEM; + } + rc = pmix_fd_read(read_fd, msg.msg_str_len, str); + if (PMIX_SUCCESS != rc) { + pmix_show_help("help-pfexec-linux.txt", "syscall fail", + true, + pmix_globals.hostname, app->cmd, + "pmix_fd_read", __FILE__, __LINE__); + free(str); + return rc; + } + } + + /* Print out what we got. We already have a rendered string, + so use pmix_show_help_norender(). */ + if (msg.msg_str_len > 0) { + fprintf(stderr, "%s\n", str); + free(str); + str = NULL; + } + + /* If msg.fatal is true, then the child exited with an error. + Otherwise, whatever we just printed was a warning, so loop + around and see what else is on the pipe (or if the pipe + closed, indicating that the child launched + successfully). */ + if (msg.fatal) { + close(read_fd); + if (NULL != str) { + free(str); + } + return PMIX_ERR_SYS_OTHER; + } + if (NULL != str) { + free(str); + str = NULL; + } + } + + /* If we got here, it means that the pipe closed without + indication of a fatal error, meaning that the child process + launched successfully. */ + close(read_fd); + return PMIX_SUCCESS; +} + + +/** + * Fork/exec the specified processes + */ +static int fork_proc(pmix_app_t *app, pmix_pfexec_child_t *child, char **env) +{ + int p[2]; + + /* A pipe is used to communicate between the parent and child to + indicate whether the exec ultimately succeeded or failed. The + child sets the pipe to be close-on-exec; the child only ever + writes anything to the pipe if there is an error (e.g., + executable not found, exec() fails, etc.). The parent does a + blocking read on the pipe; if the pipe closed with no data, + then the exec() succeeded. If the parent reads something from + the pipe, then the child was letting us know why it failed. */ + if (pipe(p) < 0) { + PMIX_ERROR_LOG(PMIX_ERR_SYS_OTHER); + return PMIX_ERR_SYS_OTHER; + } + + /* Fork off the child */ + child->pid = fork(); + + if (child->pid < 0) { + PMIX_ERROR_LOG(PMIX_ERR_SYS_OTHER); + return PMIX_ERR_SYS_OTHER; + } + + if (child->pid == 0) { + close(p[0]); + do_child(app, env, child, p[1]); + /* Does not return */ + } + + close(p[1]); + return do_parent(app, child, p[0]); +} + + +/** + * Launch all processes allocated to the current node. + */ + +static pmix_status_t spawn_proc(const pmix_info_t job_info[], size_t ninfo, + const pmix_app_t apps[], size_t napps) +{ + pmix_status_t rc; + pmix_lock_t mylock; + pmix_pfexec_fork_caddy_t *scd; + + pmix_output_verbose(5, pmix_pfexec_base_framework.framework_output, + "%s pfexec:linux spawning child job", + PMIX_NAME_PRINT(&pmix_globals.myid)); + + PMIX_CONSTRUCT_LOCK(&mylock); + PMIX_PFEXEC_SPAWN(scd, job_info, ninfo, apps, napps, fork_proc, (void*)&mylock); + PMIX_WAIT_THREAD(&mylock); + if (PMIX_SUCCESS == mylock.status) { + mylock.status = PMIX_OPERATION_SUCCEEDED; + } + rc = mylock.status; + PMIX_DESTRUCT_LOCK(&mylock); + PMIX_RELEASE(scd); + + return rc; +} diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/linux/pfexec_linux.h b/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/linux/pfexec_linux.h new file mode 100644 index 0000000000..0578f3a704 --- /dev/null +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/linux/pfexec_linux.h @@ -0,0 +1,42 @@ +/* + * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2019 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +/** + * @file: + */ + +#ifndef PMIX_PFEXEC_LINUX_H +#define PMIX_PFEXEC_LINUX_H + +#include "pmix_config.h" + +#include "src/mca/mca.h" + +#include "src/mca/pfexec/pfexec.h" + +BEGIN_C_DECLS + +/* + * PFEXEC Linux module + */ +PMIX_EXPORT extern pmix_pfexec_base_module_t pmix_pfexec_linux_module; +PMIX_EXPORT extern pmix_pfexec_base_component_t mca_pfexec_linux_component; + +END_C_DECLS + +#endif /* PMIX_PFEXEC_H */ diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/linux/pfexec_linux_component.c b/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/linux/pfexec_linux_component.c new file mode 100644 index 0000000000..cf1a485f7c --- /dev/null +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/linux/pfexec_linux_component.c @@ -0,0 +1,98 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2015 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2017-2019 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + * These symbols are in a file by themselves to provide nice linker + * semantics. Since linkers generally pull in symbols by object + * files, keeping these symbols as the only symbols in this file + * prevents utility programs such as "ompi_info" from having to import + * entire components just to query their version and parameters. + */ + +#include "pmix_config.h" +#include "pmix_common.h" + +#include +#ifdef HAVE_UNISTD_H +#include +#endif +#include + +#include "src/mca/mca.h" +#include "src/mca/base/base.h" + +#include "src/mca/pfexec/pfexec.h" +#include "src/mca/pfexec/linux/pfexec_linux.h" + +static pmix_status_t component_open(void); +static pmix_status_t component_close(void); +static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority); + +/* + * Instantiate the public struct with all of our public information + * and pointers to our public functions in it + */ + +pmix_pfexec_base_component_t mca_pfexec_linux_component = { + /* First, the mca_component_t struct containing meta information + about the component itself */ + .version = { + PMIX_PFEXEC_BASE_VERSION_1_0_0, + /* Component name and version */ + .pmix_mca_component_name = "linux", + PMIX_MCA_BASE_MAKE_VERSION(component, + PMIX_MAJOR_VERSION, + PMIX_MINOR_VERSION, + PMIX_RELEASE_VERSION), + + /* Component open and close functions */ + .pmix_mca_open_component = component_open, + .pmix_mca_close_component = component_close, + .pmix_mca_query_component = component_query, + }, +}; + + + +static pmix_status_t component_open(void) +{ + return PMIX_SUCCESS; +} + +static pmix_status_t component_close(void) +{ + return PMIX_SUCCESS; +} + +static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority) +{ + /* the base open/select logic protects us against operation when + * we are NOT in a daemon, so we don't have to check that here + */ + + /* we have built some logic into the configure.m4 file that checks + * to see if we have "fork" support and only builds this component + * if we do. Hence, we only get here if we CAN build - in which + * case, we definitely should be considered for selection + */ + *priority = 10; /* let others override us - we are the linux */ + *module = (pmix_mca_base_module_t *) &pmix_pfexec_linux_module; + return PMIX_SUCCESS; +} diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/pfexec.h b/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/pfexec.h new file mode 100644 index 0000000000..6f1b5c6dd0 --- /dev/null +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/pfexec/pfexec.h @@ -0,0 +1,92 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2011-2015 Los Alamos National Security, LLC. All rights + * reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +/** + * @file + * + * The PMIx Fork/Exec Subsystem + * + */ + +#ifndef PMIX_MCA_PFEXEC_H +#define PMIX_MCA_PFEXEC_H + +#include "pmix_config.h" +#include "pmix_common.h" +#include "src/include/types.h" + +#include "src/mca/mca.h" + +BEGIN_C_DECLS + +/* + * pfexec module functions + */ + +/** + * Locally fork/exec the provided process + */ +typedef pmix_status_t (*pmix_pfexec_base_module_spawn_process_fn_t)(const pmix_info_t job_info[], size_t ninfo, + const pmix_app_t apps[], size_t napps); + +/** + * Kill the local process we started + */ +typedef pmix_status_t (*pmix_pfexec_base_module_kill_process_fn_t)(pmix_rank_t rank); + +/** + * Signal local process we started + */ +typedef pmix_status_t (*pmix_pfexec_base_module_signal_process_fn_t)(pmix_rank_t rank, int signum); + +/** + * pfexec module version + */ +typedef struct { + pmix_pfexec_base_module_spawn_process_fn_t spawn_proc; + pmix_pfexec_base_module_kill_process_fn_t kill_proc; + pmix_pfexec_base_module_signal_process_fn_t signal_proc; +} pmix_pfexec_base_module_t; + +/** + * pfexec component + */ +typedef struct { + /** component version */ + pmix_mca_base_component_t version; + /** component data */ + pmix_mca_base_component_data_t base_data; +} pmix_pfexec_base_component_t; + + +/** + * Macro for use in modules that are of type pfexec + */ +#define PMIX_PFEXEC_BASE_VERSION_1_0_0 \ + PMIX_MCA_BASE_VERSION_1_0_0("pfexec", 1, 0, 0) + +/* Global structure for accessing PFEXEC functions +*/ +PMIX_EXPORT extern pmix_pfexec_base_module_t pmix_pfexec; /* holds selected module's function pointers */ + +END_C_DECLS + +#endif /* MCA_PFEXEC_H */ diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/pif/base/pif_base_components.c b/opal/mca/pmix/pmix4x/pmix/src/mca/pif/base/pif_base_components.c index 803e45c2a0..15ddc7d66c 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/pif/base/pif_base_components.c +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/pif/base/pif_base_components.c @@ -40,6 +40,7 @@ PMIX_MCA_BASE_FRAMEWORK_DECLARE(pmix, pif, NULL, pmix_pif_base_register, pmix_pi static int pmix_pif_base_register (pmix_mca_base_register_flag_t flags) { + (void)flags; pmix_if_do_not_resolve = false; (void) pmix_mca_base_framework_var_register (&pmix_pif_base_framework, "do_not_resolve", "If nonzero, do not attempt to resolve interfaces", diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/pif/linux_ipv6/pif_linux_ipv6.c b/opal/mca/pmix/pmix4x/pmix/src/mca/pif/linux_ipv6/pif_linux_ipv6.c index f0bb2db9f5..daa96aa48c 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/pif/linux_ipv6/pif_linux_ipv6.c +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/pif/linux_ipv6/pif_linux_ipv6.c @@ -80,7 +80,9 @@ static int if_linux_ipv6_open(void) { FILE *f; if ((f = fopen("/proc/net/if_inet6", "r"))) { - char ifname[IF_NAMESIZE]; + /* IF_NAMESIZE is normally 16 on Linux, + but the next scanf allows up to 21 bytes */ + char ifname[21]; unsigned int idx, pfxlen, scope, dadstat; struct in6_addr a6; int iter; diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/plog/base/plog_base_frame.c b/opal/mca/pmix/pmix4x/pmix/src/mca/plog/base/plog_base_frame.c index 906b8ccaa7..656b5b8322 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/plog/base/plog_base_frame.c +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/plog/base/plog_base_frame.c @@ -38,6 +38,7 @@ pmix_plog_API_module_t pmix_plog = { static char *order = NULL; static int pmix_plog_register(pmix_mca_base_register_flag_t flags) { + (void)flags; pmix_mca_base_var_register("pmix", "plog", "base", "order", "Comma-delimited, prioritized list of logging channels", PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/base/base.h b/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/base/base.h index d832bf5478..73698b606e 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/base/base.h +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/base/base.h @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ @@ -99,6 +99,18 @@ typedef struct { } pmix_pnet_job_t; PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_pnet_job_t); +typedef struct { + pmix_object_t super; + /* provide access to the component + * APIs that are managing this + * fabric plane */ + pmix_pnet_module_t *module; + /* allow the component to add + * whatever structures it needs */ + void *payload; +} pmix_pnet_fabric_t; +PMIX_EXPORT PMIX_CLASS_DECLARATION(pmix_pnet_fabric_t); + /* framework globals */ struct pmix_pnet_globals_t { pmix_lock_t lock; diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/base/pnet_base_fns.c b/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/base/pnet_base_fns.c index e4a78b0b91..f0a9b91cf7 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/base/pnet_base_fns.c +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/base/pnet_base_fns.c @@ -56,7 +56,7 @@ pmix_status_t pmix_pnet_base_allocate(char *nspace, if (NULL == nspace || NULL == ilist) { return PMIX_ERR_BAD_PARAM; } - if (PMIX_PROC_IS_GATEWAY(pmix_globals.mypeer)) { + if (PMIX_PROC_IS_SCHEDULER(pmix_globals.mypeer)) { nptr = NULL; /* find this nspace - note that it may not have * been registered yet */ @@ -207,7 +207,8 @@ pmix_status_t pmix_pnet_base_setup_fork(const pmix_proc_t *proc, char ***env) PMIX_LIST_FOREACH(active, &pmix_pnet_globals.actives, pmix_pnet_base_active_module_t) { if (NULL != active->module->setup_fork) { - if (PMIX_SUCCESS != (rc = active->module->setup_fork(nptr, proc, env))) { + rc = active->module->setup_fork(nptr, proc, env); + if (PMIX_SUCCESS != rc && PMIX_ERR_NOT_AVAILABLE != rc) { return rc; } } diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/base/pnet_base_frame.c b/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/base/pnet_base_frame.c index 0c8295fae7..76f60b126f 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/base/pnet_base_frame.c +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/base/pnet_base_frame.c @@ -11,7 +11,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015-2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ @@ -181,3 +181,12 @@ static void rdes(pmix_pnet_resource_t *p) PMIX_CLASS_INSTANCE(pmix_pnet_resource_t, pmix_list_item_t, rcon, rdes); + +static void ftcon(pmix_pnet_fabric_t *p) +{ + p->module = NULL; + p->payload = NULL; +} +PMIX_CLASS_INSTANCE(pmix_pnet_fabric_t, + pmix_object_t, + ftcon, NULL); diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/opa/configure.m4 b/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/opa/configure.m4 deleted file mode 100644 index d822ffaf74..0000000000 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/opa/configure.m4 +++ /dev/null @@ -1,110 +0,0 @@ -# -*- shell-script -*- -# -# Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana -# University Research and Technology -# Corporation. All rights reserved. -# Copyright (c) 2004-2005 The University of Tennessee and The University -# of Tennessee Research Foundation. All rights -# reserved. -# Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, -# University of Stuttgart. All rights reserved. -# Copyright (c) 2004-2005 The Regents of the University of California. -# All rights reserved. -# Copyright (c) 2010 Cisco Systems, Inc. All rights reserved. -# Copyright (c) 2013 Sandia National Laboratories. All rights reserved. -# Copyright (c) 2014-2018 Intel, Inc. All rights reserved. -# $COPYRIGHT$ -# -# Additional copyrights may follow -# -# $HEADER$ -# - -# MCA_pnet_opa_CONFIG([action-if-can-compile], -# [action-if-cant-compile]) -# ------------------------------------------------ -AC_DEFUN([MCA_pmix_pnet_opa_CONFIG],[ - AC_CONFIG_FILES([src/mca/pnet/opa/Makefile]) - - PMIX_CHECK_PSM2([pnet_opa], - [pnet_opa_happy="yes"], - [pnet_opa_happy="no"]) - - AC_ARG_WITH([opamgt], - [AC_HELP_STRING([--with-opamgt(=DIR)], - [Build OmniPath Fabric Management support (optionally adding DIR/include, DIR/include/opamgt, DIR/lib, and DIR/lib64 to the search path for headers and libraries])], [], [with_opamgt=no]) - - AC_ARG_WITH([opamgt-libdir], - [AC_HELP_STRING([--with-opamgt-libdir=DIR], - [Search for OmniPath Fabric Management libraries in DIR])]) - - pmix_check_opamgt_save_CPPFLAGS="$CPPFLAGS" - pmix_check_opamgt_save_LDFLAGS="$LDFLAGS" - pmix_check_opamgt_save_LIBS="$LIBS" - - pmix_check_opamgt_libdir= - pmix_check_opamgt_dir= - - AC_MSG_CHECKING([if opamgt requested]) - AS_IF([test "$with_opamgt" == "no"], - [AC_MSG_RESULT([no]) - pmix_check_opamgt_happy=no], - [AC_MSG_RESULT([yes]) - PMIX_CHECK_WITHDIR([opamgt-libdir], [$with_opamgt_libdir], [libopamgt.*]) - AS_IF([test ! -z "$with_opamgt" && test "$with_opamgt" != "yes"], - [pmix_check_opamgt_dir="$with_opamgt" - AS_IF([test ! -d "$pmix_check_opamgt_dir" || test ! -f "$pmix_check_opamgt_dir/opamgt.h"], - [$pmix_check_opamgt_dir=$pmix_check_opamgt_dir/include - AS_IF([test ! -d "$pmix_check_opamgt_dir" || test ! -f "$pmix_check_opamgt_dir/opamgt.h"], - [$pmix_check_opamgt_dir=$pmix_check_opamgt_dir/opamgt - AS_IF([test ! -d "$pmix_check_opamgt_dir" || test ! -f "$pmix_check_opamgt_dir/opamgt.h"], - [AC_MSG_WARN([OmniPath Fabric Management support requested, but]) - AC_MSG_WARN([required header file opamgt.h not found. Locations tested:]) - AC_MSG_WARN([ $with_opamgt]) - AC_MSG_WARN([ $with_opamgt/include]) - AC_MSG_WARN([ $with_opamgt/include/opamgt]) - AC_MSG_ERROR([Cannot continue])])])])], - [pmix_check_opamgt_dir="/usr/include/opamgt"]) - - AS_IF([test ! -z "$with_opamgt_libdir" && test "$with_opamgt_libdir" != "yes"], - [pmix_check_opamgt_libdir="$with_opamgt_libdir"]) - - # no easy way to check this, so let's ensure that the - # full opamgt install was done, including the iba support - AS_IF([test ! -d "$pmix_check_opamgt_dir/iba" || test ! -f "$pmix_check_opamgt_dir/iba/vpi.h"], - [pmix_check_opamgt_happy="no"], - [PMIX_CHECK_PACKAGE([pnet_opamgt], - [opamgt.h], - [opamgt], - [omgt_query_sa], - [], - [$pmix_check_opamgt_dir], - [$pmix_check_opamgt_libdir], - [pmix_check_opamgt_happy="yes" - pnet_opa_CFLAGS="$pnet_opa_CFLAGS $pnet_opamgt_CFLAGS" - pnet_opa_CPPFLAGS="$pnet_opa_CPPFLAGS $pnet_opamgt_CPPFLAGS" - pnet_opa_LDFLAGS="$pnet_opa_LDFLAGS $pnet_opamgt_LDFLAGS" - pnet_opa_LIBS="$pnet_opa_LIBS $pnet_opamgt_LIBS"], - [pmix_check_opamgt_happy="no"])]) - ]) - - AS_IF([test "$pmix_check_opamgt_happy" = "yes"], - [pmix_want_opamgt=1], - [pmix_want_opamgt=0]) - AC_DEFINE_UNQUOTED([PMIX_WANT_OPAMGT], [$pmix_want_opamgt], - [Whether or not to include OmniPath Fabric Manager support]) - - CPPFLAGS="$pmix_check_opamgt_save_CPPFLAGS" - LDFLAGS="$pmix_check_opamgt_save_LDFLAGS" - LIBS="$pmix_check_opamgt_save_LIBS" - - AS_IF([test "$pnet_opa_happy" = "yes"], - [$1], - [$2]) - - # substitute in the things needed to build psm2 - AC_SUBST([pnet_opa_CFLAGS]) - AC_SUBST([pnet_opa_CPPFLAGS]) - AC_SUBST([pnet_opa_LDFLAGS]) - AC_SUBST([pnet_opa_LIBS]) -])dnl diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/opa/pnet_opa.c b/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/opa/pnet_opa.c deleted file mode 100644 index 6deb3487c5..0000000000 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/opa/pnet_opa.c +++ /dev/null @@ -1,855 +0,0 @@ -/* - * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. - * Copyright (c) 2016 IBM Corporation. All rights reserved. - * - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#include - -#include -#ifdef HAVE_UNISTD_H -#include -#endif -#ifdef HAVE_SYS_TYPES_H -#include -#endif -#ifdef HAVE_SYS_STAT_H -#include -#endif -#ifdef HAVE_FCNTL_H -#include -#endif -#include - -#if PMIX_WANT_OPAMGT -#include -#include -#endif - -#include - -#include "src/mca/base/pmix_mca_base_var.h" -#include "src/class/pmix_list.h" -#include "src/include/pmix_socket_errno.h" -#include "src/include/pmix_globals.h" -#include "src/class/pmix_list.h" -#include "src/util/alfg.h" -#include "src/util/argv.h" -#include "src/util/error.h" -#include "src/util/output.h" -#include "src/util/pmix_environ.h" -#include "src/mca/preg/preg.h" -#include "src/hwloc/hwloc-internal.h" - -#include "src/mca/pnet/pnet.h" -#include "src/mca/pnet/base/base.h" -#include "pnet_opa.h" - -static pmix_status_t opa_init(void); -static void opa_finalize(void); -static pmix_status_t allocate(pmix_namespace_t *nptr, - pmix_info_t info[], size_t ninfo, - pmix_list_t *ilist); -static pmix_status_t setup_local_network(pmix_namespace_t *nptr, - pmix_info_t info[], - size_t ninfo); -static pmix_status_t setup_fork(pmix_namespace_t *nptr, - const pmix_proc_t *proc, - char ***env); -static void child_finalized(pmix_proc_t *peer); -static void local_app_finalized(pmix_namespace_t *nptr); -static void deregister_nspace(pmix_namespace_t *nptr); -static pmix_status_t collect_inventory(pmix_info_t directives[], size_t ndirs, - pmix_inventory_cbfunc_t cbfunc, void *cbdata); -static pmix_status_t deliver_inventory(pmix_info_t info[], size_t ninfo, - pmix_info_t directives[], size_t ndirs, - pmix_op_cbfunc_t cbfunc, void *cbdata); - -pmix_pnet_module_t pmix_opa_module = { - .name = "opa", - .init = opa_init, - .finalize = opa_finalize, - .allocate = allocate, - .setup_local_network = setup_local_network, - .setup_fork = setup_fork, - .child_finalized = child_finalized, - .local_app_finalized = local_app_finalized, - .deregister_nspace = deregister_nspace, - .collect_inventory = collect_inventory, - .deliver_inventory = deliver_inventory -}; - -/* local object definitions */ -typedef struct { - pmix_list_item_t super; - char *name; - char *value; -} opa_attr_t; -static void atcon(opa_attr_t *p) -{ - p->name = NULL; - p->value = NULL; -} -static void atdes(opa_attr_t *p) -{ - if (NULL != p->name) { - free(p->name); - } - if (NULL != p->value) { - free(p->value); - } -} -static PMIX_CLASS_INSTANCE(opa_attr_t, - pmix_list_item_t, - atcon, atdes); - -typedef struct { - pmix_list_item_t super; - char *device; - pmix_list_t attributes; -} opa_resource_t; -static void rcon(opa_resource_t *p) -{ - p->device = NULL; - PMIX_CONSTRUCT(&p->attributes, pmix_list_t); -} -static void rdes(opa_resource_t *p) -{ - if (NULL != p->device) { - free(p->device); - } - PMIX_LIST_DESTRUCT(&p->attributes); -} -static PMIX_CLASS_INSTANCE(opa_resource_t, - pmix_list_item_t, - rcon, rdes); - - -static pmix_status_t opa_init(void) -{ - pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, - "pnet: opa init"); - return PMIX_SUCCESS; -} - -static void opa_finalize(void) -{ - pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, - "pnet: opa finalize"); -} - -/* some network transports require a little bit of information to - * "pre-condition" them - i.e., to setup their individual transport - * connections so they can generate their endpoint addresses. This - * function provides a means for doing so. The resulting info is placed - * into the app_context's env array so it will automatically be pushed - * into the environment of every MPI process when launched. - */ - -static inline void transports_use_rand(uint64_t* unique_key) { - pmix_rng_buff_t rng; - pmix_srand(&rng,(unsigned int)time(NULL)); - unique_key[0] = pmix_rand(&rng); - unique_key[1] = pmix_rand(&rng); -} - -static char* transports_print(uint64_t *unique_key) -{ - unsigned int *int_ptr; - size_t i, j, string_key_len, written_len; - char *string_key = NULL, *format = NULL; - - /* string is two 64 bit numbers printed in hex with a dash between - * and zero padding. - */ - string_key_len = (sizeof(uint64_t) * 2) * 2 + strlen("-") + 1; - string_key = (char*) malloc(string_key_len); - if (NULL == string_key) { - return NULL; - } - - string_key[0] = '\0'; - written_len = 0; - - /* get a format string based on the length of an unsigned int. We - * want to have zero padding for sizeof(unsigned int) * 2 - * characters -- when printing as a hex number, each byte is - * represented by 2 hex characters. Format will contain something - * that looks like %08lx, where the number 8 might be a different - * number if the system has a different sized long (8 would be for - * sizeof(int) == 4)). - */ - if (0 > asprintf(&format, "%%0%dx", (int)(sizeof(unsigned int)) * 2)) { - return NULL; - } - - /* print the first number */ - int_ptr = (unsigned int*) &unique_key[0]; - for (i = 0 ; i < sizeof(uint64_t) / sizeof(unsigned int) ; ++i) { - if (0 == int_ptr[i]) { - /* inject some energy */ - for (j=0; j < sizeof(unsigned int); j++) { - int_ptr[i] |= j << j; - } - } - snprintf(string_key + written_len, - string_key_len - written_len, - format, int_ptr[i]); - written_len = strlen(string_key); - } - - /* print the middle dash */ - snprintf(string_key + written_len, string_key_len - written_len, "-"); - written_len = strlen(string_key); - - /* print the second number */ - int_ptr = (unsigned int*) &unique_key[1]; - for (i = 0 ; i < sizeof(uint64_t) / sizeof(unsigned int) ; ++i) { - if (0 == int_ptr[i]) { - /* inject some energy */ - for (j=0; j < sizeof(unsigned int); j++) { - int_ptr[i] |= j << j; - } - } - snprintf(string_key + written_len, - string_key_len - written_len, - format, int_ptr[i]); - written_len = strlen(string_key); - } - free(format); - - return string_key; -} - -/* NOTE: if there is any binary data to be transferred, then - * this function MUST pack it for transport as the host will - * not know how to do so */ -static pmix_status_t allocate(pmix_namespace_t *nptr, - pmix_info_t info[], size_t ninfo, - pmix_list_t *ilist) -{ - uint64_t unique_key[2]; - char *string_key, *cs_env; - int fd_rand; - size_t bytes_read, n, m, p; - pmix_kval_t *kv; - bool envars = false, seckeys = false, netalloc = false; - pmix_status_t rc; - pmix_proc_t pname; - pmix_coord_t coord; - pmix_buffer_t bucket; - pmix_info_t *iptr; - pmix_pnet_node_t *nd; - pmix_pnet_local_procs_t *lp; - - pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, - "pnet:opa:allocate for nspace %s", nptr->nspace); - - if (NULL == info) { - return PMIX_ERR_TAKE_NEXT_OPTION; - } - - for (n=0; n < ninfo; n++) { - if (PMIX_CHECK_KEY(&info[n], PMIX_SETUP_APP_ENVARS)) { - envars = PMIX_INFO_TRUE(&info[n]); - } else if (PMIX_CHECK_KEY(&info[n], PMIX_SETUP_APP_ALL)) { - envars = PMIX_INFO_TRUE(&info[n]); - seckeys = PMIX_INFO_TRUE(&info[n]); - } else if (PMIX_CHECK_KEY(&info[n], PMIX_SETUP_APP_NONENVARS)) { - seckeys = PMIX_INFO_TRUE(&info[n]); - } else if (PMIX_CHECK_KEY(&info[n], PMIX_ALLOC_NETWORK)) { - iptr = (pmix_info_t*)info[n].value.data.darray->array; - m = info[n].value.data.darray->size; - for (p=0; p < m; p++) { - if (PMIX_CHECK_KEY(&iptr[p], PMIX_ALLOC_NETWORK_SEC_KEY)) { - seckeys = PMIX_INFO_TRUE(&iptr[p]); - } else if (PMIX_CHECK_KEY(&iptr[p], PMIX_ALLOC_NETWORK_ID)) { - /* need to track the request by this ID */ - } else if (PMIX_CHECK_KEY(&iptr[p], PMIX_SETUP_APP_ENVARS)) { - envars = PMIX_INFO_TRUE(&iptr[p]); - } else if (PMIX_CHECK_KEY(&iptr[p], PMIX_SETUP_APP_ALL)) { - envars = PMIX_INFO_TRUE(&iptr[p]); - seckeys = PMIX_INFO_TRUE(&iptr[p]); - } else if (PMIX_CHECK_KEY(&iptr[p], PMIX_SETUP_APP_NONENVARS)) { - seckeys = PMIX_INFO_TRUE(&iptr[p]); - } - } - netalloc = true; - } - } - - if (seckeys) { - pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, - "pnet: opa providing seckeys"); - /* put the number here - or else create an appropriate string. this just needs to - * eventually be a string variable - */ - if(-1 == (fd_rand = open("/dev/urandom", O_RDONLY))) { - transports_use_rand(unique_key); - } else { - bytes_read = read(fd_rand, (char *) unique_key, 16); - if(bytes_read != 16) { - transports_use_rand(unique_key); - } - close(fd_rand); - } - - if (NULL == (string_key = transports_print(unique_key))) { - PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE); - return PMIX_ERR_OUT_OF_RESOURCE; - } - - if (PMIX_SUCCESS != pmix_mca_base_var_env_name("opa_precondition_transports", &cs_env)) { - PMIX_ERROR_LOG(PMIX_ERR_OUT_OF_RESOURCE); - free(string_key); - return PMIX_ERR_OUT_OF_RESOURCE; - } - - kv = PMIX_NEW(pmix_kval_t); - if (NULL == kv) { - free(string_key); - free(cs_env); - return PMIX_ERR_OUT_OF_RESOURCE; - } - kv->key = strdup(PMIX_SET_ENVAR); - kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); - if (NULL == kv->value) { - free(string_key); - free(cs_env); - PMIX_RELEASE(kv); - return PMIX_ERR_OUT_OF_RESOURCE; - } - kv->value->type = PMIX_ENVAR; - PMIX_ENVAR_LOAD(&kv->value->data.envar, cs_env, string_key, ':'); - pmix_list_append(ilist, &kv->super); - free(cs_env); - free(string_key); - } - - if (envars) { - pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, - "pnet: opa harvesting envars %s excluding %s", - (NULL == mca_pnet_opa_component.incparms) ? "NONE" : mca_pnet_opa_component.incparms, - (NULL == mca_pnet_opa_component.excparms) ? "NONE" : mca_pnet_opa_component.excparms); - /* harvest envars to pass along */ - if (NULL != mca_pnet_opa_component.include) { - rc = pmix_pnet_base_harvest_envars(mca_pnet_opa_component.include, - mca_pnet_opa_component.exclude, - ilist); - if (PMIX_SUCCESS != rc) { - return rc; - } - } - } - - if (netalloc) { - /* assign a simulated coordinate to each process. For now, we - * assume there is one device per node. Thus, the coordinate of - * all procs on a node will be the network coord of the device - * on that node. We'll assign device coordinates with a simple - * round-robin algo */ - coord.y = 0; - coord.z = 0; - PMIX_LOAD_NSPACE(pname.nspace, nptr->nspace); - PMIX_CONSTRUCT(&bucket, pmix_buffer_t); - PMIX_LIST_FOREACH(nd, &pmix_pnet_globals.nodes, pmix_pnet_node_t) { - /* find the job on this node */ - PMIX_LIST_FOREACH(lp, &nd->local_jobs, pmix_pnet_local_procs_t) { - if (0 == strcmp(nptr->nspace, lp->nspace)) { - /* assign the coord for each proc - in our case, - * we shall assign an x-coord based on local rank - * and the y-coord will represent the node */ - for (n=0; n < lp->np; n++) { - coord.x = n; - pname.rank = lp->ranks[n]; - /* pack this value */ - PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &bucket, &pname, 1, PMIX_PROC); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_DESTRUCT(&bucket); - return rc; - } - PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &bucket, &coord, 1, PMIX_COORD); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_DESTRUCT(&bucket); - return rc; - } - } - break; - } - } - coord.y++; - } - /* pass that up */ - kv = PMIX_NEW(pmix_kval_t); - if (NULL == kv) { - PMIX_DESTRUCT(&bucket); - return PMIX_ERR_OUT_OF_RESOURCE; - } - kv->key = strdup(PMIX_PNET_OPA_BLOB); - kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); - if (NULL == kv->value) { - PMIX_DESTRUCT(&bucket); - PMIX_RELEASE(kv); - return PMIX_ERR_OUT_OF_RESOURCE; - } - kv->value->type = PMIX_BYTE_OBJECT; - /* unload the buffer into a byte object */ - PMIX_UNLOAD_BUFFER(&bucket, kv->value->data.bo.bytes, kv->value->data.bo.size); - pmix_list_append(ilist, &kv->super); - } - - /* we don't currently manage OPA resources */ - return PMIX_ERR_TAKE_NEXT_OPTION; -} - -static pmix_status_t setup_local_network(pmix_namespace_t *nptr, - pmix_info_t info[], - size_t ninfo) -{ - size_t n; - pmix_kval_t *kv; - - - pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, - "pnet: opa setup_local_network for nspace %s", nptr->nspace); - - if (NULL != info) { - for (n=0; n < ninfo; n++) { - if (0 == strncmp(info[n].key, PMIX_PNET_OPA_BLOB, PMIX_MAX_KEYLEN)) { - /* the byte object contains a packed blob that needs to be - * cached until we determine we have local procs for this - * nspace */ - kv = PMIX_NEW(pmix_kval_t); - if (NULL == kv) { - return PMIX_ERR_NOMEM; - } - kv->key = strdup(info[n].key); - kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); - if (NULL == kv->value) { - PMIX_RELEASE(kv); - return PMIX_ERR_NOMEM; - } - pmix_value_xfer(kv->value, &info[n].value); - if (PMIX_ENVAR == kv->value->type) { - pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, - "pnet:opa:setup_local_network adding %s=%s to environment", - kv->value->data.envar.envar, kv->value->data.envar.value); - } else { - pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, - "pnet:opa:setup_local_network loading blob"); - } - pmix_list_append(&nptr->setup_data, &kv->super); - } - } - } - - return PMIX_SUCCESS; -} - -static pmix_status_t setup_fork(pmix_namespace_t *nptr, - const pmix_proc_t *proc, - char ***env) -{ - pmix_kval_t *kv, *next; - pmix_data_array_t dinfo; - pmix_info_t info[2], stinfo; - int cnt; - pmix_status_t rc; - pmix_buffer_t bkt; - pmix_proc_t pname; - pmix_coord_t coord; - - pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, - "pnet: opa setup fork for nspace: %s", nptr->nspace); - - /* if there are any cached nspace prep blobs, execute them, - * ensuring that we only do so once per nspace - note that - * we don't expect to find any envars here, though we could - * have included some if we needed to set them per-client */ - PMIX_LIST_FOREACH_SAFE(kv, next, &nptr->setup_data, pmix_kval_t) { - if (0 == strcmp(kv->key, PMIX_PNET_OPA_BLOB)) { - pmix_list_remove_item(&nptr->setup_data, &kv->super); - /* setup to unpack the blob */ - PMIX_CONSTRUCT(&bkt,pmix_buffer_t); - PMIX_LOAD_BUFFER(pmix_globals.mypeer, &bkt, - kv->value->data.bo.bytes, - kv->value->data.bo.size); - /* there will be an entry for each proc in the nspace */ - PMIX_INFO_CONSTRUCT(&stinfo); - PMIX_LOAD_KEY(stinfo.key, PMIX_PROC_DATA); - stinfo.value.type = PMIX_DATA_ARRAY; - stinfo.value.data.darray = &dinfo; - dinfo.type = PMIX_INFO; - dinfo.size = 2; - dinfo.array = info; - /* unpack all the entries */ - cnt = 1; - PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, - &bkt, &pname, &cnt, PMIX_PROC); - while (PMIX_SUCCESS == rc) { - /* unpack the coord of this proc */ - cnt = 1; - PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, - &bkt, &coord, &cnt, PMIX_COORD); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - break; - } - /* cache the info on the job */ - PMIX_INFO_LOAD(&info[0], PMIX_RANK, &pname.rank, PMIX_PROC_RANK); - PMIX_INFO_LOAD(&info[1], PMIX_NETWORK_COORDINATE, &coord, PMIX_COORD); - PMIX_GDS_CACHE_JOB_INFO(rc, pmix_globals.mypeer, nptr, - &stinfo, 1); - /* unpack next entry */ - cnt = 1; - PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, - &bkt, &pname, &cnt, PMIX_PROC); - } - PMIX_RELEASE(kv); - break; - } - } - return PMIX_SUCCESS; -} - -static void child_finalized(pmix_proc_t *peer) -{ - pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, - "pnet:opa child finalized"); -} - -static void local_app_finalized(pmix_namespace_t *nptr) -{ - pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, - "pnet:opa app finalized"); - -} - -static void deregister_nspace(pmix_namespace_t *nptr) -{ - pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, - "pnet:opa deregister nspace"); - -} - -static pmix_status_t collect_inventory(pmix_info_t directives[], size_t ndirs, - pmix_inventory_cbfunc_t cbfunc, void *cbdata) -{ - pmix_inventory_rollup_t *cd = (pmix_inventory_rollup_t*)cbdata; -#if PMIX_HAVE_HWLOC - hwloc_obj_t obj; -#endif - unsigned n; - pmix_status_t rc; - pmix_kval_t *kv; - pmix_buffer_t bucket, pbkt; - bool found = false; - pmix_byte_object_t pbo; - char nodename[PMIX_MAXHOSTNAMELEN], *foo; - - pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, - "pnet:opa collect inventory"); - - /* setup the bucket - we will pass the results as a blob */ - PMIX_CONSTRUCT(&bucket, pmix_buffer_t); - /* pack our node name */ - gethostname(nodename, sizeof(nodename)); - foo = &nodename[0]; - PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &bucket, &foo, 1, PMIX_STRING); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_DESTRUCT(&bucket); - return rc; - } - -#if PMIX_HAVE_HWLOC - if (NULL == pmix_hwloc_topology) { - goto query; - } - - /* search the topology for OPA devices */ - obj = hwloc_get_next_osdev(pmix_hwloc_topology, NULL); - while (NULL != obj) { - if (obj->attr->osdev.type != HWLOC_OBJ_OSDEV_OPENFABRICS || - 0 != strncmp(obj->name, "hfi", 3)) { - obj = hwloc_get_next_osdev(pmix_hwloc_topology, obj); - continue; - } - found = true; - if (9 < pmix_output_get_verbosity(pmix_pnet_base_framework.framework_output)) { - /* dump the discovered node resources */ - pmix_output(0, "OPA resource discovered on node: %s", nodename); - pmix_output(0, "\tDevice name: %s", obj->name); - for (n=0; n < obj->infos_count; n++) { - pmix_output(0, "\t\t%s: %s", obj->infos[n].name, obj->infos[n].value); - } - } - /* pack the name of the device */ - PMIX_CONSTRUCT(&pbkt, pmix_buffer_t); - PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &pbkt, &obj->name, 1, PMIX_STRING); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_DESTRUCT(&pbkt); - PMIX_DESTRUCT(&bucket); - return rc; - } - /* pack the number of attributes */ - PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &pbkt, &obj->infos_count, 1, PMIX_UINT); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_DESTRUCT(&pbkt); - PMIX_DESTRUCT(&bucket); - return rc; - } - /* pack each descriptive object */ - for (n=0; n < obj->infos_count; n++) { - PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &pbkt, &obj->infos[n].name, 1, PMIX_STRING); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_DESTRUCT(&pbkt); - PMIX_DESTRUCT(&bucket); - return rc; - } - PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &pbkt, &obj->infos[n].value, 1, PMIX_STRING); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_DESTRUCT(&pbkt); - PMIX_DESTRUCT(&bucket); - return rc; - } - } - /* extract the resulting blob - this is a device unit */ - PMIX_UNLOAD_BUFFER(&pbkt, pbo.bytes, pbo.size); - /* now load that into the blob */ - PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &bucket, &pbo, 1, PMIX_BYTE_OBJECT); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_BYTE_OBJECT_DESTRUCT(&pbo); - PMIX_DESTRUCT(&bucket); - return rc; - } - obj = hwloc_get_next_osdev(pmix_hwloc_topology, obj); - } - - query: -#if 0 -#if PMIX_WANT_OPAMGT - if (PMIX_PROC_IS_GATEWAY(pmix_globals.mypeer)) { - /* collect the switch information from the FM */ - OMGT_STATUS_T status = OMGT_STATUS_SUCCESS; - struct omgt_port * port = NULL; - omgt_sa_selector_t selector; - - /* create a session */ - status = omgt_open_port_by_num(&port, 1 /* hfi */, 1 /* port */, NULL); - if (OMGT_STATUS_SUCCESS != status) { - pmix_output_verbose(1, pmix_pnet_base_framework.framework_output, - "Unable to open port to FM"); - goto complete; - } - /* specify how and what we want to query by */ - selector.InputType = InputTypeLid; - selector.InputValue.PortInfoRecord.Lid = 1; - - } -#endif -#endif - /* if we found any devices, then return the blob */ - if (!found) { - PMIX_DESTRUCT(&bucket); - return PMIX_ERR_TAKE_NEXT_OPTION; - } - - /* extract the resulting blob */ - PMIX_UNLOAD_BUFFER(&bucket, pbo.bytes, pbo.size); - kv = PMIX_NEW(pmix_kval_t); - kv->key = strdup(PMIX_OPA_INVENTORY_KEY); - PMIX_VALUE_CREATE(kv->value, 1); - pmix_value_load(kv->value, &pbo, PMIX_BYTE_OBJECT); - PMIX_BYTE_OBJECT_DESTRUCT(&pbo); - pmix_list_append(&cd->payload, &kv->super); - -#else // have_hwloc -#if 0 -#if PMIX_WANT_OPAMGT - if (PMIX_PROC_IS_GATEWAY(pmix_globals.mypeer)) { - /* query the FM for the inventory */ - } - - complete: - /* if we found any devices, then return the blob */ - if (!found) { - PMIX_DESTRUCT(&bucket); - return PMIX_ERR_TAKE_NEXT_OPTION; - } - - /* extract the resulting blob */ - PMIX_UNLOAD_BUFFER(&bucket, pbo.bytes, pbo.size); - kv = PMIX_NEW(pmix_kval_t); - kv->key = strdup(PMIX_OPA_INVENTORY_KEY); - PMIX_VALUE_CREATE(kv->value, 1); - pmix_value_load(kv->value, &pbo, PMIX_BYTE_OBJECT); - PMIX_BYTE_OBJECT_DESTRUCT(&pbo); - pmix_list_append(&cd->payload, &kv->super); - -#endif -#endif - return PMIX_ERR_TAKE_NEXT_OPTION; -#endif // have_hwloc - - return PMIX_SUCCESS; -} - -static pmix_status_t deliver_inventory(pmix_info_t info[], size_t ninfo, - pmix_info_t directives[], size_t ndirs, - pmix_op_cbfunc_t cbfunc, void *cbdata) -{ - pmix_buffer_t bkt, pbkt; - size_t n; - int32_t cnt; - unsigned m, nattrs; - char *hostname; - pmix_byte_object_t pbo; - pmix_pnet_node_t *nd, *ndptr; - pmix_pnet_resource_t *lt, *lst; - opa_attr_t *attr; - opa_resource_t *res; - pmix_status_t rc; - - pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, - "pnet:opa deliver inventory"); - - for (n=0; n < ninfo; n++) { - if (0 == strncmp(info[n].key, PMIX_OPA_INVENTORY_KEY, PMIX_MAX_KEYLEN)) { - /* this is our inventory in the form of a blob */ - PMIX_CONSTRUCT(&bkt,pmix_buffer_t); - PMIX_LOAD_BUFFER(pmix_globals.mypeer, &bkt, - info[n].value.data.bo.bytes, - info[n].value.data.bo.size); - /* first is the host this came from */ - cnt = 1; - PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, - &bkt, &hostname, &cnt, PMIX_STRING); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - /* must _not_ destruct bkt as we don't - * own the bytes! */ - return rc; - } - /* do we already have this node? */ - nd = NULL; - PMIX_LIST_FOREACH(ndptr, &pmix_pnet_globals.nodes, pmix_pnet_node_t) { - if (0 == strcmp(hostname, ndptr->name)) { - nd = ndptr; - break; - } - } - if (NULL == nd) { - nd = PMIX_NEW(pmix_pnet_node_t); - nd->name = strdup(hostname); - pmix_list_append(&pmix_pnet_globals.nodes, &nd->super); - } - /* does this node already have an OPA entry? */ - lst = NULL; - PMIX_LIST_FOREACH(lt, &nd->resources, pmix_pnet_resource_t) { - if (0 == strcmp(lt->name, "opa")) { - lst = lt; - break; - } - } - if (NULL == lst) { - lst = PMIX_NEW(pmix_pnet_resource_t); - lst->name = strdup("opa"); - pmix_list_append(&nd->resources, &lst->super); - } - /* each device was packed as a "blob" */ - cnt = 1; - PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, - &bkt, &pbo, &cnt, PMIX_BYTE_OBJECT); - while (PMIX_SUCCESS == rc) { - /* load the blob for unpacking */ - PMIX_CONSTRUCT(&pbkt, pmix_buffer_t); - PMIX_LOAD_BUFFER(pmix_globals.mypeer, &pbkt, - pbo.bytes, pbo.size); - - res = PMIX_NEW(opa_resource_t); - /* starts with the name of the device */ - cnt = 1; - PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, - &pbkt, &res->device, &cnt, PMIX_STRING); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_DESTRUCT(&pbkt); - PMIX_RELEASE(res); - return rc; - } - /* next comes the numbers of attributes for that device */ - cnt = 1; - PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, - &pbkt, &nattrs, &cnt, PMIX_UINT); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_DESTRUCT(&pbkt); - PMIX_RELEASE(res); - return rc; - } - for (m=0; m < nattrs; m++) { - attr = PMIX_NEW(opa_attr_t); - /* unpack the name of the attribute */ - cnt = 1; - PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, - &pbkt, &attr->name, &cnt, PMIX_STRING); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_DESTRUCT(&pbkt); - PMIX_RELEASE(attr); - PMIX_RELEASE(res); - return rc; - } - /* unpack the attribute value */ - cnt = 1; - PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, - &pbkt, &attr->value, &cnt, PMIX_STRING); - if (PMIX_SUCCESS != rc) { - PMIX_ERROR_LOG(rc); - PMIX_DESTRUCT(&pbkt); - PMIX_RELEASE(attr); - PMIX_RELEASE(res); - return rc; - } - pmix_list_append(&res->attributes, &attr->super); - } - pmix_list_append(&lst->resources, &res->super); - PMIX_DESTRUCT(&pbkt); - - /* get the next device unit */ - cnt = 1; - PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, - &bkt, &pbo, &cnt, PMIX_BYTE_OBJECT); - } - if (5 < pmix_output_get_verbosity(pmix_pnet_base_framework.framework_output)) { - /* dump the resulting node resources */ - pmix_output(0, "OPA resources for node: %s", nd->name); - PMIX_LIST_FOREACH(lt, &nd->resources, pmix_pnet_resource_t) { - if (0 == strcmp(lt->name, "opa")) { - PMIX_LIST_FOREACH(res, <->resources, opa_resource_t) { - pmix_output(0, "\tDevice: %s", res->device); - PMIX_LIST_FOREACH(attr, &res->attributes, opa_attr_t) { - pmix_output(0, "\t\t%s: %s", attr->name, attr->value); - } - } - } - } - } - } - } - - return PMIX_SUCCESS; -} diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/opa/pnet_opa.h b/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/opa/pnet_opa.h deleted file mode 100644 index 5fe18a4de8..0000000000 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/opa/pnet_opa.h +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. - * - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - */ - -#ifndef PMIX_PNET_OPA_H -#define PMIX_PNET_OPA_H - -#include - - -#include "src/mca/pnet/pnet.h" - -BEGIN_C_DECLS - -typedef struct { - pmix_pnet_base_component_t super; - char *incparms; - char *excparms; - char **include; - char **exclude; - int radix; -} pmix_pnet_opa_component_t; - -/* the component must be visible data for the linker to find it */ -PMIX_EXPORT extern pmix_pnet_opa_component_t mca_pnet_opa_component; -extern pmix_pnet_module_t pmix_opa_module; - -/* define a key for any blob we need to send in a launch msg */ -#define PMIX_PNET_OPA_BLOB "pmix.pnet.opa.blob" - -/* define an inventory key */ -#define PMIX_OPA_INVENTORY_KEY "pmix.opa.inventory" - -END_C_DECLS - -#endif diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/opa/pnet_opa_component.c b/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/opa/pnet_opa_component.c deleted file mode 100644 index 4fb38a50e7..0000000000 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/opa/pnet_opa_component.c +++ /dev/null @@ -1,126 +0,0 @@ -/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ -/* - * Copyright (c) 2004-2008 The Trustees of Indiana University and Indiana - * University Research and Technology - * Corporation. All rights reserved. - * Copyright (c) 2004-2005 The University of Tennessee and The University - * of Tennessee Research Foundation. All rights - * reserved. - * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, - * University of Stuttgart. All rights reserved. - * Copyright (c) 2004-2005 The Regents of the University of California. - * All rights reserved. - * Copyright (c) 2015 Los Alamos National Security, LLC. All rights - * reserved. - * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. - * $COPYRIGHT$ - * - * Additional copyrights may follow - * - * $HEADER$ - * - * These symbols are in a file by themselves to provide nice linker - * semantics. Since linkers generally pull in symbols by object - * files, keeping these symbols as the only symbols in this file - * prevents utility programs such as "ompi_info" from having to import - * entire components just to query their version and parameters. - */ - -#include -#include "pmix_common.h" - -#include "src/util/argv.h" -#include "src/mca/pnet/pnet.h" -#include "pnet_opa.h" - -static pmix_status_t component_open(void); -static pmix_status_t component_close(void); -static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority); -static pmix_status_t component_register(void); - -/* - * Instantiate the public struct with all of our public information - * and pointers to our public functions in it - */ -pmix_pnet_opa_component_t mca_pnet_opa_component = { - .super = { - .base = { - PMIX_PNET_BASE_VERSION_1_0_0, - - /* Component name and version */ - .pmix_mca_component_name = "opa", - PMIX_MCA_BASE_MAKE_VERSION(component, - PMIX_MAJOR_VERSION, - PMIX_MINOR_VERSION, - PMIX_RELEASE_VERSION), - - /* Component open and close functions */ - .pmix_mca_open_component = component_open, - .pmix_mca_close_component = component_close, - .pmix_mca_register_component_params = component_register, - .pmix_mca_query_component = component_query, - }, - .data = { - /* The component is checkpoint ready */ - PMIX_MCA_BASE_METADATA_PARAM_CHECKPOINT - } - }, - .include = NULL, - .exclude = NULL, - .radix = 64 -}; - -static pmix_status_t component_register(void) -{ - pmix_mca_base_component_t *component = &mca_pnet_opa_component.super.base; - - mca_pnet_opa_component.incparms = "HFI_*,PSM2_*"; - (void)pmix_mca_base_component_var_register(component, "include_envars", - "Comma-delimited list of envars to harvest (\'*\' and \'?\' supported)", - PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, - PMIX_INFO_LVL_2, - PMIX_MCA_BASE_VAR_SCOPE_LOCAL, - &mca_pnet_opa_component.incparms); - if (NULL != mca_pnet_opa_component.incparms) { - mca_pnet_opa_component.include = pmix_argv_split(mca_pnet_opa_component.incparms, ','); - } - - mca_pnet_opa_component.excparms = NULL; - (void)pmix_mca_base_component_var_register(component, "exclude_envars", - "Comma-delimited list of envars to exclude (\'*\' and \'?\' supported)", - PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, - PMIX_INFO_LVL_2, - PMIX_MCA_BASE_VAR_SCOPE_LOCAL, - &mca_pnet_opa_component.excparms); - if (NULL != mca_pnet_opa_component.excparms) { - mca_pnet_opa_component.exclude = pmix_argv_split(mca_pnet_opa_component.excparms, ','); - } - - (void)pmix_mca_base_component_var_register(component, "radix", - "Radix for simulating the network coordinates", - PMIX_MCA_BASE_VAR_TYPE_INT, NULL, 0, 0, - PMIX_INFO_LVL_2, - PMIX_MCA_BASE_VAR_SCOPE_LOCAL, - &mca_pnet_opa_component.radix); - - return PMIX_SUCCESS; -} - -static pmix_status_t component_open(void) -{ - return PMIX_SUCCESS; -} - - -static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority) -{ - *priority = 10; - *module = (pmix_mca_base_module_t *)&pmix_opa_module; - return PMIX_SUCCESS; -} - - -static pmix_status_t component_close(void) -{ - return PMIX_SUCCESS; -} diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/pnet.h b/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/pnet.h index 92f102e81e..eaf79ed9c7 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/pnet.h +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/pnet.h @@ -27,6 +27,7 @@ #define PMIX_PNET_H #include +#include #include "src/class/pmix_list.h" #include "src/mca/mca.h" @@ -146,11 +147,54 @@ typedef pmix_status_t (*pmix_pnet_base_module_deliver_inventory_fn_t)(pmix_info_ pmix_op_cbfunc_t cbfunc, void *cbdata); +/* Register to provide cost information + * + * Scan the provided directives to identify if the module + * should service this request - directives could include + * ordered specification of fabric type or a direct request + * for a specific component + */ +typedef pmix_status_t (*pmix_pnet_base_module_register_fabric_fn_t)(pmix_fabric_t *fabric, + const pmix_info_t directives[], + size_t ndirs); + + +/* Deregister the fabric, giving the associated module a chance to cleanup */ +typedef pmix_status_t (*pmix_pnet_base_deregister_fabric_fn_t)(pmix_fabric_t *fabric); + +/* Get the number of vertices in the fabric */ +typedef pmix_status_t (*pmix_pnet_base_module_get_num_verts_fn_t)(pmix_fabric_t *fabric, + uint32_t *nverts); + +/* Get the cost of communicating from the src to the dest + * index - i.e., return the [src,dest] location in the + * communication cost array */ +typedef pmix_status_t (*pmix_pnet_base_module_get_cost_fn_t)(pmix_fabric_t *fabric, + uint32_t src, uint32_t dest, + uint16_t *cost); + +/* Get the identifier and nodename corresponding to the provided + * index of the communication cost array - caller must provide + * the address of an allocated pmix_value_t structure */ +typedef pmix_status_t (*pmix_pnet_base_module_get_vertex_fn_t)(pmix_fabric_t *fabric, + uint32_t i, + pmix_value_t *identifier, + char **nodename); + +/* Get the index in the communication cost array corresponding + * to the provided identifier and the node upon which it resides */ +typedef pmix_status_t (*pmix_pnet_base_module_get_index_fn_t)(pmix_fabric_t *fabric, + pmix_value_t *identifier, + uint32_t *i, + char **nodename); /** - * Base structure for a PNET module + * Base structure for a PNET module. Each component should malloc a + * copy of the module structure for each fabric plane they support. */ typedef struct { char *name; + /* provide a pointer to plane-specific metadata */ + void *plane; /* init/finalize */ pmix_pnet_base_module_init_fn_t init; pmix_pnet_base_module_fini_fn_t finalize; @@ -162,6 +206,12 @@ typedef struct { pmix_pnet_base_module_dregister_nspace_fn_t deregister_nspace; pmix_pnet_base_module_collect_inventory_fn_t collect_inventory; pmix_pnet_base_module_deliver_inventory_fn_t deliver_inventory; + pmix_pnet_base_module_register_fabric_fn_t register_fabric; + pmix_pnet_base_deregister_fabric_fn_t deregister_fabric; + pmix_pnet_base_module_get_num_verts_fn_t get_num_vertices; + pmix_pnet_base_module_get_cost_fn_t get_cost; + pmix_pnet_base_module_get_vertex_fn_t get_vertex; + pmix_pnet_base_module_get_index_fn_t get_index; } pmix_pnet_module_t; diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/tcp/pnet_tcp.h b/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/tcp/pnet_tcp.h index 81271b01f0..efc34429f9 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/tcp/pnet_tcp.h +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/tcp/pnet_tcp.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2018 Intel, Inc. All rights reserved. + * Copyright (c) 2018-2019 Intel, Inc. All rights reserved. * * $COPYRIGHT$ * @@ -26,6 +26,8 @@ typedef struct { char *excparms; char **include; char **exclude; + int nverts; + uint16_t **costmatrix; } pmix_pnet_tcp_component_t; /* the component must be visible data for the linker to find it */ diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/test/pnet_test.c b/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/test/pnet_test.c index a00cff1c35..1d4f1bac38 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/test/pnet_test.c +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/test/pnet_test.c @@ -32,6 +32,7 @@ #include "src/include/pmix_socket_errno.h" #include "src/include/pmix_globals.h" #include "src/class/pmix_list.h" +#include "src/class/pmix_pointer_array.h" #include "src/util/alfg.h" #include "src/util/argv.h" #include "src/util/error.h" @@ -63,7 +64,22 @@ static pmix_status_t collect_inventory(pmix_info_t directives[], size_t ndirs, static pmix_status_t deliver_inventory(pmix_info_t info[], size_t ninfo, pmix_info_t directives[], size_t ndirs, pmix_op_cbfunc_t cbfunc, void *cbdata); - +static pmix_status_t register_fabric(pmix_fabric_t *fabric, + const pmix_info_t directives[], + size_t ndirs); +static pmix_status_t deregister_fabric(pmix_fabric_t *fabric); +static pmix_status_t get_num_verts(pmix_fabric_t *fabric, uint32_t *nverts); +static pmix_status_t get_cost(pmix_fabric_t *fabric, + uint32_t src, uint32_t dest, + uint16_t *cost); +static pmix_status_t get_vertex(pmix_fabric_t *fabric, + uint32_t i, + pmix_value_t *identifier, + char **nodename); +static pmix_status_t get_index(pmix_fabric_t *fabric, + pmix_value_t *identifier, + uint32_t *i, + char **nodename); pmix_pnet_module_t pmix_test_module = { .name = "test", .init = test_init, @@ -75,20 +91,414 @@ pmix_pnet_module_t pmix_test_module = { .local_app_finalized = local_app_finalized, .deregister_nspace = deregister_nspace, .collect_inventory = collect_inventory, - .deliver_inventory = deliver_inventory + .deliver_inventory = deliver_inventory, + .register_fabric = register_fabric, + .deregister_fabric = deregister_fabric, + .get_num_vertices = get_num_verts, + .get_cost = get_cost, + .get_vertex = get_vertex, + .get_index = get_index }; +/* internal tracking structures */ +typedef struct { + pmix_list_item_t super; + char *name; + int index; + void *node; // pointer to node hosting this nic + void *s; // pointer to switch hosting this port, or + // pointer to switch this nic is attached to + void *plane; // pointer to plane this NIC is attached to + void *link; // nic this nic is connected to +} pnet_nic_t; +static void ncon(pnet_nic_t *p) +{ + p->name = NULL; + p->index = -1; + p->node = NULL; + p->s = NULL; + p->plane = NULL; + p->link = NULL; +} +static void ndes(pnet_nic_t *p) +{ + if (NULL != p->name) { + free(p->name); + } +} +static PMIX_CLASS_INSTANCE(pnet_nic_t, + pmix_list_item_t, + ncon, ndes); + +typedef struct { + pmix_list_item_t super; + char *name; + int index; + void *left; // switch to the left of this one in the ring + pnet_nic_t leftport; + void *right; // switch to the right of this one in the ring + pnet_nic_t rightport; + pmix_list_t ports; // NICs included in the switch +} pnet_switch_t; +static void scon(pnet_switch_t *p) +{ + p->name = NULL; + p->index = -1; + p->left = NULL; + p->right = NULL; + PMIX_CONSTRUCT(&p->leftport, pnet_nic_t); + PMIX_CONSTRUCT(&p->rightport, pnet_nic_t); + PMIX_CONSTRUCT(&p->ports, pmix_list_t); +} +static void sdes(pnet_switch_t *p) +{ + if (NULL != p->name) { + free(p->name); + } + PMIX_DESTRUCT(&p->leftport); + PMIX_DESTRUCT(&p->rightport); + PMIX_LIST_DESTRUCT(&p->ports); +} +static PMIX_CLASS_INSTANCE(pnet_switch_t, + pmix_list_item_t, + scon, sdes); + +typedef struct { + pmix_list_item_t super; + /* use an atomic lock for this object */ + pmix_atomic_lock_t atomlock; + char *name; + int index; + bool dense; + int nswitches; + uint64_t nverts; + uint16_t **costmatrix; + pmix_list_t switches; + uint64_t revision; +} pnet_plane_t; +static void pcon(pnet_plane_t *p) +{ + pmix_atomic_lock_init(&p->atomlock, 0); + p->name = NULL; + p->index = -1; + p->dense = false; + p->nswitches = 0; + p->nverts = 0; + p->costmatrix = NULL; + PMIX_CONSTRUCT(&p->switches, pmix_list_t); + p->revision = 0; +} +static void pdes(pnet_plane_t *p) +{ + uint64_t n; + + if (NULL != p->name) { + free(p->name); + } + if (NULL != p->costmatrix) { + for (n=0; n < p->nverts; n++) { + free(p->costmatrix[n]); + } + free(p->costmatrix); + } + PMIX_LIST_DESTRUCT(&p->switches); +} +static PMIX_CLASS_INSTANCE(pnet_plane_t, + pmix_list_item_t, + pcon, pdes); + +typedef struct { + pmix_list_item_t super; + char *name; + pmix_list_t nics; +} pnet_node_t; +static void ndcon(pnet_node_t *p) +{ + p->name = NULL; + PMIX_CONSTRUCT(&p->nics, pmix_list_t); +} +static void nddes(pnet_node_t *p) +{ + if (NULL != p->name) { + free(p->name); + } + PMIX_LIST_DESTRUCT(&p->nics); +} +static PMIX_CLASS_INSTANCE(pnet_node_t, + pmix_list_item_t, + ndcon, nddes); + +/* internal variables */ +static pmix_list_t myplanes; +static pmix_list_t mynodes; +static pmix_pointer_array_t myfabrics; +static pmix_pointer_array_t mynics; +static char **myenvlist = NULL; +static char **myvalues = NULL; + static pmix_status_t test_init(void) { + int n, m, r, ns, nplane, nnodes, nports; + uint64_t n64, m64; + char **system, **ptr; + pnet_plane_t *p; + pnet_switch_t *s, *s2; + pnet_nic_t *nic, *nic2; + pnet_node_t *node; + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, "pnet: test init"); + + PMIX_CONSTRUCT(&myplanes, pmix_list_t); + PMIX_CONSTRUCT(&mynodes, pmix_list_t); + PMIX_CONSTRUCT(&myfabrics, pmix_pointer_array_t); + pmix_pointer_array_init(&myfabrics, 1, INT_MAX, 1); + PMIX_CONSTRUCT(&mynics, pmix_pointer_array_t); + pmix_pointer_array_init(&mynics, 8, INT_MAX, 8); + + /* if we have a config file, read it now */ + if (NULL != mca_pnet_test_component.cfg_file) { + + } else if (NULL != mca_pnet_test_component.nverts) { + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet: test creating system configuration"); + /* the system description is configured as nodes and fabric planes + * delineated by semi-colons */ + system = pmix_argv_split(mca_pnet_test_component.nverts, ';'); + /* there can be multiple planes defined, but only one set of + * nodes. The nodes description contains the #nodes - we assume + * that each node has a single NIC attached to each fabric plane. + * Thus, the nodes entry has a single field associated with it that + * contains the number of nodes in the system. + * + * Similarly, we assume that each switch in the plane contains + * a port to connect to each node in the system. For simplicity, + * we assume a ring connection topology between the switches and + * reserve one port on each switch to connect to its "left" peer + * and another to connect to its "right" peer. + * + * Thus, the #NICS in a node equals the number of planes in the + * overall system. The #ports in a switch equals the #nodes in + * the system plus two for cross-switch communications. + */ + for (r=0; NULL != system[r]; r++) { + if (0 == strncasecmp(system[r], "nodes", 5)) { + /* the number of nodes must follow the colon after "nodes" */ + nnodes = strtoul(&system[r][6], NULL, 10); + for (n=0; n < nnodes; n++) { + node = PMIX_NEW(pnet_node_t); + if (0 > asprintf(&node->name, "test%03d", n)) { + return PMIX_ERR_NOMEM; + } + pmix_list_append(&mynodes, &node->super); + } + } else if (0 == strncasecmp(system[r], "plane", 5)) { + /* create a plane object */ + p = PMIX_NEW(pnet_plane_t); + /* the plane contains a flag indicating how the nodes + * are to be distributed across the plane plus the + * number of switches in the plane */ + ptr = pmix_argv_split(&system[r][6], ':'); + if (1 == pmix_argv_count(ptr)) { + /* default to dense */ + p->dense = true; + p->nswitches = strtoul(ptr[0], NULL, 10); + } else { + if ('d' == ptr[0][0] || 'D' == ptr[0][0]) { + p->dense = true; + } + p->nswitches = strtoul(ptr[1], NULL, 10); + } + pmix_argv_free(ptr); + pmix_list_append(&myplanes, &p->super); + } else { + PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); + pmix_argv_free(system); + return PMIX_ERR_BAD_PARAM; + } + } + /* setup the ports in each switch for each plane */ + nplane = 0; + PMIX_LIST_FOREACH(p, &myplanes, pnet_plane_t) { + /* assign a name to the plane */ + if (0 > asprintf(&p->name, "plane%03d", nplane)) { + return PMIX_ERR_NOMEM; + } + /* setup the ports on the switches */ + nports = nnodes / p->nswitches; + /* if it didn't divide evenly, then we have to add + * one to each switch to ensure we have enough ports */ + if (0 != nnodes % p->nswitches) { + ++nports; + } + for (n=0; n < p->nswitches; n++) { + s = PMIX_NEW(pnet_switch_t); + if (0 > asprintf(&s->name, "%s:switch%03d", p->name, n)) { + return PMIX_ERR_NOMEM; + } + s->index = n; + pmix_list_append(&p->switches, &s->super); + if (0 > asprintf(&s->leftport.name, "%s:port000", s->name)) { + return PMIX_ERR_NOMEM; + } + if (0 > asprintf(&s->rightport.name, "%s:port%03d", s->name, nports+1)) { + return PMIX_ERR_NOMEM; + } + for (m=0; m < nports; m++) { + nic = PMIX_NEW(pnet_nic_t); + if (0 > asprintf(&nic->name, "%s:port%03d", s->name, m+1)) { + return PMIX_ERR_NOMEM; + } + nic->s = s; + nic->plane = p; + pmix_list_append(&s->ports, &nic->super); + } + } + + /* link the switch ring - first nic on each switch connects + * to the left, last nic on each switch connects to + * the right */ + s = (pnet_switch_t*)pmix_list_get_first(&p->switches); + s->left = pmix_list_get_last(&p->switches); + s->right = pmix_list_get_next(&s->super); + /* setup his NICs to point to the right place */ + s2 = (pnet_switch_t*)s->left; + s->leftport.link = &s2->rightport; + s2->rightport.link = &s->leftport; + + s2 = (pnet_switch_t*)s->right; + s->rightport.link = &s2->leftport; + s2->leftport.link = &s->rightport; + + /* progress the search */ + s = (pnet_switch_t*)pmix_list_get_next(&s->super); + while (s != (pnet_switch_t*)pmix_list_get_last(&p->switches)) { + s->left = pmix_list_get_prev(&s->super); + s->right = pmix_list_get_next(&s->super); + /* setup his NICs to point to the right place */ + s2 = (pnet_switch_t*)s->left; + s->leftport.link = &s2->rightport; + s2->rightport.link = &s->leftport; + + s2 = (pnet_switch_t*)s->right; + s->rightport.link = &s2->leftport; + s2->leftport.link = &s->rightport; + s2->left = s; + + /* progress the search */ + s = (pnet_switch_t*)pmix_list_get_next(&s->super); + } + /* s now points to the last item on the list */ + s->right = pmix_list_get_first(&p->switches); + s2 = (pnet_switch_t*)s->left; + s->leftport.link = &s2->rightport; + s2->rightport.link = &s->leftport; + s2 = (pnet_switch_t*)s->right; + s->rightport.link = &s2->leftport; + s2->leftport.link = &s->rightport; + + /* now cycle across the nodes and setup their connections + * to the switches */ + if (p->dense) { + /* connect each successive node to the same switch + * until that switch is full - then move to the next */ + s = (pnet_switch_t*)pmix_list_get_first(&p->switches); + nic = (pnet_nic_t*)pmix_list_get_first(&s->ports); + n = 0; + ns = pmix_list_get_size(&s->ports); + PMIX_LIST_FOREACH(node, &mynodes, pnet_node_t) { + nic2 = PMIX_NEW(pnet_nic_t); + if (0 > asprintf(&nic2->name, "%s:nic%03d", node->name, n)) { + return PMIX_ERR_NOMEM; + } + ++n; + --ns; + nic2->node = node; + nic2->s = s; + nic2->plane = p; + nic2->index = pmix_pointer_array_add(&mynics, nic2); + PMIX_RETAIN(nic2); + pmix_list_append(&node->nics, &nic2->super); + nic2->link = nic; + nic->link = nic2; + if (0 == ns) { + /* move to the next switch */ + s = (pnet_switch_t*)pmix_list_get_next(&s->super); + nic = (pnet_nic_t*)pmix_list_get_first(&s->ports); + ns = pmix_list_get_size(&s->ports); + } + } + } + + /* setup the cost matrix - we assume switch-to-switch hops + * have a cost of 1, as do all node-to-switch hops */ + p->nverts = nnodes; // we ignore the switch ports for now + p->costmatrix = (uint16_t**)malloc(p->nverts * sizeof(uint16_t*)); + for (n64=0; n64 < p->nverts; n64++) { + p->costmatrix[n64] = malloc(p->nverts * sizeof(uint16_t)); + } + /* fill the matrix with the #hops between each NIC, keeping it symmetric */ + for (n64=0; n64 < p->nverts; n64++) { + p->costmatrix[n64][n64] = 0; + nic = (pnet_nic_t*)pmix_pointer_array_get_item(&mynics, n64); + if (NULL == nic) { + PMIX_ERROR_LOG(PMIX_ERR_NOT_FOUND); + continue; + } + for (m64=n64+1; m64 < p->nverts; m64++) { + nic2 = (pnet_nic_t*)pmix_pointer_array_get_item(&mynics, m64); + if (NULL == nic2) { + PMIX_ERROR_LOG(PMIX_ERR_NOT_FOUND); + continue; + } + /* if they are on the same switch, then cost is 2 */ + if (nic->s == nic2->s) { + p->costmatrix[n64][m64] = 2; + } else { + /* the cost is increased by the distance + * between switches */ + s = (pnet_switch_t*)nic->s; + s2 = (pnet_switch_t*)nic2->s; + if (s->index > s2->index) { + p->costmatrix[n64][m64] = 2 + s->index - s2->index; + } else { + p->costmatrix[n64][m64] = 2 + s2->index - s->index; + } + } + p->costmatrix[m64][n64] = p->costmatrix[n64][m64]; + } + } + ++nplane; + } + pmix_argv_free(system); + } + return PMIX_SUCCESS; } static void test_finalize(void) { + pmix_pnet_fabric_t *ft; + pnet_nic_t *nic; + int n; + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, "pnet: test finalize"); + + for (n=0; n < myfabrics.size; n++) { + if (NULL != (ft = (pmix_pnet_fabric_t*)pmix_pointer_array_get_item(&myfabrics, n))) { + PMIX_RELEASE(ft); + } + } + PMIX_DESTRUCT(&myfabrics); + PMIX_LIST_DESTRUCT(&mynodes); + for (n=0; n < mynics.size; n++) { + if (NULL != (nic = (pnet_nic_t*)pmix_pointer_array_get_item(&mynics, n))) { + PMIX_RELEASE(nic); + } + } + PMIX_DESTRUCT(&mynics); + PMIX_LIST_DESTRUCT(&myplanes); } /* NOTE: if there is any binary data to be transferred, then @@ -101,23 +511,25 @@ static pmix_status_t allocate(pmix_namespace_t *nptr, pmix_kval_t *kv; bool seckey = false, envars = false; pmix_list_t mylist; - size_t n, nreqs=0; - pmix_info_t *requests = NULL; - char *idkey = NULL; + size_t n, m, p, q, nreqs=0; + pmix_info_t *requests = NULL, *iptr, *ip2; + char *idkey = NULL, **locals; uint64_t unique_key = 12345; pmix_buffer_t buf; pmix_status_t rc; - pmix_pnet_job_t *jptr, *job; - pmix_pnet_node_t *nd; - pmix_pnet_local_procs_t *lptr, *lp; + char **nodes = NULL, **procs = NULL; + pmix_data_array_t *darray, *d2, *d3; + pmix_rank_t rank; + pnet_node_t *nd, *nd2; + uint32_t *u32; pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, - "pnet:test:allocate for nspace %s key %s", - nptr->nspace, info->key); + "pnet:test:allocate for nspace %s", + nptr->nspace); - /* if I am not the gateway, then ignore this call - should never + /* if I am not the scheduler, then ignore this call - should never * happen, but check to be safe */ - if (!PMIX_PROC_IS_GATEWAY(pmix_globals.mypeer)) { + if (!PMIX_PROC_IS_SCHEDULER(pmix_globals.mypeer)) { return PMIX_SUCCESS; } @@ -127,43 +539,71 @@ static pmix_status_t allocate(pmix_namespace_t *nptr, /* check directives to see if a crypto key and/or * network resource allocations requested */ for (n=0; n < ninfo; n++) { + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:test:allocate processing key %s", + info[n].key); if (PMIX_CHECK_KEY(&info[n], PMIX_SETUP_APP_ENVARS) || PMIX_CHECK_KEY(&info[n], PMIX_SETUP_APP_ALL)) { envars = PMIX_INFO_TRUE(&info[n]); - } else if (PMIX_CHECK_KEY(&info[n], PMIX_ALLOC_NETWORK_ID)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_ALLOC_NETWORK)) { /* this info key includes an array of pmix_info_t, each providing * a key (that is to be used as the key for the allocated ports) and * a number of ports to allocate for that key */ - if (PMIX_DATA_ARRAY != info->value.type || - NULL == info->value.data.darray || - PMIX_INFO != info->value.data.darray->type || - NULL == info->value.data.darray->array) { - /* just process something for test */ - goto process; + if (PMIX_DATA_ARRAY != info[n].value.type || + NULL == info[n].value.data.darray || + PMIX_INFO != info[n].value.data.darray->type || + NULL == info[n].value.data.darray->array) { + pmix_output(0, "NOTHING IN ARRAY"); + requests = NULL; + nreqs = 0; + } else { + requests = (pmix_info_t*)info[n].value.data.darray->array; + nreqs = info[n].value.data.darray->size; + } + } else if (PMIX_CHECK_KEY(&info[n], PMIX_PROC_MAP)) { + rc = pmix_preg.parse_procs(info[n].value.data.string, &procs); + if (PMIX_SUCCESS != rc) { + return PMIX_ERR_BAD_PARAM; + } + } else if (PMIX_CHECK_KEY(&info[n], PMIX_NODE_MAP)) { + rc = pmix_preg.parse_nodes(info[n].value.data.string, &nodes); + if (PMIX_SUCCESS != rc) { + return PMIX_ERR_BAD_PARAM; } - requests = (pmix_info_t*)info->value.data.darray->array; - nreqs = info->value.data.darray->size; } } + PMIX_CONSTRUCT(&mylist, pmix_list_t); + if (envars) { + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:test:allocate adding envar for nspace %s", + nptr->nspace); + kv = PMIX_NEW(pmix_kval_t); if (NULL == kv) { - return PMIX_ERR_NOMEM; + rc = PMIX_ERR_NOMEM; + goto cleanup; } kv->key = strdup(PMIX_SET_ENVAR); kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); if (NULL == kv->value) { PMIX_RELEASE(kv); - return PMIX_ERR_NOMEM; + rc = PMIX_ERR_NOMEM; + goto cleanup; } kv->value->type = PMIX_ENVAR; PMIX_ENVAR_LOAD(&kv->value->data.envar, "PMIX_TEST_ENVAR", "1", ':'); - pmix_list_append(ilist, &kv->super); + pmix_list_append(&mylist, &kv->super); } if (NULL == requests) { - return PMIX_ERR_TAKE_NEXT_OPTION; + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:test:allocate no requests for nspace %s", + nptr->nspace); + + rc = PMIX_ERR_TAKE_NEXT_OPTION; + goto complete; } pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, @@ -172,7 +612,7 @@ static pmix_status_t allocate(pmix_namespace_t *nptr, /* cycle thru the provided array and get the ID key */ for (n=0; n < nreqs; n++) { - if (0 == strncmp(requests[n].key, PMIX_ALLOC_NETWORK_ID, PMIX_MAX_KEYLEN)) { + if (PMIX_CHECK_KEY(&requests[n], PMIX_ALLOC_NETWORK_ID)) { /* check for bozo error */ if (PMIX_STRING != requests[n].value.type || NULL == requests[n].value.data.string) { @@ -180,107 +620,173 @@ static pmix_status_t allocate(pmix_namespace_t *nptr, return PMIX_ERR_BAD_PARAM; } idkey = requests[n].value.data.string; - } else if (0 == strncasecmp(requests[n].key, PMIX_ALLOC_NETWORK_SEC_KEY, PMIX_MAX_KEYLEN)) { + } else if (PMIX_CHECK_KEY(&requests[n], PMIX_ALLOC_NETWORK_SEC_KEY)) { seckey = PMIX_INFO_TRUE(&requests[n]); } } - process: /* if they didn't give us a test key, just create one */ if (NULL == idkey) { idkey = "TESTKEY"; } - PMIX_CONSTRUCT(&mylist, pmix_list_t); /* must include the idkey */ kv = PMIX_NEW(pmix_kval_t); if (NULL == kv) { - return PMIX_ERR_NOMEM; + rc = PMIX_ERR_NOMEM; + goto cleanup; } kv->key = strdup(PMIX_ALLOC_NETWORK_ID); kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); if (NULL == kv->value) { PMIX_RELEASE(kv); - return PMIX_ERR_NOMEM; + rc = PMIX_ERR_NOMEM; + goto cleanup; } kv->value->type = PMIX_STRING; kv->value->data.string = strdup(idkey); pmix_list_append(&mylist, &kv->super); if (seckey) { + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:test:allocate assigning network security key for nspace %s", + nptr->nspace); + kv = PMIX_NEW(pmix_kval_t); if (NULL == kv) { - return PMIX_ERR_NOMEM; + rc = PMIX_ERR_NOMEM; + goto cleanup; } kv->key = strdup(PMIX_ALLOC_NETWORK_SEC_KEY); kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); if (NULL == kv->value) { PMIX_RELEASE(kv); - return PMIX_ERR_NOMEM; + rc = PMIX_ERR_NOMEM; + goto cleanup; } kv->value->type = PMIX_BYTE_OBJECT; kv->value->data.bo.bytes = (char*)malloc(sizeof(uint64_t)); if (NULL == kv->value->data.bo.bytes) { PMIX_RELEASE(kv); - return PMIX_ERR_NOMEM; + rc = PMIX_ERR_NOMEM; + goto cleanup; } memcpy(kv->value->data.bo.bytes, &unique_key, sizeof(uint64_t)); kv->value->data.bo.size = sizeof(uint64_t); pmix_list_append(&mylist, &kv->super); } - /* find the info on this job, if available */ - job = NULL; - PMIX_LIST_FOREACH(jptr, &pmix_pnet_globals.jobs, pmix_pnet_job_t) { - if (0 == strcmp(jptr->nspace, nptr->nspace)) { - job = jptr; - break; - } - } - if (NULL != job) { - pmix_output(0, "ALLOCATE RESOURCES FOR JOB %s", job->nspace); - for (n=0; (int)n < job->nodes.size; n++) { - if (NULL == (nd = (pmix_pnet_node_t*)pmix_pointer_array_get_item(&job->nodes, n))) { - continue; - } - lp = NULL; - PMIX_LIST_FOREACH(lptr, &nd->local_jobs, pmix_pnet_local_procs_t) { - if (0 == strcmp(job->nspace, lptr->nspace)) { - lp = lptr; - break; - } - } - if (NULL == lp) { - pmix_output(0, "\t NODE %s 0 RANKS", nd->name); - } else { - pmix_output(0, "\tNODE %s %d RANKS", nd->name, (int)lp->np); - } - } + if (NULL == procs || NULL == nodes) { + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:test:allocate missing proc/node map for nspace %s", + nptr->nspace); + /* not an error - continue to next active component */ + rc = PMIX_ERR_TAKE_NEXT_OPTION; + goto complete; } + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:test:allocate assigning endpoints for nspace %s", + nptr->nspace); + + /* cycle across the nodes and add the endpoints + * for each proc on the node - we assume the same + * list of static endpoints on each node */ + for (n=0; NULL != nodes[n]; n++) { + pmix_output(0, "WORKING NODE %s", nodes[n]); + /* split the procs for this node */ + locals = pmix_argv_split(procs[n], ','); + if (NULL == locals) { + /* aren't any on this node */ + pmix_output(0, "NO PROCS ON THIS NODE"); + continue; + } + /* find this node in our list */ + nd = NULL; + PMIX_LIST_FOREACH(nd2, &mynodes, pnet_node_t) { + if (0 == strcmp(nd2->name, nodes[n])) { + nd = nd2; + break; + } + } + if (NULL == nd) { + /* we don't have this node in our list */ + pmix_output(0, "DO NOT KNOW THIS NODE"); + rc = PMIX_ERR_NOT_FOUND; + goto cleanup; + } + kv = PMIX_NEW(pmix_kval_t); + if (NULL == kv) { + rc = PMIX_ERR_NOMEM; + goto cleanup; + } + kv->key = strdup(PMIX_ALLOC_NETWORK_ENDPTS); + kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); + if (NULL == kv->value) { + PMIX_RELEASE(kv); + rc = PMIX_ERR_NOMEM; + goto cleanup; + } + kv->value->type = PMIX_DATA_ARRAY; + /* for each proc, we will assign an endpt + * for each NIC on the node */ + q = pmix_argv_count(locals); + PMIX_DATA_ARRAY_CREATE(darray, q, PMIX_INFO); + kv->value->data.darray = darray; + iptr = (pmix_info_t*)darray->array; + q = pmix_list_get_size(&nd->nics); + for (m=0; NULL != locals[m]; m++) { + /* each proc can have multiple endpoints depending + * on the number of NICs available on the node. So + * we package the endpoints for each proc as a data + * array with the first element being the proc ID + * and the remaining elements being the assigned + * endpoints for that proc in priority order */ + PMIX_LOAD_KEY(iptr[m].key, PMIX_PROC_DATA); + PMIX_DATA_ARRAY_CREATE(d2, 2, PMIX_INFO); + iptr[m].value.type = PMIX_DATA_ARRAY; + iptr[m].value.data.darray = d2; + ip2 = (pmix_info_t*)d2->array; + /* start with the rank */ + rank = m; + PMIX_INFO_LOAD(&ip2[0], PMIX_RANK, &rank, PMIX_PROC_RANK); + /* the second element in this array will itself + * be a data array of endpts */ + PMIX_DATA_ARRAY_CREATE(d3, q, PMIX_UINT32); + PMIX_LOAD_KEY(ip2[1].key, PMIX_NETWORK_ENDPT); + ip2[1].value.type = PMIX_DATA_ARRAY; + ip2[1].value.data.darray = d3; + u32 = (uint32_t*)d3->array; + for (p=0; p < q; p++) { + u32[p] = 3180 + (m * 4) + p; + } + } + pmix_argv_free(locals); + pmix_list_append(&mylist, &kv->super); + } + + complete: + /* pack all our results into a buffer for xmission to the backend */ n = pmix_list_get_size(&mylist); if (0 < n) { PMIX_CONSTRUCT(&buf, pmix_buffer_t); - /* pack the number of kvals for ease on the remote end */ - PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &buf, &n, 1, PMIX_SIZE); /* cycle across the list and pack the kvals */ while (NULL != (kv = (pmix_kval_t*)pmix_list_remove_first(&mylist))) { PMIX_BFROPS_PACK(rc, pmix_globals.mypeer, &buf, kv, 1, PMIX_KVAL); PMIX_RELEASE(kv); if (PMIX_SUCCESS != rc) { PMIX_DESTRUCT(&buf); - PMIX_LIST_DESTRUCT(&mylist); - return rc; + goto cleanup; } } - PMIX_LIST_DESTRUCT(&mylist); kv = PMIX_NEW(pmix_kval_t); kv->key = strdup("pmix-pnet-test-blob"); kv->value = (pmix_value_t*)malloc(sizeof(pmix_value_t)); if (NULL == kv->value) { PMIX_RELEASE(kv); PMIX_DESTRUCT(&buf); - return PMIX_ERR_NOMEM; + rc = PMIX_ERR_NOMEM; + goto cleanup; } kv->value->type = PMIX_BYTE_OBJECT; PMIX_UNLOAD_BUFFER(&buf, kv->value->data.bo.bytes, kv->value->data.bo.size); @@ -288,125 +794,92 @@ static pmix_status_t allocate(pmix_namespace_t *nptr, pmix_list_append(ilist, &kv->super); } - return PMIX_SUCCESS; + cleanup: + PMIX_LIST_DESTRUCT(&mylist); + if (NULL != nodes) { + pmix_argv_free(nodes); + } + if (NULL != procs) { + pmix_argv_free(procs); + } + return rc; } static pmix_status_t setup_local_network(pmix_namespace_t *nptr, pmix_info_t info[], size_t ninfo) { - size_t n, m, nkvals; - char *nodestring, **nodes; - pmix_proc_t *procs; - size_t nprocs; + size_t n, nvals; pmix_buffer_t bkt; int32_t cnt; pmix_kval_t *kv; pmix_status_t rc; - pmix_info_t *jinfo, stinfo; char *idkey = NULL; + uint64_t seckey = 0; + pmix_info_t *iptr; pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, "pnet:test:setup_local_network"); - /* get the list of nodes in this job - returns a regex */ - pmix_output(0, "pnet:test setup_local_network NSPACE %s", (NULL == nptr) ? "NULL" : nptr->nspace); - if (NULL == nptr) { - return PMIX_SUCCESS; - } - pmix_preg.resolve_nodes(nptr->nspace, &nodestring); - if (NULL == nodestring) { - return PMIX_SUCCESS; - } - pmix_preg.parse_nodes(nodestring, &nodes); // get an argv array of node names - pmix_output(0, "pnet:test setup_local_network NODES %s", (NULL == nodes) ? "NULL" : "NON-NULL"); - if (NULL == nodes) { - free(nodestring); - return PMIX_SUCCESS; - } - for (n=0; NULL != nodes[n]; n++) { - pmix_output(0, "pnet:test setup_local_network NODE: %s", nodes[n]); - } - - for (n=0; NULL != nodes[n]; n++) { - /* get an array of pmix_proc_t containing the names of the procs on that node */ - pmix_preg.resolve_peers(nodes[n], nptr->nspace, &procs, &nprocs); - if (NULL == procs) { - continue; - } - for (m=0; m < nprocs; m++) { - pmix_output(0, "pnet:test setup_local_network NODE %s: peer %s:%d", nodes[n], procs[m].nspace, procs[m].rank); - } - /* do stuff */ - free(procs); - } - if (NULL != info) { - for (n=0; n < ninfo; n++) { - /* look for my key */ - if (0 == strncmp(info[n].key, "pmix-pnet-test-blob", PMIX_MAX_KEYLEN)) { - /* this macro NULLs and zero's the incoming bo */ - PMIX_LOAD_BUFFER(pmix_globals.mypeer, &bkt, - info[n].value.data.bo.bytes, - info[n].value.data.bo.size); - /* unpack the number of kvals */ - cnt = 1; - PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, - &bkt, &nkvals, &cnt, PMIX_SIZE); - /* the data gets stored as a pmix_data_array_t on the provided key */ - PMIX_INFO_CONSTRUCT(&stinfo); - pmix_strncpy(stinfo.key, idkey, PMIX_MAX_KEYLEN); - stinfo.value.type = PMIX_DATA_ARRAY; - PMIX_DATA_ARRAY_CREATE(stinfo.value.data.darray, nkvals, PMIX_INFO); - jinfo = (pmix_info_t*)stinfo.value.data.darray->array; - - /* cycle thru the blob and extract the kvals */ - kv = PMIX_NEW(pmix_kval_t); - cnt = 1; - PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, - &bkt, kv, &cnt, PMIX_KVAL); - m = 0; - while (PMIX_SUCCESS == rc) { - pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, - "recvd KEY %s %s", kv->key, - (PMIX_STRING == kv->value->type) ? kv->value->data.string : "NON-STRING"); - /* xfer the value to the info */ - pmix_strncpy(jinfo[m].key, kv->key, PMIX_MAX_KEYLEN); - PMIX_BFROPS_VALUE_XFER(rc, pmix_globals.mypeer, - &jinfo[m].value, kv->value); - /* if this is the ID key, save it */ - if (NULL == idkey && - 0 == strncmp(kv->key, PMIX_ALLOC_NETWORK_ID, PMIX_MAX_KEYLEN)) { - idkey = strdup(kv->value->data.string); - } - ++m; - PMIX_RELEASE(kv); - kv = PMIX_NEW(pmix_kval_t); - cnt = 1; - PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, - &bkt, kv, &cnt, PMIX_KVAL); - } - /* restore the incoming data */ - info[n].value.data.bo.bytes = bkt.base_ptr; - info[n].value.data.bo.size = bkt.bytes_used; - bkt.base_ptr = NULL; - bkt.bytes_used = 0; - - /* if they didn't include a network ID, then this is an error */ - if (NULL == idkey) { - PMIX_INFO_FREE(jinfo, nkvals); - return PMIX_ERR_BAD_PARAM; - } - /* cache the info on the job */ - PMIX_GDS_CACHE_JOB_INFO(rc, pmix_globals.mypeer, nptr, - &stinfo, 1); - PMIX_INFO_DESTRUCT(&stinfo); - } - } - } - if (NULL != idkey) { - free(idkey); + for (n=0; n < ninfo; n++) { + /* look for my key */ + if (PMIX_CHECK_KEY(&info[n], "pmix-pnet-test-blob")) { + /* this macro NULLs and zero's the incoming bo */ + PMIX_LOAD_BUFFER(pmix_globals.mypeer, &bkt, + info[n].value.data.bo.bytes, + info[n].value.data.bo.size); + /* cycle thru the blob and extract the kvals */ + kv = PMIX_NEW(pmix_kval_t); + cnt = 1; + PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, + &bkt, kv, &cnt, PMIX_KVAL); + while (PMIX_SUCCESS == rc) { + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "recvd KEY %s %s", kv->key, PMIx_Data_type_string(kv->value->type)); + /* check for the network ID */ + if (PMIX_CHECK_KEY(kv, PMIX_ALLOC_NETWORK_ID)) { + idkey = strdup(kv->value->data.string); + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:test:setup_local_network idkey %s", idkey); + } else if (PMIX_CHECK_KEY(kv, PMIX_SET_ENVAR)) { + /* if this is an envar we are to set, save it on our + * list - we will supply it when setup_fork is called */ + pmix_argv_append_nosize(&myenvlist, kv->value->data.envar.envar); + pmix_argv_append_nosize(&myvalues, kv->value->data.envar.value); + } else if (PMIX_CHECK_KEY(kv, PMIX_ALLOC_NETWORK_SEC_KEY)) { + /* our network security key was stored as a byte object but + * is really just a uint64_t */ + memcpy(&seckey, kv->value->data.bo.bytes, sizeof(uint64_t)); + } else if (PMIX_CHECK_KEY(kv, PMIX_ALLOC_NETWORK_ENDPTS)) { + iptr = (pmix_info_t*)kv->value->data.darray->array; + nvals = kv->value->data.darray->size; + /* each element in this array is itself an array containing + * the rank and the endpts assigned to that rank. This is + * precisely the data we need to cache for the job, so + * just do so) */ + PMIX_GDS_CACHE_JOB_INFO(rc, pmix_globals.mypeer, nptr, iptr, nvals); + if (PMIX_SUCCESS != rc) { + PMIX_RELEASE(kv); + return rc; + } + } + PMIX_RELEASE(kv); + kv = PMIX_NEW(pmix_kval_t); + cnt = 1; + PMIX_BFROPS_UNPACK(rc, pmix_globals.mypeer, + &bkt, kv, &cnt, PMIX_KVAL); + } + PMIX_RELEASE(kv); + /* restore the incoming data */ + info[n].value.data.bo.bytes = bkt.base_ptr; + info[n].value.data.bo.size = bkt.bytes_used; + bkt.base_ptr = NULL; + bkt.bytes_used = 0; + } + } } + return PMIX_SUCCESS; } @@ -414,46 +887,17 @@ static pmix_status_t setup_fork(pmix_namespace_t *nptr, const pmix_proc_t *proc, char ***env) { - pmix_cb_t cb; - pmix_status_t rc; - pmix_kval_t *kv; - uint16_t localrank; + int n; - PMIX_CONSTRUCT(&cb, pmix_cb_t); - - cb.key = strdup(PMIX_LOCAL_RANK); - /* this data isn't going anywhere, so we don't require a copy */ - cb.copy = false; - /* scope is irrelevant as the info we seek must be local */ - cb.scope = PMIX_SCOPE_UNDEF; - /* ask for the value for the given proc */ - cb.proc = (pmix_proc_t*)proc; - - PMIX_GDS_FETCH_KV(rc, pmix_globals.mypeer, &cb); - if (PMIX_SUCCESS != rc) { - if (PMIX_ERR_INVALID_NAMESPACE != rc) { - PMIX_ERROR_LOG(rc); + /* if we have any envars to contribute, do so here */ + if (NULL != myenvlist) { + for (n=0; NULL != myenvlist[n]; n++) { + pmix_output_verbose(2, pmix_pnet_base_framework.framework_output, + "pnet:test:setup_fork setenv: %s=%s", + myenvlist[n], myvalues[n]); + pmix_setenv(myenvlist[n], myvalues[n], true, env); } - PMIX_DESTRUCT(&cb); - return rc; } - /* should just be the one value on the list */ - if (1 != pmix_list_get_size(&cb.kvs)) { - PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); - PMIX_DESTRUCT(&cb); - return PMIX_ERR_BAD_PARAM; - } - kv = (pmix_kval_t*)pmix_list_get_first(&cb.kvs); - if (PMIX_UINT16 != kv->value->type) { - PMIX_ERROR_LOG(PMIX_ERR_BAD_PARAM); - PMIX_DESTRUCT(&cb); - return PMIX_ERR_BAD_PARAM; - } - localrank = kv->value->data.uint16; - - pmix_output(0, "pnet:test LOCAL RANK FOR PROC %s: %d", PMIX_NAME_PRINT(proc), (int)localrank); - - PMIX_DESTRUCT(&cb); return PMIX_SUCCESS; } @@ -489,3 +933,260 @@ static pmix_status_t deliver_inventory(pmix_info_t info[], size_t ninfo, return PMIX_ERR_NOT_SUPPORTED; } + +static pmix_status_t register_fabric(pmix_fabric_t *fabric, + const pmix_info_t directives[], + size_t ndirs) +{ + pmix_pnet_fabric_t *ft; + pnet_plane_t *p, *p2; + char *pln = NULL; + size_t n; + + if (NULL == fabric) { + return PMIX_ERR_BAD_PARAM; + } + /* see what plane they wanted */ + for (n=0; n < ndirs; n++) { + if (PMIX_CHECK_KEY(&directives[n], PMIX_NETWORK_PLANE)) { + pln = directives[n].value.data.string; + break; + } + } + if (NULL == pln) { + /* just use the first on our list */ + p = (pnet_plane_t*)pmix_list_get_first(&myplanes); + } else { + /* find it */ + p = NULL; + PMIX_LIST_FOREACH(p2, &myplanes, pnet_plane_t) { + if (0 == strcmp(pln, p2->name)) { + p = p2; + break; + } + } + } + if (NULL == p) { + return PMIX_ERR_NOT_FOUND; + } + + ft = PMIX_NEW(pmix_pnet_fabric_t); + ft->module = &pmix_test_module; + ft->payload = p; + + /* pass to the user-level object */ + fabric->module = ft; + fabric->revision = p->revision; + + return PMIX_SUCCESS; +} + +static pmix_status_t deregister_fabric(pmix_fabric_t *fabric) +{ + fabric->module = NULL; + return PMIX_SUCCESS; +} + +static pmix_status_t get_num_verts(pmix_fabric_t *fabric, uint32_t *nverts) +{ + pmix_pnet_fabric_t *ft = (pmix_pnet_fabric_t*)fabric->module; + pnet_plane_t *p = (pnet_plane_t*)ft->payload; + int rc; + + rc = pmix_atomic_trylock(&p->atomlock); + if (0 != rc) { + return PMIX_ERR_RESOURCE_BUSY; + } + /* if fabric data has been updated since the last time + * this was accessed, let them know */ + if (fabric->revision != p->revision) { + /* update the revision */ + fabric->revision = p->revision; + pmix_atomic_unlock(&p->atomlock); + return PMIX_FABRIC_UPDATED; + } + + /* this component only looks at node-resident NICs as + * we assume switch-to-switch is done over dedicated + * ports in a ring topology */ + *nverts = p->nverts; + pmix_atomic_unlock(&p->atomlock); + return PMIX_SUCCESS; +} + +static pmix_status_t get_cost(pmix_fabric_t *fabric, + uint32_t src, uint32_t dest, + uint16_t *cost) +{ + pmix_pnet_fabric_t *ft = (pmix_pnet_fabric_t*)fabric->module; + pnet_plane_t *p = (pnet_plane_t*)ft->payload; + int rc; + + rc = pmix_atomic_trylock(&p->atomlock); + if (0 != rc) { + return PMIX_ERR_RESOURCE_BUSY; + } + /* if fabric data has been updated since the last time + * this was accessed, let them know */ + if (fabric->revision != p->revision) { + /* update the revision */ + fabric->revision = p->revision; + pmix_atomic_unlock(&p->atomlock); + return PMIX_FABRIC_UPDATED; + } + if (src >= p->nverts || dest >= p->nverts) { + pmix_atomic_unlock(&p->atomlock); + return PMIX_ERR_BAD_PARAM; + } + + *cost = p->costmatrix[src][dest]; + pmix_atomic_unlock(&p->atomlock); + return PMIX_SUCCESS; +} + +static pmix_status_t get_vertex(pmix_fabric_t *fabric, + uint32_t i, + pmix_value_t *identifier, + char **nodename) +{ + pmix_pnet_fabric_t *ft = (pmix_pnet_fabric_t*)fabric->module; + pnet_plane_t *p = (pnet_plane_t*)ft->payload; + pnet_nic_t *nic; + pnet_plane_t *pln; + pnet_switch_t *sw; + pnet_node_t *node; + pmix_info_t *info; + size_t n; + int rc; + + if (NULL == p) { + return PMIX_ERR_NOT_SUPPORTED; + } + + rc = pmix_atomic_trylock(&p->atomlock); + if (0 != rc) { + return PMIX_ERR_RESOURCE_BUSY; + } + /* if fabric data has been updated since the last time + * this was accessed, let them know */ + if (fabric->revision != p->revision) { + /* update the revision */ + fabric->revision = p->revision; + pmix_atomic_unlock(&p->atomlock); + return PMIX_FABRIC_UPDATED; + } + if (i >= p->nverts) { + pmix_atomic_unlock(&p->atomlock); + return PMIX_ERR_BAD_PARAM; + } + + /* find NIC that corresponds to this index */ + nic = (pnet_nic_t*)pmix_pointer_array_get_item(&mynics, i); + if (NULL == nic) { + pmix_atomic_unlock(&p->atomlock); + return PMIX_ERR_NOT_FOUND; + } + node = (pnet_node_t*)nic->node; + *nodename = strdup(node->name); + /* the value we pass back will be a data array containing + * info on the switch this NIC is connected to and the + * plane it is on */ + identifier->type = PMIX_DATA_ARRAY; + PMIX_DATA_ARRAY_CREATE(identifier->data.darray, 3, PMIX_INFO); + info = (pmix_info_t*)identifier->data.darray->array; + n = 0; + pln = (pnet_plane_t*)nic->plane; + PMIX_INFO_LOAD(&info[n], PMIX_NETWORK_PLANE, pln->name, PMIX_STRING); + ++n; + sw = (pnet_switch_t*)nic->s; + PMIX_INFO_LOAD(&info[n], PMIX_NETWORK_SWITCH, sw->name, PMIX_STRING); + ++n; + PMIX_INFO_LOAD(&info[n], PMIX_NETWORK_NIC, nic->name, PMIX_STRING); + + pmix_atomic_unlock(&p->atomlock); + return PMIX_SUCCESS; +} + +static pmix_status_t get_index(pmix_fabric_t *fabric, + pmix_value_t *identifier, + uint32_t *i, + char **nodename) +{ + pmix_pnet_fabric_t *ft = (pmix_pnet_fabric_t*)fabric->module; + pnet_plane_t *p = (pnet_plane_t*)ft->payload; + pnet_node_t *node; + pnet_nic_t *nic; + int rc, m; + pmix_status_t ret; + pmix_info_t *info; + char *nc=NULL; + size_t n; + + if (NULL == p) { + return PMIX_ERR_NOT_SUPPORTED; + } + + rc = pmix_atomic_trylock(&p->atomlock); + if (0 != rc) { + return PMIX_ERR_RESOURCE_BUSY; + } + /* if fabric data has been updated since the last time + * this was accessed, let them know */ + if (fabric->revision != p->revision) { + /* update the revision */ + fabric->revision = p->revision; + pmix_atomic_unlock(&p->atomlock); + return PMIX_FABRIC_UPDATED; + } + + /* see what they gave us */ + if (PMIX_DATA_ARRAY == identifier->type) { + if (PMIX_INFO != identifier->data.darray->type) { + ret = PMIX_ERR_BAD_PARAM; + goto cleanup; + } + info = (pmix_info_t*)identifier->data.darray->array; + for (n=0; n < identifier->data.darray->size; n++) { + if (PMIX_CHECK_KEY(&info[n], PMIX_NETWORK_NIC)) { + nc = info[n].value.data.string; + } + } + if (NULL == nc) { + ret = PMIX_ERR_BAD_PARAM; + goto cleanup; + } + /* find the NIC */ + for (m=0; m < mynics.size; m++) { + nic = (pnet_nic_t*)pmix_pointer_array_get_item(&mynics, m); + if (NULL == nic) { + continue; + } + if (0 == strcmp(nc, nic->name)) { + *i = m; + node = (pnet_node_t*)nic->node; + *nodename = strdup(node->name); + ret = PMIX_SUCCESS; + goto cleanup; + } + } + ret = PMIX_ERR_NOT_FOUND; + } else if (PMIX_UINT32 == identifier->type) { + /* they gave us the vertex number - in our case, + * that is the NIC id */ + *i = identifier->data.uint32; + nic = (pnet_nic_t*)pmix_pointer_array_get_item(&mynics, *i); + if (NULL == nic) { + ret = PMIX_ERR_NOT_FOUND; + goto cleanup; + } + node = (pnet_node_t*)nic->node; + *nodename = strdup(node->name); + ret = PMIX_SUCCESS; + } else { + ret = PMIX_ERR_BAD_PARAM; + } + + cleanup: + pmix_atomic_unlock(&p->atomlock); + return ret; +} diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/test/pnet_test.h b/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/test/pnet_test.h index 8601bc355b..9004319596 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/test/pnet_test.h +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/test/pnet_test.h @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * * $COPYRIGHT$ * @@ -20,8 +20,9 @@ BEGIN_C_DECLS typedef struct { pmix_pnet_base_component_t super; - char **include; - char **exclude; + char *cfg_file; + char *nverts; + uint16_t **costmatrix; } pmix_pnet_test_component_t; /* the component must be visible data for the linker to find it */ diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/test/pnet_test_component.c b/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/test/pnet_test_component.c index 06e360163f..4f55da7537 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/test/pnet_test_component.c +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/test/pnet_test_component.c @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -36,6 +36,7 @@ static pmix_status_t component_open(void); static pmix_status_t component_close(void); static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority); +static pmix_status_t component_register(void); /* * Instantiate the public struct with all of our public information @@ -57,16 +58,38 @@ pmix_pnet_test_component_t mca_pnet_test_component = { .pmix_mca_open_component = component_open, .pmix_mca_close_component = component_close, .pmix_mca_query_component = component_query, + .pmix_mca_register_component_params = component_register }, .data = { /* The component is checkpoint ready */ PMIX_MCA_BASE_METADATA_PARAM_CHECKPOINT } }, - .include = NULL, - .exclude = NULL + .cfg_file = NULL, + .nverts = NULL, + .costmatrix = NULL }; +static pmix_status_t component_register(void) +{ + pmix_mca_base_component_t *component = &mca_pnet_test_component.super.base; + + (void)pmix_mca_base_component_var_register(component, "cfg_file", + "Comma-delimited list of files containing descriptions of the test fabric, one plane per file", + PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, + PMIX_INFO_LVL_2, + PMIX_MCA_BASE_VAR_SCOPE_READONLY, + &mca_pnet_test_component.cfg_file); + + (void)pmix_mca_base_component_var_register(component, "nverts", + "Comma-delimited list of number of vertices in each fabric plane (if no cfg file given)", + PMIX_MCA_BASE_VAR_TYPE_STRING, NULL, 0, 0, + PMIX_INFO_LVL_2, + PMIX_MCA_BASE_VAR_SCOPE_READONLY, + &mca_pnet_test_component.nverts); + return PMIX_SUCCESS; +} + static pmix_status_t component_open(void) { int index; diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/preg/native/preg_native.c b/opal/mca/pmix/pmix4x/pmix/src/mca/preg/native/preg_native.c index 0d31f96435..0c9d6188a0 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/preg/native/preg_native.c +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/preg/native/preg_native.c @@ -1,5 +1,5 @@ /* - * Copyright (c) 2015-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * Copyright (c) 2016-2019 IBM Corporation. All rights reserved. * Copyright (c) 2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. @@ -893,7 +893,7 @@ static pmix_status_t regex_parse_value_range(char *base, char *range, for (found = false, i = 0; i < len; ++i) { if (isdigit((int) range[i])) { if (!found) { - start = atoi(range + i); + start = strtol(range + i, NULL, 10); found = true; break; } diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/opa/Makefile.am b/opal/mca/pmix/pmix4x/pmix/src/mca/psec/dummy_handshake/Makefile.am similarity index 64% rename from opal/mca/pmix/pmix4x/pmix/src/mca/pnet/opa/Makefile.am rename to opal/mca/pmix/pmix4x/pmix/src/mca/psec/dummy_handshake/Makefile.am index fe01cde836..1dd3853eb2 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/pnet/opa/Makefile.am +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/psec/dummy_handshake/Makefile.am @@ -12,8 +12,8 @@ # All rights reserved. # Copyright (c) 2012 Los Alamos National Security, Inc. All rights reserved. # Copyright (c) 2013-2019 Intel, Inc. All rights reserved. -# Copyright (c) 2017 Research Organization for Information Science -# and Technology (RIST). All rights reserved. +# Copyright (c) 2019 Mellanox Technologies, Inc. +# All rights reserved. # $COPYRIGHT$ # # Additional copyrights may follow @@ -21,24 +21,24 @@ # $HEADER$ # -AM_CPPFLAGS = $(pnet_opa_CPPFLAGS) +if MCA_BUILD_PSEC_DUMMY_HANDSHAKE -headers = pnet_opa.h +headers = psec_dummy_handshake.h sources = \ - pnet_opa_component.c \ - pnet_opa.c + psec_dummy_handshake_component.c \ + psec_dummy_handshake.c # Make the output library in this directory, and name it either # mca__.la (for DSO builds) or libmca__.la # (for static builds). -if MCA_BUILD_pmix_pnet_opa_DSO +if MCA_BUILD_pmix_psec_dummy_handshake_DSO lib = lib_sources = -component = mca_pnet_opa.la +component = mca_psec_dummy_handshake.la component_sources = $(headers) $(sources) else -lib = libmca_pnet_opa.la +lib = libmca_psec_dummy_handshake.la lib_sources = $(headers) $(sources) component = component_sources = @@ -46,14 +46,14 @@ endif mcacomponentdir = $(pmixlibdir) mcacomponent_LTLIBRARIES = $(component) -mca_pnet_opa_la_SOURCES = $(component_sources) -mca_pnet_opa_la_LIBADD = $(pnet_opa_LIBS) -mca_pnet_opa_la_LDFLAGS = -module -avoid-version $(pnet_opa_LDFLAGS) +mca_psec_dummy_handshake_la_SOURCES = $(component_sources) +mca_psec_dummy_handshake_la_LDFLAGS = -module -avoid-version if NEED_LIBPMIX -mca_pnet_opa_la_LIBADD += $(top_builddir)/src/libpmix.la +mca_psec_dummy_handshake_la_LIBADD = $(top_builddir)/src/libpmix.la endif noinst_LTLIBRARIES = $(lib) -libmca_pnet_opa_la_SOURCES = $(lib_sources) -libmca_pnet_opa_la_LIBADD = $(pnet_opa_LIBS) -libmca_pnet_opa_la_LDFLAGS = -module -avoid-version $(pnet_opa_LDFLAGS) +libmca_psec_dummy_handshake_la_SOURCES = $(lib_sources) +libmca_psec_dummy_handshake_la_LDFLAGS = -module -avoid-version + +endif diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/psec/dummy_handshake/psec_dummy_handshake.c b/opal/mca/pmix/pmix4x/pmix/src/mca/psec/dummy_handshake/psec_dummy_handshake.c new file mode 100644 index 0000000000..0e707049f6 --- /dev/null +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/psec/dummy_handshake/psec_dummy_handshake.c @@ -0,0 +1,169 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2019 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include + +#include +#ifdef HAVE_SYS_TYPES_H +#include +#endif + +#include + +#include "src/include/pmix_globals.h" +#include "src/util/error.h" +#include "src/util/output.h" + +#include "src/mca/psec/base/base.h" +#include "psec_dummy_handshake.h" + +#include "src/mca/ptl/base/base.h" + +#define PMIX_PSEC_DUMMY_HNDSHK_STR "PMIX_PSEC_DUMMY_HANDSHAKE_STRING" + +static pmix_status_t simple_init(void); +static void simple_finalize(void); +static pmix_status_t create_cred(struct pmix_peer_t *peer, + const pmix_info_t directives[], size_t ndirs, + pmix_info_t **info, size_t *ninfo, + pmix_byte_object_t *cred); +static pmix_status_t client_hndshk(int sd); +static pmix_status_t server_hndshk(int sd); + +pmix_psec_module_t pmix_dummy_handshake_module = { + .name = "dummy_handshake", + /** init/finalize */ + .init = simple_init, + .finalize = simple_finalize, + /** Client-side */ + .create_cred = create_cred, + .client_handshake = client_hndshk, + /** Server-side */ + .validate_cred = NULL, + .server_handshake = server_hndshk +}; + +static pmix_status_t simple_init(void) +{ + pmix_output_verbose(2, pmix_psec_base_framework.framework_output, + "psec: simple init"); + return PMIX_SUCCESS; +} + +static void simple_finalize(void) +{ + pmix_output_verbose(2, pmix_psec_base_framework.framework_output, + "psec: simple finalize"); +} + +static pmix_status_t create_cred(struct pmix_peer_t *peer, + const pmix_info_t directives[], size_t ndirs, + pmix_info_t **info, size_t *ninfo, + pmix_byte_object_t *cred) +{ + char mycred[] = "dymmy_cred"; + + pmix_output_verbose(2, pmix_psec_base_framework.framework_output, + "psec: simple create_cred"); + + /* ensure initialization */ + PMIX_BYTE_OBJECT_CONSTRUCT(cred); + + cred->bytes = strdup(mycred); + cred->size = strlen(mycred) + 1; + + return PMIX_SUCCESS; +} + +static pmix_status_t server_hndshk(int sd) +{ + pmix_status_t rc, status = PMIX_SUCCESS; + char *hndshk_msg = NULL; + size_t size; + + pmix_output_verbose(2, pmix_psec_base_framework.framework_output, + "psec: simple server_hndshk"); + + asprintf(&hndshk_msg, "%s", PMIX_PSEC_DUMMY_HNDSHK_STR); + size = strlen(hndshk_msg); + + /* send size of handshake message */ + if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(sd, (char*)&size, + sizeof(size)))) { + goto exit; + } + /* send handshake message */ + if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(sd, hndshk_msg, + size))) { + goto exit; + } + /* recv hadshake status from client */ + if (PMIX_SUCCESS != (rc = pmix_ptl_base_recv_blocking(sd, (char*)&status, + sizeof(status)))) { + goto exit; + } + rc = status; + pmix_output(0, "[%s:%d] psec handshake status %d recv from client", + __FILE__, __LINE__, status); + +exit: + if (NULL != hndshk_msg) { + free(hndshk_msg); + } + + return rc; +} + +static pmix_status_t client_hndshk(int sd) +{ + char *hndshk_msg = NULL; + size_t size; + pmix_status_t rc, status = PMIX_SUCCESS; + + pmix_output_verbose(2, pmix_psec_base_framework.framework_output, + "psec: simple client_hndshk"); + + /* recv size of handshake message */ + if (PMIX_SUCCESS != (rc = pmix_ptl_base_recv_blocking(sd, (char*)&size, + sizeof(size_t)))) { + return rc; + } + hndshk_msg = (char*)malloc(size); + /* recv handshake message */ + if (PMIX_SUCCESS != (rc = pmix_ptl_base_recv_blocking(sd, (char*)hndshk_msg, + size))) { + free(hndshk_msg); + return rc; + } + /* verifying handshake data */ + if (size != strlen(PMIX_PSEC_DUMMY_HNDSHK_STR)) { + rc = PMIX_ERR_HANDSHAKE_FAILED; + goto exit; + } + if (0 != strncmp(hndshk_msg, PMIX_PSEC_DUMMY_HNDSHK_STR, size)) { + rc = PMIX_ERR_HANDSHAKE_FAILED; + goto exit; + } + + /* send hadshake status to the server */ + status = PMIX_SUCCESS; + if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(sd, (char*)&status, + sizeof(status)))) { + goto exit; + } + pmix_output(0, "[%s:%d] psec handshake status %d sent to server", + __FILE__, __LINE__, status); +exit: + if (NULL != hndshk_msg) { + free(hndshk_msg); + } + return rc; +} diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/psec/dummy_handshake/psec_dummy_handshake.h b/opal/mca/pmix/pmix4x/pmix/src/mca/psec/dummy_handshake/psec_dummy_handshake.h new file mode 100644 index 0000000000..5a8ea51910 --- /dev/null +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/psec/dummy_handshake/psec_dummy_handshake.h @@ -0,0 +1,28 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2019 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef PMIX_SIMPLE_H +#define PMIX_SIMPLE_H + +#include + + +#include "src/mca/psec/psec.h" + +BEGIN_C_DECLS + +/* the component must be visible data for the linker to find it */ +PMIX_EXPORT extern pmix_psec_base_component_t mca_psec_dummy_handshake_component; +extern pmix_psec_module_t pmix_dummy_handshake_module; + +END_C_DECLS + +#endif diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/psec/dummy_handshake/psec_dummy_handshake_component.c b/opal/mca/pmix/pmix4x/pmix/src/mca/psec/dummy_handshake/psec_dummy_handshake_component.c new file mode 100644 index 0000000000..fd826b817c --- /dev/null +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/psec/dummy_handshake/psec_dummy_handshake_component.c @@ -0,0 +1,72 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2019 Mellanox Technologies, Inc. + * All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include +#include "pmix_common.h" + +#include "src/mca/base/pmix_mca_base_var.h" +#include "src/mca/psec/psec.h" +#include "psec_dummy_handshake.h" + +static pmix_status_t component_open(void); +static pmix_status_t component_close(void); +static pmix_status_t component_query(pmix_mca_base_module_t **module, int *priority); +static pmix_psec_module_t* assign_module(void); + +/* + * Instantiate the public struct with all of our public information + * and pointers to our public functions in it + */ +pmix_psec_base_component_t mca_psec_dummy_handshake_component = { + .base = { + PMIX_PSEC_BASE_VERSION_1_0_0, + + /* Component name and version */ + .pmix_mca_component_name = "dummy_handshake", + PMIX_MCA_BASE_MAKE_VERSION(component, + PMIX_MAJOR_VERSION, + PMIX_MINOR_VERSION, + PMIX_RELEASE_VERSION), + + /* Component open and close functions */ + .pmix_mca_open_component = component_open, + .pmix_mca_close_component = component_close, + .pmix_mca_query_component = component_query, + }, + .data = { + /* The component is checkpoint ready */ + PMIX_MCA_BASE_METADATA_PARAM_CHECKPOINT + }, + .assign_module = assign_module +}; + +static int component_open(void) +{ + return PMIX_SUCCESS; +} + +static int component_query(pmix_mca_base_module_t **module, int *priority) +{ + *priority = 100; + *module = (pmix_mca_base_module_t *)&pmix_dummy_handshake_module; + return PMIX_SUCCESS; +} + + +static int component_close(void) +{ + return PMIX_SUCCESS; +} + +static pmix_psec_module_t* assign_module(void) +{ + return &pmix_dummy_handshake_module; +} diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/psec/psec.h b/opal/mca/pmix/pmix4x/pmix/src/mca/psec/psec.h index 4057681f6f..e088cd5ff4 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/psec/psec.h +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/psec/psec.h @@ -5,6 +5,7 @@ * * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -155,23 +156,12 @@ PMIX_EXPORT pmix_psec_module_t* pmix_psec_base_assign_module(const char *options pmix_output_verbose(2, pmix_globals.debug_output, \ "credential validated"); \ } \ - /* send them the result */ \ - if (PMIX_SUCCESS != (_r = pmix_ptl_base_send_blocking((p)->sd, (char*)&(_r), sizeof(int)))) { \ - PMIX_ERROR_LOG(_r); \ - } \ (r) = _r; \ } else if (NULL != (p)->nptr->compat.psec->server_handshake) { \ - /* execute the handshake if the security mode calls for it */ \ + /* request the handshake if the security mode calls for it */ \ pmix_output_verbose(2, pmix_globals.debug_output, \ - "executing handshake"); \ + "requesting handshake"); \ _r = PMIX_ERR_READY_FOR_HANDSHAKE; \ - if (PMIX_SUCCESS != (_r = pmix_ptl_base_send_blocking((p)->sd, (char*)&(_r), sizeof(int)))) { \ - PMIX_ERROR_LOG(_r); \ - } else { \ - if (PMIX_SUCCESS != (_r = p->nptr->compat.psec->server_handshake((p)->sd))) { \ - PMIX_ERROR_LOG(_r); \ - } \ - } \ (r) = _r; \ } else { \ /* this is not allowed */ \ @@ -179,6 +169,21 @@ PMIX_EXPORT pmix_psec_module_t* pmix_psec_base_assign_module(const char *options } \ } while(0) + +#define PMIX_PSEC_SERVER_HANDSHAKE_IFNEED(r, p, d, nd, in, nin, c) \ + if(PMIX_ERR_READY_FOR_HANDSHAKE == r) { \ + int _r; \ + /* execute the handshake if the security mode calls for it */ \ + pmix_output_verbose(2, pmix_globals.debug_output, \ + "executing handshake"); \ + if (PMIX_SUCCESS != (_r = p->nptr->compat.psec->server_handshake((p)->sd))) { \ + PMIX_ERROR_LOG(_r); \ + } \ + /* Update the reply status */ \ + (r) = _r; \ + } + + /**** COMPONENT STRUCTURE DEFINITION ****/ /* define a component-level API for initializing the component */ diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/psensor/base/psensor_base_frame.c b/opal/mca/pmix/pmix4x/pmix/src/mca/psensor/base/psensor_base_frame.c index d10bab1cb1..9454f64e06 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/psensor/base/psensor_base_frame.c +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/psensor/base/psensor_base_frame.c @@ -48,6 +48,7 @@ static bool use_separate_thread = false; static int pmix_psensor_register(pmix_mca_base_register_flag_t flags) { + (void)flags; (void) pmix_mca_base_var_register("pmix", "psensor", "base", "use_separate_thread", "Use a separate thread for monitoring local procs", PMIX_MCA_BASE_VAR_TYPE_BOOL, NULL, 0, 0, diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/base/ptl_base_frame.c b/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/base/ptl_base_frame.c index 2e6a101752..a6a8ce03f0 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/base/ptl_base_frame.c +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/base/ptl_base_frame.c @@ -63,6 +63,7 @@ static size_t max_msg_size = PMIX_MAX_MSG_SIZE; static int pmix_ptl_register(pmix_mca_base_register_flag_t flags) { + (void)flags; pmix_mca_base_var_register("pmix", "ptl", "base", "max_msg_size", "Max size (in Mbytes) of a client/server msg", PMIX_MCA_BASE_VAR_TYPE_SIZE_T, NULL, 0, 0, diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/base/ptl_base_listener.c b/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/base/ptl_base_listener.c index ddc284f8ab..bff30b4c36 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/base/ptl_base_listener.c +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/base/ptl_base_listener.c @@ -181,6 +181,7 @@ void pmix_ptl_base_stop_listening(void) static void* listen_thread(void *obj) { + (void)obj; int rc, max, accepted_connections; socklen_t addrlen = sizeof(struct sockaddr_storage); pmix_pending_connection_t *pending_connection; diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/base/ptl_base_sendrecv.c b/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/base/ptl_base_sendrecv.c index 9857bc8a11..839e3ae5d4 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/base/ptl_base_sendrecv.c +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/base/ptl_base_sendrecv.c @@ -50,12 +50,14 @@ static void _notify_complete(pmix_status_t status, void *cbdata) { + (void)status; pmix_event_chain_t *chain = (pmix_event_chain_t*)cbdata; PMIX_RELEASE(chain); } static void lcfn(pmix_status_t status, void *cbdata) { + (void)status; pmix_peer_t *peer = (pmix_peer_t*)cbdata; PMIX_RELEASE(peer); } @@ -385,6 +387,8 @@ static pmix_status_t read_bytes(int sd, char **buf, size_t *remain) */ void pmix_ptl_base_send_handler(int sd, short flags, void *cbdata) { + (void)sd; + (void)flags; pmix_peer_t *peer = (pmix_peer_t*)cbdata; pmix_ptl_send_t *msg = peer->send_msg; pmix_status_t rc; @@ -463,6 +467,7 @@ void pmix_ptl_base_send_handler(int sd, short flags, void *cbdata) void pmix_ptl_base_recv_handler(int sd, short flags, void *cbdata) { + (void)flags; pmix_status_t rc; pmix_peer_t *peer = (pmix_peer_t*)cbdata; pmix_ptl_recv_t *msg = NULL; @@ -625,6 +630,8 @@ void pmix_ptl_base_recv_handler(int sd, short flags, void *cbdata) void pmix_ptl_base_send(int sd, short args, void *cbdata) { + (void)sd; + (void)args; pmix_ptl_queue_t *queue = (pmix_ptl_queue_t*)cbdata; pmix_ptl_send_t *snd; @@ -681,6 +688,8 @@ void pmix_ptl_base_send(int sd, short args, void *cbdata) void pmix_ptl_base_send_recv(int fd, short args, void *cbdata) { + (void)fd; + (void)args; pmix_ptl_sr_t *ms = (pmix_ptl_sr_t*)cbdata; pmix_ptl_posted_recv_t *req; pmix_ptl_send_t *snd; @@ -759,6 +768,8 @@ void pmix_ptl_base_send_recv(int fd, short args, void *cbdata) void pmix_ptl_base_process_msg(int fd, short flags, void *cbdata) { + (void)fd; + (void)flags; pmix_ptl_recv_t *msg = (pmix_ptl_recv_t*)cbdata; pmix_ptl_posted_recv_t *rcv; pmix_buffer_t buf; diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/base/ptl_base_stubs.c b/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/base/ptl_base_stubs.c index f2334e21a1..376ad3b8cd 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/base/ptl_base_stubs.c +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/base/ptl_base_stubs.c @@ -124,6 +124,8 @@ pmix_status_t pmix_ptl_base_connect_to_peer(struct pmix_peer_t *peer, static void post_recv(int fd, short args, void *cbdata) { + (void)fd; + (void)args; pmix_ptl_posted_recv_t *req = (pmix_ptl_posted_recv_t*)cbdata; pmix_ptl_recv_t *msg, *nmsg; pmix_buffer_t buf; @@ -161,6 +163,7 @@ pmix_status_t pmix_ptl_base_register_recv(struct pmix_peer_t *peer, pmix_ptl_cbfunc_t cbfunc, pmix_ptl_tag_t tag) { + (void)peer; pmix_ptl_posted_recv_t *req; req = PMIX_NEW(pmix_ptl_posted_recv_t); @@ -179,6 +182,8 @@ pmix_status_t pmix_ptl_base_register_recv(struct pmix_peer_t *peer, static void cancel_recv(int fd, short args, void *cbdata) { + (void)fd; + (void)args; pmix_ptl_posted_recv_t *req = (pmix_ptl_posted_recv_t*)cbdata; pmix_ptl_posted_recv_t *rcv; @@ -196,6 +201,7 @@ static void cancel_recv(int fd, short args, void *cbdata) pmix_status_t pmix_ptl_base_cancel_recv(struct pmix_peer_t *peer, pmix_ptl_tag_t tag) { + (void)peer; pmix_ptl_posted_recv_t *req; req = PMIX_NEW(pmix_ptl_posted_recv_t); diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/ptl_types.h b/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/ptl_types.h index 0017c5b813..2d7f2ea666 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/ptl_types.h +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/ptl_types.h @@ -12,7 +12,7 @@ * All rights reserved. * Copyright (c) 2007-2011 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2012-2013 Los Alamos National Security, Inc. All rights reserved. - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -78,6 +78,8 @@ typedef uint16_t pmix_proc_type_t; #define PMIX_PROC_CLIENT_TOOL (PMIX_PROC_TOOL | PMIX_PROC_CLIENT | PMIX_PROC_CLIENT_TOOL_ACT) #define PMIX_PROC_GATEWAY_ACT 0x4000 #define PMIX_PROC_GATEWAY (PMIX_PROC_SERVER | PMIX_PROC_GATEWAY_ACT) +#define PMIX_PROC_SCHEDULER_ACT 0x8000 +#define PMIX_PROC_SCHEDULER (PMIX_PROC_SERVER | PMIX_PROC_SCHEDULER_ACT) /* defins some convenience macros for testing proc type */ #define PMIX_PROC_IS_CLIENT(p) (PMIX_PROC_CLIENT & (p)->proc_type) @@ -90,6 +92,7 @@ typedef uint16_t pmix_proc_type_t; #define PMIX_PROC_IS_LAUNCHER(p) (PMIX_PROC_LAUNCHER_ACT & (p)->proc_type) #define PMIX_PROC_IS_CLIENT_TOOL(p) (PMIX_PROC_CLIENT_TOOL_ACT & (p)->proc_type) #define PMIX_PROC_IS_GATEWAY(p) (PMIX_PROC_GATEWAY_ACT & (p)->proc_type) +#define PMIX_PROC_IS_SCHEDULER(p) (PMIX_PROC_SCHEDULER_ACT & (p)->proc_type) /**** MESSAGING STRUCTURES ****/ @@ -110,7 +113,10 @@ typedef uint32_t pmix_ptl_tag_t; typedef struct { int32_t pindex; pmix_ptl_tag_t tag; - size_t nbytes; + uint32_t nbytes; +#if SIZEOF_SIZE_T == 8 + uint32_t padding; +#endif } pmix_ptl_hdr_t; /* define the messaging cbfunc */ diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/tcp/ptl_tcp.c b/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/tcp/ptl_tcp.c index 65b33987e2..b34e227726 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/tcp/ptl_tcp.c +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/tcp/ptl_tcp.c @@ -128,7 +128,7 @@ static pmix_status_t connect_to_peer(struct pmix_peer_t *peer, char *evar, **uri, *suri = NULL, *suri2 = NULL; char *filename, *nspace=NULL; pmix_rank_t rank = PMIX_RANK_WILDCARD; - char *p, *p2, *server_nspace = NULL, *rendfile = NULL; + char *p = NULL, *p2, *server_nspace = NULL, *rendfile = NULL; int sd, rc; size_t n; char myhost[PMIX_MAXHOSTNAMELEN]; @@ -1298,7 +1298,10 @@ static pmix_status_t recv_connect_ack(int sd, uint8_t myflag) if (NULL == pmix_client_globals.myserver->nptr) { pmix_client_globals.myserver->nptr = PMIX_NEW(pmix_namespace_t); } - pmix_ptl_base_recv_blocking(sd, (char*)nspace, PMIX_MAX_NSLEN+1); + rc = pmix_ptl_base_recv_blocking(sd, (char*)nspace, PMIX_MAX_NSLEN+1); + if (PMIX_SUCCESS != rc) { + return rc; + } if (NULL != pmix_client_globals.myserver->nptr->nspace) { free(pmix_client_globals.myserver->nptr->nspace); } @@ -1307,7 +1310,10 @@ static pmix_status_t recv_connect_ack(int sd, uint8_t myflag) free(pmix_client_globals.myserver->info->pname.nspace); } pmix_client_globals.myserver->info->pname.nspace = strdup(nspace); - pmix_ptl_base_recv_blocking(sd, (char*)&u32, sizeof(uint32_t)); + rc = pmix_ptl_base_recv_blocking(sd, (char*)&u32, sizeof(uint32_t)); + if (PMIX_SUCCESS != rc) { + return rc; + } pmix_client_globals.myserver->info->pname.rank = htonl(u32); pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, @@ -1317,7 +1323,18 @@ static pmix_status_t recv_connect_ack(int sd, uint8_t myflag) pmix_client_globals.myserver->info->pname.rank); /* get the returned status from the security handshake */ - pmix_ptl_base_recv_blocking(sd, (char*)&reply, sizeof(pmix_status_t)); + rc = pmix_ptl_base_recv_blocking(sd, (char*)&u32, sizeof(pmix_status_t)); + if (PMIX_SUCCESS != rc) { + if (sockopt) { + /* return the socket to normal */ + if (0 != setsockopt(sd, SOL_SOCKET, SO_RCVTIMEO, &save, sz)) { + return PMIX_ERR_UNREACH; + } + } + return rc; + } + + reply = ntohl(u32); if (PMIX_SUCCESS != reply) { /* see if they want us to do the handshake */ if (PMIX_ERR_READY_FOR_HANDSHAKE == reply) { diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/tcp/ptl_tcp.h b/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/tcp/ptl_tcp.h index f5373f6506..5813bc7085 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/tcp/ptl_tcp.h +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/tcp/ptl_tcp.h @@ -9,7 +9,7 @@ * University of Stuttgart. All rights reserved. * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. - * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * Copyright (c) 2018 IBM Corporation. All rights reserved. * $COPYRIGHT$ * @@ -48,6 +48,7 @@ typedef struct { struct sockaddr_storage connection; char *session_filename; char *nspace_filename; + char *pid_filename; char *system_filename; char *rendezvous_filename; int wait_to_connect; diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/tcp/ptl_tcp_component.c b/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/tcp/ptl_tcp_component.c index 56a683703f..d642b4116c 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/tcp/ptl_tcp_component.c +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/tcp/ptl_tcp_component.c @@ -16,6 +16,7 @@ * Copyright (c) 2017-2018 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2018-2019 IBM Corporation. All rights reserved. + * Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -117,6 +118,7 @@ static pmix_status_t setup_fork(const pmix_proc_t *proc, char ***env); .disable_ipv6_family = true, .session_filename = NULL, .nspace_filename = NULL, + .pid_filename = NULL, .system_filename = NULL, .rendezvous_filename = NULL, .wait_to_connect = 4, @@ -279,6 +281,14 @@ static pmix_status_t component_open(void) 0 != strcmp(mca_ptl_tcp_component.report_uri, "+")) { urifile = strdup(mca_ptl_tcp_component.report_uri); } + + if (PMIX_PROC_IS_SERVER(pmix_globals.mypeer) || + PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { + if (NULL != (tdir = getenv("PMIX_LAUNCHER_RENDEZVOUS_FILE"))) { + mca_ptl_tcp_component.rendezvous_filename = strdup(tdir); + } + } + return PMIX_SUCCESS; } @@ -297,6 +307,10 @@ pmix_status_t component_close(void) unlink(mca_ptl_tcp_component.nspace_filename); free(mca_ptl_tcp_component.nspace_filename); } + if (NULL != mca_ptl_tcp_component.pid_filename) { + unlink(mca_ptl_tcp_component.pid_filename); + free(mca_ptl_tcp_component.pid_filename); + } if (NULL != mca_ptl_tcp_component.rendezvous_filename) { unlink(mca_ptl_tcp_component.rendezvous_filename); free(mca_ptl_tcp_component.rendezvous_filename); @@ -337,6 +351,14 @@ static pmix_status_t setup_fork(const pmix_proc_t *proc, char ***env) * tool connections - in that case, we will take a non-loopback * device by default, if one is available after filtering directives * + * If we are a tool and were give a rendezvous file, then we first + * check to see if it already exists. If it does, then this is the + * connection info we are to use. If it doesn't, then this is the + * name of the file we are to use to store our listener info. + * + * If we are a server and are given a rendezvous file, then that is + * is the name of the file we are to use to store our listener info. + * * NOTE: we accept MCA parameters, but info keys override them */ static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, @@ -416,6 +438,9 @@ static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, system_tool = PMIX_INFO_TRUE(&info[n]); } else if (PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer) && PMIX_CHECK_KEY(&info[n], PMIX_LAUNCHER_RENDEZVOUS_FILE)) { + if (NULL != mca_ptl_tcp_component.rendezvous_filename) { + free(mca_ptl_tcp_component.rendezvous_filename); + } mca_ptl_tcp_component.rendezvous_filename = strdup(info[n].value.data.string); } } @@ -681,7 +706,15 @@ static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, /* if we were given a rendezvous file, then drop it */ if (NULL != mca_ptl_tcp_component.rendezvous_filename) { FILE *fp; - + /* if we are a tool and the file already exists, then we + * just use it as providing the rendezvous info for our + * server */ + if (PMIX_PROC_IS_TOOL(pmix_globals.mypeer)) { + struct stat buf; + if (0 == stat(mca_ptl_tcp_component.rendezvous_filename, &buf)) { + goto nextstep; + } + } pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "WRITING RENDEZVOUS FILE %s", mca_ptl_tcp_component.rendezvous_filename); @@ -710,6 +743,7 @@ static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, } } + nextstep: /* if we are going to support tools, then drop contact file(s) */ if (system_tool) { FILE *fp; @@ -750,10 +784,10 @@ static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, FILE *fp; pid_t mypid; - /* first output to a file based on pid */ + /* first output to a std file */ mypid = getpid(); - if (0 > asprintf(&mca_ptl_tcp_component.session_filename, "%s/pmix.%s.tool.%d", - mca_ptl_tcp_component.session_tmpdir, myhost, mypid)) { + if (0 > asprintf(&mca_ptl_tcp_component.session_filename, "%s/pmix.%s.tool", + mca_ptl_tcp_component.session_tmpdir, myhost)) { CLOSE_THE_SOCKET(lt->socket); goto sockerror; } @@ -784,6 +818,40 @@ static pmix_status_t setup_listener(pmix_info_t info[], size_t ninfo, goto sockerror; } + /* now output to a file based on pid */ + mypid = getpid(); + if (0 > asprintf(&mca_ptl_tcp_component.pid_filename, "%s/pmix.%s.tool.%d", + mca_ptl_tcp_component.session_tmpdir, myhost, mypid)) { + CLOSE_THE_SOCKET(lt->socket); + goto sockerror; + } + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "WRITING TOOL FILE %s", + mca_ptl_tcp_component.pid_filename); + fp = fopen(mca_ptl_tcp_component.pid_filename, "w"); + if (NULL == fp) { + pmix_output(0, "Impossible to open the file %s in write mode\n", mca_ptl_tcp_component.pid_filename); + PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE); + CLOSE_THE_SOCKET(lt->socket); + free(mca_ptl_tcp_component.pid_filename); + mca_ptl_tcp_component.pid_filename = NULL; + goto sockerror; + } + + /* output my URI */ + fprintf(fp, "%s\n", lt->uri); + /* add a flag that indicates we accept v2.1 protocols */ + fprintf(fp, "%s\n", PMIX_VERSION); + fclose(fp); + /* set the file mode */ + if (0 != chmod(mca_ptl_tcp_component.pid_filename, S_IRUSR | S_IWUSR | S_IRGRP)) { + PMIX_ERROR_LOG(PMIX_ERR_FILE_OPEN_FAILURE); + CLOSE_THE_SOCKET(lt->socket); + free(mca_ptl_tcp_component.pid_filename); + mca_ptl_tcp_component.pid_filename = NULL; + goto sockerror; + } + /* now output it into a file based on my nspace */ if (0 > asprintf(&mca_ptl_tcp_component.nspace_filename, "%s/pmix.%s.tool.%s", @@ -957,7 +1025,7 @@ static void connection_handler(int sd, short args, void *cbdata) pmix_ptl_hdr_t hdr; pmix_peer_t *peer; pmix_rank_t rank=0; - pmix_status_t rc; + pmix_status_t rc, reply; char *msg, *mg, *version; char *sec, *bfrops, *gds; pmix_bfrop_buffer_type_t bftype; @@ -1622,22 +1690,13 @@ static void connection_handler(int sd, short args, void *cbdata) /* validate the connection */ cred.bytes = pnd->cred; cred.size = pnd->len; - PMIX_PSEC_VALIDATE_CONNECTION(rc, peer, NULL, 0, NULL, NULL, &cred); - if (PMIX_SUCCESS != rc) { - pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, - "validation of client connection failed"); - info->proc_cnt--; - pmix_pointer_array_set_item(&pmix_server_globals.clients, peer->index, NULL); - PMIX_RELEASE(peer); - /* send an error reply to the client */ - goto error; - } + PMIX_PSEC_VALIDATE_CONNECTION(reply, peer, NULL, 0, NULL, NULL, &cred); pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, - "client connection validated"); + "client connection validated with status=%d", reply); /* tell the client all is good */ - u32 = htonl(PMIX_SUCCESS); + u32 = htonl(reply); if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t)))) { PMIX_ERROR_LOG(rc); info->proc_cnt--; @@ -1647,6 +1706,22 @@ static void connection_handler(int sd, short args, void *cbdata) PMIX_RELEASE(pnd); return; } + /* If needed perform the handshake. The macro will update reply */ + PMIX_PSEC_SERVER_HANDSHAKE_IFNEED(reply, peer, NULL, 0, NULL, NULL, &cred); + + /* It is possible that connection validation failed + * We need to reply to the client first and cleanup after */ + if (PMIX_SUCCESS != reply) { + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "validation of client connection failed"); + info->proc_cnt--; + pmix_pointer_array_set_item(&pmix_server_globals.clients, peer->index, NULL); + PMIX_RELEASE(peer); + /* send an error reply to the client */ + goto error; + } + + /* send the client's array index */ u32 = htonl(peer->index); if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t)))) { @@ -1709,7 +1784,7 @@ static void process_cbfunc(int sd, short args, void *cbdata) pmix_namespace_t *nptr; pmix_rank_info_t *info; pmix_peer_t *peer; - int rc; + pmix_status_t rc, reply; uint32_t u32; pmix_info_t ginfo; pmix_byte_object_t cred; @@ -1868,8 +1943,23 @@ static void process_cbfunc(int sd, short args, void *cbdata) /* validate the connection */ cred.bytes = pnd->cred; cred.size = pnd->len; - PMIX_PSEC_VALIDATE_CONNECTION(rc, peer, NULL, 0, NULL, NULL, &cred); - if (PMIX_SUCCESS != rc) { + PMIX_PSEC_VALIDATE_CONNECTION(reply, peer, NULL, 0, NULL, NULL, &cred); + /* communicate the result to the other side */ + u32 = htonl(reply); + if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t)))) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE(peer); + pmix_list_remove_item(&pmix_server_globals.nspaces, &nptr->super); + PMIX_RELEASE(nptr); // will release the info object + CLOSE_THE_SOCKET(pnd->sd); + goto done; + } + + /* If needed perform the handshake. The macro will update reply */ + PMIX_PSEC_SERVER_HANDSHAKE_IFNEED(reply, peer, NULL, 0, NULL, NULL, &cred); + + /* If verification wasn't successful - stop here */ + if (PMIX_SUCCESS != reply) { pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "validation of tool credentials failed: %s", PMIx_Error_string(rc)); diff --git a/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/usock/ptl_usock_component.c b/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/usock/ptl_usock_component.c index 7cb073db76..604f697e99 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/usock/ptl_usock_component.c +++ b/opal/mca/pmix/pmix4x/pmix/src/mca/ptl/usock/ptl_usock_component.c @@ -12,10 +12,11 @@ * All rights reserved. * Copyright (c) 2015 Los Alamos National Security, LLC. All rights * reserved. - * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * Copyright (c) 2017 Research Organization for Information Science * and Technology (RIST). All rights reserved. * Copyright (c) 2018 IBM Corporation. All rights reserved. + * Copyright (c) 2019 Mellanox Technologies, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -150,7 +151,11 @@ pmix_status_t component_close(void) static int component_query(pmix_mca_base_module_t **module, int *priority) { + if (PMIX_PROC_IS_TOOL(pmix_globals.mypeer)) { + return PMIX_ERR_NOT_SUPPORTED; + } *module = (pmix_mca_base_module_t*)&pmix_ptl_usock_module; + *priority = mca_ptl_usock_component.super.priority; return PMIX_SUCCESS; } @@ -339,7 +344,7 @@ static void connection_handler(int sd, short args, void *cbdata) { pmix_pending_connection_t *pnd = (pmix_pending_connection_t*)cbdata; char *msg, *ptr, *nspace, *version, *sec, *bfrops, *gds; - pmix_status_t rc; + pmix_status_t rc, reply; unsigned int rank; pmix_usock_hdr_t hdr; pmix_namespace_t *nptr, *tmp; @@ -354,6 +359,7 @@ static void connection_handler(int sd, short args, void *cbdata) unsigned int msglen; pmix_info_t ginfo; pmix_byte_object_t cred; + uint32_t u32; /* acquire the object */ PMIX_ACQUIRE_OBJECT(pnd); @@ -687,12 +693,34 @@ static void connection_handler(int sd, short args, void *cbdata) * record it here for future use */ nptr->compat.ptl = &pmix_ptl_usock_module; - /* validate the connection - the macro will send the status result to the client */ - PMIX_PSEC_VALIDATE_CONNECTION(rc, psave, NULL, 0, NULL, 0, &cred); /* now done with the msg */ free(msg); - if (PMIX_SUCCESS != rc) { + /* validate the connection - the macro will send the status result to the client */ + PMIX_PSEC_VALIDATE_CONNECTION(reply, psave, NULL, 0, NULL, 0, &cred); + pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, + "client connection validated with status=%d", reply); + + /* Communicate the result of validation to the client */ + u32 = htonl(reply); + if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&u32, sizeof(uint32_t)))) { + PMIX_ERROR_LOG(rc); + info->proc_cnt--; + PMIX_RELEASE(info); + pmix_pointer_array_set_item(&pmix_server_globals.clients, psave->index, NULL); + PMIX_RELEASE(psave); + /* error reply was sent by the above macro */ + CLOSE_THE_SOCKET(pnd->sd); + PMIX_RELEASE(pnd); + return; + } + + /* If needed perform the handshake. The macro will update reply */ + PMIX_PSEC_SERVER_HANDSHAKE_IFNEED(reply, psave, NULL, 0, NULL, 0, &cred); + + /* It is possible that connection validation failed + * We need to reply to the client first and cleanup after */ + if (PMIX_SUCCESS != reply) { pmix_output_verbose(2, pmix_ptl_base_framework.framework_output, "validation of client credentials failed: %s", PMIx_Error_string(rc)); @@ -706,6 +734,8 @@ static void connection_handler(int sd, short args, void *cbdata) return; } + + /* send the client's array index */ if (PMIX_SUCCESS != (rc = pmix_ptl_base_send_blocking(pnd->sd, (char*)&psave->index, sizeof(int)))) { PMIX_ERROR_LOG(rc); diff --git a/opal/mca/pmix/pmix4x/pmix/src/runtime/pmix_init.c b/opal/mca/pmix/pmix4x/pmix/src/runtime/pmix_init.c index 020b602f44..62a9c34e11 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/runtime/pmix_init.c +++ b/opal/mca/pmix/pmix4x/pmix/src/runtime/pmix_init.c @@ -166,6 +166,7 @@ int pmix_rte_init(pmix_proc_type_t type, /* setup the globals structure */ gethostname(hostname, PMIX_MAXHOSTNAMELEN); pmix_globals.hostname = strdup(hostname); + pmix_globals.pid = getpid(); memset(&pmix_globals.myid.nspace, 0, PMIX_MAX_NSLEN+1); pmix_globals.myid.rank = PMIX_RANK_INVALID; PMIX_CONSTRUCT(&pmix_globals.events, pmix_events_t); @@ -302,6 +303,10 @@ int pmix_rte_init(pmix_proc_type_t type, } /* open the ptl and select the active plugins */ + if (NULL != (evar = getenv("PMIX_PTL_MODULE"))) { + /* convert to an MCA param, but don't overwrite something already there */ + pmix_setenv("PMIX_MCA_ptl", evar, false, &environ); + } if (PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_ptl_base_framework, 0)) ) { error = "pmix_ptl_base_open"; goto return_error; @@ -317,6 +322,10 @@ int pmix_rte_init(pmix_proc_type_t type, } /* open the psec and select the active plugins */ + if (NULL != (evar = getenv("PMIX_SECURITY_MODE"))) { + /* convert to an MCA param, but don't overwrite something already there */ + pmix_setenv("PMIX_MCA_psec", evar, false, &environ); + } if (PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_psec_base_framework, 0))) { error = "pmix_psec_base_open"; goto return_error; @@ -327,6 +336,10 @@ int pmix_rte_init(pmix_proc_type_t type, } /* open the gds and select the active plugins */ + if (NULL != (evar = getenv("PMIX_GDS_MODULE"))) { + /* convert to an MCA param, but don't overwrite something already there */ + pmix_setenv("PMIX_MCA_gds", evar, false, &environ); + } if (PMIX_SUCCESS != (ret = pmix_mca_base_framework_open(&pmix_gds_base_framework, 0)) ) { error = "pmix_gds_base_open"; goto return_error; diff --git a/opal/mca/pmix/pmix4x/pmix/src/runtime/pmix_progress_threads.c b/opal/mca/pmix/pmix4x/pmix/src/runtime/pmix_progress_threads.c index 83b1582c00..b8b0d530a9 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/runtime/pmix_progress_threads.c +++ b/opal/mca/pmix/pmix4x/pmix/src/runtime/pmix_progress_threads.c @@ -3,6 +3,8 @@ * Copyright (c) 2015 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2017-2019 Research Organization for Information Science * and Technology (RIST). All rights reserved. + * Copyright (c) 2019 Mellanox Technologies, Inc. + * All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -239,7 +241,7 @@ static void stop_progress_engine(pmix_progress_tracker_t *trk) trk->ev_active = false; /* break the event loop - this will cause the loop to exit upon completion of any current event */ - pmix_event_base_loopbreak(trk->ev_base); + pmix_event_base_loopexit(trk->ev_base); pmix_thread_join(&trk->engine, NULL); } diff --git a/opal/mca/pmix/pmix4x/pmix/src/server/Makefile.include b/opal/mca/pmix/pmix4x/pmix/src/server/Makefile.include index c2d9301125..0417bb335c 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/server/Makefile.include +++ b/opal/mca/pmix/pmix4x/pmix/src/server/Makefile.include @@ -1,6 +1,6 @@ # -*- makefile -*- # -# Copyright (c) 2014-2017 Intel, Inc. All rights reserved. +# Copyright (c) 2014-2019 Intel, Inc. All rights reserved. # Copyright (c) 2014 Artem Y. Polyakov . # All rights reserved. # Copyright (c) 2016 Cisco Systems, Inc. All rights reserved. @@ -19,4 +19,5 @@ headers += \ sources += \ server/pmix_server.c \ server/pmix_server_ops.c \ - server/pmix_server_get.c + server/pmix_server_get.c \ + server/pmix_sched.c diff --git a/opal/mca/pmix/pmix4x/pmix/src/server/pmix_sched.c b/opal/mca/pmix/pmix4x/pmix/src/server/pmix_sched.c new file mode 100644 index 0000000000..b73e69a7c5 --- /dev/null +++ b/opal/mca/pmix/pmix4x/pmix/src/server/pmix_sched.c @@ -0,0 +1,199 @@ +/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil -*- */ +/* + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2015 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2014-2015 Artem Y. Polyakov . + * All rights reserved. + * Copyright (c) 2016 Mellanox Technologies, Inc. + * All rights reserved. + * Copyright (c) 2016 IBM Corporation. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#include +#include + +#include +#include +#include + +#include +#include +#include "src/include/pmix_globals.h" + +#ifdef HAVE_STRING_H +#include +#endif +#include +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_SYS_SOCKET_H +#include +#endif +#ifdef HAVE_SYS_UN_H +#include +#endif +#ifdef HAVE_SYS_UIO_H +#include +#endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#include PMIX_EVENT_HEADER + +#include "src/class/pmix_list.h" +#include "src/mca/pnet/base/base.h" +#include "src/util/argv.h" +#include "src/util/error.h" +#include "src/util/output.h" +#include "src/util/pmix_environ.h" + +#include "pmix_server_ops.h" + +PMIX_EXPORT pmix_status_t PMIx_server_register_fabric(pmix_fabric_t *fabric, + const pmix_info_t directives[], + size_t ndirs) +{ + pmix_pnet_base_active_module_t *active; + pmix_status_t rc; + + PMIX_ACQUIRE_THREAD(&pmix_pnet_globals.lock); + + if (0 == pmix_list_get_size(&pmix_pnet_globals.actives)) { + PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock); + return PMIX_ERR_NOT_SUPPORTED; + } + + /* scan across active modules until one returns success */ + PMIX_LIST_FOREACH(active, &pmix_pnet_globals.actives, pmix_pnet_base_active_module_t) { + if (NULL != active->module->register_fabric) { + rc = active->module->register_fabric(fabric, directives, ndirs); + if (PMIX_SUCCESS == rc || PMIX_ERR_TAKE_NEXT_OPTION != rc) { + PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock); + return rc; + } + } + } + + /* unlock prior to return */ + PMIX_RELEASE_THREAD(&pmix_pnet_globals.lock); + + return PMIX_ERR_NOT_FOUND; +} + +PMIX_EXPORT pmix_status_t PMIx_server_deregister_fabric(pmix_fabric_t *fabric) +{ + pmix_status_t rc = PMIX_SUCCESS; + pmix_pnet_fabric_t *active; + pmix_pnet_module_t *module; + + /* it is possible that multiple threads could call this, so + * check to see if it has already been initialized - if so, + * then just return success */ + if (NULL == fabric || NULL == fabric->module) { + return PMIX_SUCCESS; + } + active = (pmix_pnet_fabric_t*)fabric->module; + module = (pmix_pnet_module_t*)active->module; + + if (NULL != module->deregister_fabric) { + rc = module->deregister_fabric(fabric); + } + return rc; +} + +PMIX_EXPORT pmix_status_t PMIx_server_get_num_vertices(pmix_fabric_t *fabric, + uint32_t *nverts) +{ + pmix_status_t ret; + pmix_pnet_fabric_t *ft; + pmix_pnet_module_t *module; + + if (NULL == fabric || NULL == fabric->module) { + return PMIX_ERR_BAD_PARAM; + } + ft = (pmix_pnet_fabric_t*)fabric->module; + module = (pmix_pnet_module_t*)ft->module; + + if (NULL == module->get_num_vertices) { + return PMIX_ERR_NOT_SUPPORTED; + } + ret = module->get_num_vertices(fabric, nverts); + + return ret; +} + +PMIX_EXPORT pmix_status_t PMIx_server_get_comm_cost(pmix_fabric_t *fabric, + uint32_t src, uint32_t dest, + uint16_t *cost) +{ + pmix_status_t ret; + pmix_pnet_fabric_t *ft; + pmix_pnet_module_t *module; + + if (NULL == fabric || NULL == fabric->module) { + return PMIX_ERR_BAD_PARAM; + } + ft = (pmix_pnet_fabric_t*)fabric->module; + module = (pmix_pnet_module_t*)ft->module; + + if (NULL == module->get_cost) { + return PMIX_ERR_NOT_SUPPORTED; + } + + ret = module->get_cost(fabric, src, dest, cost); + + return ret; +} + +PMIX_EXPORT pmix_status_t PMIx_server_get_vertex_info(pmix_fabric_t *fabric, + uint32_t i, pmix_value_t *vertex, + char **nodename) +{ + pmix_status_t ret; + pmix_pnet_fabric_t *ft; + pmix_pnet_module_t *module; + + if (NULL == fabric || NULL == fabric->module) { + return PMIX_ERR_BAD_PARAM; + } + ft = (pmix_pnet_fabric_t*)fabric->module; + module = (pmix_pnet_module_t*)ft->module; + + if (NULL == module->get_vertex) { + return PMIX_ERR_NOT_SUPPORTED; + } + + ret = module->get_vertex(fabric, i, vertex, nodename); + + return ret; +} + +PMIX_EXPORT pmix_status_t PMIx_server_get_index(pmix_fabric_t *fabric, + pmix_value_t *vertex, uint32_t *i, + char **nodename) +{ + pmix_status_t ret; + pmix_pnet_fabric_t *ft; + pmix_pnet_module_t *module; + + if (NULL == fabric || NULL == fabric->module) { + return PMIX_ERR_BAD_PARAM; + } + ft = (pmix_pnet_fabric_t*)fabric->module; + module = (pmix_pnet_module_t*)ft->module; + + if (NULL == module->get_index) { + return PMIX_ERR_NOT_SUPPORTED; + } + + ret = module->get_index(fabric, vertex, i, nodename); + + return ret; +} diff --git a/opal/mca/pmix/pmix4x/pmix/src/server/pmix_server.c b/opal/mca/pmix/pmix4x/pmix/src/server/pmix_server.c index 0fcdd5a50d..3d4a11d5c4 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/server/pmix_server.c +++ b/opal/mca/pmix/pmix4x/pmix/src/server/pmix_server.c @@ -154,9 +154,23 @@ pmix_status_t pmix_server_initialize(void) pmix_server_globals.base_verbose); } + /* get our available security modules */ + security_mode = pmix_psec_base_get_available_modules(); + + /* get our available ptl modules */ + ptl_mode = pmix_ptl_base_get_available_modules(); + + /* get our available bfrop modules */ + bfrops_mode = pmix_bfrops_base_get_available_modules(); + + /* get available gds modules */ + gds_mode = pmix_gds_base_get_available_modules(); + return PMIX_SUCCESS; } +static pmix_server_module_t myhostserver = {0}; + PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, pmix_info_t info[], size_t ninfo) { @@ -173,6 +187,7 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, PMIX_SERVER_TOOL_SUPPORT, PMIX_SERVER_SYSTEM_SUPPORT, PMIX_SERVER_GATEWAY, + PMIX_SERVER_SCHEDULER, NULL }; char *evar; @@ -185,17 +200,25 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, "pmix:server init called"); /* setup the function pointers */ - pmix_host_server = *module; + if (NULL == module) { + pmix_host_server = myhostserver; + } else { + pmix_host_server = *module; + } if (NULL != info) { for (n=0; n < ninfo; n++) { - if (0 == strncmp(info[n].key, PMIX_SERVER_GATEWAY, PMIX_MAX_KEYLEN)) { + if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_GATEWAY)) { if (PMIX_INFO_TRUE(&info[n])) { ptype |= PMIX_PROC_GATEWAY; } - } else if (0 == strncmp(info[n].key, PMIX_SERVER_TMPDIR, PMIX_MAX_KEYLEN)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_SCHEDULER)) { + if (PMIX_INFO_TRUE(&info[n])) { + ptype |= PMIX_PROC_SCHEDULER; + } + } else if (PMIX_CHECK_KEY(&info[n], PMIX_SERVER_TMPDIR)) { pmix_server_globals.tmpdir = strdup(info[n].value.data.string); - } else if (0 == strncmp(info[n].key, PMIX_SYSTEM_TMPDIR, PMIX_MAX_KEYLEN)) { + } else if (PMIX_CHECK_KEY(&info[n], PMIX_SYSTEM_TMPDIR)) { pmix_server_globals.system_tmpdir = strdup(info[n].value.data.string); } } @@ -223,13 +246,6 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, return rc; } - /* setup the server-specific globals */ - if (PMIX_SUCCESS != (rc = pmix_server_initialize())) { - PMIX_ERROR_LOG(rc); - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - /* assign our internal bfrops module */ pmix_globals.mypeer->nptr->compat.bfrops = pmix_bfrops_base_assign_module(NULL); if (NULL == pmix_globals.mypeer->nptr->compat.bfrops) { @@ -272,17 +288,12 @@ PMIX_EXPORT pmix_status_t PMIx_server_init(pmix_server_module_t *module, PMIX_RETAIN(pmix_globals.mypeer->nptr); pmix_client_globals.myserver->nptr = pmix_globals.mypeer->nptr; - /* get our available security modules */ - security_mode = pmix_psec_base_get_available_modules(); - - /* get our available ptl modules */ - ptl_mode = pmix_ptl_base_get_available_modules(); - - /* get our available bfrop modules */ - bfrops_mode = pmix_bfrops_base_get_available_modules(); - - /* get available gds modules */ - gds_mode = pmix_gds_base_get_available_modules(); + /* setup the server-specific globals */ + if (PMIX_SUCCESS != (rc = pmix_server_initialize())) { + PMIX_ERROR_LOG(rc); + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } /* check the info keys for info we * need to provide to every client and @@ -457,6 +468,10 @@ PMIX_EXPORT pmix_status_t PMIx_server_finalize(void) (void)pmix_progress_thread_pause(NULL); } + /* flush anything that is still trying to be written out */ + pmix_iof_static_dump_output(&pmix_client_globals.iof_stdout); + pmix_iof_static_dump_output(&pmix_client_globals.iof_stderr); + pmix_ptl_base_stop_listening(); for (i=0; i < pmix_server_globals.clients.size; i++) { @@ -526,6 +541,13 @@ PMIX_EXPORT pmix_status_t PMIx_server_finalize(void) return PMIX_SUCCESS; } +static void opcbfunc(pmix_status_t status, void *cbdata) +{ + pmix_lock_t *lock = (pmix_lock_t*)cbdata; + lock->status = status; + PMIX_WAKEUP_THREAD(lock); +} + static void _register_nspace(int sd, short args, void *cbdata) { pmix_setup_caddy_t *cd = (pmix_setup_caddy_t*)cbdata; @@ -585,9 +607,7 @@ static void _register_nspace(int sd, short args, void *cbdata) cd->info, cd->ninfo); release: - if (NULL != cd->opcbfunc) { - cd->opcbfunc(rc, cd->cbdata); - } + cd->opcbfunc(rc, cd->cbdata); PMIX_RELEASE(cd); } @@ -597,6 +617,8 @@ PMIX_EXPORT pmix_status_t PMIx_server_register_nspace(const pmix_nspace_t nspace pmix_op_cbfunc_t cbfunc, void *cbdata) { pmix_setup_caddy_t *cd; + pmix_status_t rc; + pmix_lock_t mylock; PMIX_ACQUIRE_THREAD(&pmix_global_lock); if (pmix_globals.init_cntr <= 0) { @@ -616,6 +638,22 @@ PMIX_EXPORT pmix_status_t PMIx_server_register_nspace(const pmix_nspace_t nspace cd->info = info; } + /* if the provided callback is NULL, then substitute + * our own internal cbfunc and block here */ + if (NULL == cbfunc) { + PMIX_CONSTRUCT_LOCK(&mylock); + cd->opcbfunc = opcbfunc; + cd->cbdata = &mylock; + PMIX_THREADSHIFT(cd, _register_nspace); + PMIX_WAIT_THREAD(&mylock); + rc = mylock.status; + PMIX_DESTRUCT_LOCK(&mylock); + if (PMIX_SUCCESS == rc) { + rc = PMIX_OPERATION_SUCCEEDED; + } + return rc; + } + /* we have to push this into our event library to avoid * potential threading issues */ PMIX_THREADSHIFT(cd, _register_nspace); @@ -753,9 +791,7 @@ static void _deregister_nspace(int sd, short args, void *cbdata) } /* release the caller */ - if (NULL != cd->opcbfunc) { - cd->opcbfunc(rc, cd->cbdata); - } + cd->opcbfunc(rc, cd->cbdata); PMIX_RELEASE(cd); } @@ -764,6 +800,7 @@ PMIX_EXPORT void PMIx_server_deregister_nspace(const pmix_nspace_t nspace, void *cbdata) { pmix_setup_caddy_t *cd; + pmix_lock_t mylock; pmix_output_verbose(2, pmix_server_globals.base_output, "pmix:server deregister nspace %s", @@ -784,6 +821,18 @@ PMIX_EXPORT void PMIx_server_deregister_nspace(const pmix_nspace_t nspace, cd->opcbfunc = cbfunc; cd->cbdata = cbdata; + /* if the provided callback is NULL, then substitute + * our own internal cbfunc and block here */ + if (NULL == cbfunc) { + PMIX_CONSTRUCT_LOCK(&mylock); + cd->opcbfunc = opcbfunc; + cd->cbdata = &mylock; + PMIX_THREADSHIFT(cd, _deregister_nspace); + PMIX_WAIT_THREAD(&mylock); + PMIX_DESTRUCT_LOCK(&mylock); + return; + } + /* we have to push this into our event library to avoid * potential threading issues */ PMIX_THREADSHIFT(cd, _deregister_nspace); @@ -1060,9 +1109,7 @@ static void _register_client(int sd, short args, void *cbdata) cleanup: /* let the caller know we are done */ - if (NULL != cd->opcbfunc) { - cd->opcbfunc(rc, cd->cbdata); - } + cd->opcbfunc(rc, cd->cbdata); PMIX_RELEASE(cd); } @@ -1071,6 +1118,8 @@ PMIX_EXPORT pmix_status_t PMIx_server_register_client(const pmix_proc_t *proc, pmix_op_cbfunc_t cbfunc, void *cbdata) { pmix_setup_caddy_t *cd; + pmix_status_t rc; + pmix_lock_t mylock; PMIX_ACQUIRE_THREAD(&pmix_global_lock); if (pmix_globals.init_cntr <= 0) { @@ -1095,6 +1144,22 @@ PMIX_EXPORT pmix_status_t PMIx_server_register_client(const pmix_proc_t *proc, cd->opcbfunc = cbfunc; cd->cbdata = cbdata; + /* if the provided callback is NULL, then substitute + * our own internal cbfunc and block here */ + if (NULL == cbfunc) { + PMIX_CONSTRUCT_LOCK(&mylock); + cd->opcbfunc = opcbfunc; + cd->cbdata = &mylock; + PMIX_THREADSHIFT(cd, _register_client); + PMIX_WAIT_THREAD(&mylock); + rc = mylock.status; + PMIX_DESTRUCT_LOCK(&mylock); + if (PMIX_SUCCESS == rc) { + rc = PMIX_OPERATION_SUCCEEDED; + } + return rc; + } + /* we have to push this into our event library to avoid * potential threading issues */ PMIX_THREADSHIFT(cd, _register_client); @@ -1175,9 +1240,7 @@ static void _deregister_client(int sd, short args, void *cbdata) } cleanup: - if (NULL != cd->opcbfunc) { - cd->opcbfunc(PMIX_SUCCESS, cd->cbdata); - } + cd->opcbfunc(PMIX_SUCCESS, cd->cbdata); PMIX_RELEASE(cd); } @@ -1185,6 +1248,7 @@ PMIX_EXPORT void PMIx_server_deregister_client(const pmix_proc_t *proc, pmix_op_cbfunc_t cbfunc, void *cbdata) { pmix_setup_caddy_t *cd; + pmix_lock_t mylock; PMIX_ACQUIRE_THREAD(&pmix_global_lock); if (pmix_globals.init_cntr <= 0) { @@ -1212,6 +1276,18 @@ PMIX_EXPORT void PMIx_server_deregister_client(const pmix_proc_t *proc, cd->opcbfunc = cbfunc; cd->cbdata = cbdata; + /* if the provided callback is NULL, then substitute + * our own internal cbfunc and block here */ + if (NULL == cbfunc) { + PMIX_CONSTRUCT_LOCK(&mylock); + cd->opcbfunc = opcbfunc; + cd->cbdata = &mylock; + PMIX_THREADSHIFT(cd, _deregister_client); + PMIX_WAIT_THREAD(&mylock); + PMIX_DESTRUCT_LOCK(&mylock); + return; + } + /* we have to push this into our event library to avoid * potential threading issues */ PMIX_THREADSHIFT(cd, _deregister_client); @@ -2323,7 +2399,7 @@ static void _mdxcbfunc(int sd, short argc, void *cbdata) } // Skip the data if we didn't collect it - if (PMIX_COLLECT_YES != tracker->collect_type) { + if (PMIX_COLLECT_YES != tracker->collect_type || NULL == scd->data) { rc = PMIX_SUCCESS; goto finish_collective; } diff --git a/opal/mca/pmix/pmix4x/pmix/src/server/pmix_server_ops.c b/opal/mca/pmix/pmix4x/pmix/src/server/pmix_server_ops.c index ebec2d0346..851a2f793e 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/server/pmix_server_ops.c +++ b/opal/mca/pmix/pmix4x/pmix/src/server/pmix_server_ops.c @@ -174,7 +174,7 @@ pmix_status_t pmix_server_commit(pmix_peer_t *peer, pmix_buffer_t *buf) pmix_strncpy(proc.nspace, nptr->nspace, PMIX_MAX_NSLEN); proc.rank = info->pname.rank; - pmix_output_verbose(2, pmix_server_globals.base_output, + pmix_output_verbose(2, pmix_server_globals.fence_output, "%s:%d EXECUTE COMMIT FOR %s:%d", pmix_globals.myid.nspace, pmix_globals.myid.rank, @@ -389,6 +389,7 @@ static pmix_server_trkr_t* new_tracker(char *id, pmix_proc_t *procs, pmix_server_trkr_t *trk; size_t i; bool all_def; + pmix_rank_t ns_local = 0; pmix_namespace_t *nptr, *ns; pmix_rank_info_t *info; pmix_nspace_caddy_t *nm; @@ -452,6 +453,7 @@ static pmix_server_trkr_t* new_tracker(char *id, pmix_proc_t *procs, pmix_output_verbose(5, pmix_server_globals.base_output, "new_tracker: unknown nspace %s", procs[i].nspace); + trk->local = false; continue; } /* check and add uniq ns into trk nslist */ @@ -480,6 +482,7 @@ static pmix_server_trkr_t* new_tracker(char *id, pmix_proc_t *procs, * of the loop */ } /* is this one of my local ranks? */ + ns_local = 0; PMIX_LIST_FOREACH(info, &nptr->ranks, pmix_rank_info_t) { if (procs[i].rank == info->pname.rank || PMIX_RANK_WILDCARD == procs[i].rank) { @@ -487,12 +490,26 @@ static pmix_server_trkr_t* new_tracker(char *id, pmix_proc_t *procs, "adding local proc %s.%d to tracker", info->pname.nspace, info->pname.rank); /* track the count */ - ++trk->nlocal; + ns_local++; if (PMIX_RANK_WILDCARD != procs[i].rank) { break; } } } + + trk->nlocal += ns_local; + if (!ns_local) { + trk->local = false; + } else if (PMIX_RANK_WILDCARD == procs[i].rank) { + /* If proc is a wildcard we need to additionally check + * that all of the processes in the namespace were + * locally found. + * Otherwise this tracker is not local + */ + if (ns_local != nptr->nprocs) { + trk->local = false; + } + } } if (all_def) { trk->def_complete = true; @@ -799,11 +816,6 @@ pmix_status_t pmix_server_fence(pmix_server_caddy_t *cd, pmix_output_verbose(2, pmix_server_globals.fence_output, "recvd FENCE"); - if (NULL == pmix_host_server.fence_nb) { - PMIX_ERROR_LOG(PMIX_ERR_NOT_SUPPORTED); - return PMIX_ERR_NOT_SUPPORTED; - } - /* unpack the number of procs */ cnt = 1; PMIX_BFROPS_UNPACK(rc, cd->peer, buf, &nprocs, &cnt, PMIX_SIZE); @@ -966,7 +978,6 @@ pmix_status_t pmix_server_fence(pmix_server_caddy_t *cd, /* add this contributor to the tracker so they get * notified when we are done */ - PMIX_RETAIN(cd); pmix_list_append(&trk->local_cbs, &cd->super); /* if a timeout was specified, set it */ if (0 < tv.tv_sec) { @@ -986,6 +997,37 @@ pmix_status_t pmix_server_fence(pmix_server_caddy_t *cd, pmix_list_get_size(&trk->local_cbs) == trk->nlocal) { pmix_output_verbose(2, pmix_server_globals.fence_output, "fence complete"); + /* if this is a purely local fence (i.e., all participants are local), + * then it is done and we notify accordingly */ + if (trk->local) { + /* the modexcbfunc thread-shifts the call prior to processing, + * so it is okay to call it directly from here. The switchyard + * will acknowledge successful acceptance of the fence request, + * but the client still requires a return from the callback in + * that scenario, so we leave this caddy on the list of local cbs */ + trk->modexcbfunc(PMIX_SUCCESS, NULL, 0, trk, NULL, NULL); + rc = PMIX_SUCCESS; + goto cleanup; + } + /* this fence involves non-local procs - check if the + * host supports it */ + if (NULL == pmix_host_server.fence_nb) { + rc = PMIX_ERR_NOT_SUPPORTED; + /* clear the caddy from this tracker so it can be + * released upon return - the switchyard will send an + * error to this caller, and so the fence completion + * function doesn't need to do so */ + pmix_list_remove_item(&trk->local_cbs, &cd->super); + cd->trk = NULL; + /* we need to ensure that all other local participants don't + * just hang waiting for the error return, so execute + * the fence completion function - it threadshifts the call + * prior to processing, so it is okay to call it directly + * from here */ + trk->host_called = false; // the host will not be calling us back + trk->modexcbfunc(rc, NULL, 0, trk, NULL, NULL); + goto cleanup; + } /* if the user asked us to collect data, then we have * to provide any locally collected data to the host * server so they can circulate it - only take data @@ -997,6 +1039,18 @@ pmix_status_t pmix_server_fence(pmix_server_caddy_t *cd, if (PMIX_SUCCESS != (rc = _collect_data(trk, &bucket))) { PMIX_ERROR_LOG(rc); PMIX_DESTRUCT(&bucket); + /* clear the caddy from this tracker so it can be + * released upon return - the switchyard will send an + * error to this caller, and so the fence completion + * function doesn't need to do so */ + pmix_list_remove_item(&trk->local_cbs, &cd->super); + cd->trk = NULL; + /* we need to ensure that all other local participants don't + * just hang waiting for the error return, so execute + * the fence completion function - it threadshifts the call + * prior to processing, so it is okay to call it directly + * from here */ + trk->modexcbfunc(rc, NULL, 0, trk, NULL, NULL); goto cleanup; } /* now unload the blob and pass it upstairs */ @@ -1006,10 +1060,29 @@ pmix_status_t pmix_server_fence(pmix_server_caddy_t *cd, rc = pmix_host_server.fence_nb(trk->pcs, trk->npcs, trk->info, trk->ninfo, data, sz, trk->modexcbfunc, trk); - if (PMIX_SUCCESS != rc) { - pmix_list_remove_item(&pmix_server_globals.collectives, &trk->super); - PMIX_RELEASE(trk); + if (PMIX_SUCCESS != rc && PMIX_OPERATION_SUCCEEDED != rc) { + /* clear the caddy from this tracker so it can be + * released upon return - the switchyard will send an + * error to this caller, and so the fence completion + * function doesn't need to do so */ + pmix_list_remove_item(&trk->local_cbs, &cd->super); cd->trk = NULL; + /* we need to ensure that all other local participants don't + * just hang waiting for the error return, so execute + * the fence completion function - it threadshifts the call + * prior to processing, so it is okay to call it directly + * from here */ + trk->host_called = false; // the host will not be calling us back + trk->modexcbfunc(rc, NULL, 0, trk, NULL, NULL); + } else if (PMIX_OPERATION_SUCCEEDED == rc) { + /* the operation was atomically completed and the host will + * not be calling us back - ensure we notify all participants. + * the modexcbfunc thread-shifts the call prior to processing, + * so it is okay to call it directly from here */ + trk->host_called = false; // the host will not be calling us back + trk->modexcbfunc(PMIX_SUCCESS, NULL, 0, trk, NULL, NULL); + /* ensure that the switchyard doesn't release the caddy */ + rc = PMIX_SUCCESS; } } @@ -1658,10 +1731,29 @@ pmix_status_t pmix_server_disconnect(pmix_server_caddy_t *cd, pmix_list_get_size(&trk->local_cbs) == trk->nlocal) { trk->host_called = true; rc = pmix_host_server.disconnect(trk->pcs, trk->npcs, trk->info, trk->ninfo, cbfunc, trk); - if (PMIX_SUCCESS != rc) { - /* remove this contributor from the list - they will be notified - * by the switchyard */ + if (PMIX_SUCCESS != rc && PMIX_OPERATION_SUCCEEDED != rc) { + /* clear the caddy from this tracker so it can be + * released upon return - the switchyard will send an + * error to this caller, and so the op completion + * function doesn't need to do so */ pmix_list_remove_item(&trk->local_cbs, &cd->super); + cd->trk = NULL; + /* we need to ensure that all other local participants don't + * just hang waiting for the error return, so execute + * the op completion function - it threadshifts the call + * prior to processing, so it is okay to call it directly + * from here */ + trk->host_called = false; // the host will not be calling us back + cbfunc(rc, trk); + } else if (PMIX_OPERATION_SUCCEEDED == rc) { + /* the operation was atomically completed and the host will + * not be calling us back - ensure we notify all participants. + * the cbfunc thread-shifts the call prior to processing, + * so it is okay to call it directly from here */ + trk->host_called = false; // the host will not be calling us back + cbfunc(PMIX_SUCCESS, trk); + /* ensure that the switchyard doesn't release the caddy */ + rc = PMIX_SUCCESS; } } else { rc = PMIX_SUCCESS; @@ -1808,10 +1900,29 @@ pmix_status_t pmix_server_connect(pmix_server_caddy_t *cd, pmix_list_get_size(&trk->local_cbs) == trk->nlocal) { trk->host_called = true; rc = pmix_host_server.connect(trk->pcs, trk->npcs, trk->info, trk->ninfo, cbfunc, trk); - if (PMIX_SUCCESS != rc) { - /* remove this contributor from the list - they will be notified - * by the switchyard */ + if (PMIX_SUCCESS != rc && PMIX_OPERATION_SUCCEEDED != rc) { + /* clear the caddy from this tracker so it can be + * released upon return - the switchyard will send an + * error to this caller, and so the op completion + * function doesn't need to do so */ pmix_list_remove_item(&trk->local_cbs, &cd->super); + cd->trk = NULL; + /* we need to ensure that all other local participants don't + * just hang waiting for the error return, so execute + * the op completion function - it threadshifts the call + * prior to processing, so it is okay to call it directly + * from here */ + trk->host_called = false; // the host will not be calling us back + cbfunc(rc, trk); + } else if (PMIX_OPERATION_SUCCEEDED == rc) { + /* the operation was atomically completed and the host will + * not be calling us back - ensure we notify all participants. + * the cbfunc thread-shifts the call prior to processing, + * so it is okay to call it directly from here */ + trk->host_called = false; // the host will not be calling us back + cbfunc(PMIX_SUCCESS, trk); + /* ensure that the switchyard doesn't release the caddy */ + rc = PMIX_SUCCESS; } } else { rc = PMIX_SUCCESS; @@ -3716,10 +3827,6 @@ static void _grpcbfunc(int sd, short argc, void *cbdata) PMIX_ACQUIRE_OBJECT(scd); - pmix_output_verbose(2, pmix_server_globals.connect_output, - "server:grpcbfunc processing WITH %d MEMBERS", - (NULL == trk) ? 0 : (int)pmix_list_get_size(&trk->local_cbs)); - if (NULL == trk) { /* give them a release if they want it - this should * never happen, but protect against the possibility */ @@ -3730,6 +3837,10 @@ static void _grpcbfunc(int sd, short argc, void *cbdata) return; } + pmix_output_verbose(2, pmix_server_globals.connect_output, + "server:grpcbfunc processing WITH %d MEMBERS", + (int)pmix_list_get_size(&trk->local_cbs)); + /* if the timer is active, clear it */ if (trk->event_active) { pmix_event_del(&trk->ev); @@ -4359,7 +4470,7 @@ static void tcon(pmix_server_trkr_t *t) { t->event_active = false; t->host_called = false; - t->local = false; + t->local = true; t->id = NULL; memset(t->pname.nspace, 0, PMIX_MAX_NSLEN+1); t->pname.rank = PMIX_RANK_UNDEF; diff --git a/opal/mca/pmix/pmix4x/pmix/src/tool/pmix_tool.c b/opal/mca/pmix/pmix4x/pmix/src/tool/pmix_tool.c index 62978fb6f3..df79883ddc 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/tool/pmix_tool.c +++ b/opal/mca/pmix/pmix4x/pmix/src/tool/pmix_tool.c @@ -59,6 +59,7 @@ #include "src/runtime/pmix_rte.h" #include "src/mca/bfrops/base/base.h" #include "src/mca/gds/base/base.h" +#include "src/mca/pfexec/base/base.h" #include "src/mca/pnet/base/base.h" #include "src/mca/ptl/base/base.h" #include "src/mca/psec/psec.h" @@ -265,6 +266,7 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, bool nspace_in_enviro = false; bool rank_given = false; bool fwd_stdin = false; + bool connect_optional = false; pmix_info_t ginfo; size_t n; pmix_ptl_posted_recv_t *rcv; @@ -321,13 +323,17 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, rank_given = true; } else if (0 == strncmp(info[n].key, PMIX_FWD_STDIN, PMIX_MAX_KEYLEN)) { /* they want us to forward our stdin to someone */ - fwd_stdin = true; + fwd_stdin = PMIX_INFO_TRUE(&info[n]); } else if (0 == strncmp(info[n].key, PMIX_LAUNCHER, PMIX_MAX_KEYLEN)) { - ptype |= PMIX_PROC_LAUNCHER; + if (PMIX_INFO_TRUE(&info[n])) { + ptype |= PMIX_PROC_LAUNCHER; + } } else if (0 == strncmp(info[n].key, PMIX_SERVER_TMPDIR, PMIX_MAX_KEYLEN)) { pmix_server_globals.tmpdir = strdup(info[n].value.data.string); } else if (0 == strncmp(info[n].key, PMIX_SYSTEM_TMPDIR, PMIX_MAX_KEYLEN)) { pmix_server_globals.system_tmpdir = strdup(info[n].value.data.string); + } else if (0 == strncmp(info[n].key, PMIX_TOOL_CONNECT_OPTIONAL, PMIX_MAX_KEYLEN)) { + connect_optional = PMIX_INFO_TRUE(&info[n]); } } } @@ -400,24 +406,6 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, } } - /* if we are a launcher, then we also need to act as a server, - * so setup the server-related structures here */ - if (PMIX_PROC_LAUNCHER_ACT & ptype) { - if (PMIX_SUCCESS != (rc = pmix_server_initialize())) { - PMIX_ERROR_LOG(rc); - if (NULL != nspace) { - free(nspace); - } - if (gdsfound) { - PMIX_INFO_DESTRUCT(&ginfo); - } - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; - } - /* setup the function pointers */ - memset(&pmix_host_server, 0, sizeof(pmix_server_module_t)); - } - /* setup the runtime - this init's the globals, * opens and initializes the required frameworks */ if (PMIX_SUCCESS != (rc = pmix_rte_init(ptype, info, ninfo, @@ -571,20 +559,54 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, return PMIX_ERR_INIT; } + /* if we are a launcher, then we also need to act as a server, + * so setup the server-related structures here */ + if (PMIX_PROC_LAUNCHER_ACT & ptype) { + if (PMIX_SUCCESS != (rc = pmix_server_initialize())) { + PMIX_ERROR_LOG(rc); + if (NULL != nspace) { + free(nspace); + } + if (gdsfound) { + PMIX_INFO_DESTRUCT(&ginfo); + } + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + /* setup the function pointers */ + memset(&pmix_host_server, 0, sizeof(pmix_server_module_t)); + } + if (do_not_connect) { /* ensure we mark that we are not connected */ pmix_globals.connected = false; /* it is an error if we were not given an nspace/rank */ if (!nspace_given || !rank_given) { PMIX_RELEASE_THREAD(&pmix_global_lock); - return PMIX_ERR_INIT; + goto regattr; } } else { /* connect to the server */ rc = pmix_ptl_base_connect_to_peer((struct pmix_peer_t*)pmix_client_globals.myserver, info, ninfo); - if (PMIX_SUCCESS != rc){ - PMIX_RELEASE_THREAD(&pmix_global_lock); - return rc; + if (PMIX_SUCCESS != rc) { + if (!connect_optional) { + PMIX_RELEASE_THREAD(&pmix_global_lock); + return rc; + } + /* if connection was optional, then we need to self-assign + * a namespace and rank for ourselves. Use our hostname:pid + * for the nspace, and rank clearly is 0 */ + snprintf(pmix_globals.myid.nspace, PMIX_MAX_NSLEN-1, "%s:%lu", pmix_globals.hostname, (unsigned long)pmix_globals.pid); + pmix_globals.myid.rank = 0; + nspace_given = false; + rank_given = false; + /* also setup the client myserver to point to ourselves */ + pmix_client_globals.myserver->nptr->nspace = strdup(pmix_globals.myid.nspace); + pmix_client_globals.myserver->info = PMIX_NEW(pmix_rank_info_t); + pmix_client_globals.myserver->info->pname.nspace = strdup(pmix_globals.myid.nspace); + pmix_client_globals.myserver->info->pname.rank = pmix_globals.myid.rank; + pmix_client_globals.myserver->info->uid = pmix_globals.uid; + pmix_client_globals.myserver->info->gid = pmix_globals.gid; } } if (!nspace_given) { @@ -736,7 +758,7 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, /* quick check to see if we got something back. If this * is a launcher that is being executed multiple times * in a job-script, then the original registration data - * will have been deleted after the first invocation. In + * may have been deleted after the first invocation. In * such a case, we simply regenerate it locally as it is * well-known */ pmix_cb_t cb; @@ -768,6 +790,14 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, } PMIX_RELEASE_THREAD(&pmix_global_lock); + /* setup the fork/exec framework */ + if (PMIX_SUCCESS != (rc = pmix_mca_base_framework_open(&pmix_pfexec_base_framework, 0)) ) { + return rc; + } + if (PMIX_SUCCESS != (rc = pmix_pfexec_base_select()) ) { + return rc; + } + /* if we are acting as a server, then start listening */ if (PMIX_PROC_IS_LAUNCHER(pmix_globals.mypeer)) { /* start listening for connections */ @@ -777,6 +807,7 @@ PMIX_EXPORT int PMIx_tool_init(pmix_proc_t *proc, } } + regattr: /* register the tool supported attrs */ rc = pmix_register_tool_attrs(); return rc; @@ -790,7 +821,7 @@ pmix_status_t pmix_tool_init_info(void) char hostname[PMIX_MAX_NSLEN]; pmix_strncpy(wildcard.nspace, pmix_globals.myid.nspace, PMIX_MAX_NSLEN); - wildcard.rank = pmix_globals.myid.rank; + wildcard.rank = PMIX_RANK_WILDCARD; /* the jobid is just our nspace */ kptr = PMIX_NEW(pmix_kval_t); @@ -1145,8 +1176,6 @@ PMIX_EXPORT pmix_status_t PMIx_tool_finalize(void) /* flush anything that is still trying to be written out */ pmix_iof_static_dump_output(&pmix_client_globals.iof_stdout); pmix_iof_static_dump_output(&pmix_client_globals.iof_stderr); - PMIX_DESTRUCT(&pmix_client_globals.iof_stdout); - PMIX_DESTRUCT(&pmix_client_globals.iof_stderr); /* if we are connected, then disconnect */ if (pmix_globals.connected) { @@ -1230,6 +1259,7 @@ PMIX_EXPORT pmix_status_t PMIx_tool_finalize(void) } /* shutdown services */ + (void)pmix_mca_base_framework_close(&pmix_pfexec_base_framework); pmix_rte_finalize(); if (NULL != pmix_globals.mypeer) { PMIX_RELEASE(pmix_globals.mypeer); diff --git a/opal/mca/pmix/pmix4x/pmix/src/tools/pattrs/pattrs.c b/opal/mca/pmix/pmix4x/pmix/src/tools/pattrs/pattrs.c index b09a3041dd..ce44a6ee78 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/tools/pattrs/pattrs.c +++ b/opal/mca/pmix/pmix4x/pmix/src/tools/pattrs/pattrs.c @@ -270,7 +270,7 @@ int main(int argc, char **argv) pmix_info_t *info; mylock_t mylock; pmix_cmd_line_t cmd_line; - char **fns; + char **fns = NULL; size_t n, m; myquery_data_t mq; pmix_query_t query; diff --git a/opal/mca/pmix/pmix4x/pmix/src/util/Makefile.include b/opal/mca/pmix/pmix4x/pmix/src/util/Makefile.include index 95ac91d95b..02831df901 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/util/Makefile.include +++ b/opal/mca/pmix/pmix4x/pmix/src/util/Makefile.include @@ -51,7 +51,9 @@ headers += \ util/name_fns.h \ util/net.h \ util/pif.h \ - util/parse_options.h + util/parse_options.h \ + util/context_fns.h \ + util/pmix_pty.h sources += \ util/alfg.c \ @@ -76,7 +78,9 @@ sources += \ util/name_fns.c \ util/net.c \ util/pif.c \ - util/parse_options.c + util/parse_options.c \ + util/context_fns.c \ + util/pmix_pty.c libpmix_la_LIBADD += \ util/keyval/libpmixutilkeyval.la diff --git a/opal/mca/pmix/pmix4x/pmix/src/util/context_fns.c b/opal/mca/pmix/pmix4x/pmix/src/util/context_fns.c new file mode 100644 index 0000000000..5ddf705914 --- /dev/null +++ b/opal/mca/pmix/pmix4x/pmix/src/util/context_fns.c @@ -0,0 +1,104 @@ +/* + * Copyright (c) 2004-2007 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2008 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. + * Copyright (c) 2008-2010 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2014 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +#include "pmix_config.h" +#include "pmix_common.h" + +#include +#include +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#ifdef HAVE_SYS_STAT_H +#include +#endif +#ifdef HAVE_SYS_PARAM_H +#include +#endif +#ifdef HAVE_UNISTD_H +#include +#endif +#ifdef HAVE_NETDB_H +#include +#endif +#include + +#include "src/util/basename.h" +#include "src/util/path.h" +#include "src/util/pmix_environ.h" + +#include "src/util/context_fns.h" + +int pmix_util_check_context_cwd(pmix_app_t *app) +{ + /* If we want to chdir and the chdir fails (for any reason -- such + as if the dir doesn't exist, it isn't a dir, we don't have + permissions, etc.), then return error. */ + if (NULL != app->cwd && 0 != chdir(app->cwd)) { + return PMIX_ERR_BAD_PARAM; + } + + /* All happy */ + return PMIX_SUCCESS; +} + +int pmix_util_check_context_app(pmix_app_t *app, char **env) +{ + char *tmp; + + /* Here's the possibilities: + + 1. The caller specified an absolute pathname for the executable. + We simply need to verify that it exists and we can run it. + + 2. The caller specified a relative pathname for the executable. + Ditto with #1 -- based on the cwd, we need to verify that it + exists and we can run it. + + 3. The caller specified a naked filename. We need to search the + path, find a match, and verify that we can run it. + */ + + tmp = pmix_basename(app->cmd); + if (strlen(tmp) == strlen(app->cmd)) { + /* If this is a naked executable -- no relative or absolute + pathname -- then search the PATH for it */ + free(tmp); + tmp = pmix_path_findv(app->cmd, X_OK, env, app->cwd); + if (NULL == tmp) { + return PMIX_ERR_NOT_FOUND; + } + free(app->cmd); + app->cmd = tmp; + } else { + free(tmp); + if (0 != access(app->cmd, X_OK)) { + return PMIX_ERR_NO_PERMISSIONS; + } + } + + /* All was good */ + return PMIX_SUCCESS; +} diff --git a/opal/mca/pmix/pmix4x/pmix/src/util/context_fns.h b/opal/mca/pmix/pmix4x/pmix/src/util/context_fns.h new file mode 100644 index 0000000000..021259d517 --- /dev/null +++ b/opal/mca/pmix/pmix4x/pmix/src/util/context_fns.h @@ -0,0 +1,38 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2008 Sun Microsystems, Inc. All rights reserved. + * Copyright (c) 2019 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +/** @file: + * + */ + +#ifndef _PMIX_CONTEXT_FNS_H_ +#define _PMIX_CONTEXT_FNS_H_ + +#include "pmix_config.h" +#include "pmix_common.h" + +BEGIN_C_DECLS + +PMIX_EXPORT int pmix_util_check_context_app(pmix_app_t *app, char **env); + +PMIX_EXPORT int pmix_util_check_context_cwd(pmix_app_t *app); + +END_C_DECLS +#endif diff --git a/opal/mca/pmix/pmix4x/pmix/src/util/error.c b/opal/mca/pmix/pmix4x/pmix/src/util/error.c index da2730665c..49ed8a78be 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/util/error.c +++ b/opal/mca/pmix/pmix4x/pmix/src/util/error.c @@ -234,6 +234,8 @@ PMIX_EXPORT const char* PMIx_Error_string(pmix_status_t errnum) case PMIX_GROUP_CONTEXT_ID_ASSIGNED: return "GROUP-CONTEXT-ID-ASSIGNED"; + case PMIX_FABRIC_UPDATED: + return "FABRIC-UPDATED"; case PMIX_ERR_REPEAT_ATTR_REGISTRATION: return "REPEAT-ATTRIBUTE-REGISTRATION"; diff --git a/opal/mca/pmix/pmix4x/pmix/src/util/output.c b/opal/mca/pmix/pmix4x/pmix/src/util/output.c index 8648f1a0b7..cf73f50700 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/util/output.c +++ b/opal/mca/pmix/pmix4x/pmix/src/util/output.c @@ -10,7 +10,7 @@ * Copyright (c) 2004-2006 The Regents of the University of California. * All rights reserved. * Copyright (c) 2007-2008 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2014-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2014-2019 Intel, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ @@ -332,15 +332,10 @@ PMIX_EXPORT void pmix_output(int output_id, const char *format, ...) /* * Send a message to a stream if the verbose level is high enough */ - PMIX_EXPORT void pmix_output_verbose(int level, int output_id, const char *format, ...) + PMIX_EXPORT bool pmix_output_check_verbosity(int level, int output_id) { - if (output_id >= 0 && output_id < PMIX_OUTPUT_MAX_STREAMS && - info[output_id].ldi_verbose_level >= level) { - va_list arglist; - va_start(arglist, format); - output(output_id, format, arglist); - va_end(arglist); - } + return (output_id >= 0 && output_id < PMIX_OUTPUT_MAX_STREAMS && + info[output_id].ldi_verbose_level >= level); } diff --git a/opal/mca/pmix/pmix4x/pmix/src/util/output.h b/opal/mca/pmix/pmix4x/pmix/src/util/output.h index c3274bab7d..5e8fa677b5 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/util/output.h +++ b/opal/mca/pmix/pmix4x/pmix/src/util/output.h @@ -11,7 +11,7 @@ * All rights reserved. * Copyright (c) 2007-2011 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2010 Oracle and/or its affiliates. All rights reserved. - * Copyright (c) 2015-2017 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Intel, Inc. All rights reserved. * Copyright (c) 2016 Research Organization for Information Science * and Technology (RIST). All rights reserved. * $COPYRIGHT$ @@ -414,12 +414,13 @@ PMIX_EXPORT void pmix_output(int output_id, const char *format, ...) __pmix_attr * * @see pmix_output_set_verbosity() */ -PMIX_EXPORT void pmix_output_verbose(int verbose_level, int output_id, - const char *format, ...) __pmix_attribute_format__(__printf__, 3, 4); +#define pmix_output_verbose(verbose_level, output_id, ...) \ + if (pmix_output_check_verbosity(verbose_level, output_id)) { \ + pmix_output(output_id, __VA_ARGS__); \ + } + +PMIX_EXPORT bool pmix_output_check_verbosity(int verbose_level, int output_id); -/** -* Same as pmix_output_verbose(), but takes a va_list form of varargs. -*/ PMIX_EXPORT void pmix_output_vverbose(int verbose_level, int output_id, const char *format, va_list ap) __pmix_attribute_format__(__printf__, 3, 0); diff --git a/opal/mca/pmix/pmix4x/pmix/src/util/pmix_pty.c b/opal/mca/pmix/pmix4x/pmix/src/util/pmix_pty.c new file mode 100644 index 0000000000..505087049d --- /dev/null +++ b/opal/mca/pmix/pmix4x/pmix/src/util/pmix_pty.c @@ -0,0 +1,262 @@ +/* + * Copyright (c) 2004-2005 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2005 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2018 Cisco Systems, Inc. All rights reserved + * Copyright (c) 2019 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ +/*- + * Copyright (c) 1990, 1993 + * The Regents of the University of California. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +#include + +#ifdef HAVE_SYS_CDEFS_H +# include +#endif +#ifdef HAVE_SYS_TYPES_H +#include +#endif +#include +#ifdef HAVE_SYS_IOCTL_H +#include +#endif +#ifdef HAVE_FCNTL_H +#include +#endif +#ifdef HAVE_TERMIOS_H +# include +#else +# ifdef HAVE_TERMIO_H +# include +# endif +#endif +#include +#ifdef HAVE_UNISTD_H +# include +#endif +#include +# include +#ifdef HAVE_GRP_H +#include +#endif +#ifdef HAVE_PTY_H +#include +#endif +#ifdef HAVE_UTMP_H +#include +#endif + +#ifdef HAVE_PTSNAME +# include +# ifdef HAVE_STROPTS_H +# include +# endif +#endif + +#ifdef HAVE_UTIL_H +#include +#endif + +#include "src/util/pmix_pty.h" + +/* The only public interface is openpty - all others are to support + openpty() */ + +#if PMIX_ENABLE_PTY_SUPPORT == 0 + +int pmix_openpty(int *amaster, int *aslave, char *name, + void *termp, void *winpp) +{ + return -1; +} + +#elif defined(HAVE_OPENPTY) + +int pmix_openpty(int *amaster, int *aslave, char *name, + struct termios *termp, struct winsize *winp) +{ + return openpty(amaster, aslave, name, termp, winp); +} + +#else + +/* implement openpty in terms of ptym_open and ptys_open */ + +static int ptym_open(char *pts_name); +static int ptys_open(int fdm, char *pts_name); + +int pmix_openpty(int *amaster, int *aslave, char *name, + struct termios *termp, struct winsize *winp) +{ + char line[20]; + *amaster = ptym_open(line); + if (*amaster < 0) { + return -1; + } + *aslave = ptys_open(*amaster, line); + if (*aslave < 0) { + close(*amaster); + return -1; + } + if (name) { + // We don't know the max length of name, but we do know the + // max length of the source, so at least use that. + pmix_string_copy(name, line, sizeof(line)); + } +#ifndef TCSAFLUSH +#define TCSAFLUSH TCSETAF +#endif + if (termp) { + (void) tcsetattr(*aslave, TCSAFLUSH, termp); + } +#ifdef TIOCSWINSZ + if (winp) { + (void) ioctl(*aslave, TIOCSWINSZ, (char *) winp); + } +#endif + return 0; +} + + +static int ptym_open(char *pts_name) +{ + int fdm; +#ifdef HAVE_PTSNAME + char *ptr; + +#ifdef _AIX + strcpy(pts_name, "/dev/ptc"); +#else + strcpy(pts_name, "/dev/ptmx"); +#endif + fdm = open(pts_name, O_RDWR); + if (fdm < 0) { + return -1; + } + if (grantpt(fdm) < 0) { /* grant access to slave */ + close(fdm); + return -2; + } + if (unlockpt(fdm) < 0) { /* clear slave's lock flag */ + close(fdm); + return -3; + } + ptr = ptsname(fdm); + if (ptr == NULL) { /* get slave's name */ + close(fdm); + return -4; + } + strcpy(pts_name, ptr); /* return name of slave */ + return fdm; /* return fd of master */ +#else + char *ptr1, *ptr2; + + strcpy(pts_name, "/dev/ptyXY"); + /* array index: 012345689 (for references in following code) */ + for (ptr1 = "pqrstuvwxyzPQRST"; *ptr1 != 0; ptr1++) { + pts_name[8] = *ptr1; + for (ptr2 = "0123456789abcdef"; *ptr2 != 0; ptr2++) { + pts_name[9] = *ptr2; + /* try to open master */ + fdm = open(pts_name, O_RDWR); + if (fdm < 0) { + if (errno == ENOENT) { /* different from EIO */ + return -1; /* out of pty devices */ + } else { + continue; /* try next pty device */ + } + } + pts_name[5] = 't'; /* chage "pty" to "tty" */ + return fdm; /* got it, return fd of master */ + } + } + return -1; /* out of pty devices */ +#endif +} + + +static int ptys_open(int fdm, char *pts_name) +{ + int fds; +#ifdef HAVE_PTSNAME + /* following should allocate controlling terminal */ + fds = open(pts_name, O_RDWR); + if (fds < 0) { + close(fdm); + return -5; + } +#if defined(__SVR4) && defined(__sun) + if (ioctl(fds, I_PUSH, "ptem") < 0) { + close(fdm); + close(fds); + return -6; + } + if (ioctl(fds, I_PUSH, "ldterm") < 0) { + close(fdm); + close(fds); + return -7; + } +#endif + + return fds; +#else + int gid; + struct group *grptr; + + grptr = getgrnam("tty"); + if (grptr != NULL) { + gid = grptr->gr_gid; + } else { + gid = -1; /* group tty is not in the group file */ + } + /* following two functions don't work unless we're root */ + chown(pts_name, getuid(), gid); + chmod(pts_name, S_IRUSR | S_IWUSR | S_IWGRP); + fds = open(pts_name, O_RDWR); + if (fds < 0) { + close(fdm); + return -1; + } + return fds; +#endif +} + +#endif /* #ifdef HAVE_OPENPTY */ diff --git a/opal/mca/pmix/pmix4x/pmix/src/util/pmix_pty.h b/opal/mca/pmix/pmix4x/pmix/src/util/pmix_pty.h new file mode 100644 index 0000000000..b7480dcc8b --- /dev/null +++ b/opal/mca/pmix/pmix4x/pmix/src/util/pmix_pty.h @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2004-2006 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2006 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2019 Intel, Inc. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + */ + +#ifndef PMIX_UTIL_PTY_H +#define PMIX_UTIL_PTY_H + +#include +#include "pmix_common.h" + +#ifdef HAVE_UTIL_H +#include +#endif +#ifdef HAVE_LIBUTIL_H +#include +#endif +#ifdef HAVE_TERMIOS_H +# include +#else +# ifdef HAVE_TERMIO_H +# include +# endif +#endif + +BEGIN_C_DECLS + +#if PMIX_ENABLE_PTY_SUPPORT + +PMIX_EXPORT int pmix_openpty(int *amaster, int *aslave, char *name, + struct termios *termp, struct winsize *winp); + +#else + +PMIX_EXPORT int pmix_openpty(int *amaster, int *aslave, char *name, + void *termp, void *winpp); + +#endif + +END_C_DECLS + +#endif /* PMIX_UTIL_PTY_H */ diff --git a/opal/mca/pmix/pmix4x/pmix/src/util/show_help.c b/opal/mca/pmix/pmix4x/pmix/src/util/show_help.c index ae95ac691b..3f0a6e22d6 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/util/show_help.c +++ b/opal/mca/pmix/pmix4x/pmix/src/util/show_help.c @@ -12,7 +12,7 @@ * Copyright (c) 2008 Cisco Systems, Inc. All rights reserved. * Copyright (c) 2015 Research Organization for Information Science * and Technology (RIST). All rights reserved. - * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -48,15 +48,6 @@ static char **search_dirs = NULL; /* * Local functions */ -static int pmix_show_vhelp_internal(const char *filename, const char *topic, - int want_error_header, va_list arglist); -static int pmix_show_help_internal(const char *filename, const char *topic, - int want_error_header, ...); - -pmix_show_help_fn_t pmix_show_help = pmix_show_help_internal; -pmix_show_vhelp_fn_t pmix_show_vhelp = pmix_show_vhelp_internal; - - int pmix_show_help_init(void) { pmix_output_stream_t lds; @@ -337,8 +328,8 @@ char *pmix_show_help_string(const char *filename, const char *topic, return output; } -static int pmix_show_vhelp_internal(const char *filename, const char *topic, - int want_error_header, va_list arglist) +int pmix_show_vhelp(const char *filename, const char *topic, + int want_error_header, va_list arglist) { char *output; @@ -355,18 +346,25 @@ static int pmix_show_vhelp_internal(const char *filename, const char *topic, return (NULL == output) ? PMIX_ERROR : PMIX_SUCCESS; } -static int pmix_show_help_internal(const char *filename, const char *topic, - int want_error_header, ...) +int pmix_show_help(const char *filename, const char *topic, + int want_error_header, ...) { va_list arglist; - int rc; + char *output; - /* Convert it to a single string */ va_start(arglist, want_error_header); - rc = pmix_show_vhelp(filename, topic, want_error_header, arglist); + output = pmix_show_help_vstring(filename, topic, want_error_header, + arglist); va_end(arglist); - return rc; + /* If nothing came back, there's nothing to do */ + if (NULL == output) { + return PMIX_SUCCESS; + } + + fprintf(stderr, "%s\n", output); + free(output); + return PMIX_SUCCESS; } int pmix_show_help_add_dir(const char *directory) diff --git a/opal/mca/pmix/pmix4x/pmix/src/util/show_help.h b/opal/mca/pmix/pmix4x/pmix/src/util/show_help.h index 1129a762a7..811dfeffb8 100644 --- a/opal/mca/pmix/pmix4x/pmix/src/util/show_help.h +++ b/opal/mca/pmix/pmix4x/pmix/src/util/show_help.h @@ -10,7 +10,7 @@ * Copyright (c) 2004-2005 The Regents of the University of California. * All rights reserved. * Copyright (c) 2008-2011 Cisco Systems, Inc. All rights reserved. - * Copyright (c) 2016-2018 Intel, Inc. All rights reserved. + * Copyright (c) 2016-2019 Intel, Inc. All rights reserved. * $COPYRIGHT$ * * Additional copyrights may follow @@ -135,17 +135,15 @@ PMIX_EXPORT int pmix_show_help_finalize(void); * promotion to va_start() has undefined behavior (according to clang * warnings on MacOS High Sierra). */ -typedef int (*pmix_show_help_fn_t)(const char *filename, const char *topic, - int want_error_header, ...); -PMIX_EXPORT extern pmix_show_help_fn_t pmix_show_help; +PMIX_EXPORT int pmix_show_help(const char *filename, const char *topic, + int want_error_header, ...); /** * This function does the same thing as pmix_show_help(), but accepts * a va_list form of varargs. */ -typedef int (*pmix_show_vhelp_fn_t)(const char *filename, const char *topic, - int want_error_header, va_list ap); -PMIX_EXPORT extern pmix_show_vhelp_fn_t pmix_show_vhelp; +PMIX_EXPORT int pmix_show_vhelp(const char *filename, const char *topic, + int want_error_header, va_list ap); /** * This function does the same thing as pmix_show_help(), but returns diff --git a/opal/mca/pmix/pmix4x/pmix/test/pmix_test.c b/opal/mca/pmix/pmix4x/pmix/test/pmix_test.c index f5b5e3bb5e..2535f42e8f 100644 --- a/opal/mca/pmix/pmix4x/pmix/test/pmix_test.c +++ b/opal/mca/pmix/pmix4x/pmix/test/pmix_test.c @@ -121,7 +121,7 @@ int main(int argc, char **argv) while (NULL != pch) { ns_nprocs = (int)strtol(pch, NULL, 10); if (params.nprocs < (uint32_t)(launched+ns_nprocs)) { - TEST_ERROR(("Total number of processes doesn't correspond number specified by ns_dist parameter.")); + TEST_ERROR(("srv #%d: Total number of processes doesn't correspond number specified by ns_dist parameter.", my_server_id)); FREE_TEST_PARAMS(params); return PMIX_ERROR; } @@ -133,7 +133,7 @@ int main(int argc, char **argv) } } if (params.lsize != (uint32_t)launched) { - TEST_ERROR(("Total number of processes doesn't correspond number specified by ns_dist parameter.")); + TEST_ERROR(("srv #%d: Total number of processes doesn't correspond number specified by ns_dist parameter.", my_server_id)); cli_kill_all(); test_fail = 1; } @@ -153,13 +153,13 @@ int main(int argc, char **argv) } if( !test_terminated() ){ - TEST_ERROR(("Test exited by a timeout!")); + TEST_ERROR(("srv #%d: Test exited by a timeout!", my_server_id)); cli_kill_all(); test_fail = 1; } if( test_abort ){ - TEST_ERROR(("Test was aborted!")); + TEST_ERROR(("srv #%d: Test was aborted!", my_server_id)); /* do not simply kill the clients as that generates * event notifications which these tests then print * out, flooding the log */ @@ -174,10 +174,13 @@ int main(int argc, char **argv) /* deregister the errhandler */ PMIx_Deregister_event_handler(0, op_callbk, NULL); + TEST_VERBOSE(("srv #%d: call cli_wait_all!", my_server_id)); cli_wait_all(1.0); - test_fail += server_finalize(¶ms); + TEST_VERBOSE(("srv #%d: call server_finalize!", my_server_id)); + test_fail += server_finalize(¶ms, test_fail); + TEST_VERBOSE(("srv #%d: exit seqence!", my_server_id)); FREE_TEST_PARAMS(params); pmix_argv_free(client_argv); pmix_argv_free(client_env); diff --git a/opal/mca/pmix/pmix4x/pmix/test/server_callbacks.c b/opal/mca/pmix/pmix4x/pmix/test/server_callbacks.c index ae16129ecf..392ce2c0a1 100644 --- a/opal/mca/pmix/pmix4x/pmix/test/server_callbacks.c +++ b/opal/mca/pmix/pmix4x/pmix/test/server_callbacks.c @@ -142,7 +142,7 @@ pmix_status_t dmodex_fn(const pmix_proc_t *proc, { TEST_VERBOSE(("Getting data for %s:%d", proc->nspace, proc->rank)); - /* return not_found fot single server mode */ + /* return not_found for single server mode */ if ((pmix_list_get_size(server_list) == 1) && (my_server_id == 0)) { return PMIX_ERR_NOT_FOUND; } diff --git a/opal/mca/pmix/pmix4x/pmix/test/simple/Makefile.am b/opal/mca/pmix/pmix4x/pmix/test/simple/Makefile.am index 359c688741..bedd620356 100644 --- a/opal/mca/pmix/pmix4x/pmix/test/simple/Makefile.am +++ b/opal/mca/pmix/pmix4x/pmix/test/simple/Makefile.am @@ -25,7 +25,7 @@ headers = simptest.h noinst_PROGRAMS = simptest simpclient simppub simpdyn simpft simpdmodex \ test_pmix simptool simpdie simplegacy simptimeout \ - gwtest gwclient stability quietclient simpjctrl simpio + gwtest gwclient stability quietclient simpjctrl simpio simpsched simptest_SOURCES = \ simptest.c @@ -128,3 +128,9 @@ simpio_SOURCES = \ simpio_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) simpio_LDADD = \ $(top_builddir)/src/libpmix.la + +simpsched_SOURCES = \ + simpsched.c +simpsched_LDFLAGS = $(PMIX_PKG_CONFIG_LDFLAGS) +simpsched_LDADD = \ + $(top_builddir)/src/libpmix.la diff --git a/opal/mca/pmix/pmix4x/pmix/test/simple/simpsched.c b/opal/mca/pmix/pmix4x/pmix/test/simple/simpsched.c new file mode 100644 index 0000000000..354bf76d6a --- /dev/null +++ b/opal/mca/pmix/pmix4x/pmix/test/simple/simpsched.c @@ -0,0 +1,246 @@ +/* + * Copyright (c) 2004-2010 The Trustees of Indiana University and Indiana + * University Research and Technology + * Corporation. All rights reserved. + * Copyright (c) 2004-2011 The University of Tennessee and The University + * of Tennessee Research Foundation. All rights + * reserved. + * Copyright (c) 2004-2005 High Performance Computing Center Stuttgart, + * University of Stuttgart. All rights reserved. + * Copyright (c) 2004-2005 The Regents of the University of California. + * All rights reserved. + * Copyright (c) 2006-2013 Los Alamos National Security, LLC. + * All rights reserved. + * Copyright (c) 2009-2012 Cisco Systems, Inc. All rights reserved. + * Copyright (c) 2011 Oak Ridge National Labs. All rights reserved. + * Copyright (c) 2013-2019 Intel, Inc. All rights reserved. + * Copyright (c) 2015-2019 Research Organization for Information Science + * and Technology (RIST). All rights reserved. + * Copyright (c) 2016 IBM Corporation. All rights reserved. + * $COPYRIGHT$ + * + * Additional copyrights may follow + * + * $HEADER$ + * + */ + +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#if PMIX_HAVE_HWLOC +#include +#endif + +#include "src/class/pmix_list.h" +#include "src/util/pmix_environ.h" +#include "src/util/output.h" +#include "src/util/printf.h" +#include "src/util/argv.h" + +#include "simptest.h" + +static pmix_server_module_t mymodule = {0}; + +typedef struct { + mylock_t lock; + pmix_status_t status; + pmix_info_t *info; + size_t ninfo; +} mycaddy_t; + +static void local_cbfunc(pmix_status_t status, void *cbdata) +{ + mylock_t *lock = (mylock_t*)cbdata; + lock->status = status; + DEBUG_WAKEUP_THREAD(lock); +} + +static void setup_cbfunc(pmix_status_t status, + pmix_info_t info[], size_t ninfo, + void *provided_cbdata, + pmix_op_cbfunc_t cbfunc, void *cbdata) +{ + mycaddy_t *mq = (mycaddy_t*)provided_cbdata; + size_t n; + + /* print out what came back */ + pmix_output(0, "SETUP_APP RETURNED %d INFO", (int)ninfo); + /* transfer it to the caddy for return to the main thread */ + if (0 < ninfo) { + PMIX_INFO_CREATE(mq->info, ninfo); + mq->ninfo = ninfo; + for (n=0; n < ninfo; n++) { + fprintf(stderr, "Key %s Type %s(%d)\n", info[n].key, PMIx_Data_type_string(info[n].value.type), info[n].value.type); + PMIX_INFO_XFER(&mq->info[n], &info[n]); + } + } + + /* let the library release the data and cleanup from + * the operation */ + if (NULL != cbfunc) { + cbfunc(PMIX_SUCCESS, cbdata); + } + + DEBUG_WAKEUP_THREAD(&mq->lock); +} + +int main(int argc, char **argv) +{ + pmix_info_t *info, *iptr; + pmix_status_t rc; + pmix_fabric_t myfabric; + uint32_t nverts, n32, m32; + uint16_t cost; + pmix_value_t val; + size_t ninfo; + int exit_code=0; + char *nodename; + size_t n; + char *hosts, *procs; + char *regex, *ppn; + mylock_t lock; + mycaddy_t cd; + + /* smoke test */ + if (PMIX_SUCCESS != 0) { + fprintf(stderr, "ERROR IN COMPUTING CONSTANTS: PMIX_SUCCESS = %d\n", PMIX_SUCCESS); + exit(1); + } + + fprintf(stderr, "Testing version %s\n", PMIx_Get_version()); + + ninfo = 1; + PMIX_INFO_CREATE(info, ninfo); + PMIX_INFO_LOAD(&info[0], PMIX_SERVER_SCHEDULER, NULL, PMIX_BOOL); + if (PMIX_SUCCESS != (rc = PMIx_server_init(&mymodule, info, ninfo))) { + fprintf(stderr, "Init failed with error %d\n", rc); + return rc; + } + PMIX_INFO_FREE(info, ninfo); + + /* register a fabric */ + rc = PMIx_server_register_fabric(&myfabric, NULL, 0); + if (PMIX_SUCCESS != rc) { + fprintf(stderr, "Fabric registration failed with error: %s\n", PMIx_Error_string(rc)); + goto cleanup; + } + + /* get the number of vertices in the fabric */ + rc = PMIx_server_get_num_vertices(&myfabric, &nverts); + if (PMIX_SUCCESS != rc) { + fprintf(stderr, "Fabric get_num_vertices failed with error: %s\n", PMIx_Error_string(rc)); + goto cleanup; + } + fprintf(stderr, "Number of fabric vertices: %u\n", nverts); + + for (n32=0; n32 < nverts; n32++) { + fprintf(stderr, "%u:", n32); + for (m32=0; m32 < nverts; m32++) { + rc = PMIx_server_get_comm_cost(&myfabric, n32, m32, &cost); + fprintf(stderr, " %u", cost); + } + fprintf(stderr, "\n"); + } + + rc = PMIx_server_get_vertex_info(&myfabric, nverts/2, &val, &nodename); + if (PMIX_SUCCESS != rc) { + fprintf(stderr, "Fabric get vertex info failed with error: %s\n", PMIx_Error_string(rc)); + goto cleanup; + } + if (PMIX_DATA_ARRAY != val.type) { + fprintf(stderr, "Fabric get vertex info returned wrong type: %s\n", PMIx_Data_type_string(val.type)); + goto cleanup; + } + fprintf(stderr, "Vertex info for index %u on node %s:\n", nverts/2, nodename); + info = (pmix_info_t*)val.data.darray->array; + for (n=0; n < val.data.darray->size; n++) { + fprintf(stderr, "\t%s:\t%s\n", info[n].key, info[n].value.data.string); + } + PMIX_VALUE_DESTRUCT(&val); + free(nodename); + + PMIX_INFO_CREATE(info, 1); + PMIX_INFO_LOAD(&info[0], PMIX_NETWORK_NIC, "test002:nic002", PMIX_STRING); + val.type = PMIX_DATA_ARRAY; + PMIX_DATA_ARRAY_CREATE(val.data.darray, 1, PMIX_INFO); + val.data.darray->array = info; + rc = PMIx_server_get_index(&myfabric, &val, &n32, &nodename); + if (PMIX_SUCCESS != rc) { + fprintf(stderr, "Fabric get index failed with error: %s\n", PMIx_Error_string(rc)); + goto cleanup; + } + fprintf(stderr, "Index %u on host %s\n", n32, nodename); + + /* setup an application */ + PMIX_INFO_CREATE(iptr, 4); + hosts = "test000,test001,test002"; + PMIx_generate_regex(hosts, ®ex); + PMIX_INFO_LOAD(&iptr[0], PMIX_NODE_MAP, regex, PMIX_STRING); + free(regex); + + procs = "0,1,2;3,4,5;6,7"; + PMIx_generate_ppn(procs, &ppn); + PMIX_INFO_LOAD(&iptr[1], PMIX_PROC_MAP, ppn, PMIX_STRING); + free(ppn); + + PMIX_LOAD_KEY(iptr[2].key, PMIX_ALLOC_NETWORK); + iptr[2].value.type = PMIX_DATA_ARRAY; + PMIX_DATA_ARRAY_CREATE(iptr[2].value.data.darray, 2, PMIX_INFO); + info = (pmix_info_t*)iptr[2].value.data.darray->array; + PMIX_INFO_LOAD(&info[0], PMIX_ALLOC_NETWORK_ID, "SIMPSCHED.net", PMIX_STRING); + PMIX_INFO_LOAD(&info[1], PMIX_ALLOC_NETWORK_SEC_KEY, NULL, PMIX_BOOL); + + PMIX_INFO_LOAD(&iptr[3], PMIX_SETUP_APP_ENVARS, NULL, PMIX_BOOL); + + DEBUG_CONSTRUCT_LOCK(&cd.lock); + if (PMIX_SUCCESS != (rc = PMIx_server_setup_application("SIMPSCHED", iptr, 4, + setup_cbfunc, &cd))) { + pmix_output(0, "[%s:%d] PMIx_server_setup_application failed: %s", __FILE__, __LINE__, PMIx_Error_string(rc)); + DEBUG_DESTRUCT_LOCK(&cd.lock); + goto cleanup; + } + DEBUG_WAIT_THREAD(&cd.lock); + DEBUG_DESTRUCT_LOCK(&cd.lock); + + /* setup the local subsystem */ + DEBUG_CONSTRUCT_LOCK(&lock); + if (PMIX_SUCCESS != (rc = PMIx_server_setup_local_support("SIMPSCHED", cd.info, cd.ninfo, + local_cbfunc, &lock))) { + pmix_output(0, "[%s:%d] PMIx_server_setup_local_support failed: %s", __FILE__, __LINE__, PMIx_Error_string(rc)); + DEBUG_DESTRUCT_LOCK(&lock); + goto cleanup; + } + DEBUG_WAIT_THREAD(&lock); + DEBUG_DESTRUCT_LOCK(&lock); + + cleanup: + if (PMIX_SUCCESS != rc) { + exit_code = rc; + } + /* finalize the server library */ + if (PMIX_SUCCESS != (rc = PMIx_server_finalize())) { + fprintf(stderr, "Finalize failed with error %d\n", rc); + exit_code = rc; + } + + if (0 == exit_code) { + fprintf(stderr, "Test finished OK!\n"); + } else { + fprintf(stderr, "TEST FAILED WITH ERROR %d\n", exit_code); + } + + return exit_code; +} diff --git a/opal/mca/pmix/pmix4x/pmix/test/simple/simptest.c b/opal/mca/pmix/pmix4x/pmix/test/simple/simptest.c index 1473a277fd..30f55011a6 100644 --- a/opal/mca/pmix/pmix4x/pmix/test/simple/simptest.c +++ b/opal/mca/pmix/pmix4x/pmix/test/simple/simptest.c @@ -413,6 +413,12 @@ int main(int argc, char **argv) if (NULL == executable) { executable = strdup("./simpclient"); } + /* check for executable existence and permissions */ + if (0 != access(executable, X_OK)) { + fprintf(stderr, "Executable %s not found or missing executable permissions\n", executable); + exit(1); + } + if (cross_version && nprocs < 2) { fprintf(stderr, "Cross-version testing requires at least two clients\n"); exit(1); @@ -702,41 +708,131 @@ static void set_namespace(int nprocs, char *ranks, char *nspace, { char *regex, *ppn; char hostname[PMIX_MAXHOSTNAMELEN]; + int n, m, k; + pmix_data_array_t *array; + pmix_info_t *info; gethostname(hostname, sizeof(hostname)); - x->ninfo = 7; + x->ninfo = 16 + nprocs; PMIX_INFO_CREATE(x->info, x->ninfo); - (void)strncpy(x->info[0].key, PMIX_UNIV_SIZE, PMIX_MAX_KEYLEN); - x->info[0].value.type = PMIX_UINT32; - x->info[0].value.data.uint32 = nprocs; + n = 0; - (void)strncpy(x->info[1].key, PMIX_SPAWNED, PMIX_MAX_KEYLEN); - x->info[1].value.type = PMIX_UINT32; - x->info[1].value.data.uint32 = 0; + (void)strncpy(x->info[n].key, PMIX_UNIV_SIZE, PMIX_MAX_KEYLEN); + x->info[n].value.type = PMIX_UINT32; + x->info[n].value.data.uint32 = nprocs; + ++n; - (void)strncpy(x->info[2].key, PMIX_LOCAL_SIZE, PMIX_MAX_KEYLEN); - x->info[2].value.type = PMIX_UINT32; - x->info[2].value.data.uint32 = nprocs; + (void)strncpy(x->info[n].key, PMIX_SPAWNED, PMIX_MAX_KEYLEN); + x->info[n].value.type = PMIX_UINT32; + x->info[n].value.data.uint32 = 0; + ++n; - (void)strncpy(x->info[3].key, PMIX_LOCAL_PEERS, PMIX_MAX_KEYLEN); - x->info[3].value.type = PMIX_STRING; - x->info[3].value.data.string = strdup(ranks); + (void)strncpy(x->info[n].key, PMIX_LOCAL_SIZE, PMIX_MAX_KEYLEN); + x->info[n].value.type = PMIX_UINT32; + x->info[n].value.data.uint32 = nprocs; + ++n; + + (void)strncpy(x->info[n].key, PMIX_LOCAL_PEERS, PMIX_MAX_KEYLEN); + x->info[n].value.type = PMIX_STRING; + x->info[n].value.data.string = strdup(ranks); + ++n; PMIx_generate_regex(hostname, ®ex); - (void)strncpy(x->info[4].key, PMIX_NODE_MAP, PMIX_MAX_KEYLEN); - x->info[4].value.type = PMIX_STRING; - x->info[4].value.data.string = regex; + (void)strncpy(x->info[n].key, PMIX_NODE_MAP, PMIX_MAX_KEYLEN); + x->info[n].value.type = PMIX_STRING; + x->info[n].value.data.string = regex; + ++n; PMIx_generate_ppn(ranks, &ppn); - (void)strncpy(x->info[5].key, PMIX_PROC_MAP, PMIX_MAX_KEYLEN); - x->info[5].value.type = PMIX_STRING; - x->info[5].value.data.string = ppn; + (void)strncpy(x->info[n].key, PMIX_PROC_MAP, PMIX_MAX_KEYLEN); + x->info[n].value.type = PMIX_STRING; + x->info[n].value.data.string = ppn; + ++n; - (void)strncpy(x->info[6].key, PMIX_JOB_SIZE, PMIX_MAX_KEYLEN); - x->info[6].value.type = PMIX_UINT32; - x->info[6].value.data.uint32 = nprocs; + (void)strncpy(x->info[n].key, PMIX_JOB_SIZE, PMIX_MAX_KEYLEN); + x->info[n].value.type = PMIX_UINT32; + x->info[n].value.data.uint32 = nprocs; + ++n; + (void)strncpy(x->info[n].key, PMIX_JOBID, PMIX_MAX_KEYLEN); + x->info[n].value.type = PMIX_STRING; + x->info[n].value.data.string = strdup("1234"); + ++n; + + (void)strncpy(x->info[n].key, PMIX_NPROC_OFFSET, PMIX_MAX_KEYLEN); + x->info[n].value.type = PMIX_UINT32; + x->info[n].value.data.uint32 = 0; + ++n; + + (void)strncpy(x->info[n].key, PMIX_NODEID, PMIX_MAX_KEYLEN); + x->info[n].value.type = PMIX_UINT32; + x->info[n].value.data.uint32 = 0; + ++n; + + (void)strncpy(x->info[n].key, PMIX_NODE_SIZE, PMIX_MAX_KEYLEN); + x->info[n].value.type = PMIX_UINT32; + x->info[n].value.data.uint32 = nprocs; + ++n; + + (void)strncpy(x->info[n].key, PMIX_NUM_NODES, PMIX_MAX_KEYLEN); + x->info[n].value.type = PMIX_UINT32; + x->info[n].value.data.uint32 = 1; + ++n; + + (void)strncpy(x->info[n].key, PMIX_UNIV_SIZE, PMIX_MAX_KEYLEN); + x->info[n].value.type = PMIX_UINT32; + x->info[n].value.data.uint32 = nprocs; + ++n; + + (void)strncpy(x->info[n].key, PMIX_MAX_PROCS, PMIX_MAX_KEYLEN); + x->info[n].value.type = PMIX_UINT32; + x->info[n].value.data.uint32 = nprocs; + ++n; + + (void)strncpy(x->info[n].key, PMIX_JOB_NUM_APPS, PMIX_MAX_KEYLEN); + x->info[n].value.type = PMIX_UINT32; + x->info[n].value.data.uint32 = 1; + ++n; + + (void)strncpy(x->info[n].key, PMIX_LOCALLDR, PMIX_MAX_KEYLEN); + x->info[n].value.type = PMIX_PROC_RANK; + x->info[n].value.data.uint32 = 0; + ++n; + + /* add the proc-specific data */ + for (m=0; m < nprocs; m++) { + (void)strncpy(x->info[n].key, PMIX_PROC_DATA, PMIX_MAX_KEYLEN); + x->info[n].value.type = PMIX_DATA_ARRAY; + PMIX_DATA_ARRAY_CREATE(array, 5, PMIX_INFO); + x->info[n].value.data.darray = array; + info = (pmix_info_t*)array->array; + k = 0; + (void)strncpy(info[k].key, PMIX_RANK, PMIX_MAX_KEYLEN); + info[k].value.type = PMIX_PROC_RANK; + info[k].value.data.rank = m; + ++k; + (void)strncpy(info[k].key, PMIX_GLOBAL_RANK, PMIX_MAX_KEYLEN); + info[k].value.type = PMIX_PROC_RANK; + info[k].value.data.rank = m; + ++k; + (void)strncpy(info[k].key, PMIX_LOCAL_RANK, PMIX_MAX_KEYLEN); + info[k].value.type = PMIX_UINT16; + info[k].value.data.uint16 = m; + ++k; + + (void)strncpy(info[k].key, PMIX_NODE_RANK, PMIX_MAX_KEYLEN); + info[k].value.type = PMIX_UINT16; + info[k].value.data.uint16 = m; + ++k; + + (void)strncpy(info[k].key, PMIX_NODEID, PMIX_MAX_KEYLEN); + info[k].value.type = PMIX_UINT32; + info[k].value.data.uint32 = 0; + ++k; + /* move to next proc */ + ++n; + } PMIx_server_register_nspace(nspace, nprocs, x->info, x->ninfo, cbfunc, x); } diff --git a/opal/mca/pmix/pmix4x/pmix/test/test_common.c b/opal/mca/pmix/pmix4x/pmix/test/test_common.c index 0b66f9da82..6c67e00128 100644 --- a/opal/mca/pmix/pmix4x/pmix/test/test_common.c +++ b/opal/mca/pmix/pmix4x/pmix/test/test_common.c @@ -60,7 +60,8 @@ void parse_cmd(int argc, char **argv, test_params *params) } else if (0 == strcmp(argv[i], "--h") || 0 == strcmp(argv[i], "-h")) { /* print help */ fprintf(stderr, "usage: pmix_test [-h] [-e foo] [-b] [-c] [-nb]\n"); - fprintf(stderr, "\t-n provides information about the job size (for checking purposes)\n"); + fprintf(stderr, "\t-n the job size (for checking purposes)\n"); + fprintf(stderr, "\t-s number of servers to emulate\n"); fprintf(stderr, "\t-e foo use foo as test client\n"); fprintf(stderr, "\t-v verbose output\n"); fprintf(stderr, "\t-t <> set timeout\n"); diff --git a/opal/mca/pmix/pmix4x/pmix/test/test_common.h b/opal/mca/pmix/pmix4x/pmix/test/test_common.h index 15426dc535..2402388308 100644 --- a/opal/mca/pmix/pmix4x/pmix/test/test_common.h +++ b/opal/mca/pmix/pmix4x/pmix/test/test_common.h @@ -25,6 +25,7 @@ #include #include #include +#include #include "src/include/pmix_globals.h" #include "src/class/pmix_list.h" @@ -50,10 +51,15 @@ extern FILE *file; #define STRIPPED_FILE_NAME (strrchr(__FILE__, '/') ? strrchr(__FILE__, '/') + 1 : __FILE__) -#define TEST_OUTPUT(x) { \ - fprintf(file,"==%d== %s:%s: %s\n", getpid(), STRIPPED_FILE_NAME, __func__, \ - pmix_test_output_prepare x ); \ - fflush(file); \ +#define TEST_OUTPUT(x) { \ + struct timeval tv; \ + gettimeofday(&tv, NULL); \ + double ts = tv.tv_sec + 1E-6*tv.tv_usec; \ + fprintf(file,"==%d== [%lf] %s:%s: %s\n", \ + getpid(), ts,STRIPPED_FILE_NAME, \ + __func__, \ + pmix_test_output_prepare x ); \ + fflush(file); \ } // Write output without adding anything to it. @@ -65,9 +71,15 @@ extern FILE *file; // Always write errors to the stderr #define TEST_ERROR(x) { \ - fprintf(stderr,"==%d== ERROR [%s:%d:%s]: %s\n", getpid(), STRIPPED_FILE_NAME, __LINE__, __func__, \ - pmix_test_output_prepare x ); \ - fflush(stderr); \ + struct timeval tv; \ + gettimeofday(&tv, NULL); \ + double ts = tv.tv_sec + 1E-6*tv.tv_usec; \ + fprintf(stderr, \ + "==%d== [%lf] ERROR [%s:%d:%s]: %s\n", \ + getpid(), ts, \ + STRIPPED_FILE_NAME, __LINE__, __func__, \ + pmix_test_output_prepare x ); \ + fflush(stderr); \ } #define TEST_VERBOSE_ON() (pmix_test_verbose = 1) diff --git a/opal/mca/pmix/pmix4x/pmix/test/test_resolve_peers.c b/opal/mca/pmix/pmix4x/pmix/test/test_resolve_peers.c index c88e4ac879..cf0b8a20d9 100644 --- a/opal/mca/pmix/pmix4x/pmix/test/test_resolve_peers.c +++ b/opal/mca/pmix/pmix4x/pmix/test/test_resolve_peers.c @@ -118,9 +118,9 @@ int test_resolve_peers(char *my_nspace, int my_rank, test_params params) /* disconnect from the processes of this namespace. */ rc = PMIx_Disconnect(procs, 2, NULL, 0); if (PMIX_SUCCESS == rc) { - TEST_VERBOSE(("%s:%d: Disconnect from %s succeeded %s.", my_nspace, my_rank, nspace)); + TEST_VERBOSE(("%s:%d: Disconnect from %s succeeded.", my_nspace, my_rank, nspace)); } else { - TEST_ERROR(("%s:%d: Disconnect from %s failed %s.", my_nspace, my_rank, nspace)); + TEST_ERROR(("%s:%d: Disconnect from %s failed.", my_nspace, my_rank, nspace)); return PMIX_ERROR; } } diff --git a/opal/mca/pmix/pmix4x/pmix/test/test_server.c b/opal/mca/pmix/pmix4x/pmix/test/test_server.c index 04f79089d8..aa897fb5ab 100644 --- a/opal/mca/pmix/pmix4x/pmix/test/test_server.c +++ b/opal/mca/pmix/pmix4x/pmix/test/test_server.c @@ -150,7 +150,7 @@ static void set_namespace(int local_size, int univ_size, PMIx_generate_regex(NODE_NAME, ®ex); pmix_strncpy(info[4].key, PMIX_NODE_MAP, PMIX_MAX_KEYLEN); info[4].value.type = PMIX_STRING; - info[4].value.data.string = regex; + info[4].value.data.string = strdup(regex); /* generate the global proc map */ fill_seq_ranks_array(univ_size, 0, &ranks); @@ -161,7 +161,7 @@ static void set_namespace(int local_size, int univ_size, free(ranks); pmix_strncpy(info[5].key, PMIX_PROC_MAP, PMIX_MAX_KEYLEN); info[5].value.type = PMIX_STRING; - info[5].value.data.string = ppn; + info[5].value.data.string = strdup(ppn); pmix_strncpy(info[6].key, PMIX_JOB_SIZE, PMIX_MAX_KEYLEN); info[6].value.type = PMIX_UINT32; @@ -671,6 +671,8 @@ static void _dmdx_cb(int status, char *data, size_t sz, void *cbdata) msg_hdr.src_id = my_server_id; msg_hdr.size = sz; msg_hdr.dst_id = *sender_id; + TEST_VERBOSE(("srv #%d: DMDX RESPONSE: receiver=%d, size=%d,", + my_server_id, *sender_id, sz)); free(sender_id); server_send_msg(&msg_hdr, data, sz); @@ -825,10 +827,10 @@ error: return rc; } -int server_finalize(test_params *params) +int server_finalize(test_params *params, int local_fail) { int rc = PMIX_SUCCESS; - int total_ret = 0; + int total_ret = local_fail; if (0 != (rc = server_barrier())) { total_ret++; @@ -841,15 +843,11 @@ int server_finalize(test_params *params) } if (params->nservers && 0 == my_server_id) { - int ret; /* wait for all servers are finished */ - ret = srv_wait_all(10.0); - if (!pmix_list_is_empty(server_list)) { - total_ret += ret; - } + total_ret += srv_wait_all(10.0); PMIX_LIST_RELEASE(server_list); TEST_VERBOSE(("SERVER %d FINALIZE PID:%d with status %d", - my_server_id, getpid(), ret)); + my_server_id, getpid(), total_ret)); if (0 == total_ret) { TEST_OUTPUT(("Test finished OK!")); } else { @@ -917,18 +915,19 @@ int server_launch_clients(int local_size, int univ_size, int base_rank, /* fork/exec the test */ for (n = 0; n < local_size; n++) { proc.rank = base_rank + rank_counter; + rc = PMIx_server_register_client(&proc, myuid, mygid, NULL, NULL, NULL); + if (PMIX_SUCCESS != rc && PMIX_OPERATION_SUCCEEDED != rc) { + TEST_ERROR(("Server register client failed with error %d", rc)); + PMIx_server_finalize(); + cli_kill_all(); + return 0; + } if (PMIX_SUCCESS != (rc = PMIx_server_setup_fork(&proc, client_env))) {//n TEST_ERROR(("Server fork setup failed with error %d", rc)); PMIx_server_finalize(); cli_kill_all(); return rc; } - if (PMIX_SUCCESS != (rc = PMIx_server_register_client(&proc, myuid, mygid, NULL, NULL, NULL))) {//n - TEST_ERROR(("Server fork setup failed with error %d", rc)); - PMIx_server_finalize(); - cli_kill_all(); - return 0; - } cli_info[cli_counter].pid = fork(); if (cli_info[cli_counter].pid < 0) { diff --git a/opal/mca/pmix/pmix4x/pmix/test/test_server.h b/opal/mca/pmix/pmix4x/pmix/test/test_server.h index fbf1b2bbd4..f02df71fe9 100644 --- a/opal/mca/pmix/pmix4x/pmix/test/test_server.h +++ b/opal/mca/pmix/pmix4x/pmix/test/test_server.h @@ -65,7 +65,7 @@ extern server_info_t *my_server_info; extern pmix_list_t *server_nspace; int server_init(test_params *params); -int server_finalize(test_params *params); +int server_finalize(test_params *params, int local_fail); int server_barrier(void); int server_fence_contrib(char *data, size_t ndata, pmix_modex_cbfunc_t cbfunc, void *cbdata);